├── pubsub.selector ├── processor ├── newrelic.go ├── rancher.go ├── nomad.go ├── customjson.go ├── cloudflare.go ├── aws.go ├── teamcity.go ├── observium.go ├── zabbix.go ├── alertmanager.go ├── site24x7.go ├── winevent.go ├── datadog.go ├── gitlab.go ├── kube.go └── google.go ├── common ├── output.go ├── input.go ├── processor.go ├── inputs.go ├── processors.go ├── utils.go ├── event.go ├── sre.go └── outputs.go ├── events.go ├── test ├── cloudflare.json ├── observium.json ├── aws.rds.json ├── site24x7.json ├── zabbix.json ├── datadog.template ├── test.sh ├── aws.autoscaling.json ├── winevent-http.json ├── alertmanager.json ├── aws.acm.json ├── google.json ├── gitlab-job.json ├── aws.ec2.volume.json ├── datadog-recovered.json ├── aws.ec2.network.json ├── aws.elasticloadbalancing.json ├── aws.ec2.tags.json ├── datadog-test.json ├── datadog-triggered.json ├── aws.route53.json ├── aws.elasticloadbalancing2.json ├── aws.json └── k8s.json ├── vc2telegram.jsonata ├── kafka.message ├── .gitignore ├── .travis.yml ├── datadog2slack.jsonata ├── datadog2gitlab.jsonata ├── pubsub.message ├── workchat.selector ├── gitlab.projects ├── gitlab.variables ├── newrelic.attributes ├── telegram.selector ├── newrelic.message ├── LICENSE ├── _telegram.selector ├── collector.message ├── slack.selector ├── certs.sh ├── input ├── pubsub.go ├── nomad.go └── http.go ├── output ├── collector.go ├── kafka.go ├── newrelic.go ├── pubsub.go ├── grafana.go ├── datadog.go └── gitlab.go ├── zbx_export_mediatypes.xml ├── datadog.name ├── zbx_export_mediatypes.yaml ├── datadog.attributes ├── grafana.attributes └── workchat.message /pubsub.selector: -------------------------------------------------------------------------------- 1 | events -------------------------------------------------------------------------------- /processor/newrelic.go: -------------------------------------------------------------------------------- 1 | package processor 2 | -------------------------------------------------------------------------------- /common/output.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | type Output interface { 4 | Send(event *Event) 5 | Name() string 6 | } 7 | -------------------------------------------------------------------------------- /events.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/devopsext/events/cmd" 4 | 5 | func main() { 6 | cmd.Execute() 7 | } 8 | -------------------------------------------------------------------------------- /common/input.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type Input interface { 8 | Start(wg *sync.WaitGroup, outputs *Outputs) 9 | } 10 | -------------------------------------------------------------------------------- /test/cloudflare.json: -------------------------------------------------------------------------------- 1 | { 2 | "text": "Hello World! This is a test message sent from https://cloudflare.com. If you can see this, your webhook is configured correctly." 3 | } -------------------------------------------------------------------------------- /vc2telegram.jsonata: -------------------------------------------------------------------------------- 1 | { 2 | "Subject": Subject, 3 | "VmName": VmName, 4 | "CreatedTime": CreatedTime, 5 | "DestESXiHostName": DestESXiHostName, 6 | "FullFormattedMessage": $substring(Message, 0, 500) 7 | } 8 | -------------------------------------------------------------------------------- /kafka.message: -------------------------------------------------------------------------------- 1 | Someone make {{.operation}} for{{if .object}} 2 | {{if eq .kind "Pod"}}{{.object.status.phase}}{{end}} 3 | {{ range $key, $value := .object.metadata.labels }}{{ $key}}=>{{ $value }}{{end}}{{end}} 4 | {{.kind}}:{{.location}} 5 | -------------------------------------------------------------------------------- /common/processor.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "net/http" 4 | 5 | type Processor interface { 6 | EventType() string 7 | HandleEvent(e *Event) error 8 | } 9 | 10 | type HttpProcessor interface { 11 | Processor 12 | HandleHttpRequest(w http.ResponseWriter, r *http.Request) error 13 | } 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | .git 3 | bin/ 4 | pkg/ 5 | src/ 6 | vscode.sh 7 | events.key 8 | events.crt 9 | events.ca 10 | coverage.txt 11 | pubsub.credentials 12 | __debug_bin 13 | events 14 | .idea/ 15 | *credential* 16 | checkpoint/ 17 | vcenter_ca.crt 18 | test/vcenter/new 19 | __debug* 20 | /go.work 21 | /go.work.sum 22 | -------------------------------------------------------------------------------- /test/observium.json: -------------------------------------------------------------------------------- 1 | { 2 | "TITLE": "Om-nom-nom! Antonovka!", 3 | "ALERT_STATE": "SYSLOG", 4 | "ALERT_URL": "http:/whatever.url", 5 | "ALERT_UNIXTIME": 1661869111, 6 | "DEVICE_HOSTNAME": "localhost", 7 | "DEVICE_LOCATION": "Alaska", 8 | "METRICS": "This the exact message from syslog or metric and value for alerts" 9 | } 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - "1.17.x" 5 | 6 | go_import_path: github.com/devopsext/events 7 | 8 | before_install: 9 | - env GO111MODULE=on 10 | 11 | install: 12 | - go get -t -v ./... 13 | 14 | script: 15 | - go test -v -race -coverprofile=coverage.txt -covermode=atomic ./... 16 | - go build 17 | 18 | after_success: 19 | - bash <(curl -s https://codecov.io/bash) -------------------------------------------------------------------------------- /datadog2slack.jsonata: -------------------------------------------------------------------------------- 1 | { 2 | "Title": datadog.event.title, 3 | "Message": "*" & $fromMillis(datadog.date) & "*: " &datadog.alert.status & " <" & datadog.link & "|Datadog Monitor Link>", 4 | "ImageURL": image, 5 | "Channel": slack.channel, 6 | "ParentTS": slack.thread, 7 | "QuoteColor": datadog.alert.type = "error" ? "#FF0000" : datadog.alert.type = "success" ? "#00FF00" : "" 8 | } 9 | -------------------------------------------------------------------------------- /datadog2gitlab.jsonata: -------------------------------------------------------------------------------- 1 | ( 2 | { 3 | 'DATADOG_TRANSITION': datadog.alert.transition, 4 | 'DATADOG_ID': datadog.alert.id, 5 | 'DATADOG_TAGS': datadog.tags, 6 | 'SLACK_CHANNEL': slack.channel, 7 | 'SLACK_THREAD': slack.thread, 8 | 'DATE_FROM': $fromMillis(datadog.date - 1000 * 60 * 10), 9 | 'DATE_TO': $fromMillis(datadog.date), 10 | 'TEST': test ? "true" : "" 11 | } 12 | ) -------------------------------------------------------------------------------- /pubsub.message: -------------------------------------------------------------------------------- 1 | {{- define "text"}} 2 | {{- if eq .type "K8sEvent"}} 3 | {{- if not (.data.user.name | regexMatch "(system:serviceaccount:*|system:*)")}} 4 | {{ toJSON .}} 5 | {{- end}} 6 | {{- end}} 7 | {{- if eq .type "AlertmanagerEvent"}}{{ toJSON .}}{{- end}} 8 | {{- if eq .type "GitlabEvent"}}{{ toJSON .}}{{- end}} 9 | {{- end}} 10 | {{- define "pubsub-message"}}{{template "text" .}}{{- end}} -------------------------------------------------------------------------------- /workchat.selector: -------------------------------------------------------------------------------- 1 | {{- define "render"}}{{$url := getVar "URL"}}{{$token := (index . 0)}}{{$recipient := (print "%7B%22thread_key%22%3A%22" (index . 1) "%22%7D")}}{{printf $url $token $recipient}}{{"\n"}}{{end}} 2 | {{- define "rules"}} 3 | {{- if .type | regexMatch ".*"}}{{$a := getEnv "EVENTS_WORKCHAT_OUT_BOT_PLATFORM" | split "="}}{{template "render" $a}}{{end}} 4 | {{- end}} 5 | {{- define "workchat-selector"}}{{template "rules" .}}{{end}} -------------------------------------------------------------------------------- /gitlab.projects: -------------------------------------------------------------------------------- 1 | {{- define "rules"}} 2 | {{- if hasKey .via "Slack"}} 3 | {{- if eq .type "DataDogEvent"}} 4 | {{- if (.data.tags | regexMatch ".*sre-anomaly:(quality).*")}}{{getEnv "EVENTS_GITLAB_OUT_ANOMALY_SRE"}}{{end}} 5 | {{- if (.data.tags | regexMatch ".*sre-anomaly:(met).*")}}{{getEnv "EVENTS_GITLAB_OUT_ANOMALY_PP"}}{{end}} 6 | {{- end}} 7 | {{- end}} 8 | {{- end}} 9 | {{- define "gitlab-projects"}}{{template "rules" .}}{{end}} -------------------------------------------------------------------------------- /gitlab.variables: -------------------------------------------------------------------------------- 1 | {{- define "text"}} 2 | {{- if hasKey .via "Slack"}} 3 | {{- if eq .type "DataDogEvent"}} 4 | {{- $slack := dict "channel" .via.Slack.channel "thread" .via.Slack.ts }} 5 | {{- jsonata (dict "datadog" .data "slack" $slack "test" (.data.alert.title | regexMatch ".*TEST.*")) "/datadog2gitlab.jsonata"}} 6 | {{- end}} 7 | {{- end}} 8 | {{- end}} 9 | {{- define "gitlab-variables"}}{{- if .data}}{{template "text" .}}{{end}}{{- end}} -------------------------------------------------------------------------------- /common/inputs.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "reflect" 5 | "sync" 6 | ) 7 | 8 | type Inputs struct { 9 | list []Input 10 | } 11 | 12 | func (is *Inputs) Add(i Input) { 13 | 14 | if reflect.ValueOf(i).IsNil() { 15 | return 16 | } 17 | is.list = append(is.list, i) 18 | } 19 | 20 | func (is *Inputs) Start(wg *sync.WaitGroup, ots *Outputs) { 21 | 22 | for _, i := range is.list { 23 | 24 | if i != nil { 25 | (i).Start(wg, ots) 26 | } 27 | } 28 | } 29 | 30 | func NewInputs() Inputs { 31 | return Inputs{} 32 | } 33 | -------------------------------------------------------------------------------- /test/aws.rds.json: -------------------------------------------------------------------------------- 1 | { 2 | "account": "098093904626", 3 | "detail": { 4 | "Date": "2022-05-19T01:08:25.750Z", 5 | "EventCategories": [ 6 | "backup" 7 | ], 8 | "EventID": "RDS-EVENT-0001", 9 | "Message": "Backing up DB instance", 10 | "SourceArn": "arn:aws:rds:eu-west-2:098093904626:db:invaxa-prod-crm-bridge-postgres", 11 | "SourceIdentifier": "invaxa-prod-crm-bridge-postgres", 12 | "SourceType": "DB_INSTANCE" 13 | }, 14 | "detail-type": "RDS DB Instance Event", 15 | "region": "eu-west-2", 16 | "source": "aws.rds", 17 | "time": "2022-05-19T01:08:25Z", 18 | "version": "0" 19 | } -------------------------------------------------------------------------------- /test/site24x7.json: -------------------------------------------------------------------------------- 1 | { 2 | "MONITOR_DASHBOARD_LINK": "https://site24x7.com", 3 | "MONITORTYPE": "URL", 4 | "MONITOR_ID": 195603000000025001, 5 | "STATUS": "DOWN", 6 | "MONITORNAME": "Zylker Monitor ", 7 | "FAILED_LOCATIONS": "California-US", 8 | "INCIDENT_REASON": "Service Unavailable", 9 | "GROUP_TAGS": [ 10 | "ZylkerGrp", 11 | "URL" 12 | ], 13 | "MONITORURL": "http://zylker.com", 14 | "MONITOR_GROUPNAME": "Zylker Web Group ", 15 | "POLLFREQUENCY": 1, 16 | "TAGS": [ 17 | "zylker", 18 | "website" 19 | ], 20 | "INCIDENT_TIME": "18 Mar 2022 08:51:19 GMT", 21 | "INCIDENT_TIME_ISO": "2022-03-18T01:51:19-0700" 22 | } -------------------------------------------------------------------------------- /common/processors.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "reflect" 5 | ) 6 | 7 | type Processors struct { 8 | list []Processor 9 | } 10 | 11 | func (ps *Processors) Add(p Processor) { 12 | 13 | if reflect.ValueOf(p).IsNil() { 14 | return 15 | } 16 | ps.list = append(ps.list, p) 17 | } 18 | 19 | func (ps *Processors) Find(eventType string) Processor { 20 | for _, p := range ps.list { 21 | if p.EventType() == eventType { 22 | return p 23 | } 24 | } 25 | return nil 26 | } 27 | 28 | func (ps *Processors) FindHttpProcessor(eventType string) HttpProcessor { 29 | for _, p := range ps.list { 30 | hp, ok := p.(HttpProcessor) 31 | if ok && hp.EventType() == eventType { 32 | return hp 33 | } 34 | } 35 | return nil 36 | } 37 | 38 | func NewProcessors() *Processors { 39 | return &Processors{} 40 | } 41 | -------------------------------------------------------------------------------- /newrelic.attributes: -------------------------------------------------------------------------------- 1 | {{- define "text"}} 2 | {{- if eq .type "K8sEvent"}} 3 | {{printf "{\"type\":\"%s\",\"channel\":\"%s\",\"kind\":\"%s\",\"location\":\"%s\"}" .type .channel .data.kind .data.location}} 4 | {{- end}} 5 | {{- if eq .type "AlertmanagerEvent"}} 6 | {{printf "{\"type\":\"%s\",\"channel\":\"%s\",\"alert\":\"%s\"}" .type .channel .data.labels.alertname}} 7 | {{- end}} 8 | {{- if eq .type "GitlabEvent"}} 9 | {{- if .data.project}}{{printf "{\"type\":\"%s\",\"channel\":\"%s\",\"project\":\"%s / %s@%s\"}" .type .channel .data.project.namespace .data.project.name .data.object_attributes.ref}} 10 | {{else}}{{printf "{\"type\":\"%s\",\"channel\":\"%s\",\"project\":\"%s@%s\"}" .type .channel .data.project_name .data.ref}}{{end}} 11 | {{- end}} 12 | {{- end}} 13 | {{- define "newrelic-attributes"}}{{template "text" .}}{{- end}} -------------------------------------------------------------------------------- /test/zabbix.json: -------------------------------------------------------------------------------- 1 | { 2 | "AlertURL": "https://zabbix.fqdn/tr_events.php?triggerid=479500\u0026eventid=209471288", 3 | "Environment": "zabbix.fqdn", 4 | "EventDate": "2023.12.04", 5 | "EventID": "209471288", 6 | "EventNSeverity": "2", 7 | "EventName": "xdata-click-01.prod.env have queries which running more than 600 sec", 8 | "EventOpData": "4s 698ms", 9 | "EventTags": "Application:Clickhouse, Team:DBA", 10 | "EventTime": "08:13:24", 11 | "EventType": "ZabbixEvent", 12 | "HostName": "xdata-click-01.prod.env", 13 | "ItemID": "2855331", 14 | "ItemLastValue": "4s 698ms", 15 | "Status": "RESOLVED", 16 | "TriggerDescription": "", 17 | "TriggerExpression": "last(/xdata-click-01.prod.env/ch_params.LongestRunningQuery) \u003e= {$MAX_QUERY_TIME}", 18 | "TriggerName": "xdata-click-01.prod.env have queries which running more than 600 sec" 19 | } -------------------------------------------------------------------------------- /telegram.selector: -------------------------------------------------------------------------------- 1 | {{- define "render"}}{{printf (getEnv .)}}{{"\n"}}{{end}} 2 | {{- define "rules"}} 3 | {{- if eq .type "K8sEvent"}}{{template "render" "EVENTS_TELEGRAM_OUT_BOT_TEST"}}{{end}} 4 | {{- if eq .type "GitlabEvent"}}{{template "render" "EVENTS_TELEGRAM_OUT_BOT_DEVOPS"}}{{end}} 5 | {{- if eq .type "AlertmanagerEvent"}}{{template "render" "EVENTS_TELEGRAM_OUT_BOT_SRE"}}{{end}} 6 | {{- if eq .type "DataDogEvent"}}{{template "render" "EVENTS_TELEGRAM_OUT_BOT_SRE"}}{{end}} 7 | {{- if eq .type "Site24x7Event"}}{{template "render" "EVENTS_TELEGRAM_OUT_BOT_SRE"}}{{end}} 8 | {{- if eq .type "CloudflareEvent"}}{{template "render" "EVENTS_TELEGRAM_OUT_BOT_SRE"}}{{end}} 9 | {{- if eq .type "GoogleEvent"}}{{template "render" "EVENTS_TELEGRAM_OUT_BOT_SRE"}}{{end}} 10 | {{- if eq .type "AWSEvent"}}{{template "render" "EVENTS_TELEGRAM_OUT_BOT_TEST"}}{{end}} 11 | {{- end}} 12 | {{- define "telegram-selector"}}{{template "rules" .}}{{end}} -------------------------------------------------------------------------------- /processor/rancher.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/devopsext/events/common" 7 | sreCommon "github.com/devopsext/sre/common" 8 | ) 9 | 10 | type RancherProcessor struct { 11 | outputs *common.Outputs 12 | logger sreCommon.Logger 13 | meter sreCommon.Meter 14 | } 15 | 16 | func RancherProcessorType() string { 17 | return "Rancher" 18 | } 19 | 20 | func (p *RancherProcessor) EventType() string { 21 | return common.AsEventType(RancherProcessorType()) 22 | } 23 | 24 | func (p *RancherProcessor) HandleEvent(e *common.Event) error { 25 | return nil 26 | } 27 | 28 | func (p *RancherProcessor) HandleHttpRequest(w http.ResponseWriter, r *http.Request) error { 29 | return nil 30 | } 31 | 32 | func NewRancherProcessor(outputs *common.Outputs, observability *common.Observability) *RancherProcessor { 33 | return &RancherProcessor{ 34 | outputs: outputs, 35 | logger: observability.Logs(), 36 | meter: observability.Metrics(), 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /newrelic.message: -------------------------------------------------------------------------------- 1 | {{- define "text"}} 2 | {{- if eq .type "K8sEvent"}} 3 | {{- if not (.data.user.name | regexMatch "(system:serviceaccount:*|system:*)")}} 4 | {{- printf "%s" (upper .data.operation)}} 5 | {{- end}} 6 | {{- end}} 7 | {{- if eq .type "AlertmanagerEvent"}}{{- printf "%s" (upper .data.status)}}{{- end}} 8 | {{- if eq .type "GitlabEvent"}} 9 | {{- $match := getEnv "EVENTS_GITLAB_RUNNERS"}}{{$ok := false}} 10 | {{- if .data.builds}} 11 | {{- range .data.builds}} 12 | {{- if and (.runner.description | regexMatch $match) (not (empty .finished_at))}}{{$ok = true}}{{end}} 13 | {{- end}} 14 | {{- else}} 15 | {{- if and (.data.runner.description | regexMatch $match) (not (empty .data.build_duration))}}{{$ok = true}}{{end}} 16 | {{- end}} 17 | {{- if $ok}} 18 | {{- printf "%s" (upper .data.object_kind)}} 19 | {{- end}} 20 | {{- end}} 21 | {{- end}} 22 | {{- define "newrelic-message"}}{{- if .data}}{{template "text" .}}{{end}}{{- end}} -------------------------------------------------------------------------------- /test/datadog.template: -------------------------------------------------------------------------------- 1 | { 2 | "id": "$ID", 3 | "date": $DATE, 4 | "last_updated": $LAST_UPDATED, 5 | "link": "$LINK", 6 | "priority": "$PRIORITY", 7 | "snapshot": "$SNAPSHOT", 8 | "event": { 9 | "type": "$EVENT_TYPE", 10 | "msg": "$EVENT_MSG", 11 | "title": "$EVENT_TITLE" 12 | }, 13 | "alert": { 14 | "id": "$ALERT_ID", 15 | "metric": "$ALERT_METRIC", 16 | "priority": "$ALERT_PRIORITY", 17 | "query": "$ALERT_QUERY", 18 | "scope": "$ALERT_SCOPE", 19 | "status": "$ALERT_STATUS", 20 | "title": "$ALERT_TITLE", 21 | "transition": "$ALERT_TRANSITION", 22 | "type": "$ALERT_TYPE" 23 | }, 24 | "incident": { 25 | "title": "$INCIDENT_TITLE" 26 | }, 27 | "metric": { 28 | "namespace": "$METRIC_NAMESPACE" 29 | }, 30 | "security": { 31 | "rule_name": "$SECURITY_RULE_NAME" 32 | }, 33 | "org": { 34 | "id": "$ORG_ID", 35 | "name": "$ORG_NAME" 36 | }, 37 | "tags": "$TAGS", 38 | "text_only_msg": "$TEXT_ONLY_MSG", 39 | "user": "$USER", 40 | "username": "$USERNAME" 41 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 DevOpsExt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | © 2020 GitHub, Inc. -------------------------------------------------------------------------------- /common/utils.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "reflect" 8 | "strings" 9 | 10 | "github.com/devopsext/utils" 11 | ) 12 | 13 | func AsEventType(s string) string { 14 | return fmt.Sprintf("%sEvent", s) 15 | } 16 | 17 | func JsonMarshal(t interface{}) ([]byte, error) { 18 | buffer := &bytes.Buffer{} 19 | encoder := json.NewEncoder(buffer) 20 | encoder.SetEscapeHTML(false) 21 | err := encoder.Encode(t) 22 | return buffer.Bytes(), err 23 | } 24 | 25 | func Content(s string) string { 26 | 27 | b, err := utils.Content(s) 28 | if err != nil { 29 | return s 30 | } 31 | return string(b) 32 | } 33 | 34 | func DeDotMap(in map[string]string) map[string]string { 35 | if len(in) == 0 { 36 | return in 37 | } 38 | ret := make(map[string]string, len(in)) 39 | for key, value := range in { 40 | nKey := strings.ReplaceAll(key, ".", "_") 41 | ret[nKey] = value 42 | } 43 | return ret 44 | } 45 | 46 | func InterfaceContains(items interface{}, item interface{}) bool { 47 | v := reflect.ValueOf(items) 48 | if v.Kind() == reflect.Map { 49 | for _, key := range v.MapKeys() { 50 | if key.Interface() == item { 51 | return true 52 | } 53 | } 54 | } 55 | return false 56 | } 57 | -------------------------------------------------------------------------------- /test/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #curl -sk -X POST -H "Content-type: application/json" -H "X-Gitlab-Event: Job Hook" -d @gitlab-job.json "http://localhost:80/gitlab" 4 | #curl -sk -X POST -H "Content-type: application/json" -H "X-Gitlab-Event: Pipeline Hook" -d @gitlab-pipeline.json "http://localhost:80/gitlab" 5 | 6 | curl -sk -X POST -H "Content-type: application/json" -d @k8s.json "http://localhost:8081/k8s" 7 | 8 | #curl -sk -X POST -H "Content-type: application/json" -d @alertmanager.json "http://localhost:80/alertmanager" 9 | #curl -sk -X POST -H "Content-type: application/json" -d @zabbix.json "http://localhost:80/zabbix" 10 | 11 | #curl -sk -X POST -H "Content-type: application/json" -d @datadog-triggered.json "http://localhost:8081/datadog" 12 | #curl -sk -X POST -H "Content-type: application/json" -d @datadog-recovered.json "http://localhost:8081/datadog" 13 | 14 | #curl -sk -X POST -H "Content-type: application/json" -d @site24x7.json "http://localhost:80/site24x7" 15 | 16 | #curl -sk -X POST -H "Content-type: application/json" -d @cloudflare.json "http://localhost:80/cloudflare" 17 | 18 | #curl -sk -X POST -H "Content-type: application/json" -d @google.json "http://localhost:80/google" 19 | 20 | #curl -sk -X POST -H "Content-type: application/json" -d @aws.json "http://localhost:80/aws.amazon.com" 21 | -------------------------------------------------------------------------------- /_telegram.selector: -------------------------------------------------------------------------------- 1 | {{- define "render"}}{{$url := getVar "URL"}}{{printf $url (index . 0) (index . 1)}}{{"\n"}}{{end}} 2 | {{- define "rules"}} 3 | {{- if eq .type "K8sEvent"}} 4 | {{- if not (.data.location | regexMatch "(kube-system.ack-controlplane-healthcheck)")}} 5 | {{$a := getEnv "EVENTS_TELEGRAM_OUT_BOT_DEVOPS" | split "="}}{{template "render" $a}} 6 | {{- end}} 7 | {{- end}} 8 | {{- if eq .type "GitlabEvent"}} 9 | {{$a := getEnv "EVENTS_TELEGRAM_OUT_BOT_DEVOPS" | split "="}}{{template "render" $a}} 10 | {{- end}} 11 | {{- if eq .type "AlertmanagerEvent"}} 12 | {{$a := getEnv "EVENTS_TELEGRAM_OUT_BOT_PLATFORM" | split "="}}{{template "render" $a}} 13 | {{- end}} 14 | {{- if eq .type "DataDogEvent"}} 15 | {{$a := getEnv "EVENTS_TELEGRAM_OUT_BOT_PLATFORM" | split "="}}{{template "render" $a}} 16 | {{- end}} 17 | {{- if eq .type "Site24x7Event"}} 18 | {{$a := getEnv "EVENTS_TELEGRAM_OUT_BOT_PLATFORM" | split "="}}{{template "render" $a}} 19 | {{- end}} 20 | {{- if eq .type "CloudflareEvent"}} 21 | {{$a := getEnv "EVENTS_TELEGRAM_OUT_BOT_PLATFORM" | split "="}}{{template "render" $a}} 22 | {{- end}} 23 | {{- if eq .type "GoogleEvent"}} 24 | {{$a := getEnv "EVENTS_TELEGRAM_OUT_BOT_PLATFORM" | split "="}}{{template "render" $a}} 25 | {{- end}} 26 | {{- end}} 27 | {{- define "telegram-selector"}}{{template "rules" .}}{{end}} -------------------------------------------------------------------------------- /test/aws.autoscaling.json: -------------------------------------------------------------------------------- 1 | { 2 | "account": "157576696349", 3 | "detail": { 4 | "ActivityId": "8c360395-3daa-8195-e6a7-ca357cfcdee6", 5 | "AutoScalingGroupName": "ml-eks-eu_MlGPUNodes", 6 | "Cause": "At 2022-05-21T06:30:28Z instance i-033fe5a336b18d2f6 was taken out of service in response to a user request, shrinking the capacity from 1 to 0.", 7 | "Description": "Terminating EC2 instance: i-033fe5a336b18d2f6", 8 | "Destination": "EC2", 9 | "Details": { 10 | "Availability Zone": "eu-central-1a", 11 | "Subnet ID": "subnet-053d0c0c789e7aa18" 12 | }, 13 | "EC2InstanceId": "i-033fe5a336b18d2f6", 14 | "EndTime": "2022-05-21T06:36:52.425Z", 15 | "Origin": "AutoScalingGroup", 16 | "RequestId": "8c360395-3daa-8195-e6a7-ca357cfcdee6", 17 | "StartTime": "2022-05-21T06:30:28.768Z", 18 | "StatusCode": "InProgress", 19 | "StatusMessage": "" 20 | }, 21 | "detail-type": "EC2 Instance Terminate Successful", 22 | "id": "5e47e4c8-13cb-999b-db0f-1c5044a5c8d7", 23 | "region": "eu-central-1", 24 | "resources": [ 25 | "arn:aws:autoscaling:eu-central-1:157576696349:autoScalingGroup:7cbb2751-b0f2-4e90-b4ba-e57c2fa47135:autoScalingGroupName/ml-eks-eu_MlGPUNodes", 26 | "arn:aws:ec2:eu-central-1:157576696349:instance/i-033fe5a336b18d2f6" 27 | ], 28 | "source": "aws.autoscaling", 29 | "time": "2022-05-21T06:36:52Z", 30 | "version": "0" 31 | } -------------------------------------------------------------------------------- /test/winevent-http.json: -------------------------------------------------------------------------------- 1 | { 2 | "metrics": [ 3 | { 4 | "fields": { 5 | "Data_param1": "MetaTrader Data Center", 6 | "EventRecordID": 97068, 7 | "Keywords": "Classic", 8 | "LevelText": "Information", 9 | "Message": "The MetaTrader Data Center service entered the stopped state." 10 | }, 11 | "name": "win_eventlog", 12 | "tags": { 13 | "city": "London", 14 | "country": "", 15 | "host": "DC-TEST", 16 | "mt": "Test1", 17 | "provider": "AWS" 18 | }, 19 | "timestamp": 1654888805 20 | }, 21 | { 22 | "fields": { 23 | "Data_param1": "MetaTrader Data Center", 24 | "EventRecordID": 97069, 25 | "Keywords": "Classic", 26 | "LevelText": "Information", 27 | "Message": "The MetaTrader Data Center service entered the running state." 28 | }, 29 | "name": "win_eventlog", 30 | "tags": { 31 | "city": "London", 32 | "country": "", 33 | "host": "DC-TEST", 34 | "mt": "Test1", 35 | "provider": "AWS" 36 | }, 37 | "timestamp": 1654888808 38 | } 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /common/event.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | 7 | sreCommon "github.com/devopsext/sre/common" 8 | ) 9 | 10 | type Event struct { 11 | Time time.Time `json:"time"` 12 | Channel string `json:"channel"` 13 | Type string `json:"type"` 14 | Data interface{} `json:"data"` 15 | Via map[string]interface{} `json:"via,omitempty"` 16 | logger sreCommon.Logger 17 | } 18 | 19 | func (e *Event) JsonBytes() ([]byte, error) { 20 | 21 | bytes, err := json.Marshal(e) 22 | if err != nil { 23 | return []byte{}, err 24 | } 25 | return bytes, nil 26 | } 27 | 28 | func (e *Event) JsonObject() (interface{}, error) { 29 | 30 | bytes, err := e.JsonBytes() 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | var object interface{} 36 | if err := json.Unmarshal(bytes, &object); err != nil { 37 | return "", err 38 | } 39 | return object, nil 40 | } 41 | 42 | func (e *Event) JsonMap() (map[string]interface{}, error) { 43 | 44 | bytes, err := e.JsonBytes() 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | var m map[string]interface{} 50 | if err := json.Unmarshal(bytes, &m); err != nil { 51 | return nil, err 52 | } 53 | return m, nil 54 | } 55 | 56 | func (e *Event) SetLogger(logger sreCommon.Logger) { 57 | e.logger = logger 58 | } 59 | 60 | func (e *Event) SetTime(time time.Time) { 61 | e.Time = time 62 | } 63 | -------------------------------------------------------------------------------- /test/alertmanager.json: -------------------------------------------------------------------------------- 1 | { 2 | "receiver": "events", 3 | "status": "firing", 4 | "alerts": [ 5 | { 6 | "status": "firing", 7 | "labels": { 8 | "alertname": "Process Open FDS 2", 9 | "app": "prometheus", 10 | "instance": "10.42.0.5:9090", 11 | "kubernetes_namespace": "default", 12 | "kubernetes_pod_name": "prometheus-0", 13 | "severity": "some", 14 | "unit": "short", 15 | "minutes": "10", 16 | "statefulset_kubernetes_io_pod_name": "prometheus-0" 17 | }, 18 | "annotations": { 19 | "summary": "High process Open FDS" 20 | }, 21 | "startsAt": "2022-02-01T15:10:00.056441315Z", 22 | "endsAt": "0001-01-01T00:00:00Z", 23 | "generatorURL": "http://prometheus-0:9090/graph?g0.expr=rate(process_cpu_seconds_total[1m]) > 0.004&g0.tab=1", 24 | "fingerprint": "f8767e67485c740c" 25 | } 26 | ], 27 | "groupLabels": { 28 | "alertname": "Process Open FDS" 29 | }, 30 | "commonLabels": { 31 | "alertname": "Process Open FDS", 32 | "app": "prometheus", 33 | "instance": "10.42.0.5:9090", 34 | "kubernetes_namespace": "default", 35 | "kubernetes_pod_name": "prometheus-0", 36 | "severity": "some", 37 | "statefulset_kubernetes_io_pod_name": "prometheus-0" 38 | }, 39 | "commonAnnotations": { 40 | "summary": "High process Open FDS" 41 | }, 42 | "externalURL": "http://alertmanager-db66d4578-dm696:9093", 43 | "version": "4", 44 | "groupKey": "{}/{}:{alertname=\"Process Open FDS\"}" 45 | } 46 | -------------------------------------------------------------------------------- /common/sre.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | sre "github.com/devopsext/sre/common" 5 | ) 6 | 7 | type Observability struct { 8 | logs *sre.Logs 9 | traces *sre.Traces 10 | metrics *sre.Metrics 11 | events *sre.Events 12 | } 13 | 14 | func (o *Observability) Info(obj interface{}, args ...interface{}) { 15 | if o.logs != nil { 16 | o.logs.Info(obj, args...) 17 | } 18 | } 19 | 20 | func (o *Observability) Warn(obj interface{}, args ...interface{}) { 21 | if o.logs != nil { 22 | o.logs.Warn(obj, args...) 23 | } 24 | } 25 | 26 | func (o *Observability) Panic(obj interface{}, args ...interface{}) { 27 | if o.logs != nil { 28 | o.logs.Panic(obj, args...) 29 | } 30 | } 31 | func (o *Observability) Debug(obj interface{}, args ...interface{}) { 32 | if o.logs != nil { 33 | o.logs.Debug(obj, args...) 34 | } 35 | } 36 | 37 | func (o *Observability) Error(obj interface{}, args ...interface{}) { 38 | if o.logs != nil { 39 | o.logs.Error(obj, args...) 40 | } 41 | } 42 | 43 | func (o *Observability) Logs() *sre.Logs { 44 | return o.logs 45 | } 46 | 47 | func (o *Observability) Traces() *sre.Traces { 48 | return o.traces 49 | } 50 | 51 | func (o *Observability) Metrics() *sre.Metrics { 52 | return o.metrics 53 | } 54 | 55 | func (o *Observability) Events() *sre.Events { 56 | return o.events 57 | } 58 | 59 | func NewObservability(logs *sre.Logs, traces *sre.Traces, metrics *sre.Metrics, events *sre.Events) *Observability { 60 | 61 | return &Observability{ 62 | logs: logs, 63 | traces: traces, 64 | metrics: metrics, 65 | events: events, 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /processor/nomad.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/devopsext/events/common" 7 | sreCommon "github.com/devopsext/sre/common" 8 | nomad "github.com/hashicorp/nomad/api" 9 | ) 10 | 11 | const channel = "nomad" 12 | 13 | type NomadProcessor struct { 14 | outputs *common.Outputs 15 | logger sreCommon.Logger 16 | meter sreCommon.Meter 17 | } 18 | 19 | func (p *NomadProcessor) HandleEvent(e *common.Event) error { 20 | if e == nil { 21 | p.logger.Debug("Event is not defined") 22 | return nil 23 | } 24 | 25 | labels := make(map[string]string) 26 | labels["event_channel"] = e.Channel 27 | labels["processor"] = p.EventType() 28 | 29 | requests := p.meter.Counter("nomad", "requests", "Count of all nomad processor requests", labels, "processor") 30 | requests.Inc() 31 | 32 | p.outputs.Send(e) 33 | return nil 34 | } 35 | 36 | func NomadProcessorType() string { 37 | return "Nomad" 38 | } 39 | 40 | func (p *NomadProcessor) EventType() string { 41 | return common.AsEventType(NomadProcessorType()) 42 | } 43 | 44 | func (p *NomadProcessor) ProcessEvent(ne nomad.Event) error { 45 | ce := &common.Event{ 46 | Channel: channel, 47 | Type: p.EventType(), 48 | Data: ne, 49 | } 50 | ce.SetTime(time.Now().UTC()) 51 | err := p.HandleEvent(ce) 52 | if err != nil { 53 | return err 54 | } 55 | return nil 56 | } 57 | 58 | func NewNomadProcessor(outputs *common.Outputs, observability *common.Observability) *NomadProcessor { 59 | return &NomadProcessor{ 60 | outputs: outputs, 61 | logger: observability.Logs(), 62 | meter: observability.Metrics(), 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /common/outputs.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "encoding/json" 5 | "reflect" 6 | "regexp" 7 | 8 | sreCommon "github.com/devopsext/sre/common" 9 | "github.com/devopsext/utils" 10 | ) 11 | 12 | type Outputs struct { 13 | list []Output 14 | logger sreCommon.Logger 15 | } 16 | 17 | func (ots *Outputs) Add(o Output) { 18 | 19 | if reflect.ValueOf(o).IsNil() { 20 | return 21 | } 22 | ots.list = append(ots.list, o) 23 | } 24 | 25 | func (ots *Outputs) send(e *Event, _ []Output, pattern string) { 26 | 27 | if e == nil { 28 | if ots.logger != nil { 29 | ots.logger.Warn("Event is not found") 30 | } 31 | return 32 | } 33 | 34 | if utils.IsEmpty(pattern) { 35 | if ots.logger != nil { 36 | ots.logger.Warn("Patter is empty") 37 | } 38 | return 39 | } 40 | 41 | jsonString, err := json.Marshal(e) 42 | if err != nil { 43 | if ots.logger != nil { 44 | ots.logger.Error(err) 45 | } 46 | return 47 | } 48 | 49 | if ots.logger != nil { 50 | ots.logger.Debug("Original event => %s", string(jsonString)) 51 | } 52 | 53 | for _, o := range ots.list { 54 | 55 | if o != nil { 56 | matched, err := regexp.MatchString(pattern, o.Name()) 57 | if err != nil { 58 | if ots.logger != nil { 59 | ots.logger.Error(err) 60 | } 61 | continue 62 | } 63 | if !matched { 64 | continue 65 | } 66 | o.Send(e) 67 | } else { 68 | if ots.logger != nil { 69 | ots.logger.Warn("Output is not defined") 70 | } 71 | } 72 | } 73 | } 74 | 75 | func (ots *Outputs) Send(e *Event) { 76 | ots.send(e, []Output{}, ".*") 77 | } 78 | 79 | func (ots *Outputs) SendForward(e *Event, exclude []Output, pattern string) { 80 | ots.send(e, exclude, pattern) 81 | } 82 | 83 | func NewOutputs(logger sreCommon.Logger) Outputs { 84 | return Outputs{ 85 | logger: logger, 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /test/aws.acm.json: -------------------------------------------------------------------------------- 1 | { 2 | "account": "157576696349", 3 | "detail": { 4 | "awsRegion": "us-east-1", 5 | "eventCategory": "Management", 6 | "eventID": "a2db2694-5632-4300-8dca-0e9101b1c026", 7 | "eventName": "DeleteCertificate", 8 | "eventSource": "acm.amazonaws.com", 9 | "eventTime": "2022-05-16T16:16:46Z", 10 | "eventType": "AwsApiCall", 11 | "eventVersion": "1.08", 12 | "managementEvent": true, 13 | "readOnly": false, 14 | "recipientAccountId": "157576696349", 15 | "requestID": "97bc8462-7922-429b-ad9c-8bf863a61756", 16 | "requestParameters": { 17 | "certificateArn": "arn:aws:acm:us-east-1:157576696349:certificate/af843155-4fb9-43cb-b39b-7b6f41e454e2" 18 | }, 19 | "responseElements": null, 20 | "sessionCredentialFromConsole": "true", 21 | "sourceIPAddress": "AWS Internal", 22 | "userAgent": "AWS Internal", 23 | "userIdentity": { 24 | "accessKeyId": "WWWWSJMCNSIOTYLZQANY", 25 | "accountId": "157576696349", 26 | "arn": "arn:aws:sts::157576696349:assumed-role/sso/some.user", 27 | "principalId": "AROASJMCNSIOTZSUL5HOJ:some.user", 28 | "sessionContext": { 29 | "attributes": { 30 | "creationDate": "2022-05-16T15:49:39Z", 31 | "mfaAuthenticated": "false" 32 | }, 33 | "sessionIssuer": { 34 | "accountId": "157576696349", 35 | "arn": "arn:aws:iam::157576696349:role/sso", 36 | "principalId": "AROASJMCNSIOTZSUL5HOJ", 37 | "type": "Role", 38 | "userName": "sso" 39 | }, 40 | "webIdFederationData": {} 41 | }, 42 | "type": "AssumedRole" 43 | } 44 | }, 45 | "detail-type": "AWS API Call via CloudTrail", 46 | "id": "1284ecbe-85d0-1424-445c-164090951aca", 47 | "region": "us-east-1", 48 | "resources": [], 49 | "source": "aws.acm", 50 | "time": "2022-05-16T16:16:46Z", 51 | "version": "0" 52 | } -------------------------------------------------------------------------------- /collector.message: -------------------------------------------------------------------------------- 1 | {{- define "description"}} 2 | {{- if .orchestration | regexMatch ".*(k8s|rke|acs|eks|gke).*"}} 3 | {{- if .object}} 4 | {{- if eq .kind "Namespace"}}{{.kind}} => {{.object.metadata.creationTimestamp}}{{- end}} 5 | {{- if eq .kind "Node"}}{{.kind}} => {{.object.metadata.creationTimestamp}}{{- end}} 6 | {{- if eq .kind "ReplicaSet"}}{{.kind}} => {{.object.metadata.creationTimestamp}}{{end}} 7 | {{- if eq .kind "StatefulSet"}}{{.kind}} => {{.object.metadata.creationTimestamp}}{{end}} 8 | {{- if eq .kind "DaemonSet"}}{{.kind}} => {{.object.metadata.creationTimestamp}}{{end}} 9 | {{- if eq .kind "Secret"}}{{.kind}} => {{.object.metadata.creationTimestamp}}{{end}} 10 | {{- if eq .kind "Ingress"}}{{.kind}} => {{.object.metadata.creationTimestamp}}{{end}} 11 | {{- if eq .kind "CronJob"}}{{.kind}} => {{.object.metadata.creationTimestamp}}{{end}} 12 | {{- if eq .kind "Job"}}{{.kind}} => {{.object.metadata.creationTimestamp}}{{end}} 13 | {{- if eq .kind "ConfigMap"}}{{.kind}} => {{.object.metadata.creationTimestamp}}{{end}} 14 | {{- if eq .kind "Role"}}{{.kind}} => {{.object.metadata.creationTimestamp}}{{end}} 15 | {{- if eq .kind "Deployment"}}{{.kind}} => {{.object.metadata.creationTimestamp}}{{end}} 16 | {{- if eq .kind "Service"}}{{.kind}} => {{.object.metadata.creationTimestamp}}{{end}} 17 | {{- if eq .kind "Pod"}}{{.kind}} => {{.object.metadata.creationTimestamp}}{{end}} 18 | {{- else}}{{.kind}} => is not supported yet{{- end}} 19 | {{- end}} 20 | {{- if .orchestration | regexMatch "rancher.*"}} 21 | {{.kind}} => rancher 22 | {{- end}} 23 | {{- end}} 24 | {{define "collector-message"}}{{- if not (.user.name | regexMatch "(system:serviceaccount:*|system:*)")}}{{- .operation}},source={{.orchestration}},location={{.location}} description="{{template "description" .}}",duration=0i {{printf "%.0f" (timeNano .time)}}{{- end}}{{end}} 25 | -------------------------------------------------------------------------------- /test/google.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "test", 3 | "incident": { 4 | "incident_id": "12345", 5 | "scoping_project_id": "12345", 6 | "scoping_project_number": "12345", 7 | "url": "http://www.example.com", 8 | "started_at": 0, 9 | "ended_at": 0, 10 | "state": "OPEN", 11 | "summary": "Something is going to happen (Test Incident)", 12 | "apigee_url": "http://www.example.com", 13 | "observed_value": "1.0", 14 | 15 | "resource": { 16 | "type": "example_resource", 17 | "labels": { 18 | "example": "label" 19 | } 20 | }, 21 | 22 | "resource_type_display_name": "Example Resource Type", 23 | "resource_id": "12345", 24 | "resource_display_name": "Example Resource", 25 | "resource_name": "projects/12345/example_resources/12345", 26 | 27 | "metric": { 28 | "type": "test.googleapis.com/metric", 29 | "displayName": "Test Metric", 30 | "labels": { 31 | "example": "label" 32 | } 33 | }, 34 | 35 | "metadata": { 36 | "system_labels": { 37 | "example": "label" 38 | }, 39 | "user_labels": { 40 | "example": "label" 41 | } 42 | }, 43 | 44 | "policy_name": "projects/12345/alertPolicies/12345", 45 | "policy_user_labels": { 46 | "example": "label" 47 | }, 48 | 49 | "documentation": "Test documentation", 50 | 51 | "condition": { 52 | "name": "projects/12345/alertPolicies/12345/conditions/12345", 53 | "displayName": "Example condition", 54 | 55 | "conditionThreshold": { 56 | "filter": "metric.type=\"test.googleapis.com/metric\" resource.type=\"example_resource\"", 57 | "comparison": "COMPARISON_GT", 58 | "thresholdValue": 0.5, 59 | "duration": "0s", 60 | "trigger": { 61 | "count": 1 62 | } 63 | } 64 | }, 65 | 66 | "condition_name": "Example condition", 67 | "threshold_value": "0.5" 68 | } 69 | } -------------------------------------------------------------------------------- /test/gitlab-job.json: -------------------------------------------------------------------------------- 1 | { 2 | "object_kind": "build", 3 | "ref": "0.1.2-1", 4 | "tag": true, 5 | "before_sha": "0000000000000000000000000000000000000000", 6 | "sha": "fc21b3776d62ef2ddc1db323e50f5d4c0090f4d3", 7 | "build_id": 3855491, 8 | "build_name": "rollback:cluster", 9 | "build_stage": "rollback", 10 | "build_status": "manual", 11 | "build_created_at": "2022-02-01T15:06:00.854Z", 12 | "build_started_at": null, 13 | "build_finished_at": null, 14 | "build_duration": 20.3, 15 | "build_queued_duration": null, 16 | "build_allow_failure": true, 17 | "build_failure_reason": "unknown_failure", 18 | "pipeline_id": 826389, 19 | "runner": { 20 | "id": 815, 21 | "description": "shared-prod-env", 22 | "runner_type": "instance_type", 23 | "active": true, 24 | "is_shared": true, 25 | "tags": [ 26 | "prodenv-k8s", 27 | "prodenv", 28 | "psp", 29 | "prodenv-bi", 30 | "prod.env" 31 | ] 32 | }, 33 | "project_id": 4249, 34 | "project_name": "SRE / events", 35 | "user": { 36 | "id": 72, 37 | "name": "Some User", 38 | "username": "some.user", 39 | "avatar_url": "https://secure.gravatar.com/avatar/044638361df43cc74637cf14926e92d9?s=80\u0026d=identicon", 40 | "email": "some@email.com" 41 | }, 42 | "commit": { 43 | "id": 826389, 44 | "sha": "fc21b3776d62ef2ddc1db323e50f5d4c0090f4d3", 45 | "message": "Update .gitlab-ci.yml", 46 | "author_name": "Some User", 47 | "author_email": "some@email.com", 48 | "author_url": "https://gitlab.com/some.user", 49 | "status": "success", 50 | "duration": 140, 51 | "started_at": "2022-01-31T16:33:02.926Z", 52 | "finished_at": "2022-01-31T16:35:50.817Z" 53 | }, 54 | "repository": { 55 | "name": "events", 56 | "url": "git@gitlab.com:sre/events.git", 57 | "description": "Events is about notifications that happens in different tier of infrastructure, including Kubernetes, Alertmanager", 58 | "homepage": "https://gitlab.com/sre/events", 59 | "git_http_url": "https://gitlab.com/sre/events.git", 60 | "git_ssh_url": "git@gitlab.com:sre/events.git", 61 | "visibility_level": 0 62 | }, 63 | "environment": null 64 | } -------------------------------------------------------------------------------- /slack.selector: -------------------------------------------------------------------------------- 1 | {{- define "render"}}{{printf "%s\n" (getEnv .)}}{{end}} 2 | {{- define "rules"}} 3 | {{- if eq .type "K8sEvent"}}{{template "render" "EVENTS_SLACK_OUT_CHANNEL_EVENTS"}}{{end}} 4 | {{- if and (eq .type "KubeEvent") (eq .data.Type "Error")}}{{/* template "render" "EVENTS_SLACK_OUT_CHANNEL_ALERTS" */}}{{end}} 5 | {{- if and (eq .type "KubeEvent") (ne .data.Type "Error")}}{{/* template "render" "EVENTS_SLACK_OUT_CHANNEL_EVENTS" */}}{{end}} 6 | {{- if eq .type "GitlabEvent"}}{{template "render" "EVENTS_SLACK_OUT_CHANNEL_EVENTS"}}{{end}} 7 | {{- if eq .type "TeamcityEvent"}}{{template "render" "EVENTS_SLACK_OUT_CHANNEL_EVENTS"}}{{end}} 8 | {{- if eq .type "AlertmanagerEvent"}}{{template "render" "EVENTS_SLACK_OUT_CHANNEL_EVENTS"}}{{end}} 9 | {{- if eq .type "DataDogEvent"}} 10 | {{- if (.data.alert.title | regexMatch ".*TEST.*")}} 11 | {{- template "render" "EVENTS_SLACK_OUT_CHANNEL_TEST"}} 12 | {{- else}} 13 | {{- if (eq .data.alert.transition "No data")}} 14 | {{- template "render" "EVENTS_SLACK_OUT_CHANNEL_ALERTS"}} 15 | {{- else}} 16 | {{- template "render" "EVENTS_SLACK_OUT_CHANNEL_ANOMALY"}} 17 | {{- end}} 18 | {{- end}} 19 | {{- end}} 20 | {{- if eq .type "Site24x7Event"}}{{template "render" "EVENTS_SLACK_OUT_CHANNEL_EVENTS"}}{{end}} 21 | {{- if eq .type "CloudflareEvent"}}{{template "render" "EVENTS_SLACK_OUT_CHANNEL_EVENTS"}}{{end}} 22 | {{- if eq .type "GoogleEvent"}}{{template "render" "EVENTS_SLACK_OUT_CHANNEL_EVENTS"}}{{end}} 23 | {{- if eq .type "AWSEvent"}}{{template "render" "EVENTS_SLACK_OUT_CHANNEL_EVENTS"}}{{end}} 24 | {{- if eq .type "VCenterEvent"}}{{template "render" "EVENTS_SLACK_OUT_CHANNEL_EVENTS"}}{{end}} 25 | {{- if eq .type "ZabbixEvent" -}} 26 | {{- if or (eq .data.EventNSeverity "5") (eq .data.EventNSeverity "4") -}} 27 | {{template "render" "EVENTS_SLACK_OUT_CHANNEL_ZABBIX"}} 28 | {{- end -}} 29 | {{- end -}} 30 | {{- if and (eq .type "ObserviumEvent") (ne .data.ALERT_STATE "ALERT REMINDER")}}{{template "render" "EVENTS_SLACK_OUT_CHANNEL_EVENTS"}}{{end}} 31 | {{- if eq .type "NomadEvent"}}{{template "render" "EVENTS_SLACK_OUT_CHANNEL_EVENTS"}}{{end}} 32 | {{- end}} 33 | {{- define "slack-selector"}}{{template "rules" .}}{{end}} -------------------------------------------------------------------------------- /test/aws.ec2.volume.json: -------------------------------------------------------------------------------- 1 | { 2 | "account": "157576696349", 3 | "detail": { 4 | "awsRegion": "eu-central-1", 5 | "eventCategory": "Management", 6 | "eventID": "ca40438c-de7f-4114-8305-f516a30feabe", 7 | "eventName": "AttachVolume", 8 | "eventSource": "ec2.amazonaws.com", 9 | "eventTime": "2022-05-20T12:44:32Z", 10 | "eventType": "AwsApiCall", 11 | "eventVersion": "1.08", 12 | "managementEvent": true, 13 | "readOnly": false, 14 | "recipientAccountId": "157576696349", 15 | "requestID": "dfd36cbe-9c74-4097-b7a0-d498c4ec74ea", 16 | "requestParameters": { 17 | "deleteOnTermination": false, 18 | "device": "/dev/xvdcr", 19 | "instanceId": "i-05b6827827cbae830", 20 | "volumeId": "vol-070770e03635b13e7" 21 | }, 22 | "responseElements": { 23 | "attachTime": 1653050672014, 24 | "deleteOnTermination": false, 25 | "device": "/dev/xvdcr", 26 | "instanceId": "i-05b6827827cbae830", 27 | "requestId": "dfd36cbe-9c74-4097-b7a0-d498c4ec74ea", 28 | "status": "attaching", 29 | "volumeId": "vol-070770e03635b13e7" 30 | }, 31 | "sourceIPAddress": "eks.amazonaws.com", 32 | "userAgent": "eks.amazonaws.com", 33 | "userIdentity": { 34 | "accountId": "157576696349", 35 | "arn": "arn:aws:sts::157576696349:assumed-role/eksctl-bi-cluster-ServiceRole-1UVAASNN0I53U/1652473055054864119", 36 | "invokedBy": "eks.amazonaws.com", 37 | "principalId": "AROASJMCNSIOYFMYI7LDP:1652473055054864119", 38 | "sessionContext": { 39 | "attributes": { 40 | "creationDate": "2022-05-20T12:31:04Z", 41 | "mfaAuthenticated": "false" 42 | }, 43 | "sessionIssuer": { 44 | "accountId": "157576696349", 45 | "arn": "arn:aws:iam::157576696349:role/eksctl-bi-cluster-ServiceRole-1UVAASNN0I53U", 46 | "principalId": "AROASJMCNSIOYFMYI7LDP", 47 | "type": "Role", 48 | "userName": "eksctl-bi-cluster-ServiceRole-1UVAASNN0I53U" 49 | }, 50 | "webIdFederationData": {} 51 | }, 52 | "type": "AssumedRole" 53 | } 54 | }, 55 | "detail-type": "AWS API Call via CloudTrail", 56 | "id": "cb6f1474-8ac4-b2c2-3dc9-855bbc1cbbf5", 57 | "region": "eu-central-1", 58 | "resources": [], 59 | "source": "aws.ec2", 60 | "time": "2022-05-20T12:44:32Z", 61 | "version": "0" 62 | } -------------------------------------------------------------------------------- /certs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function stdGenerateCertificates() { 4 | 5 | local SERVICE="$1" 6 | local NAMESPACE="$2" 7 | local KEY_FILE="$3" 8 | local CRT_FILE="$4" 9 | 10 | if [[ "$SERVICE" == "" ]] || [[ "$NAMESPACE" == "" ]]; then 11 | echo "Namespace/Service is not specified. Generation skipped." 12 | return 13 | fi 14 | 15 | if [[ "$KEY_FILE" == "" ]] || [[ "$CRT_FILE" == "" ]]; then 16 | echo "Key/Crt file name is not specified. Generation skipped." 17 | return 18 | fi 19 | 20 | tmpdir=$(mktemp -d) 21 | 22 | cat <>"${tmpdir}/csr.conf" 23 | [req] 24 | req_extensions = v3_req 25 | distinguished_name = req_distinguished_name 26 | [req_distinguished_name] 27 | [ v3_req ] 28 | basicConstraints = CA:FALSE 29 | keyUsage = nonRepudiation, digitalSignature, keyEncipherment 30 | extendedKeyUsage = serverAuth 31 | subjectAltName = @alt_names 32 | [alt_names] 33 | DNS.1 = ${SERVICE} 34 | DNS.2 = ${SERVICE}.${NAMESPACE} 35 | DNS.3 = ${SERVICE}.${NAMESPACE}.svc 36 | DNS.4 = ${SERVICE}.${NAMESPACE}.svc.cluster.local 37 | EOF 38 | 39 | local __out="" 40 | 41 | openssl genrsa -out "${KEY_FILE}" 4096 >__openSsl.out 2>&1 42 | __out=$(cat __openSsl.out) 43 | if [[ ! "$?" -eq 0 ]]; then 44 | echo "$__out" 45 | return 1 46 | else 47 | echo "'openssl genrsa' output:\n$__out" 48 | fi 49 | 50 | openssl req -new -key "${KEY_FILE}" -subj "/CN=${SERVICE}.${NAMESPACE}.svc.cluster.local" -out "${tmpdir}/${SERVICE}.csr" -config "${tmpdir}/csr.conf" >__openSsl.out 2>&1 51 | __out=$(cat __openSsl.out) 52 | if [[ ! "$?" -eq 0 ]]; then 53 | echo "$__out" 54 | return 1 55 | else 56 | echo "'openssl req -new -key' output:\n$__out" 57 | fi 58 | 59 | openssl x509 -signkey "${KEY_FILE}" -in "${tmpdir}/${SERVICE}.csr" -req -days 365 -out "${CRT_FILE}" >__openSsl.out 2>&1 60 | __out=$(cat __openSsl.out) 61 | if [[ ! "$?" -eq 0 ]]; then 62 | echo "$__out" 63 | return 1 64 | else 65 | echo "'openssl x509 -signkey' output:\n$__out" 66 | fi 67 | 68 | rm -rf __openSsl.out 69 | } 70 | 71 | stdGenerateCertificates "events" "sre" "events.key" "events.crt" 72 | 73 | echo "Crt..." 74 | cat "events.crt" | base64 | tr -d '\n' 75 | echo "" 76 | echo "Key..." 77 | cat "events.key" | base64 | tr -d '\n' 78 | echo "" 79 | echo "Bundle..." 80 | cat "events.crt" | base64 | tr -d '\n' 81 | echo "" -------------------------------------------------------------------------------- /test/datadog-recovered.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "6430249883109262377", 3 | "date": 1647535626000, 4 | "last_updated": 1647535626000, 5 | "link": "https://app.datadoghq.eu/event/event?id=6430249883109262377", 6 | "priority": "normal", 7 | "snapshot": "https://p.datadoghq.eu/snapshot/view/dd-snapshots-eu1-prod/org_1000060640/2022-03-17/06c559becf60b1b86d7240564fb81765856101f83.png", 8 | "event": { 9 | "type": "query_alert_monitor", 10 | "msg": "%%%\n@webhook-events\n\nHI\n\nTest notification triggered by some.user@email.com.\n\n[![Metric Graph](https://p.datadoghq.eu/snapshot/view/dd-snapshots-eu1-prod/org_1000060640/2022-03-17/06c559becf60b1b86d7240564fb81765856101f8.png)](https://app.datadoghq.eu/monitors/4883885?to_ts=1647535925000&from_ts=1647534725000)\n\n**sre.quality** over ***** was **>= 75.0** on average during the **last 5m**.\n\nThe monitor was last triggered at Thu Mar 17 2022 16:47:05 UTC.\n\n- - -\n\n[[Monitor Status](https://app.datadoghq.eu/monitors/4883885?to_ts=1647535925000&from_ts=1647534725000)] \u00b7 [[Edit Monitor](https://app.datadoghq.eu/monitors#4883885/edit)]\n%%%", 11 | "title": "[P1] [Recovered on ] [TEST] Test" 12 | }, 13 | "alert": { 14 | "id": "155619019", 15 | "metric": "sre.quality", 16 | "priority": "P1", 17 | "query": "avg(last_5m):avg:sre.availability{service_from:site24x7.com} by {service} < 50", 18 | "scope": "", 19 | "status": "sre.availability over *** was >= 55.0 on average during the last 5m**.", 20 | "title": "[TEST] Test", 21 | "transition": "Recovered", 22 | "type": "success" 23 | }, 24 | "incident": { 25 | "title": "null" 26 | }, 27 | "metric": { 28 | "namespace": "sre" 29 | }, 30 | "security": { 31 | "rule_name": "null" 32 | }, 33 | "org": { 34 | "id": "1000060640", 35 | "name": "Some" 36 | }, 37 | "tags": "monitor,sre-anomaly:quality,some-othertag", 38 | "text_only_msg": "\n@webhook-events\n\nHI\n\nTest notification triggered by some.user@email.com.\n\nsre.quality over * was >= 75.0 on average during the last 5m.\n\nMetric value: 0.0\n\nMetric Graph: https://app.datadoghq.eu/monitors/4883885?to_ts=1647535925000&from_ts=1647534725000 \u00b7 Monitor Status: https://app.datadoghq.eu/monitors/4883885? \u00b7 Edit Monitor: https://app.datadoghq.eu/monitors#4883885/edit \u00b7 Event URL: https://app.datadoghq.eu/event/event?id=6430249883109262377", 39 | "user": "null", 40 | "username": "" 41 | } -------------------------------------------------------------------------------- /test/aws.ec2.network.json: -------------------------------------------------------------------------------- 1 | { 2 | "account": "157576696349", 3 | "detail": { 4 | "awsRegion": "eu-central-1", 5 | "eventCategory": "Management", 6 | "eventID": "8cfebd47-9ed1-4ae4-a480-822532594df6", 7 | "eventName": "AttachNetworkInterface", 8 | "eventSource": "ec2.amazonaws.com", 9 | "eventTime": "2022-05-20T13:28:09Z", 10 | "eventType": "AwsApiCall", 11 | "eventVersion": "1.08", 12 | "managementEvent": true, 13 | "readOnly": false, 14 | "recipientAccountId": "157576696349", 15 | "requestID": "9e9930ea-000a-40a7-a9aa-410d9da6970a", 16 | "requestParameters": { 17 | "deviceIndex": 2, 18 | "instanceId": "i-0a73fa680fa5f76b2", 19 | "networkInterfaceId": "eni-08cbd043527f58cbe" 20 | }, 21 | "responseElements": { 22 | "attachmentId": "eni-attach-0e7a5cd8d7d080de4", 23 | "requestId": "9e9930ea-000a-40a7-a9aa-410d9da6970a" 24 | }, 25 | "sourceIPAddress": "52.28.127.184", 26 | "tlsDetails": { 27 | "cipherSuite": "ECDHE-RSA-AES128-GCM-SHA256", 28 | "clientProvidedHostHeader": "ec2.eu-central-1.amazonaws.com", 29 | "tlsVersion": "TLSv1.2" 30 | }, 31 | "userAgent": "aws-sdk-go/1.21.7 (go1.12.17; linux; amd64)", 32 | "userIdentity": { 33 | "accessKeyId": "WWWWSJMCNSIOTYLZQANY", 34 | "accountId": "157576696349", 35 | "arn": "arn:aws:sts::157576696349:assumed-role/eksctl-bi-nodegroup-default-NodeInstanceRole/i-0a73fa680fa5f76b2", 36 | "principalId": "AROASJMCNSIO3BGXXRURK:i-0a73fa680fa5f76b2", 37 | "sessionContext": { 38 | "attributes": { 39 | "creationDate": "2022-05-20T12:55:02Z", 40 | "mfaAuthenticated": "false" 41 | }, 42 | "ec2RoleDelivery": "1.0", 43 | "sessionIssuer": { 44 | "accountId": "157576696349", 45 | "arn": "arn:aws:iam::157576696349:role/eksctl-bi-nodegroup-default-NodeInstanceRole", 46 | "principalId": "AROASJMCNSIO3BGXXRURK", 47 | "type": "Role", 48 | "userName": "eksctl-bi-nodegroup-default-NodeInstanceRole" 49 | }, 50 | "webIdFederationData": {} 51 | }, 52 | "type": "AssumedRole" 53 | } 54 | }, 55 | "detail-type": "AWS API Call via CloudTrail", 56 | "id": "5e10008c-3f0d-f99b-6481-bd8bb46d2328", 57 | "region": "eu-central-1", 58 | "resources": [], 59 | "source": "aws.ec2", 60 | "time": "2022-05-20T13:28:09Z", 61 | "version": "0" 62 | } -------------------------------------------------------------------------------- /test/aws.elasticloadbalancing.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0", 3 | "id": "e1ecb274-cf53-1e74-6533-ac0ce824d2eb", 4 | "detail-type": "AWS API Call via CloudTrail", 5 | "source": "aws.elasticloadbalancing", 6 | "account": "098011104626", 7 | "time": "2022-05-14T17:06:32Z", 8 | "region": "eu-west-2", 9 | "resources": [], 10 | "detail": { 11 | "eventVersion": "1.08", 12 | "userIdentity": { 13 | "type": "AssumedRole", 14 | "principalId": "AWWARNVW2GLZNXEMOGPGJ:ecs-service-scheduler", 15 | "arn": "arn:aws:sts::098011104626:assumed-role/AWSServiceRoleForECS/ecs-service-scheduler", 16 | "accountId": "098011104626", 17 | "accessKeyId": "AWIARNVW2GLZGPNU4WNW", 18 | "sessionContext": { 19 | "sessionIssuer": { 20 | "type": "Role", 21 | "principalId": "AWWARNVW2GLZNXEMOGPGJ", 22 | "arn": "arn:aws:iam::098011104626:role/aws-service-role/ecs.amazonaws.com/AWSServiceRoleForECS", 23 | "accountId": "098011104626", 24 | "userName": "AWSServiceRoleForECS" 25 | }, 26 | "webIdFederationData": {}, 27 | "attributes": { 28 | "creationDate": "2022-05-14T17:06:32Z", 29 | "mfaAuthenticated": "false" 30 | } 31 | }, 32 | "invokedBy": "ecs.amazonaws.com" 33 | }, 34 | "eventTime": "2022-05-14T17:06:32Z", 35 | "eventSource": "elasticloadbalancing.amazonaws.com", 36 | "eventName": "DeregisterTargets", 37 | "awsRegion": "eu-west-2", 38 | "sourceIPAddress": "ecs.amazonaws.com", 39 | "userAgent": "ecs.amazonaws.com", 40 | "requestParameters": { 41 | "targetGroupArn": "arn:aws:elasticloadbalancing:eu-west-2:098011104626:targetgroup/kafka-exporter-prod-tg/fd84b6820271c649", 42 | "targets": [ 43 | { 44 | "id": "i-09ae152cd23f2e331", 45 | "port": 9308 46 | } 47 | ] 48 | }, 49 | "responseElements": null, 50 | "requestID": "e1635181-f7ff-43c2-9367-e8a2082f20af", 51 | "eventID": "baa3baec-1e71-4123-9cdc-a5e7cfa4b560", 52 | "readOnly": false, 53 | "eventType": "AwsApiCall", 54 | "apiVersion": "2015-12-01", 55 | "managementEvent": true, 56 | "recipientAccountId": "098011104626", 57 | "eventCategory": "Management" 58 | } 59 | } -------------------------------------------------------------------------------- /test/aws.ec2.tags.json: -------------------------------------------------------------------------------- 1 | { 2 | "account": "157576696349", 3 | "detail": { 4 | "awsRegion": "eu-central-1", 5 | "eventCategory": "Management", 6 | "eventID": "0ce21697-9760-49ba-a0e2-6acfa390b7ce", 7 | "eventName": "CreateTags", 8 | "eventSource": "ec2.amazonaws.com", 9 | "eventTime": "2022-05-20T13:28:08Z", 10 | "eventType": "AwsApiCall", 11 | "eventVersion": "1.08", 12 | "managementEvent": true, 13 | "readOnly": false, 14 | "recipientAccountId": "157576696349", 15 | "requestID": "160ecb7c-298c-449c-b15c-3731b24390fa", 16 | "requestParameters": { 17 | "resourcesSet": { 18 | "items": [ 19 | { 20 | "resourceId": "eni-08cbd043527f58cbe" 21 | } 22 | ] 23 | }, 24 | "tagSet": { 25 | "items": [ 26 | { 27 | "key": "node.k8s.amazonaws.com/instance_id", 28 | "value": "i-0a73fa680fa5f76b2" 29 | } 30 | ] 31 | } 32 | }, 33 | "responseElements": { 34 | "_return": true, 35 | "requestId": "160ecb7c-298c-449c-b15c-3731b24390fa" 36 | }, 37 | "sourceIPAddress": "52.28.127.184", 38 | "tlsDetails": { 39 | "cipherSuite": "ECDHE-RSA-AES128-GCM-SHA256", 40 | "clientProvidedHostHeader": "ec2.eu-central-1.amazonaws.com", 41 | "tlsVersion": "TLSv1.2" 42 | }, 43 | "userAgent": "aws-sdk-go/1.21.7 (go1.12.17; linux; amd64)", 44 | "userIdentity": { 45 | "accessKeyId": "WWWWSJMCNSIOTYLZQANY", 46 | "accountId": "157576696349", 47 | "arn": "arn:aws:sts::157576696349:assumed-role/eksctl-bi-nodegroup-default-NodeInstanceRole/i-0a73fa680fa5f76b2", 48 | "principalId": "AROASJMCNSIO3BGXXRURK:i-0a73fa680fa5f76b2", 49 | "sessionContext": { 50 | "attributes": { 51 | "creationDate": "2022-05-20T12:55:02Z", 52 | "mfaAuthenticated": "false" 53 | }, 54 | "ec2RoleDelivery": "1.0", 55 | "sessionIssuer": { 56 | "accountId": "157576696349", 57 | "arn": "arn:aws:iam::157576696349:role/eksctl-bi-nodegroup-default-NodeInstanceRole", 58 | "principalId": "AROASJMCNSIO3BGXXRURK", 59 | "type": "Role", 60 | "userName": "eksctl-bi-nodegroup-default-NodeInstanceRole" 61 | }, 62 | "webIdFederationData": {} 63 | }, 64 | "type": "AssumedRole" 65 | } 66 | }, 67 | "detail-type": "AWS API Call via CloudTrail", 68 | "id": "2cbbeefb-bb24-231d-e46f-10d6e2805187", 69 | "region": "eu-central-1", 70 | "resources": [], 71 | "source": "aws.ec2", 72 | "time": "2022-05-20T13:28:08Z", 73 | "version": "0" 74 | } -------------------------------------------------------------------------------- /test/datadog-test.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": { 3 | "alert": { 4 | "id": "155619019", 5 | "metric": "sre.quality", 6 | "priority": "P1", 7 | "query": "avg(last_5m):avg:sre.availability{service_from:site24x7.com} by {service} < 50", 8 | "scope": "product:pp", 9 | "status": "sre.availability over *** was \u003c 50.0 on average during the last 5m**.", 10 | "title": "[TEST] Average SRE availability \u003c 50% on product:pp", 11 | "transition": "Triggered", 12 | "type": "error" 13 | }, 14 | "date": 1658322714000, 15 | "event": { 16 | "msg": "%%%\nAverage over SRE quality over [pp] is less than 70%\n@webhook-sre-events\n\nTest notification triggered by tsv.\n\n[![Metric Graph](https://p.datadoghq.eu/snapshot/view/dd-snapshots-eu1-prod/org_1000060640/2022-07-20/c88354658a86ac96833bfa4214aa4fb2abe6de8b.png)](https://app.datadoghq.eu/monitors/5561901?to_ts=1658323014000\u0026group=product%3App\u0026from_ts=1658321814000)\n\n**sre.quality** over **product:pp** was **\u003c 70.0** on average during the **last 5m**.\n\nThe monitor was last triggered at Wed Jul 20 2022 13:11:54 UTC.\n\n- - -\n\n[[Monitor Status](https://app.datadoghq.eu/monitors/5561901?to_ts=1658323014000\u0026group=product%3App\u0026from_ts=1658321814000)] · [[Edit Monitor](https://app.datadoghq.eu/monitors#5561901/edit)]\n%%%", 17 | "title": "[P1] [Triggered on {product:pp}] [TEST] Average SRE quality \u003c 70%", 18 | "type": "query_alert_monitor" 19 | }, 20 | "id": "6611227196640466337", 21 | "incident": { 22 | "title": "null" 23 | }, 24 | "last_updated": 1658322714000, 25 | "link": "https://app.datadoghq.eu/event/event?id=6611227196640466337", 26 | "metric": { 27 | "namespace": "sre" 28 | }, 29 | "org": { 30 | "id": "1000060640", 31 | "name": "Some" 32 | }, 33 | "priority": "normal", 34 | "security": { 35 | "rule_name": "null" 36 | }, 37 | "snapshot": "https://stsci-opo.org/STScI-01G8GZHGM987XCHJZ846YRHWGF.tif", 38 | "tags": "monitor,product:pp,sre-anomaly:availability,!service_from", 39 | "text_only_msg":"\nAverage over SRE quality over [pp] is less than 70%\n@webhook-sre-events\n\nTest notification triggered by tsv.\n\nsre.quality over product:pp was \u003c 70.0 on average during the last 5m.\n\nMetric value: 0.0\n\nMetric Graph: https: //app.datadoghq.eu/monitors/5561901?to_ts=1658323014000\u0026group=product%3App\u0026from_ts=1658321814000 · Monitor Status: https://app.datadoghq.eu/monitors/5561901?group=product%3App · Edit Monitor: https://app.datadoghq.eu/monitors#5561901/edit · Event URL: https://app.datadoghq.eu/event/event?id=6611227196640466337","user":"null","username":"" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /test/datadog-triggered.json: -------------------------------------------------------------------------------- 1 | { 2 | "alert": { 3 | "id": "155619019", 4 | "metric": "sre.quality", 5 | "priority": "P1", 6 | "query": "avg(last_5m):avg:sre.availability{service_from:site24x7.com} by {service} < 50", 7 | "scope": "product:pp", 8 | "status": "sre.availability over *** was \u003c 50.0 on average during the last 5m**.", 9 | "title": "[TEST] Average SRE availability \u003c 50% on product:pp", 10 | "transition": "Triggered", 11 | "type": "error" 12 | }, 13 | "date": 1658322714000, 14 | "event": { 15 | "msg": "%%%\nAverage over SRE quality over [pp] is less than 70%\n@webhook-sre-events\n\nTest notification triggered by tsv.\n\n[![Metric Graph](https://p.datadoghq.eu/snapshot/view/dd-snapshots-eu1-prod/org_1000060640/2022-07-20/c88354658a86ac96833bfa4214aa4fb2abe6de8b.png)](https://app.datadoghq.eu/monitors/5561901?to_ts=1658323014000\u0026group=product%3App\u0026from_ts=1658321814000)\n\n**sre.quality** over **product:pp** was **\u003c 70.0** on average during the **last 5m**.\n\nThe monitor was last triggered at Wed Jul 20 2022 13:11:54 UTC.\n\n- - -\n\n[[Monitor Status](https://app.datadoghq.eu/monitors/5561901?to_ts=1658323014000\u0026group=product%3App\u0026from_ts=1658321814000)] · [[Edit Monitor](https://app.datadoghq.eu/monitors#5561901/edit)]\n%%%", 16 | "title": "[P1] [Triggered on {product:pp}] [TEST] Average SRE quality \u003c 70%", 17 | "type": "query_alert_monitor" 18 | }, 19 | "id": "6611227196640466337", 20 | "incident": { 21 | "title": "null" 22 | }, 23 | "last_updated": 1658322714000, 24 | "link": "https://app.datadoghq.eu/event/event?id=6611227196640466337", 25 | "metric": { 26 | "namespace": "sre" 27 | }, 28 | "org": { 29 | "id": "1000060640", 30 | "name": "Some" 31 | }, 32 | "priority": "normal", 33 | "security": { 34 | "rule_name": "null" 35 | }, 36 | "snapshot": "https://p.datadoghq.eu/snapshot/view/dd-snapshots-eu1-prod/org_1000060640/2022-07-20/c88354658a86ac96833bfa4214aa4fb2abe6de8b.png", 37 | "tags": "monitor,product:pp,sre-anomaly:availability,!service_from", 38 | "text_only_msg":"\nAverage over SRE quality over [pp] is less than 70%\n@webhook-sre-events\n\nTest notification triggered by tsv.\n\nsre.quality over product:pp was \u003c 70.0 on average during the last 5m.\n\nMetric value: 0.0\n\nMetric Graph: https: //app.datadoghq.eu/monitors/5561901?to_ts=1658323014000\u0026group=product%3App\u0026from_ts=1658321814000 · Monitor Status: https://app.datadoghq.eu/monitors/5561901?group=product%3App · Edit Monitor: https://app.datadoghq.eu/monitors#5561901/edit · Event URL: https://app.datadoghq.eu/event/event?id=6611227196640466337","user":"null","username":""} 39 | -------------------------------------------------------------------------------- /test/aws.route53.json: -------------------------------------------------------------------------------- 1 | { 2 | "account": "098093904626", 3 | "detail": { 4 | "additionalEventData": { 5 | "Note": "Do not use to reconstruct hosted zone" 6 | }, 7 | "apiVersion": "2013-04-01", 8 | "awsRegion": "us-east-1", 9 | "eventCategory": "Management", 10 | "eventID": "368502eb-9b22-4f3d-8787-4470877211f8", 11 | "eventName": "ChangeResourceRecordSets", 12 | "eventSource": "route53.amazonaws.com", 13 | "eventTime": "2022-05-18T13:33:29Z", 14 | "eventType": "AwsApiCall", 15 | "eventVersion": "1.08", 16 | "managementEvent": true, 17 | "readOnly": false, 18 | "recipientAccountId": "098093904626", 19 | "requestID": "c705dc59-a35b-4592-b83f-c57738f6aafb", 20 | "requestParameters": { 21 | "changeBatch": { 22 | "changes": [ 23 | { 24 | "action": "UPSERT", 25 | "resourceRecordSet": { 26 | "name": "apiportal.invaxasystems.net", 27 | "resourceRecords": [ 28 | { 29 | "value": "18.170.51.73" 30 | }, 31 | { 32 | "value": "13.41.70.178" 33 | } 34 | ], 35 | "tTL": 60, 36 | "type": "A" 37 | } 38 | } 39 | ], 40 | "comment": "Automatic DNS update" 41 | }, 42 | "hostedZoneId": "Z0737640100OEZQN74GN4" 43 | }, 44 | "responseElements": { 45 | "changeInfo": { 46 | "comment": "Automatic DNS update", 47 | "id": "/change/C02974592SK3C68OT8S7F", 48 | "status": "PENDING", 49 | "submittedAt": "May 18, 2022 1:33:29 PM" 50 | } 51 | }, 52 | "sourceIPAddress": "18.170.78.200", 53 | "tlsDetails": { 54 | "cipherSuite": "ECDHE-RSA-AES128-GCM-SHA256", 55 | "clientProvidedHostHeader": "route53.amazonaws.com", 56 | "tlsVersion": "TLSv1.2" 57 | }, 58 | "userAgent": "Boto3/1.18.55 Python/3.6.15 Linux/4.14.255-273-220.498.amzn2.x86_64 exec-env/AWS_Lambda_python3.6 Botocore/1.21.56", 59 | "userIdentity": { 60 | "accessKeyId": "WWWWSJMCNSIOTYLZQANY", 61 | "accountId": "098093904626", 62 | "arn": "arn:aws:sts::098093904626:assumed-role/invaxa-staging-traefik-lambda-role/invaxa-prod-TraefikDNS", 63 | "principalId": "AROARNVW2GLZAXTKCPJKH:invaxa-prod-TraefikDNS", 64 | "sessionContext": { 65 | "attributes": { 66 | "creationDate": "2022-05-18T13:33:26Z", 67 | "mfaAuthenticated": "false" 68 | }, 69 | "sessionIssuer": { 70 | "accountId": "098093904626", 71 | "arn": "arn:aws:iam::098093904626:role/invaxa-staging-traefik-lambda-role", 72 | "principalId": "AROARNVW2GLZAXTKCPJKH", 73 | "type": "Role", 74 | "userName": "invaxa-staging-traefik-lambda-role" 75 | }, 76 | "webIdFederationData": {} 77 | }, 78 | "type": "AssumedRole" 79 | } 80 | }, 81 | "detail-type": "AWS API Call via CloudTrail", 82 | "region": "us-east-1", 83 | "source": "aws.route53", 84 | "time": "2022-05-18T13:33:29Z", 85 | "version": "0" 86 | } -------------------------------------------------------------------------------- /input/pubsub.go: -------------------------------------------------------------------------------- 1 | package input 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "os" 7 | "sync" 8 | 9 | "cloud.google.com/go/pubsub" 10 | "github.com/devopsext/events/common" 11 | sreCommon "github.com/devopsext/sre/common" 12 | "github.com/devopsext/utils" 13 | "google.golang.org/api/option" 14 | ) 15 | 16 | type PubSubInputOptions struct { 17 | Credentials string 18 | ProjectID string 19 | Subscription string 20 | } 21 | 22 | type PubSubInput struct { 23 | options PubSubInputOptions 24 | client *pubsub.Client 25 | ctx context.Context 26 | processors *common.Processors 27 | eventer sreCommon.Eventer 28 | logger sreCommon.Logger 29 | meter sreCommon.Meter 30 | } 31 | 32 | func (ps *PubSubInput) Start(wg *sync.WaitGroup, outputs *common.Outputs) { 33 | wg.Add(1) 34 | go func(wg *sync.WaitGroup) { 35 | defer wg.Done() 36 | ps.logger.Info("Start pubsub input...") 37 | 38 | sub := ps.client.Subscription(ps.options.Subscription) 39 | ps.logger.Info("PubSub input is up. Listening...") 40 | 41 | err := sub.Receive(ps.ctx, func(ctx context.Context, m *pubsub.Message) { 42 | 43 | labels := make(map[string]string) 44 | labels["subscription"] = sub.String() 45 | labels["input"] = "pubsub" 46 | 47 | requests := ps.meter.Counter("pubsub", "requests", "Count of all pubsub input requests", labels, "input") 48 | errors := ps.meter.Counter("pubsub", "errors", "Count of all pubsub input errors", labels, "input") 49 | 50 | requests.Inc() 51 | 52 | var event common.Event 53 | if err := json.Unmarshal(m.Data, &event); err != nil { 54 | errors.Inc() 55 | return 56 | } 57 | 58 | p := ps.processors.Find(event.Type) 59 | if p == nil { 60 | ps.logger.Debug("PubSub processor is not found for %s", event.Type) 61 | return 62 | } 63 | 64 | event.SetLogger(ps.logger) 65 | 66 | err := p.HandleEvent(&event) 67 | if err != nil { 68 | errors.Inc() 69 | } 70 | }) 71 | 72 | if err != nil { 73 | ps.logger.Error(err) 74 | } 75 | }(wg) 76 | } 77 | 78 | func NewPubSubInput(options PubSubInputOptions, processors *common.Processors, observability *common.Observability) *PubSubInput { 79 | logger := observability.Logs() 80 | if utils.IsEmpty(options.Credentials) || utils.IsEmpty(options.ProjectID) || utils.IsEmpty(options.Subscription) { 81 | logger.Debug("PubSub input credentials, project ID or subscription is not defined. Skipped") 82 | return nil 83 | } 84 | 85 | var o option.ClientOption 86 | if _, err := os.Stat(options.Credentials); err == nil { 87 | o = option.WithCredentialsFile(options.Credentials) 88 | } else { 89 | o = option.WithCredentialsJSON([]byte(options.Credentials)) 90 | } 91 | 92 | ctx := context.Background() 93 | client, err := pubsub.NewClient(ctx, options.ProjectID, o) 94 | if err != nil { 95 | logger.Error(err) 96 | return nil 97 | } 98 | 99 | return &PubSubInput{ 100 | options: options, 101 | client: client, 102 | ctx: ctx, 103 | processors: processors, 104 | eventer: observability.Events(), 105 | logger: observability.Logs(), 106 | meter: observability.Metrics(), 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /processor/customjson.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "encoding/json" 5 | errPkg "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | 10 | "github.com/devopsext/events/common" 11 | sreCommon "github.com/devopsext/sre/common" 12 | "github.com/devopsext/utils" 13 | "github.com/prometheus/alertmanager/template" 14 | ) 15 | 16 | type CustomJsonProcessor struct { 17 | outputs *common.Outputs 18 | logger sreCommon.Logger 19 | meter sreCommon.Meter 20 | } 21 | 22 | type CustomJsonResponse struct { 23 | Message string 24 | } 25 | 26 | func CustomJsonProcessorType() string { 27 | return "CustomJson" 28 | } 29 | 30 | func (p *CustomJsonProcessor) EventType() string { 31 | return common.AsEventType(CustomJsonProcessorType()) 32 | } 33 | 34 | func (p *CustomJsonProcessor) HandleEvent(e *common.Event) error { 35 | return nil 36 | } 37 | 38 | func (p *CustomJsonProcessor) HandleHttpRequest(w http.ResponseWriter, r *http.Request) error { 39 | 40 | labels := make(map[string]string) 41 | labels["path"] = r.URL.Path 42 | labels["processor"] = p.EventType() 43 | 44 | requests := p.meter.Counter("customjson", "requests", "Count of all customjson processor requests", labels, "processor") 45 | requests.Inc() 46 | 47 | errors := p.meter.Counter("customjson", "errors", "Count of all customjson processor errors", labels, "processor") 48 | 49 | var body []byte 50 | if r.Body != nil { 51 | if data, err := ioutil.ReadAll(r.Body); err == nil { 52 | body = data 53 | } 54 | } 55 | 56 | if len(body) == 0 { 57 | errors.Inc() 58 | err := errPkg.New("empty body") 59 | p.logger.Error(err) 60 | http.Error(w, err.Error(), http.StatusBadRequest) 61 | return err 62 | } 63 | 64 | p.logger.Debug("Body => %s", body) 65 | 66 | var response *CustomJsonResponse 67 | errorString := "" 68 | data := template.Data{} 69 | if err := json.Unmarshal(body, &data); err != nil { 70 | errorString = err.Error() 71 | p.logger.Error("Can't decode body: %v", err) 72 | response = &CustomJsonResponse{ 73 | Message: errorString, 74 | } 75 | } else { 76 | 77 | response = &CustomJsonResponse{ 78 | Message: "OK", 79 | } 80 | } 81 | 82 | resp, err := json.Marshal(response) 83 | if err != nil { 84 | errors.Inc() 85 | p.logger.Error("Can't encode response: %v", err) 86 | http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError) 87 | return err 88 | } 89 | 90 | if _, err := w.Write(resp); err != nil { 91 | errors.Inc() 92 | p.logger.Error("Can't write response: %v", err) 93 | http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) 94 | return err 95 | } 96 | 97 | if !utils.IsEmpty(errorString) { 98 | errors.Inc() 99 | err := errPkg.New(errorString) 100 | p.logger.Error(errorString) 101 | http.Error(w, fmt.Sprint(errorString), http.StatusInternalServerError) 102 | return err 103 | } 104 | return nil 105 | } 106 | 107 | func NewCustomJsonProcessor(outputs *common.Outputs, observability *common.Observability) *CustomJsonProcessor { 108 | 109 | return &CustomJsonProcessor{ 110 | outputs: outputs, 111 | logger: observability.Logs(), 112 | meter: observability.Metrics(), 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /test/aws.elasticloadbalancing2.json: -------------------------------------------------------------------------------- 1 | { 2 | "account": "157576696349", 3 | "detail": { 4 | "apiVersion": "2012-06-01", 5 | "awsRegion": "eu-central-1", 6 | "eventCategory": "Management", 7 | "eventID": "f2668e3a-cfb5-47ae-80fc-eaae031ad1b2", 8 | "eventName": "RegisterInstancesWithLoadBalancer", 9 | "eventSource": "elasticloadbalancing.amazonaws.com", 10 | "eventTime": "2022-05-21T13:00:04Z", 11 | "eventType": "AwsApiCall", 12 | "eventVersion": "1.08", 13 | "managementEvent": true, 14 | "readOnly": false, 15 | "recipientAccountId": "157576696349", 16 | "requestID": "1789bd5e-ef2b-424e-b231-6068fb693f1a", 17 | "requestParameters": { 18 | "instances": [ 19 | { 20 | "instanceId": "i-05b6827827cbae830" 21 | } 22 | ], 23 | "loadBalancerName": "a206fc323729711eab43a06dc0a5ffa2" 24 | }, 25 | "responseElements": { 26 | "instances": [ 27 | { 28 | "instanceId": "i-07cbf43e745c5a752" 29 | }, 30 | { 31 | "instanceId": "i-0e922e216d91f8b71" 32 | }, 33 | { 34 | "instanceId": "i-0f4fe288fc4ab750e" 35 | }, 36 | { 37 | "instanceId": "i-05b6827827cbae830" 38 | }, 39 | { 40 | "instanceId": "i-0845c0bbc075d40fc" 41 | }, 42 | { 43 | "instanceId": "i-065a496cb26a7dcb6" 44 | }, 45 | { 46 | "instanceId": "i-07a51d94288c2c60d" 47 | }, 48 | { 49 | "instanceId": "i-06596ccb3f69f6b2f" 50 | }, 51 | { 52 | "instanceId": "i-06fcf6631752a735c" 53 | }, 54 | { 55 | "instanceId": "i-0814f6075c37d0a10" 56 | }, 57 | { 58 | "instanceId": "i-0a73fa680fa5f76b2" 59 | }, 60 | { 61 | "instanceId": "i-00a8a6ebcd041e8da" 62 | }, 63 | { 64 | "instanceId": "i-0155bec052b32cfe5" 65 | } 66 | ] 67 | }, 68 | "sourceIPAddress": "eks.amazonaws.com", 69 | "userAgent": "eks.amazonaws.com", 70 | "userIdentity": { 71 | "accessKeyId": "WWWWSJMCNSIOTYLZQANY", 72 | "accountId": "157576696349", 73 | "arn": "arn:aws:sts::157576696349:assumed-role/eksctl-bi-cluster-ServiceRole-1UVAASNN0I53U/1653084984187285429", 74 | "invokedBy": "eks.amazonaws.com", 75 | "principalId": "AROASJMCNSIOYFMYI7LDP:1653084984187285429", 76 | "sessionContext": { 77 | "attributes": { 78 | "creationDate": "2022-05-21T12:48:08Z", 79 | "mfaAuthenticated": "false" 80 | }, 81 | "sessionIssuer": { 82 | "accountId": "157576696349", 83 | "arn": "arn:aws:iam::157576696349:role/eksctl-bi-cluster-ServiceRole-1UVAASNN0I53U", 84 | "principalId": "AROASJMCNSIOYFMYI7LDP", 85 | "type": "Role", 86 | "userName": "eksctl-bi-cluster-ServiceRole-1UVAASNN0I53U" 87 | }, 88 | "webIdFederationData": {} 89 | }, 90 | "type": "AssumedRole" 91 | } 92 | }, 93 | "detail-type": "AWS API Call via CloudTrail", 94 | "id": "8491b445-b2ff-0a37-07a3-a1b978f1f6e8", 95 | "region": "eu-central-1", 96 | "resources": [], 97 | "source": "aws.elasticloadbalancing", 98 | "time": "2022-05-21T13:00:04Z", 99 | "version": "0" 100 | } -------------------------------------------------------------------------------- /output/collector.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "net" 5 | "strings" 6 | "sync" 7 | 8 | "github.com/devopsext/events/common" 9 | sreCommon "github.com/devopsext/sre/common" 10 | toolsRender "github.com/devopsext/tools/render" 11 | "github.com/devopsext/utils" 12 | ) 13 | 14 | type CollectorOutputOptions struct { 15 | Address string 16 | Message string 17 | } 18 | 19 | type CollectorOutput struct { 20 | wg *sync.WaitGroup 21 | options CollectorOutputOptions 22 | connection *net.UDPConn 23 | message *toolsRender.TextTemplate 24 | logger sreCommon.Logger 25 | meter sreCommon.Meter 26 | } 27 | 28 | func (c *CollectorOutput) Name() string { 29 | return "Collector" 30 | } 31 | 32 | func (c *CollectorOutput) Send(event *common.Event) { 33 | 34 | c.wg.Add(1) 35 | go func() { 36 | defer c.wg.Done() 37 | 38 | if event == nil { 39 | c.logger.Debug("Event is empty") 40 | return 41 | } 42 | 43 | b, err := c.message.RenderObject(event) 44 | if err != nil { 45 | c.logger.Error(err) 46 | return 47 | } 48 | 49 | message := strings.TrimSpace(string(b)) 50 | if utils.IsEmpty(message) { 51 | c.logger.Debug("Collector message is empty") 52 | return 53 | } 54 | 55 | labels := make(map[string]string) 56 | labels["event_channel"] = event.Channel 57 | labels["event_type"] = event.Type 58 | labels["collector_address"] = c.options.Address 59 | labels["output"] = c.Name() 60 | 61 | requests := c.meter.Counter("collector", "requests", "Count of all collector requests", labels, "output") 62 | requests.Inc() 63 | 64 | c.logger.Debug("Collector message => %s", message) 65 | 66 | _, err = c.connection.Write(b) 67 | if err != nil { 68 | errors := c.meter.Counter("collector", "errors", "Count of all collector errors", labels, "output") 69 | errors.Inc() 70 | c.logger.Error(err) 71 | } 72 | }() 73 | } 74 | 75 | func makeCollectorOutputConnection(address string, logger sreCommon.Logger) *net.UDPConn { 76 | 77 | if utils.IsEmpty(address) { 78 | logger.Debug("Collector address is not defined. Skipped.") 79 | return nil 80 | } 81 | 82 | localAddr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0") 83 | if err != nil { 84 | logger.Error(err) 85 | return nil 86 | } 87 | 88 | serverAddr, err := net.ResolveUDPAddr("udp", address) 89 | if err != nil { 90 | logger.Error(err) 91 | return nil 92 | } 93 | 94 | connection, err := net.DialUDP("udp", localAddr, serverAddr) 95 | if err != nil { 96 | logger.Error(err) 97 | return nil 98 | } 99 | 100 | return connection 101 | } 102 | 103 | func NewCollectorOutput(wg *sync.WaitGroup, options CollectorOutputOptions, 104 | templateOptions toolsRender.TemplateOptions, observability *common.Observability) *CollectorOutput { 105 | 106 | logger := observability.Logs() 107 | connection := makeCollectorOutputConnection(options.Address, logger) 108 | if connection == nil { 109 | logger.Error("no connection") 110 | return nil 111 | } 112 | 113 | messageOpts := toolsRender.TemplateOptions{ 114 | Name: "collector-message", 115 | Content: common.Content(options.Message), 116 | TimeFormat: templateOptions.TimeFormat, 117 | } 118 | message, err := toolsRender.NewTextTemplate(messageOpts, observability) 119 | if err != nil { 120 | logger.Error(err) 121 | return nil 122 | } 123 | 124 | return &CollectorOutput{ 125 | wg: wg, 126 | options: options, 127 | message: message, 128 | connection: connection, 129 | logger: logger, 130 | meter: observability.Metrics(), 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /input/nomad.go: -------------------------------------------------------------------------------- 1 | package input 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "time" 7 | 8 | "github.com/devopsext/events/common" 9 | "github.com/devopsext/events/processor" 10 | sreCommon "github.com/devopsext/sre/common" 11 | "github.com/devopsext/utils" 12 | nomad "github.com/hashicorp/nomad/api" 13 | ) 14 | 15 | type NomadInputOptions struct { 16 | Address string 17 | Token string 18 | Topics []string 19 | } 20 | 21 | type NomadInput struct { 22 | options NomadInputOptions 23 | client *nomad.Client 24 | ctx context.Context 25 | processors *common.Processors 26 | eventer sreCommon.Eventer 27 | logger sreCommon.Logger 28 | meter sreCommon.Meter 29 | // requests sreCommon.Counter 30 | // errors sreCommon.Counter 31 | } 32 | 33 | func (n *NomadInput) Start(wg *sync.WaitGroup, outputs *common.Outputs) { 34 | wg.Add(1) 35 | go func(wg *sync.WaitGroup) { 36 | defer wg.Done() 37 | 38 | p := n.processors.Find("NomadEvent").(*processor.NomadProcessor) 39 | if p == nil { 40 | n.logger.Debug("Nomad processor is not found for NomadEvent") 41 | return 42 | } 43 | 44 | n.logger.Info("Start nomad input...") 45 | 46 | topics := make(map[nomad.Topic][]string) 47 | for _, topic := range n.options.Topics { 48 | topics[nomad.Topic(topic)] = []string{"*"} 49 | } 50 | 51 | q := &nomad.QueryOptions{} 52 | 53 | stream := n.client.EventStream() 54 | 55 | for { 56 | eventCh, err := stream.Stream(n.ctx, topics, 0, q) 57 | if err != nil { 58 | n.logger.Error(err) 59 | time.Sleep(5 * time.Second) 60 | continue 61 | } 62 | 63 | chanOk := true 64 | for { 65 | select { 66 | case es, ok := <-eventCh: 67 | if !ok { 68 | n.logger.Error("Stream channel closed, restarting") 69 | chanOk = false 70 | time.Sleep(2 * time.Second) 71 | break 72 | } 73 | if es.Err != nil { 74 | n.logger.Error("Stream channel return error '%v', restarting", es.Err) 75 | chanOk = false 76 | time.Sleep(2 * time.Second) 77 | break 78 | } 79 | for _, ne := range es.Events { 80 | err := p.ProcessEvent(ne) 81 | if err != nil { 82 | n.logger.Error(err) 83 | } 84 | } 85 | } 86 | if !chanOk { 87 | break 88 | } 89 | } 90 | n.logger.Debug("Restart nomad input...") 91 | } 92 | }(wg) 93 | } 94 | 95 | func NewNomadInput(options NomadInputOptions, processors *common.Processors, observability *common.Observability) *NomadInput { 96 | logger := observability.Logs() 97 | if utils.IsEmpty(options.Address) || utils.IsEmpty(options.Token) { 98 | logger.Debug("Nomad input address or token is not defined. Skipped") 99 | return nil 100 | } 101 | 102 | config := nomad.DefaultConfig() 103 | config.Address = options.Address 104 | config.SecretID = options.Token 105 | client, err := nomad.NewClient(config) 106 | if err != nil { 107 | logger.Error(err) 108 | return nil 109 | } 110 | 111 | return &NomadInput{ 112 | options: options, 113 | client: client, 114 | processors: processors, 115 | ctx: context.Background(), 116 | eventer: observability.Events(), 117 | logger: observability.Logs(), 118 | meter: observability.Metrics(), 119 | // this is not used anywhere - counters never increase! Check back later when Nomad is added back 120 | // requests: meter.Counter("nomad", "requests", "Count of all nomad input requests", map[string]string{}, "input", "nomad"), 121 | // errors: meter.Counter("nomad", "errors", "Count of all nomad input errors", map[string]string{}, "input", "nomad"), 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /processor/cloudflare.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "encoding/json" 5 | errPkg "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "strings" 10 | "time" 11 | 12 | "github.com/devopsext/events/common" 13 | sreCommon "github.com/devopsext/sre/common" 14 | ) 15 | 16 | type CloudflareProcessor struct { 17 | outputs *common.Outputs 18 | logger sreCommon.Logger 19 | meter sreCommon.Meter 20 | } 21 | 22 | type CloudflareRequest struct { 23 | Text string `json:"text"` 24 | } 25 | 26 | type CloudflareResponse struct { 27 | Message string 28 | } 29 | 30 | func CloudflareProcessorType() string { 31 | return "Cloudflare" 32 | } 33 | 34 | func (p *CloudflareProcessor) EventType() string { 35 | return common.AsEventType(CloudflareProcessorType()) 36 | } 37 | 38 | func (p *CloudflareProcessor) send(channel string, o interface{}, t *time.Time) { 39 | 40 | e := &common.Event{ 41 | Channel: channel, 42 | Type: p.EventType(), 43 | Data: o, 44 | } 45 | if t != nil && (*t).UnixNano() > 0 { 46 | e.SetTime((*t).UTC()) 47 | } else { 48 | e.SetTime(time.Now().UTC()) 49 | } 50 | e.SetLogger(p.logger) 51 | p.outputs.Send(e) 52 | } 53 | 54 | func (p *CloudflareProcessor) HandleEvent(e *common.Event) error { 55 | 56 | if e == nil { 57 | p.logger.Debug("Event is not defined") 58 | return nil 59 | } 60 | 61 | labels := make(map[string]string) 62 | labels["event_channel"] = e.Channel 63 | labels["processor"] = p.EventType() 64 | 65 | requests := p.meter.Counter("cloudflare", "requests", "Count of all cloudflare processor requests", labels, "processor") 66 | requests.Inc() 67 | 68 | p.outputs.Send(e) 69 | return nil 70 | } 71 | 72 | func (p *CloudflareProcessor) HandleHttpRequest(w http.ResponseWriter, r *http.Request) error { 73 | 74 | channel := strings.TrimLeft(r.URL.Path, "/") 75 | 76 | labels := make(map[string]string) 77 | labels["path"] = r.URL.Path 78 | labels["processor"] = p.EventType() 79 | 80 | requests := p.meter.Counter("cloudflare", "requests", "Count of all cloudflare processor requests", labels, "processor") 81 | requests.Inc() 82 | 83 | errors := p.meter.Counter("cloudflare", "errors", "Count of all cloudflare processor errors", labels, "processor") 84 | 85 | var body []byte 86 | if r.Body != nil { 87 | if data, err := ioutil.ReadAll(r.Body); err == nil { 88 | body = data 89 | } 90 | } 91 | 92 | if len(body) == 0 { 93 | errors.Inc() 94 | err := errPkg.New("empty body") 95 | p.logger.Error(err) 96 | http.Error(w, err.Error(), http.StatusBadRequest) 97 | return err 98 | } 99 | 100 | p.logger.Debug("Body => %s", body) 101 | 102 | var Cloudflare CloudflareRequest 103 | if err := json.Unmarshal(body, &Cloudflare); err != nil { 104 | errors.Inc() 105 | p.logger.Error(err) 106 | http.Error(w, "Error unmarshaling message", http.StatusInternalServerError) 107 | return err 108 | } 109 | 110 | p.send(channel, Cloudflare, nil) 111 | 112 | response := &CloudflareResponse{ 113 | Message: "OK", 114 | } 115 | 116 | resp, err := json.Marshal(response) 117 | if err != nil { 118 | errors.Inc() 119 | p.logger.Error("Can't encode response: %v", err) 120 | http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError) 121 | return err 122 | } 123 | 124 | if _, err := w.Write(resp); err != nil { 125 | errors.Inc() 126 | p.logger.Error("Can't write response: %v", err) 127 | http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) 128 | return err 129 | } 130 | return nil 131 | } 132 | 133 | func NewCloudflareProcessor(outputs *common.Outputs, observability *common.Observability) *CloudflareProcessor { 134 | 135 | return &CloudflareProcessor{ 136 | outputs: outputs, 137 | logger: observability.Logs(), 138 | meter: observability.Metrics(), 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /test/aws.json: -------------------------------------------------------------------------------- 1 | { 2 | "account": "157576696349", 3 | "detail": { 4 | "awsRegion": "eu-central-1", 5 | "eventCategory": "Management", 6 | "eventID": "3fcd5045-c9da-441c-a72d-9ee5aeff977d", 7 | "eventName": "AssignPrivateIpAddresses", 8 | "eventSource": "ec2.amazonaws.com", 9 | "eventTime": "2022-05-23T09:02:13Z", 10 | "eventType": "AwsApiCall", 11 | "eventVersion": "1.08", 12 | "managementEvent": true, 13 | "readOnly": false, 14 | "recipientAccountId": "157576696349", 15 | "requestID": "dce3bc79-b263-45a2-bb61-ff2dbba4404f", 16 | "requestParameters": { 17 | "ipv4Prefixes": {}, 18 | "networkInterfaceId": "eni-0e3c28c663ad5a4cd", 19 | "privateIpAddressesSet": {}, 20 | "secondaryPrivateIpAddressCount": 14 21 | }, 22 | "responseElements": { 23 | "_return": true, 24 | "assignedIpv4PrefixSet": {}, 25 | "assignedPrivateIpAddressesSet": { 26 | "assignedPrivateIpAddressSetType": [ 27 | { 28 | "privateIpAddress": "10.176.78.113" 29 | }, 30 | { 31 | "privateIpAddress": "10.176.78.216" 32 | }, 33 | { 34 | "privateIpAddress": "10.176.78.125" 35 | }, 36 | { 37 | "privateIpAddress": "10.176.78.30" 38 | }, 39 | { 40 | "privateIpAddress": "10.176.78.191" 41 | }, 42 | { 43 | "privateIpAddress": "10.176.78.160" 44 | }, 45 | { 46 | "privateIpAddress": "10.176.78.163" 47 | }, 48 | { 49 | "privateIpAddress": "10.176.78.4" 50 | }, 51 | { 52 | "privateIpAddress": "10.176.78.228" 53 | }, 54 | { 55 | "privateIpAddress": "10.176.78.165" 56 | }, 57 | { 58 | "privateIpAddress": "10.176.78.166" 59 | }, 60 | { 61 | "privateIpAddress": "10.176.78.73" 62 | }, 63 | { 64 | "privateIpAddress": "10.176.78.174" 65 | }, 66 | { 67 | "privateIpAddress": "10.176.78.143" 68 | } 69 | ] 70 | }, 71 | "networkInterfaceId": "eni-0e3c28c663ad5a4cd", 72 | "requestId": "dce3bc79-b263-45a2-bb61-ff2dbba4404f" 73 | }, 74 | "sourceIPAddress": "52.28.127.184", 75 | "tlsDetails": { 76 | "cipherSuite": "ECDHE-RSA-AES128-GCM-SHA256", 77 | "clientProvidedHostHeader": "ec2.eu-central-1.amazonaws.com", 78 | "tlsVersion": "TLSv1.2" 79 | }, 80 | "userAgent": "aws-sdk-go/1.21.7 (go1.12.17; linux; amd64)", 81 | "userIdentity": { 82 | "accessKeyId": "ASIASJMCNSIOX5UQKOOU", 83 | "accountId": "157576696349", 84 | "arn": "arn:aws:sts::157576696349:assumed-role/eksctl-bi-nodegroup-default-NodeInstanceRole/i-0a73fa680fa5f76b2", 85 | "principalId": "AROASJMCNSIO3BGXXRURK:i-0a73fa680fa5f76b2", 86 | "sessionContext": { 87 | "attributes": { 88 | "creationDate": "2022-05-23T03:46:37Z", 89 | "mfaAuthenticated": "false" 90 | }, 91 | "ec2RoleDelivery": "1.0", 92 | "sessionIssuer": { 93 | "accountId": "157576696349", 94 | "arn": "arn:aws:iam::157576696349:role/eksctl-bi-nodegroup-default-NodeInstanceRole", 95 | "principalId": "AROASJMCNSIO3BGXXRURK", 96 | "type": "Role", 97 | "userName": "eksctl-bi-nodegroup-default-NodeInstanceRole" 98 | }, 99 | "webIdFederationData": {} 100 | }, 101 | "type": "AssumedRole" 102 | } 103 | }, 104 | "detail-type": "AWS API Call via CloudTrail", 105 | "id": "a139b498-aaf8-dd6f-5795-028ba6436e47", 106 | "region": "eu-central-1", 107 | "resources": [], 108 | "source": "aws.ec2", 109 | "time": "2022-05-23T09:02:13Z", 110 | "version": "0" 111 | } -------------------------------------------------------------------------------- /zbx_export_mediatypes.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.4 4 | 2022-08-17T07:39:38Z 5 | 6 | 7 | zabbixEvent 8 | WEBHOOK 9 | 10 | 11 | AlertURL 12 | http://zabbix.fqdn/tr_events.php?triggerid={TRIGGER.ID}&eventid={EVENT.ID} 13 | 14 | 15 | EventDate 16 | {EVENT.DATE} 17 | 18 | 19 | EventID 20 | {EVENT.ID} 21 | 22 | 23 | EventName 24 | {EVENT.NAME} 25 | 26 | 27 | EventNSeverity 28 | {EVENT.NSEVERITY} 29 | 30 | 31 | EventOpData 32 | {EVENT.OPDATA} 33 | 34 | 35 | EventTags 36 | {EVENT.TAGS} 37 | 38 | 39 | EventTime 40 | {EVENT.TIME} 41 | 42 | 43 | EventType 44 | ZabbixEvent 45 | 46 | 47 | HostName 48 | {HOST.NAME} 49 | 50 | 51 | ItemID 52 | {ITEM.ID} 53 | 54 | 55 | ItemLastValue 56 | {ITEM.LASTVALUE} 57 | 58 | 59 | Status 60 | {EVENT.Status} 61 | 62 | 63 | TriggerDescription 64 | {TRIGGER.DESCRIPTION} 65 | 66 | 67 | TriggerExpression 68 | {TRIGGER.EXPRESSION} 69 | 70 | 71 | TriggerName 72 | {TRIGGER.NAME} 73 | 74 | 75 | 95 | YES 96 | SRE Events 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /processor/aws.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "encoding/json" 5 | errPkg "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "strings" 10 | "time" 11 | 12 | "github.com/devopsext/events/common" 13 | sreCommon "github.com/devopsext/sre/common" 14 | ) 15 | 16 | type AWSProcessor struct { 17 | outputs *common.Outputs 18 | logger sreCommon.Logger 19 | meter sreCommon.Meter 20 | } 21 | 22 | type AWSRequest struct { 23 | Version string `json:"version"` 24 | Source string `json:"source"` 25 | Account string `json:"account"` 26 | Time time.Time `json:"time"` 27 | Region string `json:"region"` 28 | DetailType string `json:"detail-type,omitempty"` 29 | Detail interface{} `json:"detail,omitempty"` 30 | } 31 | 32 | type AWSResponse struct { 33 | Message string 34 | } 35 | 36 | func AWSProcessorType() string { 37 | return "AWS" 38 | } 39 | 40 | func (p *AWSProcessor) EventType() string { 41 | return common.AsEventType(AWSProcessorType()) 42 | } 43 | 44 | func (p *AWSProcessor) send(channel string, o interface{}, t *time.Time) { 45 | 46 | e := &common.Event{ 47 | Channel: channel, 48 | Type: p.EventType(), 49 | Data: o, 50 | } 51 | if t != nil && (*t).UnixNano() > 0 { 52 | e.SetTime((*t).UTC()) 53 | } else { 54 | e.SetTime(time.Now().UTC()) 55 | } 56 | e.SetLogger(p.logger) 57 | p.outputs.Send(e) 58 | } 59 | 60 | func (p *AWSProcessor) HandleEvent(e *common.Event) error { 61 | 62 | if e == nil { 63 | p.logger.Debug("Event is not defined") 64 | return nil 65 | } 66 | 67 | labels := make(map[string]string) 68 | labels["event_channel"] = e.Channel 69 | labels["processor"] = p.EventType() 70 | 71 | requests := p.meter.Counter("aws", "requests", "Count of all aws processor requests", labels, "processor") 72 | requests.Inc() 73 | p.outputs.Send(e) 74 | return nil 75 | } 76 | 77 | func (p *AWSProcessor) HandleHttpRequest(w http.ResponseWriter, r *http.Request) error { 78 | 79 | channel := strings.TrimLeft(r.URL.Path, "/") 80 | 81 | labels := make(map[string]string) 82 | labels["path"] = r.URL.Path 83 | labels["processor"] = p.EventType() 84 | 85 | requests := p.meter.Counter("aws", "requests", "Count of all aws processor requests", labels, "processor") 86 | requests.Inc() 87 | 88 | errors := p.meter.Counter("aws", "errors", "Count of all aws processor errors", labels, "processor") 89 | 90 | var body []byte 91 | if r.Body != nil { 92 | if data, err := ioutil.ReadAll(r.Body); err == nil { 93 | body = data 94 | } 95 | } 96 | 97 | if len(body) == 0 { 98 | errors.Inc() 99 | err := errPkg.New("empty body") 100 | p.logger.Error(err) 101 | http.Error(w, err.Error(), http.StatusBadRequest) 102 | return err 103 | } 104 | 105 | p.logger.Debug("Body => %s", body) 106 | 107 | var request AWSRequest 108 | if err := json.Unmarshal(body, &request); err != nil { 109 | errors.Inc() 110 | p.logger.Error(err) 111 | http.Error(w, "Error unmarshaling message", http.StatusInternalServerError) 112 | return err 113 | } 114 | 115 | if request.Time.UnixMilli() > 0 { 116 | t := request.Time 117 | p.send(channel, request, &t) 118 | } else { 119 | p.send(channel, request, nil) 120 | } 121 | 122 | response := &AWSResponse{ 123 | Message: "OK", 124 | } 125 | 126 | resp, err := json.Marshal(response) 127 | if err != nil { 128 | errors.Inc() 129 | p.logger.Error("Can't encode response: %v", err) 130 | http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError) 131 | return err 132 | } 133 | 134 | if _, err := w.Write(resp); err != nil { 135 | errors.Inc() 136 | p.logger.Error("Can't write response: %v", err) 137 | http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) 138 | return err 139 | } 140 | return nil 141 | } 142 | 143 | func NewAWSProcessor(outputs *common.Outputs, observability *common.Observability) *AWSProcessor { 144 | 145 | return &AWSProcessor{ 146 | outputs: outputs, 147 | logger: observability.Logs(), 148 | meter: observability.Metrics(), 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /processor/teamcity.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "encoding/json" 5 | errPkg "errors" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "strings" 10 | "time" 11 | 12 | "github.com/devopsext/events/common" 13 | sreCommon "github.com/devopsext/sre/common" 14 | "github.com/go-playground/webhooks/v6/gitlab" 15 | ) 16 | 17 | type TeamcityProcessor struct { 18 | outputs *common.Outputs 19 | logger sreCommon.Logger 20 | meter sreCommon.Meter 21 | hook *gitlab.Webhook 22 | } 23 | 24 | type TeamcityEvent struct { 25 | Timestamp time.Time `json:"timestamp"` 26 | BuildEvent string `json:"build_event"` 27 | BuildName string `json:"build_name"` 28 | BuildStatusUrl string `json:"build_status_url,omitempty"` 29 | TriggeredBy string `json:"triggered_by,omitempty"` 30 | BuildResult string `json:"build_result,omitempty"` 31 | Target string `json:"target,omitempty"` 32 | } 33 | 34 | type TeamcityResponse struct { 35 | Message string 36 | } 37 | 38 | func TeamcityProcessorType() string { 39 | return "Teamcity" 40 | } 41 | 42 | func (p TeamcityProcessor) EventType() string { 43 | return common.AsEventType(TeamcityProcessorType()) 44 | } 45 | 46 | func (p TeamcityProcessor) HandleEvent(e *common.Event) error { 47 | if e == nil { 48 | p.logger.Debug("Event is not defined") 49 | return nil 50 | } 51 | 52 | labels := make(map[string]string) 53 | labels["event_channel"] = e.Channel 54 | labels["processor"] = p.EventType() 55 | 56 | requests := p.meter.Counter("teamcity", "requests", "Count of all teamcity processor requests", labels, "processor") 57 | requests.Inc() 58 | 59 | p.outputs.Send(e) 60 | return nil 61 | } 62 | 63 | func (p TeamcityProcessor) HandleHttpRequest(w http.ResponseWriter, r *http.Request) error { 64 | channel := strings.TrimLeft(r.URL.Path, "/") 65 | 66 | labels := make(map[string]string) 67 | labels["path"] = r.URL.Path 68 | labels["processor"] = p.EventType() 69 | 70 | requests := p.meter.Counter("teamcity", "requests", "Count of all teamcity processor requests", labels, "processor") 71 | requests.Inc() 72 | 73 | errors := p.meter.Counter("teamcity", "errors", "Count of all teamcity processor errors", labels, "processor") 74 | 75 | var body []byte 76 | if r.Body != nil { 77 | if data, err := io.ReadAll(r.Body); err == nil { 78 | body = data 79 | } 80 | } 81 | 82 | if len(body) == 0 { 83 | errors.Inc() 84 | err := errPkg.New("empty body") 85 | p.logger.Error(err) 86 | http.Error(w, err.Error(), http.StatusBadRequest) 87 | return err 88 | } 89 | 90 | p.logger.Debug("Body => %s", body) 91 | 92 | var tc TeamcityEvent 93 | if err := json.Unmarshal(body, &tc); err != nil { 94 | errors.Inc() 95 | p.logger.Error(err) 96 | http.Error(w, "Error unmarshaling message", http.StatusInternalServerError) 97 | return err 98 | } 99 | 100 | p.send(channel, tc, &tc.Timestamp) 101 | 102 | response := &TeamcityResponse{ 103 | Message: "OK", 104 | } 105 | 106 | resp, err := json.Marshal(response) 107 | if err != nil { 108 | errors.Inc() 109 | p.logger.Error("Can't encode response: %v", err) 110 | http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError) 111 | return err 112 | } 113 | 114 | if _, err := w.Write(resp); err != nil { 115 | errors.Inc() 116 | p.logger.Error("Can't write response: %v", err) 117 | http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) 118 | return err 119 | } 120 | return nil 121 | } 122 | 123 | func (p TeamcityProcessor) send(channel string, tc TeamcityEvent, t *time.Time) { 124 | e := &common.Event{ 125 | Channel: channel, 126 | Type: p.EventType(), 127 | Data: tc, 128 | } 129 | if t != nil && (*t).UnixNano() > 0 { 130 | e.SetTime((*t).UTC()) 131 | } else { 132 | e.SetTime(time.Now().UTC()) 133 | } 134 | e.SetLogger(p.logger) 135 | p.outputs.Send(e) 136 | } 137 | 138 | func NewTeamcityProcessor(outputs *common.Outputs, observability *common.Observability) *TeamcityProcessor { 139 | return &TeamcityProcessor{ 140 | outputs: outputs, 141 | logger: observability.Logs(), 142 | meter: observability.Metrics(), 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /processor/observium.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "encoding/json" 5 | errPkg "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "strings" 10 | "time" 11 | 12 | "github.com/devopsext/events/common" 13 | sreCommon "github.com/devopsext/sre/common" 14 | ) 15 | 16 | type ObserviumEventProcessor struct { 17 | outputs *common.Outputs 18 | logger sreCommon.Logger 19 | meter sreCommon.Meter 20 | } 21 | 22 | type ObserviumRequest struct { 23 | Title string `json:"TITLE"` 24 | AlertState string `json:"ALERT_STATE"` 25 | AlertURL string `json:"ALERT_URL"` 26 | AlertUnixTime int64 `json:"ALERT_UNIXTIME"` 27 | DeviceHostname string `json:"DEVICE_HOSTNAME"` 28 | DeviceLocation string `json:"DEVICE_LOCATION"` 29 | Metrics string `json:"METRICS"` 30 | } 31 | 32 | type ObserviumResponse struct { 33 | Message string 34 | } 35 | 36 | func ObserviumEventProcessorType() string { 37 | return "Observium" 38 | } 39 | 40 | func (p *ObserviumEventProcessor) EventType() string { 41 | return common.AsEventType(ObserviumEventProcessorType()) 42 | } 43 | 44 | func (p *ObserviumEventProcessor) send(channel string, o interface{}, t *time.Time) { 45 | 46 | e := &common.Event{ 47 | Channel: channel, 48 | Type: p.EventType(), 49 | Data: o, 50 | } 51 | if t != nil && (*t).UnixNano() > 0 { 52 | e.SetTime((*t).UTC()) 53 | } else { 54 | e.SetTime(time.Now().UTC()) 55 | } 56 | e.SetLogger(p.logger) 57 | p.outputs.Send(e) 58 | } 59 | 60 | func (p *ObserviumEventProcessor) HandleEvent(e *common.Event) error { 61 | 62 | if e == nil { 63 | p.logger.Debug("Event is not defined") 64 | return nil 65 | } 66 | 67 | labels := make(map[string]string) 68 | labels["event_channel"] = e.Channel 69 | labels["processor"] = p.EventType() 70 | 71 | requests := p.meter.Counter("observium", "requests", "Count of all observium processor requests", labels, "processor") 72 | requests.Inc() 73 | 74 | p.outputs.Send(e) 75 | return nil 76 | } 77 | 78 | func (p *ObserviumEventProcessor) HandleHttpRequest(w http.ResponseWriter, r *http.Request) error { 79 | 80 | channel := strings.TrimLeft(r.URL.Path, "/") 81 | 82 | labels := make(map[string]string) 83 | labels["path"] = r.URL.Path 84 | labels["processor"] = p.EventType() 85 | 86 | requests := p.meter.Counter("observium", "requests", "Count of all observium processor requests", labels, "processor") 87 | requests.Inc() 88 | 89 | errors := p.meter.Counter("observium", "errors", "Count of all observium processor errors", labels, "processor") 90 | 91 | var body []byte 92 | if r.Body != nil { 93 | if data, err := ioutil.ReadAll(r.Body); err == nil { 94 | body = data 95 | } 96 | } 97 | 98 | if len(body) == 0 { 99 | errors.Inc() 100 | err := errPkg.New("empty body") 101 | p.logger.Error(err) 102 | http.Error(w, err.Error(), http.StatusBadRequest) 103 | return err 104 | } 105 | 106 | p.logger.Debug("Body => %s", body) 107 | 108 | var observiumEvent ObserviumRequest 109 | if err := json.Unmarshal(body, &observiumEvent); err != nil { 110 | errors.Inc() 111 | p.logger.Error("Error decoding incoming message: %s", body) 112 | p.logger.Error(err) 113 | http.Error(w, "Error unmarshaling message", http.StatusInternalServerError) 114 | return err 115 | } 116 | 117 | t := time.Unix(observiumEvent.AlertUnixTime, 0) 118 | 119 | p.send(channel, observiumEvent, &t) 120 | 121 | response := &ObserviumResponse{ 122 | Message: "OK", 123 | } 124 | 125 | resp, err := json.Marshal(response) 126 | if err != nil { 127 | errors.Inc() 128 | p.logger.Error("Can't encode response: %v", err) 129 | http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError) 130 | return err 131 | } 132 | 133 | if _, err := w.Write(resp); err != nil { 134 | errors.Inc() 135 | p.logger.Error("Can't write response: %v", err) 136 | http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) 137 | return err 138 | } 139 | return nil 140 | } 141 | 142 | func NewObserviumEventProcessor(outputs *common.Outputs, observability *common.Observability) *ObserviumEventProcessor { 143 | 144 | return &ObserviumEventProcessor{ 145 | outputs: outputs, 146 | logger: observability.Logs(), 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /processor/zabbix.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "encoding/json" 5 | errPkg "errors" 6 | "io" 7 | "net/http" 8 | "strings" 9 | "time" 10 | 11 | "github.com/devopsext/events/common" 12 | sreCommon "github.com/devopsext/sre/common" 13 | ) 14 | 15 | type ZabbixProcessor struct { 16 | outputs *common.Outputs 17 | logger sreCommon.Logger 18 | meter sreCommon.Meter 19 | } 20 | 21 | type ZabbixEvent struct { 22 | AlertURL string `json:"AlertURL,omitempty"` 23 | Environment string `json:"Environment,omitempty"` 24 | EventDate string `json:"EventDate,omitempty"` 25 | EventID string `json:"EventID,omitempty"` 26 | EventName string `json:"EventName,omitempty"` 27 | EventNSeverity string `json:"EventNSeverity,omitempty"` 28 | EventOpData string `json:"EventOpData,omitempty"` 29 | Status string `json:"Status,omitempty"` 30 | EventTags string `json:"EventTags,omitempty"` 31 | EventTime string `json:"EventTime,omitempty"` 32 | EventType string `json:"EventType,omitempty"` 33 | HostName string `json:"HostName,omitempty"` 34 | ItemID string `json:"ItemID,omitempty"` 35 | ItemLastValue string `json:"ItemLastValue,omitempty"` 36 | TriggerDescription string `json:"TriggerDescription,omitempty"` 37 | TriggerExpression string `json:"TriggerExpression,omitempty"` 38 | TriggerName string `json:"TriggerName,omitempty"` 39 | } 40 | 41 | func ZabbixProcessorType() string { 42 | return "Zabbix" 43 | } 44 | 45 | func (p *ZabbixProcessor) EventType() string { 46 | return common.AsEventType(ZabbixProcessorType()) 47 | } 48 | 49 | func (p *ZabbixProcessor) send(channel string, o interface{}, t *time.Time) { 50 | 51 | e := &common.Event{ 52 | Channel: channel, 53 | Type: p.EventType(), 54 | Data: o, 55 | } 56 | if t != nil && (*t).UnixNano() > 0 { 57 | e.SetTime((*t).UTC()) 58 | } else { 59 | e.SetTime(time.Now().UTC()) 60 | } 61 | e.SetLogger(p.logger) 62 | p.outputs.Send(e) 63 | } 64 | 65 | func (p *ZabbixProcessor) HandleEvent(e *common.Event) error { 66 | 67 | if e == nil { 68 | p.logger.Debug("Event is not defined") 69 | return nil 70 | } 71 | 72 | labels := make(map[string]string) 73 | labels["event_channel"] = e.Channel 74 | labels["processor"] = p.EventType() 75 | 76 | requests := p.meter.Counter("zabbix", "requests", "Count of all zabbix processor requests", labels, "processor") 77 | requests.Inc() 78 | 79 | p.outputs.Send(e) 80 | return nil 81 | } 82 | 83 | func (p *ZabbixProcessor) HandleHttpRequest(w http.ResponseWriter, r *http.Request) error { 84 | 85 | channel := strings.TrimLeft(r.URL.Path, "/") 86 | 87 | labels := make(map[string]string) 88 | labels["path"] = r.URL.Path 89 | labels["processor"] = p.EventType() 90 | 91 | requests := p.meter.Counter("zabbix", "requests", "Count of all zabbix processor requests", labels, "processor") 92 | requests.Inc() 93 | 94 | errors := p.meter.Counter("zabbix", "errors", "Count of all zabbix processor errors", labels, "processor") 95 | 96 | var body []byte 97 | if r.Body != nil { 98 | if data, err := io.ReadAll(r.Body); err == nil { 99 | body = data 100 | } 101 | } 102 | 103 | if len(body) == 0 { 104 | errors.Inc() 105 | err := errPkg.New("empty body") 106 | p.logger.Error(err) 107 | http.Error(w, err.Error(), http.StatusBadRequest) 108 | return err 109 | } 110 | 111 | p.logger.Debug("Body => %s", body) 112 | 113 | var request ZabbixEvent 114 | 115 | err := json.Unmarshal(body, &request) 116 | if err != nil { 117 | errors.Inc() 118 | p.logger.Error(err) 119 | http.Error(w, err.Error(), http.StatusBadRequest) 120 | return err 121 | } 122 | 123 | EventDateTime, err := time.Parse(time.RFC3339Nano, strings.ReplaceAll(request.EventDate, ".", "-")+"T"+request.EventTime+"Z") 124 | if err != nil { 125 | p.send(channel, request, nil) 126 | return nil 127 | } 128 | p.send(channel, request, &EventDateTime) 129 | return nil 130 | } 131 | 132 | func NewZabbixProcessor(outputs *common.Outputs, observability *common.Observability) *ZabbixProcessor { 133 | 134 | return &ZabbixProcessor{ 135 | outputs: outputs, 136 | logger: observability.Logs(), 137 | meter: observability.Metrics(), 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /processor/alertmanager.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "encoding/json" 5 | errPkg "errors" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "strings" 10 | 11 | "golang.org/x/text/cases" 12 | "golang.org/x/text/language" 13 | 14 | "github.com/devopsext/events/common" 15 | sreCommon "github.com/devopsext/sre/common" 16 | "github.com/devopsext/utils" 17 | "github.com/prometheus/alertmanager/template" 18 | ) 19 | 20 | type AlertmanagerProcessor struct { 21 | outputs *common.Outputs 22 | logger sreCommon.Logger 23 | meter sreCommon.Meter 24 | } 25 | 26 | type AlertmanagerResponse struct { 27 | Message string 28 | } 29 | 30 | func AlertmanagerProcessorType() string { 31 | return "Alertmanager" 32 | } 33 | 34 | func (p *AlertmanagerProcessor) EventType() string { 35 | return common.AsEventType(AlertmanagerProcessorType()) 36 | } 37 | 38 | func (p *AlertmanagerProcessor) prepareStatus(status string) string { 39 | lower := cases.Lower(language.English) 40 | title := cases.Title(language.English) 41 | return title.String(lower.String(status)) 42 | } 43 | 44 | func (p *AlertmanagerProcessor) send(channel string, data *template.Data) { 45 | 46 | for _, alert := range data.Alerts { 47 | 48 | e := &common.Event{ 49 | Channel: channel, 50 | Type: p.EventType(), 51 | Data: alert, 52 | } 53 | e.SetTime(alert.StartsAt.UTC()) 54 | e.SetLogger(p.logger) 55 | 56 | p.outputs.Send(e) 57 | } 58 | } 59 | 60 | func (p *AlertmanagerProcessor) HandleEvent(e *common.Event) error { 61 | 62 | if e == nil { 63 | p.logger.Debug("Event is not defined") 64 | return nil 65 | } 66 | 67 | labels := make(map[string]string) 68 | labels["event_channel"] = e.Channel 69 | labels["processor"] = p.EventType() 70 | 71 | requests := p.meter.Counter("alertmanager", "requests", "Count of all alertmanager processor requests", labels, "processor") 72 | requests.Inc() 73 | p.outputs.Send(e) 74 | return nil 75 | } 76 | 77 | func (p *AlertmanagerProcessor) HandleHttpRequest(w http.ResponseWriter, r *http.Request) error { 78 | 79 | channel := strings.TrimLeft(r.URL.Path, "/") 80 | 81 | labels := make(map[string]string) 82 | labels["path"] = r.URL.Path 83 | labels["processor"] = p.EventType() 84 | 85 | requests := p.meter.Counter("alertmanager", "requests", "Count of all alertmanager processor requests", labels, "processor") 86 | requests.Inc() 87 | 88 | errors := p.meter.Counter("alertmanager", "errors", "Count of all alertmanager processor errors", labels, "processor") 89 | 90 | var body []byte 91 | if r.Body != nil { 92 | if data, err := io.ReadAll(r.Body); err == nil { 93 | body = data 94 | } 95 | } 96 | 97 | if len(body) == 0 { 98 | errors.Inc() 99 | err := errPkg.New("empty body") 100 | p.logger.Error(err) 101 | http.Error(w, err.Error(), http.StatusBadRequest) 102 | return err 103 | } 104 | 105 | p.logger.Debug("Body => %s", body) 106 | 107 | var response *AlertmanagerResponse 108 | errorString := "" 109 | data := template.Data{} 110 | if err := json.Unmarshal(body, &data); err != nil { 111 | p.logger.Error("Can't decode body: %v", err) 112 | response = &AlertmanagerResponse{ 113 | Message: err.Error(), 114 | } 115 | errorString = response.Message 116 | } else { 117 | p.send(channel, &data) 118 | response = &AlertmanagerResponse{ 119 | Message: "OK", 120 | } 121 | } 122 | 123 | resp, err := json.Marshal(response) 124 | if err != nil { 125 | errors.Inc() 126 | p.logger.Error("Can't encode response: %v", err) 127 | http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError) 128 | return err 129 | } 130 | 131 | if _, err := w.Write(resp); err != nil { 132 | errors.Inc() 133 | p.logger.Error("Can't write response: %v", err) 134 | http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) 135 | return err 136 | } 137 | 138 | if !utils.IsEmpty(errorString) { 139 | errors.Inc() 140 | p.logger.Error(errorString) 141 | http.Error(w, fmt.Sprint(errorString), http.StatusInternalServerError) 142 | return err 143 | } 144 | return nil 145 | } 146 | 147 | func NewAlertmanagerProcessor(outputs *common.Outputs, observability *common.Observability) *AlertmanagerProcessor { 148 | 149 | return &AlertmanagerProcessor{ 150 | outputs: outputs, 151 | logger: observability.Logs(), 152 | meter: observability.Metrics(), 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /datadog.name: -------------------------------------------------------------------------------- 1 | {{- define "kube-header"}} 2 | {{- $t := timeFormat .time "02.01.06 15:04:05"}} 3 | {{- printf "%s: %s => %s %s %s" "ERROR" .data.source .data.location (index (regexFindSubmatch "([a-zA-Z-]+)-kube" .channel) 1 ) $t }} 4 | {{- end}} 5 | {{- define "k8s-header"}} 6 | {{- $t := timeFormat .time "02.01.06 15:04:05"}} 7 | {{- $op := .data.operation}} 8 | {{- if eq .data.operation "Update"}} 9 | 10 | {{- $oldImages := ""}}{{- $newImages := ""}} 11 | {{- if regexMatch .data.kind "StatefulSet|Deployment|DaemonSet|ReplicaSet|Pod"}} 12 | {{- range .data.object.old.spec.template.spec.containers}} 13 | {{- $oldImages = printf "%s%s => %s\n" $oldImages .name .image}} 14 | {{- end}} 15 | {{- range .data.object.new.spec.template.spec.containers}} 16 | {{- $newImages = printf "%s%s => %s\n" $newImages .name .image}} 17 | {{- end}} 18 | {{- else if eq .data.kind "Application"}} 19 | {{- $oldImages = printf "%s:%s\n" .data.object.old.spec.source.helm.valuesObject.image.repository .data.object.old.spec.source.helm.valuesObject.image.tag }} 20 | {{- $newImages = printf "%s:%s\n" .data.object.new.spec.source.helm.valuesObject.image.repository .data.object.new.spec.source.helm.valuesObject.image.tag }} 21 | {{- end}} 22 | {{- if ne $oldImages $newImages}} 23 | {{- $op = "Release"}} 24 | {{- end}} 25 | {{- if ne .data.object.old.spec.replicas .data.object.new.spec.replicas}} 26 | {{- if ne $op "Release"}} 27 | {{- $op = "Scale"}} 28 | {{- end}} 29 | {{- end}} 30 | 31 | {{- end}} 32 | {{- printf "%s: %s / %s %s: %s by %s" (toUpper $op) .data.kind .data.location .channel $t .data.user.name}} 33 | {{- end}} 34 | {{- define "alertmanager-header"}} 35 | {{- $t := timeFormat .time "02.01.06 15:04:05"}} 36 | {{- printf "%s: %s\n%s: %s" (toUpper .data.status) .data.labels.alertname .channel $t}} 37 | {{- end}} 38 | {{- define "alertmanager-body"}} 39 | {{- if eq .status "firing"}}{{- printf "\nStartsAt: %s" (timeFormat .startsAt "02.01.06 15:04:05") }}{{end}} 40 | {{- if eq .status "resolved"}}{{- printf "\nendsAt: %s" (timeFormat .endsAt "02.01.06 15:04:05") }}{{end}} 41 | {{- end}} 42 | {{- define "gitlab-header"}} 43 | {{- $t := timeFormat .time "02.01.06 15:04:05"}} 44 | {{- if .data.project}}{{- printf "%s: %s / %s@%s %s: %s by %s" (toUpper .data.object_kind) .data.project.namespace .data.project.name .data.object_attributes.ref .channel $t .data.user.username}} 45 | {{- else}}{{- printf "%s: %s@%s %s: %s by %s" (toUpper .data.object_kind) .data.project_name .data.ref .channel $t .data.user.username}}{{end}} 46 | {{- end}} 47 | {{- define "text"}} 48 | {{- if eq .type "KubeEvent"}} 49 | {{- if or (regexMatch .data.reason "(NodeNotReady|Evicted|FailedScheduling|BackOff|EvictionThresholdMet)") (and (eq .data.reason "Failed") (or (regexMatch .data.message "^Failed to pull image.*") (regexMatch .data.message "^Failed to start container.*")))}} 50 | {{template "kube-header" .}} 51 | {{- end}} 52 | {{- end}} 53 | {{- if eq .type "K8sEvent"}} 54 | {{- $pass := false}} 55 | {{- $text := ""}} 56 | {{- if and (eq .data.operation "Update") (regexMatch "(system|eks|kube-admin).*" .data.user.name)}} 57 | {{- if (regexMatch "(Deployment|DaemonSet|StatefulSet|Node|Ingress)" .data.kind )}}{{- $pass = false}}{{- else}}{{- $pass = true}}{{- end}} 58 | {{- else}}{{- $pass = true}} 59 | {{- end}} 60 | {{- if (.data.object)}} 61 | {{- if or (not (regexMatch "StatefulSet|DaemonSet|ReplicaSet" .data.kind )) (and (regexMatch "StatefulSet|DaemonSet|ReplicaSet" .data.kind ) (not (regexMatch ".*argocd-application-controller" .data.user.name ))) }} 62 | {{template "k8s-header" .}} 63 | {{- end}} 64 | {{- end}} 65 | {{- if (gt (len $text) 0)}} 66 | {{printf "%s" $text}} 67 | {{- end}} 68 | {{- end}} 69 | {{- if eq .type "AlertmanagerEvent"}} 70 | {{template "alertmanager-header" .}}{{template "alertmanager-body" .data}} 71 | {{- end}} 72 | {{- if eq .type "GitlabEvent"}} 73 | {{- $match := getEnv "EVENTS_GITLAB_RUNNERS"}}{{$ok := false}} 74 | {{- if .data.builds}} 75 | {{- range .data.builds}} 76 | {{- if and .runner (.runner.description | regexMatch $match) (not (empty .finished_at))}}{{$ok = true}}{{end}} 77 | {{- end}} 78 | {{- else}} 79 | {{- if and .data.runner (.data.runner.description | regexMatch $match) (not (empty .data.build_duration))}}{{$ok = true}}{{end}} 80 | {{- end}} 81 | {{- if $ok}}{{template "gitlab-header" .}}{{- end}} 82 | {{- end}} 83 | {{- end}} 84 | {{- define "datadog-name"}}{{template "text" .}}{{- end}} -------------------------------------------------------------------------------- /output/kafka.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "strings" 5 | "sync" 6 | "time" 7 | 8 | sreCommon "github.com/devopsext/sre/common" 9 | toolsRender "github.com/devopsext/tools/render" 10 | "github.com/devopsext/utils" 11 | 12 | "github.com/devopsext/events/common" 13 | 14 | "github.com/IBM/sarama" 15 | ) 16 | 17 | type KafkaOutputOptions struct { 18 | ClientID string 19 | Message string 20 | Brokers string 21 | Topic string 22 | FlushFrequency int 23 | FlushMaxMessages int 24 | NetMaxOpenRequests int 25 | NetDialTimeout int 26 | NetReadTimeout int 27 | NetWriteTimeout int 28 | } 29 | 30 | type KafkaOutput struct { 31 | wg *sync.WaitGroup 32 | producer *sarama.AsyncProducer 33 | message *toolsRender.TextTemplate 34 | options KafkaOutputOptions 35 | logger sreCommon.Logger 36 | meter sreCommon.Meter 37 | } 38 | 39 | func (k *KafkaOutput) Name() string { 40 | return "Kafka" 41 | } 42 | 43 | func (k *KafkaOutput) Send(event *common.Event) { 44 | 45 | k.wg.Add(1) 46 | go func() { 47 | defer k.wg.Done() 48 | 49 | if event == nil { 50 | k.logger.Debug("Event is empty") 51 | return 52 | } 53 | 54 | b, err := k.message.RenderObject(event) 55 | if err != nil { 56 | k.logger.Error(err) 57 | return 58 | } 59 | 60 | message := strings.TrimSpace(string(b)) 61 | if utils.IsEmpty(message) { 62 | k.logger.Debug("Kafka message is empty") 63 | return 64 | } 65 | 66 | labels := make(map[string]string) 67 | labels["event_channel"] = event.Channel 68 | labels["event_type"] = event.Type 69 | labels["kafka_client_id"] = k.options.ClientID 70 | labels["kafka_brokers"] = k.options.Brokers 71 | labels["kafka_topic"] = k.options.Topic 72 | labels["output"] = k.Name() 73 | 74 | requests := k.meter.Counter("kafka", "requests", "Count of all kafka requests", labels, "output") 75 | requests.Inc() 76 | k.logger.Debug("Kafka message => %s", message) 77 | 78 | (*k.producer).Input() <- &sarama.ProducerMessage{ 79 | Topic: k.options.Topic, 80 | Value: sarama.ByteEncoder(b), 81 | } 82 | 83 | for err = range (*k.producer).Errors() { 84 | errors := k.meter.Counter("kafka", "errors", "Count of all kafka errors", labels, "output") 85 | errors.Inc() 86 | } 87 | }() 88 | } 89 | 90 | func makeKafkaProducer(wg *sync.WaitGroup, brokers string, topic string, config *sarama.Config, logger sreCommon.Logger) *sarama.AsyncProducer { 91 | 92 | brks := strings.Split(brokers, ",") 93 | if len(brks) == 0 || utils.IsEmpty(brokers) { 94 | 95 | logger.Debug("Kafka brokers are not defined. Skipped.") 96 | return nil 97 | } 98 | 99 | if utils.IsEmpty(topic) { 100 | logger.Debug("Kafka topic is not defined. Skipped.") 101 | return nil 102 | } 103 | 104 | logger.Info("Start %s for %s...", config.ClientID, topic) 105 | 106 | producer, err := sarama.NewAsyncProducer(brks, config) 107 | if err != nil { 108 | logger.Error(err) 109 | return nil 110 | } 111 | return &producer 112 | } 113 | 114 | func NewKafkaOutput(wg *sync.WaitGroup, options KafkaOutputOptions, templateOptions toolsRender.TemplateOptions, observability *common.Observability) *KafkaOutput { 115 | 116 | config := sarama.NewConfig() 117 | config.Version = sarama.V1_1_1_0 118 | 119 | if !utils.IsEmpty(options.ClientID) { 120 | config.ClientID = options.ClientID 121 | } 122 | 123 | config.Producer.Return.Successes = true 124 | config.Producer.Return.Errors = true 125 | config.Producer.Flush.Frequency = time.Second * time.Duration(options.FlushFrequency) 126 | config.Producer.Flush.MaxMessages = options.FlushMaxMessages 127 | 128 | config.Net.MaxOpenRequests = options.NetMaxOpenRequests 129 | config.Net.DialTimeout = time.Second * time.Duration(options.NetDialTimeout) 130 | config.Net.ReadTimeout = time.Second * time.Duration(options.NetReadTimeout) 131 | config.Net.WriteTimeout = time.Second * time.Duration(options.NetWriteTimeout) 132 | 133 | logger := observability.Logs() 134 | producer := makeKafkaProducer(wg, options.Brokers, options.Topic, config, logger) 135 | if producer == nil { 136 | logger.Error("no producer") 137 | return nil 138 | } 139 | 140 | messageOpts := toolsRender.TemplateOptions{ 141 | Name: "kafka-message", 142 | Content: common.Content(options.Message), 143 | TimeFormat: templateOptions.TimeFormat, 144 | } 145 | message, err := toolsRender.NewTextTemplate(messageOpts, observability) 146 | if err != nil { 147 | logger.Error(err) 148 | return nil 149 | } 150 | 151 | return &KafkaOutput{ 152 | wg: wg, 153 | producer: producer, 154 | message: message, 155 | options: options, 156 | logger: logger, 157 | meter: observability.Metrics(), 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /output/newrelic.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | "sync" 7 | 8 | "github.com/devopsext/events/common" 9 | sreCommon "github.com/devopsext/sre/common" 10 | sreProvider "github.com/devopsext/sre/provider" 11 | toolsRender "github.com/devopsext/tools/render" 12 | "github.com/devopsext/utils" 13 | ) 14 | 15 | type NewRelicOutputOptions struct { 16 | Name string 17 | Message string 18 | AttributesSelector string 19 | } 20 | 21 | type NewRelicOutput struct { 22 | wg *sync.WaitGroup 23 | name *toolsRender.TextTemplate 24 | message *toolsRender.TextTemplate 25 | attributes *toolsRender.TextTemplate 26 | options NewRelicOutputOptions 27 | logger sreCommon.Logger 28 | meter sreCommon.Meter 29 | newrelicEventer *sreProvider.NewRelicEventer 30 | newrelicOptions *sreProvider.NewRelicOptions 31 | } 32 | 33 | func (n *NewRelicOutput) Name() string { 34 | return "NewRelic" 35 | } 36 | 37 | func (n *NewRelicOutput) getAttributes(o interface{}) (map[string]string, error) { 38 | 39 | attrs := make(map[string]string) 40 | if n.attributes == nil { 41 | return attrs, nil 42 | } 43 | 44 | a, err := n.attributes.RenderObject(o) 45 | if err != nil { 46 | return attrs, err 47 | } 48 | 49 | m := string(a) 50 | if utils.IsEmpty(m) { 51 | return attrs, nil 52 | } 53 | 54 | n.logger.Debug("NewRelic raw attributes => %s", m) 55 | 56 | var object map[string]interface{} 57 | 58 | if err := json.Unmarshal([]byte(m), &object); err != nil { 59 | return attrs, err 60 | } 61 | 62 | for k, v := range object { 63 | vs, ok := v.(string) 64 | if ok { 65 | attrs[k] = vs 66 | } 67 | } 68 | return attrs, nil 69 | } 70 | 71 | func (r *NewRelicOutput) Send(event *common.Event) { 72 | 73 | r.wg.Add(1) 74 | go func() { 75 | defer r.wg.Done() 76 | 77 | if event == nil { 78 | r.logger.Debug("Event is empty") 79 | return 80 | } 81 | 82 | if event.Data == nil { 83 | r.logger.Error("Event data is empty") 84 | return 85 | } 86 | 87 | jsonObject, err := event.JsonObject() 88 | if err != nil { 89 | r.logger.Error(err) 90 | return 91 | } 92 | 93 | b, err := r.message.RenderObject(jsonObject) 94 | if err != nil { 95 | r.logger.Error(err) 96 | return 97 | } 98 | 99 | message := strings.TrimSpace(string(b)) 100 | if utils.IsEmpty(message) { 101 | r.logger.Debug("NewRelic message is empty") 102 | return 103 | } 104 | 105 | labels := make(map[string]string) 106 | labels["event_channel"] = event.Channel 107 | labels["event_type"] = event.Type 108 | labels["newrelic_environment"] = r.newrelicOptions.Environment 109 | labels["newrelic_service_name"] = r.newrelicOptions.ServiceName 110 | labels["output"] = r.Name() 111 | 112 | requests := r.meter.Counter("newrelic", "requests", "Count of all newrelic requests", labels, "output") 113 | requests.Inc() 114 | r.logger.Debug("NewRelic message => %s", message) 115 | 116 | attributes, err := r.getAttributes(jsonObject) 117 | if err != nil { 118 | r.logger.Error(err) 119 | } 120 | 121 | err = r.newrelicEventer.At(message, "", attributes, event.Time) 122 | if err != nil { 123 | errors := r.meter.Counter("newrelic", "errors", "Count of all newrelic errors", labels, "output") 124 | errors.Inc() 125 | } 126 | }() 127 | } 128 | 129 | func NewNewRelicOutput(wg *sync.WaitGroup, 130 | options NewRelicOutputOptions, 131 | templateOptions toolsRender.TemplateOptions, 132 | observability *common.Observability, 133 | newrelicEventer *sreProvider.NewRelicEventer) *NewRelicOutput { 134 | 135 | logger := observability.Logs() 136 | if newrelicEventer == nil { 137 | logger.Debug("NewRelic eventer is not defined. Skipped") 138 | return nil 139 | } 140 | 141 | messageOpts := toolsRender.TemplateOptions{ 142 | Name: "newrelic-message", 143 | Content: common.Content(options.Message), 144 | TimeFormat: templateOptions.TimeFormat, 145 | } 146 | message, err := toolsRender.NewTextTemplate(messageOpts, observability) 147 | if err != nil { 148 | logger.Error(err) 149 | return nil 150 | } 151 | 152 | attributesOpts := toolsRender.TemplateOptions{ 153 | Name: "newrelic-attributes", 154 | Content: common.Content(options.AttributesSelector), 155 | TimeFormat: templateOptions.TimeFormat, 156 | } 157 | attributes, err := toolsRender.NewTextTemplate(attributesOpts, observability) 158 | if err != nil { 159 | logger.Error(err) 160 | } 161 | 162 | return &NewRelicOutput{ 163 | wg: wg, 164 | message: message, 165 | attributes: attributes, 166 | options: options, 167 | logger: logger, 168 | meter: observability.Metrics(), 169 | newrelicEventer: newrelicEventer, 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /processor/site24x7.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "encoding/json" 5 | errPkg "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "strings" 10 | "time" 11 | 12 | "github.com/devopsext/events/common" 13 | sreCommon "github.com/devopsext/sre/common" 14 | ) 15 | 16 | type Site24x7Processor struct { 17 | outputs *common.Outputs 18 | logger sreCommon.Logger 19 | meter sreCommon.Meter 20 | } 21 | 22 | type Site24x7Request struct { 23 | MonitorID int64 `json:"MONITOR_ID"` 24 | MonitorDashboardLink string `json:"MONITOR_DASHBOARD_LINK"` 25 | MonitorType string `json:"MONITORTYPE"` 26 | MonitorName string `json:"MONITORNAME"` 27 | MonitorURL string `json:"MONITORURL"` 28 | MonitorGroupName string `json:"MONITOR_GROUPNAME"` 29 | 30 | IncidentReason string `json:"INCIDENT_REASON"` 31 | IncidentTime string `json:"INCIDENT_TIME"` 32 | IncidentTimeISO string `json:"INCIDENT_TIME_ISO"` 33 | 34 | PollFrequency int `json:"POLLFREQUENCY"` 35 | Status string `json:"STATUS"` 36 | FailedLocations string `json:"FAILED_LOCATIONS"` 37 | GroupTags []string `json:"GROUP_TAGS,omitempty"` 38 | Tags []string `json:"TAGS,omitempty"` 39 | } 40 | 41 | type Site24x7Response struct { 42 | Message string 43 | } 44 | 45 | func Site24x7ProcessorType() string { 46 | return "Site24x7" 47 | } 48 | 49 | func (p *Site24x7Processor) EventType() string { 50 | return common.AsEventType(Site24x7ProcessorType()) 51 | } 52 | 53 | func (p *Site24x7Processor) send(channel string, o interface{}, t *time.Time) { 54 | 55 | e := &common.Event{ 56 | Channel: channel, 57 | Type: p.EventType(), 58 | Data: o, 59 | } 60 | if t != nil && (*t).UnixNano() > 0 { 61 | e.SetTime((*t).UTC()) 62 | } else { 63 | e.SetTime(time.Now().UTC()) 64 | } 65 | e.SetLogger(p.logger) 66 | p.outputs.Send(e) 67 | } 68 | 69 | func (p *Site24x7Processor) HandleEvent(e *common.Event) error { 70 | 71 | if e == nil { 72 | p.logger.Debug("Event is not defined") 73 | return nil 74 | } 75 | 76 | labels := make(map[string]string) 77 | labels["event_channel"] = e.Channel 78 | labels["processor"] = p.EventType() 79 | 80 | requests := p.meter.Counter("site24x7", "requests", "Count of all site24x7 processor requests", labels, "processor") 81 | requests.Inc() 82 | 83 | p.outputs.Send(e) 84 | return nil 85 | } 86 | 87 | func (p *Site24x7Processor) HandleHttpRequest(w http.ResponseWriter, r *http.Request) error { 88 | 89 | channel := strings.TrimLeft(r.URL.Path, "/") 90 | 91 | labels := make(map[string]string) 92 | labels["path"] = r.URL.Path 93 | labels["processor"] = p.EventType() 94 | 95 | requests := p.meter.Counter("site24x7", "requests", "Count of all site24x7 processor requests", labels, "processor") 96 | requests.Inc() 97 | 98 | errors := p.meter.Counter("site24x7", "errors", "Count of all site24x7 processor errors", labels, "processor") 99 | 100 | var body []byte 101 | if r.Body != nil { 102 | if data, err := ioutil.ReadAll(r.Body); err == nil { 103 | body = data 104 | } 105 | } 106 | 107 | if len(body) == 0 { 108 | errors.Inc() 109 | err := errPkg.New("empty body") 110 | p.logger.Error(err) 111 | http.Error(w, err.Error(), http.StatusBadRequest) 112 | return err 113 | } 114 | 115 | p.logger.Debug("Body => %s", body) 116 | 117 | var site24x7 Site24x7Request 118 | if err := json.Unmarshal(body, &site24x7); err != nil { 119 | errors.Inc() 120 | p.logger.Error(err) 121 | http.Error(w, "Error unmarshaling message", http.StatusInternalServerError) 122 | return err 123 | } 124 | 125 | // 2022-03-18T01:51:19-0700" 126 | t, err := time.Parse("2006-01-02T15:04:05-0700", site24x7.IncidentTimeISO) 127 | if err != nil { 128 | errors.Inc() 129 | p.logger.Error(err) 130 | http.Error(w, "Error incident time ISO format", http.StatusInternalServerError) 131 | return err 132 | } 133 | p.send(channel, site24x7, &t) 134 | 135 | response := &Site24x7Response{ 136 | Message: "OK", 137 | } 138 | 139 | resp, err := json.Marshal(response) 140 | if err != nil { 141 | errors.Inc() 142 | p.logger.Error("Can't encode response: %v", err) 143 | http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError) 144 | return err 145 | } 146 | 147 | if _, err := w.Write(resp); err != nil { 148 | errors.Inc() 149 | p.logger.Error("Can't write response: %v", err) 150 | http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) 151 | return err 152 | } 153 | return nil 154 | } 155 | 156 | func NewSite24x7Processor(outputs *common.Outputs, observability *common.Observability) *Site24x7Processor { 157 | 158 | return &Site24x7Processor{ 159 | outputs: outputs, 160 | logger: observability.Logs(), 161 | meter: observability.Metrics(), 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /output/pubsub.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "strings" 7 | "sync" 8 | 9 | "cloud.google.com/go/pubsub" 10 | "github.com/devopsext/events/common" 11 | sreCommon "github.com/devopsext/sre/common" 12 | toolsRender "github.com/devopsext/tools/render" 13 | "github.com/devopsext/utils" 14 | "google.golang.org/api/option" 15 | ) 16 | 17 | type PubSubOutputOptions struct { 18 | Credentials string 19 | ProjectID string 20 | Message string 21 | TopicSelector string 22 | } 23 | 24 | type PubSubOutput struct { 25 | wg *sync.WaitGroup 26 | client *pubsub.Client 27 | ctx context.Context 28 | message *toolsRender.TextTemplate 29 | selector *toolsRender.TextTemplate 30 | options PubSubOutputOptions 31 | meter sreCommon.Meter 32 | logger sreCommon.Logger 33 | } 34 | 35 | func (ps *PubSubOutput) Name() string { 36 | return "PubSub" 37 | } 38 | 39 | func (ps *PubSubOutput) Send(event *common.Event) { 40 | 41 | ps.wg.Add(1) 42 | go func() { 43 | defer ps.wg.Done() 44 | 45 | if event == nil { 46 | ps.logger.Debug("Event is empty") 47 | return 48 | } 49 | 50 | if event.Data == nil { 51 | ps.logger.Error("Event data is empty") 52 | return 53 | } 54 | 55 | jsonObject, err := event.JsonObject() 56 | if err != nil { 57 | ps.logger.Error(err) 58 | return 59 | } 60 | 61 | topics := "" 62 | if ps.selector != nil { 63 | b, err := ps.selector.RenderObject(jsonObject) 64 | if err != nil { 65 | ps.logger.Debug(err) 66 | } else { 67 | topics = string(b) 68 | } 69 | } 70 | 71 | if utils.IsEmpty(topics) { 72 | ps.logger.Error("PubSub topics are not found") 73 | return 74 | } 75 | 76 | b, err := ps.message.RenderObject(jsonObject) 77 | if err != nil { 78 | ps.logger.Error(err) 79 | return 80 | } 81 | 82 | message := strings.TrimSpace(string(b)) 83 | if utils.IsEmpty(message) { 84 | ps.logger.Debug("PubSub message is empty") 85 | return 86 | } 87 | 88 | ps.logger.Debug("PubSub message => %s", message) 89 | 90 | arr := strings.Split(topics, "\n") 91 | for _, topic := range arr { 92 | topic = strings.TrimSpace(topic) 93 | if utils.IsEmpty(topic) { 94 | continue 95 | } 96 | 97 | labels := make(map[string]string) 98 | labels["event_channel"] = event.Channel 99 | labels["event_type"] = event.Type 100 | labels["pubsub_project_id"] = ps.options.ProjectID 101 | labels["pubsub_topic"] = ps.options.TopicSelector 102 | labels["output"] = ps.Name() 103 | 104 | requests := ps.meter.Counter("pubsub", "requests", "Count of all pubsub requests", labels, "output") 105 | requests.Inc() 106 | 107 | t := ps.client.Topic(topic) 108 | serverID, err := t.Publish(ps.ctx, &pubsub.Message{Data: []byte(message)}).Get(ps.ctx) 109 | if err != nil { 110 | errors := ps.meter.Counter("pubsub", "errors", "Count of all pubsub errors", labels, "output") 111 | errors.Inc() 112 | ps.logger.Error(err) 113 | continue 114 | } 115 | ps.logger.Debug("PubSub server ID => %s", serverID) 116 | } 117 | }() 118 | } 119 | 120 | func NewPubSubOutput(wg *sync.WaitGroup, 121 | options PubSubOutputOptions, 122 | templateOptions toolsRender.TemplateOptions, 123 | observability *common.Observability) *PubSubOutput { 124 | logger := observability.Logs() 125 | if utils.IsEmpty(options.Credentials) || utils.IsEmpty(options.ProjectID) { 126 | logger.Debug("PubSub output credentials or project ID is not defined. Skipped") 127 | return nil 128 | } 129 | 130 | var o option.ClientOption 131 | if _, err := os.Stat(options.Credentials); err == nil { 132 | o = option.WithCredentialsFile(options.Credentials) 133 | } else { 134 | o = option.WithCredentialsJSON([]byte(options.Credentials)) 135 | } 136 | 137 | ctx := context.Background() 138 | client, err := pubsub.NewClient(ctx, options.ProjectID, o) 139 | if err != nil { 140 | logger.Error(err) 141 | return nil 142 | } 143 | 144 | messageOpts := toolsRender.TemplateOptions{ 145 | Name: "pubsub-message", 146 | Content: common.Content(options.Message), 147 | TimeFormat: templateOptions.TimeFormat, 148 | } 149 | message, err := toolsRender.NewTextTemplate(messageOpts, observability) 150 | if err != nil { 151 | logger.Error(err) 152 | return nil 153 | } 154 | 155 | selectorOpts := toolsRender.TemplateOptions{ 156 | Name: "pubsub-selector", 157 | Content: common.Content(options.TopicSelector), 158 | TimeFormat: templateOptions.TimeFormat, 159 | } 160 | selector, err := toolsRender.NewTextTemplate(selectorOpts, observability) 161 | if err != nil { 162 | logger.Error(err) 163 | } 164 | 165 | return &PubSubOutput{ 166 | wg: wg, 167 | client: client, 168 | ctx: ctx, 169 | message: message, 170 | selector: selector, 171 | options: options, 172 | logger: logger, 173 | meter: observability.Metrics(), 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /zbx_export_mediatypes.yaml: -------------------------------------------------------------------------------- 1 | zabbix_export: 2 | version: '6.0' 3 | date: '2022-09-16T08:45:03Z' 4 | media_types: 5 | - 6 | name: 'ZabbixEvent - sre' 7 | type: WEBHOOK 8 | parameters: 9 | - 10 | name: AlertURL 11 | value: 'https://zabbix.fqdn/tr_events.php?triggerid={TRIGGER.ID}&eventid={EVENT.ID}' 12 | - 13 | name: Environment 14 | value: zabbix.test.env 15 | - 16 | name: EventDate 17 | value: '{EVENT.DATE}' 18 | - 19 | name: EventID 20 | value: '{EVENT.ID}' 21 | - 22 | name: EventName 23 | value: '{EVENT.NAME}' 24 | - 25 | name: EventNSeverity 26 | value: '{EVENT.NSEVERITY}' 27 | - 28 | name: EventOpData 29 | value: '{EVENT.OPDATA}' 30 | - 31 | name: EventStatus 32 | value: '{EVENT.Status}' 33 | - 34 | name: EventTags 35 | value: '{EVENT.TAGS}' 36 | - 37 | name: EventTime 38 | value: '{EVENT.TIME}' 39 | - 40 | name: EventType 41 | value: ZabbixEvent 42 | - 43 | name: HostName 44 | value: '{HOST.NAME}' 45 | - 46 | name: ItemID 47 | value: '{ITEM.ID}' 48 | - 49 | name: ItemLastValue 50 | value: '{ITEM.LASTVALUE}' 51 | - 52 | name: TriggerDescription 53 | value: '{TRIGGER.DESCRIPTION}' 54 | - 55 | name: TriggerExpression 56 | value: '{TRIGGER.EXPRESSION}' 57 | - 58 | name: TriggerName 59 | value: '{TRIGGER.NAME}' 60 | script: | 61 | try { 62 | Zabbix.Log(4, '[ Events webhook ] Started with params: ' + value); 63 | params = JSON.parse(value) 64 | req = new CurlHttpRequest() 65 | 66 | req.AddHeader('Content-Type: application/json'); 67 | 68 | resp = req.Post('https://catcher.fqdn/events?source=zabbix', 69 | JSON.stringify(params) 70 | ); 71 | 72 | if (req.Status() != 200) { 73 | throw 'Response code: ' + req.Status(); 74 | } 75 | return resp; 76 | } 77 | catch (error) { 78 | Zabbix.Log(3, '[ Events webhook ] issue creation failed : ' + error); 79 | throw 'Failed with error: ' + error; 80 | } 81 | process_tags: 'YES' 82 | description: 'SRE Events ' 83 | message_templates: 84 | - 85 | event_source: TRIGGERS 86 | operation_mode: PROBLEM 87 | subject: 'Problem: {EVENT.NAME}' 88 | message: | 89 | Problem started at {EVENT.TIME} on {EVENT.DATE} 90 | Problem name: {EVENT.NAME} 91 | Host: {HOST.NAME} 92 | Severity: {EVENT.SEVERITY} 93 | Operational data: {EVENT.OPDATA} 94 | Original problem ID: {EVENT.ID} 95 | {TRIGGER.URL} 96 | - 97 | event_source: TRIGGERS 98 | operation_mode: RECOVERY 99 | subject: 'Resolved: {EVENT.RECOVERY.NAME}' 100 | message: | 101 | Problem has been resolved at {EVENT.RECOVERY.TIME} on {EVENT.RECOVERY.DATE} 102 | Problem name: {EVENT.RECOVERY.NAME} 103 | Host: {HOST.NAME} 104 | Severity: {EVENT.SEVERITY} 105 | Original problem ID: {EVENT.ID} 106 | {TRIGGER.URL} 107 | - 108 | event_source: TRIGGERS 109 | operation_mode: UPDATE 110 | subject: 'Updated problem: {EVENT.NAME}' 111 | message: | 112 | {USER.FULLNAME} {EVENT.UPDATE.ACTION} problem at {EVENT.UPDATE.DATE} {EVENT.UPDATE.TIME}. 113 | {EVENT.UPDATE.MESSAGE} 114 | 115 | Current problem status is {EVENT.STATUS}, acknowledged: {EVENT.ACK.STATUS}. 116 | - 117 | event_source: DISCOVERY 118 | operation_mode: PROBLEM 119 | subject: 'Discovery: {DISCOVERY.DEVICE.STATUS} {DISCOVERY.DEVICE.IPADDRESS}' 120 | message: | 121 | Discovery rule: {DISCOVERY.RULE.NAME} 122 | 123 | Device IP: {DISCOVERY.DEVICE.IPADDRESS} 124 | Device DNS: {DISCOVERY.DEVICE.DNS} 125 | Device status: {DISCOVERY.DEVICE.STATUS} 126 | Device uptime: {DISCOVERY.DEVICE.UPTIME} 127 | 128 | Device service name: {DISCOVERY.SERVICE.NAME} 129 | Device service port: {DISCOVERY.SERVICE.PORT} 130 | Device service status: {DISCOVERY.SERVICE.STATUS} 131 | Device service uptime: {DISCOVERY.SERVICE.UPTIME} 132 | - 133 | event_source: AUTOREGISTRATION 134 | operation_mode: PROBLEM 135 | subject: 'Autoregistration: {HOST.HOST}' 136 | message: | 137 | Host name: {HOST.HOST} 138 | Host IP: {HOST.IP} 139 | Agent port: {HOST.PORT} 140 | -------------------------------------------------------------------------------- /output/grafana.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | "sync" 7 | "time" 8 | 9 | "golang.org/x/time/rate" 10 | 11 | "github.com/devopsext/events/common" 12 | sreCommon "github.com/devopsext/sre/common" 13 | sreProvider "github.com/devopsext/sre/provider" 14 | toolsRender "github.com/devopsext/tools/render" 15 | "github.com/devopsext/utils" 16 | ) 17 | 18 | type GrafanaOutputOptions struct { 19 | Name string 20 | Message string 21 | AttributesSelector string 22 | } 23 | 24 | type GrafanaOutput struct { 25 | wg *sync.WaitGroup 26 | message *toolsRender.TextTemplate 27 | attributes *toolsRender.TextTemplate 28 | options GrafanaOutputOptions 29 | logger sreCommon.Logger 30 | meter sreCommon.Meter 31 | grafanaEventer *sreProvider.GrafanaEventer 32 | rateLimiter *rate.Limiter 33 | } 34 | 35 | func (g *GrafanaOutput) Name() string { 36 | return "Grafana" 37 | } 38 | 39 | func (g *GrafanaOutput) getAttributes(o interface{}) (map[string]string, error) { 40 | 41 | attrs := make(map[string]string) 42 | if g.attributes == nil { 43 | return attrs, nil 44 | } 45 | 46 | a, err := g.attributes.RenderObject(o) 47 | if err != nil { 48 | return attrs, err 49 | } 50 | 51 | m := string(a) 52 | if utils.IsEmpty(m) { 53 | return attrs, nil 54 | } 55 | 56 | g.logger.Debug("Grafana raw attributes => %s", m) 57 | 58 | var object map[string]interface{} 59 | 60 | if err := json.Unmarshal([]byte(m), &object); err != nil { 61 | return attrs, err 62 | } 63 | 64 | for k, v := range object { 65 | vs, ok := v.(string) 66 | if ok { 67 | attrs[k] = vs 68 | } 69 | } 70 | 71 | return attrs, nil 72 | } 73 | 74 | func (g *GrafanaOutput) Send(event *common.Event) { 75 | 76 | g.wg.Add(1) 77 | go func() { 78 | defer g.wg.Done() 79 | 80 | if event == nil { 81 | g.logger.Debug("Event is empty") 82 | return 83 | } 84 | if event.Data == nil { 85 | g.logger.Error("Event data is empty") 86 | return 87 | } 88 | 89 | jsonObject, err := event.JsonObject() 90 | if err != nil { 91 | g.logger.Error(err) 92 | return 93 | } 94 | 95 | b, err := g.message.RenderObject(jsonObject) 96 | if err != nil { 97 | g.logger.Error(err) 98 | return 99 | } 100 | message := strings.TrimSpace(string(b)) 101 | if utils.IsEmpty(message) { 102 | g.logger.Debug("Grafana message is empty") 103 | return 104 | } 105 | g.logger.Debug("Grafana message => %s", message) 106 | 107 | attributes, err := g.getAttributes(jsonObject) 108 | if err != nil { 109 | g.logger.Error(err) 110 | } 111 | g.logger.Debug("Grafana attributes => %s", attributes) 112 | 113 | labels := make(map[string]string) 114 | labels["event_channel"] = event.Channel 115 | labels["event_type"] = event.Type 116 | labels["output"] = g.Name() 117 | 118 | rateLimiterIn := g.meter.Counter("grafana", "rl_in", "Count of all grafana requests before waiting", labels, "output", "rate_limiter") 119 | rateLimiterIn.Inc() 120 | r := g.rateLimiter.Reserve() 121 | // TODO increment another counter events_grafana_output_ratelimiter_wait_time_total by r.Delay*time.Millisecond 122 | time.Sleep(r.Delay()) 123 | 124 | rateLimiterOut := g.meter.Counter("grafana", "rl_out", "Count of all grafana requests after waiting", labels, "output", "rate_limiter") 125 | rateLimiterOut.Inc() 126 | 127 | requests := g.meter.Counter("grafana", "requests", "Count of all grafana requests", labels, "output") 128 | requests.Inc() 129 | 130 | err = g.grafanaEventer.Interval(message, message, attributes, event.Time, event.Time) 131 | if err != nil { 132 | errors := g.meter.Counter("grafana", "errors", "Count of all grafana errors", labels, "output") 133 | errors.Inc() 134 | } 135 | }() 136 | } 137 | 138 | func NewGrafanaOutput(wg *sync.WaitGroup, 139 | options GrafanaOutputOptions, 140 | templateOptions toolsRender.TemplateOptions, 141 | observability *common.Observability, 142 | grafanaEventer *sreProvider.GrafanaEventer) *GrafanaOutput { 143 | 144 | logger := observability.Logs() 145 | if grafanaEventer == nil { 146 | logger.Debug("Grafana eventer is not defined. Skipped") 147 | return nil 148 | } 149 | 150 | messageOpts := toolsRender.TemplateOptions{ 151 | Name: "grafana-message", 152 | Content: common.Content(options.Message), 153 | TimeFormat: templateOptions.TimeFormat, 154 | } 155 | message, err := toolsRender.NewTextTemplate(messageOpts, observability) 156 | if err != nil { 157 | logger.Error(err) 158 | return nil 159 | } 160 | 161 | selectorOpts := toolsRender.TemplateOptions{ 162 | Name: "grafana-attributes", 163 | Content: common.Content(options.AttributesSelector), 164 | TimeFormat: templateOptions.TimeFormat, 165 | } 166 | attributes, err := toolsRender.NewTextTemplate(selectorOpts, observability) 167 | if err != nil { 168 | logger.Error(err) 169 | } 170 | 171 | return &GrafanaOutput{ 172 | rateLimiter: rate.NewLimiter(rate.Every(time.Minute/time.Duration(30)), 1), 173 | wg: wg, 174 | message: message, 175 | attributes: attributes, 176 | options: options, 177 | logger: logger, 178 | meter: observability.Metrics(), 179 | grafanaEventer: grafanaEventer, 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /processor/winevent.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "encoding/json" 5 | errPkg "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "strings" 10 | "time" 11 | 12 | "github.com/devopsext/events/common" 13 | sreCommon "github.com/devopsext/sre/common" 14 | ) 15 | 16 | type WinEventPubsubRequest struct { 17 | Time string `json:"time"` 18 | Channel string `json:"channel"` 19 | Original *WinEventOriginalRequest `json:"data"` 20 | } 21 | type WinEventOriginalRequest struct { 22 | Events []WinEvent `json:"metrics"` 23 | } 24 | type WinEventTags struct { 25 | Message string `json:"Message"` 26 | MessageLevel string `json:"LevelText"` 27 | Keywords string `json:"Keywords"` 28 | EventID string `json:"EventRecordID"` 29 | Provider string `json:"provider"` 30 | City string `json:"city"` 31 | Country string `json:"country,omitempty"` 32 | MT string `json:"mt"` 33 | Host string `json:"host"` 34 | } 35 | type WinEvent struct { 36 | Name string `json:"name"` 37 | Tags *WinEventTags `json:"tags"` 38 | Timestamp int64 `json:"timestamp"` 39 | } 40 | 41 | type WinEventResponse struct { 42 | Message string 43 | } 44 | 45 | type WinEventProcessor struct { 46 | outputs *common.Outputs 47 | logger sreCommon.Logger 48 | meter sreCommon.Meter 49 | } 50 | 51 | func WinEventProcessorType() string { 52 | return "Win" 53 | } 54 | 55 | func (p *WinEventProcessor) EventType() string { 56 | return common.AsEventType(WinEventProcessorType()) 57 | } 58 | 59 | func (p *WinEventProcessor) HandleEvent(e *common.Event) error { 60 | 61 | labels := make(map[string]string) 62 | labels["event_channel"] = e.Channel 63 | labels["processor"] = p.EventType() 64 | 65 | requests := p.meter.Counter("winevent", "requests", "Count of all winevent processor requests", labels, "processor") 66 | requests.Inc() 67 | 68 | errors := p.meter.Counter("winevent", "errors", "Count of all winevent processor errors", labels, "processor") 69 | 70 | if e == nil { 71 | errors.Inc() 72 | p.logger.Debug("Event is not defined") 73 | return nil 74 | } 75 | if js, err := e.JsonBytes(); err == nil { 76 | var PubSubEvents WinEventPubsubRequest 77 | if err := json.Unmarshal(js, &PubSubEvents); err != nil { 78 | errors.Inc() 79 | p.logger.Error("Failed while unmarshalling: %s", err) 80 | return err 81 | } 82 | p.logger.Debug("After repeatitive unmarshall we got: %s", PubSubEvents) 83 | for _, event := range PubSubEvents.Original.Events { 84 | newEvent := &common.Event{ 85 | Channel: e.Channel, 86 | Type: e.Type, 87 | Data: event, 88 | } 89 | t := time.UnixMilli(event.Timestamp * 1000) 90 | newEvent.SetTime(t.UTC()) 91 | requests.Inc() 92 | p.outputs.Send(newEvent) 93 | } 94 | } 95 | return nil 96 | } 97 | 98 | func (p *WinEventProcessor) HandleHttpRequest(w http.ResponseWriter, r *http.Request) error { 99 | 100 | channel := strings.TrimLeft(r.URL.Path, "/") 101 | 102 | labels := make(map[string]string) 103 | labels["path"] = r.URL.Path 104 | labels["processor"] = p.EventType() 105 | 106 | requests := p.meter.Counter("winevent", "requests", "Count of all winevent processor requests", labels, "processor") 107 | requests.Inc() 108 | 109 | errors := p.meter.Counter("winevent", "errors", "Count of all winevent processor errors", labels, "processor") 110 | 111 | var body []byte 112 | if r.Body != nil { 113 | if data, err := ioutil.ReadAll(r.Body); err == nil { 114 | body = data 115 | } 116 | } 117 | 118 | if len(body) == 0 { 119 | errors.Inc() 120 | err := errPkg.New("empty body") 121 | p.logger.Error(err) 122 | http.Error(w, err.Error(), http.StatusBadRequest) 123 | return err 124 | } 125 | p.logger.Debug("Body => %s", body) 126 | 127 | var WinEvents WinEventOriginalRequest 128 | if err := json.Unmarshal(body, &WinEvents); err != nil { 129 | errors.Inc() 130 | p.logger.Error(err) 131 | http.Error(w, "Error unmarshaling message", http.StatusInternalServerError) 132 | return err 133 | } 134 | for _, event := range WinEvents.Events { 135 | t := time.UnixMilli(event.Timestamp) 136 | p.send(channel, event, &t) 137 | } 138 | 139 | response := &WinEventResponse{ 140 | Message: "OK", 141 | } 142 | 143 | resp, err := json.Marshal(response) 144 | if err != nil { 145 | errors.Inc() 146 | p.logger.Error("Can't encode response: %v", err) 147 | http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError) 148 | return err 149 | } 150 | 151 | if _, err := w.Write(resp); err != nil { 152 | errors.Inc() 153 | p.logger.Error("Can't write response: %v", err) 154 | http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) 155 | return err 156 | } 157 | return nil 158 | } 159 | 160 | func (p *WinEventProcessor) send(channel string, event interface{}, t *time.Time) { 161 | e := &common.Event{ 162 | Channel: channel, 163 | Type: p.EventType(), 164 | Data: event, 165 | } 166 | if t != nil && t.UnixNano() > 0 { 167 | e.SetTime(t.UTC()) 168 | } else { 169 | e.SetTime(time.Now().UTC()) 170 | } 171 | e.SetLogger(p.logger) 172 | p.outputs.Send(e) 173 | } 174 | 175 | func NewWinEventProcessor(outputs *common.Outputs, observability *common.Observability) *WinEventProcessor { 176 | 177 | return &WinEventProcessor{ 178 | outputs: outputs, 179 | logger: observability.Logs(), 180 | meter: observability.Metrics(), 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /output/datadog.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | "sync" 7 | "time" 8 | 9 | "github.com/devopsext/events/common" 10 | sreCommon "github.com/devopsext/sre/common" 11 | sreProvider "github.com/devopsext/sre/provider" 12 | toolsRender "github.com/devopsext/tools/render" 13 | "github.com/devopsext/utils" 14 | ) 15 | 16 | type DataDogOutputOptions struct { 17 | Name string 18 | Message string 19 | AttributesSelector string 20 | } 21 | 22 | type DataDogOutput struct { 23 | wg *sync.WaitGroup 24 | name *toolsRender.TextTemplate 25 | message *toolsRender.TextTemplate 26 | attributes *toolsRender.TextTemplate 27 | options DataDogOutputOptions 28 | logger sreCommon.Logger 29 | meter sreCommon.Meter 30 | datadogEventer *sreProvider.DataDogEventer 31 | } 32 | 33 | func (d *DataDogOutput) Name() string { 34 | return "Datadog" 35 | } 36 | 37 | func (d *DataDogOutput) getAttributes(o interface{}) (map[string]string, error) { 38 | 39 | attrs := make(map[string]string) 40 | if d.attributes == nil { 41 | d.logger.Debug("output attributes not set") 42 | return attrs, nil 43 | } 44 | 45 | a, err := d.attributes.RenderObject(o) 46 | if err != nil { 47 | d.logger.Debug("error execute attributes template: %s", err) 48 | return attrs, err 49 | } 50 | 51 | m := string(a) 52 | if utils.IsEmpty(m) { 53 | d.logger.Debug("attributes are empty") 54 | return attrs, nil 55 | } 56 | 57 | d.logger.Debug("DataDog raw attributes => %s", m) 58 | 59 | var object map[string]interface{} 60 | 61 | if err := json.Unmarshal([]byte(m), &object); err != nil { 62 | return attrs, err 63 | } 64 | 65 | for k, v := range object { 66 | vs, ok := v.(string) 67 | if ok { 68 | attrs[k] = vs 69 | } 70 | } 71 | 72 | return attrs, nil 73 | } 74 | 75 | func (d *DataDogOutput) Send(event *common.Event) { 76 | d.wg.Add(1) 77 | go func() { 78 | defer d.wg.Done() 79 | 80 | if event == nil { 81 | d.logger.Debug("Event is empty") 82 | return 83 | } 84 | 85 | if event.Data == nil { 86 | d.logger.Error("Event data is empty") 87 | return 88 | } 89 | 90 | jsonObject, err := event.JsonObject() 91 | if err != nil { 92 | d.logger.Error(err) 93 | return 94 | } 95 | 96 | a, err := d.name.RenderObject(jsonObject) 97 | if err != nil { 98 | d.logger.Error(err) 99 | return 100 | } 101 | 102 | name := strings.TrimSpace(string(a)) 103 | if utils.IsEmpty(a) { 104 | d.logger.Debug("DataDog name is empty") 105 | return 106 | } 107 | 108 | b, err := d.message.RenderObject(jsonObject) 109 | if err != nil { 110 | d.logger.Error(err) 111 | return 112 | } 113 | 114 | message := strings.TrimSpace(string(b)) 115 | if utils.IsEmpty(message) { 116 | d.logger.Debug("DataDog message is empty") 117 | return 118 | } 119 | 120 | labels := make(map[string]string) 121 | labels["event_channel"] = event.Channel 122 | labels["event_type"] = event.Type 123 | labels["output"] = d.Name() 124 | 125 | requests := d.meter.Counter("datadog", "requests", "Count of all datadog requests", labels, "output") 126 | requests.Inc() 127 | d.logger.Debug("DataDog message => %s%s", name, message) 128 | 129 | attributes, err := d.getAttributes(jsonObject) 130 | if err != nil { 131 | d.logger.Error(err) 132 | } 133 | d.logger.Debug("Name: %s, Message: %s, Attributes: %s, Time: %s", name, message, strings.Join(utils.MapToArray(attributes), ","), event.Time.Format(time.RFC822)) 134 | 135 | err = d.datadogEventer.At(name, message, attributes, event.Time) 136 | if err != nil { 137 | errors := d.meter.Counter("datadog", "errors", "Count of all datadog errors", labels, "output") 138 | errors.Inc() 139 | } 140 | }() 141 | } 142 | 143 | func NewDataDogOutput(wg *sync.WaitGroup, 144 | options DataDogOutputOptions, 145 | templateOptions toolsRender.TemplateOptions, 146 | observability *common.Observability, 147 | datadogEventer *sreProvider.DataDogEventer) *DataDogOutput { 148 | 149 | logger := observability.Logs() 150 | if datadogEventer == nil { 151 | logger.Debug("DataDog eventer is not defined. Skipped") 152 | return nil 153 | } 154 | 155 | nameOpts := toolsRender.TemplateOptions{ 156 | Name: "datadog-name", 157 | Content: common.Content(options.Name), 158 | TimeFormat: templateOptions.TimeFormat, 159 | } 160 | name, err := toolsRender.NewTextTemplate(nameOpts, observability) 161 | if err != nil { 162 | logger.Error(err) 163 | return nil 164 | } 165 | 166 | messageOpts := toolsRender.TemplateOptions{ 167 | Name: "datadog-message", 168 | Content: common.Content(options.Message), 169 | TimeFormat: templateOptions.TimeFormat, 170 | } 171 | message, err := toolsRender.NewTextTemplate(messageOpts, observability) 172 | if err != nil { 173 | logger.Error(err) 174 | return nil 175 | } 176 | 177 | attributesOpts := toolsRender.TemplateOptions{ 178 | Name: "datadog-attributes", 179 | Content: common.Content(options.AttributesSelector), 180 | TimeFormat: templateOptions.TimeFormat, 181 | } 182 | attributes, err := toolsRender.NewTextTemplate(attributesOpts, observability) 183 | if err != nil { 184 | logger.Error(err) 185 | } 186 | 187 | return &DataDogOutput{ 188 | wg: wg, 189 | name: name, 190 | message: message, 191 | attributes: attributes, 192 | options: options, 193 | logger: logger, 194 | meter: observability.Metrics(), 195 | datadogEventer: datadogEventer, 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /processor/datadog.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "encoding/json" 5 | errPkg "errors" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "strings" 10 | "time" 11 | 12 | "github.com/devopsext/events/common" 13 | sreCommon "github.com/devopsext/sre/common" 14 | ) 15 | 16 | type DataDogProcessor struct { 17 | outputs *common.Outputs 18 | logger sreCommon.Logger 19 | meter sreCommon.Meter 20 | } 21 | 22 | type DataDogEvent struct { 23 | Type string `json:"type"` 24 | Msg string `json:"msg"` 25 | Title string `json:"title"` 26 | } 27 | 28 | type DataDogAlert struct { 29 | ID string `json:"id"` 30 | Metric string `json:"metric"` 31 | Priority string `json:"priority"` 32 | Query string `json:"query"` 33 | Scope string `json:"scope,omitempty"` 34 | Status string `json:"status"` 35 | Title string `json:"title"` 36 | Transition string `json:"transition"` 37 | Type string `json:"type"` 38 | } 39 | 40 | type DataDogIncident struct { 41 | Title string `json:"title"` 42 | } 43 | 44 | type DataDogMetric struct { 45 | Namespace string `json:"namespace"` 46 | } 47 | 48 | type DataDogSecurity struct { 49 | RuleName string `json:"rule_name"` 50 | } 51 | 52 | type DataDogOrg struct { 53 | ID string `json:"id"` 54 | Name string `json:"name"` 55 | } 56 | 57 | type DataDogRequest struct { 58 | ID string `json:"id"` 59 | Date int64 `json:"date"` 60 | LastUpdated int64 `json:"last_updated"` 61 | Link string `json:"link"` 62 | Priority string `json:"priority"` 63 | Snapshot string `json:"snapshot"` 64 | Event *DataDogEvent `json:"event"` 65 | Alert *DataDogAlert `json:"alert,omitempty"` 66 | Incident *DataDogIncident `json:"incident,omitempty"` 67 | Metric *DataDogMetric `json:"metric,omitempty"` 68 | Security *DataDogSecurity `json:"security,omitempty"` 69 | Org *DataDogOrg `json:"org,omitempty"` 70 | Tags string `json:"tags,omitempty"` 71 | TextOnlyMsg string `json:"text_only_msg,omitempty"` 72 | User string `json:"user,omitempty"` 73 | UserName string `json:"username,omitempty"` 74 | } 75 | 76 | type DataDogResponse struct { 77 | Message string 78 | } 79 | 80 | func DataDogProcessorType() string { 81 | return "DataDog" 82 | } 83 | 84 | func (p *DataDogProcessor) EventType() string { 85 | return common.AsEventType(DataDogProcessorType()) 86 | } 87 | 88 | func (p *DataDogProcessor) send(channel string, o interface{}, t *time.Time) { 89 | 90 | e := &common.Event{ 91 | Channel: channel, 92 | Type: p.EventType(), 93 | Data: o, 94 | } 95 | if t != nil && (*t).UnixNano() > 0 { 96 | e.SetTime((*t).UTC()) 97 | } else { 98 | e.SetTime(time.Now().UTC()) 99 | } 100 | e.SetLogger(p.logger) 101 | p.outputs.Send(e) 102 | } 103 | 104 | func (p *DataDogProcessor) HandleEvent(e *common.Event) error { 105 | 106 | if e == nil { 107 | p.logger.Debug("Event is not defined") 108 | return nil 109 | } 110 | 111 | labels := make(map[string]string) 112 | labels["event_channel"] = e.Channel 113 | labels["processor"] = p.EventType() 114 | 115 | requests := p.meter.Counter("datadog", "requests", "Count of all datadog processor requests", labels, "processor", "datadog") 116 | requests.Inc() 117 | 118 | p.outputs.Send(e) 119 | return nil 120 | } 121 | 122 | func (p *DataDogProcessor) HandleHttpRequest(w http.ResponseWriter, r *http.Request) error { 123 | 124 | channel := strings.TrimLeft(r.URL.Path, "/") 125 | 126 | labels := make(map[string]string) 127 | labels["path"] = r.URL.Path 128 | labels["processor"] = p.EventType() 129 | 130 | requests := p.meter.Counter("datadog", "requests", "Count of all datadog processor requests", labels, "processor") 131 | requests.Inc() 132 | 133 | errors := p.meter.Counter("datadog", "errors", "Count of all datadog processor errors", labels, "processor") 134 | 135 | var body []byte 136 | if r.Body != nil { 137 | if data, err := io.ReadAll(r.Body); err == nil { 138 | body = data 139 | } 140 | } 141 | 142 | if len(body) == 0 { 143 | errors.Inc() 144 | err := errPkg.New("empty body") 145 | p.logger.Error(err) 146 | http.Error(w, err.Error(), http.StatusBadRequest) 147 | return err 148 | } 149 | 150 | p.logger.Debug("Body => %s", body) 151 | 152 | var datadog DataDogRequest 153 | if err := json.Unmarshal(body, &datadog); err != nil { 154 | errors.Inc() 155 | p.logger.Error(err) 156 | http.Error(w, "Error unmarshaling message", http.StatusInternalServerError) 157 | return err 158 | } 159 | 160 | t := time.UnixMilli(datadog.LastUpdated) 161 | p.send(channel, datadog, &t) 162 | 163 | response := &DataDogResponse{ 164 | Message: "OK", 165 | } 166 | 167 | resp, err := json.Marshal(response) 168 | if err != nil { 169 | errors.Inc() 170 | p.logger.Error("Can't encode response: %v", err) 171 | http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError) 172 | return err 173 | } 174 | 175 | if _, err := w.Write(resp); err != nil { 176 | errors.Inc() 177 | p.logger.Error("Can't write response: %v", err) 178 | http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) 179 | return err 180 | } 181 | return nil 182 | } 183 | 184 | func NewDataDogProcessor(outputs *common.Outputs, observability *common.Observability) *DataDogProcessor { 185 | 186 | return &DataDogProcessor{ 187 | outputs: outputs, 188 | logger: observability.Logs(), 189 | meter: observability.Metrics(), 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /datadog.attributes: -------------------------------------------------------------------------------- 1 | {{- define "gitlab-commit"}} 2 | {{- if .commit.author}}{{- printf "\\nCommit author => %s" .commit.author.name}}{{else}}{{- printf "\\nCommit author => %s" .commit.author_name}}{{end}} 3 | {{- printf "\\nCommit message => "}}{{- regexReplaceAll "\n" "\\n" .commit.message}} 4 | {{- if .commit.url}}{{- printf "\\n%s" .commit.url}}{{else}}{{- printf "\\n%s/-/commit/%s" .repository.homepage .commit.sha}}{{end}} 5 | {{- end}} 6 | {{- define "gitlab-stages"}} 7 | {{- $match := .match}} 8 | {{- range .stages}} 9 | {{- if and (.runner.description | regexMatch $match) (not (empty .finished_at))}} 10 | {{- printf "\\n%s/%s [%s] => %s" .stage .name .user.username .runner.description}} 11 | {{- end}} 12 | {{- end}} 13 | {{- end}} 14 | {{- define "gitlab-pipeline"}} 15 | {{- $match := getEnv "EVENTS_GITLAB_RUNNERS"}}{{$ok := false}} 16 | {{- range .data.builds}} 17 | {{- if and (.runner.description | regexMatch $match) (not (empty .finished_at))}}{{$ok = true}}{{end}} 18 | {{- end}} 19 | {{- if $ok}} 20 | {{- template "gitlab-commit" (dict "repository" "" "commit" .data.commit)}} 21 | {{- template "gitlab-stages" (dict "stages" (reverse .data.builds) "match" $match)}} 22 | {{- printf "\\n%s/-/pipelines/%.0f" .data.project.web_url .data.object_attributes.id}} 23 | {{- end}} 24 | {{- end}} 25 | {{- define "gitlab-build"}} 26 | {{- $match := getEnv "EVENTS_GITLAB_RUNNERS"}} 27 | {{- if and (.data.runner.description | regexMatch $match) (not (empty .data.build_duration))}} 28 | {{- template "gitlab-commit" (dict "repository" .data.repository "commit" .data.commit)}} 29 | {{- printf "\\n%s/%s => %s" .data.build_stage .data.build_name .data.runner.description}} 30 | {{- printf "\\n%s/-/jobs/%.0f" .data.repository.homepage .data.build_id}} 31 | {{- end}} 32 | {{- end}} 33 | {{- define "text"}} 34 | {{- if eq .type "KubeEvent"}} 35 | {{- $type := .data.type}} 36 | {{- if or (regexMatch .data.reason "(NodeNotReady|Evicted|FailedScheduling|BackOff|EvictionThresholdMet)") (and (eq .data.reason "Failed") (or (regexMatch .data.message "^Failed to pull image.*") (regexMatch .data.message "^Failed to start container.*")))}} 37 | {{- $type = "Error"}} 38 | {{- end}} 39 | {{printf "{\"alert_type\":\"%s\",\"aggregation_key\":\"%s\",\"type\":\"%s\",\"source_type_name\":\"kubernetes\",\"location\":\"%s\",\"reason\":\"%s\"}" (toLower $type) (index (regexFindSubmatch "([a-zA-Z-]+)-kube" .channel) 1 ) .type .data.location .data.reason}} 40 | {{- end}} 41 | {{- if eq .type "K8sEvent"}} 42 | {{- $severity := ""}} 43 | {{- $op := .data.operation}} 44 | 45 | {{- if eq .data.operation "Delete"}} 46 | {{- $severity = "warning"}} 47 | {{- else if eq .data.operation "Update"}} 48 | {{- $oldImages := ""}}{{- $newImages := ""}} 49 | 50 | {{- if regexMatch "StatefulSet|Deployment|DaemonSet|ReplicaSet|Pod" .data.kind}} 51 | {{- range .data.object.old.spec.template.spec.containers}} 52 | {{- $oldImages = printf "%s%s => %s\n" $oldImages .name .image}} 53 | {{- end}} 54 | {{- range .data.object.new.spec.template.spec.containers}} 55 | {{- $newImages = printf "%s%s => %s\n" $newImages .name .image}} 56 | {{- end}} 57 | {{- else if eq .data.kind "Application"}} 58 | {{- $oldImages = printf "%s:%s\n" .data.object.old.spec.source.helm.valuesObject.image.repository .data.object.old.spec.source.helm.valuesObject.image.tag }} 59 | {{- $newImages = printf "%s:%s\n" .data.object.new.spec.source.helm.valuesObject.image.repository .data.object.new.spec.source.helm.valuesObject.image.tag }} 60 | {{- end}} 61 | 62 | {{- if ne $oldImages $newImages}} 63 | {{- $op = "Release"}} 64 | {{- $severity = "warning"}} 65 | {{- end}} 66 | 67 | {{- if ne .data.object.old.spec.replicas .data.object.new.spec.replicas}} 68 | {{- if ne $op "Release"}} 69 | {{- $op = "Scale"}} 70 | {{- end}} 71 | {{- end}} 72 | {{- else }} 73 | {{- $severity = "info"}} 74 | {{- end}} 75 | 76 | {{printf "{\"alert_type\":\"%s\",\"aggregation_key\":\"%s\",\"type\":\"%s\",\"source_type_name\":\"kubernetes\",\"kind\":\"%s\",\"location\":\"%s\",\"operation\":\"%s\"}" $severity .channel .type .data.kind .data.location $op}} 77 | 78 | {{- end}} 79 | {{- if eq .type "AlertmanagerEvent"}} 80 | {{printf "{\"type\":\"%s\",\"source_type_name\":\"%s\",\"alert\":\"%s\"}" .type .channel .data.labels.alertname}} 81 | {{- end}} 82 | {{- if eq .type "GitlabEvent"}} 83 | {{- $match := getEnv "EVENTS_GITLAB_RUNNERS"}}{{$ok := false}} 84 | {{- if .data.builds}} 85 | {{- range .data.builds}} 86 | {{- if and .runner (.runner.description | regexMatch $match) (not (empty .finished_at))}}{{$ok = true}}{{end}} 87 | {{- end}} 88 | {{- else}} 89 | {{- if and .data.runner (.data.runner.description | regexMatch $match) (not (empty .data.build_duration))}}{{$ok = true}}{{end}} 90 | {{- end}} 91 | {{- if $ok}} 92 | { 93 | "source_type_name":"gitlab", 94 | {{- if .data.project}}{{printf "\"namespace\":\"%s\",\"project\":\"%s\"" .data.project.namespace .data.project.name}} 95 | {{- else}}{{printf "\"project\":\"%s\"" .data.project_name}}{{end}}, 96 | "text":"{{- if eq .data.object_kind "pipeline"}}{{template "gitlab-pipeline" .}}{{end}} 97 | {{- if eq .data.object_kind "build"}}{{template "gitlab-build" .}}{{end}}" 98 | } 99 | {{- end}} 100 | {{- end}} 101 | {{- end}} 102 | {{- define "datadog-attributes"}}{{template "text" .}}{{- end}} -------------------------------------------------------------------------------- /processor/gitlab.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | errPkg "errors" 7 | "fmt" 8 | "io" 9 | "net/http" 10 | "strings" 11 | "time" 12 | 13 | "github.com/devopsext/events/common" 14 | sreCommon "github.com/devopsext/sre/common" 15 | "github.com/go-playground/webhooks/v6/gitlab" 16 | ) 17 | 18 | type GitlabProcessor struct { 19 | outputs *common.Outputs 20 | logger sreCommon.Logger 21 | meter sreCommon.Meter 22 | hook *gitlab.Webhook 23 | } 24 | 25 | type GitlabResponse struct { 26 | Message string 27 | } 28 | 29 | func GitlabProcessorType() string { 30 | return "Gitlab" 31 | } 32 | 33 | func (p *GitlabProcessor) EventType() string { 34 | return common.AsEventType(GitlabProcessorType()) 35 | } 36 | 37 | func (p *GitlabProcessor) send(channel string, o interface{}, t *time.Time) { 38 | 39 | e := &common.Event{ 40 | Channel: channel, 41 | Type: p.EventType(), 42 | Data: o, 43 | } 44 | if t != nil && (*t).UnixNano() > 0 { 45 | e.SetTime((*t).UTC()) 46 | } else { 47 | e.SetTime(time.Now().UTC()) 48 | } 49 | e.SetLogger(p.logger) 50 | p.outputs.Send(e) 51 | } 52 | 53 | func (p *GitlabProcessor) HandleEvent(e *common.Event) error { 54 | 55 | if e == nil { 56 | p.logger.Debug("Event is not defined") 57 | return nil 58 | } 59 | 60 | labels := make(map[string]string) 61 | labels["event_channel"] = e.Channel 62 | labels["processor"] = p.EventType() 63 | 64 | requests := p.meter.Counter("gitlab", "requests", "Count of all gitlab processor requests", labels, "processor") 65 | requests.Inc() 66 | 67 | p.outputs.Send(e) 68 | return nil 69 | } 70 | 71 | func (p *GitlabProcessor) HandleHttpRequest(w http.ResponseWriter, r *http.Request) error { 72 | 73 | channel := strings.TrimLeft(r.URL.Path, "/") 74 | 75 | labels := make(map[string]string) 76 | labels["path"] = r.URL.Path 77 | labels["processor"] = p.EventType() 78 | 79 | requests := p.meter.Counter("gitlab", "requests", "Count of all gitlab processor requests", labels, "processor") 80 | requests.Inc() 81 | 82 | errors := p.meter.Counter("gitlab", "errors", "Count of all gitlab processor errors", labels, "processor") 83 | 84 | var body []byte 85 | if r.Body != nil { 86 | if data, err := io.ReadAll(r.Body); err == nil { 87 | body = data 88 | } 89 | } 90 | 91 | if len(body) == 0 { 92 | errors.Inc() 93 | err := errPkg.New("empty body") 94 | p.logger.Error(err) 95 | http.Error(w, err.Error(), http.StatusBadRequest) 96 | return err 97 | } 98 | 99 | p.logger.Debug("Body => %s", body) 100 | 101 | r.Body.Close() 102 | r.Body = io.NopCloser(bytes.NewBuffer(body)) 103 | 104 | events := []gitlab.Event{gitlab.PushEvents, gitlab.TagEvents, gitlab.IssuesEvents, gitlab.ConfidentialIssuesEvents, gitlab.CommentEvents, 105 | gitlab.MergeRequestEvents, gitlab.WikiPageEvents, gitlab.PipelineEvents, gitlab.BuildEvents, gitlab.JobEvents, gitlab.SystemHookEvents} 106 | payload, err := p.hook.Parse(r, events...) 107 | if err != nil { 108 | errors.Inc() 109 | p.logger.Error(err) 110 | http.Error(w, err.Error(), http.StatusBadRequest) 111 | return err 112 | } 113 | 114 | switch pl := payload.(type) { 115 | case gitlab.PushEventPayload: 116 | p.send(channel, payload.(gitlab.PushEventPayload), nil) 117 | case gitlab.TagEventPayload: 118 | p.send(channel, payload.(gitlab.TagEventPayload), nil) 119 | case gitlab.IssueEventPayload: 120 | event := payload.(gitlab.IssueEventPayload) 121 | p.send(channel, event, &event.ObjectAttributes.CreatedAt.Time) 122 | case gitlab.ConfidentialIssueEventPayload: 123 | event := payload.(gitlab.ConfidentialIssueEventPayload) 124 | p.send(channel, event, &event.ObjectAttributes.CreatedAt.Time) 125 | case gitlab.CommentEventPayload: 126 | event := payload.(gitlab.CommentEventPayload) 127 | p.send(channel, event, &event.ObjectAttributes.CreatedAt.Time) 128 | case gitlab.MergeRequestEventPayload: 129 | event := payload.(gitlab.MergeRequestEventPayload) 130 | p.send(channel, event, &event.ObjectAttributes.CreatedAt.Time) 131 | case gitlab.WikiPageEventPayload: 132 | event := payload.(gitlab.WikiPageEventPayload) 133 | p.send(channel, event, &event.ObjectAttributes.CreatedAt.Time) 134 | case gitlab.PipelineEventPayload: 135 | event := payload.(gitlab.PipelineEventPayload) 136 | p.send(channel, event, &event.ObjectAttributes.CreatedAt.Time) 137 | case gitlab.BuildEventPayload: 138 | event := payload.(gitlab.BuildEventPayload) 139 | p.send(channel, event, &event.BuildStartedAt.Time) 140 | case gitlab.JobEventPayload: 141 | event := payload.(gitlab.JobEventPayload) 142 | p.send(channel, event, &event.BuildStartedAt.Time) 143 | case gitlab.SystemHookPayload: 144 | p.send(channel, payload.(gitlab.SystemHookPayload), nil) 145 | default: 146 | p.logger.Debug("Not supported %s", pl) 147 | } 148 | 149 | response := &GitlabResponse{ 150 | Message: "OK", 151 | } 152 | 153 | resp, err := json.Marshal(response) 154 | if err != nil { 155 | errors.Inc() 156 | p.logger.Error("Can't encode response: %v", err) 157 | http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError) 158 | return err 159 | } 160 | 161 | if _, err := w.Write(resp); err != nil { 162 | errors.Inc() 163 | p.logger.Error("Can't write response: %v", err) 164 | http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) 165 | return err 166 | } 167 | return nil 168 | } 169 | 170 | func NewGitlabProcessor(outputs *common.Outputs, observability *common.Observability) *GitlabProcessor { 171 | 172 | logger := observability.Logs() 173 | hook, err := gitlab.New() 174 | if err != nil { 175 | logger.Debug("Gitlab processor is disabled.") 176 | return nil 177 | } 178 | 179 | return &GitlabProcessor{ 180 | outputs: outputs, 181 | logger: logger, 182 | hook: hook, 183 | meter: observability.Metrics(), 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /grafana.attributes: -------------------------------------------------------------------------------- 1 | {{- define "grafana-attributes" -}} 2 | {{- if .data -}} 3 | {{- $defaultTagSet := (list .channel .type) -}} 4 | {{- $tagSet := list -}} 5 | {{- $tagMap := dict -}} 6 | 7 | {{- if eq .type "GitlabEvent" -}} 8 | {{- if not (contains "exness.io" .data.repository.git_http_url) -}} 9 | {{- $gitlab_owner := "gatech" -}} 10 | {{- $tagSet = list $gitlab_owner -}} 11 | {{- end -}} 12 | {{- end -}} 13 | 14 | {{- if eq .type "K8sEvent" -}} 15 | {{- if or (not (regexMatch "StatefulSet|DaemonSet|ReplicaSet" .data.kind )) (and (regexMatch "StatefulSet|DaemonSet|ReplicaSet" .data.kind ) (not (regexMatch ".*argocd-application-controller" .data.user.name ))) }} 16 | {{- $severity := ""}} 17 | {{- $op := .data.operation}} 18 | {{- if eq .data.operation "Delete"}} 19 | {{- $severity = "warning"}} 20 | {{- else if eq .data.operation "Update"}} 21 | 22 | {{- $oldImages := ""}}{{- $newImages := ""}} 23 | {{- if regexMatch "StatefulSet|Deployment|DaemonSet|ReplicaSet|Pod" .data.kind }} 24 | {{- range .data.object.old.spec.template.spec.containers}} 25 | {{- $oldImages = printf "%s%s => %s\n" $oldImages .name .image}} 26 | {{- end}} 27 | {{- range .data.object.new.spec.template.spec.containers}} 28 | {{- $newImages = printf "%s%s => %s\n" $newImages .name .image}} 29 | {{- end}} 30 | {{- else if eq .data.kind "Application"}} 31 | {{- range $element := .data.object.old.status.summary.images}} 32 | {{- $oldImages = printf "%s%s\n" $oldImages $element}} 33 | {{- end}} 34 | {{- range $element := .data.object.new.status.summary.images}} 35 | {{- $newImages = printf "%s%s\n" $newImages $element}} 36 | {{- end}} 37 | {{- end}} 38 | {{- if ne $oldImages $newImages}} 39 | {{- $op = "Release"}} 40 | {{- $severity = "warning"}} 41 | {{- end}} 42 | {{- if ne .data.object.old.spec.replicas .data.object.new.spec.replicas}} 43 | {{- if ne $op "Release"}} 44 | {{- $op = "Scale"}} 45 | {{- end}} 46 | {{- end}} 47 | 48 | {{- else }} 49 | {{- $severity = "info"}} 50 | {{- end}} 51 | 52 | {{ $tagSet = (list $severity) -}} 53 | {{ $tagSet = (list $op) -}} 54 | {{- end}} 55 | {{- end -}} 56 | 57 | {{- if eq .type "TeamcityEvent" -}} 58 | {{ $tagSet = (concat (splitList "," .data.target) (list .data.build_name .data.build_event)) -}} 59 | {{- end -}} 60 | 61 | {{- if eq .type "WinEvent" -}} 62 | {{ $tagSet = (list .data.tags.host .data.tags.city .data.tags.mt .data.tags.provider) -}} 63 | {{- end -}} 64 | 65 | {{ if eq .type "ObserviumEvent" -}} 66 | {{ $tagSet = (list .data.DEVICE_HOSTNAME .data.DEVICE_LOCATION) -}} 67 | {{- end -}} 68 | 69 | {{- if eq .type "AWSEvent" -}} 70 | {{ $tagSet = (list .data.source .data.region .data.account) -}} 71 | {{- end -}} 72 | 73 | {{- if eq .type "KubeEvent" -}} 74 | {{ $tagSet = (list .data.reason .data.type .data.location) -}} 75 | {{- if .data.object}} 76 | {{- $tagSet = append $tagSet .data.object.kind -}} 77 | {{- $tagSet = append $tagSet .data.object.namespace -}} 78 | {{- end -}} 79 | {{- end -}} 80 | 81 | {{- if eq .type "NomadEvent"}} 82 | {{- if ne .data.Type "AllocationUpdated" }} 83 | {{- $tagSet = (list .data.Topic .data.Type) -}} 84 | {{- if eq .data.Topic "Allocation"}}{{$tagSet = append $tagSet .data.Payload.Allocation.Name -}}{{- end -}} 85 | {{- if eq .data.Topic "Job"}}{{$tagSet = append $tagSet .data.Payload.Job.Name -}}{{- end -}} 86 | {{- if eq .data.Topic "Deployment"}}{{$tagSet = append $tagSet .data.Payload.Deployment.JobID -}}{{- end -}} 87 | {{- if eq .data.Topic "Evaluation"}}{{$tagSet = append $tagSet .data.Payload.Evaluation.JobID -}}{{- end -}} 88 | {{- if eq .data.Topic "Node"}}{{$tagSet = append $tagSet .data.Payload.Node.Name -}}{{- end -}} 89 | {{- end}} 90 | {{- end}} 91 | 92 | {{- if eq .type "ZabbixEvent" -}} 93 | {{- $defaultTagSet = (list .type) -}} 94 | {{- $Severity := "NotClassified"}} 95 | {{- if eq .data.EventNSeverity "1" -}}{{- $Severity = "Information" -}}{{- end -}} 96 | {{- if eq .data.EventNSeverity "2" -}}{{- $Severity = "Warning" -}}{{- end -}} 97 | {{- if eq .data.EventNSeverity "3" -}}{{- $Severity = "Average" -}}{{- end -}} 98 | {{- if eq .data.EventNSeverity "4" -}}{{- $Severity = "High" -}}{{- end -}} 99 | {{- if eq .data.EventNSeverity "5" -}}{{- $Severity = "Disaster" -}}{{- end -}} 100 | {{- $tagSet = (list .data.HostName .data.Environment $Severity) -}} 101 | {{- end -}} 102 | 103 | {{- if eq .type "VCenterEvent"}} 104 | {{- $tagSet = (list .data.DestESXiHostName .data.OrigESXiHostName .data.VmName .data.AlarmName .data.Subject .data.DebugSubject .data.DestLocation ) -}} 105 | 106 | {{- if and (not (.data.Subject | regexMatch "(TaskEvent|VmAcquiredTicketEvent)")) (.data.Subject | regexMatch "(com.vmware.vc.sdrs.*|Vm.*)") -}} 107 | {{- $tagMap = (dict "VCenterDebug" "false") -}} 108 | {{- else -}} 109 | {{- $tagMap = (dict "VCenterDebug" "true") -}} 110 | {{- end -}} 111 | {{- end -}} 112 | 113 | {{/* Now let's merge and groom everything and make a shiny new JSON */}} 114 | {{- $grafanaTags := dict -}} 115 | 116 | {{/* Go through the tagList, drop "" keys with compact() and " " with condition, */}} 117 | {{/* then fill the tags dictionary with keys from the list and empty values */}} 118 | {{- range (compact (concat $tagSet $defaultTagSet)) -}} 119 | {{- if not (eq . " ") -}}{{- $_ := set $grafanaTags . "" -}}{{- end -}} 120 | {{- end -}} 121 | {{- toRawJson (merge $grafanaTags $tagMap) -}} 122 | 123 | {{- end -}} 124 | {{- end -}} -------------------------------------------------------------------------------- /output/gitlab.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | "sync" 7 | 8 | "github.com/devopsext/events/common" 9 | sreCommon "github.com/devopsext/sre/common" 10 | toolsRender "github.com/devopsext/tools/render" 11 | "github.com/devopsext/utils" 12 | "github.com/xanzy/go-gitlab" 13 | ) 14 | 15 | type GitlabOutputOptions struct { 16 | BaseURL string 17 | Token string 18 | Variables string 19 | Projects string 20 | } 21 | 22 | type GitlabOutput struct { 23 | wg *sync.WaitGroup 24 | client *gitlab.Client 25 | projects *toolsRender.TextTemplate 26 | variables *toolsRender.TextTemplate 27 | options GitlabOutputOptions 28 | logger sreCommon.Logger 29 | meter sreCommon.Meter 30 | } 31 | 32 | func (g *GitlabOutput) Name() string { 33 | return "Gitlab" 34 | } 35 | 36 | func (g *GitlabOutput) getVariables(o interface{}) (map[string]string, error) { 37 | 38 | attrs := make(map[string]string) 39 | if g.variables == nil { 40 | return attrs, nil 41 | } 42 | 43 | a, err := g.variables.RenderObject(o) 44 | if err != nil { 45 | return attrs, err 46 | } 47 | 48 | m := string(a) 49 | if utils.IsEmpty(m) { 50 | return attrs, nil 51 | } 52 | 53 | g.logger.Debug("Gitlab raw variables => %s", m) 54 | 55 | var object map[string]interface{} 56 | 57 | if err := json.Unmarshal([]byte(m), &object); err != nil { 58 | return attrs, err 59 | } 60 | 61 | for k, v := range object { 62 | vs, ok := v.(string) 63 | if ok { 64 | attrs[k] = vs 65 | } 66 | } 67 | return attrs, nil 68 | } 69 | 70 | // "https://some.host.domain/group/subgroup/project/-/pipelines/893667" 71 | /*func (g *GitlabOutput) getProject(s string) string { 72 | 73 | u, err := url.Parse(s) 74 | if err != nil { 75 | return "" 76 | } 77 | arr := strings.Split(u.Path, "/-/") 78 | if len(arr) > 0 { 79 | return arr[0] 80 | } 81 | return "" 82 | }*/ 83 | 84 | // projects = TOKEN=PROJECT_ID@REF 85 | func (g *GitlabOutput) Send(event *common.Event) { 86 | 87 | g.wg.Add(1) 88 | go func() { 89 | defer g.wg.Done() 90 | 91 | if event == nil { 92 | g.logger.Debug("Event is empty") 93 | return 94 | } 95 | 96 | if event.Data == nil { 97 | g.logger.Error("Event data is empty") 98 | return 99 | } 100 | 101 | jsonObject, err := event.JsonObject() 102 | if err != nil { 103 | g.logger.Error(err) 104 | return 105 | } 106 | 107 | projects := "" 108 | if g.projects != nil { 109 | b, err := g.projects.RenderObject(jsonObject) 110 | if err != nil { 111 | g.logger.Debug(err) 112 | } else { 113 | projects = string(b) 114 | } 115 | } 116 | 117 | if utils.IsEmpty(projects) { 118 | g.logger.Debug("Gitlab projects are not found") 119 | return 120 | } 121 | 122 | variables, err := g.getVariables(jsonObject) 123 | if err != nil { 124 | g.logger.Error(err) 125 | } 126 | 127 | arr := strings.Split(projects, "\n") 128 | for _, project := range arr { 129 | 130 | project = strings.TrimSpace(project) 131 | if utils.IsEmpty(project) { 132 | continue 133 | } 134 | pair := strings.SplitN(project, "=", 2) 135 | token := g.options.Token 136 | 137 | if len(pair) == 2 && !utils.IsEmpty(pair[0]) { 138 | token = pair[0] 139 | project = pair[1] 140 | } 141 | 142 | pair = strings.SplitN(project, "@", 2) 143 | if len(pair) < 2 { 144 | continue 145 | } 146 | 147 | id := pair[0] 148 | ref := pair[1] 149 | if utils.IsEmpty(ref) { 150 | ref = "main" 151 | } 152 | 153 | labels := make(map[string]string) 154 | labels["event_channel"] = event.Channel 155 | labels["event_type"] = event.Type 156 | labels["gitlab_project_id"] = project 157 | labels["gitlab_ref"] = ref 158 | labels["output"] = g.Name() 159 | 160 | requests := g.meter.Counter("gitlab", "requests", "Count of all gitlab requests", labels, "output") 161 | requests.Inc() 162 | 163 | opt := &gitlab.RunPipelineTriggerOptions{Ref: &ref, Token: &token, Variables: variables} 164 | pipeline, response, err := g.client.PipelineTriggers.RunPipelineTrigger(id, opt) 165 | 166 | errors := g.meter.Counter("gitlab", "errors", "Count of all gitlab errors", labels, "output") 167 | if err != nil { 168 | errors.Inc() 169 | g.logger.Error(err) 170 | continue 171 | } 172 | 173 | if response.StatusCode < 200 || response.StatusCode >= 300 { 174 | errors.Inc() 175 | g.logger.Error("Gitlab response: %s", response.Status) 176 | continue 177 | } 178 | g.logger.Debug("Gitlab pipeline => %s", pipeline.WebURL) 179 | } 180 | }() 181 | } 182 | 183 | func NewGitlabOutput(wg *sync.WaitGroup, 184 | options GitlabOutputOptions, 185 | templateOptions toolsRender.TemplateOptions, 186 | observability *common.Observability) *GitlabOutput { 187 | 188 | logger := observability.Logs() 189 | if utils.IsEmpty(options.BaseURL) { 190 | logger.Debug("Gitlab base URL is not defined. Skipped") 191 | return nil 192 | } 193 | 194 | client, err := gitlab.NewClient(options.Token, gitlab.WithBaseURL(options.BaseURL)) 195 | if err != nil { 196 | logger.Error(err) 197 | return nil 198 | } 199 | 200 | projectsOpts := toolsRender.TemplateOptions{ 201 | Name: "gitlab-projects", 202 | Content: common.Content(options.Projects), 203 | TimeFormat: templateOptions.TimeFormat, 204 | } 205 | projects, err := toolsRender.NewTextTemplate(projectsOpts, observability) 206 | if err != nil { 207 | logger.Error(err) 208 | return nil 209 | } 210 | 211 | variablesOpts := toolsRender.TemplateOptions{ 212 | Name: "gitlab-variables", 213 | Content: common.Content(options.Variables), 214 | TimeFormat: templateOptions.TimeFormat, 215 | } 216 | variables, err := toolsRender.NewTextTemplate(variablesOpts, observability) 217 | if err != nil { 218 | logger.Error(err) 219 | } 220 | 221 | return &GitlabOutput{ 222 | wg: wg, 223 | client: client, 224 | projects: projects, 225 | variables: variables, 226 | options: options, 227 | logger: logger, 228 | meter: observability.Metrics(), 229 | } 230 | } 231 | -------------------------------------------------------------------------------- /processor/kube.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "encoding/json" 5 | errPkg "errors" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "strings" 10 | "time" 11 | 12 | "github.com/devopsext/events/common" 13 | sreCommon "github.com/devopsext/sre/common" 14 | v1 "k8s.io/api/core/v1" 15 | ) 16 | 17 | type KubeProcessor struct { 18 | outputs *common.Outputs 19 | logger sreCommon.Logger 20 | meter sreCommon.Meter 21 | } 22 | 23 | type KubeData struct { 24 | Type string `json:"type"` 25 | Source string `json:"source"` 26 | Reason string `json:"reason"` 27 | Location string `json:"location"` 28 | Message string `json:"message"` 29 | Object interface{} `json:"object,omitempty"` 30 | } 31 | 32 | type EnhancedObjectReference struct { 33 | v1.ObjectReference `json:",inline"` 34 | Labels map[string]string `json:"labels,omitempty"` 35 | Annotations map[string]string `json:"annotations,omitempty"` 36 | } 37 | 38 | // EnhancedEvent Original file https://github.com/opsgenie/kubernetes-event-exporter/blob/master/pkg/kube/event.go 39 | type EnhancedEvent struct { 40 | v1.Event `json:",inline"` 41 | InvolvedObject EnhancedObjectReference `json:"involvedObject"` 42 | } 43 | 44 | func (p *KubeProcessor) send(channel string, e *EnhancedEvent) error { 45 | ce := &common.Event{ 46 | Channel: channel, 47 | Type: p.EventType(), 48 | Data: KubeData{ 49 | Reason: e.Reason, 50 | Message: e.Message, 51 | Type: e.Type, 52 | Location: fmt.Sprintf("%s/%s", e.Namespace, e.Name), 53 | Source: fmt.Sprintf("%s/%s", e.Source.Host, e.Source.Component), 54 | Object: e.InvolvedObject, 55 | }, 56 | } 57 | ce.SetTime(time.Now().UTC()) 58 | ce.SetLogger(p.logger) 59 | p.outputs.Send(ce) 60 | return nil 61 | } 62 | 63 | func (p *KubeProcessor) processEvent( 64 | w http.ResponseWriter, 65 | channel string, 66 | e *EnhancedEvent, 67 | ) error { 68 | 69 | labels := make(map[string]string) 70 | labels["event_channel"] = channel 71 | labels["processor"] = p.EventType() 72 | 73 | errors := p.meter.Counter("kube", "errors", "Count of all kube processor requests", labels, "processor") 74 | 75 | if err := p.send(channel, e); err != nil { 76 | errors.Inc() 77 | p.logger.Error("Can't send event: %v", err) 78 | http.Error(w, fmt.Sprintf("couldn't send event: %v", err), http.StatusInternalServerError) 79 | return err 80 | } 81 | if _, err := w.Write([]byte("OK")); err != nil { 82 | errors.Inc() 83 | p.logger.Error("Can't write response: %v", err) 84 | http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) 85 | return err 86 | } 87 | return nil 88 | } 89 | 90 | func (p *KubeProcessor) HandleHttpRequest(w http.ResponseWriter, r *http.Request) error { 91 | channel := strings.TrimLeft(r.URL.Path, "/") 92 | 93 | labels := make(map[string]string) 94 | labels["path"] = r.URL.Path 95 | labels["processor"] = p.EventType() 96 | 97 | requests := p.meter.Counter("kube", "requests", "Count of all kube processor requests", labels, "processor") 98 | requests.Inc() 99 | 100 | errors := p.meter.Counter("kube", "errors", "Count of all kube processor errors", labels, "processor") 101 | 102 | var body []byte 103 | if r.Body != nil { 104 | if data, err := io.ReadAll(r.Body); err == nil { 105 | body = data 106 | } 107 | } 108 | 109 | if len(body) == 0 { 110 | errors.Inc() 111 | err := errPkg.New("empty body") 112 | p.logger.Error(err) 113 | http.Error(w, err.Error(), http.StatusBadRequest) 114 | return err 115 | } 116 | 117 | p.logger.Debug("Body => %s", body) 118 | 119 | var e *EnhancedEvent 120 | if err := json.Unmarshal(body, &e); err == nil { 121 | return p.processEvent(w, channel, e) 122 | } 123 | errorString := fmt.Sprintf("Could not parse body as EnhancedEvent: %s", body) 124 | errors.Inc() 125 | err := errPkg.New(errorString) 126 | p.logger.Error(errorString) 127 | http.Error(w, fmt.Sprint(errorString), http.StatusInternalServerError) 128 | return err 129 | } 130 | 131 | func KubeProcessorType() string { 132 | return "Kube" 133 | } 134 | 135 | func (p *KubeProcessor) EventType() string { 136 | return common.AsEventType(KubeProcessorType()) 137 | } 138 | 139 | func (p *KubeProcessor) HandleEvent(e *common.Event) error { 140 | if e == nil { 141 | p.logger.Debug("Event is not defined") 142 | return nil 143 | } 144 | return nil 145 | } 146 | 147 | // DeDot replaces all dots in the labels and annotations with underscores. This is required for example in the 148 | // elasticsearch sink. The dynamic mapping generation interprets dots in JSON keys as path in an object. 149 | // For reference see this logstash filter: https://www.elastic.co/guide/en/logstash/current/plugins-filters-de_dot.html 150 | func (e EnhancedEvent) DeDot() EnhancedEvent { 151 | c := e 152 | c.Labels = common.DeDotMap(e.Labels) 153 | c.Annotations = common.DeDotMap(e.Annotations) 154 | c.InvolvedObject.Labels = common.DeDotMap(e.InvolvedObject.Labels) 155 | c.InvolvedObject.Annotations = common.DeDotMap(e.InvolvedObject.Annotations) 156 | return c 157 | } 158 | 159 | // ToJSON does not return an error because we are %99 confident it is JSON serializable. 160 | func (e EnhancedEvent) ToJSON() []byte { 161 | b, _ := json.Marshal(e) 162 | return b 163 | } 164 | 165 | func (e EnhancedEvent) GetTimestampMs() int64 { 166 | timestamp := e.FirstTimestamp.Time 167 | if timestamp.IsZero() { 168 | timestamp = e.EventTime.Time 169 | } 170 | 171 | return timestamp.UnixNano() / (int64(time.Millisecond) / int64(time.Nanosecond)) 172 | } 173 | 174 | func (e EnhancedEvent) GetTimestampISO8601() string { 175 | timestamp := e.FirstTimestamp.Time 176 | if timestamp.IsZero() { 177 | timestamp = e.EventTime.Time 178 | } 179 | 180 | layout := "2006-01-02T15:04:05.000Z" 181 | return timestamp.Format(layout) 182 | } 183 | 184 | func NewKubeProcessor(outputs *common.Outputs, observability *common.Observability) *KubeProcessor { 185 | return &KubeProcessor{ 186 | outputs: outputs, 187 | logger: observability.Logs(), 188 | meter: observability.Metrics(), 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /processor/google.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "encoding/json" 5 | errPkg "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "strings" 10 | "time" 11 | 12 | "github.com/devopsext/events/common" 13 | sreCommon "github.com/devopsext/sre/common" 14 | ) 15 | 16 | type GoogleProcessor struct { 17 | outputs *common.Outputs 18 | logger sreCommon.Logger 19 | meter sreCommon.Meter 20 | } 21 | 22 | type GoogleResource struct { 23 | Type string `json:"type"` 24 | Labels map[string]string `json:"labels"` 25 | } 26 | 27 | type GoogleMetric struct { 28 | Type string `json:"type"` 29 | DisplayName string `json:"displayName"` 30 | Labels map[string]string `json:"labels"` 31 | } 32 | 33 | type GoogleMetadata struct { 34 | SystemLabels map[string]string `json:"system_labels"` 35 | UserLabels map[string]string `json:"user_labels"` 36 | } 37 | 38 | type GoogleConditionThreshold struct { 39 | Filter string `json:"filter"` 40 | Comparison string `json:"comparison"` 41 | ThresholdValue float32 `json:"thresholdValue"` 42 | Duration string `json:"duration"` 43 | Trigger interface{} `json:"trigger"` 44 | } 45 | 46 | type GoogleCondition struct { 47 | Name string `json:"name"` 48 | DisplayName string `json:"displayName"` 49 | ConditionThreshold *GoogleConditionThreshold `json:"conditionThreshold"` 50 | } 51 | 52 | type GoogleIncident struct { 53 | IncidentID string `json:"incident_id"` 54 | ScopingProjectID string `json:"scoping_project_id"` 55 | ScopingProjectNumber string `json:"scoping_project_number"` 56 | URL string `json:"url"` 57 | StartedAt int64 `json:"started_at"` 58 | EndedAt int64 `json:"ended_at,omitempty"` 59 | State string `json:"state"` 60 | Summary string `json:"summary"` 61 | ApigeeURL string `json:"apigee_url"` 62 | ObservedValue string `json:"observed_value"` 63 | Resource *GoogleResource `json:"resource"` 64 | ResourceTypeDisplayName string `json:"resource_type_display_name"` 65 | ResourceID string `json:"resource_id"` 66 | ResourceDisplayName string `json:"resource_display_name"` 67 | ResourceName string `json:"resource_name"` 68 | Metric *GoogleMetric `json:"metric"` 69 | Metadata *GoogleMetadata `json:"metadata"` 70 | PolicyName string `json:"policy_name"` 71 | PolicyUserLabels map[string]string `json:"policy_user_labels"` 72 | Documentation string `json:"documentation"` 73 | Condition *GoogleCondition `json:"condition"` 74 | ConditionName string `json:"condition_name"` 75 | ThresholdValue string `json:"threshold_value"` 76 | } 77 | 78 | type GoogleRequest struct { 79 | Version string `json:"version"` 80 | Incident *GoogleIncident `json:"incident"` 81 | } 82 | 83 | type GoogleResponse struct { 84 | Message string 85 | } 86 | 87 | func GoogleProcessorType() string { 88 | return "Google" 89 | } 90 | 91 | func (p *GoogleProcessor) EventType() string { 92 | return common.AsEventType(GoogleProcessorType()) 93 | } 94 | 95 | func (p *GoogleProcessor) send(channel string, o interface{}, t *time.Time) { 96 | 97 | e := &common.Event{ 98 | Channel: channel, 99 | Type: p.EventType(), 100 | Data: o, 101 | } 102 | if t != nil && (*t).UnixNano() > 0 { 103 | e.SetTime((*t).UTC()) 104 | } else { 105 | e.SetTime(time.Now().UTC()) 106 | } 107 | e.SetLogger(p.logger) 108 | p.outputs.Send(e) 109 | } 110 | 111 | func (p *GoogleProcessor) HandleEvent(e *common.Event) error { 112 | 113 | if e == nil { 114 | p.logger.Debug("Event is not defined") 115 | return nil 116 | } 117 | 118 | labels := make(map[string]string) 119 | labels["event_channel"] = e.Channel 120 | labels["processor"] = p.EventType() 121 | 122 | requests := p.meter.Counter("google", "requests", "Count of all google processor requests", labels, "processor") 123 | requests.Inc() 124 | 125 | p.outputs.Send(e) 126 | return nil 127 | } 128 | 129 | func (p *GoogleProcessor) HandleHttpRequest(w http.ResponseWriter, r *http.Request) error { 130 | 131 | channel := strings.TrimLeft(r.URL.Path, "/") 132 | 133 | labels := make(map[string]string) 134 | labels["path"] = r.URL.Path 135 | labels["processor"] = p.EventType() 136 | 137 | requests := p.meter.Counter("google", "requests", "Count of all google processor requests", labels, "processor") 138 | requests.Inc() 139 | 140 | errors := p.meter.Counter("google", "errors", "Count of all google processor errors", labels, "processor") 141 | 142 | var body []byte 143 | if r.Body != nil { 144 | if data, err := ioutil.ReadAll(r.Body); err == nil { 145 | body = data 146 | } 147 | } 148 | 149 | if len(body) == 0 { 150 | errors.Inc() 151 | err := errPkg.New("empty body") 152 | p.logger.Error(err) 153 | http.Error(w, err.Error(), http.StatusBadRequest) 154 | return err 155 | } 156 | 157 | p.logger.Debug("Body => %s", body) 158 | 159 | var request GoogleRequest 160 | if err := json.Unmarshal(body, &request); err != nil { 161 | errors.Inc() 162 | p.logger.Error(err) 163 | http.Error(w, "Error unmarshaling message", http.StatusInternalServerError) 164 | return err 165 | } 166 | 167 | if request.Incident.StartedAt > 0 { 168 | t := time.UnixMilli(request.Incident.StartedAt) 169 | p.send(channel, request, &t) 170 | } else { 171 | p.send(channel, request, nil) 172 | } 173 | 174 | response := &GoogleResponse{ 175 | Message: "OK", 176 | } 177 | 178 | resp, err := json.Marshal(response) 179 | if err != nil { 180 | errors.Inc() 181 | p.logger.Error("Can't encode response: %v", err) 182 | http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError) 183 | return err 184 | } 185 | 186 | if _, err := w.Write(resp); err != nil { 187 | errors.Inc() 188 | p.logger.Error("Can't write response: %v", err) 189 | http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) 190 | return err 191 | } 192 | return nil 193 | } 194 | 195 | func NewGoogleProcessor(outputs *common.Outputs, observability *common.Observability) *GoogleProcessor { 196 | 197 | return &GoogleProcessor{ 198 | outputs: outputs, 199 | logger: observability.Logs(), 200 | meter: observability.Metrics(), 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /workchat.message: -------------------------------------------------------------------------------- 1 | {{- define "k8s-header"}} 2 | {{- $t := timeFormat .time "02.01.06 15:04:05"}} 3 | {{- printf "*%s*: %s / %s\n*%s*: %s\nby _%s_" (toUpper .data.operation) .data.kind .data.location .channel $t .data.user.name}} 4 | {{- end}} 5 | {{- define "k8s-namespace"}}{{- end}} 6 | {{- define "k8s-node"}}{{- end}} 7 | {{- define "k8s-replicaset"}}{{- end}} 8 | {{- define "k8s-statefulset"}} 9 | {{- printf "\n*Replicas* => %.0f\n*Selector* => %s" .spec.replicas .spec.selector}} 10 | {{- template "k8s-pod" .spec.template}} 11 | {{- end}} 12 | {{- define "k8s-daemonset"}}{{- end}} 13 | {{- define "k8s-secret"}}{{- end}} 14 | {{- define "k8s-ingress"}}{{- end}} 15 | {{- define "k8s-cronjob"}}{{- end}} 16 | {{- define "k8s-job"}}{{- end}} 17 | {{- define "k8s-configmap"}}{{- end}} 18 | {{- define "k8s-role"}}{{- end}} 19 | {{- define "k8s-deployment"}} 20 | {{- printf "\n*Replicas* => %.0f\n*Selector* => %s" .spec.replicas .spec.selector}} 21 | {{- template "k8s-pod" .spec.template}} 22 | {{- end}} 23 | {{- define "k8s-service"}} 24 | {{- printf "\n*%s* => %s" .spec.type .spec.clusterIP}} 25 | {{- range .spec.ports}} 26 | {{- printf "\n*%s* %.0f => %.0f" .protocol .port .targetPort}} 27 | {{- end}} 28 | {{- printf "\n*Selector* => %s" .spec.selector}} 29 | {{- end}} 30 | {{- define "k8s-pod"}} 31 | {{- range .spec.containers}} 32 | {{- printf "\n*%s* => %s" .name .image}} 33 | {{- range .ports}} 34 | {{- printf " [%s: %.0f]" .protocol .containerPort}} 35 | {{- end}} 36 | {{- end}} 37 | {{- end}} 38 | {{- define "alertmanager-header"}} 39 | {{- $t := timeFormat .time "02.01.06 15:04:05"}} 40 | {{- printf "*%s*: %s\n*%s*: %s" (toUpper .data.status) .data.labels.alertname .channel $t}} 41 | {{- end}} 42 | {{- define "alertmanager-body"}} 43 | {{- if eq .status "firing"}}{{- printf "\n*StartsAt*: %s" (timeFormat .startsAt "02.01.06 15:04:05") }}{{end}} 44 | {{- if eq .status "resolved"}}{{- printf "\n*endsAt*: %s" (timeFormat .endsAt "02.01.06 15:04:05") }}{{end}} 45 | {{- end}} 46 | {{- define "gitlab-header"}} 47 | {{- $t := timeFormat .time "02.01.06 15:04:05"}} 48 | {{- if .data.project}}{{- printf "*%s*: %s / %s@%s\n*%s*: %s\nby _%s_" (toUpper .data.object_kind) .data.project.namespace .data.project.name .data.object_attributes.ref .channel $t .data.user.username}} 49 | {{- else}}{{- printf "*%s*: %s@%s\n*%s*: %s\nby _%s_" (toUpper .data.object_kind) .data.project_name .data.ref .channel $t .data.user.username}}{{end}} 50 | {{- end}} 51 | {{- define "gitlab-commit"}} 52 | {{- if .commit.author}}{{- printf "\n*Commit author* => %s" .commit.author.name}}{{else}}{{- printf "\n*Commit author* => %s" .commit.author_name}}{{end}} 53 | {{- printf "\n*Commit message* => %s" .commit.message}} 54 | {{- if .commit.url}}{{- printf "\n%s" .commit.url}}{{else}}{{- printf "\n%s/-/commit/%s" .repository.homepage .commit.sha}}{{end}} 55 | {{- end}} 56 | {{- define "gitlab-stages"}} 57 | {{- $match := .match}} 58 | {{- range .stages}} 59 | {{- if and (.runner.description | regexMatch $match) (not (empty .finished_at))}} 60 | {{- printf "\n*%s/%s* [%s] => %s" .stage .name .user.username .runner.description}} 61 | {{- end}} 62 | {{- end}} 63 | {{- end}} 64 | {{- define "gitlab-pipeline"}} 65 | {{- $match := getEnv "EVENTS_GITLAB_RUNNERS"}}{{$ok := false}} 66 | {{- range .data.builds}} 67 | {{- if and (.runner.description | regexMatch $match) (not (empty .finished_at))}}{{$ok = true}}{{end}} 68 | {{- end}} 69 | {{- if $ok}} 70 | {{- template "gitlab-header" .}} 71 | {{- template "gitlab-commit" (dict "repository" "" "commit" .data.commit)}} 72 | {{- template "gitlab-stages" (dict "stages" (reverse .data.builds) "match" $match)}} 73 | {{- printf "\n%s/-/pipelines/%.0f" .data.project.web_url .data.object_attributes.id}} 74 | {{- end}} 75 | {{- end}} 76 | {{- define "gitlab-build"}} 77 | {{- $match := getEnv "EVENTS_GITLAB_RUNNERS"}} 78 | {{- if and (.data.runner.description | regexMatch $match) (not (empty .data.build_duration))}} 79 | {{- template "gitlab-header" .}} 80 | {{- template "gitlab-commit" (dict "repository" .data.repository "commit" .data.commit)}} 81 | {{- printf "\n*%s/%s* => %s" .data.build_stage .data.build_name .data.runner.description}} 82 | {{- printf "\n%s/-/jobs/%.0f" .data.repository.homepage .data.build_id}} 83 | {{- end}} 84 | {{- end}} 85 | {{- define "text"}} 86 | {{- if eq .type "K8sEvent"}} 87 | {{- if not (.data.user.name | regexMatch "(system:serviceaccount:*|system:*)")}} 88 | {{- if .data.object}} 89 | {{- if eq .data.kind "Namespace"}}{{template "k8s-header" .}}{{template "k8s-namespace" .data.object}}{{end}} 90 | {{- if eq .data.kind "Node"}}{{template "k8s-header" .}}{{template "k8s-node" .data.object}}{{end}} 91 | {{- if eq .data.kind "ReplicaSet"}}{{template "k8s-header" .}}{{template "k8s-replicaset" .data.object}}{{end}} 92 | {{- if eq .data.kind "StatefulSet"}}{{template "k8s-header" .}}{{template "k8s-statefulset" .data.object}}{{end}} 93 | {{- if eq .data.kind "DaemonSet"}}{{template "k8s-header" .}}{{template "k8s-daemonset" .data.object}}{{end}} 94 | {{- if eq .data.kind "Secret"}}{{template "k8s-header" .}}{{template "k8s-secret" .data.object}}{{end}} 95 | {{- if eq .data.kind "Ingress"}}{{template "k8s-header" .}}{{template "k8s-ingress" .data.object}}{{end}} 96 | {{- if eq .data.kind "CronJob"}}{{template "k8s-header" .}}{{template "k8s-cronjob" .data.object}}{{end}} 97 | {{- if eq .data.kind "Job"}}{{template "k8s-header" .}}{{template "k8s-job" .data.object}}{{end}} 98 | {{- if eq .data.kind "ConfigMap"}}{{template "k8s-header" .}}{{template "k8s-configmap" .data.object}}{{end}} 99 | {{- if eq .data.kind "Role"}}{{template "k8s-header" .}}{{template "k8s-role" .data.object}}{{end}} 100 | {{- if eq .data.kind "Deployment"}}{{template "k8s-header" .}}{{template "k8s-deployment" .data.object}}{{end}} 101 | {{- if eq .data.kind "Service"}}{{template "k8s-header" .}}{{template "k8s-service" .data.object}}{{end}} 102 | {{- if eq .data.kind "Pod"}}{{template "k8s-header" .}}{{template "k8s-pod" .data.object}}{{end}} 103 | {{- else}}{{template "k8s-header" .}}{{end}} 104 | {{- end}} 105 | {{- end}} 106 | {{- if eq .type "AlertmanagerEvent"}} 107 | {{template "alertmanager-header" .}}{{template "alertmanager-body" .data}} 108 | {{- end}} 109 | {{- if eq .type "GitlabEvent"}} 110 | {{- if eq .data.object_kind "pipeline"}}{{template "gitlab-pipeline" .}}{{end}} 111 | {{- if eq .data.object_kind "build"}}{{template "gitlab-build" .}}{{end}} 112 | {{- end}} 113 | {{- end}} 114 | {{- define "workchat-message"}}{{template "text" .}}{{- end}} 115 | -------------------------------------------------------------------------------- /input/http.go: -------------------------------------------------------------------------------- 1 | package input 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "fmt" 7 | "net" 8 | "net/http" 9 | "os" 10 | "strings" 11 | "sync" 12 | 13 | "github.com/devopsext/events/common" 14 | "github.com/devopsext/events/processor" 15 | sreCommon "github.com/devopsext/sre/common" 16 | "github.com/devopsext/utils" 17 | ) 18 | 19 | type HttpInputOptions struct { 20 | HealthcheckURL string 21 | K8sURL string 22 | KubeURL string 23 | WinEventURL string 24 | RancherURL string 25 | AlertmanagerURL string 26 | GitlabURL string 27 | DataDogURL string 28 | Site24x7URL string 29 | CloudflareURL string 30 | GoogleURL string 31 | AWSURL string 32 | ZabbixURL string 33 | CustomJsonURL string 34 | VCenterURL string 35 | ObserviumEventURL string 36 | TeamcityURL string 37 | 38 | ServerName string 39 | Listen string 40 | Tls bool 41 | Insecure bool 42 | Cert string 43 | Key string 44 | Chain string 45 | HeaderTraceID string 46 | } 47 | 48 | type HttpInput struct { 49 | options HttpInputOptions 50 | processors *common.Processors 51 | logger sreCommon.Logger 52 | meter sreCommon.Meter 53 | } 54 | 55 | type HttpProcessHandleFunc = func(w http.ResponseWriter, r *http.Request) 56 | 57 | func (h *HttpInput) processURL(url string, mux *http.ServeMux, p common.HttpProcessor) { 58 | 59 | urls := strings.Split(url, ",") 60 | for _, url := range urls { 61 | 62 | mux.HandleFunc(url, func(w http.ResponseWriter, r *http.Request) { 63 | 64 | labels := make(map[string]string) 65 | path := r.URL.Path 66 | 67 | labels["path"] = path 68 | labels["input"] = "http" 69 | requests := h.meter.Counter("http", "requests", "Count of all http input requests", labels, "input") 70 | 71 | requests.Inc() 72 | 73 | err := p.HandleHttpRequest(w, r) 74 | if err != nil { 75 | errors := h.meter.Counter("http", "errors", "Count of all http input errors", labels, "input") 76 | errors.Inc() 77 | } 78 | }) 79 | } 80 | } 81 | 82 | func (h *HttpInput) Start(wg *sync.WaitGroup, outputs *common.Outputs) { 83 | 84 | wg.Add(1) 85 | go func(wg *sync.WaitGroup) { 86 | 87 | defer wg.Done() 88 | h.logger.Info("Start http input...") 89 | 90 | var caPool *x509.CertPool 91 | var certificates []tls.Certificate 92 | 93 | if h.options.Tls { 94 | 95 | // load certififcate 96 | var cert []byte 97 | if _, err := os.Stat(h.options.Cert); err == nil { 98 | 99 | cert, err = os.ReadFile(h.options.Cert) 100 | if err != nil { 101 | h.logger.Panic(err) 102 | } 103 | } else { 104 | cert = []byte(h.options.Cert) 105 | } 106 | 107 | // load key 108 | var key []byte 109 | if _, err := os.Stat(h.options.Key); err == nil { 110 | key, err = os.ReadFile(h.options.Key) 111 | if err != nil { 112 | h.logger.Panic(err) 113 | } 114 | } else { 115 | key = []byte(h.options.Key) 116 | } 117 | 118 | // make pair from certificate and pair 119 | pair, err := tls.X509KeyPair(cert, key) 120 | if err != nil { 121 | h.logger.Panic(err) 122 | } 123 | 124 | certificates = append(certificates, pair) 125 | 126 | // load CA chain 127 | var chain []byte 128 | if _, err := os.Stat(h.options.Chain); err == nil { 129 | chain, err = os.ReadFile(h.options.Chain) 130 | if err != nil { 131 | h.logger.Panic(err) 132 | } 133 | } else { 134 | chain = []byte(h.options.Chain) 135 | } 136 | 137 | // make pool of chains 138 | caPool = x509.NewCertPool() 139 | if !caPool.AppendCertsFromPEM(chain) { 140 | h.logger.Debug("CA chain is invalid") 141 | } 142 | } 143 | 144 | mux := http.NewServeMux() 145 | if !utils.IsEmpty(h.options.HealthcheckURL) { 146 | mux.HandleFunc(h.options.HealthcheckURL, func(w http.ResponseWriter, r *http.Request) { 147 | if _, err := w.Write([]byte("OK")); err != nil { 148 | h.logger.Error("Can't write response: %v", err) 149 | http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) 150 | } 151 | }) 152 | } 153 | 154 | processors := h.getProcessors(h.processors, outputs) 155 | for u, p := range processors { 156 | h.processURL(u, mux, p) 157 | } 158 | 159 | listener, err := net.Listen("tcp", h.options.Listen) 160 | if err != nil { 161 | h.logger.Panic(err) 162 | } 163 | 164 | h.logger.Info("Http input is up. Listening...") 165 | 166 | srv := &http.Server{ 167 | Handler: mux, 168 | ErrorLog: nil, 169 | } 170 | 171 | if h.options.Tls { 172 | 173 | srv.TLSConfig = &tls.Config{ 174 | Certificates: certificates, 175 | RootCAs: caPool, 176 | InsecureSkipVerify: h.options.Insecure, 177 | ServerName: h.options.ServerName, 178 | } 179 | 180 | err = srv.ServeTLS(listener, "", "") 181 | if err != nil { 182 | h.logger.Panic(err) 183 | } 184 | } else { 185 | err = srv.Serve(listener) 186 | if err != nil { 187 | h.logger.Panic(err) 188 | } 189 | } 190 | }(wg) 191 | } 192 | 193 | func (h *HttpInput) setProcessor(m map[string]common.HttpProcessor, url string, t string) { 194 | 195 | if !utils.IsEmpty(url) { 196 | p := h.processors.FindHttpProcessor(common.AsEventType(t)) 197 | if p != nil { 198 | m[url] = p 199 | } 200 | } 201 | } 202 | 203 | func (h *HttpInput) getProcessors(_ *common.Processors, _ *common.Outputs) map[string]common.HttpProcessor { 204 | 205 | m := make(map[string]common.HttpProcessor) 206 | h.setProcessor(m, h.options.K8sURL, processor.K8sProcessorType()) 207 | h.setProcessor(m, h.options.KubeURL, processor.KubeProcessorType()) 208 | h.setProcessor(m, h.options.WinEventURL, processor.WinEventProcessorType()) 209 | h.setProcessor(m, h.options.ObserviumEventURL, processor.ObserviumEventProcessorType()) 210 | h.setProcessor(m, h.options.AlertmanagerURL, processor.AlertmanagerProcessorType()) 211 | h.setProcessor(m, h.options.GitlabURL, processor.GitlabProcessorType()) 212 | h.setProcessor(m, h.options.RancherURL, processor.RancherProcessorType()) 213 | h.setProcessor(m, h.options.DataDogURL, processor.DataDogProcessorType()) 214 | h.setProcessor(m, h.options.Site24x7URL, processor.Site24x7ProcessorType()) 215 | h.setProcessor(m, h.options.CloudflareURL, processor.CloudflareProcessorType()) 216 | h.setProcessor(m, h.options.GoogleURL, processor.GoogleProcessorType()) 217 | h.setProcessor(m, h.options.AWSURL, processor.AWSProcessorType()) 218 | h.setProcessor(m, h.options.ZabbixURL, processor.ZabbixProcessorType()) 219 | h.setProcessor(m, h.options.VCenterURL, processor.VCenterProcessorType()) 220 | h.setProcessor(m, h.options.CustomJsonURL, processor.CustomJsonProcessorType()) 221 | h.setProcessor(m, h.options.TeamcityURL, processor.TeamcityProcessorType()) 222 | return m 223 | } 224 | 225 | func NewHttpInput(options HttpInputOptions, processors *common.Processors, observability *common.Observability) *HttpInput { 226 | 227 | return &HttpInput{ 228 | options: options, 229 | processors: processors, 230 | logger: observability.Logs(), 231 | meter: observability.Metrics(), 232 | } 233 | } 234 | -------------------------------------------------------------------------------- /test/k8s.json: -------------------------------------------------------------------------------- 1 | { 2 | "kind": "AdmissionReview", 3 | "apiVersion": "admission.k8s.io/v1beta1", 4 | "request": { 5 | "uid": "23172a7a-f4c6-11e9-953e-0050568aa55b", 6 | "kind": { 7 | "group": "", 8 | "version": "v1", 9 | "kind": "Pod" 10 | }, 11 | "resource": { 12 | "group": "", 13 | "version": "v1", 14 | "resource": "pods" 15 | }, 16 | "namespace": "nodegroup", 17 | "operation": "CREATE", 18 | "userInfo": { 19 | "username": "eks:vpc-resource-controller", 20 | "uid": "380bb127-e96f-11e8-ae7d-0050568a9a8e", 21 | "groups": [ 22 | "system:serviceaccounts", 23 | "system:serviceaccounts:kube-system", 24 | "system:authenticated" 25 | ] 26 | }, 27 | "object": { 28 | "metadata": { 29 | "name": "someservice-php-order-1571746740-glbhp", 30 | "generateName": "someservice-php-order-1571746740-", 31 | "namespace": "nodegroup", 32 | "uid": "23171eb4-f4c6-11e9-953e-0050568aa55b", 33 | "creationTimestamp": "2019-10-22T12:19:04Z", 34 | "labels": { 35 | "controller-uid": "231132ee-f4c6-11e9-953e-0050568aa55b", 36 | "job-name": "someservice-php-order-1571746740", 37 | "k8s-app": "someservice-php-order", 38 | "platform.collector/injected": "true", 39 | "version": "v0.4" 40 | }, 41 | "annotations": { 42 | "app": "someservice-php-order", 43 | "prometheus.io/path": "/metrics", 44 | "prometheus.io/port": "60000", 45 | "prometheus.io/scrape": "true" 46 | }, 47 | "ownerReferences": [ 48 | { 49 | "apiVersion": "batch/v1", 50 | "kind": "Job", 51 | "name": "someservice-php-order-1571746740", 52 | "uid": "231132ee-f4c6-11e9-953e-0050568aa55b", 53 | "controller": true, 54 | "blockOwnerDeletion": true 55 | } 56 | ] 57 | }, 58 | "spec": { 59 | "volumes": [ 60 | { 61 | "name": "someservice-php-env-file-volume", 62 | "configMap": { 63 | "name": "someservice-php-env-file", 64 | "defaultMode": 420 65 | } 66 | }, 67 | { 68 | "name": "default-token-mn7zd", 69 | "secret": { 70 | "secretName": "default-token-mn7zd", 71 | "defaultMode": 420 72 | } 73 | }, 74 | { 75 | "name": "dockersock", 76 | "hostPath": { 77 | "path": "/var/run/docker.sock", 78 | "type": "" 79 | } 80 | }, 81 | { 82 | "name": "platform-collector-token", 83 | "secret": { 84 | "secretName": "platform-collector-token", 85 | "defaultMode": 420 86 | } 87 | } 88 | ], 89 | "containers": [ 90 | { 91 | "name": "someservice-php-order", 92 | "image": "someregistry.com/someservice-php:v0.4", 93 | "command": [ 94 | "/bin/bash", 95 | "-c", 96 | "cd /var/www ; php -d memory_limit=512M artisan transform:order; echo \"Done\"; sleep 3" 97 | ], 98 | "env": [ 99 | { 100 | "name": "POD_NAME", 101 | "valueFrom": { 102 | "fieldRef": { 103 | "apiVersion": "v1", 104 | "fieldPath": "metadata.name" 105 | } 106 | } 107 | } 108 | ], 109 | "resources": { 110 | "limits": { 111 | "cpu": "1", 112 | "memory": "200Mi" 113 | }, 114 | "requests": { 115 | "cpu": "500m", 116 | "memory": "150Mi" 117 | } 118 | }, 119 | "volumeMounts": [ 120 | { 121 | "name": "someservice-php-env-file-volume", 122 | "mountPath": "/env" 123 | }, 124 | { 125 | "name": "default-token-mn7zd", 126 | "readOnly": true, 127 | "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" 128 | } 129 | ], 130 | "terminationMessagePath": "/dev/termination-log", 131 | "terminationMessagePolicy": "File", 132 | "imagePullPolicy": "IfNotPresent" 133 | }, 134 | { 135 | "name": "collector", 136 | "image": "collector/pod:1.9.3.11-1.1.0", 137 | "env": [ 138 | { 139 | "name": "KAFKA_BROKERS", 140 | "value": "broker:9092" 141 | }, 142 | { 143 | "name": "POD_NAME", 144 | "valueFrom": { 145 | "fieldRef": { 146 | "apiVersion": "v1", 147 | "fieldPath": "metadata.name" 148 | } 149 | } 150 | }, 151 | { 152 | "name": "COLLECTOR_GLOBAL_TAGS_ORCHESTRATION", 153 | "value": "k8s.test.env" 154 | } 155 | ], 156 | "resources": {}, 157 | "volumeMounts": [ 158 | { 159 | "name": "platform-collector-token", 160 | "readOnly": true, 161 | "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" 162 | }, 163 | { 164 | "name": "dockersock", 165 | "readOnly": true, 166 | "mountPath": "/var/run/docker.sock" 167 | } 168 | ], 169 | "terminationMessagePath": "/dev/termination-log", 170 | "terminationMessagePolicy": "File", 171 | "imagePullPolicy": "Always" 172 | } 173 | ], 174 | "restartPolicy": "Never", 175 | "terminationGracePeriodSeconds": 30, 176 | "dnsPolicy": "ClusterFirst", 177 | "nodeSelector": { 178 | "platform.isolation/nodegroup": "nodegroup" 179 | }, 180 | "serviceAccountName": "default", 181 | "serviceAccount": "default", 182 | "securityContext": {}, 183 | "imagePullSecrets": [ 184 | { 185 | "name": "registry.some.io" 186 | } 187 | ], 188 | "schedulerName": "default-scheduler", 189 | "tolerations": [ 190 | { 191 | "key": "node.kubernetes.io/not-ready", 192 | "operator": "Exists", 193 | "effect": "NoExecute", 194 | "tolerationSeconds": 300 195 | }, 196 | { 197 | "key": "node.kubernetes.io/unreachable", 198 | "operator": "Exists", 199 | "effect": "NoExecute", 200 | "tolerationSeconds": 300 201 | } 202 | ], 203 | "priority": 0 204 | }, 205 | "status": { 206 | "phase": "Pending", 207 | "qosClass": "Burstable" 208 | } 209 | }, 210 | "oldObject": null, 211 | "dryRun": false 212 | } 213 | } --------------------------------------------------------------------------------