├── .gitignore ├── Makefile ├── README.md ├── backlog ├── backlog.go ├── backlog_test.go └── util.go ├── clickhouse ├── upload.go ├── util.go └── util_test.go ├── config └── config.go ├── doc └── schema.jpg ├── etc ├── config.yaml ├── debian │ ├── postinst.sh │ └── prerm.sh ├── examples │ ├── clickhouse │ │ └── table_schema.sql │ ├── example_config.yaml │ ├── nginx.conf │ └── rsyslog │ │ ├── 01-nginx-tcp.conf │ │ └── 30-nginx-zmq.conf ├── make-deb-package.go └── nginx-log-collector.service ├── go.mod ├── go.sum ├── nginx-log-collector.go ├── parser ├── error_log.go ├── error_log_test.go └── testdata │ ├── error1 │ └── errorphp ├── processor ├── access_log.go ├── convert.go ├── error_log.go ├── functions │ ├── calculateSHA1.go │ ├── calculateSHA1_test.go │ ├── dispatch.go │ ├── ipToUint32.go │ ├── ipToUint32_test.go │ ├── limitMaxLength.go │ ├── limitMaxLength_test.go │ ├── splitAndStore.go │ ├── toArray.go │ ├── toArray_test.go │ └── types.go ├── processor.go ├── tag_processor.go └── transformer.go ├── receiver ├── http.go ├── tcp.go └── zmq.go ├── service └── service.go ├── uploader └── uploader.go └── utils ├── datetime.go └── limiter.go /.gitignore: -------------------------------------------------------------------------------- 1 | build/ -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GOPACKAGES?=$(shell find . -name '*.go' -not -path "./vendor/*" -exec dirname {} \;| sort | uniq) 2 | GOFILES?=$(shell find . -type f -name '*.go' -not -path "./vendor/*") 3 | 4 | VERSION=$(shell date +%s)-$(shell git describe --abbrev=8 --dirty --always --tags) 5 | 6 | all: help 7 | 8 | .PHONY: help build fmt clean test coverage check vet lint 9 | 10 | help: 11 | @echo "build - build project" 12 | @echo "run - run project with local config" 13 | @echo "deb - build project & make deb package" 14 | @echo "fmt - format application sources" 15 | @echo "clean - remove artifacts" 16 | @echo "test - run tests" 17 | @echo "coverage - run tests with coverage" 18 | @echo "check - check code style" 19 | @echo "vet - run go vet" 20 | @echo "lint - run golint" 21 | 22 | fmt: 23 | go fmt $(GOPACKAGES) 24 | 25 | build: clean 26 | go build -ldflags '-X main.Version=$(VERSION)' -o build/nginx-log-collector nginx-log-collector.go 27 | 28 | deb: build 29 | go run -ldflags '-X main.Version=$(VERSION)' ./etc/make-deb-package.go 30 | 31 | run: 32 | go run nginx-log-collector.go -config ./etc/examples/example_config.yaml 33 | 34 | clean: 35 | go clean 36 | rm -rf ./build/ 37 | 38 | test: clean 39 | go test -v $(GOPACKAGES) 40 | 41 | coverage: clean 42 | go test -v -cover $(GOPACKAGES) 43 | 44 | check: vet lint 45 | 46 | vet: 47 | go vet $(GOPACKAGES) 48 | 49 | lint: 50 | ls $(GOFILES) | xargs -L1 golint 51 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nginx log collector 2 | 3 | make help 4 | 5 | ### Work scheme 6 | ![schema.jpg](doc/schema.jpg?v3) 7 | 8 | ### For ClickHouse server: 9 | "logs_cluster" (from table_schema.sql) get from clickhouse_remote_servers.xml between "remote_servers" and "shard" 10 | -------------------------------------------------------------------------------- /backlog/backlog.go: -------------------------------------------------------------------------------- 1 | package backlog 2 | 3 | import ( 4 | "encoding/binary" 5 | "hash/crc32" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | "path/filepath" 10 | "strconv" 11 | "strings" 12 | "sync" 13 | "time" 14 | 15 | "github.com/pkg/errors" 16 | "github.com/rs/zerolog" 17 | "nginx-log-collector/clickhouse" 18 | "nginx-log-collector/config" 19 | "nginx-log-collector/utils" 20 | "gopkg.in/alexcesaro/statsd.v2" 21 | ) 22 | 23 | const ( 24 | backlogSuffix = ".backlog" 25 | writeSuffix = ".writing" 26 | checkInterval = 30 * time.Second 27 | maxConcurrentHttpRequests = 32 28 | ) 29 | 30 | type Backlog struct { 31 | dir string 32 | 33 | logger zerolog.Logger 34 | metrics *statsd.Client 35 | makeMu *sync.Mutex 36 | wg *sync.WaitGroup 37 | limiter utils.Limiter 38 | } 39 | 40 | func New(cfg config.Backlog, metrics *statsd.Client, logger *zerolog.Logger) (*Backlog, error) { 41 | err := os.MkdirAll(cfg.Dir, 0755) 42 | if err != nil { 43 | return nil, errors.Wrap(err, "unable to create backlog directory") 44 | } 45 | files, err := ioutil.ReadDir(cfg.Dir) 46 | if err != nil { 47 | return nil, errors.Wrap(err, "unable to read backlog directory") 48 | } 49 | for _, f := range files { 50 | fName := f.Name() 51 | if strings.HasSuffix(fName, writeSuffix) { 52 | // remove incomplete files 53 | path := filepath.Join(cfg.Dir, fName) 54 | err = os.Remove(path) 55 | if err != nil { 56 | return nil, errors.Wrap(err, "unable to remove incomplete file") 57 | } 58 | } 59 | } 60 | 61 | wg := &sync.WaitGroup{} 62 | wg.Add(1) 63 | 64 | requestsLimit := maxConcurrentHttpRequests 65 | if cfg.MaxConcurrentHttpRequests > 0 { 66 | requestsLimit = cfg.MaxConcurrentHttpRequests 67 | 68 | } 69 | 70 | return &Backlog{ 71 | dir: cfg.Dir, 72 | makeMu: &sync.Mutex{}, 73 | wg: wg, 74 | metrics: metrics.Clone(statsd.Prefix("backlog")), 75 | logger: logger.With().Str("component", "backlog").Logger(), 76 | limiter: utils.NewLimiter(requestsLimit), 77 | }, nil 78 | } 79 | 80 | func (b *Backlog) Start(done <-chan struct{}) { 81 | b.logger.Info().Msg("starting") 82 | defer b.wg.Done() 83 | 84 | // don't wait for the first tick 85 | b.check(done) 86 | 87 | ticker := time.NewTicker(checkInterval) 88 | defer ticker.Stop() 89 | 90 | for { 91 | select { 92 | case <-done: 93 | return 94 | case <-ticker.C: 95 | b.check(done) 96 | } 97 | } 98 | } 99 | 100 | func (b *Backlog) Stop() { 101 | b.logger.Info().Msg("stopping") 102 | b.wg.Wait() 103 | } 104 | 105 | func (b *Backlog) processFile(filename string) { 106 | b.logger.Info().Str("file", filename).Msg("starting backlog job") 107 | b.metrics.Increment("job_start") 108 | path := filepath.Join(b.dir, filename) 109 | file, err := os.Open(path) 110 | if err != nil { 111 | b.logger.Error().Err(err).Msg("unable to open backlog file") 112 | b.metrics.Increment("open_error") 113 | return 114 | } 115 | if !checkCrc(file) { 116 | file.Close() 117 | b.logger.Error().Msg("invalid crc32 checksum") 118 | if err = os.Remove(path); err != nil { 119 | b.logger.Fatal().Err(err).Msg("unable to remove invalid backlog file") 120 | b.metrics.Increment("remove_error") 121 | } 122 | return 123 | } 124 | 125 | file.Seek(4, 0) // crc offset 126 | url := readUrl(file) 127 | 128 | err = clickhouse.UploadReader(url, file) 129 | 130 | file.Close() 131 | 132 | if err != nil { 133 | b.logger.Error().Err(err).Msg("unable to upload backlog file") 134 | b.metrics.Increment("upload_error") 135 | } else { 136 | if err = os.Remove(path); err != nil { 137 | b.logger.Fatal().Err(err).Msg("unable to remove finished backlog file") 138 | b.metrics.Increment("upload_remove_error") 139 | } 140 | } 141 | 142 | } 143 | 144 | func (b *Backlog) check(done <-chan struct{}) { 145 | b.logger.Debug().Msg("starting backlog check") 146 | files, err := ioutil.ReadDir(b.dir) 147 | if err != nil { 148 | b.logger.Error().Err(err).Msg("unable to read backlog directory") 149 | return 150 | } 151 | 152 | wg := &sync.WaitGroup{} 153 | for _, f := range files { 154 | select { 155 | case <-done: 156 | return 157 | default: 158 | } 159 | if !strings.HasSuffix(f.Name(), backlogSuffix) { 160 | continue 161 | } 162 | 163 | b.limiter.Enter() 164 | wg.Add(1) 165 | go func(name string) { 166 | b.processFile(name) 167 | b.limiter.Leave() 168 | wg.Done() 169 | }(f.Name()) 170 | } 171 | wg.Wait() 172 | return 173 | } 174 | 175 | func (b *Backlog) MakeNewBacklogJob(url string, data []byte) error { 176 | b.makeMu.Lock() 177 | defer b.makeMu.Unlock() 178 | timestamp := strconv.FormatInt(time.Now().Unix(), 10) 179 | file, err := ioutil.TempFile(b.dir, timestamp+"_*"+writeSuffix) 180 | if err != nil { 181 | b.metrics.Increment("tmp_file_create_error") 182 | return errors.Wrap(err, "unable to create tmp file") 183 | } 184 | defer file.Close() 185 | 186 | serializedUrl := serializeString(url) 187 | crcBuf := calcCrc(serializedUrl, data) 188 | 189 | file.Write(crcBuf) 190 | file.Write(serializedUrl) 191 | file.Write(data) 192 | 193 | file.Sync() 194 | 195 | if err := b.Rename(file.Name()); err != nil { 196 | b.metrics.Increment("tmp_file_rename_error") 197 | return errors.Wrap(err, "unable to finish backlog job") 198 | } 199 | b.metrics.Increment("backlog_job_created") 200 | return nil 201 | } 202 | 203 | func (b *Backlog) Rename(oldPath string) error { 204 | newPath := baseFileName(oldPath) + backlogSuffix 205 | return os.Rename(oldPath, newPath) 206 | } 207 | 208 | func (b *Backlog) GetLimiter() utils.Limiter { 209 | return b.limiter 210 | } 211 | 212 | func serializeString(s string) []byte { 213 | b := make([]byte, len(s)+4) 214 | binary.BigEndian.PutUint32(b, uint32(len(s))) 215 | copy(b[4:], s) 216 | return b 217 | } 218 | 219 | func calcCrc(serializedUrl []byte, data []byte) []byte { 220 | h := crc32.NewIEEE() 221 | h.Write(serializedUrl) 222 | h.Write(data) 223 | crcBuf := make([]byte, 4) 224 | binary.BigEndian.PutUint32(crcBuf, h.Sum32()) 225 | return crcBuf 226 | } 227 | 228 | func readUrl(file *os.File) string { 229 | strLenBuf := make([]byte, 4) 230 | file.Read(strLenBuf) 231 | strLen := binary.BigEndian.Uint32(strLenBuf) 232 | 233 | urlBuf := make([]byte, strLen) 234 | file.Read(urlBuf) 235 | return string(urlBuf) 236 | } 237 | 238 | func checkCrc(file *os.File) bool { 239 | crcBuf := make([]byte, 4) 240 | file.Read(crcBuf) 241 | expectedCrc := binary.BigEndian.Uint32(crcBuf) 242 | h := crc32.NewIEEE() 243 | 244 | io.Copy(h, file) 245 | return expectedCrc == h.Sum32() 246 | } 247 | 248 | func baseFileName(path string) string { 249 | for i := len(path) - 1; i >= 0 && !os.IsPathSeparator(path[i]); i-- { 250 | if path[i] == '.' { 251 | return path[:i] 252 | } 253 | } 254 | return path 255 | } 256 | -------------------------------------------------------------------------------- /backlog/backlog_test.go: -------------------------------------------------------------------------------- 1 | package backlog 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestBaseName(t *testing.T) { 10 | table := []struct { 11 | input string 12 | expected string 13 | }{ 14 | {"file", "file"}, 15 | {"file.1.xx", "file.1"}, 16 | {"file.xx", "file"}, 17 | {"/foo/bar/file.xx", "/foo/bar/file"}, 18 | } 19 | 20 | for _, p := range table { 21 | assert.Equal(t, p.expected, baseFileName(p.input)) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /backlog/util.go: -------------------------------------------------------------------------------- 1 | package backlog 2 | -------------------------------------------------------------------------------- /clickhouse/upload.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "net/http" 9 | "time" 10 | 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | const TIMEOUT = time.Minute * 5 15 | 16 | func Upload(uploadUrl string, data []byte) error { 17 | return UploadReader(uploadUrl, bytes.NewReader(data)) 18 | } 19 | 20 | func UploadReader(uploadUrl string, data io.Reader) error { 21 | req, err := http.NewRequest("POST", uploadUrl, data) 22 | client := &http.Client{Timeout: TIMEOUT} 23 | resp, err := client.Do(req) 24 | if err != nil { 25 | return errors.Wrap(err, "upload http error") 26 | } 27 | defer resp.Body.Close() 28 | 29 | body, _ := ioutil.ReadAll(resp.Body) 30 | 31 | if resp.StatusCode != 200 { 32 | return fmt.Errorf("clickhouse response status %d: %s", resp.StatusCode, string(body)) 33 | } 34 | return nil 35 | } 36 | -------------------------------------------------------------------------------- /clickhouse/util.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "strconv" 7 | "strings" 8 | 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | func MakeUrl(dsn, table string, skipUnknownFields bool, allowErrorRatio int) (string, error) { 13 | if !strings.HasSuffix(dsn, "/") { 14 | dsn += "/" 15 | } 16 | u, err := url.Parse(dsn) 17 | if err != nil { 18 | return "", errors.Wrap(err, "unable to make uploader url") 19 | } 20 | 21 | q := u.Query() 22 | q.Set("query", fmt.Sprintf("INSERT INTO %s FORMAT JSONEachRow", table)) 23 | if skipUnknownFields { 24 | q.Set("input_format_skip_unknown_fields", "1") 25 | } 26 | if allowErrorRatio > 0 { 27 | q.Set("input_format_allow_errors_ratio", strconv.Itoa(allowErrorRatio)) 28 | } 29 | 30 | u.RawQuery = q.Encode() 31 | return u.String(), nil 32 | } 33 | -------------------------------------------------------------------------------- /clickhouse/util_test.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestMakeUrl(t *testing.T) { 10 | table := []struct { 11 | inputDSN string 12 | inputTable string 13 | expected string 14 | }{ 15 | {"http://host:333", "db.table", "http://host:333/?input_format_skip_unknown_fields=1&query=INSERT+INTO+db.table+FORMAT+JSONEachRow"}, 16 | {"http://host:333/", "db.table", "http://host:333/?input_format_skip_unknown_fields=1&query=INSERT+INTO+db.table+FORMAT+JSONEachRow"}, 17 | } 18 | 19 | for _, p := range table { 20 | url, err := MakeUrl(p.inputDSN, p.inputTable, true, 0) 21 | assert.Nil(t, err) 22 | assert.Equal(t, p.expected, url) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "nginx-log-collector/processor/functions" 5 | ) 6 | 7 | type Backlog struct { 8 | Dir string `yaml:"dir"` 9 | MaxConcurrentHttpRequests int `yaml:"max_concurrent_http_requests"` 10 | } 11 | 12 | type CollectedLog struct { 13 | Tag string `yaml:"tag"` 14 | Format string `yaml:"format"` 15 | AllowErrorRatio int `yaml:"allow_error_ratio"` 16 | BufferSize int `yaml:"buffer_size"` 17 | 18 | Transformers functions.FunctionSignatureMap `yaml:"transformers"` 19 | Upload Upload `yaml:"upload"` 20 | 21 | Audit bool `yaml:"audit"` // debug feature 22 | } 23 | 24 | type HttpReceiver struct { 25 | Enabled bool `yaml:"enabled"` 26 | Url string `yaml:"url"` 27 | } 28 | 29 | type Logging struct { 30 | Level string `yaml:"level"` 31 | Path string `yaml:"path"` 32 | } 33 | 34 | type PProf struct { 35 | Addr string `yaml:"addr"` 36 | Enabled bool `yaml:"enabled"` 37 | } 38 | 39 | type Processor struct { 40 | Workers int `yaml:"workers"` 41 | } 42 | 43 | type Statsd struct { 44 | Addr string `yaml:"addr"` 45 | Enabled bool `yaml:"enabled"` 46 | Prefix string `yaml:"prefix"` 47 | } 48 | 49 | type TCPReceiver struct { 50 | Addr string `yaml:"addr"` 51 | } 52 | 53 | type Upload struct { 54 | Table string `yaml:"table"` 55 | DSN string `yaml:"dsn"` 56 | } 57 | 58 | type Config struct { 59 | Backlog Backlog `yaml:"backlog"` 60 | CollectedLogs []CollectedLog `yaml:"collected_logs"` 61 | HttpReceiver HttpReceiver `yaml:"httpReceiver"` 62 | Logging Logging `yaml:"logging"` 63 | PProf PProf `yaml:"pprof"` 64 | Processor Processor `yaml:"processor"` 65 | TCPReceiver TCPReceiver `yaml:"tcpReceiver"` 66 | Statsd Statsd `yaml:"statsd"` 67 | GoMaxProcs int `yaml:"gomaxprocs"` 68 | } 69 | -------------------------------------------------------------------------------- /doc/schema.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avito-tech/nginx-log-collector/6f19aa81e35096e1ab92223f96226d2c52625138/doc/schema.jpg -------------------------------------------------------------------------------- /etc/config.yaml: -------------------------------------------------------------------------------- 1 | processor: 2 | workers: 8 3 | 4 | receiver: 5 | addr: 0.0.0.0:4444 6 | 7 | logging: 8 | level: info 9 | path: /var/log/nginx-log-collector/main.log 10 | 11 | statsd: 12 | prefix: resources.monitoring.nginx_log_collector 13 | addr: statsd-aggregator:8125 14 | enabled: true 15 | 16 | pprof: 17 | enabled: true 18 | addr: 0.0.0.0:6060 19 | 20 | backlog: 21 | dir: /var/lib/nginx-log-collector/backlog/ 22 | 23 | collected_logs: 24 | - tag: "nginx:" 25 | format: access # access | error 26 | buffer_size: 104857600 27 | transformers: # possible functions: ipToUint32 | limitMaxLength(int) | toArray | splitAndStore 28 | http_x_real_ip: 29 | ipToUint32: 30 | upstream_response_time: 31 | toArray: 32 | http_referer: 33 | limitMaxLength: 800 34 | request_uri: 35 | splitAndStore: 36 | delimiter: "?" 37 | store_to: 38 | request_uri: 0 39 | request_args: 1 40 | upload: 41 | table: nginx.access_log 42 | dsn: http://localhost:8123/ 43 | 44 | - tag: "nginx_error:" 45 | format: error # access | error 46 | buffer_size: 1048576 47 | upload: 48 | table: nginx.error_log 49 | dsn: http://localhost:8123/ 50 | 51 | 52 | - tag: "iac_logs:" 53 | format: access 54 | buffer_size: 4048576 55 | audit: true # Advanced traffic analysis data log 56 | upload: 57 | table: services.iac_logs 58 | dsn: http://user:passwd@localhost:8123/ 59 | -------------------------------------------------------------------------------- /etc/debian/postinst.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Initial installation: $1 == 1 6 | # Upgrade: $1 == 2, and configured to restart on upgrade 7 | #if [ $1 -eq 1 ] ; then 8 | if ! getent group "log-collector" > /dev/null 2>&1 ; then 9 | groupadd -r "log-collector" 10 | fi 11 | if ! getent passwd "log-collector" > /dev/null 2>&1 ; then 12 | useradd -r -g log-collector -d /usr/share/log-collector -s /sbin/nologin \ 13 | -c "log collector user" log-collector 14 | fi 15 | chown -R log-collector /var/lib/nginx-log-collector 16 | chown -R log-collector /var/log/nginx-log-collector 17 | /bin/systemctl daemon-reload 18 | /bin/systemctl enable nginx-log-collector.service 19 | #fi -------------------------------------------------------------------------------- /etc/debian/prerm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Initial installation: $1 == 1 6 | # Upgrade: $1 == 2, and configured to restart on upgrade 7 | #if [ $1 -eq 1 ] ; then 8 | /bin/systemctl stop nginx-log-collector.service 9 | #fi -------------------------------------------------------------------------------- /etc/examples/clickhouse/table_schema.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE IF NOT EXISTS nginx; 2 | 3 | set allow_experimental_low_cardinality_type=1; 4 | 5 | CREATE TABLE nginx.access_log_shard 6 | ( 7 | event_datetime DateTime, 8 | event_date Date, 9 | server_name LowCardinality(String), 10 | remote_user String, 11 | http_x_real_ip UInt32, 12 | status UInt16, 13 | scheme LowCardinality(String), 14 | request_method LowCardinality(String), 15 | request_uri String, 16 | server_protocol LowCardinality(String), 17 | body_bytes_sent UInt64, 18 | request_bytes UInt64, 19 | http_referer String, 20 | http_user_agent LowCardinality(String), 21 | request_time Float32, 22 | upstream_response_time Array(Float32), 23 | hostname LowCardinality(String), 24 | host LowCardinality(String) 25 | ) 26 | ENGINE = MergeTree(event_date, (hostname, request_uri, event_date), 8192) 27 | 28 | 29 | CREATE TABLE nginx.access_log 30 | ( 31 | event_datetime DateTime, 32 | event_date Date, 33 | server_name LowCardinality(String), 34 | remote_user String, 35 | http_x_real_ip UInt32, 36 | status UInt16, 37 | scheme LowCardinality(String), 38 | request_method LowCardinality(String), 39 | request_uri String, 40 | server_protocol LowCardinality(String), 41 | body_bytes_sent UInt64, 42 | request_bytes UInt64, 43 | http_referer String, 44 | http_user_agent LowCardinality(String), 45 | request_time Float32, 46 | upstream_response_time Array(Float32), 47 | hostname LowCardinality(String), 48 | host LowCardinality(String) 49 | ) 50 | ENGINE = Distributed('logs_cluster', 'nginx', 'access_log_shard', rand()) 51 | 52 | 53 | CREATE TABLE nginx.error_log 54 | ( 55 | event_datetime DateTime, 56 | event_date Date, 57 | server_name LowCardinality(String), 58 | http_referer String, 59 | pid UInt32, 60 | sid UInt32, 61 | tid UInt64, 62 | host LowCardinality(String), 63 | client String, 64 | request String, 65 | message String, 66 | login String, 67 | upstream String, 68 | subrequest String, 69 | hostname LowCardinality(String) 70 | ) 71 | ENGINE = ReplicatedMergeTree('/clickhouse/tables/logs_replicator/nginx.error2_log', _SET_ME_, event_date, (server_name, request, event_date), 8192) 72 | -------------------------------------------------------------------------------- /etc/examples/example_config.yaml: -------------------------------------------------------------------------------- 1 | processor: 2 | workers: 8 3 | 4 | httpReceiver: 5 | enabled: true 6 | url: 0.0.0.0:4446 7 | 8 | tcpReceiver: 9 | addr: 0.0.0.0:4444 10 | 11 | logging: 12 | level: debug 13 | 14 | statsd: 15 | # prefix: resources.monitoring.nginx_log_collector 16 | prefix: complex.delete_me.nginx_log_collector 17 | addr: localhost:2003 18 | enabled: false 19 | 20 | pprof: 21 | enabled: true 22 | addr: 0.0.0.0:6060 23 | 24 | backlog: 25 | dir: /tmp/backlog 26 | 27 | 28 | collected_logs: 29 | - tag: "nginx:" 30 | format: access # access | error 31 | allow_error_ratio: 10 32 | buffer_size: 8388608 33 | transformers: # possible functions: ipToUint32 | limitMaxLength(int) 34 | ip: ipToUint32 35 | remote_addr: ipToUint32 36 | request_uri: limitMaxLength(100) 37 | upstream_response_time: toArray 38 | upload: 39 | table: nginx.access_log 40 | dsn: http://localhost:8123/ 41 | 42 | - tag: "nginx_error:" 43 | format: error # access | error 44 | buffer_size: 8388608 45 | # transformers: # possible functions: ipToUint32 | limitMaxLength(int) 46 | # ip: ipToUint32 47 | # request_uri: limitMaxLength(100) 48 | upload: 49 | table: nginx.error_log 50 | dsn: http://localhost:8123/ 51 | -------------------------------------------------------------------------------- /etc/examples/nginx.conf: -------------------------------------------------------------------------------- 1 | user www-data; 2 | worker_processes auto; 3 | pid /run/nginx.pid; 4 | include /etc/nginx/modules-enabled/*.conf; 5 | 6 | events { 7 | worker_connections 768; 8 | # multi_accept on; 9 | } 10 | 11 | http { 12 | 13 | ## 14 | # Basic Settings 15 | ## 16 | 17 | sendfile on; 18 | tcp_nopush on; 19 | tcp_nodelay on; 20 | keepalive_timeout 65; 21 | types_hash_max_size 2048; 22 | # server_tokens off; 23 | 24 | # server_names_hash_bucket_size 64; 25 | # server_name_in_redirect off; 26 | 27 | include /etc/nginx/mime.types; 28 | default_type application/octet-stream; 29 | 30 | ## 31 | # SSL Settings 32 | ## 33 | 34 | ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE 35 | ssl_prefer_server_ciphers on; 36 | 37 | ## 38 | # Logging Settings 39 | ## 40 | log_format avito_json escape=json 41 | '{' 42 | '"event_datetime": "$time_iso8601", ' 43 | '"server_name": "$server_name", ' 44 | '"remote_addr": "$remote_addr", ' 45 | '"remote_user": "$remote_user", ' 46 | '"http_x_real_ip": "$http_x_real_ip", ' 47 | '"status": "$status", ' 48 | '"scheme": "$scheme", ' 49 | '"request_method": "$request_method", ' 50 | '"request_uri": "$request_uri", ' 51 | '"server_protocol": "$server_protocol", ' 52 | '"body_bytes_sent": $body_bytes_sent, ' 53 | '"http_referer": "$http_referer", ' 54 | '"http_user_agent": "$http_user_agent", ' 55 | '"request_bytes": "$request_length", ' 56 | '"request_time": "$request_time", ' 57 | '"upstream_response_time": "$upstream_response_time", ' 58 | '"hostname": "$hostname", ' 59 | '"host": "$host"' 60 | '}'; 61 | 62 | access_log syslog:server=unix:/var/run/nginx_log.sock,nohostname,tag=nginx avito_json; #ClickHouse 63 | error_log syslog:server=unix:/var/run/nginx_log.sock,nohostname,tag=nginx_error; #ClickHouse 64 | 65 | ## 66 | # Gzip Settings 67 | ## 68 | 69 | gzip on; 70 | 71 | # gzip_vary on; 72 | # gzip_proxied any; 73 | # gzip_comp_level 6; 74 | # gzip_buffers 16 8k; 75 | # gzip_http_version 1.1; 76 | # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; 77 | 78 | ## 79 | # Virtual Host Configs 80 | ## 81 | 82 | include /etc/nginx/conf.d/*.conf; 83 | include /etc/nginx/sites-enabled/*; 84 | } 85 | 86 | 87 | #mail { 88 | # # See sample authentication script at: 89 | # # http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript 90 | # 91 | # # auth_http localhost/auth.php; 92 | # # pop3_capabilities "TOP" "USER"; 93 | # # imap_capabilities "IMAP4rev1" "UIDPLUS"; 94 | # 95 | # server { 96 | # listen localhost:110; 97 | # protocol pop3; 98 | # proxy on; 99 | # } 100 | # 101 | # server { 102 | # listen localhost:143; 103 | # protocol imap; 104 | # proxy on; 105 | # } 106 | #} 107 | -------------------------------------------------------------------------------- /etc/examples/rsyslog/01-nginx-tcp.conf: -------------------------------------------------------------------------------- 1 | input( 2 | type="imuxsock" 3 | Socket="/var/run/nginx_log.sock" 4 | UseSysTimeStamp="off" 5 | UseSpecialParser="off" 6 | ParseHostname="off" 7 | ruleset="fwd" 8 | ) 9 | 10 | template( 11 | name="TSV" 12 | type="string" 13 | string="%HOSTNAME%\t%syslogtag%\t%msg%" 14 | ) 15 | 16 | ruleset(name="fwd") { 17 | $RepeatedMsgReduction off 18 | action( 19 | type="omfwd" 20 | name="fwd_to_logserver" 21 | target="localhost" 22 | port="4444" 23 | protocol="tcp" 24 | template="TSV" 25 | action.resumeretrycount="20" 26 | action.resumeInterval="10" 27 | queue.spoolDirectory="/var/spool/rsyslog" 28 | queue.filename="fwd_to_logserver" # enable disk-assisted queue (for restarts) 29 | queue.type="fixedArray" 30 | queue.size="250000" 31 | queue.discardmark="245000" 32 | queue.dequeueBatchSize="4096" 33 | queue.workerThreads="4" 34 | queue.workerThreadMinimumMessages="60000" 35 | queue.discardseverity="0" # discard all 36 | ) 37 | 38 | 39 | stop 40 | } 41 | -------------------------------------------------------------------------------- /etc/examples/rsyslog/30-nginx-zmq.conf: -------------------------------------------------------------------------------- 1 | module( 2 | load="omczmq" 3 | authenticator="off" 4 | ) 5 | module( 6 | load="impstats" 7 | interval="1" # how often to generate stats 8 | resetCounters="on" # to get deltas (e.g. # of messages submitted in the last 10 seconds) 9 | log.file="/tmp/stats" # file to write those stats to 10 | log.syslog="off" # don't send stats through the normal processing pipeline. More on that in a bit 11 | ) 12 | 13 | 14 | input(type="imuxsock" 15 | Socket="/var/run/nginx_log.sock" UseSysTimeStamp="off" useSpecialParser="off" parseHostname="off") 16 | 17 | template (name="TSV" type="string" 18 | string="%HOSTNAME%\t%syslogtag%\t%msg%") 19 | 20 | 21 | ruleset(name="zmq") { 22 | action( 23 | name="to_zeromq" 24 | type="omczmq" 25 | socktype="PUSH" 26 | endpoints="tcp://localhost:4444" 27 | template="TSV" 28 | sendhwm="2000000" 29 | ) 30 | } 31 | 32 | 33 | if $syslogtag contains 'nginx:' then { 34 | $RepeatedMsgReduction off 35 | call zmq 36 | 37 | stop 38 | } 39 | -------------------------------------------------------------------------------- /etc/make-deb-package.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "github.com/goreleaser/nfpm" 8 | _ "github.com/goreleaser/nfpm/deb" 9 | ) 10 | 11 | // should be filled by go build 12 | var Version = "0.0.0-devel" 13 | 14 | func defaultInfo() nfpm.Info { 15 | return nfpm.WithDefaults(nfpm.Info{ 16 | Name: "nginx-log-collector", 17 | Arch: "amd64", 18 | Description: "Collects nginx logs from rsyslog and saves them into clickhouse dbms", 19 | Maintainer: "Oleg Matrokhin ", 20 | Version: Version, 21 | Overridables: nfpm.Overridables{ 22 | Files: map[string]string{ 23 | "./build/nginx-log-collector": "/usr/bin/nginx-log-collector", 24 | }, 25 | EmptyFolders: []string{ 26 | "/var/lib/nginx-log-collector/backlog/", 27 | "/var/log/nginx-log-collector/", 28 | }, 29 | ConfigFiles: map[string]string{ 30 | "./etc/config.yaml": "/etc/nginx-log-collector/config.yaml", 31 | "./etc/nginx-log-collector.service": "/lib/systemd/system/nginx-log-collector.service", 32 | }, 33 | Scripts: nfpm.Scripts{ 34 | PostInstall: "./etc/debian/postinst.sh", 35 | PreRemove: "./etc/debian/prerm.sh", 36 | }, 37 | }, 38 | }) 39 | } 40 | 41 | func main() { 42 | pkg, err := nfpm.Get("deb") 43 | if err != nil { 44 | log.Fatalln(err) 45 | } 46 | i := defaultInfo() 47 | 48 | f, err := os.Create("./build/nginx-log-collector-" + Version + ".deb") 49 | if err != nil { 50 | log.Fatalln(err) 51 | } 52 | 53 | pkg.Package(i, f) 54 | f.Close() 55 | } 56 | -------------------------------------------------------------------------------- /etc/nginx-log-collector.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Collects nginx logs from rsyslog and saves them into clickhouse dbms 3 | Wants=network-online.target 4 | After=network-online.target 5 | 6 | [Service] 7 | User=log-collector 8 | Group=log-collector 9 | Type=simple 10 | PermissionsStartOnly=true 11 | ExecStart=/usr/bin/nginx-log-collector -config /etc/nginx-log-collector/config.yaml 12 | Restart=on-failure 13 | 14 | [Install] 15 | WantedBy=multi-user.target 16 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module nginx-log-collector 2 | 3 | require ( 4 | github.com/BurntSushi/toml v0.3.1 // indirect 5 | github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 // indirect 6 | github.com/buger/jsonparser v0.0.0-20180910192245-6acdf747ae99 7 | github.com/davecgh/go-spew v1.1.1 // indirect 8 | github.com/goreleaser/nfpm v0.9.5 9 | github.com/imdario/mergo v0.3.6 // indirect 10 | github.com/kr/pretty v0.1.0 // indirect 11 | github.com/mattn/go-zglob v0.0.0-20180803001819-2ea3427bfa53 // indirect 12 | github.com/pkg/errors v0.8.0 13 | github.com/pmezard/go-difflib v1.0.0 // indirect 14 | github.com/rs/zerolog v1.9.1 15 | github.com/stretchr/testify v1.2.2 16 | github.com/valyala/fastjson v0.0.0-20180829103600-37952265e1c0 17 | gopkg.in/alexcesaro/statsd.v2 v2.0.0 18 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect 19 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 20 | gopkg.in/yaml.v2 v2.2.1 21 | ) 22 | 23 | go 1.13 24 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 h1:oMCHnXa6CCCafdPDbMh/lWRhRByN0VFLvv+g+ayx1SI= 4 | github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= 5 | github.com/buger/jsonparser v0.0.0-20180910192245-6acdf747ae99 h1:yxtDQw7A+kLZZaufGxZtKDkKXbk+/7dguKjFUdlXocg= 6 | github.com/buger/jsonparser v0.0.0-20180910192245-6acdf747ae99/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= 7 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 8 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/goreleaser/nfpm v0.9.5 h1:ntRGZSucXRjoCk6FdwJaXcCZxZZu7YoqX7UH5IC13l4= 10 | github.com/goreleaser/nfpm v0.9.5/go.mod h1:kn0Dps10Osi7V2icEXFTBRZhmiuGPUizzZVw/WQtQ/k= 11 | github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= 12 | github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 13 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 14 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 15 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 16 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 17 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 18 | github.com/mattn/go-zglob v0.0.0-20180803001819-2ea3427bfa53 h1:tGfIHhDghvEnneeRhODvGYOt305TPwingKt6p90F4MU= 19 | github.com/mattn/go-zglob v0.0.0-20180803001819-2ea3427bfa53/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= 20 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= 21 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 22 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 23 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 24 | github.com/rs/zerolog v1.9.1 h1:AjV/SFRF0+gEa6rSjkh0Eji/DnkrJKVpPho6SW5g4mU= 25 | github.com/rs/zerolog v1.9.1/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= 26 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 27 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 28 | github.com/valyala/fastjson v0.0.0-20180829103600-37952265e1c0 h1:CHfx7L6F8ODsyZSpt17JAaMyyeg7N+Gizas3+cRiyuQ= 29 | github.com/valyala/fastjson v0.0.0-20180829103600-37952265e1c0/go.mod h1:nV6MsjxL2IMJQUoHDIrjEI7oLyeqK6aBD7EFWPsvP8o= 30 | gopkg.in/alexcesaro/statsd.v2 v2.0.0 h1:FXkZSCZIH17vLCO5sO2UucTHsH9pc+17F6pl3JVCwMc= 31 | gopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7SLonnE4drU= 32 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 33 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 34 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 35 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= 36 | gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= 37 | gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= 38 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 39 | -------------------------------------------------------------------------------- /nginx-log-collector.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "io/ioutil" 6 | "math/rand" 7 | "net/http" 8 | _ "net/http/pprof" 9 | "os" 10 | "os/signal" 11 | "runtime" 12 | "strings" 13 | "syscall" 14 | "time" 15 | 16 | "github.com/rs/zerolog" 17 | "github.com/rs/zerolog/log" 18 | "nginx-log-collector/config" 19 | "nginx-log-collector/service" 20 | "gopkg.in/alexcesaro/statsd.v2" 21 | "gopkg.in/natefinch/lumberjack.v2" 22 | "gopkg.in/yaml.v2" 23 | ) 24 | 25 | // should be filled by go build 26 | var Version = "0.0.0-devel" 27 | 28 | func loadConfig(configFile string) *config.Config { 29 | data, err := ioutil.ReadFile(configFile) 30 | if err != nil { 31 | log.Fatal().Err(err).Msg("unable to read config file") 32 | } 33 | 34 | cfg := &config.Config{} 35 | if err := yaml.Unmarshal(data, cfg); err != nil { 36 | log.Fatal().Err(err).Msg("unable to parse config") 37 | } 38 | return cfg 39 | } 40 | 41 | func setupLogger(cfg config.Logging) (*zerolog.Logger, error) { 42 | lvl, err := zerolog.ParseLevel(strings.ToLower(cfg.Level)) 43 | if err != nil { 44 | return nil, err 45 | } 46 | zerolog.SetGlobalLevel(lvl) 47 | 48 | var z zerolog.Logger 49 | 50 | if cfg.Path != "" && cfg.Path != "stdout" { 51 | _, err := os.OpenFile(cfg.Path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) 52 | if err != nil { 53 | return nil, err 54 | } 55 | z = zerolog.New(&lumberjack.Logger{ 56 | Filename: cfg.Path, 57 | MaxSize: 200, // megabytes 58 | MaxAge: 180, // days 59 | }) 60 | } else { 61 | z = zerolog.New(os.Stdout) 62 | } 63 | zt := z.With().Timestamp().Logger() 64 | return &zt, nil 65 | } 66 | 67 | func setupStatsD(cfg config.Statsd) (*statsd.Client, error) { 68 | hostname, err := os.Hostname() 69 | if err != nil { 70 | return nil, err 71 | } 72 | hostname = strings.Replace(hostname, ".", "_", -1) 73 | return statsd.New( 74 | statsd.Address(cfg.Addr), 75 | statsd.Prefix(cfg.Prefix+".host."+hostname), 76 | statsd.Mute(!cfg.Enabled), 77 | ) 78 | } 79 | 80 | func main() { 81 | rand.Seed(time.Now().UnixNano()) 82 | 83 | configFile := flag.String("config", "", "Config path") 84 | flag.Parse() 85 | 86 | if *configFile == "" { 87 | log.Fatal().Msg("-config flag should be set") 88 | } 89 | cfg := loadConfig(*configFile) 90 | 91 | logger, err := setupLogger(cfg.Logging) 92 | if err != nil { 93 | log.Fatal().Err(err).Msg("unable to setup logging") 94 | } 95 | 96 | logger.Info().Str("version", Version).Msg("start") 97 | 98 | if cfg.GoMaxProcs > 0 { 99 | runtime.GOMAXPROCS(cfg.GoMaxProcs) 100 | logger.Info().Int("gomaxprocs", cfg.GoMaxProcs).Msg("gomaxprocs set") 101 | } 102 | 103 | metrics, err := setupStatsD(cfg.Statsd) 104 | if err != nil { 105 | logger.Fatal().Err(err).Msg("unable to setup statsd client") 106 | } 107 | 108 | done := make(chan struct{}, 1) 109 | go func() { 110 | c := make(chan os.Signal, 1) 111 | signal.Notify(c, os.Interrupt, syscall.SIGTERM) 112 | <-c 113 | logger.Info().Msg("got signal; exiting") 114 | close(done) 115 | }() 116 | 117 | s, err := service.New(cfg, metrics, logger) 118 | if err != nil { 119 | logger.Fatal().Err(err).Msg("unable to init service") 120 | } 121 | 122 | if cfg.PProf.Enabled { 123 | logger.Info().Msg("starting pprof server") 124 | go func() { 125 | logger.Warn().Err( 126 | http.ListenAndServe(cfg.PProf.Addr, nil), 127 | ).Msg("pprof server error") 128 | }() 129 | } 130 | 131 | s.Start(done) 132 | 133 | logger.Info().Msg("exit") 134 | } 135 | -------------------------------------------------------------------------------- /parser/error_log.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "sort" 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | var nginxErrorLogVariables = []string{ 13 | ", client: ", 14 | ", server: ", 15 | ", login: ", 16 | ", upstream: ", 17 | ", request: ", 18 | ", subrequest: ", 19 | ", host: ", 20 | ", referrer: ", 21 | } 22 | 23 | func NginxErrorLogMessage(msg []byte, out map[string]interface{}) error { 24 | msg = bytes.TrimPrefix(msg, []byte(" ")) 25 | if len(msg) < 19 { 26 | return errors.New("line too short") 27 | } 28 | // TODO check field names 29 | 30 | text := string(msg) 31 | 32 | // XXX skip timestamp parsing because of missing timezone. Use current server time instead 33 | // timestamp, err := time.ParseInLocation("2006/01/02 15:04:05", text[:19], time.Local) 34 | // if err != nil { 35 | // return err 36 | // } 37 | // out["event_datetime"] = timestamp.Format(dateTimeFmt) 38 | // out["event_date"] = timestamp.Format("2006-01-02") 39 | 40 | var p1, p2 int 41 | 42 | p1 = strings.IndexByte(text, '[') 43 | p2 = strings.IndexByte(text, ']') 44 | 45 | if p1 < 0 || p2 < 0 || p2 <= p1 { 46 | return errors.New("can't find error level") 47 | } 48 | 49 | out["level"] = text[p1+1 : p2] 50 | 51 | text = text[p2+2:] 52 | 53 | p1 = strings.IndexByte(text, '#') 54 | if p1 < 0 { 55 | return errors.New("PID not found") 56 | } 57 | 58 | var err error 59 | out["pid"], err = strconv.Atoi(text[:p1]) 60 | if err != nil { 61 | return fmt.Errorf("wrong PID: %s", text[:p1]) 62 | } 63 | 64 | text = text[p1+1:] 65 | 66 | p1 = strings.IndexByte(text, ':') 67 | if p1 < 0 { 68 | return errors.New("TID not found") 69 | } 70 | out["tid"], err = strconv.Atoi(text[:p1]) 71 | if err != nil { 72 | return fmt.Errorf("wrong TID: %s", text[:p1]) 73 | } 74 | 75 | text = text[p1+2:] 76 | 77 | if text[0] == '*' { 78 | p1 = strings.IndexByte(text, ' ') 79 | if p1 < 0 { 80 | return errors.New("SID not found") 81 | } 82 | out["sid"], err = strconv.Atoi(text[1:p1]) 83 | if err != nil { 84 | return fmt.Errorf("wrong SID: %s", text[1:p1]) 85 | } 86 | 87 | text = text[p1+1:] 88 | } 89 | 90 | indexes := make([]int, 0, len(nginxErrorLogVariables)+1) 91 | 92 | for i := 0; i < len(nginxErrorLogVariables); i++ { 93 | p1 = strings.LastIndex(text, nginxErrorLogVariables[i]) 94 | if p1 < 0 { 95 | continue 96 | } 97 | indexes = append(indexes, p1) 98 | } 99 | 100 | if len(indexes) == 0 { 101 | out["message"] = text 102 | return nil 103 | } 104 | 105 | indexes = append(indexes, len(text)) 106 | 107 | sort.Ints(indexes) 108 | 109 | out["message"] = text[:indexes[0]] 110 | 111 | for i := 0; i < len(indexes)-1; i++ { 112 | s := text[indexes[i]:indexes[i+1]] 113 | p1 = strings.IndexByte(s, ':') 114 | v := s[p1+2:] 115 | if len(v) > 0 && v[0] == '"' { 116 | v = v[1:] 117 | } 118 | if len(v) > 0 && v[len(v)-1] == '"' { 119 | v = v[:len(v)-1] 120 | } 121 | k := s[2:p1] 122 | if k == "server" { 123 | k = "server_name" 124 | } else if k == "referrer" { 125 | k = "http_referer" 126 | } 127 | out[k] = v 128 | } 129 | 130 | return nil 131 | } 132 | -------------------------------------------------------------------------------- /parser/error_log_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "io/ioutil" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestParsing(t *testing.T) { 11 | table := []struct { 12 | inputFile string 13 | expectedHost string 14 | }{ 15 | {"error1", "g.avito.ru"}, 16 | {"errorphp", "www.avito.ru"}, 17 | } 18 | 19 | for _, p := range table { 20 | data, err := ioutil.ReadFile("./testdata/" + p.inputFile) 21 | assert.Nil(t, err) 22 | 23 | out := make(map[string]interface{}) 24 | err = NginxErrorLogMessage(data, out) 25 | assert.Nil(t, err) 26 | assert.Equal(t, p.expectedHost, out["host"]) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /parser/testdata/error1: -------------------------------------------------------------------------------- 1 | 2018/04/11 21:57:59 [error] 702#702: *693176 connect() failed (111: Connection refused) while connecting to upstream, client: 10.8.232.43, server: grafana, request: "GET /grafana/api/org HTTP/1.1", upstream: "http://127.0.0.1:3000/api/org", host: "g.test.ru" 2 | -------------------------------------------------------------------------------- /parser/testdata/errorphp: -------------------------------------------------------------------------------- 1 | 2018/09/20 13:37:39 [error] 55836#0: *6417865223 FastCGI sent in stderr: "PHP message: PHP Fatal error: Uncaught Exception\Warning: Imagick::__construct(): HTTP request failed! HTTP/1.1 404 Not Found 2 | in /home/www-data/tags/rc-201809201033/release3/lib/Controller/Common/X.php:17 3 | Stack trace: 4 | #0 /home/www-data/tags/rc-201809201033/release3/lib/bootstrap.php(214): Bootstrap::errorHandler(2, 'Imagick::__cons...', '/home/www-data/...', 17, Array) 5 | #1 /home/www-data/tags/rc-201809201033/release3/lib/Utils/ShutdownHandler.php(16): Bootstrap::shutdownHandler() 6 | #2 [internal function]: Utils\ShutdownHandler->Utils\{closure}() 7 | #3 {main}" 8 | 9 | thrown in /home/www-data/tags/rc-201809201033/release3/lib/Controller/Common/X.php on line 17" while reading upstream, client: 10.0.0.205, server: www.test.ru, request: "GET /img/share/auto/1 HTTP/1.0", upstream: "fastcgi://127.0.0.1:9000", host: "www.test.ru" 10 | -------------------------------------------------------------------------------- /processor/access_log.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/buger/jsonparser" 7 | "github.com/pkg/errors" 8 | "github.com/valyala/fastjson" 9 | 10 | "nginx-log-collector/processor/functions" 11 | "nginx-log-collector/utils" 12 | ) 13 | 14 | type AccessLogConverter struct { 15 | transformers []transformer 16 | } 17 | 18 | var datetimeTransformers = []*utils.DatetimeTransformer{ 19 | { 20 | "2006-01-02T15:04:05.000000000Z07:00", 21 | "2006-01-02T15:04:05.000000000", 22 | time.Local, 23 | }, 24 | { 25 | time.RFC3339, 26 | dateTimeFmt, 27 | time.Local, 28 | }, 29 | { 30 | "2006-01-02T15:04:05.999999999", 31 | "2006-01-02T15:04:05.999999999", 32 | time.UTC, 33 | }, 34 | { 35 | "2006-01-02T15:04:05.999999", 36 | "2006-01-02T15:04:05.999999", 37 | time.UTC, 38 | }, 39 | { 40 | "2006-01-02T15:04:05.999", 41 | "2006-01-02T15:04:05.999", 42 | time.UTC, 43 | }, 44 | } 45 | 46 | func NewAccessLogConverter(transformerMap functions.FunctionSignatureMap) (*AccessLogConverter, error) { 47 | transformers, err := parseTransformersMap(transformerMap) 48 | if err != nil { 49 | return nil, errors.Wrap(err, "unable to create access_log converter") 50 | } 51 | return &AccessLogConverter{ 52 | transformers: transformers, 53 | }, nil 54 | } 55 | 56 | func (a *AccessLogConverter) Convert(msg []byte, _ string) ([]byte, error) { 57 | if err := fastjson.ValidateBytes(msg); err != nil { 58 | return nil, errors.Wrap(err, "invalid json") 59 | } 60 | 61 | val, err := jsonparser.GetUnsafeString(msg, dateTimeField) 62 | if err != nil { 63 | return nil, errors.Wrap(err, "unable to get datetime field") 64 | } 65 | 66 | // try to transform source datetime to naive datetime and date strings using several datetime formats 67 | parsed, transformer, err := utils.TryDatetimeFormats(val, datetimeTransformers) 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | msg, err = jsonparser.Set(msg, []byte(`"`+parsed.Format(transformer.FormatDst)+`"`), dateTimeField) 73 | if err != nil { 74 | return nil, errors.Wrap(err, "unable to set datetime field") 75 | } 76 | msg, err = jsonparser.Set(msg, []byte(`"`+parsed.Format(dateFmt)+`"`), dateField) 77 | if err != nil { 78 | return nil, errors.Wrap(err, "unable to set date field") 79 | } 80 | 81 | return a.transform(msg) 82 | } 83 | 84 | func (a *AccessLogConverter) transform(msg []byte) ([]byte, error) { 85 | for _, tr := range a.transformers { 86 | val, err := jsonparser.GetUnsafeString(msg, tr.fieldNameSrc) 87 | if err != nil { 88 | continue 89 | } 90 | 91 | callResult := tr.function.Call(val) 92 | for _, chunk := range callResult { 93 | var fieldName string 94 | if chunk.DstFieldName != nil { 95 | fieldName = *chunk.DstFieldName 96 | } else { 97 | fieldName = tr.fieldNameSrc 98 | } 99 | 100 | msg, err = jsonparser.Set(msg, chunk.Value, fieldName) 101 | if err != nil { 102 | return nil, errors.Wrap(err, "unable to set field") 103 | } 104 | } 105 | } 106 | return msg, nil 107 | } 108 | -------------------------------------------------------------------------------- /processor/convert.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "fmt" 5 | 6 | "nginx-log-collector/config" 7 | ) 8 | 9 | const ( 10 | dateTimeField = "event_datetime" 11 | dateField = "event_date" 12 | dateTimeFmt = "2006-01-02 15:04:05" 13 | dateFmt = "2006-01-02" 14 | ) 15 | 16 | type Converter interface { 17 | Convert([]byte, string) ([]byte, error) 18 | } 19 | 20 | func NewConverter(cfg config.CollectedLog) (Converter, error) { 21 | switch cfg.Format { 22 | case "access": 23 | return NewAccessLogConverter(cfg.Transformers) 24 | case "error": 25 | return NewErrorLogConverter(cfg.Transformers) 26 | default: 27 | return nil, fmt.Errorf("unknown log format: %s", cfg.Format) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /processor/error_log.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | 7 | "github.com/pkg/errors" 8 | "nginx-log-collector/parser" 9 | "nginx-log-collector/processor/functions" 10 | ) 11 | 12 | type ErrorLogConverter struct { 13 | transformers []transformer 14 | } 15 | 16 | func NewErrorLogConverter(transformerMap functions.FunctionSignatureMap) (*ErrorLogConverter, error) { 17 | transformers, err := parseTransformersMap(transformerMap) 18 | if err != nil { 19 | return nil, errors.Wrap(err, "unable to create error_log converter") 20 | } 21 | return &ErrorLogConverter{ 22 | transformers: transformers, 23 | }, nil 24 | } 25 | 26 | func (e *ErrorLogConverter) Convert(msg []byte, hostname string) ([]byte, error) { 27 | now := time.Now() 28 | v := make(map[string]interface{}) 29 | err := parser.NginxErrorLogMessage(msg, v) 30 | if err != nil { 31 | return nil, err 32 | } 33 | v["hostname"] = hostname 34 | v[dateField] = now.Format(dateFmt) 35 | v[dateTimeField] = now.Format(dateTimeFmt) 36 | 37 | e.transform(v) 38 | return json.Marshal(v) 39 | } 40 | 41 | func (e *ErrorLogConverter) transform(v map[string]interface{}) { 42 | for _, tr := range e.transformers { 43 | value, found := v[tr.fieldNameSrc] 44 | if !found { 45 | continue 46 | } 47 | strValue, ok := value.(string) 48 | if !ok { 49 | continue 50 | } 51 | 52 | callResult := tr.function.Call(strValue) 53 | for _, chunk := range callResult { 54 | var fieldName string 55 | if chunk.DstFieldName != nil { 56 | fieldName = *chunk.DstFieldName 57 | } else { 58 | fieldName = tr.fieldNameSrc 59 | } 60 | 61 | v[fieldName] = chunk.Value 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /processor/functions/calculateSHA1.go: -------------------------------------------------------------------------------- 1 | package functions 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha1" 6 | "fmt" 7 | ) 8 | 9 | type calculateSHA1 struct { 10 | StoreTo *string `yaml:"store_to,omitempty"` 11 | } 12 | 13 | func (f *calculateSHA1) Call(value string) FunctionResult { 14 | b := bytes.Buffer{} 15 | result := FunctionPartialResult{DstFieldName: f.StoreTo} 16 | 17 | b.WriteByte('"') 18 | hash := sha1.Sum([]byte(value)) 19 | hashPrintable := fmt.Sprintf("%x", hash) 20 | b.WriteString(hashPrintable) 21 | b.WriteByte('"') 22 | 23 | result.Value = b.Bytes() 24 | return FunctionResult{result} 25 | } 26 | -------------------------------------------------------------------------------- /processor/functions/calculateSHA1_test.go: -------------------------------------------------------------------------------- 1 | package functions 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestCalculateSHA1(t *testing.T) { 10 | type testCase struct { 11 | input string 12 | expected string 13 | } 14 | table := []testCase{ 15 | { 16 | input: "The quick brown fox jumps over the lazy dog", 17 | expected: `"2fd4e1c67a2d28fced849ee1bb76e7391b93eb12"`, 18 | }, 19 | { 20 | input: "", 21 | expected: `"da39a3ee5e6b4b0d3255bfef95601890afd80709"`, 22 | }, 23 | } 24 | 25 | for _, p := range table { 26 | callable := &calculateSHA1{} 27 | v := callable.Call(p.input) 28 | assert.Equal(t, len(v), 1) 29 | 30 | assert.Equal(t, p.expected, string(v[0].Value)) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /processor/functions/dispatch.go: -------------------------------------------------------------------------------- 1 | package functions 2 | 3 | import ( 4 | "fmt" 5 | "gopkg.in/yaml.v2" 6 | ) 7 | 8 | func Dispatch(signature FunctionSignature) (Callable, error) { 9 | if len(signature) != 1 { 10 | return nil, fmt.Errorf("function signature must have exactly one key, got %d instead", len(signature)) 11 | } 12 | 13 | var ( 14 | callable Callable 15 | err error 16 | functionName string 17 | functionExtra interface{} 18 | ) 19 | 20 | for key, value := range signature { 21 | functionName = key 22 | functionExtra = value 23 | break 24 | } 25 | 26 | if functionName == "ipToUint32" { 27 | callable, err = validateIpToUint32(functionExtra) 28 | } else if functionName == "limitMaxLength" { 29 | callable, err = validateLimitMaxLength(functionExtra) 30 | } else if functionName == "splitAndStore" { 31 | callable, err = validateSplitAndStore(functionExtra) 32 | } else if functionName == "toArray" { 33 | callable, err = validateToArray(functionExtra) 34 | } else if functionName == "calculateSHA1" { 35 | callable, err = validateCalculateSHA1(functionExtra) 36 | } else { 37 | err = fmt.Errorf("unknown function name: %s", functionName) 38 | } 39 | 40 | return callable, err 41 | } 42 | 43 | func expectValueEmpty(data interface{}) error { 44 | if data == nil { 45 | return nil 46 | } else { 47 | switch dataType := data.(type) { 48 | case string: 49 | if value := data.(string); value != "" { 50 | return fmt.Errorf("expects empty value, got \"%s\" instead", value) 51 | } else { 52 | return nil 53 | } 54 | default: 55 | return fmt.Errorf("expects empty value, got type %T instead", dataType) 56 | } 57 | } 58 | } 59 | 60 | func expectValuePositiveInt(data interface{}) error { 61 | switch dataType := data.(type) { 62 | case int: 63 | if value := data.(int); value <= 0 { 64 | return fmt.Errorf("expects positive integer value, got %d instead", value) 65 | } else { 66 | return nil 67 | } 68 | default: 69 | return fmt.Errorf("expects positive integer value, got %T instead", dataType) 70 | } 71 | } 72 | 73 | func validateIpToUint32(data interface{}) (*ipToUint32, error) { 74 | if err := expectValueEmpty(data); err != nil { 75 | return nil, fmt.Errorf("ipToUint32 %s", err.Error()) 76 | } else { 77 | return &ipToUint32{}, nil 78 | } 79 | } 80 | 81 | func validateLimitMaxLength(data interface{}) (*limitMaxLength, error) { 82 | if err := expectValuePositiveInt(data); err != nil { 83 | return nil, fmt.Errorf("limitMaxLength %s", err.Error()) 84 | } else { 85 | return &limitMaxLength{maxLength: data.(int)}, nil 86 | } 87 | } 88 | 89 | func validateSplitAndStore(data interface{}) (*splitAndStore, error) { 90 | var result splitAndStore 91 | 92 | if out, err := yaml.Marshal(data); err != nil { 93 | return nil, err 94 | } else if err := yaml.Unmarshal(out, &result); err != nil { 95 | return nil, err 96 | } else { 97 | return &result, nil 98 | } 99 | } 100 | 101 | func validateToArray(data interface{}) (*toArray, error) { 102 | if err := expectValueEmpty(data); err != nil { 103 | return nil, fmt.Errorf("ipToUint32 %s", err.Error()) 104 | } else { 105 | return &toArray{}, nil 106 | } 107 | } 108 | 109 | func validateCalculateSHA1(data interface{}) (*calculateSHA1, error) { 110 | var result calculateSHA1 111 | 112 | if out, err := yaml.Marshal(data); err != nil { 113 | return nil, err 114 | } else if err := yaml.Unmarshal(out, &result); err != nil { 115 | return nil, err 116 | } else { 117 | return &result, nil 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /processor/functions/ipToUint32.go: -------------------------------------------------------------------------------- 1 | package functions 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "net" 7 | "strconv" 8 | "strings" 9 | ) 10 | 11 | type ipToUint32 struct{} 12 | 13 | func (f *ipToUint32) Call(ip string) FunctionResult { 14 | b := bytes.Buffer{} 15 | result := FunctionPartialResult{} 16 | 17 | b.WriteByte('"') 18 | if strings.Contains(ip, ":") { // XXX ignore ipv6 for now 19 | b.WriteByte('0') 20 | } else if parsed := net.ParseIP(ip); parsed == nil { 21 | b.WriteByte('0') 22 | } else { 23 | b.WriteString(strconv.FormatUint(uint64(binary.BigEndian.Uint32(parsed[12:16])), 10)) 24 | } 25 | b.WriteByte('"') 26 | 27 | result.Value = b.Bytes() 28 | return FunctionResult{result} 29 | } 30 | -------------------------------------------------------------------------------- /processor/functions/ipToUint32_test.go: -------------------------------------------------------------------------------- 1 | package functions 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestIpToUint32(t *testing.T) { 10 | table := []struct { 11 | input string 12 | expected string 13 | }{ 14 | {"2001:0db8:0000:0042:0000:8a2e:0370:7334", `"0"`}, 15 | {"not ip", `"0"`}, 16 | {"", `"0"`}, 17 | {"127.0.0.1", `"2130706433"`}, 18 | {"0.0.0.1", `"1"`}, 19 | {"255.255.255.255", `"4294967295"`}, 20 | } 21 | 22 | for _, p := range table { 23 | callable := &ipToUint32{} 24 | assert.Equal(t, p.expected, string(callable.Call(p.input)[0].Value)) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /processor/functions/limitMaxLength.go: -------------------------------------------------------------------------------- 1 | package functions 2 | 3 | import ( 4 | "bytes" 5 | ) 6 | 7 | const MIDDLE = "<...>" 8 | 9 | type limitMaxLength struct { 10 | maxLength int 11 | } 12 | 13 | func (f *limitMaxLength) Call(value string) FunctionResult { 14 | // len(MIDDLE) = 5 15 | b := bytes.Buffer{} 16 | result := FunctionPartialResult{} 17 | 18 | b.WriteByte('"') 19 | if f.maxLength < 5 || len(value) <= f.maxLength { 20 | // string is too short - no need to do anything 21 | b.WriteString(value) 22 | } else { 23 | rawMaxLen := f.maxLength - 5 24 | 25 | leftLen := rawMaxLen / 2 26 | rightLen := rawMaxLen - leftLen 27 | 28 | b.WriteString(value[:leftLen]) 29 | b.WriteString(MIDDLE) 30 | b.WriteString(value[len(value)-rightLen:]) 31 | } 32 | b.WriteByte('"') 33 | 34 | result.Value = b.Bytes() 35 | return FunctionResult{result} 36 | } 37 | -------------------------------------------------------------------------------- /processor/functions/limitMaxLength_test.go: -------------------------------------------------------------------------------- 1 | package functions 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestTruncateMid(t *testing.T) { 10 | table := []struct { 11 | input string 12 | maxLength int 13 | expected string 14 | }{ 15 | {"foobar2", 5, `"<...>"`}, 16 | {"foobar2", 6, `"<...>2"`}, 17 | {"foobar22", 7, `"f<...>2"`}, 18 | {"foobar22", 6, `"<...>2"`}, 19 | {"xxxxxjfkljffeyyyyy", 15, `"xxxxx<...>yyyyy"`}, 20 | {"ok_message", 100, `"ok_message"`}, 21 | } 22 | 23 | for _, p := range table { 24 | callable := &limitMaxLength{maxLength: p.maxLength} 25 | v := callable.Call(p.input) 26 | assert.Equal(t, len(v), 1) 27 | 28 | assert.Equal(t, p.expected, string(v[0].Value)) 29 | assert.True(t, len(v)-2 <= p.maxLength) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /processor/functions/splitAndStore.go: -------------------------------------------------------------------------------- 1 | package functions 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | ) 7 | 8 | type splitAndStore struct { 9 | Delimiter string `yaml:"delimiter"` 10 | StoreTo map[string]int `yaml:"store_to"` 11 | } 12 | 13 | func (f *splitAndStore) Call(value string) FunctionResult { 14 | result := make(FunctionResult, 0, len(f.StoreTo)) 15 | 16 | parts := strings.Split(value, f.Delimiter) 17 | partsMap := make(map[int]string, len(parts)) 18 | for i, part := range parts { 19 | partsMap[i] = part 20 | } 21 | 22 | for fieldName, index := range f.StoreTo { 23 | b := bytes.Buffer{} 24 | dstFieldName := fieldName 25 | valuePart := partsMap[index] // empty string if there is no split part with such index 26 | 27 | b.WriteByte('"') 28 | b.WriteString(valuePart) 29 | b.WriteByte('"') 30 | 31 | result = append(result, FunctionPartialResult{ 32 | Value: b.Bytes(), 33 | DstFieldName: &dstFieldName, 34 | }) 35 | } 36 | 37 | return result 38 | } 39 | -------------------------------------------------------------------------------- /processor/functions/toArray.go: -------------------------------------------------------------------------------- 1 | package functions 2 | 3 | import ( 4 | "bytes" 5 | "strconv" 6 | "strings" 7 | ) 8 | 9 | type toArray struct{} 10 | 11 | func (f *toArray) Call(value string) FunctionResult { 12 | b := bytes.Buffer{} 13 | needComma := false 14 | result := FunctionPartialResult{} 15 | 16 | b.WriteByte('[') 17 | for _, n := range strings.Split(value, " ") { 18 | if n != "" { 19 | if needComma { 20 | b.WriteByte(',') 21 | } 22 | if _, err := strconv.ParseFloat(n, 32); err == nil { 23 | b.WriteString(n) 24 | needComma = true 25 | } 26 | } 27 | } 28 | b.WriteByte(']') 29 | 30 | result.Value = b.Bytes() 31 | return FunctionResult{result} 32 | } 33 | -------------------------------------------------------------------------------- /processor/functions/toArray_test.go: -------------------------------------------------------------------------------- 1 | package functions 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestToArray(t *testing.T) { 10 | table := []struct { 11 | input string 12 | expected string 13 | }{ 14 | {"200 3300 4000", "[200,3300,4000]"}, 15 | {"299.33 ", "[299.33]"}, 16 | {"20 2020 ", "[20,2020]"}, 17 | } 18 | 19 | for _, p := range table { 20 | callable := &toArray{} 21 | assert.Equal(t, p.expected, string(callable.Call(p.input)[0].Value)) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /processor/functions/types.go: -------------------------------------------------------------------------------- 1 | package functions 2 | 3 | // Callable is basic interface of any function 4 | type Callable interface { 5 | Call(string) FunctionResult 6 | } 7 | 8 | // functionPartialResult is the part of the result of a single function call; there can be several values returned 9 | type FunctionPartialResult struct { 10 | Value []byte // returned value 11 | DstFieldName *string // name of the field returned value will be stored to (optional) 12 | } 13 | 14 | // functionResult represents all the values returned by function 15 | type FunctionResult []FunctionPartialResult 16 | 17 | // FunctionSignature is signature of a single function as it is represented in config 18 | type FunctionSignature map[string]interface{} 19 | 20 | // FunctionSignatureMap is configuration of all functions as it is represented in config 21 | type FunctionSignatureMap map[string]FunctionSignature 22 | -------------------------------------------------------------------------------- /processor/processor.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "sync" 7 | "time" 8 | 9 | "github.com/pkg/errors" 10 | "github.com/rs/zerolog" 11 | "gopkg.in/alexcesaro/statsd.v2" 12 | 13 | "nginx-log-collector/config" 14 | ) 15 | 16 | const ( 17 | flushInterval = 30 * time.Second 18 | queueCheckInterval = 30 * time.Second 19 | ) 20 | 21 | type Result struct { 22 | Tag string 23 | Data []byte 24 | Lines int 25 | } 26 | 27 | type Processor struct { 28 | metrics *statsd.Client 29 | 30 | tagContexts map[string]TagContext 31 | 32 | resultChan chan Result 33 | 34 | logger zerolog.Logger 35 | wg *sync.WaitGroup 36 | workersCnt int 37 | } 38 | 39 | type TagContext struct { 40 | Config config.CollectedLog 41 | Converter Converter 42 | } 43 | 44 | func New(cfg config.Processor, logs []config.CollectedLog, metrics *statsd.Client, logger *zerolog.Logger) (*Processor, error) { 45 | tagContexts := make(map[string]TagContext, len(logs)) 46 | for _, l := range logs { 47 | converter, err := NewConverter(l) 48 | if err != nil { 49 | return nil, errors.Wrap(err, "unable to create converter") 50 | } 51 | if l.BufferSize <= 0 { 52 | return nil, fmt.Errorf("bad buffer size: %d for tag %s", l.BufferSize, l.Tag) 53 | 54 | } 55 | tagContexts[l.Tag] = TagContext{Config: l, Converter: converter} 56 | } 57 | 58 | return &Processor{ 59 | tagContexts: tagContexts, 60 | metrics: metrics.Clone(statsd.Prefix("processor")), 61 | resultChan: make(chan Result, 1000), 62 | wg: &sync.WaitGroup{}, 63 | workersCnt: cfg.Workers, 64 | logger: logger.With().Str("component", "processor").Logger(), 65 | }, nil 66 | } 67 | 68 | func (p *Processor) Start(done <-chan struct{}, msgChanList ...chan []byte) { 69 | p.logger.Info().Msg("starting") 70 | for i := 0; i < p.workersCnt; i++ { 71 | p.wg.Add(1) 72 | go p.Worker(done, p.aggregateChan(msgChanList)) 73 | } 74 | 75 | p.wg.Add(1) 76 | go p.queueMonitoring(done) 77 | <-done 78 | p.logger.Debug().Msg("got done") 79 | } 80 | 81 | func (p *Processor) Stop() { 82 | p.logger.Info().Msg("stopping") 83 | p.wg.Wait() 84 | p.logger.Debug().Msg("stopping [close phase]") 85 | close(p.resultChan) 86 | } 87 | 88 | func (p *Processor) ResultChan() chan Result { 89 | return p.resultChan 90 | } 91 | 92 | func (p *Processor) Worker(done <-chan struct{}, msgChan <-chan []byte) { 93 | defer p.wg.Done() 94 | 95 | tpMap := make(map[string]*tagProcessor) 96 | for tag, tagContext := range p.tagContexts { 97 | tp := newTagProcessor(tagContext.Config.BufferSize, tag) 98 | tpMap[tag] = tp 99 | p.wg.Add(1) 100 | go tp.flusher(p.resultChan, done, p.wg) 101 | } 102 | 103 | for rawMsg := range msgChan { 104 | // format is defined in rsyslog 105 | s := bytes.SplitN(rawMsg, []byte{'\t'}, 3) 106 | if len(s) != 3 { 107 | p.logger.Error().Bytes("msg", rawMsg).Int("len", len(s)).Msg("wrong message format") 108 | p.metrics.Increment("format_error") 109 | continue 110 | } 111 | 112 | hostname, tag, msg := string(s[0]), string(s[1]), s[2] 113 | tagContext, found := p.tagContexts[tag] 114 | if !found { 115 | p.logger.Warn().Str("host", hostname).Str("tag", tag).Msg("wrong tag") 116 | p.metrics.Increment("tag_error") 117 | continue 118 | } 119 | 120 | converted, err := tagContext.Converter.Convert(msg, hostname) 121 | if err != nil { 122 | logEvent := p.logger.Error().Str("host", hostname).Err(err) 123 | // AD-17284: always log the message 124 | logEvent = logEvent.Bytes("msg", msg) 125 | 126 | logEvent.Msg("convert error") 127 | p.metrics.Increment("convert_error") 128 | continue 129 | } 130 | 131 | tp := tpMap[tag] 132 | tp.writeLine(converted, p.resultChan) 133 | if tagContext.Config.Audit { 134 | p.logger.Error().Str("tag", tag).Msgf("write to buffer: %s", string(converted)) 135 | } 136 | } 137 | 138 | for _, tp := range tpMap { 139 | tp.mu.Lock() 140 | tp.flush(p.resultChan) 141 | tp.mu.Unlock() 142 | } 143 | 144 | p.logger.Debug().Msg("processor worker done") 145 | } 146 | 147 | // aggregateChan aggregates list of channels to single channel 148 | func (p *Processor) aggregateChan(msgChanList []chan []byte) chan []byte { 149 | bufferSize := 0 150 | for _, msgChan := range msgChanList { 151 | bufferSize += cap(msgChan) 152 | } 153 | 154 | aggregate := make(chan []byte, bufferSize) 155 | var wg sync.WaitGroup 156 | wg.Add(len(msgChanList)) 157 | for _, msgChan := range msgChanList { 158 | go func(msgChan <-chan []byte) { 159 | for msg := range msgChan { 160 | aggregate <- msg 161 | } 162 | wg.Done() 163 | }(msgChan) 164 | } 165 | go func() { 166 | wg.Wait() 167 | close(aggregate) 168 | }() 169 | 170 | return aggregate 171 | } 172 | 173 | func (p *Processor) queueMonitoring(done <-chan struct{}) { 174 | defer p.wg.Done() 175 | 176 | ticker := time.NewTicker(queueCheckInterval) 177 | defer ticker.Stop() 178 | 179 | for { 180 | select { 181 | case <-ticker.C: 182 | p.logger.Debug().Int("result_queue_len", len(p.resultChan)).Msg("queue stats") 183 | p.metrics.Count("result_queue_len", len(p.resultChan)) 184 | case <-done: 185 | p.logger.Debug().Msg("queueMonitoring exit") 186 | return 187 | } 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /processor/tag_processor.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "bytes" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | type tagProcessor struct { 10 | mu *sync.Mutex 11 | buffer *bytes.Buffer 12 | nextFlushAt time.Time 13 | tag string 14 | bufSize int 15 | linesInBuf int 16 | } 17 | 18 | func newTagProcessor(bufferSize int, tag string) *tagProcessor { 19 | b := make([]byte, 0, bufferSize) 20 | return &tagProcessor{ 21 | buffer: bytes.NewBuffer(b), 22 | nextFlushAt: time.Now().Add(flushInterval), 23 | tag: tag, 24 | mu: &sync.Mutex{}, 25 | bufSize: bufferSize, 26 | linesInBuf: 0, 27 | } 28 | } 29 | 30 | // XXX mutex should be taken 31 | func (t *tagProcessor) flush(resultChan chan Result) { 32 | t.nextFlushAt = time.Now().Add(flushInterval) 33 | b := t.buffer.Bytes() 34 | if len(b) == 0 { 35 | return 36 | } 37 | clone := make([]byte, len(b)) // TODO sync pool? 38 | copy(clone, b) 39 | resultChan <- Result{ 40 | Tag: t.tag, 41 | Data: clone, 42 | Lines: t.linesInBuf, 43 | } 44 | t.buffer.Reset() 45 | t.linesInBuf = 0 46 | } 47 | 48 | func (t *tagProcessor) writeLine(data []byte, resultChan chan Result) { 49 | t.mu.Lock() 50 | 51 | if t.buffer.Len()+len(data) > t.bufSize { 52 | t.flush(resultChan) 53 | } 54 | t.buffer.Write(data) 55 | t.linesInBuf += 1 56 | 57 | t.mu.Unlock() 58 | } 59 | 60 | func (t *tagProcessor) flusher(resultChan chan Result, done <-chan struct{}, wg *sync.WaitGroup) { 61 | defer wg.Done() 62 | ticker := time.NewTicker(flushInterval) 63 | defer ticker.Stop() 64 | 65 | for { 66 | select { 67 | case <-ticker.C: 68 | t.mu.Lock() 69 | if time.Now().After(t.nextFlushAt) { 70 | t.flush(resultChan) 71 | } 72 | t.mu.Unlock() 73 | case <-done: 74 | return 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /processor/transformer.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "nginx-log-collector/processor/functions" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | type transformer struct { 10 | fieldNameSrc string 11 | function functions.Callable 12 | } 13 | 14 | func parseTransformersMap(transformersMap functions.FunctionSignatureMap) ([]transformer, error) { 15 | transformers := make([]transformer, 0, len(transformersMap)) 16 | 17 | for fieldNameSrc, functionSignature := range transformersMap { 18 | if callable, err := functions.Dispatch(functionSignature); err != nil { 19 | return nil, errors.Wrapf(err, "unable to convert expression for field %s to function", fieldNameSrc) 20 | } else { 21 | transformers = append(transformers, transformer{ 22 | fieldNameSrc: fieldNameSrc, 23 | function: callable, 24 | }) 25 | } 26 | } 27 | 28 | return transformers, nil 29 | } 30 | -------------------------------------------------------------------------------- /receiver/http.go: -------------------------------------------------------------------------------- 1 | package receiver 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "context" 7 | "encoding/json" 8 | "fmt" 9 | "io" 10 | "net/http" 11 | "strings" 12 | "sync" 13 | "time" 14 | 15 | "github.com/rs/zerolog" 16 | "gopkg.in/alexcesaro/statsd.v2" 17 | 18 | "nginx-log-collector/config" 19 | "nginx-log-collector/utils" 20 | ) 21 | 22 | type HttpReceiver struct { 23 | config *config.HttpReceiver 24 | metrics *statsd.Client 25 | msgChan chan []byte 26 | logger zerolog.Logger 27 | wg *sync.WaitGroup 28 | } 29 | 30 | type processedLogEntry struct { 31 | EventDateTime string `json:"event_datetime"` 32 | EventDate string `json:"event_date"` 33 | Hostname string `json:"hostname"` 34 | Message string `json:"message"` 35 | SetupID string `json:"request_id"` 36 | Severity string `json:"severity"` 37 | User string `json:"user"` 38 | RowNumber int `json:"row_number"` 39 | } 40 | 41 | const ( 42 | httpReadTimeout = 30 * time.Second 43 | httpWriteTimeout = 5 * time.Second 44 | httpShutdownTimeout = 5 * time.Second 45 | 46 | dateFormat = "2006-01-02" 47 | headerHostname = "X-Log-Source" 48 | headerSetupID = "X-Setup-Id" 49 | multipartFormMaxSize = 100 * 1024 * 1024 50 | tag = "puppet:" 51 | ) 52 | 53 | var ( 54 | datetimeTransformers = []*utils.DatetimeTransformer{ 55 | { 56 | "2006-01-02 15:04:05 -0700", 57 | time.RFC3339, 58 | time.Local, 59 | }, 60 | { 61 | "2006-01-02 15:04:05 0700", 62 | time.RFC3339, 63 | time.Local, 64 | }, 65 | } 66 | ) 67 | 68 | func NewHttpReceiver(cfg *config.HttpReceiver, metrics *statsd.Client, logger *zerolog.Logger) (*HttpReceiver, error) { 69 | httpReceiver := &HttpReceiver{ 70 | config: cfg, 71 | metrics: metrics.Clone(statsd.Prefix("receiver.http")), 72 | msgChan: make(chan []byte, 100000), 73 | wg: &sync.WaitGroup{}, 74 | logger: logger.With().Str("component", "receiver.http").Logger(), 75 | } 76 | return httpReceiver, nil 77 | } 78 | 79 | func (h *HttpReceiver) MsgChan() chan []byte { 80 | return h.msgChan 81 | } 82 | 83 | func (h *HttpReceiver) Start(done <-chan struct{}) { 84 | h.logger.Info().Msg("Starting") 85 | 86 | router := http.NewServeMux() 87 | router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 88 | h.handle(w, r) 89 | }) 90 | 91 | server := &http.Server{ 92 | Addr: h.config.Url, 93 | Handler: router, 94 | ReadTimeout: httpReadTimeout, 95 | WriteTimeout: httpWriteTimeout, 96 | } 97 | 98 | h.wg.Add(1) 99 | go h.gracefulShutdownSentry(done, server) 100 | 101 | h.wg.Add(1) 102 | go h.queueStats(done) 103 | 104 | err := server.ListenAndServe() 105 | if err != nil && err != http.ErrServerClosed { 106 | h.logger.Fatal().Err(err).Msgf("Could not listen on %s", h.config.Url) 107 | return 108 | } 109 | } 110 | 111 | func (h *HttpReceiver) Stop() { 112 | h.logger.Info().Msg("stopping") 113 | h.wg.Wait() 114 | close(h.msgChan) 115 | } 116 | 117 | // gracefulShutdownSentry tries to gracefully shutdown http server once done channel is closed 118 | func (h *HttpReceiver) gracefulShutdownSentry(done <-chan struct{}, server *http.Server) { 119 | defer h.wg.Done() 120 | 121 | <-done 122 | h.logger.Warn().Msgf("Http server is shutting down with time of %.2f seconds", httpShutdownTimeout.Seconds()) 123 | 124 | ctx, cancel := context.WithTimeout(context.Background(), httpShutdownTimeout) 125 | defer cancel() 126 | 127 | server.SetKeepAlivesEnabled(false) 128 | if err := server.Shutdown(ctx); err != nil { 129 | h.logger.Error().Msgf("Failed to shutdown gracefully: %v", err) 130 | } 131 | } 132 | 133 | // handle processes single request 134 | func (h *HttpReceiver) handle(w http.ResponseWriter, r *http.Request) { 135 | defer r.Body.Close() 136 | 137 | hostname := r.Header.Get(headerHostname) 138 | if hostname == "" { 139 | w.WriteHeader(http.StatusBadRequest) 140 | _, _ = w.Write([]byte("Missing or empty " + headerHostname + " header")) 141 | return 142 | } 143 | 144 | contentType := r.Header.Get("Content-Type") 145 | if contentType == "" { 146 | contentType = "" 147 | } 148 | 149 | setupID := r.Header.Get(headerSetupID) 150 | 151 | if strings.HasPrefix(contentType, "multipart/form-data") { 152 | err := r.ParseMultipartForm(multipartFormMaxSize) 153 | if err != nil { 154 | h.logger.Error().Err(err).Msg("Failed to parse form") 155 | 156 | w.WriteHeader(http.StatusInternalServerError) 157 | _, _ = w.Write([]byte("Failed to parse form: " + err.Error())) 158 | return 159 | } 160 | 161 | for _, fileHeaders := range r.MultipartForm.File { 162 | for _, fileHeader := range fileHeaders { 163 | file, err := fileHeader.Open() 164 | if err != nil { 165 | h.logger.Error().Err(err).Str("filename", fileHeader.Filename).Msg("Failed to open the file") 166 | continue 167 | } 168 | 169 | h.processContent(file, hostname, setupID) 170 | _ = file.Close() 171 | } 172 | } 173 | } else if strings.HasPrefix(contentType, "text/plain") { 174 | h.processContent(r.Body, hostname, setupID) 175 | } else { 176 | w.WriteHeader(http.StatusBadRequest) 177 | _, _ = w.Write([]byte("Only 'text/plain' and 'multipart/form-data' content types are supported, got " + contentType)) 178 | return 179 | } 180 | 181 | w.WriteHeader(http.StatusNoContent) 182 | } 183 | 184 | // processContent processes request body or posted file 185 | func (h *HttpReceiver) processContent(content io.Reader, hostname, setupID string) { 186 | hasData := true 187 | reader := bufio.NewReader(content) 188 | rowNumber := 0 189 | 190 | var ( 191 | logEntryCurr *processedLogEntry 192 | logEntryNext *processedLogEntry 193 | ) 194 | 195 | for hasData { 196 | line, err := reader.ReadString('\n') 197 | hasData = err != io.EOF 198 | if err != nil && hasData { 199 | h.logger.Error().Err(err).Str("line", line).Msg("Got an error while reading the line") 200 | } 201 | 202 | line = strings.TrimSuffix(line, "\n") 203 | logEntryNext, err = h.transformLine(line) 204 | 205 | if logEntryCurr != nil { 206 | if logEntryNext == nil { 207 | logEntryCurr.Message += "\n" + line 208 | } 209 | } 210 | 211 | if logEntryNext != nil { // new log entry is up 212 | if logEntryCurr != nil { // flush previous log entry 213 | h.sendLogEntry(logEntryCurr) 214 | } 215 | 216 | // switch to the new one 217 | logEntryCurr = logEntryNext 218 | logEntryCurr.Hostname = hostname 219 | logEntryCurr.RowNumber = rowNumber 220 | logEntryCurr.SetupID = setupID 221 | rowNumber++ 222 | } else if logEntryCurr != nil { // another row of multiline log entry 223 | logEntryCurr.Message += "\n" + line 224 | } else if err != nil { 225 | h.logger.Error().Err(err).Str("line", line).Msg("Got an error while processing the line") 226 | } 227 | 228 | if !hasData && logEntryCurr != nil { // also flush the last log entry 229 | h.sendLogEntry(logEntryCurr) 230 | } 231 | } 232 | } 233 | 234 | // queueStats sends metric with length of message once per amount of time 235 | func (h *HttpReceiver) queueStats(done <-chan struct{}) { 236 | defer h.wg.Done() 237 | 238 | ticker := time.NewTicker(queueCheckInterval) 239 | defer ticker.Stop() 240 | 241 | for { 242 | select { 243 | case <-done: 244 | h.logger.Debug().Msg("queueStats stopped") 245 | return 246 | case <-ticker.C: 247 | queueSize := len(h.msgChan) 248 | h.logger.Debug().Int("http_msg_chan_len", queueSize).Msg("http queue stats") 249 | h.metrics.Count("http_msg_chan_len", queueSize) 250 | } 251 | } 252 | } 253 | 254 | // sendLogEntry prepares the bytes buffer and sends it 255 | func (h *HttpReceiver) sendLogEntry(logEntry *processedLogEntry) { 256 | data, err := json.Marshal(logEntry) 257 | if err != nil { 258 | h.logger.Error().Err(err).Str("logEntry", fmt.Sprintf("%+v", logEntry)).Msg("Failed to marshal log entry") 259 | return 260 | } 261 | 262 | buffer := bytes.Buffer{} 263 | buffer.WriteString(logEntry.Hostname) 264 | buffer.WriteByte('\t') 265 | buffer.WriteString(tag) 266 | buffer.WriteByte('\t') 267 | buffer.Write(data) 268 | 269 | h.msgChan <- buffer.Bytes() 270 | } 271 | 272 | // transformLine parses log line and transforms it to access log format 273 | // expected line format: 274 | // 2020-04-24 18:14:42 +0300 Puppet (info): Applying configuration version '1587741267' 275 | func (h *HttpReceiver) transformLine(line string) (*processedLogEntry, error) { 276 | // "2020-04-24", "18:14:42", "+0300", "Puppet", "(info):", "Applying configuration version '1587741267'" 277 | parts := strings.SplitN(line, " ", 6) 278 | if len(parts) != 6 { 279 | return nil, fmt.Errorf("wrong line structure") 280 | } 281 | 282 | // "(info):" 283 | severity := parts[4] 284 | if !strings.HasPrefix(severity, "(") || !strings.HasSuffix(severity, "):") { 285 | return nil, fmt.Errorf("wrong severity format: %s", severity) 286 | } 287 | severity = severity[1 : len(severity)-2] 288 | 289 | // "2020-04-24", "18:14:42", "+0300" 290 | datetime := strings.Join(parts[0:3], " ") 291 | parsed, transformer, err := utils.TryDatetimeFormats(datetime, datetimeTransformers) 292 | if err != nil { 293 | return nil, err 294 | } 295 | 296 | // "Puppet" 297 | user := parts[3] 298 | if user == "" { 299 | return nil, fmt.Errorf("wrong user format: %s", user) 300 | } 301 | 302 | // "Applying configuration version '1587741267'" 303 | message := parts[5] // message can be empty 304 | 305 | return &processedLogEntry{ 306 | EventDateTime: parsed.Format(transformer.FormatDst), 307 | EventDate: parsed.Format(dateFormat), 308 | Message: message, 309 | Severity: severity, 310 | User: user, 311 | }, nil 312 | } 313 | -------------------------------------------------------------------------------- /receiver/tcp.go: -------------------------------------------------------------------------------- 1 | package receiver 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "net" 7 | "sync" 8 | "time" 9 | 10 | "github.com/pkg/errors" 11 | "github.com/rs/zerolog" 12 | "gopkg.in/alexcesaro/statsd.v2" 13 | ) 14 | 15 | const ( 16 | tcpReadTimeout = 30 * time.Second 17 | queueCheckInterval = 30 * time.Second 18 | ) 19 | 20 | type TCPReceiver struct { 21 | msgChan chan []byte 22 | listener *net.TCPListener 23 | 24 | metrics *statsd.Client 25 | logger zerolog.Logger 26 | wg *sync.WaitGroup 27 | } 28 | 29 | func NewTCPReceiver(addr string, metrics *statsd.Client, logger *zerolog.Logger) (*TCPReceiver, error) { 30 | resolvedAddr, err := net.ResolveTCPAddr("tcp", addr) 31 | if err != nil { 32 | return nil, errors.Wrap(err, "unable to resolve addr") 33 | } 34 | 35 | listener, err := net.ListenTCP("tcp", resolvedAddr) 36 | if err != nil { 37 | return nil, errors.Wrap(err, "unable to listen") 38 | } 39 | 40 | wg := &sync.WaitGroup{} 41 | wg.Add(1) 42 | 43 | msgChan := make(chan []byte, 100000) 44 | return &TCPReceiver{ 45 | msgChan: msgChan, 46 | listener: listener, 47 | metrics: metrics.Clone(statsd.Prefix("receiver.tcp")), 48 | wg: wg, 49 | logger: logger.With().Str("component", "receiver.tcp").Logger(), 50 | }, nil 51 | } 52 | 53 | func (t *TCPReceiver) MsgChan() chan []byte { 54 | return t.msgChan 55 | } 56 | 57 | func (t *TCPReceiver) Start(done <-chan struct{}) { 58 | t.logger.Info().Msg("starting") 59 | 60 | go t.queueMonitoring(done) 61 | 62 | defer t.listener.Close() 63 | for { 64 | conn, err := t.listener.Accept() 65 | if err != nil { 66 | select { 67 | case <-done: 68 | return 69 | default: 70 | } 71 | t.logger.Warn().Err(err).Msg("unable to accept connection") 72 | continue 73 | } 74 | t.wg.Add(1) 75 | go t.handle(conn, done) 76 | } 77 | } 78 | 79 | func (t *TCPReceiver) handle(conn net.Conn, done <-chan struct{}) { 80 | defer conn.Close() 81 | defer t.wg.Done() 82 | t.metrics.Increment("accepted") 83 | reader := bufio.NewReader(conn) 84 | var cnt uint64 85 | for { 86 | select { 87 | case <-done: 88 | return 89 | default: 90 | } 91 | err := conn.SetReadDeadline(time.Now().Add(tcpReadTimeout)) 92 | if err != nil { 93 | t.logger.Warn().Err(err).Msg("set deadline error") 94 | } 95 | 96 | line, err := reader.ReadBytes('\n') 97 | if err != nil { 98 | if err == io.EOF { 99 | if len(line) > 0 { 100 | t.logger.Warn().Str("line", string(line)).Msg("unfinished line") 101 | t.metrics.Increment("line_error") 102 | } 103 | } else { 104 | t.logger.Debug().Err(err).Msg("read error (can be ignored)") // it's ok 105 | } 106 | break 107 | } 108 | t.msgChan <- line[:len(line)-1] 109 | cnt++ 110 | if cnt%100 == 0 { 111 | t.metrics.Count("lines", 100) 112 | } 113 | if cnt%10000 == 0 { 114 | t.logger.Debug().Msg("10k lines processed") 115 | cnt = 0 116 | } 117 | } 118 | } 119 | 120 | func (t *TCPReceiver) queueMonitoring(done <-chan struct{}) { 121 | defer t.wg.Done() 122 | 123 | ticker := time.NewTicker(queueCheckInterval) 124 | defer ticker.Stop() 125 | 126 | for { 127 | select { 128 | case <-ticker.C: 129 | t.logger.Debug().Int("tcp_msg_chan_len", len(t.msgChan)).Msg("tcp queue stats") 130 | t.metrics.Count("tcp_msg_chan_len", len(t.msgChan)) 131 | case <-done: 132 | t.logger.Debug().Msg("queueMonitoring exit") 133 | return 134 | } 135 | } 136 | } 137 | 138 | func (t *TCPReceiver) Stop() { 139 | t.listener.Close() 140 | t.logger.Info().Msg("stopping") 141 | t.wg.Wait() 142 | close(t.msgChan) 143 | } 144 | -------------------------------------------------------------------------------- /receiver/zmq.go: -------------------------------------------------------------------------------- 1 | package receiver 2 | 3 | /* 4 | 5 | import ( 6 | "sync" 7 | "time" 8 | 9 | "github.com/pebbe/zmq4" 10 | "github.com/pkg/errors" 11 | "github.com/rs/zerolog/log" 12 | "gopkg.in/alexcesaro/statsd.v2" 13 | ) 14 | 15 | const ( 16 | queueCheckInterval = 30 * time.Second 17 | ) 18 | 19 | type ZmqReceiver struct { 20 | msgChan chan []byte 21 | socket *zmq4.Socket 22 | metrics *statsd.Client 23 | wg *sync.WaitGroup 24 | poller *zmq4.Poller 25 | } 26 | 27 | func NewZmqReceiver(addr string, metrics *statsd.Client) (*ZmqReceiver, error) { 28 | socket, err := zmq4.NewSocket(zmq4.PULL) 29 | if err != nil { 30 | return nil, errors.Wrap(err, "unable to init zmq socket") 31 | } 32 | err = socket.Bind(addr) 33 | if err != nil { 34 | return nil, errors.Wrapf(err, "unable to bind zmq socket with addr: %s", addr) 35 | } 36 | poller := zmq4.NewPoller() 37 | poller.Add(socket, zmq4.POLLIN) 38 | 39 | wg := &sync.WaitGroup{} 40 | wg.Add(1) 41 | 42 | msgChan := make(chan []byte, 100000) 43 | return &ZmqReceiver{ 44 | msgChan: msgChan, 45 | socket: socket, 46 | metrics: metrics.Clone(statsd.Prefix("receiver.zmq")), 47 | wg: wg, 48 | poller: poller, 49 | }, nil 50 | } 51 | 52 | func (z *ZmqReceiver) MsgChan() chan []byte { 53 | return z.msgChan 54 | } 55 | 56 | func (z *ZmqReceiver) Start(done <-chan struct{}) { 57 | defer z.wg.Done() 58 | log.Info().Msg("zmq receiver starting") 59 | 60 | ticker := time.NewTicker(queueCheckInterval) 61 | defer ticker.Stop() 62 | z.wg.Add(1) 63 | go z.queueMonitoring(ticker.C, done) 64 | 65 | var cnt uint64 66 | for { 67 | select { 68 | case <-done: 69 | log.Debug().Msg("zmq done") 70 | return 71 | default: 72 | sockets, err := z.poller.Poll(time.Second) 73 | if err != nil { 74 | z.metrics.Increment("poll_errors") 75 | log.Warn().Err(err).Msg("zmq poll error; ignoring message") 76 | continue 77 | } 78 | if len(sockets) == 0 { 79 | continue 80 | } 81 | msg, err := sockets[0].Socket.RecvBytes(0) 82 | if err != nil { 83 | z.metrics.Increment("recv_errors") 84 | log.Warn().Err(err).Msg("zmq RecvBytes error; ignoring message") 85 | continue 86 | } 87 | z.msgChan <- msg 88 | cnt++ 89 | if cnt%100 == 0 { 90 | log.Debug().Msg("100 done") 91 | z.metrics.Count("messages", cnt) 92 | cnt = 0 93 | } 94 | } 95 | } 96 | } 97 | 98 | func (z *ZmqReceiver) queueMonitoring(ticker <-chan time.Time, done <-chan struct{}) { 99 | defer z.wg.Done() 100 | for { 101 | select { 102 | case <-ticker: 103 | log.Debug().Int("msg_chan_len", len(z.msgChan)).Msg("queue stats") 104 | z.metrics.Count("msg_chan_len", len(z.msgChan)) 105 | case <-done: 106 | return 107 | } 108 | } 109 | log.Debug().Msg("queueMonitoring exit") 110 | } 111 | 112 | func (z *ZmqReceiver) Stop() { 113 | log.Info().Msg("zmq receiver stopping") 114 | z.wg.Wait() 115 | z.poller.RemoveBySocket(z.socket) 116 | z.socket.Close() 117 | close(z.msgChan) 118 | } 119 | */ 120 | -------------------------------------------------------------------------------- /service/service.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "github.com/rs/zerolog" 6 | "gopkg.in/alexcesaro/statsd.v2" 7 | 8 | "nginx-log-collector/backlog" 9 | "nginx-log-collector/config" 10 | "nginx-log-collector/processor" 11 | "nginx-log-collector/receiver" 12 | "nginx-log-collector/uploader" 13 | ) 14 | 15 | type Service struct { 16 | httpReceiver *receiver.HttpReceiver 17 | tcpReceiver *receiver.TCPReceiver 18 | processor *processor.Processor 19 | uploader *uploader.Uploader 20 | backlog *backlog.Backlog 21 | 22 | logger zerolog.Logger 23 | metrics *statsd.Client 24 | } 25 | 26 | func New(cfg *config.Config, metrics *statsd.Client, logger *zerolog.Logger) (*Service, error) { 27 | httpReceiver, err := receiver.NewHttpReceiver(&cfg.HttpReceiver, metrics, logger) 28 | if err != nil { 29 | return nil, errors.Wrap(err, "http receiver init error") 30 | } 31 | 32 | tcpReceiver, err := receiver.NewTCPReceiver(cfg.TCPReceiver.Addr, metrics, logger) 33 | if err != nil { 34 | return nil, errors.Wrap(err, "tcp receiver init error") 35 | } 36 | 37 | proc, err := processor.New(cfg.Processor, cfg.CollectedLogs, metrics, logger) 38 | if err != nil { 39 | return nil, errors.Wrap(err, "processor init error") 40 | } 41 | 42 | bl, err := backlog.New(cfg.Backlog, metrics, logger) 43 | if err != nil { 44 | return nil, errors.Wrap(err, "backlog init error") 45 | } 46 | 47 | upl, err := uploader.New(cfg.CollectedLogs, bl, metrics, logger) 48 | if err != nil { 49 | return nil, errors.Wrap(err, "uploader init error") 50 | } 51 | 52 | return &Service{ 53 | httpReceiver: httpReceiver, 54 | tcpReceiver: tcpReceiver, 55 | processor: proc, 56 | uploader: upl, 57 | backlog: bl, 58 | logger: logger.With().Str("component", "service").Logger(), 59 | metrics: metrics.Clone(statsd.Prefix("service")), 60 | }, nil 61 | } 62 | 63 | func (s *Service) Start(done <-chan struct{}) { 64 | s.logger.Info().Msg("starting") 65 | 66 | sDone := make(chan struct{}) 67 | if s.httpReceiver != nil { 68 | go s.httpReceiver.Start(sDone) 69 | } 70 | go s.tcpReceiver.Start(sDone) 71 | go s.processor.Start(sDone, s.httpReceiver.MsgChan(), s.tcpReceiver.MsgChan()) 72 | go s.uploader.Start(sDone, s.processor.ResultChan()) 73 | go s.backlog.Start(done) 74 | 75 | <-done 76 | close(sDone) 77 | 78 | s.logger.Info().Msg("stopping service") 79 | 80 | if s.httpReceiver != nil { 81 | s.httpReceiver.Stop() 82 | s.logger.Info().Msg("http receiver stopped") 83 | } 84 | 85 | s.tcpReceiver.Stop() 86 | s.logger.Info().Msg("tcp receiver stopped") 87 | 88 | s.processor.Stop() 89 | s.logger.Info().Msg("processor stopped") 90 | 91 | s.uploader.Stop() 92 | s.logger.Info().Msg("uploader stopped") 93 | 94 | s.backlog.Stop() 95 | s.logger.Info().Msg("backlog stopped") 96 | } 97 | -------------------------------------------------------------------------------- /uploader/uploader.go: -------------------------------------------------------------------------------- 1 | package uploader 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "github.com/pkg/errors" 8 | "github.com/rs/zerolog" 9 | "gopkg.in/alexcesaro/statsd.v2" 10 | 11 | "nginx-log-collector/backlog" 12 | "nginx-log-collector/clickhouse" 13 | "nginx-log-collector/config" 14 | "nginx-log-collector/processor" 15 | ) 16 | 17 | const maxResultChanLen = 10 18 | 19 | type Uploader struct { 20 | backlog *backlog.Backlog 21 | tagContexts map[string]TagContext 22 | logger zerolog.Logger 23 | metrics *statsd.Client 24 | wg *sync.WaitGroup 25 | } 26 | 27 | type TagContext struct { 28 | Config config.CollectedLog 29 | URL string 30 | } 31 | 32 | func New(logs []config.CollectedLog, bl *backlog.Backlog, metrics *statsd.Client, logger *zerolog.Logger) (*Uploader, error) { 33 | tagContexts := make(map[string]TagContext) 34 | for _, l := range logs { 35 | uploadUrl, err := clickhouse.MakeUrl(l.Upload.DSN, l.Upload.Table, true, l.AllowErrorRatio) 36 | if err != nil { 37 | return nil, errors.Wrap(err, "unable to create uploader") 38 | } 39 | 40 | tagContexts[l.Tag] = TagContext{Config: l, URL: uploadUrl} 41 | } 42 | 43 | wg := &sync.WaitGroup{} 44 | wg.Add(1) 45 | return &Uploader{ 46 | tagContexts: tagContexts, 47 | wg: wg, 48 | backlog: bl, 49 | metrics: metrics.Clone(statsd.Prefix("uploader")), 50 | logger: logger.With().Str("component", "uploader").Logger(), 51 | }, nil 52 | } 53 | 54 | func (u *Uploader) Start(done <-chan struct{}, resultChan chan processor.Result) { 55 | defer u.wg.Done() 56 | u.logger.Info().Msg("starting") 57 | limiter := u.backlog.GetLimiter() 58 | isDone := false 59 | for result := range resultChan { 60 | tagContext, found := u.tagContexts[result.Tag] 61 | if !found { 62 | u.metrics.Increment("tag_missing_error") 63 | u.logger.Warn().Str("tag", result.Tag).Msg("tag missing in uploader") 64 | continue 65 | } 66 | 67 | if !isDone { 68 | select { 69 | case <-done: 70 | isDone = true 71 | default: 72 | } 73 | } 74 | 75 | u.metrics.Gauge("upload_result_chan_len", len(resultChan)) 76 | 77 | if isDone || len(resultChan) > maxResultChanLen { 78 | u.logger.Info().Msg("flushing to backlog") 79 | 80 | if tagContext.Config.Audit { 81 | // level is error because global log level is error 82 | u.logger.Error().Str("tag", result.Tag).Msgf("make new backlog job: %s", string(result.Data)) 83 | } 84 | 85 | if err := u.backlog.MakeNewBacklogJob(tagContext.URL, result.Data); err != nil { 86 | u.logger.Fatal().Err(err).Msg("unable to create backlog job") 87 | } 88 | continue 89 | } 90 | 91 | limiter.Enter() 92 | u.wg.Add(1) 93 | go func(url string, data []byte, tag string, lines int) { 94 | tagTrimmed := tag[:len(tag)-1] // trim : 95 | 96 | u.metrics.Increment(fmt.Sprintf("uploading.batches.%s", tagTrimmed)) 97 | u.metrics.Count(fmt.Sprintf("uploading.lines.%s", tagTrimmed), lines) 98 | 99 | err := clickhouse.Upload(url, data) 100 | if tagContext.Config.Audit { 101 | // level is error because global log level is error 102 | u.logger.Error().Str("tag", tag).Err(err).Msgf("upload: %s", string(data)) 103 | } 104 | if err != nil { 105 | u.logger.Error().Str("tag", tag).Str("url", tagContext.URL).Err(err).Msg("upload error; creating backlog job") 106 | u.metrics.Increment("upload_error") 107 | if err := u.backlog.MakeNewBacklogJob(url, data); err != nil { 108 | u.logger.Fatal().Err(err).Msg("unable to create backlog job") 109 | } 110 | u.metrics.Increment(fmt.Sprintf("failed.batches.%s", tagTrimmed)) 111 | u.metrics.Count(fmt.Sprintf("failed.lines.%s", tagTrimmed), lines) 112 | } else { 113 | u.metrics.Increment(fmt.Sprintf("ok.batches.%s", tagTrimmed)) 114 | u.metrics.Count(fmt.Sprintf("ok.lines.%s", tagTrimmed), lines) 115 | } 116 | 117 | // old-style metric for compatibility 118 | u.metrics.Increment(fmt.Sprintf("upload_tag_%s_", tagTrimmed)) // trim : 119 | 120 | limiter.Leave() 121 | u.wg.Done() 122 | 123 | }(tagContext.URL, result.Data, result.Tag, result.Lines) 124 | } 125 | <-done 126 | } 127 | 128 | func (u *Uploader) Stop() { 129 | u.logger.Info().Msg("stopping") 130 | u.wg.Wait() 131 | } 132 | -------------------------------------------------------------------------------- /utils/datetime.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | "github.com/pkg/errors" 8 | ) 9 | 10 | // DatetimeTransformer contains info needed to transform 11 | // datetime string from one format to another 12 | type DatetimeTransformer struct { 13 | FormatSrc string 14 | FormatDst string 15 | Location *time.Location 16 | } 17 | 18 | func TryDatetimeFormats(datetime string, transformers []*DatetimeTransformer) (parsed time.Time, matched *DatetimeTransformer, err error) { 19 | errorMessages := make([]string, 0, len(transformers)) 20 | for _, transformer := range transformers { 21 | t, err := time.Parse(transformer.FormatSrc, datetime) 22 | if err != nil { 23 | errorMessages = append(errorMessages, err.Error()) 24 | } else { 25 | return t.In(transformer.Location), transformer, nil 26 | } 27 | } 28 | 29 | if len(errorMessages) > 0 { 30 | err = errors.Wrap(errors.New(strings.Join(errorMessages, "\n")), "unable to parse datetime field") 31 | } else { 32 | err = errors.New("Empty transformers list") 33 | } 34 | 35 | return time.Time{}, nil, err 36 | } 37 | -------------------------------------------------------------------------------- /utils/limiter.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | type Limiter chan struct{} 4 | 5 | func (l Limiter) Enter() { l <- struct{}{} } 6 | func (l Limiter) Leave() { <-l } 7 | 8 | func NewLimiter(l int) Limiter { 9 | return make(chan struct{}, l) 10 | } 11 | --------------------------------------------------------------------------------