├── rotate-metrics.sh ├── entrypoint.sh ├── model ├── writer.go ├── sort.go ├── constructor_test.go ├── utils_test.go ├── constructors.go ├── structs.go ├── writer_test.go ├── reader_test.go └── reader.go ├── log_rotate_conf.sh ├── Dockerfile ├── Rakefile ├── Gopkg.toml ├── .travis.yml ├── bin ├── promrec │ └── main.go └── promplay │ └── main.go ├── .gitignore ├── README.md ├── Gopkg.lock └── LICENSE /rotate-metrics.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | while true 4 | do 5 | logrotate /etc/logrotate.d/metrics 6 | sleep 300 # 5 minutes 7 | done 8 | -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | CONF_LOG_FILE=/etc/logrotate.d/metrics 4 | 5 | touch $CONF_LOG_FILE 6 | 7 | ./log_rotate_conf.sh "$ROTATION_FILE_LOG" "${ROTATION_PERIOD:=daily}" "${ROTATION_COUNT:=10}" "${ROTATION_SIZE:=-1}" $CONF_LOG_FILE 8 | 9 | source ./rotate-metrics.sh & 10 | 11 | # shellcheck disable=SC2086 12 | /promqueen/promrec $PROM_ARGS -------------------------------------------------------------------------------- /model/writer.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "io" 5 | "sync" 6 | "encoding/binary" 7 | 8 | "github.com/sirupsen/logrus" 9 | ) 10 | 11 | var mutex = &sync.Mutex{} 12 | 13 | // WriteFrame writes the frame with the given uri to the WriteSeeker 14 | func WriteFrame(w io.Writer, frame *Frame) error { 15 | mutex.Lock() 16 | defer mutex.Unlock() 17 | 18 | err := binary.Write(w, binary.BigEndian, frame.Header) 19 | if err != nil { 20 | return err 21 | } 22 | err = binary.Write(w, binary.BigEndian, frame.Data) 23 | if err != nil { 24 | return err 25 | } 26 | logrus.Debugf("Written data to WriteSeeker: Data %s", frame.Data) 27 | 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /model/sort.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "regexp" 5 | "strconv" 6 | ) 7 | 8 | var reNumber *regexp.Regexp 9 | 10 | func init() { 11 | reNumber, _ = regexp.Compile("[0-9]+") 12 | } 13 | 14 | // ByNumber helper struct to sort by last number all the log files 15 | type ByNumber []string 16 | 17 | func (s ByNumber) Len() int { 18 | return len(s) 19 | } 20 | func (s ByNumber) Swap(i, j int) { 21 | s[i], s[j] = s[j], s[i] 22 | } 23 | func (s ByNumber) Less(i, j int) bool { 24 | si := reNumber.FindString(s[i]) 25 | sj := reNumber.FindString(s[j]) 26 | di, _ := strconv.ParseInt(si, 10, 32) 27 | dj, _ := strconv.ParseInt(sj, 10, 32) 28 | return di < dj 29 | } 30 | -------------------------------------------------------------------------------- /log_rotate_conf.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | LOGFILE=$5 4 | 5 | echo "LOG { 6 | ROTATION_PERIOD 7 | ROTATION_COUNT 8 | ROTATION_SIZE 9 | copytruncate 10 | delaycompress 11 | compress 12 | notifempty 13 | missingok 14 | sharedscripts 15 | }" > "$LOGFILE" 16 | 17 | 18 | sed -i "s+LOG+$1+" "$LOGFILE" 19 | sed -i "s/ROTATION_PERIOD/$2/" "$LOGFILE" 20 | sed -i "s/ROTATION_COUNT/rotate $3/" "$LOGFILE" 21 | 22 | if [[ $4 =~ ^[0-9]+[kMG]$ ]]; then 23 | sed -i "s/ROTATION_SIZE/maxsize $4/" "$LOGFILE" 24 | 25 | elif [[ $4 == -1 ]]; then 26 | sed -i '4d' "$LOGFILE" 27 | 28 | else 29 | echo "rotation size $4 is not valid. Skipped configuration" 30 | sed -i '4d' "$LOGFILE" 31 | fi -------------------------------------------------------------------------------- /model/constructor_test.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestFrameCreation(t *testing.T) { 11 | uri := "http://testtest:9090/net" 12 | name := "test" 13 | frame := NewFrame(uri, name, []byte("foobar")) 14 | assert.Equal(t, frame.Header.Size, int64(6), "length should be equal") 15 | assert.Equal(t, frame.Data, []byte("foobar"), "data contained should be equal") 16 | assert.NotZero(t, frame.Header.Timestamp, "timestamp should be setted and different from zero") 17 | assert.Condition(t, func() bool { 18 | return time.Now().Unix() >= frame.Header.Timestamp 19 | }, "current timestamp should be greater or equal that frame creation timestamp") 20 | } 21 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.15 AS build 2 | 3 | RUN apt update && apt install -yq go-dep 4 | 5 | RUN mkdir -p $GOPATH/src/github.com/Cleafy 6 | 7 | WORKDIR $GOPATH/src/github.com/Cleafy/promqueen 8 | COPY . . 9 | RUN dep ensure 10 | 11 | WORKDIR $GOPATH/src/github.com/Cleafy/promqueen/bin/promrec 12 | RUN GOOS=linux GOARCH=386 CGO_ENABLED=0 go build -o "$GOPATH/bin/promrec" -a -ldflags "-extldflags '-static'" github.com/Cleafy/promqueen/bin/promrec 13 | 14 | FROM golang:1.15 15 | 16 | WORKDIR /promqueen 17 | 18 | COPY --from=build $GOPATH/bin/promrec /promqueen 19 | 20 | ENV DEBIAN_FRONTEND=noninteractive 21 | RUN apt-get update && apt-get install -y logrotate findutils && rm -rf /var/lib/apt/lists/* 22 | 23 | ARG METRICS_DIR="/var/log/cleafy/metrics" 24 | 25 | RUN mkdir -p $METRICS_DIR 26 | 27 | COPY log_rotate_conf.sh . 28 | COPY rotate-metrics.sh . 29 | COPY entrypoint.sh . 30 | 31 | ENTRYPOINT ["./entrypoint.sh"] -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | PKG = 'github.com/cleafy/promqueen'.freeze 2 | 3 | task :ensure do 4 | ln_sf ENV['PWD'], "#{ENV['GOPATH']}/src/#{PKG}" 5 | sh "cd #{ENV['GOPATH']}/src/#{PKG} && dep ensure" 6 | cd ENV['PWD'] 7 | Dir.chdir('vendor') do 8 | Dir['*/*/*'].each do |d| 9 | puts "Linking #{d} into #{ENV['GOPATH']}" 10 | rm_rf "#{ENV['GOPATH']}/src/#{d}" 11 | mkdir_p "#{ENV['GOPATH']}/src/#{File.dirname(d)}" 12 | ln_sf "#{Dir.pwd}/#{d}", "#{ENV['GOPATH']}/src/#{d}" 13 | end 14 | end 15 | end 16 | 17 | task :update do 18 | ln_sf ENV['PWD'], "#{ENV['GOPATH']}/src/#{PKG}" 19 | sh "cd #{ENV['GOPATH']}/src/#{PKG} && dep ensure -update" 20 | cd ENV['PWD'] 21 | Dir.chdir('vendor') do 22 | Dir['*/*/*'].each do |d| 23 | puts "Linking #{d} into #{ENV['GOPATH']}" 24 | rm_rf "#{ENV['GOPATH']}/src/#{d}" 25 | mkdir_p "#{ENV['GOPATH']}/src/#{File.dirname(d)}" 26 | ln_sf "#{Dir.pwd}/#{d}", "#{ENV['GOPATH']}/src/#{d}" 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /model/utils_test.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import "reflect" 4 | 5 | func dataSize(v reflect.Value) int { 6 | if v.Kind() == reflect.Slice { 7 | if s := sizeof(v.Type().Elem()); s >= 0 { 8 | return s * v.Len() 9 | } 10 | return -1 11 | } 12 | return sizeof(v.Type()) 13 | } 14 | 15 | func sizeof(t reflect.Type) int { 16 | switch t.Kind() { 17 | case reflect.Array: 18 | if s := sizeof(t.Elem()); s >= 0 { 19 | return s * t.Len() 20 | } 21 | 22 | case reflect.Struct: 23 | sum := 0 24 | for i, n := 0, t.NumField(); i < n; i++ { 25 | s := sizeof(t.Field(i).Type) 26 | if s < 0 { 27 | return -1 28 | } 29 | sum += s 30 | } 31 | return sum 32 | 33 | case reflect.Bool, 34 | reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, 35 | reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, 36 | reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128: 37 | return int(t.Size()) 38 | } 39 | 40 | return -1 41 | } 42 | -------------------------------------------------------------------------------- /model/constructors.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | var ( 8 | magic = [3]byte{0x83, 0xF1, 0xF1} 9 | // this should be bumped every time the format is not compatible anymore 10 | version = [3]byte{0x00, 0x00, 0x00} 11 | ) 12 | 13 | // CheckVersion verifies that the binary format is compatible with the current release 14 | func CheckVersion(header *FrameHeader) bool { 15 | return header.Magic == magic && header.Version == version 16 | } 17 | 18 | // NewFrame generates a new Frame from a given byte data 19 | func NewFrame(name string, uri string, data []byte) *Frame { 20 | var u, n [52]byte 21 | copy(u[:], uri) 22 | copy(n[:], name) 23 | 24 | return &Frame{ 25 | Header: &FrameHeader{ 26 | Magic: magic, 27 | Version: version, 28 | Size: int64(len(data)), 29 | Timestamp: time.Now().Unix(), 30 | Name: n, 31 | URI: u, 32 | }, 33 | Data: data, 34 | } 35 | } 36 | 37 | // NewEmptyFrame generates a new empty frame 38 | func NewEmptyFrame() *Frame { 39 | return NewFrame("", "", nil) 40 | } 41 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | 2 | # Gopkg.toml example 3 | # 4 | # Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md 5 | # for detailed Gopkg.toml documentation. 6 | # 7 | # required = ["github.com/user/thing/cmd/thing"] 8 | # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] 9 | # 10 | # [[constraint]] 11 | # name = "github.com/user/project" 12 | # version = "1.0.0" 13 | # 14 | # [[constraint]] 15 | # name = "github.com/user/project2" 16 | # branch = "dev" 17 | # source = "github.com/myfork/project2" 18 | # 19 | # [[override]] 20 | # name = "github.com/x/y" 21 | # version = "2.4.0" 22 | 23 | 24 | [[constraint]] 25 | branch = "master" 26 | name = "github.com/mattetti/filebuffer" 27 | 28 | [[constraint]] 29 | branch = "master" 30 | name = "github.com/prometheus/client_model" 31 | 32 | [[constraint]] 33 | branch = "master" 34 | name = "github.com/prometheus/common" 35 | 36 | [[constraint]] 37 | name = "github.com/prometheus/prometheus" 38 | version = "<=1.5.3" 39 | 40 | [[constraint]] 41 | name = "github.com/sirupsen/logrus" 42 | version = "1.0.3" 43 | 44 | [[constraint]] 45 | name = "github.com/stretchr/testify" 46 | version = "1.1.4" 47 | 48 | [[constraint]] 49 | name = "gopkg.in/alecthomas/kingpin.v2" 50 | version = "2.2.5" 51 | 52 | [[constraint]] 53 | name = "gopkg.in/h2non/filetype.v1" 54 | version = "1.0.3" 55 | -------------------------------------------------------------------------------- /model/structs.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import "strings" 4 | 5 | // Collection represents the file that contains all the subsequent frames 6 | type Collection struct { 7 | Data []*Frame 8 | } 9 | 10 | // Header.URIString converts the backing URI to a string 11 | func (frame *Frame) URIString() string { 12 | return strings.TrimRight(string(frame.Header.URI[:]), "\x00") 13 | } 14 | 15 | // Header.URIString converts the backing URI to a string 16 | func (frame *Frame) NameString() string { 17 | return strings.TrimRight(string(frame.Header.Name[:]), "\x00") 18 | } 19 | 20 | // FrameHeaderLength total header length size for each frame 21 | const FrameHeaderLength = 128 22 | 23 | // FrameHeader represents the header of each Frame 24 | // - a Size that represents how big is the the Data section 25 | // - a Timestamp that represents when the Frame is snapshotted 26 | // - a Name that represents the service that has been snapshotted 27 | // - an URL that represents the service location 28 | type FrameHeader struct { 29 | Magic [3]byte 30 | Version [3]byte 31 | Reserved [2]byte 32 | Size int64 33 | Timestamp int64 34 | Name [52]byte 35 | URI [52]byte 36 | } 37 | 38 | // Frame represents one of the frame of the Collection file. It contains: 39 | // - the Data slice that contains the data of the frame 40 | type Frame struct { 41 | Header *FrameHeader 42 | Data []byte 43 | } 44 | -------------------------------------------------------------------------------- /model/writer_test.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "testing" 7 | 8 | "github.com/mattetti/filebuffer" 9 | "github.com/sirupsen/logrus" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | // This test depends on a working reader 14 | func TestWriteFrame(t *testing.T) { 15 | backing := make([]byte, 0) 16 | buffer := filebuffer.New(backing) 17 | uri := "http://ciao:8080/v1/metrics" 18 | name := "foobar" 19 | 20 | err := WriteFrame(buffer, NewFrame(name, uri, []byte("Foo1Bar"))) 21 | assert.Empty(t, err, "error1 should be empty") 22 | err = WriteFrame(buffer, NewFrame(name, uri, []byte("Foo2Bar"))) 23 | assert.Empty(t, err, "error2 should be empty") 24 | err = WriteFrame(buffer, NewFrame(name, uri, []byte("Foo3Bar"))) 25 | assert.Empty(t, err, "error3 should be empty") 26 | 27 | // restart the 28 | buffer.Seek(0, io.SeekStart) 29 | collection := ReadAll(buffer) 30 | 31 | for _, frame := range collection.Data { 32 | assert.True(t, CheckVersion(frame.Header)) 33 | } 34 | assert.Equal(t, 3, len(collection.Data), "there should be exactly 3 frames") 35 | 36 | for _, frame := range collection.Data { 37 | assert.Equal(t, uri, frame.URIString(), "saved uri should be equal") 38 | assert.Equal(t, name, frame.NameString(), "saved uri should be equal") 39 | } 40 | logrus.Debugf("collection data %+v", collection.Data) 41 | assert.Equal(t, "Foo1Bar", string(collection.Data[0].Data), "data should be equal") 42 | assert.Equal(t, "Foo2Bar", string(collection.Data[1].Data), "data should be equal") 43 | assert.Equal(t, "Foo3Bar", string(collection.Data[2].Data), "data should be equal") 44 | } 45 | 46 | func init() { 47 | // Output to stdout instead of the default stderr 48 | // Can be any io.Writer, see below for File example 49 | logrus.SetOutput(os.Stdout) 50 | 51 | // Only log the warning severity or above. 52 | // logrus.SetLevel(logrus.DebugLevel) 53 | } 54 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.8 4 | install: 5 | - go get -u github.com/golang/dep/cmd/dep 6 | - dep ensure 7 | script: 8 | - go test -v github.com/Cleafy/promqueen/model 9 | - GOOS=linux GOARCH=386 CGO_ENABLED=0 go build -o "$GOPATH/bin/promrec.elf" -a -ldflags "-extldflags '-static' -X main.Version=$(git describe --tags --abbrev=0)" github.com/Cleafy/promqueen/bin/promrec 10 | - GOOS=linux GOARCH=386 CGO_ENABLED=0 go build -o "$GOPATH/bin/promplay.elf" -a -ldflags "-extldflags '-static' -X main.Version=$(git describe --tags --abbrev=0)" github.com/Cleafy/promqueen/bin/promplay 11 | - GOOS=windows GOARCH=386 CGO_ENABLED=0 go build -o "$GOPATH/bin/promrec.exe" -a -ldflags "-extldflags '-static' -X main.Version=$(git describe --tags --abbrev=0)" github.com/Cleafy/promqueen/bin/promrec 12 | - GOOS=windows GOARCH=386 CGO_ENABLED=0 go build -o "$GOPATH/bin/promplay.exe" -a -ldflags "-extldflags '-static'-X main.Version=$(git describe --tags --abbrev=0)" github.com/Cleafy/promqueen/bin/promplay 13 | - GOOS=darwin GOARCH=386 CGO_ENABLED=0 go build -o "$GOPATH/bin/promrec" -a -ldflags "-extldflags '-static' -X main.Version=$(git describe --tags --abbrev=0)" github.com/Cleafy/promqueen/bin/promrec 14 | - GOOS=darwin GOARCH=386 CGO_ENABLED=0 go build -o "$GOPATH/bin/promplay" -a -ldflags "-extldflags '-static' -X main.Version=$(git describe --tags --abbrev=0)" github.com/Cleafy/promqueen/bin/promplay 15 | deploy: 16 | provider: releases 17 | api_key: 18 | secure: G1FtHnchisi9145HM8gA6v7X+JzKO7kD3dL12OPegMVOVnjE5c73aMDALuqogl0c4N00IAlrMoj1NaddW0xQKYULqG0X3mswVA3Bmgcrl93XKAxCmymzYZSoq1Q2vcehXihmbpZQfl76DPrJwKIYrFWVtJoE9tH3yrkxpsGmTh3YgLtRr1PpCuUE5xFILDwEu1AsIlQeMzLxERkykSSMq2iNtABPsaPKLU0NUmkN4/WM2i/7bzMQE+lYQnZe7p3TjTf7/+9ht9KyHWH49I+6kTDsANMTH+Q6tHZNggBTtSloStCQmm1L7IQkIpYw3gDzVNfNQWuhKwICE0Z8iWX3VFKiolRSC9nfuf6LEaNyFvMhB7wcf+9rF5thZVf+cZVjh7wSGoYPAPvh3ld0vm35uX0B7MA6LjAFNbURpt8no1urNuJirTzyP9H6Ym0KfOCjWLxDtUIDzZZ89zieP1atoaCmP+/4rbS74p0WPp32OnrY6uAym9TQPuHWp7S3Iq9DBOHxHidFEC4pmWJgT4wD5fDQk/g2kleidtcq28sPVrxKUZFyN578z9FIEi80AHkl8bL4iKLFDFsklt8IyWodl6rbx0flFosjVJ1rUr+IDMuFxmnkJbHamg2W5fysbtHplWDpODlDj2KQyYJC4ltZyDe4TS4sGr9wwQKoGNEphWU= 19 | file: 20 | - "$GOPATH/bin/promrec" 21 | - "$GOPATH/bin/promplay" 22 | - "$GOPATH/bin/promrec.elf" 23 | - "$GOPATH/bin/promplay.elf" 24 | - "$GOPATH/bin/promrec.exe" 25 | - "$GOPATH/bin/promplay.exe" 26 | on: 27 | repo: Cleafy/promqueen 28 | branch: master 29 | tags: true 30 | -------------------------------------------------------------------------------- /model/reader_test.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "testing" 7 | 8 | "github.com/mattetti/filebuffer" 9 | "github.com/sirupsen/logrus" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | var data = []byte{ 14 | 0x42, 0x42, 0x42, 0x42, 15 | } 16 | 17 | var datalen byte = byte(len(data)) 18 | 19 | var frameSample = []byte{ 20 | 0x83, 0xF1, 0xF1, // Magic 21 | 0x00, 0x00, 0x00, // Version 22 | 0x00, 0x00, // Reserved 23 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, datalen, // Size 24 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Timestamp 25 | } // len(frameSample) == 20 26 | 27 | func init() { 28 | f := make([]byte, FrameHeaderLength) 29 | copy(f, frameSample) 30 | frameSample = f 31 | frameSample = append(frameSample, data...) 32 | } 33 | 34 | func TestReadFrame(t *testing.T) { 35 | buffer := filebuffer.New(frameSample) 36 | _, err := ReadFrame(buffer) 37 | assert.Empty(t, err, "should not be any error") 38 | } 39 | 40 | func TestRead2Frames(t *testing.T) { 41 | tmp := append(frameSample, frameSample...) 42 | 43 | assert.Equal(t, 44 | len(frameSample)*2, 45 | len(tmp), 46 | "tmp length should be the sum of all the lenghts") 47 | 48 | buffer := filebuffer.New(tmp) 49 | 50 | _, err := ReadFrame(buffer) 51 | assert.Empty(t, err, "should not be any error") 52 | 53 | _, err = ReadFrame(buffer) 54 | assert.Empty(t, err, "should not be any error") 55 | } 56 | 57 | func TestReadAll(t *testing.T) { 58 | tmp := append(frameSample, frameSample...) 59 | 60 | assert.Equal(t, 61 | len(frameSample)*2, 62 | len(tmp), 63 | "tmp length should be the sum of all the lenghts") 64 | 65 | buffer := filebuffer.New(tmp) 66 | 67 | collection := ReadAll(buffer) 68 | 69 | for _, frame := range collection.Data { 70 | assert.True(t, CheckVersion(frame.Header), "the header version should be correct") 71 | } 72 | assert.Equal(t, 2, len(collection.Data), "there should be two frames") 73 | } 74 | 75 | func TestNewFrameReader(t *testing.T) { 76 | tmp := append(frameSample, frameSample...) 77 | 78 | assert.Equal(t, 79 | len(frameSample)*2, 80 | len(tmp), 81 | "tmp length should be the sum of all the lenghts") 82 | 83 | buffer := filebuffer.New(tmp) 84 | 85 | frameChannel := NewMultiReader([]io.Reader{buffer}) 86 | 87 | frame1 := <-frameChannel 88 | frame2 := <-frameChannel 89 | logrus.Debugf("%+v %+v", frame1, frame2) 90 | 91 | assert.Equal(t, frame1, frame2, "The two frames should be equal") 92 | _, ok := <-frameChannel 93 | assert.False(t, ok, "frameChannel should be closed") 94 | } 95 | 96 | func init() { 97 | // Output to stdout instead of the default stderr 98 | // Can be any io.Writer, see below for File example 99 | logrus.SetOutput(os.Stdout) 100 | 101 | // Only log the warning severity or above. 102 | // logrus.SetLevel(logrus.DebugLevel) 103 | } 104 | -------------------------------------------------------------------------------- /bin/promrec/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "compress/gzip" 5 | "io" 6 | "net/http" 7 | "net/http/httputil" 8 | "os" 9 | "time" 10 | 11 | "github.com/Cleafy/promqueen/model" 12 | "github.com/sirupsen/logrus" 13 | "gopkg.in/alecthomas/kingpin.v2" 14 | ) 15 | 16 | var ( 17 | debug = kingpin.Flag("debug", "Enable debug mode.").Bool() 18 | enableGZIP = kingpin.Flag("gzip", "Enable gzip mode.").Bool() 19 | interval = kingpin.Flag("interval", "Timeout waiting for ping.").Default("60s").OverrideDefaultFromEnvar("ACTION_INTERVAL").Short('i').Duration() 20 | umap = kingpin.Flag("umap", "stringmap [eg. service.name=http://get.uri:port/uri].").Short('u').StringMap() 21 | output = kingpin.Flag("output", "Output file.").Short('o').OverrideDefaultFromEnvar("OUTPUT_FILE").Default("metrics").String() 22 | maxIntervalsNumber = kingpin.Flag("maxIntervalsNumber", "Max number of intervals").Short('n').Default("120").Int() 23 | Version = "0.0.10" 24 | filewriter io.WriteCloser 25 | ) 26 | 27 | func closeIfNotNil(wc io.WriteCloser) { 28 | if wc != nil { 29 | if err := wc.Close(); err != nil { 30 | logrus.Info(err) 31 | } 32 | } 33 | } 34 | 35 | func writerFor() (io.Writer, error) { 36 | if _, err := os.Stat(*output); !os.IsNotExist(err) && filewriter != nil { 37 | return filewriter, nil 38 | } 39 | 40 | file, err := os.OpenFile(*output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 41 | if err != nil { 42 | return nil, err 43 | } 44 | closeIfNotNil(filewriter) 45 | if *enableGZIP { 46 | filewriter = gzip.NewWriter(file) 47 | } else { 48 | filewriter = file 49 | } 50 | return filewriter, nil 51 | } 52 | 53 | func main() { 54 | kingpin.Version(Version) 55 | kingpin.Parse() 56 | 57 | if *debug { 58 | logrus.SetLevel(logrus.DebugLevel) 59 | } 60 | 61 | if len(*umap) <= 0 { 62 | kingpin.Usage() 63 | return 64 | } 65 | 66 | ticker := time.NewTicker(*interval) 67 | intervalsCount := 0 68 | 69 | for range ticker.C { 70 | if (*maxIntervalsNumber > 0) { 71 | if (intervalsCount > *maxIntervalsNumber) { 72 | os.Exit(0) 73 | } 74 | } 75 | 76 | for sname, url := range *umap { 77 | writer, err := writerFor() 78 | if err != nil { 79 | logrus.Errorf("writeFor failed with %v", err) 80 | continue 81 | } 82 | 83 | resp, err := http.Get(url) 84 | if err != nil { 85 | logrus.Errorf("http.Get: %v", err) 86 | continue 87 | } 88 | defer resp.Body.Close() 89 | 90 | dump, err := httputil.DumpResponse(resp, true) 91 | if err != nil { 92 | logrus.Errorf("httputil.DumpResponse: %v", err) 93 | continue 94 | } 95 | 96 | frame := model.NewFrame(sname, url, dump) 97 | 98 | err = model.WriteFrame(writer, frame) 99 | if err != nil { 100 | logrus.Errorf("model.WriteFrame failed with %v", err) 101 | continue 102 | } 103 | } 104 | 105 | intervalsCount += 1 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /model/reader.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "io" 7 | "unsafe" 8 | 9 | "github.com/sirupsen/logrus" 10 | ) 11 | 12 | // NewFrameReader returns a channel of Frames. The channel is closed whenever 13 | // there are no other frames or the FrameReader encounter an error reading a frame 14 | func NewMultiReader(r []io.Reader) <-chan Frame { 15 | chframe := make(chan Frame) 16 | 17 | go func() { 18 | defer close(chframe) 19 | windex := 0 20 | for windex < len(r) { 21 | frame, err := ReadFrame(r[windex]) 22 | if err != nil || !CheckVersion(frame.Header) { 23 | windex++ 24 | if windex >= len(r) { 25 | break 26 | } 27 | continue 28 | } 29 | chframe <- *frame 30 | } 31 | logrus.Infof("Frames ended") 32 | }() 33 | return chframe 34 | } 35 | 36 | // ReadAll reads all the Collection (Header, Frame*) and returns in a compound 37 | // structure. 38 | // NOTE: the NewFrameReader streaming implementation should be preferred 39 | func ReadAll(r io.Reader) *Collection { 40 | frames := make([]*Frame, 0) 41 | 42 | for { 43 | frame, err := ReadFrame(r) 44 | if err != nil { 45 | break 46 | } 47 | frames = append(frames, frame) 48 | } 49 | 50 | return &Collection{ 51 | Data: frames, 52 | } 53 | } 54 | 55 | func readNextBytes(reader io.Reader, number int64) ([]byte, error) { 56 | bytes := make([]byte, number) 57 | 58 | _, err := reader.Read(bytes) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | return bytes, nil 64 | } 65 | 66 | // ReadFrameHeader reads the next FrameHeader from the reader 67 | func ReadFrameHeader(r io.Reader) (*FrameHeader, error) { 68 | header := &FrameHeader{} 69 | 70 | // read the frame Size 71 | data, err := readNextBytes(r, int64(unsafe.Sizeof(*header))) 72 | if err != nil { 73 | return nil, err 74 | } 75 | buffer := bytes.NewBuffer(data) 76 | 77 | err = binary.Read(buffer, binary.BigEndian, header) 78 | if err != nil { 79 | return nil, err 80 | } 81 | logrus.Debugf("ReadFrame: frame.Header %d", header) 82 | 83 | return header, nil 84 | } 85 | 86 | // ReadFrame reads the next frame from the Reader or returns an error in 87 | // case it cannot interpret the Frame 88 | func ReadFrame(r io.Reader) (frame *Frame, err error) { 89 | defer func() { 90 | if e := recover(); e != nil { 91 | if e.(error).Error() != "EOF" { 92 | logrus.Errorf("Errors occured while reading frame %v, MESSAGE: %v", frame.NameString, e) 93 | } 94 | } 95 | }() 96 | 97 | frame = NewEmptyFrame() 98 | frame.Header, err = ReadFrameHeader(r) 99 | 100 | if err != nil { 101 | panic(err) 102 | } 103 | 104 | // generate the correct framesize for .Data 105 | frame.Data = make([]byte, frame.Header.Size) 106 | 107 | // read the frame Data 108 | data, err := readNextBytes(r, int64(len(frame.Data))) 109 | if err != nil { 110 | panic(err) 111 | } 112 | 113 | buffer := bytes.NewBuffer(data) 114 | 115 | err = binary.Read(buffer, binary.BigEndian, frame.Data) 116 | if err != nil { 117 | panic(err) 118 | } 119 | 120 | logrus.Debugf("ReadFrame: frame.Data %d", frame.Data) 121 | 122 | return 123 | } 124 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/go,vim,emacs,intellij+all 3 | 4 | ### Emacs ### 5 | # -*- mode: gitignore; -*- 6 | *~ 7 | \#*\# 8 | /.emacs.desktop 9 | /.emacs.desktop.lock 10 | *.elc 11 | auto-save-list 12 | tramp 13 | .\#* 14 | 15 | # Org-mode 16 | .org-id-locations 17 | *_archive 18 | 19 | # flymake-mode 20 | *_flymake.* 21 | 22 | # eshell files 23 | /eshell/history 24 | /eshell/lastdir 25 | 26 | # elpa packages 27 | /elpa/ 28 | 29 | # reftex files 30 | *.rel 31 | 32 | # AUCTeX auto folder 33 | /auto/ 34 | 35 | # cask packages 36 | .cask/ 37 | dist/ 38 | 39 | # Flycheck 40 | flycheck_*.el 41 | 42 | # server auth directory 43 | /server/ 44 | 45 | # projectiles files 46 | .projectile 47 | projectile-bookmarks.eld 48 | 49 | # directory configuration 50 | .dir-locals.el 51 | 52 | # saveplace 53 | places 54 | 55 | # url cache 56 | url/cache/ 57 | 58 | # cedet 59 | ede-projects.el 60 | 61 | # smex 62 | smex-items 63 | 64 | # company-statistics 65 | company-statistics-cache.el 66 | 67 | # anaconda-mode 68 | anaconda-mode/ 69 | 70 | ### Go ### 71 | # Binaries for programs and plugins 72 | *.exe 73 | *.dll 74 | *.so 75 | *.dylib 76 | 77 | # Test binary, build with `go test -c` 78 | *.test 79 | 80 | # Output of the go coverage tool, specifically when used with LiteIDE 81 | *.out 82 | 83 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 84 | .glide/ 85 | 86 | # Golang project vendor packages which should be ignored 87 | vendor/ 88 | 89 | ### Intellij+all ### 90 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 91 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 92 | 93 | # User-specific stuff: 94 | .idea/**/workspace.xml 95 | .idea/**/tasks.xml 96 | .idea/dictionaries 97 | 98 | # Sensitive or high-churn files: 99 | .idea/**/dataSources/ 100 | .idea/**/dataSources.ids 101 | .idea/**/dataSources.xml 102 | .idea/**/dataSources.local.xml 103 | .idea/**/sqlDataSources.xml 104 | .idea/**/dynamic.xml 105 | .idea/**/uiDesigner.xml 106 | 107 | # Gradle: 108 | .idea/**/gradle.xml 109 | .idea/**/libraries 110 | 111 | # CMake 112 | cmake-build-debug/ 113 | 114 | # Mongo Explorer plugin: 115 | .idea/**/mongoSettings.xml 116 | 117 | ## File-based project format: 118 | *.iws 119 | 120 | ## Plugin-specific files: 121 | 122 | # IntelliJ 123 | /out/ 124 | 125 | # mpeltonen/sbt-idea plugin 126 | .idea_modules/ 127 | 128 | # JIRA plugin 129 | atlassian-ide-plugin.xml 130 | 131 | # Cursive Clojure plugin 132 | .idea/replstate.xml 133 | 134 | # Crashlytics plugin (for Android Studio and IntelliJ) 135 | com_crashlytics_export_strings.xml 136 | crashlytics.properties 137 | crashlytics-build.properties 138 | fabric.properties 139 | 140 | ### Intellij+all Patch ### 141 | # Ignores the whole idea folder 142 | # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 143 | 144 | .idea/ 145 | 146 | ### Vim ### 147 | # swap 148 | [._]*.s[a-v][a-z] 149 | [._]*.sw[a-p] 150 | [._]s[a-v][a-z] 151 | [._]sw[a-p] 152 | # session 153 | Session.vim 154 | # temporary 155 | .netrwhist 156 | # auto-generated tag files 157 | tags 158 | 159 | # End of https://www.gitignore.io/api/go,vim,emacs,intellij+all 160 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PromQueen 2 | **PromQueen** made possible to record _prometheus_ metrics offline. 3 | **PromQueen** can, therefore, backfill the recorded data inside a native _prometheus_ database. 4 | 5 | **PromQueen** is composed of two primary tools: 6 | 7 | - `promrec` tapes the metrics in a specified output file. 8 | - `promplay` backfills the _prometheus_ database from scratch. 9 | 10 | ## Build instructions (Linux/OSX) 11 | 12 | Clone this repository in your **$GOPATH**: 13 | 14 | ``` 15 | $ mkdir -p $GOPATH/src/github.com/Cleafy 16 | $ cd $GOPATH/src/github.com/Cleafy 17 | $ git clone https://github.com/Cleafy/promqueen.git 18 | ``` 19 | 20 | Use Go package manager ***dep*** to install the required dependencies: 21 | 22 | ``` 23 | $ cd $GOPATH/src/github.com/Cleafy/promqueen 24 | $ dep ensure 25 | ``` 26 | 27 | To build `promrec`: 28 | 29 | ``` 30 | $ cd $GOPATH/src/github.com/Cleafy/promqueen/bin/promrec 31 | $ go build 32 | ``` 33 | 34 | To build `promplay`: 35 | 36 | ``` 37 | $ cd $GOPATH/src/github.com/Cleafy/promqueen/bin/promplay 38 | $ go build 39 | ``` 40 | 41 | ## Usage 42 | 43 | ### PromREC 44 | 45 | ``` 46 | usage: promrec [] 47 | 48 | Flags: 49 | --help Show context-sensitive help (also try --help-long and --help-man). 50 | --debug Enable debug mode. 51 | --gzip Enable gzip mode. 52 | -i, --interval=60s Timeout waiting for ping. 53 | -u, --umap=UMAP ... stringmap [eg. service.name=http://get.uri:port/uri]. 54 | -o, --output="metrics" Output file. 55 | --version Show application version. 56 | ``` 57 | 58 | ### PromPLAY 59 | 60 | ``` 61 | usage: promplay [] 62 | 63 | Flags: 64 | --help Show context-sensitive help (also try --help-long and --help-man). 65 | --debug Enable debug mode. (VERY VERBOSE!) 66 | --verbose (-v) Enable info-level message 67 | --nopromcfg Disable the generation of the prometheus cfg file (prometheus.yml) 68 | -d, --dir="/tmp" Input directory. 69 | --version Show application version. 70 | --storage.path="data" Directory path to create and fill the data store under. 71 | --storage.retention-period=360h 72 | Period of time to store data for 73 | --storage.checkpoint-interval=30m 74 | Period of time to store data for 75 | --storage.checkpoint-dirty-series-limit=10000 76 | Period of time to store data for 77 | ``` 78 | 79 | ### Environment variables 80 | 81 | ```PROM_ARGS```: The argument for the promqueen service. Output, interval and at least one service is mandatory. 82 | - E.g. --output=/var/log/promqueen/metrics/metrics.prom --interval=30s -u serviceName1=URL1 -u serviceName2=URL2 ... 83 | 84 | 85 | ```ROTATION_FILE_LOG```: where the rotation should occurr. Must be the same of the "output" parameter in PROM_ARGS 86 | 87 | 88 | ```ROTATION_PERIOD```: how frequently a rotation will occurr. Default: "daily" 89 | 90 | ```ROTATION_COUNT```: how many rotation will be retained. Default: 10 91 | 92 | ```ROTATION_SIZE```: how big each rotation file will be in bytes. -1 means no limit. Default: -1. E.g. 100M 93 | 94 | 95 | ``` 96 | docker run -d --network=host --name promqueen \ 97 | -e ROTATION_FILE_LOG="/var/log/promqueen/metrics/metrics.prom" \ 98 | -e PROM_ARGS="--output=/var/log/promqueen/metrics/metrics.prom --interval=30s -u service1=URL1 -u service2=URL2" \ 99 | promqueen_image 100 | ``` 101 | 102 | ### Notes 103 | 104 | As of today **PromQueen** only supports backfilling inside _prometheus_ local storage. New storage types such as influxdb are not supported. 105 | -------------------------------------------------------------------------------- /Gopkg.lock: -------------------------------------------------------------------------------- 1 | # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. 2 | 3 | 4 | [[projects]] 5 | branch = "master" 6 | name = "github.com/alecthomas/template" 7 | packages = [ 8 | ".", 9 | "parse" 10 | ] 11 | revision = "a0175ee3bccc567396460bf5acd36800cb10c49c" 12 | 13 | [[projects]] 14 | branch = "master" 15 | name = "github.com/alecthomas/units" 16 | packages = ["."] 17 | revision = "2efee857e7cfd4f3d0138cc3cbb1b4966962b93a" 18 | 19 | [[projects]] 20 | branch = "master" 21 | name = "github.com/beorn7/perks" 22 | packages = ["quantile"] 23 | revision = "3a771d992973f24aa725d07868b467d1ddfceafb" 24 | 25 | [[projects]] 26 | name = "github.com/cleafy/promqueen" 27 | packages = ["model"] 28 | revision = "880d284cb214968ed48c13dbc34bb8a657b399b4" 29 | version = "v0.0.7" 30 | 31 | [[projects]] 32 | name = "github.com/davecgh/go-spew" 33 | packages = ["spew"] 34 | revision = "346938d642f2ec3594ed81d874461961cd0faa76" 35 | version = "v1.1.0" 36 | 37 | [[projects]] 38 | name = "github.com/golang/protobuf" 39 | packages = ["proto"] 40 | revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" 41 | version = "v1.1.0" 42 | 43 | [[projects]] 44 | branch = "master" 45 | name = "github.com/golang/snappy" 46 | packages = ["."] 47 | revision = "2e65f85255dbc3072edf28d6b5b8efc472979f5a" 48 | 49 | [[projects]] 50 | branch = "master" 51 | name = "github.com/mattetti/filebuffer" 52 | packages = ["."] 53 | revision = "3a1e8e5a6548ea21cce597660d0c0dcc8c77a520" 54 | 55 | [[projects]] 56 | name = "github.com/mattn/go-colorable" 57 | packages = ["."] 58 | revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" 59 | version = "v0.0.9" 60 | 61 | [[projects]] 62 | name = "github.com/mattn/go-isatty" 63 | packages = ["."] 64 | revision = "e1f7b56ace729e4a73a29a6b4fac6cd5fcda7ab3" 65 | version = "v0.0.9" 66 | 67 | [[projects]] 68 | name = "github.com/matttproud/golang_protobuf_extensions" 69 | packages = ["pbutil"] 70 | revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" 71 | version = "v1.0.1" 72 | 73 | [[projects]] 74 | name = "github.com/pmezard/go-difflib" 75 | packages = ["difflib"] 76 | revision = "792786c7400a136282c1664665ae0a8db921c6c2" 77 | version = "v1.0.0" 78 | 79 | [[projects]] 80 | name = "github.com/prometheus/client_golang" 81 | packages = ["prometheus"] 82 | revision = "c5b7fccd204277076155f10851dad72b76a49317" 83 | version = "v0.8.0" 84 | 85 | [[projects]] 86 | branch = "master" 87 | name = "github.com/prometheus/client_model" 88 | packages = ["go"] 89 | revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" 90 | 91 | [[projects]] 92 | branch = "master" 93 | name = "github.com/prometheus/common" 94 | packages = [ 95 | "expfmt", 96 | "internal/bitbucket.org/ww/goautoneg", 97 | "log", 98 | "model" 99 | ] 100 | revision = "7600349dcfe1abd18d72d3a1770870d9800a7801" 101 | 102 | [[projects]] 103 | branch = "master" 104 | name = "github.com/prometheus/procfs" 105 | packages = [ 106 | ".", 107 | "internal/util", 108 | "nfs", 109 | "xfs" 110 | ] 111 | revision = "61aaa706c6d4fda9365b6273c96839eb7e27f6e4" 112 | 113 | [[projects]] 114 | name = "github.com/prometheus/prometheus" 115 | packages = [ 116 | "storage", 117 | "storage/local", 118 | "storage/local/chunk", 119 | "storage/local/codable", 120 | "storage/local/index", 121 | "storage/metric", 122 | "util/flock", 123 | "util/testutil" 124 | ] 125 | revision = "1edf99ce5dc52e8e680176c22864e5acc67b9243" 126 | version = "v1.5.3" 127 | 128 | [[projects]] 129 | name = "github.com/sirupsen/logrus" 130 | packages = ["."] 131 | revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc" 132 | version = "v1.0.5" 133 | 134 | [[projects]] 135 | name = "github.com/stretchr/testify" 136 | packages = ["assert"] 137 | revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71" 138 | version = "v1.2.1" 139 | 140 | [[projects]] 141 | branch = "master" 142 | name = "github.com/syndtr/goleveldb" 143 | packages = [ 144 | "leveldb", 145 | "leveldb/cache", 146 | "leveldb/comparer", 147 | "leveldb/errors", 148 | "leveldb/filter", 149 | "leveldb/iterator", 150 | "leveldb/journal", 151 | "leveldb/memdb", 152 | "leveldb/opt", 153 | "leveldb/storage", 154 | "leveldb/table", 155 | "leveldb/util" 156 | ] 157 | revision = "5d6fca44a948d2be89a9702de7717f0168403d3d" 158 | 159 | [[projects]] 160 | branch = "master" 161 | name = "golang.org/x/crypto" 162 | packages = ["ssh/terminal"] 163 | revision = "ab813273cd59e1333f7ae7bff5d027d4aadf528c" 164 | 165 | [[projects]] 166 | branch = "master" 167 | name = "golang.org/x/net" 168 | packages = ["context"] 169 | revision = "1e491301e022f8f977054da4c2d852decd59571f" 170 | 171 | [[projects]] 172 | branch = "master" 173 | name = "golang.org/x/sys" 174 | packages = [ 175 | "unix", 176 | "windows", 177 | "windows/registry", 178 | "windows/svc/eventlog" 179 | ] 180 | revision = "c11f84a56e43e20a78cee75a7c034031ecf57d1f" 181 | 182 | [[projects]] 183 | name = "gopkg.in/VividCortex/ewma.v1" 184 | packages = ["."] 185 | revision = "b24eb346a94c3ba12c1da1e564dbac1b498a77ce" 186 | version = "v1.1.1" 187 | 188 | [[projects]] 189 | name = "gopkg.in/alecthomas/kingpin.v2" 190 | packages = ["."] 191 | revision = "947dcec5ba9c011838740e680966fd7087a71d0d" 192 | version = "v2.2.6" 193 | 194 | [[projects]] 195 | name = "gopkg.in/cheggaaa/pb.v2" 196 | packages = [ 197 | ".", 198 | "termutil" 199 | ] 200 | revision = "11d8df0cfef0bcb94e149d1189e5739e275f4df6" 201 | version = "v2.0.7" 202 | 203 | [[projects]] 204 | name = "gopkg.in/fatih/color.v1" 205 | packages = ["."] 206 | revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" 207 | version = "v1.7.0" 208 | 209 | [[projects]] 210 | name = "gopkg.in/h2non/filetype.v1" 211 | packages = [ 212 | ".", 213 | "matchers", 214 | "types" 215 | ] 216 | revision = "cc14fdc9ca0e4c2bafad7458f6ff79fd3947cfbb" 217 | version = "v1.0.5" 218 | 219 | [[projects]] 220 | name = "gopkg.in/mattn/go-colorable.v0" 221 | packages = ["."] 222 | revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" 223 | version = "v0.0.9" 224 | 225 | [[projects]] 226 | name = "gopkg.in/mattn/go-isatty.v0" 227 | packages = ["."] 228 | revision = "e1f7b56ace729e4a73a29a6b4fac6cd5fcda7ab3" 229 | version = "v0.0.9" 230 | 231 | [[projects]] 232 | name = "gopkg.in/mattn/go-runewidth.v0" 233 | packages = ["."] 234 | revision = "3ee7d812e62a0804a7d0a324e0249ca2db3476d3" 235 | version = "v0.0.4" 236 | 237 | [solve-meta] 238 | analyzer-name = "dep" 239 | analyzer-version = 1 240 | inputs-digest = "c385f6eea735682131d521721aa367bfc71b66d47094a7ebd90ede7504ce56fa" 241 | solver-name = "gps-cdcl" 242 | solver-version = 1 243 | -------------------------------------------------------------------------------- /bin/promplay/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "compress/gzip" 6 | "flag" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | "os" 11 | "path/filepath" 12 | "sort" 13 | "strings" 14 | "time" 15 | 16 | cm "github.com/Cleafy/promqueen/model" 17 | 18 | "github.com/mattetti/filebuffer" 19 | dto "github.com/prometheus/client_model/go" 20 | "github.com/prometheus/common/expfmt" 21 | "github.com/prometheus/common/model" 22 | "github.com/prometheus/prometheus/storage/local" 23 | "github.com/sirupsen/logrus" 24 | kingpin "gopkg.in/alecthomas/kingpin.v2" 25 | pb "gopkg.in/cheggaaa/pb.v2" 26 | filetype "gopkg.in/h2non/filetype.v1" 27 | ) 28 | 29 | var replayType = filetype.NewType("rep", "application/replay") 30 | 31 | func replayMatcher(buf []byte) bool { 32 | header, err := cm.ReadFrameHeader(filebuffer.New(buf)) 33 | if err == nil { 34 | return cm.CheckVersion(header) 35 | } 36 | logrus.Errorf("Malformed frame header!") 37 | return false 38 | } 39 | 40 | var ( 41 | debug = kingpin.Flag("debug", "Enable debug mode. More verbose than --verbose").Default("false").Bool() 42 | verbose = kingpin.Flag("verbose", "Enable info-only mode").Short('v').Default("false").Bool() 43 | nopromcfg = kingpin.Flag("nopromcfg", "Disable the generation of the prometheus cfg file (prometheus.yml)").Bool() 44 | dir = kingpin.Flag("dir", "Input directory.").Short('d').OverrideDefaultFromEnvar("INPUT_DIRECTORY").Default(".").String() 45 | memoryChunk = kingpin.Flag("memoryChunk", "Maximum number of chunks in memory").Default("100000000").Int() 46 | maxChunkToPersist = kingpin.Flag("maxChunkToPersist", "Maximum number of chunks waiting, in memory, to be written on the disk").Default("10000").Int() 47 | framereader = make(<-chan cm.Frame) 48 | Version = "0.0.10" 49 | cfgMemoryStorage = local.MemorySeriesStorageOptions{ 50 | MemoryChunks: 0, 51 | MaxChunksToPersist: 0, 52 | //PersistenceStoragePath: 53 | //PersistenceRetentionPeriod: 54 | //CheckpointInterval: time.Minute*30, 55 | //CheckpointDirtySeriesLimit: 10000, 56 | Dirty: true, 57 | PedanticChecks: true, 58 | SyncStrategy: local.Always, 59 | } 60 | ) 61 | 62 | func osfile2fname(fss []os.FileInfo, dir string) []string { 63 | out := make([]string, len(fss)) 64 | for i, fin := range fss { 65 | out[i] = dir + "/" + fin.Name() 66 | } 67 | return out 68 | } 69 | 70 | func generateFramereader() int { 71 | defer func() { 72 | if e := recover(); e != nil { 73 | logrus.Errorf("Frame reader generation failed!, MESSAGE: %v", e) 74 | } 75 | }() 76 | 77 | logrus.Infoln("Preliminary file read started...") 78 | var count int = 0 79 | // 1. Check for every file that is GZip or csave format and create the filemap 80 | files, err := ioutil.ReadDir(*dir) 81 | if err != nil { 82 | panic(err) 83 | } 84 | readers := make([]io.Reader, 0) 85 | 86 | fnames := osfile2fname(files, *dir) 87 | sort.Sort(sort.Reverse(cm.ByNumber(fnames))) 88 | 89 | logrus.Debugf("fnames: %v", fnames) 90 | 91 | for _, path := range fnames { 92 | logrus.Debugf("filepath: %v", path) 93 | ftype, err := filetype.MatchFile(path) 94 | if err != nil { 95 | logrus.Debugf("err %v", err) 96 | continue 97 | } 98 | if ftype.MIME.Value == "application/replay" { 99 | f, _ := os.Open(path) 100 | 101 | count += len(cm.ReadAll(f).Data) 102 | f.Seek(0, 0) 103 | 104 | readers = append(readers, f) 105 | } 106 | if ftype.MIME.Value == "application/gzip" { 107 | filename := filepath.Base(path) 108 | ungzip(path, "./tmp/"+trimSuffix(filename, ".gz")) 109 | 110 | f, _ := os.Open("./tmp/" + trimSuffix(filename, ".gz")) 111 | 112 | count += len(cm.ReadAll(f).Data) 113 | f.Seek(0, 0) 114 | 115 | readers = append(readers, f) 116 | } 117 | } 118 | framereader = cm.NewMultiReader(readers) 119 | return count 120 | } 121 | 122 | func trimSuffix(s, suffix string) string { 123 | if strings.HasSuffix(s, suffix) { 124 | s = s[:len(s)-len(suffix)] 125 | } 126 | return s 127 | } 128 | 129 | func updateURLTimestamp(timestamp int64, name string, url string, body io.Reader) io.Reader { 130 | dec := expfmt.NewDecoder(body, expfmt.FmtText) 131 | pr, pw := io.Pipe() 132 | enc := expfmt.NewEncoder(pw, expfmt.FmtText) 133 | //ts := timestamp * 1000 134 | 135 | go func() { 136 | count := 0 137 | 138 | for { 139 | var metrics dto.MetricFamily 140 | err := dec.Decode(&metrics) 141 | if err == io.EOF { 142 | break 143 | } 144 | if err != nil { 145 | logrus.Error(err) 146 | break 147 | } 148 | 149 | lpName := "job" 150 | urlName := "url" 151 | 152 | for _, metric := range metrics.GetMetric() { 153 | lp := dto.LabelPair{ 154 | Name: &lpName, 155 | Value: &name, 156 | } 157 | metric.Label = append(metric.Label, &lp) 158 | urlp := dto.LabelPair{ 159 | Name: &urlName, 160 | Value: &url, 161 | } 162 | metric.Label = append(metric.Label, &urlp) 163 | } 164 | 165 | enc.Encode(&metrics) 166 | 167 | count++ 168 | } 169 | 170 | logrus.Printf("%d metrics unmarshalled for %s", count, url) 171 | pw.Close() 172 | }() 173 | 174 | return pr 175 | } 176 | 177 | func ungzip(source, target string) { 178 | defer func() { 179 | if e := recover(); e != nil { 180 | logrus.Errorf("Errors during decompression of %v", source) 181 | } 182 | }() 183 | 184 | reader, err := os.Open(source) 185 | if err != nil { 186 | panic(err) 187 | } 188 | defer reader.Close() 189 | 190 | archive, err := gzip.NewReader(reader) 191 | if err != nil { 192 | panic(err) 193 | } 194 | defer archive.Close() 195 | 196 | target = filepath.Join(target, archive.Name) 197 | writer, err := os.Create(target) 198 | if err != nil { 199 | panic(err) 200 | } 201 | defer writer.Close() 202 | 203 | _, err = io.Copy(writer, archive) 204 | if err != nil { 205 | panic(err) 206 | } 207 | } 208 | 209 | func main() { 210 | 211 | kingpin.Version(Version) 212 | 213 | kingpin.Flag("storage.path", "Directory path to create and fill the data store under.").Default("data").StringVar(&cfgMemoryStorage.PersistenceStoragePath) 214 | kingpin.Flag("storage.retention-period", "Period of time to store data for").Default("360h").DurationVar(&cfgMemoryStorage.PersistenceRetentionPeriod) 215 | 216 | kingpin.Flag("storage.checkpoint-interval", "Period of time to store data for").Default("30m").DurationVar(&cfgMemoryStorage.CheckpointInterval) 217 | kingpin.Flag("storage.checkpoint-dirty-series-limit", "Period of time to store data for").Default("10000").IntVar(&cfgMemoryStorage.CheckpointDirtySeriesLimit) 218 | 219 | kingpin.Parse() 220 | 221 | if *debug { 222 | logrus.SetLevel(logrus.DebugLevel) 223 | flag.Set("log.level", "debug") 224 | } 225 | 226 | if !*verbose { 227 | logrus.SetLevel(logrus.ErrorLevel) 228 | flag.Set("log.level", "error") 229 | } 230 | 231 | // create temp directory to store ungzipped files 232 | os.Mkdir("./tmp", 0700) 233 | defer os.RemoveAll("./tmp") 234 | 235 | logrus.Infoln("Prefilling into", cfgMemoryStorage.PersistenceStoragePath) 236 | 237 | cfgMemoryStorage.MaxChunksToPersist = *maxChunkToPersist 238 | cfgMemoryStorage.MemoryChunks = *memoryChunk 239 | 240 | localStorage := local.NewMemorySeriesStorage(&cfgMemoryStorage) 241 | 242 | sampleAppender := localStorage 243 | 244 | logrus.Infoln("Starting the storage engine") 245 | if err := localStorage.Start(); err != nil { 246 | logrus.Errorln("Error opening memory series storage:", err) 247 | os.Exit(1) 248 | } 249 | defer func() { 250 | if err := localStorage.Stop(); err != nil { 251 | logrus.Errorln("Error stopping storage:", err) 252 | } 253 | }() 254 | 255 | filetype.AddMatcher(replayType, replayMatcher) 256 | 257 | count := generateFramereader() 258 | 259 | logrus.Debug("frameReader %+v", framereader) 260 | 261 | sout := bufio.NewWriter(os.Stdout) 262 | defer sout.Flush() 263 | 264 | r := &http.Request{} 265 | 266 | bar := pb.ProgressBarTemplate(`{{ red "Frames processed:" }} {{bar . | green}} {{rtime . "ETA %s" | blue }} {{percent . }}`).Start(count) 267 | defer bar.Finish() 268 | 269 | for frame := range framereader { 270 | bar.Increment() 271 | 272 | response, err := http.ReadResponse(bufio.NewReader(filebuffer.New(frame.Data)), r) 273 | if err != nil { 274 | logrus.Errorf("Errors occured while reading frame %d, MESSAGE: %v", frame.NameString, err) 275 | continue 276 | } 277 | bytesReader := updateURLTimestamp(frame.Header.Timestamp, frame.NameString(), frame.URIString(), response.Body) 278 | 279 | sdec := expfmt.SampleDecoder{ 280 | Dec: expfmt.NewDecoder(bytesReader, expfmt.FmtText), 281 | Opts: &expfmt.DecodeOptions{ 282 | Timestamp: model.TimeFromUnix(frame.Header.Timestamp), 283 | }, 284 | } 285 | 286 | decSamples := make(model.Vector, 0, 1) 287 | tempSamples := make(model.Vector, 0, 1) 288 | 289 | for err := sdec.Decode(&tempSamples); err == nil; err = sdec.Decode(&tempSamples) { 290 | decSamples = append(decSamples, tempSamples...) 291 | } 292 | 293 | logrus.Infoln("Ingested", len(decSamples), "metrics") 294 | 295 | for sampleAppender.NeedsThrottling() { 296 | logrus.Debugln("THROTTLING: Waiting 100ms for appender to be ready for more data") 297 | time.Sleep(time.Millisecond * 100) 298 | } 299 | 300 | var ( 301 | numOutOfOrder = 0 302 | numDuplicates = 0 303 | ) 304 | 305 | for _, s := range model.Samples(decSamples) { 306 | 307 | if err := sampleAppender.Append(s); err != nil { 308 | switch err { 309 | case local.ErrOutOfOrderSample: 310 | numOutOfOrder++ 311 | logrus.WithFields(logrus.Fields{ 312 | "sample": s, 313 | "error": err, 314 | }).Error("Sample discarded") 315 | case local.ErrDuplicateSampleForTimestamp: 316 | numDuplicates++ 317 | logrus.WithFields(logrus.Fields{ 318 | "sample": s, 319 | "error": err, 320 | }).Error("Sample discarded") 321 | default: 322 | logrus.WithFields(logrus.Fields{ 323 | "sample": s, 324 | "error": err, 325 | }).Error("Sample discarded") 326 | } 327 | } 328 | } 329 | } 330 | 331 | // Generate the prometheus.yml in case it does not exist 332 | promcfgpath := cfgMemoryStorage.PersistenceStoragePath + "/../prometheus.yml" 333 | if _, err := os.Stat(promcfgpath); os.IsNotExist(err) && !*nopromcfg { 334 | if err = ioutil.WriteFile(promcfgpath, []byte("global: {}"), os.ModeExclusive|0644); err != nil { 335 | logrus.Error(err) 336 | } 337 | } 338 | 339 | logrus.Info("Exiting! :)") 340 | } 341 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | --------------------------------------------------------------------------------