├── .gitignore ├── Godeps ├── Makefile ├── gateway ├── config.go ├── config_test.go ├── gateway.go ├── sample.toml └── sender.go ├── main.go └── readme.md /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | -------------------------------------------------------------------------------- /Godeps: -------------------------------------------------------------------------------- 1 | github.com/influxdata/influxdb b7bb7e8359642b6e071735b50ae41f5eb343fd42 2 | go.uber.org/zap fbae0281ffd546fa6d1959fec6075ac5da7fb577 3 | gopkg.in/natefinch/lumberjack.v2 dd45e6a67c53f673bb49ca8a001fd3a63ceb640e 4 | github.com/BurntSushi/toml b26d9c308763d68093482582cea63d69be07a0f0 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: b 2 | 3 | b: 4 | CGO_ENABLED=0 go build -o bin/influxdb-gateway main.go 5 | 6 | install: 7 | go install . 8 | -------------------------------------------------------------------------------- /gateway/config.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "io/ioutil" 5 | 6 | "github.com/BurntSushi/toml" 7 | ) 8 | 9 | type Config struct { 10 | Sender SenderConfig `toml:"sender"` 11 | } 12 | 13 | func LoadConfig(path string) (c Config, err error) { 14 | data, err := ioutil.ReadFile(path) 15 | if err != nil { 16 | return 17 | } 18 | err = toml.Unmarshal(data, &c) 19 | return 20 | } 21 | -------------------------------------------------------------------------------- /gateway/config_test.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestLoadConfig(t *testing.T) { 8 | c, err := LoadConfig("sample.toml") 9 | if err != nil { 10 | t.Error(err) 11 | } else if c.Sender.Addr == "" { 12 | t.Error("addr is null") 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /gateway/gateway.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | 6 | "github.com/influxdata/influxdb/models" 7 | "github.com/influxdata/influxdb/services/meta" 8 | "github.com/influxdata/influxdb/services/udp" 9 | ) 10 | 11 | type Service interface { 12 | Open() (err error) 13 | Close() error 14 | } 15 | 16 | type Gateway struct { 17 | Services []Service 18 | PointsWriter interface { 19 | WritePoints(database, retentionPolicy string, consistencyLevel models.ConsistencyLevel, points []models.Point) error 20 | } 21 | MetaClient interface { 22 | CreateDatabase(name string) (*meta.DatabaseInfo, error) 23 | } 24 | Logger zap.Logger 25 | } 26 | 27 | type FakeMetaClient struct{} 28 | 29 | func (f *FakeMetaClient) CreateDatabase(name string) (*meta.DatabaseInfo, error) { 30 | return nil, nil 31 | } 32 | 33 | func New(c Config, log zap.Logger) (*Gateway, error) { 34 | pointsWriter, err := NewSender(c.Sender) 35 | pointsWriter.Logger = log 36 | if err != nil { 37 | return nil, err 38 | } 39 | gateway := &Gateway{ 40 | MetaClient: &FakeMetaClient{}, 41 | PointsWriter: pointsWriter, 42 | Logger: log, 43 | } 44 | for _, conf := range c.Sender.UDPs { 45 | gateway.AppendUDPService(conf) 46 | } 47 | 48 | return gateway, nil 49 | } 50 | 51 | func (g *Gateway) AppendUDPService(conf udp.Config) { 52 | if !conf.Enabled { 53 | return 54 | } 55 | srv := udp.NewService(conf) 56 | srv.PointsWriter = g.PointsWriter 57 | srv.MetaClient = g.MetaClient 58 | srv.Logger = g.Logger 59 | g.Services = append(g.Services, srv) 60 | } 61 | 62 | func (g *Gateway) Open() (err error) { 63 | for _, s := range g.Services { 64 | err = s.Open() 65 | if err != nil { 66 | return 67 | } 68 | } 69 | return nil 70 | } 71 | 72 | func (g *Gateway) Close() error { 73 | for _, s := range g.Services { 74 | err := s.Close() 75 | if err != nil { 76 | g.Logger.Error(err.Error()) 77 | continue 78 | } 79 | } 80 | return nil 81 | } 82 | -------------------------------------------------------------------------------- /gateway/sample.toml: -------------------------------------------------------------------------------- 1 | [sender] 2 | addr = "http://localhost:6666" 3 | username = "" 4 | password = "" 5 | user-agent = "influxdb-gateway" 6 | timeout = 1 7 | gzip = true 8 | precision = "ns" 9 | consistency = "one" 10 | udp = [ 11 | {enabled = true,bind-address = ":9100",database = "db",batch-size = 10000,batch-pending = 100,batch-timeout = "1s",read-buffer = 0}, 12 | ] 13 | -------------------------------------------------------------------------------- /gateway/sender.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "crypto/tls" 7 | "errors" 8 | "fmt" 9 | "io/ioutil" 10 | "net/http" 11 | "net/url" 12 | "time" 13 | 14 | "go.uber.org/zap" 15 | 16 | "github.com/influxdata/influxdb/models" 17 | "github.com/influxdata/influxdb/services/udp" 18 | ) 19 | 20 | const ( 21 | DefaultPrecision = "ns" 22 | DefaultTimeout = 1 23 | DefaultConsistency = "one" 24 | ) 25 | 26 | type Sender struct { 27 | url *url.URL 28 | username string 29 | password string 30 | userAgent string 31 | httpClient *http.Client 32 | gzip bool 33 | precision string 34 | consistency string 35 | Logger zap.Logger 36 | } 37 | 38 | type SenderConfig struct { 39 | Addr string `toml:"addr"` 40 | Username string `toml:"username"` 41 | Password string `toml:"password"` 42 | UserAgent string `toml:"user-agent"` 43 | Timeout int `toml:"timeout"` 44 | Gzip bool `toml:"gzip"` 45 | InsecureSkipVerify bool `toml:"insucure-skip-verify"` 46 | Precision string `toml:"precision"` // ns | s | ms | n 47 | Consistency string `toml:"consistency"` // all | any | one | quorum 48 | UDPs []udp.Config `toml:"udp"` 49 | } 50 | 51 | func NewSender(c SenderConfig) (*Sender, error) { 52 | if c.UserAgent == "" { 53 | c.UserAgent = "InfluxDB-Gateway" 54 | } 55 | if c.Timeout == 0 { 56 | c.Timeout = DefaultTimeout 57 | } 58 | if c.Precision == "" { 59 | c.Precision = DefaultPrecision 60 | } 61 | if c.Consistency == "" { 62 | c.Consistency = DefaultConsistency 63 | } 64 | 65 | u, err := url.Parse(c.Addr) 66 | if err != nil { 67 | return nil, err 68 | } else if u.Scheme != "http" && u.Scheme != "https" { 69 | m := fmt.Sprintf("Unsupported protocol scheme: %s, your address"+ 70 | " must start with http:// or https://", u.Scheme) 71 | return nil, errors.New(m) 72 | } 73 | 74 | tr := &http.Transport{ 75 | TLSClientConfig: &tls.Config{ 76 | InsecureSkipVerify: c.InsecureSkipVerify, 77 | }, 78 | } 79 | 80 | return &Sender{ 81 | url: u, 82 | username: c.Username, 83 | password: c.Password, 84 | userAgent: c.UserAgent, 85 | gzip: c.Gzip, 86 | precision: c.Precision, 87 | consistency: c.Consistency, 88 | httpClient: &http.Client{ 89 | Timeout: time.Duration(c.Timeout) * time.Second, 90 | Transport: tr, 91 | }, 92 | }, nil 93 | } 94 | 95 | func (s *Sender) WritePoints(database, retentionPolicy string, consistencyLevel models.ConsistencyLevel, points []models.Point) error { 96 | go func() { 97 | var b bytes.Buffer 98 | if s.gzip { 99 | writer := gzip.NewWriter(&b) 100 | for _, p := range points { 101 | if _, err := writer.Write([]byte(p.PrecisionString(s.precision))); err != nil { 102 | s.Logger.Error(err.Error()) 103 | return 104 | } 105 | if _, err := writer.Write([]byte("\n")); err != nil { 106 | s.Logger.Error(err.Error()) 107 | return 108 | } 109 | } 110 | if err := writer.Flush(); err != nil { 111 | s.Logger.Error(err.Error()) 112 | return 113 | } 114 | if err := writer.Close(); err != nil { 115 | s.Logger.Error(err.Error()) 116 | return 117 | } 118 | } else { 119 | for _, p := range points { 120 | if _, err := b.WriteString(p.PrecisionString(s.precision)); err != nil { 121 | s.Logger.Error(err.Error()) 122 | return 123 | } 124 | if err := b.WriteByte('\n'); err != nil { 125 | s.Logger.Error(err.Error()) 126 | return 127 | } 128 | } 129 | } 130 | 131 | u := s.url 132 | u.Path = "write" 133 | req, err := http.NewRequest("POST", u.String(), &b) 134 | if err != nil { 135 | s.Logger.Error(err.Error()) 136 | return 137 | } 138 | req.Header.Set("Content-Type", "") 139 | req.Header.Set("User-Agent", s.userAgent) 140 | if s.gzip { 141 | req.Header.Set("Content-Encoding", "gzip") 142 | } 143 | if s.username != "" { 144 | req.SetBasicAuth(s.username, s.password) 145 | } 146 | 147 | params := req.URL.Query() 148 | params.Set("db", database) 149 | params.Set("rp", retentionPolicy) 150 | params.Set("precision", s.precision) 151 | params.Set("consistency", s.consistency) 152 | 153 | req.URL.RawQuery = params.Encode() 154 | 155 | resp, err := s.httpClient.Do(req) 156 | if err != nil { 157 | s.Logger.Error(err.Error()) 158 | return 159 | } 160 | defer resp.Body.Close() 161 | 162 | body, err := ioutil.ReadAll(resp.Body) 163 | if err != nil { 164 | s.Logger.Error(err.Error()) 165 | return 166 | } 167 | if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { 168 | s.Logger.Error(string(body)) 169 | return 170 | } 171 | }() 172 | 173 | return nil 174 | } 175 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "os" 6 | "os/signal" 7 | "syscall" 8 | 9 | "go.uber.org/zap" 10 | 11 | "github.com/pingliu/influxdb-gateway/gateway" 12 | 13 | lumberjack "gopkg.in/natefinch/lumberjack.v2" 14 | ) 15 | 16 | var ( 17 | configFilePath string 18 | logFilePath string 19 | logger zap.Logger 20 | ) 21 | 22 | func init() { 23 | flag.StringVar(&configFilePath, "config-file-path", "/etc/influxdb-gateway.toml", "config file path") 24 | flag.StringVar(&logFilePath, "log-file-path", "/var/log/influxdb-gateway.log", "log file path") 25 | flag.Parse() 26 | 27 | logger = zap.New( 28 | zap.NewTextEncoder(), 29 | zap.Output(zap.AddSync(&lumberjack.Logger{ 30 | Filename: logFilePath, 31 | MaxSize: 100, 32 | MaxBackups: 5, 33 | MaxAge: 7, 34 | })), 35 | ) 36 | } 37 | 38 | func main() { 39 | c, err := gateway.LoadConfig(configFilePath) 40 | if err != nil { 41 | logger.Fatal(err.Error()) 42 | } 43 | gateway, err := gateway.New(c, logger) 44 | if err != nil { 45 | logger.Fatal(err.Error()) 46 | } 47 | 48 | err = gateway.Open() 49 | if err != nil { 50 | logger.Fatal(err.Error()) 51 | } 52 | 53 | signalCh := make(chan os.Signal, 1) 54 | signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGKILL) 55 | logger.Info("Listening for signals") 56 | select { 57 | case <-signalCh: 58 | gateway.Close() 59 | logger.Info("Signal received, shutdown...") 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Influxdb Gateway 2 | 3 | ### 功能描述 4 | 5 | * 采用udp协议接受数据 6 | * 对数据进行格式检查 7 | * 对数据进行gzip,解决跨机房流量问题 8 | 9 | ### 使用方式 10 | 11 | ``` 12 | cd $GOPATH/src/github.com/influxdata/influxdb 13 | git checkout v1.2.0 14 | 15 | go get github.com/uber-go/zap 16 | mkdir $GOPATH/src/go.uber.org/ 17 | cd $GOPATH/src/go.uber.org/ 18 | mv $GOPATH/src/github.com/uber-go/zap . 19 | cd zap 20 | git checkout fbae0281ffd546fa6d1959fec6075ac5da7fb577 21 | 22 | cd $GOPATH/src/github.com/pingliu/influxdb-gateway 23 | make all 24 | ./bin/influxdb-gateway -h #查看使用方式 25 | ``` 26 | 27 | ### influxdb 高可用 28 | [InfluxDB Proxy](https://github.com/shell909090/influx-proxy) 29 | --------------------------------------------------------------------------------