├── .travis.yml ├── LICENSE ├── README.md ├── configuration_example.yml ├── go.mod └── http ├── client.go ├── config.go ├── enc.go ├── http.go └── url.go /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | env: 4 | - GO111MODULE="on" 5 | 6 | go: 7 | - 1.14.7 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 StackState 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | beats-output-http 2 | ================= 3 | 4 | Outputter for the Elastic Beats platform that simply 5 | POSTs events to an HTTP endpoint. 6 | 7 | [![Build Status](https://travis-ci.org/raboof/beats-output-http.svg?branch=master)](https://travis-ci.org/raboof/beats-output-http) 8 | 9 | Usage 10 | ===== 11 | 12 | To add support for this output plugin to a beat, you 13 | have to import this plugin into your main beats package, 14 | like this: 15 | 16 | ``` 17 | package main 18 | 19 | import ( 20 | "os" 21 | 22 | _ "github.com/raboof/beats-output-http/http" 23 | 24 | "github.com/elastic/beats/filebeat/cmd" 25 | ) 26 | 27 | func main() { 28 | if err := cmd.RootCmd.Execute(); err != nil { 29 | os.Exit(1) 30 | } 31 | } 32 | 33 | ``` 34 | 35 | Then configure the http output plugin in filebeat.yaml: 36 | 37 | ``` 38 | output: 39 | http: 40 | hosts: ["some.example.com:80/foo"] 41 | ``` 42 | -------------------------------------------------------------------------------- /configuration_example.yml: -------------------------------------------------------------------------------- 1 | #-------------------------- HTTP output ------------------------------ 2 | #output.http: 3 | # hosts: ["${STSURL}connbeat?api_key=${APIKEY}"] 4 | # 5 | # Optional further settings: 6 | # protocol: "https" 7 | # path: "foo" 8 | # parameters: "xyz" 9 | # proxy_url: "xyz" 10 | # loadbalance: true 11 | # compression_level: 9 12 | # format: "json_lines" 13 | # content_type: "text/plain" 14 | # max_retries: 3 15 | # timeout: 90 seconds 16 | # tls: 17 | # enabled: false 18 | # verification_mode: "full" 19 | # supported_protocols: [...] 20 | # cipher_suites: [...] 21 | # curve_types: [...] 22 | # certificate_authorities: [...] 23 | # certificate: ... 24 | # key: ... 25 | # key_passphrase: ... 26 | # 27 | # BASIC authentication: 28 | # username: "alice" 29 | # password: "secret" 30 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/raboof/beats-output-http 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/elastic/beats/v7 v7.10.1 7 | ) 8 | 9 | // needed because elastic wants these replacements, and https://github.com/golang/go/issues/30354#issuecomment-466479708 10 | replace ( 11 | github.com/Azure/go-autorest => github.com/Azure/go-autorest v12.2.0+incompatible 12 | github.com/Microsoft/go-winio => github.com/bi-zone/go-winio v0.4.15 13 | github.com/Shopify/sarama => github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec 14 | github.com/cucumber/godog => github.com/cucumber/godog v0.8.1 15 | github.com/docker/docker => github.com/docker/engine v0.0.0-20191113042239-ea84732a7725 16 | github.com/docker/go-plugins-helpers => github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f 17 | github.com/dop251/goja => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 18 | github.com/dop251/goja_nodejs => github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 19 | github.com/fsnotify/fsevents => github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 20 | github.com/fsnotify/fsnotify => github.com/adriansr/fsnotify v0.0.0-20180417234312-c9bbe1f46f1d 21 | github.com/google/gopacket => github.com/adriansr/gopacket v1.1.18-0.20200327165309-dd62abfa8a41 22 | github.com/insomniacslk/dhcp => github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 // indirect 23 | github.com/kardianos/service => github.com/blakerouse/service v1.1.1-0.20200924160513-057808572ffa 24 | github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c 25 | golang.org/x/tools => golang.org/x/tools v0.0.0-20200602230032-c00d67ef29d0 // release 1.14 26 | ) 27 | -------------------------------------------------------------------------------- /http/client.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | "net/url" 11 | "time" 12 | 13 | "github.com/elastic/beats/v7/libbeat/beat" 14 | "github.com/elastic/beats/v7/libbeat/outputs" 15 | "github.com/elastic/beats/v7/libbeat/outputs/outil" 16 | "github.com/elastic/beats/v7/libbeat/publisher" 17 | "github.com/elastic/elastic-agent-libs/mapstr" 18 | "github.com/elastic/elastic-agent-libs/transport" 19 | "github.com/elastic/elastic-agent-libs/transport/tlscommon" 20 | ) 21 | 22 | // Client struct 23 | type Client struct { 24 | Connection 25 | tlsConfig *tlscommon.TLSConfig 26 | params map[string]string 27 | // additional configs 28 | compressionLevel int 29 | proxyURL *url.URL 30 | batchPublish bool 31 | observer outputs.Observer 32 | headers map[string]string 33 | format string 34 | } 35 | 36 | // ClientSettings struct 37 | type ClientSettings struct { 38 | URL string 39 | Proxy *url.URL 40 | TLS *tlscommon.TLSConfig 41 | Username, Password string 42 | Parameters map[string]string 43 | Index outil.Selector 44 | Pipeline *outil.Selector 45 | Timeout time.Duration 46 | CompressionLevel int 47 | Observer outputs.Observer 48 | BatchPublish bool 49 | Headers map[string]string 50 | ContentType string 51 | Format string 52 | } 53 | 54 | // Connection struct 55 | type Connection struct { 56 | URL string 57 | Username string 58 | Password string 59 | http *http.Client 60 | connected bool 61 | encoder bodyEncoder 62 | ContentType string 63 | } 64 | 65 | type eventRaw map[string]json.RawMessage 66 | 67 | type event struct { 68 | Timestamp time.Time `json:"@timestamp"` 69 | Fields mapstr.M `json:"-"` 70 | } 71 | 72 | // NewClient instantiate a client. 73 | func NewClient(s ClientSettings) (*Client, error) { 74 | proxy := http.ProxyFromEnvironment 75 | if s.Proxy != nil { 76 | proxy = http.ProxyURL(s.Proxy) 77 | } 78 | logger.Info("HTTP URL: %s", s.URL) 79 | var dialer, tlsDialer transport.Dialer 80 | var err error 81 | 82 | dialer = transport.NetDialer(s.Timeout) 83 | tlsDialer = transport.TLSDialer(dialer, s.TLS, s.Timeout) 84 | 85 | if st := s.Observer; st != nil { 86 | dialer = transport.StatsDialer(dialer, st) 87 | tlsDialer = transport.StatsDialer(tlsDialer, st) 88 | } 89 | params := s.Parameters 90 | var encoder bodyEncoder 91 | compression := s.CompressionLevel 92 | if compression == 0 { 93 | switch s.Format { 94 | case "json": 95 | encoder = newJSONEncoder(nil) 96 | case "json_lines": 97 | encoder = newJSONLinesEncoder(nil) 98 | } 99 | } else { 100 | switch s.Format { 101 | case "json": 102 | encoder, err = newGzipEncoder(compression, nil) 103 | case "json_lines": 104 | encoder, err = newGzipLinesEncoder(compression, nil) 105 | } 106 | if err != nil { 107 | return nil, err 108 | } 109 | } 110 | client := &Client{ 111 | Connection: Connection{ 112 | URL: s.URL, 113 | Username: s.Username, 114 | Password: s.Password, 115 | ContentType: s.ContentType, 116 | http: &http.Client{ 117 | Transport: &http.Transport{ 118 | Dial: dialer.Dial, 119 | DialTLS: tlsDialer.Dial, 120 | Proxy: proxy, 121 | }, 122 | Timeout: s.Timeout, 123 | }, 124 | encoder: encoder, 125 | }, 126 | params: params, 127 | compressionLevel: compression, 128 | proxyURL: s.Proxy, 129 | batchPublish: s.BatchPublish, 130 | headers: s.Headers, 131 | format: s.Format, 132 | } 133 | 134 | return client, nil 135 | } 136 | 137 | // Clone clones a client. 138 | func (client *Client) Clone() *Client { 139 | // when cloning the connection callback and params are not copied. A 140 | // client's close is for example generated for topology-map support. With params 141 | // most likely containing the ingest node pipeline and default callback trying to 142 | // create install a template, we don't want these to be included in the clone. 143 | c, _ := NewClient( 144 | ClientSettings{ 145 | URL: client.URL, 146 | Proxy: client.proxyURL, 147 | TLS: client.tlsConfig, 148 | Username: client.Username, 149 | Password: client.Password, 150 | Parameters: client.params, 151 | Timeout: client.http.Timeout, 152 | CompressionLevel: client.compressionLevel, 153 | BatchPublish: client.batchPublish, 154 | Headers: client.headers, 155 | ContentType: client.ContentType, 156 | Format: client.format, 157 | }, 158 | ) 159 | return c 160 | } 161 | 162 | // Connect establishes a connection to the clients sink. 163 | func (conn *Connection) Connect() error { 164 | conn.connected = true 165 | return nil 166 | } 167 | 168 | // Close closes a connection. 169 | func (conn *Connection) Close() error { 170 | conn.connected = false 171 | return nil 172 | } 173 | 174 | func (client *Client) String() string { 175 | return client.URL 176 | } 177 | 178 | // Publish sends events to the clients sink. 179 | func (client *Client) Publish(_ context.Context, batch publisher.Batch) error { 180 | events := batch.Events() 181 | rest, err := client.publishEvents(events) 182 | if len(rest) == 0 { 183 | batch.ACK() 184 | } else { 185 | batch.RetryEvents(rest) 186 | } 187 | return err 188 | } 189 | 190 | // PublishEvents posts all events to the http endpoint. On error a slice with all 191 | // events not published will be returned. 192 | func (client *Client) publishEvents(data []publisher.Event) ([]publisher.Event, error) { 193 | begin := time.Now() 194 | if len(data) == 0 { 195 | return nil, nil 196 | } 197 | if !client.connected { 198 | return data, ErrNotConnected 199 | } 200 | var failedEvents []publisher.Event 201 | sendErr := error(nil) 202 | if client.batchPublish { 203 | // Publish events in bulk 204 | logger.Debugf("Publishing events in batch.") 205 | sendErr = client.BatchPublishEvent(data) 206 | if sendErr != nil { 207 | return data, sendErr 208 | } 209 | } else { 210 | logger.Debugf("Publishing events one by one.") 211 | for index, event := range data { 212 | sendErr = client.PublishEvent(event) 213 | if sendErr != nil { 214 | // return the rest of the data with the error 215 | failedEvents = data[index:] 216 | break 217 | } 218 | } 219 | } 220 | logger.Debugf("PublishEvents: %d metrics have been published over HTTP in %v.", len(data), time.Now().Sub(begin)) 221 | if len(failedEvents) > 0 { 222 | return failedEvents, sendErr 223 | } 224 | return nil, nil 225 | } 226 | 227 | // BatchPublishEvent publish a single event to output. 228 | func (client *Client) BatchPublishEvent(data []publisher.Event) error { 229 | if !client.connected { 230 | return ErrNotConnected 231 | } 232 | var events = make([]eventRaw, len(data)) 233 | for i, event := range data { 234 | events[i] = makeEvent(&event.Content) 235 | } 236 | status, _, err := client.request("POST", client.params, events, client.headers) 237 | if err != nil { 238 | logger.Warn("Fail to insert a single event: %s", err) 239 | if err == ErrJSONEncodeFailed { 240 | // don't retry unencodable values 241 | return nil 242 | } 243 | } 244 | switch { 245 | case status == 500 || status == 400: //server error or bad input, don't retry 246 | return nil 247 | case status >= 300: 248 | // retry 249 | return err 250 | } 251 | return nil 252 | } 253 | 254 | // PublishEvent publish a single event to output. 255 | func (client *Client) PublishEvent(data publisher.Event) error { 256 | if !client.connected { 257 | return ErrNotConnected 258 | } 259 | event := data 260 | logger.Debugf("Publish event: %s", event) 261 | status, _, err := client.request("POST", client.params, makeEvent(&event.Content), client.headers) 262 | if err != nil { 263 | logger.Warn("Fail to insert a single event: %s", err) 264 | if err == ErrJSONEncodeFailed { 265 | // don't retry unencodable values 266 | return nil 267 | } 268 | } 269 | switch { 270 | case status == 500 || status == 400: //server error or bad input, don't retry 271 | return nil 272 | case status >= 300: 273 | // retry 274 | return err 275 | } 276 | if !client.connected { 277 | return ErrNotConnected 278 | } 279 | return nil 280 | } 281 | 282 | func (conn *Connection) request(method string, params map[string]string, body interface{}, headers map[string]string) (int, []byte, error) { 283 | urlStr := addToURL(conn.URL, params) 284 | logger.Debugf("%s %s %v", method, urlStr, body) 285 | 286 | if body == nil { 287 | return conn.execRequest(method, urlStr, nil, headers) 288 | } 289 | 290 | if err := conn.encoder.Marshal(body); err != nil { 291 | logger.Warn("Failed to json encode body (%v): %#v", err, body) 292 | return 0, nil, ErrJSONEncodeFailed 293 | } 294 | return conn.execRequest(method, urlStr, conn.encoder.Reader(), headers) 295 | } 296 | 297 | func (conn *Connection) execRequest(method, url string, body io.Reader, headers map[string]string) (int, []byte, error) { 298 | req, err := http.NewRequest(method, url, body) 299 | if err != nil { 300 | logger.Warn("Failed to create request: %v", err) 301 | return 0, nil, err 302 | } 303 | if body != nil { 304 | conn.encoder.AddHeader(&req.Header, conn.ContentType) 305 | } 306 | return conn.execHTTPRequest(req, headers) 307 | } 308 | 309 | func (conn *Connection) execHTTPRequest(req *http.Request, headers map[string]string) (int, []byte, error) { 310 | req.Header.Add("Accept", "application/json") 311 | for key, value := range headers { 312 | req.Header.Add(key, value) 313 | } 314 | if conn.Username != "" || conn.Password != "" { 315 | req.SetBasicAuth(conn.Username, conn.Password) 316 | } 317 | resp, err := conn.http.Do(req) 318 | if err != nil { 319 | conn.connected = false 320 | return 0, nil, err 321 | } 322 | defer closing(resp.Body) 323 | 324 | status := resp.StatusCode 325 | if status >= 300 { 326 | conn.connected = false 327 | return status, nil, fmt.Errorf("%v", resp.Status) 328 | } 329 | obj, err := ioutil.ReadAll(resp.Body) 330 | if err != nil { 331 | conn.connected = false 332 | return status, nil, err 333 | } 334 | return status, obj, nil 335 | } 336 | 337 | func closing(c io.Closer) { 338 | err := c.Close() 339 | if err != nil { 340 | logger.Warn("Close failed with: %v", err) 341 | } 342 | } 343 | 344 | // this should ideally be in enc.go 345 | func makeEvent(v *beat.Event) map[string]json.RawMessage { 346 | // Inline not supported, 347 | // HT: https://stackoverflow.com/questions/49901287/embed-mapstringstring-in-go-json-marshaling-without-extra-json-property-inlin 348 | type event0 event // prevent recursion 349 | e := event{Timestamp: v.Timestamp.UTC(), Fields: v.Fields} 350 | b, err := json.Marshal(event0(e)) 351 | if err != nil { 352 | logger.Warn("Error encoding event to JSON: %v", err) 353 | } 354 | 355 | var eventMap map[string]json.RawMessage 356 | err = json.Unmarshal(b, &eventMap) 357 | if err != nil { 358 | logger.Warn("Error decoding JSON to map: %v", err) 359 | } 360 | // Add the individual fields to the map, flatten "Fields" 361 | for j, k := range e.Fields { 362 | b, err = json.Marshal(k) 363 | if err != nil { 364 | logger.Warn("Error encoding map to JSON: %v", err) 365 | } 366 | eventMap[j] = b 367 | } 368 | return eventMap 369 | } 370 | -------------------------------------------------------------------------------- /http/config.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/elastic/elastic-agent-libs/transport/tlscommon" 8 | ) 9 | 10 | type httpConfig struct { 11 | Protocol string `config:"protocol"` 12 | Path string `config:"path"` 13 | Params map[string]string `config:"parameters"` 14 | Username string `config:"username"` 15 | Password string `config:"password"` 16 | ProxyURL string `config:"proxy_url"` 17 | LoadBalance bool `config:"loadbalance"` 18 | BatchPublish bool `config:"batch_publish"` 19 | BatchSize int `config:"batch_size"` 20 | CompressionLevel int `config:"compression_level" validate:"min=0, max=9"` 21 | TLS *tlscommon.Config `config:"tls"` 22 | MaxRetries int `config:"max_retries"` 23 | Timeout time.Duration `config:"timeout"` 24 | Headers map[string]string `config:"headers"` 25 | ContentType string `config:"content_type"` 26 | Backoff backoff `config:"backoff"` 27 | Format string `config:"format"` 28 | } 29 | 30 | type backoff struct { 31 | Init time.Duration 32 | Max time.Duration 33 | } 34 | 35 | var ( 36 | defaultConfig = httpConfig{ 37 | Protocol: "", 38 | Path: "", 39 | Params: nil, 40 | ProxyURL: "", 41 | Username: "", 42 | Password: "", 43 | BatchPublish: false, 44 | BatchSize: 2048, 45 | Timeout: 90 * time.Second, 46 | CompressionLevel: 0, 47 | TLS: nil, 48 | MaxRetries: 3, 49 | LoadBalance: false, 50 | Backoff: backoff{ 51 | Init: 1 * time.Second, 52 | Max: 60 * time.Second, 53 | }, 54 | Format: "json", 55 | } 56 | ) 57 | 58 | func (c *httpConfig) Validate() error { 59 | if c.ProxyURL != "" { 60 | if _, err := parseProxyURL(c.ProxyURL); err != nil { 61 | return err 62 | } 63 | } 64 | if c.Format != "json" && c.Format != "json_lines" { 65 | return fmt.Errorf("Unsupported config option format: %s", c.Format) 66 | } 67 | 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /http/enc.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "encoding/json" 7 | "io" 8 | "net/http" 9 | "reflect" 10 | ) 11 | 12 | type bodyEncoder interface { 13 | bulkBodyEncoder 14 | Reader() io.Reader 15 | Marshal(doc interface{}) error 16 | } 17 | 18 | type bulkBodyEncoder interface { 19 | bulkWriter 20 | 21 | AddHeader(*http.Header, string) 22 | Reset() 23 | } 24 | 25 | type bulkWriter interface { 26 | Add(meta, obj interface{}) error 27 | AddRaw(raw interface{}) error 28 | } 29 | 30 | type jsonEncoder struct { 31 | buf *bytes.Buffer 32 | } 33 | 34 | type jsonLinesEncoder struct { 35 | buf *bytes.Buffer 36 | } 37 | 38 | type gzipEncoder struct { 39 | buf *bytes.Buffer 40 | gzip *gzip.Writer 41 | } 42 | 43 | type gzipLinesEncoder struct { 44 | buf *bytes.Buffer 45 | gzip *gzip.Writer 46 | } 47 | 48 | func newJSONEncoder(buf *bytes.Buffer) *jsonEncoder { 49 | if buf == nil { 50 | buf = bytes.NewBuffer(nil) 51 | } 52 | return &jsonEncoder{buf} 53 | } 54 | 55 | func (b *jsonEncoder) Reset() { 56 | b.buf.Reset() 57 | } 58 | 59 | func (b *jsonEncoder) AddHeader(header *http.Header, contentType string) { 60 | if (contentType == "") { 61 | header.Add("Content-Type", "application/json; charset=UTF-8") 62 | } else { 63 | header.Add("Content-Type", contentType) 64 | } 65 | } 66 | 67 | func (b *jsonEncoder) Reader() io.Reader { 68 | return b.buf 69 | } 70 | 71 | func (b *jsonEncoder) Marshal(obj interface{}) error { 72 | b.Reset() 73 | enc := json.NewEncoder(b.buf) 74 | return enc.Encode(obj) 75 | } 76 | 77 | func (b *jsonEncoder) AddRaw(raw interface{}) error { 78 | enc := json.NewEncoder(b.buf) 79 | return enc.Encode(raw) 80 | } 81 | 82 | func (b *jsonEncoder) Add(meta, obj interface{}) error { 83 | enc := json.NewEncoder(b.buf) 84 | pos := b.buf.Len() 85 | 86 | if err := enc.Encode(meta); err != nil { 87 | b.buf.Truncate(pos) 88 | return err 89 | } 90 | if err := enc.Encode(obj); err != nil { 91 | b.buf.Truncate(pos) 92 | return err 93 | } 94 | return nil 95 | } 96 | 97 | func newJSONLinesEncoder(buf *bytes.Buffer) *jsonLinesEncoder { 98 | if buf == nil { 99 | buf = bytes.NewBuffer(nil) 100 | } 101 | return &jsonLinesEncoder{buf} 102 | } 103 | 104 | func (b *jsonLinesEncoder) Reset() { 105 | b.buf.Reset() 106 | } 107 | 108 | func (b *jsonLinesEncoder) AddHeader(header *http.Header, contentType string) { 109 | if (contentType == "") { 110 | header.Add("Content-Type", "application/x-ndjson; charset=UTF-8") 111 | } else { 112 | header.Add("Content-Type", contentType) 113 | } 114 | } 115 | 116 | func (b *jsonLinesEncoder) Reader() io.Reader { 117 | return b.buf 118 | } 119 | 120 | func (b *jsonLinesEncoder) Marshal(obj interface{}) error { 121 | b.Reset() 122 | return b.AddRaw(obj) 123 | } 124 | 125 | func (b *jsonLinesEncoder) AddRaw(obj interface{}) error { 126 | enc := json.NewEncoder(b.buf) 127 | 128 | // single event 129 | if reflect.TypeOf(obj).Kind() == reflect.Map { 130 | return enc.Encode(obj) 131 | } 132 | 133 | // batch of events 134 | for _, item := range obj.([]eventRaw) { 135 | err := enc.Encode(item) 136 | if err != nil { 137 | return err 138 | } 139 | } 140 | return nil 141 | } 142 | 143 | func (b *jsonLinesEncoder) Add(meta, obj interface{}) error { 144 | pos := b.buf.Len() 145 | 146 | if err := b.AddRaw(meta); err != nil { 147 | b.buf.Truncate(pos) 148 | return err 149 | } 150 | if err := b.AddRaw(obj); err != nil { 151 | b.buf.Truncate(pos) 152 | return err 153 | } 154 | 155 | return nil 156 | } 157 | 158 | func newGzipEncoder(level int, buf *bytes.Buffer) (*gzipEncoder, error) { 159 | if buf == nil { 160 | buf = bytes.NewBuffer(nil) 161 | } 162 | w, err := gzip.NewWriterLevel(buf, level) 163 | if err != nil { 164 | return nil, err 165 | } 166 | 167 | return &gzipEncoder{buf, w}, nil 168 | } 169 | 170 | func (b *gzipEncoder) Reset() { 171 | b.buf.Reset() 172 | b.gzip.Reset(b.buf) 173 | } 174 | 175 | func (b *gzipEncoder) Reader() io.Reader { 176 | b.gzip.Close() 177 | return b.buf 178 | } 179 | 180 | func (b *gzipEncoder) AddHeader(header *http.Header, contentType string) { 181 | if (contentType == "") { 182 | header.Add("Content-Type", "application/json; charset=UTF-8") 183 | } else { 184 | header.Add("Content-Type", contentType) 185 | } 186 | header.Add("Content-Encoding", "gzip") 187 | } 188 | 189 | func (b *gzipEncoder) Marshal(obj interface{}) error { 190 | b.Reset() 191 | enc := json.NewEncoder(b.gzip) 192 | err := enc.Encode(obj) 193 | return err 194 | } 195 | 196 | func (b *gzipEncoder) AddRaw(raw interface{}) error { 197 | enc := json.NewEncoder(b.gzip) 198 | return enc.Encode(raw) 199 | } 200 | 201 | func (b *gzipEncoder) Add(meta, obj interface{}) error { 202 | enc := json.NewEncoder(b.gzip) 203 | pos := b.buf.Len() 204 | 205 | if err := enc.Encode(meta); err != nil { 206 | b.buf.Truncate(pos) 207 | return err 208 | } 209 | if err := enc.Encode(obj); err != nil { 210 | b.buf.Truncate(pos) 211 | return err 212 | } 213 | 214 | b.gzip.Flush() 215 | return nil 216 | } 217 | 218 | func newGzipLinesEncoder(level int, buf *bytes.Buffer) (*gzipLinesEncoder, error) { 219 | if buf == nil { 220 | buf = bytes.NewBuffer(nil) 221 | } 222 | w, err := gzip.NewWriterLevel(buf, level) 223 | if err != nil { 224 | return nil, err 225 | } 226 | 227 | return &gzipLinesEncoder{buf, w}, nil 228 | } 229 | 230 | func (b *gzipLinesEncoder) Reset() { 231 | b.buf.Reset() 232 | b.gzip.Reset(b.buf) 233 | } 234 | 235 | func (b *gzipLinesEncoder) Reader() io.Reader { 236 | b.gzip.Close() 237 | return b.buf 238 | } 239 | 240 | func (b *gzipLinesEncoder) AddHeader(header *http.Header, contentType string) { 241 | if (contentType == "") { 242 | header.Add("Content-Type", "application/x-ndjson; charset=UTF-8") 243 | } else { 244 | header.Add("Content-Type", contentType) 245 | } 246 | header.Add("Content-Encoding", "gzip") 247 | } 248 | 249 | func (b *gzipLinesEncoder) Marshal(obj interface{}) error { 250 | b.Reset() 251 | return b.AddRaw(obj) 252 | } 253 | 254 | func (b *gzipLinesEncoder) AddRaw(obj interface{}) error { 255 | enc := json.NewEncoder(b.gzip) 256 | 257 | // single event 258 | if reflect.TypeOf(obj).Kind() == reflect.Map { 259 | return enc.Encode(obj) 260 | } 261 | 262 | // batch of events 263 | for _, item := range obj.([]eventRaw) { 264 | err := enc.Encode(item) 265 | if err != nil { 266 | return err 267 | } 268 | } 269 | return nil 270 | } 271 | 272 | func (b *gzipLinesEncoder) Add(meta, obj interface{}) error { 273 | pos := b.buf.Len() 274 | 275 | if err := b.AddRaw(meta); err != nil { 276 | b.buf.Truncate(pos) 277 | return err 278 | } 279 | if err := b.AddRaw(obj); err != nil { 280 | b.buf.Truncate(pos) 281 | return err 282 | } 283 | 284 | b.gzip.Flush() 285 | return nil 286 | } 287 | -------------------------------------------------------------------------------- /http/http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/elastic/beats/v7/libbeat/beat" 7 | "github.com/elastic/beats/v7/libbeat/common" 8 | "github.com/elastic/beats/v7/libbeat/outputs" 9 | conf "github.com/elastic/elastic-agent-libs/config" 10 | "github.com/elastic/elastic-agent-libs/logp" 11 | "github.com/elastic/elastic-agent-libs/transport/tlscommon" 12 | ) 13 | 14 | func init() { 15 | outputs.RegisterType("http", MakeHTTP) 16 | } 17 | 18 | var ( 19 | logger = logp.NewLogger("output.http") 20 | // ErrNotConnected indicates failure due to client having no valid connection 21 | ErrNotConnected = errors.New("not connected") 22 | // ErrJSONEncodeFailed indicates encoding failures 23 | ErrJSONEncodeFailed = errors.New("json encode failed") 24 | ) 25 | 26 | func MakeHTTP( 27 | _ outputs.IndexManager, 28 | beat beat.Info, 29 | observer outputs.Observer, 30 | cfg *conf.C, 31 | ) (outputs.Group, error) { 32 | config := defaultConfig 33 | if err := cfg.Unpack(&config); err != nil { 34 | return outputs.Fail(err) 35 | } 36 | tlsConfig, err := tlscommon.LoadTLSConfig(config.TLS) 37 | if err != nil { 38 | return outputs.Fail(err) 39 | } 40 | hosts, err := outputs.ReadHostList(cfg) 41 | if err != nil { 42 | return outputs.Fail(err) 43 | } 44 | proxyURL, err := parseProxyURL(config.ProxyURL) 45 | if err != nil { 46 | return outputs.Fail(err) 47 | } 48 | if proxyURL != nil { 49 | logger.Info("Using proxy URL: %s", proxyURL) 50 | } 51 | params := config.Params 52 | if len(params) == 0 { 53 | params = nil 54 | } 55 | clients := make([]outputs.NetworkClient, len(hosts)) 56 | for i, host := range hosts { 57 | logger.Info("Making client for host: " + host) 58 | hostURL, err := common.MakeURL(config.Protocol, config.Path, host, 80) 59 | if err != nil { 60 | logger.Error("Invalid host param set: %s, Error: %v", host, err) 61 | return outputs.Fail(err) 62 | } 63 | logger.Info("Final host URL: " + hostURL) 64 | var client outputs.NetworkClient 65 | client, err = NewClient(ClientSettings{ 66 | URL: hostURL, 67 | Proxy: proxyURL, 68 | TLS: tlsConfig, 69 | Username: config.Username, 70 | Password: config.Password, 71 | Parameters: params, 72 | Timeout: config.Timeout, 73 | CompressionLevel: config.CompressionLevel, 74 | Observer: observer, 75 | BatchPublish: config.BatchPublish, 76 | Headers: config.Headers, 77 | ContentType: config.ContentType, 78 | Format: config.Format, 79 | }) 80 | 81 | if err != nil { 82 | return outputs.Fail(err) 83 | } 84 | client = outputs.WithBackoff(client, config.Backoff.Init, config.Backoff.Max) 85 | clients[i] = client 86 | } 87 | return outputs.SuccessNet(config.LoadBalance, config.BatchSize, config.MaxRetries, clients) 88 | } 89 | -------------------------------------------------------------------------------- /http/url.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "github.com/elastic/beats/v7/libbeat/common" 5 | "net/url" 6 | "strings" 7 | ) 8 | 9 | func addToURL(urlStr string, params map[string]string) string { 10 | if strings.HasSuffix(urlStr, "/") { 11 | urlStr = strings.TrimSuffix(urlStr, "/") 12 | } 13 | if len(params) == 0 { 14 | return urlStr 15 | } 16 | values := url.Values{} 17 | for key, val := range params { 18 | values.Add(key, val) 19 | } 20 | return common.EncodeURLParams(urlStr, values) 21 | } 22 | 23 | func parseProxyURL(raw string) (*url.URL, error) { 24 | if raw == "" { 25 | return nil, nil 26 | } 27 | parsedUrl, err := url.Parse(raw) 28 | if err == nil && strings.HasPrefix(parsedUrl.Scheme, "http") { 29 | return parsedUrl, err 30 | } 31 | // Proxy was bogus. Try prepending "http://" to it and 32 | // see if that parses correctly. 33 | return url.Parse("http://" + raw) 34 | } 35 | --------------------------------------------------------------------------------