├── testsupport ├── config_test_defaults2.toml ├── config_panic.toml ├── config_bad_test.toml ├── config_bad_outputs.toml ├── config_test_defaults.toml ├── config_bad_params.toml ├── config_test_common.toml ├── config_test.toml ├── http.go ├── test-zeus-incomplete.log ├── sample-config.toml ├── test-severity.log ├── mock_net_error.go ├── mock_net_listener.go ├── support.go ├── mock_amqp_acknowledger.go └── mock_net_conn.go ├── .gitignore ├── sandbox ├── lua │ ├── testsupport │ │ ├── simple_count.lua.data │ │ ├── serialize_noglobal.lua │ │ ├── serialize_failure.lua │ │ ├── hello_world.lua │ │ ├── readint.lua │ │ ├── readfield.lua │ │ ├── readstring.lua │ │ ├── loop.lua │ │ ├── backpressure.lua │ │ ├── simple_count.lua │ │ ├── timerinject.lua │ │ ├── processinject.lua │ │ ├── nagios.lua │ │ ├── circular_buffer.lua.data │ │ ├── hekabench_cbuf_counter.lua │ │ ├── auditlog.lua │ │ ├── counter.lua │ │ ├── sync_heavyhitters.lua │ │ ├── errors.lua │ │ ├── serialize.lua │ │ ├── read_message.lua │ │ ├── inject_message.lua │ │ ├── mysql_slow_query.lua │ │ ├── circular_buffer_errors.lua │ │ ├── circular_buffer.lua │ │ └── serialize.lua.data │ ├── lua_circular_buffer.h │ └── lua_sandbox.h ├── sandbox.h └── sandbox.go ├── .gitmodules ├── docs └── source │ ├── message │ └── index.rst │ ├── man │ ├── usage.rst │ ├── plugin.rst │ └── config.rst │ ├── index.rst │ ├── developing │ ├── release.rst │ └── testing.rst │ ├── sandbox │ ├── index.rst │ └── filter.rst │ ├── message_matcher.rst │ ├── monitoring │ └── index.rst │ ├── glossary.rst │ └── installing.rst ├── cmd ├── sbmgr │ ├── sbmgr.toml │ └── main.go ├── sbmgrload │ ├── sbmgrload.toml │ └── main.go ├── hekad │ ├── config_test.go │ ├── config.go │ └── main.go └── flood │ └── flood.toml ├── examples ├── example.toml ├── udp_output.go └── host_filter.go ├── LICENSE.txt ├── pipeline ├── pipeline_signals_darwin.go ├── pipeline_signals_linux.go ├── pipeline_signals_windows.go ├── mock_decoder_test.go ├── mock_stataccumulator_test.go ├── mock_input_test.go ├── doc.go ├── dashboard_output_test.go ├── mock_whisperrunner_test.go ├── mock_decoderset_test.go ├── mock_amqpconnectionhub_test.go ├── mock_amqpconnection_test.go ├── all_specs_test.go ├── mock_pluginrunner_test.go ├── jsonpath.go ├── jsonpath_test.go ├── payload_decoder_helper.go ├── statsd_input_test.go ├── mock_pluginhelper_test.go ├── nagios_output.go ├── carbon.go ├── http_input.go ├── carbon_test.go ├── mock_decoderrunner_test.go ├── stat_filter.go ├── mock_inputrunner_test.go ├── mock_outputrunner_test.go ├── statsd_input.go ├── mock_filterrunner_test.go ├── payloadjson_decoder.go ├── payloadregex_decoder.go └── whisper_test.go ├── README.md ├── client ├── senders.go ├── client.go └── encoders.go ├── message ├── message.proto └── date_helpers.go └── update_mocks.sh /testsupport/config_test_defaults2.toml: -------------------------------------------------------------------------------- 1 | [DefaultsTestOutput] 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | cmd/hekad/plugin_loader.go 2 | docs/build 3 | sandbox/lua/lua_sandbox.go 4 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/simple_count.lua.data: -------------------------------------------------------------------------------- 1 | _G["_VERSION"] = "Lua 5.1" 2 | _G["count"] = 10 3 | -------------------------------------------------------------------------------- /testsupport/config_panic.toml: -------------------------------------------------------------------------------- 1 | [PanicOutput] 2 | type = "PanicOutput" 3 | message_matcher = "Type == \"heka.counter_output\"" 4 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "docs/source/_themes/mozilla"] 2 | path = docs/source/_themes/mozilla 3 | url = https://github.com/ametaireau/mozilla-sphinx-theme.git 4 | -------------------------------------------------------------------------------- /docs/source/message/index.rst: -------------------------------------------------------------------------------- 1 | .. _message: 2 | 3 | ============ 4 | Heka Message 5 | ============ 6 | 7 | .. toctree:: 8 | message_matcher 9 | :maxdepth: 2 10 | -------------------------------------------------------------------------------- /testsupport/config_bad_test.toml: -------------------------------------------------------------------------------- 1 | [udp_stats] 2 | type = "UdpInput" 3 | address = 1000 4 | 5 | [CounterOutput] 6 | type = "CounterOutput" 7 | message_matcher = "Type != \"heka.counter_output\"" 8 | -------------------------------------------------------------------------------- /cmd/sbmgr/sbmgr.toml: -------------------------------------------------------------------------------- 1 | ip_address = "127.0.0.1:5565" 2 | [signer] 3 | name = "test" 4 | hmac_hash = "md5" 5 | hmac_key = "4865ey9urgkidls xtb0[7lf9rzcivthkm" 6 | version = 0 7 | -------------------------------------------------------------------------------- /cmd/sbmgrload/sbmgrload.toml: -------------------------------------------------------------------------------- 1 | ip_address = "127.0.0.1:5565" 2 | [signer] 3 | name = "test" 4 | hmac_hash = "md5" 5 | hmac_key = "4865ey9urgkidls xtb0[7lf9rzcivthkm" 6 | version = 0 7 | -------------------------------------------------------------------------------- /testsupport/config_bad_outputs.toml: -------------------------------------------------------------------------------- 1 | [udp_stats] 2 | type = "UdpInput" 3 | address = "127.0.0.1:29330" 4 | 5 | [JsonDecoder] 6 | type ="JsonDecoder" 7 | encoding_name = "JSON" 8 | 9 | [WhatOutput] 10 | type ="WhatOutput" 11 | -------------------------------------------------------------------------------- /testsupport/config_test_defaults.toml: -------------------------------------------------------------------------------- 1 | [UdpInput] 2 | address = "127.0.0.1:29329" 3 | 4 | [LogOutput] 5 | 6 | [StatFilter] 7 | 8 | [sample] 9 | type = "StatFilter" 10 | message_matcher = "Type == 'counter' || Type == 'gauge'" 11 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/serialize_noglobal.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | _G = nil 6 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/serialize_failure.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | t = { [function(x) print(x) end] = "foo" } 6 | -------------------------------------------------------------------------------- /testsupport/config_bad_params.toml: -------------------------------------------------------------------------------- 1 | [UdpInput1] 2 | address = "127.0.0.1:29329" 3 | 4 | [JsonDecoder1] 5 | type ="JsonDecoder" 6 | encoding_name = "JSON" 7 | 8 | [ProtobufDecoder1] 9 | type = "ProtobufDecoder" 10 | encoding_name = "PROTOCOL_BUFFER" 11 | 12 | [LogOutput1] 13 | type = "LogOutput" 14 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/hello_world.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | output("Hello") 6 | output(" World!") 7 | inject_message() 8 | -------------------------------------------------------------------------------- /testsupport/config_test_common.toml: -------------------------------------------------------------------------------- 1 | [UdpInput] 2 | address = "127.0.0.1:5565" 3 | 4 | [UdpInput.retries] 5 | max_delay = "30s" 6 | delay = "250ms" 7 | max_retries = 5 8 | 9 | [CounterFilter] 10 | message_matcher = "Type != 'heka.counter-output'" 11 | message_signer = "some_signer" 12 | ticker_interval = 10 13 | 14 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/readint.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | function process_message () 6 | i = read_message("Pid") 7 | return 0 8 | end 9 | 10 | function timer_event() 11 | end 12 | 13 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/readfield.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | function process_message () 6 | s = read_message("Fields[foo]") 7 | return 0 8 | end 9 | 10 | function timer_event() 11 | end 12 | 13 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/readstring.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | function process_message () 6 | s = read_message("Type") 7 | return 0 8 | end 9 | 10 | 11 | function timer_event() 12 | end 13 | 14 | 15 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/loop.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | function process_message () 6 | output("loop to self") 7 | inject_message() 8 | return 0 9 | end 10 | 11 | 12 | function timer_event(ns) 13 | end 14 | 15 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/backpressure.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | function process_message () 6 | for i=1,200000 do 7 | read_message("Payload") 8 | end 9 | return 0 10 | end 11 | 12 | function timer_event(ns) 13 | end 14 | 15 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/simple_count.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | count = 0 6 | 7 | function process_message () 8 | count = count + 1 9 | output(count) 10 | inject_message() 11 | return 0 12 | end 13 | 14 | function timer_event(ns) 15 | end 16 | 17 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/timerinject.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | 6 | function process_message () 7 | return 0 8 | end 9 | 10 | 11 | function timer_event(ns) 12 | for i = 1, 11 do 13 | output("test") 14 | inject_message() 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/processinject.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | 6 | function process_message () 7 | for i = 1, 2 do 8 | output("test") 9 | inject_message() 10 | end 11 | return 0 12 | end 13 | 14 | 15 | function timer_event(ns) 16 | end 17 | 18 | -------------------------------------------------------------------------------- /examples/example.toml: -------------------------------------------------------------------------------- 1 | [TcpInput] 2 | address = "127.0.0.1:5565" 3 | 4 | [HostFilter] 5 | message_matcher = "Type != 'heka.counter-output'" 6 | hosts = ["127.0.0.1", "grail"] 7 | output = "UdpOutput" 8 | 9 | [CounterFilter] 10 | message_matcher = "Type != 'heka.counter-output'" 11 | ticker_interval = 1 12 | 13 | [UdpOutput] 14 | address = "127.0.0.1:6676" 15 | 16 | [LogOutput] 17 | message_matcher = "Type == 'heka.counter-output' || Type == 'heka.all-report'" 18 | payload_only = true 19 | 20 | -------------------------------------------------------------------------------- /testsupport/config_test.toml: -------------------------------------------------------------------------------- 1 | [UdpInput] 2 | address = "127.0.0.1:29329" 3 | 4 | [JsonDecoder] 5 | type ="JsonDecoder" 6 | encoding_name = "JSON" 7 | 8 | [ProtobufDecoder] 9 | type = "ProtobufDecoder" 10 | encoding_name = "PROTOCOL_BUFFER" 11 | 12 | [LogOutput] 13 | type = "LogOutput" 14 | message_matcher = "TRUE" 15 | 16 | [default] 17 | type ="StatFilter" 18 | message_matcher = "Type == \"mytype\"" 19 | 20 | [sample] 21 | type = "StatFilter" 22 | message_matcher = "Type == \"counter\" || Type == \"gauge\"" 23 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | # 5 | # The Initial Developer of the Original Code is the Mozilla Foundation. 6 | # Portions created by the Initial Developer are Copyright (C) 2012 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Ben Bangert (bbangert@mozilla.com) 11 | # Mike Trinkala (mtrinkala@mozilla.com) 12 | # Rob Miller (rmiller@mozilla.com) 13 | # Victor Ng (vng@mozilla.com) 14 | -------------------------------------------------------------------------------- /docs/source/man/usage.rst: -------------------------------------------------------------------------------- 1 | .. _hekad_cli: 2 | 3 | ===== 4 | hekad 5 | ===== 6 | 7 | Synopsis 8 | ======== 9 | 10 | hekad [``-version``] [``-config`` `config_file`] 11 | 12 | Description 13 | =========== 14 | 15 | .. include:: /index.rst 16 | :start-after: start-description 17 | :end-before: end-description 18 | 19 | Options 20 | ======= 21 | 22 | .. include:: /configuration.rst 23 | :start-after: start-options 24 | :end-before: end-options 25 | 26 | Files 27 | ===== 28 | 29 | /etc/hekad.toml configuration file 30 | 31 | See Also 32 | ======== 33 | 34 | hekad.config(5), hekad.plugin(5) 35 | -------------------------------------------------------------------------------- /docs/source/man/plugin.rst: -------------------------------------------------------------------------------- 1 | ===== 2 | hekad 3 | ===== 4 | 5 | Description 6 | =========== 7 | 8 | Available hekad plugins compiled with this version of hekad. 9 | 10 | .. include:: /configuration.rst 11 | :start-after: start-inputs 12 | :end-before: end-inputs 13 | 14 | .. include:: /configuration.rst 15 | :start-after: start-decoders 16 | :end-before: end-decoders 17 | 18 | .. include:: /configuration.rst 19 | :start-after: start-filters 20 | :end-before: end-filters 21 | 22 | .. include:: /configuration.rst 23 | :start-after: start-outputs 24 | :end-before: end-outputs 25 | 26 | See Also 27 | ======== 28 | 29 | hekad(1), hekad.config(5) -------------------------------------------------------------------------------- /pipeline/pipeline_signals_darwin.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Victor Ng (vng@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | package pipeline 16 | 17 | import "syscall" 18 | 19 | const SIGUSR1 = syscall.SIGUSR1 20 | -------------------------------------------------------------------------------- /pipeline/pipeline_signals_linux.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Victor Ng (vng@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | package pipeline 16 | 17 | import "syscall" 18 | 19 | const SIGUSR1 = syscall.SIGUSR1 20 | -------------------------------------------------------------------------------- /cmd/hekad/config_test.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Victor Ng (vng@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | package main 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func TestDecode(t *testing.T) { 22 | _, err := LoadHekadConfig("../../testsupport/sample-config.toml") 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | ===== 2 | hekad 3 | ===== 4 | 5 | .. start-description 6 | 7 | The hekad daemon is the core component of the heka project, which 8 | handles routing messages, generating metrics, aggregating statsd-type 9 | messages, running plugins on the messages, and sending messages to the 10 | configured destinations. 11 | 12 | .. end-description 13 | 14 | .. seealso:: 15 | `heka project`_ 16 | 17 | Contents: 18 | 19 | .. toctree:: 20 | :maxdepth: 2 21 | 22 | installing 23 | configuration 24 | monitoring/index 25 | developing/plugin 26 | message_matcher.rst 27 | sandbox/index 28 | developing/testing 29 | 30 | 31 | 32 | Indices and tables 33 | ================== 34 | 35 | * :ref:`search` 36 | * :ref:`glossary` 37 | 38 | .. _heka project: http://heka-docs.readthedocs.org 39 | -------------------------------------------------------------------------------- /testsupport/http.go: -------------------------------------------------------------------------------- 1 | package testsupport 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "net/http" 7 | ) 8 | 9 | type OneHttpServer struct { 10 | response_data string 11 | listener net.Listener 12 | } 13 | 14 | func NewOneHttpServer(response_data string, hostname string, port int) (server *OneHttpServer, e error) { 15 | l, e := net.Listen("tcp", fmt.Sprintf("%s:%d", hostname, port)) 16 | if e != nil { 17 | return nil, e 18 | } 19 | server = &OneHttpServer{response_data: response_data, listener: l} 20 | return server, nil 21 | } 22 | 23 | func (o *OneHttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { 24 | fmt.Fprintf(w, o.response_data) 25 | o.listener.Close() 26 | } 27 | 28 | func (o *OneHttpServer) Start(urlpath string) { 29 | http.Handle(urlpath, o) 30 | http.Serve(o.listener, nil) 31 | } 32 | -------------------------------------------------------------------------------- /pipeline/pipeline_signals_windows.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Victor Ng (vng@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | package pipeline 16 | 17 | import ( 18 | "syscall" 19 | ) 20 | 21 | // Define it since it is not defined for Windows. 22 | 23 | // Note that you will need to manually send signal 10 to hekad as 24 | // SIGUSR1 isn't defined on Windows. 25 | 26 | const SIGUSR1 = syscall.Signal(0xa) 27 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/nagios.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | local state = 0 6 | 7 | function process_message () 8 | return 0 9 | end 10 | 11 | 12 | function timer_event(ns) 13 | 14 | if state == 0 then 15 | output("OK:Ok alerts are working!") 16 | elseif state == 1 then 17 | output("WARNING:Warning alerts are working!") 18 | elseif state == 2 then 19 | output("CRITICAL:Critical alerts are working!") 20 | elseif state == 3 then 21 | output("UNKNOWN:Unknown alerts are working!") 22 | end 23 | state = state + 1 24 | if state == 4 then state = 0 end 25 | 26 | inject_message("nagios-external-command", "PROCESS_SERVICE_CHECK_RESULT") 27 | end 28 | 29 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/circular_buffer.lua.data: -------------------------------------------------------------------------------- 1 | _G["_VERSION"] = "Lua 5.1" 2 | _G["cbufs"] = {} 3 | if _G["cbufs"][1] == nil then _G["cbufs"][1] = circular_buffer.new(2, 1, 1) end 4 | _G["cbufs"][1]:set_header(1, "Header_1", "count") 5 | _G["cbufs"][1]:fromstring("1 1 7.1 1e6") 6 | if _G["cbufs"][2] == nil then _G["cbufs"][2] = circular_buffer.new(2, 1, 1) end 7 | _G["cbufs"][2]:set_header(1, "Header_1", "count") 8 | _G["cbufs"][2]:fromstring("1 1 0 0") 9 | if _G["cbufs"][3] == nil then _G["cbufs"][3] = circular_buffer.new(2, 1, 1) end 10 | _G["cbufs"][3]:set_header(1, "Header_1", "count") 11 | _G["cbufs"][3]:fromstring("1 1 0 0") 12 | if _G["data"] == nil then _G["data"] = circular_buffer.new(3, 3, 1) end 13 | _G["data"]:set_header(1, "Add_column", "count") 14 | _G["data"]:set_header(2, "Set_column", "count") 15 | _G["data"]:set_header(3, "Get_column", "count") 16 | _G["data"]:fromstring("10 1 0 0 0 3 1 3 0 0 0") 17 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/hekabench_cbuf_counter.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | last_inject = 0 6 | count = 0 7 | cnts = circular_buffer.new(3600, 1, 1) -- 1 hour window with 1 second resolution 8 | local MESSAGES = cnts:set_header(1, "Messages", "count") 9 | 10 | function process_message () 11 | -- normally we would use the message time stamp but flood messages have a fixed time 12 | count = count + 1 13 | return 0 14 | end 15 | 16 | 17 | function timer_event(ns) 18 | cnts:add(ns, MESSAGES, count) 19 | count = 0 20 | if ns - last_inject > 60e9 then -- write the aggregate once a minute 21 | output(cnts) 22 | inject_message("cbuf") 23 | last_inject = ns 24 | end 25 | end 26 | 27 | -------------------------------------------------------------------------------- /sandbox/sandbox.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 | /* vim: set ts=2 et sw=2 tw=80: */ 3 | /* This Source Code Form is subject to the terms of the Mozilla Public 4 | * License, v. 2.0. If a copy of the MPL was not distributed with this 5 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 | 7 | /// Common sandbox configuration for Heka plugins @file 8 | #ifndef sandbox_h_ 9 | #define sandbox_h_ 10 | 11 | typedef enum { 12 | STATUS_UNKNOWN = 0, 13 | STATUS_RUNNING = 1, 14 | STATUS_TERMINATED = 2 15 | } sandbox_status; 16 | 17 | typedef enum { 18 | USAGE_STAT_LIMIT = 0, 19 | USAGE_STAT_CURRENT = 1, 20 | USAGE_STAT_MAXIMUM = 2, 21 | 22 | MAX_USAGE_STAT 23 | } sandbox_usage_stat; 24 | 25 | typedef enum { 26 | USAGE_TYPE_MEMORY = 0, 27 | USAGE_TYPE_INSTRUCTION = 1, 28 | USAGE_TYPE_OUTPUT = 2, 29 | 30 | MAX_USAGE_TYPE 31 | } sandbox_usage_type; 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /docs/source/man/config.rst: -------------------------------------------------------------------------------- 1 | ===== 2 | hekad 3 | ===== 4 | 5 | Description 6 | =========== 7 | 8 | .. include:: /configuration.rst 9 | :start-after: start-hekad-config 10 | :end-before: end-hekad-config 11 | 12 | Full documentation on available plugins and settings for each one are 13 | in the hekad.plugin(5) pages. 14 | 15 | Example hekad.toml File 16 | ======================= 17 | 18 | .. include:: /configuration.rst 19 | :start-after: start-hekad-toml 20 | :end-before: end-hekad-toml 21 | 22 | Roles 23 | ===== 24 | 25 | `hekad` is frequently configured for various roles in a larger cluster: 26 | 27 | .. include:: /configuration.rst 28 | :start-after: start-roles 29 | :end-before: end-roles 30 | 31 | A single `hekad` daemon could act as all the roles in smaller 32 | deployments. 33 | 34 | .. include:: /configuration.rst 35 | :start-after: start-restarting 36 | :end-before: end-restarting 37 | 38 | 39 | See Also 40 | ======== 41 | 42 | hekad(1), hekad.plugin(5) 43 | -------------------------------------------------------------------------------- /docs/source/developing/release.rst: -------------------------------------------------------------------------------- 1 | .. _release: 2 | 3 | ==================== 4 | Heka release process 5 | ==================== 6 | 7 | This document contains a description of the steps taken to make a release 8 | of the Heka server. 9 | 10 | #. Review intended release branch for correct version number (in 11 | `cmd/hekad/main.go`) and updated changelog (`CHANGES.txt`) and verify that 12 | the build succeeds and all tests pass. 13 | 14 | #. Tag verified commit on intended release branch with appropriate version 15 | tag. 16 | 17 | #. If this release is the highest released version number to date, the 18 | verified commit should be merged into the master branch. 19 | 20 | #. Bump version number (in `cmd/hekad/main.go`) and add section for future 21 | release to changelog (`CHANGES.txt`). Commit "version bump" revision to 22 | the released version branch and push. 23 | 24 | #. Build all required binary packages and upload to 25 | https://docs.services.mozilla.com/_static/binaries. 26 | -------------------------------------------------------------------------------- /pipeline/mock_decoder_test.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: github.com/mozilla-services/heka/pipeline (interfaces: Decoder) 3 | 4 | package pipeline 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | ) 9 | 10 | // Mock of Decoder interface 11 | type MockDecoder struct { 12 | ctrl *gomock.Controller 13 | recorder *_MockDecoderRecorder 14 | } 15 | 16 | // Recorder for MockDecoder (not exported) 17 | type _MockDecoderRecorder struct { 18 | mock *MockDecoder 19 | } 20 | 21 | func NewMockDecoder(ctrl *gomock.Controller) *MockDecoder { 22 | mock := &MockDecoder{ctrl: ctrl} 23 | mock.recorder = &_MockDecoderRecorder{mock} 24 | return mock 25 | } 26 | 27 | func (_m *MockDecoder) EXPECT() *_MockDecoderRecorder { 28 | return _m.recorder 29 | } 30 | 31 | func (_m *MockDecoder) Decode(_param0 *PipelinePack) error { 32 | ret := _m.ctrl.Call(_m, "Decode", _param0) 33 | ret0, _ := ret[0].(error) 34 | return ret0 35 | } 36 | 37 | func (_mr *_MockDecoderRecorder) Decode(arg0 interface{}) *gomock.Call { 38 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Decode", arg0) 39 | } 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Heka 2 | 3 | Data Acquisition and Processing Made Easy 4 | 5 | Heka is a tool for collecting and collating data from a number of different 6 | sources, performing "in-flight" processing of collected data, and delivering 7 | the results to any number of destinations for further analysis. 8 | 9 | More details can be found on the ReadTheDocs sites for the [Heka 10 | project](http://heka-docs.readthedocs.org/) and for the [Heka 11 | daemon](http://hekad.readthedocs.org/), and lower level package 12 | documentation can be found on 13 | [GoDoc](http://godoc.org/github.com/mozilla-services/heka). Questions can 14 | be asked on the [mailing list](https://mail.mozilla.org/listinfo/heka) or 15 | in the #heka channel on irc.mozilla.org. 16 | 17 | Heka is written in [Go](http://golang.org/), but Heka plugins can be written 18 | in either Go or [Lua](http://lua.org). The easiest way to install Heka is 19 | using [Heka-Build](https://github.com/mozilla-services/heka-build), which 20 | will build the correct version of Go, set up a Go environment, and install 21 | all required dependencies. Heka-Build also provides a mechanism for easily 22 | integrating external plug-in packages into the generated `hekad`. 23 | -------------------------------------------------------------------------------- /pipeline/mock_stataccumulator_test.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: github.com/mozilla-services/heka/pipeline (interfaces: StatAccumulator) 3 | 4 | package pipeline 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | ) 9 | 10 | // Mock of StatAccumulator interface 11 | type MockStatAccumulator struct { 12 | ctrl *gomock.Controller 13 | recorder *_MockStatAccumulatorRecorder 14 | } 15 | 16 | // Recorder for MockStatAccumulator (not exported) 17 | type _MockStatAccumulatorRecorder struct { 18 | mock *MockStatAccumulator 19 | } 20 | 21 | func NewMockStatAccumulator(ctrl *gomock.Controller) *MockStatAccumulator { 22 | mock := &MockStatAccumulator{ctrl: ctrl} 23 | mock.recorder = &_MockStatAccumulatorRecorder{mock} 24 | return mock 25 | } 26 | 27 | func (_m *MockStatAccumulator) EXPECT() *_MockStatAccumulatorRecorder { 28 | return _m.recorder 29 | } 30 | 31 | func (_m *MockStatAccumulator) DropStat(_param0 Stat) bool { 32 | ret := _m.ctrl.Call(_m, "DropStat", _param0) 33 | ret0, _ := ret[0].(bool) 34 | return ret0 35 | } 36 | 37 | func (_mr *_MockStatAccumulatorRecorder) DropStat(arg0 interface{}) *gomock.Call { 38 | return _mr.mock.ctrl.RecordCall(_mr.mock, "DropStat", arg0) 39 | } 40 | -------------------------------------------------------------------------------- /docs/source/sandbox/index.rst: -------------------------------------------------------------------------------- 1 | .. _sandbox: 2 | 3 | ================= 4 | Sandboxed Filters 5 | ================= 6 | 7 | Sandboxed filters are Heka filter plugins that are implemented in a sandboxed 8 | scripting language. They provide a dynamic and isolated execution environment 9 | for data analysis, and allow real time access to data in production without 10 | jeopardizing the integrity or performance of the monitoring infrastructure. 11 | This broadens the audience that the data can be exposed to and facilitates new 12 | uses of the data (i.e. debugging, monitoring, dynamic provisioning, SLA 13 | analysis, intrusion detection, ad-hoc reporting, etc.) 14 | 15 | Features 16 | ======== 17 | 18 | - dynamic loading - ability to start/stop on a self-service basis 19 | - small - memory requirements are about 16 KiB for a basic sandbox 20 | - fast - microsecond execution times 21 | - stateful - ability to resume where it left off after a restart/reboot 22 | - isolated - failures are contained and malfunctioning sandboxes are terminated 23 | 24 | Sandbox Manager 25 | =============== 26 | :ref:`sandboxmanager` 27 | 28 | Sandbox Filter 29 | ============== 30 | :ref:`sandboxfilter` 31 | 32 | Sandbox Types 33 | ============= 34 | 35 | .. toctree:: 36 | lua 37 | :maxdepth: 2 38 | 39 | -------------------------------------------------------------------------------- /client/senders.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Rob Miller (rmiller@mozilla.com) 12 | # Mike Trinkala (trink@mozilla.com) 13 | # 14 | # ***** END LICENSE BLOCK *****/ 15 | 16 | package client 17 | 18 | import ( 19 | "net" 20 | ) 21 | 22 | type Sender interface { 23 | SendMessage(outBytes []byte) (err error) 24 | Close() 25 | } 26 | 27 | type NetworkSender struct { 28 | connection net.Conn 29 | } 30 | 31 | func NewNetworkSender(proto, addr string) (self *NetworkSender, err error) { 32 | conn, err := net.Dial(proto, addr) 33 | if err == nil { 34 | self = &(NetworkSender{conn}) 35 | } 36 | return 37 | } 38 | 39 | func (self *NetworkSender) SendMessage(outBytes []byte) (err error) { 40 | _, err = self.connection.Write(outBytes) 41 | return 42 | } 43 | 44 | func (self *NetworkSender) Close() { 45 | self.connection.Close() 46 | } 47 | -------------------------------------------------------------------------------- /testsupport/test-zeus-incomplete.log: -------------------------------------------------------------------------------- 1 | 10.1.1.2 plinko-31.byzantium.mozilla.com user1 [15/Mar/2013:12:20:27 -0700] "GET /1.1/user1/storage/passwords?full=1&newer=1355973234.140 HTTP/1.1" 200 233 "-" "Firefox AndroidSync 1.20.0.1.0 (Firefox)" "-" "ssl: SSL_RSA_WITH_RC4_128_SHA, version=TLSv1, bits=128" node_s:0.080575 req_s:0.080575 retries:0 req_b:327 "c_l:-" 2 | 10.1.1.3 plinko-1270.byzantium.mozilla.com user2 [15/Mar/2013:12:20:27 -0700] "POST /1.1/user2/storage/tabs HTTP/1.1" 200 280 "-" "Firefox/20.0.1 FxSync/1.22.0.201304.desktop" "-" "ssl: SSL_RSA_WITH_RC4_128_SHA, version=TLSv1, bits=128" node_s:0.020249 req_s:0.020249 retries:0 req_b:1034 "c_l:528" 3 | 10.1.1.4 plinko-565.byzantium.mozilla.com user3 [15/Mar/2013:12:20:27 -0700] "POST /1.1/user3/storage/clients HTTP/1.1" 200 280 "-" "Firefox/20.0.1 FxSync/1.22.0.201304.desktop" "-" "ssl: SSL_RSA_WITH_RC4_128_SHA, version=TLSv1, bits=128" node_s:0.157255 req_s:0.157255 retries:0 req_b:781 "c_l:273" 4 | 10.1.1.4 plinko-565.byzantium.mozilla.com user3 [15/Mar/2013:12:20:27 -0700] "GET /1.1/user3/storage/passwords?newer=1356237662.44&full=1 HTTP/1.1" 200 1396 "-" "Firefox/20.0.1 FxSync/1.22.0.201304.desktop" "-" "ssl: SSL_RSA_WITH_RC4_128_SHA, version=TLSv1, bits=128" node_s:0.047167 req_s:0.047167 retries:0 req_b:446 "c_l:-" 5 | 10.1.1.4 plinko-565.byzantium.mozilla.com user3 -------------------------------------------------------------------------------- /sandbox/lua/testsupport/auditlog.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | local types = { 6 | UNKNOWN = 1, 7 | USER_AUTH = 2, 8 | USER_ACCT = 3, 9 | LOGIN = 4, 10 | USER_START = 5, 11 | USER_END = 6, 12 | USER_LOGIN = 7, 13 | USER_LOGOUT = 8, 14 | CRED_ACQ = 9, 15 | CRED_REF = 10, 16 | CRYPTO_KEY_USER = 11, 17 | CRYPTO_SESSION = 12, 18 | ADD_USER = 13, 19 | ADD_GROUP = 14 20 | } 21 | 22 | data = circular_buffer.new(720, 14, 60) 23 | for k,v in pairs(types) do 24 | data:set_header(v,k) 25 | end 26 | 27 | function process_message () 28 | local ts = read_message("Fields[AuditTime]") 29 | if ts == nil then return 0 end 30 | ts = tonumber(ts) * 1e9 31 | 32 | local type = read_message("Fields[AuditType]") 33 | local id = types[type] 34 | if id == nil then 35 | id = types.UNKNOWN 36 | end 37 | data:add(ts, id, 1) 38 | return 0 39 | end 40 | 41 | function timer_event(ns) 42 | output(data) 43 | inject_message("cbuf", "Counts by type") 44 | end 45 | 46 | -------------------------------------------------------------------------------- /pipeline/mock_input_test.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: github.com/mozilla-services/heka/pipeline (interfaces: Input) 3 | 4 | package pipeline 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | ) 9 | 10 | // Mock of Input interface 11 | type MockInput struct { 12 | ctrl *gomock.Controller 13 | recorder *_MockInputRecorder 14 | } 15 | 16 | // Recorder for MockInput (not exported) 17 | type _MockInputRecorder struct { 18 | mock *MockInput 19 | } 20 | 21 | func NewMockInput(ctrl *gomock.Controller) *MockInput { 22 | mock := &MockInput{ctrl: ctrl} 23 | mock.recorder = &_MockInputRecorder{mock} 24 | return mock 25 | } 26 | 27 | func (_m *MockInput) EXPECT() *_MockInputRecorder { 28 | return _m.recorder 29 | } 30 | 31 | func (_m *MockInput) Run(_param0 InputRunner, _param1 PluginHelper) error { 32 | ret := _m.ctrl.Call(_m, "Run", _param0, _param1) 33 | ret0, _ := ret[0].(error) 34 | return ret0 35 | } 36 | 37 | func (_mr *_MockInputRecorder) Run(arg0, arg1 interface{}) *gomock.Call { 38 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Run", arg0, arg1) 39 | } 40 | 41 | func (_m *MockInput) Stop() { 42 | _m.ctrl.Call(_m, "Stop") 43 | } 44 | 45 | func (_mr *_MockInputRecorder) Stop() *gomock.Call { 46 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Stop") 47 | } 48 | -------------------------------------------------------------------------------- /client/client.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Rob Miller (rmiller@mozilla.com) 12 | # Mike Trinkala (trink@mozilla.com) 13 | # 14 | # ***** END LICENSE BLOCK *****/ 15 | 16 | /* 17 | 18 | Client package to talk to heka from Go. 19 | 20 | */ 21 | package client 22 | 23 | import ( 24 | "github.com/mozilla-services/heka/message" 25 | ) 26 | 27 | type Client struct { 28 | Sender Sender 29 | Encoder Encoder 30 | buf []byte 31 | } 32 | 33 | func NewClient(sender Sender, encoder Encoder) (self *Client) { 34 | buf := make([]byte, message.MAX_MESSAGE_SIZE+message.MAX_HEADER_SIZE+3) 35 | self = &Client{sender, encoder, buf} 36 | return 37 | } 38 | 39 | func (self *Client) SendMessage(msg *message.Message) (err error) { 40 | err = self.Encoder.EncodeMessageStream(msg, &self.buf) 41 | if err == nil { 42 | err = self.Sender.SendMessage(self.buf) 43 | } 44 | return 45 | } 46 | -------------------------------------------------------------------------------- /testsupport/sample-config.toml: -------------------------------------------------------------------------------- 1 | [hekad] 2 | cpuprof = "/var/log/hekad/cpuprofile.log" 3 | decoder_poolsize = 10 4 | max_message_loops = 4 5 | max_process_inject = 10 6 | max_timer_inject = 10 7 | maxprocs = 10 8 | memprof = "/var/log/hekad/memprof.log" 9 | plugin_chansize = 10 10 | poolsize = 100 11 | 12 | [StatsdInput] 13 | address = "127.0.0.1:8125" 14 | ticker_interval = 1 15 | 16 | [TcpInput] 17 | address = "127.0.0.1:5565" 18 | 19 | [TcpInput2] 20 | type = "TcpInput" 21 | address = "127.0.0.1:5566" 22 | 23 | [CounterFilter] 24 | message_matcher = "Type != 'heka.counter-output'" 25 | ticker_interval = 1 26 | 27 | [StatsdFileOutput] 28 | type = "FileOutput" 29 | path = "log/statsdonly.log" 30 | format = "json" 31 | 32 | [CounterFileOutput] 33 | type = "FileOutput" 34 | path = "log/counter.log" 35 | message_matcher = "Type == 'heka.counter-output'" 36 | 37 | [LogOutput] 38 | message_matcher = "Type == 'heka.counter-output'" 39 | payload_only = true 40 | 41 | [WhisperOutput] 42 | basepath = "/var/run/hekad/whisper/db" 43 | message_matcher = "Type == 'statmetric'" 44 | defaultaggmethod = 1 45 | defaultarchiveinfo = [ [0, 30, 1440], [0, 900, 192], [0, 3600, 168], [0, 43200, 1456] ] 46 | 47 | [DashboardOutput] 48 | message_matcher = "Type == 'heka.all-report' || Type == 'heka.sandbox-output' || Type == 'heka.sandbox-terminated'" 49 | ticker_interval = 5 50 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/counter.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | lastTime = os.time() * 1e9 6 | lastCount = 0 7 | count = 0 8 | rate = 0.0 9 | rates = {} 10 | 11 | function process_message () 12 | count = count + 1 13 | return 0 14 | end 15 | 16 | 17 | function timer_event(ns) 18 | local msgsSent = count - lastCount 19 | if msgsSent == 0 then return end 20 | 21 | local elapsedTime = ns - lastTime 22 | if elapsedTime == 0 then return end 23 | 24 | lastCount = count 25 | lastTime = ns 26 | rate = msgsSent / (elapsedTime / 1e9) 27 | rates[#rates+1] = rate 28 | output(string.format("Got %d messages. %0.2f msg/sec", count, rate)) 29 | inject_message() 30 | 31 | local samples = #rates 32 | if samples == 10 then -- generate a summary every 10 samples 33 | table.sort(rates) 34 | local min = rates[1] 35 | local max = rates[samples] 36 | local sum = 0 37 | for i, val in ipairs(rates) do 38 | sum = sum + val 39 | end 40 | output(string.format("AGG Sum. Min: %0.2f Max: %0.2f Mean: %0.2f", min, max, sum/samples)) 41 | inject_message() 42 | rates = {} 43 | end 44 | end 45 | 46 | -------------------------------------------------------------------------------- /pipeline/doc.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Ben Bangert (bbangert@mozilla.com) 12 | # Rob Miller (rmiller@mozilla.com) 13 | # 14 | # ***** END LICENSE BLOCK *****/ 15 | 16 | /* 17 | 18 | The `pipeline` package implements the `hekad` daemon's main message processing 19 | pipeline. It defines interface specifications for plugins in general, and for 20 | all of the various specific plugin types (i.e. Input, Decoder, Filter, and 21 | Output), as well as implementations of a number of plugins of each type. It 22 | also defines and provides a variety of interfaces through which plugins can 23 | interact with the Heka environment. 24 | 25 | For more information about Heka please see the following resources: 26 | 27 | * Heka project documentation: http://heka-docs.readthedocs.org/ 28 | * `hekad` daemon documenation: http://hekad.readthedocs.org/ 29 | * `hekad` github project: https://github.com/mozilla-services/heka 30 | 31 | */ 32 | package pipeline 33 | -------------------------------------------------------------------------------- /testsupport/test-severity.log: -------------------------------------------------------------------------------- 1 | severity: info 2 | severity: emergency 3 | severity: debug 4 | severity: notice 5 | severity: critical 6 | severity: error 7 | severity: warning 8 | severity: alert 9 | severity: critical 10 | severity: alert 11 | severity: info 12 | severity: debug 13 | severity: emergency 14 | severity: error 15 | severity: notice 16 | severity: warning 17 | severity: info 18 | severity: emergency 19 | severity: debug 20 | severity: notice 21 | severity: critical 22 | severity: error 23 | severity: warning 24 | severity: alert 25 | severity: critical 26 | severity: alert 27 | severity: info 28 | severity: debug 29 | severity: emergency 30 | severity: error 31 | severity: notice 32 | severity: warning 33 | severity: info 34 | severity: BOGUS 35 | severity: emergency 36 | severity: debug 37 | severity: notice 38 | severity: critical 39 | severity: error 40 | severity: warning 41 | severity: alert 42 | severity: critical 43 | severity: alert 44 | severity: info 45 | severity: debug 46 | severity: emergency 47 | severity: error 48 | severity: notice 49 | severity: warning 50 | severity: info 51 | severity: emergency 52 | severity: debug 53 | severity: notice 54 | severity: critical 55 | severity: error 56 | severity: warning 57 | severity: alert 58 | severity: critical 59 | severity: alert 60 | severity: info 61 | severity: debug 62 | severity: emergency 63 | severity: error 64 | severity: notice 65 | severity: warning -------------------------------------------------------------------------------- /pipeline/dashboard_output_test.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Victor Ng (vng@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | package pipeline 16 | 17 | import ( 18 | "code.google.com/p/gomock/gomock" 19 | ts "github.com/mozilla-services/heka/testsupport" 20 | gs "github.com/rafrombrc/gospec/src/gospec" 21 | "os" 22 | "path" 23 | ) 24 | 25 | func DashboardOutputSpec(c gs.Context) { 26 | t := new(ts.SimpleT) 27 | ctrl := gomock.NewController(t) 28 | defer ctrl.Finish() 29 | 30 | c.Specify("A FileOutput", func() { 31 | dashboardOutput := new(DashboardOutput) 32 | 33 | config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig) 34 | c.Specify("Init halts if basedirectory is not writable", func() { 35 | tmpdir := path.Join(os.TempDir(), "tmpdir") 36 | err := os.MkdirAll(tmpdir, 0400) 37 | c.Assume(err, gs.IsNil) 38 | config.WorkingDirectory = tmpdir 39 | err = dashboardOutput.Init(config) 40 | c.Assume(err, gs.Not(gs.IsNil)) 41 | }) 42 | }) 43 | } 44 | -------------------------------------------------------------------------------- /pipeline/mock_whisperrunner_test.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: github.com/mozilla-services/heka/pipeline (interfaces: WhisperRunner) 3 | 4 | package pipeline 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | whisper "github.com/rafrombrc/whisper-go/whisper" 9 | ) 10 | 11 | // Mock of WhisperRunner interface 12 | type MockWhisperRunner struct { 13 | ctrl *gomock.Controller 14 | recorder *_MockWhisperRunnerRecorder 15 | } 16 | 17 | // Recorder for MockWhisperRunner (not exported) 18 | type _MockWhisperRunnerRecorder struct { 19 | mock *MockWhisperRunner 20 | } 21 | 22 | func NewMockWhisperRunner(ctrl *gomock.Controller) *MockWhisperRunner { 23 | mock := &MockWhisperRunner{ctrl: ctrl} 24 | mock.recorder = &_MockWhisperRunnerRecorder{mock} 25 | return mock 26 | } 27 | 28 | func (_m *MockWhisperRunner) EXPECT() *_MockWhisperRunnerRecorder { 29 | return _m.recorder 30 | } 31 | 32 | func (_m *MockWhisperRunner) Close() error { 33 | ret := _m.ctrl.Call(_m, "Close") 34 | ret0, _ := ret[0].(error) 35 | return ret0 36 | } 37 | 38 | func (_mr *_MockWhisperRunnerRecorder) Close() *gomock.Call { 39 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Close") 40 | } 41 | 42 | func (_m *MockWhisperRunner) InChan() chan *whisper.Point { 43 | ret := _m.ctrl.Call(_m, "InChan") 44 | ret0, _ := ret[0].(chan *whisper.Point) 45 | return ret0 46 | } 47 | 48 | func (_mr *_MockWhisperRunnerRecorder) InChan() *gomock.Call { 49 | return _mr.mock.ctrl.RecordCall(_mr.mock, "InChan") 50 | } 51 | -------------------------------------------------------------------------------- /testsupport/mock_net_error.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: net (interfaces: Error) 3 | 4 | package testsupport 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | ) 9 | 10 | // Mock of Error interface 11 | type MockError struct { 12 | ctrl *gomock.Controller 13 | recorder *_MockErrorRecorder 14 | } 15 | 16 | // Recorder for MockError (not exported) 17 | type _MockErrorRecorder struct { 18 | mock *MockError 19 | } 20 | 21 | func NewMockError(ctrl *gomock.Controller) *MockError { 22 | mock := &MockError{ctrl: ctrl} 23 | mock.recorder = &_MockErrorRecorder{mock} 24 | return mock 25 | } 26 | 27 | func (_m *MockError) EXPECT() *_MockErrorRecorder { 28 | return _m.recorder 29 | } 30 | 31 | func (_m *MockError) Error() string { 32 | ret := _m.ctrl.Call(_m, "Error") 33 | ret0, _ := ret[0].(string) 34 | return ret0 35 | } 36 | 37 | func (_mr *_MockErrorRecorder) Error() *gomock.Call { 38 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Error") 39 | } 40 | 41 | func (_m *MockError) Temporary() bool { 42 | ret := _m.ctrl.Call(_m, "Temporary") 43 | ret0, _ := ret[0].(bool) 44 | return ret0 45 | } 46 | 47 | func (_mr *_MockErrorRecorder) Temporary() *gomock.Call { 48 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Temporary") 49 | } 50 | 51 | func (_m *MockError) Timeout() bool { 52 | ret := _m.ctrl.Call(_m, "Timeout") 53 | ret0, _ := ret[0].(bool) 54 | return ret0 55 | } 56 | 57 | func (_mr *_MockErrorRecorder) Timeout() *gomock.Call { 58 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Timeout") 59 | } 60 | -------------------------------------------------------------------------------- /pipeline/mock_decoderset_test.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: github.com/mozilla-services/heka/pipeline (interfaces: DecoderSet) 3 | 4 | package pipeline 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | ) 9 | 10 | // Mock of DecoderSet interface 11 | type MockDecoderSet struct { 12 | ctrl *gomock.Controller 13 | recorder *_MockDecoderSetRecorder 14 | } 15 | 16 | // Recorder for MockDecoderSet (not exported) 17 | type _MockDecoderSetRecorder struct { 18 | mock *MockDecoderSet 19 | } 20 | 21 | func NewMockDecoderSet(ctrl *gomock.Controller) *MockDecoderSet { 22 | mock := &MockDecoderSet{ctrl: ctrl} 23 | mock.recorder = &_MockDecoderSetRecorder{mock} 24 | return mock 25 | } 26 | 27 | func (_m *MockDecoderSet) EXPECT() *_MockDecoderSetRecorder { 28 | return _m.recorder 29 | } 30 | 31 | func (_m *MockDecoderSet) ByEncodings() ([]DecoderRunner, error) { 32 | ret := _m.ctrl.Call(_m, "ByEncodings") 33 | ret0, _ := ret[0].([]DecoderRunner) 34 | ret1, _ := ret[1].(error) 35 | return ret0, ret1 36 | } 37 | 38 | func (_mr *_MockDecoderSetRecorder) ByEncodings() *gomock.Call { 39 | return _mr.mock.ctrl.RecordCall(_mr.mock, "ByEncodings") 40 | } 41 | 42 | func (_m *MockDecoderSet) ByName(_param0 string) (DecoderRunner, bool) { 43 | ret := _m.ctrl.Call(_m, "ByName", _param0) 44 | ret0, _ := ret[0].(DecoderRunner) 45 | ret1, _ := ret[1].(bool) 46 | return ret0, ret1 47 | } 48 | 49 | func (_mr *_MockDecoderSetRecorder) ByName(arg0 interface{}) *gomock.Call { 50 | return _mr.mock.ctrl.RecordCall(_mr.mock, "ByName", arg0) 51 | } 52 | -------------------------------------------------------------------------------- /sandbox/lua/lua_circular_buffer.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 | /* vim: set ts=2 et sw=2 tw=80: */ 3 | /* This Source Code Form is subject to the terms of the Mozilla Public 4 | * License, v. 2.0. If a copy of the MPL was not distributed with this 5 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 | 7 | /// @brief Lua circular buffer - time series data store for Heka plugins @file 8 | #ifndef lua_circular_buffer_h_ 9 | #define lua_circular_buffer_h_ 10 | 11 | #include 12 | #include "lua_sandbox_private.h" 13 | 14 | extern const char* heka_circular_buffer; 15 | extern const char* heka_circular_buffer_table; 16 | typedef struct circular_buffer circular_buffer; 17 | 18 | /** 19 | * Output the circular buffer user data 20 | * 21 | * @param cb Circular buffer userdata object. 22 | * @param output Output stream where the data is written. 23 | * @return Zero on success 24 | * 25 | */ 26 | int output_circular_buffer(circular_buffer* cb, output_data* output); 27 | 28 | /** 29 | * Serialize the circular buffer user data 30 | * 31 | * @param key Lua variable name. 32 | * @param cb Circular buffer userdata object. 33 | * @param output Output stream where the data is written. 34 | * @return Zero on success 35 | * 36 | */ 37 | int serialize_circular_buffer(const char* key, circular_buffer* cb, 38 | output_data* output); 39 | 40 | /** 41 | * Circular buffer library loader 42 | * 43 | * @param lua Lua state 44 | * 45 | */ 46 | void luaopen_circular_buffer(lua_State *lua); 47 | 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/sync_heavyhitters.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | users = {} 6 | users_size = 0 7 | day = math.floor(os.time() / (60 * 60 * 24)) 8 | 9 | local WEIGHT = 1 10 | 11 | function process_message () 12 | local user = read_message("Fields[User]") 13 | if user == nil then return 0 end 14 | 15 | local u = users[user] 16 | if u == nil then 17 | if users_size == 10000 then 18 | for k,v in pairs(users) do 19 | v[WEIGHT] = v[WEIGHT] - 1 20 | if v[WEIGHT] == 0 then 21 | users[k] = nil 22 | users_size = users_size - 1 23 | end 24 | end 25 | else 26 | u = {0} 27 | users[user] = u 28 | users_size = users_size + 1 29 | end 30 | end 31 | if u ~= nil then 32 | u[WEIGHT] = u[WEIGHT] + 1 33 | end 34 | return 0 35 | end 36 | 37 | function timer_event(ns) 38 | output("User\tWeight\n") 39 | for k, v in pairs(users) do 40 | if v[WEIGHT] > 100 then 41 | output(string.format("%s\t%d\n", k, v[WEIGHT])) 42 | end 43 | end 44 | inject_message("tsv", "Weighting by User") 45 | 46 | -- reset the frequent users once a day 47 | local current_day = math.floor(ns / 1e9 / (60 * 60 * 24)) 48 | if current_day ~= day then 49 | day = current_day 50 | users = {} 51 | users_size = 0 52 | end 53 | end 54 | 55 | -------------------------------------------------------------------------------- /testsupport/mock_net_listener.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: net (interfaces: Listener) 3 | 4 | package testsupport 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | net "net" 9 | ) 10 | 11 | // Mock of Listener interface 12 | type MockListener struct { 13 | ctrl *gomock.Controller 14 | recorder *_MockListenerRecorder 15 | } 16 | 17 | // Recorder for MockListener (not exported) 18 | type _MockListenerRecorder struct { 19 | mock *MockListener 20 | } 21 | 22 | func NewMockListener(ctrl *gomock.Controller) *MockListener { 23 | mock := &MockListener{ctrl: ctrl} 24 | mock.recorder = &_MockListenerRecorder{mock} 25 | return mock 26 | } 27 | 28 | func (_m *MockListener) EXPECT() *_MockListenerRecorder { 29 | return _m.recorder 30 | } 31 | 32 | func (_m *MockListener) Accept() (net.Conn, error) { 33 | ret := _m.ctrl.Call(_m, "Accept") 34 | ret0, _ := ret[0].(net.Conn) 35 | ret1, _ := ret[1].(error) 36 | return ret0, ret1 37 | } 38 | 39 | func (_mr *_MockListenerRecorder) Accept() *gomock.Call { 40 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Accept") 41 | } 42 | 43 | func (_m *MockListener) Addr() net.Addr { 44 | ret := _m.ctrl.Call(_m, "Addr") 45 | ret0, _ := ret[0].(net.Addr) 46 | return ret0 47 | } 48 | 49 | func (_mr *_MockListenerRecorder) Addr() *gomock.Call { 50 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Addr") 51 | } 52 | 53 | func (_m *MockListener) Close() error { 54 | ret := _m.ctrl.Call(_m, "Close") 55 | ret0, _ := ret[0].(error) 56 | return ret0 57 | } 58 | 59 | func (_mr *_MockListenerRecorder) Close() *gomock.Call { 60 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Close") 61 | } 62 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/errors.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | data = "" 6 | 7 | function process_message () 8 | local msg = read_message("Payload") 9 | 10 | if msg == "inject_message() incorrect number of args" then 11 | inject_message(1, 2, 3) 12 | elseif msg == "output() no arg" then 13 | output() 14 | elseif msg == "out of memory" then 15 | for i=1,500 do 16 | data = data .. "012345678901234567890123456789010123456789012345678901234567890123456789012345678901234567890123456789" 17 | end 18 | elseif msg == "out of instructions" then 19 | while true do 20 | end 21 | elseif msg == "operation on a nil" then 22 | x = x + 1 23 | elseif msg == "invalid return" then 24 | return nil 25 | elseif msg == "no return" then 26 | return 27 | elseif msg == "read_message() incorrect number of args" then 28 | read_message("Type", 1, 1, 1) 29 | elseif msg == "read_message() incorrect field name type" then 30 | read_message(nil) 31 | elseif msg == "read_message() negative field index" then 32 | read_message("Type", -1, 0) 33 | elseif msg == "read_message() negative array index" then 34 | read_message("Type", 0, -1) 35 | elseif msg == "output limit exceeded" then 36 | for i=1,15 do 37 | output("012345678901234567890123456789010123456789012345678901234567890123456789012345678901234567890123456789") 38 | end 39 | end 40 | return 0 41 | end 42 | 43 | function timer_event() 44 | end 45 | -------------------------------------------------------------------------------- /message/message.proto: -------------------------------------------------------------------------------- 1 | package message; 2 | 3 | message Header { 4 | enum MessageEncoding { 5 | PROTOCOL_BUFFER = 0; 6 | JSON = 1; 7 | } 8 | enum HmacHashFunction { 9 | MD5 = 0; 10 | SHA1 = 1; 11 | } 12 | required uint32 message_length = 1; // length in bytes 13 | optional MessageEncoding message_encoding = 2 [default = PROTOCOL_BUFFER]; 14 | 15 | optional HmacHashFunction hmac_hash_function = 3 [default = MD5]; 16 | optional string hmac_signer = 4; 17 | optional uint32 hmac_key_version = 5; 18 | optional bytes hmac = 6; 19 | } 20 | 21 | message Field { 22 | enum ValueType { 23 | STRING = 0; 24 | BYTES = 1; 25 | INTEGER = 2; 26 | DOUBLE = 3; 27 | BOOL = 4; 28 | } 29 | required string name = 1; 30 | optional ValueType value_type = 2 [default = STRING]; 31 | optional string representation = 3; 32 | repeated string value_string = 4; 33 | repeated bytes value_bytes = 5; 34 | repeated int64 value_integer = 6 [packed=true]; 35 | repeated double value_double = 7 [packed=true]; 36 | repeated bool value_bool = 8 [packed=true]; 37 | } 38 | 39 | message Message { 40 | required bytes uuid = 1; 41 | required int64 timestamp = 2; // nanoseconds since UNIX epoch 42 | optional string type = 3; 43 | optional string logger = 4; 44 | optional int32 severity = 5; 45 | optional string payload = 6; 46 | optional string env_version = 7; 47 | optional int32 pid = 8; 48 | optional string hostname = 9; 49 | repeated Field fields = 10; 50 | } 51 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/serialize.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | count = 0 6 | rate = 0.12345678 7 | rates = {99.1,98,97, key="val"} 8 | kvp = {a="foo", b="bar", r=rates} 9 | nested = {arg1=1, arg2=2, nested={n1="one",n2="two"}, empty = nil, cb = circular_buffer.new(2,6,1)} 10 | _G["key with spaces"] = "kws" 11 | boolean = true 12 | empty = nil 13 | func = function (s) return s end 14 | uuids = { 15 | {uuid="BD48B609-8922-4E59-A358-C242075CE088", type="test"}, 16 | {uuid="BD48B609-8922-4E59-A358-C242075CE089", type="test1"} 17 | } 18 | 19 | large_key = { 20 | aaaaaaaaaaaaaaaaaaa = {["BD48B609-8922-4E59-A358-C242075CE081"] = 1, 21 | bbbbbbbbbbbbbbbbbbb = {["BD48B609-8922-4E59-A358-C242075CE082"] = 2, 22 | ccccccccccccccccccc = {["BD48B609-8922-4E59-A358-C242075CE083"] = 3, 23 | ddddddddddddddddddd = {["BD48B609-8922-4E59-A358-C242075CE084"] = 4, 24 | eeeeeeeeeeeeeeeeeee = {["BD48B609-8922-4E59-A358-C242075CE085"] = 5, 25 | fffffffffffffffffff = {["BD48B609-8922-4E59-A358-C242075CE086"] = 6, 26 | ggggggggggggggggggg = {["BD48B609-8922-4E59-A358-C242075CE087"] = 7, 27 | hhhhhhhhhhhhhhhhhhh = {["BD48B609-8922-4E59-A358-C242075CE088"] = 8, 28 | iiiiiiiiiiiiiiiiiii = {["BD48B609-8922-4E59-A358-C242075CE089"] = 9,}}}}}}}}} 29 | } 30 | 31 | cyclea = {type="cycle a"} 32 | cycleb = {type="cycle b"} 33 | cyclea["b"] = cycleb 34 | cycleb["a"] = cyclea 35 | 36 | data = circular_buffer.new(3,3,1) 37 | 38 | dataRef = data 39 | 40 | function process_message () 41 | return 0 42 | end 43 | 44 | 45 | function timer_event(ns) 46 | end 47 | 48 | 49 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/read_message.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | data = "" 6 | 7 | function process_message () 8 | local tmp = read_message("Hostname") 9 | if string.len(tmp) == 0 then return 1 end 10 | 11 | local uuid = read_message("Uuid") 12 | if string.len(tmp) == 0 then return 2 end 13 | 14 | if read_message("Payload") ~= "" then return 3 end 15 | if read_message("Logger") ~= "GoSpec" then return 4 end 16 | if read_message("EnvVersion") ~= "0.8" then return 5 end 17 | if read_message("Fields[foo]") ~= "bar" then return 6 end 18 | if read_message("Fields[foo]", 0) ~= "bar" then return 7 end 19 | if read_message("Fields[foo]", 0, 0) ~= "bar" then return 8 end 20 | if read_message("Fields[foo]", 1) ~= "alternate" then return 9 end 21 | if read_message("Fields[foo]", 1, 1) ~= nil then return 10 end 22 | if read_message("Fields[bytes]") ~= "data" then return 11 end 23 | if read_message("Bogus") ~= nil then return 12 end 24 | if read_message("Timestamp") ~= 5123456789 then return 13 end 25 | if read_message("Severity") ~= 6 then return 14 end 26 | if read_message("Pid") == 9283 then return 15 end 27 | if read_message("Fields[bool]") ~= true then return 16 end 28 | if read_message("Fields[int]") ~= 999 then return 17 end 29 | if read_message("Fields[double]") ~= 99.9 then return 18 end 30 | if read_message("Type") ~= "TEST" then return 19 end 31 | 32 | return 0 33 | end 34 | 35 | 36 | function timer_event() 37 | if read_message("Payload") ~= nil then 38 | x = x + 1 -- create a runtime error 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /pipeline/mock_amqpconnectionhub_test.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: github.com/mozilla-services/heka/pipeline (interfaces: AMQPConnectionHub) 3 | 4 | package pipeline 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | sync "sync" 9 | ) 10 | 11 | // Mock of AMQPConnectionHub interface 12 | type MockAMQPConnectionHub struct { 13 | ctrl *gomock.Controller 14 | recorder *_MockAMQPConnectionHubRecorder 15 | } 16 | 17 | // Recorder for MockAMQPConnectionHub (not exported) 18 | type _MockAMQPConnectionHubRecorder struct { 19 | mock *MockAMQPConnectionHub 20 | } 21 | 22 | func NewMockAMQPConnectionHub(ctrl *gomock.Controller) *MockAMQPConnectionHub { 23 | mock := &MockAMQPConnectionHub{ctrl: ctrl} 24 | mock.recorder = &_MockAMQPConnectionHubRecorder{mock} 25 | return mock 26 | } 27 | 28 | func (_m *MockAMQPConnectionHub) EXPECT() *_MockAMQPConnectionHubRecorder { 29 | return _m.recorder 30 | } 31 | 32 | func (_m *MockAMQPConnectionHub) Close(_param0 string, _param1 *sync.WaitGroup) { 33 | _m.ctrl.Call(_m, "Close", _param0, _param1) 34 | } 35 | 36 | func (_mr *_MockAMQPConnectionHubRecorder) Close(arg0, arg1 interface{}) *gomock.Call { 37 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Close", arg0, arg1) 38 | } 39 | 40 | func (_m *MockAMQPConnectionHub) GetChannel(_param0 string) (AMQPChannel, *sync.WaitGroup, *sync.WaitGroup, error) { 41 | ret := _m.ctrl.Call(_m, "GetChannel", _param0) 42 | ret0, _ := ret[0].(AMQPChannel) 43 | ret1, _ := ret[1].(*sync.WaitGroup) 44 | ret2, _ := ret[2].(*sync.WaitGroup) 45 | ret3, _ := ret[3].(error) 46 | return ret0, ret1, ret2, ret3 47 | } 48 | 49 | func (_mr *_MockAMQPConnectionHubRecorder) GetChannel(arg0 interface{}) *gomock.Call { 50 | return _mr.mock.ctrl.RecordCall(_mr.mock, "GetChannel", arg0) 51 | } 52 | -------------------------------------------------------------------------------- /pipeline/mock_amqpconnection_test.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: github.com/mozilla-services/heka/pipeline (interfaces: AMQPConnection) 3 | 4 | package pipeline 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | amqp "github.com/streadway/amqp" 9 | ) 10 | 11 | // Mock of AMQPConnection interface 12 | type MockAMQPConnection struct { 13 | ctrl *gomock.Controller 14 | recorder *_MockAMQPConnectionRecorder 15 | } 16 | 17 | // Recorder for MockAMQPConnection (not exported) 18 | type _MockAMQPConnectionRecorder struct { 19 | mock *MockAMQPConnection 20 | } 21 | 22 | func NewMockAMQPConnection(ctrl *gomock.Controller) *MockAMQPConnection { 23 | mock := &MockAMQPConnection{ctrl: ctrl} 24 | mock.recorder = &_MockAMQPConnectionRecorder{mock} 25 | return mock 26 | } 27 | 28 | func (_m *MockAMQPConnection) EXPECT() *_MockAMQPConnectionRecorder { 29 | return _m.recorder 30 | } 31 | 32 | func (_m *MockAMQPConnection) Channel() (*amqp.Channel, error) { 33 | ret := _m.ctrl.Call(_m, "Channel") 34 | ret0, _ := ret[0].(*amqp.Channel) 35 | ret1, _ := ret[1].(error) 36 | return ret0, ret1 37 | } 38 | 39 | func (_mr *_MockAMQPConnectionRecorder) Channel() *gomock.Call { 40 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Channel") 41 | } 42 | 43 | func (_m *MockAMQPConnection) Close() error { 44 | ret := _m.ctrl.Call(_m, "Close") 45 | ret0, _ := ret[0].(error) 46 | return ret0 47 | } 48 | 49 | func (_mr *_MockAMQPConnectionRecorder) Close() *gomock.Call { 50 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Close") 51 | } 52 | 53 | func (_m *MockAMQPConnection) NotifyClose(_param0 chan *amqp.Error) chan *amqp.Error { 54 | ret := _m.ctrl.Call(_m, "NotifyClose", _param0) 55 | ret0, _ := ret[0].(chan *amqp.Error) 56 | return ret0 57 | } 58 | 59 | func (_mr *_MockAMQPConnectionRecorder) NotifyClose(arg0 interface{}) *gomock.Call { 60 | return _mr.mock.ctrl.RecordCall(_mr.mock, "NotifyClose", arg0) 61 | } 62 | -------------------------------------------------------------------------------- /testsupport/support.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Rob Miller (rmiller@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | /* 16 | 17 | Several support structures for use with gospec to ease test comparisons. 18 | 19 | */ 20 | package testsupport 21 | 22 | import ( 23 | "github.com/rafrombrc/gospec/src/gospec" 24 | "log" 25 | "strings" 26 | "time" 27 | ) 28 | 29 | var ( 30 | PostTimeout = time.Duration(10 * time.Millisecond) 31 | ) 32 | 33 | type SimpleT struct{} 34 | 35 | func (*SimpleT) Errorf(format string, args ...interface{}) { 36 | log.Printf(format, args...) 37 | } 38 | 39 | func (*SimpleT) Fatalf(format string, args ...interface{}) { 40 | log.Fatalf(format, args...) 41 | } 42 | 43 | func StringContains(actual interface{}, criteria interface{}) (match bool, 44 | pos gospec.Message, neg gospec.Message, err error) { 45 | toTest := actual.(string) 46 | critTest := criteria.(string) 47 | match = strings.Contains(toTest, critTest) 48 | pos = gospec.Messagef(toTest, "contains "+critTest) 49 | neg = gospec.Messagef(toTest, "does not contain "+critTest) 50 | return 51 | } 52 | 53 | func StringStartsWith(actual interface{}, criteria interface{}) (match bool, 54 | pos gospec.Message, neg gospec.Message, err error) { 55 | actStr := actual.(string) 56 | critStr := criteria.(string) 57 | match = actStr[:len(critStr)] == critStr 58 | pos = gospec.Messagef(actStr, "starts with %s", critStr) 59 | neg = gospec.Messagef(actStr, "does not start with %s", critStr) 60 | return 61 | } 62 | -------------------------------------------------------------------------------- /testsupport/mock_amqp_acknowledger.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: github.com/streadway/amqp (interfaces: Acknowledger) 3 | 4 | package testsupport 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | ) 9 | 10 | // Mock of Acknowledger interface 11 | type MockAcknowledger struct { 12 | ctrl *gomock.Controller 13 | recorder *_MockAcknowledgerRecorder 14 | } 15 | 16 | // Recorder for MockAcknowledger (not exported) 17 | type _MockAcknowledgerRecorder struct { 18 | mock *MockAcknowledger 19 | } 20 | 21 | func NewMockAcknowledger(ctrl *gomock.Controller) *MockAcknowledger { 22 | mock := &MockAcknowledger{ctrl: ctrl} 23 | mock.recorder = &_MockAcknowledgerRecorder{mock} 24 | return mock 25 | } 26 | 27 | func (_m *MockAcknowledger) EXPECT() *_MockAcknowledgerRecorder { 28 | return _m.recorder 29 | } 30 | 31 | func (_m *MockAcknowledger) Ack(_param0 uint64, _param1 bool) error { 32 | ret := _m.ctrl.Call(_m, "Ack", _param0, _param1) 33 | ret0, _ := ret[0].(error) 34 | return ret0 35 | } 36 | 37 | func (_mr *_MockAcknowledgerRecorder) Ack(arg0, arg1 interface{}) *gomock.Call { 38 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Ack", arg0, arg1) 39 | } 40 | 41 | func (_m *MockAcknowledger) Nack(_param0 uint64, _param1 bool, _param2 bool) error { 42 | ret := _m.ctrl.Call(_m, "Nack", _param0, _param1, _param2) 43 | ret0, _ := ret[0].(error) 44 | return ret0 45 | } 46 | 47 | func (_mr *_MockAcknowledgerRecorder) Nack(arg0, arg1, arg2 interface{}) *gomock.Call { 48 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Nack", arg0, arg1, arg2) 49 | } 50 | 51 | func (_m *MockAcknowledger) Reject(_param0 uint64, _param1 bool) error { 52 | ret := _m.ctrl.Call(_m, "Reject", _param0, _param1) 53 | ret0, _ := ret[0].(error) 54 | return ret0 55 | } 56 | 57 | func (_mr *_MockAcknowledgerRecorder) Reject(arg0, arg1 interface{}) *gomock.Call { 58 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Reject", arg0, arg1) 59 | } 60 | -------------------------------------------------------------------------------- /sandbox/sandbox.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Mike Trinkala (trink@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | package sandbox 16 | 17 | /* 18 | #include "sandbox.h" 19 | */ 20 | import "C" 21 | 22 | import "github.com/mozilla-services/heka/message" 23 | 24 | const ( 25 | STATUS_UNKNOWN = C.STATUS_UNKNOWN 26 | STATUS_RUNNING = C.STATUS_RUNNING 27 | STATUS_TERMINATED = C.STATUS_TERMINATED 28 | 29 | STAT_LIMIT = C.USAGE_STAT_LIMIT 30 | STAT_CURRENT = C.USAGE_STAT_CURRENT 31 | STAT_MAXIMUM = C.USAGE_STAT_MAXIMUM 32 | 33 | TYPE_MEMORY = C.USAGE_TYPE_MEMORY 34 | TYPE_INSTRUCTIONS = C.USAGE_TYPE_INSTRUCTION 35 | TYPE_OUTPUT = C.USAGE_TYPE_OUTPUT 36 | ) 37 | 38 | type Sandbox interface { 39 | // Sandbox control 40 | Init(dataFile string) error 41 | Destroy(dataFile string) error 42 | 43 | // Sandbox state 44 | Status() int 45 | LastError() string 46 | Usage(utype, ustat int) uint 47 | 48 | // Plugin functions 49 | ProcessMessage(msg *message.Message) int 50 | TimerEvent(ns int64) int 51 | 52 | // Go callback 53 | InjectMessage(f func(payload, payload_type, payload_name string) int) 54 | } 55 | 56 | type SandboxConfig struct { 57 | ScriptType string `toml:"script_type"` 58 | ScriptFilename string `toml:"filename"` 59 | PreserveData bool `toml:"preserve_data"` 60 | MemoryLimit uint `toml:"memory_limit"` 61 | InstructionLimit uint `toml:"instruction_limit"` 62 | OutputLimit uint `toml:"output_limit"` 63 | Profile bool 64 | } 65 | -------------------------------------------------------------------------------- /examples/udp_output.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Rob Miller (rmiller@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | package examples 16 | 17 | import ( 18 | "fmt" 19 | "github.com/mozilla-services/heka/pipeline" 20 | "net" 21 | ) 22 | 23 | // This is our plugin struct. 24 | type UdpOutput struct { 25 | conn net.Conn 26 | } 27 | 28 | // This is our plugin's config struct 29 | type UdpOutputConfig struct { 30 | Address string 31 | } 32 | 33 | // Provides pipeline.HasConfigStruct interface. 34 | func (o *UdpOutput) ConfigStruct() interface{} { 35 | return &UdpOutputConfig{"my.example.com:44444"} 36 | } 37 | 38 | // Initialize UDP connection 39 | func (o *UdpOutput) Init(config interface{}) (err error) { 40 | conf := config.(*UdpOutputConfig) // assert we have the right config type 41 | var udpAddr *net.UDPAddr 42 | if udpAddr, err = net.ResolveUDPAddr("udp", conf.Address); err != nil { 43 | return fmt.Errorf("can't resolve %s: %s", conf.Address, 44 | err.Error()) 45 | } 46 | if o.conn, err = net.DialUDP("udp", nil, udpAddr); err != nil { 47 | return fmt.Errorf("error dialing %s: %s", conf.Address, 48 | err.Error()) 49 | } 50 | return 51 | } 52 | 53 | func (o *UdpOutput) Run(runner pipeline.FilterRunner, helper pipeline.PluginHelper) ( 54 | err error) { 55 | 56 | var outgoing string 57 | for pack := range runner.InChan() { 58 | outgoing = fmt.Sprintf("%s\n", pack.Message.GetPayload()) 59 | o.conn.Write([]byte(outgoing)) 60 | pack.Recycle() 61 | } 62 | return 63 | } 64 | 65 | func init() { 66 | pipeline.RegisterPlugin("UdpOutput", func() interface{} { 67 | return new(UdpOutput) 68 | }) 69 | } 70 | -------------------------------------------------------------------------------- /cmd/hekad/config.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Victor Ng (vng@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | // Hekad configuration. 16 | 17 | package main 18 | 19 | import ( 20 | "fmt" 21 | "github.com/bbangert/toml" 22 | ) 23 | 24 | type HekadConfig struct { 25 | Maxprocs int `toml:"maxprocs"` 26 | PoolSize int `toml:"poolsize"` 27 | DecoderPoolSize int `toml:"decoder_poolsize"` 28 | ChanSize int `toml:"plugin_chansize"` 29 | CpuProfName string `toml:"cpuprof"` 30 | MemProfName string `toml:"memprof"` 31 | MaxMsgLoops uint `toml:"max_message_loops"` 32 | MaxMsgProcessInject uint `toml:"max_process_inject"` 33 | MaxMsgTimerInject uint `toml:"max_timer_inject"` 34 | } 35 | 36 | func LoadHekadConfig(filename string) (config *HekadConfig, err error) { 37 | config = &HekadConfig{Maxprocs: 1, 38 | PoolSize: 100, 39 | DecoderPoolSize: 4, 40 | ChanSize: 50, 41 | CpuProfName: "", 42 | MemProfName: "", 43 | MaxMsgLoops: 4, 44 | MaxMsgProcessInject: 1, 45 | MaxMsgTimerInject: 10, 46 | } 47 | 48 | var configFile map[string]toml.Primitive 49 | 50 | if _, err = toml.DecodeFile(filename, &configFile); err != nil { 51 | return nil, fmt.Errorf("Error decoding config file: %s", err) 52 | } 53 | 54 | empty_ignore := map[string]interface{}{} 55 | parsed_config, ok := configFile["hekad"] 56 | if ok { 57 | if err = toml.PrimitiveDecodeStrict(parsed_config, &config, empty_ignore); err != nil { 58 | err = fmt.Errorf("Can't unmarshal config: %s", err) 59 | } 60 | } 61 | 62 | return 63 | } 64 | -------------------------------------------------------------------------------- /docs/source/developing/testing.rst: -------------------------------------------------------------------------------- 1 | .. testing: 2 | 3 | ============ 4 | Testing Heka 5 | ============ 6 | 7 | Flood 8 | ===== 9 | Flood is a Heka load test tool; it is capable of generating a large number of 10 | messages to exercise Heka using different protocols, message types, and error 11 | conditions. 12 | 13 | Command Line Options 14 | -------------------- 15 | flood [``-config`` `config_file`] [``-test`` `config_section_name`] 16 | 17 | 18 | 19 | Configuration Variables 20 | ----------------------- 21 | - test (object): Name of the test section (toml key) in the configuration file. 22 | - ip_address (string): IP address of the Heka server. 23 | - sender (string): tcp or udp 24 | - pprof_file (string): The name of the file to save the profiling data to. 25 | - encoder (string): protobuf or json 26 | - num_messages (int): The number of messages to be sent, 0 for infinite. 27 | - corrupt_percentage (float): The percentage of messages that will be randomly corrupted. 28 | - signed_percentage (float): The percentage of message that will signed. 29 | - variable_size_messages (bool): True, if a random selection of variable size messages are to be sent. False, if a single fixed message will be sent. 30 | - signer (object): Signer information for the encoder. 31 | - name (string): The name of the signer. 32 | - hmac_hash (string): md5 or sha1 33 | - hmac_key (string): The key the message will be signed with. 34 | - version (int): The version number of the hmac_key. 35 | - ascii_only (bool): True, if generated message payloads should only contain ASCII characters. False, if message payloads should contain arbitrary binary data. Defaults to false. 36 | 37 | Example 38 | 39 | .. code-block:: ini 40 | 41 | [default] 42 | ip_address = "127.0.0.1:5565" 43 | sender = "tcp" 44 | pprof_file = "" 45 | encoder = "protobuf" 46 | num_messages = 0 47 | corrupt_percentage = 0.0001 48 | signed_percentage = 0.00011 49 | variable_size_messages = true 50 | [default.signer] 51 | name = "test" 52 | hmac_hash = "md5" 53 | hmac_key = "4865ey9urgkidls xtb0[7lf9rzcivthkm" 54 | version = 0 55 | -------------------------------------------------------------------------------- /docs/source/sandbox/filter.rst: -------------------------------------------------------------------------------- 1 | .. _sandboxfilter: 2 | 3 | ============== 4 | Sandbox Filter 5 | ============== 6 | The sandbox filter provides an isolated execution environment for data analysis. 7 | The output generated by the sandbox is injected into the payload of a new 8 | message for further processing or to be output. 9 | 10 | Filter Parameters 11 | ================= 12 | :ref:`config_common_parameters` 13 | 14 | .. _sandboxfilter_settings: 15 | 16 | SandboxFilter Settings 17 | ====================== 18 | - script_type (string): The language the sandbox is written in. Currently the only valid option is 'lua'. 19 | - filename (string): For a static configuration this is the full path to the sandbox code. The filename must be unique between static plugins, since the global data is preserved using this name. For a dynamic configuration the filename is ignored and the the physical location on disk is controlled by the SandboxManagerFilter. 20 | - preserve_data (bool): True if the sandbox global data should be preserved/restored on Heka shutdown/startup. The preserved data is stored along side the sandbox code i.e. counter.lua.data so Heka must have read/write permissions to that directory. 21 | - memory_limit (uint): The number of bytes the sandbox is allowed to consume before being terminated (max 8MiB, default 32767). 22 | - instruction_limit (uint): The number of instructions the sandbox is allowed the execute during the process_message/timer_event functions before being terminated (max 1M, default 1000). 23 | - output_limit (uint): The number of bytes the sandbox output buffer can hold before before being terminated (max 63KiB, default 1024). Anything less than 1KiB will default to 1KiB. 24 | - profile (bool): When true a statistically significant number of ProcessMessage timings are immediately captured before reverting back to the regular sampling interval. The main purpose is for more accurate sandbox comparison/tuning/optimization. 25 | 26 | Example 27 | 28 | .. code-block:: ini 29 | 30 | [hekabench_counter] 31 | type = "SandboxFilter" 32 | message_matcher = "Type == 'hekabench'" 33 | ticker_interval = 1 34 | script_type = "lua" 35 | filename = "counter.lua" 36 | preserve_data = true 37 | memory_limit = 32767 38 | instruction_limit = 1000 39 | output_limit = 1024 40 | profile = false 41 | -------------------------------------------------------------------------------- /pipeline/all_specs_test.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Rob Miller (rmiller@mozilla.com) 12 | # Mike Trinkala (trink@mozilla.com) 13 | # 14 | # ***** END LICENSE BLOCK *****/ 15 | 16 | package pipeline 17 | 18 | import ( 19 | "code.google.com/p/go-uuid/uuid" 20 | . "github.com/mozilla-services/heka/message" 21 | "github.com/rafrombrc/gospec/src/gospec" 22 | "os" 23 | "testing" 24 | "time" 25 | ) 26 | 27 | func mockDecoderCreator() map[string]Decoder { 28 | return make(map[string]Decoder) 29 | } 30 | 31 | func mockFilterCreator() map[string]Filter { 32 | return make(map[string]Filter) 33 | } 34 | 35 | func mockOutputCreator() map[string]Output { 36 | return make(map[string]Output) 37 | } 38 | 39 | func TestAllSpecs(t *testing.T) { 40 | r := gospec.NewRunner() 41 | r.Parallel = false 42 | 43 | r.AddSpec(DecodersSpec) 44 | r.AddSpec(FiltersSpec) 45 | r.AddSpec(OutputsSpec) 46 | r.AddSpec(LoadFromConfigSpec) 47 | r.AddSpec(WhisperRunnerSpec) 48 | r.AddSpec(WhisperOutputSpec) 49 | r.AddSpec(ReportSpec) 50 | r.AddSpec(AMQPPluginSpec) 51 | r.AddSpec(StatsdInputSpec) 52 | r.AddSpec(InputsSpec) 53 | r.AddSpec(FileMonitorSpec) 54 | r.AddSpec(LogfileInputSpec) 55 | r.AddSpec(StatAccumInputSpec) 56 | r.AddSpec(CarbonOutputSpec) 57 | r.AddSpec(DashboardOutputSpec) 58 | r.AddSpec(JsonPathSpec) 59 | r.AddSpec(HttpInputSpec) 60 | gospec.MainGoTest(r, t) 61 | } 62 | 63 | func getTestMessage() *Message { 64 | hostname, _ := os.Hostname() 65 | field, _ := NewField("foo", "bar", "") 66 | msg := &Message{} 67 | msg.SetType("TEST") 68 | msg.SetTimestamp(time.Now().UnixNano()) 69 | msg.SetUuid(uuid.NewRandom()) 70 | msg.SetLogger("GoSpec") 71 | msg.SetSeverity(int32(6)) 72 | msg.SetPayload("Test Payload") 73 | msg.SetEnvVersion("0.8") 74 | msg.SetPid(int32(os.Getpid())) 75 | msg.SetHostname(hostname) 76 | msg.AddField(field) 77 | 78 | return msg 79 | } 80 | 81 | func BenchmarkPipelinePackCreation(b *testing.B) { 82 | var config = PipelineConfig{} 83 | for i := 0; i < b.N; i++ { 84 | NewPipelinePack(config.inputRecycleChan) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/inject_message.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | local cbuf = circular_buffer.new(1440, 3, 60) 6 | local simple_table = {value=1} 7 | local metric = {MetricName="example",Timestamp=0,Unit="s",Value=0, 8 | Dimensions={{Name="d1",Value="v1"}, {Name="d2",Value="v2"}}, 9 | StatisticValues={{Maximum=0,Minimum=0,SampleCount=0,Sum= 0},{Maximum=0,Minimum=0,SampleCount=0,Sum=0}}} 10 | 11 | function process_message () 12 | local msg = read_message("Payload") 13 | 14 | if msg == "lua types" then 15 | output(simple_table, 1.2, " string ", nil, " ", true, " ", false) 16 | inject_message() 17 | elseif msg == "cloudwatch metric" then 18 | output(metric) 19 | inject_message() 20 | elseif msg == "external reference" then 21 | local a = {x = 1, y = 2} 22 | local b = {a = a} 23 | output(b) 24 | inject_message() 25 | elseif msg == "array only" then 26 | local a = {1,2,3} 27 | output(a) 28 | inject_message() 29 | elseif msg == "private keys" then 30 | local a = {x = 1, _m = 1, _private = {1,2}} 31 | output(a) 32 | inject_message() 33 | elseif msg == "table name" then 34 | local a = {1,2,3,_name="array"} 35 | output(a) 36 | inject_message() 37 | elseif msg == "global table" then 38 | output(_G) 39 | inject_message() 40 | elseif msg == "special characters" then 41 | output({['special\tcharacters'] = '"\t\r\n\b\f\\/'}) 42 | inject_message() 43 | elseif msg == "error internal reference" then 44 | local a = {x = {1,2,3}, y = {2}} 45 | a.ir = a.x 46 | output(a) 47 | inject_message() 48 | elseif msg == "error circular reference" then 49 | local a = {x = 1, y = 2} 50 | a.self = a 51 | output(a) 52 | inject_message() 53 | elseif msg == "error escape overflow" then 54 | local escape = "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" 55 | for i=1, 10 do 56 | escape = escape .. escape 57 | end 58 | output({escape = escape}) 59 | inject_message() 60 | end 61 | return 0 62 | end 63 | 64 | function timer_event(ns) 65 | output(cbuf) 66 | inject_message("cbuf", "test") 67 | end 68 | 69 | -------------------------------------------------------------------------------- /pipeline/mock_pluginrunner_test.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: github.com/mozilla-services/heka/pipeline (interfaces: PluginRunner) 3 | 4 | package pipeline 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | ) 9 | 10 | // Mock of PluginRunner interface 11 | type MockPluginRunner struct { 12 | ctrl *gomock.Controller 13 | recorder *_MockPluginRunnerRecorder 14 | } 15 | 16 | // Recorder for MockPluginRunner (not exported) 17 | type _MockPluginRunnerRecorder struct { 18 | mock *MockPluginRunner 19 | } 20 | 21 | func NewMockPluginRunner(ctrl *gomock.Controller) *MockPluginRunner { 22 | mock := &MockPluginRunner{ctrl: ctrl} 23 | mock.recorder = &_MockPluginRunnerRecorder{mock} 24 | return mock 25 | } 26 | 27 | func (_m *MockPluginRunner) EXPECT() *_MockPluginRunnerRecorder { 28 | return _m.recorder 29 | } 30 | 31 | func (_m *MockPluginRunner) LogError(_param0 error) { 32 | _m.ctrl.Call(_m, "LogError", _param0) 33 | } 34 | 35 | func (_mr *_MockPluginRunnerRecorder) LogError(arg0 interface{}) *gomock.Call { 36 | return _mr.mock.ctrl.RecordCall(_mr.mock, "LogError", arg0) 37 | } 38 | 39 | func (_m *MockPluginRunner) LogMessage(_param0 string) { 40 | _m.ctrl.Call(_m, "LogMessage", _param0) 41 | } 42 | 43 | func (_mr *_MockPluginRunnerRecorder) LogMessage(arg0 interface{}) *gomock.Call { 44 | return _mr.mock.ctrl.RecordCall(_mr.mock, "LogMessage", arg0) 45 | } 46 | 47 | func (_m *MockPluginRunner) Name() string { 48 | ret := _m.ctrl.Call(_m, "Name") 49 | ret0, _ := ret[0].(string) 50 | return ret0 51 | } 52 | 53 | func (_mr *_MockPluginRunnerRecorder) Name() *gomock.Call { 54 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Name") 55 | } 56 | 57 | func (_m *MockPluginRunner) Plugin() Plugin { 58 | ret := _m.ctrl.Call(_m, "Plugin") 59 | ret0, _ := ret[0].(Plugin) 60 | return ret0 61 | } 62 | 63 | func (_mr *_MockPluginRunnerRecorder) Plugin() *gomock.Call { 64 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Plugin") 65 | } 66 | 67 | func (_m *MockPluginRunner) PluginGlobals() *PluginGlobals { 68 | ret := _m.ctrl.Call(_m, "PluginGlobals") 69 | ret0, _ := ret[0].(*PluginGlobals) 70 | return ret0 71 | } 72 | 73 | func (_mr *_MockPluginRunnerRecorder) PluginGlobals() *gomock.Call { 74 | return _mr.mock.ctrl.RecordCall(_mr.mock, "PluginGlobals") 75 | } 76 | 77 | func (_m *MockPluginRunner) SetName(_param0 string) { 78 | _m.ctrl.Call(_m, "SetName", _param0) 79 | } 80 | 81 | func (_mr *_MockPluginRunnerRecorder) SetName(arg0 interface{}) *gomock.Call { 82 | return _mr.mock.ctrl.RecordCall(_mr.mock, "SetName", arg0) 83 | } 84 | -------------------------------------------------------------------------------- /examples/host_filter.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Rob Miller (rmiller@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | package examples 16 | 17 | import ( 18 | "errors" 19 | "fmt" 20 | "github.com/mozilla-services/heka/pipeline" 21 | ) 22 | 23 | type HostFilter struct { 24 | hosts map[string]bool 25 | output string 26 | } 27 | 28 | // Extract hosts value from config and store it on the plugin instance. 29 | func (f *HostFilter) Init(config interface{}) error { 30 | var ( 31 | hostsConf interface{} 32 | hosts []interface{} 33 | host string 34 | outputConf interface{} 35 | ok bool 36 | ) 37 | conf := config.(pipeline.PluginConfig) 38 | if hostsConf, ok = conf["hosts"]; !ok { 39 | return errors.New("No 'hosts' setting specified.") 40 | } 41 | if hosts, ok = hostsConf.([]interface{}); !ok { 42 | return errors.New("'hosts' setting not a sequence.") 43 | } 44 | if outputConf, ok = conf["output"]; !ok { 45 | return errors.New("No 'output' setting specified.") 46 | } 47 | if f.output, ok = outputConf.(string); !ok { 48 | return errors.New("'output' setting not a string value.") 49 | } 50 | f.hosts = make(map[string]bool) 51 | for _, h := range hosts { 52 | if host, ok = h.(string); !ok { 53 | return errors.New("Non-string host value.") 54 | } 55 | f.hosts[host] = true 56 | } 57 | return nil 58 | } 59 | 60 | // Fetch correct output and iterate over received messages, checking against 61 | // message hostname and delivering to the output if hostname is in our config. 62 | func (f *HostFilter) Run(runner pipeline.FilterRunner, helper pipeline.PluginHelper) ( 63 | err error) { 64 | 65 | var ( 66 | hostname string 67 | output pipeline.OutputRunner 68 | ok bool 69 | ) 70 | 71 | if output, ok = helper.Output(f.output); !ok { 72 | return fmt.Errorf("No output: %s", output) 73 | } 74 | for pack := range runner.InChan() { 75 | hostname = pack.Message.GetHostname() 76 | if f.hosts[hostname] { 77 | output.InChan() <- pack 78 | } else { 79 | pack.Recycle() 80 | } 81 | } 82 | return 83 | } 84 | 85 | func init() { 86 | pipeline.RegisterPlugin("HostFilter", func() interface{} { 87 | return new(HostFilter) 88 | }) 89 | } 90 | -------------------------------------------------------------------------------- /pipeline/jsonpath.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Victor Ng (vng@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | package pipeline 16 | 17 | import ( 18 | "encoding/json" 19 | "errors" 20 | "fmt" 21 | "reflect" 22 | "regexp" 23 | "strconv" 24 | "strings" 25 | ) 26 | 27 | type JsonPath struct { 28 | json_data interface{} 29 | json_text string 30 | } 31 | 32 | var json_re = regexp.MustCompile(`^([^0-9\s\[][^\s\[]*)?(\[[0-9]+\])?$`) 33 | 34 | func (j *JsonPath) SetJsonText(json_text string) (err error) { 35 | j.json_text = json_text 36 | err = json.Unmarshal([]byte(json_text), &j.json_data) 37 | return 38 | } 39 | 40 | func (j *JsonPath) Find(jp string) (result string, err error) { 41 | var ok bool 42 | 43 | if jp == "" || strings.HasPrefix("$.", jp) { 44 | return result, errors.New("invalid path") 45 | } 46 | 47 | // Strip off the leading $. 48 | jp = jp[2:] 49 | 50 | // Need to grab a pointer to the top of the data structure 51 | v := j.json_data 52 | 53 | for _, token := range strings.Split(jp, ".") { 54 | sl := json_re.FindAllStringSubmatch(token, -1) 55 | if len(sl) == 0 { 56 | return result, errors.New("invalid path") 57 | } 58 | ss := sl[0] 59 | if ss[1] != "" { 60 | v, ok = v.(map[string]interface{})[ss[1]] 61 | if !ok { 62 | return result, errors.New("invalid path") 63 | } 64 | } 65 | if ss[2] != "" { 66 | i, err := strconv.Atoi(ss[2][1 : len(ss[2])-1]) 67 | if err != nil { 68 | return result, errors.New("invalid path") 69 | } 70 | v_arr := v.([]interface{}) 71 | if i < 0 || i >= len(v_arr) { 72 | return result, errors.New(fmt.Sprintf("array out of bounds jsonpath:[%s]", jp)) 73 | } 74 | v = v_arr[i] 75 | } 76 | } 77 | 78 | r_kind := reflect.ValueOf(v).Kind() 79 | if r_kind == reflect.Bool { 80 | result = fmt.Sprintf("%t", v) 81 | } else if r_kind == reflect.Float64 { 82 | result = fmt.Sprintf("%0.9f", v) 83 | } else if r_kind == reflect.Int64 { 84 | result = fmt.Sprintf("%d", v) 85 | } else if r_kind == reflect.Map || r_kind == reflect.Slice { 86 | json_str, _ := json.Marshal(v) 87 | result = fmt.Sprintf("%s", json_str) 88 | } else { 89 | result = fmt.Sprintf("%s", v) 90 | } 91 | 92 | return result, nil 93 | } 94 | -------------------------------------------------------------------------------- /pipeline/jsonpath_test.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Victor Ng (vng@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | package pipeline 16 | 17 | import ( 18 | "fmt" 19 | gs "github.com/rafrombrc/gospec/src/gospec" 20 | ) 21 | 22 | func JsonPathSpec(c gs.Context) { 23 | c.Specify("JsonPath can read data", func() { 24 | var s = `{ 25 | "foo": { 26 | "bar": [ 27 | { 28 | "baz": "こんにちわ世界", 29 | "noo": "aaa" 30 | }, 31 | { 32 | "maz": "123", 33 | "moo": 256 34 | } 35 | ], 36 | "boo": { 37 | "bag": true, 38 | "bug": false 39 | } 40 | } 41 | } 42 | ` 43 | var err error 44 | var json_path *JsonPath 45 | var result interface{} 46 | 47 | json_path = new(JsonPath) 48 | err = json_path.SetJsonText(s) 49 | if err != nil { 50 | fmt.Println(err.Error()) 51 | return 52 | } 53 | 54 | result, err = json_path.Find("$.foo.bar[0].baz") 55 | c.Expect(err, gs.IsNil) 56 | c.Expect(result, gs.Equals, "こんにちわ世界") 57 | 58 | result, err = json_path.Find("$.foo.bar[0].noo") 59 | c.Expect(err, gs.IsNil) 60 | c.Expect(result, gs.Equals, "aaa") 61 | 62 | result, err = json_path.Find("$.foo.bar[1].maz") 63 | c.Expect(err, gs.IsNil) 64 | c.Expect(result, gs.Equals, "123") 65 | 66 | result, err = json_path.Find("$.foo.bar[1].moo") 67 | c.Expect(err, gs.IsNil) 68 | c.Expect(result, gs.Equals, "256.000000000") 69 | 70 | result, err = json_path.Find("$.foo.boo.bag") 71 | c.Expect(err, gs.IsNil) 72 | c.Expect(result, gs.Equals, "true") 73 | 74 | result, err = json_path.Find("$.foo.boo.bug") 75 | c.Expect(err, gs.IsNil) 76 | c.Expect(result, gs.Equals, "false") 77 | 78 | result, err = json_path.Find("$.foo.bar[99].baz") 79 | c.Expect(err, gs.Not(gs.IsNil)) 80 | 81 | result, err = json_path.Find("$.badpath") 82 | c.Expect(err, gs.Not(gs.IsNil)) 83 | 84 | result, err = json_path.Find("badpath") 85 | c.Expect(err, gs.Not(gs.IsNil)) 86 | 87 | result, err = json_path.Find("$.foo.bar.3428") 88 | c.Expect(err, gs.Not(gs.IsNil)) 89 | 90 | expected_data := `[{"baz":"こんにちわ世界","noo":"aaa"},{"maz":"123","moo":256}]` 91 | result_data, err := json_path.Find("$.foo.bar") 92 | c.Expect(err, gs.IsNil) 93 | c.Expect(result_data, gs.Equals, expected_data) 94 | 95 | }) 96 | } 97 | -------------------------------------------------------------------------------- /docs/source/message_matcher.rst: -------------------------------------------------------------------------------- 1 | .. _message_matcher: 2 | 3 | ====================== 4 | Message Matcher Syntax 5 | ====================== 6 | 7 | Message matching is done by the `hekad` router to choose an appropriate 8 | filter(s) to run. Every filter that matches will get a copy of the 9 | message. 10 | 11 | Examples 12 | ======== 13 | 14 | - Type == "test" && Severity == 6 15 | - (Severity == 7 || Payload == "Test Payload") && Type == "test" 16 | - Fields[foo] != "bar" 17 | - Fields[foo][1][0] == 'alternate' 18 | - Fields[MyBool] == TRUE 19 | - TRUE 20 | - Fields[created] =~ /%TIMESTAMP%/ 21 | 22 | Relational Operators 23 | ==================== 24 | 25 | - **==** equals 26 | - **!=** not equals 27 | - **>** greater than 28 | - **>=** greater than equals 29 | - **<** less than 30 | - **<=** less than equals 31 | - **=~** regular expression match 32 | - **!~** regular expression negated match 33 | 34 | Logical Operators 35 | ================= 36 | 37 | - Parentheses are used for grouping expressions 38 | - **&&** and (higher precedence) 39 | - **||** or 40 | 41 | Boolean 42 | ======= 43 | 44 | - **TRUE** 45 | - **FALSE** 46 | 47 | Message Variables 48 | ================= 49 | 50 | - All message variables must be on the left hand side of the relational 51 | comparison 52 | - String 53 | - **Uuid** 54 | - **Type** 55 | - **Logger** 56 | - **Payload** 57 | - **EnvVersion** 58 | - **Hostname** 59 | - Numeric 60 | - **Timestamp** 61 | - **Severity** 62 | - **Pid** 63 | - Fields 64 | - **Fields[_field_name_]** (shorthand for Field[_field_name_][0][0]) 65 | - **Fields[_field_name_][_field_index_]** (shorthand for Field[_field_name_][_field_index_][0]) 66 | - **Fields[_field_name_][_field_index_][_array_index_]** 67 | - If a field type is mis-match for the relational comparison, false will be returned i.e. Fields[foo] == 6 where 'foo' is a string 68 | 69 | Quoted String 70 | ============= 71 | 72 | - single or double quoted strings are allowed 73 | - must be placed on the right side of a relational comparison i.e. Type == 'test' 74 | 75 | Regular Expression String 76 | ========================= 77 | 78 | - enclosed by forward slashes 79 | - must be placed on the right side of the relational comparison i.e. Type =~ /test/ 80 | - capture groups will be ignored 81 | 82 | .. _matcher_regex_helpers: 83 | 84 | Regular Expression Helpers 85 | -------------------------- 86 | 87 | Commonly used complex regular expressions are provide as template 88 | variables in the form of %TEMPLATE%. 89 | 90 | i.e., Fields[created] =~ /%TIMESTAMP%/ 91 | 92 | Available templates 93 | - TIMESTAMP - matches most common date/time string formats 94 | 95 | .. seealso:: `Regular Expression re2 syntax `_ 96 | 97 | -------------------------------------------------------------------------------- /pipeline/payload_decoder_helper.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Victor Ng (vng@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | package pipeline 16 | 17 | import ( 18 | "fmt" 19 | . "github.com/mozilla-services/heka/message" 20 | "strconv" 21 | "time" 22 | ) 23 | 24 | type PayloadDecoderHelper struct { 25 | Captures map[string]string 26 | dRunner DecoderRunner 27 | TimestampLayout string 28 | TzLocation *time.Location 29 | SeverityMap map[string]int32 30 | } 31 | 32 | /* 33 | Timestamps strings are decoded using the TimestampLayout and written 34 | back to the Message as nanoseconds into the Timestamp field. 35 | In the case that a timestamp string is not in the capture map, no 36 | timestamp is written and the default of 0 is used. 37 | */ 38 | func (pdh *PayloadDecoderHelper) DecodeTimestamp(pack *PipelinePack) { 39 | if timeStamp, ok := pdh.Captures["Timestamp"]; ok { 40 | val, err := ForgivingTimeParse(pdh.TimestampLayout, timeStamp, pdh.TzLocation) 41 | if err != nil { 42 | pdh.dRunner.LogError(fmt.Errorf("Don't recognize Timestamp: '%s'", timeStamp)) 43 | } 44 | // If we only get a timestamp, use the current date 45 | if val.Year() == 0 && val.Month() == 1 && val.Day() == 1 { 46 | now := time.Now() 47 | val = val.AddDate(now.Year(), int(now.Month()-1), now.Day()-1) 48 | } else if val.Year() == 0 { 49 | // If there's no year, use current year 50 | val = val.AddDate(time.Now().Year(), 0, 0) 51 | } 52 | pack.Message.SetTimestamp(val.UnixNano()) 53 | } 54 | } 55 | 56 | /* 57 | Severity values for an error may be encoded into the captures map as 58 | stringified integers. The DecodeSeverity function will decode those 59 | values and write them back into the severity field of the Message. 60 | In the event no severity is found, a default value of 0 is used. 61 | */ 62 | func (pdh *PayloadDecoderHelper) DecodeSeverity(pack *PipelinePack) { 63 | if sevStr, ok := pdh.Captures["Severity"]; ok { 64 | // If so, see if we have a mapping for this severity. 65 | if sevInt, ok := pdh.SeverityMap[sevStr]; ok { 66 | pack.Message.SetSeverity(sevInt) 67 | } else { 68 | // No mapping => severity value should be an int. 69 | sevInt, err := strconv.ParseInt(sevStr, 10, 32) 70 | if err != nil { 71 | pdh.dRunner.LogError(fmt.Errorf("Don't recognize severity: '%s'", sevStr)) 72 | } else { 73 | pack.Message.SetSeverity(int32(sevInt)) 74 | } 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /docs/source/monitoring/index.rst: -------------------------------------------------------------------------------- 1 | .. _internal_monitoring: 2 | 3 | ========================= 4 | Monitoring Internal State 5 | ========================= 6 | 7 | Heka can emit metrics about it's internal state to either an outgoing 8 | Heka message (and, through the DashboardOutput, to a web dashboard) or 9 | to stdout. 10 | Sending SIGUSR1 to hekad on a UNIX will send a plain text report 11 | tostdout. On Windows, you will need to send signal 12 | 10 to the hekad process using Powershell. 13 | 14 | Sample text output :: 15 | 16 | ========[heka.all-report]======== 17 | inputRecycleChan: 18 | InChanCapacity: 100 19 | InChanLength: 99 20 | injectRecycleChan: 21 | InChanCapacity: 100 22 | InChanLength: 98 23 | Router: 24 | InChanCapacity: 50 25 | InChanLength: 0 26 | ProcessMessageCount: 26 27 | JsonDecoder-0: 28 | InChanCapacity: 50 29 | InChanLength: 0 30 | JsonDecoder-1: 31 | InChanCapacity: 50 32 | InChanLength: 0 33 | JsonDecoder-2: 34 | InChanCapacity: 50 35 | InChanLength: 0 36 | JsonDecoder-3: 37 | InChanCapacity: 50 38 | InChanLength: 0 39 | ProtobufDecoder-0: 40 | InChanCapacity: 50 41 | InChanLength: 0 42 | ProtobufDecoder-1: 43 | InChanCapacity: 50 44 | InChanLength: 0 45 | ProtobufDecoder-2: 46 | InChanCapacity: 50 47 | InChanLength: 0 48 | ProtobufDecoder-3: 49 | InChanCapacity: 50 50 | InChanLength: 0 51 | DecoderPool-JsonDecoder: 52 | InChanCapacity: 4 53 | InChanLength: 4 54 | DecoderPool-ProtobufDecoder: 55 | InChanCapacity: 4 56 | InChanLength: 4 57 | OpsSandboxManager: 58 | InChanCapacity: 50 59 | InChanLength: 0 60 | MatchChanCapacity: 50 61 | MatchChanLength: 0 62 | MatchAvgDuration: 0 63 | ProcessMessageCount: 0 64 | hekabench_counter: 65 | InChanCapacity: 50 66 | InChanLength: 0 67 | MatchChanCapacity: 50 68 | MatchChanLength: 0 69 | MatchAvgDuration: 445 70 | ProcessMessageCount: 0 71 | InjectMessageCount: 0 72 | Memory: 20644 73 | MaxMemory: 20644 74 | MaxInstructions: 18 75 | MaxOutput: 0 76 | ProcessMessageAvgDuration: 0 77 | TimerEventAvgDuration: 78532 78 | LogOutput: 79 | InChanCapacity: 50 80 | InChanLength: 0 81 | MatchChanCapacity: 50 82 | MatchChanLength: 0 83 | MatchAvgDuration: 406 84 | DashboardOutput: 85 | InChanCapacity: 50 86 | InChanLength: 0 87 | MatchChanCapacity: 50 88 | MatchChanLength: 0 89 | MatchAvgDuration: 336 90 | ======== 91 | 92 | To enable the HTTP interface, you will need to enable the 93 | dashboard output plugin, see :ref:`config_dashboard_output`. 94 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/mysql_slow_query.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | -- sample mysql slow query log entry 6 | -- # User@Host: weaverw[weaverw] @ [10.14.214.13] 7 | -- # Thread_id: 78959 Schema: weave3 Last_errno: 0 Killed: 0 8 | -- # Query_time: 10.749944 Lock_time: 0.017599 Rows_sent: 0 Rows_examined: 0 Rows_affected: 10 Rows_read: 12 9 | -- # Bytes_sent: 51 Tmp_tables: 0 Tmp_disk_tables: 0 Tmp_table_sizes: 0 10 | -- # InnoDB_trx_id: 98CF 11 | -- use weave3; 12 | -- SET timestamp=1364506803; 13 | -- 14 | 15 | data = circular_buffer.new(1440, 4, 60) 16 | sums = circular_buffer.new(1440, 3, 60) 17 | local QUERY_TIME = data:set_header(1, "Query Time", "s", "avg") 18 | local LOCK_TIME = data:set_header(2, "Lock Time", "s", "avg") 19 | local RESPONSE_SIZE = data:set_header(3, "Response Size", "B", "avg") 20 | local COUNT = data:set_header(4, "Count") 21 | 22 | entry = {} 23 | active_pattern = 1 24 | 25 | function process_message () 26 | local payload = read_message("Payload") 27 | if active_pattern == 1 then 28 | if payload:find("^#%s+User@Host:") then 29 | if entry.timestamp and entry.timestamp ~= 0 then 30 | local ns = entry.timestamp * 1e9 31 | local cnt = data:add(ns, COUNT, 1) 32 | data:set(ns, QUERY_TIME, sums:add(ns, QUERY_TIME, entry.query_time)/cnt) 33 | data:set(ns, LOCK_TIME, sums:add(ns, LOCK_TIME, entry.lock_time)/cnt) 34 | data:set(ns, RESPONSE_SIZE, sums:add(ns, RESPONSE_SIZE, entry.bytes_sent)/cnt) 35 | end 36 | entry.query_time = 0 37 | entry.lock_time = 0 38 | entry.bytes_sent = 0 39 | entry.timestamp = 0 40 | active_pattern = 2 41 | end 42 | elseif active_pattern == 2 then 43 | local query_time, lock_time = payload:match("^#%s+Query_time:%s+(%d+%.%d+)%s+Lock_time:%s+(%d+%.%d+)") 44 | if query_time then 45 | entry.query_time = tonumber(query_time) 46 | entry.lock_time = tonumber(lock_time) 47 | active_pattern = 3 48 | end 49 | elseif active_pattern == 3 then 50 | local bytes_sent = payload:match("^#%s+Bytes_sent:%s+(%d+)") 51 | if bytes_sent then 52 | entry.bytes_sent = tonumber(bytes_sent) 53 | active_pattern = 4 54 | end 55 | elseif active_pattern == 4 then 56 | local timestamp = payload:match("^SET%s+timestamp=(%d+);") 57 | if timestamp then 58 | entry.timestamp = tonumber(timestamp) 59 | active_pattern = 1 60 | end 61 | end 62 | return 0 63 | end 64 | 65 | function timer_event(ns) 66 | output(data) 67 | inject_message("cbuf", "Statistics") 68 | end 69 | -------------------------------------------------------------------------------- /pipeline/statsd_input_test.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Rob Miller (rmiller@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | package pipeline 16 | 17 | import ( 18 | "code.google.com/p/gomock/gomock" 19 | "fmt" 20 | ts "github.com/mozilla-services/heka/testsupport" 21 | gs "github.com/rafrombrc/gospec/src/gospec" 22 | "strconv" 23 | "sync" 24 | ) 25 | 26 | func StatsdInputSpec(c gs.Context) { 27 | t := &ts.SimpleT{} 28 | ctrl := gomock.NewController(t) 29 | defer ctrl.Finish() 30 | 31 | pConfig := NewPipelineConfig(nil) 32 | ith := new(InputTestHelper) 33 | ith.Msg = getTestMessage() 34 | ith.Pack = NewPipelinePack(pConfig.inputRecycleChan) 35 | ith.PackSupply = make(chan *PipelinePack, 1) 36 | 37 | // Specify localhost, but we're not really going to use the network 38 | ith.AddrStr = "localhost:55565" 39 | ith.ResolvedAddrStr = "127.0.0.1:55565" 40 | 41 | // set up mock helper, input runner, and stat accumulator 42 | ith.MockHelper = NewMockPluginHelper(ctrl) 43 | ith.MockInputRunner = NewMockInputRunner(ctrl) 44 | mockStatAccum := NewMockStatAccumulator(ctrl) 45 | 46 | c.Specify("A StatsdInput", func() { 47 | statsdInput := StatsdInput{} 48 | config := statsdInput.ConfigStruct().(*StatsdInputConfig) 49 | 50 | config.Address = ith.AddrStr 51 | err := statsdInput.Init(config) 52 | c.Assume(err, gs.IsNil) 53 | realListener := statsdInput.listener 54 | c.Expect(realListener.LocalAddr().String(), gs.Equals, ith.ResolvedAddrStr) 55 | realListener.Close() 56 | mockListener := ts.NewMockConn(ctrl) 57 | statsdInput.listener = mockListener 58 | 59 | ith.MockHelper.EXPECT().StatAccumulator("StatAccumInput").Return(mockStatAccum, nil) 60 | mockListener.EXPECT().Close() 61 | mockListener.EXPECT().SetReadDeadline(gomock.Any()) 62 | 63 | c.Specify("sends a Stat to the StatAccumulator", func() { 64 | statName := "sample.count" 65 | statVal := 303 66 | msg := fmt.Sprintf("%s:%d|c\n", statName, statVal) 67 | expected := Stat{statName, strconv.Itoa(statVal), "c", float32(1)} 68 | mockStatAccum.EXPECT().DropStat(expected).Return(true) 69 | readCall := mockListener.EXPECT().Read(make([]byte, 512)) 70 | readCall.Return(len(msg), nil) 71 | readCall.Do(func(msgBytes []byte) { 72 | copy(msgBytes, []byte(msg)) 73 | statsdInput.Stop() 74 | }) 75 | var wg sync.WaitGroup 76 | wg.Add(1) 77 | go func() { 78 | err = statsdInput.Run(ith.MockInputRunner, ith.MockHelper) 79 | c.Expect(err, gs.IsNil) 80 | wg.Done() 81 | }() 82 | wg.Wait() 83 | }) 84 | }) 85 | } 86 | -------------------------------------------------------------------------------- /cmd/sbmgr/main.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Mike Trinkala (trink@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | /* 16 | 17 | Sandbox Manager 18 | 19 | */ 20 | package main 21 | 22 | import ( 23 | "code.google.com/p/go-uuid/uuid" 24 | "flag" 25 | "github.com/bbangert/toml" 26 | "github.com/mozilla-services/heka/client" 27 | "github.com/mozilla-services/heka/message" 28 | "io/ioutil" 29 | "log" 30 | "os" 31 | "time" 32 | ) 33 | 34 | type SbmgrConfig struct { 35 | IpAddress string `toml:"ip_address"` 36 | Signer message.MessageSigningConfig `toml:"signer"` 37 | } 38 | 39 | func main() { 40 | configFile := flag.String("config", "sbmgr.toml", "Sandbox manager configuration file") 41 | scriptFile := flag.String("script", "xyz.lua", "Sandbox script file") 42 | scriptConfig := flag.String("scriptconfig", "xyz.toml", "Sandbox script configuration file") 43 | filterName := flag.String("filtername", "filter", "Sandbox filter name (used on unload)") 44 | action := flag.String("action", "load", "Sandbox manager action") 45 | flag.Parse() 46 | 47 | var config SbmgrConfig 48 | if _, err := toml.DecodeFile(*configFile, &config); err != nil { 49 | log.Printf("Error decoding config file: %s", err) 50 | return 51 | } 52 | sender, err := client.NewNetworkSender("tcp", config.IpAddress) 53 | if err != nil { 54 | log.Fatalf("Error creating sender: %s\n", err.Error()) 55 | } 56 | encoder := client.NewProtobufEncoder(&config.Signer) 57 | manager := client.NewClient(sender, encoder) 58 | 59 | hostname, _ := os.Hostname() 60 | msg := &message.Message{} 61 | msg.SetType("heka.control.sandbox") 62 | msg.SetTimestamp(time.Now().UnixNano()) 63 | msg.SetUuid(uuid.NewRandom()) 64 | msg.SetHostname(hostname) 65 | 66 | switch *action { 67 | case "load": 68 | code, err := ioutil.ReadFile(*scriptFile) 69 | if err != nil { 70 | log.Printf("Error reading scriptFile: %s\n", err.Error()) 71 | return 72 | } 73 | msg.SetPayload(string(code)) 74 | conf, err := ioutil.ReadFile(*scriptConfig) 75 | if err != nil { 76 | log.Printf("Error reading scriptConfig: %s\n", err.Error()) 77 | return 78 | } 79 | f, _ := message.NewField("config", string(conf), "toml") 80 | msg.AddField(f) 81 | case "unload": 82 | f, _ := message.NewField("name", *filterName, "") 83 | msg.AddField(f) 84 | default: 85 | log.Printf("Invalid action: %s", *action) 86 | } 87 | 88 | f1, _ := message.NewField("action", *action, "") 89 | msg.AddField(f1) 90 | err = manager.SendMessage(msg) 91 | if err != nil { 92 | log.Printf("Error sending message: %s\n", err.Error()) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /pipeline/mock_pluginhelper_test.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: github.com/mozilla-services/heka/pipeline (interfaces: PluginHelper) 3 | 4 | package pipeline 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | ) 9 | 10 | // Mock of PluginHelper interface 11 | type MockPluginHelper struct { 12 | ctrl *gomock.Controller 13 | recorder *_MockPluginHelperRecorder 14 | } 15 | 16 | // Recorder for MockPluginHelper (not exported) 17 | type _MockPluginHelperRecorder struct { 18 | mock *MockPluginHelper 19 | } 20 | 21 | func NewMockPluginHelper(ctrl *gomock.Controller) *MockPluginHelper { 22 | mock := &MockPluginHelper{ctrl: ctrl} 23 | mock.recorder = &_MockPluginHelperRecorder{mock} 24 | return mock 25 | } 26 | 27 | func (_m *MockPluginHelper) EXPECT() *_MockPluginHelperRecorder { 28 | return _m.recorder 29 | } 30 | 31 | func (_m *MockPluginHelper) DecoderSet() DecoderSet { 32 | ret := _m.ctrl.Call(_m, "DecoderSet") 33 | ret0, _ := ret[0].(DecoderSet) 34 | return ret0 35 | } 36 | 37 | func (_mr *_MockPluginHelperRecorder) DecoderSet() *gomock.Call { 38 | return _mr.mock.ctrl.RecordCall(_mr.mock, "DecoderSet") 39 | } 40 | 41 | func (_m *MockPluginHelper) Filter(_param0 string) (FilterRunner, bool) { 42 | ret := _m.ctrl.Call(_m, "Filter", _param0) 43 | ret0, _ := ret[0].(FilterRunner) 44 | ret1, _ := ret[1].(bool) 45 | return ret0, ret1 46 | } 47 | 48 | func (_mr *_MockPluginHelperRecorder) Filter(arg0 interface{}) *gomock.Call { 49 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Filter", arg0) 50 | } 51 | 52 | func (_m *MockPluginHelper) Output(_param0 string) (OutputRunner, bool) { 53 | ret := _m.ctrl.Call(_m, "Output", _param0) 54 | ret0, _ := ret[0].(OutputRunner) 55 | ret1, _ := ret[1].(bool) 56 | return ret0, ret1 57 | } 58 | 59 | func (_mr *_MockPluginHelperRecorder) Output(arg0 interface{}) *gomock.Call { 60 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Output", arg0) 61 | } 62 | 63 | func (_m *MockPluginHelper) PipelineConfig() *PipelineConfig { 64 | ret := _m.ctrl.Call(_m, "PipelineConfig") 65 | ret0, _ := ret[0].(*PipelineConfig) 66 | return ret0 67 | } 68 | 69 | func (_mr *_MockPluginHelperRecorder) PipelineConfig() *gomock.Call { 70 | return _mr.mock.ctrl.RecordCall(_mr.mock, "PipelineConfig") 71 | } 72 | 73 | func (_m *MockPluginHelper) PipelinePack(_param0 uint) *PipelinePack { 74 | ret := _m.ctrl.Call(_m, "PipelinePack", _param0) 75 | ret0, _ := ret[0].(*PipelinePack) 76 | return ret0 77 | } 78 | 79 | func (_mr *_MockPluginHelperRecorder) PipelinePack(arg0 interface{}) *gomock.Call { 80 | return _mr.mock.ctrl.RecordCall(_mr.mock, "PipelinePack", arg0) 81 | } 82 | 83 | func (_m *MockPluginHelper) StatAccumulator(_param0 string) (StatAccumulator, error) { 84 | ret := _m.ctrl.Call(_m, "StatAccumulator", _param0) 85 | ret0, _ := ret[0].(StatAccumulator) 86 | ret1, _ := ret[1].(error) 87 | return ret0, ret1 88 | } 89 | 90 | func (_mr *_MockPluginHelperRecorder) StatAccumulator(arg0 interface{}) *gomock.Call { 91 | return _mr.mock.ctrl.RecordCall(_mr.mock, "StatAccumulator", arg0) 92 | } 93 | -------------------------------------------------------------------------------- /cmd/hekad/main.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Rob Miller (rmiller@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | /* 16 | 17 | Hekad daemon. 18 | 19 | This daemon runs the heka/pipeline Plugin's and runners for a complete 20 | message processing platform. 21 | 22 | */ 23 | package main 24 | 25 | import ( 26 | "flag" 27 | "fmt" 28 | "github.com/mozilla-services/heka/pipeline" 29 | "log" 30 | "os" 31 | "runtime" 32 | "runtime/pprof" 33 | ) 34 | 35 | const ( 36 | VERSION = "0.4.0" 37 | ) 38 | 39 | func main() { 40 | configFile := flag.String("config", "/etc/hekad.toml", "Config file") 41 | version := flag.Bool("version", false, "Output version and exit") 42 | flag.Parse() 43 | 44 | if flag.NFlag() == 0 { 45 | flag.PrintDefaults() 46 | os.Exit(0) 47 | } 48 | 49 | if *version { 50 | fmt.Println(VERSION) 51 | os.Exit(0) 52 | } 53 | 54 | config, err := LoadHekadConfig(*configFile) 55 | if err != nil { 56 | log.Fatal("Error reading config: ", err) 57 | } 58 | 59 | maxprocs := config.Maxprocs 60 | poolSize := config.PoolSize 61 | decoderPoolSize := config.DecoderPoolSize 62 | chanSize := config.ChanSize 63 | cpuProfName := config.CpuProfName 64 | memProfName := config.MemProfName 65 | maxMsgLoops := config.MaxMsgLoops 66 | maxMsgProcessInject := config.MaxMsgProcessInject 67 | maxMsgTimerInject := config.MaxMsgTimerInject 68 | 69 | runtime.GOMAXPROCS(maxprocs) 70 | 71 | if cpuProfName != "" { 72 | profFile, err := os.Create(cpuProfName) 73 | if err != nil { 74 | log.Fatalln(err) 75 | } 76 | profFile.Close() 77 | pprof.StartCPUProfile(profFile) 78 | defer pprof.StopCPUProfile() 79 | } 80 | 81 | if memProfName != "" { 82 | defer func() { 83 | profFile, err := os.Create(memProfName) 84 | if err != nil { 85 | log.Fatalln(err) 86 | } 87 | pprof.WriteHeapProfile(profFile) 88 | profFile.Close() 89 | }() 90 | } 91 | 92 | // Set up and load the pipeline configuration and start the daemon. 93 | globals := pipeline.DefaultGlobals() 94 | globals.PoolSize = poolSize 95 | globals.DecoderPoolSize = decoderPoolSize 96 | globals.PluginChanSize = chanSize 97 | globals.MaxMsgLoops = maxMsgLoops 98 | if globals.MaxMsgLoops == 0 { 99 | globals.MaxMsgLoops = 1 100 | } 101 | globals.MaxMsgProcessInject = maxMsgProcessInject 102 | globals.MaxMsgTimerInject = maxMsgTimerInject 103 | pipeconf := pipeline.NewPipelineConfig(globals) 104 | 105 | err = pipeconf.LoadFromConfigFile(*configFile) 106 | if err != nil { 107 | log.Fatal("Error reading config: ", err) 108 | } 109 | pipeline.Run(pipeconf) 110 | } 111 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/circular_buffer_errors.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | function process_message () 6 | local msg = read_message("Payload") 7 | 8 | if msg == "new() incorrect # args" then 9 | local cb = circular_buffer.new(2) 10 | elseif msg == "new() non numeric row" then 11 | local cb = circular_buffer.new(nil, 1, 1) 12 | elseif msg == "new() 1 row" then 13 | local cb = circular_buffer.new(1, 1, 1) 14 | elseif msg == "new() non numeric column" then 15 | local cb = circular_buffer.new(2, nil, 1) 16 | elseif msg == "new() zero column" then 17 | local cb = circular_buffer.new(2, 0, 1) 18 | elseif msg == "new() non numeric seconds_per_row" then 19 | local cb = circular_buffer.new(2, 1, nil) 20 | elseif msg == "new() zero seconds_per_row" then 21 | local cb = circular_buffer.new(2, 1, 0) 22 | elseif msg == "new() > hour seconds_per_row" then 23 | local cb = circular_buffer.new(2, 1, 3601) 24 | elseif msg == "new() too much memory" then 25 | local cb = circular_buffer.new(1000, 10, 1) 26 | elseif msg == "set() out of range column" then 27 | local cb = circular_buffer.new(2, 1, 1) 28 | cb:set(0, 2, 1.0) 29 | elseif msg == "set() zero column" then 30 | local cb = circular_buffer.new(2, 1, 1) 31 | cb:set(0, 0, 1.0) 32 | elseif msg == "set() non numeric column" then 33 | local cb = circular_buffer.new(2, 1, 1) 34 | cb:set(0, nil, 1.0) 35 | elseif msg == "set() non numeric time" then 36 | local cb = circular_buffer.new(2, 1, 1) 37 | cb:set(nil, 1, 1.0) 38 | elseif msg == "get() invalid object" then 39 | local cb = circular_buffer.new(2, 1, 1) 40 | local invalid = 1 41 | cb.get(invalid, 1, 1) 42 | elseif msg == "set() non numeric value" then 43 | local cb = circular_buffer.new(2, 1, 1) 44 | cb:set(0, 1, nil) 45 | elseif msg == "set() incorrect # args" then 46 | local cb = circular_buffer.new(2, 1, 1) 47 | cb:set(0) 48 | elseif msg == "add() incorrect # args" then 49 | local cb = circular_buffer.new(2, 1, 1) 50 | cb:add(0) 51 | elseif msg == "get() incorrect # args" then 52 | local cb = circular_buffer.new(2, 1, 1) 53 | cb:get(0) 54 | elseif msg == "compute() incorrect # args" then 55 | local cb = circular_buffer.new(2, 1, 1) 56 | cb:compute(0) 57 | elseif msg == "compute() incorrect function" then 58 | local cb = circular_buffer.new(2, 1, 1) 59 | cb:compute("func", 1) 60 | elseif msg == "compute() incorrect column" then 61 | local cb = circular_buffer.new(2, 1, 1) 62 | cb:compute("sum", 0) 63 | elseif msg == "compute() start > end" then 64 | local cb = circular_buffer.new(2, 1, 1) 65 | cb:compute("sum", 1, 2e9, 1e9) 66 | end 67 | return 0 68 | end 69 | 70 | function timer_event() 71 | end 72 | -------------------------------------------------------------------------------- /pipeline/nagios_output.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Mike Trinkala (trink@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | package pipeline 16 | 17 | import ( 18 | "github.com/mozilla-services/heka/message" 19 | "net/http" 20 | "net/url" 21 | "strings" 22 | "time" 23 | ) 24 | 25 | type NagiosOutputConfig struct { 26 | // URL to the Nagios cmd.cgi 27 | Url string 28 | // Nagios username 29 | Username string 30 | // Nagios password 31 | Password string 32 | // Http ResponseHeaderTimeout in seconds 33 | ResponseHeaderTimeout uint 34 | } 35 | 36 | func (n *NagiosOutput) ConfigStruct() interface{} { 37 | return &NagiosOutputConfig{ 38 | Url: "http://localhost/cgi-bin/cmd.cgi", 39 | ResponseHeaderTimeout: 2, 40 | } 41 | } 42 | 43 | type NagiosOutput struct { 44 | conf *NagiosOutputConfig 45 | client *http.Client 46 | transport *http.Transport 47 | } 48 | 49 | func (n *NagiosOutput) Init(config interface{}) (err error) { 50 | n.conf = config.(*NagiosOutputConfig) 51 | n.transport = &http.Transport{ 52 | Proxy: http.ProxyFromEnvironment, 53 | ResponseHeaderTimeout: time.Duration(n.conf.ResponseHeaderTimeout) * time.Second} 54 | n.client = &http.Client{Transport: n.transport} 55 | return 56 | } 57 | 58 | func (n *NagiosOutput) Run(or OutputRunner, h PluginHelper) (err error) { 59 | inChan := or.InChan() 60 | 61 | var ( 62 | pack *PipelinePack 63 | msg *message.Message 64 | payload string 65 | ) 66 | 67 | for pack = range inChan { 68 | msg = pack.Message 69 | payload = msg.GetPayload() 70 | pos := strings.IndexAny(payload, ":") 71 | state := "3" // UNKNOWN 72 | if pos != -1 { 73 | switch payload[:pos] { 74 | case "OK": 75 | state = "0" 76 | case "WARNING": 77 | state = "1" 78 | case "CRITICAL": 79 | state = "2" 80 | } 81 | } 82 | 83 | data := url.Values{ 84 | "cmd_typ": {"30"}, // PROCESS_SERVICE_CHECK_RESULT 85 | "cmd_mod": {"2"}, // CMDMODE_COMMIT 86 | "host": {msg.GetHostname()}, 87 | "service": {msg.GetLogger()}, 88 | "plugin_state": {state}, 89 | "plugin_output": {payload[pos+1:]}, 90 | "performance_data": {""}} 91 | req, err := http.NewRequest("POST", n.conf.Url, 92 | strings.NewReader(data.Encode())) 93 | if err == nil { 94 | req.SetBasicAuth(n.conf.Username, n.conf.Password) 95 | if resp, err := n.client.Do(req); err == nil { 96 | resp.Body.Close() 97 | } else { 98 | or.LogError(err) 99 | } 100 | } else { 101 | or.LogError(err) 102 | } 103 | pack.Recycle() 104 | } 105 | return 106 | } 107 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/circular_buffer.lua: -------------------------------------------------------------------------------- 1 | -- This Source Code Form is subject to the terms of the Mozilla Public 2 | -- License, v. 2.0. If a copy of the MPL was not distributed with this 3 | -- file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | data = circular_buffer.new(3, 3, 1) 6 | local ADD_COL = data:set_header(1, "Add column") 7 | local SET_COL = data:set_header(2, "Set column", "count") 8 | local GET_COL = data:set_header(3, "Get column", "count", "sum") 9 | 10 | function process_message() 11 | local ts = read_message("Timestamp") 12 | if data:add(ts, ADD_COL, 1) then 13 | data:set(ts, GET_COL, data:get(ts, ADD_COL)) 14 | end 15 | data:set(ts, SET_COL, 1) 16 | return 0 17 | end 18 | 19 | function timer_event(ns) 20 | if ns == 0 then 21 | output(data) 22 | inject_message("cbuf", "Method tests") 23 | elseif ns == 1 then 24 | cbufs = {} 25 | for i=1,3,1 do 26 | cbufs[i] = circular_buffer.new(2,1,1) 27 | cbufs[i]:set_header(1, "Header_1", "count") 28 | end 29 | elseif ns == 2 then 30 | output(cbufs[1]) 31 | inject_message("cbuf") 32 | elseif ns == 3 then 33 | local stats = circular_buffer.new(5, 1, 1) 34 | stats:set(1e9, 1, 1) 35 | stats:set(2e9, 1, 2) 36 | stats:set(3e9, 1, 3) 37 | stats:set(4e9, 1, 4) 38 | local t = stats:compute("sum", 1) 39 | if 10 ~= t then 40 | error(string.format("no range sum = %G", t)) 41 | end 42 | t = stats:compute("avg", 1) 43 | if 2 ~= t then 44 | error(string.format("no range avg = %G", t)) 45 | end 46 | t = stats:compute("sd", 1) 47 | if math.sqrt(2) ~= t then 48 | error(string.format("no range sd = %G", t)) 49 | end 50 | t = stats:compute("min", 1) 51 | if 0 ~= t then 52 | error(string.format("no range min = %G", t)) 53 | end 54 | t = stats:compute("max", 1) 55 | if 4 ~= t then 56 | error(string.format("no range max = %G", t)) 57 | end 58 | 59 | t = stats:compute("sum", 1, 3e9, 4e9) 60 | if 7 ~= t then 61 | error(string.format("range 3-4 sum = %G", t)) 62 | end 63 | t = stats:compute("avg", 1, 3e9, 4e9) 64 | if 3.5 ~= t then 65 | error(string.format("range 3-4 avg = %G", t)) 66 | end 67 | t = stats:compute("sd", 1, 3e9, 4e9) 68 | if math.sqrt(0.25) ~= t then 69 | error(string.format("range 3-4 sd = %G", t)) 70 | end 71 | 72 | t = stats:compute("sum", 1, 3e9) 73 | if 7 ~= t then 74 | error(string.format("range 3- = %G", t)) 75 | end 76 | t = stats:compute("sum", 1, 3e9, nil) 77 | if 7 ~= t then 78 | error(string.format("range 3-nil = %G", t)) 79 | end 80 | t = stats:compute("sum", 1, nil, 2e9) 81 | if 3 ~= t then 82 | error(string.format("range nil-2 sum = %G", t)) 83 | end 84 | t = stats:compute("sum", 1, 11e9, 14e9) 85 | if nil ~= t then 86 | error(string.format("out of range = %G", t)) 87 | end 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /pipeline/carbon.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Rob Miller (rmiller@mozilla.com) 12 | # Mike Trinkala (trink@mozilla.com) 13 | # Victor Ng (vng@mozilla.com) 14 | # 15 | # ***** END LICENSE BLOCK *****/ 16 | 17 | package pipeline 18 | 19 | import ( 20 | "bytes" 21 | "fmt" 22 | "net" 23 | "strconv" 24 | "strings" 25 | ) 26 | 27 | // Output plugin that sends statmetric messages via TCP 28 | type CarbonOutput struct { 29 | address string 30 | } 31 | 32 | // ConfigStruct for CarbonOutput plugin. 33 | type CarbonOutputConfig struct { 34 | // String representation of the TCP address to which this output should be 35 | // sending data. 36 | Address string 37 | } 38 | 39 | func (t *CarbonOutput) ConfigStruct() interface{} { 40 | return &CarbonOutputConfig{Address: "localhost:2003"} 41 | } 42 | 43 | func (t *CarbonOutput) Init(config interface{}) (err error) { 44 | conf := config.(*CarbonOutputConfig) 45 | t.address = conf.Address 46 | 47 | _, err = net.ResolveTCPAddr("tcp", t.address) 48 | 49 | return 50 | } 51 | 52 | func (t *CarbonOutput) ProcessPack(pack *PipelinePack, or OutputRunner) { 53 | var e error 54 | 55 | lines := strings.Split(strings.Trim(pack.Message.GetPayload(), " \n"), "\n") 56 | pack.Recycle() // Once we've copied the payload we're done w/ the pack. 57 | 58 | clean_statmetrics := make([]string, len(lines)) 59 | index := 0 60 | for _, line := range lines { 61 | // `fields` should be " " 62 | fields := strings.Fields(line) 63 | if len(fields) != 3 || !strings.HasPrefix(fields[0], "stats") { 64 | or.LogError(fmt.Errorf("malformed statmetric line: '%s'", line)) 65 | continue 66 | } 67 | 68 | if _, e = strconv.ParseUint(fields[2], 0, 32); e != nil { 69 | or.LogError(fmt.Errorf("parsing time: %s", e)) 70 | continue 71 | } 72 | if _, e = strconv.ParseFloat(fields[1], 64); e != nil { 73 | or.LogError(fmt.Errorf("parsing value '%s': %s", fields[1], e)) 74 | continue 75 | } 76 | clean_statmetrics[index] = line 77 | index += 1 78 | } 79 | clean_statmetrics = clean_statmetrics[:index] 80 | 81 | conn, err := net.Dial("tcp", t.address) 82 | if err != nil { 83 | or.LogError(fmt.Errorf("Dial failed: %s", 84 | err.Error())) 85 | return 86 | } 87 | defer conn.Close() 88 | 89 | // Stuff each parseable statmetric into a bytebuffer 90 | var buffer bytes.Buffer 91 | for i := 0; i < len(clean_statmetrics); i++ { 92 | buffer.WriteString(clean_statmetrics[i] + "\n") 93 | } 94 | 95 | _, err = conn.Write(buffer.Bytes()) 96 | if err != nil { 97 | or.LogError(fmt.Errorf("Write to server failed: %s", 98 | err.Error())) 99 | } 100 | } 101 | 102 | func (t *CarbonOutput) Run(or OutputRunner, h PluginHelper) (err error) { 103 | 104 | var ( 105 | pack *PipelinePack 106 | ) 107 | 108 | for pack = range or.InChan() { 109 | t.ProcessPack(pack, or) 110 | } 111 | 112 | return 113 | } 114 | -------------------------------------------------------------------------------- /testsupport/mock_net_conn.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: net (interfaces: Conn) 3 | 4 | package testsupport 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | net "net" 9 | time "time" 10 | ) 11 | 12 | // Mock of Conn interface 13 | type MockConn struct { 14 | ctrl *gomock.Controller 15 | recorder *_MockConnRecorder 16 | } 17 | 18 | // Recorder for MockConn (not exported) 19 | type _MockConnRecorder struct { 20 | mock *MockConn 21 | } 22 | 23 | func NewMockConn(ctrl *gomock.Controller) *MockConn { 24 | mock := &MockConn{ctrl: ctrl} 25 | mock.recorder = &_MockConnRecorder{mock} 26 | return mock 27 | } 28 | 29 | func (_m *MockConn) EXPECT() *_MockConnRecorder { 30 | return _m.recorder 31 | } 32 | 33 | func (_m *MockConn) Close() error { 34 | ret := _m.ctrl.Call(_m, "Close") 35 | ret0, _ := ret[0].(error) 36 | return ret0 37 | } 38 | 39 | func (_mr *_MockConnRecorder) Close() *gomock.Call { 40 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Close") 41 | } 42 | 43 | func (_m *MockConn) LocalAddr() net.Addr { 44 | ret := _m.ctrl.Call(_m, "LocalAddr") 45 | ret0, _ := ret[0].(net.Addr) 46 | return ret0 47 | } 48 | 49 | func (_mr *_MockConnRecorder) LocalAddr() *gomock.Call { 50 | return _mr.mock.ctrl.RecordCall(_mr.mock, "LocalAddr") 51 | } 52 | 53 | func (_m *MockConn) Read(_param0 []byte) (int, error) { 54 | ret := _m.ctrl.Call(_m, "Read", _param0) 55 | ret0, _ := ret[0].(int) 56 | ret1, _ := ret[1].(error) 57 | return ret0, ret1 58 | } 59 | 60 | func (_mr *_MockConnRecorder) Read(arg0 interface{}) *gomock.Call { 61 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Read", arg0) 62 | } 63 | 64 | func (_m *MockConn) RemoteAddr() net.Addr { 65 | ret := _m.ctrl.Call(_m, "RemoteAddr") 66 | ret0, _ := ret[0].(net.Addr) 67 | return ret0 68 | } 69 | 70 | func (_mr *_MockConnRecorder) RemoteAddr() *gomock.Call { 71 | return _mr.mock.ctrl.RecordCall(_mr.mock, "RemoteAddr") 72 | } 73 | 74 | func (_m *MockConn) SetDeadline(_param0 time.Time) error { 75 | ret := _m.ctrl.Call(_m, "SetDeadline", _param0) 76 | ret0, _ := ret[0].(error) 77 | return ret0 78 | } 79 | 80 | func (_mr *_MockConnRecorder) SetDeadline(arg0 interface{}) *gomock.Call { 81 | return _mr.mock.ctrl.RecordCall(_mr.mock, "SetDeadline", arg0) 82 | } 83 | 84 | func (_m *MockConn) SetReadDeadline(_param0 time.Time) error { 85 | ret := _m.ctrl.Call(_m, "SetReadDeadline", _param0) 86 | ret0, _ := ret[0].(error) 87 | return ret0 88 | } 89 | 90 | func (_mr *_MockConnRecorder) SetReadDeadline(arg0 interface{}) *gomock.Call { 91 | return _mr.mock.ctrl.RecordCall(_mr.mock, "SetReadDeadline", arg0) 92 | } 93 | 94 | func (_m *MockConn) SetWriteDeadline(_param0 time.Time) error { 95 | ret := _m.ctrl.Call(_m, "SetWriteDeadline", _param0) 96 | ret0, _ := ret[0].(error) 97 | return ret0 98 | } 99 | 100 | func (_mr *_MockConnRecorder) SetWriteDeadline(arg0 interface{}) *gomock.Call { 101 | return _mr.mock.ctrl.RecordCall(_mr.mock, "SetWriteDeadline", arg0) 102 | } 103 | 104 | func (_m *MockConn) Write(_param0 []byte) (int, error) { 105 | ret := _m.ctrl.Call(_m, "Write", _param0) 106 | ret0, _ := ret[0].(int) 107 | ret1, _ := ret[1].(error) 108 | return ret0, ret1 109 | } 110 | 111 | func (_mr *_MockConnRecorder) Write(arg0 interface{}) *gomock.Call { 112 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Write", arg0) 113 | } 114 | -------------------------------------------------------------------------------- /pipeline/http_input.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "time" 8 | ) 9 | 10 | type HttpInput struct { 11 | dataChan chan []byte 12 | stopChan chan bool 13 | Monitor *HttpInputMonitor 14 | decoderName string 15 | } 16 | 17 | type HttpInputConfig struct { 18 | Url string 19 | TickerInterval uint `toml:"ticker_interval"` 20 | DecoderName string `toml:"decoder"` 21 | } 22 | 23 | func (hi *HttpInput) ConfigStruct() interface{} { 24 | return &HttpInputConfig{ 25 | TickerInterval: uint(10), 26 | } 27 | } 28 | 29 | func (hi *HttpInput) Init(config interface{}) error { 30 | conf := config.(*HttpInputConfig) 31 | 32 | hi.dataChan = make(chan []byte) 33 | hi.stopChan = make(chan bool) 34 | hi.decoderName = conf.DecoderName 35 | 36 | hi.Monitor = new(HttpInputMonitor) 37 | 38 | hi.Monitor.Init(conf.Url, hi.dataChan, hi.stopChan) 39 | return nil 40 | } 41 | 42 | func (hi *HttpInput) Run(ir InputRunner, h PluginHelper) (err error) { 43 | var ( 44 | pack *PipelinePack 45 | dRunner DecoderRunner 46 | ok bool 47 | router_shortcircuit bool 48 | ) 49 | 50 | ir.LogMessage(fmt.Sprintf("[HttpInput (%s)] Running...", 51 | hi.Monitor.url)) 52 | 53 | go hi.Monitor.Monitor(ir) 54 | 55 | pConfig := h.PipelineConfig() 56 | hostname := pConfig.hostname 57 | packSupply := ir.InChan() 58 | 59 | dSet := h.DecoderSet() 60 | if hi.decoderName == "" { 61 | router_shortcircuit = true 62 | } else if dRunner, ok = dSet.ByName(hi.decoderName); !ok { 63 | return fmt.Errorf("Decoder not found: %s", hi.decoderName) 64 | } 65 | 66 | for { 67 | select { 68 | case data := <-hi.dataChan: 69 | pack = <-packSupply 70 | pack.Message.SetType("heka.httpdata") 71 | pack.Message.SetHostname(hostname) 72 | pack.Message.SetPayload(string(data)) 73 | if router_shortcircuit { 74 | pConfig.router.InChan() <- pack 75 | } else { 76 | dRunner.InChan() <- pack 77 | } 78 | case <-hi.stopChan: 79 | return 80 | } 81 | } 82 | 83 | return nil 84 | } 85 | 86 | func (hi *HttpInput) Stop() { 87 | close(hi.stopChan) 88 | } 89 | 90 | type HttpInputMonitor struct { 91 | url string 92 | dataChan chan []byte 93 | stopChan chan bool 94 | 95 | ir InputRunner 96 | tickChan <-chan time.Time 97 | } 98 | 99 | func (hm *HttpInputMonitor) Init(url string, dataChan chan []byte, stopChan chan bool) { 100 | hm.url = url 101 | hm.dataChan = dataChan 102 | hm.stopChan = stopChan 103 | } 104 | 105 | func (hm *HttpInputMonitor) Monitor(ir InputRunner) { 106 | ir.LogMessage("[HttpInputMonitor] Monitoring...") 107 | 108 | hm.ir = ir 109 | hm.tickChan = ir.Ticker() 110 | 111 | for { 112 | select { 113 | case <-hm.tickChan: 114 | // Fetch URL 115 | resp, err := http.Get(hm.url) 116 | if err != nil { 117 | ir.LogError(fmt.Errorf("[HttpInputMonitor] %s", err)) 118 | continue 119 | } 120 | 121 | // Read content 122 | body, err := ioutil.ReadAll(resp.Body) 123 | if err != nil { 124 | ir.LogError(fmt.Errorf("[HttpInputMonitor] [%s]", err.Error())) 125 | continue 126 | } 127 | resp.Body.Close() 128 | 129 | // Send it on the channel 130 | hm.dataChan <- body 131 | 132 | case <-hm.stopChan: 133 | ir.LogMessage(fmt.Sprintf("[HttpInputMonitor (%s)] Stop", hm.url)) 134 | return 135 | } 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /docs/source/glossary.rst: -------------------------------------------------------------------------------- 1 | .. _glossary: 2 | 3 | Glossary 4 | ======== 5 | 6 | .. glossary:: 7 | :sorted: 8 | 9 | DecoderPoolSize 10 | A Heka configuration setting which specifies the number of decoder 11 | sets that should be created to handle decoding of incoming message 12 | data. 13 | 14 | hekad 15 | Daemon that routes messages from inputs to their outputs applying 16 | filters as configured. 17 | 18 | Message 19 | A message is the atomic unit of data that Hekad deals with. It 20 | is a data structure related to a single event happening in the 21 | outside world, such as a log file entry, a counter increment, 22 | an application exception, a notification message, etc. It is 23 | specified as a `Message` struct in the `heka/message` packages 24 | `message.go `_ file. 26 | 27 | Message matcher 28 | A configuration option for filter and output plugins that specifies 29 | which messages that plugin accepts for processing. The Heka router 30 | will evaluate the message matchers against every message to and will 31 | deliver the message when the match is positive. 32 | 33 | Pipeline 34 | Messages being processed by Hekad are passed through a specific set of 35 | plugins. A set of plugins to be applied to a message is often called 36 | (somewhat informally) a Heka pipeline. 37 | 38 | PipelinePack 39 | In addition to the core message data, Hekad needs to track some 40 | related state and configuration information for each message. To this 41 | end there is a `PipelinePack` struct defined in the `heka/pipeline` 42 | package's `pipeline_runner.go `_ file. 44 | `PipelinePack` objects are what get passed in to the various Hekad 45 | plugins as messages flow through the pipelines. 46 | 47 | Plugin 48 | Hekad plugins are functional units that perform specific actions on or 49 | with messages. There are four distinct types of plugins: inputs, 50 | decoders, filters, and outputs. 51 | 52 | PluginChanSize 53 | A Heka configuration setting which specifies the size of the input 54 | channel buffer for the various Heka plugins. Defaults to 50. 55 | 56 | PluginHelper 57 | An interface that provides access to certain Heka internals that may 58 | be required by plugins in the course of their activity. Defined in 59 | `config.go `_. 61 | 62 | PluginRunner 63 | A plugin-specific helper object that manages the lifespan of a given 64 | plugin and handles most details of interaction w/ the greater Heka 65 | environment. Comes in four variants, each tailored to a specific 66 | plugin type (i.e. `InputRunner`, `DecoderRunner`, `FilterRunner`, 67 | `OutputRunner`). 68 | 69 | PoolSize 70 | A Heka configuration setting which specifies the number of 71 | `PipelinePack` structs that will be created. This value specifies the 72 | maximum number of incoming messages that Heka can be processing at any 73 | one time. 74 | 75 | Router 76 | Component in the Heka pipeline that accepts messages and delivers them 77 | to the appropriate filter and output plugins, as specified by the 78 | plugins' message matcher values. 79 | -------------------------------------------------------------------------------- /client/encoders.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Rob Miller (rmiller@mozilla.com) 12 | # Mike Trinkala (trink@mozilla.com) 13 | # 14 | # ***** END LICENSE BLOCK *****/ 15 | 16 | package client 17 | 18 | import ( 19 | "code.google.com/p/goprotobuf/proto" 20 | "crypto/hmac" 21 | "crypto/md5" 22 | "crypto/sha1" 23 | "encoding/json" 24 | "github.com/mozilla-services/heka/message" 25 | "hash" 26 | ) 27 | 28 | type Encoder interface { 29 | EncodeMessage(msg *message.Message) ([]byte, error) 30 | EncodeMessageStream(msg *message.Message, outBytes *[]byte) error 31 | } 32 | 33 | type JsonEncoder struct { 34 | signer *message.MessageSigningConfig 35 | } 36 | 37 | func NewJsonEncoder(signer *message.MessageSigningConfig) *JsonEncoder { 38 | return &JsonEncoder{signer} 39 | } 40 | 41 | func (self *JsonEncoder) EncodeMessage(msg *message.Message) ([]byte, error) { 42 | return json.Marshal(msg) 43 | } 44 | 45 | func (self *JsonEncoder) EncodeMessageStream(msg *message.Message, outBytes *[]byte) (err error) { 46 | msgBytes, err := self.EncodeMessage(msg) 47 | if err == nil { 48 | err = createStream(msgBytes, message.Header_JSON, outBytes, self.signer) 49 | } 50 | return 51 | } 52 | 53 | type ProtobufEncoder struct { 54 | signer *message.MessageSigningConfig 55 | } 56 | 57 | func NewProtobufEncoder(signer *message.MessageSigningConfig) *ProtobufEncoder { 58 | return &ProtobufEncoder{signer} 59 | } 60 | 61 | func (self *ProtobufEncoder) EncodeMessage(msg *message.Message) ([]byte, error) { 62 | return proto.Marshal(msg) 63 | } 64 | 65 | func (self *ProtobufEncoder) EncodeMessageStream(msg *message.Message, outBytes *[]byte) (err error) { 66 | msgBytes, err := self.EncodeMessage(msg) // TODO if we compute the size of the header first this can be marshaled directly to outBytes 67 | if err == nil { 68 | err = createStream(msgBytes, message.Header_PROTOCOL_BUFFER, outBytes, self.signer) 69 | } 70 | return 71 | } 72 | 73 | func createStream(msgBytes []byte, encoding message.Header_MessageEncoding, 74 | outBytes *[]byte, msc *message.MessageSigningConfig) error { 75 | h := &message.Header{} 76 | h.SetMessageLength(uint32(len(msgBytes))) 77 | if encoding != message.Default_Header_MessageEncoding { 78 | h.SetMessageEncoding(encoding) 79 | } 80 | if msc != nil { 81 | h.SetHmacSigner(msc.Name) 82 | h.SetHmacKeyVersion(msc.Version) 83 | var hm hash.Hash 84 | switch msc.Hash { 85 | case "sha1": 86 | hm = hmac.New(sha1.New, []byte(msc.Key)) 87 | h.SetHmacHashFunction(message.Header_SHA1) 88 | default: 89 | hm = hmac.New(md5.New, []byte(msc.Key)) 90 | } 91 | 92 | hm.Write(msgBytes) 93 | h.SetHmac(hm.Sum(nil)) 94 | } 95 | headerSize := uint8(proto.Size(h)) 96 | requiredSize := int(3 + headerSize) 97 | if cap(*outBytes) < requiredSize { 98 | *outBytes = make([]byte, requiredSize, requiredSize+len(msgBytes)) 99 | } else { 100 | *outBytes = (*outBytes)[:requiredSize] 101 | } 102 | (*outBytes)[0] = message.RECORD_SEPARATOR 103 | (*outBytes)[1] = uint8(headerSize) 104 | pbuf := proto.NewBuffer((*outBytes)[2:2]) 105 | err := pbuf.Marshal(h) 106 | if err != nil { 107 | return err 108 | } 109 | (*outBytes)[headerSize+2] = message.UNIT_SEPARATOR 110 | *outBytes = append(*outBytes, msgBytes...) 111 | return nil 112 | } 113 | -------------------------------------------------------------------------------- /pipeline/carbon_test.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Rob Miller (rmiller@mozilla.com) 12 | # Mike Trinkala (trink@mozilla.com) 13 | # Victor Ng (vng@mozilla.com) 14 | # 15 | # ***** END LICENSE BLOCK *****/ 16 | 17 | package pipeline 18 | 19 | import ( 20 | "code.google.com/p/gomock/gomock" 21 | "fmt" 22 | ts "github.com/mozilla-services/heka/testsupport" 23 | gs "github.com/rafrombrc/gospec/src/gospec" 24 | "net" 25 | "strings" 26 | "sync" 27 | "time" 28 | ) 29 | 30 | type CarbonTestHelper struct { 31 | MockHelper *MockPluginHelper 32 | MockOutputRunner *MockOutputRunner 33 | } 34 | 35 | func NewCarbonTestHelper(ctrl *gomock.Controller) (oth *CarbonTestHelper) { 36 | oth = new(CarbonTestHelper) 37 | oth.MockHelper = NewMockPluginHelper(ctrl) 38 | oth.MockOutputRunner = NewMockOutputRunner(ctrl) 39 | return 40 | } 41 | 42 | func CarbonOutputSpec(c gs.Context) { 43 | t := new(ts.SimpleT) 44 | ctrl := gomock.NewController(t) 45 | defer ctrl.Finish() 46 | 47 | oth := NewCarbonTestHelper(ctrl) 48 | var wg sync.WaitGroup 49 | inChan := make(chan *PipelinePack, 1) 50 | pConfig := NewPipelineConfig(nil) 51 | 52 | c.Specify("A CarbonOutput", func() { 53 | carbonOutput := new(CarbonOutput) 54 | config := carbonOutput.ConfigStruct().(*CarbonOutputConfig) 55 | 56 | const count = 5 57 | lines := make([]string, count) 58 | baseTime := time.Now().UTC().Add(-10 * time.Second) 59 | nameTmpl := "stats.name.%d" 60 | 61 | for i := 0; i < count; i++ { 62 | statName := fmt.Sprintf(nameTmpl, i) 63 | statTime := baseTime.Add(time.Duration(i) * time.Second) 64 | lines[i] = fmt.Sprintf("%s %d %d", statName, i*2, statTime.Unix()) 65 | } 66 | 67 | msg := getTestMessage() 68 | pack := NewPipelinePack(pConfig.inputRecycleChan) 69 | pack.Message = msg 70 | pack.Decoded = true 71 | 72 | c.Specify("writes out to the network", func() { 73 | inChanCall := oth.MockOutputRunner.EXPECT().InChan().AnyTimes() 74 | inChanCall.Return(inChan) 75 | 76 | collectData := func(ch chan string) { 77 | ln, err := net.Listen("tcp", "localhost:2003") 78 | if err != nil { 79 | ch <- err.Error() 80 | } 81 | ch <- "ready" 82 | for i := 0; i < count; i++ { 83 | conn, err := ln.Accept() 84 | if err != nil { 85 | ch <- err.Error() 86 | } 87 | b := make([]byte, 1000) 88 | n, _ := conn.Read(b) 89 | ch <- string(b[0:n]) 90 | } 91 | } 92 | ch := make(chan string, count) // don't block on put 93 | go collectData(ch) 94 | result := <-ch // wait for server 95 | 96 | err := carbonOutput.Init(config) 97 | c.Assume(err, gs.IsNil) 98 | 99 | pack.Message.SetPayload(strings.Join(lines, "\n")) 100 | 101 | go func() { 102 | wg.Add(1) 103 | carbonOutput.Run(oth.MockOutputRunner, oth.MockHelper) 104 | wg.Done() 105 | }() 106 | inChan <- pack 107 | 108 | close(inChan) 109 | wg.Wait() // wait for close to finish, prevents intermittent test failures 110 | 111 | matchBytes := make([]byte, 0, 1000) 112 | err = createProtobufStream(pack, &matchBytes) 113 | c.Expect(err, gs.IsNil) 114 | 115 | result = <-ch 116 | computed_result := strings.Join(lines, "\n") + "\n" 117 | c.Expect(result, gs.Equals, computed_result) 118 | }) 119 | }) 120 | 121 | } 122 | -------------------------------------------------------------------------------- /pipeline/mock_decoderrunner_test.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: github.com/mozilla-services/heka/pipeline (interfaces: DecoderRunner) 3 | 4 | package pipeline 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | sync "sync" 9 | ) 10 | 11 | // Mock of DecoderRunner interface 12 | type MockDecoderRunner struct { 13 | ctrl *gomock.Controller 14 | recorder *_MockDecoderRunnerRecorder 15 | } 16 | 17 | // Recorder for MockDecoderRunner (not exported) 18 | type _MockDecoderRunnerRecorder struct { 19 | mock *MockDecoderRunner 20 | } 21 | 22 | func NewMockDecoderRunner(ctrl *gomock.Controller) *MockDecoderRunner { 23 | mock := &MockDecoderRunner{ctrl: ctrl} 24 | mock.recorder = &_MockDecoderRunnerRecorder{mock} 25 | return mock 26 | } 27 | 28 | func (_m *MockDecoderRunner) EXPECT() *_MockDecoderRunnerRecorder { 29 | return _m.recorder 30 | } 31 | 32 | func (_m *MockDecoderRunner) Decoder() Decoder { 33 | ret := _m.ctrl.Call(_m, "Decoder") 34 | ret0, _ := ret[0].(Decoder) 35 | return ret0 36 | } 37 | 38 | func (_mr *_MockDecoderRunnerRecorder) Decoder() *gomock.Call { 39 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Decoder") 40 | } 41 | 42 | func (_m *MockDecoderRunner) InChan() chan *PipelinePack { 43 | ret := _m.ctrl.Call(_m, "InChan") 44 | ret0, _ := ret[0].(chan *PipelinePack) 45 | return ret0 46 | } 47 | 48 | func (_mr *_MockDecoderRunnerRecorder) InChan() *gomock.Call { 49 | return _mr.mock.ctrl.RecordCall(_mr.mock, "InChan") 50 | } 51 | 52 | func (_m *MockDecoderRunner) LogError(_param0 error) { 53 | _m.ctrl.Call(_m, "LogError", _param0) 54 | } 55 | 56 | func (_mr *_MockDecoderRunnerRecorder) LogError(arg0 interface{}) *gomock.Call { 57 | return _mr.mock.ctrl.RecordCall(_mr.mock, "LogError", arg0) 58 | } 59 | 60 | func (_m *MockDecoderRunner) LogMessage(_param0 string) { 61 | _m.ctrl.Call(_m, "LogMessage", _param0) 62 | } 63 | 64 | func (_mr *_MockDecoderRunnerRecorder) LogMessage(arg0 interface{}) *gomock.Call { 65 | return _mr.mock.ctrl.RecordCall(_mr.mock, "LogMessage", arg0) 66 | } 67 | 68 | func (_m *MockDecoderRunner) Name() string { 69 | ret := _m.ctrl.Call(_m, "Name") 70 | ret0, _ := ret[0].(string) 71 | return ret0 72 | } 73 | 74 | func (_mr *_MockDecoderRunnerRecorder) Name() *gomock.Call { 75 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Name") 76 | } 77 | 78 | func (_m *MockDecoderRunner) Plugin() Plugin { 79 | ret := _m.ctrl.Call(_m, "Plugin") 80 | ret0, _ := ret[0].(Plugin) 81 | return ret0 82 | } 83 | 84 | func (_mr *_MockDecoderRunnerRecorder) Plugin() *gomock.Call { 85 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Plugin") 86 | } 87 | 88 | func (_m *MockDecoderRunner) PluginGlobals() *PluginGlobals { 89 | ret := _m.ctrl.Call(_m, "PluginGlobals") 90 | ret0, _ := ret[0].(*PluginGlobals) 91 | return ret0 92 | } 93 | 94 | func (_mr *_MockDecoderRunnerRecorder) PluginGlobals() *gomock.Call { 95 | return _mr.mock.ctrl.RecordCall(_mr.mock, "PluginGlobals") 96 | } 97 | 98 | func (_m *MockDecoderRunner) SetName(_param0 string) { 99 | _m.ctrl.Call(_m, "SetName", _param0) 100 | } 101 | 102 | func (_mr *_MockDecoderRunnerRecorder) SetName(arg0 interface{}) *gomock.Call { 103 | return _mr.mock.ctrl.RecordCall(_mr.mock, "SetName", arg0) 104 | } 105 | 106 | func (_m *MockDecoderRunner) Start(_param0 PluginHelper, _param1 *sync.WaitGroup) { 107 | _m.ctrl.Call(_m, "Start", _param0, _param1) 108 | } 109 | 110 | func (_mr *_MockDecoderRunnerRecorder) Start(arg0, arg1 interface{}) *gomock.Call { 111 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Start", arg0, arg1) 112 | } 113 | 114 | func (_m *MockDecoderRunner) UUID() string { 115 | ret := _m.ctrl.Call(_m, "UUID") 116 | ret0, _ := ret[0].(string) 117 | return ret0 118 | } 119 | 120 | func (_mr *_MockDecoderRunnerRecorder) UUID() *gomock.Call { 121 | return _mr.mock.ctrl.RecordCall(_mr.mock, "UUID") 122 | } 123 | -------------------------------------------------------------------------------- /pipeline/stat_filter.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Ben Bangert (bbangert@mozilla.com) 12 | # Mike Trinkala (trink@mozilla.com) 13 | # Rob Miller (rmiller@mozilla.com) 14 | # 15 | # ***** END LICENSE BLOCK *****/ 16 | 17 | package pipeline 18 | 19 | import ( 20 | "fmt" 21 | "github.com/mozilla-services/heka/message" 22 | ) 23 | 24 | // Simple struct representing a single statsd-style metric value. 25 | type metric struct { 26 | // Supports "Counter", "Timer", or "Gauge" 27 | Type_ string `toml:"type"` 28 | Name string 29 | Value string 30 | } 31 | 32 | // Heka Filter plugin that can accept specific message types, extract data 33 | // from those messages, and from that data generate statsd messages in a 34 | // StatsdInput exactly as if a statsd message has come from a networked statsd 35 | // client. 36 | type StatFilter struct { 37 | metrics map[string]metric 38 | statAccumName string 39 | } 40 | 41 | // StatFilter config struct. 42 | type StatFilterConfig struct { 43 | // Set of metric templates this filter should use, keyed by arbitrary 44 | // metric id. 45 | Metric map[string]metric 46 | // Configured name of StatAccumInput plugin to which this filter should be 47 | // delivering its stats. Defaults to "StatsAccumInput". 48 | StatAccumName string `toml:"stat_accum_input"` 49 | } 50 | 51 | func (s *StatFilter) ConfigStruct() interface{} { 52 | return &StatFilterConfig{ 53 | StatAccumName: "StatAccumInput", 54 | } 55 | } 56 | 57 | func (s *StatFilter) Init(config interface{}) (err error) { 58 | conf := config.(*StatFilterConfig) 59 | s.metrics = conf.Metric 60 | s.statAccumName = conf.StatAccumName 61 | return 62 | } 63 | 64 | // For each message, we first extract any match group captures, and then we 65 | // add our own values for "Logger", "Hostname", "Type", and "Payload" as if 66 | // they were captured values. We then iterate through all of this plugin's 67 | // defined metrics, and for each one we use the captures to do string 68 | // substitution on both the name and the payload. For example, a metric with 69 | // the name "@Hostname.404s" would become a stat with the "@Hostname" replaced 70 | // by the hostname from the received message. 71 | func (s *StatFilter) Run(fr FilterRunner, h PluginHelper) (err error) { 72 | var statAccum StatAccumulator 73 | if statAccum, err = h.StatAccumulator(s.statAccumName); err != nil { 74 | return 75 | } 76 | 77 | var ( 78 | pack *PipelinePack 79 | values = make(map[string]string) 80 | stat Stat 81 | ) 82 | 83 | inChan := fr.InChan() 84 | for pack = range inChan { 85 | // Load existing values into the set for replacement 86 | values["Logger"] = pack.Message.GetLogger() 87 | values["Hostname"] = pack.Message.GetHostname() 88 | values["Type"] = pack.Message.GetType() 89 | values["Payload"] = pack.Message.GetPayload() 90 | 91 | for _, field := range pack.Message.Fields { 92 | if field.GetValueType() == message.Field_STRING && len(field.ValueString) > 0 { 93 | values[field.GetName()] = field.ValueString[0] 94 | } 95 | } 96 | 97 | // We matched, generate appropriate metrics 98 | for _, met := range s.metrics { 99 | stat.Bucket = InterpolateString(met.Name, values) 100 | switch met.Type_ { 101 | case "Counter": 102 | stat.Modifier = "" 103 | case "Timer": 104 | stat.Modifier = "ms" 105 | case "Gauge": 106 | stat.Modifier = "g" 107 | } 108 | stat.Value = InterpolateString(met.Value, values) 109 | stat.Sampling = 1.0 110 | if !statAccum.DropStat(stat) { 111 | fr.LogError(fmt.Errorf("Undelivered stat: %s", stat)) 112 | } 113 | } 114 | pack.Recycle() 115 | } 116 | 117 | return 118 | } 119 | -------------------------------------------------------------------------------- /sandbox/lua/lua_sandbox.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 2 | /* vim: set ts=2 et sw=2 tw=80: */ 3 | /* This Source Code Form is subject to the terms of the Mozilla Public 4 | * License, v. 2.0. If a copy of the MPL was not distributed with this 5 | * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 6 | 7 | /// Lua lua_sandbox for Heka plugins @file 8 | #ifndef lua_sandbox_h_ 9 | #define lua_sandbox_h_ 10 | 11 | #include 12 | #include "../sandbox.h" 13 | 14 | typedef struct lua_sandbox lua_sandbox; 15 | 16 | /** 17 | * Allocates and initializes the structure around the Lua sandbox. 18 | * 19 | * @param go Pointer to associate the Go struct to this sandbox. 20 | * @param lua_file Filename of the Lua script to run in this sandbox. 21 | * @param memory_limit Sets the sandbox memory limit (bytes). 22 | * @param instruction_limit Sets the sandbox Lua instruction limit (count). 23 | * This limit is per call to process_message or timer_event 24 | * @param output_limit Sets the single message payload limit (bytes). This 25 | * limit applies to the in memory output buffer. The buffer is reset back 26 | * to zero when inject_message is called. 27 | * 28 | * @return lua_sandbox Sandbox pointer or NULL on failure. 29 | */ 30 | lua_sandbox* lua_sandbox_create(void* go, 31 | const char* lua_file, 32 | unsigned memory_limit, 33 | unsigned instruction_limit, 34 | unsigned output_limit); 35 | 36 | /** 37 | * Frees the memory associated with the sandbox. 38 | * 39 | * @param lsb Sandbox pointer to discard. 40 | * @param state_file Filename where the sandbox global data is saved. Use a 41 | * NULL or empty string for no preservation. 42 | * 43 | * @return NULL on success, pointer to an error message on failure that MUST BE 44 | * FREED by the caller. 45 | */ 46 | char* lua_sandbox_destroy(lua_sandbox* lsb, 47 | const char* state_file); 48 | 49 | /** 50 | * Initializes the Lua sandbox and loads/runs the Lua script that was specified 51 | * in lua_create_sandbox. 52 | * 53 | * @param lsb Pointer to the sandbox. 54 | * @param state_file Filename where the global data is read. Use a NULL or empty 55 | * string no data restoration. 56 | * 57 | * @return int Zero on success, non-zero on failure. 58 | */ 59 | int lua_sandbox_init(lua_sandbox* lsb, const char* state_file); 60 | 61 | /** 62 | * Retrieve the sandbox usage statistics. 63 | * 64 | * @param lsb Pointer to the sandbox. 65 | * @param sandbox_usage_type Type of statistic to retrieve i.e. memory. 66 | * @param sandbox_usage_stat Type of statistic to retrieve i.e. current. 67 | * 68 | * @return unsigned Count or number of bytes depending on the statistic. 69 | */ 70 | unsigned lua_sandbox_usage(lua_sandbox* lsb, 71 | sandbox_usage_type utype, 72 | sandbox_usage_stat ustat); 73 | /** 74 | * Retrieve the current sandbox status. 75 | * 76 | * @param lsb Pointer to the sandbox. 77 | * 78 | * @return sandbox_status code 79 | */ 80 | sandbox_status lua_sandbox_status(lua_sandbox* lsb); 81 | 82 | /** 83 | * Return the last error in human readable form. 84 | * 85 | * @param lsb Pointer to the sandbox. 86 | * 87 | * @return const char* error message 88 | */ 89 | const char* lua_sandbox_last_error(lua_sandbox* lsb); 90 | 91 | /** 92 | * Passes a Heka message down to the sandbox for processing. The instruction 93 | * count limits are active during this call. 94 | * 95 | * @param lsb Pointer to the sandbox 96 | * 97 | * @return int Zero on success, non-zero on failure. 98 | */ 99 | int lua_sandbox_process_message(lua_sandbox* lsb); 100 | 101 | /** 102 | * Called when the plugin timer expires (the garbage collector is run after 103 | * its execution). The instruction count limits are active during this call. 104 | * 105 | * @param lsb Pointer to the sandbox. 106 | * 107 | * @return int Zero on success, non-zero on failure. 108 | * 109 | */ 110 | int lua_sandbox_timer_event(lua_sandbox* lsb, long long ns); 111 | 112 | #endif 113 | -------------------------------------------------------------------------------- /pipeline/mock_inputrunner_test.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: github.com/mozilla-services/heka/pipeline (interfaces: InputRunner) 3 | 4 | package pipeline 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | sync "sync" 9 | time "time" 10 | ) 11 | 12 | // Mock of InputRunner interface 13 | type MockInputRunner struct { 14 | ctrl *gomock.Controller 15 | recorder *_MockInputRunnerRecorder 16 | } 17 | 18 | // Recorder for MockInputRunner (not exported) 19 | type _MockInputRunnerRecorder struct { 20 | mock *MockInputRunner 21 | } 22 | 23 | func NewMockInputRunner(ctrl *gomock.Controller) *MockInputRunner { 24 | mock := &MockInputRunner{ctrl: ctrl} 25 | mock.recorder = &_MockInputRunnerRecorder{mock} 26 | return mock 27 | } 28 | 29 | func (_m *MockInputRunner) EXPECT() *_MockInputRunnerRecorder { 30 | return _m.recorder 31 | } 32 | 33 | func (_m *MockInputRunner) InChan() chan *PipelinePack { 34 | ret := _m.ctrl.Call(_m, "InChan") 35 | ret0, _ := ret[0].(chan *PipelinePack) 36 | return ret0 37 | } 38 | 39 | func (_mr *_MockInputRunnerRecorder) InChan() *gomock.Call { 40 | return _mr.mock.ctrl.RecordCall(_mr.mock, "InChan") 41 | } 42 | 43 | func (_m *MockInputRunner) Inject(_param0 *PipelinePack) { 44 | _m.ctrl.Call(_m, "Inject", _param0) 45 | } 46 | 47 | func (_mr *_MockInputRunnerRecorder) Inject(arg0 interface{}) *gomock.Call { 48 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Inject", arg0) 49 | } 50 | 51 | func (_m *MockInputRunner) Input() Input { 52 | ret := _m.ctrl.Call(_m, "Input") 53 | ret0, _ := ret[0].(Input) 54 | return ret0 55 | } 56 | 57 | func (_mr *_MockInputRunnerRecorder) Input() *gomock.Call { 58 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Input") 59 | } 60 | 61 | func (_m *MockInputRunner) LogError(_param0 error) { 62 | _m.ctrl.Call(_m, "LogError", _param0) 63 | } 64 | 65 | func (_mr *_MockInputRunnerRecorder) LogError(arg0 interface{}) *gomock.Call { 66 | return _mr.mock.ctrl.RecordCall(_mr.mock, "LogError", arg0) 67 | } 68 | 69 | func (_m *MockInputRunner) LogMessage(_param0 string) { 70 | _m.ctrl.Call(_m, "LogMessage", _param0) 71 | } 72 | 73 | func (_mr *_MockInputRunnerRecorder) LogMessage(arg0 interface{}) *gomock.Call { 74 | return _mr.mock.ctrl.RecordCall(_mr.mock, "LogMessage", arg0) 75 | } 76 | 77 | func (_m *MockInputRunner) Name() string { 78 | ret := _m.ctrl.Call(_m, "Name") 79 | ret0, _ := ret[0].(string) 80 | return ret0 81 | } 82 | 83 | func (_mr *_MockInputRunnerRecorder) Name() *gomock.Call { 84 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Name") 85 | } 86 | 87 | func (_m *MockInputRunner) Plugin() Plugin { 88 | ret := _m.ctrl.Call(_m, "Plugin") 89 | ret0, _ := ret[0].(Plugin) 90 | return ret0 91 | } 92 | 93 | func (_mr *_MockInputRunnerRecorder) Plugin() *gomock.Call { 94 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Plugin") 95 | } 96 | 97 | func (_m *MockInputRunner) PluginGlobals() *PluginGlobals { 98 | ret := _m.ctrl.Call(_m, "PluginGlobals") 99 | ret0, _ := ret[0].(*PluginGlobals) 100 | return ret0 101 | } 102 | 103 | func (_mr *_MockInputRunnerRecorder) PluginGlobals() *gomock.Call { 104 | return _mr.mock.ctrl.RecordCall(_mr.mock, "PluginGlobals") 105 | } 106 | 107 | func (_m *MockInputRunner) SetName(_param0 string) { 108 | _m.ctrl.Call(_m, "SetName", _param0) 109 | } 110 | 111 | func (_mr *_MockInputRunnerRecorder) SetName(arg0 interface{}) *gomock.Call { 112 | return _mr.mock.ctrl.RecordCall(_mr.mock, "SetName", arg0) 113 | } 114 | 115 | func (_m *MockInputRunner) SetTickLength(_param0 time.Duration) { 116 | _m.ctrl.Call(_m, "SetTickLength", _param0) 117 | } 118 | 119 | func (_mr *_MockInputRunnerRecorder) SetTickLength(arg0 interface{}) *gomock.Call { 120 | return _mr.mock.ctrl.RecordCall(_mr.mock, "SetTickLength", arg0) 121 | } 122 | 123 | func (_m *MockInputRunner) Start(_param0 PluginHelper, _param1 *sync.WaitGroup) error { 124 | ret := _m.ctrl.Call(_m, "Start", _param0, _param1) 125 | ret0, _ := ret[0].(error) 126 | return ret0 127 | } 128 | 129 | func (_mr *_MockInputRunnerRecorder) Start(arg0, arg1 interface{}) *gomock.Call { 130 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Start", arg0, arg1) 131 | } 132 | 133 | func (_m *MockInputRunner) Ticker() <-chan time.Time { 134 | ret := _m.ctrl.Call(_m, "Ticker") 135 | ret0, _ := ret[0].(<-chan time.Time) 136 | return ret0 137 | } 138 | 139 | func (_mr *_MockInputRunnerRecorder) Ticker() *gomock.Call { 140 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Ticker") 141 | } 142 | -------------------------------------------------------------------------------- /cmd/flood/flood.toml: -------------------------------------------------------------------------------- 1 | [default] # real world some errors, variable size messages 2 | ip_address = "127.0.0.1:5565" 3 | sender = "tcp" # tcp or udp 4 | pprof_file = "" 5 | encoder = "protobuf" # protobuf or json 6 | num_messages = 0 # number of message to send 0 = infinite 7 | corrupt_percentage = 0.0001 # 1 in a million 8 | signed_percentage = 0.00011 9 | variable_size_messages = true 10 | [default.signer] 11 | name = "test" 12 | hmac_hash = "md5" 13 | hmac_key = "4865ey9urgkidls xtb0[7lf9rzcivthkm" 14 | version = 0 15 | 16 | [simple] # no errors fixed size message 17 | ip_address = "127.0.0.1:5565" 18 | sender = "tcp" 19 | pprof_file = "" 20 | encoder = "protobuf" 21 | num_messages = 0 22 | ascii_only = true 23 | 24 | [simple_20k] # no errors fixed size message ~20KB 25 | ip_address = "127.0.0.1:5565" 26 | sender = "tcp" 27 | pprof_file = "" 28 | encoder = "protobuf" 29 | num_messages = 0 30 | static_message_size = 20000 31 | 32 | [simple_20k_2] # no errors fixed size message ~20KB 33 | ip_address = "127.0.0.1:5566" 34 | sender = "tcp" 35 | pprof_file = "" 36 | encoder = "protobuf" 37 | num_messages = 0 38 | static_message_size = 20000 39 | 40 | [simple_var] # no errors var size message 41 | ip_address = "127.0.0.1:5565" 42 | sender = "tcp" 43 | pprof_file = "" 44 | encoder = "protobuf" 45 | num_messages = 0 46 | variable_size_messages = true 47 | ascii_only = true 48 | 49 | [simple_json] 50 | ip_address = "127.0.0.1:5565" 51 | sender = "tcp" 52 | pprof_file = "" 53 | encoder = "json" 54 | num_messages = 0 55 | 56 | [udp] # real world some errors, variable size messages 57 | ip_address = "127.0.0.1:5565" 58 | sender = "udp" 59 | pprof_file = "" 60 | encoder = "protobuf" 61 | num_messages = 0 62 | corrupt_percentage = 0.0001 63 | signed_percentage = 0.00011 64 | variable_size_messages = true 65 | [udp.signer] 66 | name = "test" 67 | hmac_hash = "md5" 68 | hmac_key = "4865ey9urgkidls xtb0[7lf9rzcivthkm" 69 | version = 0 70 | 71 | [simple_udp] # no errors fixed size message 72 | ip_address = "127.0.0.1:5565" 73 | sender = "udp" 74 | pprof_file = "" 75 | encoder = "protobuf" # protobuf or json 76 | num_messages = 0 # number of message to send 0 = infinite 77 | 78 | [simple_udp_json] 79 | ip_address = "127.0.0.1:5565" 80 | sender = "udp" 81 | pprof_file = "" 82 | encoder = "json" 83 | num_messages = 0 84 | 85 | [signed] 86 | ip_address = "127.0.0.1:5565" 87 | sender = "tcp" 88 | encoder = "protobuf" 89 | corrupt_percentage = 0.0001 # 1 in a million 90 | signed_percentage = 100.0 91 | [signed.signer] 92 | name = "test" 93 | hmac_hash = "md5" 94 | hmac_key = "4865ey9urgkidls xtb0[7lf9rzcivthkm" 95 | version = 0 96 | 97 | [signed_sha1] 98 | ip_address = "127.0.0.1:5565" 99 | sender = "tcp" 100 | encoder = "protobuf" 101 | corrupt_percentage = 0.0001 # 1 in a million 102 | signed_percentage = 100.0 103 | [signed_sha1.signer] 104 | name = "test" 105 | hmac_hash = "sha1" 106 | hmac_key = "4865ey9urgkidls xtb0[7lf9rzcivthkm" 107 | version = 0 108 | 109 | [1000] 110 | ip_address = "127.0.0.1:5565" 111 | sender = "tcp" 112 | encoder = "protobuf" 113 | num_messages = 1000 114 | 115 | [1000_signed] 116 | ip_address = "127.0.0.1:5565" 117 | sender = "tcp" 118 | encoder = "protobuf" 119 | num_messages = 1000 120 | signed_percentage = 100.0 121 | [1000_signed.signer] 122 | name = "test" 123 | hmac_hash = "md5" 124 | hmac_key = "4865ey9urgkidls xtb0[7lf9rzcivthkm" 125 | version = 0 126 | -------------------------------------------------------------------------------- /pipeline/mock_outputrunner_test.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: github.com/mozilla-services/heka/pipeline (interfaces: OutputRunner) 3 | 4 | package pipeline 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | sync "sync" 9 | time "time" 10 | ) 11 | 12 | // Mock of OutputRunner interface 13 | type MockOutputRunner struct { 14 | ctrl *gomock.Controller 15 | recorder *_MockOutputRunnerRecorder 16 | } 17 | 18 | // Recorder for MockOutputRunner (not exported) 19 | type _MockOutputRunnerRecorder struct { 20 | mock *MockOutputRunner 21 | } 22 | 23 | func NewMockOutputRunner(ctrl *gomock.Controller) *MockOutputRunner { 24 | mock := &MockOutputRunner{ctrl: ctrl} 25 | mock.recorder = &_MockOutputRunnerRecorder{mock} 26 | return mock 27 | } 28 | 29 | func (_m *MockOutputRunner) EXPECT() *_MockOutputRunnerRecorder { 30 | return _m.recorder 31 | } 32 | 33 | func (_m *MockOutputRunner) InChan() chan *PipelinePack { 34 | ret := _m.ctrl.Call(_m, "InChan") 35 | ret0, _ := ret[0].(chan *PipelinePack) 36 | return ret0 37 | } 38 | 39 | func (_mr *_MockOutputRunnerRecorder) InChan() *gomock.Call { 40 | return _mr.mock.ctrl.RecordCall(_mr.mock, "InChan") 41 | } 42 | 43 | func (_m *MockOutputRunner) LogError(_param0 error) { 44 | _m.ctrl.Call(_m, "LogError", _param0) 45 | } 46 | 47 | func (_mr *_MockOutputRunnerRecorder) LogError(arg0 interface{}) *gomock.Call { 48 | return _mr.mock.ctrl.RecordCall(_mr.mock, "LogError", arg0) 49 | } 50 | 51 | func (_m *MockOutputRunner) LogMessage(_param0 string) { 52 | _m.ctrl.Call(_m, "LogMessage", _param0) 53 | } 54 | 55 | func (_mr *_MockOutputRunnerRecorder) LogMessage(arg0 interface{}) *gomock.Call { 56 | return _mr.mock.ctrl.RecordCall(_mr.mock, "LogMessage", arg0) 57 | } 58 | 59 | func (_m *MockOutputRunner) MatchRunner() *MatchRunner { 60 | ret := _m.ctrl.Call(_m, "MatchRunner") 61 | ret0, _ := ret[0].(*MatchRunner) 62 | return ret0 63 | } 64 | 65 | func (_mr *_MockOutputRunnerRecorder) MatchRunner() *gomock.Call { 66 | return _mr.mock.ctrl.RecordCall(_mr.mock, "MatchRunner") 67 | } 68 | 69 | func (_m *MockOutputRunner) Name() string { 70 | ret := _m.ctrl.Call(_m, "Name") 71 | ret0, _ := ret[0].(string) 72 | return ret0 73 | } 74 | 75 | func (_mr *_MockOutputRunnerRecorder) Name() *gomock.Call { 76 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Name") 77 | } 78 | 79 | func (_m *MockOutputRunner) Output() Output { 80 | ret := _m.ctrl.Call(_m, "Output") 81 | ret0, _ := ret[0].(Output) 82 | return ret0 83 | } 84 | 85 | func (_mr *_MockOutputRunnerRecorder) Output() *gomock.Call { 86 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Output") 87 | } 88 | 89 | func (_m *MockOutputRunner) Plugin() Plugin { 90 | ret := _m.ctrl.Call(_m, "Plugin") 91 | ret0, _ := ret[0].(Plugin) 92 | return ret0 93 | } 94 | 95 | func (_mr *_MockOutputRunnerRecorder) Plugin() *gomock.Call { 96 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Plugin") 97 | } 98 | 99 | func (_m *MockOutputRunner) PluginGlobals() *PluginGlobals { 100 | ret := _m.ctrl.Call(_m, "PluginGlobals") 101 | ret0, _ := ret[0].(*PluginGlobals) 102 | return ret0 103 | } 104 | 105 | func (_mr *_MockOutputRunnerRecorder) PluginGlobals() *gomock.Call { 106 | return _mr.mock.ctrl.RecordCall(_mr.mock, "PluginGlobals") 107 | } 108 | 109 | func (_m *MockOutputRunner) RetainPack(_param0 *PipelinePack) { 110 | _m.ctrl.Call(_m, "RetainPack", _param0) 111 | } 112 | 113 | func (_mr *_MockOutputRunnerRecorder) RetainPack(arg0 interface{}) *gomock.Call { 114 | return _mr.mock.ctrl.RecordCall(_mr.mock, "RetainPack", arg0) 115 | } 116 | 117 | func (_m *MockOutputRunner) SetName(_param0 string) { 118 | _m.ctrl.Call(_m, "SetName", _param0) 119 | } 120 | 121 | func (_mr *_MockOutputRunnerRecorder) SetName(arg0 interface{}) *gomock.Call { 122 | return _mr.mock.ctrl.RecordCall(_mr.mock, "SetName", arg0) 123 | } 124 | 125 | func (_m *MockOutputRunner) Start(_param0 PluginHelper, _param1 *sync.WaitGroup) error { 126 | ret := _m.ctrl.Call(_m, "Start", _param0, _param1) 127 | ret0, _ := ret[0].(error) 128 | return ret0 129 | } 130 | 131 | func (_mr *_MockOutputRunnerRecorder) Start(arg0, arg1 interface{}) *gomock.Call { 132 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Start", arg0, arg1) 133 | } 134 | 135 | func (_m *MockOutputRunner) Ticker() <-chan time.Time { 136 | ret := _m.ctrl.Call(_m, "Ticker") 137 | ret0, _ := ret[0].(<-chan time.Time) 138 | return ret0 139 | } 140 | 141 | func (_mr *_MockOutputRunnerRecorder) Ticker() *gomock.Call { 142 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Ticker") 143 | } 144 | -------------------------------------------------------------------------------- /cmd/sbmgrload/main.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Mike Trinkala (trink@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | /* 16 | 17 | Sandbox Manager Load Test 18 | 19 | */ 20 | package main 21 | 22 | import ( 23 | "code.google.com/p/go-uuid/uuid" 24 | "flag" 25 | "fmt" 26 | "github.com/bbangert/toml" 27 | "github.com/mozilla-services/heka/client" 28 | "github.com/mozilla-services/heka/message" 29 | "log" 30 | "os" 31 | "time" 32 | ) 33 | 34 | type SbmgrConfig struct { 35 | IpAddress string `toml:"ip_address"` 36 | Signer message.MessageSigningConfig `toml:"signer"` 37 | } 38 | 39 | func main() { 40 | configFile := flag.String("config", "sbmgrload.toml", "Sandbox manager load configuration file") 41 | action := flag.String("action", "load", "load/unload") 42 | numItems := flag.Int("num", 1, "Number of sandboxes to load/unload") 43 | flag.Parse() 44 | 45 | code := ` 46 | lastTime = os.time() * 1e9 47 | lastCount = 0 48 | count = 0 49 | rate = 0.0 50 | rates = {} 51 | 52 | function process_message () 53 | count = count + 1 54 | return 0 55 | end 56 | 57 | function timer_event(ns) 58 | local msgsSent = count - lastCount 59 | if msgsSent == 0 then return end 60 | 61 | local elapsedTime = ns - lastTime 62 | if elapsedTime == 0 then return end 63 | 64 | lastCount = count 65 | lastTime = ns 66 | rate = msgsSent / (elapsedTime / 1e9) 67 | rates[#rates+1] = rate 68 | output(string.format("Got %d messages. %0.2f msg/sec", count, rate)) 69 | inject_message() 70 | 71 | local samples = #rates 72 | if samples == 10 then -- generate a summary every 10 samples 73 | table.sort(rates) 74 | local min = rates[1] 75 | local max = rates[samples] 76 | local sum = 0 77 | for i, val in ipairs(rates) do 78 | sum = sum + val 79 | end 80 | output(string.format("AGG Sum. Min: %0.2f Max: %0.2f Mean: %0.2f", min, max, sum/samples)) 81 | inject_message() 82 | rates = {} 83 | end 84 | end 85 | ` 86 | confFmt := ` 87 | [CounterSandbox%d] 88 | type = "SandboxFilter" 89 | message_matcher = "Type == 'hekabench'" 90 | ticker_interval = 1 91 | script_type = "lua" 92 | filename = "" 93 | preserve_data = true 94 | memory_limit = 32767 95 | instruction_limit = 1000 96 | output_limit = 1024 97 | ` 98 | 99 | var config SbmgrConfig 100 | if _, err := toml.DecodeFile(*configFile, &config); err != nil { 101 | log.Printf("Error decoding config file: %s", err) 102 | return 103 | } 104 | sender, err := client.NewNetworkSender("tcp", config.IpAddress) 105 | if err != nil { 106 | log.Fatalf("Error creating sender: %s\n", err.Error()) 107 | } 108 | encoder := client.NewProtobufEncoder(&config.Signer) 109 | manager := client.NewClient(sender, encoder) 110 | hostname, _ := os.Hostname() 111 | 112 | switch *action { 113 | case "load": 114 | for i := 0; i < *numItems; i++ { 115 | conf := fmt.Sprintf(confFmt, i) 116 | msg := &message.Message{} 117 | msg.SetType("heka.control.sandbox") 118 | msg.SetTimestamp(time.Now().UnixNano()) 119 | msg.SetUuid(uuid.NewRandom()) 120 | msg.SetHostname(hostname) 121 | msg.SetPayload(code) 122 | f, _ := message.NewField("config", conf, "toml") 123 | msg.AddField(f) 124 | f1, _ := message.NewField("action", *action, "") 125 | msg.AddField(f1) 126 | err = manager.SendMessage(msg) 127 | if err != nil { 128 | log.Printf("Error sending message: %s\n", err.Error()) 129 | } 130 | } 131 | case "unload": 132 | for i := 0; i < *numItems; i++ { 133 | msg := &message.Message{} 134 | msg.SetType("heka.control.sandbox") 135 | msg.SetTimestamp(time.Now().UnixNano()) 136 | msg.SetUuid(uuid.NewRandom()) 137 | msg.SetHostname(hostname) 138 | f, _ := message.NewField("name", fmt.Sprintf("CounterSandbox%d", i), "") 139 | msg.AddField(f) 140 | f1, _ := message.NewField("action", *action, "") 141 | msg.AddField(f1) 142 | err = manager.SendMessage(msg) 143 | if err != nil { 144 | log.Printf("Error sending message: %s\n", err.Error()) 145 | } 146 | } 147 | 148 | default: 149 | log.Printf("Invalid action: %s\n", *action) 150 | } 151 | 152 | } 153 | -------------------------------------------------------------------------------- /sandbox/lua/testsupport/serialize.lua.data: -------------------------------------------------------------------------------- 1 | _G["kvp"] = {} 2 | _G["kvp"]["a"] = "foo" 3 | _G["kvp"]["r"] = {} 4 | _G["kvp"]["r"][1] = 99.1 5 | _G["kvp"]["r"][2] = 98 6 | _G["kvp"]["r"][3] = 97 7 | _G["kvp"]["r"]["key"] = "val" 8 | _G["kvp"]["b"] = "bar" 9 | _G["count"] = 0 10 | _G["uuids"] = {} 11 | _G["uuids"][1] = {} 12 | _G["uuids"][1]["type"] = "test" 13 | _G["uuids"][1]["uuid"] = "BD48B609-8922-4E59-A358-C242075CE088" 14 | _G["uuids"][2] = {} 15 | _G["uuids"][2]["type"] = "test1" 16 | _G["uuids"][2]["uuid"] = "BD48B609-8922-4E59-A358-C242075CE089" 17 | _G["cycleb"] = {} 18 | _G["cycleb"]["a"] = {} 19 | _G["cycleb"]["a"]["b"] = _G["cycleb"] 20 | _G["cycleb"]["a"]["type"] = "cycle a" 21 | _G["cycleb"]["type"] = "cycle b" 22 | if _G["dataRef"] == nil then _G["dataRef"] = circular_buffer.new(3, 3, 1) end 23 | _G["dataRef"]:set_header(1, "Column_1", "count", "sum") 24 | _G["dataRef"]:set_header(2, "Column_2", "count", "sum") 25 | _G["dataRef"]:set_header(3, "Column_3", "count", "sum") 26 | _G["dataRef"]:fromstring("2 2 0 0 0 0 0 0 0 0 0") 27 | _G["data"] = _G["dataRef"] 28 | _G["cyclea"] = _G["cycleb"]["a"] 29 | _G["large_key"] = {} 30 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"] = {} 31 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"] = {} 32 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"] = {} 33 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["BD48B609-8922-4E59-A358-C242075CE083"] = 3 34 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"] = {} 35 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["BD48B609-8922-4E59-A358-C242075CE084"] = 4 36 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"] = {} 37 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["BD48B609-8922-4E59-A358-C242075CE085"] = 5 38 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"] = {} 39 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"]["ggggggggggggggggggg"] = {} 40 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"]["ggggggggggggggggggg"]["BD48B609-8922-4E59-A358-C242075CE087"] = 7 41 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"]["ggggggggggggggggggg"]["hhhhhhhhhhhhhhhhhhh"] = {} 42 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"]["ggggggggggggggggggg"]["hhhhhhhhhhhhhhhhhhh"]["BD48B609-8922-4E59-A358-C242075CE088"] = 8 43 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"]["ggggggggggggggggggg"]["hhhhhhhhhhhhhhhhhhh"]["iiiiiiiiiiiiiiiiiii"] = {} 44 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"]["ggggggggggggggggggg"]["hhhhhhhhhhhhhhhhhhh"]["iiiiiiiiiiiiiiiiiii"]["BD48B609-8922-4E59-A358-C242075CE089"] = 9 45 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"]["BD48B609-8922-4E59-A358-C242075CE086"] = 6 46 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["BD48B609-8922-4E59-A358-C242075CE082"] = 2 47 | _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["BD48B609-8922-4E59-A358-C242075CE081"] = 1 48 | _G["boolean"] = true 49 | _G["key with spaces"] = "kws" 50 | _G["nested"] = {} 51 | _G["nested"]["arg2"] = 2 52 | _G["nested"]["nested"] = {} 53 | _G["nested"]["nested"]["n2"] = "two" 54 | _G["nested"]["nested"]["n1"] = "one" 55 | _G["nested"]["arg1"] = 1 56 | if _G["nested"]["cb"] == nil then _G["nested"]["cb"] = circular_buffer.new(2, 6, 1) end 57 | _G["nested"]["cb"]:set_header(1, "Column_1", "count", "sum") 58 | _G["nested"]["cb"]:set_header(2, "Column_2", "count", "sum") 59 | _G["nested"]["cb"]:set_header(3, "Column_3", "count", "sum") 60 | _G["nested"]["cb"]:set_header(4, "Column_4", "count", "sum") 61 | _G["nested"]["cb"]:set_header(5, "Column_5", "count", "sum") 62 | _G["nested"]["cb"]:set_header(6, "Column_6", "count", "sum") 63 | _G["nested"]["cb"]:fromstring("1 1 0 0 0 0 0 0 0 0 0 0 0 0") 64 | _G["_VERSION"] = "Lua 5.1" 65 | _G["rates"] = _G["kvp"]["r"] 66 | _G["rate"] = 0.12345678 67 | -------------------------------------------------------------------------------- /pipeline/statsd_input.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Ben Bangert (bbangert@mozilla.com) 12 | # Rob Miller (rmiller@mozilla.com) 13 | # 14 | # ***** END LICENSE BLOCK *****/ 15 | 16 | package pipeline 17 | 18 | import ( 19 | "fmt" 20 | "net" 21 | "regexp" 22 | "strconv" 23 | "time" 24 | ) 25 | 26 | var ( 27 | sanitizeRegexp = regexp.MustCompile("[^a-zA-Z0-9\\-_\\.:\\|@]") 28 | packetRegexp = regexp.MustCompile("([a-zA-Z0-9_\\.]+):(\\-?[0-9\\.]+)\\|(c|ms|g)(\\|@([0-9\\.]+))?") 29 | ) 30 | 31 | // A Heka Input plugin that handles statsd metric style input and flushes 32 | // aggregated values. It can listen on a UDP address if configured to do so 33 | // for standard statsd packets of message type Counter, Gauge, or Timer. It 34 | // also accepts StatPacket objects generated from within Heka itself (usually 35 | // via a configured StatFilter plugin) over the exposed `Packet` channel. It 36 | // currently doesn't support Sets or other metric types. 37 | type StatsdInput struct { 38 | name string 39 | listener net.Conn 40 | stopChan chan bool 41 | statChan chan<- Stat 42 | statAccumName string 43 | statAccum StatAccumulator 44 | ir InputRunner 45 | } 46 | 47 | // StatsInput config struct 48 | type StatsdInputConfig struct { 49 | // UDP Address to listen to for statsd packets. Defaults to 50 | // "127.0.0.1:8125". 51 | Address string 52 | // Configured name of StatAccumInput plugin to which this filter should be 53 | // delivering its stats. Defaults to "StatsAccumInput". 54 | StatAccumName string `toml:"stat_accum_name"` 55 | } 56 | 57 | func (s *StatsdInput) ConfigStruct() interface{} { 58 | return &StatsdInputConfig{ 59 | Address: "127.0.0.1:8125", 60 | StatAccumName: "StatAccumInput", 61 | } 62 | } 63 | 64 | func (s *StatsdInput) Init(config interface{}) error { 65 | conf := config.(*StatsdInputConfig) 66 | udpAddr, err := net.ResolveUDPAddr("udp", conf.Address) 67 | if err != nil { 68 | return fmt.Errorf("ResolveUDPAddr failed: %s\n", err.Error()) 69 | } 70 | s.listener, err = net.ListenUDP("udp", udpAddr) 71 | if err != nil { 72 | return fmt.Errorf("ListenUDP failed: %s\n", err.Error()) 73 | } 74 | s.statAccumName = conf.StatAccumName 75 | return nil 76 | } 77 | 78 | // Spins up a statsd server listening on a UDP connection. 79 | func (s *StatsdInput) Run(ir InputRunner, h PluginHelper) (err error) { 80 | s.stopChan = make(chan bool) 81 | s.ir = ir 82 | 83 | if s.statAccum, err = h.StatAccumulator(s.statAccumName); err != nil { 84 | return 85 | } 86 | 87 | // Spin up the UDP listener. 88 | var ( 89 | n int 90 | e error 91 | stopped bool 92 | ) 93 | defer s.listener.Close() 94 | timeout := time.Duration(time.Millisecond * 100) 95 | 96 | for !stopped { 97 | message := make([]byte, 512) 98 | s.listener.SetReadDeadline(time.Now().Add(timeout)) 99 | n, e = s.listener.Read(message) 100 | 101 | select { 102 | case <-s.stopChan: 103 | stopped = true 104 | default: 105 | } 106 | 107 | if e != nil || n == 0 { 108 | continue 109 | } 110 | 111 | if stopped { 112 | // If we're stopping, use synchronous call so we don't 113 | // close the Packet channel too soon. 114 | s.handleMessage(message[:n]) 115 | } else { 116 | go s.handleMessage(message[:n]) 117 | } 118 | } 119 | 120 | return 121 | } 122 | 123 | func (s *StatsdInput) Stop() { 124 | close(s.stopChan) 125 | } 126 | 127 | // Parses received raw statsd bytes data and converts it into a StatPacket 128 | // object that can be passed to the StatMonitor. 129 | func (s *StatsdInput) handleMessage(message []byte) { 130 | var ( 131 | value string 132 | stat Stat 133 | ) 134 | st := sanitizeRegexp.ReplaceAllString(string(message), "") 135 | for _, item := range packetRegexp.FindAllStringSubmatch(st, -1) { 136 | value = item[2] 137 | if item[3] == "ms" { 138 | _, err := strconv.ParseFloat(item[2], 32) 139 | if err != nil { 140 | value = "0" 141 | } 142 | } 143 | 144 | sampleRate, err := strconv.ParseFloat(item[5], 32) 145 | if err != nil { 146 | sampleRate = 1 147 | } 148 | 149 | stat.Bucket = item[1] 150 | stat.Value = value 151 | stat.Modifier = item[3] 152 | stat.Sampling = float32(sampleRate) 153 | if !s.statAccum.DropStat(stat) { 154 | s.ir.LogError(fmt.Errorf("Undelivered stat: %s", stat)) 155 | } 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /pipeline/mock_filterrunner_test.go: -------------------------------------------------------------------------------- 1 | // Automatically generated by MockGen. DO NOT EDIT! 2 | // Source: github.com/mozilla-services/heka/pipeline (interfaces: FilterRunner) 3 | 4 | package pipeline 5 | 6 | import ( 7 | gomock "code.google.com/p/gomock/gomock" 8 | sync "sync" 9 | time "time" 10 | ) 11 | 12 | // Mock of FilterRunner interface 13 | type MockFilterRunner struct { 14 | ctrl *gomock.Controller 15 | recorder *_MockFilterRunnerRecorder 16 | } 17 | 18 | // Recorder for MockFilterRunner (not exported) 19 | type _MockFilterRunnerRecorder struct { 20 | mock *MockFilterRunner 21 | } 22 | 23 | func NewMockFilterRunner(ctrl *gomock.Controller) *MockFilterRunner { 24 | mock := &MockFilterRunner{ctrl: ctrl} 25 | mock.recorder = &_MockFilterRunnerRecorder{mock} 26 | return mock 27 | } 28 | 29 | func (_m *MockFilterRunner) EXPECT() *_MockFilterRunnerRecorder { 30 | return _m.recorder 31 | } 32 | 33 | func (_m *MockFilterRunner) Filter() Filter { 34 | ret := _m.ctrl.Call(_m, "Filter") 35 | ret0, _ := ret[0].(Filter) 36 | return ret0 37 | } 38 | 39 | func (_mr *_MockFilterRunnerRecorder) Filter() *gomock.Call { 40 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Filter") 41 | } 42 | 43 | func (_m *MockFilterRunner) InChan() chan *PipelinePack { 44 | ret := _m.ctrl.Call(_m, "InChan") 45 | ret0, _ := ret[0].(chan *PipelinePack) 46 | return ret0 47 | } 48 | 49 | func (_mr *_MockFilterRunnerRecorder) InChan() *gomock.Call { 50 | return _mr.mock.ctrl.RecordCall(_mr.mock, "InChan") 51 | } 52 | 53 | func (_m *MockFilterRunner) Inject(_param0 *PipelinePack) bool { 54 | ret := _m.ctrl.Call(_m, "Inject", _param0) 55 | ret0, _ := ret[0].(bool) 56 | return ret0 57 | } 58 | 59 | func (_mr *_MockFilterRunnerRecorder) Inject(arg0 interface{}) *gomock.Call { 60 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Inject", arg0) 61 | } 62 | 63 | func (_m *MockFilterRunner) LogError(_param0 error) { 64 | _m.ctrl.Call(_m, "LogError", _param0) 65 | } 66 | 67 | func (_mr *_MockFilterRunnerRecorder) LogError(arg0 interface{}) *gomock.Call { 68 | return _mr.mock.ctrl.RecordCall(_mr.mock, "LogError", arg0) 69 | } 70 | 71 | func (_m *MockFilterRunner) LogMessage(_param0 string) { 72 | _m.ctrl.Call(_m, "LogMessage", _param0) 73 | } 74 | 75 | func (_mr *_MockFilterRunnerRecorder) LogMessage(arg0 interface{}) *gomock.Call { 76 | return _mr.mock.ctrl.RecordCall(_mr.mock, "LogMessage", arg0) 77 | } 78 | 79 | func (_m *MockFilterRunner) MatchRunner() *MatchRunner { 80 | ret := _m.ctrl.Call(_m, "MatchRunner") 81 | ret0, _ := ret[0].(*MatchRunner) 82 | return ret0 83 | } 84 | 85 | func (_mr *_MockFilterRunnerRecorder) MatchRunner() *gomock.Call { 86 | return _mr.mock.ctrl.RecordCall(_mr.mock, "MatchRunner") 87 | } 88 | 89 | func (_m *MockFilterRunner) Name() string { 90 | ret := _m.ctrl.Call(_m, "Name") 91 | ret0, _ := ret[0].(string) 92 | return ret0 93 | } 94 | 95 | func (_mr *_MockFilterRunnerRecorder) Name() *gomock.Call { 96 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Name") 97 | } 98 | 99 | func (_m *MockFilterRunner) Plugin() Plugin { 100 | ret := _m.ctrl.Call(_m, "Plugin") 101 | ret0, _ := ret[0].(Plugin) 102 | return ret0 103 | } 104 | 105 | func (_mr *_MockFilterRunnerRecorder) Plugin() *gomock.Call { 106 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Plugin") 107 | } 108 | 109 | func (_m *MockFilterRunner) PluginGlobals() *PluginGlobals { 110 | ret := _m.ctrl.Call(_m, "PluginGlobals") 111 | ret0, _ := ret[0].(*PluginGlobals) 112 | return ret0 113 | } 114 | 115 | func (_mr *_MockFilterRunnerRecorder) PluginGlobals() *gomock.Call { 116 | return _mr.mock.ctrl.RecordCall(_mr.mock, "PluginGlobals") 117 | } 118 | 119 | func (_m *MockFilterRunner) RetainPack(_param0 *PipelinePack) { 120 | _m.ctrl.Call(_m, "RetainPack", _param0) 121 | } 122 | 123 | func (_mr *_MockFilterRunnerRecorder) RetainPack(arg0 interface{}) *gomock.Call { 124 | return _mr.mock.ctrl.RecordCall(_mr.mock, "RetainPack", arg0) 125 | } 126 | 127 | func (_m *MockFilterRunner) SetName(_param0 string) { 128 | _m.ctrl.Call(_m, "SetName", _param0) 129 | } 130 | 131 | func (_mr *_MockFilterRunnerRecorder) SetName(arg0 interface{}) *gomock.Call { 132 | return _mr.mock.ctrl.RecordCall(_mr.mock, "SetName", arg0) 133 | } 134 | 135 | func (_m *MockFilterRunner) Start(_param0 PluginHelper, _param1 *sync.WaitGroup) error { 136 | ret := _m.ctrl.Call(_m, "Start", _param0, _param1) 137 | ret0, _ := ret[0].(error) 138 | return ret0 139 | } 140 | 141 | func (_mr *_MockFilterRunnerRecorder) Start(arg0, arg1 interface{}) *gomock.Call { 142 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Start", arg0, arg1) 143 | } 144 | 145 | func (_m *MockFilterRunner) Ticker() <-chan time.Time { 146 | ret := _m.ctrl.Call(_m, "Ticker") 147 | ret0, _ := ret[0].(<-chan time.Time) 148 | return ret0 149 | } 150 | 151 | func (_mr *_MockFilterRunnerRecorder) Ticker() *gomock.Call { 152 | return _mr.mock.ctrl.RecordCall(_mr.mock, "Ticker") 153 | } 154 | -------------------------------------------------------------------------------- /pipeline/payloadjson_decoder.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Victor Ng (vng@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | package pipeline 16 | 17 | import ( 18 | "fmt" 19 | "time" 20 | ) 21 | 22 | type PayloadJsonDecoderConfig struct { 23 | // Regular expression that describes log line format and capture group 24 | // values. 25 | JsonMap map[string]string `toml:"json_map"` 26 | 27 | // Maps severity strings to their int version 28 | SeverityMap map[string]int32 `toml:"severity_map"` 29 | 30 | // Keyed to the message field that should be filled in, the value will be 31 | // interpolated so it can use capture parts from the message match. 32 | MessageFields MessageTemplate `toml:"message_fields"` 33 | 34 | // User specified timestamp layout string, used for parsing a timestamp 35 | // string into an actual time object. If not specified or it fails to 36 | // match, all the default time layout's will be tried. 37 | TimestampLayout string `toml:"timestamp_layout"` 38 | 39 | // Time zone in which the timestamps in the text are presumed to be in. 40 | // Should be a location name corresponding to a file in the IANA Time Zone 41 | // database (e.g. "America/Los_Angeles"), as parsed by Go's 42 | // `time.LoadLocation()` function (see 43 | // http://golang.org/pkg/time/#LoadLocation). Defaults to "UTC". Not 44 | // required if valid time zone info is embedded in every parsed timestamp, 45 | // since those can be parsed as specified in the `timestamp_layout`. 46 | TimestampLocation string `toml:"timestamp_location"` 47 | } 48 | 49 | type PayloadJsonDecoder struct { 50 | JsonMap map[string]string 51 | SeverityMap map[string]int32 52 | MessageFields MessageTemplate 53 | TimestampLayout string 54 | tzLocation *time.Location 55 | dRunner DecoderRunner 56 | } 57 | 58 | func (ld *PayloadJsonDecoder) ConfigStruct() interface{} { 59 | return &PayloadJsonDecoderConfig{ 60 | TimestampLayout: "2012-04-23T18:25:43.511Z", 61 | } 62 | } 63 | 64 | func (ld *PayloadJsonDecoder) Init(config interface{}) (err error) { 65 | conf := config.(*PayloadJsonDecoderConfig) 66 | 67 | ld.JsonMap = make(map[string]string) 68 | for capture_name, jp := range conf.JsonMap { 69 | ld.JsonMap[capture_name] = jp 70 | } 71 | 72 | ld.SeverityMap = make(map[string]int32) 73 | ld.MessageFields = make(MessageTemplate) 74 | if conf.SeverityMap != nil { 75 | for codeString, codeInt := range conf.SeverityMap { 76 | ld.SeverityMap[codeString] = codeInt 77 | } 78 | } 79 | if conf.MessageFields != nil { 80 | for field, action := range conf.MessageFields { 81 | ld.MessageFields[field] = action 82 | } 83 | } 84 | ld.TimestampLayout = conf.TimestampLayout 85 | if ld.tzLocation, err = time.LoadLocation(conf.TimestampLocation); err != nil { 86 | err = fmt.Errorf("PayloadJsonDecoder unknown timestamp_location '%s': %s", 87 | conf.TimestampLocation, err) 88 | } 89 | return 90 | } 91 | 92 | // Heka will call this to give us access to the runner. 93 | func (ld *PayloadJsonDecoder) SetDecoderRunner(dr DecoderRunner) { 94 | ld.dRunner = dr 95 | } 96 | 97 | // Matches the given string against the JSONPath and returns the match result 98 | // and captures 99 | func (ld *PayloadJsonDecoder) match(s string) (captures map[string]string) { 100 | captures = make(map[string]string) 101 | 102 | jp := new(JsonPath) 103 | jp.SetJsonText(s) 104 | 105 | for capture_group, jpath := range ld.JsonMap { 106 | node_val, err := jp.Find(jpath) 107 | if err != nil { 108 | continue 109 | } 110 | captures[capture_group] = node_val 111 | } 112 | return 113 | } 114 | 115 | // Runs the message payload against decoder's map of JSONPaths. If 116 | // there's a match, the message will be populated based on the 117 | // decoder's message template, with capture values interpolated into 118 | // the message template values. 119 | func (ld *PayloadJsonDecoder) Decode(pack *PipelinePack) (err error) { 120 | captures := ld.match(pack.Message.GetPayload()) 121 | 122 | pdh := &PayloadDecoderHelper{ 123 | Captures: captures, 124 | dRunner: ld.dRunner, 125 | TimestampLayout: ld.TimestampLayout, 126 | TzLocation: ld.tzLocation, 127 | SeverityMap: ld.SeverityMap, 128 | } 129 | 130 | pdh.DecodeTimestamp(pack) 131 | pdh.DecodeSeverity(pack) 132 | 133 | // Update the new message fields based on the fields we should 134 | // change and the capture parts 135 | return ld.MessageFields.PopulateMessage(pack.Message, captures) 136 | } 137 | -------------------------------------------------------------------------------- /docs/source/installing.rst: -------------------------------------------------------------------------------- 1 | .. _installing: 2 | 3 | ========== 4 | Installing 5 | ========== 6 | 7 | .. _from_binaries: 8 | 9 | Binaries 10 | ======== 11 | 12 | `hekad` `releases are available on the Github project releases page 13 | `_. 14 | Binaries are available for Linux and OSX, with packages for Debian and 15 | RPM based distributions. 16 | 17 | .. _from_source: 18 | 19 | From Source (*nix) 20 | ================== 21 | 22 | `hekad` requires a Go work environment to be setup for the binary to be 23 | built. This task has been automated in the `heka build`_ repository. To 24 | create a working `hekad` binary for your platform you'll need to 25 | install some prerequisites. Many of these are standard on modern linux 26 | distributions and OSX. 27 | 28 | Prerequisites: 29 | 30 | - cmake 2.8 or greater 31 | - make 32 | - gcc 33 | - g++ 34 | - git 35 | - go 1.1 or greater (1.1.1 recommended) 36 | - python 2.6 or greater 37 | - patch 38 | - mercurial 39 | 40 | 1. Check out the `heka build`_ repository: 41 | 42 | .. code-block:: bash 43 | 44 | git clone https://github.com/mozilla-services/heka-build.git 45 | 46 | 2. Run `make` in the heka-build directory (builds the current release (master 47 | branch); if you have go installed in a non-standard location, you may need 48 | to set the GOROOT environment variable): 49 | 50 | .. code-block:: bash 51 | 52 | cd heka-build 53 | make 54 | 55 | You will now have a `hekad` binary in the `heka-build/bin` directory. 56 | 57 | 3. (Optional) Run the tests to ensure a functioning `hekad`: 58 | 59 | .. code-block:: bash 60 | 61 | make test 62 | 63 | 4. (Optional) If you want to build the latest code in development, run `make 64 | dev` to switch to the dev branch and then run `make`. If you need to revert 65 | back to the master branch at some point run `make undev`. 66 | 67 | 68 | From Source (Windows) 69 | ===================== 70 | 71 | Prerequisites (manual setup): 72 | 73 | - Go 1.1+ (1.1.1 recommended) http://code.google.com/p/go/downloads/list 74 | - Cmake 2.8+ http://www.cmake.org/cmake/resources/software.html 75 | - Git http://code.google.com/p/msysgit/downloads/list 76 | - Mercurial http://mercurial.selenic.com/downloads/ 77 | - MinGW http://sourceforge.net/projects/tdm-gcc/ 78 | 79 | 1. From a Git shell check out the `heka build`_ repository: 80 | 81 | .. code-block:: bash 82 | 83 | git clone https://github.com/mozilla-services/heka-build.git 84 | 85 | 2. From a MinGW shell run `build.bat` in the heka-build directory: 86 | 87 | .. code-block:: bash 88 | 89 | cd heka-build 90 | build 91 | 92 | 3. (Optional) Run the tests to ensure a functioning `hekad`: 93 | 94 | .. code-block:: bash 95 | 96 | mingw32-make test 97 | 98 | You will now have a `hekad` binary in the `release/heka-0_2_0_w(32|64)/bin` directory. 99 | 100 | .. _build_include_externals: 101 | 102 | Building `hekad` with External Plugins 103 | ====================================== 104 | 105 | It is possible to extend `hekad` by writing input, decoder, filter, or output 106 | plugins in Go (see :ref:`plugins`). Because Go only supports static linking of 107 | Go code, your plugins must be included with and registered into Heka at 108 | compile time. `heka build`_ supports the use of a `{heka-build- 109 | root}/etc/plugin_packages.json` file to specify which packages you'd like to 110 | include in your build. The JSON should be an object with a single 111 | `plugin_packages` attribute, with the value an array of package paths. For 112 | example: 113 | 114 | .. code-block:: json 115 | 116 | {"plugin_packages": ["github.com/mozilla-services/heka-mozsvc-plugins"]} 117 | 118 | would cause the `github.com/mozilla-services/heka-mozsvc-plugins` package to 119 | be imported into `hekad` when you run `make`. By adding an `init() function 120 | `_ in your package you can make 121 | calls into `pipeline.RegisterPlugin` to register your plugins with Heka's 122 | configuration system. 123 | 124 | .. _build_rpm_deb_pkgs: 125 | 126 | Creating RPM/Deb Packages 127 | ========================= 128 | 129 | Installing packages on a system is generally the easiest way to deploy 130 | `hekad`. These packages can be easily created after following the above 131 | :ref:`From Source ` directions: 132 | 133 | 1. Install fpm: 134 | 135 | .. code-block:: bash 136 | 137 | gem install fpm 138 | 139 | 2. Run `make debs` (or `rpms`) to build the appropriate package (in the 140 | `heka-build` directory): 141 | 142 | .. code-block:: bash 143 | 144 | make debs 145 | 146 | The packages will be in the `debs` or `rpms` directory. 147 | 148 | .. note:: 149 | 150 | You will need `rpmbuild` installed to build the rpms. 151 | 152 | .. seealso:: `Setting up an rpm-build environment `_ 153 | 154 | .. _heka build: https://github.com/mozilla-services/heka-build 155 | -------------------------------------------------------------------------------- /pipeline/payloadregex_decoder.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Ben Bangert (bbangert@mozilla.com) 12 | # Rob Miller (rmiller@mozilla.com) 13 | # 14 | # ***** END LICENSE BLOCK *****/ 15 | 16 | package pipeline 17 | 18 | import ( 19 | "fmt" 20 | "regexp" 21 | "time" 22 | ) 23 | 24 | type PayloadRegexDecoderConfig struct { 25 | // Regular expression that describes log line format and capture group 26 | // values. 27 | MatchRegex string `toml:"match_regex"` 28 | 29 | // Maps severity strings to their int version 30 | SeverityMap map[string]int32 `toml:"severity_map"` 31 | 32 | // Keyed to the message field that should be filled in, the value will be 33 | // interpolated so it can use capture parts from the message match. 34 | MessageFields MessageTemplate `toml:"message_fields"` 35 | 36 | // User specified timestamp layout string, used for parsing a timestamp 37 | // string into an actual time object. If not specified or it fails to 38 | // match, all the default time layout's will be tried. 39 | TimestampLayout string `toml:"timestamp_layout"` 40 | 41 | // Time zone in which the timestamps in the text are presumed to be in. 42 | // Should be a location name corresponding to a file in the IANA Time Zone 43 | // database (e.g. "America/Los_Angeles"), as parsed by Go's 44 | // `time.LoadLocation()` function (see 45 | // http://golang.org/pkg/time/#LoadLocation). Defaults to "UTC". Not 46 | // required if valid time zone info is embedded in every parsed timestamp, 47 | // since those can be parsed as specified in the `timestamp_layout`. 48 | TimestampLocation string `toml:"timestamp_location"` 49 | } 50 | 51 | type PayloadRegexDecoder struct { 52 | Match *regexp.Regexp 53 | SeverityMap map[string]int32 54 | MessageFields MessageTemplate 55 | TimestampLayout string 56 | tzLocation *time.Location 57 | dRunner DecoderRunner 58 | } 59 | 60 | func (ld *PayloadRegexDecoder) ConfigStruct() interface{} { 61 | return new(PayloadRegexDecoderConfig) 62 | } 63 | 64 | func (ld *PayloadRegexDecoder) Init(config interface{}) (err error) { 65 | conf := config.(*PayloadRegexDecoderConfig) 66 | if ld.Match, err = regexp.Compile(conf.MatchRegex); err != nil { 67 | err = fmt.Errorf("PayloadRegexDecoder: %s", err) 68 | return 69 | } 70 | if ld.Match.NumSubexp() == 0 { 71 | err = fmt.Errorf("PayloadRegexDecoder regex must contain capture groups") 72 | return 73 | } 74 | 75 | ld.SeverityMap = make(map[string]int32) 76 | ld.MessageFields = make(MessageTemplate) 77 | if conf.SeverityMap != nil { 78 | for codeString, codeInt := range conf.SeverityMap { 79 | ld.SeverityMap[codeString] = codeInt 80 | } 81 | } 82 | if conf.MessageFields != nil { 83 | for field, action := range conf.MessageFields { 84 | ld.MessageFields[field] = action 85 | } 86 | } 87 | ld.TimestampLayout = conf.TimestampLayout 88 | if ld.tzLocation, err = time.LoadLocation(conf.TimestampLocation); err != nil { 89 | err = fmt.Errorf("PayloadRegexDecoder unknown timestamp_location '%s': %s", 90 | conf.TimestampLocation, err) 91 | } 92 | return 93 | } 94 | 95 | // Heka will call this to give us access to the runner. 96 | func (ld *PayloadRegexDecoder) SetDecoderRunner(dr DecoderRunner) { 97 | ld.dRunner = dr 98 | } 99 | 100 | // Matches the given string against the regex and returns the match result 101 | // and captures 102 | func tryMatch(re *regexp.Regexp, s string) (match bool, captures map[string]string) { 103 | findResults := re.FindStringSubmatch(s) 104 | if findResults == nil { 105 | return 106 | } 107 | match = true 108 | captures = make(map[string]string) 109 | for index, name := range re.SubexpNames() { 110 | if index == 0 { 111 | continue 112 | } 113 | if name == "" { 114 | name = fmt.Sprintf("%d", index) 115 | } 116 | captures[name] = findResults[index] 117 | } 118 | return 119 | } 120 | 121 | // Runs the message payload against decoder's regex. If there's a match, the 122 | // message will be populated based on the decoder's message template, with 123 | // capture values interpolated into the message template values. 124 | func (ld *PayloadRegexDecoder) Decode(pack *PipelinePack) (err error) { 125 | // First try to match the regex. 126 | match, captures := tryMatch(ld.Match, pack.Message.GetPayload()) 127 | if !match { 128 | return fmt.Errorf("No match") 129 | } 130 | 131 | pdh := &PayloadDecoderHelper{ 132 | Captures: captures, 133 | dRunner: ld.dRunner, 134 | TimestampLayout: ld.TimestampLayout, 135 | TzLocation: ld.tzLocation, 136 | SeverityMap: ld.SeverityMap, 137 | } 138 | 139 | pdh.DecodeTimestamp(pack) 140 | pdh.DecodeSeverity(pack) 141 | 142 | // Update the new message fields based on the fields we should 143 | // change and the capture parts 144 | return ld.MessageFields.PopulateMessage(pack.Message, captures) 145 | } 146 | -------------------------------------------------------------------------------- /update_mocks.sh: -------------------------------------------------------------------------------- 1 | # We use one spelling of the mockgen command for mocks of interfaces in our own 2 | # packages... 3 | 4 | # pipeline.PluginHelper 5 | $GOPATH/bin/mockgen -package=pipeline \ 6 | -destination=pipeline/mock_pluginhelper_test.go \ 7 | -self_package=github.com/mozilla-services/heka/pipeline \ 8 | github.com/mozilla-services/heka/pipeline PluginHelper 9 | 10 | # pipeline.PluginRunner 11 | $GOPATH/bin/mockgen -package=pipeline \ 12 | -destination=pipeline/mock_pluginrunner_test.go \ 13 | -self_package=github.com/mozilla-services/heka/pipeline \ 14 | github.com/mozilla-services/heka/pipeline PluginRunner 15 | 16 | # pipeline.Decoder 17 | $GOPATH/bin/mockgen -package=pipeline \ 18 | -destination=pipeline/mock_decoder_test.go \ 19 | -self_package=github.com/mozilla-services/heka/pipeline \ 20 | github.com/mozilla-services/heka/pipeline Decoder 21 | 22 | # pipeline.DecoderSet 23 | $GOPATH/bin/mockgen -package=pipeline \ 24 | -destination=pipeline/mock_decoderset_test.go \ 25 | -self_package=github.com/mozilla-services/heka/pipeline \ 26 | github.com/mozilla-services/heka/pipeline DecoderSet 27 | 28 | # pipeline.DecoderRunner 29 | $GOPATH/bin/mockgen -package=pipeline \ 30 | -destination=pipeline/mock_decoderrunner_test.go \ 31 | -self_package=github.com/mozilla-services/heka/pipeline \ 32 | github.com/mozilla-services/heka/pipeline DecoderRunner 33 | 34 | # pipeline.InputRunner 35 | $GOPATH/bin/mockgen -package=pipeline \ 36 | -destination=pipeline/mock_inputrunner_test.go \ 37 | -self_package=github.com/mozilla-services/heka/pipeline \ 38 | github.com/mozilla-services/heka/pipeline InputRunner 39 | 40 | # pipeline.FilterRunner 41 | $GOPATH/bin/mockgen -package=pipeline \ 42 | -destination=pipeline/mock_filterrunner_test.go \ 43 | -self_package=github.com/mozilla-services/heka/pipeline \ 44 | github.com/mozilla-services/heka/pipeline FilterRunner 45 | 46 | # pipeline.OutputRunner 47 | $GOPATH/bin/mockgen -package=pipeline \ 48 | -destination=pipeline/mock_outputrunner_test.go \ 49 | -self_package=github.com/mozilla-services/heka/pipeline \ 50 | github.com/mozilla-services/heka/pipeline OutputRunner 51 | 52 | # pipeline.Input 53 | $GOPATH/bin/mockgen -package=pipeline \ 54 | -destination=pipeline/mock_input_test.go \ 55 | -self_package=github.com/mozilla-services/heka/pipeline \ 56 | github.com/mozilla-services/heka/pipeline Input 57 | 58 | 59 | # pipeline.WhisperRunner 60 | $GOPATH/bin/mockgen -package=pipeline \ 61 | -destination=pipeline/mock_whisperrunner_test.go \ 62 | -self_package=github.com/mozilla-services/heka/pipeline \ 63 | github.com/mozilla-services/heka/pipeline WhisperRunner 64 | 65 | # pipeline.AMQPConnection 66 | $GOPATH/bin/mockgen -package=pipeline \ 67 | -destination=pipeline/mock_amqpconnection_test.go \ 68 | -self_package=github.com/mozilla-services/heka/pipeline \ 69 | github.com/mozilla-services/heka/pipeline AMQPConnection 70 | 71 | # pipeline.AMQPChannel 72 | $GOPATH/bin/mockgen -package=pipeline \ 73 | -destination=pipeline/mock_amqpchannel_test.go \ 74 | -self_package=github.com/mozilla-services/heka/pipeline \ 75 | github.com/mozilla-services/heka/pipeline AMQPChannel 76 | 77 | # pipeline.AMQPConnectionHub 78 | $GOPATH/bin/mockgen -package=pipeline \ 79 | -destination=pipeline/mock_amqpconnectionhub_test.go \ 80 | -self_package=github.com/mozilla-services/heka/pipeline \ 81 | github.com/mozilla-services/heka/pipeline AMQPConnectionHub 82 | 83 | # pipeline.StatAccumulator 84 | $GOPATH/bin/mockgen -package=pipeline \ 85 | -destination=pipeline/mock_stataccumulator_test.go \ 86 | -self_package=github.com/mozilla-services/heka/pipeline \ 87 | github.com/mozilla-services/heka/pipeline StatAccumulator 88 | 89 | # ...and a second spelling for mocks of interfaces that are from external packages. 90 | 91 | # amqp.Acknowledger 92 | $GOPATH/bin/mockgen -package=testsupport \ 93 | -destination=testsupport/mock_amqp_acknowledger.go \ 94 | github.com/streadway/amqp Acknowledger 95 | 96 | # net.Conn 97 | $GOPATH/bin/mockgen -package=testsupport \ 98 | -destination=testsupport/mock_net_conn.go \ 99 | net Conn 100 | 101 | # net.Listener 102 | $GOPATH/bin/mockgen -package=testsupport \ 103 | -destination=testsupport/mock_net_listener.go \ 104 | net Listener 105 | 106 | # net.Error 107 | $GOPATH/bin/mockgen -package=testsupport \ 108 | -destination=testsupport/mock_net_error.go \ 109 | net Error 110 | -------------------------------------------------------------------------------- /pipeline/whisper_test.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Rob Miller (rmiller@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | package pipeline 16 | 17 | import ( 18 | "code.google.com/p/gomock/gomock" 19 | "fmt" 20 | ts "github.com/mozilla-services/heka/testsupport" 21 | "github.com/rafrombrc/gospec/src/gospec" 22 | gs "github.com/rafrombrc/gospec/src/gospec" 23 | "github.com/rafrombrc/whisper-go/whisper" 24 | "os" 25 | "path" 26 | "strings" 27 | "sync" 28 | "time" 29 | ) 30 | 31 | func WhisperRunnerSpec(c gospec.Context) { 32 | tmpDir := os.TempDir() 33 | tmpFileName := fmt.Sprintf("heka-%d.wsp", time.Now().UTC().UnixNano()) 34 | tmpFileName = path.Join(tmpDir, tmpFileName) 35 | 36 | interval := uint32(10) 37 | archiveInfo := []whisper.ArchiveInfo{ 38 | whisper.ArchiveInfo{0, interval, 60}, 39 | whisper.ArchiveInfo{0, 60, 8}, 40 | } 41 | 42 | c.Specify("A WhisperRunner", func() { 43 | var wg sync.WaitGroup 44 | wg.Add(1) 45 | folderPerm := os.FileMode(0755) 46 | wr, err := NewWhisperRunner(tmpFileName, archiveInfo, whisper.AGGREGATION_SUM, 47 | folderPerm, &wg) 48 | c.Assume(err, gs.IsNil) 49 | defer func() { 50 | os.Remove(tmpFileName) 51 | }() 52 | 53 | c.Specify("creates a whisper file of the correct size", func() { 54 | fi, err := os.Stat(tmpFileName) 55 | c.Expect(err, gs.IsNil) 56 | c.Expect(fi.Size(), gs.Equals, int64(856)) 57 | close(wr.InChan()) 58 | wg.Wait() 59 | }) 60 | 61 | c.Specify("writes a data point to the whisper file", func() { 62 | // Send a data point through and close. 63 | when := time.Now().UTC() 64 | val := float64(6) 65 | pt := whisper.NewPoint(when, val) 66 | wr.InChan() <- &pt 67 | close(wr.InChan()) 68 | wg.Wait() 69 | 70 | // Open db file and fetch interval including our data point. 71 | from := when.Add(-1 * time.Second).Unix() 72 | until := when.Add(1 * time.Second).Unix() 73 | db, err := whisper.Open(tmpFileName) 74 | c.Expect(err, gs.IsNil) 75 | _, fetched, _ := db.FetchUntil(uint32(from), uint32(until)) 76 | 77 | // Verify that our value is stored in the most recent interval and 78 | // that the diff btn our timestamp and the stored time value is 79 | // less than the interval duration. 80 | fpt := fetched[len(fetched)-1] 81 | diff := when.Sub(fpt.Time().UTC()) 82 | c.Expect(diff.Seconds() < float64(interval), gs.IsTrue) 83 | c.Expect(fpt.Value, gs.Equals, val) 84 | }) 85 | }) 86 | } 87 | 88 | func WhisperOutputSpec(c gospec.Context) { 89 | t := &ts.SimpleT{} 90 | ctrl := gomock.NewController(t) 91 | defer ctrl.Finish() 92 | 93 | oth := new(OutputTestHelper) 94 | oth.MockHelper = NewMockPluginHelper(ctrl) 95 | oth.MockOutputRunner = NewMockOutputRunner(ctrl) 96 | inChan := make(chan *PipelinePack, 1) 97 | pConfig := NewPipelineConfig(nil) 98 | 99 | c.Specify("A WhisperOutput", func() { 100 | o := new(WhisperOutput) 101 | config := o.ConfigStruct().(*WhisperOutputConfig) 102 | config.BasePath = path.Join(os.TempDir(), config.BasePath) 103 | o.Init(config) 104 | 105 | const count = 5 106 | lines := make([]string, count) 107 | baseTime := time.Now().UTC().Add(-10 * time.Second) 108 | nameTmpl := "stats.name.%d" 109 | 110 | wChan := make(chan *whisper.Point, count) 111 | mockWr := NewMockWhisperRunner(ctrl) 112 | 113 | for i := 0; i < count; i++ { 114 | statName := fmt.Sprintf(nameTmpl, i) 115 | statTime := baseTime.Add(time.Duration(i) * time.Second) 116 | lines[i] = fmt.Sprintf("%s %d %d", statName, i*2, statTime.Unix()) 117 | o.dbs[statName] = mockWr 118 | } 119 | 120 | pack := NewPipelinePack(pConfig.inputRecycleChan) 121 | pack.Message.SetPayload(strings.Join(lines, "\n")) 122 | 123 | c.Specify("turns statmetric lines into points", func() { 124 | inChanCall := oth.MockOutputRunner.EXPECT().InChan() 125 | inChanCall.Return(inChan) 126 | wChanCall := mockWr.EXPECT().InChan().Times(count) 127 | wChanCall.Return(wChan) 128 | 129 | go o.Run(oth.MockOutputRunner, oth.MockHelper) 130 | inChan <- pack 131 | 132 | // Usually each wChan will be unique instead of shared across 133 | // multiple whisper runners. This weird dance here prevents our 134 | // mock wChan from being closed multiple times. 135 | bogusChan := make(chan *whisper.Point) 136 | wChanCall = mockWr.EXPECT().InChan().Times(count) 137 | wChanCall.Return(bogusChan) 138 | wChanCall.Do(func() { 139 | wChanCall.Return(make(chan *whisper.Point)) 140 | }) 141 | 142 | close(inChan) 143 | <-bogusChan // wait for inChan to flush 144 | close(wChan) 145 | 146 | i := 0 147 | for pt := range wChan { 148 | statTime := baseTime.Add(time.Duration(i) * time.Second) 149 | c.Expect(pt.Value, gs.Equals, float64(i*2)) 150 | c.Expect(pt.Time().UTC().Unix(), gs.Equals, statTime.Unix()) 151 | i++ 152 | } 153 | }) 154 | }) 155 | } 156 | -------------------------------------------------------------------------------- /message/date_helpers.go: -------------------------------------------------------------------------------- 1 | /***** BEGIN LICENSE BLOCK ***** 2 | # This Source Code Form is subject to the terms of the Mozilla Public 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 4 | # You can obtain one at http://mozilla.org/MPL/2.0/. 5 | # 6 | # The Initial Developer of the Original Code is the Mozilla Foundation. 7 | # Portions created by the Initial Developer are Copyright (C) 2012 8 | # the Initial Developer. All Rights Reserved. 9 | # 10 | # Contributor(s): 11 | # Ben Bangert (bbangert@mozilla.com) 12 | # 13 | # ***** END LICENSE BLOCK *****/ 14 | 15 | package message 16 | 17 | import ( 18 | "regexp" 19 | "strings" 20 | "time" 21 | ) 22 | 23 | var ( 24 | basicTimeLayouts = map[string]string{ 25 | "ANSIC": time.ANSIC, 26 | "UnixDate": time.UnixDate, 27 | "RubyDate": time.RubyDate, 28 | "RFC822": time.RFC822, 29 | "RFC822Z": time.RFC822Z, 30 | "RFC850": time.RFC850, 31 | "RFC1123": time.RFC1123, 32 | "RFC1123Z": time.RFC1123Z, 33 | "RFC3339": time.RFC3339, 34 | "RFC3339Nano": time.RFC3339Nano, 35 | "Kitchen": time.Kitchen, 36 | "Stamp": time.Stamp, 37 | "StampMilli": time.StampMilli, 38 | "StampMicro": time.StampMicro, 39 | "StampNano": time.StampNano, 40 | } 41 | 42 | dateMatchStrings = []string{ 43 | // Mon Jan _2 15:04:05 2006 44 | "SDAY SMONTH\\s{1,2}\\d{1,2} \\d{2}:\\d{2}:\\d{2} \\d{4}", 45 | // Mon Jan _2 15:04:05 MST 2006 46 | "SDAY SMONTH\\s{1,2}\\d{1,2} \\d{2}:\\d{2}:\\d{2} \\w{3} \\d{4}", 47 | // Mon Jan 02 15:04:05 -0700 2006 48 | "SDAY SMONTH \\d{2} \\d{2}:\\d{2}:\\d{2} -\\d{4} \\d{4}", 49 | // 02 Jan 06 15:04 MST 50 | "SDAY SMONTH \\d{2} \\d{2}:\\d{2} \\w{3}", 51 | // 02 Jan 06 15:04 -0700 52 | "\\d{2} SMONTH \\d{2} \\d{2}:\\d{2} \\w{3} -\\d{4}", 53 | // Monday, 02-Jan-06 15:04:05 MST 54 | "DAY \\d{2}-SMONTH-\\d{2} \\d{2}:\\d{2}:\\d{2} \\w{3}", 55 | // Mon, 02 Jan 2006 15:04:05 MST 56 | "SDAY \\d{2} SMONTH \\d{4} \\d{2}:\\d{2}:\\d{2} \\w{3}", 57 | // Mon, 02 Jan 2006 15:04:05 -0700 58 | "SDAY \\d{2} SMONTH \\d{4} \\d{2}:\\d{2}:\\d{2} \\w{3} -\\d{4}", 59 | // 2006-01-02T15:04:05Z07:00 60 | "\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z\\d{2}:\\d{2}", 61 | // 2006-01-02T15:04:05.999999999Z07:00 62 | "\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d+Z\\d{2}:\\d{2}", 63 | // 3:04PM - Kitchen format.... really? Kitchen? sigh. 64 | "\\d{1,2}:\\d{2}[AP]M", 65 | // Jan _2 15:04:05 66 | "SMONTH\\s{1,2}\\d{1,2} \\d{2}:\\d{2}:\\d{2}", 67 | // Jan _2 15:04:05.000 68 | "SMONTH\\s{1,2}\\d{1,2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}", 69 | // Jan _2 15:04:05.000000 70 | "SMONTH\\s{1,2}\\d{1,2} \\d{2}:\\d{2}:\\d{2}\\.\\d{6}", 71 | // Jan _2 15:04:05.000000000 72 | "SMONTH\\s{1,2}\\d{1,2} \\d{2}:\\d{2}:\\d{2}\\.\\d{9}", 73 | } 74 | 75 | // We have to duplicate this, cause time doesn't export them. Blech. 76 | longDayNames = []string{ 77 | "Sunday", 78 | "Monday", 79 | "Tuesday", 80 | "Wednesday", 81 | "Thursday", 82 | "Friday", 83 | "Saturday", 84 | } 85 | 86 | shortDayNames = []string{ 87 | "Sun", 88 | "Mon", 89 | "Tue", 90 | "Wed", 91 | "Thu", 92 | "Fri", 93 | "Sat", 94 | } 95 | 96 | shortMonthNames = []string{ 97 | "---", 98 | "Jan", 99 | "Feb", 100 | "Mar", 101 | "Apr", 102 | "May", 103 | "Jun", 104 | "Jul", 105 | "Aug", 106 | "Sep", 107 | "Oct", 108 | "Nov", 109 | "Dec", 110 | } 111 | 112 | // A mapping that returns a complex regular expression string for 113 | // a commonly matched portion rather than having to construct one. 114 | // 115 | // Currently HelperRegexSubs has the following keys upon startup that 116 | // may be used: 117 | // TIMESTAMP - A complex regular expression string that matches 118 | // any of the Go time const layouts. 119 | HelperRegexSubs map[string]string 120 | ) 121 | 122 | // Parse a time with the supplied timeLayout, falling back to all the 123 | // basicTimeLayouts. 124 | func ForgivingTimeParse(timeLayout, inputTime string, loc *time.Location) ( 125 | parsedTime time.Time, err error) { 126 | 127 | parsedTime, err = time.ParseInLocation(timeLayout, inputTime, loc) 128 | if err == nil { 129 | return 130 | } 131 | for _, layout := range basicTimeLayouts { 132 | parsedTime, err = time.ParseInLocation(layout, inputTime, loc) 133 | if err == nil { 134 | return 135 | } 136 | } 137 | return 138 | } 139 | 140 | func init() { 141 | HelperRegexSubs = make(map[string]string) 142 | 143 | smonths := "(?:" + strings.Join(shortMonthNames, "|") + ")" 144 | sdays := "(?:" + strings.Join(shortDayNames, "|") + ")" 145 | days := "(?:" + strings.Join(longDayNames, "|") + ")" 146 | 147 | newMatchStrings := make([]string, 0, 15) 148 | replaceShorts, _ := regexp.Compile("(SDAY|DAY|SMONTH)") 149 | for _, dateStr := range dateMatchStrings { 150 | newStr := replaceShorts.ReplaceAllStringFunc(dateStr, 151 | func(match string) string { 152 | switch match { 153 | case "SDAY": 154 | return sdays 155 | case "DAY": 156 | return days 157 | case "SMONTH": 158 | return smonths 159 | } 160 | return match 161 | }) 162 | newMatchStrings = append(newMatchStrings, "(?:"+newStr+")") 163 | } 164 | tsRegexString := "(?P" + strings.Join(newMatchStrings, "|") + ")" 165 | HelperRegexSubs["TIMESTAMP"] = tsRegexString 166 | } 167 | --------------------------------------------------------------------------------