├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ ├── config.yml │ ├── feature_request.md │ └── bug_report.md ├── codecov.yaml ├── CODEOWNERS ├── workflows │ ├── lint.yml │ ├── gosec.yml │ └── license.yml ├── dependabot.yml └── PULL_REQUEST_TEMPLATE.md ├── operator ├── builtin │ ├── input │ │ ├── file │ │ │ ├── testdata │ │ │ │ ├── default.yaml │ │ │ │ ├── id_custom.yaml │ │ │ │ ├── extra_field.yaml │ │ │ │ ├── encoding_lower.yaml │ │ │ │ ├── encoding_upper.yaml │ │ │ │ ├── include_invalid.yaml │ │ │ │ ├── include_one.yaml │ │ │ │ ├── poll_interval_1ms.yaml │ │ │ │ ├── poll_interval_1s.yaml │ │ │ │ ├── include_glob.yaml │ │ │ │ ├── max_log_size_mb_lower.yaml │ │ │ │ ├── max_log_size_mb_upper.yaml │ │ │ │ ├── max_log_size_mib_lower.yaml │ │ │ │ ├── max_log_size_mib_upper.yaml │ │ │ │ ├── poll_interval_no_units.yaml │ │ │ │ ├── start_at_string.yaml │ │ │ │ ├── fingerprint_size_1KB.yaml │ │ │ │ ├── fingerprint_size_1KiB.yaml │ │ │ │ ├── fingerprint_size_float.yaml │ │ │ │ ├── include_inline.yaml │ │ │ │ ├── max_log_size_invalid_unit.yaml │ │ │ │ ├── poll_interval_1000ms.yaml │ │ │ │ ├── fingerprint_size_1kb_lower.yaml │ │ │ │ ├── fingerprint_size_1kib_lower.yaml │ │ │ │ ├── fingerprint_size_no_units.yaml │ │ │ │ ├── include_glob_double_asterisk.yaml │ │ │ │ ├── exclude_one.yaml │ │ │ │ ├── max_concurrent_large.yaml │ │ │ │ ├── multiline_line_end_special.yaml │ │ │ │ ├── exclude_glob.yaml │ │ │ │ ├── exclude_inline.yaml │ │ │ │ ├── exclude_invalid.yaml │ │ │ │ ├── include_file_name_on.yaml │ │ │ │ ├── include_file_path_no.yaml │ │ │ │ ├── include_file_path_on.yaml │ │ │ │ ├── label_regex.yaml │ │ │ │ ├── multiline_line_end_string.yaml │ │ │ │ ├── multiline_line_start_special.yaml │ │ │ │ ├── include_file_name_lower.yaml │ │ │ │ ├── include_file_name_upper.yaml │ │ │ │ ├── include_file_name_yes.yaml │ │ │ │ ├── include_file_path_lower.yaml │ │ │ │ ├── include_file_path_off.yaml │ │ │ │ ├── include_file_path_upper.yaml │ │ │ │ ├── include_file_path_yes.yaml │ │ │ │ ├── include_glob_double_asterisk_nested.yaml │ │ │ │ ├── include_glob_double_asterisk_prefix.yaml │ │ │ │ ├── include_multi.yaml │ │ │ │ ├── multiline_line_start_string.yaml │ │ │ │ ├── include_file_path_nonbool.yaml │ │ │ │ ├── exclude_glob_double_asterisk.yaml │ │ │ │ ├── multiline_extra_field.yaml │ │ │ │ ├── exclude_multi.yaml │ │ │ │ ├── exclude_glob_double_asterisk_nested.yaml │ │ │ │ └── exclude_glob_double_asterisk_prefix.yaml │ │ │ ├── positional_scanner.go │ │ │ ├── finder.go │ │ │ └── fingerprint.go │ │ ├── windows │ │ │ ├── testdata │ │ │ │ └── security │ │ │ │ │ ├── driver_started │ │ │ │ │ ├── details.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── message.out │ │ │ │ │ ├── service_started │ │ │ │ │ ├── details.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── message.out │ │ │ │ │ ├── event_processing │ │ │ │ │ ├── details.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── message.out │ │ │ │ │ ├── service_shutdown │ │ │ │ │ ├── details.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── message.out │ │ │ │ │ ├── audit_success │ │ │ │ │ ├── message.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── details.out │ │ │ │ │ ├── time_change │ │ │ │ │ ├── message.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── details.out │ │ │ │ │ ├── domain_policy_changed │ │ │ │ │ ├── message.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── details.out │ │ │ │ │ ├── logon │ │ │ │ │ └── message.out │ │ │ │ │ ├── user_account_changed │ │ │ │ │ ├── message.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── details.out │ │ │ │ │ ├── user_account_created │ │ │ │ │ ├── message.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── details.out │ │ │ │ │ ├── user_account_enabled │ │ │ │ │ ├── message.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── details.out │ │ │ │ │ ├── object_added │ │ │ │ │ ├── message.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── details.out │ │ │ │ │ ├── account_name_changed │ │ │ │ │ ├── message.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── details.out │ │ │ │ │ ├── special_logon │ │ │ │ │ ├── message.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── details.out │ │ │ │ │ ├── local_group_changed │ │ │ │ │ ├── message.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── details.out │ │ │ │ │ ├── audit_settings_changed │ │ │ │ │ ├── message.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── details.out │ │ │ │ │ ├── per_user_audit_policy_table_created │ │ │ │ │ ├── message.out │ │ │ │ │ ├── details.out │ │ │ │ │ └── message.in │ │ │ │ │ ├── user_added_to_global_group │ │ │ │ │ ├── message.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── details.out │ │ │ │ │ ├── user_password_reset_attempt │ │ │ │ │ ├── message.out │ │ │ │ │ ├── message.in │ │ │ │ │ └── details.out │ │ │ │ │ ├── credential_validate_attempt │ │ │ │ │ ├── message.out │ │ │ │ │ ├── details.out │ │ │ │ │ └── message.in │ │ │ │ │ └── query_blank_password │ │ │ │ │ ├── message.out │ │ │ │ │ ├── details.out │ │ │ │ │ └── message.in │ │ │ ├── os_test.go │ │ │ ├── api_test.go │ │ │ ├── event_test.go │ │ │ ├── publisher.go │ │ │ └── publisher_test.go │ │ ├── goflow │ │ │ ├── testdata │ │ │ │ ├── netflowv5.yaml │ │ │ │ └── netflowv5.sh │ │ │ └── goflow_test.go │ │ ├── azure │ │ │ ├── eventhub │ │ │ │ ├── parse.go │ │ │ │ └── event_hub.go │ │ │ ├── event_hub_persist_test.go │ │ │ ├── event_hub_persist.go │ │ │ └── event_hub_parse.go │ │ ├── stdin │ │ │ └── stdin_test.go │ │ ├── generate │ │ │ └── generate_test.go │ │ ├── aws │ │ │ └── cloudwatch │ │ │ │ ├── cloudwatch_persist.go │ │ │ │ └── cloudwatch_persist_test.go │ │ ├── stanza │ │ │ └── stanza_test.go │ │ └── http │ │ │ └── auth.go │ ├── output │ │ ├── googlecloud │ │ │ ├── client_test.go │ │ │ ├── client.go │ │ │ ├── severity.go │ │ │ └── severity_test.go │ │ ├── drop │ │ │ ├── drop_test.go │ │ │ └── drop.go │ │ ├── stdout │ │ │ ├── stdout_test.go │ │ │ └── stdout.go │ │ └── forward │ │ │ └── forward_test.go │ ├── transformer │ │ ├── noop │ │ │ ├── noop_test.go │ │ │ └── noop.go │ │ ├── ratelimit │ │ │ └── rate_limit_test.go │ │ └── hostmetadata │ │ │ └── host_metadata_test.go │ └── parser │ │ ├── xml │ │ └── element.go │ │ ├── severity │ │ └── severity.go │ │ └── time │ │ └── time.go ├── operator.go ├── helper │ ├── persister_test.go │ ├── labeler.go │ ├── identifier.go │ ├── host_identifier_test.go │ ├── output.go │ ├── operatortest │ │ └── operatortest.go │ ├── duration.go │ └── labeler_test.go ├── build_context_test.go ├── flusher │ └── flusher_test.go └── buffer │ ├── disk_metadata_test.go │ └── util_test.go ├── docs ├── images │ ├── logo_small.png │ └── stanza_plugins.png ├── examples │ ├── k8s │ │ ├── aks │ │ │ ├── assets │ │ │ │ └── entries.png │ │ │ └── README.md │ │ ├── eks │ │ │ ├── assets │ │ │ │ └── entries.png │ │ │ └── README.md │ │ ├── gke │ │ │ ├── assets │ │ │ │ └── entries.png │ │ │ └── README.md │ │ ├── events │ │ │ ├── assets │ │ │ │ └── events.png │ │ │ ├── config.yaml │ │ │ ├── log_credentials.json │ │ │ ├── service_account.yaml │ │ │ └── USAGE.md │ │ └── onprem │ │ │ ├── assets │ │ │ └── entries.png │ │ │ └── README.md │ ├── simple_plugins │ │ ├── config.yaml │ │ └── plugins │ │ │ ├── repeater.yaml │ │ │ └── decorator.yaml │ ├── tomcat │ │ ├── access.log │ │ └── config.yaml │ └── scenarios │ │ └── custom_parsing.md ├── types │ ├── bytesize.md │ ├── on_error.md │ ├── duration.md │ ├── expression.md │ ├── entry.md │ └── flusher.md ├── operators │ ├── stdout.md │ ├── forward_output.md │ ├── stdin.md │ ├── file_output.md │ ├── drop_output.md │ ├── rate_limit.md │ ├── forward_input.md │ ├── filter.md │ ├── udp_input.md │ ├── generate_input.md │ ├── stanza_input.md │ └── README.md ├── faq.md ├── proxy.md └── MIRRORS.md ├── cmd └── stanza │ ├── init_linux.go │ ├── init_windows.go │ ├── main.go │ ├── testdata │ ├── simple_plugins │ │ ├── config.yaml │ │ └── plugins │ │ │ ├── repeater.yaml │ │ │ └── decorator.yaml │ └── tomcat │ │ ├── access.log │ │ └── config.yaml │ ├── version.go │ ├── default_paths.go │ ├── service.go │ ├── graph.go │ └── offsets_test.go ├── .gitignore ├── pipeline ├── pipeline.go ├── node_test.go ├── config.go └── node.go ├── version └── version.go ├── logger ├── logger.go ├── emitter.go ├── parser.go └── core.go ├── revive └── config.toml ├── MAINTAINERS.md ├── errors └── details.go ├── license.yaml ├── entry ├── nil_field.go ├── nil_field_test.go ├── label_field.go ├── resource_field.go └── severity_test.go ├── .kitchen.yml ├── agent ├── agent.go └── config.go ├── Dockerfile ├── testutil ├── operator_builder.go ├── database.go ├── pipeline.go └── util.go └── database └── database.go /.dockerignore: -------------------------------------------------------------------------------- 1 | artifacts/ 2 | .git/ 3 | dev/ 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/default.yaml: -------------------------------------------------------------------------------- 1 | type: file_input -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/driver_started/details.out: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/service_started/details.out: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/id_custom.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | id: test_id -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/event_processing/details.out: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/service_shutdown/details.out: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/extra_field.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | hello: world -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/encoding_lower.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | encoding: "utf-16le" -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/encoding_upper.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | encoding: "UTF-16lE" -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_invalid.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: "justwords" -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_one.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - one.log -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/poll_interval_1ms.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | poll_interval: 1ms -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/poll_interval_1s.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | poll_interval: 1s -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/audit_success/message.out: -------------------------------------------------------------------------------- 1 | Windows is starting up. -------------------------------------------------------------------------------- /docs/images/logo_small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/observIQ/stanza/HEAD/docs/images/logo_small.png -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_glob.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - "*.log" -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/max_log_size_mb_lower.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | max_log_size: 1mib -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/max_log_size_mb_upper.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | max_log_size: 1MiB -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/max_log_size_mib_lower.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | max_log_size: 1mib -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/max_log_size_mib_upper.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | max_log_size: 1MiB -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/poll_interval_no_units.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | poll_interval: 1 -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/start_at_string.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | start_at: "beginning" -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/time_change/message.out: -------------------------------------------------------------------------------- 1 | The system time was changed. -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/fingerprint_size_1KB.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | fingerprint_size: 1KB -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/fingerprint_size_1KiB.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | fingerprint_size: 1KiB -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/fingerprint_size_float.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | fingerprint_size: 1.1kb -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_inline.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: [ "a.log", "b.log" ] -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/max_log_size_invalid_unit.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | max_log_size: 1TOFU -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/poll_interval_1000ms.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | poll_interval: 1000ms -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/domain_policy_changed/message.out: -------------------------------------------------------------------------------- 1 | Domain Policy was changed. -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/logon/message.out: -------------------------------------------------------------------------------- 1 | An account was successfully logged on. -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/user_account_changed/message.out: -------------------------------------------------------------------------------- 1 | A user account was changed. -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/user_account_created/message.out: -------------------------------------------------------------------------------- 1 | A user account was created. -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/user_account_enabled/message.out: -------------------------------------------------------------------------------- 1 | A user account was enabled. -------------------------------------------------------------------------------- /docs/images/stanza_plugins.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/observIQ/stanza/HEAD/docs/images/stanza_plugins.png -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/fingerprint_size_1kb_lower.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | fingerprint_size: 1kb -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/fingerprint_size_1kib_lower.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | fingerprint_size: 1kib -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/fingerprint_size_no_units.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | fingerprint_size: 1000 -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/object_added/message.out: -------------------------------------------------------------------------------- 1 | An object was added to the COM+ Catalog. -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_glob_double_asterisk.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - "**.log" -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/account_name_changed/message.out: -------------------------------------------------------------------------------- 1 | The name of an account was changed -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/service_shutdown/message.in: -------------------------------------------------------------------------------- 1 | The event logging service has shut down. -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/service_shutdown/message.out: -------------------------------------------------------------------------------- 1 | The event logging service has shut down. -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/special_logon/message.out: -------------------------------------------------------------------------------- 1 | Special privileges assigned to new logon. -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/driver_started/message.in: -------------------------------------------------------------------------------- 1 | The Windows Firewall Driver started successfully. -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/local_group_changed/message.out: -------------------------------------------------------------------------------- 1 | A security-enabled local group was changed. -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/exclude_one.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - "*.log" 4 | exclude: 5 | - one.log -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/max_concurrent_large.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | max_concurrent_files: 9223372036854775807 -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/multiline_line_end_special.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | multiline: 3 | line_end_pattern: '%' -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/audit_settings_changed/message.out: -------------------------------------------------------------------------------- 1 | Auditing settings on object were changed. -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/driver_started/message.out: -------------------------------------------------------------------------------- 1 | The Windows Firewall Driver started successfully. -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/event_processing/message.in: -------------------------------------------------------------------------------- 1 | Audit events have been dropped by the transport. 0 -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/event_processing/message.out: -------------------------------------------------------------------------------- 1 | Audit events have been dropped by the transport. 0 -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/service_started/message.in: -------------------------------------------------------------------------------- 1 | The Windows Firewall service started successfully. -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/service_started/message.out: -------------------------------------------------------------------------------- 1 | The Windows Firewall service started successfully. -------------------------------------------------------------------------------- /docs/examples/k8s/aks/assets/entries.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/observIQ/stanza/HEAD/docs/examples/k8s/aks/assets/entries.png -------------------------------------------------------------------------------- /docs/examples/k8s/eks/assets/entries.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/observIQ/stanza/HEAD/docs/examples/k8s/eks/assets/entries.png -------------------------------------------------------------------------------- /docs/examples/k8s/gke/assets/entries.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/observIQ/stanza/HEAD/docs/examples/k8s/gke/assets/entries.png -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/exclude_glob.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - "*.log" 4 | exclude: 5 | - "not*.log" -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/exclude_inline.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: [ "*.log" ] 3 | exclude: [ "a.log", "b.log" ] -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/exclude_invalid.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - "*.log" 4 | exclude: "aRandomString" -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_file_name_on.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - one.log 4 | include_file_name: on -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_file_path_no.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - one.log 4 | include_file_path: no -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_file_path_on.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - one.log 4 | include_file_path: on -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/label_regex.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | label_regex: "^(?P[a-zA-z]+ [A-Z]+): (?P.*)" -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/multiline_line_end_string.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | multiline: 3 | line_end_pattern: 'Start' -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/multiline_line_start_special.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | multiline: 3 | line_start_pattern: '%' -------------------------------------------------------------------------------- /docs/examples/k8s/events/assets/events.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/observIQ/stanza/HEAD/docs/examples/k8s/events/assets/events.png -------------------------------------------------------------------------------- /docs/examples/k8s/onprem/assets/entries.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/observIQ/stanza/HEAD/docs/examples/k8s/onprem/assets/entries.png -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_file_name_lower.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - one.log 4 | include_file_name: true -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_file_name_upper.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - one.log 4 | include_file_name: TRUE -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_file_name_yes.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - one.log 4 | include_file_name: yes -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_file_path_lower.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - one.log 4 | include_file_path: true -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_file_path_off.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - one.log 4 | include_file_path: off -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_file_path_upper.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - one.log 4 | include_file_path: TRUE -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_file_path_yes.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - one.log 4 | include_file_path: yes -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_glob_double_asterisk_nested.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - "directory/**/*.log" -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_glob_double_asterisk_prefix.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - "**/directory/**/*.log" -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_multi.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - one.log 4 | - two.log 5 | - three.log -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/multiline_line_start_string.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | multiline: 3 | line_start_pattern: 'Start' -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/include_file_path_nonbool.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - one.log 4 | include_file_path: asdf -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/per_user_audit_policy_table_created/message.out: -------------------------------------------------------------------------------- 1 | The Per-user audit policy table was created. -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/user_added_to_global_group/message.out: -------------------------------------------------------------------------------- 1 | A member was added to a security-enabled global group. -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/user_password_reset_attempt/message.out: -------------------------------------------------------------------------------- 1 | An attempt was made to reset an account's password. -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/exclude_glob_double_asterisk.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - "*.log" 4 | exclude: 5 | - "not**.log" -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/multiline_extra_field.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | multiline: 3 | that_random_field: "this should go nowhere" -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/credential_validate_attempt/message.out: -------------------------------------------------------------------------------- 1 | The computer attempted to validate the credentials for an account. -------------------------------------------------------------------------------- /.github/codecov.yaml: -------------------------------------------------------------------------------- 1 | coverage: 2 | range: 50..90 3 | round: nearest 4 | status: 5 | project: off 6 | patch: off 7 | ignore: 8 | - "testutil/*" 9 | -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/exclude_multi.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - "*.log" 4 | exclude: 5 | - one.log 6 | - two.log 7 | - three.log -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/query_blank_password/message.out: -------------------------------------------------------------------------------- 1 | An attempt was made to query the existence of a blank password for an account. -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/exclude_glob_double_asterisk_nested.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - "*.log" 4 | exclude: 5 | - "directory/**/not*.log" -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/per_user_audit_policy_table_created/details.out: -------------------------------------------------------------------------------- 1 | { 2 | "Number of Elements": "0", 3 | "Policy ID": "0xAE48" 4 | } -------------------------------------------------------------------------------- /operator/builtin/input/file/testdata/exclude_glob_double_asterisk_prefix.yaml: -------------------------------------------------------------------------------- 1 | type: file_input 2 | include: 3 | - "*.log" 4 | exclude: 5 | - "**/directory/**/not*.log" -------------------------------------------------------------------------------- /operator/builtin/input/goflow/testdata/netflowv5.yaml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | - type: goflow_input 3 | mode: netflow_v5 4 | port: 2056 5 | - type: file_output 6 | path: /testdata/out.log 7 | 8 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/audit_success/message.in: -------------------------------------------------------------------------------- 1 | Windows is starting up. 2 | 3 | This event is logged when LSASS.EXE starts and the auditing subsystem is initialized. -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/per_user_audit_policy_table_created/message.in: -------------------------------------------------------------------------------- 1 | The Per-user audit policy table was created. 2 | 3 | Number of Elements: 0 4 | Policy ID: 0xAE48 -------------------------------------------------------------------------------- /cmd/stanza/init_linux.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | // Load linux only packages when importing input operators 5 | _ "github.com/observiq/stanza/operator/builtin/input/journald" 6 | ) 7 | -------------------------------------------------------------------------------- /cmd/stanza/init_windows.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | // Load windows only packages when importing input operators 5 | _ "github.com/observiq/stanza/operator/builtin/input/windows" 6 | ) 7 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # These owners will be the default owners for everything in 2 | # the repo. Unless a later match takes precedence, 3 | # review when someone opens a pull request. 4 | * @jsirianni 5 | 6 | -------------------------------------------------------------------------------- /cmd/stanza/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | func main() { 8 | rootCmd := NewRootCmd() 9 | err := rootCmd.Execute() 10 | if err != nil { 11 | os.Exit(1) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/audit_success/details.out: -------------------------------------------------------------------------------- 1 | { 2 | "Additional Context": [ 3 | "This event is logged when LSASS.EXE starts and the auditing subsystem is initialized." 4 | ] 5 | } -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/credential_validate_attempt/details.out: -------------------------------------------------------------------------------- 1 | { 2 | "Authentication Package": "MICROSOFT_AUTHENTICATION_PACKAGE_V1_0", 3 | "Error Code": "0x0", 4 | "Logon Account": "Someone", 5 | "Source Workstation": "WIN-322E2C550UP" 6 | } -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/credential_validate_attempt/message.in: -------------------------------------------------------------------------------- 1 | The computer attempted to validate the credentials for an account. 2 | 3 | Authentication Package: MICROSOFT_AUTHENTICATION_PACKAGE_V1_0 4 | Logon Account: Someone 5 | Source Workstation: WIN-322E2C550UP 6 | Error Code: 0x0 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dev/* 2 | .DS_Store 3 | tmp/* 4 | local/* 5 | **/coverage.txt 6 | **/coverage.html 7 | artifacts/* 8 | **/.vscode/* 9 | gen/ 10 | **/testdata/*.log 11 | **/*.msi 12 | **/*.exe 13 | **/*zip 14 | **/wix 15 | **/.vagrant 16 | **/wix.dynamic.json 17 | stanza-plugins/ 18 | **/plugins 19 | **/.kitchen 20 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/user_account_enabled/message.in: -------------------------------------------------------------------------------- 1 | A user account was enabled. 2 | 3 | Subject: 4 | Security ID: SYSTEM 5 | Account Name: WIN-322E2C550UP$ 6 | Account Domain: WORKGROUP 7 | Logon ID: 0x3E7 8 | 9 | Target Account: 10 | Security ID: WIN-322E2C550UP\Someone 11 | Account Name: Someone 12 | Account Domain: WIN-322E2C550UP -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | on: 3 | pull_request: 4 | 5 | jobs: 6 | lint: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Checkout Sources 10 | uses: actions/checkout@v4 11 | - name: Run Revive Action by pulling pre-built image 12 | uses: docker://morphy/revive-action:v2 13 | with: 14 | config: revive/config.toml 15 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/user_password_reset_attempt/message.in: -------------------------------------------------------------------------------- 1 | An attempt was made to reset an account's password. 2 | 3 | Subject: 4 | Security ID: SYSTEM 5 | Account Name: WIN-322E2C550UP$ 6 | Account Domain: WORKGROUP 7 | Logon ID: 0x3E7 8 | 9 | Target Account: 10 | Security ID: WIN-322E2C550UP\Someone 11 | Account Name: Someone 12 | Account Domain: WIN-322E2C550UP -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/user_account_enabled/details.out: -------------------------------------------------------------------------------- 1 | { 2 | "Subject": { 3 | "Account Domain": "WORKGROUP", 4 | "Account Name": "WIN-322E2C550UP$", 5 | "Logon ID": "0x3E7", 6 | "Security ID": "SYSTEM" 7 | }, 8 | "Target Account": { 9 | "Account Domain": "WIN-322E2C550UP", 10 | "Account Name": "Someone", 11 | "Security ID": "WIN-322E2C550UP\\Someone" 12 | } 13 | } -------------------------------------------------------------------------------- /pipeline/pipeline.go: -------------------------------------------------------------------------------- 1 | //go:generate mockery --name=^(Pipeline)$ --output=../testutil --outpkg=testutil --case=snake 2 | 3 | package pipeline 4 | 5 | import "github.com/observiq/stanza/operator" 6 | 7 | // Pipeline is a collection of connected operators that exchange entries 8 | type Pipeline interface { 9 | Start() error 10 | Stop() error 11 | Operators() []operator.Operator 12 | Render() ([]byte, error) 13 | } 14 | -------------------------------------------------------------------------------- /docs/examples/k8s/events/config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: stanza-agent 6 | namespace: default 7 | data: 8 | config.yaml: | 9 | pipeline: 10 | - type: kubernetes_events 11 | cluster_name: CHANGE_ME 12 | - credentials_file: /stanza_home/log_destinations/google_cloud/log_credentials.json 13 | project_id: CHANGE_ME 14 | type: google_cloud_output 15 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/os_test.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package windows 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | 10 | "github.com/observiq/stanza/operator" 11 | ) 12 | 13 | func TestWindowsOnly(t *testing.T) { 14 | _, ok := operator.Lookup("windows_eventlog_input") 15 | require.False(t, ok, "'windows_eventlog_input' should only be available on windows") 16 | } 17 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/user_password_reset_attempt/details.out: -------------------------------------------------------------------------------- 1 | { 2 | "Subject": { 3 | "Account Domain": "WORKGROUP", 4 | "Account Name": "WIN-322E2C550UP$", 5 | "Logon ID": "0x3E7", 6 | "Security ID": "SYSTEM" 7 | }, 8 | "Target Account": { 9 | "Account Domain": "WIN-322E2C550UP", 10 | "Account Name": "Someone", 11 | "Security ID": "WIN-322E2C550UP\\Someone" 12 | } 13 | } -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | # Setting the limit to zero should limit 5 | # Dependabot to security updates only. 6 | open-pull-requests-limit: 0 7 | directory: "/" 8 | schedule: 9 | interval: "monthly" 10 | day: "monday" 11 | labels: 12 | - "dependencies" 13 | - "security" 14 | commit-message: 15 | prefix: "deps" 16 | include: "scope" -------------------------------------------------------------------------------- /docs/examples/k8s/events/log_credentials.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "service_account", 3 | "project_id": "", 4 | "private_key_id": "", 5 | "private_key": "", 6 | "client_email": "", 7 | "client_id": "", 8 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 9 | "token_uri": "https://oauth2.googleapis.com/token", 10 | "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", 11 | "client_x509_cert_url": "" 12 | } 13 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/object_added/message.in: -------------------------------------------------------------------------------- 1 | An object was added to the COM+ Catalog. 2 | 3 | Subject: 4 | Security ID: SYSTEM 5 | Account Name: SYSTEM 6 | Account Domain: NT AUTHORITY 7 | Logon ID: 0x3E7 8 | 9 | Object: 10 | COM+ Catalog Collection: Roles 11 | Object Name: 12 | ApplId = {01234567-ABCD-ABCD-ABCD-0123456789AB} 13 | Name = Administrators 14 | Object Details: 15 | Description = Administrators group -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/account_name_changed/message.in: -------------------------------------------------------------------------------- 1 | The name of an account was changed: 2 | 3 | Subject: 4 | Security ID: SYSTEM 5 | Account Name: WIN-322E2C550UP$ 6 | Account Domain: WORKGROUP 7 | Logon ID: 0x3E7 8 | 9 | Target Account: 10 | Security ID: BUILTIN\Power Users 11 | Account Domain: Builtin 12 | Old Account Name: Power Users 13 | New Account Name: Power Users 14 | 15 | Additional Information: 16 | Privileges: - -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/query_blank_password/details.out: -------------------------------------------------------------------------------- 1 | { 2 | "Additional Information": { 3 | "Caller Workstation": "WIN-322E2C550UP", 4 | "Target Account Domain": "WIN-322E2C550UP", 5 | "Target Account Name": "Administrator" 6 | }, 7 | "Subject": { 8 | "Account Domain": "WIN-322E2C550UP", 9 | "Account Name": "Someone", 10 | "Logon ID": "0x2EECB", 11 | "Security ID": "WIN-322E2C550UP\\Someone" 12 | } 13 | } -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/query_blank_password/message.in: -------------------------------------------------------------------------------- 1 | An attempt was made to query the existence of a blank password for an account. 2 | 3 | Subject: 4 | Security ID: WIN-322E2C550UP\Someone 5 | Account Name: Someone 6 | Account Domain: WIN-322E2C550UP 7 | Logon ID: 0x2EECB 8 | 9 | Additional Information: 10 | Caller Workstation: WIN-322E2C550UP 11 | Target Account Name: Administrator 12 | Target Account Domain: WIN-322E2C550UP -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/local_group_changed/message.in: -------------------------------------------------------------------------------- 1 | A security-enabled local group was changed. 2 | 3 | Subject: 4 | Security ID: SYSTEM 5 | Account Name: WIN-322E2C550UP$ 6 | Account Domain: WORKGROUP 7 | Logon ID: 0x3E7 8 | 9 | Group: 10 | Security ID: BUILTIN\Administrators 11 | Group Name: Administrators 12 | Group Domain: Builtin 13 | 14 | Changed Attributes: 15 | SAM Account Name: - 16 | SID History: - 17 | 18 | Additional Information: 19 | Privileges: - -------------------------------------------------------------------------------- /docs/types/bytesize.md: -------------------------------------------------------------------------------- 1 | # ByteSize 2 | 3 | ByteSizes are a type that allows specifying a number of bytes in a human-readable format. See the examples for details. 4 | 5 | 6 | ## Examples 7 | 8 | ### Various ways to specify 5000 bytes 9 | 10 | ```yaml 11 | - type: some_operator 12 | bytes: 5000 13 | ``` 14 | 15 | ```yaml 16 | - type: some_operator 17 | bytes: 5kb 18 | ``` 19 | 20 | ```yaml 21 | - type: some_operator 22 | bytes: 4.88KiB 23 | ``` 24 | 25 | ```yaml 26 | - type: some_operator 27 | bytes: 5e3 28 | ``` 29 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/user_added_to_global_group/message.in: -------------------------------------------------------------------------------- 1 | A member was added to a security-enabled global group. 2 | 3 | Subject: 4 | Security ID: SYSTEM 5 | Account Name: WIN-322E2C550UP$ 6 | Account Domain: WORKGROUP 7 | Logon ID: 0x3E7 8 | 9 | Member: 10 | Security ID: WIN-322E2C550UP\Someone 11 | Account Name: - 12 | 13 | Group: 14 | Security ID: WIN-322E2C550UP\None 15 | Group Name: None 16 | Group Domain: WIN-322E2C550UP 17 | 18 | Additional Information: 19 | Privileges: - -------------------------------------------------------------------------------- /docs/examples/simple_plugins/config.yaml: -------------------------------------------------------------------------------- 1 | # This example configuration uses two plugins that are defined 2 | # in the ./plugins directory. See those files for details 3 | pipeline: 4 | # repeater is a plugin defined by ./plugins/repeater.yaml 5 | - type: repeater 6 | 7 | # decorator is a plugin defined by ./plugins/decorator.yaml 8 | # It adds the label "decorated" to each entry that passes through 9 | # it with the value specified here 10 | - type: decorator 11 | value: my_decorated_value 12 | 13 | - type: stdout 14 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/account_name_changed/details.out: -------------------------------------------------------------------------------- 1 | { 2 | "Additional Information": { 3 | "Privileges": "-" 4 | }, 5 | "Subject": { 6 | "Account Domain": "WORKGROUP", 7 | "Account Name": "WIN-322E2C550UP$", 8 | "Logon ID": "0x3E7", 9 | "Security ID": "SYSTEM" 10 | }, 11 | "Target Account": { 12 | "Account Domain": "Builtin", 13 | "New Account Name": "Power Users", 14 | "Old Account Name": "Power Users", 15 | "Security ID": "BUILTIN\\Power Users" 16 | } 17 | } -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/object_added/details.out: -------------------------------------------------------------------------------- 1 | { 2 | "Object": { 3 | "COM+ Catalog Collection": "Roles", 4 | "Object Details": [ 5 | "Description = Administrators group" 6 | ], 7 | "Object Name": [ 8 | "ApplId = {01234567-ABCD-ABCD-ABCD-0123456789AB}", 9 | "Name = Administrators" 10 | ] 11 | }, 12 | "Subject": { 13 | "Account Domain": "NT AUTHORITY", 14 | "Account Name": "SYSTEM", 15 | "Logon ID": "0x3E7", 16 | "Security ID": "SYSTEM" 17 | } 18 | } -------------------------------------------------------------------------------- /cmd/stanza/testdata/simple_plugins/config.yaml: -------------------------------------------------------------------------------- 1 | # This example configuration uses two plugins that are defined 2 | # in the ./plugins directory. See those files for details 3 | pipeline: 4 | # repeater is a plugin defined by ./plugins/repeater.yaml 5 | - type: repeater 6 | 7 | # decorator is a plugin defined by ./plugins/decorator.yaml 8 | # It adds the label "decorated" to each entry that passes through 9 | # it with the value specified here 10 | - type: decorator 11 | value: my_decorated_value 12 | 13 | - type: stdout 14 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import "fmt" 4 | 5 | var ( 6 | // GitCommit set externally of the git commit this was built on 7 | GitCommit string 8 | 9 | // GitTag set externally of the git tag this was built on 10 | GitTag string 11 | ) 12 | 13 | // GetVersion returns the version of the stanza library 14 | func GetVersion() string { 15 | if GitTag != "" { 16 | return GitTag 17 | } 18 | 19 | if GitCommit != "" { 20 | return fmt.Sprintf("git-commit: %s", GitCommit) 21 | } 22 | 23 | return "unknown" 24 | } 25 | -------------------------------------------------------------------------------- /cmd/stanza/version.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | 7 | "github.com/observiq/stanza/version" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | // NewVersionCommand returns the cli command for version 12 | func NewVersionCommand() *cobra.Command { 13 | return &cobra.Command{ 14 | Use: "version", 15 | Args: cobra.NoArgs, 16 | Short: "Print the stanza version", 17 | Run: func(_ *cobra.Command, _ []string) { 18 | fmt.Println(version.GetVersion(), runtime.GOOS, runtime.GOARCH) 19 | }, 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | ) 6 | 7 | // Logger is a wrapped logger used by the stanza agent. 8 | type Logger struct { 9 | *zap.SugaredLogger 10 | *Emitter 11 | } 12 | 13 | // New will create a new logger. 14 | func New(sugared *zap.SugaredLogger) *Logger { 15 | baseLogger := sugared.Desugar() 16 | emitter := newEmitter() 17 | core := newCore(baseLogger.Core(), emitter) 18 | wrappedLogger := zap.New(core).Sugar() 19 | 20 | return &Logger{ 21 | wrappedLogger, 22 | emitter, 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/special_logon/message.in: -------------------------------------------------------------------------------- 1 | Special privileges assigned to new logon. 2 | 3 | Subject: 4 | Security ID: SYSTEM 5 | Account Name: SYSTEM 6 | Account Domain: NT AUTHORITY 7 | Logon ID: 0x3E7 8 | 9 | Privileges: SeAssignPrimaryTokenPrivilege 10 | SeTcbPrivilege 11 | SeSecurityPrivilege 12 | SeTakeOwnershipPrivilege 13 | SeLoadDriverPrivilege 14 | SeBackupPrivilege 15 | SeRestorePrivilege 16 | SeDebugPrivilege 17 | SeAuditPrivilege 18 | SeSystemEnvironmentPrivilege 19 | SeImpersonatePrivilege -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/local_group_changed/details.out: -------------------------------------------------------------------------------- 1 | { 2 | "Additional Information": { 3 | "Privileges": "-" 4 | }, 5 | "Changed Attributes": { 6 | "SAM Account Name": "-", 7 | "SID History": "-" 8 | }, 9 | "Group": { 10 | "Group Domain": "Builtin", 11 | "Group Name": "Administrators", 12 | "Security ID": "BUILTIN\\Administrators" 13 | }, 14 | "Subject": { 15 | "Account Domain": "WORKGROUP", 16 | "Account Name": "WIN-322E2C550UP$", 17 | "Logon ID": "0x3E7", 18 | "Security ID": "SYSTEM" 19 | } 20 | } -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/user_added_to_global_group/details.out: -------------------------------------------------------------------------------- 1 | { 2 | "Additional Information": { 3 | "Privileges": "-" 4 | }, 5 | "Group": { 6 | "Group Domain": "WIN-322E2C550UP", 7 | "Group Name": "None", 8 | "Security ID": "WIN-322E2C550UP\\None" 9 | }, 10 | "Member": { 11 | "Account Name": "-", 12 | "Security ID": "WIN-322E2C550UP\\Someone" 13 | }, 14 | "Subject": { 15 | "Account Domain": "WORKGROUP", 16 | "Account Name": "WIN-322E2C550UP$", 17 | "Logon ID": "0x3E7", 18 | "Security ID": "SYSTEM" 19 | } 20 | } -------------------------------------------------------------------------------- /operator/builtin/input/goflow/testdata/netflowv5.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | CONFIG_FILE="/testdata/netflowv5.yaml" 4 | LOG_FILE="/testdata/stanza.log" 5 | STDOUT_FILE="/testdata/stdout.log" 6 | OUTPUT_FILE="/testdata/out.log" 7 | 8 | # clear the log if it exists, is is crucial that each test 9 | # starts with empty files 10 | > "${LOG_FILE}" 11 | > "${STDOUT_FILE}" 12 | > "${OUTPUT_FILE}" 13 | 14 | chmod 0666 $LOG_FILE 15 | chmod 0666 $STDOUT_FILE 16 | chmod 0666 $OUTPUT_FILE 17 | 18 | /stanza_home/stanza \ 19 | --config "${CONFIG_FILE}" \ 20 | --log_file "${LOG_FILE}" >"${STDOUT_FILE}" 2>&1 21 | -------------------------------------------------------------------------------- /docs/examples/tomcat/access.log: -------------------------------------------------------------------------------- 1 | 10.66.2.46 - - [13/Mar/2019:10:43:00 -0400] "GET / HTTP/1.1" 404 - 2 | 10.66.2.46 - - [13/Mar/2019:10:43:01 -0400] "GET /favicon.ico HTTP/1.1" 404 - 3 | 10.66.2.46 - - [13/Mar/2019:10:43:08 -0400] "GET /manager HTTP/1.1" 302 - 4 | 10.66.2.46 - - [13/Mar/2019:10:43:08 -0400] "GET /manager/ HTTP/1.1" 403 3420 5 | 10.66.2.46 - - [13/Mar/2019:11:00:26 -0400] "GET /manager/html HTTP/1.1" 401 2473 6 | 10.66.2.46 - tomcat [13/Mar/2019:11:00:53 -0400] "GET /manager/html HTTP/1.1" 200 11936 7 | 10.66.2.46 - - [13/Mar/2019:11:00:53 -0400] "GET /manager/images/asf-logo.svg HTTP/1.1" 200 19698 8 | -------------------------------------------------------------------------------- /cmd/stanza/testdata/tomcat/access.log: -------------------------------------------------------------------------------- 1 | 10.66.2.46 - - [13/Mar/2019:10:43:00 -0400] "GET / HTTP/1.1" 404 - 2 | 10.66.2.46 - - [13/Mar/2019:10:43:01 -0400] "GET /favicon.ico HTTP/1.1" 404 - 3 | 10.66.2.46 - - [13/Mar/2019:10:43:08 -0400] "GET /manager HTTP/1.1" 302 - 4 | 10.66.2.46 - - [13/Mar/2019:10:43:08 -0400] "GET /manager/ HTTP/1.1" 403 3420 5 | 10.66.2.46 - - [13/Mar/2019:11:00:26 -0400] "GET /manager/html HTTP/1.1" 401 2473 6 | 10.66.2.46 - tomcat [13/Mar/2019:11:00:53 -0400] "GET /manager/html HTTP/1.1" 200 11936 7 | 10.66.2.46 - - [13/Mar/2019:11:00:53 -0400] "GET /manager/images/asf-logo.svg HTTP/1.1" 200 19698 8 | -------------------------------------------------------------------------------- /docs/examples/simple_plugins/plugins/repeater.yaml: -------------------------------------------------------------------------------- 1 | # This plugin is registered as the type 'repeater'. 2 | # The type comes from the filename. 3 | # It will generate exactly 5 entries with the same 4 | # timestamp and the content "test record" 5 | pipeline: 6 | - type: generate_input 7 | static: true 8 | entry: 9 | timestamp: "2006-01-02T15:04:05Z" 10 | record: "test record" 11 | count: 5 12 | # The output is parameterized with go templates 13 | # so that it can use the output that is configured for the 14 | # plugin in the top-level pipeline 15 | output: {{ .output }} 16 | -------------------------------------------------------------------------------- /cmd/stanza/testdata/simple_plugins/plugins/repeater.yaml: -------------------------------------------------------------------------------- 1 | # This plugin is registered as the type 'repeater'. 2 | # The type comes from the filename. 3 | # It will generate exactly 5 entries with the same 4 | # timestamp and the content "test record" 5 | pipeline: 6 | - type: generate_input 7 | static: true 8 | entry: 9 | timestamp: "2006-01-02T15:04:05Z" 10 | record: "test record" 11 | count: 5 12 | # The output is parameterized with go templates 13 | # so that it can use the output that is configured for the 14 | # plugin in the top-level pipeline 15 | output: {{ .output }} 16 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/special_logon/details.out: -------------------------------------------------------------------------------- 1 | { 2 | "Privileges": [ 3 | "SeAssignPrimaryTokenPrivilege", 4 | "SeTcbPrivilege", 5 | "SeSecurityPrivilege", 6 | "SeTakeOwnershipPrivilege", 7 | "SeLoadDriverPrivilege", 8 | "SeBackupPrivilege", 9 | "SeRestorePrivilege", 10 | "SeDebugPrivilege", 11 | "SeAuditPrivilege", 12 | "SeSystemEnvironmentPrivilege", 13 | "SeImpersonatePrivilege" 14 | ], 15 | "Subject": { 16 | "Account Domain": "NT AUTHORITY", 17 | "Account Name": "SYSTEM", 18 | "Logon ID": "0x3E7", 19 | "Security ID": "SYSTEM" 20 | } 21 | } -------------------------------------------------------------------------------- /docs/operators/stdout.md: -------------------------------------------------------------------------------- 1 | ## `stdout` operator 2 | 3 | The `stdout` operator will write entries to stdout in JSON format. This is particularly useful for debugging a config file 4 | or running one-time batch processing jobs. 5 | 6 | ### Configuration Fields 7 | 8 | | Field | Default | Description | 9 | | --- | --- | --- | 10 | | `id` | required | A unique identifier for the operator | 11 | 12 | 13 | ### Example Configurations 14 | 15 | #### Simple configuration 16 | 17 | Configuration: 18 | ```yaml 19 | - id: my_stdout 20 | type: stdout 21 | ``` 22 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/audit_settings_changed/message.in: -------------------------------------------------------------------------------- 1 | Auditing settings on object were changed. 2 | 3 | Subject: 4 | Security ID: SYSTEM 5 | Account Name: WIN-PS00R22J635$ 6 | Account Domain: WORKGROUP 7 | Logon ID: 0x3E7 8 | 9 | Object: 10 | Object Server: Security 11 | Object Type: File 12 | Object Name: C:\Windows\Temp\winre\ExtractedFromWim 13 | Handle ID: 0x72c 14 | 15 | Process Information: 16 | Process ID: 0x2ec 17 | Process Name: C:\Windows\System32\oobe\Setup.exe 18 | 19 | Auditing Settings: 20 | Original Security Descriptor: 21 | New Security Descriptor: S:ARAI(AU;SAFA;DCLCRPCRSDWDWO;;;WD) -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 7 | 8 | ## Description of Changes 9 | 10 | ## **Please check that the PR fulfills these requirements** 11 | - [ ] Tests for the changes have been added (for bug fixes / features) 12 | - [ ] Docs have been added / updated (for bug fixes / features) 13 | - [ ] CI passes 14 | -------------------------------------------------------------------------------- /pipeline/node_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "testing" 5 | 6 | _ "github.com/observiq/stanza/operator/builtin/input/generate" 7 | _ "github.com/observiq/stanza/operator/builtin/transformer/noop" 8 | "github.com/observiq/stanza/testutil" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestNodeDOTID(t *testing.T) { 13 | operator := testutil.NewMockOperator("test") 14 | operator.On("Outputs").Return(nil) 15 | node := createOperatorNode(operator) 16 | require.Equal(t, operator.ID(), node.DOTID()) 17 | } 18 | 19 | func TestCreateNodeID(t *testing.T) { 20 | nodeID := createNodeID("test_id") 21 | require.Equal(t, int64(5795108767401590291), nodeID) 22 | } 23 | -------------------------------------------------------------------------------- /docs/types/on_error.md: -------------------------------------------------------------------------------- 1 | # `on_error` parameter 2 | The `on_error` parameter determines the error handling strategy an operator should use when it fails to process an entry. There are 2 supported values: `drop` and `send`. 3 | 4 | Regardless of the method selected, all processing errors will be logged by the operator. 5 | 6 | ### `drop` 7 | In this mode, if an operator fails to process an entry, it will drop the entry altogether. This will stop the entry from being sent further down the pipeline. 8 | 9 | ### `send` 10 | In this mode, if an operator fails to process an entry, it will still send the entry down the pipeline. This may result in downstream operators receiving entries in an undesired format. -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /revive/config.toml: -------------------------------------------------------------------------------- 1 | ignoreGeneratedHeader = false 2 | severity = "warning" 3 | confidence = 0.8 4 | errorCode = 1 5 | warningCode = 1 6 | 7 | [rule.blank-imports] 8 | [rule.context-as-argument] 9 | [rule.context-keys-type] 10 | [rule.dot-imports] 11 | [rule.error-return] 12 | [rule.error-strings] 13 | [rule.error-naming] 14 | [rule.if-return] 15 | [rule.increment-decrement] 16 | [rule.var-naming] 17 | [rule.var-declaration] 18 | [rule.range] 19 | [rule.receiver-naming] 20 | [rule.time-naming] 21 | [rule.unexported-return] 22 | [rule.indent-error-flow] 23 | [rule.errorf] 24 | [rule.empty-block] 25 | [rule.superfluous-else] 26 | [rule.unreachable-code] 27 | [rule.exported] 28 | arguments=["disableStutteringCheck"] 29 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/time_change/message.in: -------------------------------------------------------------------------------- 1 | The system time was changed. 2 | 3 | Subject: 4 | Security ID: SYSTEM 5 | Account Name: WIN-322E2C550UP$ 6 | Account Domain: WORKGROUP 7 | Logon ID: 0x3E7 8 | 9 | Process Information: 10 | Process ID: 0x474 11 | Name: C:\Program Files\VMware\VMware Tools\vmtoolsd.exe 12 | 13 | Previous Time: ‎2020‎-‎01‎-‎01T23:55:29.757489600Z 14 | New Time: ‎2020‎-‎01‎-‎02T03:51:44.145000000Z 15 | 16 | This event is generated when the system time is changed. It is normal for the Windows Time Service, which runs with System privilege, to change the system time on a regular basis. Other system time changes may be indicative of attempts to tamper with the computer. -------------------------------------------------------------------------------- /operator/builtin/output/googlecloud/client_test.go: -------------------------------------------------------------------------------- 1 | package googlecloud 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | logging "cloud.google.com/go/logging/apiv2" 8 | "github.com/stretchr/testify/require" 9 | "golang.org/x/oauth2/google" 10 | "google.golang.org/api/option" 11 | ) 12 | 13 | func TestNewClient(t *testing.T) { 14 | json := `{"type": "service_account"}` 15 | credentials, err := google.CredentialsFromJSON(context.Background(), []byte(json), loggingScope) 16 | require.NoError(t, err) 17 | 18 | options := option.WithCredentials(credentials) 19 | client, err := newClient(context.Background(), options) 20 | require.NoError(t, err) 21 | 22 | _, ok := client.(*logging.Client) 23 | require.True(t, ok) 24 | } 25 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | # The Stanza Maintainers 2 | 3 | This file lists the maintainers of the Stanza project. The responsibilities of maintainers are listed in the [GOVERNANCE.md](GOVERNANCE.md) file. 4 | 5 | ## Project Maintainers 6 | | Name | GitHub ID | Affiliation | 7 | | ---- | --------- | ----------- | 8 | | [Mike Kelly](mailto:mike.kelly@observiq.com) | [mkelly](https://github.com/mkelly) | observIQ | 9 | | [Dan Jaglowski](mailto:dan.jaglowski@observiq.com) | [djaglowski](https://github.com/djaglowski) | observIQ | 10 | | [Joseph Howell](mailto:joseph.howell@observiq.com) | [jhowellbm](https://github.com/jhowellbm) | observIQ | 11 | | [Joe Sirianni](mailto:joe.sirianni@observiq.com) | [jhowellbm](https://github.com/jsirianni) | observIQ | 12 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/audit_settings_changed/details.out: -------------------------------------------------------------------------------- 1 | { 2 | "Auditing Settings": { 3 | "New Security Descriptor": "S:ARAI(AU;SAFA;DCLCRPCRSDWDWO;;;WD)", 4 | "Original Security Descriptor": "-" 5 | }, 6 | "Object": { 7 | "Handle ID": "0x72c", 8 | "Object Name": "C:\\Windows\\Temp\\winre\\ExtractedFromWim", 9 | "Object Server": "Security", 10 | "Object Type": "File" 11 | }, 12 | "Process Information": { 13 | "Process ID": "0x2ec", 14 | "Process Name": "C:\\Windows\\System32\\oobe\\Setup.exe" 15 | }, 16 | "Subject": { 17 | "Account Domain": "WORKGROUP", 18 | "Account Name": "WIN-PS00R22J635$", 19 | "Logon ID": "0x3E7", 20 | "Security ID": "SYSTEM" 21 | } 22 | } -------------------------------------------------------------------------------- /operator/builtin/input/azure/eventhub/parse.go: -------------------------------------------------------------------------------- 1 | package eventhub 2 | 3 | import ( 4 | "context" 5 | 6 | azhub "github.com/Azure/azure-event-hubs-go/v3" 7 | "github.com/observiq/stanza/operator/builtin/input/azure" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | // handleEvent handles an event recieved by an Event Hub consumer. 12 | func (e *EventHubInput) handleEvent(ctx context.Context, event *azhub.Event) error { 13 | e.WG.Add(1) 14 | defer e.WG.Done() 15 | 16 | entry, err := e.NewEntry(nil) 17 | if err != nil { 18 | e.Errorw("", zap.Error(err)) 19 | return err 20 | } 21 | 22 | if err := azure.ParseEvent(*event, entry); err != nil { 23 | e.Errorw("", zap.Error(err)) 24 | return err 25 | } 26 | 27 | e.Write(ctx, entry) 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/api_test.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package windows 4 | 5 | import ( 6 | "syscall" 7 | ) 8 | 9 | // MockProc is a mocked syscall procedure. 10 | type MockProc struct { 11 | call func(a ...uintptr) (uintptr, uintptr, error) 12 | } 13 | 14 | // Call will return the result of the embedded call function. 15 | func (m MockProc) Call(a ...uintptr) (uintptr, uintptr, error) { 16 | return m.call(a...) 17 | } 18 | 19 | // SimpleMockProc returns a mock proc that will always return the supplied arguments when called. 20 | func SimpleMockProc(r1 uintptr, r2 uintptr, err syscall.Errno) SyscallProc { 21 | return MockProc{ 22 | call: func(a ...uintptr) (uintptr, uintptr, error) { 23 | return r1, r2, err 24 | }, 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Environment:** 27 | - OS 28 | - Stanza Version or Commit Hash 29 | 30 | **Additional context** 31 | Add any other context about the problem here. 32 | -------------------------------------------------------------------------------- /docs/types/duration.md: -------------------------------------------------------------------------------- 1 | # Durations 2 | 3 | Durations are lengths of time that are specified as part of a pluign configuration using a number or string. 4 | 5 | If a number is specified, it will be interpreted as a number of seconds. 6 | 7 | If a string is specified, it will be interpreted according to Golang's [`time.ParseDuration`](https://golang.org/src/time/format.go?s=40541:40587#L1369) documentation. 8 | 9 | ## Examples 10 | 11 | ### Various ways to specify a duration of 1 minute 12 | 13 | ```yaml 14 | - type: some_operator 15 | duration: 1m 16 | ``` 17 | 18 | ```yaml 19 | - type: some_operator 20 | duration: 60s 21 | ``` 22 | 23 | ```yaml 24 | - type: some_operator 25 | duration: 60 26 | ``` 27 | 28 | ```yaml 29 | - type: some_operator 30 | duration: 60.0 31 | ``` 32 | -------------------------------------------------------------------------------- /errors/details.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import "go.uber.org/zap/zapcore" 4 | 5 | // ErrorDetails is a map of details for an agent error. 6 | type ErrorDetails map[string]string 7 | 8 | // MarshalLogObject will define the representation of details when logging. 9 | func (d ErrorDetails) MarshalLogObject(encoder zapcore.ObjectEncoder) error { 10 | for key, value := range d { 11 | encoder.AddString(key, value) 12 | } 13 | return nil 14 | } 15 | 16 | // createDetails will create details for an error from key/value pairs. 17 | func createDetails(keyValues []string) ErrorDetails { 18 | details := make(ErrorDetails) 19 | if len(keyValues) > 0 { 20 | for i := 0; i+1 < len(keyValues); i += 2 { 21 | details[keyValues[i]] = keyValues[i+1] 22 | } 23 | } 24 | return details 25 | } 26 | -------------------------------------------------------------------------------- /docs/examples/k8s/events/service_account.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: ServiceAccount 3 | apiVersion: v1 4 | metadata: 5 | name: stanza-agent 6 | namespace: default 7 | --- 8 | kind: ClusterRole 9 | apiVersion: rbac.authorization.k8s.io/v1 10 | metadata: 11 | name: stanza-agent 12 | rules: 13 | - apiGroups: ["", "apps", "batch"] 14 | resources: 15 | - pods 16 | - namespaces 17 | - replicasets 18 | - jobs 19 | - events 20 | verbs: ["get", "list", "watch"] 21 | --- 22 | kind: ClusterRoleBinding 23 | apiVersion: rbac.authorization.k8s.io/v1 24 | metadata: 25 | name: stanza-agent 26 | roleRef: 27 | apiGroup: rbac.authorization.k8s.io 28 | kind: ClusterRole 29 | name: stanza-agent 30 | subjects: 31 | - kind: ServiceAccount 32 | name: stanza-agent 33 | namespace: default 34 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/time_change/details.out: -------------------------------------------------------------------------------- 1 | { 2 | "Additional Context": [ 3 | "This event is generated when the system time is changed. It is normal for the Windows Time Service, which runs with System privilege, to change the system time on a regular basis. Other system time changes may be indicative of attempts to tamper with the computer." 4 | ], 5 | "New Time": "‎2020‎-‎01‎-‎02T03:51:44.145000000Z", 6 | "Previous Time": "‎2020‎-‎01‎-‎01T23:55:29.757489600Z", 7 | "Process Information": { 8 | "Name": "C:\\Program Files\\VMware\\VMware Tools\\vmtoolsd.exe", 9 | "Process ID": "0x474" 10 | }, 11 | "Subject": { 12 | "Account Domain": "WORKGROUP", 13 | "Account Name": "WIN-322E2C550UP$", 14 | "Logon ID": "0x3E7", 15 | "Security ID": "SYSTEM" 16 | } 17 | } -------------------------------------------------------------------------------- /docs/faq.md: -------------------------------------------------------------------------------- 1 | # Frequently asked questions 2 | 3 | ## How do I configure the agent? 4 | The agent is configured using a YAML file. Use the `--config` flag to tell stanza where to find this file. 5 | 6 | This file defines a collection of operators that make up a `pipeline`. Each operator possesses a `type` field, and can optionally be given an `id` as well. 7 | 8 | ```yaml 9 | pipeline: 10 | - type: udp_input 11 | listen_address: :5141 12 | 13 | - type: syslog_parser 14 | parse_from: message 15 | protocol: rfc5424 16 | 17 | - type: elastic_output 18 | ``` 19 | 20 | ## What is a plugin? 21 | 22 | A plugin is a templated set of operators. Read more about plugins [here](/docs/plugins.md). 23 | 24 | 25 | ## Does Stanza support HTTP proxies? 26 | 27 | Yes. Read about the details [here](/docs/proxy.md). 28 | -------------------------------------------------------------------------------- /operator/builtin/input/stdin/stdin_test.go: -------------------------------------------------------------------------------- 1 | package stdin 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/observiq/stanza/operator" 8 | "github.com/observiq/stanza/testutil" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestStdin(t *testing.T) { 13 | cfg := NewStdinInputConfig("") 14 | cfg.OutputIDs = []string{"fake"} 15 | 16 | op, err := cfg.Build(testutil.NewBuildContext(t)) 17 | require.NoError(t, err) 18 | 19 | fake := testutil.NewFakeOutput(t) 20 | op[0].SetOutputs([]operator.Operator{fake}) 21 | 22 | r, w, err := os.Pipe() 23 | require.NoError(t, err) 24 | 25 | stdin := op[0].(*StdinInput) 26 | stdin.stdin = r 27 | 28 | require.NoError(t, stdin.Start()) 29 | defer stdin.Stop() 30 | 31 | w.WriteString("test") 32 | w.Close() 33 | fake.ExpectRecord(t, "test") 34 | } 35 | -------------------------------------------------------------------------------- /license.yaml: -------------------------------------------------------------------------------- 1 | # minimum confidence percentage used during license classification 2 | threshold: .90 3 | 4 | # all permitted licenses - if no list is specified, all licenses are assumed to be allowed 5 | allow: 6 | - "MIT" 7 | - "Apache-2.0" 8 | - "BSD-3-Clause" 9 | - "BSD-2-Clause" 10 | - "Zlib" 11 | 12 | exceptions: 13 | licenseNotPermitted: 14 | # MPL is approved as long as the source is not modified 15 | - path: "github.com/hashicorp/go-uuid" 16 | licenses: ["MPL-2.0"] 17 | - path: "github.com/hashicorp/errwrap" 18 | licenses: ["MPL-2.0"] 19 | - path: "github.com/hashicorp/go-multierror" 20 | licenses: ["MPL-2.0"] 21 | 22 | # ISC 23 | - path: "github.com/davecgh/go-spew" 24 | licenses: ["ISC"] 25 | - path: "github.com/libp2p/go-reuseport" 26 | licenses: ["ISC"] -------------------------------------------------------------------------------- /entry/nil_field.go: -------------------------------------------------------------------------------- 1 | package entry 2 | 3 | // NilField is a struct that implements Field, but 4 | // does nothing for all its operations. It is useful 5 | // as a default no-op field to avoid nil checks. 6 | type NilField struct{} 7 | 8 | // Get will return always return nil 9 | func (l NilField) Get(_ *Entry) (interface{}, bool) { 10 | return nil, true 11 | } 12 | 13 | // Set will do nothing and return no error 14 | func (l NilField) Set(_ *Entry, _ interface{}) error { 15 | return nil 16 | } 17 | 18 | // Delete will do nothing and return no error 19 | func (l NilField) Delete(_ *Entry) (interface{}, bool) { 20 | return nil, true 21 | } 22 | 23 | func (l NilField) String() string { 24 | return "$nil" 25 | } 26 | 27 | // NewNilField will create a new nil field 28 | func NewNilField() Field { 29 | return Field{NilField{}} 30 | } 31 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/domain_policy_changed/message.in: -------------------------------------------------------------------------------- 1 | Domain Policy was changed. 2 | 3 | Change Type: Logoff Policy modified 4 | 5 | Subject: 6 | Security ID: WIN-322E2C550UP\Someone 7 | Account Name: Someone 8 | Account Domain: WIN-322E2C550UP 9 | Logon ID: 0x2F62B 10 | 11 | Domain: 12 | Domain Name: WIN-322E2C550UP 13 | Domain ID: WIN-322E2C550UP\ 14 | 15 | Changed Attributes: 16 | Min. Password Age: - 17 | Max. Password Age: - 18 | Force Logoff: - 19 | Lockout Threshold: - 20 | Lockout Observation Window: - 21 | Lockout Duration: - 22 | Password Properties: - 23 | Min. Password Length: - 24 | Password History Length: - 25 | Machine Account Quota: - 26 | Mixed Domain Mode: - 27 | Domain Behavior Version: - 28 | OEM Information: - 29 | 30 | Additional Information: 31 | Privileges: - -------------------------------------------------------------------------------- /operator/builtin/input/windows/event_test.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package windows 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestEventCloseWhenAlreadyClosed(t *testing.T) { 12 | event := NewEvent(0) 13 | err := event.Close() 14 | require.NoError(t, err) 15 | } 16 | 17 | func TestEventCloseSyscallFailure(t *testing.T) { 18 | event := NewEvent(5) 19 | closeProc = SimpleMockProc(0, 0, ErrorNotSupported) 20 | err := event.Close() 21 | require.Error(t, err) 22 | require.Contains(t, err.Error(), "failed to close event handle") 23 | } 24 | 25 | func TestEventCloseSuccess(t *testing.T) { 26 | event := NewEvent(5) 27 | closeProc = SimpleMockProc(1, 0, ErrorSuccess) 28 | err := event.Close() 29 | require.NoError(t, err) 30 | require.Equal(t, uintptr(0), event.handle) 31 | } 32 | -------------------------------------------------------------------------------- /.github/workflows/gosec.yml: -------------------------------------------------------------------------------- 1 | name: Gosec 2 | on: 3 | workflow_dispatch: 4 | schedule: 5 | # ┌───────────── minute (0 - 59) 6 | # │ ┌───────────── hour (0 - 23) 7 | # │ │ ┌───────────── day of the month (1 - 31) 8 | # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) 9 | # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) 10 | # │ │ │ │ │ 11 | # │ │ │ │ │ 12 | # │ │ │ │ │ 13 | # * * * * * 14 | - cron: '30 1 * * *' 15 | pull_request: 16 | 17 | jobs: 18 | tests: 19 | runs-on: ubuntu-latest 20 | env: 21 | GO111MODULE: on 22 | steps: 23 | - name: Checkout Source 24 | uses: actions/checkout@v4 25 | 26 | - name: Run Gosec Security Scanner 27 | uses: securego/gosec@v2.8.1 28 | with: 29 | args: './...' 30 | -------------------------------------------------------------------------------- /operator/builtin/output/googlecloud/client.go: -------------------------------------------------------------------------------- 1 | package googlecloud 2 | 3 | import ( 4 | "context" 5 | 6 | api "cloud.google.com/go/logging/apiv2" 7 | "github.com/googleapis/gax-go/v2" 8 | "google.golang.org/api/option" 9 | "google.golang.org/genproto/googleapis/logging/v2" 10 | ) 11 | 12 | // Client is an interface for writing entries to google cloud logging 13 | type Client interface { 14 | WriteLogEntries(ctx context.Context, req *logging.WriteLogEntriesRequest, opts ...gax.CallOption) (*logging.WriteLogEntriesResponse, error) 15 | Close() error 16 | } 17 | 18 | // ClientBuilder is a function that builds a client 19 | type ClientBuilder = func(ctx context.Context, opts ...option.ClientOption) (Client, error) 20 | 21 | // newClient creates a new client 22 | func newClient(ctx context.Context, opts ...option.ClientOption) (Client, error) { 23 | return api.NewClient(ctx, opts...) 24 | } 25 | -------------------------------------------------------------------------------- /operator/builtin/input/generate/generate_test.go: -------------------------------------------------------------------------------- 1 | package generate 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/observiq/stanza/entry" 7 | "github.com/observiq/stanza/operator" 8 | "github.com/observiq/stanza/testutil" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestInputGenerate(t *testing.T) { 13 | cfg := NewGenerateInputConfig("test_operator_id") 14 | cfg.OutputIDs = []string{"fake"} 15 | cfg.Count = 5 16 | cfg.Entry = entry.Entry{ 17 | Record: "test message", 18 | } 19 | 20 | ops, err := cfg.Build(testutil.NewBuildContext(t)) 21 | require.NoError(t, err) 22 | op := ops[0] 23 | 24 | fake := testutil.NewFakeOutput(t) 25 | err = op.SetOutputs([]operator.Operator{fake}) 26 | require.NoError(t, err) 27 | 28 | require.NoError(t, op.Start()) 29 | defer op.Stop() 30 | 31 | for i := 0; i < 5; i++ { 32 | fake.ExpectRecord(t, "test message") 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /entry/nil_field_test.go: -------------------------------------------------------------------------------- 1 | package entry 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestNilFieldGet(t *testing.T) { 10 | entry := &Entry{} 11 | nilField := NewNilField() 12 | value, ok := nilField.Get(entry) 13 | require.True(t, ok) 14 | require.Nil(t, value) 15 | } 16 | 17 | func TestNilFieldSet(t *testing.T) { 18 | entry := &Entry{} 19 | nilField := NewNilField() 20 | err := nilField.Set(entry, "value") 21 | require.NoError(t, err) 22 | require.Equal(t, *entry, Entry{}) 23 | } 24 | 25 | func TestNilFieldDelete(t *testing.T) { 26 | entry := &Entry{} 27 | nilField := NewNilField() 28 | value, ok := nilField.Delete(entry) 29 | require.True(t, ok) 30 | require.Nil(t, value) 31 | require.Equal(t, *entry, Entry{}) 32 | } 33 | 34 | func TestNilFieldString(t *testing.T) { 35 | nilField := NewNilField() 36 | require.Equal(t, "$nil", nilField.String()) 37 | } 38 | -------------------------------------------------------------------------------- /.kitchen.yml: -------------------------------------------------------------------------------- 1 | provisioner: 2 | name: shell 3 | script: 'build/package/test/provision.sh' 4 | root_path: '/home/vagrant/' 5 | 6 | verifier: 7 | # cinc 8 | name: inspec 9 | 10 | platforms: 11 | # RHEL based 12 | - name: centos-7 13 | - name: rockylinux-8.5 14 | - name: almalinux-8.5 15 | - name: oracle-8.5 16 | - name: fedora-30 17 | - name: fedora-34 18 | # Debian based 19 | - name: debian-9 20 | - name: debian-10 21 | - name: debian-11 22 | - name: ubuntu-16.04 23 | - name: ubuntu-18.04 24 | - name: ubuntu-20.04 25 | - name: ubuntu-20.10 26 | 27 | driver: 28 | name: vagrant 29 | provider: virtualbox 30 | synced_folders: 31 | - ["./artifacts", "/home/vagrant/dist"] 32 | customize: 33 | memory: 1024 34 | vagrantfiles: 35 | - build/package/test/Vagrantfile 36 | 37 | suites: 38 | - name: default 39 | verifier: 40 | inspec_tests: 41 | - build/package/test/ 42 | -------------------------------------------------------------------------------- /docs/types/expression.md: -------------------------------------------------------------------------------- 1 | # Expressions 2 | 3 | Expressions give the config flexibility by allowing dynamic business logic rules to be included in static configs. 4 | Most notably, expressions can be used to route messages and add new fields based on the contents of the log entry 5 | being processed. 6 | 7 | For reference documentation of the expression language, see [here](https://github.com/antonmedv/expr/blob/master/docs/Language-Definition.md). 8 | 9 | Available to the expressions are a few special variables: 10 | - `$record` contains the entry's record 11 | - `$labels` contains the entry's labels 12 | - `$resource` contains the entry's resource 13 | - `$timestamp` contains the entry's timestamp 14 | - `env()` is a function that allows you to read environment variables 15 | 16 | ## Examples 17 | 18 | ### Add a label from an environment variable 19 | 20 | ```yaml 21 | - type: metadata 22 | labels: 23 | stack: 'EXPR(env("STACK"))' 24 | ``` 25 | -------------------------------------------------------------------------------- /operator/builtin/input/aws/cloudwatch/cloudwatch_persist.go: -------------------------------------------------------------------------------- 1 | package cloudwatch 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | 7 | "github.com/observiq/stanza/operator/helper" 8 | ) 9 | 10 | // Persister ensures data is persisted across shutdowns 11 | type Persister struct { 12 | DB helper.Persister 13 | } 14 | 15 | // Read is a helper function to get persisted data 16 | func (p *Persister) Read(key string) (int64, error) { 17 | var startTime int64 18 | buffer := bytes.NewBuffer(p.DB.Get(key)) 19 | err := binary.Read(buffer, binary.BigEndian, &startTime) 20 | if err != nil && err.Error() != "EOF" { 21 | return 0, err 22 | } 23 | return startTime, nil 24 | } 25 | 26 | // Write is a helper function to set persisted data 27 | func (p *Persister) Write(key string, value int64) { 28 | var buf = make([]byte, 8) 29 | // #nosec G115 - Value will not be negative 30 | binary.BigEndian.PutUint64(buf, uint64(value)) 31 | p.DB.Set(key, buf) 32 | } 33 | -------------------------------------------------------------------------------- /operator/builtin/input/azure/event_hub_persist_test.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestPersistenceKey(t *testing.T) { 10 | type TestKey struct { 11 | namespace string 12 | name string 13 | consumerGroup string 14 | partitionID string 15 | } 16 | 17 | cases := []struct { 18 | name string 19 | input TestKey 20 | expected string 21 | }{ 22 | { 23 | "basic", 24 | TestKey{ 25 | namespace: "stanza", 26 | name: "devel", 27 | consumerGroup: "$Default", 28 | partitionID: "0", 29 | }, 30 | "stanza-devel-$Default-0", 31 | }, 32 | } 33 | 34 | for _, tc := range cases { 35 | t.Run(tc.name, func(t *testing.T) { 36 | p := Persister{} 37 | out := p.persistenceKey(tc.input.namespace, tc.input.name, tc.input.consumerGroup, tc.input.partitionID) 38 | require.Equal(t, tc.expected, out) 39 | }) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /agent/agent.go: -------------------------------------------------------------------------------- 1 | package agent 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/observiq/stanza/database" 7 | "github.com/observiq/stanza/pipeline" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | // LogAgent is an entity that handles log monitoring. 12 | type LogAgent struct { 13 | database database.Database 14 | pipeline pipeline.Pipeline 15 | 16 | startOnce sync.Once 17 | stopOnce sync.Once 18 | 19 | *zap.SugaredLogger 20 | } 21 | 22 | // Start will start the log monitoring process 23 | func (a *LogAgent) Start() (err error) { 24 | a.startOnce.Do(func() { 25 | err = a.pipeline.Start() 26 | if err != nil { 27 | return 28 | } 29 | }) 30 | return 31 | } 32 | 33 | // Stop will stop the log monitoring process 34 | func (a *LogAgent) Stop() (err error) { 35 | a.stopOnce.Do(func() { 36 | err = a.pipeline.Stop() 37 | if err != nil { 38 | return 39 | } 40 | 41 | err = a.database.Close() 42 | if err != nil { 43 | return 44 | } 45 | }) 46 | return 47 | } 48 | -------------------------------------------------------------------------------- /.github/workflows/license.yml: -------------------------------------------------------------------------------- 1 | name: license 2 | on: 3 | pull_request: 4 | 5 | jobs: 6 | build: 7 | name: Scan Licenses 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Check out source code 11 | uses: actions/checkout@v1 12 | 13 | - name: Set up Go 14 | uses: actions/setup-go@v5 15 | with: 16 | go-version-file: "go.mod" 17 | 18 | - name: Install License Scanner 19 | run: go install github.com/uw-labs/lichen@v0.1.7 20 | 21 | # License scanner requires a built binary 22 | - name: Build Stanza 23 | run: make build-all 24 | 25 | - name: Scan Licenses Linux 26 | run: lichen --config=./license.yaml "./artifacts/stanza_linux_amd64" 27 | 28 | - name: Scan Licenses Windows 29 | run: lichen --config=./license.yaml "./artifacts/stanza_windows_amd64" 30 | 31 | - name: Scan Licenses MacOS 32 | run: lichen --config=./license.yaml "./artifacts/stanza_darwin_amd64" 33 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/domain_policy_changed/details.out: -------------------------------------------------------------------------------- 1 | { 2 | "Additional Information": { 3 | "Privileges": "-" 4 | }, 5 | "Change Type": "Logoff Policy modified", 6 | "Changed Attributes": { 7 | "Domain Behavior Version": "-", 8 | "Force Logoff": "-", 9 | "Lockout Duration": "-", 10 | "Lockout Observation Window": "-", 11 | "Lockout Threshold": "-", 12 | "Machine Account Quota": "-", 13 | "Max. Password Age": "-", 14 | "Min. Password Age": "-", 15 | "Min. Password Length": "-", 16 | "Mixed Domain Mode": "-", 17 | "OEM Information": "-", 18 | "Password History Length": "-", 19 | "Password Properties": "-" 20 | }, 21 | "Domain": { 22 | "Domain ID": "WIN-322E2C550UP\\", 23 | "Domain Name": "WIN-322E2C550UP" 24 | }, 25 | "Subject": { 26 | "Account Domain": "WIN-322E2C550UP", 27 | "Account Name": "Someone", 28 | "Logon ID": "0x2F62B", 29 | "Security ID": "WIN-322E2C550UP\\Someone" 30 | } 31 | } -------------------------------------------------------------------------------- /logger/emitter.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/observiq/stanza/entry" 7 | ) 8 | 9 | // Receiver is a channel that receives internal stanza logs. 10 | type Receiver chan entry.Entry 11 | 12 | // Emitter emits internal logs to registered receivers. 13 | type Emitter struct { 14 | receivers []Receiver 15 | mux sync.RWMutex 16 | } 17 | 18 | // AddReceiver will add a receiver to the emitter. 19 | func (e *Emitter) AddReceiver(receiver Receiver) { 20 | e.mux.Lock() 21 | e.receivers = append(e.receivers, receiver) 22 | e.mux.Unlock() 23 | } 24 | 25 | // Emit emits an entry to all receivers. 26 | func (e *Emitter) emit(entry entry.Entry) { 27 | e.mux.RLock() 28 | defer e.mux.RUnlock() 29 | for _, receiver := range e.receivers { 30 | select { 31 | case receiver <- entry: 32 | default: 33 | } 34 | } 35 | } 36 | 37 | // newEmitter creates a new emitter. 38 | func newEmitter() *Emitter { 39 | return &Emitter{ 40 | receivers: make([]Receiver, 0, 2), 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /operator/builtin/input/file/positional_scanner.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | ) 7 | 8 | // PositionalScanner is a scanner that maintains position 9 | type PositionalScanner struct { 10 | pos int64 11 | *bufio.Scanner 12 | } 13 | 14 | // NewPositionalScanner creates a new positional scanner 15 | func NewPositionalScanner(r io.Reader, maxLogSize int, startOffset int64, splitFunc bufio.SplitFunc) *PositionalScanner { 16 | ps := &PositionalScanner{ 17 | pos: startOffset, 18 | Scanner: bufio.NewScanner(r), 19 | } 20 | 21 | buf := make([]byte, 0, 16384) 22 | ps.Scanner.Buffer(buf, maxLogSize) 23 | 24 | scanFunc := func(data []byte, atEOF bool) (advance int, token []byte, err error) { 25 | advance, token, err = splitFunc(data, atEOF) 26 | ps.pos += int64(advance) 27 | return 28 | } 29 | ps.Scanner.Split(scanFunc) 30 | return ps 31 | } 32 | 33 | // Pos returns the current position of the scanner 34 | func (ps *PositionalScanner) Pos() int64 { 35 | return ps.pos 36 | } 37 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/user_account_changed/message.in: -------------------------------------------------------------------------------- 1 | A user account was changed. 2 | 3 | Subject: 4 | Security ID: SYSTEM 5 | Account Name: WIN-322E2C550UP$ 6 | Account Domain: WORKGROUP 7 | Logon ID: 0x3E7 8 | 9 | Target Account: 10 | Security ID: WIN-322E2C550UP\Someone 11 | Account Name: Someone 12 | Account Domain: WIN-322E2C550UP 13 | 14 | Changed Attributes: 15 | SAM Account Name: Someone 16 | Display Name: 17 | User Principal Name: - 18 | Home Directory: 19 | Home Drive: 20 | Script Path: 21 | Profile Path: 22 | User Workstations: 23 | Password Last Set: 12/4/2019 11:15:46 AM 24 | Account Expires: 25 | Primary Group ID: 513 26 | AllowedToDelegateTo: - 27 | Old UAC Value: 0x14 28 | New UAC Value: 0x10 29 | User Account Control: 30 | 'Password Not Required' - Disabled 31 | User Parameters: - 32 | SID History: - 33 | Logon Hours: All 34 | 35 | Additional Information: 36 | Privileges: - -------------------------------------------------------------------------------- /cmd/stanza/default_paths.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "runtime" 8 | "strings" 9 | ) 10 | 11 | var agentName = "stanza" 12 | 13 | func defaultPluginDir() string { 14 | if stat, err := os.Stat("./plugins"); err == nil { 15 | if stat.IsDir() { 16 | return "./plugins" 17 | } 18 | } 19 | 20 | return filepath.Join(agentHome(), "plugins") 21 | } 22 | 23 | func defaultConfig() string { 24 | if _, err := os.Stat("./config.yaml"); err == nil { 25 | return "./config.yaml" 26 | } 27 | 28 | return filepath.Join(agentHome(), "config.yaml") 29 | } 30 | 31 | func agentHome() string { 32 | if home := os.Getenv(strings.ToUpper(agentName) + "_HOME"); home != "" { 33 | return home 34 | } 35 | 36 | switch runtime.GOOS { 37 | case "windows": 38 | return filepath.Join(`C:\`, agentName) 39 | case "darwin": 40 | home, _ := os.UserHomeDir() 41 | return filepath.Join(home, agentName) 42 | case "linux": 43 | return filepath.Join("/opt", agentName) 44 | default: 45 | panic(fmt.Sprintf("Unsupported GOOS %s", runtime.GOOS)) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/user_account_created/message.in: -------------------------------------------------------------------------------- 1 | A user account was created. 2 | 3 | Subject: 4 | Security ID: SYSTEM 5 | Account Name: WIN-322E2C550UP$ 6 | Account Domain: WORKGROUP 7 | Logon ID: 0x3E7 8 | 9 | New Account: 10 | Security ID: WIN-322E2C550UP\Someone 11 | Account Name: Someone 12 | Account Domain: WIN-322E2C550UP 13 | 14 | Attributes: 15 | SAM Account Name: Someone 16 | Display Name: 17 | User Principal Name: - 18 | Home Directory: 19 | Home Drive: 20 | Script Path: 21 | Profile Path: 22 | User Workstations: 23 | Password Last Set: 24 | Account Expires: 25 | Primary Group ID: 513 26 | Allowed To Delegate To: - 27 | Old UAC Value: 0x0 28 | New UAC Value: 0x15 29 | User Account Control: 30 | Account Disabled 31 | 'Password Not Required' - Enabled 32 | 'Normal Account' - Enabled 33 | User Parameters: 34 | SID History: - 35 | Logon Hours: All 36 | 37 | Additional Information: 38 | Privileges - -------------------------------------------------------------------------------- /operator/builtin/output/drop/drop_test.go: -------------------------------------------------------------------------------- 1 | package drop 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/observiq/stanza/entry" 8 | "github.com/observiq/stanza/testutil" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestBuildValid(t *testing.T) { 13 | cfg := NewDropOutputConfig("test") 14 | ctx := testutil.NewBuildContext(t) 15 | ops, err := cfg.Build(ctx) 16 | require.NoError(t, err) 17 | op := ops[0] 18 | require.IsType(t, &DropOutput{}, op) 19 | } 20 | 21 | func TestBuildIvalid(t *testing.T) { 22 | cfg := NewDropOutputConfig("test") 23 | ctx := testutil.NewBuildContext(t) 24 | ctx.Logger = nil 25 | _, err := cfg.Build(ctx) 26 | require.Error(t, err) 27 | require.Contains(t, err.Error(), "build context is missing a logger") 28 | } 29 | 30 | func TestProcess(t *testing.T) { 31 | cfg := NewDropOutputConfig("test") 32 | ctx := testutil.NewBuildContext(t) 33 | ops, err := cfg.Build(ctx) 34 | require.NoError(t, err) 35 | op := ops[0] 36 | 37 | entry := entry.New() 38 | result := op.Process(context.Background(), entry) 39 | require.Nil(t, result) 40 | } 41 | -------------------------------------------------------------------------------- /docs/examples/scenarios/custom_parsing.md: -------------------------------------------------------------------------------- 1 | # Custom Parsing 2 | 3 | Once you have Stanza installed and running from the [quickstart guide](./README.md#quick-start), you can use custom parsing to send logs via Stanza. 4 | 5 | Stanza supports several parsing operators that can be used together to parse log messages. These parsing operators include: 6 | - [JSON](/docs/operators/json_parser.md) 7 | - [Regex](/docs/operators/regex_parser.md) 8 | - [CSV](/docs/operators/csv_parser.md) 9 | - [Syslog](/docs/operators/syslog_parser.md) 10 | - [Key Values](/docs/operators/key_value_parser.md) 11 | - [Severity](/docs/operators/severity_parser.md) 12 | - [Time](/docs/operators/time_parser.md) 13 | - [URI](/docs/operators/uri_parser.md) 14 | 15 | To see more details and examples of each parser, follow the links to their individual documentation. 16 | 17 | ## Next Steps 18 | 19 | - Learn more about [plugins](/docs/plugins.md). 20 | - Read up on how to write a stanza [pipeline](/docs/pipeline.md). 21 | - Check out stanza's list of [operators](/docs/operators/README.md). 22 | - Check out the [FAQ](/docs/faq.md). 23 | -------------------------------------------------------------------------------- /docs/examples/simple_plugins/plugins/decorator.yaml: -------------------------------------------------------------------------------- 1 | version: 0.0.0 2 | title: Decorator 3 | description: A decorator plugin 4 | parameters: 5 | - name: value 6 | label: Value 7 | description: A value to decorate the entries 8 | type: string 9 | required: true 10 | 11 | # This plugin is registered as the type 'decorator'. 12 | # The type comes from the filename. 13 | # It take any entries sent to it, and add the label 14 | # 'decorated' to those entries with a value specified 15 | # by the argument 'value' in the top-level pipeline 16 | pipeline: 17 | # The input parameter is replaced with the ID of the 18 | # operator in the top-level config so that the plugin 19 | # graph can be connected properly. 20 | - id: {{ .input }} 21 | type: metadata 22 | labels: 23 | # The value parameter comes from the configuration 24 | # of the plugin in the top-level config 25 | decorated: {{ .value }} 26 | # The output is parameterized with go templates 27 | # so that it can use the output that is configured for the 28 | # plugin in the top-level pipeline 29 | output: {{ .output }} 30 | -------------------------------------------------------------------------------- /cmd/stanza/testdata/simple_plugins/plugins/decorator.yaml: -------------------------------------------------------------------------------- 1 | version: 0.0.0 2 | title: Decorator 3 | description: A decorator plugin 4 | parameters: 5 | - name: value 6 | label: Value 7 | description: A value to decorate the entries 8 | type: string 9 | required: true 10 | 11 | # This plugin is registered as the type 'decorator'. 12 | # The type comes from the filename. 13 | # It take any entries sent to it, and add the label 14 | # 'decorated' to those entries with a value specified 15 | # by the argument 'value' in the top-level pipeline 16 | pipeline: 17 | # The input parameter is replaced with the ID of the 18 | # operator in the top-level config so that the plugin 19 | # graph can be connected properly. 20 | - id: {{ .input }} 21 | type: metadata 22 | labels: 23 | # The value parameter comes from the configuration 24 | # of the plugin in the top-level config 25 | decorated: {{ .value }} 26 | # The output is parameterized with go templates 27 | # so that it can use the output that is configured for the 28 | # plugin in the top-level pipeline 29 | output: {{ .output }} 30 | -------------------------------------------------------------------------------- /docs/operators/forward_output.md: -------------------------------------------------------------------------------- 1 | ## `forward_output` operator 2 | 3 | The `forward_output` operator sends logs to another Stanza instance running `forward_input`. 4 | 5 | ### Configuration Fields 6 | 7 | | Field | Default | Description | 8 | | --- | --- | --- | 9 | | `id` | `forward_output` | A unique identifier for the operator | 10 | | `address` | required | The address that the downstream Stanza instance is listening on | 11 | | `buffer` | | A [buffer](/docs/types/buffer.md) block indicating how to buffer entries before flushing | 12 | | `flusher` | | A [flusher](/docs/types/flusher.md) block configuring flushing behavior | 13 | 14 | 15 | ### Example Configurations 16 | 17 | #### Simple configuration 18 | 19 | Configuration: 20 | ```yaml 21 | - type: forward_output 22 | address: "http://downstream_server:25535" 23 | ``` 24 | -------------------------------------------------------------------------------- /operator/operator.go: -------------------------------------------------------------------------------- 1 | //go:generate mockery --name=^(Operator)$ --output=../testutil --outpkg=testutil --case=snake 2 | 3 | package operator 4 | 5 | import ( 6 | "context" 7 | 8 | "github.com/observiq/stanza/entry" 9 | "go.uber.org/zap" 10 | ) 11 | 12 | // Operator is a log monitoring component. 13 | type Operator interface { 14 | // ID returns the id of the operator. 15 | ID() string 16 | // Type returns the type of the operator. 17 | Type() string 18 | 19 | // Start will start the operator. 20 | Start() error 21 | // Stop will stop the operator. 22 | Stop() error 23 | 24 | // CanOutput indicates if the operator will output entries to other operators. 25 | CanOutput() bool 26 | // Outputs returns the list of connected outputs. 27 | Outputs() []Operator 28 | // SetOutputs will set the connected outputs. 29 | SetOutputs([]Operator) error 30 | 31 | // CanProcess indicates if the operator will process entries from other operators. 32 | CanProcess() bool 33 | // Process will process an entry from an operator. 34 | Process(context.Context, *entry.Entry) error 35 | // Logger returns the operator's logger 36 | Logger() *zap.SugaredLogger 37 | } 38 | -------------------------------------------------------------------------------- /operator/helper/persister_test.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/observiq/stanza/database" 8 | "github.com/observiq/stanza/testutil" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestPersisterCache(t *testing.T) { 13 | stubDatabase := database.NewStubDatabase() 14 | persister := NewScopedDBPersister(stubDatabase, "test") 15 | persister.Set("key", []byte("value")) 16 | value := persister.Get("key") 17 | require.Equal(t, []byte("value"), value) 18 | } 19 | 20 | func TestPersisterLoad(t *testing.T) { 21 | tempDir := testutil.NewTempDir(t) 22 | db, err := database.OpenDatabase(filepath.Join(tempDir, "test.db")) 23 | defer func() { 24 | if err := db.Close(); err != nil { 25 | t.Error(err.Error()) 26 | } 27 | }() 28 | persister := NewScopedDBPersister(db, "test") 29 | persister.Set("key", []byte("value")) 30 | 31 | err = persister.Sync() 32 | require.NoError(t, err) 33 | 34 | newPersister := NewScopedDBPersister(db, "test") 35 | err = newPersister.Load() 36 | require.NoError(t, err) 37 | 38 | value := newPersister.Get("key") 39 | require.Equal(t, []byte("value"), value) 40 | } 41 | -------------------------------------------------------------------------------- /operator/build_context_test.go: -------------------------------------------------------------------------------- 1 | package operator 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestBuildContext(t *testing.T) { 10 | t.Run("PrependNamespace", func(t *testing.T) { 11 | bc := BuildContext{ 12 | Namespace: "$.test", 13 | } 14 | 15 | t.Run("Standard", func(t *testing.T) { 16 | id := bc.PrependNamespace("testid") 17 | require.Equal(t, "$.test.testid", id) 18 | }) 19 | 20 | t.Run("AlreadyPrefixed", func(t *testing.T) { 21 | id := bc.PrependNamespace("$.myns.testid") 22 | require.Equal(t, "$.myns.testid", id) 23 | }) 24 | }) 25 | 26 | t.Run("WithSubNamespace", func(t *testing.T) { 27 | bc := BuildContext{ 28 | Namespace: "$.ns", 29 | } 30 | bc2 := bc.WithSubNamespace("subns") 31 | require.Equal(t, "$.ns.subns", bc2.Namespace) 32 | require.Equal(t, "$.ns", bc.Namespace) 33 | }) 34 | 35 | t.Run("WithDefaultOutputIDs", func(t *testing.T) { 36 | bc := BuildContext{ 37 | DefaultOutputIDs: []string{"orig"}, 38 | } 39 | bc2 := bc.WithDefaultOutputIDs([]string{"id1", "id2"}) 40 | require.Equal(t, []string{"id1", "id2"}, bc2.DefaultOutputIDs) 41 | require.Equal(t, []string{"orig"}, bc.DefaultOutputIDs) 42 | 43 | }) 44 | } 45 | -------------------------------------------------------------------------------- /docs/operators/stdin.md: -------------------------------------------------------------------------------- 1 | ## `stdin` operator 2 | 3 | The `stdin` generates entries from lines written to stdin. 4 | 5 | ### Configuration Fields 6 | 7 | | Field | Default | Description | 8 | | --- | --- | --- | 9 | | `id` | `generate_input` | A unique identifier for the operator | 10 | | `output` | Next in pipeline | The connected operator(s) that will receive all outbound entries | 11 | | `write_to` | $ | A [field](/docs/types/field.md) that will be set to the path of the file the entry was read from | 12 | 13 | ### Example Configurations 14 | 15 | #### Mock a file input 16 | 17 | Configuration: 18 | ```yaml 19 | - type: stdin 20 | ``` 21 | 22 | Command: 23 | ```bash 24 | echo "test" | stanza -c ./config.yaml 25 | ``` 26 | 27 | Output records: 28 | ```json 29 | { 30 | "timestamp": "2020-11-10T11:09:56.505467-05:00", 31 | "severity": 0, 32 | "record": "test" 33 | } 34 | ``` 35 | -------------------------------------------------------------------------------- /operator/builtin/input/stanza/stanza_test.go: -------------------------------------------------------------------------------- 1 | package stanza 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/observiq/stanza/operator" 8 | "github.com/observiq/stanza/testutil" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestStanzaOperator(t *testing.T) { 13 | cfg := NewInputConfig("test") 14 | cfg.OutputIDs = []string{"fake"} 15 | 16 | bc := testutil.NewBuildContext(t) 17 | ops, err := cfg.Build(bc) 18 | require.NoError(t, err) 19 | op := ops[0] 20 | 21 | fake := testutil.NewFakeOutput(t) 22 | op.SetOutputs([]operator.Operator{fake}) 23 | 24 | require.NoError(t, op.Start()) 25 | defer op.Stop() 26 | 27 | bc.Logger.Errorw("test failure", "key", "value") 28 | 29 | expectedRecord := map[string]interface{}{ 30 | "message": "test failure", 31 | "key": "value", 32 | } 33 | 34 | select { 35 | case e := <-fake.Received: 36 | require.Equal(t, expectedRecord, e.Record) 37 | 38 | case <-time.After(time.Second): 39 | require.FailNow(t, "timed out") 40 | } 41 | } 42 | 43 | func TestStanzaOperatorBUildFailure(t *testing.T) { 44 | cfg := NewInputConfig("") 45 | cfg.OperatorType = "" 46 | bc := testutil.NewBuildContext(t) 47 | _, err := cfg.Build(bc) 48 | require.Error(t, err) 49 | } 50 | -------------------------------------------------------------------------------- /operator/builtin/output/stdout/stdout_test.go: -------------------------------------------------------------------------------- 1 | package stdout 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "testing" 8 | "time" 9 | 10 | "github.com/observiq/stanza/entry" 11 | "github.com/observiq/stanza/operator/helper" 12 | "github.com/observiq/stanza/testutil" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | func TestStdoutOperator(t *testing.T) { 17 | cfg := StdoutConfig{ 18 | OutputConfig: helper.OutputConfig{ 19 | BasicConfig: helper.BasicConfig{ 20 | OperatorID: "test_operator_id", 21 | OperatorType: "stdout", 22 | }, 23 | }, 24 | } 25 | 26 | ops, err := cfg.Build(testutil.NewBuildContext(t)) 27 | require.NoError(t, err) 28 | op := ops[0] 29 | 30 | var buf bytes.Buffer 31 | op.(*StdoutOperator).encoder = json.NewEncoder(&buf) 32 | 33 | ts := time.Unix(1591042864, 0) 34 | e := &entry.Entry{ 35 | Timestamp: ts, 36 | Record: "test record", 37 | } 38 | err = op.Process(context.Background(), e) 39 | require.NoError(t, err) 40 | 41 | marshalledTimestamp, err := json.Marshal(ts) 42 | require.NoError(t, err) 43 | 44 | expected := `{"timestamp":` + string(marshalledTimestamp) + `,"severity":0,"record":"test record"}` + "\n" 45 | require.Equal(t, expected, buf.String()) 46 | } 47 | -------------------------------------------------------------------------------- /operator/builtin/input/http/auth.go: -------------------------------------------------------------------------------- 1 | package httpevents 2 | 3 | import "net/http" 4 | 5 | type authMiddleware interface { 6 | auth(next http.Handler) http.Handler 7 | name() string 8 | } 9 | 10 | type authToken struct { 11 | tokenHeader string 12 | tokens []string 13 | } 14 | 15 | func (a authToken) auth(next http.Handler) http.Handler { 16 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 17 | token := r.Header.Get(a.tokenHeader) 18 | 19 | for _, validToken := range a.tokens { 20 | if validToken == token { 21 | next.ServeHTTP(w, r) 22 | return 23 | } 24 | } 25 | w.WriteHeader(http.StatusForbidden) 26 | }) 27 | } 28 | 29 | func (a authToken) name() string { 30 | return "token-auth" 31 | } 32 | 33 | type authBasic struct { 34 | username string 35 | password string 36 | } 37 | 38 | func (a authBasic) auth(next http.Handler) http.Handler { 39 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 40 | u, p, ok := r.BasicAuth() 41 | if ok { 42 | if u == a.username && p == a.password { 43 | next.ServeHTTP(w, r) 44 | return 45 | } 46 | } 47 | w.WriteHeader(http.StatusForbidden) 48 | }) 49 | } 50 | 51 | func (a authBasic) name() string { 52 | return "basic-auth" 53 | } 54 | -------------------------------------------------------------------------------- /docs/examples/k8s/gke/README.md: -------------------------------------------------------------------------------- 1 | # Google Kubernetes Engine Logs and Events w/ Google Cloud Logging 2 | 3 | Stanza can be deployed to Google Kubernetes Engine for log and event collection. Container logs 4 | are gathered from each Kubernetes Node's filesystem. Events are collected from the Kubernetes 5 | API Server. 6 | 7 | ## Architecture 8 | 9 | 1. Service account with permission to the Kubernetes API server 10 | 2. Config map: Contains the Stanza configurations 11 | 3. Persistent volume: Allows the Stanza events agent database to persist between restarts and pod evictions 12 | 4. Statefulset: A single replica statefulset for reading Kubernetes events 13 | 5. Daemonset: For reading logs from each Kubernetes node 14 | 15 | ## Prerequisites 16 | 17 | 1. Google Cloud account with Cloud Logging API enabled 18 | 2. Google GKE cluster with [write permission to cloud logging](https://developers.google.com/identity/protocols/oauth2/scopes#logging) 19 | 3. Edit `agent.yaml`'s configmap (at the top) to include: 20 | - Your cluster name: an arbitrary value that will be added to each log entry as a label 21 | 22 | ## Deployment Steps 23 | 24 | Deploy Stanza 25 | ```bash 26 | kubectl apply -f agent.yaml 27 | ``` 28 | 29 | ## Validate 30 | 31 | Log into Google Cloud Logging 32 | 33 | ![Events](./assets/entries.png) 34 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/publisher.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package windows 4 | 5 | import ( 6 | "fmt" 7 | "syscall" 8 | ) 9 | 10 | // Publisher is a windows event metadata publisher. 11 | type Publisher struct { 12 | handle uintptr 13 | } 14 | 15 | // Open will open the publisher handle using the supplied provider. 16 | func (p *Publisher) Open(provider string) error { 17 | if p.handle != 0 { 18 | return fmt.Errorf("publisher handle is already open") 19 | } 20 | 21 | utf16, err := syscall.UTF16PtrFromString(provider) 22 | if err != nil { 23 | return fmt.Errorf("failed to convert provider to utf16: %s", err) 24 | } 25 | 26 | handle, err := evtOpenPublisherMetadata(0, utf16, nil, 0, 0) 27 | if err != nil { 28 | return fmt.Errorf("failed to open publisher handle: %s", err) 29 | } 30 | 31 | p.handle = handle 32 | return nil 33 | } 34 | 35 | // Close will close the publisher handle. 36 | func (p *Publisher) Close() error { 37 | if p.handle == 0 { 38 | return nil 39 | } 40 | 41 | if err := evtClose(p.handle); err != nil { 42 | return fmt.Errorf("failed to close publisher: %s", err) 43 | } 44 | 45 | p.handle = 0 46 | return nil 47 | } 48 | 49 | // NewPublisher will create a new publisher with an empty handle. 50 | func NewPublisher() Publisher { 51 | return Publisher{ 52 | handle: 0, 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/user_account_changed/details.out: -------------------------------------------------------------------------------- 1 | { 2 | "Additional Information": { 3 | "Privileges": "-" 4 | }, 5 | "Changed Attributes": { 6 | "Account Expires": "\u003cnever\u003e", 7 | "AllowedToDelegateTo": "-", 8 | "Display Name": "\u003cvalue not set\u003e", 9 | "Home Directory": "\u003cvalue not set\u003e", 10 | "Home Drive": "\u003cvalue not set\u003e", 11 | "Logon Hours": "All", 12 | "New UAC Value": "0x10", 13 | "Old UAC Value": "0x14", 14 | "Password Last Set": "12/4/2019 11:15:46 AM", 15 | "Primary Group ID": "513", 16 | "Profile Path": "\u003cvalue not set\u003e", 17 | "SAM Account Name": "Someone", 18 | "SID History": "-", 19 | "Script Path": "\u003cvalue not set\u003e", 20 | "User Account Control": [ 21 | "'Password Not Required' - Disabled" 22 | ], 23 | "User Parameters": "-", 24 | "User Principal Name": "-", 25 | "User Workstations": "\u003cvalue not set\u003e" 26 | }, 27 | "Subject": { 28 | "Account Domain": "WORKGROUP", 29 | "Account Name": "WIN-322E2C550UP$", 30 | "Logon ID": "0x3E7", 31 | "Security ID": "SYSTEM" 32 | }, 33 | "Target Account": { 34 | "Account Domain": "WIN-322E2C550UP", 35 | "Account Name": "Someone", 36 | "Security ID": "WIN-322E2C550UP\\Someone" 37 | } 38 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.17 as stage 2 | 3 | ARG plugins_url="https://github.com/observiq/stanza-plugins/releases/latest/download/stanza-plugins.zip" 4 | # arm cross builds do not have these symlinks in palce 5 | RUN \ 6 | ln -s /usr/bin/dpkg-split /usr/sbin/dpkg-split && \ 7 | ln -s /usr/bin/dpkg-deb /usr/sbin/dpkg-deb && \ 8 | ln -s /bin/tar /usr/sbin/tar && \ 9 | ln -s /bin/rm /usr/sbin/rm && \ 10 | echo "resolvconf resolvconf/linkify-resolvconf boolean false" | debconf-set-selections 11 | # unzip is required because tar does not work on arm 12 | RUN apt-get update && apt-get install unzip -y 13 | WORKDIR /stanza/artifacts 14 | RUN curl -fL "${plugins_url}" -o stanza-plugins.zip 15 | RUN unzip stanza-plugins.zip 16 | WORKDIR /stanza 17 | COPY . . 18 | RUN make build 19 | RUN mv "artifacts/stanza_$(go env GOOS)_$(go env GOARCH)" artifacts/stanza 20 | 21 | 22 | FROM gcr.io/observiq-container-images/stanza-base:v1.1.0 23 | 24 | RUN mkdir -p /stanza_home 25 | ENV STANZA_HOME=/stanza_home 26 | RUN echo "pipeline:\n" >> /stanza_home/config.yaml 27 | COPY --from=stage /stanza/artifacts/stanza /stanza_home/stanza 28 | COPY --from=stage /stanza/artifacts/plugins /stanza_home/plugins 29 | ENTRYPOINT /stanza_home/stanza \ 30 | --config /stanza_home/config.yaml \ 31 | --database /stanza_home/stanza.db \ 32 | --plugin_dir /stanza_home/plugins 33 | -------------------------------------------------------------------------------- /docs/types/entry.md: -------------------------------------------------------------------------------- 1 | # Entry 2 | 3 | Entry is the base representation of log data as it moves through a pipeline. All operators either create, modify, or consume entries. 4 | 5 | ## Structure 6 | | Field | Description | 7 | | --- | --- | 8 | | `timestamp` | The timestamp associated with the log (RFC 3339). | 9 | | `severity` | The [severity](/docs/types/field.md) of the log. | 10 | | `severity_text` | The original text that was interpreted as a [severity](/docs/types/field.md). | 11 | | `resource` | A map of key/value pairs that describe the resource from which the log originated. | 12 | | `labels` | A map of key/value pairs that provide additional context to the log. This value is often used by a consumer to filter logs. | 13 | | `record` | The contents of the log. This value is often modified and restructured in the pipeline. | 14 | -------------------------------------------------------------------------------- /logger/parser.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "github.com/observiq/stanza/entry" 5 | "go.uber.org/zap/zapcore" 6 | ) 7 | 8 | // parseEntry will create a stanza entry from a zapcore entry. 9 | func parseEntry(zapEntry zapcore.Entry, fields []zapcore.Field) entry.Entry { 10 | return entry.Entry{ 11 | Timestamp: zapEntry.Time, 12 | Record: parseRecord(zapEntry, fields), 13 | Severity: parseSeverity(zapEntry), 14 | } 15 | } 16 | 17 | // parseRecord will parse a record from a zapcore entry. 18 | func parseRecord(zapEntry zapcore.Entry, fields []zapcore.Field) map[string]interface{} { 19 | encoder := zapcore.NewMapObjectEncoder() 20 | encoder.AddString("message", zapEntry.Message) 21 | 22 | for _, field := range fields { 23 | field.AddTo(encoder) 24 | } 25 | 26 | return encoder.Fields 27 | } 28 | 29 | // parseSeverity will parse a stanza severity from a zapcore entry. 30 | func parseSeverity(zapEntry zapcore.Entry) entry.Severity { 31 | switch zapEntry.Level { 32 | case zapcore.DebugLevel: 33 | return entry.Debug 34 | case zapcore.InfoLevel: 35 | return entry.Info 36 | case zapcore.WarnLevel: 37 | return entry.Warning 38 | case zapcore.ErrorLevel: 39 | return entry.Error 40 | case zapcore.PanicLevel: 41 | return entry.Critical 42 | case zapcore.FatalLevel: 43 | return entry.Catastrophe 44 | default: 45 | return entry.Default 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /docs/operators/file_output.md: -------------------------------------------------------------------------------- 1 | ## `file_output` operator 2 | 3 | The `file_output` operator will write log entries to a file. By default, they will be written as JSON-formatted lines, but if a `Format` is provided, that format will be used as a template to render each log line 4 | 5 | ### Configuration Fields 6 | 7 | | Field | Default | Description | 8 | | --- | --- | --- | 9 | | `id` | `file_output` | A unique identifier for the operator | 10 | | `path` | required | A path to write the entries to | 11 | | `format` | | A [go template](https://golang.org/pkg/text/template/) that will be used to render each entry into a log line | 12 | 13 | 14 | ### Example Configurations 15 | 16 | #### Simple configuration 17 | 18 | Configuration: 19 | ```yaml 20 | - type: file_output 21 | path: /tmp/output.json 22 | ``` 23 | 24 | #### Custom format 25 | 26 | Configuration: 27 | ```yaml 28 | - type: file_output 29 | path: /tmp/output.log 30 | format: "Time: {{.Timestamp}} Record: {{.Record}}\n" 31 | ``` 32 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/testdata/security/user_account_created/details.out: -------------------------------------------------------------------------------- 1 | { 2 | "Additional Information": {}, 3 | "Attributes": { 4 | "Account Expires": "\u003cnever\u003e", 5 | "Allowed To Delegate To": "-", 6 | "Display Name": "\u003cvalue not set\u003e", 7 | "Home Directory": "\u003cvalue not set\u003e", 8 | "Home Drive": "\u003cvalue not set\u003e", 9 | "Logon Hours": "All", 10 | "New UAC Value": "0x15", 11 | "Old UAC Value": "0x0", 12 | "Password Last Set": "\u003cnever\u003e", 13 | "Primary Group ID": "513", 14 | "Profile Path": "\u003cvalue not set\u003e", 15 | "SAM Account Name": "Someone", 16 | "SID History": "-", 17 | "Script Path": "\u003cvalue not set\u003e", 18 | "User Account Control": [ 19 | "Account Disabled", 20 | "'Password Not Required' - Enabled", 21 | "'Normal Account' - Enabled" 22 | ], 23 | "User Parameters": "\u003cvalue not set\u003e", 24 | "User Principal Name": "-", 25 | "User Workstations": "\u003cvalue not set\u003e" 26 | }, 27 | "New Account": { 28 | "Account Domain": "WIN-322E2C550UP", 29 | "Account Name": "Someone", 30 | "Security ID": "WIN-322E2C550UP\\Someone" 31 | }, 32 | "Subject": { 33 | "Account Domain": "WORKGROUP", 34 | "Account Name": "WIN-322E2C550UP$", 35 | "Logon ID": "0x3E7", 36 | "Security ID": "SYSTEM" 37 | } 38 | } -------------------------------------------------------------------------------- /docs/types/flusher.md: -------------------------------------------------------------------------------- 1 | # Flushers 2 | 3 | Flushers handle reading entries from buffers in chunks, flushing them to their final destination, and retrying on failure. 4 | 5 | In most cases, the default options will work well, but they may be need tuning for optimal performance or for reducing load 6 | on the destination API. 7 | 8 | For example, if you hit an API limit on the number of requests per second, consider decreasing `max_concurrent` and 9 | increasing `max_chunk_entries`. This will make fewer, larger requests which should increase efficiency at the cost of 10 | some latency. 11 | 12 | Or, if you have low load and don't care about the higher latency, consider increasing `max_wait` so that entries are sent 13 | less often in larger requests. 14 | 15 | ## Flusher configuration 16 | 17 | Flushers are configured with the `flusher` block on output plugins. 18 | 19 | | Field | Default | Description | 20 | | --- | --- | --- | 21 | | `max_concurrent` | `16` | The maximum number of goroutines flushing entries concurrently | 22 | -------------------------------------------------------------------------------- /docs/operators/drop_output.md: -------------------------------------------------------------------------------- 1 | ## `drop_output` operator 2 | 3 | The `drop_output` operator does nothing. Useful for discarding entries while troubleshooting Stanza during development. 4 | 5 | ### Configuration Fields 6 | 7 | Operator `drop_output` does not have configuration 8 | 9 | ### Example Configurations 10 | 11 | Configuration 12 | ```yaml 13 | pipeline: 14 | - type: stdin 15 | - type: drop_output 16 | ``` 17 | 18 | Send a message: 19 | ```bash 20 | echo "hello world" | ./stanza -c ./config.yaml 21 | ``` 22 | 23 | There will be no output: 24 | ```json 25 | {"level":"info","timestamp":"2021-08-20T20:09:55.057-0400","message":"Starting stanza agent"} 26 | {"level":"info","timestamp":"2021-08-20T20:09:55.057-0400","message":"Stanza agent started"} 27 | {"level":"info","timestamp":"2021-08-20T20:09:55.057-0400","message":"Stdin has been closed","operator_id":"$.stdin","operator_type":"stdin"} 28 | ``` 29 | 30 | Compare with `stdout` output operator: 31 | ```json 32 | {"level":"info","timestamp":"2021-08-20T20:09:28.314-0400","message":"Starting stanza agent"} 33 | {"level":"info","timestamp":"2021-08-20T20:09:28.314-0400","message":"Stanza agent started"} 34 | {"timestamp":"2021-08-20T20:09:28.314776719-04:00","severity":0,"record":"hello world"} 35 | {"level":"info","timestamp":"2021-08-20T20:09:28.314-0400","message":"Stdin has been closed","operator_id":"$.stdin","operator_type":"stdin"} 36 | ``` 37 | 38 | -------------------------------------------------------------------------------- /operator/builtin/transformer/noop/noop_test.go: -------------------------------------------------------------------------------- 1 | package noop 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/observiq/stanza/entry" 8 | "github.com/observiq/stanza/operator" 9 | "github.com/observiq/stanza/testutil" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestBuildValid(t *testing.T) { 14 | cfg := NewNoopOperatorConfig("test") 15 | ops, err := cfg.Build(testutil.NewBuildContext(t)) 16 | require.NoError(t, err) 17 | op := ops[0] 18 | require.IsType(t, &NoopOperator{}, op) 19 | } 20 | 21 | func TestBuildIvalid(t *testing.T) { 22 | cfg := NewNoopOperatorConfig("test") 23 | ctx := testutil.NewBuildContext(t) 24 | ctx.Logger = nil 25 | _, err := cfg.Build(ctx) 26 | require.Error(t, err) 27 | require.Contains(t, err.Error(), "build context is missing a logger") 28 | } 29 | 30 | func TestProcess(t *testing.T) { 31 | cfg := NewNoopOperatorConfig("test") 32 | cfg.OutputIDs = []string{"fake"} 33 | ops, err := cfg.Build(testutil.NewBuildContext(t)) 34 | require.NoError(t, err) 35 | op := ops[0] 36 | 37 | fake := testutil.NewFakeOutput(t) 38 | op.SetOutputs([]operator.Operator{fake}) 39 | 40 | entry := entry.New() 41 | entry.AddLabel("label", "value") 42 | entry.AddResourceKey("resource", "value") 43 | 44 | expected := entry.Copy() 45 | err = op.Process(context.Background(), entry) 46 | require.NoError(t, err) 47 | 48 | fake.ExpectEntry(t, expected) 49 | } 50 | -------------------------------------------------------------------------------- /docs/proxy.md: -------------------------------------------------------------------------------- 1 | # Connecting through a proxy 2 | 3 | Stanza supports sending logs through a HTTP proxy. To enable this, set the environment variables HTTP_PROXY and HTTPS_PROXY to the address of your proxy server. 4 | 5 | For example: 6 | 7 | ```bash 8 | export HTTP_PROXY=http://user:password@myproxy:3128 9 | export HTTPS_PROXY=http://user:password@myproxy:3128 10 | stanza -c ./config.yaml 11 | ``` 12 | 13 | To set this for the Stanza service on Linux, the service file can be modified with `systemctl edit stanza`, and add the following lines in the `[Service]` section. 14 | 15 | ```service 16 | [Service] 17 | Environment=HTTP_PROXY=http://user:password@myproxy:3128 18 | Environment=HTTPS_PROXY=http://user:password@myproxy:3128 19 | ``` 20 | 21 | ## Using a self-signed certificate 22 | 23 | If your proxy uses a self-signed certificate, it will need to be installed in the OS's trusted certificate store. For an example of how to do that on RedHat Linux, see [here](https://www.redhat.com/sysadmin/ca-certificates-cli) under the heading "Updating ca-certificates to validate sites with an internal CA certificate". 24 | 25 | ## Known issues 26 | 27 | `go-grpc`, which is used for the Google Cloud Output operator, does not currently support sending a proxy `CONNECT` call over HTTPS, so proxy URLs must start with `http://`. Once the `CONNECT` request is made, a TLS tunnel will still be established, encrypting all the logs that are sent. 28 | -------------------------------------------------------------------------------- /operator/builtin/input/aws/cloudwatch/cloudwatch_persist_test.go: -------------------------------------------------------------------------------- 1 | package cloudwatch 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/observiq/stanza/database" 8 | "github.com/observiq/stanza/operator/helper" 9 | "github.com/observiq/stanza/testutil" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestPersisterCache(t *testing.T) { 14 | stubDatabase := database.NewStubDatabase() 15 | persister := Persister{ 16 | DB: helper.NewScopedDBPersister(stubDatabase, "test"), 17 | } 18 | persister.Write("key", int64(1620666055012)) 19 | value, readErr := persister.Read("key") 20 | require.NoError(t, readErr) 21 | require.Equal(t, int64(1620666055012), value) 22 | } 23 | 24 | func TestPersisterLoad(t *testing.T) { 25 | tempDir := testutil.NewTempDir(t) 26 | db, openDbErr := database.OpenDatabase(filepath.Join(tempDir, "test.db")) 27 | require.NoError(t, openDbErr) 28 | defer func() { 29 | if err := db.Close(); err != nil { 30 | t.Error(err.Error()) 31 | } 32 | }() 33 | persister := Persister{ 34 | DB: helper.NewScopedDBPersister(db, "test"), 35 | } 36 | persister.Write("key", 1620666055012) 37 | 38 | syncErr := persister.DB.Sync() 39 | require.NoError(t, syncErr) 40 | 41 | loadErr := persister.DB.Load() 42 | require.NoError(t, loadErr) 43 | 44 | value, readErr := persister.Read("key") 45 | require.NoError(t, readErr) 46 | require.Equal(t, int64(1620666055012), value) 47 | } 48 | -------------------------------------------------------------------------------- /docs/operators/rate_limit.md: -------------------------------------------------------------------------------- 1 | ## `rate_limit` operator 2 | 3 | The `rate_limit` operator limits the rate of entries that can pass through it. This is useful if you want to limit 4 | throughput of the agent, or in conjunction with operators like `generate_input`, which will otherwise 5 | send as fast as possible. 6 | 7 | ### Configuration Fields 8 | 9 | | Field | Default | Description | 10 | | --- | --- | --- | 11 | | `id` | `rate_limit` | A unique identifier for the operator | 12 | | `output` | Next in pipeline | The connected operator(s) that will receive all outbound entries | 13 | | `rate` | | The number of logs to allow per second | 14 | | `interval` | | A [duration](/docs/types/duration.md) that indicates the time between sent entries | 15 | | `burst` | 0 | The max number of entries to "save up" for spikes of load | 16 | 17 | Exactly one of `rate` or `interval` must be specified. 18 | 19 | ### Example Configurations 20 | 21 | 22 | #### Limit throughput to 10 entries per second 23 | 24 | Configuration: 25 | ```yaml 26 | - type: rate_limit 27 | rate: 10 28 | ``` 29 | -------------------------------------------------------------------------------- /operator/helper/labeler.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "github.com/observiq/stanza/entry" 5 | ) 6 | 7 | // NewLabelerConfig creates a new labeler config with default values 8 | func NewLabelerConfig() LabelerConfig { 9 | return LabelerConfig{ 10 | Labels: make(map[string]ExprStringConfig), 11 | } 12 | } 13 | 14 | // LabelerConfig is the configuration of a labeler 15 | type LabelerConfig struct { 16 | Labels map[string]ExprStringConfig `json:"labels" yaml:"labels"` 17 | } 18 | 19 | // Build will build a labeler from the supplied configuration 20 | func (c LabelerConfig) Build() (Labeler, error) { 21 | labeler := Labeler{ 22 | labels: make(map[string]*ExprString), 23 | } 24 | 25 | for k, v := range c.Labels { 26 | exprString, err := v.Build() 27 | if err != nil { 28 | return labeler, err 29 | } 30 | 31 | labeler.labels[k] = exprString 32 | } 33 | 34 | return labeler, nil 35 | } 36 | 37 | // Labeler is a helper that adds labels to an entry 38 | type Labeler struct { 39 | labels map[string]*ExprString 40 | } 41 | 42 | // Label will add labels to an entry 43 | func (l *Labeler) Label(e *entry.Entry) error { 44 | if len(l.labels) == 0 { 45 | return nil 46 | } 47 | 48 | env := GetExprEnv(e) 49 | defer PutExprEnv(env) 50 | 51 | for k, v := range l.labels { 52 | rendered, err := v.Render(env) 53 | if err != nil { 54 | return err 55 | } 56 | e.AddLabel(k, rendered) 57 | } 58 | 59 | return nil 60 | } 61 | -------------------------------------------------------------------------------- /entry/label_field.go: -------------------------------------------------------------------------------- 1 | package entry 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // LabelField is the path to an entry label 9 | type LabelField struct { 10 | key string 11 | } 12 | 13 | // Get will return the label value and a boolean indicating if it exists 14 | func (l LabelField) Get(entry *Entry) (interface{}, bool) { 15 | if entry.Labels == nil { 16 | return "", false 17 | } 18 | val, ok := entry.Labels[l.key] 19 | return val, ok 20 | } 21 | 22 | // Set will set the label value on an entry 23 | func (l LabelField) Set(entry *Entry, val interface{}) error { 24 | if entry.Labels == nil { 25 | entry.Labels = make(map[string]string, 1) 26 | } 27 | 28 | str, ok := val.(string) 29 | if !ok { 30 | return fmt.Errorf("cannot set a label to a non-string value") 31 | } 32 | entry.Labels[l.key] = str 33 | return nil 34 | } 35 | 36 | // Delete will delete a label from an entry 37 | func (l LabelField) Delete(entry *Entry) (interface{}, bool) { 38 | if entry.Labels == nil { 39 | return "", false 40 | } 41 | 42 | val, ok := entry.Labels[l.key] 43 | delete(entry.Labels, l.key) 44 | return val, ok 45 | } 46 | 47 | func (l LabelField) String() string { 48 | if strings.Contains(l.key, ".") { 49 | return fmt.Sprintf(`$labels['%s']`, l.key) 50 | } 51 | return "$labels." + l.key 52 | } 53 | 54 | // NewLabelField will creat a new label field from a key 55 | func NewLabelField(key string) Field { 56 | return Field{LabelField{key}} 57 | } 58 | -------------------------------------------------------------------------------- /logger/core.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import "go.uber.org/zap/zapcore" 4 | 5 | // Core is a zap Core used for logging 6 | type Core struct { 7 | core zapcore.Core 8 | emitter *Emitter 9 | } 10 | 11 | // With adds contextual fields to the underlying core. 12 | func (c *Core) With(fields []zapcore.Field) zapcore.Core { 13 | return &Core{ 14 | core: c.core.With(fields), 15 | emitter: c.emitter, 16 | } 17 | } 18 | 19 | // Enabled will check if the supplied log level is enabled. 20 | func (c *Core) Enabled(level zapcore.Level) bool { 21 | return c.core.Enabled(level) 22 | } 23 | 24 | // Check checks the entry and determines if the core should write it. 25 | func (c *Core) Check(zapEntry zapcore.Entry, checkedEntry *zapcore.CheckedEntry) *zapcore.CheckedEntry { 26 | if !c.Enabled(zapEntry.Level) { 27 | return checkedEntry 28 | } 29 | return checkedEntry.AddCore(zapEntry, c) 30 | } 31 | 32 | // Write sends an entry to the emitter before logging. 33 | func (c *Core) Write(zapEntry zapcore.Entry, fields []zapcore.Field) error { 34 | stanzaEntry := parseEntry(zapEntry, fields) 35 | c.emitter.emit(stanzaEntry) 36 | return c.core.Write(zapEntry, fields) 37 | } 38 | 39 | // Sync will sync the underlying core. 40 | func (c *Core) Sync() error { 41 | return c.core.Sync() 42 | } 43 | 44 | // newCore creates a new core. 45 | func newCore(core zapcore.Core, emitter *Emitter) *Core { 46 | return &Core{ 47 | core: core, 48 | emitter: emitter, 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /operator/flusher/flusher_test.go: -------------------------------------------------------------------------------- 1 | package flusher 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "math/rand" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/require" 11 | "go.uber.org/zap/zaptest" 12 | ) 13 | 14 | func TestFlusher(t *testing.T) { 15 | 16 | // Override setting for test 17 | maxElapsedTime = 5 * time.Second 18 | 19 | outChan := make(chan struct{}, 100) 20 | flusherCfg := NewConfig() 21 | flusher := flusherCfg.Build(zaptest.NewLogger(t).Sugar()) 22 | 23 | failed := errors.New("test failure") 24 | for i := 0; i < 100; i++ { 25 | flusher.Do(func(_ context.Context) error { 26 | // Fail randomly but still expect the entries to come through 27 | if rand.Int()%5 == 0 { 28 | return failed 29 | } 30 | outChan <- struct{}{} 31 | return nil 32 | }) 33 | } 34 | 35 | for i := 0; i < 100; i++ { 36 | select { 37 | case <-time.After(5 * time.Second): 38 | require.FailNow(t, "timed out") 39 | case <-outChan: 40 | } 41 | } 42 | } 43 | 44 | func TestMaxElapsedTime(t *testing.T) { 45 | 46 | // Override setting for test 47 | maxElapsedTime = 1 * time.Second 48 | 49 | flusherCfg := NewConfig() 50 | flusher := flusherCfg.Build(zaptest.NewLogger(t).Sugar()) 51 | 52 | start := time.Now() 53 | flusher.flushWithRetry(context.Background(), func(_ context.Context) error { 54 | return errors.New("never flushes") 55 | }) 56 | require.WithinDuration(t, start.Add(maxElapsedTime), time.Now(), maxElapsedTime) 57 | } 58 | -------------------------------------------------------------------------------- /operator/builtin/output/drop/drop.go: -------------------------------------------------------------------------------- 1 | package drop 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/observiq/stanza/entry" 7 | "github.com/observiq/stanza/operator" 8 | "github.com/observiq/stanza/operator/helper" 9 | ) 10 | 11 | func init() { 12 | operator.Register("drop_output", func() operator.Builder { return NewDropOutputConfig("") }) 13 | } 14 | 15 | // NewDropOutputConfig creates a new drop output config with default values 16 | func NewDropOutputConfig(operatorID string) *DropOutputConfig { 17 | return &DropOutputConfig{ 18 | OutputConfig: helper.NewOutputConfig(operatorID, "drop_output"), 19 | } 20 | } 21 | 22 | // DropOutputConfig is the configuration of a drop output operator. 23 | type DropOutputConfig struct { 24 | helper.OutputConfig `yaml:",inline"` 25 | } 26 | 27 | // Build will build a drop output operator. 28 | func (c DropOutputConfig) Build(context operator.BuildContext) ([]operator.Operator, error) { 29 | outputOperator, err := c.OutputConfig.Build(context) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | dropOutput := &DropOutput{ 35 | OutputOperator: outputOperator, 36 | } 37 | 38 | return []operator.Operator{dropOutput}, nil 39 | } 40 | 41 | // DropOutput is an operator that consumes and ignores incoming entries. 42 | type DropOutput struct { 43 | helper.OutputOperator 44 | } 45 | 46 | // Process will drop the incoming entry. 47 | func (p *DropOutput) Process(_ context.Context, _ *entry.Entry) error { 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /operator/builtin/input/azure/event_hub_persist.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/Azure/azure-event-hubs-go/v3/persist" 8 | "github.com/observiq/stanza/operator/helper" 9 | ) 10 | 11 | // Persister implements persist.CheckpointPersister 12 | type Persister struct { 13 | DB helper.Persister 14 | } 15 | 16 | // Write records an Azure Event Hub Checkpoint to the Stanza persistence backend 17 | func (p *Persister) Write(namespace, name, consumerGroup, partitionID string, checkpoint persist.Checkpoint) error { 18 | key := p.persistenceKey(namespace, name, consumerGroup, partitionID) 19 | value, err := json.Marshal(checkpoint) 20 | if err != nil { 21 | return err 22 | } 23 | 24 | p.DB.Set(key, value) 25 | return p.DB.Sync() 26 | } 27 | 28 | // Read retrieves an Azure Event Hub Checkpoint from the Stanza persistence backend 29 | func (p *Persister) Read(namespace, name, consumerGroup, partitionID string) (persist.Checkpoint, error) { 30 | key := p.persistenceKey(namespace, name, consumerGroup, partitionID) 31 | value := p.DB.Get(key) 32 | 33 | if len(value) < 1 { 34 | return persist.Checkpoint{}, nil 35 | } 36 | 37 | var checkpoint persist.Checkpoint 38 | err := json.Unmarshal(value, &checkpoint) 39 | return checkpoint, err 40 | } 41 | 42 | func (p *Persister) persistenceKey(namespace, name, consumerGroup, partitionID string) string { 43 | x := fmt.Sprintf("%s-%s-%s-%s", namespace, name, consumerGroup, partitionID) 44 | return x 45 | } 46 | -------------------------------------------------------------------------------- /operator/builtin/transformer/ratelimit/rate_limit_test.go: -------------------------------------------------------------------------------- 1 | package ratelimit 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "testing" 7 | "time" 8 | 9 | "github.com/observiq/stanza/entry" 10 | "github.com/observiq/stanza/operator" 11 | "github.com/observiq/stanza/testutil" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestRateLimit(t *testing.T) { 16 | cfg := NewRateLimitConfig("my_rate_limit") 17 | cfg.OutputIDs = []string{"fake"} 18 | cfg.Burst = 1 19 | cfg.Rate = 100 20 | 21 | ops, err := cfg.Build(testutil.NewBuildContext(t)) 22 | require.NoError(t, err) 23 | op := ops[0] 24 | 25 | fake := testutil.NewFakeOutput(t) 26 | 27 | err = op.SetOutputs([]operator.Operator{fake}) 28 | require.NoError(t, err) 29 | 30 | err = op.Start() 31 | defer op.Stop() 32 | require.NoError(t, err) 33 | 34 | var wg sync.WaitGroup 35 | wg.Add(1) 36 | go func() { 37 | defer wg.Done() 38 | for { 39 | _, ok := <-fake.Received 40 | if !ok { 41 | return 42 | } 43 | } 44 | }() 45 | 46 | // Warm up 47 | for i := 0; i < 100; i++ { 48 | err := op.Process(context.Background(), entry.New()) 49 | require.NoError(t, err) 50 | } 51 | 52 | // Measure 53 | start := time.Now() 54 | for i := 0; i < 500; i++ { 55 | err := op.Process(context.Background(), entry.New()) 56 | require.NoError(t, err) 57 | } 58 | elapsed := time.Since(start) 59 | 60 | close(fake.Received) 61 | wg.Wait() 62 | 63 | require.InEpsilon(t, elapsed.Nanoseconds(), 5*time.Second.Nanoseconds(), 0.6) 64 | } 65 | -------------------------------------------------------------------------------- /docs/operators/forward_input.md: -------------------------------------------------------------------------------- 1 | ## `foward_input` operator 2 | 3 | The `foward_input` operator receives logs from another Stanza instance running `forward_output`. 4 | 5 | ### Configuration Fields 6 | 7 | | Field | Default | Description | 8 | | --- | --- | --- | 9 | | `id` | `forward_output` | A unique identifier for the operator | 10 | | `listen_address` | `:80` | The IP address and port to listen on | 11 | | `tls` | | A block for configuring the server to listen with TLS | 12 | | `read_timeout` | `5s` | Maximum duration for reading the entire request, including the body. | 13 | 14 | #### TLS block configuration 15 | 16 | | Field | Default | Description | 17 | | --- | --- | --- | 18 | | `cert_file` | | The location of the certificate file | 19 | | `key_file` | | The location of the key file | 20 | 21 | 22 | ### Example Configurations 23 | 24 | #### Simple configuration 25 | 26 | Configuration: 27 | ```yaml 28 | - type: forward_input 29 | listen_address: ":25535" 30 | ``` 31 | 32 | #### TLS configuration 33 | 34 | Configuration: 35 | ```yaml 36 | - type: forward_input 37 | listen_address: ":25535" 38 | tls: 39 | cert_file: /tmp/public.crt 40 | key_file: /tmp/private.key 41 | ``` 42 | -------------------------------------------------------------------------------- /operator/buffer/disk_metadata_test.go: -------------------------------------------------------------------------------- 1 | package buffer 2 | 3 | import ( 4 | "bytes" 5 | "strconv" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestMetadata(t *testing.T) { 12 | t.Run("binaryRoundTrip", func(t *testing.T) { 13 | cases := [...]Metadata{ 14 | 0: { 15 | read: []*readEntry{}, 16 | unreadStartOffset: 0, 17 | unreadCount: 0, 18 | deadRangeStart: 0, 19 | deadRangeLength: 0, 20 | }, 21 | 1: { 22 | read: []*readEntry{}, 23 | unreadStartOffset: 0, 24 | unreadCount: 50, 25 | deadRangeStart: 0, 26 | deadRangeLength: 0, 27 | }, 28 | 2: { 29 | read: []*readEntry{ 30 | { 31 | flushed: false, 32 | length: 10, 33 | startOffset: 0, 34 | }, 35 | }, 36 | unreadStartOffset: 10, 37 | unreadCount: 50, 38 | deadRangeStart: 0, 39 | deadRangeLength: 0, 40 | }, 41 | 3: { 42 | read: []*readEntry{}, 43 | unreadStartOffset: 0, 44 | unreadCount: 50, 45 | deadRangeStart: 10, 46 | deadRangeLength: 100, 47 | }, 48 | } 49 | 50 | for i, md := range cases { 51 | t.Run(strconv.Itoa(i), func(t *testing.T) { 52 | var buf bytes.Buffer 53 | err := md.MarshalBinary(&buf) 54 | require.NoError(t, err) 55 | 56 | md2 := Metadata{} 57 | err = md2.UnmarshalBinary(&buf) 58 | require.NoError(t, err) 59 | 60 | require.Equal(t, md, md2) 61 | }) 62 | } 63 | }) 64 | } 65 | -------------------------------------------------------------------------------- /operator/builtin/output/googlecloud/severity.go: -------------------------------------------------------------------------------- 1 | package googlecloud 2 | 3 | import ( 4 | "github.com/observiq/stanza/entry" 5 | sev "google.golang.org/genproto/googleapis/logging/type" 6 | ) 7 | 8 | var fastSev = map[entry.Severity]sev.LogSeverity{ 9 | entry.Catastrophe: sev.LogSeverity_EMERGENCY, 10 | entry.Emergency: sev.LogSeverity_EMERGENCY, 11 | entry.Alert: sev.LogSeverity_ALERT, 12 | entry.Critical: sev.LogSeverity_CRITICAL, 13 | entry.Error: sev.LogSeverity_ERROR, 14 | entry.Warning: sev.LogSeverity_WARNING, 15 | entry.Notice: sev.LogSeverity_NOTICE, 16 | entry.Info: sev.LogSeverity_INFO, 17 | entry.Debug: sev.LogSeverity_DEBUG, 18 | entry.Trace: sev.LogSeverity_DEBUG, 19 | entry.Default: sev.LogSeverity_DEFAULT, 20 | } 21 | 22 | func convertSeverity(s entry.Severity) sev.LogSeverity { 23 | if logSev, ok := fastSev[s]; ok { 24 | return logSev 25 | } 26 | 27 | switch { 28 | case s >= entry.Emergency: 29 | return sev.LogSeverity_EMERGENCY 30 | case s >= entry.Alert: 31 | return sev.LogSeverity_ALERT 32 | case s >= entry.Critical: 33 | return sev.LogSeverity_CRITICAL 34 | case s >= entry.Error: 35 | return sev.LogSeverity_ERROR 36 | case s >= entry.Warning: 37 | return sev.LogSeverity_WARNING 38 | case s >= entry.Notice: 39 | return sev.LogSeverity_NOTICE 40 | case s >= entry.Info: 41 | return sev.LogSeverity_INFO 42 | case s > entry.Default: 43 | return sev.LogSeverity_DEBUG 44 | default: 45 | return sev.LogSeverity_DEFAULT 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /entry/resource_field.go: -------------------------------------------------------------------------------- 1 | package entry 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // ResourceField is the path to an entry's resource key 9 | type ResourceField struct { 10 | key string 11 | } 12 | 13 | // Get will return the resource value and a boolean indicating if it exists 14 | func (r ResourceField) Get(entry *Entry) (interface{}, bool) { 15 | if entry.Resource == nil { 16 | return "", false 17 | } 18 | val, ok := entry.Resource[r.key] 19 | return val, ok 20 | } 21 | 22 | // Set will set the resource value on an entry 23 | func (r ResourceField) Set(entry *Entry, val interface{}) error { 24 | if entry.Resource == nil { 25 | entry.Resource = make(map[string]string, 1) 26 | } 27 | 28 | str, ok := val.(string) 29 | if !ok { 30 | return fmt.Errorf("cannot set a resource to a non-string value") 31 | } 32 | entry.Resource[r.key] = str 33 | return nil 34 | } 35 | 36 | // Delete will delete a resource key from an entry 37 | func (r ResourceField) Delete(entry *Entry) (interface{}, bool) { 38 | if entry.Resource == nil { 39 | return "", false 40 | } 41 | 42 | val, ok := entry.Resource[r.key] 43 | delete(entry.Resource, r.key) 44 | return val, ok 45 | } 46 | 47 | func (r ResourceField) String() string { 48 | if strings.Contains(r.key, ".") { 49 | return fmt.Sprintf(`$resource['%s']`, r.key) 50 | } 51 | return "$resource." + r.key 52 | } 53 | 54 | // NewResourceField will creat a new resource field from a key 55 | func NewResourceField(key string) Field { 56 | return Field{ResourceField{key}} 57 | } 58 | -------------------------------------------------------------------------------- /pipeline/config.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "github.com/observiq/stanza/operator" 5 | ) 6 | 7 | // Config is the configuration of a pipeline. 8 | type Config []operator.Config 9 | 10 | // BuildOperators builds the operators from the list of configs into operators 11 | func (c Config) BuildOperators(bc operator.BuildContext) ([]operator.Operator, error) { 12 | operators := make([]operator.Operator, 0, len(c)) 13 | for i, builder := range c { 14 | nbc := getBuildContextWithDefaultOutput(c, i, bc) 15 | op, err := builder.Build(nbc) 16 | if err != nil { 17 | return nil, err 18 | } 19 | operators = append(operators, op...) 20 | } 21 | return operators, nil 22 | } 23 | 24 | // BuildPipeline will build a pipeline from the config. 25 | func (c Config) BuildPipeline(bc operator.BuildContext, defaultOperator operator.Operator) (*DirectedPipeline, error) { 26 | if defaultOperator != nil { 27 | bc.DefaultOutputIDs = []string{defaultOperator.ID()} 28 | } 29 | 30 | operators, err := c.BuildOperators(bc) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | if defaultOperator != nil { 36 | operators = append(operators, defaultOperator) 37 | } 38 | 39 | return NewDirectedPipeline(operators) 40 | } 41 | 42 | func getBuildContextWithDefaultOutput(configs []operator.Config, i int, bc operator.BuildContext) operator.BuildContext { 43 | if i+1 >= len(configs) { 44 | return bc 45 | } 46 | 47 | id := configs[i+1].ID() 48 | id = bc.PrependNamespace(id) 49 | return bc.WithDefaultOutputIDs([]string{id}) 50 | } 51 | -------------------------------------------------------------------------------- /operator/helper/identifier.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "github.com/observiq/stanza/entry" 5 | ) 6 | 7 | // NewIdentifierConfig creates a new identifier config with default values 8 | func NewIdentifierConfig() IdentifierConfig { 9 | return IdentifierConfig{ 10 | Resource: make(map[string]ExprStringConfig), 11 | } 12 | } 13 | 14 | // IdentifierConfig is the configuration of a resource identifier 15 | type IdentifierConfig struct { 16 | Resource map[string]ExprStringConfig `json:"resource" yaml:"resource"` 17 | } 18 | 19 | // Build will build an identifier from the supplied configuration 20 | func (c IdentifierConfig) Build() (Identifier, error) { 21 | identifier := Identifier{ 22 | resource: make(map[string]*ExprString), 23 | } 24 | 25 | for k, v := range c.Resource { 26 | exprString, err := v.Build() 27 | if err != nil { 28 | return identifier, err 29 | } 30 | 31 | identifier.resource[k] = exprString 32 | } 33 | 34 | return identifier, nil 35 | } 36 | 37 | // Identifier is a helper that adds values to the resource of an entry 38 | type Identifier struct { 39 | resource map[string]*ExprString 40 | } 41 | 42 | // Identify will add values to the resource of an entry 43 | func (i *Identifier) Identify(e *entry.Entry) error { 44 | if len(i.resource) == 0 { 45 | return nil 46 | } 47 | 48 | env := GetExprEnv(e) 49 | defer PutExprEnv(env) 50 | 51 | for k, v := range i.resource { 52 | rendered, err := v.Render(env) 53 | if err != nil { 54 | return err 55 | } 56 | e.AddResourceKey(k, rendered) 57 | } 58 | 59 | return nil 60 | } 61 | -------------------------------------------------------------------------------- /docs/operators/filter.md: -------------------------------------------------------------------------------- 1 | ## `filter` operator 2 | 3 | The `filter` operator filters incoming entries that match an expression. 4 | 5 | ### Configuration Fields 6 | 7 | | Field | Default | Description | 8 | | --- | --- | --- | 9 | | `id` | `filter` | A unique identifier for the operator | 10 | | `output` | Next in pipeline | The connected operator(s) that will receive all outbound entries | 11 | | `expr` | required | Incoming entries that match this [expression](/docs/types/expression.md) will be dropped | 12 | | `drop_ratio` | 1.0 | The probability a matching entry is dropped (used for sampling). A value of 1.0 will drop 100% of matching entries, while a value of 0.0 will drop 0%. | 13 | 14 | ### Examples 15 | 16 | #### Filter entries based on a regex pattern 17 | 18 | ```yaml 19 | - type: filter 20 | expr: '$record.message matches "^LOG: .* END$"' 21 | output: my_output 22 | ``` 23 | 24 | #### Filter entries based on a label value 25 | 26 | ```yaml 27 | - type: filter 28 | expr: '$labels.env == "production"' 29 | output: my_output 30 | ``` 31 | 32 | #### Filter entries based on an environment variable 33 | 34 | ```yaml 35 | - type: filter 36 | expr: '$record.message == env("MY_ENV_VARIABLE")' 37 | output: my_output 38 | ``` 39 | -------------------------------------------------------------------------------- /pipeline/node.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "hash/fnv" 5 | 6 | "github.com/observiq/stanza/operator" 7 | ) 8 | 9 | // OperatorNode is a basic node that represents an operator in a pipeline. 10 | type OperatorNode struct { 11 | operator operator.Operator 12 | id int64 13 | outputIDs map[string]int64 14 | } 15 | 16 | // Operator returns the operator of the node. 17 | func (b OperatorNode) Operator() operator.Operator { 18 | return b.operator 19 | } 20 | 21 | // ID returns the node id. 22 | func (b OperatorNode) ID() int64 { 23 | return b.id 24 | } 25 | 26 | // DOTID returns the id used to represent this node in a dot graph. 27 | func (b OperatorNode) DOTID() string { 28 | return b.operator.ID() 29 | } 30 | 31 | // OutputIDs returns a map of output operator ids to node ids. 32 | func (b OperatorNode) OutputIDs() map[string]int64 { 33 | return b.outputIDs 34 | } 35 | 36 | // createOperatorNode will create an operator node. 37 | func createOperatorNode(operator operator.Operator) OperatorNode { 38 | id := createNodeID(operator.ID()) 39 | outputIDs := make(map[string]int64) 40 | if operator.CanOutput() { 41 | for _, output := range operator.Outputs() { 42 | outputIDs[output.ID()] = createNodeID(output.ID()) 43 | } 44 | } 45 | return OperatorNode{operator, id, outputIDs} 46 | } 47 | 48 | // createNodeID generates a node id from an operator id. 49 | func createNodeID(operatorID string) int64 { 50 | hash := fnv.New64a() 51 | _, _ = hash.Write([]byte(operatorID)) 52 | 53 | // #nosec G115 - Hash will not exceed int64 54 | return int64(hash.Sum64()) 55 | } 56 | -------------------------------------------------------------------------------- /testutil/operator_builder.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.0. DO NOT EDIT. 2 | 3 | package testutil 4 | 5 | import ( 6 | operator "github.com/observiq/stanza/operator" 7 | mock "github.com/stretchr/testify/mock" 8 | ) 9 | 10 | // OperatorBuilder is an autogenerated mock type for the Builder type 11 | type OperatorBuilder struct { 12 | mock.Mock 13 | } 14 | 15 | // Build provides a mock function with given fields: _a0 16 | func (_m *OperatorBuilder) Build(_a0 operator.BuildContext) (operator.Operator, error) { 17 | ret := _m.Called(_a0) 18 | 19 | var r0 operator.Operator 20 | if rf, ok := ret.Get(0).(func(operator.BuildContext) operator.Operator); ok { 21 | r0 = rf(_a0) 22 | } else { 23 | if ret.Get(0) != nil { 24 | r0 = ret.Get(0).(operator.Operator) 25 | } 26 | } 27 | 28 | var r1 error 29 | if rf, ok := ret.Get(1).(func(operator.BuildContext) error); ok { 30 | r1 = rf(_a0) 31 | } else { 32 | r1 = ret.Error(1) 33 | } 34 | 35 | return r0, r1 36 | } 37 | 38 | // ID provides a mock function with given fields: 39 | func (_m *OperatorBuilder) ID() string { 40 | ret := _m.Called() 41 | 42 | var r0 string 43 | if rf, ok := ret.Get(0).(func() string); ok { 44 | r0 = rf() 45 | } else { 46 | r0 = ret.Get(0).(string) 47 | } 48 | 49 | return r0 50 | } 51 | 52 | // Type provides a mock function with given fields: 53 | func (_m *OperatorBuilder) Type() string { 54 | ret := _m.Called() 55 | 56 | var r0 string 57 | if rf, ok := ret.Get(0).(func() string); ok { 58 | r0 = rf() 59 | } else { 60 | r0 = ret.Get(0).(string) 61 | } 62 | 63 | return r0 64 | } 65 | -------------------------------------------------------------------------------- /operator/builtin/transformer/noop/noop.go: -------------------------------------------------------------------------------- 1 | package noop 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/observiq/stanza/entry" 7 | "github.com/observiq/stanza/operator" 8 | "github.com/observiq/stanza/operator/helper" 9 | ) 10 | 11 | func init() { 12 | operator.Register("noop", func() operator.Builder { return NewNoopOperatorConfig("") }) 13 | } 14 | 15 | // NewNoopOperatorConfig creates a new noop operator config with default values 16 | func NewNoopOperatorConfig(operatorID string) *NoopOperatorConfig { 17 | return &NoopOperatorConfig{ 18 | TransformerConfig: helper.NewTransformerConfig(operatorID, "noop"), 19 | } 20 | } 21 | 22 | // NoopOperatorConfig is the configuration of a noop operator. 23 | type NoopOperatorConfig struct { 24 | helper.TransformerConfig `yaml:",inline"` 25 | } 26 | 27 | // Build will build a noop operator. 28 | func (c NoopOperatorConfig) Build(context operator.BuildContext) ([]operator.Operator, error) { 29 | transformerOperator, err := c.TransformerConfig.Build(context) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | noopOperator := &NoopOperator{ 35 | TransformerOperator: transformerOperator, 36 | } 37 | 38 | return []operator.Operator{noopOperator}, nil 39 | } 40 | 41 | // NoopOperator is an operator that performs no operations on an entry. 42 | type NoopOperator struct { 43 | helper.TransformerOperator 44 | } 45 | 46 | // Process will forward the entry to the next output without any alterations. 47 | func (p *NoopOperator) Process(ctx context.Context, entry *entry.Entry) error { 48 | p.Write(ctx, entry) 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /operator/builtin/transformer/hostmetadata/host_metadata_test.go: -------------------------------------------------------------------------------- 1 | package hostmetadata 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "testing" 7 | 8 | "github.com/observiq/stanza/entry" 9 | "github.com/observiq/stanza/operator" 10 | "github.com/observiq/stanza/testutil" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | type hostMetadataBenchmark struct { 15 | name string 16 | cfgMod func(*HostMetadataConfig) 17 | } 18 | 19 | func (g *hostMetadataBenchmark) Run(b *testing.B) { 20 | cfg := NewHostMetadataConfig(g.name) 21 | g.cfgMod(cfg) 22 | ops, err := cfg.Build(testutil.NewBuildContext(b)) 23 | require.NoError(b, err) 24 | op := ops[0] 25 | 26 | fake := testutil.NewFakeOutput(b) 27 | op.(*HostMetadata).OutputOperators = []operator.Operator{fake} 28 | 29 | b.ResetTimer() 30 | var wg sync.WaitGroup 31 | wg.Add(1) 32 | go func() { 33 | defer wg.Done() 34 | for i := 0; i < b.N; i++ { 35 | e := entry.New() 36 | op.Process(context.Background(), e) 37 | } 38 | err = op.Stop() 39 | require.NoError(b, err) 40 | }() 41 | 42 | wg.Add(1) 43 | go func() { 44 | defer wg.Done() 45 | for i := 0; i < b.N; i++ { 46 | <-fake.Received 47 | } 48 | }() 49 | 50 | wg.Wait() 51 | } 52 | 53 | func BenchmarkHostMetadata(b *testing.B) { 54 | cases := []hostMetadataBenchmark{ 55 | { 56 | "Default", 57 | func(cfg *HostMetadataConfig) {}, 58 | }, 59 | { 60 | "NoHostname", 61 | func(cfg *HostMetadataConfig) { 62 | cfg.IncludeHostname = false 63 | }, 64 | }, 65 | } 66 | 67 | for _, tc := range cases { 68 | b.Run(tc.name, tc.Run) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /testutil/database.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.0. DO NOT EDIT. 2 | 3 | package testutil 4 | 5 | import ( 6 | bbolt "go.etcd.io/bbolt" 7 | 8 | mock "github.com/stretchr/testify/mock" 9 | ) 10 | 11 | // Database is an autogenerated mock type for the Database type 12 | type Database struct { 13 | mock.Mock 14 | } 15 | 16 | // Close provides a mock function with given fields: 17 | func (_m *Database) Close() error { 18 | ret := _m.Called() 19 | 20 | var r0 error 21 | if rf, ok := ret.Get(0).(func() error); ok { 22 | r0 = rf() 23 | } else { 24 | r0 = ret.Error(0) 25 | } 26 | 27 | return r0 28 | } 29 | 30 | // Sync provides a mock function with given fields: 31 | func (_m *Database) Sync() error { 32 | ret := _m.Called() 33 | 34 | var r0 error 35 | if rf, ok := ret.Get(0).(func() error); ok { 36 | r0 = rf() 37 | } else { 38 | r0 = ret.Error(0) 39 | } 40 | 41 | return r0 42 | } 43 | 44 | // Update provides a mock function with given fields: _a0 45 | func (_m *Database) Update(_a0 func(*bbolt.Tx) error) error { 46 | ret := _m.Called(_a0) 47 | 48 | var r0 error 49 | if rf, ok := ret.Get(0).(func(func(*bbolt.Tx) error) error); ok { 50 | r0 = rf(_a0) 51 | } else { 52 | r0 = ret.Error(0) 53 | } 54 | 55 | return r0 56 | } 57 | 58 | // View provides a mock function with given fields: _a0 59 | func (_m *Database) View(_a0 func(*bbolt.Tx) error) error { 60 | ret := _m.Called(_a0) 61 | 62 | var r0 error 63 | if rf, ok := ret.Get(0).(func(func(*bbolt.Tx) error) error); ok { 64 | r0 = rf(_a0) 65 | } else { 66 | r0 = ret.Error(0) 67 | } 68 | 69 | return r0 70 | } 71 | -------------------------------------------------------------------------------- /operator/buffer/util_test.go: -------------------------------------------------------------------------------- 1 | package buffer 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/observiq/stanza/entry" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func intEntry(i int) *entry.Entry { 13 | e := entry.New() 14 | e.Timestamp = time.Date(2006, 01, 02, 03, 04, 05, 06, time.UTC) 15 | e.Record = float64(i) 16 | return e 17 | } 18 | 19 | func writeN(t testing.TB, buffer Buffer, n, start int) { 20 | ctx := context.Background() 21 | for i := start; i < n+start; i++ { 22 | err := buffer.Add(ctx, intEntry(i)) 23 | require.NoError(t, err) 24 | } 25 | } 26 | 27 | func readN(t testing.TB, buffer Buffer, n, start int) Clearer { 28 | entries := make([]*entry.Entry, n) 29 | f, readCount, err := buffer.Read(entries) 30 | require.NoError(t, err) 31 | require.Equal(t, n, readCount) 32 | for i := 0; i < n; i++ { 33 | require.Equal(t, intEntry(start+i), entries[i]) 34 | } 35 | return f 36 | } 37 | 38 | func readWaitN(t testing.TB, buffer Buffer, n, start int) Clearer { 39 | entries := make([]*entry.Entry, n) 40 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) 41 | defer cancel() 42 | f, readCount, err := buffer.ReadWait(ctx, entries) 43 | require.NoError(t, err) 44 | require.Equal(t, n, readCount) 45 | for i := 0; i < n; i++ { 46 | require.Equal(t, intEntry(start+i), entries[i]) 47 | } 48 | return f 49 | } 50 | 51 | func flushN(t testing.TB, buffer Buffer, n, start int) { 52 | f := readN(t, buffer, n, start) 53 | f.MarkAllAsFlushed() 54 | } 55 | 56 | func panicOnErr(err error) { 57 | if err != nil { 58 | panic(err) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /docs/examples/tomcat/config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | pipeline: 3 | 4 | # Read lines from Apache Tomcat access logs 5 | # Example input line: 6 | # 10.66.2.46 - - [13/Mar/2019:10:43:00 -0400] "GET / HTTP/1.1" 404 - 7 | - type: file_input 8 | start_at: beginning 9 | include: 10 | - ./access.log 11 | labels: 12 | log_type: tomcat 13 | 14 | # Parse the logs into labeled fields 15 | # Example input: 16 | # { 17 | # "timestamp": "2020-06-13T11:00:53-04:00", 18 | # "record": "10.66.2.46 - - [13/Mar/2019:10:43:00 -0400] "GET / HTTP/1.1" 404 -" 19 | # } 20 | - type: regex_parser 21 | regex: >- 22 | (?P[^\s]+) 23 | - 24 | (?P[^\s]+) 25 | \[(?P[^\]]+)\] 26 | "(?P[A-Z]+) 27 | (?P[^\s]+)[^"]*" 28 | (?P\d+) 29 | (?P[\d-]+) 30 | timestamp: 31 | parse_from: timestamp 32 | layout: '%d/%b/%Y:%H:%M:%S %z' 33 | severity: 34 | parse_from: http_status 35 | preserve_to: http_status 36 | mapping: 37 | error: "4xx" 38 | info: 39 | - min: 300 40 | max: 399 41 | debug: 200 42 | 43 | # Write the log to stdout 44 | # Example input: 45 | # { 46 | # "timestamp": "2019-03-13T11:00:53-04:00", 47 | # "severity": 60, 48 | # "record": { 49 | # "bytes_sent": "19698", 50 | # "http_method": "GET", 51 | # "http_status": "200", 52 | # "remote_host": "10.66.2.46", 53 | # "remote_user": "-", 54 | # "url_path": "/manager/images/asf-logo.svg" 55 | # } 56 | # } 57 | - type: stdout 58 | -------------------------------------------------------------------------------- /cmd/stanza/testdata/tomcat/config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | pipeline: 3 | 4 | # Read lines from Apache Tomcat access logs 5 | # Example input line: 6 | # 10.66.2.46 - - [13/Mar/2019:10:43:00 -0400] "GET / HTTP/1.1" 404 - 7 | - type: file_input 8 | start_at: beginning 9 | include: 10 | - ./access.log 11 | labels: 12 | log_type: tomcat 13 | 14 | # Parse the logs into labeled fields 15 | # Example input: 16 | # { 17 | # "timestamp": "2020-06-13T11:00:53-04:00", 18 | # "record": "10.66.2.46 - - [13/Mar/2019:10:43:00 -0400] "GET / HTTP/1.1" 404 -" 19 | # } 20 | - type: regex_parser 21 | regex: >- 22 | (?P[^\s]+) 23 | - 24 | (?P[^\s]+) 25 | \[(?P[^\]]+)\] 26 | "(?P[A-Z]+) 27 | (?P[^\s]+)[^"]*" 28 | (?P\d+) 29 | (?P[\d-]+) 30 | timestamp: 31 | parse_from: timestamp 32 | layout: '%d/%b/%Y:%H:%M:%S %z' 33 | severity: 34 | parse_from: http_status 35 | preserve_to: http_status 36 | mapping: 37 | error: "4xx" 38 | info: 39 | - min: 300 40 | max: 399 41 | debug: 200 42 | 43 | # Write the log to stdout 44 | # Example input: 45 | # { 46 | # "timestamp": "2019-03-13T11:00:53-04:00", 47 | # "severity": 60, 48 | # "record": { 49 | # "bytes_sent": "19698", 50 | # "http_method": "GET", 51 | # "http_status": "200", 52 | # "remote_host": "10.66.2.46", 53 | # "remote_user": "-", 54 | # "url_path": "/manager/images/asf-logo.svg" 55 | # } 56 | # } 57 | - type: stdout 58 | -------------------------------------------------------------------------------- /entry/severity_test.go: -------------------------------------------------------------------------------- 1 | package entry 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestStringer(t *testing.T) { 10 | require.Equal(t, "default", Default.String()) 11 | require.Equal(t, "trace", Trace.String()) 12 | require.Equal(t, "trace2", Trace2.String()) 13 | require.Equal(t, "trace3", Trace3.String()) 14 | require.Equal(t, "trace4", Trace4.String()) 15 | require.Equal(t, "debug", Debug.String()) 16 | require.Equal(t, "debug2", Debug2.String()) 17 | require.Equal(t, "debug3", Debug3.String()) 18 | require.Equal(t, "debug4", Debug4.String()) 19 | require.Equal(t, "info", Info.String()) 20 | require.Equal(t, "info2", Info2.String()) 21 | require.Equal(t, "info3", Info3.String()) 22 | require.Equal(t, "info4", Info4.String()) 23 | require.Equal(t, "notice", Notice.String()) 24 | require.Equal(t, "warning", Warning.String()) 25 | require.Equal(t, "warning2", Warning2.String()) 26 | require.Equal(t, "warning3", Warning3.String()) 27 | require.Equal(t, "warning4", Warning4.String()) 28 | require.Equal(t, "error", Error.String()) 29 | require.Equal(t, "error2", Error2.String()) 30 | require.Equal(t, "error3", Error3.String()) 31 | require.Equal(t, "error4", Error4.String()) 32 | require.Equal(t, "critical", Critical.String()) 33 | require.Equal(t, "alert", Alert.String()) 34 | require.Equal(t, "emergency", Emergency.String()) 35 | require.Equal(t, "emergency2", Emergency2.String()) 36 | require.Equal(t, "emergency3", Emergency3.String()) 37 | require.Equal(t, "emergency4", Emergency4.String()) 38 | require.Equal(t, "catastrophe", Catastrophe.String()) 39 | require.Equal(t, "19", Severity(19).String()) 40 | } 41 | -------------------------------------------------------------------------------- /operator/helper/host_identifier_test.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/observiq/stanza/entry" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func MockHostIdentifierConfig(includeIP, includeHostname bool, ip, hostname string) HostIdentifierConfig { 11 | return HostIdentifierConfig{ 12 | IncludeIP: includeIP, 13 | IncludeHostname: includeHostname, 14 | getIP: func() (string, error) { return ip, nil }, 15 | getHostname: func() (string, error) { return hostname, nil }, 16 | } 17 | } 18 | 19 | func TestHostLabeler(t *testing.T) { 20 | cases := []struct { 21 | name string 22 | config HostIdentifierConfig 23 | expectedResource map[string]string 24 | }{ 25 | { 26 | "HostnameAndIP", 27 | MockHostIdentifierConfig(true, true, "ip", "hostname"), 28 | map[string]string{ 29 | "host.name": "hostname", 30 | "host.ip": "ip", 31 | }, 32 | }, 33 | { 34 | "HostnameNoIP", 35 | MockHostIdentifierConfig(false, true, "ip", "hostname"), 36 | map[string]string{ 37 | "host.name": "hostname", 38 | }, 39 | }, 40 | { 41 | "IPNoHostname", 42 | MockHostIdentifierConfig(true, false, "ip", "hostname"), 43 | map[string]string{ 44 | "host.ip": "ip", 45 | }, 46 | }, 47 | { 48 | "NoHostnameNoIP", 49 | MockHostIdentifierConfig(false, false, "", "test"), 50 | nil, 51 | }, 52 | } 53 | 54 | for _, tc := range cases { 55 | t.Run(tc.name, func(t *testing.T) { 56 | identifier, err := tc.config.Build() 57 | require.NoError(t, err) 58 | 59 | e := entry.New() 60 | identifier.Identify(e) 61 | require.Equal(t, tc.expectedResource, e.Resource) 62 | }) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /operator/builtin/input/goflow/goflow_test.go: -------------------------------------------------------------------------------- 1 | package goflow 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/observiq/stanza/testutil" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestBuild(t *testing.T) { 11 | cases := []struct { 12 | name string 13 | inputRecord GoflowInputConfig 14 | expectErr bool 15 | }{ 16 | { 17 | "minimal-default-mode", 18 | GoflowInputConfig{ 19 | ListenAddress: "0.0.0.0:2056", 20 | }, 21 | false, 22 | }, 23 | { 24 | "minimal-netflow-v5", 25 | GoflowInputConfig{ 26 | Mode: "netflow_v5", 27 | ListenAddress: "0.0.0.0:2056", 28 | }, 29 | false, 30 | }, 31 | { 32 | "minimal-netflow-ipfix", 33 | GoflowInputConfig{ 34 | Mode: "netflow_ipfix", 35 | ListenAddress: "0.0.0.0:2056", 36 | }, 37 | false, 38 | }, 39 | { 40 | "minimal-netflow-sflow", 41 | GoflowInputConfig{ 42 | Mode: "netflow_v5", 43 | ListenAddress: "0.0.0.0:2056", 44 | }, 45 | false, 46 | }, 47 | { 48 | "invalid mode", 49 | GoflowInputConfig{ 50 | Mode: "netflow", 51 | ListenAddress: "0.0.0.0:2056", 52 | }, 53 | true, 54 | }, 55 | { 56 | "missing-address", 57 | GoflowInputConfig{ 58 | Mode: "sflow", 59 | }, 60 | true, 61 | }, 62 | } 63 | 64 | for _, tc := range cases { 65 | t.Run(tc.name, func(t *testing.T) { 66 | cfg := NewGoflowInputConfig("test_id") 67 | cfg.ListenAddress = tc.inputRecord.ListenAddress 68 | cfg.Mode = tc.inputRecord.Mode 69 | 70 | _, err := cfg.Build(testutil.NewBuildContext(t)) 71 | if tc.expectErr { 72 | require.Error(t, err) 73 | return 74 | } 75 | require.NoError(t, err) 76 | }) 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /docs/operators/udp_input.md: -------------------------------------------------------------------------------- 1 | ## `udp_input` operator 2 | 3 | The `udp_input` operator listens for logs from UDP packets. 4 | 5 | ### Configuration Fields 6 | 7 | | Field | Default | Description | 8 | | --- | --- | --- | 9 | | `id` | `udp_input` | A unique identifier for the operator | 10 | | `output` | Next in pipeline | The connected operator(s) that will receive all outbound entries | 11 | | `listen_address` | required | A listen address of the form `:` | 12 | | `write_to` | $ | The record [field](/docs/types/field.md) written to when creating a new log entry | 13 | | `labels` | {} | A map of `key: value` labels to add to the entry's labels | 14 | | `resource` | {} | A map of `key: value` labels to add to the entry's resource | 15 | | `add_labels` | false | Adds `net.transport`, `net.peer.ip`, `net.peer.port`, `net.host.ip` and `net.host.port` labels | 16 | 17 | ### Example Configurations 18 | 19 | #### Simple 20 | 21 | Configuration: 22 | ```yaml 23 | - type: udp_input 24 | listen_adress: "0.0.0.0:54526" 25 | ``` 26 | 27 | Send a log: 28 | ```bash 29 | $ nc -u localhost 54525 < message1 31 | heredoc> message2 32 | heredoc> EOF 33 | ``` 34 | 35 | Generated entries: 36 | ```json 37 | { 38 | "timestamp": "2020-04-30T12:10:17.656726-04:00", 39 | "record": "message1\nmessage2\n" 40 | } 41 | ``` 42 | -------------------------------------------------------------------------------- /testutil/pipeline.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.0. DO NOT EDIT. 2 | 3 | package testutil 4 | 5 | import ( 6 | operator "github.com/observiq/stanza/operator" 7 | mock "github.com/stretchr/testify/mock" 8 | ) 9 | 10 | // Pipeline is an autogenerated mock type for the Pipeline type 11 | type Pipeline struct { 12 | mock.Mock 13 | } 14 | 15 | // Operators provides a mock function with given fields: 16 | func (_m *Pipeline) Operators() []operator.Operator { 17 | ret := _m.Called() 18 | 19 | var r0 []operator.Operator 20 | if rf, ok := ret.Get(0).(func() []operator.Operator); ok { 21 | r0 = rf() 22 | } else { 23 | if ret.Get(0) != nil { 24 | r0 = ret.Get(0).([]operator.Operator) 25 | } 26 | } 27 | 28 | return r0 29 | } 30 | 31 | // Render provides a mock function with given fields: 32 | func (_m *Pipeline) Render() ([]byte, error) { 33 | ret := _m.Called() 34 | 35 | var r0 []byte 36 | if rf, ok := ret.Get(0).(func() []byte); ok { 37 | r0 = rf() 38 | } else { 39 | if ret.Get(0) != nil { 40 | r0 = ret.Get(0).([]byte) 41 | } 42 | } 43 | 44 | var r1 error 45 | if rf, ok := ret.Get(1).(func() error); ok { 46 | r1 = rf() 47 | } else { 48 | r1 = ret.Error(1) 49 | } 50 | 51 | return r0, r1 52 | } 53 | 54 | // Start provides a mock function with given fields: 55 | func (_m *Pipeline) Start() error { 56 | ret := _m.Called() 57 | 58 | var r0 error 59 | if rf, ok := ret.Get(0).(func() error); ok { 60 | r0 = rf() 61 | } else { 62 | r0 = ret.Error(0) 63 | } 64 | 65 | return r0 66 | } 67 | 68 | // Stop provides a mock function with given fields: 69 | func (_m *Pipeline) Stop() error { 70 | ret := _m.Called() 71 | 72 | var r0 error 73 | if rf, ok := ret.Get(0).(func() error); ok { 74 | r0 = rf() 75 | } else { 76 | r0 = ret.Error(0) 77 | } 78 | 79 | return r0 80 | } 81 | -------------------------------------------------------------------------------- /operator/builtin/input/file/finder.go: -------------------------------------------------------------------------------- 1 | // Copyright The OpenTelemetry Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package file 16 | 17 | import ( 18 | "github.com/bmatcuk/doublestar/v3" 19 | ) 20 | 21 | // Finder is responsible for find files according to the include and exclude rules 22 | type Finder struct { 23 | Include []string `mapstructure:"include,omitempty" json:"include,omitempty" yaml:"include,omitempty"` 24 | Exclude []string `mapstructure:"exclude,omitempty" json:"exclude,omitempty" yaml:"exclude,omitempty"` 25 | } 26 | 27 | // FindFiles gets a list of paths given an array of glob patterns to include and exclude 28 | func (f Finder) FindFiles() []string { 29 | paths := make(map[string]bool, len(f.Include)) 30 | uniquePaths := make([]string, 0, len(f.Include)) 31 | for _, include := range f.Include { 32 | matches, _ := doublestar.Glob(include) // compile error checked in build 33 | INCLUDE: 34 | for _, match := range matches { 35 | for _, exclude := range f.Exclude { 36 | if itMatches, _ := doublestar.PathMatch(exclude, match); itMatches { 37 | continue INCLUDE 38 | } 39 | } 40 | 41 | if ok := paths[match]; !ok { 42 | paths[match] = true 43 | uniquePaths = append(uniquePaths, match) 44 | } 45 | } 46 | } 47 | 48 | return uniquePaths 49 | } 50 | -------------------------------------------------------------------------------- /operator/helper/output.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "github.com/observiq/stanza/errors" 5 | "github.com/observiq/stanza/operator" 6 | ) 7 | 8 | // NewOutputConfig creates a new output config 9 | func NewOutputConfig(operatorID, operatorType string) OutputConfig { 10 | return OutputConfig{ 11 | BasicConfig: NewBasicConfig(operatorID, operatorType), 12 | } 13 | } 14 | 15 | // OutputConfig provides a basic implementation of an output operator config. 16 | type OutputConfig struct { 17 | BasicConfig `mapstructure:",squash" yaml:",inline"` 18 | } 19 | 20 | // Build will build an output operator. 21 | func (c OutputConfig) Build(context operator.BuildContext) (OutputOperator, error) { 22 | basicOperator, err := c.BasicConfig.Build(context) 23 | if err != nil { 24 | return OutputOperator{}, err 25 | } 26 | 27 | outputOperator := OutputOperator{ 28 | BasicOperator: basicOperator, 29 | } 30 | 31 | return outputOperator, nil 32 | } 33 | 34 | // OutputOperator provides a basic implementation of an output operator. 35 | type OutputOperator struct { 36 | BasicOperator 37 | } 38 | 39 | // CanProcess will always return true for an output operator. 40 | func (o *OutputOperator) CanProcess() bool { 41 | return true 42 | } 43 | 44 | // CanOutput will always return false for an output operator. 45 | func (o *OutputOperator) CanOutput() bool { 46 | return false 47 | } 48 | 49 | // Outputs will always return an empty array for an output operator. 50 | func (o *OutputOperator) Outputs() []operator.Operator { 51 | return []operator.Operator{} 52 | } 53 | 54 | // SetOutputs will return an error if called. 55 | func (o *OutputOperator) SetOutputs(_ []operator.Operator) error { 56 | return errors.NewError( 57 | "Operator can not output, but is attempting to set an output.", 58 | "This is an unexpected internal error. Please submit a bug/issue.", 59 | ) 60 | } 61 | -------------------------------------------------------------------------------- /docs/examples/k8s/events/USAGE.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Events w/ Google Cloud Logging 2 | 3 | Stanza can be deployed as a Kubernetes Events collector by leveraging the [k8s_event_input](https://github.com/observIQ/stanza/blob/master/docs/operators/k8s_event_input.md) operator. [Minikube](https://minikube.sigs.k8s.io/docs/start/) 4 | can be used for this example. 5 | 6 | ## Architecture 7 | 8 | 1. Service account with permission to the Kubernetes API server 9 | 2. Config map: Contains the stanza configuration file 10 | 3. Credentials secret: Contains Google Cloud [service account credentials JSON file](https://cloud.google.com/docs/authentication/getting-started) 11 | 4. Persistent volume: Allows the stanza database to persist between restarts and pod evictions 12 | 5. Deployment: A single replica deployment for the agent 13 | 14 | ## Prerequisites 15 | 16 | 1. Google Cloud account with Cloud Logging API enabled 17 | 2. Google service account with [roles/logging.logWriter](https://cloud.google.com/logging/docs/access-control) 18 | 3. Kubernetes Cluster with a storageclass capable of providing persistent volumes 19 | 4. Edit `config.yaml` to include: 20 | - Your cluster name (this is added as a label) 21 | - Your project_id 22 | 23 | ## Deployment Steps 24 | 25 | Create the credentials secret. The file provided in this example should be replaced 26 | with your service account's credentials. 27 | ``` 28 | kubectl create secret generic stanza-agent-credentials \ 29 | --from-file=log_credentials.json 30 | ``` 31 | 32 | Create the Kubernetes Service Account 33 | ``` 34 | kubectl apply -f service_account.yaml 35 | ``` 36 | 37 | Create the config map 38 | ``` 39 | kubectl apply -f config.yaml 40 | ``` 41 | 42 | Deploy the agent 43 | ``` 44 | kubectl apply -f deployment.yaml 45 | ``` 46 | 47 | ## Validate 48 | 49 | Log into Google Cloud Logging 50 | 51 | ![Events](./assets/events.png) 52 | -------------------------------------------------------------------------------- /operator/builtin/output/stdout/stdout.go: -------------------------------------------------------------------------------- 1 | package stdout 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "io" 7 | "os" 8 | "sync" 9 | 10 | "github.com/observiq/stanza/entry" 11 | "github.com/observiq/stanza/operator" 12 | "github.com/observiq/stanza/operator/helper" 13 | ) 14 | 15 | // Stdout is a global handle to standard output 16 | var Stdout io.Writer = os.Stdout 17 | 18 | func init() { 19 | operator.Register("stdout", func() operator.Builder { return NewStdoutConfig("") }) 20 | } 21 | 22 | // NewStdoutConfig creates a new stdout config with default values 23 | func NewStdoutConfig(operatorID string) *StdoutConfig { 24 | return &StdoutConfig{ 25 | OutputConfig: helper.NewOutputConfig(operatorID, "stdout"), 26 | } 27 | } 28 | 29 | // StdoutConfig is the configuration of the Stdout operator 30 | type StdoutConfig struct { 31 | helper.OutputConfig `yaml:",inline"` 32 | } 33 | 34 | // Build will build a stdout operator. 35 | func (c StdoutConfig) Build(context operator.BuildContext) ([]operator.Operator, error) { 36 | outputOperator, err := c.OutputConfig.Build(context) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | op := &StdoutOperator{ 42 | OutputOperator: outputOperator, 43 | encoder: json.NewEncoder(Stdout), 44 | } 45 | return []operator.Operator{op}, nil 46 | } 47 | 48 | // StdoutOperator is an operator that logs entries using stdout. 49 | type StdoutOperator struct { 50 | helper.OutputOperator 51 | encoder *json.Encoder 52 | mux sync.Mutex 53 | } 54 | 55 | // Process will log entries received. 56 | func (o *StdoutOperator) Process(_ context.Context, entry *entry.Entry) error { 57 | o.mux.Lock() 58 | err := o.encoder.Encode(entry) 59 | if err != nil { 60 | o.mux.Unlock() 61 | o.Errorf("Failed to process entry: %s, $s", err, entry.Record) 62 | return err 63 | } 64 | o.mux.Unlock() 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /agent/config.go: -------------------------------------------------------------------------------- 1 | package agent 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "path/filepath" 7 | 8 | "github.com/observiq/stanza/pipeline" 9 | yaml "gopkg.in/yaml.v2" 10 | ) 11 | 12 | // Config is the configuration of the stanza log agent. 13 | type Config struct { 14 | Pipeline pipeline.Config `json:"pipeline" yaml:"pipeline"` 15 | } 16 | 17 | // NewConfigFromFile will create a new agent config from a YAML file. 18 | func NewConfigFromFile(file string) (*Config, error) { 19 | contents, err := ioutil.ReadFile(file) // #nosec - configs load based on user specified directory 20 | if err != nil { 21 | return nil, fmt.Errorf("could not find config file: %s", err) 22 | } 23 | 24 | config := Config{} 25 | if err := yaml.UnmarshalStrict(contents, &config); err != nil { 26 | return nil, fmt.Errorf("failed to read config file as yaml: %s", err) 27 | } 28 | 29 | return &config, nil 30 | } 31 | 32 | // NewConfigFromGlobs will create an agent config from multiple files matching a pattern. 33 | func NewConfigFromGlobs(globs []string) (*Config, error) { 34 | paths := make([]string, 0, len(globs)) 35 | for _, glob := range globs { 36 | matches, err := filepath.Glob(glob) 37 | if err != nil { 38 | return nil, err 39 | } 40 | paths = append(paths, matches...) 41 | } 42 | 43 | if len(paths) == 0 { 44 | return nil, fmt.Errorf("No config files found") 45 | } 46 | 47 | config := &Config{} 48 | for _, path := range paths { 49 | newConfig, err := NewConfigFromFile(path) 50 | if err != nil { 51 | return nil, fmt.Errorf("failed to load config from %s: %s", path, err) 52 | } 53 | 54 | config = mergeConfigs(config, newConfig) 55 | } 56 | 57 | return config, nil 58 | } 59 | 60 | // mergeConfigs will merge two agent configs. 61 | func mergeConfigs(dst *Config, src *Config) *Config { 62 | dst.Pipeline = append(dst.Pipeline, src.Pipeline...) 63 | return dst 64 | } 65 | -------------------------------------------------------------------------------- /docs/examples/k8s/onprem/README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes On Prem Logs and Events w/ Google Cloud Logging 2 | 3 | Stanza can be deployed to Kubernetes On Prem for log and event collection. Container logs 4 | are gathered from each Kubernetes Node's filesystem. Events are collected from the Kubernetes 5 | API Server. 6 | 7 | ## Architecture 8 | 9 | 1. Service account with permission to the Kubernetes API server 10 | 2. Config map: Contains the Stanza configurations 11 | 3. Credentials secret: Contains Google Cloud [service account credentials JSON file](https://cloud.google.com/docs/authentication/getting-started) 12 | 4. Persistent volume: Allows the Stanza events agent database to persist between restarts and pod evictions 13 | 5. Statefulset: A single replica statefulset for reading Kubernetes events 14 | 6. Daemonset: For reading logs from each Kubernetes node 15 | 16 | ## Prerequisites 17 | 18 | 1. Google Cloud account with Cloud Logging API enabled 19 | 2. Google service account with [roles/logging.logWriter](https://cloud.google.com/logging/docs/access-control) 20 | 3. Kubernetes Cluster with a storageclass capable of providing persistent volumes 21 | 4. Edit `agent.yaml`'s configmap (at the top) to include: 22 | - Your cluster name: an arbitrary value that will be added to each log entry as a label 23 | 24 | ## Deployment Steps 25 | 26 | Create the credentials secret. Download your Google service accounts JSON key and name it `log_credentials.json`. 27 | **NOTE**: The file name `log_credentials.json` is required, as that will be the name of the key that is referenced 28 | when mounting the secret. 29 | ```bash 30 | kubectl create secret generic stanza-agent-credentials \ 31 | --from-file=log_credentials.json 32 | ``` 33 | 34 | Deploy Stanza 35 | ```bash 36 | kubectl apply -f agent.yaml 37 | ``` 38 | 39 | ## Validate 40 | 41 | Log into Google Cloud Logging 42 | 43 | ![Events](./assets/entries.png) 44 | -------------------------------------------------------------------------------- /docs/examples/k8s/aks/README.md: -------------------------------------------------------------------------------- 1 | # Auzre Kubernetes Service Logs and Events w/ Google Cloud Logging 2 | 3 | Stanza can be deployed to Auzre Kubernetes Service for log and event collection. Container logs 4 | are gathered from each Kubernetes Node's filesystem. Events are collected from the Kubernetes 5 | API Server. 6 | 7 | ## Architecture 8 | 9 | 1. Service account with permission to the Kubernetes API server 10 | 2. Config map: Contains the Stanza configurations 11 | 3. Credentials secret: Contains Google Cloud [service account credentials JSON file](https://cloud.google.com/docs/authentication/getting-started) 12 | 4. Persistent volume: Allows the Stanza events agent database to persist between restarts and pod evictions 13 | 5. Statefulset: A single replica statefulset for reading Kubernetes events 14 | 6. Daemonset: For reading logs from each Kubernetes node 15 | 16 | ## Prerequisites 17 | 18 | 1. Google Cloud account with Cloud Logging API enabled 19 | 2. Google service account with [roles/logging.logWriter](https://cloud.google.com/logging/docs/access-control) 20 | 3. Kubernetes Cluster with a storageclass capable of providing persistent volumes 21 | 4. Edit `agent.yaml`'s configmap (at the top) to include: 22 | - Your cluster name: an arbitrary value that will be added to each log entry as a label 23 | 24 | ## Deployment Steps 25 | 26 | Create the credentials secret. Download your Google service accounts JSON key and name it `log_credentials.json`. 27 | **NOTE**: The file name `log_credentials.json` is required, as that will be the name of the key that is referenced 28 | when mounting the secret. 29 | ```bash 30 | kubectl create secret generic stanza-agent-credentials \ 31 | --from-file=log_credentials.json 32 | ``` 33 | 34 | Deploy Stanza 35 | ```bash 36 | kubectl apply -f agent.yaml 37 | ``` 38 | 39 | ## Validate 40 | 41 | Log into Google Cloud Logging 42 | 43 | ![Events](./assets/entries.png) 44 | -------------------------------------------------------------------------------- /operator/builtin/input/azure/event_hub_parse.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "reflect" 5 | "time" 6 | 7 | azhub "github.com/Azure/azure-event-hubs-go/v3" 8 | "github.com/mitchellh/mapstructure" 9 | "github.com/observiq/stanza/entry" 10 | ) 11 | 12 | // ParseEvent parses an Azure Event Hub event as an Entry. 13 | func ParseEvent(event azhub.Event, e *entry.Entry) error { 14 | promoteTime(event, e) 15 | return parse(event, e) 16 | } 17 | 18 | // parse parses an Azure Event Hub event into a map 19 | func parse(event azhub.Event, e *entry.Entry) error { 20 | m := make(map[string]interface{}) 21 | 22 | if len(event.Data) != 0 { 23 | m["message"] = string(event.Data) 24 | } 25 | 26 | if event.PartitionKey != nil { 27 | m["partition_key"] = event.PartitionKey 28 | } 29 | 30 | if event.Properties != nil { 31 | m["properties"] = event.Properties 32 | } 33 | 34 | // promote event.ID to resource.event_id 35 | if event.ID != "" { 36 | e.AddResourceKey("event_id", event.ID) 37 | } 38 | 39 | sysProp := make(map[string]interface{}) 40 | if event.SystemProperties != nil { 41 | if err := mapstructure.Decode(event.SystemProperties, &sysProp); err != nil { 42 | return err 43 | } 44 | for key := range sysProp { 45 | if sysProp[key] == nil || reflect.ValueOf(sysProp[key]).IsNil() { 46 | delete(sysProp, key) 47 | } 48 | } 49 | m["system_properties"] = sysProp 50 | } 51 | 52 | e.Record = m 53 | return nil 54 | } 55 | 56 | // promoteTime promotes an Azure Event Hub event's timestamp 57 | // EnqueuedTime takes precedence over IoTHubEnqueuedTime 58 | func promoteTime(event azhub.Event, e *entry.Entry) { 59 | timestamps := []*time.Time{ 60 | event.SystemProperties.EnqueuedTime, 61 | event.SystemProperties.IoTHubEnqueuedTime, 62 | } 63 | 64 | for _, t := range timestamps { 65 | if t != nil && !t.IsZero() { 66 | e.Timestamp = *t 67 | return 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /operator/builtin/input/file/fingerprint.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "os" 8 | ) 9 | 10 | const defaultFingerprintSize = 1000 // bytes 11 | const minFingerprintSize = 16 // bytes 12 | 13 | // Fingerprint is used to identify a file 14 | // A file's fingerprint is the first N bytes of the file, 15 | // where N is the fingerprintSize on the file_input operator 16 | type Fingerprint struct { 17 | // FirstBytes represents the first N bytes of a file 18 | FirstBytes []byte 19 | } 20 | 21 | // NewFingerprint creates a new fingerprint from an open file 22 | func (f *InputOperator) NewFingerprint(file *os.File) (*Fingerprint, error) { 23 | buf := make([]byte, f.fingerprintSize) 24 | 25 | n, err := file.ReadAt(buf, 0) 26 | if err != nil && err != io.EOF { 27 | return nil, fmt.Errorf("reading fingerprint bytes: %s", err) 28 | } 29 | 30 | fp := &Fingerprint{ 31 | FirstBytes: buf[:n], 32 | } 33 | 34 | return fp, nil 35 | } 36 | 37 | // Copy creates a new copy of the fingerprint 38 | func (f Fingerprint) Copy() *Fingerprint { 39 | buf := make([]byte, len(f.FirstBytes), cap(f.FirstBytes)) 40 | n := copy(buf, f.FirstBytes) 41 | return &Fingerprint{ 42 | FirstBytes: buf[:n], 43 | } 44 | } 45 | 46 | // StartsWith returns true if the fingerprints are the same 47 | // or if the new fingerprint starts with the old one 48 | // This is important functionality for tracking new files, 49 | // since their initial size is typically less than that of 50 | // a fingerprint. As the file grows, its fingerprint is updated 51 | // until it reaches a maximum size, as configured on the operator 52 | func (f Fingerprint) StartsWith(old *Fingerprint) bool { 53 | l0 := len(old.FirstBytes) 54 | if l0 == 0 { 55 | return false 56 | } 57 | l1 := len(f.FirstBytes) 58 | if l0 > l1 { 59 | return false 60 | } 61 | return bytes.Equal(old.FirstBytes[:l0], f.FirstBytes[:l0]) 62 | } 63 | -------------------------------------------------------------------------------- /docs/examples/k8s/eks/README.md: -------------------------------------------------------------------------------- 1 | # Elastic Kubernetes Service Logs and Events w/ Google Cloud Logging 2 | 3 | Stanza can be deployed to Elastic Kubernetes Service for log and event collection. Container logs 4 | are gathered from each Kubernetes Node's filesystem. Events are collected from the Kubernetes 5 | API Server. 6 | 7 | ## Architecture 8 | 9 | 1. Service account with permission to the Kubernetes API server 10 | 2. Config map: Contains the Stanza configurations 11 | 3. Credentials secret: Contains Google Cloud [service account credentials JSON file](https://cloud.google.com/docs/authentication/getting-started) 12 | 4. Persistent volume: Allows the Stanza events agent database to persist between restarts and pod evictions 13 | 5. Statefulset: A single replica statefulset for reading Kubernetes events 14 | 6. Daemonset: For reading logs from each Kubernetes node 15 | 16 | ## Prerequisites 17 | 18 | 1. Google Cloud account with Cloud Logging API enabled 19 | 2. Google service account with [roles/logging.logWriter](https://cloud.google.com/logging/docs/access-control) 20 | 3. Kubernetes Cluster with a storageclass capable of providing persistent volumes 21 | 4. Edit `agent.yaml`'s configmap (at the top) to include: 22 | - Your cluster name: an arbitrary value that will be added to each log entry as a label 23 | 24 | ## Deployment Steps 25 | 26 | Create the credentials secret. Download your Google service accounts JSON key and name it `log_credentials.json`. 27 | **NOTE**: The file name `log_credentials.json` is required, as that will be the name of the key that is referenced 28 | when mounting the secret. 29 | ```bash 30 | kubectl create secret generic stanza-agent-credentials \ 31 | --from-file=log_credentials.json 32 | ``` 33 | 34 | Deploy Stanza 35 | ```bash 36 | kubectl apply -f agent.yaml 37 | ``` 38 | 39 | ## Validate 40 | 41 | Log into Google Cloud Logging 42 | 43 | ![Events](./assets/entries.png) 44 | -------------------------------------------------------------------------------- /docs/operators/generate_input.md: -------------------------------------------------------------------------------- 1 | ## `generate_input` operator 2 | 3 | The `generate_input` operator generates log entries with a static record. This is useful for testing pipelines, especially when 4 | coupled with the [`rate_limit`](/docs/operators/rate_limit.md) operator. 5 | 6 | ### Configuration Fields 7 | 8 | | Field | Default | Description | 9 | | --- | --- | --- | 10 | | `id` | `generate_input` | A unique identifier for the operator | 11 | | `output` | Next in pipeline | The connected operator(s) that will receive all outbound entries | 12 | | `write_to` | $ | A [field](/docs/types/field.md) that will be set to the path of the file the entry was read from | 13 | | `entry` | | A [entry](/docs/types/entry.md) log entry to repeatedly generate | 14 | | `count` | 0 | The number of entries to generate before stopping. A value of 0 indicates unlimited | 15 | | `static` | `false` | If true, the timestamp of the entry will remain static after each invocation | 16 | 17 | ### Example Configurations 18 | 19 | #### Mock a file input 20 | 21 | Configuration: 22 | ```yaml 23 | - type: generate_input 24 | entry: 25 | record: 26 | message1: log1 27 | message2: log2 28 | ``` 29 | 30 | Output records: 31 | ```json 32 | { 33 | "record": { 34 | "message1": "log1", 35 | "message2": "log2" 36 | }, 37 | }, 38 | { 39 | "record": { 40 | "message1": "log1", 41 | "message2": "log2" 42 | }, 43 | }, 44 | ... 45 | ``` 46 | -------------------------------------------------------------------------------- /docs/MIRRORS.md: -------------------------------------------------------------------------------- 1 | # Creating & Using Mirrors for Stanza Releases 2 | 3 | ## Creating a Mirror 4 | Mirrors for Stanza can come in two forms: 5 | 1. Hosted websites 6 | 2. Local filesystem mirrors 7 | 8 | The only requirements for either are creating a directory layout that 9 | mirrors that of the GitHub releases, such as in the visualization below. 10 | It is suggested to use an automated synchronization process to manage 11 | keeping ths up to date, including rewriting the symlink for the latest 12 | to the highest version number. 13 | 14 | ### Mirror Tree Visualization 15 | ➜ stanza_mirror tree 16 | . 17 | ├── download 18 | │ └── v1.2.12 19 | │ ├── stanza-plugins.tar.gz 20 | │ ├── stanza-plugins.zip 21 | │ ├── stanza_darwin_amd64 22 | │ ├── stanza_linux_amd64 23 | │ ├── stanza_linux_arm64 24 | │ ├── stanza_windows_amd64 25 | │ ├── unix-install.sh 26 | │ ├── version.json 27 | │ └── windows-install.ps1 28 | └── latest 29 | └── download 30 | ├── stanza-plugins.tar.gz 31 | ├── stanza-plugins.zip 32 | ├── stanza_darwin_amd64 33 | ├── stanza_linux_amd64 34 | ├── stanza_linux_arm64 35 | ├── stanza_windows_amd64 36 | ├── unix-install.sh 37 | ├── version.json 38 | └── windows-install.ps1 39 | 40 | ## Usage Syntax with the Install Script 41 | 42 | ### Web URL 43 | ```shell 44 | # Latest 45 | ./unix-install -l http://dl.example.com/some/path 46 | # Specific Version 1.2.12 47 | ./unix-install -l http://dl.example.com/some/path -v 1.2.12 48 | ``` 49 | 50 | ### File URL 51 | ```shell 52 | ./unix-install -l file:///Users/username/Downloads/stanza_local 53 | # Specific Version 1.2.12 54 | ./unix-install -l file:///Users/username/Downloads/stanza_local -v 1.2.12 55 | ``` 56 | 57 | ## Further Information 58 | For further usage information, and other supported flags, please see the [Quick Start Guide](README.md) 59 | -------------------------------------------------------------------------------- /database/database.go: -------------------------------------------------------------------------------- 1 | //go:generate mockery --name=^(Database)$ --output=../testutil --outpkg=testutil --case=snake 2 | 3 | package database 4 | 5 | import ( 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "time" 10 | 11 | "go.etcd.io/bbolt" 12 | ) 13 | 14 | // Database is a database used to save offsets 15 | type Database interface { 16 | Close() error 17 | Sync() error 18 | Update(func(*bbolt.Tx) error) error 19 | View(func(*bbolt.Tx) error) error 20 | } 21 | 22 | // StubDatabase is an implementation of Database that 23 | // succeeds on all calls without persisting anything to disk. 24 | // This is used when --database is unspecified. 25 | type StubDatabase struct{} 26 | 27 | // Close will be ignored by the stub database 28 | func (d *StubDatabase) Close() error { return nil } 29 | 30 | // Sync will be ignored by the stub database 31 | func (d *StubDatabase) Sync() error { return nil } 32 | 33 | // Update will be ignored by the stub database 34 | func (d *StubDatabase) Update(func(tx *bbolt.Tx) error) error { return nil } 35 | 36 | // View will be ignored by the stub database 37 | func (d *StubDatabase) View(func(tx *bbolt.Tx) error) error { return nil } 38 | 39 | // NewStubDatabase creates a new StubDatabase 40 | func NewStubDatabase() *StubDatabase { 41 | return &StubDatabase{} 42 | } 43 | 44 | // OpenDatabase will open and create a database 45 | func OpenDatabase(file string) (Database, error) { 46 | if file == "" { 47 | return NewStubDatabase(), nil 48 | } 49 | 50 | if _, err := os.Stat(filepath.Dir(file)); err != nil { 51 | if os.IsNotExist(err) { 52 | err := os.MkdirAll(filepath.Dir(file), 0755) // #nosec - 0755 directory permissions are okay 53 | if err != nil { 54 | return nil, fmt.Errorf("creating database directory: %s", err) 55 | } 56 | } else { 57 | return nil, err 58 | } 59 | } 60 | 61 | options := &bbolt.Options{Timeout: 1 * time.Second} 62 | return bbolt.Open(file, 0600, options) 63 | } 64 | -------------------------------------------------------------------------------- /operator/helper/operatortest/operatortest.go: -------------------------------------------------------------------------------- 1 | // Copyright The OpenTelemetry Authors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package operatortest 16 | 17 | import ( 18 | "fmt" 19 | "io/ioutil" 20 | "path" 21 | "testing" 22 | 23 | "github.com/stretchr/testify/require" 24 | "gopkg.in/yaml.v2" 25 | ) 26 | 27 | // ConfigUnmarshalTest is used for testing golden configs 28 | type ConfigUnmarshalTest struct { 29 | Name string 30 | Expect interface{} 31 | ExpectErr bool 32 | } 33 | 34 | func configFromFileViaYaml(file string, config interface{}) error { 35 | bytes, err := ioutil.ReadFile(file) // #nosec - configs load based on user specified directory 36 | if err != nil { 37 | return fmt.Errorf("could not find config file: %s", err) 38 | } 39 | if err := yaml.Unmarshal(bytes, config); err != nil { 40 | return fmt.Errorf("failed to read config file as yaml: %s", err) 41 | } 42 | 43 | return nil 44 | } 45 | 46 | // Run Unmarshalls yaml files and compares them against the expected. 47 | func (c ConfigUnmarshalTest) Run(t *testing.T, config interface{}) { 48 | yamlConfig := config 49 | yamlErr := configFromFileViaYaml(path.Join(".", "testdata", fmt.Sprintf("%s.yaml", c.Name)), yamlConfig) 50 | 51 | if c.ExpectErr { 52 | require.Error(t, yamlErr) 53 | } else { 54 | require.NoError(t, yamlErr) 55 | require.Equal(t, c.Expect, yamlConfig) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /operator/builtin/parser/xml/element.go: -------------------------------------------------------------------------------- 1 | package xml 2 | 3 | import ( 4 | "bytes" 5 | "encoding/xml" 6 | ) 7 | 8 | // Element represents an XML element 9 | type Element struct { 10 | Tag string 11 | Content string 12 | Attributes map[string]interface{} 13 | Children []*Element 14 | Parent *Element 15 | } 16 | 17 | // convertToMap converts an element to a map 18 | func convertToMap(element *Element) map[string]interface{} { 19 | results := map[string]interface{}{} 20 | results["tag"] = element.Tag 21 | 22 | if element.Content != "" { 23 | results["content"] = element.Content 24 | } 25 | 26 | if len(element.Attributes) > 0 { 27 | results["attributes"] = element.Attributes 28 | } 29 | 30 | if len(element.Children) > 0 { 31 | results["children"] = convertToMaps(element.Children) 32 | } 33 | 34 | return results 35 | } 36 | 37 | // convertToMaps converts a slice of elements to a slice of maps 38 | func convertToMaps(elements []*Element) []map[string]interface{} { 39 | results := []map[string]interface{}{} 40 | for _, e := range elements { 41 | results = append(results, convertToMap(e)) 42 | } 43 | 44 | return results 45 | } 46 | 47 | // newElement creates a new element for the given xml start element 48 | func newElement(element xml.StartElement) *Element { 49 | return &Element{ 50 | Tag: element.Name.Local, 51 | Attributes: getAttributes(element), 52 | } 53 | } 54 | 55 | // getAttributes returns the attributes of the given element 56 | func getAttributes(element xml.StartElement) map[string]interface{} { 57 | if len(element.Attr) == 0 { 58 | return nil 59 | } 60 | 61 | attributes := map[string]interface{}{} 62 | for _, attr := range element.Attr { 63 | key := attr.Name.Local 64 | attributes[key] = attr.Value 65 | } 66 | 67 | return attributes 68 | } 69 | 70 | // getValue returns value of the given char data 71 | func getValue(data xml.CharData) string { 72 | return string(bytes.TrimSpace(data)) 73 | } 74 | -------------------------------------------------------------------------------- /docs/operators/stanza_input.md: -------------------------------------------------------------------------------- 1 | ## `stanza_input` operator 2 | 3 | The `stanza_input` operator acts as a source for Stanza's internal logs. It copies the logs rather than consumes them, so Stanza will still write to its log file or to stdout, depending on how it is configured. 4 | 5 | Care should be taken when doing any additional processing of logs coming from the `stanza_input` operator because errors from downstream processing will be passed back through the `stanza_input` operator, which can cause an infinite error loop. 6 | 7 | ### Configuration Fields 8 | 9 | | Field | Default | Description | 10 | | --- | --- | --- | 11 | | `id` | `stanza_input` | A unique identifier for the operator | 12 | | `output` | Next in pipeline | The connected operator(s) that will receive all outbound entries | 13 | | `buffer_size` | 100 | The number of entries to buffer before dropping entries because we aren't processing fast enough | 14 | 15 | 16 | ### Example Configurations 17 | 18 | #### Simple Stanza input 19 | 20 | Configuration: 21 | ```yaml 22 | - type: stanza_input 23 | ``` 24 | 25 | Sample entry output: 26 | ```yaml 27 | { 28 | "timestamp": "2020-11-06T13:55:11.314283-05:00", 29 | "severity": 60, 30 | "record": { 31 | "action": "send", 32 | "entry": { 33 | "timestamp": "2020-11-06T13:55:11.314057-05:00", 34 | "severity": 0, 35 | "record": "{\"key\":\"value\"" 36 | }, 37 | "error": "ReadMapCB: expect }, but found \u0000, error found in #10 byte of ...|y\":\"value\"|..., bigger context ...|{\"key\":\"value\"|...", 38 | "message": "Failed to process entry" 39 | } 40 | } 41 | ``` 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /operator/builtin/parser/severity/severity.go: -------------------------------------------------------------------------------- 1 | package severity 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/observiq/stanza/entry" 7 | "github.com/observiq/stanza/operator" 8 | "github.com/observiq/stanza/operator/helper" 9 | ) 10 | 11 | func init() { 12 | operator.Register("severity_parser", func() operator.Builder { return NewSeverityParserConfig("") }) 13 | } 14 | 15 | // NewSeverityParserConfig creates a new severity parser config with default values 16 | func NewSeverityParserConfig(operatorID string) *SeverityParserConfig { 17 | return &SeverityParserConfig{ 18 | TransformerConfig: helper.NewTransformerConfig(operatorID, "severity_parser"), 19 | SeverityParserConfig: helper.NewSeverityParserConfig(), 20 | } 21 | } 22 | 23 | // SeverityParserConfig is the configuration of a severity parser operator. 24 | type SeverityParserConfig struct { 25 | helper.TransformerConfig `yaml:",inline"` 26 | helper.SeverityParserConfig `yaml:",omitempty,inline"` 27 | } 28 | 29 | // Build will build a time parser operator. 30 | func (c SeverityParserConfig) Build(context operator.BuildContext) ([]operator.Operator, error) { 31 | transformerOperator, err := c.TransformerConfig.Build(context) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | severityParser, err := c.SeverityParserConfig.Build(context) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | severityOperator := &SeverityParserOperator{ 42 | TransformerOperator: transformerOperator, 43 | SeverityParser: severityParser, 44 | } 45 | 46 | return []operator.Operator{severityOperator}, nil 47 | } 48 | 49 | // SeverityParserOperator is an operator that parses time from a field to an entry. 50 | type SeverityParserOperator struct { 51 | helper.TransformerOperator 52 | helper.SeverityParser 53 | } 54 | 55 | // Process will parse time from an entry. 56 | func (p *SeverityParserOperator) Process(ctx context.Context, entry *entry.Entry) error { 57 | return p.ProcessWith(ctx, entry, p.Parse) 58 | } 59 | -------------------------------------------------------------------------------- /cmd/stanza/service.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/signal" 7 | "syscall" 8 | 9 | "github.com/kardianos/service" 10 | "github.com/observiq/stanza/agent" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | // AgentService is a service that runs the stanza agent. 15 | type AgentService struct { 16 | cancel context.CancelFunc 17 | agent *agent.LogAgent 18 | } 19 | 20 | // Start will start the stanza agent. 21 | func (a *AgentService) Start(_ service.Service) error { 22 | a.agent.Info("Starting stanza agent") 23 | if err := a.agent.Start(); err != nil { 24 | a.agent.Errorw("Failed to start stanza agent", zap.Any("error", err)) 25 | a.cancel() 26 | return nil 27 | } 28 | 29 | a.agent.Info("Stanza agent started") 30 | return nil 31 | } 32 | 33 | // Stop will stop the stanza agent. 34 | func (a *AgentService) Stop(_ service.Service) error { 35 | a.agent.Info("Stopping stanza agent") 36 | if err := a.agent.Stop(); err != nil { 37 | a.agent.Errorw("Failed to stop stanza agent gracefully", zap.Any("error", err)) 38 | a.cancel() 39 | return nil 40 | } 41 | 42 | a.agent.Info("Stanza agent stopped") 43 | a.cancel() 44 | return nil 45 | } 46 | 47 | // newAgentService creates a new agent service with the provided agent. 48 | func newAgentService(ctx context.Context, agent *agent.LogAgent, cancel context.CancelFunc) (service.Service, error) { 49 | agentService := &AgentService{cancel, agent} 50 | config := &service.Config{ 51 | Name: "stanza", 52 | DisplayName: "Stanza Log Agent", 53 | Description: "Monitors and processes log entries", 54 | Option: service.KeyValue{ 55 | "RunWait": func() { 56 | var sigChan = make(chan os.Signal, 3) 57 | signal.Notify(sigChan, syscall.SIGTERM, os.Interrupt) 58 | select { 59 | case <-sigChan: 60 | case <-ctx.Done(): 61 | } 62 | }, 63 | }, 64 | } 65 | 66 | service, err := service.New(agentService, config) 67 | if err != nil { 68 | return nil, err 69 | } 70 | 71 | return service, nil 72 | } 73 | -------------------------------------------------------------------------------- /operator/builtin/parser/time/time.go: -------------------------------------------------------------------------------- 1 | package time 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/observiq/stanza/entry" 7 | "github.com/observiq/stanza/operator" 8 | "github.com/observiq/stanza/operator/helper" 9 | ) 10 | 11 | func init() { 12 | operator.Register("time_parser", func() operator.Builder { return NewTimeParserConfig("") }) 13 | } 14 | 15 | // NewTimeParserConfig creates a new time parser config with default values 16 | func NewTimeParserConfig(operatorID string) *TimeParserConfig { 17 | return &TimeParserConfig{ 18 | TransformerConfig: helper.NewTransformerConfig(operatorID, "time_parser"), 19 | TimeParser: helper.NewTimeParser(), 20 | } 21 | } 22 | 23 | // TimeParserConfig is the configuration of a time parser operator. 24 | type TimeParserConfig struct { 25 | helper.TransformerConfig `yaml:",inline"` 26 | helper.TimeParser `yaml:",omitempty,inline"` 27 | } 28 | 29 | // Build will build a time parser operator. 30 | func (c TimeParserConfig) Build(context operator.BuildContext) ([]operator.Operator, error) { 31 | transformerOperator, err := c.TransformerConfig.Build(context) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | if err := c.TimeParser.Validate(context); err != nil { 37 | return nil, err 38 | } 39 | 40 | timeParser := &TimeParserOperator{ 41 | TransformerOperator: transformerOperator, 42 | TimeParser: c.TimeParser, 43 | } 44 | 45 | return []operator.Operator{timeParser}, nil 46 | } 47 | 48 | // TimeParserOperator is an operator that parses time from a field to an entry. 49 | type TimeParserOperator struct { 50 | helper.TransformerOperator 51 | helper.TimeParser 52 | } 53 | 54 | // CanOutput will always return true for a parser operator. 55 | func (t *TimeParserOperator) CanOutput() bool { 56 | return true 57 | } 58 | 59 | // Process will parse time from an entry. 60 | func (t *TimeParserOperator) Process(ctx context.Context, entry *entry.Entry) error { 61 | return t.ProcessWith(ctx, entry, t.TimeParser.Parse) 62 | } 63 | -------------------------------------------------------------------------------- /operator/builtin/output/googlecloud/severity_test.go: -------------------------------------------------------------------------------- 1 | package googlecloud 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/observiq/stanza/entry" 7 | "github.com/stretchr/testify/require" 8 | sev "google.golang.org/genproto/googleapis/logging/type" 9 | ) 10 | 11 | func TestConvertSeverity(t *testing.T) { 12 | testCases := []struct { 13 | name string 14 | severity entry.Severity 15 | expectedSeverity sev.LogSeverity 16 | }{ 17 | { 18 | name: "above emergency", 19 | severity: entry.Emergency + 1, 20 | expectedSeverity: sev.LogSeverity_EMERGENCY, 21 | }, 22 | { 23 | name: "above alert", 24 | severity: entry.Alert + 1, 25 | expectedSeverity: sev.LogSeverity_ALERT, 26 | }, 27 | { 28 | name: "above critical", 29 | severity: entry.Critical + 1, 30 | expectedSeverity: sev.LogSeverity_CRITICAL, 31 | }, 32 | { 33 | name: "above error", 34 | severity: entry.Error + 1, 35 | expectedSeverity: sev.LogSeverity_ERROR, 36 | }, 37 | { 38 | name: "above warning", 39 | severity: entry.Warning + 1, 40 | expectedSeverity: sev.LogSeverity_WARNING, 41 | }, 42 | { 43 | name: "above notice", 44 | severity: entry.Notice + 1, 45 | expectedSeverity: sev.LogSeverity_NOTICE, 46 | }, 47 | { 48 | name: "above info", 49 | severity: entry.Info + 1, 50 | expectedSeverity: sev.LogSeverity_INFO, 51 | }, 52 | { 53 | name: "above debug", 54 | severity: entry.Debug + 1, 55 | expectedSeverity: sev.LogSeverity_DEBUG, 56 | }, 57 | { 58 | name: "unknown", 59 | severity: -1, 60 | expectedSeverity: sev.LogSeverity_DEFAULT, 61 | }, 62 | } 63 | 64 | for _, tc := range testCases { 65 | t.Run(tc.name, func(t *testing.T) { 66 | result := convertSeverity(tc.severity) 67 | require.Equal(t, tc.expectedSeverity, result) 68 | }) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /operator/helper/duration.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | // Duration is the representation of a length of time 10 | type Duration struct { 11 | time.Duration 12 | } 13 | 14 | // NewDuration creates a new duration from a time 15 | func NewDuration(t time.Duration) Duration { 16 | return Duration{ 17 | Duration: t, 18 | } 19 | } 20 | 21 | // Raw will return the raw duration, without modification 22 | func (d *Duration) Raw() time.Duration { 23 | return d.Duration 24 | } 25 | 26 | // MarshalJSON will marshal the duration as a json string 27 | func (d Duration) MarshalJSON() ([]byte, error) { 28 | return []byte(`"` + d.Duration.String() + `"`), nil 29 | } 30 | 31 | // UnmarshalJSON will unmarshal json as a duration 32 | func (d *Duration) UnmarshalJSON(raw []byte) error { 33 | var v interface{} 34 | err := json.Unmarshal(raw, &v) 35 | if err != nil { 36 | return err 37 | } 38 | d.Duration, err = durationFromInterface(v) 39 | return err 40 | } 41 | 42 | // MarshalYAML will marshal the duration as a yaml string 43 | func (d Duration) MarshalYAML() (interface{}, error) { 44 | return d.Duration.String(), nil 45 | } 46 | 47 | // UnmarshalYAML will unmarshal yaml as a duration 48 | func (d *Duration) UnmarshalYAML(unmarshal func(interface{}) error) error { 49 | var v interface{} 50 | err := unmarshal(&v) 51 | if err != nil { 52 | return err 53 | } 54 | d.Duration, err = durationFromInterface(v) 55 | if d.Duration < 0 { 56 | d.Duration *= -1 57 | } 58 | return err 59 | } 60 | 61 | func durationFromInterface(val interface{}) (time.Duration, error) { 62 | switch value := val.(type) { 63 | case float64: 64 | return time.Duration(value * float64(time.Second)), nil 65 | case int: 66 | return time.Duration(value) * time.Second, nil 67 | case string: 68 | var err error 69 | d, err := time.ParseDuration(value) 70 | return d, err 71 | default: 72 | return 0, fmt.Errorf("cannot unmarshal value of type %T into a duration", val) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /cmd/stanza/graph.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/observiq/stanza/agent" 7 | "github.com/observiq/stanza/database" 8 | "github.com/observiq/stanza/operator" 9 | "github.com/observiq/stanza/plugin" 10 | "github.com/spf13/cobra" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | // GraphFlags are the flags that can be supplied when running the graph command 15 | type GraphFlags struct { 16 | *RootFlags 17 | } 18 | 19 | // NewGraphCommand creates a command for printing the pipeline as a graph 20 | func NewGraphCommand(rootFlags *RootFlags) *cobra.Command { 21 | return &cobra.Command{ 22 | Use: "graph", 23 | Args: cobra.NoArgs, 24 | Short: "Export a dot-formatted representation of the operator graph", 25 | Run: func(command *cobra.Command, args []string) { runGraph(command, args, rootFlags) }, 26 | } 27 | } 28 | 29 | func runGraph(_ *cobra.Command, _ []string, flags *RootFlags) { 30 | logger := newLogger(*flags).Sugar() 31 | defer func() { 32 | _ = logger.Sync() 33 | }() 34 | 35 | cfg, err := agent.NewConfigFromGlobs(flags.ConfigFiles) 36 | if err != nil { 37 | logger.Errorw("Failed to read configs from glob", zap.Any("error", err)) 38 | os.Exit(1) 39 | } 40 | 41 | if errs := plugin.RegisterPlugins(flags.PluginDir, operator.DefaultRegistry); len(errs) != 0 { 42 | logger.Errorw("Got errors parsing parsing", "errors", err) 43 | } 44 | 45 | buildContext := operator.NewBuildContext(database.NewStubDatabase(), logger) 46 | pipeline, err := cfg.Pipeline.BuildPipeline(buildContext, nil) 47 | if err != nil { 48 | logger.Errorw("Failed to build operator pipeline", zap.Any("error", err)) 49 | os.Exit(1) 50 | } 51 | 52 | dotGraph, err := pipeline.Render() 53 | if err != nil { 54 | logger.Errorw("Failed to marshal dot graph", zap.Any("error", err)) 55 | os.Exit(1) 56 | } 57 | 58 | dotGraph = append(dotGraph, '\n') 59 | _, err = stdout.Write(dotGraph) 60 | if err != nil { 61 | logger.Errorw("Failed to write dot graph to stdout", zap.Any("error", err)) 62 | os.Exit(1) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /operator/builtin/output/forward/forward_test.go: -------------------------------------------------------------------------------- 1 | package forward 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "io/ioutil" 7 | "net/http" 8 | "net/http/httptest" 9 | "testing" 10 | "time" 11 | 12 | "github.com/observiq/stanza/entry" 13 | "github.com/observiq/stanza/operator/buffer" 14 | "github.com/observiq/stanza/operator/helper" 15 | "github.com/observiq/stanza/testutil" 16 | "github.com/stretchr/testify/require" 17 | ) 18 | 19 | func TestForwardOutput(t *testing.T) { 20 | received := make(chan []byte, 1) 21 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 22 | body, _ := ioutil.ReadAll(req.Body) 23 | received <- body 24 | })) 25 | 26 | cfg := NewForwardOutputConfig("test") 27 | memoryCfg := buffer.NewMemoryBufferConfig() 28 | memoryCfg.MaxChunkDelay = helper.NewDuration(50 * time.Millisecond) 29 | cfg.BufferConfig = buffer.Config{ 30 | Builder: memoryCfg, 31 | } 32 | cfg.Address = srv.URL 33 | 34 | ops, err := cfg.Build(testutil.NewBuildContext(t)) 35 | require.NoError(t, err) 36 | forwardOutput := ops[0].(*ForwardOutput) 37 | 38 | newEntry := entry.New() 39 | newEntry.Record = "test" 40 | newEntry.Timestamp = newEntry.Timestamp.Round(time.Second) 41 | require.NoError(t, forwardOutput.Start()) 42 | defer forwardOutput.Stop() 43 | require.NoError(t, forwardOutput.Process(context.Background(), newEntry)) 44 | 45 | select { 46 | case <-time.After(time.Second): 47 | require.FailNow(t, "Timed out waiting for server to receive entry") 48 | case body := <-received: 49 | var entries []*entry.Entry 50 | require.NoError(t, json.Unmarshal(body, &entries)) 51 | require.Len(t, entries, 1) 52 | e := entries[0] 53 | require.True(t, newEntry.Timestamp.Equal(e.Timestamp)) 54 | require.Equal(t, newEntry.Record, e.Record) 55 | require.Equal(t, newEntry.Severity, e.Severity) 56 | require.Equal(t, newEntry.SeverityText, e.SeverityText) 57 | require.Equal(t, newEntry.Labels, e.Labels) 58 | require.Equal(t, newEntry.Resource, e.Resource) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /docs/operators/README.md: -------------------------------------------------------------------------------- 1 | ## What is an operator? 2 | An operator is the most basic unit of log processing. Each operator fulfills a single responsibility, such as reading lines from a file, or parsing JSON from a field. Operators are then chained together in a pipeline to achieve a desired result. 3 | 4 | For instance, a user may read lines from a file using the `file_input` operator. From there, the results of this operation may be sent to a `regex_parser` operator that creates fields based on a regex pattern. And then finally, these results may be sent to a `elastic_output` operator that writes each line to Elasticsearch. 5 | 6 | 7 | ## What operators are available? 8 | 9 | Inputs: 10 | - [File](/docs/operators/file_input.md) 11 | - [Windows Event Log](/docs/operators/windows_eventlog_input.md) 12 | - [TCP](/docs/operators/tcp_input.md) 13 | - [UDP](/docs/operators/udp_input.md) 14 | - [Journald](/docs/operators/journald_input.md) 15 | - [Generate](/docs/operators/generate_input.md) 16 | 17 | Parsers: 18 | - [CSV](/docs/operators/csv_parser.md) 19 | - [JSON](/docs/operators/json_parser.md) 20 | - [Regex](/docs/operators/regex_parser.md) 21 | - [Syslog](/docs/operators/syslog_parser.md) 22 | - [Severity](/docs/operators/severity_parser.md) 23 | - [Time](/docs/operators/time_parser.md) 24 | - [XML](/docs/operators/xml_parser.md) 25 | 26 | Outputs: 27 | - [Google Cloud Logging](/docs/operators/google_cloud_output.md) 28 | - [Elasticsearch](/docs/operators/elastic_output.md) 29 | - [Stdout](/docs/operators/stdout.md) 30 | - [File](/docs/operators/file_output.md) 31 | 32 | General purpose: 33 | - [Rate Limit](/docs/operators/rate_limit.md) 34 | - [Filter](/docs/operators/filter.md) 35 | - [Router](/docs/operators/router.md) 36 | - [Metadata](/docs/operators/metadata.md) 37 | - [Restructure](/docs/operators/restructure.md) 38 | - [Host Metadata](/docs/operators/host_metadata.md) 39 | - [Kubernetes Metadata Decorator](/docs/operators/k8s_metadata_decorator.md) 40 | 41 | Or create your own [plugins](/docs/plugins.md) for a technology-specific use case. 42 | -------------------------------------------------------------------------------- /operator/builtin/input/windows/publisher_test.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package windows 4 | 5 | import ( 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestPublisherOpenPreexisting(t *testing.T) { 12 | publisher := Publisher{handle: 5} 13 | err := publisher.Open("") 14 | require.Error(t, err) 15 | require.Contains(t, err.Error(), "publisher handle is already open") 16 | } 17 | 18 | func TestPublisherOpenInvalidUTF8(t *testing.T) { 19 | publisher := NewPublisher() 20 | invalidUTF8 := "\u0000" 21 | err := publisher.Open(invalidUTF8) 22 | require.Error(t, err) 23 | require.Contains(t, err.Error(), "failed to convert provider to utf16") 24 | } 25 | 26 | func TestPublisherOpenSyscallFailure(t *testing.T) { 27 | publisher := NewPublisher() 28 | provider := "provider" 29 | openPublisherMetadataProc = SimpleMockProc(0, 0, ErrorNotSupported) 30 | err := publisher.Open(provider) 31 | require.Error(t, err) 32 | require.Contains(t, err.Error(), "failed to open publisher handle") 33 | } 34 | 35 | func TestPublisherOpenSuccess(t *testing.T) { 36 | publisher := NewPublisher() 37 | provider := "provider" 38 | openPublisherMetadataProc = SimpleMockProc(5, 0, ErrorSuccess) 39 | err := publisher.Open(provider) 40 | require.NoError(t, err) 41 | require.Equal(t, uintptr(5), publisher.handle) 42 | } 43 | 44 | func TestPublisherCloseWhenAlreadyClosed(t *testing.T) { 45 | publisher := NewPublisher() 46 | err := publisher.Close() 47 | require.NoError(t, err) 48 | } 49 | 50 | func TestPublisherCloseSyscallFailure(t *testing.T) { 51 | publisher := Publisher{handle: 5} 52 | closeProc = SimpleMockProc(0, 0, ErrorNotSupported) 53 | err := publisher.Close() 54 | require.Error(t, err) 55 | require.Contains(t, err.Error(), "failed to close publisher") 56 | } 57 | 58 | func TestPublisherCloseSuccess(t *testing.T) { 59 | publisher := Publisher{handle: 5} 60 | closeProc = SimpleMockProc(1, 0, ErrorSuccess) 61 | err := publisher.Close() 62 | require.NoError(t, err) 63 | require.Equal(t, uintptr(0), publisher.handle) 64 | } 65 | -------------------------------------------------------------------------------- /operator/builtin/input/azure/eventhub/event_hub.go: -------------------------------------------------------------------------------- 1 | package eventhub 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/observiq/stanza/operator" 7 | "github.com/observiq/stanza/operator/builtin/input/azure" 8 | "github.com/observiq/stanza/operator/helper" 9 | ) 10 | 11 | const operatorName = "azure_event_hub_input" 12 | 13 | func init() { 14 | operator.Register(operatorName, func() operator.Builder { return NewEventHubConfig("") }) 15 | } 16 | 17 | // NewEventHubConfig creates a new Azure Event Hub input config with default values 18 | func NewEventHubConfig(operatorID string) *EventHubInputConfig { 19 | return &EventHubInputConfig{ 20 | InputConfig: helper.NewInputConfig(operatorID, operatorName), 21 | AzureConfig: azure.AzureConfig{ 22 | PrefetchCount: 1000, 23 | StartAt: "end", 24 | }, 25 | } 26 | } 27 | 28 | // EventHubInputConfig is the configuration of a Azure Event Hub input operator. 29 | type EventHubInputConfig struct { 30 | helper.InputConfig `yaml:",inline"` 31 | azure.AzureConfig `yaml:",inline"` 32 | } 33 | 34 | // Build will build a Azure Event Hub input operator. 35 | func (c *EventHubInputConfig) Build(buildContext operator.BuildContext) ([]operator.Operator, error) { 36 | if err := c.AzureConfig.Build(buildContext, c.InputConfig); err != nil { 37 | return nil, err 38 | } 39 | 40 | eventHubInput := &EventHubInput{ 41 | EventHub: azure.EventHub{ 42 | AzureConfig: c.AzureConfig, 43 | Persist: &azure.Persister{ 44 | DB: helper.NewScopedDBPersister(buildContext.Database, c.ID()), 45 | }, 46 | }, 47 | } 48 | return []operator.Operator{eventHubInput}, nil 49 | } 50 | 51 | // EventHubInput is an operator that reads input from Azure Event Hub. 52 | type EventHubInput struct { 53 | azure.EventHub 54 | } 55 | 56 | // Start will start generating log entries. 57 | func (e *EventHubInput) Start() error { 58 | e.Handler = e.handleEvent 59 | return e.StartConsumers(context.Background()) 60 | } 61 | 62 | // Stop will stop generating logs. 63 | func (e *EventHubInput) Stop() error { 64 | return e.StopConsumers() 65 | } 66 | -------------------------------------------------------------------------------- /operator/helper/labeler_test.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/observiq/stanza/entry" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestLabeler(t *testing.T) { 12 | os.Setenv("TEST_METADATA_PLUGIN_ENV", "foo") 13 | defer os.Unsetenv("TEST_METADATA_PLUGIN_ENV") 14 | 15 | cases := []struct { 16 | name string 17 | config LabelerConfig 18 | input *entry.Entry 19 | expected *entry.Entry 20 | }{ 21 | { 22 | "AddLabelLiteral", 23 | func() LabelerConfig { 24 | cfg := NewLabelerConfig() 25 | cfg.Labels = map[string]ExprStringConfig{ 26 | "label1": "value1", 27 | } 28 | return cfg 29 | }(), 30 | entry.New(), 31 | func() *entry.Entry { 32 | e := entry.New() 33 | e.Labels = map[string]string{ 34 | "label1": "value1", 35 | } 36 | return e 37 | }(), 38 | }, 39 | { 40 | "AddLabelExpr", 41 | func() LabelerConfig { 42 | cfg := NewLabelerConfig() 43 | cfg.Labels = map[string]ExprStringConfig{ 44 | "label1": `EXPR("start" + "end")`, 45 | } 46 | return cfg 47 | }(), 48 | entry.New(), 49 | func() *entry.Entry { 50 | e := entry.New() 51 | e.Labels = map[string]string{ 52 | "label1": "startend", 53 | } 54 | return e 55 | }(), 56 | }, 57 | { 58 | "AddLabelEnv", 59 | func() LabelerConfig { 60 | cfg := NewLabelerConfig() 61 | cfg.Labels = map[string]ExprStringConfig{ 62 | "label1": `EXPR(env("TEST_METADATA_PLUGIN_ENV"))`, 63 | } 64 | return cfg 65 | }(), 66 | entry.New(), 67 | func() *entry.Entry { 68 | e := entry.New() 69 | e.Labels = map[string]string{ 70 | "label1": "foo", 71 | } 72 | return e 73 | }(), 74 | }, 75 | } 76 | 77 | for _, tc := range cases { 78 | t.Run(tc.name, func(t *testing.T) { 79 | labeler, err := tc.config.Build() 80 | require.NoError(t, err) 81 | 82 | err = labeler.Label(tc.input) 83 | require.NoError(t, err) 84 | require.Equal(t, tc.expected.Labels, tc.input.Labels) 85 | }) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /testutil/util.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/observiq/stanza/logger" 11 | "github.com/observiq/stanza/operator" 12 | "go.etcd.io/bbolt" 13 | "go.uber.org/zap/zapcore" 14 | "go.uber.org/zap/zaptest" 15 | ) 16 | 17 | // NewTempDir will return a new temp directory for testing 18 | func NewTempDir(t testing.TB) string { 19 | tempDir, err := ioutil.TempDir("", "") 20 | if err != nil { 21 | t.Errorf("%v", err) 22 | t.FailNow() 23 | } 24 | 25 | t.Cleanup(func() { 26 | if err := os.RemoveAll(tempDir); err != nil { 27 | t.Errorf("%v", err) 28 | } 29 | }) 30 | 31 | return tempDir 32 | } 33 | 34 | // NewTestDatabase will return a new database for testing 35 | func NewTestDatabase(t testing.TB) *bbolt.DB { 36 | tempDir, err := ioutil.TempDir("", "") 37 | if err != nil { 38 | t.Errorf("%v", err) 39 | t.FailNow() 40 | } 41 | 42 | t.Cleanup(func() { 43 | if err := os.RemoveAll(tempDir); err != nil { 44 | t.Errorf("%v", err) 45 | } 46 | }) 47 | 48 | db, err := bbolt.Open(filepath.Join(tempDir, "test.db"), 0666, nil) 49 | if err != nil { 50 | t.Errorf("%v", err) 51 | t.FailNow() 52 | } 53 | 54 | t.Cleanup(func() { 55 | if err := db.Close(); err != nil { 56 | t.Errorf("%v", err) 57 | } 58 | }) 59 | 60 | return db 61 | } 62 | 63 | // NewBuildContext will return a new build context for testing 64 | func NewBuildContext(t testing.TB) operator.BuildContext { 65 | return operator.BuildContext{ 66 | Database: NewTestDatabase(t), 67 | Logger: logger.New(zaptest.NewLogger(t, zaptest.Level(zapcore.ErrorLevel)).Sugar()), 68 | Namespace: "$", 69 | } 70 | } 71 | 72 | // Trim removes white space from the lines of a string 73 | func Trim(s string) string { 74 | lines := strings.Split(s, "\n") 75 | trimmed := make([]string, 0, len(lines)) 76 | for _, line := range lines { 77 | if len(line) == 0 { 78 | continue 79 | } 80 | trimmed = append(trimmed, strings.Trim(line, " \t\n")) 81 | } 82 | 83 | return strings.Join(trimmed, "\n") 84 | } 85 | -------------------------------------------------------------------------------- /cmd/stanza/offsets_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | 10 | "github.com/observiq/stanza/database" 11 | "github.com/observiq/stanza/operator/helper" 12 | "github.com/stretchr/testify/require" 13 | "go.etcd.io/bbolt" 14 | ) 15 | 16 | func TestOffsets(t *testing.T) { 17 | tempDir, err := ioutil.TempDir("", "") 18 | require.NoError(t, err) 19 | defer os.RemoveAll(tempDir) 20 | 21 | databasePath := filepath.Join(tempDir, "logagent.db") 22 | configPath := filepath.Join(tempDir, "config.yaml") 23 | ioutil.WriteFile(configPath, []byte{}, 0666) 24 | 25 | // capture stdout 26 | buf := bytes.NewBuffer([]byte{}) 27 | stdout = buf 28 | 29 | // add an offset to the database 30 | db, err := database.OpenDatabase(databasePath) 31 | require.NoError(t, err) 32 | db.Update(func(tx *bbolt.Tx) error { 33 | bucket, err := tx.CreateBucketIfNotExists(helper.OffsetsBucket) 34 | require.NoError(t, err) 35 | 36 | _, err = bucket.CreateBucket([]byte("$.testoperatorid1")) 37 | require.NoError(t, err) 38 | _, err = bucket.CreateBucket([]byte("$.testoperatorid2")) 39 | require.NoError(t, err) 40 | return nil 41 | }) 42 | db.Close() 43 | 44 | // check that offsets list actually lists the operator 45 | offsetsList := NewRootCmd() 46 | offsetsList.SetArgs([]string{ 47 | "offsets", "list", 48 | "--database", databasePath, 49 | "--config", configPath, 50 | }) 51 | 52 | err = offsetsList.Execute() 53 | require.NoError(t, err) 54 | require.Equal(t, "$.testoperatorid1\n$.testoperatorid2\n", buf.String()) 55 | 56 | // clear the offsets 57 | offsetsClear := NewRootCmd() 58 | offsetsClear.SetArgs([]string{ 59 | "offsets", "clear", 60 | "--database", databasePath, 61 | "--config", configPath, 62 | "$.testoperatorid2", 63 | }) 64 | 65 | err = offsetsClear.Execute() 66 | require.NoError(t, err) 67 | 68 | // Check that offsets list only shows uncleared operator id 69 | buf.Reset() 70 | err = offsetsList.Execute() 71 | require.NoError(t, err) 72 | require.Equal(t, "$.testoperatorid1\n", buf.String()) 73 | } 74 | --------------------------------------------------------------------------------