├── benchmarks ├── benchmarks │ ├── no_mappings │ │ └── benchmark.toml │ ├── match_all_rule │ │ ├── match_everything.rule │ │ ├── simple.pipeline │ │ └── benchmark.toml │ ├── many_stages_match_all │ │ ├── match_everything.rule │ │ ├── benchmark.toml │ │ └── simple.pipeline │ ├── complex_when_no_action │ │ ├── simple.pipeline │ │ ├── complex_when.rule │ │ └── benchmark.toml │ └── grok_extract │ │ ├── simple.pipeline │ │ ├── benchmark.toml │ │ └── grok_job_extraction.rule ├── src │ └── main │ │ ├── resources │ │ ├── META-INF │ │ │ └── services │ │ │ │ └── org.openjdk.jmh.profile.Profiler │ │ └── git.properties │ │ └── assembly │ │ └── tarball.xml └── scripts │ └── benchmark-filtered.sh ├── plugin ├── src │ ├── main │ │ ├── resources │ │ │ ├── META-INF │ │ │ │ └── services │ │ │ │ │ └── org.graylog2.plugin.Plugin │ │ │ └── org.graylog.plugins.graylog-plugin-pipeline-processor │ │ │ │ └── graylog-plugin.properties │ │ └── java │ │ │ └── org │ │ │ └── graylog │ │ │ └── plugins │ │ │ └── pipelineprocessor │ │ │ ├── codegen │ │ │ ├── GeneratedRule.java │ │ │ ├── PipelineClassloader.java │ │ │ └── compiler │ │ │ │ ├── PipelineCompilationException.java │ │ │ │ └── JavaSourceFromString.java │ │ │ ├── PipelineConfig.java │ │ │ ├── events │ │ │ ├── LegacyDefaultStreamMigrated.java │ │ │ └── PipelineConnectionsChangedEvent.java │ │ │ ├── functions │ │ │ ├── dates │ │ │ │ ├── periods │ │ │ │ │ ├── Days.java │ │ │ │ │ ├── Hours.java │ │ │ │ │ ├── Weeks.java │ │ │ │ │ ├── Millis.java │ │ │ │ │ ├── Months.java │ │ │ │ │ ├── Minutes.java │ │ │ │ │ ├── Seconds.java │ │ │ │ │ ├── Years.java │ │ │ │ │ ├── PeriodParseFunction.java │ │ │ │ │ └── AbstractPeriodComponentFunction.java │ │ │ │ ├── DateConversion.java │ │ │ │ └── Now.java │ │ │ ├── syslog │ │ │ │ ├── SyslogPriority.java │ │ │ │ └── SyslogPriorityAsString.java │ │ │ ├── hashing │ │ │ │ ├── MD5.java │ │ │ │ ├── SHA1.java │ │ │ │ ├── SHA256.java │ │ │ │ ├── SHA512.java │ │ │ │ ├── CRC32.java │ │ │ │ ├── CRC32C.java │ │ │ │ ├── Murmur3_128.java │ │ │ │ └── Murmur3_32.java │ │ │ ├── strings │ │ │ │ ├── Lowercase.java │ │ │ │ ├── Uppercase.java │ │ │ │ ├── Capitalize.java │ │ │ │ ├── Uncapitalize.java │ │ │ │ └── Swapcase.java │ │ │ └── encoding │ │ │ │ ├── Base16Encode.java │ │ │ │ ├── Base64Encode.java │ │ │ │ ├── Base16Decode.java │ │ │ │ ├── Base32Decode.java │ │ │ │ ├── Base32Encode.java │ │ │ │ ├── Base64Decode.java │ │ │ │ ├── Base64UrlDecode.java │ │ │ │ ├── Base64UrlEncode.java │ │ │ │ ├── Base32HumanEncode.java │ │ │ │ └── Base32HumanDecode.java │ │ │ ├── db │ │ │ ├── memory │ │ │ │ ├── InMemoryServicesModule.java │ │ │ │ └── InMemoryPipelineStreamConnectionsService.java │ │ │ ├── PipelineService.java │ │ │ ├── RuleService.java │ │ │ ├── PipelineStreamConnectionsService.java │ │ │ └── mongodb │ │ │ │ └── MongoDbServicesModule.java │ │ │ ├── parser │ │ │ ├── errors │ │ │ │ ├── InvalidOperation.java │ │ │ │ ├── UndeclaredFunction.java │ │ │ │ ├── NonIndexableType.java │ │ │ │ ├── UndeclaredVariable.java │ │ │ │ ├── IncompatibleType.java │ │ │ │ ├── OptionalParametersMustBeNamed.java │ │ │ │ ├── IncompatibleIndexType.java │ │ │ │ ├── IncompatibleTypes.java │ │ │ │ ├── MissingRequiredParam.java │ │ │ │ ├── SyntaxError.java │ │ │ │ ├── WrongNumberOfArgs.java │ │ │ │ └── ParseError.java │ │ │ ├── ParseException.java │ │ │ └── FunctionRegistry.java │ │ │ ├── ast │ │ │ ├── expressions │ │ │ │ ├── LogicalExpression.java │ │ │ │ ├── NumericExpression.java │ │ │ │ ├── BaseExpression.java │ │ │ │ ├── StringExpression.java │ │ │ │ ├── ConstantExpression.java │ │ │ │ ├── BooleanExpression.java │ │ │ │ ├── BinaryExpression.java │ │ │ │ ├── NotExpression.java │ │ │ │ ├── OrExpression.java │ │ │ │ ├── AndExpression.java │ │ │ │ ├── LongExpression.java │ │ │ │ ├── DoubleExpression.java │ │ │ │ ├── FieldRefExpression.java │ │ │ │ ├── MessageRefExpression.java │ │ │ │ └── ArrayLiteralExpression.java │ │ │ ├── statements │ │ │ │ ├── Statement.java │ │ │ │ ├── FunctionStatement.java │ │ │ │ └── VarAssignStatement.java │ │ │ ├── exceptions │ │ │ │ ├── LocationAwareEvalException.java │ │ │ │ ├── PrecomputeFailure.java │ │ │ │ └── FunctionEvaluationException.java │ │ │ └── functions │ │ │ │ └── AbstractFunction.java │ │ │ ├── rest │ │ │ ├── BulkRuleRequest.java │ │ │ └── PipelineReverseConnections.java │ │ │ ├── simulator │ │ │ └── PipelineInterpreterTrace.java │ │ │ ├── PipelineProcessorPlugin.java │ │ │ ├── processors │ │ │ └── listeners │ │ │ │ └── InterpreterListener.java │ │ │ └── audit │ │ │ └── PipelineProcessorAuditEventTypes.java │ ├── test │ │ ├── resources │ │ │ ├── org │ │ │ │ └── graylog │ │ │ │ │ └── plugins │ │ │ │ │ └── pipelineprocessor │ │ │ │ │ ├── parser │ │ │ │ │ ├── issue185.txt │ │ │ │ │ ├── invalidDateAddition.txt │ │ │ │ │ ├── undeclaredFunction.txt │ │ │ │ │ ├── declaredFunction.txt │ │ │ │ │ ├── booleanNot.txt │ │ │ │ │ ├── undeclaredIdentifier.txt │ │ │ │ │ ├── invalidArgumentValue.txt │ │ │ │ │ ├── basicRule.txt │ │ │ │ │ ├── booleanValuedFunctionAsCondition.txt │ │ │ │ │ ├── inferVariableType.txt │ │ │ │ │ ├── optionalParamsMustBeNamed.txt │ │ │ │ │ ├── singleArgFunction.txt │ │ │ │ │ ├── messageRefQuotedField.txt │ │ │ │ │ ├── typedFieldAccess.txt │ │ │ │ │ ├── indexedAccessWrongIndexType.txt │ │ │ │ │ ├── optionalArguments.txt │ │ │ │ │ ├── indexedAccessWrongType.txt │ │ │ │ │ ├── positionalArguments.txt │ │ │ │ │ ├── indexedAccess.txt │ │ │ │ │ ├── mismatchedNumericTypes.txt │ │ │ │ │ ├── messageRef.txt │ │ │ │ │ ├── mapArrayLiteral.txt │ │ │ │ │ ├── invalidArgType.txt │ │ │ │ │ ├── pipelineDeclaration.txt │ │ │ │ │ ├── arithmetic.txt │ │ │ │ │ ├── nestedFieldAccess.txt │ │ │ │ │ └── dateArithmetic.txt │ │ │ │ │ ├── functions │ │ │ │ │ ├── routeToStream.txt │ │ │ │ │ ├── removeFromStream.txt │ │ │ │ │ ├── clonedMessageWithInvalidTimestamp.txt │ │ │ │ │ ├── ipMatchingIssue28.txt │ │ │ │ │ ├── routeToStreamRemoveDefault.txt │ │ │ │ │ ├── fieldRenaming.txt │ │ │ │ │ ├── evalErrorSuppressed.txt │ │ │ │ │ ├── split.txt │ │ │ │ │ ├── keyValueFailure.txt │ │ │ │ │ ├── removeFromStreamRetainDefault.txt │ │ │ │ │ ├── grok.txt │ │ │ │ │ ├── datesUnixTimestamps.txt │ │ │ │ │ ├── substring.txt │ │ │ │ │ ├── ipMatching.txt │ │ │ │ │ ├── newlyCreatedMessage.txt │ │ │ │ │ ├── json.txt │ │ │ │ │ ├── timezones.txt │ │ │ │ │ ├── clonedMessage.txt │ │ │ │ │ ├── regexMatch.txt │ │ │ │ │ ├── digests.txt │ │ │ │ │ ├── keyValue.txt │ │ │ │ │ ├── urls.txt │ │ │ │ │ ├── jsonpath.txt │ │ │ │ │ ├── fieldPrefixSuffix.txt │ │ │ │ │ ├── encodings.txt │ │ │ │ │ ├── strings.txt │ │ │ │ │ ├── comparisons.txt │ │ │ │ │ └── dateArithmetic.txt │ │ │ │ │ └── codegen │ │ │ │ │ └── runCodegen.txt │ │ │ └── log4j2-test.xml │ │ └── java │ │ │ └── org │ │ │ └── graylog │ │ │ └── plugins │ │ │ └── pipelineprocessor │ │ │ ├── functions │ │ │ └── messages │ │ │ │ └── StreamCacheServiceTest.java │ │ │ └── parser │ │ │ └── CodegenPipelineRuleParserTest.java │ ├── web │ │ ├── simulator │ │ │ ├── SimulatorActions.js │ │ │ ├── SimulationResults.css │ │ │ ├── SimulationTrace.css │ │ │ ├── SimulationChanges.css │ │ │ ├── SimulationTrace.jsx │ │ │ ├── SimulatorStore.js │ │ │ └── SimulationPreview.jsx │ │ ├── pipelines │ │ │ ├── PipelineConnectionsActions.js │ │ │ ├── PipelinesActions.jsx │ │ │ ├── Pipeline.css │ │ │ ├── ProcessingTimelineComponent.css │ │ │ ├── NewPipeline.jsx │ │ │ ├── PipelineConnectionsList.jsx │ │ │ ├── PipelineDetails.jsx │ │ │ └── PipelinesOverviewPage.jsx │ │ ├── rules │ │ │ ├── RuleForm.css │ │ │ ├── RuleHelper.css │ │ │ ├── RulesActions.jsx │ │ │ └── RulesComponent.jsx │ │ ├── logic │ │ │ └── SourceGenerator.js │ │ └── index.jsx │ └── deb │ │ └── control │ │ └── control ├── .eslintrc ├── build.config.js ├── webpack.config.js └── package.json ├── .travis.yml ├── .github └── ISSUE_TEMPLATE.md ├── CONTRIBUTING.md └── .gitignore /benchmarks/benchmarks/no_mappings/benchmark.toml: -------------------------------------------------------------------------------- 1 | name = "Empty" -------------------------------------------------------------------------------- /benchmarks/benchmarks/match_all_rule/match_everything.rule: -------------------------------------------------------------------------------- 1 | rule "match everything" 2 | when 3 | true 4 | then 5 | end -------------------------------------------------------------------------------- /benchmarks/benchmarks/many_stages_match_all/match_everything.rule: -------------------------------------------------------------------------------- 1 | rule "match everything" 2 | when 3 | true 4 | then 5 | end -------------------------------------------------------------------------------- /benchmarks/benchmarks/complex_when_no_action/simple.pipeline: -------------------------------------------------------------------------------- 1 | pipeline "simple" 2 | stage 0 match all 3 | rule "complex when" 4 | end -------------------------------------------------------------------------------- /benchmarks/benchmarks/grok_extract/simple.pipeline: -------------------------------------------------------------------------------- 1 | pipeline "grok" 2 | stage 0 match all 3 | rule "grok jenkins extraction" 4 | end -------------------------------------------------------------------------------- /benchmarks/benchmarks/match_all_rule/simple.pipeline: -------------------------------------------------------------------------------- 1 | pipeline "simple" 2 | stage 0 match all 3 | rule "match everything" 4 | end -------------------------------------------------------------------------------- /plugin/src/main/resources/META-INF/services/org.graylog2.plugin.Plugin: -------------------------------------------------------------------------------- 1 | org.graylog.plugins.pipelineprocessor.PipelineProcessorPlugin 2 | -------------------------------------------------------------------------------- /benchmarks/src/main/resources/META-INF/services/org.openjdk.jmh.profile.Profiler: -------------------------------------------------------------------------------- 1 | org.graylog.benchmarks.pipeline.PipelinePerformanceBenchmarks$MetricsProfiler -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/issue185.txt: -------------------------------------------------------------------------------- 1 | rule "issue-185" 2 | when 3 | true 4 | then 5 | let a = "\s+$" 6 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidDateAddition.txt: -------------------------------------------------------------------------------- 1 | rule "cannot add dates" 2 | when 3 | now() + now() == now() 4 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/undeclaredFunction.txt: -------------------------------------------------------------------------------- 1 | rule "undeclared function" 2 | when false == unknown() 3 | then 4 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/declaredFunction.txt: -------------------------------------------------------------------------------- 1 | rule "using declared function 'nein'" 2 | when true == nein() 3 | then 4 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/booleanNot.txt: -------------------------------------------------------------------------------- 1 | rule "booleanNot" 2 | when 3 | !false == false 4 | then 5 | trigger_test(); 6 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/undeclaredIdentifier.txt: -------------------------------------------------------------------------------- 1 | rule "undeclared variable" 2 | when true 3 | then 4 | one_arg(one: x); 5 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidArgumentValue.txt: -------------------------------------------------------------------------------- 1 | rule "invalid arg" 2 | when now_in_tz("123") // this isn't a valid tz 3 | then 4 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/routeToStream.txt: -------------------------------------------------------------------------------- 1 | rule "stream routing" 2 | when true 3 | then 4 | route_to_stream(name: "some name"); 5 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/removeFromStream.txt: -------------------------------------------------------------------------------- 1 | rule "stream routing" 2 | when true 3 | then 4 | remove_from_stream(name: "some name"); 5 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/basicRule.txt: -------------------------------------------------------------------------------- 1 | rule "something" 2 | when double_valued_func() > 1.0d AND false == true 3 | then 4 | double_valued_func(); 5 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/booleanValuedFunctionAsCondition.txt: -------------------------------------------------------------------------------- 1 | rule "bool function as top level" 2 | when doch() 3 | then 4 | trigger_test(); 5 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/inferVariableType.txt: -------------------------------------------------------------------------------- 1 | rule "infer" 2 | when true 3 | then 4 | let x = one_arg("string"); 5 | one_arg(x); 6 | end 7 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/optionalParamsMustBeNamed.txt: -------------------------------------------------------------------------------- 1 | rule "optionalParamsMustBeNamed" 2 | when 3 | optional(false, "string", 3) 4 | then 5 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/singleArgFunction.txt: -------------------------------------------------------------------------------- 1 | rule "single arg" 2 | when one_arg("arg") == one_arg(one: "arg") 3 | then 4 | trigger_test(); 5 | end -------------------------------------------------------------------------------- /plugin/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "babel-eslint", 3 | "ecmaFeatures": { 4 | "classes": true, 5 | "jsx": true, 6 | }, 7 | "extends": [ 8 | "graylog", 9 | ], 10 | } 11 | 12 | -------------------------------------------------------------------------------- /benchmarks/benchmarks/grok_extract/benchmark.toml: -------------------------------------------------------------------------------- 1 | name = "Grok extraction benchmark" 2 | 3 | [[streams]] 4 | name = "default" 5 | description = "All incoming messages" 6 | pipelines = ["grok"] 7 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/clonedMessageWithInvalidTimestamp.txt: -------------------------------------------------------------------------------- 1 | rule "operate on cloned message" 2 | when true 3 | then 4 | let cloned = clone_message(); 5 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatchingIssue28.txt: -------------------------------------------------------------------------------- 1 | rule "IP subnet" 2 | when 3 | cidr_match("10.20.30.0/24", to_ip($message.source)) 4 | then 5 | trigger_test(); 6 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRefQuotedField.txt: -------------------------------------------------------------------------------- 1 | rule "test" 2 | when to_string($message.`@specialfieldname`, "empty") == "string" 3 | then 4 | trigger_test(); 5 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/typedFieldAccess.txt: -------------------------------------------------------------------------------- 1 | rule "typed field access" 2 | when 3 | to_long(customObject("1").id, 0) < 2 4 | then 5 | trigger_test(); 6 | end 7 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccessWrongIndexType.txt: -------------------------------------------------------------------------------- 1 | rule "indexed array and map access" 2 | when 3 | ["first"][true] == "first" 4 | then 5 | trigger_test(); 6 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/optionalArguments.txt: -------------------------------------------------------------------------------- 1 | rule "optional function arguments" 2 | when 3 | optional(d: 3, a: true, b: "string") 4 | then 5 | trigger_test(); 6 | end -------------------------------------------------------------------------------- /plugin/src/web/simulator/SimulatorActions.js: -------------------------------------------------------------------------------- 1 | import Reflux from 'reflux'; 2 | 3 | const SimulatorActions = Reflux.createActions({ 4 | simulate: { asyncResult: true }, 5 | }); 6 | 7 | export default SimulatorActions; 8 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/routeToStreamRemoveDefault.txt: -------------------------------------------------------------------------------- 1 | rule "stream routing" 2 | when true 3 | then 4 | route_to_stream(name: "some name", remove_from_default: true); 5 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccessWrongType.txt: -------------------------------------------------------------------------------- 1 | rule "indexed array and map access" 2 | when 3 | one_arg("not an array")[0] == "first" 4 | then 5 | trigger_test(); 6 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/positionalArguments.txt: -------------------------------------------------------------------------------- 1 | rule "positional args" 2 | when concat("a", 1, true) == concat(one: "a", two: 1, three: true) 3 | then 4 | trigger_test(); 5 | end -------------------------------------------------------------------------------- /plugin/src/web/simulator/SimulationResults.css: -------------------------------------------------------------------------------- 1 | .message-preview-wrapper { 2 | margin-left: 15px; 3 | margin-right: 15px; 4 | } 5 | 6 | .message-preview-wrapper dl { 7 | margin-top: 5px; 8 | margin-bottom: 0; 9 | } -------------------------------------------------------------------------------- /benchmarks/benchmarks/complex_when_no_action/complex_when.rule: -------------------------------------------------------------------------------- 1 | rule "complex when" 2 | when 3 | has_field("message") == (1 < 2) && 2 < 3 && (true == true == true == true == true == true == true == true != false) && !false == false 4 | then 5 | end -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | dist: trusty 3 | language: java 4 | jdk: 5 | - oraclejdk8 6 | install: 7 | - mvn install -DskipTests=true -Dmaven.javadoc.skip=true -Dskip.web.build=true -B -V 8 | script: 9 | - mvn test -Dskip.web.build=true -B 10 | -------------------------------------------------------------------------------- /plugin/src/deb/control/control: -------------------------------------------------------------------------------- 1 | Package: [[name]] 2 | Version: [[version]] 3 | Architecture: all 4 | Maintainer: Kay Roepke 5 | Section: web 6 | Priority: optional 7 | Depends: graylog-server | graylog-radio 8 | Description: [[description]] 9 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/indexedAccess.txt: -------------------------------------------------------------------------------- 1 | rule "indexed array and map access" 2 | when 3 | ["first","second"][0] == "first" and {third: "a value"}["third"] == "a value" 4 | then 5 | trigger_test(); 6 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/mismatchedNumericTypes.txt: -------------------------------------------------------------------------------- 1 | rule "incompatible numeric types inference" 2 | when 3 | 1.0 + 10 == 11 // error: no automatic long -> double conversion! 4 | then 5 | trigger_test(); 6 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/messageRef.txt: -------------------------------------------------------------------------------- 1 | rule "message field ref" 2 | when to_long(value: $message.responseCode, default: 200) >= 500 3 | then 4 | set_field(field: "response_category", value: "server_error"); 5 | end 6 | -------------------------------------------------------------------------------- /plugin/src/web/simulator/SimulationTrace.css: -------------------------------------------------------------------------------- 1 | .dl-simulation-trace { 2 | padding-top: 15px; 3 | } 4 | 5 | .dl-simulation-trace dt { 6 | width: 80px; 7 | } 8 | 9 | .dl-simulation-trace dd { 10 | margin-left: 100px; 11 | margin-bottom: 5px; 12 | } 13 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/mapArrayLiteral.txt: -------------------------------------------------------------------------------- 1 | rule "mapliteral" 2 | when sort(keys({some_identifier: 1, `something with spaces`: "some expression"})) == ["some_identifier", "something with spaces"] 3 | then 4 | trigger_test(); 5 | end -------------------------------------------------------------------------------- /plugin/build.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = { 4 | // Make sure that this is the correct path to the web interface part of the Graylog server repository. 5 | web_src_path: path.resolve(__dirname, '../../graylog2-server/graylog2-web-interface'), 6 | }; -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/fieldRenaming.txt: -------------------------------------------------------------------------------- 1 | rule "fieldRenaming" 2 | when true 3 | then 4 | 5 | rename_field("no_such_field", "field_1"); 6 | rename_field("field_a", "field_2"); 7 | rename_field("field_b", "field_b"); 8 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/evalErrorSuppressed.txt: -------------------------------------------------------------------------------- 1 | rule "suppressing exceptions/nulls" 2 | when 3 | is_null(to_ip($message.does_not_exist, "d.f.f.f")) && is_not_null($message.this_field_was_set) 4 | then 5 | trigger_test(); 6 | end 7 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Problem description 2 | 3 | ### Steps to reproduce the problem 4 | 5 | 1. ... 6 | 7 | ### Environment 8 | 9 | * Graylog Version: 10 | * Pipeline Processor plugin version: 11 | * Elasticsearch Version: 12 | * MongoDB Version: 13 | * Operating System: 14 | * Browser version: 15 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/invalidArgType.txt: -------------------------------------------------------------------------------- 1 | rule "invalid arg type" 2 | when one_arg(0d) == "0" // one_arg needs a String argument, but 0d is Double 3 | then 4 | let x = double_valued_func(); 5 | one_arg(x); // this needs a String argument, but x resolves to Double 6 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/split.txt: -------------------------------------------------------------------------------- 1 | rule "split" 2 | when 3 | true 4 | then 5 | set_field("limit_0", split("_", "foo_bar_baz")); 6 | set_field("limit_1", split(":", "foo:bar:baz", 1)); 7 | set_field("limit_2", split("\\|", "foo|bar|baz", 2)); 8 | trigger_test(); 9 | end -------------------------------------------------------------------------------- /benchmarks/benchmarks/match_all_rule/benchmark.toml: -------------------------------------------------------------------------------- 1 | name = "Simple match all" 2 | 3 | [[streams]] 4 | name = "default" 5 | description = "All incoming messages" 6 | pipelines = ["simple"] 7 | 8 | [[streams]] 9 | name = "unused" 10 | description = "A stream that never contains messages" 11 | pipelines = [] 12 | 13 | -------------------------------------------------------------------------------- /plugin/src/web/pipelines/PipelineConnectionsActions.js: -------------------------------------------------------------------------------- 1 | import Reflux from 'reflux'; 2 | 3 | const PipelineConnectionsActions = Reflux.createActions({ 4 | list: { asyncResult: true }, 5 | connectToStream: { asyncResult: true }, 6 | connectToPipeline: { asyncResult: true }, 7 | }); 8 | 9 | export default PipelineConnectionsActions; 10 | -------------------------------------------------------------------------------- /benchmarks/benchmarks/complex_when_no_action/benchmark.toml: -------------------------------------------------------------------------------- 1 | name = "Simple match all" 2 | 3 | [[streams]] 4 | name = "default" 5 | description = "All incoming messages" 6 | pipelines = ["simple"] 7 | 8 | [[streams]] 9 | name = "unused" 10 | description = "A stream that never contains messages" 11 | pipelines = [] 12 | 13 | -------------------------------------------------------------------------------- /benchmarks/benchmarks/many_stages_match_all/benchmark.toml: -------------------------------------------------------------------------------- 1 | name = "Simple match all" 2 | 3 | [[streams]] 4 | name = "default" 5 | description = "All incoming messages" 6 | pipelines = ["many stages"] 7 | 8 | [[streams]] 9 | name = "unused" 10 | description = "A stream that never contains messages" 11 | pipelines = [] 12 | 13 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/keyValueFailure.txt: -------------------------------------------------------------------------------- 1 | rule "kv" 2 | when true 3 | then 4 | set_fields(key_value( 5 | value: "dup_first=1 dup_first=2", 6 | allow_dup_keys: false 7 | )); 8 | set_fields(key_value( 9 | value: "dup_last=", 10 | ignore_empty_values: false 11 | )); 12 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/removeFromStreamRetainDefault.txt: -------------------------------------------------------------------------------- 1 | rule "stream routing" 2 | when true 3 | then 4 | remove_from_stream(name: "some name"); 5 | // if a message is taken off all stream it was on, the default stream will be added back to avoid dropping the message 6 | remove_from_stream(id: "000000000000000000000001"); 7 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/pipelineDeclaration.txt: -------------------------------------------------------------------------------- 1 | pipeline "cisco" 2 | stage 1 match all 3 | rule "check_ip_whitelist" 4 | rule "cisco_device" 5 | stage 2 match either 6 | rule "parse_cisco_time" 7 | rule "extract_src_dest" 8 | rule "normalize_src_dest" 9 | rule "lookup_ips" 10 | rule "resolve_ips" 11 | end 12 | -------------------------------------------------------------------------------- /plugin/webpack.config.js: -------------------------------------------------------------------------------- 1 | const PluginWebpackConfig = require('graylog-web-plugin').PluginWebpackConfig; 2 | const loadBuildConfig = require('graylog-web-plugin').loadBuildConfig; 3 | const path = require('path'); 4 | 5 | module.exports = new PluginWebpackConfig('org.graylog.plugins.pipelineprocessor.PipelineProcessorPlugin', loadBuildConfig(path.resolve(__dirname, './build.config')), { 6 | }); 7 | -------------------------------------------------------------------------------- /plugin/src/web/rules/RuleForm.css: -------------------------------------------------------------------------------- 1 | :local(.usedInPipelines) { 2 | margin: 0; 3 | padding: 0; 4 | } 5 | 6 | :local(.usedInPipelines li:not(:last-child)) { 7 | float: left; 8 | } 9 | 10 | :local(.usedInPipelines li:not(:last-child):after) { 11 | content: ','; 12 | margin-right: 5px; 13 | } 14 | 15 | :local(.usedInPipelines li:last-child:after) { 16 | content: '.'; 17 | } -------------------------------------------------------------------------------- /plugin/src/web/pipelines/PipelinesActions.jsx: -------------------------------------------------------------------------------- 1 | import Reflux from 'reflux'; 2 | 3 | const RulesActions = Reflux.createActions({ 4 | 'delete': { asyncResult: true }, 5 | 'list': { asyncResult: true }, 6 | 'get': { asyncResult: true }, 7 | 'save': { asyncResult: true }, 8 | 'update': { asyncResult: true }, 9 | 'parse' : { asyncResult: true }, 10 | }); 11 | 12 | export default RulesActions; -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/GeneratedRule.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.codegen; 2 | 3 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 4 | 5 | public interface GeneratedRule { 6 | 7 | String name(); 8 | 9 | boolean when(EvaluationContext context); 10 | 11 | void then(EvaluationContext context); 12 | 13 | } 14 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/grok.txt: -------------------------------------------------------------------------------- 1 | rule "grok" 2 | when true 3 | then 4 | let matches = grok(pattern: "%{GREEDY:timestamp;date;yyyy-MM-dd'T'HH:mm:ss.SSSX}", value: "2015-07-31T10:05:36.773Z"); 5 | set_fields(matches); 6 | 7 | // only named captures 8 | let matches1 = grok("%{NUM:num}", "10", true); 9 | set_fields(matches1); 10 | end 11 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/arithmetic.txt: -------------------------------------------------------------------------------- 1 | rule "arithmetic operators" 2 | when 3 | 1.0 + 1.0 == 2.0 && 4 | 8 * 2 > 15 && 5 | double_valued_func() / 20.0 == 0.0 && 6 | 21 % 20 == 1 && 7 | 10.0 / 20.0 == 0.5 && 8 | +10.0 / -5.0 == -2.0 && 9 | -double_valued_func() == -0.0 && 10 | double_valued_func() + 1.0 > 0.0 11 | then 12 | trigger_test(); 13 | end -------------------------------------------------------------------------------- /plugin/src/web/rules/RuleHelper.css: -------------------------------------------------------------------------------- 1 | :local(.clickableRow) { 2 | cursor: pointer; 3 | } 4 | 5 | :local(.functionTableCell) { 6 | width: 300px; 7 | } 8 | 9 | :local(.marginQuickReferenceText) { 10 | margin-top: 5px; 11 | } 12 | 13 | :local(.marginTab) { 14 | margin-top: 10px; 15 | } 16 | 17 | :local(.exampleFunction) { 18 | white-space: pre-wrap; 19 | } 20 | 21 | :local(.adjustedTableCellWidth) { 22 | width: 1%; 23 | } -------------------------------------------------------------------------------- /benchmarks/benchmarks/grok_extract/grok_job_extraction.rule: -------------------------------------------------------------------------------- 1 | rule "grok jenkins extraction" 2 | when 3 | to_string($message.source) == "jenkins.torch.sh" && 4 | regex("#\\d+", to_string($message.message)).matches == true && !has_field("something_that_doesnt_exist") 5 | then 6 | let fields = grok("%{NOTSPACE:job_name}%{SPACE:unwanted}#%{NUMBER:job_number}\\sStarted by\\s%{USERNAME:user}", to_string($message.message), true); 7 | 8 | set_fields(fields); 9 | end -------------------------------------------------------------------------------- /plugin/src/web/rules/RulesActions.jsx: -------------------------------------------------------------------------------- 1 | import Reflux from 'reflux'; 2 | 3 | const RulesActions = Reflux.createActions({ 4 | delete: { asyncResult: true }, 5 | list: { asyncResult: true }, 6 | get: { asyncResult: true }, 7 | save: { asyncResult: true }, 8 | update: { asyncResult: true }, 9 | parse: { asyncResult: true }, 10 | multiple: { asyncResult: true }, 11 | loadFunctions: { asyncResult: true }, 12 | }); 13 | 14 | export default RulesActions; -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineConfig.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor; 2 | 3 | import com.github.joschi.jadconfig.Parameter; 4 | 5 | import org.graylog2.plugin.PluginConfigBean; 6 | 7 | public class PipelineConfig implements PluginConfigBean { 8 | 9 | @Parameter("cached_stageiterators") 10 | private boolean cachedStageIterators = true; 11 | 12 | @Parameter("generate_native_code") 13 | private boolean generateNativeCode = false; 14 | } 15 | -------------------------------------------------------------------------------- /plugin/src/web/logic/SourceGenerator.js: -------------------------------------------------------------------------------- 1 | const SourceGenerator = { 2 | generatePipeline(pipeline) { 3 | let source = `pipeline "${pipeline.title}"\n`; 4 | pipeline.stages.forEach(stage => { 5 | source += `stage ${stage.stage} match ${stage.match_all ? 'all' : 'either'}\n`; 6 | stage.rules.forEach(rule => { 7 | source += `rule "${rule}"\n`; 8 | }); 9 | }); 10 | source += 'end'; 11 | 12 | return source; 13 | }, 14 | }; 15 | 16 | export default SourceGenerator; 17 | -------------------------------------------------------------------------------- /plugin/src/web/pipelines/Pipeline.css: -------------------------------------------------------------------------------- 1 | .pipeline-dl { 2 | margin-bottom: 0; 3 | } 4 | 5 | dl.pipeline-dl > dt { 6 | text-align: left; 7 | width: 140px; 8 | } 9 | 10 | dl.pipeline-dl > dt:after { 11 | content: ':'; 12 | } 13 | 14 | dl.pipeline-dl > dd { 15 | margin-left: 100px; 16 | } 17 | 18 | .row-margin-top { 19 | margin-top: 10px; 20 | } 21 | 22 | .description-margin-top { 23 | margin-top: 5px; 24 | } 25 | 26 | .pipeline-no-connections-warning { 27 | margin-bottom: 13px; 28 | } 29 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/codegen/runCodegen.txt: -------------------------------------------------------------------------------- 1 | rule "grok jenkins extraction" 2 | when 3 | to_string($message.source) == "jenkins.torch.sh" && 4 | (regex("#\\d+", to_string($message.message)).matches == true || !has_field("something_that_doesnt_exist")) 5 | then 6 | let number = 1; 7 | let string = "sadfasdf"; 8 | let fields = {some_identifier: 1, `something with spaces`: "some expression"}; 9 | let ary = [1,3,4,5,"object", string]; 10 | set_fields(fields); 11 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/datesUnixTimestamps.txt: -------------------------------------------------------------------------------- 1 | rule "dates" 2 | when 3 | parse_unix_milliseconds(0) == parse_date("1970-01-01T00:00:00.000Z", "yyyy-MM-dd'T'HH:mm:ss.SSSZ") && 4 | parse_unix_milliseconds(1516272143555) == parse_date("2018-01-18T10:42:23.555Z", "yyyy-MM-dd'T'HH:mm:ss.SSSZ") && 5 | parse_unix_milliseconds(1516272143555, "Europe/Berlin") == parse_date(value: "2018-01-18T11:42:23.555", pattern: "yyyy-MM-dd'T'HH:mm:ss.SSS", timezone: "Europe/Berlin") 6 | then 7 | trigger_test(); 8 | end -------------------------------------------------------------------------------- /plugin/src/main/resources/org.graylog.plugins.graylog-plugin-pipeline-processor/graylog-plugin.properties: -------------------------------------------------------------------------------- 1 | # The plugin version 2 | version=${project.version} 3 | 4 | # The required Graylog server version 5 | graylog.version=${graylog.version} 6 | 7 | # When set to true (the default) the plugin gets a separate class loader 8 | # when loading the plugin. When set to false, the plugin shares a class loader 9 | # with other plugins that have isolated=false. 10 | # 11 | # Do not disable this unless this plugin depends on another plugin! 12 | isolated=false 13 | -------------------------------------------------------------------------------- /plugin/src/web/rules/RulesComponent.jsx: -------------------------------------------------------------------------------- 1 | import PropTypes from 'prop-types'; 2 | import React from 'react'; 3 | 4 | import { Spinner } from 'components/common'; 5 | import RuleList from './RuleList'; 6 | 7 | const RulesComponent = React.createClass({ 8 | propTypes: { 9 | rules: PropTypes.array, 10 | }, 11 | 12 | render() { 13 | if (!this.props.rules) { 14 | return ; 15 | } 16 | 17 | return ( 18 |
19 | 20 |
21 | ); 22 | }, 23 | }); 24 | 25 | export default RulesComponent; -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/substring.txt: -------------------------------------------------------------------------------- 1 | rule "substrings" 2 | when 3 | substring("abc", 0, 2) == "ab" && 4 | substring("abc", 0, 2) == "ab" && 5 | substring("abc", 2, 0) == "" && 6 | substring("abc", 2, 4) == "c" && 7 | substring("abc", 4, 6) == "" && 8 | substring("abc", 2, 2) == "" && 9 | substring("abc", -2, -1) == "b" && 10 | substring("abc", -4, 2) == "ab" && 11 | substring("abc", 1) == "bc" && 12 | substring("abc", 0, -1) == "ab" 13 | then 14 | trigger_test(); 15 | end -------------------------------------------------------------------------------- /plugin/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "PipelineProcessor", 3 | "version": "3.0.0-SNAPSHOT", 4 | "description": "", 5 | "repository": { 6 | "type": "git", 7 | "url": "graylog-plugin-pipeline-processor" 8 | }, 9 | "scripts": { 10 | "build": "webpack --bail" 11 | }, 12 | "keywords": [ 13 | "graylog" 14 | ], 15 | "author": "Graylog, Inc. ", 16 | "license": "GPL-3.0", 17 | "dependencies": {}, 18 | "devDependencies": { 19 | "graylog-web-plugin": "file:../../graylog2-server/graylog2-web-interface/packages/graylog-web-plugin" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/ipMatching.txt: -------------------------------------------------------------------------------- 1 | rule "ip handling" 2 | when 3 | cidr_match("192.0.0.0/8", to_ip("192.168.1.50")) && 4 | ! cidr_match("191.0.0.0/8", to_ip("192.168.1.50")) && 5 | is_ip(to_ip("127.0.0.1")) == true && 6 | is_ip("foobar") == false && 7 | is_ip(1234) == false && 8 | is_ip(12.34) == false && 9 | is_ip(true) == false 10 | then 11 | set_field("ip_anon", to_string(to_ip($message.ip).anonymized)); 12 | set_field("ipv6_anon", to_string(to_ip("2001:db8::1").anonymized)); 13 | trigger_test(); 14 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/newlyCreatedMessage.txt: -------------------------------------------------------------------------------- 1 | rule "operate on newly created message" 2 | when true 3 | then 4 | let x = create_message("new", "synthetic", now()); 5 | 6 | set_field(field: "removed_again", value: "foo", message: x); 7 | set_field(field: "only_in", value: "new message", message: x); 8 | set_fields(fields: { multi: "new message" }, message: x); 9 | set_field(field: "has_source", value: has_field("source", x), message: x); 10 | route_to_stream(name: "some stream", message: x); 11 | remove_field("removed_again", x); 12 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/json.txt: -------------------------------------------------------------------------------- 1 | rule "json" 2 | when 3 | true 4 | then 5 | let json = parse_json(to_string($message.flat_json)); 6 | set_fields(to_map(json)); 7 | 8 | // Don't fail on invalid input 9 | let invalid_json = parse_json("#FOOBAR#"); 10 | set_fields(to_map(invalid_json)); 11 | 12 | // Don't fail on empty input 13 | let empty_json = parse_json(""); 14 | set_fields(to_map(empty_json)); 15 | 16 | // Don't fail on nested input 17 | let nested_json = parse_json(to_string($message.nested_json)); 18 | set_fields(to_map(nested_json)); 19 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/timezones.txt: -------------------------------------------------------------------------------- 1 | // now() is fixed, test uses different millisprovider! 2 | 3 | rule "timezones" 4 | when 5 | now("CET") == now("UTC") && 6 | now("utc") == now("UTC") && 7 | now("Europe/Moscow") == now("europe/moscow") && 8 | now("europe/MoSCOw") == now("msk") && 9 | to_string(now("europe/MoSCOw").zone) == "Europe/Moscow" && 10 | to_string(now("cet").zone) == "CET" && 11 | to_string(now("Etc/gmt-14").zone) == "Etc/GMT-14" && 12 | to_string(now("").zone) == "UTC" && 13 | to_string(now("invalid-timezone").zone) == "UTC" 14 | then 15 | trigger_test(); 16 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/clonedMessage.txt: -------------------------------------------------------------------------------- 1 | rule "operate on cloned message" 2 | when true 3 | then 4 | let x = clone_message(); 5 | let new = create_message("foo", "source"); 6 | let cloned = clone_message(new); 7 | 8 | set_field(field: "removed_again", value: "foo", message: x); 9 | set_field(field: "only_in", value: "new message", message: x); 10 | set_fields(fields: { multi: "new message" }, message: x); 11 | set_field(field: "has_source", value: has_field("source", x), message: x); 12 | route_to_stream(name: "some stream", message: x); 13 | remove_field("removed_again", x); 14 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/regexMatch.txt: -------------------------------------------------------------------------------- 1 | rule "regexMatch" 2 | when 3 | regex("^.*(cde\\.)(:(\\d+))?.*$", "abcde.fg").matches == true && 4 | regex(".*(cde\\.)(:(\\d+))?.*", "abcde.fg").matches == true && 5 | regex("(cde\\.)(:(\\d+))?", "abcde.fg").matches == true && 6 | regex("^(cde\\.)(:(\\d+))?$", "abcde.fg").matches == false 7 | then 8 | let result = regex("(cd\\.e)", "abcd.efg"); 9 | set_field("group_1", result["0"]); 10 | let result = regex("(cd\\.e)", "abcd.efg", ["name"]); 11 | set_field("named_group", result["name"]); 12 | set_field("matched_regex", result.matches); 13 | end -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/PipelineClassloader.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.codegen; 2 | 3 | import java.util.concurrent.atomic.AtomicLong; 4 | 5 | public class PipelineClassloader extends ClassLoader { 6 | 7 | public static AtomicLong loadedClasses = new AtomicLong(); 8 | 9 | @Override 10 | public Class loadClass(String name) throws ClassNotFoundException { 11 | loadedClasses.incrementAndGet(); 12 | return super.loadClass(name); 13 | } 14 | 15 | public void defineClass(String className, byte[] bytes) { 16 | super.defineClass(className, bytes, 0, bytes.length); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Please follow [the instructions on graylog.org](https://www.graylog.org/contributing-to-graylog/). 2 | 3 | #### Code of Conduct 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | Please read and understand the [Code of Conduct](https://github.com/Graylog2/graylog-plugin-pipeline-processor/blob/master/CODE_OF_CONDUCT.md). 13 | -------------------------------------------------------------------------------- /benchmarks/scripts/benchmark-filtered.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | JAR=${uberjar.name}-${git.commit.id.describe}.jar 4 | DEFAULT_JAVA_OPTS="-Xms512m -Xmx512m -server -XX:+ResizeTLAB -XX:+UseConcMarkSweepGC -XX:+CMSConcurrentMTEnabled -XX:+CMSClassUnloadingEnabled -XX:+UseParNewGC -XX:-OmitStackTraceInFastThrow" 5 | JAVA_OPTS="${JAVA_OPTS:="$DEFAULT_JAVA_OPTS"}" 6 | 7 | # default java 8 | JAVA_CMD=${JAVA_CMD:=$(which java)} 9 | 10 | if [ -n "$JAVA_HOME" ] 11 | then 12 | # try to use $JAVA_HOME 13 | if [ -x "$JAVA_HOME"/bin/java ] 14 | then 15 | JAVA_CMD="$JAVA_HOME"/bin/java 16 | else 17 | die "$JAVA_HOME"/bin/java is not executable 18 | fi 19 | fi 20 | 21 | $JAVA_CMD $JAVA_OPTS -jar $JAR $@ -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/digests.txt: -------------------------------------------------------------------------------- 1 | rule "digests" 2 | when 3 | crc32("graylog") == "e3018c57" && 4 | crc32c("graylog") == "82390e89" && 5 | md5("graylog") == "6f9efb466e043b9f3635827ce446e13c" && 6 | murmur3_32("graylog") == "67285534" && 7 | murmur3_128("graylog") == "945d5b1aaa8fdfe9b880b31e814972b3" && 8 | sha1("graylog") == "6d88bccf40bf65b911fe79d78c7af98e382f0c1a" && 9 | sha256("graylog") == "4bbdd5a829dba09d7a7ff4c1367be7d36a017b4267d728d31bd264f63debeaa6" && 10 | sha512("graylog") == "f6cb3a96450fb9c9174299a651333c926cd67b6f5c25d8daeede1589ffa006f4dd31da4f0625b7f281051a34c8352b3a9c1a9babf90020360e911a380b5c3f4f" 11 | then 12 | trigger_test(); 13 | end -------------------------------------------------------------------------------- /plugin/src/web/pipelines/ProcessingTimelineComponent.css: -------------------------------------------------------------------------------- 1 | .pipeline-stage { 2 | border: 1px solid #666; 3 | border-radius: 4px; 4 | display: inline-block; 5 | margin-right: 15px; 6 | padding: 20px; 7 | text-align: center; 8 | width: 120px; 9 | } 10 | 11 | .pipeline-stage.idle-stage { 12 | background-color: #E3E5E5; 13 | border-color: #D0D4D4; 14 | } 15 | 16 | .pipeline-stage.used-stage { 17 | background-color: #FFFFFF; 18 | } 19 | 20 | .pipeline-name { 21 | max-width: 300px; 22 | overflow-x: hidden; 23 | text-overflow: ellipsis; 24 | white-space: nowrap; 25 | width: 300px; 26 | } 27 | 28 | .stream-list { 29 | max-width: 150px; 30 | width: 150px; 31 | word-wrap: break-word; 32 | } -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/events/LegacyDefaultStreamMigrated.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.events; 2 | 3 | import com.fasterxml.jackson.annotation.JsonAutoDetect; 4 | import com.fasterxml.jackson.annotation.JsonCreator; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | import com.google.auto.value.AutoValue; 7 | 8 | @AutoValue 9 | @JsonAutoDetect 10 | public abstract class LegacyDefaultStreamMigrated { 11 | @JsonProperty 12 | public abstract boolean migrationDone(); 13 | 14 | @JsonCreator 15 | public static LegacyDefaultStreamMigrated create(@JsonProperty("migration_done") boolean migrationDone) { 16 | return new AutoValue_LegacyDefaultStreamMigrated(migrationDone); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Days.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.functions.dates.periods; 2 | 3 | import org.joda.time.Period; 4 | 5 | import javax.annotation.Nonnull; 6 | 7 | public class Days extends AbstractPeriodComponentFunction { 8 | 9 | public static final String NAME = "days"; 10 | 11 | @Nonnull 12 | @Override 13 | protected Period getPeriod(int period) { 14 | return Period.days(period); 15 | } 16 | 17 | @Nonnull 18 | @Override 19 | protected String getName() { 20 | return NAME; 21 | } 22 | 23 | @Nonnull 24 | @Override 25 | protected String getDescription() { 26 | return "Create a period with a specified number of days."; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Hours.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.functions.dates.periods; 2 | 3 | import org.joda.time.Period; 4 | 5 | import javax.annotation.Nonnull; 6 | 7 | public class Hours extends AbstractPeriodComponentFunction { 8 | 9 | public static final String NAME = "hours"; 10 | 11 | @Nonnull 12 | @Override 13 | protected Period getPeriod(int period) { 14 | return Period.hours(period); 15 | } 16 | 17 | @Nonnull 18 | @Override 19 | protected String getName() { 20 | return NAME; 21 | } 22 | 23 | @Nonnull 24 | @Override 25 | protected String getDescription() { 26 | return "Create a period with a specified number of hours."; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Weeks.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.functions.dates.periods; 2 | 3 | import org.joda.time.Period; 4 | 5 | import javax.annotation.Nonnull; 6 | 7 | public class Weeks extends AbstractPeriodComponentFunction { 8 | 9 | public static final String NAME = "weeks"; 10 | 11 | @Nonnull 12 | @Override 13 | protected Period getPeriod(int period) { 14 | return Period.weeks(period); 15 | } 16 | 17 | @Nonnull 18 | @Override 19 | protected String getName() { 20 | return NAME; 21 | } 22 | 23 | @Nonnull 24 | @Override 25 | protected String getDescription() { 26 | return "Create a period with a specified number of weeks."; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /plugin/src/web/simulator/SimulationChanges.css: -------------------------------------------------------------------------------- 1 | .simulation-changes { 2 | padding-top: 15px; 3 | } 4 | 5 | .simulation-changes dl { 6 | margin-bottom: 10px; 7 | margin-top: 5px; 8 | } 9 | 10 | .simulation-changes dt, .simulation-changes dd { 11 | padding-left: 20px; 12 | } 13 | 14 | .original-message-changes { 15 | margin-top: 10px; 16 | } 17 | 18 | .added-fields dl { 19 | background-color: #dff0d8; 20 | color: #3c763d; 21 | } 22 | 23 | .removed-fields dl { 24 | background-color: #f2dede; 25 | color: #a94442; 26 | } 27 | 28 | .mutated-fields dl { 29 | background-color: #d9edf7; 30 | color: #31708f; 31 | } 32 | 33 | .field-value { 34 | font-family: monospace; 35 | } 36 | 37 | .mutated-fields .removed-field { 38 | text-decoration: line-through; 39 | } -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Millis.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.functions.dates.periods; 2 | 3 | import org.joda.time.Period; 4 | 5 | import javax.annotation.Nonnull; 6 | 7 | public class Millis extends AbstractPeriodComponentFunction { 8 | 9 | public static final String NAME = "millis"; 10 | 11 | @Nonnull 12 | @Override 13 | protected Period getPeriod(int period) { 14 | return Period.millis(period); 15 | } 16 | 17 | @Nonnull 18 | @Override 19 | protected String getName() { 20 | return NAME; 21 | } 22 | 23 | @Nonnull 24 | @Override 25 | protected String getDescription() { 26 | return "Create a period with a specified number of millis."; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Months.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.functions.dates.periods; 2 | 3 | import org.joda.time.Period; 4 | 5 | import javax.annotation.Nonnull; 6 | 7 | public class Months extends AbstractPeriodComponentFunction { 8 | 9 | public static final String NAME = "months"; 10 | 11 | @Nonnull 12 | @Override 13 | protected Period getPeriod(int period) { 14 | return Period.months(period); 15 | } 16 | 17 | @Nonnull 18 | @Override 19 | protected String getName() { 20 | return NAME; 21 | } 22 | 23 | @Nonnull 24 | @Override 25 | protected String getDescription() { 26 | return "Create a period with a specified number of months."; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Minutes.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.functions.dates.periods; 2 | 3 | import org.joda.time.Period; 4 | 5 | import javax.annotation.Nonnull; 6 | 7 | public class Minutes extends AbstractPeriodComponentFunction { 8 | 9 | public static final String NAME = "minutes"; 10 | 11 | @Nonnull 12 | @Override 13 | protected Period getPeriod(int period) { 14 | return Period.minutes(period); 15 | } 16 | 17 | @Nonnull 18 | @Override 19 | protected String getName() { 20 | return NAME; 21 | } 22 | 23 | @Nonnull 24 | @Override 25 | protected String getDescription() { 26 | return "Create a period with a specified number of minutes."; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Seconds.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.functions.dates.periods; 2 | 3 | import org.joda.time.Period; 4 | 5 | import javax.annotation.Nonnull; 6 | 7 | public class Seconds extends AbstractPeriodComponentFunction { 8 | 9 | public static final String NAME = "seconds"; 10 | 11 | @Nonnull 12 | @Override 13 | protected Period getPeriod(int period) { 14 | return Period.seconds(period); 15 | } 16 | 17 | @Nonnull 18 | @Override 19 | protected String getName() { 20 | return NAME; 21 | } 22 | 23 | @Nonnull 24 | @Override 25 | protected String getDescription() { 26 | return "Create a period with a specified number of seconds."; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /plugin/src/test/resources/log4j2-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/nestedFieldAccess.txt: -------------------------------------------------------------------------------- 1 | // Test nested field/bean access with camel case and snake case 2 | rule "nested field access" 3 | when 4 | beanObject("1", "john", "doe").id == "1" && 5 | beanObject("1", "john", "doe").theName.firstName == "john" && 6 | beanObject("1", "john", "doe").theName.lastName == "doe" && 7 | beanObject("1", "john", "doe").theName.first_name == "john" && 8 | beanObject("1", "john", "doe").theName.last_name == "doe" && 9 | beanObject("1", "john", "doe").the_name.firstName == "john" && 10 | beanObject("1", "john", "doe").the_name.lastName == "doe" && 11 | beanObject("1", "john", "doe").the_name.first_name == "john" && 12 | beanObject("1", "john", "doe").the_name.last_name == "doe" 13 | then 14 | trigger_test(); 15 | end 16 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/keyValue.txt: -------------------------------------------------------------------------------- 1 | rule "kv" 2 | when true 3 | then 4 | set_fields(key_value( 5 | value: "a='1' =2 \n 'c'=3 [d]=44 a=4 \"e\"=4 [f=1][[g]:3] h=", 6 | delimiters: " \t\n\r[", 7 | kv_delimiters: "=:", 8 | ignore_empty_values: true, 9 | trim_key_chars: "\"[]<>'", 10 | trim_value_chars: "']", 11 | allow_dup_keys: true, // the default 12 | handle_dup_keys: "," // meaning concat, default "take_first" 13 | )); 14 | 15 | set_fields(key_value( 16 | value: "dup_first=1 dup_first=2", 17 | handle_dup_keys: "take_first" 18 | )); 19 | set_fields(key_value( 20 | value: "dup_last=1 dup_last=2", 21 | handle_dup_keys: "take_last" 22 | )); 23 | end -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryServicesModule.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.db.memory; 2 | 3 | import org.graylog.plugins.pipelineprocessor.db.PipelineService; 4 | import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; 5 | import org.graylog.plugins.pipelineprocessor.db.RuleService; 6 | import org.graylog2.plugin.PluginModule; 7 | 8 | public class InMemoryServicesModule extends PluginModule { 9 | @Override 10 | protected void configure() { 11 | bind(RuleService.class).to(InMemoryRuleService.class).asEagerSingleton(); 12 | bind(PipelineService.class).to(InMemoryPipelineService.class).asEagerSingleton(); 13 | bind(PipelineStreamConnectionsService.class).to(InMemoryPipelineStreamConnectionsService.class).asEagerSingleton(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/Years.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.functions.dates.periods; 2 | 3 | import org.joda.time.Period; 4 | 5 | import javax.annotation.Nonnull; 6 | 7 | import static com.google.common.primitives.Ints.saturatedCast; 8 | 9 | public class Years extends AbstractPeriodComponentFunction { 10 | 11 | public static final String NAME = "years"; 12 | 13 | @Override 14 | @Nonnull 15 | protected Period getPeriod(int period) { 16 | return Period.years(saturatedCast(period)); 17 | } 18 | 19 | @Override 20 | @Nonnull 21 | protected String getName() { 22 | return NAME; 23 | } 24 | 25 | @Nonnull 26 | @Override 27 | protected String getDescription() { 28 | return "Create a period with a specified number of years."; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /benchmarks/src/main/resources/git.properties: -------------------------------------------------------------------------------- 1 | git.tags=${git.tags} 2 | git.branch=${git.branch} 3 | git.dirty=${git.dirty} 4 | git.remote.origin.url=${git.remote.origin.url} 5 | git.commit.id=${git.commit.id} 6 | git.commit.id.abbrev=${git.commit.id.abbrev} 7 | git.commit.id.describe=${git.commit.id.describe} 8 | git.commit.id.describe-short=${git.commit.id.describe-short} 9 | git.commit.user.name=${git.commit.user.name} 10 | git.commit.user.email=${git.commit.user.email} 11 | git.commit.message.full=${git.commit.message.full} 12 | git.commit.message.short=${git.commit.message.short} 13 | git.commit.time=${git.commit.time} 14 | git.closest.tag.name=${git.closest.tag.name} 15 | git.closest.tag.commit.count=${git.closest.tag.commit.count} 16 | git.build.user.name=${git.build.user.name} 17 | git.build.user.email=${git.build.user.email} 18 | git.build.time=${git.build.time} 19 | git.build.host=${git.build.host} 20 | git.build.version=${git.build.version} -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/urls.txt: -------------------------------------------------------------------------------- 1 | rule "urls" 2 | when 3 | is_url("foobar") == false && 4 | is_url(1234) == false && 5 | is_url(12.34) == false && 6 | is_url(true) == false && 7 | is_url(to_url("http://example.org/")) == true 8 | then 9 | let url = to_url("https://admin:s3cr31@some.host.with.lots.of.subdomains.com:9999/path1/path2/three?q1=something&with_spaces=hello%20graylog&equal=can=containanotherone#anchorstuff"); 10 | set_fields({ 11 | protocol: url.protocol, 12 | authority: url.authority, 13 | user_info: url.userInfo, 14 | host: url.host, 15 | port: url.port, 16 | path: url.path, 17 | file: url.file, 18 | fragment: url.fragment, 19 | query: url.query, 20 | q1: url.queryParams.q1, 21 | with_spaces: url.queryParams.with_spaces, 22 | equal: url.queryParams.equal 23 | }); 24 | trigger_test(); 25 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/jsonpath.txt: -------------------------------------------------------------------------------- 1 | rule "jsonpath" 2 | when 3 | is_json(parse_json("{}")) == true && 4 | is_json("foobar") == false && 5 | is_json(1234) == false && 6 | is_json(12.34) == false && 7 | is_json(true) == false 8 | then 9 | let x = parse_json(to_string($message.message)); 10 | let new_fields = select_jsonpath(x, 11 | { author_first: "$['store']['book'][0]['author']", 12 | author_last: "$['store']['book'][-1:]['author']" 13 | }); 14 | set_fields(new_fields); 15 | 16 | // Don't fail on empty input 17 | let invalid_json = parse_json("#FOOBAR#"); 18 | let invalid_json_fields = select_jsonpath(invalid_json, { some_field: "$.message" }); 19 | set_fields(invalid_json_fields); 20 | 21 | // Don't fail on missing field 22 | let missing_fields = select_jsonpath(x, { some_field: "$.i_dont_exist", this_should_exist: "$['store']['book'][-1:]['author']" }); 23 | set_fields(missing_fields); 24 | end -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/parser/dateArithmetic.txt: -------------------------------------------------------------------------------- 1 | // now() is fixed to "2010-07-30T18:03:25+02:00" to provide a better testing experience 2 | rule "date math" 3 | when 4 | now() + years(1) > now() && 5 | now() + months(1) > now() && 6 | now() + weeks(1) > now() && 7 | now() + days(1) > now() && 8 | now() + hours(1) > now() && 9 | now() + minutes(1) > now() && 10 | now() + seconds(1) > now() && 11 | now() + millis(1) > now() && 12 | now() + period("P1YT1M") > now() && 13 | 14 | now() - years(1) < now() && 15 | now() - months(1) < now() && 16 | now() - weeks(1) < now() && 17 | now() - days(1) < now() && 18 | now() - hours(1) < now() && 19 | now() - minutes(1) < now() && 20 | now() - seconds(1) < now() && 21 | now() - millis(1) < now() && 22 | now() - period("P1YT1M") < now() 23 | 24 | then 25 | set_field("interval", now() - (now() - days(1))); // is a duration of 1 day 26 | trigger_test(); 27 | end -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/InvalidOperation.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.parser.errors; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | 5 | import org.antlr.v4.runtime.ParserRuleContext; 6 | import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; 7 | 8 | public class InvalidOperation extends ParseError { 9 | private final Expression expr; 10 | 11 | private final String message; 12 | 13 | public InvalidOperation(ParserRuleContext ctx, Expression expr, String message) { 14 | super("invalid_operation", ctx); 15 | this.expr = expr; 16 | this.message = message; 17 | } 18 | 19 | @JsonProperty("reason") 20 | @Override 21 | public String toString() { 22 | return "Invalid operation: " + message; 23 | } 24 | 25 | public Expression getExpression() { 26 | return expr; 27 | } 28 | 29 | public String getMessage() { 30 | return message; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /plugin/src/test/java/org/graylog/plugins/pipelineprocessor/functions/messages/StreamCacheServiceTest.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.functions.messages; 2 | 3 | import com.google.common.eventbus.EventBus; 4 | 5 | import org.graylog2.plugin.streams.Stream; 6 | import org.graylog2.streams.StreamService; 7 | import org.junit.Test; 8 | 9 | import java.util.Collection; 10 | import java.util.concurrent.Executors; 11 | 12 | import static org.assertj.core.api.Assertions.assertThat; 13 | import static org.mockito.Mockito.mock; 14 | 15 | public class StreamCacheServiceTest { 16 | @Test 17 | public void getByName() throws Exception { 18 | final StreamCacheService streamCacheService = new StreamCacheService(new EventBus(), mock(StreamService.class), Executors.newSingleThreadScheduledExecutor()); 19 | 20 | // make sure getByName always returns a collection 21 | final Collection streams = streamCacheService.getByName("nonexisting"); 22 | assertThat(streams).isNotNull().isEmpty(); 23 | } 24 | 25 | } -------------------------------------------------------------------------------- /benchmarks/benchmarks/many_stages_match_all/simple.pipeline: -------------------------------------------------------------------------------- 1 | pipeline "many stages" 2 | stage 0 match all 3 | rule "match everything" 4 | stage 1 match all 5 | rule "match everything" 6 | stage 2 match all 7 | rule "match everything" 8 | stage 3 match all 9 | rule "match everything" 10 | stage 4 match all 11 | rule "match everything" 12 | stage 5 match all 13 | rule "match everything" 14 | stage 6 match all 15 | rule "match everything" 16 | stage 7 match all 17 | rule "match everything" 18 | stage 8 match all 19 | rule "match everything" 20 | stage 9 match all 21 | rule "match everything" 22 | stage 10 match all 23 | rule "match everything" 24 | stage 11 match all 25 | rule "match everything" 26 | stage 12 match all 27 | rule "match everything" 28 | stage 13 match all 29 | rule "match everything" 30 | stage 14 match all 31 | rule "match everything" 32 | stage 15 match all 33 | rule "match everything" 34 | stage 16 match all 35 | rule "match everything" 36 | stage 17 match all 37 | rule "match everything" 38 | stage 18 match all 39 | rule "match everything" 40 | stage 19 match all 41 | rule "match everything" 42 | end -------------------------------------------------------------------------------- /plugin/src/web/simulator/SimulationTrace.jsx: -------------------------------------------------------------------------------- 1 | import PropTypes from 'prop-types'; 2 | import React from 'react'; 3 | 4 | import NumberUtils from 'util/NumberUtils'; 5 | 6 | const SimulationTrace = React.createClass({ 7 | propTypes: { 8 | simulationResults: PropTypes.object.isRequired, 9 | }, 10 | 11 | componentDidMount() { 12 | this.style.use(); 13 | }, 14 | 15 | componentWillUnmount() { 16 | this.style.unuse(); 17 | }, 18 | 19 | style: require('!style/useable!css!./SimulationTrace.css'), 20 | 21 | render() { 22 | const simulationTrace = this.props.simulationResults.simulation_trace; 23 | 24 | const traceEntries = []; 25 | 26 | simulationTrace.forEach((trace, idx) => { 27 | traceEntries.push(
{NumberUtils.formatNumber(trace.time)} μs
); 28 | traceEntries.push(
{trace.message}
); 29 | }); 30 | 31 | return ( 32 |
33 | {traceEntries} 34 |
35 | ); 36 | }, 37 | }); 38 | 39 | export default SimulationTrace; 40 | -------------------------------------------------------------------------------- /plugin/src/web/index.jsx: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line no-unused-vars 2 | import webpackEntry from 'webpack-entry'; 3 | 4 | import packageJson from '../../package.json'; 5 | import { PluginManifest, PluginStore } from 'graylog-web-plugin/plugin'; 6 | import PipelinesOverviewPage from 'pipelines/PipelinesOverviewPage'; 7 | import PipelineDetailsPage from 'pipelines/PipelineDetailsPage'; 8 | import SimulatorPage from 'simulator/SimulatorPage'; 9 | import RulesPage from 'rules/RulesPage'; 10 | import RuleDetailsPage from 'rules/RuleDetailsPage'; 11 | 12 | PluginStore.register(new PluginManifest(packageJson, { 13 | routes: [ 14 | { path: '/system/pipelines', component: PipelinesOverviewPage }, 15 | { path: '/system/pipelines/rules', component: RulesPage }, 16 | { path: '/system/pipelines/rules/:ruleId', component: RuleDetailsPage }, 17 | { path: '/system/pipelines/simulate', component: SimulatorPage }, 18 | { path: '/system/pipelines/:pipelineId', component: PipelineDetailsPage }, 19 | ], 20 | 21 | systemnavigation: [ 22 | { path: '/system/pipelines', description: 'Pipelines', permissions: 'inputs:create' }, 23 | ], 24 | })); 25 | -------------------------------------------------------------------------------- /plugin/src/web/pipelines/NewPipeline.jsx: -------------------------------------------------------------------------------- 1 | import PropTypes from 'prop-types'; 2 | import React from 'react'; 3 | import { Row, Col } from 'react-bootstrap'; 4 | 5 | import history from 'util/History'; 6 | import PipelineDetails from './PipelineDetails'; 7 | 8 | import Routes from 'routing/Routes'; 9 | 10 | const NewPipeline = React.createClass({ 11 | propTypes: { 12 | onChange: PropTypes.func.isRequired, 13 | }, 14 | 15 | _onChange(newPipeline) { 16 | this.props.onChange(newPipeline, this._goToPipeline); 17 | }, 18 | 19 | _goToPipeline(pipeline) { 20 | history.push(Routes.pluginRoute('SYSTEM_PIPELINES_PIPELINEID')(pipeline.id)); 21 | }, 22 | 23 | _goBack() { 24 | history.goBack(); 25 | }, 26 | 27 | render() { 28 | return ( 29 | 30 | 31 |

32 | Give a name and description to the new pipeline. You can add stages to it when you save the changes. 33 |

34 | 35 | 36 | 37 | ); 38 | }, 39 | }); 40 | 41 | export default NewPipeline; 42 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LogicalExpression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.expressions; 18 | 19 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 20 | 21 | public interface LogicalExpression extends Expression { 22 | 23 | boolean evaluateBool(EvaluationContext context); 24 | } 25 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/Statement.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.statements; 18 | 19 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 20 | 21 | public interface Statement { 22 | 23 | // TODO should this have a return value at all? 24 | Object evaluate(EvaluationContext context); 25 | } 26 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/fieldPrefixSuffix.txt: -------------------------------------------------------------------------------- 1 | rule "prefixsuffix" 2 | when true 3 | then 4 | // plain set field 5 | set_field("field", "1"); 6 | // both prefix and suffix, doesn't touch the above 7 | set_field("field", "2", "prae_", "_sueff"); 8 | 9 | // combinations of optional prefix, suffix 10 | set_field(field: "field", value: "3", suffix: "_sueff"); 11 | set_field(field: "field", value: "4", prefix: "prae_"); 12 | 13 | // set multiple fields with the same prefix 14 | set_fields( 15 | fields: { 16 | field1: "5", 17 | field2: "6" 18 | }, 19 | prefix: "pre_", 20 | suffix: "_suff" 21 | ); 22 | 23 | // set multiple fields with the same prefix, suffix optional 24 | set_fields( 25 | fields: { 26 | field1: "7", 27 | field2: "8" 28 | }, 29 | prefix: "pre_" 30 | ); 31 | // set multiple fields with the same suffix, prefix optional 32 | set_fields( 33 | fields: { 34 | field1: "9", 35 | field2: "10" 36 | }, 37 | suffix: "_suff" 38 | ); 39 | end 40 | 41 | -------------------------------------------------------------------------------- /plugin/src/web/simulator/SimulatorStore.js: -------------------------------------------------------------------------------- 1 | import Reflux from 'reflux'; 2 | import URLUtils from 'util/URLUtils'; 3 | import fetch from 'logic/rest/FetchProvider'; 4 | 5 | import MessageFormatter from 'logic/message/MessageFormatter'; 6 | import ObjectUtils from 'util/ObjectUtils'; 7 | 8 | import SimulatorActions from './SimulatorActions'; 9 | 10 | const urlPrefix = '/plugins/org.graylog.plugins.pipelineprocessor'; 11 | 12 | const SimulatorStore = Reflux.createStore({ 13 | listenables: [SimulatorActions], 14 | 15 | simulate(stream, messageFields, inputId) { 16 | const url = URLUtils.qualifyUrl(`${urlPrefix}/system/pipelines/simulate`); 17 | const simulation = { 18 | stream_id: stream.id, 19 | message: messageFields, 20 | input_id: inputId, 21 | }; 22 | 23 | let promise = fetch('POST', url, simulation); 24 | promise = promise.then(response => { 25 | const formattedResponse = ObjectUtils.clone(response); 26 | formattedResponse.messages = response.messages.map(msg => MessageFormatter.formatMessageSummary(msg)); 27 | 28 | return formattedResponse; 29 | }); 30 | 31 | SimulatorActions.simulate.promise(promise); 32 | }, 33 | }); 34 | 35 | export default SimulatorStore; 36 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NumericExpression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.expressions; 18 | 19 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 20 | 21 | public interface NumericExpression extends Expression { 22 | 23 | boolean isIntegral(); 24 | 25 | long evaluateLong(EvaluationContext context); 26 | 27 | double evaluateDouble(EvaluationContext context); 28 | } 29 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.db; 18 | 19 | import org.graylog2.database.NotFoundException; 20 | 21 | import java.util.Collection; 22 | 23 | public interface PipelineService { 24 | PipelineDao save(PipelineDao pipeline); 25 | 26 | PipelineDao load(String id) throws NotFoundException; 27 | 28 | Collection loadAll(); 29 | 30 | void delete(String id); 31 | } 32 | -------------------------------------------------------------------------------- /benchmarks/src/main/assembly/tarball.xml: -------------------------------------------------------------------------------- 1 | 4 | benchmarks-tarball 5 | 6 | tar.gz 7 | 8 | 9 | 10 | ${project.basedir}/benchmarks 11 | 12 | **/* 13 | 14 | benchmarks 15 | true 16 | 17 | 18 | 19 | 20 | ${project.build.directory}/${uberjar.name}.jar 21 | ${uberjar.name}-${git.commit.id.describe}.jar 22 | 23 | 24 | scripts/benchmark-filtered.sh 25 | benchmark.sh 26 | 755 27 | true 28 | 29 | 30 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/encodings.txt: -------------------------------------------------------------------------------- 1 | rule "digests" 2 | when 3 | base16_encode(value: "graylog") == "677261796C6F67" && 4 | base16_decode(value: "677261796C6F67") == "graylog" && 5 | base16_encode(value: "graylog", omit_padding: true) == "677261796C6F67" && 6 | base16_decode(value: "677261796C6F67", omit_padding: true) == "graylog" && 7 | base32_encode(value: "graylog") == "CTP62UBCDTJG====" && 8 | base32_decode(value: "CTP62UBCDTJG====") == "graylog" && 9 | base32_encode(value: "graylog", omit_padding: true) == "CTP62UBCDTJG" && 10 | base32_decode(value: "CTP62UBCDTJG", omit_padding: true) == "graylog" && 11 | base32human_encode(value: "graylog") == "M5ZGC6LMN5TQ====" && 12 | base32human_decode(value: "M5ZGC6LMN5TQ====") == "graylog" && 13 | base32human_encode(value: "graylog", omit_padding: true) == "M5ZGC6LMN5TQ" && 14 | base32human_decode(value: "M5ZGC6LMN5TQ", omit_padding: true) == "graylog" && 15 | base64_encode(value: "graylog") == "Z3JheWxvZw==" && 16 | base64_decode(value: "Z3JheWxvZw==") == "graylog" && 17 | base64url_encode(value: "graylog", omit_padding: true) == "Z3JheWxvZw" && 18 | base64url_decode(value: "Z3JheWxvZw", omit_padding: true) == "graylog" 19 | then 20 | trigger_test(); 21 | end -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriority.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.syslog; 18 | 19 | import com.google.auto.value.AutoValue; 20 | 21 | @AutoValue 22 | public abstract class SyslogPriority { 23 | public abstract int getLevel(); 24 | 25 | public abstract int getFacility(); 26 | 27 | public static SyslogPriority create(int level, int facility) { 28 | return new AutoValue_SyslogPriority(level, facility); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/db/RuleService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.db; 18 | 19 | import org.graylog2.database.NotFoundException; 20 | 21 | import java.util.Collection; 22 | 23 | public interface RuleService { 24 | RuleDao save(RuleDao rule); 25 | 26 | RuleDao load(String id) throws NotFoundException; 27 | 28 | Collection loadAll(); 29 | 30 | void delete(String id); 31 | 32 | Collection loadNamed(Collection ruleNames); 33 | } 34 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BaseExpression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.expressions; 18 | 19 | import org.antlr.v4.runtime.Token; 20 | 21 | public abstract class BaseExpression implements Expression { 22 | 23 | private final Token startToken; 24 | 25 | public BaseExpression(Token startToken) { 26 | this.startToken = startToken; 27 | } 28 | 29 | @Override 30 | public Token getStartToken() { 31 | return startToken; 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /plugin/src/web/pipelines/PipelineConnectionsList.jsx: -------------------------------------------------------------------------------- 1 | import PropTypes from 'prop-types'; 2 | import React from 'react'; 3 | import naturalSort from 'javascript-natural-sort'; 4 | 5 | const PipelineConnectionsList = React.createClass({ 6 | propTypes: { 7 | pipeline: PropTypes.object.isRequired, 8 | connections: PropTypes.array.isRequired, 9 | streams: PropTypes.array.isRequired, 10 | streamsFormatter: PropTypes.func.isRequired, 11 | noConnectionsMessage: PropTypes.any, 12 | }, 13 | 14 | getDefaultProps() { 15 | return { 16 | noConnectionsMessage: 'Pipeline not connected to any streams', 17 | }; 18 | }, 19 | 20 | render() { 21 | const streamsUsingPipeline = this.props.connections 22 | .filter(c => c.pipeline_ids && c.pipeline_ids.includes(this.props.pipeline.id)) // Get connections for this pipeline 23 | .filter(c => this.props.streams.some(s => s.id === c.stream_id)) // Filter out deleted streams 24 | .map(c => this.props.streams.find(s => s.id === c.stream_id)) 25 | .sort((s1, s2) => naturalSort(s1.title, s2.title)); 26 | 27 | return ( 28 | 29 | {streamsUsingPipeline.length === 0 ? this.props.noConnectionsMessage : this.props.streamsFormatter(streamsUsingPipeline)} 30 | 31 | ); 32 | }, 33 | }); 34 | 35 | export default PipelineConnectionsList; 36 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/MD5.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.hashing; 18 | 19 | import org.apache.commons.codec.digest.DigestUtils; 20 | 21 | public class MD5 extends SingleArgStringFunction { 22 | 23 | public static final String NAME = "md5"; 24 | 25 | @Override 26 | protected String getDigest(String value) { 27 | return DigestUtils.md5Hex(value); 28 | } 29 | 30 | @Override 31 | protected String getName() { 32 | return NAME; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/syslog/SyslogPriorityAsString.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.syslog; 18 | 19 | import com.google.auto.value.AutoValue; 20 | 21 | @AutoValue 22 | public abstract class SyslogPriorityAsString { 23 | public abstract String getLevel(); 24 | 25 | public abstract String getFacility(); 26 | 27 | public static SyslogPriorityAsString create(String level, String facility) { 28 | return new AutoValue_SyslogPriorityAsString(level, facility); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA1.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.hashing; 18 | 19 | import org.apache.commons.codec.digest.DigestUtils; 20 | 21 | public class SHA1 extends SingleArgStringFunction { 22 | 23 | public static final String NAME = "sha1"; 24 | 25 | @Override 26 | protected String getDigest(String value) { 27 | return DigestUtils.sha1Hex(value); 28 | } 29 | 30 | @Override 31 | protected String getName() { 32 | return NAME; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA256.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.hashing; 18 | 19 | import org.apache.commons.codec.digest.DigestUtils; 20 | 21 | public class SHA256 extends SingleArgStringFunction { 22 | 23 | public static final String NAME = "sha256"; 24 | 25 | @Override 26 | protected String getDigest(String value) { 27 | return DigestUtils.sha256Hex(value); 28 | } 29 | 30 | @Override 31 | protected String getName() { 32 | return NAME; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/SHA512.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.hashing; 18 | 19 | import org.apache.commons.codec.digest.DigestUtils; 20 | 21 | public class SHA512 extends SingleArgStringFunction { 22 | 23 | public static final String NAME = "sha512"; 24 | 25 | @Override 26 | protected String getDigest(String value) { 27 | return DigestUtils.sha512Hex(value); 28 | } 29 | 30 | @Override 31 | protected String getName() { 32 | return NAME; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/LocationAwareEvalException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.exceptions; 18 | 19 | import org.antlr.v4.runtime.Token; 20 | 21 | public class LocationAwareEvalException extends RuntimeException { 22 | private final Token startToken; 23 | 24 | public LocationAwareEvalException(Token startToken, Throwable cause) { 25 | super(cause); 26 | this.startToken = startToken; 27 | } 28 | 29 | public Token getStartToken() { 30 | return startToken; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /plugin/src/test/java/org/graylog/plugins/pipelineprocessor/parser/CodegenPipelineRuleParserTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.parser; 18 | 19 | import org.graylog.plugins.pipelineprocessor.codegen.PipelineClassloader; 20 | import org.junit.Ignore; 21 | 22 | @Ignore("code generation disabled") 23 | public class CodegenPipelineRuleParserTest extends PipelineRuleParserTest { 24 | 25 | // runs the same tests as in PipelineRuleParserTest but with dynamic code generation turned on. 26 | public CodegenPipelineRuleParserTest() { 27 | classLoader = new PipelineClassloader(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/db/PipelineStreamConnectionsService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.db; 18 | 19 | import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; 20 | import org.graylog2.database.NotFoundException; 21 | 22 | import java.util.Set; 23 | 24 | public interface PipelineStreamConnectionsService { 25 | PipelineConnections save(PipelineConnections connections); 26 | 27 | PipelineConnections load(String streamId) throws NotFoundException; 28 | 29 | Set loadAll(); 30 | 31 | void delete(String streamId); 32 | } 33 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/CRC32.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.hashing; 18 | 19 | import com.google.common.hash.Hashing; 20 | 21 | import java.nio.charset.StandardCharsets; 22 | 23 | public class CRC32 extends SingleArgStringFunction { 24 | 25 | public static final String NAME = "crc32"; 26 | 27 | @Override 28 | protected String getDigest(String value) { 29 | return Hashing.crc32().hashString(value, StandardCharsets.UTF_8).toString(); 30 | } 31 | 32 | @Override 33 | protected String getName() { 34 | return NAME; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/CRC32C.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.hashing; 18 | 19 | import com.google.common.hash.Hashing; 20 | 21 | import java.nio.charset.StandardCharsets; 22 | 23 | public class CRC32C extends SingleArgStringFunction { 24 | 25 | public static final String NAME = "crc32c"; 26 | 27 | @Override 28 | protected String getDigest(String value) { 29 | return Hashing.crc32c().hashString(value, StandardCharsets.UTF_8).toString(); 30 | } 31 | 32 | @Override 33 | protected String getName() { 34 | return NAME; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/Murmur3_128.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.hashing; 18 | 19 | import com.google.common.hash.Hashing; 20 | 21 | import java.nio.charset.StandardCharsets; 22 | 23 | public class Murmur3_128 extends SingleArgStringFunction { 24 | 25 | public static final String NAME = "murmur3_128"; 26 | 27 | @Override 28 | protected String getDigest(String value) { 29 | return Hashing.murmur3_128().hashString(value, StandardCharsets.UTF_8).toString(); 30 | } 31 | 32 | @Override 33 | protected String getName() { 34 | return NAME; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/hashing/Murmur3_32.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.hashing; 18 | 19 | import com.google.common.hash.Hashing; 20 | 21 | import java.nio.charset.StandardCharsets; 22 | 23 | public class Murmur3_32 extends SingleArgStringFunction { 24 | 25 | public static final String NAME = "murmur3_32"; 26 | 27 | @Override 28 | protected String getDigest(String value) { 29 | return Hashing.murmur3_32().hashString(value, StandardCharsets.UTF_8).toString(); 30 | } 31 | 32 | @Override 33 | protected String getName() { 34 | return NAME; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/PrecomputeFailure.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.exceptions; 18 | 19 | public class PrecomputeFailure extends RuntimeException { 20 | private final String argumentName; 21 | 22 | public PrecomputeFailure(String argumentName, Exception cause) { 23 | super(cause); 24 | this.argumentName = argumentName; 25 | } 26 | 27 | public String getArgumentName() { 28 | return argumentName; 29 | } 30 | 31 | @Override 32 | public String getMessage() { 33 | return "Unable to pre-compute argument " + getArgumentName() + ": " + getCause().getMessage(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/strings.txt: -------------------------------------------------------------------------------- 1 | // various string functions 2 | rule "string tests" 3 | when 4 | contains("abcdef", "bc") && 5 | lowercase("a MIXED bag of chArs") == "a mixed bag of chars" && 6 | uppercase("a MIXED bag of chArs") == "A MIXED BAG OF CHARS" && 7 | swapcase("Capitalized") == "cAPITALIZED" && 8 | capitalize("hello") == "Hello" && 9 | capitalize("hEllo") == "HEllo" && 10 | uncapitalize("Hello") == "hello" && 11 | uncapitalize("HEllo") == "hEllo" && 12 | abbreviate("", 4) == "" && 13 | abbreviate("abcdefg", 6) == "abc..." && 14 | abbreviate("abcdefg", 7) == "abcdefg" && 15 | abbreviate("abcdefg", 8) == "abcdefg" && 16 | abbreviate("abcdefg", 4) == "a..." && 17 | concat("foo", "bar") == "foobar" && 18 | starts_with("foobar", "foo") == true && 19 | starts_with("foobar", "") == true && 20 | starts_with("", "foo") == false && 21 | starts_with("foobar", "abc") == false && 22 | starts_with("foobar", "FOO") == false && 23 | starts_with("foobar", "FOO", true) == true && 24 | ends_with("foobar", "bar") == true && 25 | ends_with("foobar", "") == true && 26 | ends_with("", "bar") == false && 27 | ends_with("foobar", "abc") == false && 28 | ends_with("foobar", "BAR") == false && 29 | ends_with("foobar", "BAR", true) == true 30 | then 31 | set_field("has_xyz", contains("abcdef", "xyz")); 32 | set_field("string_literal", "abcd\\.e\tfg\u03a9\363"); 33 | trigger_test(); 34 | end -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/rest/BulkRuleRequest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.rest; 18 | 19 | import com.fasterxml.jackson.annotation.JsonAutoDetect; 20 | import com.fasterxml.jackson.annotation.JsonCreator; 21 | import com.fasterxml.jackson.annotation.JsonProperty; 22 | import com.google.auto.value.AutoValue; 23 | 24 | import java.util.List; 25 | 26 | @AutoValue 27 | @JsonAutoDetect 28 | public abstract class BulkRuleRequest { 29 | @JsonProperty 30 | public abstract List rules(); 31 | 32 | @JsonCreator 33 | public static BulkRuleRequest create(@JsonProperty("rules") List rules) { 34 | return new AutoValue_BulkRuleRequest(rules); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/functions/AbstractFunction.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.functions; 18 | 19 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 20 | import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; 21 | 22 | /** 23 | * Helper Function implementation which evaluates and memoizes all constant FunctionArgs. 24 | * 25 | * @param the return type 26 | */ 27 | public abstract class AbstractFunction implements Function { 28 | 29 | @Override 30 | public Object preComputeConstantArgument(FunctionArgs args, String name, Expression arg) { 31 | return arg.evaluateUnsafe(EvaluationContext.emptyContext()); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /plugin/src/web/simulator/SimulationPreview.jsx: -------------------------------------------------------------------------------- 1 | import PropTypes from 'prop-types'; 2 | import React from 'react'; 3 | import { Alert } from 'react-bootstrap'; 4 | 5 | import MessageShow from 'components/search/MessageShow'; 6 | 7 | const SimulationPreview = React.createClass({ 8 | propTypes: { 9 | simulationResults: PropTypes.object.isRequired, 10 | streams: PropTypes.object.isRequired, 11 | }, 12 | 13 | render() { 14 | const messages = this.props.simulationResults.messages; 15 | 16 | if (messages.length === 0) { 17 | return ( 18 | 19 |

Message would be dropped

20 |

21 | The pipeline processor would drop such a message. That means that the message would 22 | not be stored, and would not be available for searches, alerts, outputs, or dashboards. 23 |

24 | 25 | ); 26 | } 27 | 28 | const formattedMessages = messages.map(message => { 29 | return ( 30 | 37 | ); 38 | }); 39 | 40 | return
{formattedMessages}
; 41 | }, 42 | }); 43 | 44 | export default SimulationPreview; 45 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/PeriodParseFunction.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.functions.dates.periods; 2 | 3 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 4 | import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; 5 | import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; 6 | import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; 7 | import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; 8 | import org.joda.time.Period; 9 | 10 | public class PeriodParseFunction extends AbstractFunction { 11 | 12 | public static final String NAME = "period"; 13 | private final ParameterDescriptor value = 14 | ParameterDescriptor 15 | .string("value", Period.class) 16 | .transform(Period::parse) 17 | .build(); 18 | 19 | 20 | @Override 21 | public Period evaluate(FunctionArgs args, EvaluationContext context) { 22 | return value.required(args, context); 23 | } 24 | 25 | @Override 26 | public FunctionDescriptor descriptor() { 27 | return FunctionDescriptor.builder() 28 | .name(NAME) 29 | .description("Parses a ISO 8601 period from the specified string.") 30 | .pure(true) 31 | .returnType(Period.class) 32 | .params(value) 33 | .build(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/StringExpression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.expressions; 18 | 19 | import org.antlr.v4.runtime.Token; 20 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 21 | 22 | public class StringExpression extends ConstantExpression { 23 | 24 | private final String value; 25 | 26 | public StringExpression(Token start, String value) { 27 | super(start, String.class); 28 | this.value = value; 29 | } 30 | 31 | @Override 32 | public Object evaluateUnsafe(EvaluationContext context) { 33 | return value; 34 | } 35 | 36 | @Override 37 | public String toString() { 38 | return '"' + value + '"'; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/UndeclaredFunction.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.parser.errors; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; 21 | 22 | public class UndeclaredFunction extends ParseError { 23 | private final RuleLangParser.FunctionCallContext ctx; 24 | 25 | public UndeclaredFunction(RuleLangParser.FunctionCallContext ctx) { 26 | super("undeclared_function", ctx); 27 | this.ctx = ctx; 28 | } 29 | 30 | @JsonProperty("reason") 31 | @Override 32 | public String toString() { 33 | return "Unknown function " + ctx.funcName.getText() + positionString(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/compiler/PipelineCompilationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.codegen.compiler; 18 | 19 | import java.util.List; 20 | import java.util.Locale; 21 | import java.util.stream.Collectors; 22 | 23 | import javax.tools.Diagnostic; 24 | 25 | public class PipelineCompilationException extends RuntimeException { 26 | private final List errors; 27 | 28 | public PipelineCompilationException(List errors) { 29 | this.errors = errors; 30 | } 31 | 32 | @Override 33 | public String getMessage() { 34 | return errors.stream() 35 | .map(diagnostic -> diagnostic.getMessage(Locale.ENGLISH)) 36 | .collect(Collectors.joining("\n")); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/db/mongodb/MongoDbServicesModule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.db.mongodb; 18 | 19 | import org.graylog.plugins.pipelineprocessor.db.PipelineService; 20 | import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; 21 | import org.graylog.plugins.pipelineprocessor.db.RuleService; 22 | import org.graylog2.plugin.PluginModule; 23 | 24 | public class MongoDbServicesModule extends PluginModule { 25 | @Override 26 | protected void configure() { 27 | bind(PipelineService.class).to(MongoDbPipelineService.class); 28 | bind(RuleService.class).to(MongoDbRuleService.class); 29 | bind(PipelineStreamConnectionsService.class).to(MongoDbPipelineStreamConnectionsService.class); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/NonIndexableType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.parser.errors; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; 21 | 22 | public class NonIndexableType extends ParseError { 23 | private final Class indexableType; 24 | 25 | public NonIndexableType(RuleLangParser.IndexedAccessContext ctx, Class indexableType) { 26 | super("non_indexable", ctx); 27 | this.indexableType = indexableType; 28 | } 29 | 30 | @JsonProperty("reason") 31 | @Override 32 | public String toString() { 33 | return "Cannot index value of type " + indexableType.getSimpleName() + positionString(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ConstantExpression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.expressions; 18 | 19 | import org.antlr.v4.runtime.Token; 20 | 21 | import java.util.Collections; 22 | 23 | public abstract class ConstantExpression extends BaseExpression { 24 | 25 | private final Class type; 26 | 27 | protected ConstantExpression(Token start, Class type) { 28 | super(start); 29 | this.type = type; 30 | } 31 | 32 | @Override 33 | public boolean isConstant() { 34 | return true; 35 | } 36 | 37 | @Override 38 | public Class getType() { 39 | return type; 40 | } 41 | 42 | @Override 43 | public Iterable children() { 44 | return Collections.emptySet(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Lowercase.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.strings; 18 | 19 | import org.apache.commons.lang3.StringUtils; 20 | 21 | import java.util.Locale; 22 | 23 | public class Lowercase extends StringUtilsFunction { 24 | 25 | public static final String NAME = "lowercase"; 26 | 27 | @Override 28 | protected String getName() { 29 | return NAME; 30 | } 31 | 32 | @Override 33 | protected String description() { 34 | return "Lowercases a string"; 35 | } 36 | 37 | @Override 38 | protected boolean isLocaleAware() { 39 | return true; 40 | } 41 | 42 | @Override 43 | protected String apply(String value, Locale locale) { 44 | return StringUtils.lowerCase(value, locale); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uppercase.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.strings; 18 | 19 | import org.apache.commons.lang3.StringUtils; 20 | 21 | import java.util.Locale; 22 | 23 | public class Uppercase extends StringUtilsFunction { 24 | 25 | public static final String NAME = "uppercase"; 26 | 27 | @Override 28 | protected String getName() { 29 | return NAME; 30 | } 31 | 32 | @Override 33 | protected String description() { 34 | return "Uppercases a string"; 35 | } 36 | 37 | @Override 38 | protected boolean isLocaleAware() { 39 | return true; 40 | } 41 | 42 | @Override 43 | protected String apply(String value, Locale locale) { 44 | return StringUtils.upperCase(value, locale); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/codegen/compiler/JavaSourceFromString.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.codegen.compiler; 18 | 19 | import java.net.URI; 20 | 21 | import javax.tools.SimpleJavaFileObject; 22 | import javax.validation.constraints.NotNull; 23 | 24 | import static javax.tools.JavaFileObject.Kind.SOURCE; 25 | 26 | public class JavaSourceFromString extends SimpleJavaFileObject { 27 | 28 | private final String sourceCode; 29 | 30 | public JavaSourceFromString(@NotNull String name, String sourceCode) { 31 | super(URI.create("string:///" + name.replace('.', '/') + SOURCE.extension), SOURCE); 32 | this.sourceCode = sourceCode; 33 | } 34 | 35 | @Override 36 | public CharSequence getCharContent(boolean ignoreEncodingErrors) { 37 | return sourceCode; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/parser/ParseException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.parser; 18 | 19 | import org.graylog.plugins.pipelineprocessor.parser.errors.ParseError; 20 | 21 | import java.util.Set; 22 | 23 | public class ParseException extends RuntimeException { 24 | private final Set errors; 25 | 26 | public ParseException(Set errors) { 27 | this.errors = errors; 28 | } 29 | 30 | public Set getErrors() { 31 | return errors; 32 | } 33 | 34 | @Override 35 | public String getMessage() { 36 | StringBuilder sb = new StringBuilder("Errors:\n"); 37 | for (ParseError parseError : getErrors()) { 38 | sb.append(" ").append(parseError).append("\n"); 39 | } 40 | return sb.toString(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/UndeclaredVariable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.parser.errors; 18 | 19 | import com.fasterxml.jackson.annotation.JsonIgnore; 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; 22 | 23 | public class UndeclaredVariable extends ParseError { 24 | 25 | @JsonIgnore 26 | private final RuleLangParser.IdentifierContext ctx; 27 | 28 | public UndeclaredVariable(RuleLangParser.IdentifierContext ctx) { 29 | super("undeclared_variable", ctx); 30 | this.ctx = ctx; 31 | } 32 | 33 | @JsonProperty("reason") 34 | @Override 35 | public String toString() { 36 | return "Undeclared variable " + ctx.Identifier().getText() + positionString(); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Capitalize.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.strings; 18 | 19 | import org.apache.commons.lang3.StringUtils; 20 | 21 | import java.util.Locale; 22 | 23 | public class Capitalize extends StringUtilsFunction { 24 | 25 | public static final String NAME = "capitalize"; 26 | 27 | @Override 28 | protected String getName() { 29 | return NAME; 30 | } 31 | 32 | @Override 33 | protected String description() { 34 | return "Capitalizes a String changing the first letter to title case from lower case"; 35 | } 36 | 37 | @Override 38 | protected boolean isLocaleAware() { 39 | return false; 40 | } 41 | 42 | @Override 43 | protected String apply(String value, Locale unused) { 44 | return StringUtils.capitalize(value); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.parser.errors; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; 21 | 22 | public class IncompatibleType extends ParseError { 23 | private final Class expected; 24 | private final Class actual; 25 | 26 | public IncompatibleType(RuleLangParser.MessageRefContext ctx, Class expected, Class actual) { 27 | super("incompatible_type", ctx); 28 | this.expected = expected; 29 | this.actual = actual; 30 | } 31 | 32 | @JsonProperty("reason") 33 | @Override 34 | public String toString() { 35 | return "Expected type " + expected.getSimpleName() + " but found " + actual.getSimpleName() + positionString(); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/simulator/PipelineInterpreterTrace.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.simulator; 18 | 19 | import com.fasterxml.jackson.annotation.JsonAutoDetect; 20 | import com.fasterxml.jackson.annotation.JsonCreator; 21 | import com.fasterxml.jackson.annotation.JsonProperty; 22 | import com.google.auto.value.AutoValue; 23 | 24 | @AutoValue 25 | @JsonAutoDetect 26 | public abstract class PipelineInterpreterTrace { 27 | @JsonProperty 28 | public abstract long time(); 29 | 30 | @JsonProperty 31 | public abstract String message(); 32 | 33 | @JsonCreator 34 | public static PipelineInterpreterTrace create (@JsonProperty("time") long time, 35 | @JsonProperty("message") String message) { 36 | return new AutoValue_PipelineInterpreterTrace(time, message); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Uncapitalize.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.strings; 18 | 19 | import org.apache.commons.lang3.StringUtils; 20 | 21 | import java.util.Locale; 22 | 23 | public class Uncapitalize extends StringUtilsFunction { 24 | 25 | public static final String NAME = "uncapitalize"; 26 | 27 | @Override 28 | protected String getName() { 29 | return NAME; 30 | } 31 | 32 | @Override 33 | protected String description() { 34 | return "Uncapitalizes a String changing the first letter to lower case from title case"; 35 | } 36 | 37 | @Override 38 | protected boolean isLocaleAware() { 39 | return false; 40 | } 41 | 42 | @Override 43 | protected String apply(String value, Locale unused) { 44 | return StringUtils.uncapitalize(value); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/strings/Swapcase.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.strings; 18 | 19 | import org.apache.commons.lang3.StringUtils; 20 | 21 | import java.util.Locale; 22 | 23 | public class Swapcase extends StringUtilsFunction { 24 | 25 | public static final String NAME = "swapcase"; 26 | 27 | @Override 28 | protected String getName() { 29 | return NAME; 30 | } 31 | 32 | @Override 33 | protected String description() { 34 | return "Swaps the case of a String changing upper and title case to lower case, and lower case to upper case."; 35 | } 36 | 37 | @Override 38 | protected boolean isLocaleAware() { 39 | return false; 40 | } 41 | 42 | @Override 43 | protected String apply(String value, Locale unused) { 44 | return StringUtils.swapCase(value); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/PipelineProcessorPlugin.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor; 18 | 19 | import com.google.common.collect.ImmutableList; 20 | import org.graylog.plugins.pipelineprocessor.db.mongodb.MongoDbServicesModule; 21 | import org.graylog2.plugin.Plugin; 22 | import org.graylog2.plugin.PluginMetaData; 23 | import org.graylog2.plugin.PluginModule; 24 | 25 | import java.util.Collection; 26 | 27 | /** 28 | * Implement the Plugin interface here. 29 | */ 30 | public class PipelineProcessorPlugin implements Plugin { 31 | @Override 32 | public PluginMetaData metadata() { 33 | return new PipelineProcessorMetaData(); 34 | } 35 | 36 | @Override 37 | public Collection modules () { 38 | return ImmutableList.of( 39 | new PipelineProcessorModule(), 40 | new MongoDbServicesModule() 41 | ); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/exceptions/FunctionEvaluationException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.exceptions; 18 | 19 | import org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression; 20 | 21 | public class FunctionEvaluationException extends LocationAwareEvalException { 22 | private final FunctionExpression functionExpression; 23 | private final Exception exception; 24 | 25 | public FunctionEvaluationException(FunctionExpression functionExpression, Exception exception) { 26 | super(functionExpression.getStartToken(), exception); 27 | this.functionExpression = functionExpression; 28 | this.exception = exception; 29 | } 30 | 31 | public FunctionExpression getFunctionExpression() { 32 | return functionExpression; 33 | } 34 | 35 | public Exception getException() { 36 | return exception; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BooleanExpression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.expressions; 18 | 19 | import org.antlr.v4.runtime.Token; 20 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 21 | 22 | public class BooleanExpression extends ConstantExpression implements LogicalExpression { 23 | private final boolean value; 24 | 25 | public BooleanExpression(Token start, boolean value) { 26 | super(start, Boolean.class); 27 | this.value = value; 28 | } 29 | 30 | @Override 31 | public Object evaluateUnsafe(EvaluationContext context) { 32 | return value; 33 | } 34 | 35 | 36 | @Override 37 | public boolean evaluateBool(EvaluationContext context) { 38 | return value; 39 | } 40 | 41 | @Override 42 | public String toString() { 43 | return Boolean.toString(value); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/FunctionStatement.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.statements; 18 | 19 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 20 | import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; 21 | 22 | public class FunctionStatement implements Statement { 23 | 24 | private final Expression functionExpression; 25 | 26 | public FunctionStatement(Expression functionExpression) { 27 | this.functionExpression = functionExpression; 28 | } 29 | 30 | @Override 31 | public Object evaluate(EvaluationContext context) { 32 | return functionExpression.evaluate(context); 33 | } 34 | 35 | public Expression getFunctionExpression() { 36 | return functionExpression; 37 | } 38 | 39 | @Override 40 | public String toString() { 41 | return functionExpression.toString(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/OptionalParametersMustBeNamed.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.parser.errors; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | import org.graylog.plugins.pipelineprocessor.ast.functions.Function; 21 | import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; 22 | 23 | public class OptionalParametersMustBeNamed extends ParseError { 24 | private final Function function; 25 | 26 | public OptionalParametersMustBeNamed(RuleLangParser.FunctionCallContext ctx, Function function) { 27 | super("must_name_optional_params", ctx); 28 | this.function = function; 29 | } 30 | 31 | @JsonProperty("reason") 32 | @Override 33 | public String toString() { 34 | return "Function " + function.descriptor().name() + " has optional parameters, must use named parameters to call" + positionString(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/comparisons.txt: -------------------------------------------------------------------------------- 1 | rule "comparison" 2 | when 3 | is_string("") == true && 4 | is_string("foobar") == true && 5 | is_string(false) == false && 6 | is_string(1000) == false && 7 | is_string(1.234d) == false && 8 | 9 | is_bool(true) == true && 10 | is_bool(false) == true && 11 | is_bool("foobar") == false && 12 | is_bool(1234) == false && 13 | is_bool(23.42) == false && 14 | 15 | is_double(23.42) == true && 16 | is_double(23) == false && 17 | is_double(true) == false && 18 | is_double("foobar") == false && 19 | 20 | is_long(23) == true && 21 | is_long(23.42) == false && 22 | is_long(true) == false && 23 | is_long("foobar") == false && 24 | 25 | is_number(23) == true && 26 | is_number(23.42) == true && 27 | is_number(true) == false && 28 | is_number("foobar") == false && 29 | 30 | is_collection(["foobar", "foobaz"]) == true && 31 | is_collection({foo:"bar"}) == false && 32 | is_collection("foobar") == false && 33 | is_collection(true) == false && 34 | is_collection(23) == false && 35 | is_collection(23.42) == false && 36 | is_collection("foobar") == false && 37 | 38 | is_list(["foobar", "foobaz"]) == true && 39 | is_list({foo:"bar"}) == false && 40 | is_list("foobar") == false && 41 | is_list(true) == false && 42 | is_list(23) == false && 43 | is_list(23.42) == false && 44 | is_list("foobar") == false && 45 | 46 | is_map({foo:"bar"}) == true && 47 | is_map(["foobar", "foobaz"]) == false && 48 | is_map(true) == false && 49 | is_map(23) == false && 50 | is_map(23.42) == false && 51 | is_map("foobar") == false 52 | then 53 | trigger_test(); 54 | end -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/BinaryExpression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.expressions; 18 | 19 | import com.google.common.collect.ImmutableList; 20 | 21 | import org.antlr.v4.runtime.Token; 22 | 23 | public abstract class BinaryExpression extends UnaryExpression { 24 | 25 | protected Expression left; 26 | 27 | public BinaryExpression(Token start, Expression left, Expression right) { 28 | super(start, right); 29 | this.left = left; 30 | } 31 | 32 | @Override 33 | public boolean isConstant() { 34 | return left.isConstant() && right.isConstant(); 35 | } 36 | 37 | public Expression left() { 38 | return left; 39 | } 40 | 41 | public void left(Expression left) { 42 | this.left = left; 43 | } 44 | @Override 45 | public Iterable children() { 46 | return ImmutableList.of(left, right); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/rest/PipelineReverseConnections.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.rest; 18 | 19 | import com.fasterxml.jackson.annotation.JsonAutoDetect; 20 | import com.fasterxml.jackson.annotation.JsonCreator; 21 | import com.fasterxml.jackson.annotation.JsonProperty; 22 | import com.google.auto.value.AutoValue; 23 | 24 | import java.util.Set; 25 | 26 | @AutoValue 27 | @JsonAutoDetect 28 | public abstract class PipelineReverseConnections { 29 | @JsonProperty 30 | public abstract String pipelineId(); 31 | 32 | @JsonProperty 33 | public abstract Set streamIds(); 34 | 35 | @JsonCreator 36 | public static PipelineReverseConnections create(@JsonProperty("pipeline_id") String pipelineId, 37 | @JsonProperty("stream_ids") Set streamIds) { 38 | return new AutoValue_PipelineReverseConnections(pipelineId, streamIds); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base16Encode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.encoding; 18 | 19 | import com.google.common.io.BaseEncoding; 20 | 21 | import java.nio.charset.StandardCharsets; 22 | 23 | public class Base16Encode extends BaseEncodingSingleArgStringFunction { 24 | public static final String NAME = "base16_encode"; 25 | private static final String ENCODING_NAME = "base16"; 26 | 27 | @Override 28 | protected String getEncodedValue(String value, boolean omitPadding) { 29 | BaseEncoding encoding = BaseEncoding.base16(); 30 | encoding = omitPadding ? encoding.omitPadding() : encoding; 31 | 32 | return encoding.encode(value.getBytes(StandardCharsets.UTF_8)); 33 | } 34 | 35 | @Override 36 | protected String getEncodingName() { 37 | return ENCODING_NAME; 38 | } 39 | 40 | protected String getName() { 41 | return NAME; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base64Encode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.encoding; 18 | 19 | import com.google.common.io.BaseEncoding; 20 | 21 | import java.nio.charset.StandardCharsets; 22 | 23 | public class Base64Encode extends BaseEncodingSingleArgStringFunction { 24 | public static final String NAME = "base64_encode"; 25 | private static final String ENCODING_NAME = "base64"; 26 | 27 | @Override 28 | protected String getEncodedValue(String value, boolean omitPadding) { 29 | BaseEncoding encoding = BaseEncoding.base64(); 30 | encoding = omitPadding ? encoding.omitPadding() : encoding; 31 | 32 | return encoding.encode(value.getBytes(StandardCharsets.UTF_8)); 33 | } 34 | 35 | @Override 36 | protected String getEncodingName() { 37 | return ENCODING_NAME; 38 | } 39 | 40 | protected String getName() { 41 | return NAME; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base16Decode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.encoding; 18 | 19 | import com.google.common.io.BaseEncoding; 20 | 21 | import java.nio.charset.StandardCharsets; 22 | 23 | public class Base16Decode extends BaseEncodingSingleArgStringFunction { 24 | public static final String NAME = "base16_decode"; 25 | private static final String ENCODING_NAME = "base16"; 26 | 27 | @Override 28 | protected String getEncodedValue(String value, boolean omitPadding) { 29 | BaseEncoding encoding = BaseEncoding.base16(); 30 | encoding = omitPadding ? encoding.omitPadding() : encoding; 31 | 32 | return new String(encoding.decode(value), StandardCharsets.UTF_8); 33 | } 34 | 35 | @Override 36 | protected String getEncodingName() { 37 | return ENCODING_NAME; 38 | } 39 | 40 | protected String getName() { 41 | return NAME; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base32Decode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.encoding; 18 | 19 | import com.google.common.io.BaseEncoding; 20 | 21 | import java.nio.charset.StandardCharsets; 22 | 23 | public class Base32Decode extends BaseEncodingSingleArgStringFunction { 24 | public static final String NAME = "base32_decode"; 25 | private static final String ENCODING_NAME = "base32"; 26 | 27 | @Override 28 | protected String getEncodedValue(String value, boolean omitPadding) { 29 | BaseEncoding encoding = BaseEncoding.base32Hex(); 30 | encoding = omitPadding ? encoding.omitPadding() : encoding; 31 | 32 | return new String(encoding.decode(value), StandardCharsets.UTF_8); 33 | } 34 | 35 | @Override 36 | protected String getEncodingName() { 37 | return ENCODING_NAME; 38 | } 39 | 40 | protected String getName() { 41 | return NAME; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base32Encode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.encoding; 18 | 19 | import com.google.common.io.BaseEncoding; 20 | 21 | import java.nio.charset.StandardCharsets; 22 | 23 | public class Base32Encode extends BaseEncodingSingleArgStringFunction { 24 | public static final String NAME = "base32_encode"; 25 | private static final String ENCODING_NAME = "base32"; 26 | 27 | @Override 28 | protected String getEncodedValue(String value, boolean omitPadding) { 29 | BaseEncoding encoding = BaseEncoding.base32Hex(); 30 | encoding = omitPadding ? encoding.omitPadding() : encoding; 31 | 32 | return encoding.encode(value.getBytes(StandardCharsets.UTF_8)); 33 | } 34 | 35 | @Override 36 | protected String getEncodingName() { 37 | return ENCODING_NAME; 38 | } 39 | 40 | protected String getName() { 41 | return NAME; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base64Decode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.encoding; 18 | 19 | import com.google.common.io.BaseEncoding; 20 | 21 | import java.nio.charset.StandardCharsets; 22 | 23 | public class Base64Decode extends BaseEncodingSingleArgStringFunction { 24 | public static final String NAME = "base64_decode"; 25 | private static final String ENCODING_NAME = "base64"; 26 | 27 | @Override 28 | protected String getEncodedValue(String value, boolean omitPadding) { 29 | BaseEncoding encoding = BaseEncoding.base64(); 30 | encoding = omitPadding ? encoding.omitPadding() : encoding; 31 | 32 | return new String(encoding.decode(value), StandardCharsets.UTF_8); 33 | } 34 | 35 | @Override 36 | protected String getEncodingName() { 37 | return ENCODING_NAME; 38 | } 39 | 40 | protected String getName() { 41 | return NAME; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/NotExpression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.expressions; 18 | 19 | import org.antlr.v4.runtime.Token; 20 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 21 | 22 | public class NotExpression extends UnaryExpression implements LogicalExpression { 23 | public NotExpression(Token start, Expression right) { 24 | super(start, right); 25 | } 26 | 27 | @Override 28 | public Object evaluateUnsafe(EvaluationContext context) { 29 | return evaluateBool(context); 30 | } 31 | 32 | @Override 33 | public boolean evaluateBool(EvaluationContext context) { 34 | return !((LogicalExpression)right).evaluateBool(context); 35 | } 36 | 37 | @Override 38 | public Class getType() { 39 | return Boolean.class; 40 | } 41 | 42 | @Override 43 | public String toString() { 44 | return "NOT " + right.toString(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base64UrlDecode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.encoding; 18 | 19 | import com.google.common.io.BaseEncoding; 20 | 21 | import java.nio.charset.StandardCharsets; 22 | 23 | public class Base64UrlDecode extends BaseEncodingSingleArgStringFunction { 24 | public static final String NAME = "base64url_decode"; 25 | private static final String ENCODING_NAME = "base64 (URL-safe)"; 26 | 27 | @Override 28 | protected String getEncodedValue(String value, boolean omitPadding) { 29 | BaseEncoding encoding = BaseEncoding.base64Url(); 30 | encoding = omitPadding ? encoding.omitPadding() : encoding; 31 | 32 | return new String(encoding.decode(value), StandardCharsets.UTF_8); 33 | } 34 | 35 | @Override 36 | protected String getEncodingName() { 37 | return ENCODING_NAME; 38 | } 39 | 40 | protected String getName() { 41 | return NAME; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base64UrlEncode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.encoding; 18 | 19 | import com.google.common.io.BaseEncoding; 20 | 21 | import java.nio.charset.StandardCharsets; 22 | 23 | public class Base64UrlEncode extends BaseEncodingSingleArgStringFunction { 24 | public static final String NAME = "base64url_encode"; 25 | private static final String ENCODING_NAME = "base64 (URL-safe)"; 26 | 27 | @Override 28 | protected String getEncodedValue(String value, boolean omitPadding) { 29 | BaseEncoding encoding = BaseEncoding.base64Url(); 30 | encoding = omitPadding ? encoding.omitPadding() : encoding; 31 | 32 | return encoding.encode(value.getBytes(StandardCharsets.UTF_8)); 33 | } 34 | 35 | @Override 36 | protected String getEncodingName() { 37 | return ENCODING_NAME; 38 | } 39 | 40 | protected String getName() { 41 | return NAME; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base32HumanEncode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.encoding; 18 | 19 | import com.google.common.io.BaseEncoding; 20 | 21 | import java.nio.charset.StandardCharsets; 22 | 23 | public class Base32HumanEncode extends BaseEncodingSingleArgStringFunction { 24 | public static final String NAME = "base32human_encode"; 25 | private static final String ENCODING_NAME = "base32 (human-friendly)"; 26 | 27 | @Override 28 | protected String getEncodedValue(String value, boolean omitPadding) { 29 | BaseEncoding encoding = BaseEncoding.base32(); 30 | encoding = omitPadding ? encoding.omitPadding() : encoding; 31 | 32 | return encoding.encode(value.getBytes(StandardCharsets.UTF_8)); 33 | } 34 | 35 | @Override 36 | protected String getEncodingName() { 37 | return ENCODING_NAME; 38 | } 39 | 40 | protected String getName() { 41 | return NAME; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleIndexType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.parser.errors; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; 21 | 22 | public class IncompatibleIndexType extends ParseError { 23 | private final Class expected; 24 | private final Class actual; 25 | 26 | public IncompatibleIndexType(RuleLangParser.IndexedAccessContext ctx, 27 | Class expected, 28 | Class actual) { 29 | super("incompatible_index_type", ctx); 30 | this.expected = expected; 31 | this.actual = actual; 32 | } 33 | 34 | @JsonProperty("reason") 35 | @Override 36 | public String toString() { 37 | return "Expected type " + expected.getSimpleName() + " but found " + actual.getSimpleName() + " when indexing" + positionString(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/encoding/Base32HumanDecode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.encoding; 18 | 19 | import com.google.common.io.BaseEncoding; 20 | 21 | import java.nio.charset.StandardCharsets; 22 | 23 | public class Base32HumanDecode extends BaseEncodingSingleArgStringFunction { 24 | public static final String NAME = "base32human_decode"; 25 | private static final String ENCODING_NAME = "base32 (human-friendly)"; 26 | 27 | @Override 28 | protected String getEncodedValue(String value, boolean omitPadding) { 29 | BaseEncoding encoding = BaseEncoding.base32(); 30 | encoding = omitPadding ? encoding.omitPadding() : encoding; 31 | 32 | return new String(encoding.decode(value), StandardCharsets.UTF_8); 33 | } 34 | 35 | @Override 36 | protected String getEncodingName() { 37 | return ENCODING_NAME; 38 | } 39 | 40 | protected String getName() { 41 | return NAME; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/events/PipelineConnectionsChangedEvent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.events; 18 | 19 | import com.fasterxml.jackson.annotation.JsonAutoDetect; 20 | import com.fasterxml.jackson.annotation.JsonCreator; 21 | import com.fasterxml.jackson.annotation.JsonProperty; 22 | import com.google.auto.value.AutoValue; 23 | 24 | import java.util.Set; 25 | 26 | @JsonAutoDetect 27 | @AutoValue 28 | public abstract class PipelineConnectionsChangedEvent { 29 | @JsonProperty("stream_id") 30 | public abstract String streamId(); 31 | 32 | @JsonProperty("pipeline_ids") 33 | public abstract Set pipelineIds(); 34 | 35 | @JsonCreator 36 | public static PipelineConnectionsChangedEvent create(@JsonProperty("stream_id") String streamId, 37 | @JsonProperty("pipeline_ids") Set pipelineIds) { 38 | return new AutoValue_PipelineConnectionsChangedEvent(streamId, pipelineIds); 39 | } 40 | } -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/OrExpression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.expressions; 18 | 19 | import org.antlr.v4.runtime.Token; 20 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 21 | 22 | public class OrExpression extends BinaryExpression implements LogicalExpression { 23 | public OrExpression(Token start, Expression left, 24 | Expression right) { 25 | super(start, left, right); 26 | } 27 | 28 | @Override 29 | public Object evaluateUnsafe(EvaluationContext context) { 30 | return evaluateBool(context); 31 | } 32 | 33 | @Override 34 | public boolean evaluateBool(EvaluationContext context) { 35 | return ((LogicalExpression)left).evaluateBool(context) || ((LogicalExpression)right).evaluateBool(context); 36 | } 37 | 38 | @Override 39 | public Class getType() { 40 | return Boolean.class; 41 | } 42 | 43 | @Override 44 | public String toString() { 45 | return left.toString() + " OR " + right.toString(); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/AndExpression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.expressions; 18 | 19 | import org.antlr.v4.runtime.Token; 20 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 21 | 22 | public class AndExpression extends BinaryExpression implements LogicalExpression { 23 | public AndExpression(Token start, Expression left, 24 | Expression right) { 25 | super(start, left, right); 26 | } 27 | 28 | @Override 29 | public Object evaluateUnsafe(EvaluationContext context) { 30 | return evaluateBool(context); 31 | } 32 | 33 | @Override 34 | public boolean evaluateBool(EvaluationContext context) { 35 | return ((LogicalExpression)left).evaluateBool(context) && ((LogicalExpression)right).evaluateBool(context); 36 | } 37 | 38 | @Override 39 | public Class getType() { 40 | return Boolean.class; 41 | } 42 | 43 | @Override 44 | public String toString() { 45 | return left.toString() + " AND " + right.toString(); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/parser/FunctionRegistry.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.parser; 18 | 19 | import org.graylog.plugins.pipelineprocessor.ast.functions.Function; 20 | 21 | import javax.inject.Inject; 22 | import java.util.Collection; 23 | import java.util.Map; 24 | import java.util.stream.Collectors; 25 | 26 | public class FunctionRegistry { 27 | 28 | private final Map> functions; 29 | 30 | @Inject 31 | public FunctionRegistry(Map> functions) { 32 | this.functions = functions; 33 | } 34 | 35 | 36 | public Function resolve(String name) { 37 | return functions.get(name); 38 | } 39 | 40 | public Function resolveOrError(String name) { 41 | final Function function = resolve(name); 42 | if (function == null) { 43 | return Function.ERROR_FUNCTION; 44 | } 45 | return function; 46 | } 47 | 48 | public Collection> all() { 49 | return functions.values().stream().collect(Collectors.toList()); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/statements/VarAssignStatement.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.statements; 18 | 19 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 20 | import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; 21 | 22 | public class VarAssignStatement implements Statement { 23 | private final String name; 24 | private final Expression expr; 25 | 26 | public VarAssignStatement(String name, Expression expr) { 27 | this.name = name; 28 | this.expr = expr; 29 | } 30 | 31 | @Override 32 | public Void evaluate(EvaluationContext context) { 33 | final Object result = expr.evaluate(context); 34 | context.define(name, expr.getType(), result); 35 | return null; 36 | } 37 | 38 | public String getName() { 39 | return name; 40 | } 41 | 42 | public Expression getValueExpression() { 43 | return expr; 44 | } 45 | 46 | @Override 47 | public String toString() { 48 | return "let " + name + " = " + expr.toString(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/LongExpression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.expressions; 18 | 19 | import org.antlr.v4.runtime.Token; 20 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 21 | 22 | public class LongExpression extends ConstantExpression implements NumericExpression { 23 | private final long value; 24 | 25 | public LongExpression(Token start, long value) { 26 | super(start, Long.class); 27 | this.value = value; 28 | } 29 | 30 | @Override 31 | public Object evaluateUnsafe(EvaluationContext context) { 32 | return value; 33 | } 34 | 35 | @Override 36 | public String toString() { 37 | return Long.toString(value); 38 | } 39 | 40 | @Override 41 | public boolean isIntegral() { 42 | return true; 43 | } 44 | 45 | @Override 46 | public long evaluateLong(EvaluationContext context) { 47 | return value; 48 | } 49 | 50 | @Override 51 | public double evaluateDouble(EvaluationContext context) { 52 | return value; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/DateConversion.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.functions.dates; 2 | 3 | import com.google.common.collect.ImmutableList; 4 | 5 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 6 | import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; 7 | import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; 8 | import org.joda.time.DateTime; 9 | import org.joda.time.DateTimeZone; 10 | 11 | import java.time.ZonedDateTime; 12 | import java.util.Date; 13 | 14 | public class DateConversion extends TimezoneAwareFunction { 15 | 16 | public static final String NAME = "to_date"; 17 | private final ParameterDescriptor value; 18 | 19 | public DateConversion() { 20 | value = ParameterDescriptor.object("value").description("The value to convert to a date").build(); 21 | } 22 | 23 | @Override 24 | protected DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone) { 25 | final Object datish = value.required(args, context); 26 | if (datish instanceof DateTime) { 27 | return (DateTime) datish; 28 | } 29 | if (datish instanceof Date) { 30 | return new DateTime(datish); 31 | } 32 | if (datish instanceof ZonedDateTime) { 33 | return new DateTime(((ZonedDateTime) datish).toInstant().toEpochMilli()); 34 | } 35 | return null; 36 | } 37 | 38 | @Override 39 | protected String description() { 40 | return "Converts a type to a date, useful for $message.timestamp or related message fields."; 41 | } 42 | 43 | @Override 44 | protected String getName() { 45 | return NAME; 46 | } 47 | 48 | @Override 49 | protected ImmutableList params() { 50 | return ImmutableList.of(value); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/periods/AbstractPeriodComponentFunction.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.functions.dates.periods; 2 | 3 | import com.google.common.primitives.Ints; 4 | 5 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 6 | import org.graylog.plugins.pipelineprocessor.ast.functions.AbstractFunction; 7 | import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; 8 | import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor; 9 | import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; 10 | import org.joda.time.Period; 11 | 12 | import javax.annotation.Nonnull; 13 | 14 | public abstract class AbstractPeriodComponentFunction extends AbstractFunction { 15 | 16 | private final ParameterDescriptor value = 17 | ParameterDescriptor 18 | .integer("value", Period.class) 19 | .transform(this::getPeriodOfInt) 20 | .build(); 21 | 22 | private Period getPeriodOfInt(long period) { 23 | return getPeriod(Ints.saturatedCast(period)); 24 | } 25 | 26 | @Nonnull 27 | protected abstract Period getPeriod(int period); 28 | 29 | @Override 30 | public Period evaluate(FunctionArgs args, EvaluationContext context) { 31 | return value.required(args, context); 32 | } 33 | 34 | @Override 35 | public FunctionDescriptor descriptor() { 36 | return FunctionDescriptor.builder() 37 | .name(getName()) 38 | .description(getDescription()) 39 | .pure(true) 40 | .returnType(Period.class) 41 | .params(value) 42 | .build(); 43 | } 44 | 45 | @Nonnull 46 | protected abstract String getName(); 47 | 48 | @Nonnull 49 | protected abstract String getDescription(); 50 | } 51 | -------------------------------------------------------------------------------- /plugin/src/test/resources/org/graylog/plugins/pipelineprocessor/functions/dateArithmetic.txt: -------------------------------------------------------------------------------- 1 | // now() is fixed to "2010-07-30T18:03:25+02:00" to provide a better testing experience 2 | rule "date math" 3 | when 4 | now() + years(1) > now() && 5 | now() + months(1) > now() && 6 | now() + weeks(1) > now() && 7 | now() + days(1) > now() && 8 | now() + hours(1) > now() && 9 | now() + minutes(1) > now() && 10 | now() + seconds(1) > now() && 11 | now() + millis(1) > now() && 12 | now() + period("P1YT1M") > now() && 13 | 14 | now() - years(1) < now() && 15 | now() - months(1) < now() && 16 | now() - weeks(1) < now() && 17 | now() - days(1) < now() && 18 | now() - hours(1) < now() && 19 | now() - minutes(1) < now() && 20 | now() - seconds(1) < now() && 21 | now() - millis(1) < now() && 22 | now() - period("P1YT1M") < now() && 23 | 24 | is_period(years(1)) == true && 25 | is_period(months(1)) == true && 26 | is_period(weeks(1)) == true && 27 | is_period(days(1)) == true && 28 | is_period(hours(1)) == true && 29 | is_period(minutes(1)) == true && 30 | is_period(seconds(1)) == true && 31 | is_period(millis(1)) == true && 32 | is_period(period("P1YT1M")) == true && 33 | is_period("foobar") == false && 34 | is_period(1234) == false && 35 | is_period(12.34) == false && 36 | is_period(true) == false 37 | then 38 | set_field("interval", now() - (now() - days(1))); // is a duration of 1 day 39 | set_field("long_time_ago", now() - years(10000)); 40 | set_fields({ 41 | years: years(2), 42 | months: months(2), 43 | weeks: weeks(2), 44 | days: days(2), 45 | hours: hours(2), 46 | minutes: minutes(2), 47 | seconds: seconds(2), 48 | millis: millis(2), 49 | period: period("P1YT1M") 50 | }); 51 | set_field("timestamp", to_date($message.timestamp) + hours(1)); 52 | 53 | trigger_test(); 54 | end -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/DoubleExpression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.expressions; 18 | 19 | import org.antlr.v4.runtime.Token; 20 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 21 | 22 | public class DoubleExpression extends ConstantExpression implements NumericExpression { 23 | private final double value; 24 | 25 | public DoubleExpression(Token start, double value) { 26 | super(start, Double.class); 27 | this.value = value; 28 | } 29 | 30 | @Override 31 | public Object evaluateUnsafe(EvaluationContext context) { 32 | return value; 33 | } 34 | 35 | @Override 36 | public String toString() { 37 | return Double.toString(value); 38 | } 39 | 40 | @Override 41 | public boolean isIntegral() { 42 | return false; 43 | } 44 | 45 | @Override 46 | public long evaluateLong(EvaluationContext context) { 47 | return (long) value; 48 | } 49 | 50 | @Override 51 | public double evaluateDouble(EvaluationContext context) { 52 | return value; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/functions/dates/Now.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.functions.dates; 18 | 19 | import com.google.common.collect.ImmutableList; 20 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 21 | import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs; 22 | import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; 23 | import org.joda.time.DateTime; 24 | import org.joda.time.DateTimeZone; 25 | 26 | public class Now extends TimezoneAwareFunction { 27 | 28 | public static final String NAME = "now"; 29 | 30 | @Override 31 | protected DateTime evaluate(FunctionArgs args, EvaluationContext context, DateTimeZone timezone) { 32 | return DateTime.now(timezone); 33 | } 34 | 35 | @Override 36 | protected String description() { 37 | return "Returns the current time"; 38 | } 39 | 40 | @Override 41 | protected String getName() { 42 | return NAME; 43 | } 44 | 45 | @Override 46 | protected ImmutableList params() { 47 | return ImmutableList.of(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/processors/listeners/InterpreterListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.processors.listeners; 18 | 19 | import org.graylog.plugins.pipelineprocessor.ast.Pipeline; 20 | import org.graylog.plugins.pipelineprocessor.ast.Rule; 21 | import org.graylog.plugins.pipelineprocessor.ast.Stage; 22 | import org.graylog2.plugin.Message; 23 | 24 | import java.util.Set; 25 | 26 | public interface InterpreterListener { 27 | void startProcessing(); 28 | void finishProcessing(); 29 | void processStreams(Message message, Set pipelines, Set streams); 30 | void enterStage(Stage stage); 31 | void exitStage(Stage stage); 32 | void evaluateRule(Rule rule, Pipeline pipeline); 33 | void failEvaluateRule(Rule rule, Pipeline pipeline); 34 | void satisfyRule(Rule rule, Pipeline pipeline); 35 | void dissatisfyRule(Rule rule, Pipeline pipeline); 36 | void executeRule(Rule rule, Pipeline pipeline); 37 | void failExecuteRule(Rule rule, Pipeline pipeline); 38 | void continuePipelineExecution(Pipeline pipeline, Stage stage); 39 | void stopPipelineExecution(Pipeline pipeline, Stage stage); 40 | } 41 | -------------------------------------------------------------------------------- /plugin/src/web/pipelines/PipelineDetails.jsx: -------------------------------------------------------------------------------- 1 | import PropTypes from 'prop-types'; 2 | import React from 'react'; 3 | import { Row, Col } from 'react-bootstrap'; 4 | 5 | import { Timestamp } from 'components/common'; 6 | import PipelineForm from './PipelineForm'; 7 | 8 | import { MetricContainer, CounterRate } from 'components/metrics'; 9 | 10 | const PipelineDetails = React.createClass({ 11 | propTypes: { 12 | pipeline: PropTypes.object, 13 | create: PropTypes.bool, 14 | onChange: PropTypes.func.isRequired, 15 | onCancel: PropTypes.func, 16 | }, 17 | 18 | render() { 19 | if (this.props.create) { 20 | return ; 21 | } 22 | 23 | const pipeline = this.props.pipeline; 24 | return ( 25 |
26 | 27 | 28 |
29 | 30 |
31 |

Details

32 |
33 |
Title
34 |
{pipeline.title}
35 |
Description
36 |
{pipeline.description}
37 |
Created
38 |
39 |
Last modified
40 |
41 |
Current throughput
42 |
43 | 44 | 45 | 46 |
47 |
48 | 49 | 50 |
51 |
52 | ); 53 | }, 54 | }); 55 | 56 | export default PipelineDetails; 57 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/IncompatibleTypes.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.parser.errors; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | import org.graylog.plugins.pipelineprocessor.ast.expressions.BinaryExpression; 21 | import org.graylog.plugins.pipelineprocessor.ast.expressions.Expression; 22 | import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; 23 | 24 | public class IncompatibleTypes extends ParseError { 25 | private final RuleLangParser.ExpressionContext ctx; 26 | private final BinaryExpression binaryExpr; 27 | 28 | public IncompatibleTypes(RuleLangParser.ExpressionContext ctx, BinaryExpression binaryExpr) { 29 | super("incompatible_types", ctx); 30 | this.ctx = ctx; 31 | this.binaryExpr = binaryExpr; 32 | } 33 | 34 | @JsonProperty("reason") 35 | @Override 36 | public String toString() { 37 | return "Incompatible types " + exprString(binaryExpr.left()) + " <=> " + exprString(binaryExpr.right()) + positionString(); 38 | } 39 | 40 | private String exprString(Expression e) { 41 | return "(" + e.toString() + ") : " + e.getType().getSimpleName(); 42 | } 43 | 44 | 45 | } 46 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/MissingRequiredParam.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.parser.errors; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | import org.graylog.plugins.pipelineprocessor.ast.functions.Function; 21 | import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; 22 | import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; 23 | 24 | public class MissingRequiredParam extends ParseError { 25 | private final Function function; 26 | private final ParameterDescriptor param; 27 | 28 | public MissingRequiredParam(RuleLangParser.FunctionCallContext ctx, 29 | Function function, 30 | ParameterDescriptor param) { 31 | super("missing_required_param", ctx); 32 | this.function = function; 33 | this.param = param; 34 | } 35 | 36 | @JsonProperty("reason") 37 | @Override 38 | public String toString() { 39 | return "Missing required parameter " + param.name() + 40 | " of type " + param.type().getSimpleName() + 41 | " in call to function " + function.descriptor().name() 42 | + positionString(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/FieldRefExpression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.expressions; 18 | 19 | import org.antlr.v4.runtime.Token; 20 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 21 | 22 | import java.util.Collections; 23 | 24 | public class FieldRefExpression extends BaseExpression { 25 | private final String variableName; 26 | private final Expression fieldExpr; 27 | 28 | public FieldRefExpression(Token start, String variableName, Expression fieldExpr) { 29 | super(start); 30 | this.variableName = variableName; 31 | this.fieldExpr = fieldExpr; 32 | } 33 | 34 | @Override 35 | public boolean isConstant() { 36 | return true; 37 | } 38 | 39 | @Override 40 | public Object evaluateUnsafe(EvaluationContext context) { 41 | return variableName; 42 | } 43 | 44 | @Override 45 | public Class getType() { 46 | return String.class; 47 | } 48 | 49 | @Override 50 | public String toString() { 51 | return variableName; 52 | } 53 | 54 | public String fieldName() { 55 | return variableName; 56 | } 57 | 58 | @Override 59 | public Iterable children() { 60 | return Collections.emptySet(); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /plugin/src/web/pipelines/PipelinesOverviewPage.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Row, Col, Button } from 'react-bootstrap'; 3 | import { LinkContainer } from 'react-router-bootstrap'; 4 | 5 | import { DocumentTitle, PageHeader } from 'components/common'; 6 | import DocumentationLink from 'components/support/DocumentationLink'; 7 | import ProcessingTimelineComponent from './ProcessingTimelineComponent'; 8 | 9 | import Routes from 'routing/Routes'; 10 | import DocsHelper from 'util/DocsHelper'; 11 | 12 | const PipelinesOverviewPage = React.createClass({ 13 | render() { 14 | return ( 15 | 16 |
17 | 18 | 19 | Pipelines let you transform and process messages coming from streams. Pipelines consist of stages where 20 | rules are evaluated and applied. Messages can go through one or more stages. 21 | 22 | 23 | Read more about Graylog pipelines in the . 24 | 25 | 26 | 27 | 28 | 29 | 30 |   31 | 32 | 33 | 34 |   35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 |
47 | 48 | ); 49 | }, 50 | }); 51 | 52 | export default PipelinesOverviewPage; 53 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/db/memory/InMemoryPipelineStreamConnectionsService.java: -------------------------------------------------------------------------------- 1 | package org.graylog.plugins.pipelineprocessor.db.memory; 2 | 3 | import com.google.common.collect.ImmutableSet; 4 | import com.google.common.collect.MapMaker; 5 | import org.graylog.plugins.pipelineprocessor.db.PipelineStreamConnectionsService; 6 | import org.graylog.plugins.pipelineprocessor.rest.PipelineConnections; 7 | import org.graylog2.database.NotFoundException; 8 | 9 | import java.util.Map; 10 | import java.util.Set; 11 | import java.util.concurrent.atomic.AtomicLong; 12 | 13 | public class InMemoryPipelineStreamConnectionsService implements PipelineStreamConnectionsService { 14 | 15 | // poor man's id generator 16 | private AtomicLong idGen = new AtomicLong(0); 17 | 18 | private Map store = new MapMaker().makeMap(); 19 | 20 | @Override 21 | public PipelineConnections save(PipelineConnections connections) { 22 | PipelineConnections toSave = connections.id() != null 23 | ? connections 24 | : connections.toBuilder().id(createId()).build(); 25 | store.put(toSave.id(), toSave); 26 | 27 | return toSave; 28 | } 29 | 30 | @Override 31 | public PipelineConnections load(String streamId) throws NotFoundException { 32 | final PipelineConnections connections = store.get(streamId); 33 | if (connections == null) { 34 | throw new NotFoundException("No such pipeline connections for stream " + streamId); 35 | } 36 | return connections; 37 | } 38 | 39 | @Override 40 | public Set loadAll() { 41 | return ImmutableSet.copyOf(store.values()); 42 | } 43 | 44 | @Override 45 | public void delete(String streamId) { 46 | try { 47 | final PipelineConnections connections = load(streamId); 48 | store.remove(connections.id()); 49 | } catch (NotFoundException e) { 50 | // Do nothing 51 | } 52 | } 53 | 54 | private String createId() { 55 | return String.valueOf(idGen.incrementAndGet()); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/SyntaxError.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.parser.errors; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | import org.antlr.v4.runtime.ParserRuleContext; 21 | import org.antlr.v4.runtime.RecognitionException; 22 | 23 | import javax.annotation.Nullable; 24 | 25 | public class SyntaxError extends ParseError { 26 | 27 | private final Object offendingSymbol; 28 | private final int line; 29 | private final int charPositionInLine; 30 | private final String msg; 31 | private final RecognitionException e; 32 | 33 | public SyntaxError(@Nullable Object offendingSymbol, int line, int charPositionInLine, String msg, @Nullable RecognitionException e) { 34 | super("syntax_error", new ParserRuleContext()); 35 | 36 | this.offendingSymbol = offendingSymbol; 37 | this.line = line; 38 | this.charPositionInLine = charPositionInLine; 39 | this.msg = msg; 40 | this.e = e; 41 | } 42 | 43 | @Override 44 | public int line() { 45 | return line; 46 | } 47 | 48 | @Override 49 | public int positionInLine() { 50 | return charPositionInLine; 51 | } 52 | 53 | @JsonProperty("reason") 54 | @Override 55 | public String toString() { 56 | return msg; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/WrongNumberOfArgs.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.parser.errors; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | import org.graylog.plugins.pipelineprocessor.ast.functions.Function; 21 | import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor; 22 | import org.graylog.plugins.pipelineprocessor.parser.RuleLangParser; 23 | 24 | import java.util.function.Predicate; 25 | 26 | public class WrongNumberOfArgs extends ParseError { 27 | private final Function function; 28 | private final int argCount; 29 | 30 | public WrongNumberOfArgs(RuleLangParser.FunctionCallContext ctx, 31 | Function function, 32 | int argCount) { 33 | super("wrong_number_of_arguments", ctx); 34 | this.function = function; 35 | this.argCount = argCount; 36 | } 37 | 38 | @JsonProperty("reason") 39 | @Override 40 | public String toString() { 41 | final Predicate optional = ParameterDescriptor::optional; 42 | return "Expected " + function.descriptor().params().stream().filter(optional.negate()).count() + 43 | " arguments but found " + argCount + 44 | " in call to function " + function.descriptor().name() 45 | + positionString(); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/MessageRefExpression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.expressions; 18 | 19 | import org.antlr.v4.runtime.Token; 20 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 21 | 22 | import java.util.Collections; 23 | 24 | public class MessageRefExpression extends BaseExpression { 25 | private final Expression fieldExpr; 26 | 27 | public MessageRefExpression(Token start, Expression fieldExpr) { 28 | super(start); 29 | this.fieldExpr = fieldExpr; 30 | } 31 | 32 | @Override 33 | public boolean isConstant() { 34 | return false; 35 | } 36 | 37 | @Override 38 | public Object evaluateUnsafe(EvaluationContext context) { 39 | final Object fieldName = fieldExpr.evaluateUnsafe(context); 40 | if (fieldName == null) { 41 | return null; 42 | } 43 | return context.currentMessage().getField(fieldName.toString()); 44 | } 45 | 46 | @Override 47 | public Class getType() { 48 | return Object.class; 49 | } 50 | 51 | @Override 52 | public String toString() { 53 | return "$message." + fieldExpr.toString(); 54 | } 55 | 56 | public Expression getFieldExpr() { 57 | return fieldExpr; 58 | } 59 | 60 | @Override 61 | public Iterable children() { 62 | return Collections.singleton(fieldExpr); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### OSX template 3 | .DS_Store 4 | .AppleDouble 5 | .LSOverride 6 | 7 | # Icon must end with two \r 8 | Icon 9 | 10 | # Thumbnails 11 | ._* 12 | 13 | # Files that might appear in the root of a volume 14 | .DocumentRevisions-V100 15 | .fseventsd 16 | .Spotlight-V100 17 | .TemporaryItems 18 | .Trashes 19 | .VolumeIcon.icns 20 | 21 | # Directories potentially created on remote AFP share 22 | .AppleDB 23 | .AppleDesktop 24 | Network Trash Folder 25 | Temporary Items 26 | .apdisk 27 | ### Java template 28 | *.class 29 | 30 | # Mobile Tools for Java (J2ME) 31 | .mtj.tmp/ 32 | 33 | # Package Files # 34 | *.jar 35 | *.war 36 | *.ear 37 | 38 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 39 | hs_err_pid* 40 | ### JetBrains template 41 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio 42 | 43 | *.iml 44 | 45 | ## Directory-based project format: 46 | .idea/ 47 | # if you remove the above rule, at least ignore the following: 48 | 49 | # User-specific stuff: 50 | # .idea/workspace.xml 51 | # .idea/tasks.xml 52 | # .idea/dictionaries 53 | 54 | # Sensitive or high-churn files: 55 | # .idea/dataSources.ids 56 | # .idea/dataSources.xml 57 | # .idea/sqlDataSources.xml 58 | # .idea/dynamic.xml 59 | # .idea/uiDesigner.xml 60 | 61 | # Gradle: 62 | # .idea/gradle.xml 63 | # .idea/libraries 64 | 65 | # Mongo Explorer plugin: 66 | # .idea/mongoSettings.xml 67 | 68 | ## File-based project format: 69 | *.ipr 70 | *.iws 71 | 72 | ## Plugin-specific files: 73 | 74 | # IntelliJ 75 | /out/ 76 | 77 | # mpeltonen/sbt-idea plugin 78 | .idea_modules/ 79 | 80 | # JIRA plugin 81 | atlassian-ide-plugin.xml 82 | 83 | # Crashlytics plugin (for Android Studio and IntelliJ) 84 | com_crashlytics_export_strings.xml 85 | crashlytics.properties 86 | crashlytics-build.properties 87 | 88 | ## Maven 89 | 90 | target/ 91 | pom.xml.tag 92 | pom.xml.releaseBackup 93 | pom.xml.versionsBackup 94 | pom.xml.next 95 | release.properties 96 | dependency-reduced-pom.xml 97 | buildNumber.properties 98 | .mvn/timing.properties 99 | 100 | node_modules 101 | node 102 | build 103 | plugin/cache 104 | 105 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/ast/expressions/ArrayLiteralExpression.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.ast.expressions; 18 | 19 | import com.google.common.base.Joiner; 20 | import com.google.common.collect.ImmutableList; 21 | 22 | import org.antlr.v4.runtime.Token; 23 | import org.graylog.plugins.pipelineprocessor.EvaluationContext; 24 | 25 | import java.util.List; 26 | import java.util.stream.Collectors; 27 | 28 | public class ArrayLiteralExpression extends BaseExpression { 29 | private final List elements; 30 | 31 | public ArrayLiteralExpression(Token start, List elements) { 32 | super(start); 33 | this.elements = elements; 34 | } 35 | 36 | @Override 37 | public boolean isConstant() { 38 | return elements.stream().allMatch(Expression::isConstant); 39 | } 40 | 41 | @Override 42 | public List evaluateUnsafe(EvaluationContext context) { 43 | return elements.stream() 44 | .map(expression -> expression.evaluateUnsafe(context)) 45 | .collect(Collectors.toList()); 46 | } 47 | 48 | @Override 49 | public Class getType() { 50 | return List.class; 51 | } 52 | 53 | @Override 54 | public String toString() { 55 | return "[" + Joiner.on(", ").join(elements) + "]"; 56 | } 57 | 58 | @Override 59 | public Iterable children() { 60 | return ImmutableList.copyOf(elements); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/audit/PipelineProcessorAuditEventTypes.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.audit; 18 | 19 | import com.google.common.collect.ImmutableSet; 20 | import org.graylog2.audit.PluginAuditEventTypes; 21 | 22 | import java.util.Set; 23 | 24 | public class PipelineProcessorAuditEventTypes implements PluginAuditEventTypes { 25 | private static final String NAMESPACE = "pipeline_processor:"; 26 | 27 | public static final String PIPELINE_CONNECTION_UPDATE = NAMESPACE + "pipeline_connection:update"; 28 | public static final String PIPELINE_CREATE = NAMESPACE + "pipeline:create"; 29 | public static final String PIPELINE_UPDATE = NAMESPACE + "pipeline:update"; 30 | public static final String PIPELINE_DELETE = NAMESPACE + "pipeline:delete"; 31 | public static final String RULE_CREATE = NAMESPACE + "rule:create"; 32 | public static final String RULE_UPDATE = NAMESPACE + "rule:update"; 33 | public static final String RULE_DELETE = NAMESPACE + "rule:delete"; 34 | 35 | private static final Set EVENT_TYPES = ImmutableSet.builder() 36 | .add(PIPELINE_CONNECTION_UPDATE) 37 | .add(PIPELINE_CREATE) 38 | .add(PIPELINE_UPDATE) 39 | .add(PIPELINE_DELETE) 40 | .add(RULE_CREATE) 41 | .add(RULE_UPDATE) 42 | .add(RULE_DELETE) 43 | .build(); 44 | 45 | @Override 46 | public Set auditEventTypes() { 47 | return EVENT_TYPES; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /plugin/src/main/java/org/graylog/plugins/pipelineprocessor/parser/errors/ParseError.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of Graylog Pipeline Processor. 3 | * 4 | * Graylog Pipeline Processor is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License as published by 6 | * the Free Software Foundation, either version 3 of the License, or 7 | * (at your option) any later version. 8 | * 9 | * Graylog Pipeline Processor is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with Graylog Pipeline Processor. If not, see . 16 | */ 17 | package org.graylog.plugins.pipelineprocessor.parser.errors; 18 | 19 | import com.fasterxml.jackson.annotation.JsonIgnore; 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | import org.antlr.v4.runtime.ParserRuleContext; 22 | 23 | import java.util.Objects; 24 | 25 | public abstract class ParseError { 26 | 27 | @JsonProperty 28 | private final String type; 29 | 30 | @JsonIgnore 31 | private final ParserRuleContext ctx; 32 | 33 | protected ParseError(String type, ParserRuleContext ctx) { 34 | this.type = type; 35 | this.ctx = ctx; 36 | } 37 | 38 | @JsonProperty 39 | public int line() { 40 | return ctx.getStart().getLine(); 41 | } 42 | 43 | @JsonProperty 44 | public int positionInLine() { 45 | return ctx.getStart().getCharPositionInLine(); 46 | } 47 | 48 | protected String positionString() { 49 | return " in" + 50 | " line " + line() + 51 | " pos " + positionInLine(); 52 | } 53 | 54 | @Override 55 | public boolean equals(Object o) { 56 | if (this == o) return true; 57 | if (!(o instanceof ParseError)) return false; 58 | ParseError that = (ParseError) o; 59 | return Objects.equals(type, that.type) && 60 | Objects.equals(ctx, that.ctx); 61 | } 62 | 63 | @Override 64 | public int hashCode() { 65 | return Objects.hash(type, ctx); 66 | } 67 | } 68 | --------------------------------------------------------------------------------