├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── pom.xml └── src ├── main └── java │ └── com │ └── ottogroup │ └── bi │ └── streaming │ ├── operator │ ├── json │ │ ├── JsonContentReference.java │ │ ├── JsonContentType.java │ │ ├── JsonProcessingUtils.java │ │ ├── aggregate │ │ │ ├── AggregatorConfiguration.java │ │ │ ├── ContentAggregator.java │ │ │ ├── FieldAggregationConfiguration.java │ │ │ ├── WindowedJsonContentAggregator.java │ │ │ └── functions │ │ │ │ ├── BooleanContentAggregateFunction.java │ │ │ │ ├── DoubleContentAggregateFunction.java │ │ │ │ ├── IntegerContentAggregateFunction.java │ │ │ │ ├── JsonContentAggregateFunction.java │ │ │ │ ├── StringContentAggregateFunction.java │ │ │ │ └── TimestampContentAggregateFunction.java │ │ ├── converter │ │ │ ├── JsonObjectToByteArray.java │ │ │ ├── JsonObjectToString.java │ │ │ ├── StringToByteArray.java │ │ │ └── StringToJsonObject.java │ │ ├── csv │ │ │ └── Json2CsvConverter.java │ │ ├── decode │ │ │ ├── Base64ContentDecoder.java │ │ │ └── Base64ContentDecoderConfiguration.java │ │ ├── filter │ │ │ ├── JsonContentFilter.java │ │ │ ├── RegularExpressionMatcher.java │ │ │ └── cfg │ │ │ │ ├── FieldConditionCombiner.java │ │ │ │ ├── FieldConditionCombinerConfiguration.java │ │ │ │ ├── FieldConditionConfiguration.java │ │ │ │ ├── FieldConditionOperator.java │ │ │ │ └── JsonContentFilterConfiguration.java │ │ ├── insert │ │ │ └── JsonStaticContentInsertion.java │ │ ├── partitioning │ │ │ └── JsonKeySelector.java │ │ ├── project │ │ │ ├── JsonContentProjectionMapper.java │ │ │ └── ProjectionMapperConfiguration.java │ │ └── statsd │ │ │ ├── StatsdExtractedMetricsReporter.java │ │ │ ├── StatsdExtractedMetricsReporterConfiguration.java │ │ │ ├── StatsdMetricConfig.java │ │ │ └── StatsdMetricType.java │ └── metrics │ │ └── MessageCountingMetricsReporter.java │ ├── runtime │ ├── StreamingAppConfiguration.java │ └── StreamingAppRuntime.java │ ├── sink │ ├── elasticsearch │ │ ├── ElasticsearchNodeAddress.java │ │ ├── ElasticsearchSink.java │ │ └── ElasticsearchSinkConfiguration.java │ └── kafka │ │ ├── KafkaProducerBuilder.java │ │ └── KafkaProducerConfiguration.java │ ├── source │ └── kafka │ │ ├── KafkaConsumerBuilder.java │ │ └── KafkaConsumerConfiguration.java │ └── testing │ ├── JSONFieldContentMatcher.java │ └── MatchJSONContent.java └── test ├── java └── com │ └── ottogroup │ └── bi │ └── streaming │ ├── operator │ ├── json │ │ ├── JsonProcessingUtilsTest.java │ │ ├── aggregate │ │ │ ├── WindowedJsonContentAggregatorTest.java │ │ │ └── functions │ │ │ │ ├── BooleanContentAggregateFunctionTest.java │ │ │ │ ├── DoubleContentAggregateFunctionTest.java │ │ │ │ ├── IntegerContentAggregateFunctionTest.java │ │ │ │ ├── StringContentAggregateFunctionTest.java │ │ │ │ └── TimestampContentAggregateFunctionTest.java │ │ ├── converter │ │ │ ├── JsonObjectToByteArrayTest.java │ │ │ ├── JsonObjectToStringTest.java │ │ │ ├── StringToByteArrayTest.java │ │ │ └── StringToJsonObjectTest.java │ │ ├── csv │ │ │ └── Json2CsvConverterTest.java │ │ ├── decode │ │ │ └── Base64ContentDecoderTest.java │ │ ├── filter │ │ │ ├── JsonContentFilterTest.java │ │ │ └── RegularExpressionMatcherTest.java │ │ ├── insert │ │ │ └── JsonStaticContentInsertionTest.java │ │ ├── partitioning │ │ │ └── JsonKeySelectorTest.java │ │ ├── project │ │ │ └── JsonContentProjectionMapperTest.java │ │ └── statsd │ │ │ └── StatsdExtractedMetricsReporterTest.java │ └── metrics │ │ └── MessageCountingMetricsReporterTest.java │ ├── runtime │ └── StreamingAppRuntimeTest.java │ ├── sink │ ├── elasticsearch │ │ └── ElasticsearchSinkTest.java │ └── kafka │ │ └── KafkaProducerBuilderTest.java │ ├── source │ └── kafka │ │ └── KafkaConsumerBuilderTest.java │ └── testing │ ├── JSONFieldContentMatcherTest.java │ ├── MatchJSONContentFlinkSpectorTest.java │ └── MatchJSONContentTest.java └── resources ├── invalid-test-configuration.cfg ├── log4j.properties └── valid-test-configuration.cfg /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | 3 | # Mobile Tools for Java (J2ME) 4 | .mtj.tmp/ 5 | 6 | # Package Files # 7 | *.jar 8 | *.war 9 | *.ear 10 | 11 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 12 | hs_err_pid* 13 | /target/ 14 | /.settings/ 15 | /.classpath 16 | /.project 17 | /bin/ 18 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - oraclejdk8 4 | sudo: true -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/ottogroup/flink-operator-library.svg?branch=master)](https://travis-ci.org/ottogroup/flink-operator-library) 2 | 3 | # flink-operator-library 4 | 5 | The library provides a set of [Apache Flink](https://flink.apache.org) operators and supporting utilities. Please visit the [wiki](https://github.com/ottogroup/flink-operator-library/wiki/) to learn more about the implemented components. Amongst them you will find 6 | 7 | * [a JSON based content filter](https://github.com/ottogroup/flink-operator-library/wiki/JSON-Content-Filter-operator) 8 | * [an integration with flink-spector to allow testing of JSON based operators](https://github.com/ottogroup/flink-operator-library/wiki/JSON%20Content%20Matcher%20to%20integrate%20with%20Flink%20Spector%20for%20operator%20and%20pipeline%20testing) 9 | * [JSON processing utilities](https://github.com/ottogroup/flink-operator-library/wiki/JSON%20processing%20utilities) 10 | * [an abstract runtime foundation for streaming applications to speed up development](https://github.com/ottogroup/flink-operator-library/wiki/Base-runtime-for-streaming-applications) 11 | * [a selector implementation to extract keys from JSON documents](https://github.com/ottogroup/flink-operator-library/wiki/JSON-document-backed-key-selector) 12 | * [a JSON to character separated string converter](https://github.com/ottogroup/flink-operator-library/wiki/JSON-to-CSV-conversion-operator) 13 | * [an operator to insert static content into existing JSON documents](https://github.com/ottogroup/flink-operator-library/wiki/Static-content-insertion-into-existing-JSON-documents) 14 | * [a window-based JSON content aggregator](https://github.com/ottogroup/flink-operator-library/wiki/Window-based-JSON-content-aggregation) 15 | * [some JSON content mappers](https://github.com/ottogroup/flink-operator-library/wiki/JSON-Content-Mappers) 16 | 17 | To integrate the library with maven based projects, please add these sections: 18 | 19 | ```json 20 | 21 | 22 | otto-bintray 23 | https://dl.bintray.com/ottogroup/maven 24 | 25 | 26 | ``` 27 | 28 | For use with Scala v2.10 29 | ```json 30 | 31 | 32 | com.ottogroup.bi.streaming 33 | flink-operator-library 34 | 0.3.4_2.10 35 | 36 | 37 | ``` 38 | 39 | For use with Scala v2.11 40 | ```json 41 | 42 | 43 | com.ottogroup.bi.streaming 44 | flink-operator-library 45 | 0.3.4_2.11 46 | 47 | 48 | ``` -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/JsonContentReference.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json; 18 | 19 | import java.io.Serializable; 20 | 21 | import org.apache.sling.commons.json.JSONObject; 22 | 23 | /** 24 | * Describes a reference into a {@link JSONObject} by providing a path as array of {@link String} 25 | * elements and the expected {@link JsonContentType} 26 | * @author mnxfst 27 | * @since Jan 21, 2016 28 | */ 29 | public class JsonContentReference implements Serializable { 30 | 31 | private static final long serialVersionUID = -5075723737186875253L; 32 | 33 | private String[] path = null; 34 | private JsonContentType contentType = null; 35 | private String conversionPattern = null; 36 | /** enforce strict evaluation and require the field to be present */ 37 | private boolean required = false; 38 | 39 | public JsonContentReference() { 40 | } 41 | 42 | public JsonContentReference(final String[] path, final JsonContentType contentType) { 43 | this(path, contentType, false); 44 | } 45 | 46 | public JsonContentReference(final String[] path, final JsonContentType contentType, final boolean required) { 47 | this.path = path; 48 | this.contentType = contentType; 49 | this.required = required; 50 | } 51 | 52 | public JsonContentReference(final String[] path, final JsonContentType contentType, final String conversionPattern) { 53 | this.path = path; 54 | this.contentType = contentType; 55 | this.conversionPattern = conversionPattern; 56 | } 57 | 58 | public JsonContentReference(final String[] path, final JsonContentType contentType, final String conversionPattern, final boolean required) { 59 | this.path = path; 60 | this.contentType = contentType; 61 | this.conversionPattern = conversionPattern; 62 | this.required = required; 63 | } 64 | 65 | public String[] getPath() { 66 | return path; 67 | } 68 | 69 | public void setPath(String[] path) { 70 | this.path = path; 71 | } 72 | 73 | public JsonContentType getContentType() { 74 | return contentType; 75 | } 76 | 77 | public void setContentType(JsonContentType contentType) { 78 | this.contentType = contentType; 79 | } 80 | 81 | public String getConversionPattern() { 82 | return conversionPattern; 83 | } 84 | 85 | public void setConversionPattern(String conversionPattern) { 86 | this.conversionPattern = conversionPattern; 87 | } 88 | 89 | public boolean isRequired() { 90 | return required; 91 | } 92 | 93 | public void setRequired(boolean required) { 94 | this.required = required; 95 | } 96 | 97 | 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/JsonContentType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json; 18 | 19 | import java.io.Serializable; 20 | 21 | /** 22 | * Supported value types: STRING, INTEGER, DOUBLE, TIMESTAMP, BOOLEAN 23 | * @author mnxfst 24 | * @since Jan 13, 2016 25 | */ 26 | public enum JsonContentType implements Serializable { 27 | STRING, INTEGER, DOUBLE, TIMESTAMP, BOOLEAN 28 | } -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/aggregate/AggregatorConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.aggregate; 18 | 19 | import java.io.Serializable; 20 | import java.util.ArrayList; 21 | import java.util.HashMap; 22 | import java.util.List; 23 | import java.util.Map; 24 | 25 | import org.apache.commons.lang3.StringUtils; 26 | 27 | import com.fasterxml.jackson.databind.ObjectMapper; 28 | import com.ottogroup.bi.streaming.operator.json.JsonContentReference; 29 | import com.ottogroup.bi.streaming.operator.json.JsonContentType; 30 | 31 | /** 32 | * Structure to keep all configuration required for setting up an instance of {@link WindowedJsonContentAggregator} 33 | * @author mnxfst 34 | * @since Jan 21, 2016 35 | */ 36 | public class AggregatorConfiguration implements Serializable { 37 | 38 | private static final long serialVersionUID = 7886758725763674809L; 39 | 40 | /** name that will be assigned to output element */ 41 | private String outputElement = null; 42 | /** pointer towards a number of fields to group the computation results by */ 43 | private List groupByFields = new ArrayList<>(); 44 | /** aggregation configuration for a number of fields that must be processed */ 45 | private List fieldAggregations = new ArrayList<>(); 46 | /** optional fields that must be added to each output document */ 47 | private Map optionalFields = new HashMap<>(); 48 | /** append raw messages */ 49 | private boolean raw = false; 50 | /** timestamp pattern in case the current time is requested as optional field */ 51 | private String timestampPattern = "yyyy-MM-dd'T'HH:mm:ss.SSSZ"; 52 | 53 | public AggregatorConfiguration() { 54 | } 55 | 56 | public AggregatorConfiguration(final String outputElement) { 57 | this.outputElement = outputElement; 58 | } 59 | 60 | public AggregatorConfiguration(final String outputElement, final boolean raw) { 61 | this.outputElement = outputElement; 62 | this.raw = raw; 63 | } 64 | 65 | public AggregatorConfiguration(final String outputElement, final String timestampPattern) { 66 | this.outputElement = outputElement; 67 | this.timestampPattern = timestampPattern; 68 | } 69 | 70 | public AggregatorConfiguration(final String outputElement, final String timestampPattern, final boolean raw) { 71 | this.outputElement = outputElement; 72 | this.timestampPattern = timestampPattern; 73 | this.raw = raw; 74 | } 75 | 76 | /** 77 | * Adds a new element to group results by. The method does not validate for 78 | * already existing configurations showing the same settings 79 | * @param contentReference 80 | */ 81 | public void addGroupByField(final JsonContentReference contentReference) { 82 | this.groupByFields.add(contentReference); 83 | } 84 | 85 | /** 86 | * Adds a new element to aggregate values from. The method does not validate for 87 | * already existing configurations showing the same settings 88 | * @param fieldAggregation 89 | */ 90 | public void addFieldAggregation(final FieldAggregationConfiguration fieldAggregation) { 91 | this.fieldAggregations.add(fieldAggregation); 92 | } 93 | 94 | /** 95 | * Adds the provided element to list of optional fields along with the value 96 | * that must be assigned when exporting the document. The value may be a constant value 97 | * or reference a default like {@link WindowedJsonContentAggregator#OPTIONAL_FIELD_TYPE_TIMESTAMP 98 | * or {@link WindowedJsonContentAggregator#OPTIONAL_FIELD_TYPE_TOTAL_MESSAGE_COUNT}. The first 99 | * one adds the current time the second one adds the message count of the current window 100 | * @param outputElement 101 | * @param outputValue 102 | */ 103 | public void addOptionalField(final String outputElement, final String outputValue) { 104 | if(StringUtils.isNotBlank(outputElement)) 105 | this.optionalFields.put(outputElement, outputValue); 106 | } 107 | 108 | public String getOutputElement() { 109 | return outputElement; 110 | } 111 | 112 | public void setOutputElement(String outputElement) { 113 | this.outputElement = outputElement; 114 | } 115 | 116 | public List getGroupByFields() { 117 | return groupByFields; 118 | } 119 | 120 | public void setGroupByFields(List groupByFields) { 121 | this.groupByFields = groupByFields; 122 | } 123 | 124 | public List getFieldAggregations() { 125 | return fieldAggregations; 126 | } 127 | 128 | public void setFieldAggregations(List fieldAggregations) { 129 | this.fieldAggregations = fieldAggregations; 130 | } 131 | 132 | public Map getOptionalFields() { 133 | return optionalFields; 134 | } 135 | 136 | public void setOptionalFields(Map optionalFields) { 137 | this.optionalFields = optionalFields; 138 | } 139 | 140 | public String getTimestampPattern() { 141 | return timestampPattern; 142 | } 143 | 144 | public void setTimestampPattern(String timestampPattern) { 145 | this.timestampPattern = timestampPattern; 146 | } 147 | 148 | public boolean isRaw() { 149 | return raw; 150 | } 151 | 152 | public void setRaw(boolean raw) { 153 | this.raw = raw; 154 | } 155 | 156 | public static void main(String[] args) throws Exception { 157 | AggregatorConfiguration cfg = new AggregatorConfiguration(); 158 | 159 | cfg.setOutputElement("output-element"); 160 | cfg.setRaw(true); 161 | cfg.setTimestampPattern("yyyy-MM-dd"); 162 | cfg.addFieldAggregation(new FieldAggregationConfiguration( "aggregated-field-1", 163 | new JsonContentReference(new String[]{"path", "to", "field"}, JsonContentType.STRING))); 164 | cfg.addFieldAggregation(new FieldAggregationConfiguration( "aggregated-field-1", 165 | new JsonContentReference(new String[]{"path", "to", "anotherField"}, JsonContentType.STRING))); 166 | cfg.addGroupByField(new JsonContentReference(new String[]{"path", "to", "field"}, JsonContentType.STRING)); 167 | cfg.addOptionalField("staticContent", "staticValue"); 168 | System.out.println(new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(cfg)); 169 | 170 | } 171 | 172 | } 173 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/aggregate/ContentAggregator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.aggregate; 18 | 19 | import java.io.Serializable; 20 | 21 | /** 22 | * List of available aggregation functions. The library currently provides support for 23 | *
    24 | *
  • SUM - sums up numerical field content
    • 25 | *
  • COUNT - counts the number of occurrences of a referenced field
    • 26 | *
  • MIN - holds the minimum value of a referenced field
    • 27 | *
  • MAX - holds the maximum value of a referenced field
    • 28 | *
  • AVG - holds the average value of a referenced field
    • 29 | *
30 | * Note: all functions are defined over time 31 | * @author mnxfst 32 | * @since Jan 14, 2016 33 | */ 34 | public enum ContentAggregator implements Serializable { 35 | 36 | SUM, COUNT, MIN, MAX, AVG 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/aggregate/FieldAggregationConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.aggregate; 18 | 19 | import java.io.Serializable; 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | import org.apache.sling.commons.json.JSONObject; 24 | 25 | import com.ottogroup.bi.streaming.operator.json.JsonContentReference; 26 | 27 | /** 28 | * Provides information on how to aggregate a specific element within a {@link JSONObject} 29 | * @author mnxfst 30 | * @since Jan 21, 2016 31 | */ 32 | public class FieldAggregationConfiguration implements Serializable { 33 | 34 | private static final long serialVersionUID = -8937620821459847616L; 35 | 36 | /** name that will be assigned to output element */ 37 | private String outputElement = null; 38 | /** reference that points towards the field that must be aggregated */ 39 | private JsonContentReference aggregateField = null; 40 | /** list of methods to apply on the referenced field */ 41 | private List methods = new ArrayList<>(); 42 | 43 | public FieldAggregationConfiguration() { 44 | } 45 | 46 | /** 47 | * Initializes the configuration using the provided input. It selects 48 | * {@link ContentAggregator#COUNT} as default aggregation method. 49 | * @param outputElement 50 | * @param aggregateField 51 | */ 52 | public FieldAggregationConfiguration(final String outputElement, final JsonContentReference aggregateField) { 53 | this.outputElement = outputElement; 54 | this.aggregateField = aggregateField; 55 | this.methods.add(ContentAggregator.COUNT); 56 | } 57 | 58 | /** 59 | * Adds {@link ContentAggregator} as aggregation method 60 | * @param method 61 | */ 62 | public void addAggregationMethod(final ContentAggregator method) { 63 | if(!this.methods.contains(method)) 64 | this.methods.add(method); 65 | } 66 | 67 | public String getOutputElement() { 68 | return outputElement; 69 | } 70 | 71 | public void setOutputElement(String outputElement) { 72 | this.outputElement = outputElement; 73 | } 74 | 75 | public JsonContentReference getAggregateField() { 76 | return aggregateField; 77 | } 78 | 79 | public void setAggregateField(JsonContentReference aggregateField) { 80 | this.aggregateField = aggregateField; 81 | } 82 | 83 | public List getMethods() { 84 | return methods; 85 | } 86 | 87 | public void setMethods(List methods) { 88 | this.methods = methods; 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/aggregate/functions/BooleanContentAggregateFunction.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.aggregate.functions; 18 | 19 | import org.apache.commons.lang3.tuple.MutablePair; 20 | 21 | /** 22 | * @author mnxfst 23 | * @since Jan 18, 2016 24 | */ 25 | public class BooleanContentAggregateFunction implements JsonContentAggregateFunction { 26 | 27 | private static final long serialVersionUID = -645728554568812207L; 28 | 29 | /** 30 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#sum(java.io.Serializable, java.io.Serializable) 31 | */ 32 | public Boolean sum(Boolean oldSum, Boolean value) throws Exception { 33 | throw new UnsupportedOperationException("Method 'sum' is not defined for boolean fields"); 34 | } 35 | 36 | /** 37 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#min(java.io.Serializable, java.io.Serializable) 38 | */ 39 | public Boolean min(Boolean oldMin, Boolean value) throws Exception { 40 | throw new UnsupportedOperationException("Method 'min' is not defined for boolean fields"); 41 | } 42 | 43 | /** 44 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#max(java.io.Serializable, java.io.Serializable) 45 | */ 46 | public Boolean max(Boolean oldMax, Boolean value) throws Exception { 47 | throw new UnsupportedOperationException("Method 'max' is not defined for boolean fields"); 48 | } 49 | 50 | /** 51 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#count(java.lang.Integer) 52 | */ 53 | public Integer count(Integer value) throws Exception { 54 | if(value == null) 55 | return Integer.valueOf(1); 56 | return Integer.valueOf(value.intValue() + 1); 57 | } 58 | 59 | /** 60 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#average(org.apache.commons.lang3.tuple.MutablePair, java.io.Serializable) 61 | */ 62 | public MutablePair average(MutablePair sumAndCount, Boolean value) 63 | throws Exception { 64 | throw new UnsupportedOperationException("Method 'average' is not defined for boolean fields"); 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/aggregate/functions/DoubleContentAggregateFunction.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.aggregate.functions; 18 | 19 | import org.apache.commons.lang3.tuple.MutablePair; 20 | 21 | /** 22 | * @author mnxfst 23 | * @since Jan 18, 2016 24 | */ 25 | public class DoubleContentAggregateFunction implements JsonContentAggregateFunction { 26 | 27 | private static final long serialVersionUID = -2521010219546318123L; 28 | 29 | /** 30 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#sum(java.io.Serializable, java.io.Serializable) 31 | */ 32 | public Double sum(Double oldSum, Double value) throws Exception { 33 | if(oldSum == null && value == null) 34 | return null; 35 | if(oldSum != null && value == null) 36 | return oldSum; 37 | if(oldSum == null && value != null) 38 | return Double.valueOf(value.doubleValue()); 39 | return Double.valueOf(oldSum.doubleValue() + value.doubleValue()); 40 | } 41 | 42 | /** 43 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#min(java.io.Serializable, java.io.Serializable) 44 | */ 45 | public Double min(Double oldMin, Double value) throws Exception { 46 | if(oldMin == null && value == null) 47 | return null; 48 | if(oldMin != null && value == null) 49 | return oldMin; 50 | if(oldMin == null && value != null) 51 | return Double.valueOf(value.doubleValue()); 52 | if(oldMin.doubleValue() <= value.doubleValue()) 53 | return oldMin; 54 | return Double.valueOf(value.doubleValue()); 55 | } 56 | 57 | /** 58 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#max(java.io.Serializable, java.io.Serializable) 59 | */ 60 | public Double max(Double oldMax, Double value) throws Exception { 61 | if(oldMax == null && value == null) 62 | return null; 63 | if(oldMax != null && value == null) 64 | return oldMax; 65 | if(oldMax == null && value != null) 66 | return Double.valueOf(value.doubleValue()); 67 | if(oldMax.doubleValue() >= value.doubleValue()) 68 | return oldMax; 69 | return Double.valueOf(value.doubleValue()); 70 | } 71 | 72 | /** 73 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#count(java.lang.Integer) 74 | */ 75 | public Integer count(Integer value) throws Exception { 76 | if(value == null) 77 | return Integer.valueOf(1); 78 | return Integer.valueOf(value.intValue() + 1); 79 | } 80 | 81 | /** 82 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#average(org.apache.commons.lang3.tuple.MutablePair, java.io.Serializable) 83 | */ 84 | public MutablePair average(MutablePair sumAndCount, Double value) throws Exception { 85 | if(sumAndCount == null && value == null) 86 | return new MutablePair<>(Double.valueOf(0), Integer.valueOf(0)); 87 | if(sumAndCount == null && value != null) 88 | return new MutablePair<>(Double.valueOf(value.doubleValue()), Integer.valueOf(1)); 89 | if(sumAndCount != null && value == null) 90 | return sumAndCount; 91 | sumAndCount.setLeft(sumAndCount.getLeft().doubleValue() + value.doubleValue()); 92 | sumAndCount.setRight(sumAndCount.getRight().intValue() + 1); 93 | return sumAndCount; 94 | } 95 | 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/aggregate/functions/IntegerContentAggregateFunction.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.aggregate.functions; 18 | 19 | import org.apache.commons.lang3.tuple.MutablePair; 20 | 21 | /** 22 | * @author mnxfst 23 | * @since Jan 14, 2016 24 | */ 25 | public class IntegerContentAggregateFunction implements JsonContentAggregateFunction { 26 | 27 | private static final long serialVersionUID = -7538203855813425911L; 28 | 29 | /** 30 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#sum(java.io.Serializable, java.io.Serializable) 31 | */ 32 | public Integer sum(Integer oldSum, Integer value) throws Exception { 33 | if(oldSum == null && value == null) 34 | return null; 35 | if(oldSum != null && value == null) 36 | return oldSum; 37 | if(oldSum == null && value != null) 38 | return Integer.valueOf(value.intValue()); 39 | return Integer.valueOf(oldSum.intValue() + value.intValue()); 40 | } 41 | 42 | /** 43 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#min(java.io.Serializable, java.io.Serializable) 44 | */ 45 | public Integer min(Integer oldMin, Integer value) throws Exception { 46 | if(oldMin == null && value == null) 47 | return null; 48 | if(oldMin != null && value == null) 49 | return oldMin; 50 | if(oldMin == null && value != null) 51 | return Integer.valueOf(value.intValue()); 52 | if(oldMin.intValue() <= value.intValue()) 53 | return oldMin; 54 | return Integer.valueOf(value.intValue()); 55 | } 56 | 57 | /** 58 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#max(java.io.Serializable, java.io.Serializable) 59 | */ 60 | public Integer max(Integer oldMax, Integer value) throws Exception { 61 | if(oldMax == null && value == null) 62 | return null; 63 | if(oldMax != null && value == null) 64 | return oldMax; 65 | if(oldMax == null && value != null) 66 | return Integer.valueOf(value.intValue()); 67 | if(oldMax.intValue() >= value.intValue()) 68 | return oldMax; 69 | return Integer.valueOf(value.intValue()); 70 | } 71 | 72 | /** 73 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#count(java.lang.Integer) 74 | */ 75 | public Integer count(Integer value) throws Exception { 76 | if(value == null) 77 | return Integer.valueOf(1); 78 | return Integer.valueOf(value.intValue() + 1); 79 | } 80 | 81 | /** 82 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#average(org.apache.commons.lang3.tuple.Pair, java.io.Serializable) 83 | */ 84 | public MutablePair average(MutablePair sumAndCount, Integer value) throws Exception { 85 | if(sumAndCount == null && value == null) 86 | return new MutablePair<>(Integer.valueOf(0), Integer.valueOf(0)); 87 | if(sumAndCount == null && value != null) 88 | return new MutablePair<>(Integer.valueOf(value.intValue()), Integer.valueOf(1)); 89 | if(sumAndCount != null && value == null) 90 | return sumAndCount; 91 | sumAndCount.setLeft(sumAndCount.getLeft().intValue() + value.intValue()); 92 | sumAndCount.setRight(sumAndCount.getRight().intValue() + 1); 93 | return sumAndCount; 94 | } 95 | 96 | 97 | } 98 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/aggregate/functions/JsonContentAggregateFunction.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.aggregate.functions; 18 | 19 | import java.io.Serializable; 20 | 21 | import org.apache.commons.lang3.tuple.MutablePair; 22 | 23 | /** 24 | * @author mnxfst 25 | * @since Jan 14, 2016 26 | */ 27 | public interface JsonContentAggregateFunction extends Serializable { 28 | public I sum(final I oldSum, final I value) throws Exception; 29 | public I min(final I oldMin, final I value) throws Exception; 30 | public I max(final I oldMax, final I value) throws Exception; 31 | public Integer count(final Integer value) throws Exception; 32 | public MutablePair average(final MutablePair sumAndCount, final I value) throws Exception; 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/aggregate/functions/StringContentAggregateFunction.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.aggregate.functions; 18 | 19 | import org.apache.commons.lang3.tuple.MutablePair; 20 | 21 | /** 22 | * @author mnxfst 23 | * @since Jan 18, 2016 24 | */ 25 | public class StringContentAggregateFunction implements JsonContentAggregateFunction { 26 | 27 | private static final long serialVersionUID = 6265880736555658327L; 28 | 29 | /** 30 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#sum(java.io.Serializable, java.io.Serializable) 31 | */ 32 | public String sum(String oldSum, String value) throws Exception { 33 | throw new UnsupportedOperationException("Method 'sum' is not defined for string fields"); 34 | } 35 | 36 | /** 37 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#min(java.io.Serializable, java.io.Serializable) 38 | */ 39 | public String min(String oldMin, String value) throws Exception { 40 | throw new UnsupportedOperationException("Method 'min' is not defined for string fields"); 41 | } 42 | 43 | /** 44 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#max(java.io.Serializable, java.io.Serializable) 45 | */ 46 | public String max(String oldMax, String value) throws Exception { 47 | throw new UnsupportedOperationException("Method 'max' is not defined for string fields"); 48 | } 49 | 50 | /** 51 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#count(java.lang.Integer) 52 | */ 53 | public Integer count(Integer value) throws Exception { 54 | if(value == null) 55 | return Integer.valueOf(1); 56 | return Integer.valueOf(value.intValue() + 1); 57 | } 58 | 59 | /** 60 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#average(org.apache.commons.lang3.tuple.MutablePair, java.io.Serializable) 61 | */ 62 | public MutablePair average(MutablePair sumAndCount, String value) throws Exception { 63 | throw new UnsupportedOperationException("Method 'average' is not defined for string fields"); 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/aggregate/functions/TimestampContentAggregateFunction.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.aggregate.functions; 18 | 19 | import java.util.Date; 20 | 21 | import org.apache.commons.lang3.tuple.MutablePair; 22 | 23 | /** 24 | * @author mnxfst 25 | * @since Jan 18, 2016 26 | */ 27 | public class TimestampContentAggregateFunction implements JsonContentAggregateFunction { 28 | 29 | private static final long serialVersionUID = -323695075715674183L; 30 | 31 | /** 32 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#sum(java.io.Serializable, java.io.Serializable) 33 | */ 34 | public Date sum(Date oldSum, Date value) throws Exception { 35 | throw new UnsupportedOperationException("Method 'sum' is not defined for timestamp fields"); 36 | } 37 | 38 | /** 39 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#min(java.io.Serializable, java.io.Serializable) 40 | */ 41 | public Date min(Date oldMin, Date value) throws Exception { 42 | if(oldMin == null && value == null) 43 | return null; 44 | if(oldMin != null && value == null) 45 | return oldMin; 46 | if(oldMin == null && value != null) 47 | return new Date(value.getTime()); 48 | if(oldMin.getTime() <= value.getTime()) 49 | return oldMin; 50 | return new Date(value.getTime()); 51 | } 52 | 53 | /** 54 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#max(java.io.Serializable, java.io.Serializable) 55 | */ 56 | public Date max(Date oldMax, Date value) throws Exception { 57 | if(oldMax == null && value == null) 58 | return null; 59 | if(oldMax != null && value == null) 60 | return oldMax; 61 | if(oldMax == null && value != null) 62 | return new Date(value.getTime()); 63 | if(oldMax.getTime() >= value.getTime()) 64 | return oldMax; 65 | return new Date(value.getTime()); 66 | } 67 | 68 | /** 69 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#count(java.lang.Integer) 70 | */ 71 | public Integer count(Integer value) throws Exception { 72 | if(value == null) 73 | return Integer.valueOf(1); 74 | return Integer.valueOf(value.intValue() + 1); 75 | } 76 | 77 | /** 78 | * @see com.ottogroup.bi.streaming.operator.json.aggregate.functions.JsonContentAggregateFunction#average(org.apache.commons.lang3.tuple.MutablePair, java.io.Serializable) 79 | */ 80 | public MutablePair average(MutablePair sumAndCount, Date value) throws Exception { 81 | throw new UnsupportedOperationException("Method 'average' is not defined for timestamp fields"); 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/converter/JsonObjectToByteArray.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.converter; 18 | 19 | import java.nio.charset.Charset; 20 | 21 | import org.apache.flink.api.common.functions.MapFunction; 22 | import org.apache.sling.commons.json.JSONObject; 23 | 24 | /** 25 | * Converts inbound {@link JSONObject} back to byte arrays processable by the attached kafka 26 | * @author mnxfst 27 | * @since Sep 16, 2015 28 | */ 29 | public class JsonObjectToByteArray implements MapFunction { 30 | 31 | private static final long serialVersionUID = -791573058238550981L; 32 | 33 | /** 34 | * @see org.apache.flink.api.common.functions.MapFunction#map(java.lang.Object) 35 | */ 36 | public byte[] map(JSONObject json) throws Exception { 37 | if(json == null) 38 | return new byte[0]; 39 | return json.toString().getBytes(Charset.forName("UTF-8")); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/converter/JsonObjectToString.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.streaming.operator.json.converter; 17 | 18 | import org.apache.flink.api.common.functions.MapFunction; 19 | import org.apache.sling.commons.json.JSONObject; 20 | 21 | /** 22 | * Converts an incoming {@link JSONObject} into its string representation 23 | * @author mnxfst 24 | * @since May 3, 2016 25 | * 26 | */ 27 | public class JsonObjectToString implements MapFunction { 28 | 29 | private static final long serialVersionUID = -5199858671855117214L; 30 | 31 | /** 32 | * @see org.apache.flink.api.common.functions.MapFunction#map(java.lang.Object) 33 | */ 34 | public String map(JSONObject json) throws Exception { 35 | if(json == null) 36 | return "{}"; 37 | return json.toString(); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/converter/StringToByteArray.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.streaming.operator.json.converter; 17 | 18 | import java.nio.charset.Charset; 19 | 20 | import org.apache.flink.api.common.functions.MapFunction; 21 | 22 | /** 23 | * Maps an incoming string into its byte array representation 24 | * @author mnxfst 25 | * @since May 19, 2016 26 | */ 27 | public class StringToByteArray implements MapFunction { 28 | 29 | private static final long serialVersionUID = 833403150440583363L; 30 | 31 | private String encoding = "UTF-8"; 32 | 33 | /** 34 | * Default constructor which keeps the default encoding 35 | */ 36 | public StringToByteArray() { 37 | } 38 | 39 | /** 40 | * Initializes the mapper and assigns a new encoding 41 | * @param encoding 42 | * The encoding to use when converting a string into its byte array representation 43 | */ 44 | public StringToByteArray(final String encoding) { 45 | Charset.forName(encoding); // called to check if the charset exists -> throws an exception if it does not exist 46 | this.encoding = encoding; 47 | } 48 | 49 | /** 50 | * @see org.apache.flink.api.common.functions.MapFunction#map(java.lang.Object) 51 | */ 52 | public byte[] map(String str) throws Exception { 53 | if(str != null) 54 | return str.getBytes(this.encoding); 55 | return new byte[0]; 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/converter/StringToJsonObject.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.converter; 18 | 19 | import org.apache.commons.lang3.StringUtils; 20 | import org.apache.flink.api.common.functions.FlatMapFunction; 21 | import org.apache.flink.util.Collector; 22 | import org.apache.log4j.Logger; 23 | import org.apache.sling.commons.json.JSONObject; 24 | 25 | /** 26 | * Converts incoming strings to {@link JSONObject} representations 27 | * @author mnxfst 28 | * @since Sep 14, 2015 29 | */ 30 | public class StringToJsonObject implements FlatMapFunction { 31 | 32 | private static final long serialVersionUID = 4573928723585302447L; 33 | private static Logger LOG = Logger.getLogger(StringToJsonObject.class); 34 | 35 | /** 36 | * @see org.apache.flink.api.common.functions.FlatMapFunction#flatMap(java.lang.Object, org.apache.flink.util.Collector) 37 | */ 38 | public void flatMap(String content, Collector collector) throws Exception { 39 | if(StringUtils.isNotBlank(content)) { 40 | try { 41 | if(collector != null && StringUtils.isNotBlank(content)) 42 | collector.collect(new JSONObject(content)); 43 | } catch(Exception e) { 44 | LOG.error("Failed to convert incoming string into JSON object representation. Reason: " + e.getMessage(), e); 45 | } 46 | } 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/csv/Json2CsvConverter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.csv; 18 | 19 | import java.util.ArrayList; 20 | import java.util.Iterator; 21 | import java.util.List; 22 | 23 | import org.apache.commons.lang3.StringUtils; 24 | import org.apache.flink.api.common.functions.MapFunction; 25 | import org.apache.sling.commons.json.JSONObject; 26 | 27 | import com.ottogroup.bi.streaming.operator.json.JsonContentReference; 28 | import com.ottogroup.bi.streaming.operator.json.JsonProcessingUtils; 29 | 30 | /** 31 | * For each incoming {@link JSONObject} the operator parses out the content from fields as referenced by {@link #exportDefinition}. 32 | * According to the order provided by the export definition the content gets converted into a x-separated string 33 | * where the provided {@link #separatorChar} defines the character used to separate two values. 34 | * @author mnxfst 35 | * @since Jan 28, 2016 36 | */ 37 | public class Json2CsvConverter implements MapFunction { 38 | 39 | private static final long serialVersionUID = 7188793921579053791L; 40 | 41 | private final JsonProcessingUtils jsonUtils = new JsonProcessingUtils(); 42 | private List exportDefinition = new ArrayList<>(); 43 | private final char separatorChar; 44 | private final String emptyLine; 45 | 46 | /** 47 | * Initializes the converter using the provided input 48 | * @param exportDefinition 49 | * @param separatorChar 50 | */ 51 | public Json2CsvConverter(final List exportDefinition, final char separatorChar) { 52 | this.exportDefinition.addAll(exportDefinition); 53 | this.separatorChar = separatorChar; 54 | 55 | final StringBuffer buf = new StringBuffer(); 56 | for(int i = 0; i < exportDefinition.size()-1; i++) 57 | buf.append(separatorChar); 58 | this.emptyLine = buf.toString(); 59 | } 60 | 61 | /** 62 | * @see org.apache.flink.api.common.functions.MapFunction#map(java.lang.Object) 63 | */ 64 | public String map(JSONObject value) throws Exception { 65 | return parse(value); 66 | } 67 | 68 | /** 69 | * Parses a given {@link JSONObject} into its character separated representation according 70 | * to the provided configuration 71 | * @param json 72 | * @return 73 | */ 74 | protected String parse(final JSONObject json) { 75 | 76 | // if the provided input is empty, return an empty line 77 | if(json == null) 78 | return this.emptyLine; 79 | 80 | StringBuffer result = new StringBuffer(); 81 | for(final Iterator refIter = this.exportDefinition.iterator(); refIter.hasNext();) { 82 | final JsonContentReference ref = refIter.next(); 83 | String fieldValue = null; 84 | try { 85 | fieldValue = jsonUtils.getTextFieldValue(json, ref.getPath()); 86 | } catch(Exception e) { 87 | // ignore 88 | } 89 | result.append(StringUtils.isNotBlank(fieldValue) ? fieldValue : ""); 90 | if(refIter.hasNext()) 91 | result.append(this.separatorChar); 92 | } 93 | return result.toString(); 94 | } 95 | 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/decode/Base64ContentDecoder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.decode; 18 | 19 | import java.io.UnsupportedEncodingException; 20 | import java.util.ArrayList; 21 | import java.util.Base64; 22 | import java.util.Base64.Decoder; 23 | import java.util.List; 24 | 25 | import org.apache.commons.lang3.StringUtils; 26 | import org.apache.flink.api.common.functions.MapFunction; 27 | import org.apache.sling.commons.json.JSONException; 28 | import org.apache.sling.commons.json.JSONObject; 29 | 30 | import com.ottogroup.bi.streaming.operator.json.JsonProcessingUtils; 31 | 32 | /** 33 | * Deflates {@link Base64} encoded content located at specific locations inside {@link JSONObject} documents. 34 | * The implementation handles plain values as well as contained JSON sub-structures which replace the origin 35 | * content accordingly. 36 | * @author mnxfst 37 | * @since Mar 23, 2016 38 | */ 39 | public class Base64ContentDecoder implements MapFunction { 40 | 41 | private static final long serialVersionUID = 1946831918020477606L; 42 | 43 | private List contentReferences = new ArrayList<>(); 44 | private JsonProcessingUtils jsonUtils = new JsonProcessingUtils(); 45 | 46 | /** 47 | * Initializes the mapper using the provided input 48 | * @param contentReferences 49 | */ 50 | public Base64ContentDecoder(final List contentReferences) { 51 | 52 | ///////////////////////////////////////////////////////////////// 53 | // validate and transfer provided input 54 | if(contentReferences == null || contentReferences.isEmpty()) 55 | throw new IllegalArgumentException("Missing required list of content references or list contains no elements"); 56 | 57 | for(final Base64ContentDecoderConfiguration ref : contentReferences) { 58 | if(ref == null) 59 | throw new IllegalArgumentException("Empty list elements are not permitted"); 60 | if(ref.getJsonRef() == null) 61 | throw new IllegalArgumentException("Missing required JSON field reference"); 62 | if(ref.getJsonRef().getContentType() == null) 63 | throw new IllegalArgumentException("Missing required JSON element content type"); 64 | if(ref.getJsonRef().getPath() == null || ref.getJsonRef().getPath().length < 1) 65 | throw new IllegalArgumentException("Missing required JSON element path"); 66 | this.contentReferences.add(ref); 67 | } 68 | ///////////////////////////////////////////////////////////////// 69 | } 70 | 71 | /** 72 | * @see org.apache.flink.api.common.functions.MapFunction#map(java.lang.Object) 73 | */ 74 | public JSONObject map(JSONObject value) throws Exception { 75 | return processJSON(value); 76 | } 77 | 78 | /** 79 | * Steps through {@link Base64ContentDecoderConfiguration field configurations}, reads the referenced value from the {@link JSONObject} 80 | * and attempts to decode {@link Base64} string. If the result is expected to hold a {@link JSONObject} the value is replaced inside the 81 | * original document accordingly. Otherwise the value simply replaces the existing value 82 | * @param jsonObject 83 | * @return 84 | * @throws JSONException 85 | * @throws UnsupportedEncodingException 86 | */ 87 | protected JSONObject processJSON(final JSONObject jsonObject) throws JSONException, UnsupportedEncodingException { 88 | if(jsonObject == null) 89 | return null; 90 | 91 | for(final Base64ContentDecoderConfiguration cfg : this.contentReferences) { 92 | final String value = this.jsonUtils.getTextFieldValue(jsonObject, cfg.getJsonRef().getPath(), false); 93 | final String decodedValue = decodeBase64(value, cfg.getNoValuePrefix(), cfg.getEncoding()); 94 | 95 | if(cfg.isJsonContent()) 96 | this.jsonUtils.insertField(jsonObject, cfg.getJsonRef().getPath(), new JSONObject(decodedValue), true); 97 | else 98 | this.jsonUtils.insertField(jsonObject, cfg.getJsonRef().getPath(), decodedValue, true); 99 | } 100 | return jsonObject; 101 | } 102 | 103 | /** 104 | * Decodes the provided {@link Base64} encoded string. Prior decoding a possibly existing prefix is removed from the encoded string. 105 | * The result which is received from {@link Decoder#decode(byte[])} as array of bytes is converted into a string representation which 106 | * follows the given encoding 107 | * @param base64EncodedString 108 | * The {@link Base64} encoded string 109 | * @param noValuePrefix 110 | * An optional prefix attached to string after encoding (eg to mark it as base64 value) which must be removed prior decoding 111 | * @param encoding 112 | * The encoding applied on converting the resulting byte array into a string representation 113 | * @return 114 | */ 115 | protected String decodeBase64(final String base64EncodedString, final String noValuePrefix, final String encoding) throws UnsupportedEncodingException { 116 | 117 | // if the base64 encoded string is either empty or holds only the prefix that must be removed before decoding, the method returns an empty string 118 | if(StringUtils.isBlank(base64EncodedString) || StringUtils.equalsIgnoreCase(base64EncodedString, noValuePrefix)) 119 | return ""; 120 | 121 | // remove optional prefix and decode - if the prefix does not exist, simply decode the input 122 | byte[] result = null; 123 | if(StringUtils.startsWith(base64EncodedString, noValuePrefix)) 124 | result = Base64.getDecoder().decode(StringUtils.substring(base64EncodedString, noValuePrefix.length())); 125 | else 126 | result = Base64.getDecoder().decode(base64EncodedString); 127 | 128 | // if the result array is either null or empty the method returns an empty string 129 | if(result == null || result.length < 1) 130 | return ""; 131 | 132 | // otherwise: the method tries to convert the array into a proper string representation following the given encoding 133 | return new String(result, (StringUtils.isNotBlank(encoding) ? encoding : "UTF-8")); 134 | } 135 | 136 | /** 137 | * Returns the list of {@link Base64ContentDecoderConfiguration} pointing to selected elements 138 | * inside a {@link JSONObject} that hold base64 encoded strings - implemented 139 | * for testing purpose only 140 | * @return 141 | */ 142 | protected List getContentReferences() { 143 | return this.contentReferences; 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/decode/Base64ContentDecoderConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.decode; 18 | 19 | import java.io.Serializable; 20 | 21 | import javax.validation.constraints.NotNull; 22 | 23 | import com.fasterxml.jackson.annotation.JsonProperty; 24 | import com.ottogroup.bi.streaming.operator.json.JsonContentReference; 25 | 26 | /** 27 | * Configuration required for setting up instances of type {@link Base64ContentDecoder} 28 | * @author mnxfst 29 | * @since Mar 23, 2016 30 | */ 31 | public class Base64ContentDecoderConfiguration implements Serializable { 32 | 33 | private static final long serialVersionUID = -1006181381054464078L; 34 | 35 | /** prefix that must be removed before deflating which is attached to values found in referenced fields */ 36 | @JsonProperty(value="noValuePrefix", required=false) 37 | private String noValuePrefix = null; 38 | 39 | /** encoding to be applied when converting the deflated value into a string representation */ 40 | @JsonProperty(value="encoding", required=false) 41 | private String encoding = "UTF-8"; 42 | 43 | /** reference to field the base64 encoded content is expected in */ 44 | @JsonProperty(value="jsonRef", required=true) 45 | @NotNull 46 | private JsonContentReference jsonRef = null; 47 | 48 | /** indicates if the base64 encoded string is expected to hold a JSON structure */ 49 | @JsonProperty(value="jsonContent", required=false) 50 | private boolean jsonContent = false; 51 | 52 | public Base64ContentDecoderConfiguration() { 53 | } 54 | 55 | public Base64ContentDecoderConfiguration(final JsonContentReference jsonRef, final String noValuePrefix, final String encoding, final boolean jsonContent) { 56 | this.jsonRef = jsonRef; 57 | this.noValuePrefix = noValuePrefix; 58 | this.encoding = encoding; 59 | this.jsonContent = jsonContent; 60 | } 61 | 62 | public String getNoValuePrefix() { 63 | return noValuePrefix; 64 | } 65 | 66 | public void setNoValuePrefix(String noValuePrefix) { 67 | this.noValuePrefix = noValuePrefix; 68 | } 69 | 70 | public String getEncoding() { 71 | return encoding; 72 | } 73 | 74 | public void setEncoding(String encoding) { 75 | this.encoding = encoding; 76 | } 77 | 78 | public JsonContentReference getJsonRef() { 79 | return jsonRef; 80 | } 81 | 82 | public void setJsonRef(JsonContentReference jsonRef) { 83 | this.jsonRef = jsonRef; 84 | } 85 | 86 | public boolean isJsonContent() { 87 | return jsonContent; 88 | } 89 | 90 | public void setJsonContent(boolean jsonContent) { 91 | this.jsonContent = jsonContent; 92 | } 93 | 94 | 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/filter/RegularExpressionMatcher.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.filter; 18 | 19 | import org.hamcrest.BaseMatcher; 20 | import org.hamcrest.Description; 21 | import org.hamcrest.Matcher; 22 | 23 | import com.ottogroup.bi.streaming.operator.json.filter.cfg.FieldConditionOperator; 24 | 25 | /** 26 | * Implements a {@link Matcher} that validates an incoming value against a provided regular expression. 27 | * The implementation facilitates the features of {@link String#matches(String)}. See 28 | * {@link JsonContentFilter#matcher(com.ottogroup.bi.streaming.operator.json.filter.cfg.FieldConditionOperator, Comparable, com.ottogroup.bi.streaming.operator.json.JsonContentType)} 29 | * for an example ({@link FieldConditionOperator#LIKE} selector). 30 | * @author mnxfst 31 | * @since May 12, 2016 32 | */ 33 | public class RegularExpressionMatcher extends BaseMatcher { 34 | 35 | private String regularExpression = null; 36 | 37 | private RegularExpressionMatcher(String regularExpression) { 38 | this.regularExpression = regularExpression; 39 | } 40 | 41 | /** 42 | * @see org.hamcrest.Matcher#matches(java.lang.Object) 43 | */ 44 | public boolean matches(Object item) { 45 | 46 | if(item == null && regularExpression == null) 47 | return true; 48 | 49 | if(item == null && regularExpression != null) 50 | return false; 51 | 52 | if(item != null && regularExpression == null) 53 | return false; 54 | 55 | return item.toString().matches(this.regularExpression); 56 | } 57 | 58 | /** 59 | * @see org.hamcrest.SelfDescribing#describeTo(org.hamcrest.Description) 60 | */ 61 | public void describeTo(Description description) { 62 | description.appendText("matches expression '"+regularExpression+"'"); 63 | } 64 | 65 | /** 66 | * Creates a {@link PatternMatcher} instance for the given pattern 67 | * @param pattern 68 | * @return 69 | */ 70 | public static RegularExpressionMatcher matchesPattern(final String regularExpression) { 71 | return new RegularExpressionMatcher(regularExpression); 72 | } 73 | 74 | 75 | 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/filter/cfg/FieldConditionCombiner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.streaming.operator.json.filter.cfg; 17 | 18 | import java.io.Serializable; 19 | 20 | /** 21 | * Supported types for combining field conditions 22 | * @author mnxfst 23 | * @since Apr 26, 2016 24 | */ 25 | public enum FieldConditionCombiner implements Serializable { 26 | ALL, ANY, AT_LEAST, AT_MOST, EXACTLY 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/filter/cfg/FieldConditionCombinerConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.streaming.operator.json.filter.cfg; 17 | 18 | import java.io.Serializable; 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | import javax.validation.constraints.Min; 23 | import javax.validation.constraints.NotNull; 24 | import javax.validation.constraints.Size; 25 | 26 | import com.fasterxml.jackson.annotation.JsonProperty; 27 | 28 | /** 29 | * Defines how to combine {@link JsonFieldContentMatcher} such that 30 | *
    31 | *
  • any of the listed,
    • 32 | *
  • at least n,
    • 33 | *
  • at most n or
    • 34 | *
  • exactly n
    • 35 | *
36 | * matchers must evaluate to true before the currently processed JSON object gets accepted. If no 37 | * such section is included in {@link JsonContentFilterConfiguration} any of the provided 38 | * {@link JsonContentFilterConfiguration#getFieldContentMatchers()} must evaluate to true. That 39 | * behavior is equal to listing all field content matchers here and requesting {@link FieldConditionCombiner#ANY}. 40 | * @author mnxfst 41 | * @since Apr 26, 2016 42 | * 43 | */ 44 | public class FieldConditionCombinerConfiguration implements Serializable { 45 | 46 | private static final long serialVersionUID = -1477685211279523068L; 47 | 48 | /** list of identifiers referencing content matchers that must be included into this combiner */ 49 | @NotNull 50 | @Size(min=1) 51 | @JsonProperty(value="fieldConditionRefs", required=true) 52 | private List fieldConditionRefs = new ArrayList<>(); 53 | 54 | /** how to combine the matchers */ 55 | @NotNull 56 | @JsonProperty(value="combiner", required=true) 57 | private FieldConditionCombiner combiner = FieldConditionCombiner.ANY; 58 | 59 | /** required to set the number of matchers that must evaluate to true if combiners other than ANY are selected */ 60 | @NotNull 61 | @Min(0) 62 | @JsonProperty(value="n", required=true) 63 | private int n = 0; 64 | 65 | public FieldConditionCombinerConfiguration() { 66 | } 67 | 68 | public FieldConditionCombinerConfiguration(final FieldConditionCombiner combiner, final int n, final String ... refs) { 69 | this.combiner = combiner; 70 | this.n = n; 71 | if(refs != null && refs.length > 0) { 72 | for(int i = 0; i < refs.length; i++) 73 | if(!this.fieldConditionRefs.contains(refs[i])) 74 | this.fieldConditionRefs.add(refs[i]); 75 | } 76 | } 77 | 78 | public List getFieldConditionRefs() { 79 | return fieldConditionRefs; 80 | } 81 | 82 | public void setFieldConditionRefs(List fieldConditionRefs) { 83 | this.fieldConditionRefs = fieldConditionRefs; 84 | } 85 | 86 | public FieldConditionCombiner getCombiner() { 87 | return combiner; 88 | } 89 | 90 | public void setCombiner(FieldConditionCombiner combiner) { 91 | this.combiner = combiner; 92 | } 93 | 94 | public int getN() { 95 | return n; 96 | } 97 | 98 | public void setN(int n) { 99 | this.n = n; 100 | } 101 | 102 | 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/filter/cfg/FieldConditionConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.streaming.operator.json.filter.cfg; 17 | 18 | import java.io.Serializable; 19 | 20 | import javax.validation.constraints.NotNull; 21 | 22 | import com.fasterxml.jackson.annotation.JsonProperty; 23 | import com.ottogroup.bi.streaming.operator.json.JsonContentReference; 24 | 25 | /** 26 | * Provides the configuration for a single content matcher 27 | * @author mnxfst 28 | * @since Apr 26, 2016 29 | * 30 | */ 31 | public class FieldConditionConfiguration implements Serializable { 32 | 33 | private static final long serialVersionUID = -5080138507217022078L; 34 | 35 | /** reference into json object pointing to field that must be validated */ 36 | @NotNull 37 | @JsonProperty(value="contentRef", required=true) 38 | private JsonContentReference contentRef = null; 39 | 40 | /** defines the matcher to apply on the referenced field */ 41 | @NotNull 42 | @JsonProperty(value="operator", required=true) 43 | private FieldConditionOperator operator = null; 44 | 45 | /** value to evaluate the field content against */ 46 | @NotNull 47 | @JsonProperty(value="value", required=true) 48 | private String value = null; 49 | 50 | public FieldConditionConfiguration() { 51 | } 52 | 53 | public FieldConditionConfiguration(final JsonContentReference contentRef, final FieldConditionOperator operator, final String value) { 54 | this.contentRef = contentRef; 55 | this.operator = operator; 56 | this.value = value; 57 | } 58 | 59 | public JsonContentReference getContentRef() { 60 | return contentRef; 61 | } 62 | 63 | public void setContentRef(JsonContentReference contentRef) { 64 | this.contentRef = contentRef; 65 | } 66 | 67 | public FieldConditionOperator getOperator() { 68 | return operator; 69 | } 70 | 71 | public void setOperator(FieldConditionOperator operator) { 72 | this.operator = operator; 73 | } 74 | 75 | public String getValue() { 76 | return value; 77 | } 78 | 79 | public void setValue(String value) { 80 | this.value = value; 81 | } 82 | 83 | 84 | } 85 | 86 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/filter/cfg/FieldConditionOperator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.streaming.operator.json.filter.cfg; 17 | 18 | import java.io.Serializable; 19 | 20 | /** 21 | * List of supported operators / matcher types 22 | * @author mnxfst 23 | * @since Apr 26, 2016 24 | */ 25 | public enum FieldConditionOperator implements Serializable { 26 | IS, NOT, LESS_THAN, GREATER_THAN, LIKE, IS_EMPTY, NOT_EMPTY 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/filter/cfg/JsonContentFilterConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.streaming.operator.json.filter.cfg; 17 | 18 | import java.io.Serializable; 19 | import java.util.ArrayList; 20 | import java.util.HashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | import javax.validation.constraints.NotNull; 25 | import javax.validation.constraints.Size; 26 | 27 | import com.fasterxml.jackson.annotation.JsonProperty; 28 | import com.ottogroup.bi.streaming.operator.json.filter.JsonContentFilter; 29 | 30 | /** 31 | * Provides all configuration required for setting up an instance of {@link JsonContentFilter} 32 | * @author mnxfst 33 | * @since Apr 26, 2016 34 | * 35 | */ 36 | public class JsonContentFilterConfiguration implements Serializable { 37 | 38 | private static final long serialVersionUID = 7832339652217385180L; 39 | 40 | /** list of configured field content matchers */ 41 | @NotNull 42 | @Size(min=1) 43 | @JsonProperty(value="fieldConditions", required=true) 44 | private Map fieldContentMatchers = new HashMap<>(); 45 | 46 | /** list of matcher combiner configurations - if missing, any of the content matchers must evaluate to true for incoming content */ 47 | @JsonProperty(value="fieldConditionCombiners", required=false) 48 | private List fieldContentMatcherCombiners = new ArrayList<>(); 49 | 50 | public JsonContentFilterConfiguration() { 51 | } 52 | 53 | public void addFieldContentMatcher(final String id, final FieldConditionConfiguration cfg) { 54 | this.fieldContentMatchers.put(id, cfg); 55 | } 56 | 57 | public void addFieldContentMatchersCombiner(final FieldConditionCombinerConfiguration cfg) { 58 | this.fieldContentMatcherCombiners.add(cfg); 59 | } 60 | 61 | public Map getFieldContentMatchers() { 62 | return fieldContentMatchers; 63 | } 64 | 65 | public void setFieldContentMatchers(Map fieldContentMatchers) { 66 | this.fieldContentMatchers = fieldContentMatchers; 67 | } 68 | 69 | public List getFieldContentMatcherCombiners() { 70 | return fieldContentMatcherCombiners; 71 | } 72 | 73 | public void setFieldContentMatcherCombiners( 74 | List fieldContentMatcherCombiners) { 75 | this.fieldContentMatcherCombiners = fieldContentMatcherCombiners; 76 | } 77 | 78 | 79 | 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/insert/JsonStaticContentInsertion.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.insert; 18 | 19 | import java.io.Serializable; 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | import org.apache.commons.lang3.tuple.Pair; 24 | import org.apache.flink.api.common.functions.MapFunction; 25 | import org.apache.sling.commons.json.JSONObject; 26 | 27 | import com.ottogroup.bi.streaming.operator.json.JsonContentReference; 28 | import com.ottogroup.bi.streaming.operator.json.JsonProcessingUtils; 29 | 30 | /** 31 | * Inserts static values into incoming {@link JSONObject} instances. The operator behavior is 32 | * configured during instantiation where it receives a list of key/value pairs holding paths 33 | * along with values to insert at the referenced locations. 34 | * @author mnxfst 35 | * @since Feb 16, 2016 36 | */ 37 | public class JsonStaticContentInsertion implements MapFunction { 38 | 39 | private static final long serialVersionUID = -1052281005116904913L; 40 | 41 | private JsonProcessingUtils utils = new JsonProcessingUtils(); 42 | private List> values = new ArrayList<>(); 43 | 44 | public JsonStaticContentInsertion(final List> values) throws IllegalArgumentException { 45 | 46 | /////////////////////////////////////////////////////// 47 | // validate input 48 | if(values == null) // empty input "configures" the operator to work as simple "pass-through" operator 49 | throw new IllegalArgumentException("Missing required input"); 50 | /////////////////////////////////////////////////////// 51 | 52 | for(final Pair v : values) { 53 | if(v == null) 54 | throw new IllegalArgumentException("Empty list elements are not permitted"); 55 | if(v.getLeft() == null || v.getKey().getPath() == null || v.getKey().getPath().length < 1) 56 | throw new IllegalArgumentException("Empty content referenced are not permitted"); 57 | if(v.getRight() == null) 58 | throw new IllegalArgumentException("Null is not permitted as insertion value"); 59 | this.values.add(v); 60 | } 61 | } 62 | 63 | 64 | /** 65 | * @see org.apache.flink.api.common.functions.MapFunction#map(java.lang.Object) 66 | */ 67 | public JSONObject map(JSONObject json) throws Exception { 68 | 69 | // if no values were configured or the incoming json equals 70 | // null simply let the message go through un-processed 71 | if(json == null || values.isEmpty()) 72 | return json; 73 | 74 | // otherwise step through configuration holding static values and insert them 75 | // at referenced locations 76 | for(final Pair v : values) { 77 | json = insert(json, v.getKey().getPath(), v.getValue()); 78 | } 79 | return json; 80 | } 81 | 82 | /** 83 | * Inserts the given value into the {@link JSONObject} at the referenced location 84 | * @param json 85 | * @param location 86 | * @param value 87 | * @return 88 | * @throws Exception 89 | */ 90 | protected JSONObject insert(final JSONObject json, final String[] location, final Serializable value) throws Exception { 91 | return utils.insertField(json, location, value); 92 | } 93 | 94 | /** 95 | * Returns the configured set of values that must be inserted into every incoming json object

96 | * Attention: this method is implemented for testing purpose only 97 | * @return 98 | */ 99 | protected List> getValues() { 100 | return values; 101 | } 102 | 103 | 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/partitioning/JsonKeySelector.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.partitioning; 18 | 19 | import org.apache.flink.api.java.functions.KeySelector; 20 | import org.apache.sling.commons.json.JSONObject; 21 | 22 | import com.ottogroup.bi.streaming.operator.json.JsonContentReference; 23 | import com.ottogroup.bi.streaming.operator.json.JsonProcessingUtils; 24 | 25 | /** 26 | * Provides a {@link KeySelector} implementation which takes {@link JSONObject} instances 27 | * and extracts the key value from from a configured location. The location is provided via {@link JsonContentReference}. 28 | * @author mnxfst 29 | * @since 20.04.2016 30 | */ 31 | public class JsonKeySelector implements KeySelector { 32 | 33 | private static final long serialVersionUID = -1280949832025582748L; 34 | 35 | private JsonProcessingUtils utils = new JsonProcessingUtils(); 36 | private JsonContentReference ref = null; 37 | 38 | /** 39 | * Initializes the key selector using the provided input 40 | * @param ref 41 | */ 42 | public JsonKeySelector(final JsonContentReference ref) { 43 | /////////////////////////////////////////////////// 44 | // input validation 45 | if(ref == null) 46 | throw new IllegalArgumentException("Missing required reference into JSON objects to read partitioning key from"); 47 | if(ref.getPath() == null || ref.getPath().length < 1) 48 | throw new IllegalArgumentException("Missing required path pointing to position inside JSON object to read partitioning key from"); 49 | /////////////////////////////////////////////////// 50 | this.ref = ref; 51 | } 52 | 53 | /** 54 | * @see org.apache.flink.api.java.functions.KeySelector#getKey(java.lang.Object) 55 | */ 56 | public String getKey(JSONObject value) throws Exception { 57 | if(value != null) 58 | return utils.getTextFieldValue(value, ref.getPath(), ref.isRequired()); 59 | return null; 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/project/JsonContentProjectionMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.project; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | import java.util.NoSuchElementException; 22 | 23 | import org.apache.commons.collections.list.UnmodifiableList; 24 | import org.apache.flink.api.common.functions.MapFunction; 25 | import org.apache.sling.commons.json.JSONException; 26 | import org.apache.sling.commons.json.JSONObject; 27 | 28 | import com.ottogroup.bi.streaming.operator.json.JsonProcessingUtils; 29 | 30 | /** 31 | * Reads configured fields from an incoming {@link JSONObject} using {@link JsonProcessingUtils#getTextFieldValue(JSONObject, String[])} 32 | * and inserts it into a newly created {@link JSONObject} at a configured destination. 33 | * @author mnxfst 34 | * @since Jan 27, 2016 35 | */ 36 | public class JsonContentProjectionMapper implements MapFunction { 37 | 38 | private static final long serialVersionUID = 2798661087920205902L; 39 | 40 | private final JsonProcessingUtils jsonUtils = new JsonProcessingUtils(); 41 | private final List configuration = new ArrayList<>(); 42 | 43 | /** 44 | * Initializes the mapper using the provided input 45 | * @param configuration 46 | */ 47 | public JsonContentProjectionMapper(final List configuration) { 48 | if(configuration != null && !configuration.isEmpty()) 49 | this.configuration.addAll(configuration); 50 | } 51 | 52 | /** 53 | * @see org.apache.flink.api.common.functions.MapFunction#map(java.lang.Object) 54 | */ 55 | public JSONObject map(JSONObject value) throws Exception { 56 | if(value == null) 57 | return null; 58 | else if(this.configuration.isEmpty()) 59 | return new JSONObject(); 60 | return project(value); 61 | } 62 | 63 | /** 64 | * Reads selected content from provided {@link JSONObject} and inserts it into a newly created 65 | * {@link JSONObject} instance at configuration locations 66 | * @param json 67 | * @return 68 | * @throws IllegalArgumentException 69 | * @throws NoSuchElementException 70 | * @throws JSONException 71 | */ 72 | protected JSONObject project(final JSONObject json) throws IllegalArgumentException, NoSuchElementException, JSONException { 73 | 74 | // prepare result object as new and independent instance to avoid any dirty remains inside the return value 75 | JSONObject result = new JSONObject(); 76 | 77 | // step through the configuration, read content from referenced location and write it to selected destination 78 | for(final ProjectionMapperConfiguration c : this.configuration) { 79 | Object value = null; 80 | try { 81 | value = this.jsonUtils.getFieldValue(json, c.getProjectField().getPath(), c.getProjectField().isRequired()); 82 | } catch(Exception e) { 83 | // if the field value is required return an empty object and interrupt the projection 84 | if(c.getProjectField().isRequired()) 85 | return new JSONObject(); 86 | // ignore 87 | } 88 | this.jsonUtils.insertField(result, c.getDestinationPath(), (value != null ? value : "")); 89 | } 90 | return result; 91 | } 92 | 93 | /** 94 | * Returns the configuration of this instance - implemented for testing purpose only 95 | * @return 96 | */ 97 | @SuppressWarnings("unchecked") 98 | protected List getConfiguration() { 99 | return UnmodifiableList.decorate(this.configuration); 100 | } 101 | 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/project/ProjectionMapperConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.project; 18 | 19 | import java.io.Serializable; 20 | 21 | import org.apache.sling.commons.json.JSONObject; 22 | 23 | import com.ottogroup.bi.streaming.operator.json.JsonContentReference; 24 | 25 | /** 26 | * Provides the configuration used by {@link JsonContentProjectionMapper} to project a single field from an 27 | * existing {@link JSONObject} into a newly created instance 28 | * @author mnxfst 29 | * @since Jan 27, 2016 30 | */ 31 | public class ProjectionMapperConfiguration implements Serializable { 32 | 33 | private static final long serialVersionUID = 8215744170987684540L; 34 | 35 | private JsonContentReference projectField = null; 36 | private String[] destinationPath = null; 37 | 38 | public ProjectionMapperConfiguration() { 39 | } 40 | 41 | public ProjectionMapperConfiguration(final JsonContentReference projectField, final String[] destinationPath) { 42 | this.projectField = projectField; 43 | this.destinationPath = destinationPath; 44 | } 45 | 46 | public JsonContentReference getProjectField() { 47 | return projectField; 48 | } 49 | 50 | public void setProjectField(JsonContentReference projectField) { 51 | this.projectField = projectField; 52 | } 53 | 54 | public String[] getDestinationPath() { 55 | return destinationPath; 56 | } 57 | 58 | public void setDestinationPath(String[] destinationPath) { 59 | this.destinationPath = destinationPath; 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/statsd/StatsdExtractedMetricsReporterConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.statsd; 18 | 19 | import java.io.Serializable; 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | import javax.validation.constraints.Max; 24 | import javax.validation.constraints.Min; 25 | import javax.validation.constraints.NotNull; 26 | import javax.validation.constraints.Size; 27 | 28 | import com.fasterxml.jackson.annotation.JsonProperty; 29 | 30 | /** 31 | * Provides information required for setting up an instance of type {@link StatsdExtractedMetricsReporter} 32 | * @author mnxfst 33 | * @since Mar 22, 2016 34 | */ 35 | public class StatsdExtractedMetricsReporterConfiguration implements Serializable { 36 | 37 | private static final long serialVersionUID = 7650844464087305523L; 38 | 39 | @JsonProperty(value="host", required=true) 40 | @NotNull 41 | @Size(min=1) 42 | private String host = null; 43 | 44 | @JsonProperty(value="port", required=true) 45 | @NotNull 46 | @Min(value=1) 47 | @Max(value=99999) 48 | private int port = 8125; 49 | 50 | @JsonProperty(value="prefix", required=true) 51 | private String prefix = null; 52 | 53 | @JsonProperty(value="fields", required=true) 54 | private List fields = new ArrayList<>(); 55 | 56 | public StatsdExtractedMetricsReporterConfiguration() { 57 | } 58 | 59 | public StatsdExtractedMetricsReporterConfiguration(final String host, final int port, final String prefix) { 60 | this.host = host; 61 | this.port = port; 62 | this.prefix = prefix; 63 | } 64 | 65 | public void addMetricConfig(final StatsdMetricConfig cfg) { 66 | this.fields.add(cfg); 67 | } 68 | 69 | public String getHost() { 70 | return host; 71 | } 72 | 73 | public void setHost(String host) { 74 | this.host = host; 75 | } 76 | 77 | public int getPort() { 78 | return port; 79 | } 80 | 81 | public void setPort(int port) { 82 | this.port = port; 83 | } 84 | 85 | public String getPrefix() { 86 | return prefix; 87 | } 88 | 89 | public void setPrefix(String prefix) { 90 | this.prefix = prefix; 91 | } 92 | 93 | public List getFields() { 94 | return fields; 95 | } 96 | 97 | public void setFields(List fields) { 98 | this.fields = fields; 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/statsd/StatsdMetricConfig.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.statsd; 18 | 19 | import java.io.Serializable; 20 | 21 | import javax.validation.constraints.NotNull; 22 | import javax.validation.constraints.Size; 23 | 24 | import com.fasterxml.jackson.annotation.JsonProperty; 25 | import com.ottogroup.bi.streaming.operator.json.JsonContentReference; 26 | 27 | /** 28 | * Points to a destination inside statsd/graphite to write a value to. It gives also a hint on the type 29 | * @author mnxfst 30 | * @since Mar 22, 2016 31 | */ 32 | public class StatsdMetricConfig implements Serializable { 33 | 34 | private static final long serialVersionUID = -224675879381334473L; 35 | 36 | @JsonProperty(value="path", required=true) 37 | @NotNull 38 | @Size(min=1) 39 | private String path = null; 40 | 41 | @JsonProperty(value="dynamicPathPrefix", required=false) 42 | private JsonContentReference dynamicPathPrefix = null; 43 | 44 | @JsonProperty(value="type", required=true) 45 | @NotNull 46 | private StatsdMetricType type = null; 47 | 48 | @JsonProperty(value="reportDelta", required=false) 49 | private boolean reportDelta = false; 50 | 51 | @JsonProperty(value="jsonRef", required=true) 52 | @NotNull 53 | private JsonContentReference jsonRef = null; 54 | 55 | @JsonProperty(value="scaleFactor", required=false) 56 | @NotNull 57 | private long scaleFactor = 1; 58 | 59 | public StatsdMetricConfig() { 60 | } 61 | 62 | public StatsdMetricConfig(final String path, final StatsdMetricType type, final JsonContentReference jsonRef) { 63 | this.path = path; 64 | this.type = type; 65 | this.jsonRef = jsonRef; 66 | } 67 | 68 | public StatsdMetricConfig(final String path, final JsonContentReference dynamicPathPrefix, final StatsdMetricType type, final JsonContentReference jsonRef) { 69 | this.path = path; 70 | this.dynamicPathPrefix = dynamicPathPrefix; 71 | this.type = type; 72 | this.jsonRef = jsonRef; 73 | } 74 | 75 | public StatsdMetricConfig(final String path, final StatsdMetricType type, final JsonContentReference jsonRef, final boolean reportDelta) { 76 | this.path = path; 77 | this.type = type; 78 | this.jsonRef = jsonRef; 79 | this.reportDelta = reportDelta; 80 | } 81 | 82 | public StatsdMetricConfig(final String path, final JsonContentReference dynamicPathPrefix, final StatsdMetricType type, final JsonContentReference jsonRef, final boolean reportDelta) { 83 | this.path = path; 84 | this.dynamicPathPrefix = dynamicPathPrefix; 85 | this.type = type; 86 | this.jsonRef = jsonRef; 87 | this.reportDelta = reportDelta; 88 | } 89 | 90 | public StatsdMetricConfig(final String path, final StatsdMetricType type, final JsonContentReference jsonRef, final boolean reportDelta, final long scaleFactor) { 91 | this.path = path; 92 | this.type = type; 93 | this.jsonRef = jsonRef; 94 | this.reportDelta = reportDelta; 95 | this.scaleFactor = scaleFactor; 96 | } 97 | 98 | public StatsdMetricConfig(final String path, final JsonContentReference dynamicPathPrefix, final StatsdMetricType type, final JsonContentReference jsonRef, final boolean reportDelta, final long scaleFactor) { 99 | this.path = path; 100 | this.dynamicPathPrefix = dynamicPathPrefix; 101 | this.type = type; 102 | this.jsonRef = jsonRef; 103 | this.reportDelta = reportDelta; 104 | this.scaleFactor = scaleFactor; 105 | } 106 | 107 | public long getScaleFactor() { 108 | return scaleFactor; 109 | } 110 | 111 | public void setScaleFactor(long scaleFactor) { 112 | this.scaleFactor = scaleFactor; 113 | } 114 | 115 | public JsonContentReference getJsonRef() { 116 | return jsonRef; 117 | } 118 | 119 | public void setJsonRef(JsonContentReference jsonRef) { 120 | this.jsonRef = jsonRef; 121 | } 122 | 123 | public String getPath() { 124 | return path; 125 | } 126 | 127 | public void setPath(String path) { 128 | this.path = path; 129 | } 130 | 131 | public JsonContentReference getDynamicPathPrefix() { 132 | return dynamicPathPrefix; 133 | } 134 | 135 | public void setDynamicPathPrefix(JsonContentReference dynamicPathPrefix) { 136 | this.dynamicPathPrefix = dynamicPathPrefix; 137 | } 138 | 139 | public StatsdMetricType getType() { 140 | return type; 141 | } 142 | 143 | public void setType(StatsdMetricType type) { 144 | this.type = type; 145 | } 146 | 147 | public boolean isReportDelta() { 148 | return reportDelta; 149 | } 150 | 151 | public void setReportDelta(boolean reportDelta) { 152 | this.reportDelta = reportDelta; 153 | } 154 | 155 | } 156 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/json/statsd/StatsdMetricType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.statsd; 18 | 19 | import java.io.Serializable; 20 | 21 | /** 22 | * Supported statsD metric types 23 | * @author mnxfst 24 | * @since Mar 22, 2016 25 | */ 26 | public enum StatsdMetricType implements Serializable { 27 | GAUGE, COUNTER, TIME 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/operator/metrics/MessageCountingMetricsReporter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.metrics; 18 | 19 | import org.apache.commons.lang3.StringUtils; 20 | import org.apache.flink.api.common.functions.RichFilterFunction; 21 | import org.apache.flink.configuration.Configuration; 22 | 23 | import com.timgroup.statsd.NonBlockingStatsDClient; 24 | import com.timgroup.statsd.StatsDClient; 25 | 26 | /** 27 | * Implements a message flow counter backed by statsD. It records each message floating through its 28 | * body and reports the occurrence towards a previously configured statsd instance. The call is 29 | * issued through the {@link StatsDClient#incrementCounter(String)} method. 30 | * @author mnxfst 31 | * @since 12.02.2016 32 | */ 33 | public class MessageCountingMetricsReporter extends RichFilterFunction { 34 | 35 | private static final long serialVersionUID = -6890516061476629541L; 36 | 37 | private StatsDClient statsdClient; 38 | private final String statsdHost; 39 | private final int statsdPort; 40 | private final String[] statsdMetrics; 41 | private final String statsdPrefix; 42 | 43 | /** 44 | * Initializes the message flow counter using the provided input 45 | * @param statsdHost 46 | * @param statsdPort 47 | * @param statsdPrefix 48 | * @param statsdMetrics 49 | */ 50 | public MessageCountingMetricsReporter(final String statsdHost, final int statsdPort, final String statsdPrefix, final String[] statsdMetrics) { 51 | 52 | //////////////////////////////////////////////////////// 53 | // validate provided input 54 | if(StringUtils.isBlank(statsdHost)) 55 | throw new IllegalArgumentException("Missing required statsd host"); 56 | if(statsdPort < 1) 57 | throw new IllegalArgumentException("Missing valid statsd port"); 58 | if(statsdMetrics == null || statsdMetrics.length < 1) 59 | throw new IllegalArgumentException("Missing metrics to report values to"); 60 | 61 | for(final String s : statsdMetrics) 62 | if(StringUtils.isBlank(s)) 63 | throw new IllegalArgumentException("Empty strings are not permitted as members of metrics list"); 64 | //////////////////////////////////////////////////////// 65 | 66 | this.statsdHost = statsdHost; 67 | this.statsdPort = statsdPort; 68 | this.statsdPrefix = statsdPrefix; 69 | this.statsdMetrics = statsdMetrics; 70 | } 71 | 72 | /** 73 | * @see org.apache.flink.api.common.functions.AbstractRichFunction#open(org.apache.flink.configuration.Configuration) 74 | */ 75 | public void open(Configuration parameters) throws Exception { 76 | this.statsdClient = new NonBlockingStatsDClient(this.statsdPrefix, this.statsdHost, this.statsdPort); 77 | } 78 | 79 | /** 80 | * @see org.apache.flink.api.common.functions.RichFilterFunction#filter(java.lang.Object) 81 | */ 82 | public boolean filter(T value) throws Exception { 83 | for(final String s : this.statsdMetrics) 84 | this.statsdClient.incrementCounter(s); 85 | return true; 86 | } 87 | 88 | /** 89 | * Sets the statsd client - implemented for testing purpose only 90 | * @param client 91 | */ 92 | protected void setStatsDClient(final StatsDClient client) { 93 | this.statsdClient = client; 94 | } 95 | 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/runtime/StreamingAppConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.runtime; 18 | 19 | import java.io.Serializable; 20 | 21 | import javax.validation.constraints.Min; 22 | import javax.validation.constraints.NotNull; 23 | import javax.validation.constraints.Size; 24 | 25 | import com.fasterxml.jackson.annotation.JsonProperty; 26 | 27 | /** 28 | * Must be extended by all {@link StreamingAppRuntime} based applications which 29 | * required a configuration 30 | * @author mnxfst 31 | * @since Feb 22, 2016 32 | */ 33 | public abstract class StreamingAppConfiguration implements Serializable { 34 | 35 | private static final long serialVersionUID = 9178370371151923779L; 36 | 37 | /** degree of parallelism to apply on application startup - default: 1, value must not be null and larger than zero */ 38 | @JsonProperty(value="parallelism", required = true) 39 | @NotNull 40 | @Min(1) 41 | private int parallelism = 1; 42 | 43 | /** number of retries in case the application cannot be properly executed - default: 1, value must not be null and larger than zero */ 44 | @JsonProperty(value="executionRetries", required = true) 45 | @NotNull 46 | @Min(1) 47 | private int executionRetries = 1; 48 | 49 | /** name of the application - must not be null and hold at least one character */ 50 | @JsonProperty(value="name", required=true) 51 | @NotNull 52 | @Size(min=1) 53 | private String applicationName; 54 | 55 | /** description which must not be null and hold at least one character */ 56 | @JsonProperty(value="description", required=true) 57 | @NotNull 58 | @Size(min=1) 59 | private String applicationDescription; 60 | 61 | public StreamingAppConfiguration() { 62 | } 63 | 64 | public StreamingAppConfiguration(final String name, final String description) { 65 | this.applicationName = name; 66 | this.applicationDescription = description; 67 | } 68 | 69 | public String getApplicationName() { 70 | return applicationName; 71 | } 72 | 73 | public void setApplicationName(String applicationName) { 74 | this.applicationName = applicationName; 75 | } 76 | 77 | public String getApplicationDescription() { 78 | return applicationDescription; 79 | } 80 | 81 | public void setApplicationDescription(String applicationDescription) { 82 | this.applicationDescription = applicationDescription; 83 | } 84 | 85 | public int getParallelism() { 86 | return parallelism; 87 | } 88 | 89 | public void setParallelism(int parallelism) { 90 | this.parallelism = parallelism; 91 | } 92 | 93 | public int getExecutionRetries() { 94 | return executionRetries; 95 | } 96 | 97 | public void setExecutionRetries(int executionRetries) { 98 | this.executionRetries = executionRetries; 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/sink/elasticsearch/ElasticsearchNodeAddress.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.sink.elasticsearch; 18 | 19 | import java.io.Serializable; 20 | 21 | import javax.validation.constraints.Min; 22 | import javax.validation.constraints.NotNull; 23 | import javax.validation.constraints.Size; 24 | 25 | import com.fasterxml.jackson.annotation.JsonProperty; 26 | 27 | /** 28 | * Describes an elasticsearch node address consisting of host and port information 29 | * @author mnxfst 30 | * @since Feb 22, 2016 31 | */ 32 | public class ElasticsearchNodeAddress implements Serializable { 33 | 34 | private static final long serialVersionUID = 374026005018231257L; 35 | 36 | /** host the elasticsearch server is located on - must neither be null nor be empty */ 37 | @JsonProperty(value="host", required=true) 38 | @NotNull @Size(min=1) 39 | private String host; 40 | 41 | /** port the elasticsearch server listens to - must neither be null nor must it show a values less than 1 */ 42 | @JsonProperty(value="port", required=true) 43 | @NotNull @Min(1) 44 | private Integer port; 45 | 46 | public ElasticsearchNodeAddress() { 47 | } 48 | 49 | public ElasticsearchNodeAddress(final String host, int port) { 50 | this.host = host; 51 | this.port = port; 52 | } 53 | 54 | public String getHost() { 55 | return host; 56 | } 57 | 58 | public void setHost(String host) { 59 | this.host = host; 60 | } 61 | 62 | public int getPort() { 63 | return port; 64 | } 65 | 66 | public void setPort(int port) { 67 | this.port = port; 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/sink/elasticsearch/ElasticsearchSinkConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.sink.elasticsearch; 18 | 19 | import java.io.Serializable; 20 | import java.util.ArrayList; 21 | 22 | import javax.validation.constraints.NotNull; 23 | import javax.validation.constraints.Size; 24 | 25 | import com.fasterxml.jackson.annotation.JsonProperty; 26 | import com.fasterxml.jackson.annotation.JsonRootName; 27 | 28 | /** 29 | * Provides an enclosed block to cover all required configuration options to properly set up an {@link ElasticsearchSink} instance 30 | * @author mnxfst 31 | * @since Feb 23, 2016 32 | */ 33 | @JsonRootName(value="elasticsearch") 34 | public class ElasticsearchSinkConfiguration implements Serializable { 35 | 36 | private static final long serialVersionUID = -2277569536091711140L; 37 | 38 | /** name of the cluster to establish a connection with - must not be null and hold at least one character */ 39 | @JsonProperty(value="cluster", required=true) 40 | @NotNull @Size(min=1) 41 | private String cluster = null; 42 | 43 | /** index to read data from - must not be null and hold at least one character */ 44 | @JsonProperty(value="index", required=true) 45 | @NotNull @Size(min=1) 46 | private String index = null; 47 | 48 | /** type of document what are written to cluster by this application - must not be null and hold at least one character */ 49 | @JsonProperty(value="documentType", required=true) 50 | @NotNull @Size(min=1) 51 | private String documentType = null; 52 | 53 | /** list references towards elasticsearch server nodes - must not be null */ 54 | @JsonProperty(value="servers", required=true) 55 | @NotNull 56 | private ArrayList servers = new ArrayList<>(); 57 | 58 | public ElasticsearchSinkConfiguration() { 59 | } 60 | 61 | public ElasticsearchSinkConfiguration(final String cluster, final String index, final String documentType) { 62 | this.cluster = cluster; 63 | this.index = index; 64 | this.documentType = documentType; 65 | } 66 | 67 | public String getCluster() { 68 | return cluster; 69 | } 70 | 71 | public void setCluster(String cluster) { 72 | this.cluster = cluster; 73 | } 74 | 75 | public String getIndex() { 76 | return index; 77 | } 78 | 79 | public void setIndex(String index) { 80 | this.index = index; 81 | } 82 | 83 | public String getDocumentType() { 84 | return documentType; 85 | } 86 | 87 | public void setDocumentType(String documentType) { 88 | this.documentType = documentType; 89 | } 90 | 91 | public ArrayList getServers() { 92 | return servers; 93 | } 94 | 95 | public void setServers(ArrayList servers) { 96 | this.servers = servers; 97 | } 98 | 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/sink/kafka/KafkaProducerBuilder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.sink.kafka; 18 | 19 | import java.io.Serializable; 20 | import java.util.Properties; 21 | 22 | import org.apache.commons.lang3.StringUtils; 23 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer09; 24 | import org.apache.flink.streaming.util.serialization.SerializationSchema; 25 | import org.apache.kafka.clients.producer.ProducerConfig; 26 | 27 | import com.ottogroup.bi.streaming.source.kafka.KafkaConsumerBuilder; 28 | 29 | /** 30 | * Provides support for creating {@link FlinkKafkaProducer08} instances 31 | * @author mnxfst 32 | * @since Feb 29, 2016 33 | */ 34 | public class KafkaProducerBuilder implements Serializable { 35 | 36 | private static final long serialVersionUID = 878130140393093604L; 37 | 38 | private Properties properties = new Properties(); 39 | private String topic; 40 | private String brokerList; 41 | private SerializationSchema serializationSchema; 42 | 43 | private KafkaProducerBuilder() { 44 | } 45 | 46 | /** 47 | * Returns a new {@link KafkaConsumerBuilder} instance 48 | * @return 49 | */ 50 | public static KafkaProducerBuilder getInstance() { 51 | return new KafkaProducerBuilder(); 52 | } 53 | 54 | /** 55 | * Sets the topic to produce data to 56 | * @param topic 57 | * @return 58 | */ 59 | public KafkaProducerBuilder topic(final String topic) { 60 | if(StringUtils.isNotBlank(topic)) 61 | this.topic = topic; 62 | return this; 63 | } 64 | 65 | /** 66 | * Sets the broker list to produce data to 67 | * @param topic 68 | * @return 69 | */ 70 | public KafkaProducerBuilder brokerList(final String brokerList) { 71 | if(StringUtils.isNotBlank(brokerList)) 72 | this.brokerList = brokerList; 73 | return this; 74 | } 75 | 76 | /** 77 | * Adds a new key/value pair to properties 78 | * @param key 79 | * @param value 80 | * @return 81 | */ 82 | public KafkaProducerBuilder addProperty(final String key, final String value) { 83 | if(StringUtils.isNotBlank(key) && value != null) 84 | this.properties.put(StringUtils.lowerCase(StringUtils.trim(key)), value); 85 | return this; 86 | } 87 | 88 | /** 89 | * Adds all key/value pairs to properties 90 | * @param properties 91 | * @return 92 | */ 93 | public KafkaProducerBuilder addProperties(final Properties properties) { 94 | if(properties != null && !properties.isEmpty()) 95 | this.properties.putAll(properties); 96 | return this; 97 | } 98 | 99 | 100 | /** 101 | * Sets the {@link SerializationSchema} required for writing data to kafka topic 102 | * @param serializationSchema 103 | * @return 104 | */ 105 | public KafkaProducerBuilder serializationSchema(final SerializationSchema serializationSchema) { 106 | if(serializationSchema != null) 107 | this.serializationSchema = serializationSchema; 108 | return this; 109 | } 110 | 111 | 112 | /** 113 | * Create a {@link FlinkKafkaProducer09} depending on the provided settings 114 | * @param version 115 | * @return 116 | */ 117 | public FlinkKafkaProducer09 create() { 118 | 119 | ///////////////////////////////////////////////////////////////////////// 120 | // validate provided input 121 | if(StringUtils.isBlank(this.topic)) 122 | throw new IllegalArgumentException("Missing required topic"); 123 | if(StringUtils.isBlank(this.brokerList)) 124 | throw new IllegalArgumentException("Missing required broker list"); 125 | ///////////////////////////////////////////////////////////////////////// 126 | 127 | if(!this.properties.isEmpty()) { 128 | Properties producerProperties = new Properties(); 129 | producerProperties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, this.brokerList); 130 | 131 | for(Object k : this.properties.keySet()) 132 | producerProperties.put(k, this.properties.get(k)); 133 | return new FlinkKafkaProducer09<>(this.topic, this.serializationSchema, producerProperties); 134 | } 135 | return new FlinkKafkaProducer09(this.brokerList, this.topic, this.serializationSchema); 136 | } 137 | 138 | /** 139 | * Returns the broker list - implemented for testing purpose only 140 | * @return 141 | */ 142 | protected String getBrokerList() { 143 | return this.brokerList; 144 | } 145 | 146 | /** 147 | * Returns the managed topic - implemented for testing purpose only 148 | * @return 149 | */ 150 | protected String getTopic() { 151 | return this.topic; 152 | } 153 | 154 | /** 155 | * Returns the managed topic - implemented for testing purpose only 156 | * @return 157 | */ 158 | protected Properties getProperties() { 159 | return this.properties; 160 | } 161 | 162 | } 163 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/sink/kafka/KafkaProducerConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.sink.kafka; 18 | 19 | import java.io.Serializable; 20 | 21 | import javax.validation.constraints.NotNull; 22 | import javax.validation.constraints.Size; 23 | 24 | import com.fasterxml.jackson.annotation.JsonProperty; 25 | 26 | /** 27 | * Provides required information for setting up an instance of {@link FlinkKafkaProducer08} 28 | * @author mnxfst 29 | * @since Feb 29, 2016 30 | */ 31 | public class KafkaProducerConfiguration implements Serializable { 32 | 33 | private static final long serialVersionUID = -6789016956806675242L; 34 | 35 | /** comma-separated list of host:port combinations referencing kafka brokers - must not be null and hold at least one character */ 36 | @JsonProperty(value="brokerList", required=true) 37 | @NotNull 38 | @Size(min=1) 39 | private String brokerList = null; 40 | 41 | /** name of topic to receive data from - must not be null and hold at least one character */ 42 | @JsonProperty(value="topic", required=true) 43 | @NotNull 44 | @Size(min=1) 45 | private String topic = null; 46 | 47 | public KafkaProducerConfiguration() { 48 | } 49 | 50 | /** 51 | * Initializes the instance using the provided input 52 | * @param brokerList 53 | * @param topic 54 | */ 55 | public KafkaProducerConfiguration(final String brokerList, final String topic) { 56 | this.brokerList = brokerList; 57 | this.topic = topic; 58 | } 59 | 60 | public String getBrokerList() { 61 | return brokerList; 62 | } 63 | 64 | public void setBrokerList(String brokerList) { 65 | this.brokerList = brokerList; 66 | } 67 | 68 | public String getTopic() { 69 | return topic; 70 | } 71 | 72 | public void setTopic(String topic) { 73 | this.topic = topic; 74 | } 75 | 76 | 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/source/kafka/KafkaConsumerBuilder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.source.kafka; 18 | 19 | import java.io.Serializable; 20 | import java.util.Properties; 21 | 22 | import org.apache.commons.lang3.StringUtils; 23 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer09; 24 | import org.apache.flink.streaming.util.serialization.DeserializationSchema; 25 | 26 | /** 27 | * Provides support for creating {@link FlinkKafkaConsumer} instances 28 | * @author mnxfst 29 | * @since Feb 3, 2016 30 | */ 31 | public class KafkaConsumerBuilder implements Serializable { 32 | 33 | private static final long serialVersionUID = -3497945088790519194L; 34 | 35 | public static final String KAFKA_PROPS_GROUP_ID = "group.id"; 36 | public static final String KAFKA_PROPS_AUTO_COMMIT_ENABLE = "enable.auto.commit"; 37 | public static final String KAFKA_PROPS_BOOTSTRAP_SERVERS = "bootstrap.servers"; 38 | 39 | 40 | private Properties properties = new Properties(); 41 | private String topic; 42 | private DeserializationSchema deserializationSchema; 43 | 44 | private KafkaConsumerBuilder() { 45 | } 46 | 47 | /** 48 | * Returns a new {@link KafkaConsumerBuilder} instance 49 | * @return 50 | */ 51 | public static KafkaConsumerBuilder getInstance() { 52 | return new KafkaConsumerBuilder(); 53 | } 54 | 55 | /** 56 | * Adds a new key/value pair to properties 57 | * @param key 58 | * @param value 59 | * @return 60 | */ 61 | public KafkaConsumerBuilder addProperty(final String key, final String value) { 62 | if(StringUtils.isNotBlank(key) && value != null) 63 | this.properties.put(StringUtils.lowerCase(StringUtils.trim(key)), value); 64 | return this; 65 | } 66 | 67 | /** 68 | * Adds all key/value pairs to properties 69 | * @param properties 70 | * @return 71 | */ 72 | public KafkaConsumerBuilder addProperties(final Properties properties) { 73 | if(properties != null && !properties.isEmpty()) 74 | this.properties.putAll(properties); 75 | return this; 76 | } 77 | 78 | /** 79 | * Sets the topic to consume data from 80 | * @param topic 81 | * @return 82 | */ 83 | public KafkaConsumerBuilder topic(final String topic) { 84 | if(StringUtils.isNotBlank(topic)) 85 | this.topic = topic; 86 | return this; 87 | } 88 | 89 | /** 90 | * Sets the {@link DeserializationSchema} required for reading data from kafka topic into 91 | * processable format 92 | * @param deserializationSchema 93 | * @return 94 | */ 95 | public KafkaConsumerBuilder deserializationSchema(final DeserializationSchema deserializationSchema) { 96 | if(deserializationSchema != null) 97 | this.deserializationSchema = deserializationSchema; 98 | return this; 99 | } 100 | 101 | /** 102 | * Create a {@link FlinkKafkaConsumer} depending on the provided settings 103 | * @param version 104 | * @return 105 | */ 106 | public FlinkKafkaConsumer09 create() { 107 | 108 | ///////////////////////////////////////////////////////////////////////// 109 | // validate provided input 110 | if(StringUtils.isBlank(this.topic)) 111 | throw new IllegalArgumentException("Missing required topic"); 112 | if(this.properties.isEmpty()) 113 | throw new IllegalArgumentException("Missing required properties"); 114 | if(!this.properties.containsKey(KAFKA_PROPS_AUTO_COMMIT_ENABLE)) 115 | throw new IllegalArgumentException("Missing value for required property '"+KAFKA_PROPS_AUTO_COMMIT_ENABLE+"'"); 116 | if(!this.properties.containsKey(KAFKA_PROPS_BOOTSTRAP_SERVERS)) 117 | throw new IllegalArgumentException("Missing value for required property '"+KAFKA_PROPS_BOOTSTRAP_SERVERS+"'"); 118 | if(!this.properties.containsKey(KAFKA_PROPS_GROUP_ID)) 119 | throw new IllegalArgumentException("Missing value for required property '"+KAFKA_PROPS_GROUP_ID+"'"); 120 | ///////////////////////////////////////////////////////////////////////// 121 | 122 | return new FlinkKafkaConsumer09<>(this.topic, this.deserializationSchema, this.properties); 123 | } 124 | 125 | /** 126 | * Returns the managed properties - implemented for testing purpose only 127 | * @return 128 | */ 129 | protected Properties getProperties() { 130 | return this.properties; 131 | } 132 | 133 | /** 134 | * Returns the managed topic - implemented for testing purpose only 135 | * @return 136 | */ 137 | protected String getTopic() { 138 | return this.topic; 139 | } 140 | 141 | /** 142 | * Returns the managed deserialization schema - implemented for testing purpose only 143 | * @return 144 | */ 145 | protected DeserializationSchema getDeserializationSchema() { 146 | return this.deserializationSchema; 147 | } 148 | 149 | 150 | } 151 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/source/kafka/KafkaConsumerConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.source.kafka; 18 | 19 | import java.io.Serializable; 20 | 21 | import javax.validation.constraints.NotNull; 22 | import javax.validation.constraints.Size; 23 | 24 | import com.fasterxml.jackson.annotation.JsonProperty; 25 | 26 | /** 27 | * Provides required configuration to set up a {@link FlinkKafkaConsumer} 28 | * @author mnxfst 29 | * @since Feb 23, 2016 30 | */ 31 | public class KafkaConsumerConfiguration implements Serializable { 32 | 33 | private static final long serialVersionUID = 2804346209553044550L; 34 | 35 | /** sets the auto-commit parameter for the kafka consumer - default: true */ 36 | @JsonProperty(value="autoCommit", required=false) 37 | private boolean autoCommit = false; 38 | 39 | /** comma-separated list of host:port combinations referencing kafka brokers - must not be null and hold at least one character */ 40 | @JsonProperty(value="bootstrapServers", required=true) 41 | @NotNull 42 | @Size(min=1) 43 | private String bootstrapServers = null; 44 | 45 | /** identifier to use when establishing a connection with the kafka cluster - must not be null and hold at least one character */ 46 | @JsonProperty(value="groupId", required=true) 47 | @NotNull 48 | @Size(min=1) 49 | private String groupId = null; 50 | 51 | /** name of topic to receive data from - must not be null and hold at least one character */ 52 | @JsonProperty(value="topic", required=true) 53 | @NotNull 54 | @Size(min=1) 55 | private String topic = null; 56 | 57 | public KafkaConsumerConfiguration() { 58 | } 59 | 60 | public KafkaConsumerConfiguration(final String topic, final String bootstrapServers, final String groupId, final boolean autoCommit) { 61 | this.topic = topic; 62 | this.bootstrapServers = bootstrapServers; 63 | this.groupId = groupId; 64 | this.autoCommit = autoCommit; 65 | } 66 | 67 | 68 | public boolean isAutoCommit() { 69 | return autoCommit; 70 | } 71 | 72 | public void setAutoCommit(boolean autoCommit) { 73 | this.autoCommit = autoCommit; 74 | } 75 | 76 | public String getBootstrapServers() { 77 | return bootstrapServers; 78 | } 79 | 80 | public void setBootstrapServers(String bootstrapServers) { 81 | this.bootstrapServers = bootstrapServers; 82 | } 83 | 84 | public String getGroupId() { 85 | return groupId; 86 | } 87 | 88 | public void setGroupId(String groupId) { 89 | this.groupId = groupId; 90 | } 91 | 92 | public String getTopic() { 93 | return topic; 94 | } 95 | 96 | public void setTopic(String topic) { 97 | this.topic = topic; 98 | } 99 | 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/com/ottogroup/bi/streaming/testing/JSONFieldContentMatcher.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.testing; 18 | 19 | import org.apache.commons.lang3.StringUtils; 20 | import org.apache.sling.commons.json.JSONObject; 21 | import org.hamcrest.Description; 22 | import org.hamcrest.Matcher; 23 | import org.hamcrest.TypeSafeDiagnosingMatcher; 24 | 25 | import com.ottogroup.bi.streaming.operator.json.JsonContentReference; 26 | import com.ottogroup.bi.streaming.operator.json.JsonContentType; 27 | import com.ottogroup.bi.streaming.operator.json.JsonProcessingUtils; 28 | 29 | /** 30 | * Wraps a {@link Matcher} to verify content at given location inside a {@link JSONObject}. 31 | * @author mnxfst 32 | * @since Apr 25, 2016 33 | */ 34 | public class JSONFieldContentMatcher extends TypeSafeDiagnosingMatcher { 35 | 36 | /** provides utilities to access content inside json documents */ 37 | private JsonProcessingUtils jsonUtils = new JsonProcessingUtils(); 38 | /** reference into json document to read content from that must be verified */ 39 | private JsonContentReference contentReference; 40 | /** actual matcher to be used for content verification */ 41 | private Matcher contentMatcher = null; 42 | /** helper variable to store the content path as dot separated string - used inside error messages, avoids recomputation */ 43 | private String flattenedContentPath = null; 44 | 45 | /** 46 | * Initializes the json content matcher using the provided input 47 | * @param contentReference 48 | * @param contentMatcher 49 | */ 50 | public JSONFieldContentMatcher(final JsonContentReference contentReference, final Matcher contentMatcher) { 51 | 52 | /////////////////////////////////////////////////////////// 53 | // validate provided input 54 | if(contentReference == null) 55 | throw new IllegalArgumentException("Missing required content reference"); 56 | if(contentReference.getPath() == null || contentReference.getPath().length < 1) 57 | throw new IllegalArgumentException("Missing required content path"); 58 | if(contentReference.getContentType() == JsonContentType.TIMESTAMP && StringUtils.isBlank(contentReference.getConversionPattern())) 59 | throw new IllegalArgumentException("Missing required conversion pattern for timestamp values"); 60 | if(contentMatcher == null) 61 | throw new IllegalArgumentException("Missing required content matcher"); 62 | /////////////////////////////////////////////////////////// 63 | 64 | this.contentReference = contentReference; 65 | this.contentMatcher = contentMatcher; 66 | 67 | /////////////////////////////////////////////////////////// 68 | // create flattened 69 | this.flattenedContentPath = String.join(".", contentReference.getPath()); 70 | /////////////////////////////////////////////////////////// 71 | } 72 | 73 | /** 74 | * @see org.hamcrest.SelfDescribing#describeTo(org.hamcrest.Description) 75 | */ 76 | public void describeTo(Description description) { 77 | if(description != null) { 78 | StringBuffer buf = new StringBuffer(); 79 | buf.append("path '").append(this.flattenedContentPath).append("' ").append(contentMatcher); 80 | description.appendText(buf.toString()); 81 | } 82 | } 83 | 84 | /** 85 | * @see org.hamcrest.TypeSafeDiagnosingMatcher#matchesSafely(java.lang.Object, org.hamcrest.Description) 86 | */ 87 | protected boolean matchesSafely(JSONObject item, Description mismatchDescription) { 88 | 89 | if(item == null) { 90 | this.contentMatcher.describeMismatch(null, mismatchDescription); 91 | mismatchDescription.appendText(" for path '"+this.flattenedContentPath+"' on " + item + " but should match ("); 92 | contentMatcher.describeTo(mismatchDescription); 93 | mismatchDescription.appendText(")"); 94 | return false; 95 | } 96 | 97 | try { 98 | final Object jsonFieldValue; 99 | switch(contentReference.getContentType()) { 100 | case BOOLEAN: { 101 | jsonFieldValue = this.jsonUtils.getBooleanFieldValue(item, this.contentReference.getPath(), this.contentReference.isRequired()); 102 | break; 103 | } 104 | case DOUBLE: { 105 | jsonFieldValue = this.jsonUtils.getDoubleFieldValue(item, this.contentReference.getPath(), this.contentReference.isRequired()); 106 | break; 107 | } 108 | case INTEGER: { 109 | jsonFieldValue = this.jsonUtils.getIntegerFieldValue(item, this.contentReference.getPath(), this.contentReference.isRequired()); 110 | break; 111 | } 112 | case TIMESTAMP: { 113 | jsonFieldValue = this.jsonUtils.getDateTimeFieldValue(item, this.contentReference.getPath(), this.contentReference.getConversionPattern(), this.contentReference.isRequired()); 114 | break; 115 | } 116 | default: { 117 | jsonFieldValue = this.jsonUtils.getTextFieldValue(item, this.contentReference.getPath(), this.contentReference.isRequired()); 118 | break; 119 | } 120 | } 121 | final boolean result = this.contentMatcher.matches(jsonFieldValue); 122 | if(!result) { 123 | this.contentMatcher.describeMismatch(jsonFieldValue, mismatchDescription); 124 | mismatchDescription.appendText(" for path '"+this.flattenedContentPath+"' on " + item + " but should match ("); 125 | contentMatcher.describeTo(mismatchDescription); 126 | mismatchDescription.appendText(")"); 127 | } 128 | return result; 129 | } catch (Exception e) { 130 | mismatchDescription.appendText("[content extraction failed: "+e.getMessage()+"]"); 131 | return false; 132 | } 133 | } 134 | 135 | /** 136 | * Returns the flattened content path
Note: implemented for testing purpose only 137 | * @return the flattenedContentPath 138 | */ 139 | protected String getFlattenedContentPath() { 140 | return flattenedContentPath; 141 | } 142 | 143 | } 144 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/operator/json/aggregate/functions/BooleanContentAggregateFunctionTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.aggregate.functions; 18 | 19 | import org.apache.commons.lang3.tuple.MutablePair; 20 | import org.junit.Assert; 21 | import org.junit.Test; 22 | import org.mockito.Mockito; 23 | 24 | /** 25 | * Test case for {@link BooleanContentAggregateFunction} 26 | * @author mnxfst 27 | * @since Jan 27, 2016 28 | */ 29 | public class BooleanContentAggregateFunctionTest { 30 | 31 | /** 32 | * Test case for {@link BooleanContentAggregateFunction#sum(Boolean, Boolean)} 33 | */ 34 | @Test(expected=UnsupportedOperationException.class) 35 | public void testSum() throws Exception { 36 | new BooleanContentAggregateFunction().sum(true, false); 37 | } 38 | 39 | /** 40 | * Test case for {@link BooleanContentAggregateFunction#min(Boolean, Boolean)} 41 | */ 42 | @Test(expected=UnsupportedOperationException.class) 43 | public void testMin() throws Exception { 44 | new BooleanContentAggregateFunction().min(true, false); 45 | } 46 | 47 | /** 48 | * Test case for {@link BooleanContentAggregateFunction#max(Boolean, Boolean)} 49 | */ 50 | @Test(expected=UnsupportedOperationException.class) 51 | public void testMax() throws Exception { 52 | new BooleanContentAggregateFunction().max(true, false); 53 | } 54 | 55 | /** 56 | * Test case for {@link BooleanContentAggregateFunction#average(org.apache.commons.lang3.tuple.MutablePair, Boolean)} 57 | */ 58 | @SuppressWarnings("unchecked") 59 | @Test(expected=UnsupportedOperationException.class) 60 | public void testAverage() throws Exception { 61 | new BooleanContentAggregateFunction().average(Mockito.mock(MutablePair.class), false); 62 | } 63 | 64 | /** 65 | * Test case for {@link BooleanContentAggregateFunction#count(Integer)} being provided null as input 66 | */ 67 | @Test 68 | public void testCount_withNullInput() throws Exception { 69 | Assert.assertEquals(Integer.valueOf(1), new BooleanContentAggregateFunction().count(null)); 70 | } 71 | 72 | /** 73 | * Test case for {@link BooleanContentAggregateFunction#count(Integer)} being provided a valid integer as input 74 | */ 75 | @Test 76 | public void testCount_withValidInput() throws Exception { 77 | Assert.assertEquals(Integer.valueOf(123), new BooleanContentAggregateFunction().count(Integer.valueOf(122))); 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/operator/json/aggregate/functions/DoubleContentAggregateFunctionTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.aggregate.functions; 18 | 19 | import org.apache.commons.lang3.tuple.MutablePair; 20 | import org.junit.Assert; 21 | import org.junit.Test; 22 | 23 | 24 | 25 | /** 26 | * Test case for {@link DoubleContentAggregateFunction} 27 | * @author mnxfst 28 | * @since Jan 27, 2016 29 | */ 30 | public class DoubleContentAggregateFunctionTest { 31 | 32 | /** 33 | * Test case for {@link DoubleContentAggregateFunction#sum(Double, Double)} being 34 | * provided null as input to both parameters 35 | */ 36 | @Test 37 | public void testSum_withNullForOldAndNew() throws Exception { 38 | Assert.assertNull(new DoubleContentAggregateFunction().sum(null, null)); 39 | } 40 | 41 | /** 42 | * Test case for {@link DoubleContentAggregateFunction#sum(Double, Double)} being 43 | * provided null as input to old sum parameter 44 | */ 45 | @Test 46 | public void testSum_withNullOldSum() throws Exception { 47 | Assert.assertEquals(Double.valueOf(1.23), new DoubleContentAggregateFunction().sum(null, Double.valueOf(1.23))); 48 | } 49 | 50 | /** 51 | * Test case for {@link DoubleContentAggregateFunction#sum(Double, Double)} being 52 | * provided null as input to new sum parameter 53 | */ 54 | @Test 55 | public void testSum_withNullNewSum() throws Exception { 56 | Assert.assertEquals(Double.valueOf(1.23), new DoubleContentAggregateFunction().sum(Double.valueOf(1.23), null)); 57 | } 58 | 59 | /** 60 | * Test case for {@link DoubleContentAggregateFunction#sum(Double, Double)} being 61 | * provided valid input to both parameter 62 | */ 63 | @Test 64 | public void testSum_withValidInput() throws Exception { 65 | Assert.assertEquals(Double.valueOf(1.26), new DoubleContentAggregateFunction().sum(Double.valueOf(1.23), Double.valueOf(0.03))); 66 | } 67 | 68 | /** 69 | * Test case for {@link DoubleContentAggregateFunction#min(Double, Double)} being 70 | * provided null as input to both parameters 71 | */ 72 | @Test 73 | public void testMin_withNullForOldAndNew() throws Exception { 74 | Assert.assertNull(new DoubleContentAggregateFunction().min(null, null)); 75 | } 76 | 77 | /** 78 | * Test case for {@link DoubleContentAggregateFunction#min(Double, Double)} being 79 | * provided null as input to old min parameter 80 | */ 81 | @Test 82 | public void testMin_withNullOldMin() throws Exception { 83 | Assert.assertEquals(Double.valueOf(9.87), new DoubleContentAggregateFunction().min(null, Double.valueOf(9.87))); 84 | } 85 | 86 | /** 87 | * Test case for {@link DoubleContentAggregateFunction#min(Double, Double)} being 88 | * provided null as input to new min parameter 89 | */ 90 | @Test 91 | public void testMin_withNullOldNew() throws Exception { 92 | Assert.assertEquals(Double.valueOf(9.87), new DoubleContentAggregateFunction().min(Double.valueOf(9.87), null)); 93 | } 94 | 95 | /** 96 | * Test case for {@link DoubleContentAggregateFunction#min(Double, Double)} being 97 | * provided valid input to both parameters 98 | */ 99 | @Test 100 | public void testMin_withValidInput() throws Exception { 101 | Assert.assertEquals(Double.valueOf(9.87), new DoubleContentAggregateFunction().min(Double.valueOf(9.88), Double.valueOf(9.87))); 102 | Assert.assertEquals(Double.valueOf(9.87), new DoubleContentAggregateFunction().min(Double.valueOf(9.87), Double.valueOf(9.88))); 103 | } 104 | /** 105 | * Test case for {@link DoubleContentAggregateFunction#max(Double, Double)} being 106 | * provided null as input to both parameters 107 | */ 108 | @Test 109 | public void testMax_withNullForOldAndNew() throws Exception { 110 | Assert.assertNull(new DoubleContentAggregateFunction().max(null, null)); 111 | } 112 | 113 | /** 114 | * Test case for {@link DoubleContentAggregateFunction#max(Double, Double)} being 115 | * provided null as input to old max parameter 116 | */ 117 | @Test 118 | public void testMax_withNullOldMin() throws Exception { 119 | Assert.assertEquals(Double.valueOf(9.87), new DoubleContentAggregateFunction().max(null, Double.valueOf(9.87))); 120 | } 121 | 122 | /** 123 | * Test case for {@link DoubleContentAggregateFunction#max(Double, Double)} being 124 | * provided null as input to new max parameter 125 | */ 126 | @Test 127 | public void testMax_withNullOldNew() throws Exception { 128 | Assert.assertEquals(Double.valueOf(9.87), new DoubleContentAggregateFunction().max(Double.valueOf(9.87), null)); 129 | } 130 | 131 | /** 132 | * Test case for {@link DoubleContentAggregateFunction#max(Double, Double)} being 133 | * provided valid input to both parameters 134 | */ 135 | @Test 136 | public void testMax_withValidInput() throws Exception { 137 | Assert.assertEquals(Double.valueOf(9.88), new DoubleContentAggregateFunction().max(Double.valueOf(9.87), Double.valueOf(9.88))); 138 | Assert.assertEquals(Double.valueOf(9.88), new DoubleContentAggregateFunction().max(Double.valueOf(9.88), Double.valueOf(9.87))); 139 | } 140 | 141 | /** 142 | * Test case for {@link DoubleContentAggregateFunction#count(Integer)} being 143 | * provided null as input 144 | */ 145 | @Test 146 | public void testCount_withNullInput() throws Exception { 147 | Assert.assertEquals(Integer.valueOf(1), new DoubleContentAggregateFunction().count(null)); 148 | } 149 | 150 | /** 151 | * Test case for {@link DoubleContentAggregateFunction#count(Integer)} being 152 | * provided valid input 153 | */ 154 | @Test 155 | public void testCount_withValidInput() throws Exception { 156 | Assert.assertEquals(Integer.valueOf(8), new DoubleContentAggregateFunction().count(Integer.valueOf(7))); 157 | } 158 | 159 | /** 160 | * Test case for {@link DoubleContentAggregateFunction#average(org.apache.commons.lang3.tuple.MutablePair, Double)} 161 | * being provided null to both parameters 162 | */ 163 | @Test 164 | public void testAverage_withNullInputForBothParameters() throws Exception { 165 | MutablePair result = new DoubleContentAggregateFunction().average(null, null); 166 | Assert.assertEquals(Double.valueOf(0), result.getLeft()); 167 | Assert.assertEquals(Integer.valueOf(0), result.getRight()); 168 | } 169 | 170 | /** 171 | * Test case for {@link DoubleContentAggregateFunction#average(org.apache.commons.lang3.tuple.MutablePair, Double)} 172 | * being provided null to sumAndCount variable 173 | */ 174 | @Test 175 | public void testAverage_withNullInputToSumAndCount() throws Exception { 176 | MutablePair result = new DoubleContentAggregateFunction().average(null, Double.valueOf(1.23)); 177 | Assert.assertEquals(Double.valueOf(1.23), result.getLeft()); 178 | Assert.assertEquals(Integer.valueOf(1), result.getRight()); 179 | } 180 | 181 | /** 182 | * Test case for {@link DoubleContentAggregateFunction#average(org.apache.commons.lang3.tuple.MutablePair, Double)} 183 | * being provided null to value variable 184 | */ 185 | @Test 186 | public void testAverage_withNullInputToValue() throws Exception { 187 | MutablePair result = new DoubleContentAggregateFunction().average( 188 | new MutablePair(Double.valueOf(1.34), Integer.valueOf(4)), null); 189 | Assert.assertEquals(Double.valueOf(1.34), result.getLeft()); 190 | Assert.assertEquals(Integer.valueOf(4), result.getRight()); 191 | } 192 | 193 | /** 194 | * Test case for {@link DoubleContentAggregateFunction#average(org.apache.commons.lang3.tuple.MutablePair, Double)} 195 | * being provided valid input 196 | */ 197 | @Test 198 | public void testAverage_withValidInput() throws Exception { 199 | MutablePair result = new DoubleContentAggregateFunction().average( 200 | new MutablePair(Double.valueOf(1.34), Integer.valueOf(4)), Double.valueOf(8.22)); 201 | Assert.assertEquals(Double.valueOf(9.56), result.getLeft()); 202 | Assert.assertEquals(Integer.valueOf(5), result.getRight()); 203 | } 204 | 205 | } 206 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/operator/json/aggregate/functions/IntegerContentAggregateFunctionTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.aggregate.functions; 18 | 19 | import org.apache.commons.lang3.tuple.MutablePair; 20 | import org.junit.Assert; 21 | import org.junit.Test; 22 | 23 | /** 24 | * Test case for {@link IntegerContentAggregateFunction} 25 | * @author mnxfst 26 | * @since Jan 27, 2016 27 | */ 28 | public class IntegerContentAggregateFunctionTest { 29 | 30 | /** 31 | * Test case for {@link IntegerContentAggregateFunction#sum(Integer, Integer)} being 32 | * provided null to both parameters 33 | */ 34 | @Test 35 | public void testSum_withNullInput() throws Exception { 36 | Assert.assertNull(new IntegerContentAggregateFunction().sum(null, null)); 37 | } 38 | 39 | /** 40 | * Test case for {@link IntegerContentAggregateFunction#sum(Integer, Integer)} being 41 | * provided null to old value parameter 42 | */ 43 | @Test 44 | public void testSum_withNullOldSum() throws Exception { 45 | Assert.assertEquals(Integer.valueOf(123), new IntegerContentAggregateFunction().sum(null, Integer.valueOf(123))); 46 | } 47 | 48 | /** 49 | * Test case for {@link IntegerContentAggregateFunction#sum(Integer, Integer)} being 50 | * provided null to new value parameter 51 | */ 52 | @Test 53 | public void testSum_withNullNewSum() throws Exception { 54 | Assert.assertEquals(Integer.valueOf(123), new IntegerContentAggregateFunction().sum(Integer.valueOf(123), null)); 55 | } 56 | 57 | /** 58 | * Test case for {@link IntegerContentAggregateFunction#sum(Integer, Integer)} being 59 | * provided valid values to both parameters 60 | */ 61 | @Test 62 | public void testSum_withValidInput() throws Exception { 63 | Assert.assertEquals(Integer.valueOf(123), new IntegerContentAggregateFunction().sum(Integer.valueOf(120), Integer.valueOf(3))); 64 | } 65 | 66 | /** 67 | * Test case for {@link IntegerContentAggregateFunction#min(Integer, Integer)} being 68 | * provided null to both parameters 69 | */ 70 | @Test 71 | public void testMin_withNullInput() throws Exception { 72 | Assert.assertNull(new IntegerContentAggregateFunction().min(null, null)); 73 | } 74 | 75 | /** 76 | * Test case for {@link IntegerContentAggregateFunction#min(Integer, Integer)} being 77 | * provided null to old value parameter 78 | */ 79 | @Test 80 | public void testMin_withNullOldSum() throws Exception { 81 | Assert.assertEquals(Integer.valueOf(123), new IntegerContentAggregateFunction().min(null, Integer.valueOf(123))); 82 | } 83 | 84 | /** 85 | * Test case for {@link IntegerContentAggregateFunction#min(Integer, Integer)} being 86 | * provided null to new value parameter 87 | */ 88 | @Test 89 | public void testMin_withNullNewSum() throws Exception { 90 | Assert.assertEquals(Integer.valueOf(123), new IntegerContentAggregateFunction().min(Integer.valueOf(123), null)); 91 | } 92 | 93 | /** 94 | * Test case for {@link IntegerContentAggregateFunction#min(Integer, Integer)} being 95 | * provided valid values to both parameters 96 | */ 97 | @Test 98 | public void testMin_withValidInput() throws Exception { 99 | Assert.assertEquals(Integer.valueOf(3), new IntegerContentAggregateFunction().min(Integer.valueOf(120), Integer.valueOf(3))); 100 | Assert.assertEquals(Integer.valueOf(12), new IntegerContentAggregateFunction().min(Integer.valueOf(12), Integer.valueOf(123))); 101 | } 102 | 103 | /** 104 | * Test case for {@link IntegerContentAggregateFunction#max(Integer, Integer)} being 105 | * provided null to both parameters 106 | */ 107 | @Test 108 | public void testMax_withNullInput() throws Exception { 109 | Assert.assertNull(new IntegerContentAggregateFunction().max(null, null)); 110 | } 111 | 112 | /** 113 | * Test case for {@link IntegerContentAggregateFunction#max(Integer, Integer)} being 114 | * provided null to old value parameter 115 | */ 116 | @Test 117 | public void testMax_withNullOldSum() throws Exception { 118 | Assert.assertEquals(Integer.valueOf(123), new IntegerContentAggregateFunction().max(null, Integer.valueOf(123))); 119 | } 120 | 121 | /** 122 | * Test case for {@link IntegerContentAggregateFunction#max(Integer, Integer)} being 123 | * provided null to new value parameter 124 | */ 125 | @Test 126 | public void testMax_withNullNewSum() throws Exception { 127 | Assert.assertEquals(Integer.valueOf(123), new IntegerContentAggregateFunction().max(Integer.valueOf(123), null)); 128 | } 129 | 130 | /** 131 | * Test case for {@link IntegerContentAggregateFunction#max(Integer, Integer)} being 132 | * provided valid values to both parameters 133 | */ 134 | @Test 135 | public void testMax_withValidInput() throws Exception { 136 | Assert.assertEquals(Integer.valueOf(120), new IntegerContentAggregateFunction().max(Integer.valueOf(120), Integer.valueOf(3))); 137 | Assert.assertEquals(Integer.valueOf(123), new IntegerContentAggregateFunction().max(Integer.valueOf(3), Integer.valueOf(123))); 138 | } 139 | 140 | /** 141 | * Test case for {@link IntegerContentAggregateFunction#count(Integer)} being 142 | * provided null as input 143 | */ 144 | @Test 145 | public void testCount_withNullInput() throws Exception { 146 | Assert.assertEquals(Integer.valueOf(1), new IntegerContentAggregateFunction().count(null)); 147 | } 148 | 149 | /** 150 | * Test case for {@link IntegerContentAggregateFunction#count(Integer)} being 151 | * provided valid value as input 152 | */ 153 | @Test 154 | public void testCount_withValidInput() throws Exception { 155 | Assert.assertEquals(Integer.valueOf(34), new IntegerContentAggregateFunction().count(Integer.valueOf(33))); 156 | } 157 | 158 | /** 159 | * Test case for {@link IntegerContentAggregateFunction#average(org.apache.commons.lang3.tuple.MutablePair, Integer)} 160 | * being provided null as input to both parameters 161 | */ 162 | @Test 163 | public void testAverage_withNullInput() throws Exception { 164 | MutablePair result = new IntegerContentAggregateFunction().average(null, null); 165 | Assert.assertEquals(Integer.valueOf(0), result.getLeft()); 166 | Assert.assertEquals(Integer.valueOf(0), result.getRight()); 167 | } 168 | 169 | /** 170 | * Test case for {@link IntegerContentAggregateFunction#average(org.apache.commons.lang3.tuple.MutablePair, Integer)} 171 | * being provided null as input to old sumAndCount parameter 172 | */ 173 | @Test 174 | public void testAverage_withSumAndCountNullInput() throws Exception { 175 | MutablePair result = new IntegerContentAggregateFunction().average(null, Integer.valueOf(123)); 176 | Assert.assertEquals(Integer.valueOf(123), result.getLeft()); 177 | Assert.assertEquals(Integer.valueOf(1), result.getRight()); 178 | } 179 | 180 | /** 181 | * Test case for {@link IntegerContentAggregateFunction#average(org.apache.commons.lang3.tuple.MutablePair, Integer)} 182 | * being provided null as input to new value parameter 183 | */ 184 | @Test 185 | public void testAverage_withNewValueNullInput() throws Exception { 186 | MutablePair result = new IntegerContentAggregateFunction().average( 187 | new MutablePair(Integer.valueOf(125), Integer.valueOf(3)), null); 188 | Assert.assertEquals(Integer.valueOf(125), result.getLeft()); 189 | Assert.assertEquals(Integer.valueOf(3), result.getRight()); 190 | } 191 | 192 | /** 193 | * Test case for {@link IntegerContentAggregateFunction#average(org.apache.commons.lang3.tuple.MutablePair, Integer)} 194 | * being provided valid input to both parameters 195 | */ 196 | @Test 197 | public void testAverage_withValidInput() throws Exception { 198 | MutablePair result = new IntegerContentAggregateFunction().average( 199 | new MutablePair(Integer.valueOf(125), Integer.valueOf(3)), Integer.valueOf(43)); 200 | Assert.assertEquals(Integer.valueOf(168), result.getLeft()); 201 | Assert.assertEquals(Integer.valueOf(4), result.getRight()); 202 | } 203 | } 204 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/operator/json/aggregate/functions/StringContentAggregateFunctionTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.aggregate.functions; 18 | 19 | import org.apache.commons.lang3.tuple.MutablePair; 20 | import org.junit.Assert; 21 | import org.junit.Test; 22 | import org.mockito.Mockito; 23 | 24 | /** 25 | * Test case for {@link StringContentAggregateFunction} 26 | * @author mnxfst 27 | * @since Jan 27, 2016 28 | */ 29 | public class StringContentAggregateFunctionTest { 30 | 31 | /** 32 | * Test case for {@link StringContentAggregateFunction#sum(String, String)} 33 | */ 34 | @Test(expected=UnsupportedOperationException.class) 35 | public void testSum() throws Exception { 36 | new StringContentAggregateFunction().sum("test", "string"); 37 | } 38 | 39 | /** 40 | * Test case for {@link StringContentAggregateFunction#min(String, String)} 41 | */ 42 | @Test(expected=UnsupportedOperationException.class) 43 | public void testMin() throws Exception { 44 | new StringContentAggregateFunction().min("test", "string"); 45 | } 46 | 47 | /** 48 | * Test case for {@link StringContentAggregateFunction#max(String, String)} 49 | */ 50 | @Test(expected=UnsupportedOperationException.class) 51 | public void testMax() throws Exception { 52 | new StringContentAggregateFunction().max("test", "string"); 53 | } 54 | 55 | /** 56 | * Test case for {@link StringContentAggregateFunction#average(org.apache.commons.lang3.tuple.MutablePair, String)} 57 | */ 58 | @SuppressWarnings("unchecked") 59 | @Test(expected=UnsupportedOperationException.class) 60 | public void testAverage() throws Exception { 61 | new StringContentAggregateFunction().average(Mockito.mock(MutablePair.class), "test"); 62 | } 63 | 64 | /** 65 | * Test case for {@link StringContentAggregateFunction#count(Integer)} being provided null as input 66 | */ 67 | @Test 68 | public void testCount_withNullInput() throws Exception { 69 | Assert.assertEquals(Integer.valueOf(1), new StringContentAggregateFunction().count(null)); 70 | } 71 | 72 | /** 73 | * Test case for {@link StringContentAggregateFunction#count(Integer)} being provided a valid integer as input 74 | */ 75 | @Test 76 | public void testCount_withValidInput() throws Exception { 77 | Assert.assertEquals(Integer.valueOf(123), new StringContentAggregateFunction().count(Integer.valueOf(122))); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/operator/json/aggregate/functions/TimestampContentAggregateFunctionTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.aggregate.functions; 18 | 19 | import java.util.Date; 20 | 21 | import org.apache.commons.lang3.tuple.MutablePair; 22 | import org.junit.Assert; 23 | import org.junit.Test; 24 | import org.mockito.Mockito; 25 | 26 | /** 27 | * Test case for {@link TimestampContentAggregateFunction} 28 | * @author mnxfst 29 | * @since Jan 27, 2016 30 | */ 31 | public class TimestampContentAggregateFunctionTest { 32 | 33 | /** 34 | * Test case for {@link TimestampContentAggregateFunction#sum(Date, Date)} 35 | */ 36 | @Test(expected=UnsupportedOperationException.class) 37 | public void testSum() throws Exception { 38 | new TimestampContentAggregateFunction().sum(new Date(), new Date()); 39 | } 40 | 41 | /** 42 | * Test case for {@link TimestampContentAggregateFunction#min(Date, Date)} being 43 | * provided null to both parameters 44 | */ 45 | @Test 46 | public void testMin_withNullInput() throws Exception { 47 | Assert.assertNull(new TimestampContentAggregateFunction().min(null, null)); 48 | } 49 | 50 | /** 51 | * Test case for {@link TimestampContentAggregateFunction#min(Date, Date)} being 52 | * provided null to old value parameter 53 | */ 54 | @Test 55 | public void testMin_withNullOldValue() throws Exception { 56 | final Date input = new Date(); 57 | Assert.assertEquals(input.getTime(), new TimestampContentAggregateFunction().min(null, input).getTime()); 58 | } 59 | 60 | /** 61 | * Test case for {@link TimestampContentAggregateFunction#min(Date, Date)} being 62 | * provided null to new value parameter 63 | */ 64 | @Test 65 | public void testMin_withNullNewValue() throws Exception { 66 | final Date input = new Date(); 67 | Assert.assertEquals(input.getTime(), new TimestampContentAggregateFunction().min(input, null).getTime()); 68 | } 69 | 70 | /** 71 | * Test case for {@link TimestampContentAggregateFunction#min(Date, Date)} being 72 | * provided valid input to both parameters 73 | */ 74 | @Test 75 | public void testMin_withValidInput() throws Exception { 76 | final Date smaller = new Date(); 77 | final Date larger = new Date(System.currentTimeMillis()+1); 78 | Assert.assertEquals(smaller.getTime(), new TimestampContentAggregateFunction().min(smaller, larger).getTime()); 79 | Assert.assertEquals(smaller.getTime(), new TimestampContentAggregateFunction().min(larger, smaller).getTime()); 80 | } 81 | 82 | /** 83 | * Test case for {@link TimestampContentAggregateFunction#max(Date, Date)} being 84 | * provided null to both parameters 85 | */ 86 | @Test 87 | public void testMax_withNullInput() throws Exception { 88 | Assert.assertNull(new TimestampContentAggregateFunction().max(null, null)); 89 | } 90 | 91 | /** 92 | * Test case for {@link TimestampContentAggregateFunction#max(Date, Date)} being 93 | * provided null to old value parameter 94 | */ 95 | @Test 96 | public void testMax_withNullOldValue() throws Exception { 97 | final Date input = new Date(); 98 | Assert.assertEquals(input.getTime(), new TimestampContentAggregateFunction().max(null, input).getTime()); 99 | } 100 | 101 | /** 102 | * Test case for {@link TimestampContentAggregateFunction#max(Date, Date)} being 103 | * provided null to new value parameter 104 | */ 105 | @Test 106 | public void testMax_withNullNewValue() throws Exception { 107 | final Date input = new Date(); 108 | Assert.assertEquals(input.getTime(), new TimestampContentAggregateFunction().max(input, null).getTime()); 109 | } 110 | 111 | /** 112 | * Test case for {@link TimestampContentAggregateFunction#max(Date, Date)} being 113 | * provided valid input to both parameters 114 | */ 115 | @Test 116 | public void testMax_withValidInput() throws Exception { 117 | final Date smaller = new Date(); 118 | final Date larger = new Date(System.currentTimeMillis()+1); 119 | Assert.assertEquals(larger.getTime(), new TimestampContentAggregateFunction().max(smaller, larger).getTime()); 120 | Assert.assertEquals(larger.getTime(), new TimestampContentAggregateFunction().max(larger, smaller).getTime()); 121 | } 122 | 123 | /** 124 | * Test case for {@link TimestampContentAggregateFunction#average(org.apache.commons.lang3.tuple.MutablePair, Date)} 125 | */ 126 | @SuppressWarnings("unchecked") 127 | @Test(expected=UnsupportedOperationException.class) 128 | public void testAverage() throws Exception { 129 | new TimestampContentAggregateFunction().average(Mockito.mock(MutablePair.class), new Date()); 130 | } 131 | 132 | /** 133 | * Test case for {@link TimestampContentAggregateFunction#count(Integer)} being provided null as input 134 | */ 135 | @Test 136 | public void testCount_withNullInput() throws Exception { 137 | Assert.assertEquals(Integer.valueOf(1), new TimestampContentAggregateFunction().count(null)); 138 | } 139 | 140 | /** 141 | * Test case for {@link TimestampContentAggregateFunction#count(Integer)} being provided a valid integer as input 142 | */ 143 | @Test 144 | public void testCount_withValidInput() throws Exception { 145 | Assert.assertEquals(Integer.valueOf(123), new TimestampContentAggregateFunction().count(Integer.valueOf(122))); 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/operator/json/converter/JsonObjectToByteArrayTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.converter; 18 | 19 | import org.apache.sling.commons.json.JSONObject; 20 | import org.junit.Assert; 21 | import org.junit.Test; 22 | 23 | /** 24 | * Test case for {@link JsonObjectToByteArray} 25 | * @author mnxfst 26 | * @since Jan 27, 2016 27 | */ 28 | public class JsonObjectToByteArrayTest { 29 | 30 | /** 31 | * Test case for {@link JsonObjectToByteArray#map(org.apache.sling.commons.json.JSONObject)} being 32 | * provided null as input 33 | */ 34 | @Test 35 | public void testMap_withNullInput() throws Exception { 36 | Assert.assertArrayEquals(new byte[0], new JsonObjectToByteArray().map(null)); 37 | } 38 | 39 | /** 40 | * Test case for {@link JsonObjectToByteArray#map(org.apache.sling.commons.json.JSONObject)} being 41 | * provided a valid json object 42 | */ 43 | @Test 44 | public void testMap_withValidInput() throws Exception { 45 | Assert.assertArrayEquals("{}".getBytes("UTF-8"), new JsonObjectToByteArray().map(new JSONObject())); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/operator/json/converter/JsonObjectToStringTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.streaming.operator.json.converter; 17 | 18 | import org.apache.sling.commons.json.JSONObject; 19 | import org.junit.Assert; 20 | import org.junit.Test; 21 | 22 | /** 23 | * Test case for {@link JsonObjectToString} 24 | * @author mnxfst 25 | * @since May 3, 2016 26 | * 27 | */ 28 | public class JsonObjectToStringTest { 29 | 30 | /** 31 | * Test case for {@link JsonObjectToString#map(org.apache.sling.commons.json.JSONObject)} being provided 32 | * null as input 33 | */ 34 | @Test 35 | public void testMap_withNullInput() throws Exception { 36 | Assert.assertEquals("{}", new JsonObjectToString().map(null)); 37 | } 38 | 39 | /** 40 | * Test case for {@link JsonObjectToString#map(org.apache.sling.commons.json.JSONObject)} being provided 41 | * a valid JSON document as input 42 | */ 43 | @Test 44 | public void testMap_withValidInput() throws Exception { 45 | Assert.assertEquals("{\"test\":\"value\"}", new JsonObjectToString().map(new JSONObject("{\"test\":\"value\"}"))); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/operator/json/converter/StringToByteArrayTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.streaming.operator.json.converter; 17 | 18 | import java.nio.charset.IllegalCharsetNameException; 19 | import java.nio.charset.UnsupportedCharsetException; 20 | 21 | import org.junit.Assert; 22 | import org.junit.Test; 23 | 24 | /** 25 | * Test case for {@link StringToByteArray} 26 | * @author mnxfst 27 | * @since May 19, 2016 28 | */ 29 | public class StringToByteArrayTest { 30 | 31 | /** 32 | * Test case for {@link StringToByteArray#StringToByteArray(String)} being provided 33 | * null 34 | */ 35 | @Test(expected=IllegalArgumentException.class) 36 | public void testCustomizedConstructor_withNull() { 37 | new StringToByteArray(null); 38 | } 39 | 40 | /** 41 | * Test case for {@link StringToByteArray#StringToByteArray(String)} being provided 42 | * an empty string 43 | */ 44 | @Test(expected=IllegalCharsetNameException.class) 45 | public void testCustomizedConstructor_withEmptyInput() { 46 | new StringToByteArray(""); 47 | } 48 | 49 | /** 50 | * Test case for {@link StringToByteArray#StringToByteArray(String)} being provided 51 | * an unknown charset/encoding 52 | */ 53 | @Test(expected=UnsupportedCharsetException.class) 54 | public void testCustomizedConstructor_withUnknownEncoding() { 55 | new StringToByteArray("unknown-"+System.currentTimeMillis()); 56 | } 57 | 58 | /** 59 | * Test case for {@link StringToByteArray#StringToByteArray(String)} being provided 60 | * a valid encoding 61 | */ 62 | @Test 63 | public void testCustomizedConstructor_withValidEncoding() { 64 | new StringToByteArray("UTF-8"); 65 | } 66 | 67 | /** 68 | * Test case for {@link StringToByteArray#StringToByteArray(String)} being provided 69 | * a valid encoding 70 | */ 71 | @Test 72 | public void testMap_withNullInput() throws Exception { 73 | Assert.assertArrayEquals(new byte[0], new StringToByteArray("UTF-8").map(null)); 74 | } 75 | 76 | /** 77 | * Test case for {@link StringToByteArray#StringToByteArray(String)} being provided 78 | * an empty string 79 | */ 80 | @Test 81 | public void testMap_withEmptyString() throws Exception { 82 | Assert.assertArrayEquals("".getBytes("UTF-8"), new StringToByteArray("UTF-8").map("")); 83 | } 84 | 85 | /** 86 | * Test case for {@link StringToByteArray#StringToByteArray(String)} being provided 87 | * a valid string 88 | */ 89 | @Test 90 | public void testMap_withValidString() throws Exception { 91 | Assert.assertArrayEquals("test-content".getBytes("UTF-8"), new StringToByteArray("UTF-8").map("test-content")); 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/operator/json/converter/StringToJsonObjectTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.converter; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | import org.apache.flink.api.common.functions.util.ListCollector; 23 | import org.apache.sling.commons.json.JSONObject; 24 | import org.junit.Assert; 25 | import org.junit.Test; 26 | import org.mockito.Mockito; 27 | 28 | 29 | /** 30 | * Test case for {@link StringToJsonObject} 31 | * @author mnxfst 32 | * @since Jan 27, 2016 33 | */ 34 | public class StringToJsonObjectTest { 35 | 36 | /** 37 | * Test case for {@link StringToJsonObject#flatMap(String, org.apache.flink.util.Collector)} being provided 38 | * null as input to string parameter 39 | */ 40 | @Test 41 | public void testFlatMap_withNullString() throws Exception { 42 | @SuppressWarnings("unchecked") 43 | ListCollector resultCollector = Mockito.mock(ListCollector.class); 44 | new StringToJsonObject().flatMap(null, resultCollector); 45 | Mockito.verify(resultCollector, Mockito.never()).collect(Mockito.anyObject()); 46 | } 47 | 48 | /** 49 | * Test case for {@link StringToJsonObject#flatMap(String, org.apache.flink.util.Collector)} being provided 50 | * an empty string as input to string parameter 51 | */ 52 | @Test 53 | public void testFlatMap_withEmptyString() throws Exception { 54 | @SuppressWarnings("unchecked") 55 | ListCollector resultCollector = Mockito.mock(ListCollector.class); 56 | new StringToJsonObject().flatMap("", resultCollector); 57 | Mockito.verify(resultCollector, Mockito.never()).collect(Mockito.anyObject()); 58 | } 59 | 60 | /** 61 | * Test case for {@link StringToJsonObject#flatMap(String, org.apache.flink.util.Collector)} being provided 62 | * an invalid json string as input to string parameter 63 | */ 64 | @Test 65 | public void testFlatMap_withInvalidJSONString() throws Exception { 66 | @SuppressWarnings("unchecked") 67 | ListCollector resultCollector = Mockito.mock(ListCollector.class); 68 | new StringToJsonObject().flatMap("{ test", resultCollector); 69 | Mockito.verify(resultCollector, Mockito.never()).collect(Mockito.anyObject()); 70 | } 71 | 72 | /** 73 | * Test case for {@link StringToJsonObject#flatMap(String, org.apache.flink.util.Collector)} being provided 74 | * null as input to collector parameter 75 | */ 76 | @Test 77 | public void testFlatMap_withNullCollector() throws Exception { 78 | new StringToJsonObject().flatMap("String", null); 79 | } 80 | 81 | /** 82 | * Test case for {@link StringToJsonObject#flatMap(String, org.apache.flink.util.Collector)} being provided 83 | * valid input 84 | */ 85 | @Test 86 | public void testFlatMap_withValidInput() throws Exception { 87 | List result = new ArrayList<>(); 88 | ListCollector resultCollector = new ListCollector<>(result); 89 | new StringToJsonObject().flatMap("{\"test\":\"value\"}", resultCollector); 90 | Assert.assertEquals(result.size(), 1); 91 | Assert.assertEquals("{\"test\":\"value\"}", result.get(0).toString()); 92 | } 93 | 94 | } -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/operator/json/csv/Json2CsvConverterTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.csv; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | import org.apache.sling.commons.json.JSONObject; 23 | import org.junit.Assert; 24 | import org.junit.Test; 25 | 26 | import com.ottogroup.bi.streaming.operator.json.JsonContentReference; 27 | import com.ottogroup.bi.streaming.operator.json.JsonContentType; 28 | 29 | /** 30 | * Test case for {@link Json2CsvConverter} 31 | * @author mnxfst 32 | * @since Jan 28, 2016 33 | */ 34 | public class Json2CsvConverterTest { 35 | 36 | /** 37 | * Test case for {@link Json2CsvConverter#parse(org.apache.sling.commons.json.JSONObject)} 38 | * being provided null as input 39 | */ 40 | @Test 41 | public void testParse_withNullInput() { 42 | Assert.assertTrue(new Json2CsvConverter(new ArrayList<>(), '\t').parse(null).isEmpty()); 43 | } 44 | 45 | /** 46 | * Test case for {@link Json2CsvConverter#parse(org.apache.sling.commons.json.JSONObject)} 47 | * being provided a valid object but the converter has no export configuration 48 | */ 49 | @Test 50 | public void testParse_withValidObjectButNoExportDefinition() throws Exception { 51 | Assert.assertEquals("", new Json2CsvConverter(new ArrayList<>(), '\t').parse(new JSONObject("{\"test\":\"value\"}"))); 52 | } 53 | 54 | /** 55 | * Test case for {@link Json2CsvConverter#parse(org.apache.sling.commons.json.JSONObject)} 56 | * being provided a valid object but the converter has an export configuration which references no existing field 57 | */ 58 | @Test 59 | public void testParse_withValidObjectButExportDefinitionRefNoExistingField() throws Exception { 60 | List refDef = new ArrayList<>(); 61 | refDef.add(new JsonContentReference(new String[]{"test", "field"}, JsonContentType.STRING)); 62 | Assert.assertEquals("", new Json2CsvConverter(refDef, '\t').parse(new JSONObject("{\"test\":\"value\"}"))); 63 | } 64 | 65 | /** 66 | * Test case for {@link Json2CsvConverter#parse(org.apache.sling.commons.json.JSONObject)} 67 | * being provided a valid object and the converter has a matching export definition 68 | */ 69 | @Test 70 | public void testParse_withValidObjectAndMatchingExportDefinition() throws Exception { 71 | List refDef = new ArrayList<>(); 72 | refDef.add(new JsonContentReference(new String[]{"test"}, JsonContentType.STRING)); 73 | refDef.add(new JsonContentReference(new String[]{"another"}, JsonContentType.STRING)); 74 | Assert.assertEquals("value\t1", new Json2CsvConverter(refDef, '\t').parse(new JSONObject("{\"test\":\"value\",\"another\":\"1\"}"))); 75 | } 76 | 77 | /** 78 | * Test case for {@link Json2CsvConverter#parse(org.apache.sling.commons.json.JSONObject)} 79 | * being provided a valid object and the converter has a matching export definition (reverse order) 80 | */ 81 | @Test 82 | public void testParse_withValidObjectAndMatchingExportDefinitionRevOrder() throws Exception { 83 | List refDef = new ArrayList<>(); 84 | refDef.add(new JsonContentReference(new String[]{"another"}, JsonContentType.STRING)); 85 | refDef.add(new JsonContentReference(new String[]{"test"}, JsonContentType.STRING)); 86 | Assert.assertEquals("1\tvalue", new Json2CsvConverter(refDef, '\t').parse(new JSONObject("{\"test\":\"value\",\"another\":\"1\"}"))); 87 | } 88 | 89 | /** 90 | * Test case for {@link Json2CsvConverter#flatMap(JSONObject, org.apache.flink.util.Collector)} being 91 | * provided null as input to JSON parameter 92 | */ 93 | @Test 94 | public void testFlatMap_withNullObjectInput() throws Exception { 95 | List refDef = new ArrayList<>(); 96 | refDef.add(new JsonContentReference(new String[]{"another"}, JsonContentType.STRING)); 97 | refDef.add(new JsonContentReference(new String[]{"test"}, JsonContentType.STRING)); 98 | Json2CsvConverter conv = new Json2CsvConverter(refDef, '\t'); 99 | Assert.assertEquals("\t", conv.map(null)); 100 | } 101 | 102 | /** 103 | * Test case for {@link Json2CsvConverter#flatMap(JSONObject, org.apache.flink.util.Collector)} being 104 | * provided an object where no path matches 105 | */ 106 | @Test 107 | public void testFlatMap_withObjectNoPathMatches() throws Exception { 108 | List refDef = new ArrayList<>(); 109 | refDef.add(new JsonContentReference(new String[]{"another"}, JsonContentType.STRING)); 110 | refDef.add(new JsonContentReference(new String[]{"test"}, JsonContentType.STRING)); 111 | Json2CsvConverter conv = new Json2CsvConverter(refDef, '\t'); 112 | Assert.assertEquals("\t", conv.map(new JSONObject("{\"tester\":\"value\"}"))); 113 | } 114 | 115 | /** 116 | * Test case for {@link Json2CsvConverter#flatMap(JSONObject, org.apache.flink.util.Collector)} being 117 | * provided an object which produces valid output 118 | */ 119 | @Test 120 | public void testFlatMap_withObjectAndPathMatches() throws Exception { 121 | List refDef = new ArrayList<>(); 122 | refDef.add(new JsonContentReference(new String[]{"another"}, JsonContentType.STRING)); 123 | refDef.add(new JsonContentReference(new String[]{"test"}, JsonContentType.STRING)); 124 | Json2CsvConverter conv = new Json2CsvConverter(refDef, '\t'); 125 | Assert.assertEquals("1\tvalue", conv.map(new JSONObject("{\"test\":\"value\",\"another\":\"1\"}"))); 126 | } 127 | 128 | } 129 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/operator/json/filter/RegularExpressionMatcherTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.filter; 18 | 19 | import java.util.regex.PatternSyntaxException; 20 | 21 | import org.junit.Assert; 22 | import org.junit.Test; 23 | 24 | /** 25 | * Test case for {@link RegularExpressionMatcher} 26 | * @author mnxfst 27 | * @since May 12, 2016 28 | */ 29 | public class RegularExpressionMatcherTest { 30 | 31 | /** 32 | * Test case for {@link RegularExpressionMatcher#matchesPattern(String)} being provided null as input 33 | */ 34 | @Test 35 | public void testMatchPattern_withNullInput() { 36 | Assert.assertTrue(RegularExpressionMatcher.matchesPattern(null).matches(null)); 37 | Assert.assertFalse(RegularExpressionMatcher.matchesPattern(null).matches("")); 38 | Assert.assertFalse(RegularExpressionMatcher.matchesPattern(null).matches("test")); 39 | } 40 | 41 | /** 42 | * Test case for {@link RegularExpressionMatcher#matchesPattern(String)} being provided an empty string as input 43 | */ 44 | @Test 45 | public void testMatchPattern_withEmptyInput() { 46 | Assert.assertFalse(RegularExpressionMatcher.matchesPattern("").matches("test")); 47 | Assert.assertFalse(RegularExpressionMatcher.matchesPattern("").matches(null)); 48 | Assert.assertTrue(RegularExpressionMatcher.matchesPattern("").matches("")); 49 | } 50 | 51 | /** 52 | * Test case for {@link RegularExpressionMatcher#matchesPattern(String)} being provided an invalid pattern 53 | */ 54 | @Test(expected=PatternSyntaxException.class) 55 | public void testMatchPattern_withInvalidPattern() { 56 | Assert.assertFalse(RegularExpressionMatcher.matchesPattern("dsds(").matches("test")); 57 | } 58 | 59 | /** 60 | * Test case for {@link RegularExpressionMatcher#matchesPattern(String)} being provided valid pattern 61 | */ 62 | @Test 63 | public void testMatchPattern_withValidPattern() { 64 | Assert.assertFalse(RegularExpressionMatcher.matchesPattern("(first|last)name").matches(null)); 65 | Assert.assertTrue(RegularExpressionMatcher.matchesPattern("(first|last)name").matches("firstname")); 66 | Assert.assertTrue(RegularExpressionMatcher.matchesPattern("(first|last)name").matches("lastname")); 67 | Assert.assertFalse(RegularExpressionMatcher.matchesPattern("(first|last)name").matches("noname")); 68 | Assert.assertTrue(RegularExpressionMatcher.matchesPattern("0.123").matches(Double.valueOf(0.123))); 69 | Assert.assertFalse(RegularExpressionMatcher.matchesPattern("0.123").matches(Double.valueOf(0.124))); 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/operator/json/insert/JsonStaticContentInsertionTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.streaming.operator.json.insert; 17 | 18 | import java.io.Serializable; 19 | import java.util.ArrayList; 20 | import java.util.Arrays; 21 | import java.util.Collections; 22 | import java.util.List; 23 | 24 | import org.apache.commons.lang3.tuple.ImmutablePair; 25 | import org.apache.commons.lang3.tuple.Pair; 26 | import org.apache.sling.commons.json.JSONObject; 27 | import org.hamcrest.Matchers; 28 | import org.junit.Assert; 29 | import org.junit.Test; 30 | 31 | import com.ottogroup.bi.streaming.operator.json.JsonContentReference; 32 | import com.ottogroup.bi.streaming.operator.json.JsonContentType; 33 | import com.ottogroup.bi.streaming.testing.MatchJSONContent; 34 | 35 | 36 | /** 37 | * Test case for {@link JsonStaticContentInsertion} 38 | * @author mnxfst 39 | * @since Apr 28, 2016 40 | */ 41 | public class JsonStaticContentInsertionTest { 42 | 43 | /** 44 | * Test case for {@link JsonStaticContentInsertion#JsonStaticContentInsertion(java.util.List)} being provided null 45 | * as input 46 | */ 47 | @Test(expected=IllegalArgumentException.class) 48 | public void testConstructor_withNullInput() throws Exception { 49 | new JsonStaticContentInsertion(null); 50 | } 51 | 52 | /** 53 | * Test case for {@link JsonStaticContentInsertion#JsonStaticContentInsertion(java.util.List)} being provided a 54 | * configuration which holds a "null" element 55 | */ 56 | @Test(expected=IllegalArgumentException.class) 57 | public void testConstructor_withNullListElement() throws Exception { 58 | List> values = new ArrayList<>(); 59 | values.add(new ImmutablePair(new JsonContentReference(new String[]{"valid"}, JsonContentType.STRING),"test")); 60 | values.add(null); 61 | new JsonStaticContentInsertion(values); 62 | } 63 | 64 | /** 65 | * Test case for {@link JsonStaticContentInsertion#JsonStaticContentInsertion(java.util.List)} being provided a 66 | * configuration which holds an element that is missing the required content path 67 | */ 68 | @Test(expected=IllegalArgumentException.class) 69 | public void testConstructor_withElementMissingContentPath() throws Exception { 70 | List> values = new ArrayList<>(); 71 | values.add(new ImmutablePair(null ,"test")); 72 | new JsonStaticContentInsertion(values); 73 | } 74 | 75 | /** 76 | * Test case for {@link JsonStaticContentInsertion#JsonStaticContentInsertion(java.util.List)} being provided a 77 | * configuration which holds an element that shows an empty path (null) 78 | */ 79 | @Test(expected=IllegalArgumentException.class) 80 | public void testConstructor_withElementHoldingNullContentPath() throws Exception { 81 | List> values = new ArrayList<>(); 82 | values.add(new ImmutablePair(new JsonContentReference(null, JsonContentType.STRING),"test")); 83 | new JsonStaticContentInsertion(values); 84 | } 85 | 86 | /** 87 | * Test case for {@link JsonStaticContentInsertion#JsonStaticContentInsertion(java.util.List)} being provided a 88 | * configuration which holds an element that shows an empty path (no elements) 89 | */ 90 | @Test(expected=IllegalArgumentException.class) 91 | public void testConstructor_withElementHoldingEmptyContentPath() throws Exception { 92 | List> values = new ArrayList<>(); 93 | values.add(new ImmutablePair(new JsonContentReference(new String[0], JsonContentType.STRING),"test")); 94 | new JsonStaticContentInsertion(values); 95 | } 96 | 97 | /** 98 | * Test case for {@link JsonStaticContentInsertion#JsonStaticContentInsertion(java.util.List)} being provided a 99 | * configuration which holds an element that is missing the insertion value 100 | */ 101 | @Test(expected=IllegalArgumentException.class) 102 | public void testConstructor_withNullInsertionValue() throws Exception { 103 | List> values = new ArrayList<>(); 104 | values.add(new ImmutablePair(new JsonContentReference(new String[]{"valid"}, JsonContentType.STRING), null)); 105 | new JsonStaticContentInsertion(values); 106 | } 107 | 108 | /** 109 | * Test case for {@link JsonStaticContentInsertion#JsonStaticContentInsertion(java.util.List)} being provided an empty list 110 | */ 111 | @Test 112 | public void testConstructor_withEmptyList() throws Exception { 113 | Assert.assertTrue(new JsonStaticContentInsertion(new ArrayList<>()).getValues().isEmpty()); 114 | } 115 | 116 | /** 117 | * Test case for {@link JsonStaticContentInsertion#map(JSONObject)} being provided null 118 | */ 119 | @Test 120 | public void testMap_withNullInput() throws Exception { 121 | List> values = new ArrayList<>(); 122 | values.add(new ImmutablePair(new JsonContentReference(new String[]{"valid"}, JsonContentType.STRING), "test")); 123 | Assert.assertNull(new JsonStaticContentInsertion(values).map(null)); 124 | } 125 | 126 | /** 127 | * Test case for {@link JsonStaticContentInsertion#map(JSONObject)} with empty insertion values list 128 | */ 129 | @Test 130 | public void testMap_withEmptyList() throws Exception { 131 | String inputAndExpected = "{\"test\":\"value\"}"; 132 | Assert.assertEquals(inputAndExpected, new JsonStaticContentInsertion(new ArrayList<>()).map(new JSONObject(inputAndExpected)).toString()); 133 | } 134 | 135 | /** 136 | * Test case for {@link JsonStaticContentInsertion#map(JSONObject)} with valid configuration provided during instantiation 137 | */ 138 | @Test 139 | public void testMap_withValidInsertionConfiguration() throws Exception { 140 | Assert.assertTrue( 141 | MatchJSONContent.create() 142 | .assertString("test", Matchers.is("value")) 143 | .assertString("valid", Matchers.is("test")) 144 | .matchOnSingle(new JsonStaticContentInsertion( 145 | Arrays.asList(new ImmutablePair(new JsonContentReference(new String[]{"valid"}, JsonContentType.STRING), "test"))) 146 | .map(new JSONObject("{\"test\":\"value\"}")) 147 | ) 148 | ); 149 | } 150 | 151 | /** 152 | * Test case for {@link JsonStaticContentInsertion#insert(org.apache.sling.commons.json.JSONObject, String[], java.io.Serializable)} with 153 | * valid input. Test cases for invalid input are not required as the method calling the insertion method ensure that the provided 154 | * input is valid 155 | */ 156 | @Test 157 | public void testInsert_withValidInput() throws Exception { 158 | JsonStaticContentInsertion inserter = new JsonStaticContentInsertion(Collections.>emptyList()); 159 | inserter.insert(new JSONObject("{}"), new String[]{"path", "to", "value"}, "test-value"); 160 | MatchJSONContent.create().assertString("path.to.value", Matchers.is("test-value")).onEachRecord(); 161 | } 162 | 163 | } 164 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/operator/json/partitioning/JsonKeySelectorTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.partitioning; 18 | 19 | import java.util.NoSuchElementException; 20 | 21 | import org.apache.sling.commons.json.JSONObject; 22 | import org.junit.Assert; 23 | import org.junit.Test; 24 | 25 | import com.ottogroup.bi.streaming.operator.json.JsonContentReference; 26 | import com.ottogroup.bi.streaming.operator.json.JsonContentType; 27 | 28 | /** 29 | * Test case for {@link JsonKeySelector} 30 | * @author mnxfst 31 | * @since 20.04.2016 32 | */ 33 | public class JsonKeySelectorTest { 34 | 35 | /** 36 | * Test case for {@link JsonKeySelector#JSONPartitioningKeySelector(com.ottogroup.bi.streaming.operator.json.aggregate.JsonContentReference)} 37 | * being provided null as input 38 | */ 39 | @Test(expected=IllegalArgumentException.class) 40 | public void testConstructor_withNullReferenceInput() { 41 | new JsonKeySelector(null); 42 | } 43 | 44 | /** 45 | * Test case for {@link JsonKeySelector#JSONPartitioningKeySelector(com.ottogroup.bi.streaming.operator.json.aggregate.JsonContentReference)} 46 | * being provided a reference showing a path value set to null 47 | */ 48 | @Test(expected=IllegalArgumentException.class) 49 | public void testConstructor_withNullPathInfo() { 50 | new JsonKeySelector(new JsonContentReference(null, JsonContentType.STRING)); 51 | } 52 | 53 | /** 54 | * Test case for {@link JsonKeySelector#JSONPartitioningKeySelector(com.ottogroup.bi.streaming.operator.json.aggregate.JsonContentReference)} 55 | * being provided a reference showing a path value set to an empty array 56 | */ 57 | @Test(expected=IllegalArgumentException.class) 58 | public void testConstructor_withEmptyPathInfo() { 59 | new JsonKeySelector(new JsonContentReference(new String[0], JsonContentType.STRING)); 60 | } 61 | 62 | /** 63 | * Test case for {@link JsonKeySelector#getKey(org.apache.sling.commons.json.JSONObject)} being provided null 64 | */ 65 | @Test 66 | public void testGetKey_withNullInput() throws Exception { 67 | Assert.assertNull(new JsonKeySelector(new JsonContentReference(new String[]{"test"}, JsonContentType.STRING)).getKey(null)); 68 | } 69 | 70 | /** 71 | * Test case for {@link JsonKeySelector#getKey(org.apache.sling.commons.json.JSONObject)} being provided 72 | * a valid JSON object but misses the required location referenced inside configuration 73 | */ 74 | @Test(expected=NoSuchElementException.class) 75 | public void testGetKey_withValidInputButMissingReferencedLocation() throws Exception { 76 | new JsonKeySelector( 77 | new JsonContentReference(new String[]{"test"}, JsonContentType.STRING, true)).getKey(new JSONObject("{\"key\":\"value\"}")); 78 | } 79 | 80 | /** 81 | * Test case for {@link JsonKeySelector#getKey(org.apache.sling.commons.json.JSONObject)} being provided 82 | * a valid JSON object holding the required location referenced inside configuration 83 | */ 84 | @Test 85 | public void testGetKey_withValidInputAndValueAtReferencedLocation() throws Exception { 86 | Assert.assertEquals("value", new JsonKeySelector( 87 | new JsonContentReference(new String[]{"test"}, JsonContentType.STRING, true)).getKey(new JSONObject("{\"test\":\"value\"}"))); 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/operator/json/project/JsonContentProjectionMapperTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.json.project; 18 | 19 | import java.util.ArrayList; 20 | import java.util.Collections; 21 | import java.util.List; 22 | 23 | import org.apache.sling.commons.json.JSONObject; 24 | import org.junit.Assert; 25 | import org.junit.Test; 26 | import org.mockito.Mockito; 27 | 28 | import com.ottogroup.bi.streaming.operator.json.JsonContentReference; 29 | import com.ottogroup.bi.streaming.operator.json.JsonContentType; 30 | 31 | /** 32 | * Test case for {@link JsonContentProjectionMapper} 33 | * @author mnxfst 34 | * @since Jan 27, 2016 35 | */ 36 | public class JsonContentProjectionMapperTest { 37 | 38 | /** 39 | * Test case for {@link JsonContentProjectionMapper#JsonContentProjectionMapper(java.util.List)} 40 | * being provided null as input 41 | */ 42 | @Test 43 | public void testConstructor_withNullInput() { 44 | Assert.assertTrue(new JsonContentProjectionMapper(null).getConfiguration().isEmpty()); 45 | } 46 | 47 | /** 48 | * Test case for {@link JsonContentProjectionMapper#JsonContentProjectionMapper(java.util.List)} 49 | * being provided an empty list as input 50 | */ 51 | @Test 52 | public void testConstructor_withEmptyList() { 53 | Assert.assertTrue(new JsonContentProjectionMapper(Collections.emptyList()).getConfiguration().isEmpty()); 54 | } 55 | 56 | /** 57 | * Test case for {@link JsonContentProjectionMapper#map(org.apache.sling.commons.json.JSONObject)} being 58 | * provided null as input 59 | */ 60 | @Test 61 | public void testMap_withNullInput() throws Exception { 62 | @SuppressWarnings("unchecked") 63 | List cfg = Mockito.mock(List.class); 64 | Mockito.when(cfg.isEmpty()).thenReturn(true); 65 | Assert.assertNull(new JsonContentProjectionMapper(cfg).map(null)); 66 | Mockito.verify(cfg, Mockito.times(1)).isEmpty(); 67 | } 68 | 69 | /** 70 | * Test case for {@link JsonContentProjectionMapper#map(org.apache.sling.commons.json.JSONObject)} being 71 | * provided a valid object but no configuration 72 | */ 73 | @Test 74 | public void testMap_withValidObjectEmptyConfiguration() throws Exception { 75 | Assert.assertEquals("{}", new JsonContentProjectionMapper(Collections.emptyList()). 76 | map(new JSONObject("{\"key\":\"value\"}")).toString()); 77 | } 78 | 79 | /** 80 | * Test case for {@link JsonContentProjectionMapper#map(org.apache.sling.commons.json.JSONObject)} being 81 | * provided a valid object and valid configuration 82 | */ 83 | @Test 84 | public void testMap_withValidObjectValidConfiguration() throws Exception { 85 | String inputJson = "{\"test\":\"value\", \"sub\":{\"key\":\"another-value\"}}"; 86 | List cfg = new ArrayList<>(); 87 | cfg.add(new ProjectionMapperConfiguration(new JsonContentReference(new String[]{"sub","key"}, JsonContentType.STRING, false), 88 | new String[]{"simple","structure"})); 89 | 90 | Assert.assertEquals("{\"simple\":{\"structure\":\"another-value\"}}", new JsonContentProjectionMapper(cfg). 91 | map(new JSONObject(inputJson)).toString()); 92 | } 93 | 94 | /** 95 | * Test case for {@link JsonContentProjectionMapper#project(JSONObject)} being provided 96 | * a valid object but no configuration 97 | */ 98 | @Test 99 | public void testProject_withEmptyConfiguration() throws Exception { 100 | Assert.assertEquals("{}", new JsonContentProjectionMapper(Collections.emptyList()). 101 | project(new JSONObject("{\"key\":\"value\"}")).toString()); 102 | } 103 | 104 | /** 105 | * Test case for {@link JsonContentProjectionMapper#project(JSONObject)} being provided 106 | * a valid object and a simple projection 107 | */ 108 | @Test 109 | public void testProject_withValidObjectSimpleProjection() throws Exception { 110 | String inputJson = "{\"test\":\"value\", \"sub\":{\"key\":\"another-value\"}}"; 111 | List cfg = new ArrayList<>(); 112 | cfg.add(new ProjectionMapperConfiguration(new JsonContentReference(new String[]{"sub","key"}, JsonContentType.STRING, false), 113 | new String[]{"simple","structure"})); 114 | Assert.assertEquals("{\"simple\":{\"structure\":\"another-value\"}}", new JsonContentProjectionMapper(cfg). 115 | project(new JSONObject(inputJson)).toString()); 116 | } 117 | 118 | /** 119 | * Test case for {@link JsonContentProjectionMapper#project(JSONObject)} being provided 120 | * a valid object and a simple projection pointing to a non-existing element (field not required) 121 | */ 122 | @Test 123 | public void testProject_withValidObjectSimpleProjectionNonExistingElementFieldNotRequired() throws Exception { 124 | String inputJson = "{\"test\":\"value\", \"sub\":{\"key\":\"another-value\"}}"; 125 | List cfg = new ArrayList<>(); 126 | cfg.add(new ProjectionMapperConfiguration(new JsonContentReference(new String[]{"does","not","exist"}, JsonContentType.STRING, false), 127 | new String[]{"simple","structure"})); 128 | Assert.assertEquals("{\"simple\":{\"structure\":\"\"}}", new JsonContentProjectionMapper(cfg). 129 | project(new JSONObject(inputJson)).toString()); 130 | } 131 | 132 | /** 133 | * Test case for {@link JsonContentProjectionMapper#project(JSONObject)} being provided 134 | * a valid object and a simple projection pointing to a non-existing element (field required) 135 | */ 136 | @Test 137 | public void testProject_withValidObjectSimpleProjectionNonExistingElementFieldRequired() throws Exception { 138 | String inputJson = "{\"test\":\"value\", \"sub\":{\"key\":\"another-value\"}}"; 139 | List cfg = new ArrayList<>(); 140 | cfg.add(new ProjectionMapperConfiguration(new JsonContentReference(new String[]{"does","not","exist"}, JsonContentType.STRING, true), 141 | new String[]{"simple","structure"})); 142 | Assert.assertEquals("{}", new JsonContentProjectionMapper(cfg). 143 | project(new JSONObject(inputJson)).toString()); 144 | } 145 | 146 | /** 147 | * Test case for {@link JsonContentProjectionMapper#project(JSONObject)} being provided 148 | * a valid object and a simple projection pointing to an array 149 | */ 150 | @Test 151 | public void testProject_withValidProjectionOfArray() throws Exception { 152 | String inputJson = "{\"test\":\"value\", \"sub\":[{\"key-1\":\"value-1\"},{\"key-2\":\"value-2\"}]}"; 153 | List cfg = new ArrayList<>(); 154 | cfg.add(new ProjectionMapperConfiguration(new JsonContentReference(new String[]{"sub"}, JsonContentType.STRING, true), 155 | new String[]{"simple","structure"})); 156 | Assert.assertEquals("{\"simple\":{\"structure\":[{\"key-1\":\"value-1\"},{\"key-2\":\"value-2\"}]}}", new JsonContentProjectionMapper(cfg). 157 | project(new JSONObject(inputJson)).toString()); 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/operator/metrics/MessageCountingMetricsReporterTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.operator.metrics; 18 | 19 | import org.junit.Test; 20 | import org.mockito.Mockito; 21 | 22 | import com.timgroup.statsd.StatsDClient; 23 | 24 | /** 25 | * Test case for {@link MessageCountingMetricsReporter} 26 | * @author mnxfst 27 | * @since 12.02.2016 28 | */ 29 | public class MessageCountingMetricsReporterTest { 30 | 31 | /** 32 | * Test case for {@link MessageCountingMetricsReporter#MessageCountingMetricsReporter(String, int, String, String[])} being 33 | * provided null as input to host parameter 34 | */ 35 | @Test(expected=IllegalArgumentException.class) 36 | public void testConstructor_withEmptyHost() { 37 | new MessageCountingMetricsReporter(null, 1, "prefix", new String[]{"test"}); 38 | } 39 | 40 | /** 41 | * Test case for {@link MessageCountingMetricsReporter#MessageCountingMetricsReporter(String, int, String, String[])} being 42 | * provided -1 as input to port parameter 43 | */ 44 | @Test(expected=IllegalArgumentException.class) 45 | public void testConstructor_withNegativePort() { 46 | new MessageCountingMetricsReporter("host", -1, "prefix", new String[]{"test"}); 47 | } 48 | 49 | /** 50 | * Test case for {@link MessageCountingMetricsReporter#MessageCountingMetricsReporter(String, int, String, String[])} being 51 | * provided null as input to metrics parameter 52 | */ 53 | @Test(expected=IllegalArgumentException.class) 54 | public void testConstructor_withNullMetrics() { 55 | new MessageCountingMetricsReporter("host", 1, "prefix", null); 56 | } 57 | 58 | /** 59 | * Test case for {@link MessageCountingMetricsReporter#MessageCountingMetricsReporter(String, int, String, String[])} being 60 | * provided an empty array as input to metrics parameter 61 | */ 62 | @Test(expected=IllegalArgumentException.class) 63 | public void testConstructor_withEmptyMetrics() { 64 | new MessageCountingMetricsReporter("host", 1, "prefix", new String[0]); 65 | } 66 | 67 | /** 68 | * Test case for {@link MessageCountingMetricsReporter#MessageCountingMetricsReporter(String, int, String, String[])} being 69 | * provided an array having valid entries and one empty element as input to metrics parameter 70 | */ 71 | @Test(expected=IllegalArgumentException.class) 72 | public void testConstructor_withValidEntriesOneEmpty() { 73 | new MessageCountingMetricsReporter("host", 1, "prefix", new String[]{"test1", "", "test2"}); 74 | } 75 | 76 | /** 77 | * Test case for {@link MessageCountingMetricsReporter#filter(Object)} being provided 78 | * null as input 79 | */ 80 | @Test 81 | public void testFilter_withNullInput() throws Exception { 82 | StatsDClient client = Mockito.mock(StatsDClient.class); 83 | final MessageCountingMetricsReporter counter = new MessageCountingMetricsReporter<>("localhost", 1, "prefix", new String[]{"test1","test2"}); 84 | counter.setStatsDClient(client); 85 | counter.filter(null); 86 | Mockito.verify(client).incrementCounter("test1"); 87 | Mockito.verify(client).incrementCounter("test2"); 88 | } 89 | 90 | /** 91 | * Test case for {@link MessageCountingMetricsReporter#filter(Object)} being provided 92 | * a valid string as input 93 | */ 94 | @Test 95 | public void testFilter_withValidStringAsInput() throws Exception { 96 | StatsDClient client = Mockito.mock(StatsDClient.class); 97 | final MessageCountingMetricsReporter counter = new MessageCountingMetricsReporter<>("localhost", 1, "prefix", new String[]{"test1","test2"}); 98 | counter.setStatsDClient(client); 99 | counter.filter("valid-string"); 100 | Mockito.verify(client).incrementCounter("test1"); 101 | Mockito.verify(client).incrementCounter("test2"); 102 | } 103 | 104 | } 105 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/sink/kafka/KafkaProducerBuilderTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.sink.kafka; 18 | 19 | import java.util.Properties; 20 | 21 | import org.apache.commons.lang3.StringUtils; 22 | import org.junit.Assert; 23 | import org.junit.Test; 24 | 25 | /** 26 | * Test case for {@link KafkaProducerBuilder} 27 | * @author mnxfst 28 | * @since Feb 29, 2016 29 | */ 30 | public class KafkaProducerBuilderTest { 31 | 32 | /** 33 | * Test case for {@link KafkaProducerBuilder#getInstance()} 34 | */ 35 | @Test 36 | public void testGetInstance() { 37 | Assert.assertNull(KafkaProducerBuilder.getInstance().getTopic()); 38 | Assert.assertNull(KafkaProducerBuilder.getInstance().getBrokerList()); 39 | } 40 | 41 | /** 42 | * Test case for {@link KafkaProducerBuilder#topic(String)} being provided 43 | * null as input 44 | */ 45 | @Test 46 | public void testTopic_withNullInput() { 47 | Assert.assertNull(KafkaProducerBuilder.getInstance().topic(null).getTopic()); 48 | } 49 | 50 | /** 51 | * Test case for {@link KafkaProducerBuilder#topic(String)} being provided 52 | * an empty string as input 53 | */ 54 | @Test 55 | public void testTopic_withEmptyInput() { 56 | Assert.assertTrue(StringUtils.isBlank(KafkaProducerBuilder.getInstance().topic("").getTopic())); 57 | } 58 | 59 | /** 60 | * Test case for {@link KafkaProducerBuilder#topic(String)} being provided 61 | * a valid string as input 62 | */ 63 | @Test 64 | public void testTopic_withValidInput() { 65 | Assert.assertEquals("test-topic", KafkaProducerBuilder.getInstance().topic("test-topic").getTopic()); 66 | } 67 | 68 | /** 69 | * Test case for {@link KafkaProducerBuilder#brokerList(String)} being provided 70 | * null as input 71 | */ 72 | @Test 73 | public void testBrokerList_withNullInput() { 74 | Assert.assertNull(KafkaProducerBuilder.getInstance().brokerList(null).getBrokerList()); 75 | } 76 | 77 | /** 78 | * Test case for {@link KafkaProducerBuilder#brokerList(String)} being provided 79 | * an empty string as input 80 | */ 81 | @Test 82 | public void testBrokerList_withEmptyInput() { 83 | Assert.assertTrue(StringUtils.isBlank(KafkaProducerBuilder.getInstance().brokerList("").getBrokerList())); 84 | } 85 | 86 | /** 87 | * Test case for {@link KafkaProducerBuilder#brokerList(String)} being provided 88 | * a valid string as input 89 | */ 90 | @Test 91 | public void testBrokerList_withValidInput() { 92 | Assert.assertEquals("test-broker", KafkaProducerBuilder.getInstance().brokerList("test-broker").getBrokerList()); 93 | } 94 | 95 | /** 96 | * Test case for {@link KafkaProducerBuilder#create()} missing a required topic 97 | */ 98 | @Test(expected=IllegalArgumentException.class) 99 | public void testCreate_withMissingTopic() { 100 | KafkaProducerBuilder.getInstance().brokerList("broker:2181").create(); 101 | } 102 | 103 | /** 104 | * Test case for {@link KafkaProducerBuilder#create()} missing the required broker list 105 | */ 106 | @Test(expected=IllegalArgumentException.class) 107 | public void testCreate_withMissingBrokerList() { 108 | KafkaProducerBuilder.getInstance().topic("topic").create(); 109 | } 110 | 111 | /** 112 | * Test case for {@link KafkaProducerBuilder#addProperty(String, String)} being provided null 113 | * as input to key parameter 114 | */ 115 | @Test 116 | public void testAddProperty_withNullKey() { 117 | KafkaProducerBuilder builder = KafkaProducerBuilder.getInstance(); 118 | Assert.assertTrue(builder.getProperties().isEmpty()); 119 | builder = builder.addProperty(null, ""); 120 | Assert.assertTrue(builder.getProperties().isEmpty()); 121 | } 122 | 123 | /** 124 | * Test case for {@link KafkaProducerBuilder#addProperty(String, String)} being provided null 125 | * as input to value parameter 126 | */ 127 | @Test 128 | public void testAddProperty_withValidKeyAndNullValue() { 129 | KafkaProducerBuilder builder = KafkaProducerBuilder.getInstance(); 130 | Assert.assertTrue(builder.getProperties().isEmpty()); 131 | builder = builder.addProperty("test", null); 132 | Assert.assertTrue(builder.getProperties().isEmpty()); 133 | } 134 | 135 | /** 136 | * Test case for {@link KafkaProducerBuilder#addProperty(String, String)} being provided an 137 | * empty string as input to value parameter 138 | */ 139 | @Test 140 | public void testAddProperty_withValidKeyAndEmptyValue() { 141 | KafkaProducerBuilder builder = KafkaProducerBuilder.getInstance(); 142 | Assert.assertTrue(builder.getProperties().isEmpty()); 143 | builder = builder.addProperty("test", ""); 144 | Assert.assertEquals("", builder.getProperties().get("test")); 145 | } 146 | 147 | /** 148 | * Test case for {@link KafkaProducerBuilder#addProperties(java.util.Properties)} being provided 149 | * null as input 150 | */ 151 | @Test 152 | public void testAddProperties_withNullInput() { 153 | KafkaProducerBuilder builder = KafkaProducerBuilder.getInstance(); 154 | Assert.assertTrue(builder.getProperties().isEmpty()); 155 | builder = builder.addProperties(null); 156 | Assert.assertTrue(builder.getProperties().isEmpty()); 157 | } 158 | 159 | /** 160 | * Test case for {@link KafkaProducerBuilder#addProperties(java.util.Properties)} being provided 161 | * an empty properties set as input 162 | */ 163 | @Test 164 | public void testAddProperties_withEmptyInput() { 165 | KafkaProducerBuilder builder = KafkaProducerBuilder.getInstance(); 166 | Assert.assertTrue(builder.getProperties().isEmpty()); 167 | builder = builder.addProperties(new Properties()); 168 | Assert.assertTrue(builder.getProperties().isEmpty()); 169 | } 170 | 171 | /** 172 | * Test case for {@link KafkaProducerBuilder#addProperties(java.util.Properties)} being provided 173 | * valid properties 174 | */ 175 | @Test 176 | public void testAddProperties_withFilledInput() { 177 | 178 | Properties props = new Properties(); 179 | props.put("test-1", "value-1"); 180 | props.put("test-2", "value-2"); 181 | 182 | KafkaProducerBuilder builder = KafkaProducerBuilder.getInstance(); 183 | Assert.assertTrue(builder.getProperties().isEmpty()); 184 | builder = builder.addProperties(props); 185 | Assert.assertEquals(2, builder.getProperties().size()); 186 | Assert.assertEquals("value-1", builder.getProperties().get("test-1")); 187 | Assert.assertEquals("value-2", builder.getProperties().get("test-2")); 188 | } 189 | 190 | } 191 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/testing/MatchJSONContentFlinkSpectorTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.ottogroup.bi.streaming.testing; 17 | 18 | import org.apache.flink.streaming.api.datastream.DataStream; 19 | import org.apache.sling.commons.json.JSONException; 20 | import org.apache.sling.commons.json.JSONObject; 21 | import org.flinkspector.core.quantify.OutputMatcher; 22 | import org.flinkspector.datastream.DataStreamTestBase; 23 | import org.hamcrest.Matchers; 24 | import org.junit.Test; 25 | 26 | /** 27 | * Test case to show check integration with flink-spector 28 | * @author mnxfst 29 | * @since Apr 26, 2016 30 | * 31 | */ 32 | public class MatchJSONContentFlinkSpectorTest extends DataStreamTestBase { 33 | 34 | /** 35 | * Test case to show the integration of {@link MatchJSONContent} with {@link StreamTestBase} 36 | * of flink-spector 37 | */ 38 | @Test 39 | public void testIntegration_withWorkingExample() throws JSONException { 40 | 41 | DataStream jsonStream = 42 | createTestStreamWith(new JSONObject("{\"key1\":123}")) 43 | .emit(new JSONObject("{\"key2\":\"test\"}")) 44 | .emit(new JSONObject("{\"key3\":{\"key4\":0.122}}")) 45 | .close(); 46 | 47 | OutputMatcher matcher = 48 | new MatchJSONContent() 49 | .assertInteger("key1", Matchers.is(123)) 50 | .assertString("key2", Matchers.isIn(new String[]{"test","test1"})) 51 | .assertDouble("key3.key4", Matchers.greaterThan(0.12)).atLeastNOfThem(1).onAnyRecord(); 52 | 53 | assertStream(jsonStream, matcher); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/test/java/com/ottogroup/bi/streaming/testing/MatchJSONContentTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 Otto (GmbH & Co KG) 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ottogroup.bi.streaming.testing; 18 | 19 | import java.text.SimpleDateFormat; 20 | 21 | import org.hamcrest.Matcher; 22 | import org.hamcrest.Matchers; 23 | import org.junit.Test; 24 | 25 | /** 26 | * Test case for {@link MatchJSONContent} 27 | * @author mnxfst 28 | * @since Apr 25, 2016 29 | */ 30 | public class MatchJSONContentTest{ 31 | 32 | /** 33 | * Test case for {@link MatchJSONContent#assertString(String, org.hamcrest.Matcher)} being provided an 34 | * empty path 35 | */ 36 | @Test(expected=IllegalArgumentException.class) 37 | public void testAssertString_withEmptyPath() { 38 | new MatchJSONContent().assertString(null, Matchers.is("test")); 39 | } 40 | 41 | /** 42 | * Test case for {@link MatchJSONContent#assertString(String, org.hamcrest.Matcher)} being provided an 43 | * empty matcher 44 | */ 45 | @Test(expected=IllegalArgumentException.class) 46 | public void testAssertString_withEmptyMatcher() { 47 | new MatchJSONContent().assertString("path", null); 48 | } 49 | 50 | /** 51 | * Test case for {@link MatchJSONContent#assertString(String, org.hamcrest.Matcher)} being provided 52 | * valid input 53 | */ 54 | @Test 55 | public void testAssertString_withValidInput() throws Exception { 56 | new MatchJSONContent().assertString("path", Matchers.is("value")); 57 | } 58 | 59 | /** 60 | * Test case for {@link MatchJSONContent#assertInteger(String, Matcher)} being provided an 61 | * empty path 62 | */ 63 | @Test(expected=IllegalArgumentException.class) 64 | public void testAssertInteger_withEmptyPath() { 65 | new MatchJSONContent().assertInteger(null, Matchers.is(10)); 66 | } 67 | 68 | /** 69 | * Test case for {@link MatchJSONContent#assertInteger(String, Matcher)} being provided an 70 | * empty matcher 71 | */ 72 | @Test(expected=IllegalArgumentException.class) 73 | public void testAssertInteger_withEmptyMatcher() { 74 | new MatchJSONContent().assertInteger("path", null); 75 | } 76 | 77 | /** 78 | * Test case for {@link MatchJSONContent#assertInteger(String, Matcher)} being provided 79 | * valid input 80 | */ 81 | @Test 82 | public void testAssertInteger_withValidInput() throws Exception { 83 | new MatchJSONContent().assertInteger("path", Matchers.is(10)); 84 | } 85 | 86 | /** 87 | * Test case for {@link MatchJSONContent#assertBoolean(String, Matcher)} being provided an 88 | * empty path 89 | */ 90 | @Test(expected=IllegalArgumentException.class) 91 | public void testAssertBoolean_withEmptyPath() { 92 | new MatchJSONContent().assertBoolean(null, Matchers.is(true)); 93 | } 94 | 95 | /** 96 | * Test case for {@link MatchJSONContent#assertBoolean(String, Matcher)} being provided an 97 | * empty matcher 98 | */ 99 | @Test(expected=IllegalArgumentException.class) 100 | public void testAssertBoolean_withEmptyMatcher() { 101 | new MatchJSONContent().assertBoolean("path", null); 102 | } 103 | 104 | /** 105 | * Test case for {@link MatchJSONContent#assertBoolean(String, Matcher)} being provided 106 | * valid input 107 | */ 108 | @Test 109 | public void testAssertBoolean_withValidInput() throws Exception { 110 | new MatchJSONContent().assertBoolean("path", Matchers.is(true)); 111 | } 112 | 113 | /** 114 | * Test case for {@link MatchJSONContent#assertDouble(String, Matcher)} being provided an 115 | * empty path 116 | */ 117 | @Test(expected=IllegalArgumentException.class) 118 | public void testAssertDouble_withEmptyPath() { 119 | new MatchJSONContent().assertDouble(null, Matchers.is(1.23)); 120 | } 121 | 122 | /** 123 | * Test case for {@link MatchJSONContent#assertDouble(String, Matcher)} being provided an 124 | * empty matcher 125 | */ 126 | @Test(expected=IllegalArgumentException.class) 127 | public void testAssertDouble_withEmptyMatcher() { 128 | new MatchJSONContent().assertDouble("path", null); 129 | } 130 | 131 | /** 132 | * Test case for {@link MatchJSONContent#assertDouble(String, Matcher)} being provided 133 | * valid input 134 | */ 135 | @Test 136 | public void testAssertDouble_withValidInput() throws Exception { 137 | new MatchJSONContent().assertDouble("path", Matchers.is(1.23)); 138 | } 139 | 140 | /** 141 | * Test case for {@link MatchJSONContent#assertTimestamp(String, String, Matcher)} being provided an 142 | * empty path 143 | */ 144 | @Test(expected=IllegalArgumentException.class) 145 | public void testAssertTimestamp_withEmptyPath() throws Exception { 146 | final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); 147 | new MatchJSONContent().assertTimestamp(null, "yyyy-MM-dd", Matchers.is(sdf.parse("2016-04-25"))); 148 | } 149 | 150 | /** 151 | * Test case for {@link MatchJSONContent#assertTimestamp(String, String, Matcher)} being provided an 152 | * empty format string 153 | */ 154 | @Test(expected=IllegalArgumentException.class) 155 | public void testAssertTimestamp_withEmptyFormatString() throws Exception { 156 | final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); 157 | new MatchJSONContent().assertTimestamp("path", null, Matchers.is(sdf.parse("2016-04-25"))); 158 | } 159 | 160 | /** 161 | * Test case for {@link MatchJSONContent#assertTimestamp(String, String, Matcher)} being provided an 162 | * empty matcher 163 | */ 164 | @Test(expected=IllegalArgumentException.class) 165 | public void testAssertTimestamp_withEmptyMatcher() { 166 | new MatchJSONContent().assertTimestamp("path", "yyyy-MM-dd", null); 167 | } 168 | 169 | /** 170 | * Test case for {@link MatchJSONContent#assertTimestamp(String, String, Matcher)} being provided 171 | * valid input 172 | */ 173 | @Test 174 | public void testAssertTimestamp_withValidInput() throws Exception { 175 | new MatchJSONContent().assertTimestamp("path", "yyyy-MM-dd", Matchers.is(new SimpleDateFormat("yyyy-MM-dd").parse("2016-04-25"))); 176 | } 177 | 178 | } 179 | -------------------------------------------------------------------------------- /src/test/resources/invalid-test-configuration.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "str":"testls", 3 | "intVal":5 4 | } -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=ERROR, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n 9 | -------------------------------------------------------------------------------- /src/test/resources/valid-test-configuration.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "name":"test-application", 3 | "description":"test application", 4 | "parallelism":1, 5 | "executionRetries":3, 6 | "str":"testl", 7 | "intVal":5 8 | } --------------------------------------------------------------------------------