finalConfig = new HashMap<>(env);
11 | finalConfig.put("SINK_CONNECTOR_SCHEMA_PROTO_MESSAGE_CLASS", env.getOrDefault("INPUT_SCHEMA_PROTO_CLASS", ""));
12 | finalConfig.put("SINK_CONNECTOR_SCHEMA_PROTO_KEY_CLASS", env.getOrDefault("INPUT_SCHEMA_PROTO_CLASS", ""));
13 | finalConfig.put("SINK_CONNECTOR_SCHEMA_DATA_TYPE", env.getOrDefault("INPUT_SCHEMA_DATA_TYPE", "protobuf"));
14 | finalConfig.put("SINK_METRICS_APPLICATION_PREFIX", "firehose_");
15 | finalConfig.put("SINK_CONNECTOR_SCHEMA_PROTO_ALLOW_UNKNOWN_FIELDS_ENABLE", env.getOrDefault("INPUT_SCHEMA_PROTO_ALLOW_UNKNOWN_FIELDS_ENABLE", "false"));
16 | finalConfig.put("SINK_CONNECTOR_SCHEMA_MESSAGE_MODE",
17 | env.getOrDefault("KAFKA_RECORD_PARSER_MODE", "").equals("key") ? SinkConnectorSchemaMessageMode.LOG_KEY.name() : SinkConnectorSchemaMessageMode.LOG_MESSAGE.name());
18 | return finalConfig;
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/http/request/HttpRequestMethodFactory.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.http.request;
2 |
3 | import org.raystack.firehose.config.enums.HttpSinkRequestMethodType;
4 | import org.raystack.firehose.sink.http.request.method.HttpDeleteWithBody;
5 | import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
6 | import org.apache.http.client.methods.HttpPatch;
7 | import org.apache.http.client.methods.HttpPost;
8 | import org.apache.http.client.methods.HttpPut;
9 |
10 | import java.net.URI;
11 |
12 | /**
13 | * The type Http request method factory.
14 | */
15 | public class HttpRequestMethodFactory {
16 | /**
17 | * Create http entity enclosing request base.
18 | *
19 | * @param uri the uri
20 | * @param method the method
21 | * @return the http entity enclosing request base
22 | */
23 | public static HttpEntityEnclosingRequestBase create(URI uri, HttpSinkRequestMethodType method) {
24 | switch (method) {
25 | case POST:
26 | return new HttpPost(uri);
27 | case PATCH:
28 | return new HttpPatch(uri);
29 | case DELETE:
30 | return new HttpDeleteWithBody(uri);
31 | default:
32 | return new HttpPut(uri);
33 | }
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/serializer/JsonWrappedProtoByte.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.serializer;
2 |
3 | import org.raystack.firehose.message.Message;
4 | import org.raystack.firehose.exception.DeserializerException;
5 | import com.google.gson.Gson;
6 | import com.google.gson.GsonBuilder;
7 |
8 | /**
9 | * JsonWrappedProto wrap encoded64 protobuff message into json format. The
10 | * format would look like
11 | *
12 | * {
13 | * "topic":"sample-topic",
14 | * "log_key":"CgYIyOm+xgUSBgiE6r7GBRgNIICAgIDA9/y0LigCMAM\u003d",
15 | * "log_message":"CgYIyOm+xgUSBgiE6r7GBRgNIICAgIDA9/y0LigCMAM\u003d"
16 | * }
17 | *
18 | */
19 | public class JsonWrappedProtoByte implements MessageSerializer {
20 |
21 | private Gson gson;
22 |
23 | /**
24 | * Instantiates a new Json wrapped proto byte.
25 | */
26 | public JsonWrappedProtoByte() {
27 | this.gson = new GsonBuilder().registerTypeAdapter(Message.class, new MessageJsonSerializer()).create();
28 | }
29 |
30 | /**
31 | * Serialize string.
32 | *
33 | * @param message the message
34 | * @return the string
35 | * @throws DeserializerException the deserializer exception
36 | */
37 | @Override
38 | public String serialize(Message message) throws DeserializerException {
39 | return gson.toJson(message);
40 | }
41 |
42 | }
43 |
--------------------------------------------------------------------------------
/src/test/java/org/raystack/firehose/sink/prometheus/request/PromRequestCreatorTest.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.prometheus.request;
2 |
3 |
4 | import org.raystack.firehose.config.PromSinkConfig;
5 | import org.raystack.depot.metrics.StatsDReporter;
6 | import org.raystack.stencil.Parser;
7 | import org.aeonbits.owner.ConfigFactory;
8 | import org.junit.Before;
9 | import org.junit.Test;
10 | import org.mockito.Mock;
11 |
12 | import java.util.Properties;
13 |
14 | import static org.junit.Assert.assertNotNull;
15 | import static org.mockito.MockitoAnnotations.initMocks;
16 |
17 | public class PromRequestCreatorTest {
18 |
19 | @Mock
20 | private StatsDReporter statsDReporter;
21 |
22 | @Mock
23 | private Parser protoParser;
24 |
25 | @Before
26 | public void setup() {
27 | initMocks(this);
28 | }
29 |
30 | @Test
31 | public void shouldCreateNotNullRequest() {
32 |
33 | Properties promConfigProps = new Properties();
34 |
35 | PromSinkConfig promSinkConfig = ConfigFactory.create(PromSinkConfig.class, promConfigProps);
36 | PromRequestCreator promRequestCreator = new PromRequestCreator(statsDReporter, promSinkConfig, protoParser);
37 |
38 | PromRequest request = promRequestCreator.createRequest();
39 |
40 | assertNotNull(request);
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/http/request/body/JsonBody.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.http.request.body;
2 |
3 | import java.util.ArrayList;
4 | import java.util.List;
5 |
6 | import org.raystack.firehose.exception.DeserializerException;
7 | import org.raystack.firehose.message.Message;
8 | import org.raystack.firehose.serializer.MessageSerializer;
9 |
10 | /**
11 | * JsonBody Serialize the message according to injected serialzier and return it
12 | * as List of serialized string.
13 | */
14 | public class JsonBody {
15 |
16 | private MessageSerializer jsonSerializer;
17 |
18 | /**
19 | * Instantiates a new Json body.
20 | *
21 | * @param jsonSerializer the json serializer
22 | */
23 | public JsonBody(MessageSerializer jsonSerializer) {
24 | this.jsonSerializer = jsonSerializer;
25 | }
26 |
27 | /**
28 | * Serialize list.
29 | *
30 | * @param messages the messages
31 | * @return the list
32 | * @throws DeserializerException the deserializer exception
33 | */
34 | public List serialize(List messages) throws DeserializerException {
35 | List serializedBody = new ArrayList();
36 | for (Message message : messages) {
37 | serializedBody.add(jsonSerializer.serialize(message));
38 | }
39 | return serializedBody;
40 | }
41 |
42 | }
43 |
--------------------------------------------------------------------------------
/docs/docs/advance/filters.md:
--------------------------------------------------------------------------------
1 | # Filters
2 |
3 | Following variables need to be set to enable JSON/JEXL filters.
4 |
5 | ## `FILTER_ENGINE`
6 |
7 | Defines whether to use `JSON` Schema-based filters or `JEXL`-based filters or `NO_OP` \(i.e. no filtering\)
8 |
9 | - Example value: `JSON`
10 | - Type: `optional`
11 | - Default value`: NO_OP`
12 |
13 | ## `FILTER_JSON_ESB_MESSAGE_TYPE`
14 |
15 | Defines the format type of the input ESB messages, i.e. JSON/Protobuf. This field is required only for JSON filters.
16 |
17 | - Example value: `JSON`
18 | - Type: `optional`
19 |
20 | ## `FILTER_SCHEMA_PROTO_CLASS`
21 |
22 | The fully qualified name of the proto schema so that the key/message in Kafka could be parsed.
23 |
24 | - Example value: `com.raystack.esb.driverlocation.DriverLocationLogKey`
25 | - Type: `optional`
26 |
27 | ## `FILTER_DATA_SOURCE`
28 |
29 | `key`/`message`/`none`depending on where to apply filter
30 |
31 | - Example value: `key`
32 | - Type: `optional`
33 | - Default value`: none`
34 |
35 | ## `FILTER_JEXL_EXPRESSION`
36 |
37 | JEXL filter expression
38 |
39 | - Example value: `driverLocationLogKey.getVehicleType()=="BIKE"`
40 | - Type: `optional`
41 |
42 | ## `FILTER_JSON_SCHEMA`
43 |
44 | JSON Schema string containing the filter rules to be applied.
45 |
46 | - Example value: `{"properties":{"order_number":{"const":"1253"}}}`
47 | - Type: `optional`
48 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/utils/ConsumerRebalancer.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.utils;
2 |
3 | import org.raystack.firehose.metrics.FirehoseInstrumentation;
4 | import lombok.AllArgsConstructor;
5 | import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
6 | import org.apache.kafka.common.TopicPartition;
7 |
8 | import java.util.Arrays;
9 | import java.util.Collection;
10 |
11 | /**
12 | * A callback to log when the partition rebalancing happens.
13 | */
14 | @AllArgsConstructor
15 | public class ConsumerRebalancer implements ConsumerRebalanceListener {
16 |
17 | private FirehoseInstrumentation firehoseInstrumentation;
18 |
19 | /**
20 | * Function to run On partitions revoked.
21 | *
22 | * @param partitions list of partitions
23 | */
24 | @Override
25 | public void onPartitionsRevoked(Collection partitions) {
26 | firehoseInstrumentation.logWarn("Partitions Revoked {}", Arrays.toString(partitions.toArray()));
27 | }
28 |
29 | /**
30 | * Function to run On partitions assigned.
31 | *
32 | * @param partitions list of partitions
33 | */
34 | @Override
35 | public void onPartitionsAssigned(Collection partitions) {
36 | firehoseInstrumentation.logInfo("Partitions Assigned {}", Arrays.toString(partitions.toArray()));
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/src/test/java/org/raystack/firehose/sink/blob/TestUtils.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.blob;
2 |
3 | import com.google.protobuf.Descriptors;
4 | import com.google.protobuf.DynamicMessage;
5 | import org.raystack.firehose.message.Message;
6 | import org.raystack.firehose.sink.blob.message.KafkaMetadataUtils;
7 | import org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessageUtils;
8 |
9 | import java.time.Instant;
10 |
11 | public class TestUtils {
12 |
13 | public static DynamicMessage createMetadata(String kafkaMetadataColumnName, Instant eventTimestamp, long offset, int partition, String topic) {
14 | Message message = new Message("".getBytes(), "".getBytes(), topic, partition, offset, null, eventTimestamp.toEpochMilli(), eventTimestamp.toEpochMilli());
15 | Descriptors.FileDescriptor fileDescriptor = KafkaMetadataProtoMessageUtils.createFileDescriptor(kafkaMetadataColumnName);
16 | return KafkaMetadataUtils.createKafkaMetadata(fileDescriptor, message, kafkaMetadataColumnName);
17 | }
18 |
19 | public static DynamicMessage createMessage(Instant timestamp, int orderNum) {
20 | TestProtoMessage.MessageBuilder messageBuilder = TestProtoMessage.createMessageBuilder();
21 | return messageBuilder
22 | .setCreatedTime(timestamp)
23 | .setOrderNumber(orderNum).build();
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/metrics/BigQueryMetrics.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.metrics;
2 |
3 | public class BigQueryMetrics {
4 | public enum BigQueryAPIType {
5 | TABLE_UPDATE,
6 | TABLE_CREATE,
7 | DATASET_UPDATE,
8 | DATASET_CREATE,
9 | TABLE_INSERT_ALL,
10 | }
11 |
12 | public enum BigQueryErrorType {
13 | UNKNOWN_ERROR,
14 | INVALID_SCHEMA_ERROR,
15 | OOB_ERROR,
16 | STOPPED_ERROR,
17 | }
18 |
19 | public static final String BIGQUERY_SINK_PREFIX = "bigquery_";
20 | public static final String BIGQUERY_TABLE_TAG = "table=%s";
21 | public static final String BIGQUERY_DATASET_TAG = "dataset=%s";
22 | public static final String BIGQUERY_API_TAG = "api=%s";
23 | public static final String BIGQUERY_ERROR_TAG = "error=%s";
24 | // BigQuery SINK MEASUREMENTS
25 | public static final String SINK_BIGQUERY_OPERATION_TOTAL = Metrics.APPLICATION_PREFIX + Metrics.SINK_PREFIX + BIGQUERY_SINK_PREFIX + "operation_total";
26 | public static final String SINK_BIGQUERY_OPERATION_LATENCY_MILLISECONDS = Metrics.APPLICATION_PREFIX + Metrics.SINK_PREFIX + BIGQUERY_SINK_PREFIX + "operation_latency_milliseconds";
27 | public static final String SINK_BIGQUERY_ERRORS_TOTAL = Metrics.APPLICATION_PREFIX + Metrics.SINK_PREFIX + BIGQUERY_SINK_PREFIX + "errors_total";
28 |
29 | }
30 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/config/FilterConfig.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.config;
2 |
3 | import org.raystack.firehose.config.converter.FilterDataSourceTypeConverter;
4 | import org.raystack.firehose.config.converter.FilterEngineTypeConverter;
5 | import org.raystack.firehose.config.converter.FilterMessageFormatTypeConverter;
6 | import org.raystack.firehose.config.enums.FilterDataSourceType;
7 | import org.raystack.firehose.config.enums.FilterEngineType;
8 | import org.raystack.firehose.config.enums.FilterMessageFormatType;
9 | import org.aeonbits.owner.Config;
10 |
11 | public interface FilterConfig extends Config {
12 |
13 | @Key("FILTER_ENGINE")
14 | @ConverterClass(FilterEngineTypeConverter.class)
15 | @DefaultValue("NO_OP")
16 | FilterEngineType getFilterEngine();
17 |
18 | @Key("FILTER_SCHEMA_PROTO_CLASS")
19 | String getFilterSchemaProtoClass();
20 |
21 | @Key("FILTER_ESB_MESSAGE_FORMAT")
22 | @ConverterClass(FilterMessageFormatTypeConverter.class)
23 | FilterMessageFormatType getFilterESBMessageFormat();
24 |
25 | @Key("FILTER_DATA_SOURCE")
26 | @ConverterClass(FilterDataSourceTypeConverter.class)
27 | FilterDataSourceType getFilterDataSource();
28 |
29 | @Key("FILTER_JEXL_EXPRESSION")
30 | String getFilterJexlExpression();
31 |
32 | @Key("FILTER_JSON_SCHEMA")
33 | String getFilterJsonSchema();
34 |
35 | }
36 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/prometheus/builder/HeaderBuilder.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.prometheus.builder;
2 |
3 | import java.util.Arrays;
4 | import java.util.HashMap;
5 | import java.util.Map;
6 | import java.util.stream.Collectors;
7 |
8 | import static org.raystack.firehose.sink.prometheus.PromSinkConstants.*;
9 |
10 | /**
11 | * Builder for prometheus request header.
12 | */
13 | public class HeaderBuilder {
14 |
15 | private final Map baseHeaders;
16 |
17 | /**
18 | * Instantiates a new Header builder.
19 | *
20 | * @param headerConfig the header config
21 | */
22 | public HeaderBuilder(String headerConfig) {
23 | baseHeaders = Arrays.stream(headerConfig.split(","))
24 | .filter(kv -> !kv.trim().isEmpty()).map(kv -> kv.split(":"))
25 | .collect(Collectors.toMap(kv -> kv[0], kv -> kv[1]));
26 | }
27 |
28 | /**
29 | * build prometheus request header.
30 | *
31 | * @return prometheus request header
32 | */
33 | public Map build() {
34 | Map headers = new HashMap<>(baseHeaders);
35 | headers.put(CONTENT_ENCODING, CONTENT_ENCODING_DEFAULT);
36 | headers.put(PROMETHEUS_REMOTE_WRITE_VERSION, PROMETHEUS_REMOTE_WRITE_VERSION_DEFAULT);
37 | return headers;
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcFieldFactory.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.jdbc.field;
2 |
3 | import org.raystack.firehose.sink.jdbc.field.message.JdbcDefaultMessageField;
4 | import org.raystack.firehose.sink.jdbc.field.message.JdbcCollectionField;
5 | import org.raystack.firehose.sink.jdbc.field.message.JdbcTimestampField;
6 | import com.google.protobuf.Descriptors;
7 |
8 | import java.util.Arrays;
9 | import java.util.List;
10 |
11 | /**
12 | * Jdbc field factory.
13 | */
14 | public class JdbcFieldFactory {
15 | /**
16 | * Returns the field based on configuration.
17 | *
18 | * @param columnValue the column value
19 | * @param fieldDescriptor the field descriptor
20 | * @return the field
21 | */
22 | public static JdbcField getField(Object columnValue, Descriptors.FieldDescriptor fieldDescriptor) {
23 | List jdbcFields = Arrays.asList(
24 | new JdbcCollectionField(columnValue, fieldDescriptor),
25 | new JdbcMapField(columnValue, fieldDescriptor),
26 | new JdbcTimestampField(columnValue),
27 | new JdbcDefaultMessageField(columnValue));
28 | return jdbcFields.stream()
29 | .filter(JdbcField::canProcess)
30 | .findFirst()
31 | .orElseGet(() -> new JdbcDefaultField(columnValue));
32 | }
33 |
34 | }
35 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/common/KeyOrMessageParser.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.common;
2 |
3 |
4 | import org.raystack.firehose.config.AppConfig;
5 | import org.raystack.firehose.message.Message;
6 | import com.google.protobuf.DynamicMessage;
7 | import com.google.protobuf.InvalidProtocolBufferException;
8 | import org.raystack.stencil.Parser;
9 | import lombok.AllArgsConstructor;
10 |
11 | import java.io.IOException;
12 |
13 | /**
14 | * Parser for Key or message.
15 | */
16 | @AllArgsConstructor
17 | public class KeyOrMessageParser {
18 |
19 | private Parser protoParser;
20 | private AppConfig appConfig;
21 |
22 | /**
23 | * Parse dynamic message.
24 | *
25 | * @param message the message
26 | * @return the dynamic message
27 | * @throws IOException when invalid message is encountered
28 | */
29 | public DynamicMessage parse(Message message) throws IOException {
30 | if (appConfig.getKafkaRecordParserMode().equals("key")) {
31 | return protoParse(message.getLogKey());
32 | }
33 | return protoParse(message.getLogMessage());
34 | }
35 |
36 | private DynamicMessage protoParse(byte[] data) throws IOException {
37 | try {
38 | return protoParser.parse(data);
39 | } catch (InvalidProtocolBufferException e) {
40 | throw new IOException(e);
41 | }
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/config/converter/SchemaRegistryHeadersConverter.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.config.converter;
2 |
3 | import java.lang.reflect.Method;
4 | import java.util.regex.Pattern;
5 | import java.util.stream.Collectors;
6 |
7 | import org.aeonbits.owner.Converter;
8 | import org.aeonbits.owner.Tokenizer;
9 | import org.apache.http.Header;
10 | import org.apache.http.message.BasicHeader;
11 |
12 | public class SchemaRegistryHeadersConverter implements Converter, Tokenizer {
13 |
14 | @Override
15 | public Header convert(Method method, String input) {
16 | String[] split = input.split(":");
17 | return new BasicHeader(split[0].trim(), split[1].trim());
18 | }
19 |
20 | @Override
21 | public String[] tokens(String values) {
22 | String[] headers = Pattern.compile(",").splitAsStream(values).map(String::trim)
23 | .filter(s -> {
24 | String[] args = s.split(":");
25 | return args.length == 2 && args[0].trim().length() > 0 && args[1].trim().length() > 0;
26 | })
27 | .collect(Collectors.toList())
28 | .toArray(new String[0]);
29 | if (headers.length == 0) {
30 | throw new IllegalArgumentException(String.format("provided headers %s is not valid", values));
31 | }
32 |
33 | return headers;
34 | }
35 |
36 | }
37 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/config/InfluxSinkConfig.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.config;
2 |
3 | import org.raystack.firehose.config.converter.ProtoIndexToFieldMapConverter;
4 | import org.aeonbits.owner.Config;
5 |
6 | import java.util.Properties;
7 |
8 | public interface InfluxSinkConfig extends AppConfig {
9 | @Config.Key("SINK_INFLUX_FIELD_NAME_PROTO_INDEX_MAPPING")
10 | @Config.ConverterClass(ProtoIndexToFieldMapConverter.class)
11 | Properties getSinkInfluxFieldNameProtoIndexMapping();
12 |
13 | @Config.Key("SINK_INFLUX_TAG_NAME_PROTO_INDEX_MAPPING")
14 | @Config.ConverterClass(ProtoIndexToFieldMapConverter.class)
15 | Properties getSinkInfluxTagNameProtoIndexMapping();
16 |
17 | @Config.Key("SINK_INFLUX_MEASUREMENT_NAME")
18 | String getSinkInfluxMeasurementName();
19 |
20 | @Config.Key("SINK_INFLUX_PROTO_EVENT_TIMESTAMP_INDEX")
21 | Integer getSinkInfluxProtoEventTimestampIndex();
22 |
23 | @Config.Key("SINK_INFLUX_DB_NAME")
24 | String getSinkInfluxDbName();
25 |
26 | @Config.Key("SINK_INFLUX_RETENTION_POLICY")
27 | @DefaultValue("autogen")
28 | String getSinkInfluxRetentionPolicy();
29 |
30 | @Config.Key("SINK_INFLUX_URL")
31 | String getSinkInfluxUrl();
32 |
33 | @Config.Key("SINK_INFLUX_USERNAME")
34 | String getSinkInfluxUsername();
35 |
36 | @Config.Key("SINK_INFLUX_PASSWORD")
37 | String getSinkInfluxPassword();
38 | }
39 |
40 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sinkdecorator/SinkFinal.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sinkdecorator;
2 |
3 | import org.raystack.firehose.message.Message;
4 | import org.raystack.firehose.exception.DeserializerException;
5 | import org.raystack.firehose.metrics.FirehoseInstrumentation;
6 | import org.raystack.firehose.metrics.Metrics;
7 | import org.raystack.firehose.sink.Sink;
8 |
9 | import java.io.IOException;
10 | import java.util.List;
11 |
12 | public class SinkFinal extends SinkDecorator {
13 | private final FirehoseInstrumentation firehoseInstrumentation;
14 |
15 | /**
16 | * Instantiates a new Sink decorator.
17 | *
18 | * @param sink wrapped sink object
19 | */
20 |
21 | public SinkFinal(Sink sink, FirehoseInstrumentation firehoseInstrumentation) {
22 | super(sink);
23 | this.firehoseInstrumentation = firehoseInstrumentation;
24 | }
25 |
26 | @Override
27 | public List pushMessage(List inputMessages) throws IOException, DeserializerException {
28 | List failedMessages = super.pushMessage(inputMessages);
29 | if (failedMessages.size() > 0) {
30 | firehoseInstrumentation.logInfo("Ignoring messages {}", failedMessages.size());
31 | firehoseInstrumentation.captureGlobalMessageMetrics(Metrics.MessageScope.IGNORED, failedMessages.size());
32 | }
33 | return failedMessages;
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/src/test/java/org/raystack/firehose/sinkdecorator/SinkFinalTest.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sinkdecorator;
2 |
3 | import org.raystack.firehose.message.Message;
4 | import org.raystack.firehose.metrics.FirehoseInstrumentation;
5 | import org.raystack.firehose.metrics.Metrics;
6 | import org.raystack.firehose.sink.Sink;
7 | import org.junit.Test;
8 | import org.mockito.Mockito;
9 |
10 | import java.io.IOException;
11 | import java.util.ArrayList;
12 | import java.util.List;
13 |
14 | public class SinkFinalTest {
15 |
16 | @Test
17 | public void shouldIgnoreMessages() throws IOException {
18 | Sink sink = Mockito.mock(Sink.class);
19 | FirehoseInstrumentation firehoseInstrumentation = Mockito.mock(FirehoseInstrumentation.class);
20 | SinkFinal sinkFinal = new SinkFinal(sink, firehoseInstrumentation);
21 | List messages = new ArrayList() {{
22 | add(new Message("".getBytes(), "".getBytes(), "", 0, 0));
23 | add(new Message("".getBytes(), "".getBytes(), "", 0, 0));
24 | }};
25 | Mockito.when(sink.pushMessage(messages)).thenReturn(messages);
26 |
27 | sinkFinal.pushMessage(messages);
28 | Mockito.verify(firehoseInstrumentation, Mockito.times(1)).logInfo("Ignoring messages {}", 2);
29 | Mockito.verify(firehoseInstrumentation, Mockito.times(1)).captureGlobalMessageMetrics(Metrics.MessageScope.IGNORED, 2);
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/http/request/entity/RequestEntityBuilder.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.http.request.entity;
2 |
3 | import org.raystack.firehose.exception.DeserializerException;
4 | import org.apache.http.entity.ContentType;
5 | import org.apache.http.entity.StringEntity;
6 |
7 | import java.util.Collections;
8 |
9 | /**
10 | * Request entity builder.
11 | */
12 | public class RequestEntityBuilder {
13 | private boolean wrapArray;
14 |
15 | /**
16 | * Instantiates a new Request entity builder.
17 | */
18 | public RequestEntityBuilder() {
19 | this.wrapArray = false;
20 | }
21 |
22 | public RequestEntityBuilder setWrapping(boolean isArrayWrap) {
23 | this.wrapArray = isArrayWrap;
24 | return this;
25 | }
26 |
27 | /**
28 | * Build http entity string entity.
29 | *
30 | * @param bodyContent the body content
31 | * @return the string entity
32 | * @throws DeserializerException the deserializer exception
33 | */
34 | public StringEntity buildHttpEntity(String bodyContent) throws DeserializerException {
35 | if (!wrapArray) {
36 | return new StringEntity(bodyContent, ContentType.APPLICATION_JSON);
37 | } else {
38 | String arrayWrappedBody = Collections.singletonList(bodyContent).toString();
39 | return new StringEntity(arrayWrappedBody, ContentType.APPLICATION_JSON);
40 | }
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcMapField.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.jdbc.field;
2 |
3 | import com.google.protobuf.Descriptors;
4 | import com.google.protobuf.DynamicMessage;
5 | import org.json.simple.JSONObject;
6 |
7 | import java.util.HashMap;
8 | import java.util.List;
9 |
10 | /**
11 | * Jdbc map field.
12 | */
13 | public class JdbcMapField implements JdbcField {
14 | private Object columnValue;
15 | private Descriptors.FieldDescriptor fieldDescriptor;
16 |
17 |
18 | public JdbcMapField(Object columnValue, Descriptors.FieldDescriptor fieldDescriptor) {
19 | this.columnValue = columnValue;
20 | this.fieldDescriptor = fieldDescriptor;
21 | }
22 |
23 | @Override
24 | public Object getColumn() throws RuntimeException {
25 | HashMap columnFields = new HashMap<>();
26 | List values = (List) this.columnValue;
27 | for (DynamicMessage dynamicMessage : values) {
28 | Object[] data = dynamicMessage.getAllFields().values().toArray();
29 | Object mapValue = data.length > 1 ? data[1] : "";
30 | columnFields.put((String) data[0], mapValue);
31 | }
32 | String columnEntry = JSONObject.toJSONString(columnFields);
33 | return columnEntry;
34 | }
35 |
36 | @Override
37 | public boolean canProcess() {
38 | return fieldDescriptor.isMapField();
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sinkdecorator/SinkDecorator.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sinkdecorator;
2 |
3 | import org.raystack.firehose.message.Message;
4 | import org.raystack.firehose.exception.DeserializerException;
5 | import org.raystack.firehose.sink.Sink;
6 |
7 | import java.io.IOException;
8 | import java.util.List;
9 |
10 | /**
11 | * Sink decorator provides internal processing on the use provided sink type.
12 | */
13 | public class SinkDecorator implements Sink {
14 |
15 | private final Sink sink;
16 |
17 | /**
18 | * Instantiates a new Sink decorator.
19 | *
20 | * @param sink wrapped sink object
21 | */
22 | public SinkDecorator(Sink sink) {
23 | this.sink = sink;
24 | }
25 |
26 | @Override
27 | public List pushMessage(List message) throws IOException, DeserializerException {
28 | return sink.pushMessage(message);
29 | }
30 |
31 | @Override
32 | public void close() throws IOException {
33 | sink.close();
34 | }
35 |
36 | @Override
37 | public void calculateCommittableOffsets() {
38 | sink.calculateCommittableOffsets();
39 | }
40 |
41 | @Override
42 | public boolean canManageOffsets() {
43 | return sink.canManageOffsets();
44 | }
45 |
46 | @Override
47 | public void addOffsetsAndSetCommittable(List messageList) {
48 | sink.addOffsetsAndSetCommittable(messageList);
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/.github/workflows/package.yml:
--------------------------------------------------------------------------------
1 | name: Package
2 |
3 | on:
4 | release:
5 | types: [published]
6 | workflow_dispatch:
7 |
8 | jobs:
9 | build:
10 | runs-on: ubuntu-latest
11 | steps:
12 | - uses: actions/checkout@v2
13 | - name: Set up JDK 1.8
14 | uses: actions/setup-java@v1
15 | with:
16 | java-version: 1.8
17 | - name: Grant execute permission for gradlew
18 | run: chmod +x gradlew
19 | - name: Build
20 | run: ./gradlew build
21 |
22 | publish:
23 | needs: build
24 | runs-on: ubuntu-latest
25 | steps:
26 | - uses: actions/checkout@v2
27 | - uses: actions/setup-java@v1
28 | with:
29 | java-version: 1.8
30 | - name: Get release tag
31 | id: get_version
32 | uses: battila7/get-version-action@v2
33 | - name: Publish package
34 | run: ./gradlew publish
35 | env:
36 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
37 | - name: Login to DockerHub
38 | uses: docker/login-action@v1
39 | with:
40 | username: ${{ secrets.DOCKERHUB_USERNAME }}
41 | password: ${{ secrets.DOCKERHUB_TOKEN }}
42 | - name: Build and Push
43 | uses: docker/build-push-action@v2
44 | with:
45 | context: .
46 | push: true
47 | tags: |
48 | raystack/firehose:latest
49 | raystack/firehose:${{ steps.get_version.outputs.version-without-v }}
50 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/jdbc/field/message/JdbcTimestampField.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.jdbc.field.message;
2 |
3 | import org.raystack.firehose.sink.jdbc.field.JdbcField;
4 | import com.google.protobuf.Descriptors;
5 | import com.google.protobuf.DynamicMessage;
6 | import com.google.protobuf.Timestamp;
7 |
8 | import java.time.Instant;
9 | import java.util.ArrayList;
10 | import java.util.List;
11 |
12 | public class JdbcTimestampField implements JdbcField {
13 | private Object columnValue;
14 |
15 | public JdbcTimestampField(Object columnValue) {
16 | this.columnValue = columnValue;
17 | }
18 |
19 | @Override
20 | public Object getColumn() {
21 | List fieldDescriptors = ((DynamicMessage) columnValue).getDescriptorForType().getFields();
22 | ArrayList