messages) {
40 | return messages.stream().filter(message -> message.getUnknownFields().asMap().size() > 0).collect(Collectors.toList());
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/serializer/JsonWrappedProtoByte.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.serializer;
2 |
3 | import org.raystack.firehose.message.Message;
4 | import org.raystack.firehose.exception.DeserializerException;
5 | import com.google.gson.Gson;
6 | import com.google.gson.GsonBuilder;
7 |
8 | /**
9 | * JsonWrappedProto wrap encoded64 protobuff message into json format. The
10 | * format would look like
11 | *
12 | * {
13 | * "topic":"sample-topic",
14 | * "log_key":"CgYIyOm+xgUSBgiE6r7GBRgNIICAgIDA9/y0LigCMAM\u003d",
15 | * "log_message":"CgYIyOm+xgUSBgiE6r7GBRgNIICAgIDA9/y0LigCMAM\u003d"
16 | * }
17 | *
18 | */
19 | public class JsonWrappedProtoByte implements MessageSerializer {
20 |
21 | private Gson gson;
22 |
23 | /**
24 | * Instantiates a new Json wrapped proto byte.
25 | */
26 | public JsonWrappedProtoByte() {
27 | this.gson = new GsonBuilder().registerTypeAdapter(Message.class, new MessageJsonSerializer()).create();
28 | }
29 |
30 | /**
31 | * Serialize string.
32 | *
33 | * @param message the message
34 | * @return the string
35 | * @throws DeserializerException the deserializer exception
36 | */
37 | @Override
38 | public String serialize(Message message) throws DeserializerException {
39 | return gson.toJson(message);
40 | }
41 |
42 | }
43 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/serializer/MessageJsonSerializer.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.serializer;
2 |
3 | import org.raystack.firehose.message.Message;
4 | import com.google.gson.JsonElement;
5 | import com.google.gson.JsonObject;
6 | import com.google.gson.JsonSerializationContext;
7 | import com.google.gson.JsonSerializer;
8 |
9 | import java.lang.reflect.Type;
10 |
11 | /**
12 | * Message Json serializer serialises Kafka message to json.
13 | */
14 | public class MessageJsonSerializer implements JsonSerializer {
15 |
16 | /**
17 | * Serialize kafka message into json element.
18 | *
19 | * @param message the message
20 | * @param typeOfSrc the type of src
21 | * @param context the context
22 | * @return the json element
23 | */
24 | @Override
25 | public JsonElement serialize(Message message, Type typeOfSrc, JsonSerializationContext context) {
26 | JsonObject object = new JsonObject();
27 | object.addProperty("topic", message.getTopic());
28 | object.addProperty("log_key", message.getSerializedKey());
29 | object.addProperty("log_message", message.getSerializedMessage());
30 | return object;
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/serializer/MessageSerializer.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.serializer;
2 |
3 | import org.raystack.firehose.message.Message;
4 | import org.raystack.firehose.exception.DeserializerException;
5 |
6 | /**
7 | * Serializer serialize Message into string format.
8 | */
9 | public interface MessageSerializer {
10 |
11 | /**
12 | * Serialize kafka message into string.
13 | *
14 | * @param message the message
15 | * @return serialised message
16 | * @throws DeserializerException the deserializer exception
17 | */
18 | String serialize(Message message) throws DeserializerException;
19 | }
20 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/GenericSink.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink;
2 |
3 | import org.raystack.depot.Sink;
4 | import org.raystack.depot.SinkResponse;
5 | import org.raystack.firehose.exception.DeserializerException;
6 | import org.raystack.firehose.message.FirehoseMessageUtils;
7 | import org.raystack.firehose.message.Message;
8 | import org.raystack.firehose.metrics.FirehoseInstrumentation;
9 |
10 | import java.io.IOException;
11 | import java.sql.SQLException;
12 | import java.util.ArrayList;
13 | import java.util.List;
14 | import java.util.stream.Collectors;
15 |
16 | public class GenericSink extends AbstractSink {
17 | private final List messageList = new ArrayList<>();
18 | private final Sink sink;
19 |
20 | public GenericSink(FirehoseInstrumentation firehoseInstrumentation, String sinkType, Sink sink) {
21 | super(firehoseInstrumentation, sinkType);
22 | this.sink = sink;
23 | }
24 |
25 | @Override
26 | protected List execute() throws Exception {
27 | List messages = FirehoseMessageUtils.convertToDepotMessage(messageList);
28 | SinkResponse response = sink.pushToSink(messages);
29 | return response.getErrors().keySet().stream()
30 | .map(index -> {
31 | Message message = messageList.get(index.intValue());
32 | message.setErrorInfo(response.getErrorsFor(index));
33 | return message;
34 | }).collect(Collectors.toList());
35 | }
36 |
37 | @Override
38 | protected void prepare(List messages) throws DeserializerException, IOException, SQLException {
39 | messageList.clear();
40 | messageList.addAll(messages);
41 | }
42 |
43 | @Override
44 | public void close() throws IOException {
45 | sink.close();
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/Sink.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink;
2 |
3 | import org.raystack.firehose.message.Message;
4 |
5 | import java.io.Closeable;
6 | import java.io.IOException;
7 | import java.util.List;
8 |
9 | /**
10 | * An interface for developing custom Sinks to FirehoseConsumer.
11 | */
12 | public interface Sink extends Closeable {
13 |
14 | /**
15 | * method to write batch of messages read from kafka.
16 | * The logic of how to persist the data goes in here.
17 | * in the future this method should return Response object instead of list of messages
18 | *
19 | * @param message list of {@see EsbMessage}
20 | * @return the list of failed messages
21 | * @throws IOException in case of error conditions while persisting it to the custom sink.
22 | */
23 | List pushMessage(List message) throws IOException;
24 |
25 | /**
26 | * Method that inform that sink is managing kafka offset to be committed by its own.
27 | *
28 | * @return
29 | */
30 | default boolean canManageOffsets() {
31 | return false;
32 | }
33 |
34 | /**
35 | * Method to register kafka offsets and setting it to be committed.
36 | * This method should be implemented when sink manages the commit offsets by themselves.
37 | * Or when {@link Sink#canManageOffsets()} return true
38 | *
39 | * @param messageList messages to be committed
40 | */
41 | default void addOffsetsAndSetCommittable(List messageList) {
42 |
43 | }
44 |
45 | /**
46 | * Method to calculate offsets ready to be committed.
47 | * This method should be implemented when sink manages the commit offsets by themselves.
48 | */
49 | default void calculateCommittableOffsets() {
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/SinkFactoryUtils.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink;
2 |
3 | import org.raystack.depot.message.SinkConnectorSchemaMessageMode;
4 |
5 | import java.util.HashMap;
6 | import java.util.Map;
7 |
8 | public class SinkFactoryUtils {
9 | protected static Map addAdditionalConfigsForSinkConnectors(Map env) {
10 | Map finalConfig = new HashMap<>(env);
11 | finalConfig.put("SINK_CONNECTOR_SCHEMA_PROTO_MESSAGE_CLASS", env.getOrDefault("INPUT_SCHEMA_PROTO_CLASS", ""));
12 | finalConfig.put("SINK_CONNECTOR_SCHEMA_PROTO_KEY_CLASS", env.getOrDefault("INPUT_SCHEMA_PROTO_CLASS", ""));
13 | finalConfig.put("SINK_CONNECTOR_SCHEMA_DATA_TYPE", env.getOrDefault("INPUT_SCHEMA_DATA_TYPE", "protobuf"));
14 | finalConfig.put("SINK_METRICS_APPLICATION_PREFIX", "firehose_");
15 | finalConfig.put("SINK_CONNECTOR_SCHEMA_PROTO_ALLOW_UNKNOWN_FIELDS_ENABLE", env.getOrDefault("INPUT_SCHEMA_PROTO_ALLOW_UNKNOWN_FIELDS_ENABLE", "false"));
16 | finalConfig.put("SINK_CONNECTOR_SCHEMA_MESSAGE_MODE",
17 | env.getOrDefault("KAFKA_RECORD_PARSER_MODE", "").equals("key") ? SinkConnectorSchemaMessageMode.LOG_KEY.name() : SinkConnectorSchemaMessageMode.LOG_MESSAGE.name());
18 | return finalConfig;
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/bigquery/BigquerySinkUtils.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.bigquery;
2 |
3 | import java.util.Map;
4 | import java.util.function.Function;
5 |
6 | public class BigquerySinkUtils {
7 | public static Function, String> getRowIDCreator() {
8 | return (m -> String.format("%s_%d_%d", m.get("message_topic"), m.get("message_partition"), m.get("message_offset")));
9 | }
10 |
11 | public static void addMetadataColumns(Map config) {
12 | config.put("SINK_BIGQUERY_METADATA_COLUMNS_TYPES",
13 | "message_offset=integer,message_topic=string,load_time=timestamp,message_timestamp=timestamp,message_partition=integer");
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/blob/Constants.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.blob;
2 |
3 | public class Constants {
4 | public enum WriterType {
5 | PARQUET,
6 | }
7 |
8 | public enum FilePartitionType {
9 | NONE,
10 | DAY,
11 | HOUR
12 | }
13 | }
14 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/blob/message/KafkaMetadataUtils.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.blob.message;
2 |
3 | import com.google.protobuf.Descriptors;
4 | import com.google.protobuf.DynamicMessage;
5 | import org.raystack.firehose.message.Message;
6 | import org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessage;
7 | import org.raystack.firehose.sink.blob.proto.NestedKafkaMetadataProtoMessage;
8 |
9 | import java.time.Instant;
10 |
11 | /**
12 | * KafkaMetadataUtils utility class for creating kafka metadata {@link com.google.protobuf.DynamicMessage DynamicMessage} from {@link Message}.
13 | */
14 | public class KafkaMetadataUtils {
15 |
16 | public static DynamicMessage createKafkaMetadata(Descriptors.FileDescriptor kafkaMetadataFileDescriptor, Message message, String kafkaMetadataColumnName) {
17 | Descriptors.Descriptor metadataDescriptor = kafkaMetadataFileDescriptor.findMessageTypeByName(KafkaMetadataProtoMessage.getTypeName());
18 |
19 | Instant loadTime = Instant.now();
20 | Instant messageTimestamp = Instant.ofEpochMilli(message.getTimestamp());
21 |
22 | KafkaMetadataProtoMessage.MessageBuilder messageBuilder = KafkaMetadataProtoMessage.newBuilder(metadataDescriptor)
23 | .setLoadTime(loadTime)
24 | .setMessageTimestamp(messageTimestamp)
25 | .setOffset(message.getOffset())
26 | .setPartition(message.getPartition())
27 | .setTopic(message.getTopic());
28 |
29 | DynamicMessage metadata = messageBuilder.build();
30 |
31 | if (kafkaMetadataColumnName.isEmpty()) {
32 | return metadata;
33 | }
34 |
35 | Descriptors.Descriptor nestedMetadataDescriptor = kafkaMetadataFileDescriptor.findMessageTypeByName(NestedKafkaMetadataProtoMessage.getTypeName());
36 |
37 | return NestedKafkaMetadataProtoMessage.newMessageBuilder(nestedMetadataDescriptor)
38 | .setMetadata(metadata)
39 | .setMetadataColumnName(kafkaMetadataColumnName).build();
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/blob/message/Record.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.blob.message;
2 |
3 | import com.google.protobuf.Descriptors;
4 | import com.google.protobuf.DynamicMessage;
5 | import org.raystack.firehose.sink.blob.proto.KafkaMetadataProtoMessage;
6 | import lombok.AllArgsConstructor;
7 | import lombok.Data;
8 |
9 | import java.time.Instant;
10 |
11 | @AllArgsConstructor
12 | @Data
13 | public class Record {
14 | private DynamicMessage message;
15 | private DynamicMessage metadata;
16 |
17 | public String getTopic(String fieldName) {
18 | Descriptors.Descriptor metadataDescriptor = metadata.getDescriptorForType();
19 |
20 | if (!fieldName.isEmpty()) {
21 | DynamicMessage nestedMetadataMessage = (DynamicMessage) metadata.getField(metadataDescriptor.findFieldByName(fieldName));
22 | Descriptors.Descriptor nestedMetadataMessageDescriptor = nestedMetadataMessage.getDescriptorForType();
23 | return (String) nestedMetadataMessage.getField(nestedMetadataMessageDescriptor.findFieldByName(KafkaMetadataProtoMessage.MESSAGE_TOPIC_FIELD_NAME));
24 | }
25 |
26 | return (String) metadata.getField(metadataDescriptor.findFieldByName(KafkaMetadataProtoMessage.MESSAGE_TOPIC_FIELD_NAME));
27 | }
28 |
29 | public Instant getTimestamp(String fieldName) {
30 | Descriptors.Descriptor descriptor = message.getDescriptorForType();
31 | Descriptors.FieldDescriptor timestampField = descriptor.findFieldByName(fieldName);
32 | DynamicMessage timestamp = (DynamicMessage) message.getField(timestampField);
33 | long seconds = (long) timestamp.getField(timestamp.getDescriptorForType().findFieldByName("seconds"));
34 | int nanos = (int) timestamp.getField(timestamp.getDescriptorForType().findFieldByName("nanos"));
35 | return Instant.ofEpochSecond(seconds, nanos);
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/blob/proto/TimestampMetadataProtoMessage.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.blob.proto;
2 |
3 | import com.github.os72.protobuf.dynamic.MessageDefinition;
4 | import com.google.protobuf.Timestamp;
5 |
6 | /**
7 | * TimestampMetadataProtoMessage contains proto schema proto message of Timestamp. The schema equals to {@link com.google.protobuf.Timestamp Timestamp} message.
8 | * This class provides {@link com.github.os72.protobuf.dynamic.MessageDefinition} to generate protobuf descriptor and builder of {@link com.google.protobuf.Timestamp Timestamp} proto message.
9 | *
10 | * message Timestamp {
11 | * int64 seconds = 1
12 | * int42 nanos = 2;
13 | * }
14 | *
15 | */
16 | public class TimestampMetadataProtoMessage {
17 | private static final String TYPE_NAME = "Timestamp";
18 | public static final String SECONDS_FIELD_NAME = "seconds";
19 | public static final String NANOS_FIELD_NAME = "nanos";
20 | public static final int SECONDS_FIELD_NUMBER = 1;
21 | public static final int NANOS_FIELD_NUMBER = 2;
22 |
23 | public static MessageDefinition createMessageDefinition() {
24 | return MessageDefinition.newBuilder(TYPE_NAME)
25 | .addField("optional", "int64", SECONDS_FIELD_NAME, SECONDS_FIELD_NUMBER)
26 | .addField("optional", "int32", NANOS_FIELD_NAME, NANOS_FIELD_NUMBER)
27 | .build();
28 | }
29 |
30 | public static String getTypeName() {
31 | return TYPE_NAME;
32 | }
33 |
34 | public static Timestamp.Builder newBuilder() {
35 | return Timestamp.newBuilder();
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalFileMetadata.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.blob.writer.local;
2 |
3 | import lombok.AllArgsConstructor;
4 | import lombok.EqualsAndHashCode;
5 | import lombok.Getter;
6 |
7 | @AllArgsConstructor
8 | @Getter
9 | @EqualsAndHashCode
10 | public class LocalFileMetadata {
11 | private final String basePath;
12 | private final String fullPath;
13 | private final long createdTimestampMillis;
14 | private final long recordCount;
15 | private final long size;
16 | }
17 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalFileWriter.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.blob.writer.local;
2 |
3 | import org.raystack.firehose.sink.blob.message.Record;
4 |
5 | import java.io.Closeable;
6 | import java.io.IOException;
7 |
8 | public interface LocalFileWriter extends Closeable {
9 | /**
10 | * @param record to write
11 | * @return true if write succeeds, false if the writer is closed.
12 | * @throws IOException if local file writing fails
13 | */
14 | boolean write(Record record) throws IOException;
15 |
16 | LocalFileMetadata getMetadata();
17 |
18 | LocalFileMetadata closeAndFetchMetaData() throws IOException;
19 |
20 | String getFullPath();
21 | }
22 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/blob/writer/local/LocalFileWriterFailedException.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.blob.writer.local;
2 |
3 | import java.io.IOException;
4 |
5 | public class LocalFileWriterFailedException extends RuntimeException {
6 | public LocalFileWriterFailedException(IOException e) {
7 | super(e);
8 | }
9 | }
10 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/blob/writer/local/policy/SizeBasedRotatingPolicy.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.blob.writer.local.policy;
2 |
3 | import org.raystack.firehose.sink.blob.writer.local.LocalFileMetadata;
4 |
5 | public class SizeBasedRotatingPolicy implements WriterPolicy {
6 |
7 | private final long maxSize;
8 |
9 | public SizeBasedRotatingPolicy(long maxSize) {
10 | if (maxSize <= 0) {
11 | throw new IllegalArgumentException("The max size should be a positive integer");
12 | }
13 | this.maxSize = maxSize;
14 | }
15 |
16 | @Override
17 | public boolean shouldRotate(LocalFileMetadata metadata) {
18 | return metadata.getSize() >= maxSize;
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/blob/writer/local/policy/TimeBasedRotatingPolicy.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.blob.writer.local.policy;
2 |
3 | import org.raystack.firehose.sink.blob.writer.local.LocalFileMetadata;
4 |
5 | public class TimeBasedRotatingPolicy implements WriterPolicy {
6 |
7 | private final long maxRotatingDurationMillis;
8 |
9 | public TimeBasedRotatingPolicy(long maxRotatingDurationMillis) {
10 | if (maxRotatingDurationMillis <= 0) {
11 | throw new IllegalArgumentException("The max duration should be a positive integer");
12 | }
13 | this.maxRotatingDurationMillis = maxRotatingDurationMillis;
14 | }
15 |
16 | @Override
17 | public boolean shouldRotate(LocalFileMetadata metadata) {
18 | return System.currentTimeMillis() - metadata.getCreatedTimestampMillis() >= maxRotatingDurationMillis;
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/blob/writer/local/policy/WriterPolicy.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.blob.writer.local.policy;
2 |
3 | import org.raystack.firehose.sink.blob.writer.local.LocalFileMetadata;
4 |
5 | public interface WriterPolicy {
6 | boolean shouldRotate(LocalFileMetadata metadata);
7 | }
8 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageFailedException.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.blob.writer.remote;
2 |
3 | public class BlobStorageFailedException extends RuntimeException {
4 | public BlobStorageFailedException(Throwable th) {
5 | super(th);
6 | }
7 | }
8 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/blob/writer/remote/BlobStorageWorker.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.blob.writer.remote;
2 |
3 | import org.raystack.firehose.sink.common.blobstorage.BlobStorage;
4 | import org.raystack.firehose.sink.common.blobstorage.BlobStorageException;
5 | import org.raystack.firehose.sink.blob.writer.local.LocalFileMetadata;
6 | import lombok.AllArgsConstructor;
7 |
8 | import java.nio.file.Paths;
9 | import java.time.Duration;
10 | import java.time.Instant;
11 | import java.util.concurrent.Callable;
12 |
13 | /**
14 | * Uploads a local file to object-storage and returns the total time taken.
15 | */
16 | @AllArgsConstructor
17 | public class BlobStorageWorker implements Callable {
18 |
19 | private final BlobStorage blobStorage;
20 | private final LocalFileMetadata metadata;
21 |
22 | @Override
23 | public Long call() throws BlobStorageException {
24 | Instant start = Instant.now();
25 | String objectName = Paths.get(metadata.getBasePath()).relativize(Paths.get(metadata.getFullPath())).toString();
26 | blobStorage.store(objectName, metadata.getFullPath());
27 | return Duration.between(start, Instant.now()).toMillis();
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/common/KeyOrMessageParser.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.common;
2 |
3 |
4 | import org.raystack.firehose.config.AppConfig;
5 | import org.raystack.firehose.message.Message;
6 | import com.google.protobuf.DynamicMessage;
7 | import com.google.protobuf.InvalidProtocolBufferException;
8 | import org.raystack.stencil.Parser;
9 | import lombok.AllArgsConstructor;
10 |
11 | import java.io.IOException;
12 |
13 | /**
14 | * Parser for Key or message.
15 | */
16 | @AllArgsConstructor
17 | public class KeyOrMessageParser {
18 |
19 | private Parser protoParser;
20 | private AppConfig appConfig;
21 |
22 | /**
23 | * Parse dynamic message.
24 | *
25 | * @param message the message
26 | * @return the dynamic message
27 | * @throws IOException when invalid message is encountered
28 | */
29 | public DynamicMessage parse(Message message) throws IOException {
30 | if (appConfig.getKafkaRecordParserMode().equals("key")) {
31 | return protoParse(message.getLogKey());
32 | }
33 | return protoParse(message.getLogMessage());
34 | }
35 |
36 | private DynamicMessage protoParse(byte[] data) throws IOException {
37 | try {
38 | return protoParser.parse(data);
39 | } catch (InvalidProtocolBufferException e) {
40 | throw new IOException(e);
41 | }
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorage.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.common.blobstorage;
2 |
3 | /**
4 | * Abstraction of any storage that store binary bytes as file.
5 | */
6 | public interface BlobStorage {
7 | void store(String objectName, String filePath) throws BlobStorageException;
8 |
9 | void store(String objectName, byte[] content) throws BlobStorageException;
10 | }
11 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorageException.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.common.blobstorage;
2 |
3 | import lombok.EqualsAndHashCode;
4 | import lombok.Getter;
5 |
6 | /**
7 | * Should be thrown when there is exception thrown by blob storage client.
8 | */
9 | @Getter
10 | @EqualsAndHashCode(callSuper = false)
11 | public class BlobStorageException extends Exception {
12 | private final String errorType;
13 | private final String message;
14 |
15 | public BlobStorageException(String errorType, String message, Throwable cause) {
16 | super(message, cause);
17 | this.errorType = errorType;
18 | this.message = message;
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorageFactory.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.common.blobstorage;
2 |
3 | import org.raystack.firehose.config.GCSConfig;
4 | import org.raystack.firehose.config.S3Config;
5 | import org.raystack.firehose.sink.common.blobstorage.gcs.GoogleCloudStorage;
6 | import org.raystack.firehose.sink.common.blobstorage.s3.S3;
7 | import org.aeonbits.owner.ConfigFactory;
8 |
9 | import java.io.IOException;
10 | import java.util.Map;
11 |
12 | public class BlobStorageFactory {
13 |
14 | public static BlobStorage createObjectStorage(BlobStorageType storageType, Map config) {
15 | switch (storageType) {
16 | case GCS:
17 | try {
18 | GCSConfig gcsConfig = ConfigFactory.create(GCSConfig.class, config);
19 | return new GoogleCloudStorage(gcsConfig);
20 | } catch (IOException e) {
21 | throw new IllegalArgumentException("Exception while creating GCS Storage", e);
22 | }
23 | case S3:
24 | try {
25 | S3Config s3Config = ConfigFactory.create(S3Config.class, config);
26 | return new S3(s3Config);
27 | } catch (Exception e) {
28 | throw new IllegalArgumentException("Exception while creating S3 Storage", e);
29 | }
30 |
31 | default:
32 | throw new IllegalArgumentException("Blob Storage Type " + storageType + " is not supported");
33 | }
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/common/blobstorage/BlobStorageType.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.common.blobstorage;
2 |
3 | public enum BlobStorageType {
4 | GCS,
5 | S3
6 | }
7 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/common/blobstorage/gcs/error/GCSErrorType.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.common.blobstorage.gcs.error;
2 |
3 | import java.util.HashMap;
4 | import java.util.Map;
5 |
6 | /**
7 | * Error types from exception thrown by google cloud storage client.
8 | * There might be newer error codes that thrown by gcs client, need to update this list of error.
9 | */
10 | public enum GCSErrorType {
11 | FOUND(302),
12 | SEE_OTHER(303),
13 | NOT_MODIFIED(304),
14 | TEMPORARY_REDIRECT(305),
15 | RESUME_INCOMPLETE(308),
16 | UNAUTHORIZED(401),
17 | FORBIDDEN(403),
18 | NOT_FOUND(404),
19 | METHOD_NOT_ALLOWED(405),
20 | CONFLICT(409),
21 | GONE(410),
22 | LENGTH_REQUIRED(411),
23 | PRECONDITION_FAILED(412),
24 | PAYLOAD_TOO_LARGE(413),
25 | REQUESTED_RANGE_NOT_SATISFIABLE(416),
26 | CLIENT_CLOSED_REQUEST(499),
27 | BAD_REQUEST(400),
28 | GATEWAY_TIMEOUT(504),
29 | SERVICE_UNAVAILABLE(503),
30 | BAD_GATEWAY(502),
31 | INTERNAL_SERVER_ERROR(500),
32 | TOO_MANY_REQUEST(429),
33 | REQUEST_TIMEOUT(408),
34 | DEFAULT_ERROR;
35 |
36 | private static final Map ERROR_NUMBER_TYPE_MAP = new HashMap<>();
37 |
38 | static {
39 | for (GCSErrorType errorType : values()) {
40 | ERROR_NUMBER_TYPE_MAP.put(errorType.codeValue, errorType);
41 | }
42 | }
43 |
44 | public static GCSErrorType valueOfCode(int code) {
45 | return ERROR_NUMBER_TYPE_MAP.getOrDefault(code, DEFAULT_ERROR);
46 | }
47 |
48 | private int codeValue;
49 |
50 | GCSErrorType(int codeValue) {
51 | this.codeValue = codeValue;
52 | }
53 |
54 | GCSErrorType() {
55 |
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/dlq/DLQWriterType.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.dlq;
2 |
3 | public enum DLQWriterType {
4 | KAFKA,
5 | BLOB_STORAGE,
6 | LOG
7 | }
8 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/dlq/DlqWriter.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.dlq;
2 |
3 | import org.raystack.firehose.message.Message;
4 |
5 | import java.io.IOException;
6 | import java.util.List;
7 |
8 | public interface DlqWriter {
9 |
10 | /**
11 | * Method to write messages to dead letter queues destination.
12 | * @param messages is collection of message that need to be sent to dead letter queue
13 | * @return collection of messages that failed to be processed
14 | * @throws IOException can be thrown for non retry able error
15 | */
16 | List write(List messages) throws IOException;
17 | }
18 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/dlq/blobstorage/DlqMessage.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.dlq.blobstorage;
2 |
3 | import com.fasterxml.jackson.annotation.JsonProperty;
4 | import lombok.AllArgsConstructor;
5 | import lombok.Data;
6 |
7 | @Data
8 | @AllArgsConstructor
9 | public class DlqMessage {
10 | @JsonProperty("key")
11 | private String key;
12 | @JsonProperty("value")
13 | private String value;
14 | @JsonProperty("topic")
15 | private String topic;
16 | @JsonProperty("partition")
17 | private int partition;
18 | @JsonProperty("offset")
19 | private long offset;
20 | @JsonProperty("timestamp")
21 | private long timestamp;
22 | @JsonProperty("error")
23 | private String error;
24 | }
25 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/dlq/log/LogDlqWriter.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.dlq.log;
2 |
3 | import org.raystack.firehose.message.Message;
4 | import org.raystack.firehose.metrics.FirehoseInstrumentation;
5 | import org.raystack.depot.error.ErrorInfo;
6 | import org.raystack.firehose.sink.dlq.DlqWriter;
7 | import org.apache.commons.lang3.exception.ExceptionUtils;
8 |
9 | import java.io.IOException;
10 | import java.util.LinkedList;
11 | import java.util.List;
12 |
13 | public class LogDlqWriter implements DlqWriter {
14 | private final FirehoseInstrumentation firehoseInstrumentation;
15 |
16 | public LogDlqWriter(FirehoseInstrumentation firehoseInstrumentation) {
17 | this.firehoseInstrumentation = firehoseInstrumentation;
18 | }
19 |
20 | @Override
21 | public List write(List messages) throws IOException {
22 | for (Message message : messages) {
23 | String key = message.getLogKey() == null ? "" : new String(message.getLogKey());
24 | String value = message.getLogMessage() == null ? "" : new String(message.getLogMessage());
25 |
26 | String error = "";
27 | ErrorInfo errorInfo = message.getErrorInfo();
28 | if (errorInfo != null) {
29 | if (errorInfo.getException() != null) {
30 | error = ExceptionUtils.getStackTrace(errorInfo.getException());
31 | }
32 | }
33 |
34 | firehoseInstrumentation.logInfo("key: {}\nvalue: {}\nerror: {}", key, value, error);
35 | }
36 | return new LinkedList<>();
37 | }
38 |
39 | }
40 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/elasticsearch/request/EsRequestHandler.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.elasticsearch.request;
2 |
3 | import org.raystack.firehose.config.enums.EsSinkMessageType;
4 | import org.raystack.firehose.exception.JsonParseException;
5 | import org.raystack.firehose.message.Message;
6 | import org.raystack.firehose.serializer.MessageToJson;
7 | import org.elasticsearch.action.DocWriteRequest;
8 | import org.json.simple.JSONObject;
9 | import org.json.simple.parser.JSONParser;
10 | import org.json.simple.parser.ParseException;
11 |
12 | import java.nio.charset.Charset;
13 |
14 | public abstract class EsRequestHandler {
15 | private final EsSinkMessageType messageType;
16 | private final MessageToJson jsonSerializer;
17 | private final JSONParser jsonParser;
18 |
19 | public EsRequestHandler(EsSinkMessageType messageType, MessageToJson jsonSerializer) {
20 | this.messageType = messageType;
21 | this.jsonSerializer = jsonSerializer;
22 | this.jsonParser = new JSONParser();
23 | }
24 |
25 | public abstract boolean canCreate();
26 |
27 | public abstract DocWriteRequest getRequest(Message message);
28 |
29 | String extractPayload(Message message) {
30 | if (messageType.equals(EsSinkMessageType.PROTOBUF)) {
31 | return getFieldFromJSON(jsonSerializer.serialize(message), "logMessage");
32 | }
33 | return new String(message.getLogMessage(), Charset.defaultCharset());
34 | }
35 |
36 | String getFieldFromJSON(String jsonString, String key) {
37 | try {
38 | JSONObject parse = (JSONObject) jsonParser.parse(jsonString);
39 | Object valueAtKey = parse.get(key);
40 | if (valueAtKey == null) {
41 | throw new IllegalArgumentException("Key: " + key + " not found in ESB Message");
42 | }
43 | return valueAtKey.toString();
44 | } catch (ParseException e) {
45 | throw new JsonParseException(e.getMessage(), e.getCause());
46 | } finally {
47 | jsonParser.reset();
48 | }
49 | }
50 |
51 | }
52 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/elasticsearch/request/EsRequestHandlerFactory.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.elasticsearch.request;
2 |
3 | import org.raystack.firehose.config.EsSinkConfig;
4 | import org.raystack.firehose.config.enums.EsSinkMessageType;
5 | import org.raystack.firehose.config.enums.EsSinkRequestType;
6 | import org.raystack.firehose.metrics.FirehoseInstrumentation;
7 | import org.raystack.firehose.serializer.MessageToJson;
8 | import lombok.AllArgsConstructor;
9 |
10 | import java.util.ArrayList;
11 |
12 | @AllArgsConstructor
13 | public class EsRequestHandlerFactory {
14 |
15 | private EsSinkConfig esSinkConfig;
16 | private FirehoseInstrumentation firehoseInstrumentation;
17 | private final String esIdFieldName;
18 | private final EsSinkMessageType messageType;
19 | private final MessageToJson jsonSerializer;
20 | private final String esTypeName;
21 | private final String esIndexName;
22 | private final String esRoutingKeyName;
23 |
24 | public EsRequestHandler getRequestHandler() {
25 | EsSinkRequestType esSinkRequestType = esSinkConfig.isSinkEsModeUpdateOnlyEnable() ? EsSinkRequestType.UPDATE_ONLY : EsSinkRequestType.INSERT_OR_UPDATE;
26 | firehoseInstrumentation.logInfo("ES request mode: {}", esSinkRequestType);
27 |
28 | ArrayList esRequestHandlers = new ArrayList<>();
29 | esRequestHandlers.add(new EsUpdateRequestHandler(messageType, jsonSerializer, esTypeName, esIndexName, esSinkRequestType, esIdFieldName, esRoutingKeyName));
30 | esRequestHandlers.add(new EsUpsertRequestHandler(messageType, jsonSerializer, esTypeName, esIndexName, esSinkRequestType, esIdFieldName, esRoutingKeyName));
31 |
32 | return esRequestHandlers
33 | .stream()
34 | .filter(EsRequestHandler::canCreate)
35 | .findFirst()
36 | .orElseThrow(() -> new IllegalArgumentException("Es Request Type " + esSinkRequestType.name() + " not supported"));
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/elasticsearch/request/EsUpdateRequestHandler.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.elasticsearch.request;
2 |
3 | import org.raystack.firehose.config.enums.EsSinkMessageType;
4 | import org.raystack.firehose.config.enums.EsSinkRequestType;
5 | import org.raystack.firehose.message.Message;
6 | import org.raystack.firehose.serializer.MessageToJson;
7 | import org.apache.commons.lang.StringUtils;
8 | import org.elasticsearch.action.DocWriteRequest;
9 | import org.elasticsearch.action.update.UpdateRequest;
10 | import org.elasticsearch.common.xcontent.XContentType;
11 |
12 | public class EsUpdateRequestHandler extends EsRequestHandler {
13 |
14 | private final String esTypeName;
15 | private final String esIndexName;
16 | private EsSinkRequestType esSinkRequestType;
17 | private String esIdFieldName;
18 | private String esRoutingKeyName;
19 |
20 | public EsUpdateRequestHandler(EsSinkMessageType messageType, MessageToJson jsonSerializer, String esTypeName, String esIndexName, EsSinkRequestType esSinkRequestType, String esIdFieldName, String esRoutingKeyName) {
21 | super(messageType, jsonSerializer);
22 | this.esTypeName = esTypeName;
23 | this.esIndexName = esIndexName;
24 | this.esSinkRequestType = esSinkRequestType;
25 | this.esIdFieldName = esIdFieldName;
26 | this.esRoutingKeyName = esRoutingKeyName;
27 | }
28 |
29 | @Override
30 | public boolean canCreate() {
31 | return esSinkRequestType == EsSinkRequestType.UPDATE_ONLY;
32 | }
33 |
34 | public DocWriteRequest getRequest(Message message) {
35 | String logMessage = extractPayload(message);
36 | UpdateRequest request = new UpdateRequest(esIndexName, esTypeName, getFieldFromJSON(logMessage, esIdFieldName));
37 | if (StringUtils.isNotEmpty(esRoutingKeyName)) {
38 | request.routing(getFieldFromJSON(logMessage, esRoutingKeyName));
39 | }
40 | request.doc(logMessage, XContentType.JSON);
41 | return request;
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/elasticsearch/request/EsUpsertRequestHandler.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.elasticsearch.request;
2 |
3 | import org.raystack.firehose.config.enums.EsSinkMessageType;
4 | import org.raystack.firehose.config.enums.EsSinkRequestType;
5 | import org.raystack.firehose.message.Message;
6 | import org.raystack.firehose.serializer.MessageToJson;
7 | import org.apache.commons.lang.StringUtils;
8 | import org.elasticsearch.action.DocWriteRequest;
9 | import org.elasticsearch.action.index.IndexRequest;
10 | import org.elasticsearch.common.xcontent.XContentType;
11 |
12 | public class EsUpsertRequestHandler extends EsRequestHandler {
13 | private final String esTypeName;
14 | private final String esIndexName;
15 | private EsSinkRequestType esSinkRequestType;
16 | private String esIdFieldName;
17 | private String esRoutingKeyName;
18 |
19 | public EsUpsertRequestHandler(EsSinkMessageType messageType, MessageToJson jsonSerializer, String esTypeName, String esIndexName, EsSinkRequestType esSinkRequestType, String esIdFieldName, String esRoutingKeyName) {
20 | super(messageType, jsonSerializer);
21 | this.esTypeName = esTypeName;
22 | this.esIndexName = esIndexName;
23 | this.esSinkRequestType = esSinkRequestType;
24 | this.esIdFieldName = esIdFieldName;
25 | this.esRoutingKeyName = esRoutingKeyName;
26 | }
27 |
28 | @Override
29 | public boolean canCreate() {
30 | return esSinkRequestType == EsSinkRequestType.INSERT_OR_UPDATE;
31 | }
32 |
33 | public DocWriteRequest getRequest(Message message) {
34 | String logMessage = extractPayload(message);
35 | IndexRequest request = new IndexRequest(esIndexName, esTypeName, getFieldFromJSON(logMessage, esIdFieldName));
36 | if (StringUtils.isNotEmpty(esRoutingKeyName)) {
37 | request.routing(getFieldFromJSON(logMessage, esRoutingKeyName));
38 | }
39 | request.source(logMessage, XContentType.JSON);
40 | return request;
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/http/auth/OAuth2AccessToken.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.http.auth;
2 |
3 | import org.joda.time.DateTimeUtils;
4 |
5 | public class OAuth2AccessToken {
6 | private final String accessToken;
7 | private final Long expirationTimeMs;
8 | private static final int DEFAULT_EXPIRATION_TIME = 3600;
9 | private static final long MILLIS = 1000L;
10 |
11 | public OAuth2AccessToken(String accessToken, Integer expiresIn) {
12 | this.accessToken = accessToken;
13 | expiresIn = expiresIn == null ? DEFAULT_EXPIRATION_TIME : expiresIn;
14 | this.expirationTimeMs = DateTimeUtils.currentTimeMillis() + (expiresIn * MILLIS);
15 | }
16 |
17 | public boolean isExpired() {
18 | final long oneMinute = 60L;
19 | return this.getExpiresIn() <= oneMinute;
20 | }
21 |
22 | public String toString() {
23 | return this.accessToken;
24 | }
25 |
26 | public Long getExpiresIn() {
27 | return (this.expirationTimeMs - DateTimeUtils.currentTimeMillis()) / MILLIS;
28 | }
29 | }
30 |
31 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/http/request/HttpRequestMethodFactory.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.http.request;
2 |
3 | import org.raystack.firehose.config.enums.HttpSinkRequestMethodType;
4 | import org.raystack.firehose.sink.http.request.method.HttpDeleteWithBody;
5 | import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
6 | import org.apache.http.client.methods.HttpPatch;
7 | import org.apache.http.client.methods.HttpPost;
8 | import org.apache.http.client.methods.HttpPut;
9 |
10 | import java.net.URI;
11 |
12 | /**
13 | * The type Http request method factory.
14 | */
15 | public class HttpRequestMethodFactory {
16 | /**
17 | * Create http entity enclosing request base.
18 | *
19 | * @param uri the uri
20 | * @param method the method
21 | * @return the http entity enclosing request base
22 | */
23 | public static HttpEntityEnclosingRequestBase create(URI uri, HttpSinkRequestMethodType method) {
24 | switch (method) {
25 | case POST:
26 | return new HttpPost(uri);
27 | case PATCH:
28 | return new HttpPatch(uri);
29 | case DELETE:
30 | return new HttpDeleteWithBody(uri);
31 | default:
32 | return new HttpPut(uri);
33 | }
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/http/request/body/JsonBody.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.http.request.body;
2 |
3 | import java.util.ArrayList;
4 | import java.util.List;
5 |
6 | import org.raystack.firehose.exception.DeserializerException;
7 | import org.raystack.firehose.message.Message;
8 | import org.raystack.firehose.serializer.MessageSerializer;
9 |
10 | /**
11 | * JsonBody Serialize the message according to injected serialzier and return it
12 | * as List of serialized string.
13 | */
14 | public class JsonBody {
15 |
16 | private MessageSerializer jsonSerializer;
17 |
18 | /**
19 | * Instantiates a new Json body.
20 | *
21 | * @param jsonSerializer the json serializer
22 | */
23 | public JsonBody(MessageSerializer jsonSerializer) {
24 | this.jsonSerializer = jsonSerializer;
25 | }
26 |
27 | /**
28 | * Serialize list.
29 | *
30 | * @param messages the messages
31 | * @return the list
32 | * @throws DeserializerException the deserializer exception
33 | */
34 | public List serialize(List messages) throws DeserializerException {
35 | List serializedBody = new ArrayList();
36 | for (Message message : messages) {
37 | serializedBody.add(jsonSerializer.serialize(message));
38 | }
39 | return serializedBody;
40 | }
41 |
42 | }
43 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/http/request/create/RequestCreator.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.http.request.create;
2 |
3 | import org.raystack.firehose.message.Message;
4 | import org.raystack.firehose.sink.http.request.entity.RequestEntityBuilder;
5 | import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
6 |
7 | import java.net.URISyntaxException;
8 | import java.util.List;
9 |
10 | /**
11 | * Creates http requests.
12 | */
13 | public interface RequestCreator {
14 |
15 | List create(List bodyContents, RequestEntityBuilder entity) throws URISyntaxException;
16 | }
17 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/http/request/entity/RequestEntityBuilder.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.http.request.entity;
2 |
3 | import org.raystack.firehose.exception.DeserializerException;
4 | import org.apache.http.entity.ContentType;
5 | import org.apache.http.entity.StringEntity;
6 |
7 | import java.util.Collections;
8 |
9 | /**
10 | * Request entity builder.
11 | */
12 | public class RequestEntityBuilder {
13 | private boolean wrapArray;
14 |
15 | /**
16 | * Instantiates a new Request entity builder.
17 | */
18 | public RequestEntityBuilder() {
19 | this.wrapArray = false;
20 | }
21 |
22 | public RequestEntityBuilder setWrapping(boolean isArrayWrap) {
23 | this.wrapArray = isArrayWrap;
24 | return this;
25 | }
26 |
27 | /**
28 | * Build http entity string entity.
29 | *
30 | * @param bodyContent the body content
31 | * @return the string entity
32 | * @throws DeserializerException the deserializer exception
33 | */
34 | public StringEntity buildHttpEntity(String bodyContent) throws DeserializerException {
35 | if (!wrapArray) {
36 | return new StringEntity(bodyContent, ContentType.APPLICATION_JSON);
37 | } else {
38 | String arrayWrappedBody = Collections.singletonList(bodyContent).toString();
39 | return new StringEntity(arrayWrappedBody, ContentType.APPLICATION_JSON);
40 | }
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/http/request/method/HttpDeleteWithBody.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.http.request.method;
2 |
3 | import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
4 |
5 | import java.net.URI;
6 |
7 | public class HttpDeleteWithBody extends HttpEntityEnclosingRequestBase {
8 | public static final String METHOD_NAME = "DELETE";
9 |
10 | @Override
11 | public String getMethod() {
12 | return METHOD_NAME;
13 | }
14 |
15 | public HttpDeleteWithBody(final URI uri) {
16 | super();
17 | setURI(uri);
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/http/request/uri/UriBuilder.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.http.request.uri;
2 |
3 | import org.raystack.firehose.config.enums.HttpSinkParameterSourceType;
4 | import org.raystack.firehose.message.Message;
5 | import org.raystack.firehose.proto.ProtoToFieldMapper;
6 |
7 | import java.net.URI;
8 | import java.net.URISyntaxException;
9 | import java.util.Map;
10 |
11 | /**
12 | * Builds URI based on the requirement.
13 | */
14 | public class UriBuilder {
15 |
16 | private String baseURL;
17 | private UriParser uriParser;
18 | private ProtoToFieldMapper protoToFieldMapper;
19 | private HttpSinkParameterSourceType httpSinkParameterSourceType;
20 |
21 | public UriBuilder(String baseURL, UriParser uriParser) {
22 | this.baseURL = baseURL;
23 | this.uriParser = uriParser;
24 | }
25 |
26 | public URI build() throws URISyntaxException {
27 | return new URI(baseURL);
28 | }
29 |
30 | public URI build(Message message) throws URISyntaxException {
31 | String url = uriParser.parse(message, baseURL);
32 | org.apache.http.client.utils.URIBuilder uriBuilder = new org.apache.http.client.utils.URIBuilder(url);
33 | if (protoToFieldMapper == null) {
34 | return uriBuilder.build();
35 | }
36 |
37 | // flow for parameterized URI
38 | Map paramMap = protoToFieldMapper
39 | .getFields((httpSinkParameterSourceType == HttpSinkParameterSourceType.KEY) ? message.getLogKey()
40 | : message.getLogMessage());
41 | paramMap.forEach((string, object) -> uriBuilder.addParameter(string, object.toString()));
42 | return uriBuilder.build();
43 | }
44 |
45 | public UriBuilder withParameterizedURI(ProtoToFieldMapper protoToFieldmapper, HttpSinkParameterSourceType httpSinkParameterSource) {
46 | this.protoToFieldMapper = protoToFieldmapper;
47 | this.httpSinkParameterSourceType = httpSinkParameterSource;
48 | return this;
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/influxdb/InfluxSinkFactory.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.influxdb;
2 |
3 |
4 | import org.raystack.firehose.config.InfluxSinkConfig;
5 | import org.raystack.firehose.metrics.FirehoseInstrumentation;
6 | import org.raystack.depot.metrics.StatsDReporter;
7 | import org.raystack.firehose.sink.AbstractSink;
8 | import org.raystack.stencil.client.StencilClient;
9 | import org.aeonbits.owner.ConfigFactory;
10 | import org.influxdb.InfluxDB;
11 | import org.influxdb.InfluxDBFactory;
12 |
13 | import java.util.Map;
14 |
15 | /**
16 | * Influx sink factory create influx sink.
17 | */
18 | public class InfluxSinkFactory {
19 | /**
20 | * Create Influx sink.
21 | *
22 | * @param configProperties the config properties
23 | * @param statsDReporter the statsd reporter
24 | * @param stencilClient the stencil client
25 | * @return Influx sink
26 | */
27 | public static AbstractSink create(Map configProperties, StatsDReporter statsDReporter, StencilClient stencilClient) {
28 | InfluxSinkConfig config = ConfigFactory.create(InfluxSinkConfig.class, configProperties);
29 |
30 | FirehoseInstrumentation firehoseInstrumentation = new FirehoseInstrumentation(statsDReporter, InfluxSinkFactory.class);
31 | firehoseInstrumentation.logDebug("\nInflux Url: {}\nInflux Username: {}", config.getSinkInfluxUrl(), config.getSinkInfluxUsername());
32 |
33 | InfluxDB client = InfluxDBFactory.connect(config.getSinkInfluxUrl(), config.getSinkInfluxUsername(), config.getSinkInfluxPassword());
34 | firehoseInstrumentation.logInfo("InfluxDB connection established");
35 |
36 | return new InfluxSink(new FirehoseInstrumentation(statsDReporter, InfluxSink.class), "influx.db", config, stencilClient.getParser(config.getInputSchemaProtoClass()), client, stencilClient);
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/jdbc/JdbcConnectionPool.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.jdbc;
2 |
3 | import java.sql.Connection;
4 | import java.sql.SQLException;
5 |
6 | /**
7 | * Interface for database connection pool.
8 | */
9 | public interface JdbcConnectionPool {
10 | /**
11 | * Get connection from the pool.
12 | *
13 | * @return the connection
14 | * @throws SQLException the sql exception
15 | */
16 | Connection getConnection() throws SQLException;
17 |
18 | /**
19 | * Release the connection held.
20 | *
21 | * @param connection the connection
22 | */
23 | void release(Connection connection) throws SQLException;
24 |
25 | /**
26 | * Shutdown the connection pool.
27 | *
28 | * @throws InterruptedException the interrupted exception
29 | */
30 | void shutdown() throws InterruptedException;
31 | }
32 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcDefaultField.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.jdbc.field;
2 |
3 | public class JdbcDefaultField implements JdbcField {
4 | private Object columnValue;
5 |
6 | public JdbcDefaultField(Object columnValue) {
7 | this.columnValue = columnValue;
8 | }
9 |
10 | @Override
11 | public Object getColumn() throws RuntimeException {
12 | return columnValue;
13 | }
14 |
15 | @Override
16 | public boolean canProcess() {
17 | return false;
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcField.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.jdbc.field;
2 |
3 | /**
4 | * Jdbc field for JDBC sink.
5 | */
6 | public interface JdbcField {
7 | /**
8 | * Get column value for the field.
9 | *
10 | * @return the column
11 | * @throws RuntimeException the runtime exception
12 | */
13 | Object getColumn() throws RuntimeException;
14 |
15 | /**
16 | * @return true if this jdbc field is set by config
17 | */
18 | boolean canProcess();
19 | }
20 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcFieldFactory.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.jdbc.field;
2 |
3 | import org.raystack.firehose.sink.jdbc.field.message.JdbcDefaultMessageField;
4 | import org.raystack.firehose.sink.jdbc.field.message.JdbcCollectionField;
5 | import org.raystack.firehose.sink.jdbc.field.message.JdbcTimestampField;
6 | import com.google.protobuf.Descriptors;
7 |
8 | import java.util.Arrays;
9 | import java.util.List;
10 |
11 | /**
12 | * Jdbc field factory.
13 | */
14 | public class JdbcFieldFactory {
15 | /**
16 | * Returns the field based on configuration.
17 | *
18 | * @param columnValue the column value
19 | * @param fieldDescriptor the field descriptor
20 | * @return the field
21 | */
22 | public static JdbcField getField(Object columnValue, Descriptors.FieldDescriptor fieldDescriptor) {
23 | List jdbcFields = Arrays.asList(
24 | new JdbcCollectionField(columnValue, fieldDescriptor),
25 | new JdbcMapField(columnValue, fieldDescriptor),
26 | new JdbcTimestampField(columnValue),
27 | new JdbcDefaultMessageField(columnValue));
28 | return jdbcFields.stream()
29 | .filter(JdbcField::canProcess)
30 | .findFirst()
31 | .orElseGet(() -> new JdbcDefaultField(columnValue));
32 | }
33 |
34 | }
35 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/jdbc/field/JdbcMapField.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.jdbc.field;
2 |
3 | import com.google.protobuf.Descriptors;
4 | import com.google.protobuf.DynamicMessage;
5 | import org.json.simple.JSONObject;
6 |
7 | import java.util.HashMap;
8 | import java.util.List;
9 |
10 | /**
11 | * Jdbc map field.
12 | */
13 | public class JdbcMapField implements JdbcField {
14 | private Object columnValue;
15 | private Descriptors.FieldDescriptor fieldDescriptor;
16 |
17 |
18 | public JdbcMapField(Object columnValue, Descriptors.FieldDescriptor fieldDescriptor) {
19 | this.columnValue = columnValue;
20 | this.fieldDescriptor = fieldDescriptor;
21 | }
22 |
23 | @Override
24 | public Object getColumn() throws RuntimeException {
25 | HashMap columnFields = new HashMap<>();
26 | List values = (List) this.columnValue;
27 | for (DynamicMessage dynamicMessage : values) {
28 | Object[] data = dynamicMessage.getAllFields().values().toArray();
29 | Object mapValue = data.length > 1 ? data[1] : "";
30 | columnFields.put((String) data[0], mapValue);
31 | }
32 | String columnEntry = JSONObject.toJSONString(columnFields);
33 | return columnEntry;
34 | }
35 |
36 | @Override
37 | public boolean canProcess() {
38 | return fieldDescriptor.isMapField();
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/jdbc/field/message/JdbcCollectionField.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.jdbc.field.message;
2 |
3 | import org.raystack.firehose.sink.jdbc.field.JdbcField;
4 | import com.google.gson.GsonBuilder;
5 | import com.google.protobuf.Descriptors;
6 | import com.google.protobuf.Message;
7 |
8 | import java.util.Collection;
9 | import java.util.Optional;
10 | import java.util.stream.Collectors;
11 |
12 | public class JdbcCollectionField implements JdbcField {
13 | private Object columnValue;
14 | private Descriptors.FieldDescriptor fieldDescriptor;
15 |
16 | public JdbcCollectionField(Object columnValue, Descriptors.FieldDescriptor fieldDescriptor) {
17 | this.columnValue = columnValue;
18 | this.fieldDescriptor = fieldDescriptor;
19 | }
20 |
21 | @Override
22 | public Object getColumn() throws RuntimeException {
23 | Collection collectionOfMessages = (Collection) columnValue;
24 | Optional first = collectionOfMessages.stream().findFirst();
25 | if (first.isPresent() && first.get() instanceof Message) {
26 | Object messageJsons = collectionOfMessages
27 | .stream()
28 | .map(cValue -> new JdbcDefaultMessageField(cValue).getColumn().toString())
29 | .collect(Collectors.joining(","));
30 | return "[" + messageJsons + "]";
31 | } else {
32 | return new GsonBuilder().create().toJson(collectionOfMessages);
33 | }
34 | }
35 |
36 | @Override
37 | public boolean canProcess() {
38 | return columnValue instanceof Collection && !fieldDescriptor.isMapField();
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/jdbc/field/message/JdbcDefaultMessageField.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.jdbc.field.message;
2 |
3 | import org.raystack.firehose.sink.jdbc.field.JdbcField;
4 | import com.google.protobuf.InvalidProtocolBufferException;
5 | import com.google.protobuf.Message;
6 | import com.google.protobuf.util.JsonFormat;
7 |
8 | public class JdbcDefaultMessageField implements JdbcField {
9 | private Object columnValue;
10 | private JsonFormat.Printer jsonPrinter = JsonFormat.printer()
11 | .omittingInsignificantWhitespace()
12 | .preservingProtoFieldNames()
13 | .includingDefaultValueFields();
14 |
15 | public JdbcDefaultMessageField(Object columnValue) {
16 | this.columnValue = columnValue;
17 | }
18 |
19 | @Override
20 | public Object getColumn() throws RuntimeException {
21 | try {
22 | columnValue = this.jsonPrinter.print((Message) columnValue);
23 | } catch (InvalidProtocolBufferException e) {
24 | throw new RuntimeException(e);
25 | }
26 | return columnValue;
27 | }
28 |
29 | @Override
30 | public boolean canProcess() {
31 | return columnValue instanceof Message;
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/src/main/java/org/raystack/firehose/sink/jdbc/field/message/JdbcTimestampField.java:
--------------------------------------------------------------------------------
1 | package org.raystack.firehose.sink.jdbc.field.message;
2 |
3 | import org.raystack.firehose.sink.jdbc.field.JdbcField;
4 | import com.google.protobuf.Descriptors;
5 | import com.google.protobuf.DynamicMessage;
6 | import com.google.protobuf.Timestamp;
7 |
8 | import java.time.Instant;
9 | import java.util.ArrayList;
10 | import java.util.List;
11 |
12 | public class JdbcTimestampField implements JdbcField {
13 | private Object columnValue;
14 |
15 | public JdbcTimestampField(Object columnValue) {
16 | this.columnValue = columnValue;
17 | }
18 |
19 | @Override
20 | public Object getColumn() {
21 | List fieldDescriptors = ((DynamicMessage) columnValue).getDescriptorForType().getFields();
22 | ArrayList