├── .github └── workflows │ ├── build.yml │ └── package.yml ├── .gitignore ├── LICENSE ├── README.md ├── build.gradle ├── config └── checkstyle │ ├── checkstyle.xml │ └── suppressions.xml ├── docs ├── README.md ├── SUMMARY.md ├── contribute │ ├── contribution.md │ └── development.md ├── images │ └── bigquery-json-flow-diagram.svg ├── reference │ ├── configuration │ │ ├── README.md │ │ ├── bigquery-sink.md │ │ ├── bigtable.md │ │ ├── generic.md │ │ ├── redis.md │ │ └── stencil-client.md │ ├── metrics.md │ └── sink_response.md ├── roadmap.md └── sinks │ ├── bigquery.md │ ├── bigtable.md │ └── redis.md ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle └── src ├── main └── java │ └── org │ └── raystack │ └── depot │ ├── Sink.java │ ├── SinkResponse.java │ ├── bigquery │ ├── BigQuerySink.java │ ├── BigQuerySinkFactory.java │ ├── BigQueryStorageAPISink.java │ ├── BigqueryStencilUpdateListenerFactory.java │ ├── client │ │ ├── BigQueryClient.java │ │ ├── BigQueryResponseParser.java │ │ ├── BigQueryRow.java │ │ ├── BigQueryRowWithInsertId.java │ │ ├── BigQueryRowWithoutInsertId.java │ │ └── BigQueryTableDefinition.java │ ├── converter │ │ ├── MessageRecordConverter.java │ │ ├── MessageRecordConverterCache.java │ │ └── MessageRecordConverterUtils.java │ ├── error │ │ ├── ErrorDescriptor.java │ │ ├── ErrorParser.java │ │ ├── InvalidSchemaError.java │ │ ├── OOBError.java │ │ ├── StoppedError.java │ │ └── UnknownError.java │ ├── exception │ │ ├── BQClusteringKeysException.java │ │ ├── BQDatasetLocationChangedException.java │ │ ├── BQPartitionKeyNotSpecified.java │ │ ├── BQSchemaMappingException.java │ │ ├── BQTableUpdateFailure.java │ │ └── BigQuerySinkException.java │ ├── handler │ │ ├── ErrorHandler.java │ │ ├── ErrorHandlerFactory.java │ │ ├── JsonErrorHandler.java │ │ └── NoopErrorHandler.java │ ├── json │ │ └── BigqueryJsonUpdateListener.java │ ├── models │ │ ├── BQField.java │ │ ├── Record.java │ │ └── Records.java │ ├── proto │ │ ├── BigqueryFields.java │ │ └── BigqueryProtoUpdateListener.java │ └── storage │ │ ├── BigQueryPayload.java │ │ ├── BigQueryStorageClient.java │ │ ├── BigQueryStorageClientFactory.java │ │ ├── BigQueryStorageResponseParser.java │ │ ├── BigQueryStream.java │ │ ├── BigQueryWriter.java │ │ ├── BigQueryWriterFactory.java │ │ ├── BigQueryWriterUtils.java │ │ ├── json │ │ ├── BigQueryJsonStream.java │ │ └── BigQueryJsonWriter.java │ │ └── proto │ │ ├── BigQueryProtoStorageClient.java │ │ ├── BigQueryProtoStream.java │ │ ├── BigQueryProtoUtils.java │ │ ├── BigQueryProtoWriter.java │ │ ├── BigQueryRecordMeta.java │ │ └── TimeStampUtils.java │ ├── bigtable │ ├── BigTableSink.java │ ├── BigTableSinkFactory.java │ ├── client │ │ └── BigTableClient.java │ ├── exception │ │ └── BigTableInvalidSchemaException.java │ ├── model │ │ ├── BigTableRecord.java │ │ └── BigTableSchema.java │ ├── parser │ │ ├── BigTableRecordParser.java │ │ ├── BigTableResponseParser.java │ │ └── BigTableRowKeyParser.java │ └── response │ │ └── BigTableResponse.java │ ├── common │ ├── Function3.java │ ├── Template.java │ ├── Tuple.java │ └── TupleString.java │ ├── config │ ├── BigQuerySinkConfig.java │ ├── BigTableSinkConfig.java │ ├── MetricsConfig.java │ ├── RedisSinkConfig.java │ ├── SinkConfig.java │ ├── converter │ │ ├── ConfToListConverter.java │ │ ├── ConverterUtils.java │ │ ├── JsonToPropertiesConverter.java │ │ ├── LabelMapConverter.java │ │ ├── RedisSinkDataTypeConverter.java │ │ ├── RedisSinkDeploymentTypeConverter.java │ │ ├── RedisSinkTtlTypeConverter.java │ │ ├── SchemaRegistryHeadersConverter.java │ │ ├── SchemaRegistryRefreshConverter.java │ │ ├── SinkConnectorSchemaDataTypeConverter.java │ │ └── SinkConnectorSchemaMessageModeConverter.java │ └── enums │ │ └── SinkConnectorSchemaDataType.java │ ├── error │ ├── ErrorInfo.java │ └── ErrorType.java │ ├── exception │ ├── ConfigurationException.java │ ├── DeserializerException.java │ ├── EmptyMessageException.java │ ├── InvalidTemplateException.java │ ├── ProtoNotFoundException.java │ ├── SinkException.java │ └── UnknownFieldsException.java │ ├── log │ ├── LogSink.java │ └── LogSinkFactory.java │ ├── message │ ├── Message.java │ ├── MessageParser.java │ ├── MessageParserFactory.java │ ├── MessageSchema.java │ ├── MessageUtils.java │ ├── ParsedMessage.java │ ├── SinkConnectorSchemaMessageMode.java │ ├── field │ │ ├── FieldUtils.java │ │ ├── GenericField.java │ │ ├── GenericFieldFactory.java │ │ ├── json │ │ │ ├── DefaultField.java │ │ │ └── JsonFieldFactory.java │ │ └── proto │ │ │ ├── DefaultField.java │ │ │ ├── DurationField.java │ │ │ ├── MapField.java │ │ │ ├── MessageField.java │ │ │ ├── ProtoFieldFactory.java │ │ │ ├── StructField.java │ │ │ └── TimeStampField.java │ ├── json │ │ ├── JsonMessageParser.java │ │ └── JsonParsedMessage.java │ └── proto │ │ ├── Constants.java │ │ ├── DescriptorCache.java │ │ ├── ProtoField.java │ │ ├── ProtoFieldParser.java │ │ ├── ProtoMapper.java │ │ ├── ProtoMessageParser.java │ │ ├── ProtoMessageSchema.java │ │ ├── ProtoParsedMessage.java │ │ ├── UnknownProtoFields.java │ │ └── converter │ │ └── fields │ │ ├── ByteProtoField.java │ │ ├── DefaultProtoField.java │ │ ├── DurationProtoField.java │ │ ├── EnumProtoField.java │ │ ├── FloatProtoField.java │ │ ├── IntegerProtoField.java │ │ ├── MapProtoField.java │ │ ├── MessageProtoField.java │ │ ├── ProtoField.java │ │ ├── ProtoFieldFactory.java │ │ ├── StructProtoField.java │ │ └── TimestampProtoField.java │ ├── metrics │ ├── BigQueryMetrics.java │ ├── BigTableMetrics.java │ ├── Instrumentation.java │ ├── JsonParserMetrics.java │ ├── SinkMetrics.java │ ├── StatsDReporter.java │ └── StatsDReporterBuilder.java │ ├── redis │ ├── RedisSink.java │ ├── RedisSinkFactory.java │ ├── client │ │ ├── RedisClient.java │ │ ├── RedisClientFactory.java │ │ ├── RedisClusterClient.java │ │ ├── RedisStandaloneClient.java │ │ ├── entry │ │ │ ├── RedisEntry.java │ │ │ ├── RedisHashSetFieldEntry.java │ │ │ ├── RedisKeyValueEntry.java │ │ │ └── RedisListEntry.java │ │ └── response │ │ │ ├── RedisClusterResponse.java │ │ │ ├── RedisResponse.java │ │ │ └── RedisStandaloneResponse.java │ ├── enums │ │ ├── RedisSinkDataType.java │ │ ├── RedisSinkDeploymentType.java │ │ └── RedisSinkTtlType.java │ ├── parsers │ │ ├── RedisEntryParser.java │ │ ├── RedisEntryParserFactory.java │ │ ├── RedisHashSetEntryParser.java │ │ ├── RedisKeyValueEntryParser.java │ │ ├── RedisListEntryParser.java │ │ └── RedisParser.java │ ├── record │ │ └── RedisRecord.java │ ├── ttl │ │ ├── DurationTtl.java │ │ ├── ExactTimeTtl.java │ │ ├── NoRedisTtl.java │ │ ├── RedisTTLFactory.java │ │ └── RedisTtl.java │ └── util │ │ └── RedisSinkUtils.java │ ├── stencil │ └── DepotStencilUpdateListener.java │ └── utils │ ├── DateUtils.java │ ├── JsonUtils.java │ ├── MessageConfigUtils.java │ ├── ProtoUtils.java │ ├── StencilUtils.java │ └── StringUtils.java └── test ├── java └── org │ └── raystack │ └── depot │ ├── bigquery │ ├── BigQuerySinkTest.java │ ├── TestMessageBuilder.java │ ├── TestMetadata.java │ ├── client │ │ ├── BigQueryClientTest.java │ │ ├── BigQueryResponseParserTest.java │ │ ├── BigQueryRowWithInsertIdTest.java │ │ ├── BigQueryRowWithoutInsertIdTest.java │ │ └── BigQueryTableDefinitionTest.java │ ├── converter │ │ ├── MessageRecordConverterForJsonTest.java │ │ ├── MessageRecordConverterTest.java │ │ └── MessageRecordConverterUtilsTest.java │ ├── handler │ │ └── JsonErrorHandlerTest.java │ ├── json │ │ └── BigqueryJsonUpdateListenerTest.java │ ├── models │ │ ├── BQFieldTest.java │ │ └── ProtoFieldTest.java │ ├── proto │ │ ├── BigqueryFieldsTest.java │ │ └── BigqueryProtoUpdateListenerTest.java │ └── storage │ │ ├── BigQueryStorageResponseParserTest.java │ │ ├── BigQueryWriterUtilsTest.java │ │ └── proto │ │ ├── BigQueryProtoStorageClientTest.java │ │ └── BigQueryProtoWriterTest.java │ ├── bigtable │ ├── BigTableSinkTest.java │ ├── client │ │ └── BigTableClientTest.java │ ├── model │ │ └── BigTableSchemaTest.java │ └── parser │ │ ├── BigTableRecordParserTest.java │ │ ├── BigTableResponseParserTest.java │ │ └── BigTableRowKeyParserTest.java │ ├── common │ └── TemplateTest.java │ ├── config │ ├── BigQuerySinkConfigTest.java │ ├── RedisSinkConfigTest.java │ └── converter │ │ ├── ConfToListConverterTest.java │ │ ├── ConverterUtilsTest.java │ │ ├── JsonToPropertiesConverterTest.java │ │ ├── LabelMapConverterTest.java │ │ ├── RedisSinkDataTypeConverterTest.java │ │ ├── RedisSinkDeploymentTypeConverterTest.java │ │ ├── RedisSinkTtlTypeConverterTest.java │ │ ├── SchemaRegistryHeadersConverterTest.java │ │ └── SinkConnectorSchemaMessageModeConverterTest.java │ ├── log │ └── LogSinkTest.java │ ├── message │ ├── MessageUtilsTest.java │ ├── field │ │ └── proto │ │ │ ├── DefaultFieldTest.java │ │ │ ├── DurationFieldTest.java │ │ │ ├── MapFieldTest.java │ │ │ ├── MessageFieldTest.java │ │ │ └── TimeStampFieldTest.java │ ├── json │ │ ├── JsonMessageParserTest.java │ │ └── JsonParsedMessageTest.java │ └── proto │ │ ├── ProtoFieldParserTest.java │ │ ├── ProtoMapperTest.java │ │ ├── ProtoMessageParserTest.java │ │ ├── ProtoParsedMessageTest.java │ │ ├── TestProtoUtil.java │ │ ├── UnknownProtoFieldsTest.java │ │ └── converter │ │ └── fields │ │ ├── ByteProtoFieldTest.java │ │ ├── DefaultProtoFieldTest.java │ │ ├── EnumProtoFieldTest.java │ │ ├── MessageProtoFieldTest.java │ │ ├── StructProtoFieldTest.java │ │ └── TimestampProtoFieldTest.java │ ├── metrics │ ├── InstrumentationTest.java │ └── MetricsTest.java │ ├── redis │ ├── RedisSinkTest.java │ ├── client │ │ ├── RedisClientFactoryTest.java │ │ ├── RedisClusterClientTest.java │ │ ├── RedisStandaloneClientTest.java │ │ ├── entry │ │ │ ├── RedisHashSetFieldEntryTest.java │ │ │ ├── RedisKeyValueEntryTest.java │ │ │ └── RedisListEntryTest.java │ │ └── response │ │ │ ├── RedisClusterResponseTest.java │ │ │ └── RedisStandaloneResponseTest.java │ ├── parsers │ │ ├── RedisEntryParserFactoryTest.java │ │ ├── RedisHashSetEntryParserTest.java │ │ ├── RedisKeyValueEntryParserTest.java │ │ ├── RedisListEntryParserTest.java │ │ └── RedisParserTest.java │ ├── record │ │ └── RedisRecordTest.java │ ├── ttl │ │ ├── DurationTTLTest.java │ │ ├── ExactTimeTTLTest.java │ │ └── RedisTtlFactoryTest.java │ └── util │ │ └── RedisSinkUtilsTest.java │ └── utils │ ├── JsonUtilsTest.java │ ├── ProtoUtilTest.java │ └── StringUtilsTest.java ├── proto ├── TestGrpc.proto ├── TestLogMessage.proto ├── TestMessage.proto └── TestMessageBQ.proto └── resources └── mockito-extensions └── org.mockito.plugins.MockMaker /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - main 8 | pull_request: 9 | workflow_dispatch: 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v2 16 | - name: Set up JDK 1.8 17 | uses: actions/setup-java@v1 18 | with: 19 | java-version: 1.8 20 | - name: Grant execute permission for gradlew 21 | run: chmod +x gradlew 22 | - name: Build 23 | run: ./gradlew build --stacktrace --no-daemon 24 | -------------------------------------------------------------------------------- /.github/workflows/package.yml: -------------------------------------------------------------------------------- 1 | name: Package 2 | 3 | on: 4 | release: 5 | types: [published] 6 | workflow_dispatch: 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Set up JDK 1.8 14 | uses: actions/setup-java@v1 15 | with: 16 | java-version: 1.8 17 | - name: Grant execute permission for gradlew 18 | run: chmod +x gradlew 19 | - name: Build 20 | run: ./gradlew build 21 | 22 | publish: 23 | needs: build 24 | runs-on: ubuntu-latest 25 | steps: 26 | - uses: actions/checkout@v2 27 | - name: Set up JDK 8 28 | uses: actions/setup-java@v2 29 | with: 30 | distribution: adopt 31 | java-version: 8 32 | - name: Publish depot 33 | run: | 34 | printf "$GPG_SIGNING_KEY" | base64 --decode > private.key 35 | ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -Psigning.keyId=${GPG_SIGNING_KEY_ID} -Psigning.password=${GPG_SIGNING_PASSWORD} -Psigning.secretKeyRingFile=private.key --console=verbose 36 | env: 37 | MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} 38 | MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} 39 | GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} 40 | GPG_SIGNING_KEY_ID: ${{ secrets.GPG_SIGNING_KEY_ID }} 41 | GPG_SIGNING_PASSWORD: ${{ secrets.GPG_SIGNING_PASSWORD }} 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.iml 3 | build 4 | out 5 | classes 6 | .project 7 | generated 8 | .gradle 9 | *.ipr 10 | *.iws 11 | .classpath 12 | .project 13 | .settings 14 | bin 15 | src/test/resources/__files 16 | -------------------------------------------------------------------------------- /config/checkstyle/suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 10 | 12 | 13 | -------------------------------------------------------------------------------- /docs/SUMMARY.md: -------------------------------------------------------------------------------- 1 | # Table of contents 2 | 3 | * [Introduction](README.md) 4 | * [Roadmap](roadmap.md) 5 | 6 | ## Sink 7 | * [Bigquery](sinks/bigquery.md) 8 | * [Redis](sinks/redis.md) 9 | 10 | ## Reference 11 | 12 | * [Configuration](reference/configuration/README.md) 13 | * [Generic](reference/configuration/generic.md) 14 | * [Stencil Client](reference/configuration/stencil-client.md) 15 | * [Bigquery Sink](reference/configuration/bigquery-sink.md) 16 | * [Metrics](reference/metrics.md) 17 | 18 | ## Contribute 19 | 20 | * [Contribution Process](contribute/contribution.md) 21 | * [Development Guide](contribute/development.md) 22 | 23 | -------------------------------------------------------------------------------- /docs/reference/configuration/README.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | This page contains reference for all the configurations for sink connectors. 4 | 5 | ## Table of Contents 6 | 7 | * [Generic](generic.md) 8 | * [Stencil Client](stencil-client.md) 9 | * [Bigquery Sink](bigquery-sink.md) 10 | * [Redis Sink](redis.md) 11 | * [Bigtable Sink](bigtable.md) 12 | 13 | -------------------------------------------------------------------------------- /docs/reference/configuration/generic.md: -------------------------------------------------------------------------------- 1 | # Generic 2 | 3 | All sinks require the following variables to be set 4 | 5 | ## `SINK_METRICS_APPLICATION_PREFIX` 6 | 7 | Application prefix for sink metrics. 8 | 9 | - Example value: `application_` 10 | - Type: `required` 11 | 12 | ## `SINK_CONNECTOR_SCHEMA_PROTO_MESSAGE_CLASS` 13 | 14 | Message log-message schema class 15 | 16 | - Example value: `org.raystack.schema.MessageClass` 17 | - Type: `required` 18 | 19 | ## `SINK_CONNECTOR_SCHEMA_PROTO_KEY_CLASS` 20 | 21 | Message log-key schema class 22 | 23 | - Example value: `org.raystack.schema.KeyClass` 24 | - Type: `required` 25 | 26 | ## `SINK_CONNECTOR_SCHEMA_DATA_TYPE` 27 | 28 | Message raw data type 29 | 30 | - Example value: `JSON` 31 | - Type: `required` 32 | - Default: `PROTOBUF` 33 | 34 | ## `SINK_CONNECTOR_SCHEMA_MESSAGE_MODE` 35 | 36 | The type of raw message to read from 37 | 38 | - Example value: `LOG_MESSAGE` 39 | - Type: `required` 40 | - Default: `LOG_MESSAGE` 41 | 42 | ## `SINK_CONNECTOR_SCHEMA_PROTO_ALLOW_UNKNOWN_FIELDS_ENABLE` 43 | 44 | Allow unknown fields in proto schema 45 | 46 | - Example value: `true` 47 | - Type: `required` 48 | - Default: `false` 49 | 50 | ## `METRIC_STATSD_HOST` 51 | 52 | URL of the StatsD host 53 | 54 | - Example value: `localhost` 55 | - Type: `optional` 56 | - Default value`: localhost` 57 | 58 | ## `METRIC_STATSD_PORT` 59 | 60 | Port of the StatsD host 61 | 62 | - Example value: `8125` 63 | - Type: `optional` 64 | - Default value`: 8125` 65 | 66 | ## `METRIC_STATSD_TAGS` 67 | 68 | Global tags for StatsD metrics. Tags must be comma-separated. 69 | 70 | - Example value: `team=engineering,app=myapp` 71 | - Type: `optional` 72 | -------------------------------------------------------------------------------- /docs/reference/configuration/stencil-client.md: -------------------------------------------------------------------------------- 1 | # Stencil Client 2 | 3 | Stencil, the schema registry used by sinks need the following variables to be set for the Stencil client. 4 | 5 | ## `SCHEMA_REGISTRY_STENCIL_ENABLE` 6 | 7 | Defines whether to enable Stencil Schema registry 8 | 9 | * Example value: `true` 10 | * Type: `optional` 11 | * Default value: `false` 12 | 13 | ## `SCHEMA_REGISTRY_STENCIL_URLS` 14 | 15 | Defines the URL of the Proto Descriptor set file in the Stencil Server 16 | 17 | * Example value: `http://localhost:8000/v1/namespaces/quickstart/descriptors/example/versions/latest` 18 | * Type: `optional` 19 | 20 | ## `SCHEMA_REGISTRY_STENCIL_FETCH_TIMEOUT_MS` 21 | 22 | Defines the timeout in milliseconds to fetch the Proto Descriptor set file from the Stencil Server. 23 | 24 | * Example value: `4000` 25 | * Type: `optional` 26 | * Default value: `10000` 27 | 28 | ## `SCHEMA_REGISTRY_STENCIL_FETCH_RETRIES` 29 | 30 | Defines the number of times to retry to fetch the Proto Descriptor set file from the Stencil Server. 31 | 32 | * Example value: `4` 33 | * Type: `optional` 34 | * Default value: `3` 35 | 36 | ## `SCHEMA_REGISTRY_STENCIL_FETCH_BACKOFF_MIN_MS` 37 | 38 | Defines the minimum time in milliseconds after which to back off from fetching the Proto Descriptor set file from the Stencil Server. 39 | 40 | * Example value: `70000` 41 | * Type: `optional` 42 | * Default value: `60000` 43 | 44 | ## `SCHEMA_REGISTRY_STENCIL_FETCH_AUTH_BEARER_TOKEN` 45 | 46 | Defines the token for authentication to connect to Stencil Server 47 | 48 | * Example value: `tcDpw34J8d1` 49 | * Type: `optional` 50 | 51 | ## `SCHEMA_REGISTRY_STENCIL_CACHE_AUTO_REFRESH` 52 | 53 | Defines whether to enable auto-refresh of Stencil cache. 54 | 55 | * Example value: `true` 56 | * Type: `optional` 57 | * Default value: `false` 58 | 59 | ## `SCHEMA_REGISTRY_STENCIL_CACHE_TTL_MS` 60 | 61 | Defines the minimum time in milliseconds after which to refresh the Stencil cache. 62 | 63 | * Example value: `900000` 64 | * Type: `optional` 65 | * Default value: `900000` 66 | 67 | -------------------------------------------------------------------------------- /docs/reference/metrics.md: -------------------------------------------------------------------------------- 1 | # Metrics 2 | 3 | Depot library has built-in instrumentation with statsd support. 4 | Sinks can have their own metrics, and they will be emmited while using sink connector library into other applications. 5 | 6 | ## Table of Contents 7 | 8 | * [Bigquery Sink](metrics.md#bigquery-sink) 9 | * [Bigtable Sink](metrics.md#bigtable-sink) 10 | 11 | ## Bigquery Sink 12 | 13 | ### `Bigquery Operation Total` 14 | 15 | Total number of bigquery API operation performed 16 | 17 | ### `Bigquery Operation Latency` 18 | 19 | Time taken for bigquery API operation performed 20 | 21 | ### `Bigquery Errors Total` 22 | 23 | Total numbers of error occurred on bigquery insert operation 24 | 25 | ## Bigtable Sink 26 | 27 | ### `Bigtable Operation Total` 28 | 29 | Total number of bigtable insert/update operation performed 30 | 31 | ### `Bigtable Operation Latency` 32 | 33 | Time taken for bigtable insert/update operation performed 34 | 35 | ### `Bigtable Errors Total` 36 | 37 | Total numbers of error occurred on bigtable insert/update operation 38 | 39 | 40 | -------------------------------------------------------------------------------- /docs/reference/sink_response.md: -------------------------------------------------------------------------------- 1 | # SinkResponse 2 | ```java 3 | public class SinkResponse { 4 | private final Map errors = new HashMap<>(); 5 | ... 6 | } 7 | 8 | ``` 9 | SinkResponse will be returned by sink.pushToSink(messageList) function call. 10 | The response contains error map indexed by message in the input list. 11 | 12 | ## Errors 13 | These errors are returned by sinks in the SinkResponse object. The error type are: 14 | 15 | * DESERIALIZATION_ERROR 16 | * INVALID_MESSAGE_ERROR 17 | * UNKNOWN_FIELDS_ERROR 18 | * SINK_4XX_ERROR 19 | * SINK_5XX_ERROR 20 | * SINK_RETRYABLE_ERROR 21 | * SINK_UNKNOWN_ERROR 22 | * DEFAULT_ERROR 23 | * If no error is specified (To be deprecated soon) 24 | 25 | -------------------------------------------------------------------------------- /docs/roadmap.md: -------------------------------------------------------------------------------- 1 | # Roadmap 2 | 3 | In the following section, you can learn about what features we're working on, 4 | what stage they're in, and when we expect to bring them to you. 5 | Have any questions or comments about items on the roadmap? 6 | Join the [discussions](https://github.com/raystack/depot/discussions). 7 | 8 | We’re planning to iterate on the format of the roadmap itself, 9 | and we see the potential to engage more in discussions 10 | about the future of sink connectors. 11 | If you have feedback about the roadmap section itself, 12 | such as how the issues are presented, 13 | let us know through [discussions](https://github.com/raystack/depot/discussions). 14 | 15 | ## Depot 0.1.4 16 | 17 | ### Feature enhancements 18 | 19 | - Support for Statsd Metrics 20 | - Support for LogSink 21 | - Support for [BigQuery](https://cloud.google.com/bigquery) sink 22 | -------------------------------------------------------------------------------- /docs/sinks/redis.md: -------------------------------------------------------------------------------- 1 | # Redis Sink 2 | 3 | ### Data Types 4 | Redis sink can be created in 3 different modes based on the value of [`SINK_REDIS_DATA_TYPE`](../reference/configuration/redis.md#sink_redis_data_type): HashSet, KeyValue or List 5 | - `Hashset`: For each message, an entry of the format `key : field : value` is generated and pushed to Redis. Field and value are generated on the basis of the config [`SINK_REDIS_HASHSET_FIELD_TO_COLUMN_MAPPING`](../reference/configuration/redis.md#sink_redis_hashset_field_to_column_mapping) 6 | - `List`: For each message entry of the format `key : value` is generated and pushed to Redis. The value is fetched for the Proto field name provided in the config [`SINK_REDIS_LIST_DATA_FIELD_NAME`](../reference/configuration/redis.md#sink_redis_list_data_field_name) 7 | - `KeyValue`: For each message entry of the format `key : value` is generated and pushed to Redis. The value is fetched for the proto field name provided in the config [`SINK_REDIS_KEY_VALUE_DATA_FIELD_NAME`](../reference/configuration/redis.md#sink_redis_key_value_data_field_name) 8 | 9 | The `key` is picked up from a field in the message itself. 10 | 11 | Limitation: Depot Redis sink only supports Key-Value, HashSet and List entries as of now. 12 | 13 | ### Deployment Types 14 | Redis sink, as of now, supports two different Deployment Types `Standalone` and `Cluster`. This can be configured in the Depot environment variable `SINK_REDIS_DEPLOYMENT_TYPE`. 15 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/depot/e81b52662da624b282bdf94b26eacec3e4a3b406/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'depot' 2 | 3 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/Sink.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot; 2 | 3 | import org.raystack.depot.exception.SinkException; 4 | import org.raystack.depot.message.Message; 5 | 6 | import java.io.Closeable; 7 | import java.util.List; 8 | 9 | public interface Sink extends Closeable { 10 | 11 | SinkResponse pushToSink(List messages) throws SinkException; 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/SinkResponse.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot; 2 | 3 | import org.raystack.depot.error.ErrorInfo; 4 | 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | 8 | public class SinkResponse { 9 | private final Map errors = new HashMap<>(); 10 | 11 | /** 12 | * Returns error as a map whose keys are indexes of messages that failed to be 13 | * pushed. 14 | * Each failed message index is associated with a {@link ErrorInfo}. 15 | */ 16 | public Map getErrors() { 17 | return errors; 18 | } 19 | 20 | /** 21 | * Returns error for the provided message index. If no error exists returns 22 | * {@code null}. 23 | */ 24 | public ErrorInfo getErrorsFor(long index) { 25 | return errors.get(index); 26 | } 27 | 28 | /** 29 | * Adds an error for the index. 30 | */ 31 | public void addErrors(long index, ErrorInfo errorInfo) { 32 | errors.put(index, errorInfo); 33 | } 34 | 35 | /** 36 | * Returns {@code true} if no row insertion failed, {@code false} otherwise. If 37 | * {@code false}. 38 | * {@link #getErrors()} ()} returns an empty map. 39 | */ 40 | public boolean hasErrors() { 41 | return !errors.isEmpty(); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/BigQueryStorageAPISink.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery; 2 | 3 | import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; 4 | import org.raystack.depot.Sink; 5 | import org.raystack.depot.SinkResponse; 6 | import org.raystack.depot.bigquery.storage.BigQueryPayload; 7 | import org.raystack.depot.bigquery.storage.BigQueryStorageClient; 8 | import org.raystack.depot.bigquery.storage.BigQueryStorageResponseParser; 9 | import org.raystack.depot.exception.SinkException; 10 | import org.raystack.depot.message.Message; 11 | 12 | import java.io.IOException; 13 | import java.util.List; 14 | import java.util.concurrent.ExecutionException; 15 | 16 | public class BigQueryStorageAPISink implements Sink { 17 | private final BigQueryStorageClient bigQueryStorageClient; 18 | private final BigQueryStorageResponseParser responseParser; 19 | 20 | public BigQueryStorageAPISink( 21 | BigQueryStorageClient bigQueryStorageClient, 22 | BigQueryStorageResponseParser responseParser) { 23 | this.bigQueryStorageClient = bigQueryStorageClient; 24 | this.responseParser = responseParser; 25 | } 26 | 27 | @Override 28 | public SinkResponse pushToSink(List messages) throws SinkException { 29 | SinkResponse sinkResponse = new SinkResponse(); 30 | BigQueryPayload payload = bigQueryStorageClient.convert(messages); 31 | responseParser.setSinkResponseForInvalidMessages(payload, messages, sinkResponse); 32 | if (payload.getPayloadIndexes().size() > 0) { 33 | try { 34 | AppendRowsResponse appendRowsResponse = bigQueryStorageClient.appendAndGet(payload); 35 | responseParser.setSinkResponseForErrors(payload, appendRowsResponse, messages, sinkResponse); 36 | } catch (ExecutionException e) { 37 | e.printStackTrace(); 38 | Throwable cause = e.getCause(); 39 | responseParser.setSinkResponseForException(cause, payload, messages, sinkResponse); 40 | } catch (InterruptedException e) { 41 | e.printStackTrace(); 42 | throw new SinkException("Interrupted exception occurred", e); 43 | } 44 | } 45 | return sinkResponse; 46 | } 47 | 48 | @Override 49 | public void close() throws IOException { 50 | bigQueryStorageClient.close(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/BigqueryStencilUpdateListenerFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery; 2 | 3 | import org.raystack.depot.bigquery.client.BigQueryClient; 4 | import org.raystack.depot.bigquery.converter.MessageRecordConverterCache; 5 | import org.raystack.depot.bigquery.json.BigqueryJsonUpdateListener; 6 | import org.raystack.depot.bigquery.proto.BigqueryProtoUpdateListener; 7 | import org.raystack.depot.exception.ConfigurationException; 8 | import org.raystack.depot.config.BigQuerySinkConfig; 9 | import org.raystack.depot.metrics.Instrumentation; 10 | import org.raystack.depot.metrics.StatsDReporter; 11 | import org.raystack.depot.stencil.DepotStencilUpdateListener; 12 | 13 | public class BigqueryStencilUpdateListenerFactory { 14 | public static DepotStencilUpdateListener create(BigQuerySinkConfig config, BigQueryClient bqClient, 15 | MessageRecordConverterCache converterCache, StatsDReporter statsDReporter) { 16 | switch (config.getSinkConnectorSchemaDataType()) { 17 | case JSON: 18 | return new BigqueryJsonUpdateListener(config, converterCache, bqClient, 19 | new Instrumentation(statsDReporter, BigqueryJsonUpdateListener.class)); 20 | case PROTOBUF: 21 | return new BigqueryProtoUpdateListener(config, bqClient, converterCache); 22 | default: 23 | throw new ConfigurationException("Schema Type is not supported"); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/client/BigQueryRow.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.client; 2 | 3 | import com.google.cloud.bigquery.InsertAllRequest; 4 | import org.raystack.depot.bigquery.models.Record; 5 | 6 | /** 7 | * Fetches BQ insertable row from the base record {@link Record}. The 8 | * implementations can differ if unique rows need to be inserted or not. 9 | */ 10 | public interface BigQueryRow { 11 | 12 | InsertAllRequest.RowToInsert of(Record record); 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/client/BigQueryRowWithInsertId.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.client; 2 | 3 | import com.google.cloud.bigquery.InsertAllRequest; 4 | import org.raystack.depot.bigquery.models.Record; 5 | 6 | import java.util.Map; 7 | import java.util.function.Function; 8 | 9 | public class BigQueryRowWithInsertId implements BigQueryRow { 10 | private final Function, String> rowIDCreator; 11 | 12 | public BigQueryRowWithInsertId(Function, String> rowIDCreator) { 13 | this.rowIDCreator = rowIDCreator; 14 | } 15 | 16 | @Override 17 | public InsertAllRequest.RowToInsert of(Record record) { 18 | return InsertAllRequest.RowToInsert.of(rowIDCreator.apply(record.getMetadata()), record.getColumns()); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/client/BigQueryRowWithoutInsertId.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.client; 2 | 3 | import com.google.cloud.bigquery.InsertAllRequest; 4 | import org.raystack.depot.bigquery.models.Record; 5 | 6 | public class BigQueryRowWithoutInsertId implements BigQueryRow { 7 | 8 | @Override 9 | public InsertAllRequest.RowToInsert of(Record record) { 10 | return InsertAllRequest.RowToInsert.of(record.getColumns()); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/converter/MessageRecordConverterCache.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.converter; 2 | 3 | import lombok.Data; 4 | 5 | @Data 6 | public class MessageRecordConverterCache { 7 | private MessageRecordConverter messageRecordConverter; 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/converter/MessageRecordConverterUtils.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.converter; 2 | 3 | import com.google.api.client.util.DateTime; 4 | import org.raystack.depot.common.TupleString; 5 | import org.raystack.depot.config.BigQuerySinkConfig; 6 | import org.raystack.depot.config.enums.SinkConnectorSchemaDataType; 7 | import org.raystack.depot.message.Message; 8 | import org.raystack.depot.utils.DateUtils; 9 | 10 | import java.util.List; 11 | import java.util.Map; 12 | import java.util.stream.Collectors; 13 | 14 | public class MessageRecordConverterUtils { 15 | 16 | public static final String JSON_TIME_STAMP_COLUMN = "event_timestamp"; 17 | 18 | public static void addMetadata(Map columns, Message message, BigQuerySinkConfig config) { 19 | if (config.shouldAddMetadata()) { 20 | List metadataColumnsTypes = config.getMetadataColumnsTypes(); 21 | Map metadata = message.getMetadata(metadataColumnsTypes); 22 | Map finalMetadata = metadataColumnsTypes.stream() 23 | .collect(Collectors.toMap(TupleString::getFirst, t -> { 24 | String key = t.getFirst(); 25 | String dataType = t.getSecond(); 26 | Object value = metadata.get(key); 27 | if (value instanceof Long && dataType.equals("timestamp")) { 28 | value = new DateTime((long) value); 29 | } 30 | return value; 31 | })); 32 | if (config.getBqMetadataNamespace().isEmpty()) { 33 | columns.putAll(finalMetadata); 34 | } else { 35 | columns.put(config.getBqMetadataNamespace(), finalMetadata); 36 | } 37 | 38 | } 39 | } 40 | 41 | public static void addTimeStampColumnForJson(Map columns, BigQuerySinkConfig config) { 42 | if (config.getSinkConnectorSchemaDataType() == SinkConnectorSchemaDataType.JSON 43 | && config.getSinkBigqueryAddEventTimestampEnable()) { 44 | columns.put(JSON_TIME_STAMP_COLUMN, DateUtils.formatCurrentTimeAsUTC()); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/error/ErrorDescriptor.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.error; 2 | 3 | /** 4 | * Descriptor interface that defines the various error descriptors and the 5 | * corresponding error types. 6 | */ 7 | public interface ErrorDescriptor { 8 | 9 | /** 10 | * If the implementing descriptor matches the condition as prescribed in the 11 | * concrete implementation. 12 | * 13 | * @return - true if the condition matches, false otherwise. 14 | */ 15 | boolean matches(); 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/error/ErrorParser.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.error; 2 | 3 | import com.google.cloud.bigquery.BigQueryError; 4 | 5 | import java.util.Arrays; 6 | import java.util.List; 7 | import java.util.stream.Collectors; 8 | 9 | /** 10 | * ErrorParser determines the {@link ErrorDescriptor} classes error based on the 11 | * error string supplied. 12 | */ 13 | public class ErrorParser { 14 | 15 | public static ErrorDescriptor getError(String reasonText, String msgText) { 16 | List errDescList = Arrays.asList( 17 | new InvalidSchemaError(reasonText, msgText), 18 | new OOBError(reasonText, msgText), 19 | new StoppedError(reasonText)); 20 | 21 | return errDescList 22 | .stream() 23 | .filter(ErrorDescriptor::matches) 24 | .findFirst() 25 | .orElse(new UnknownError(reasonText, msgText)); 26 | } 27 | 28 | public static List parseError(List bqErrors) { 29 | return bqErrors.stream() 30 | .map(err -> getError(err.getReason(), err.getMessage())) 31 | .collect(Collectors.toList()); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/error/InvalidSchemaError.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.error; 2 | 3 | import lombok.AllArgsConstructor; 4 | 5 | @AllArgsConstructor 6 | /** 7 | * This error returns when there is any kind of invalid input 8 | * other than an invalid query, such as missing required fields 9 | * or an invalid table schema. 10 | * 11 | * https://cloud.google.com/bigquery/docs/error-messages 12 | */ 13 | public class InvalidSchemaError implements ErrorDescriptor { 14 | 15 | private final String reason; 16 | private final String message; 17 | 18 | @Override 19 | public boolean matches() { 20 | return reason.equals("invalid") && message.contains("no such field"); 21 | } 22 | 23 | @Override 24 | public String toString() { 25 | return String.format("InvalidSchemaError: %s", message); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/error/OOBError.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.error; 2 | 3 | import lombok.AllArgsConstructor; 4 | 5 | @AllArgsConstructor 6 | /** 7 | * Out of bounds are caused when the partitioned column has a date value less 8 | * than 9 | * 5 years and more than 1 year in future 10 | */ 11 | public class OOBError implements ErrorDescriptor { 12 | 13 | private final String reason; 14 | private final String message; 15 | 16 | @Override 17 | public boolean matches() { 18 | return reason.equals("invalid") 19 | && ((message.contains("is outside the allowed bounds") && message.contains("days in the past and") 20 | && message.contains("days in the future")) 21 | || message.contains("out of range")); 22 | } 23 | 24 | @Override 25 | public String toString() { 26 | return String.format("OOBError: %s", message); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/error/StoppedError.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.error; 2 | 3 | import lombok.AllArgsConstructor; 4 | 5 | @AllArgsConstructor 6 | /** 7 | * stopped 200 This status code returns when a job is canceled. 8 | * This will be returned if a batch of insertion has some bad records 9 | * which caused the job to be cancelled. Bad records will have some *other* 10 | * error 11 | * but rest of records will be marked as "stopped" and can be sent as is 12 | * 13 | * https://cloud.google.com/bigquery/docs/error-messages 14 | */ 15 | public class StoppedError implements ErrorDescriptor { 16 | 17 | private final String reason; 18 | 19 | @Override 20 | public boolean matches() { 21 | return reason.equals("stopped"); 22 | } 23 | 24 | @Override 25 | public String toString() { 26 | return "StoppedError: BigQuery encountered an error on individual rows in the request, none of the rows are inserted. This error can be retried"; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/error/UnknownError.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.error; 2 | 3 | import lombok.AllArgsConstructor; 4 | 5 | @AllArgsConstructor 6 | /** 7 | * UnknownError is used when error factory failed to match any possible 8 | * known errors 9 | */ 10 | public class UnknownError implements ErrorDescriptor { 11 | 12 | private String reason; 13 | 14 | private String message; 15 | 16 | @Override 17 | public boolean matches() { 18 | return false; 19 | } 20 | 21 | @Override 22 | public String toString() { 23 | return String.format("%s: %s", !reason.equals("") ? reason : "UnknownError", message != null ? message : ""); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/exception/BQClusteringKeysException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.exception; 2 | 3 | public class BQClusteringKeysException extends RuntimeException { 4 | public BQClusteringKeysException(String message) { 5 | super(message); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/exception/BQDatasetLocationChangedException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.exception; 2 | 3 | public class BQDatasetLocationChangedException extends RuntimeException { 4 | public BQDatasetLocationChangedException(String message) { 5 | super(message); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/exception/BQPartitionKeyNotSpecified.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.exception; 2 | 3 | public class BQPartitionKeyNotSpecified extends RuntimeException { 4 | public BQPartitionKeyNotSpecified(String message) { 5 | super(message); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/exception/BQSchemaMappingException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.exception; 2 | 3 | public class BQSchemaMappingException extends RuntimeException { 4 | public BQSchemaMappingException(String message) { 5 | super(message); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/exception/BQTableUpdateFailure.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.exception; 2 | 3 | public class BQTableUpdateFailure extends RuntimeException { 4 | public BQTableUpdateFailure(String message, Throwable rootCause) { 5 | super(message, rootCause); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/exception/BigQuerySinkException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.exception; 2 | 3 | import lombok.EqualsAndHashCode; 4 | 5 | @EqualsAndHashCode(callSuper = false) 6 | public class BigQuerySinkException extends RuntimeException { 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/handler/ErrorHandler.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.handler; 2 | 3 | import com.google.cloud.bigquery.BigQueryError; 4 | import org.raystack.depot.bigquery.models.Record; 5 | 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | public interface ErrorHandler { 10 | default void handle(Map> errorInfoMap, List records) { 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/handler/ErrorHandlerFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.handler; 2 | 3 | import org.raystack.depot.config.BigQuerySinkConfig; 4 | import org.raystack.depot.config.enums.SinkConnectorSchemaDataType; 5 | import org.raystack.depot.metrics.Instrumentation; 6 | import org.raystack.depot.metrics.StatsDReporter; 7 | import org.raystack.depot.bigquery.client.BigQueryClient; 8 | 9 | public class ErrorHandlerFactory { 10 | public static ErrorHandler create(BigQuerySinkConfig sinkConfig, BigQueryClient bigQueryClient, 11 | StatsDReporter statsDReprter) { 12 | if (SinkConnectorSchemaDataType.JSON == sinkConfig.getSinkConnectorSchemaDataType()) { 13 | return new JsonErrorHandler( 14 | bigQueryClient, 15 | sinkConfig, new Instrumentation(statsDReprter, JsonErrorHandler.class)); 16 | } 17 | return new NoopErrorHandler(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/handler/NoopErrorHandler.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.handler; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | 5 | @Slf4j 6 | public class NoopErrorHandler implements ErrorHandler { 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/models/Record.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.models; 2 | 3 | import org.raystack.depot.error.ErrorInfo; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Builder; 6 | import lombok.EqualsAndHashCode; 7 | import lombok.Getter; 8 | import lombok.ToString; 9 | 10 | import java.util.Map; 11 | 12 | @AllArgsConstructor 13 | @Getter 14 | @EqualsAndHashCode 15 | @Builder 16 | @ToString 17 | public class Record { 18 | private final Map metadata; 19 | private final Map columns; 20 | private final long index; 21 | private final ErrorInfo errorInfo; 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/models/Records.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.models; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.EqualsAndHashCode; 5 | import lombok.Getter; 6 | import lombok.ToString; 7 | 8 | import java.util.List; 9 | 10 | @AllArgsConstructor 11 | @Getter 12 | @EqualsAndHashCode 13 | @ToString 14 | public class Records { 15 | private final List validRecords; 16 | private final List invalidRecords; 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/proto/BigqueryFields.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.proto; 2 | 3 | import com.google.cloud.bigquery.Field; 4 | import com.google.cloud.bigquery.FieldList; 5 | import com.google.cloud.bigquery.LegacySQLTypeName; 6 | import org.raystack.depot.bigquery.models.BQField; 7 | import org.raystack.depot.common.TupleString; 8 | import org.raystack.depot.message.proto.ProtoField; 9 | 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | import java.util.stream.Collectors; 13 | 14 | public class BigqueryFields { 15 | public static List getMetadataFields(List metadataColumnsTypes) { 16 | return metadataColumnsTypes.stream().map( 17 | tuple -> Field.newBuilder(tuple.getFirst(), LegacySQLTypeName.valueOf(tuple.getSecond())) 18 | .setMode(Field.Mode.NULLABLE) 19 | .build()) 20 | .collect(Collectors.toList()); 21 | } 22 | 23 | /* 24 | * throws an exception if typeName is not recognized by 25 | * LegacySQLTypeName.valueOfStric 26 | */ 27 | public static List getMetadataFieldsStrict(List metadataColumnsTypes) { 28 | return metadataColumnsTypes.stream().map( 29 | tuple -> Field 30 | .newBuilder(tuple.getFirst(), LegacySQLTypeName.valueOfStrict(tuple.getSecond().toUpperCase())) 31 | .setMode(Field.Mode.NULLABLE) 32 | .build()) 33 | .collect(Collectors.toList()); 34 | } 35 | 36 | public static Field getNamespacedMetadataField(String namespace, List metadataColumnsTypes) { 37 | return Field 38 | .newBuilder(namespace, LegacySQLTypeName.RECORD, FieldList.of(getMetadataFields(metadataColumnsTypes))) 39 | .setMode(Field.Mode.NULLABLE) 40 | .build(); 41 | } 42 | 43 | public static List generateBigquerySchema(ProtoField protoField) { 44 | if (protoField == null) { 45 | return null; 46 | } 47 | List schemaFields = new ArrayList<>(); 48 | for (ProtoField field : protoField.getFields()) { 49 | BQField bqField = new BQField(field); 50 | if (field.isNested()) { 51 | List fields = generateBigquerySchema(field); 52 | bqField.setSubFields(fields); 53 | } 54 | schemaFields.add(bqField.getField()); 55 | } 56 | return schemaFields; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/storage/BigQueryPayload.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.storage; 2 | 3 | import org.raystack.depot.bigquery.storage.proto.BigQueryRecordMeta; 4 | 5 | import java.util.ArrayList; 6 | import java.util.HashMap; 7 | import java.util.Iterator; 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.Set; 11 | 12 | public class BigQueryPayload implements Iterable { 13 | private final List recordMetadata = new ArrayList<>(); 14 | private final Map payloadIndexToInputIndex = new HashMap<>(); 15 | private Object payload; 16 | 17 | public void addMetadataRecord(BigQueryRecordMeta record) { 18 | recordMetadata.add(record); 19 | } 20 | 21 | public void putValidIndexToInputIndex(long validIndex, long inputIndex) { 22 | payloadIndexToInputIndex.put(validIndex, inputIndex); 23 | } 24 | 25 | public long getInputIndex(long payloadIndex) { 26 | return payloadIndexToInputIndex.get(payloadIndex); 27 | } 28 | 29 | public Set getPayloadIndexes() { 30 | return payloadIndexToInputIndex.keySet(); 31 | } 32 | 33 | public Iterator iterator() { 34 | return recordMetadata.iterator(); 35 | } 36 | 37 | public Object getPayload() { 38 | return payload; 39 | } 40 | 41 | public void setPayload(Object payload) { 42 | this.payload = payload; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/storage/BigQueryStorageClient.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.storage; 2 | 3 | import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; 4 | import org.raystack.depot.message.Message; 5 | 6 | import java.io.Closeable; 7 | import java.util.List; 8 | import java.util.concurrent.ExecutionException; 9 | 10 | public interface BigQueryStorageClient extends Closeable { 11 | BigQueryPayload convert(List messages); 12 | 13 | AppendRowsResponse appendAndGet(BigQueryPayload payload) throws ExecutionException, InterruptedException; 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/storage/BigQueryStorageClientFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.storage; 2 | 3 | import org.raystack.depot.bigquery.storage.proto.BigQueryProtoStorageClient; 4 | import org.raystack.depot.config.BigQuerySinkConfig; 5 | import org.raystack.depot.message.MessageParser; 6 | 7 | public class BigQueryStorageClientFactory { 8 | public static BigQueryStorageClient createBigQueryStorageClient( 9 | BigQuerySinkConfig config, 10 | MessageParser parser, 11 | BigQueryWriter bigQueryWriter) { 12 | switch (config.getSinkConnectorSchemaDataType()) { 13 | case PROTOBUF: 14 | return new BigQueryProtoStorageClient(bigQueryWriter, config, parser); 15 | default: 16 | throw new IllegalArgumentException("Invalid data type"); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/storage/BigQueryStream.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.storage; 2 | 3 | public interface BigQueryStream { 4 | } 5 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/storage/BigQueryWriter.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.storage; 2 | 3 | import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; 4 | 5 | import java.util.concurrent.ExecutionException; 6 | 7 | public interface BigQueryWriter extends AutoCloseable { 8 | 9 | void init(); 10 | 11 | AppendRowsResponse appendAndGet(BigQueryPayload payload) throws ExecutionException, InterruptedException; 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/storage/BigQueryWriterFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.storage; 2 | 3 | import com.google.api.gax.core.CredentialsProvider; 4 | import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; 5 | import com.google.cloud.bigquery.storage.v1.ProtoSchema; 6 | import org.raystack.depot.bigquery.storage.proto.BigQueryProtoWriter; 7 | import org.raystack.depot.bigquery.storage.json.BigQueryJsonWriter; 8 | import org.raystack.depot.common.Function3; 9 | import org.raystack.depot.config.BigQuerySinkConfig; 10 | import org.raystack.depot.metrics.BigQueryMetrics; 11 | import org.raystack.depot.metrics.Instrumentation; 12 | 13 | import java.util.function.Function; 14 | 15 | public class BigQueryWriterFactory { 16 | 17 | public static BigQueryWriter createBigQueryWriter( 18 | BigQuerySinkConfig config, 19 | Function bqWriterCreator, 20 | Function credCreator, 21 | Function3 streamCreator, 22 | Instrumentation instrumentation, 23 | BigQueryMetrics metrics) { 24 | switch (config.getSinkConnectorSchemaDataType()) { 25 | case PROTOBUF: 26 | return new BigQueryProtoWriter(config, bqWriterCreator, credCreator, streamCreator, instrumentation, 27 | metrics); 28 | case JSON: 29 | return new BigQueryJsonWriter(config); 30 | default: 31 | throw new IllegalArgumentException("Couldn't initialise the BQ writer"); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/storage/json/BigQueryJsonStream.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.storage.json; 2 | 3 | import com.google.cloud.bigquery.storage.v1.JsonStreamWriter; 4 | import org.raystack.depot.bigquery.storage.BigQueryStream; 5 | 6 | public class BigQueryJsonStream implements BigQueryStream { 7 | 8 | public JsonStreamWriter getStreamWriter() { 9 | return null; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/storage/json/BigQueryJsonWriter.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.storage.json; 2 | 3 | import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; 4 | import org.raystack.depot.bigquery.storage.BigQueryPayload; 5 | import org.raystack.depot.bigquery.storage.BigQueryWriter; 6 | import org.raystack.depot.config.BigQuerySinkConfig; 7 | 8 | import java.util.concurrent.ExecutionException; 9 | 10 | public class BigQueryJsonWriter implements BigQueryWriter { 11 | 12 | public BigQueryJsonWriter(BigQuerySinkConfig config) { 13 | 14 | } 15 | 16 | @Override 17 | public void init() { 18 | 19 | } 20 | 21 | @Override 22 | public AppendRowsResponse appendAndGet(BigQueryPayload payload) throws ExecutionException, InterruptedException { 23 | return null; 24 | } 25 | 26 | @Override 27 | public void close() throws Exception { 28 | 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/storage/proto/BigQueryProtoStream.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.storage.proto; 2 | 3 | import com.google.cloud.bigquery.storage.v1.StreamWriter; 4 | import org.raystack.depot.bigquery.storage.BigQueryStream; 5 | import lombok.AllArgsConstructor; 6 | import lombok.Getter; 7 | 8 | @AllArgsConstructor 9 | public class BigQueryProtoStream implements BigQueryStream { 10 | @Getter 11 | private final StreamWriter streamWriter; 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/storage/proto/BigQueryRecordMeta.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.storage.proto; 2 | 3 | import org.raystack.depot.error.ErrorInfo; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Getter; 6 | 7 | @AllArgsConstructor 8 | @Getter 9 | public class BigQueryRecordMeta { 10 | private final long inputIndex; 11 | private final ErrorInfo errorInfo; 12 | private final boolean isValid; 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigquery/storage/proto/TimeStampUtils.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.storage.proto; 2 | 3 | import com.google.protobuf.Descriptors; 4 | import org.raystack.depot.config.BigQuerySinkConfig; 5 | 6 | import java.time.Instant; 7 | import java.util.concurrent.TimeUnit; 8 | 9 | public class TimeStampUtils { 10 | private static final long FIVE_YEARS_DAYS = 1825; 11 | private static final long ONE_YEAR_DAYS = 365; 12 | private static final Instant MIN_TIMESTAMP = Instant.parse("0001-01-01T00:00:00Z"); 13 | private static final Instant MAX_TIMESTAMP = Instant.parse("9999-12-31T23:59:59.999999Z"); 14 | 15 | public static long getBQInstant(Instant instant, Descriptors.FieldDescriptor fieldDescriptor, boolean isTopLevel, 16 | BigQuerySinkConfig config) { 17 | // Timestamp should be in microseconds 18 | long timeStamp = TimeUnit.SECONDS.toMicros(instant.getEpochSecond()) 19 | + TimeUnit.NANOSECONDS.toMicros(instant.getNano()); 20 | // Partition column is always top level 21 | if (isTopLevel && fieldDescriptor.getName().equals(config.getTablePartitionKey())) { 22 | Instant currentInstant = Instant.now(); 23 | boolean isValid; 24 | boolean isPastInstant = currentInstant.isAfter(instant); 25 | if (isPastInstant) { 26 | Instant fiveYearPast = currentInstant.minusMillis(TimeUnit.DAYS.toMillis(FIVE_YEARS_DAYS)); 27 | isValid = fiveYearPast.isBefore(instant); 28 | } else { 29 | Instant oneYearFuture = currentInstant.plusMillis(TimeUnit.DAYS.toMillis(ONE_YEAR_DAYS)); 30 | isValid = oneYearFuture.isAfter(instant); 31 | 32 | } 33 | if (!isValid) { 34 | throw new UnsupportedOperationException(instant + " for field " 35 | + fieldDescriptor.getFullName() + " is outside the allowed bounds. " 36 | + "You can only stream to date range within 1825 days in the past " 37 | + "and 366 days in the future relative to the current date."); 38 | } 39 | return timeStamp; 40 | } else { 41 | // other timestamps should be in the limit specifies by BQ 42 | if (instant.isAfter(MIN_TIMESTAMP) && instant.isBefore(MAX_TIMESTAMP)) { 43 | return timeStamp; 44 | } else { 45 | throw new UnsupportedOperationException(instant 46 | + " for field " 47 | + fieldDescriptor.getFullName() 48 | + " is outside the allowed bounds in BQ."); 49 | } 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigtable/exception/BigTableInvalidSchemaException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigtable.exception; 2 | 3 | public class BigTableInvalidSchemaException extends RuntimeException { 4 | public BigTableInvalidSchemaException(String message, Throwable cause) { 5 | super(message, cause); 6 | } 7 | 8 | public BigTableInvalidSchemaException(String messsage) { 9 | super(messsage); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigtable/model/BigTableRecord.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigtable.model; 2 | 3 | import com.google.cloud.bigtable.data.v2.models.RowMutationEntry; 4 | import org.raystack.depot.error.ErrorInfo; 5 | import lombok.AllArgsConstructor; 6 | import lombok.Getter; 7 | 8 | import java.util.Map; 9 | 10 | @AllArgsConstructor 11 | @Getter 12 | public class BigTableRecord { 13 | private final RowMutationEntry rowMutationEntry; 14 | private final long index; 15 | private final ErrorInfo errorInfo; 16 | private final Map metadata; 17 | 18 | public boolean isValid() { 19 | return errorInfo == null; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigtable/model/BigTableSchema.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigtable.model; 2 | 3 | import org.raystack.depot.exception.ConfigurationException; 4 | import org.json.JSONObject; 5 | 6 | import java.util.HashSet; 7 | import java.util.Set; 8 | 9 | public class BigTableSchema { 10 | 11 | private final JSONObject columnFamilyMapping; 12 | 13 | public BigTableSchema(String columnMapping) { 14 | if (columnMapping == null || columnMapping.isEmpty()) { 15 | throw new ConfigurationException("Column Mapping should not be empty or null"); 16 | } 17 | this.columnFamilyMapping = new JSONObject(columnMapping); 18 | } 19 | 20 | public String getField(String columnFamily, String columnName) { 21 | JSONObject columns = columnFamilyMapping.getJSONObject(columnFamily); 22 | return columns.getString(columnName); 23 | } 24 | 25 | public Set getColumnFamilies() { 26 | return columnFamilyMapping.keySet(); 27 | } 28 | 29 | public Set getColumns(String family) { 30 | return columnFamilyMapping.getJSONObject(family).keySet(); 31 | } 32 | 33 | /** 34 | * Returns missing column families. 35 | * 36 | * @param existingColumnFamilies existing column families in a table. 37 | * @return set of missing column families 38 | */ 39 | public Set getMissingColumnFamilies(Set existingColumnFamilies) { 40 | Set tempSet = new HashSet<>(getColumnFamilies()); 41 | tempSet.removeAll(existingColumnFamilies); 42 | return tempSet; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigtable/parser/BigTableRowKeyParser.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigtable.parser; 2 | 3 | import org.raystack.depot.common.Template; 4 | import org.raystack.depot.message.MessageSchema; 5 | import org.raystack.depot.message.ParsedMessage; 6 | import lombok.AllArgsConstructor; 7 | 8 | @AllArgsConstructor 9 | public class BigTableRowKeyParser { 10 | private final Template keyTemplate; 11 | private final MessageSchema schema; 12 | 13 | public String parse(ParsedMessage parsedMessage) { 14 | return keyTemplate.parse(parsedMessage, schema); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/bigtable/response/BigTableResponse.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigtable.response; 2 | 3 | import com.google.cloud.bigtable.data.v2.models.MutateRowsException; 4 | import lombok.Getter; 5 | 6 | import java.util.List; 7 | 8 | @Getter 9 | public class BigTableResponse { 10 | private final List failedMutations; 11 | 12 | public BigTableResponse(MutateRowsException e) { 13 | failedMutations = e.getFailedMutations(); 14 | } 15 | 16 | public boolean hasErrors() { 17 | return !failedMutations.isEmpty(); 18 | } 19 | 20 | public int getErrorCount() { 21 | return failedMutations.size(); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/common/Function3.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.common; 2 | 3 | @FunctionalInterface 4 | public interface Function3 { 5 | R apply(T t, U u, V v); 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/common/Template.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.common; 2 | 3 | import com.google.common.base.Splitter; 4 | import org.raystack.depot.exception.InvalidTemplateException; 5 | import org.raystack.depot.message.MessageSchema; 6 | import org.raystack.depot.message.ParsedMessage; 7 | import org.raystack.depot.message.field.GenericFieldFactory; 8 | import org.raystack.depot.message.proto.converter.fields.ProtoField; 9 | import org.raystack.depot.utils.StringUtils; 10 | 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | 14 | public class Template { 15 | private final String templatePattern; 16 | private final List patternVariableFieldNames; 17 | 18 | public Template(String template) throws InvalidTemplateException { 19 | if (template == null || template.isEmpty()) { 20 | throw new InvalidTemplateException("Template cannot be empty"); 21 | } 22 | List templateStrings = new ArrayList<>(); 23 | Splitter.on(",").omitEmptyStrings().split(template).forEach(s -> templateStrings.add(s.trim())); 24 | this.templatePattern = templateStrings.get(0); 25 | this.patternVariableFieldNames = templateStrings.subList(1, templateStrings.size()); 26 | validate(); 27 | } 28 | 29 | private void validate() throws InvalidTemplateException { 30 | int validArgs = StringUtils.countVariables(templatePattern); 31 | int values = patternVariableFieldNames.size(); 32 | int variables = StringUtils.count(templatePattern, '%'); 33 | if (validArgs != values || variables != values) { 34 | throw new InvalidTemplateException(String.format( 35 | "Template is not valid, variables=%d, validArgs=%d, values=%d", variables, validArgs, values)); 36 | } 37 | } 38 | 39 | public String parse(ParsedMessage parsedMessage, MessageSchema schema) { 40 | Object[] patternVariableData = patternVariableFieldNames 41 | .stream() 42 | .map(fieldName -> fetchInternalValue(parsedMessage.getFieldByName(fieldName, schema))) 43 | .toArray(); 44 | return String.format(templatePattern, patternVariableData); 45 | } 46 | 47 | private Object fetchInternalValue(Object ob) { 48 | if (ob instanceof ProtoField) { 49 | return GenericFieldFactory.getField(ob).getString(); 50 | } else { 51 | return ob; 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/common/Tuple.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.common; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | 6 | @Data 7 | @AllArgsConstructor 8 | public class Tuple { 9 | private T first; 10 | private V second; 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/common/TupleString.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.common; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | 6 | @Data 7 | @AllArgsConstructor 8 | public class TupleString { 9 | private String first; 10 | private String second; 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/config/BigTableSinkConfig.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config; 2 | 3 | import org.aeonbits.owner.Config; 4 | 5 | @Config.DisableFeature(Config.DisableableFeature.PARAMETER_FORMATTING) 6 | public interface BigTableSinkConfig extends SinkConfig { 7 | @Key("SINK_BIGTABLE_GOOGLE_CLOUD_PROJECT_ID") 8 | String getGCloudProjectID(); 9 | 10 | @Key("SINK_BIGTABLE_INSTANCE_ID") 11 | String getInstanceId(); 12 | 13 | @Key("SINK_BIGTABLE_TABLE_ID") 14 | String getTableId(); 15 | 16 | @Key("SINK_BIGTABLE_CREDENTIAL_PATH") 17 | String getCredentialPath(); 18 | 19 | @Key("SINK_BIGTABLE_ROW_KEY_TEMPLATE") 20 | String getRowKeyTemplate(); 21 | 22 | @Key("SINK_BIGTABLE_COLUMN_FAMILY_MAPPING") 23 | String getColumnFamilyMapping(); 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/config/MetricsConfig.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config; 2 | 3 | import org.aeonbits.owner.Config; 4 | 5 | public interface MetricsConfig extends Config { 6 | 7 | @Config.Key("METRIC_STATSD_HOST") 8 | @DefaultValue("localhost") 9 | String getMetricStatsDHost(); 10 | 11 | @Config.Key("METRIC_STATSD_PORT") 12 | @DefaultValue("8125") 13 | Integer getMetricStatsDPort(); 14 | 15 | @Config.Key("METRIC_STATSD_TAGS") 16 | @DefaultValue("") 17 | String getMetricStatsDTags(); 18 | 19 | @Config.Key("METRIC_STATSD_TAGS_NATIVE_FORMAT_ENABLE") 20 | @DefaultValue("false") 21 | boolean getMetricStatsDTagsNativeFormatEnabled(); 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/config/RedisSinkConfig.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config; 2 | 3 | import org.raystack.depot.config.converter.JsonToPropertiesConverter; 4 | import org.raystack.depot.config.converter.RedisSinkDataTypeConverter; 5 | import org.raystack.depot.config.converter.RedisSinkDeploymentTypeConverter; 6 | import org.raystack.depot.config.converter.RedisSinkTtlTypeConverter; 7 | import org.raystack.depot.redis.enums.RedisSinkDataType; 8 | import org.raystack.depot.redis.enums.RedisSinkDeploymentType; 9 | import org.raystack.depot.redis.enums.RedisSinkTtlType; 10 | import org.aeonbits.owner.Config; 11 | 12 | import java.util.Properties; 13 | 14 | @Config.DisableFeature(Config.DisableableFeature.PARAMETER_FORMATTING) 15 | public interface RedisSinkConfig extends SinkConfig { 16 | @Key("SINK_REDIS_URLS") 17 | String getSinkRedisUrls(); 18 | 19 | @Key("SINK_REDIS_KEY_TEMPLATE") 20 | String getSinkRedisKeyTemplate(); 21 | 22 | @Key("SINK_REDIS_DATA_TYPE") 23 | @DefaultValue("HASHSET") 24 | @ConverterClass(RedisSinkDataTypeConverter.class) 25 | RedisSinkDataType getSinkRedisDataType(); 26 | 27 | @Key("SINK_REDIS_TTL_TYPE") 28 | @DefaultValue("DISABLE") 29 | @ConverterClass(RedisSinkTtlTypeConverter.class) 30 | RedisSinkTtlType getSinkRedisTtlType(); 31 | 32 | @Key("SINK_REDIS_TTL_VALUE") 33 | @DefaultValue("0") 34 | long getSinkRedisTtlValue(); 35 | 36 | @Key("SINK_REDIS_DEPLOYMENT_TYPE") 37 | @DefaultValue("Standalone") 38 | @ConverterClass(RedisSinkDeploymentTypeConverter.class) 39 | RedisSinkDeploymentType getSinkRedisDeploymentType(); 40 | 41 | @Key("SINK_REDIS_KEY_VALUE_DATA_FIELD_NAME") 42 | String getSinkRedisKeyValueDataFieldName(); 43 | 44 | @Key("SINK_REDIS_LIST_DATA_FIELD_NAME") 45 | String getSinkRedisListDataFieldName(); 46 | 47 | @Key("SINK_REDIS_HASHSET_FIELD_TO_COLUMN_MAPPING") 48 | @ConverterClass(JsonToPropertiesConverter.class) 49 | @DefaultValue("") 50 | Properties getSinkRedisHashsetFieldToColumnMapping(); 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/config/converter/ConfToListConverter.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.raystack.depot.common.TupleString; 4 | import org.aeonbits.owner.Converter; 5 | 6 | import java.lang.reflect.Method; 7 | 8 | public class ConfToListConverter implements Converter { 9 | public static final String ELEMENT_SEPARATOR = ","; 10 | public static final String VALUE_SEPARATOR = "="; 11 | 12 | @Override 13 | public TupleString convert(Method method, String input) { 14 | if (input.isEmpty()) { 15 | return null; 16 | } 17 | String[] split = input.split(VALUE_SEPARATOR); 18 | return new TupleString(split[0], split[1]); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/config/converter/ConverterUtils.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.raystack.depot.common.Tuple; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | public class ConverterUtils { 9 | public static final String ELEMENT_SEPARATOR = ","; 10 | private static final String VALUE_SEPARATOR = "="; 11 | private static final int MAX_LENGTH = 63; 12 | 13 | public static List> convertToList(String input) { 14 | List> result = new ArrayList<>(); 15 | String[] chunks = input.split(ELEMENT_SEPARATOR, -1); 16 | for (String chunk : chunks) { 17 | String[] entry = chunk.split(VALUE_SEPARATOR, -1); 18 | if (entry.length <= 1) { 19 | continue; 20 | } 21 | String key = entry[0].trim(); 22 | if (key.isEmpty()) { 23 | continue; 24 | } 25 | 26 | String value = entry[1].trim(); 27 | value = value.length() > MAX_LENGTH ? value.substring(0, MAX_LENGTH) : value; 28 | result.add(new Tuple<>(key, value)); 29 | } 30 | return result; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/config/converter/LabelMapConverter.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.raystack.depot.common.Tuple; 4 | import org.aeonbits.owner.Converter; 5 | 6 | import java.lang.reflect.Method; 7 | import java.util.List; 8 | import java.util.Map; 9 | import java.util.stream.Collectors; 10 | 11 | public class LabelMapConverter implements Converter> { 12 | 13 | public Map convert(Method method, String input) { 14 | List> listResult = ConverterUtils.convertToList(input); 15 | return listResult.stream().collect(Collectors.toMap(Tuple::getFirst, Tuple::getSecond)); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/config/converter/RedisSinkDataTypeConverter.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.raystack.depot.redis.enums.RedisSinkDataType; 4 | import org.aeonbits.owner.Converter; 5 | 6 | import java.lang.reflect.Method; 7 | 8 | public class RedisSinkDataTypeConverter implements Converter { 9 | @Override 10 | public RedisSinkDataType convert(Method method, String input) { 11 | return RedisSinkDataType.valueOf(input.toUpperCase()); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/config/converter/RedisSinkDeploymentTypeConverter.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.raystack.depot.redis.enums.RedisSinkDeploymentType; 4 | import org.aeonbits.owner.Converter; 5 | 6 | import java.lang.reflect.Method; 7 | 8 | public class RedisSinkDeploymentTypeConverter implements Converter { 9 | @Override 10 | public RedisSinkDeploymentType convert(Method method, String input) { 11 | return RedisSinkDeploymentType.valueOf(input.toUpperCase()); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/config/converter/RedisSinkTtlTypeConverter.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.raystack.depot.redis.enums.RedisSinkTtlType; 4 | import org.aeonbits.owner.Converter; 5 | 6 | import java.lang.reflect.Method; 7 | 8 | public class RedisSinkTtlTypeConverter implements Converter { 9 | @Override 10 | public RedisSinkTtlType convert(Method method, String input) { 11 | return RedisSinkTtlType.valueOf(input.toUpperCase()); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/config/converter/SchemaRegistryHeadersConverter.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.aeonbits.owner.Converter; 4 | import org.aeonbits.owner.Tokenizer; 5 | import org.apache.http.Header; 6 | import org.apache.http.message.BasicHeader; 7 | 8 | import java.lang.reflect.Method; 9 | import java.util.regex.Pattern; 10 | import java.util.stream.Collectors; 11 | 12 | public class SchemaRegistryHeadersConverter implements Converter
, Tokenizer { 13 | 14 | @Override 15 | public Header convert(Method method, String input) { 16 | String[] split = input.split(":"); 17 | return new BasicHeader(split[0].trim(), split[1].trim()); 18 | } 19 | 20 | @Override 21 | public String[] tokens(String values) { 22 | String[] headers = Pattern.compile(",").splitAsStream(values).map(String::trim) 23 | .filter(s -> { 24 | String[] args = s.split(":"); 25 | return args.length == 2 && args[0].trim().length() > 0 && args[1].trim().length() > 0; 26 | }) 27 | .collect(Collectors.toList()) 28 | .toArray(new String[0]); 29 | if (headers.length == 0) { 30 | throw new IllegalArgumentException(String.format("provided headers %s is not valid", values)); 31 | } 32 | 33 | return headers; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/config/converter/SchemaRegistryRefreshConverter.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.raystack.stencil.cache.SchemaRefreshStrategy; 4 | import org.aeonbits.owner.Converter; 5 | 6 | import java.lang.reflect.Method; 7 | 8 | public class SchemaRegistryRefreshConverter implements Converter { 9 | 10 | @Override 11 | public SchemaRefreshStrategy convert(Method method, String input) { 12 | if ("VERSION_BASED_REFRESH".equalsIgnoreCase(input)) { 13 | return SchemaRefreshStrategy.versionBasedRefresh(); 14 | } 15 | return SchemaRefreshStrategy.longPollingStrategy(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/config/converter/SinkConnectorSchemaDataTypeConverter.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.raystack.depot.config.enums.SinkConnectorSchemaDataType; 4 | import org.aeonbits.owner.Converter; 5 | 6 | import java.lang.reflect.Method; 7 | 8 | public class SinkConnectorSchemaDataTypeConverter implements Converter { 9 | @Override 10 | public SinkConnectorSchemaDataType convert(Method method, String input) { 11 | return SinkConnectorSchemaDataType.valueOf(input.toUpperCase()); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/config/converter/SinkConnectorSchemaMessageModeConverter.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.raystack.depot.message.SinkConnectorSchemaMessageMode; 4 | import org.aeonbits.owner.Converter; 5 | 6 | import java.lang.reflect.Method; 7 | 8 | public class SinkConnectorSchemaMessageModeConverter implements Converter { 9 | @Override 10 | public SinkConnectorSchemaMessageMode convert(Method method, String input) { 11 | return SinkConnectorSchemaMessageMode.valueOf(input.toUpperCase()); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/config/enums/SinkConnectorSchemaDataType.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.enums; 2 | 3 | public enum SinkConnectorSchemaDataType { 4 | PROTOBUF, 5 | JSON 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/error/ErrorInfo.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.error; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.EqualsAndHashCode; 6 | 7 | @AllArgsConstructor 8 | @Data 9 | public class ErrorInfo { 10 | 11 | @EqualsAndHashCode.Exclude 12 | private Exception exception; 13 | private ErrorType errorType; 14 | 15 | public String toString() { 16 | return String.format("Exception %s, ErrorType: %s", exception != null ? exception.getMessage() : "NULL", 17 | errorType.name()); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/error/ErrorType.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.error; 2 | 3 | public enum ErrorType { 4 | DESERIALIZATION_ERROR, 5 | INVALID_MESSAGE_ERROR, 6 | UNKNOWN_FIELDS_ERROR, 7 | SINK_4XX_ERROR, 8 | SINK_5XX_ERROR, 9 | SINK_RETRYABLE_ERROR, 10 | SINK_UNKNOWN_ERROR, 11 | DEFAULT_ERROR // Deprecated 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/exception/ConfigurationException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.exception; 2 | 3 | public class ConfigurationException extends RuntimeException { 4 | public ConfigurationException(String message) { 5 | super(message); 6 | } 7 | 8 | public ConfigurationException(String message, Throwable cause) { 9 | super(message, cause); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/exception/DeserializerException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.exception; 2 | 3 | /** 4 | * Deserializer exception is thrown when message from proto is not 5 | * deserializable into the Java object. 6 | */ 7 | public class DeserializerException extends RuntimeException { 8 | 9 | public DeserializerException(String message) { 10 | super(message); 11 | } 12 | 13 | public DeserializerException(String message, Exception e) { 14 | super(message, e); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/exception/EmptyMessageException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.exception; 2 | 3 | /** 4 | * Empty thrown when the message is contains zero bytes. 5 | */ 6 | public class EmptyMessageException extends DeserializerException { 7 | public EmptyMessageException() { 8 | super("log message is empty"); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/exception/InvalidTemplateException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.exception; 2 | 3 | public class InvalidTemplateException extends Exception { 4 | public InvalidTemplateException(String message) { 5 | super(message); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/exception/ProtoNotFoundException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.exception; 2 | 3 | public class ProtoNotFoundException extends RuntimeException { 4 | public ProtoNotFoundException(String message) { 5 | super(message); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/exception/SinkException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.exception; 2 | 3 | import java.io.IOException; 4 | 5 | public class SinkException extends IOException { 6 | public SinkException(String message, Throwable th) { 7 | super(message, th); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/exception/UnknownFieldsException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.exception; 2 | 3 | import com.google.protobuf.DynamicMessage; 4 | 5 | /** 6 | * UnknownFieldsException is thrown when unknown fields is detected on the log 7 | * message although the proto message was succesfuly parsed. 8 | * Unknown fields error can happen because multiple causes, and can be handled 9 | * differently depends on the use case. 10 | * Unknown fields error by default should be handled by retry the processing 11 | * because there is a probability that, message deserializer is not updated to 12 | * the latest schema 13 | * When consumer is deliberately process message using different schema and 14 | * intentionally ignore extra fields that missing from descriptor the error 15 | * handling can be disabled. 16 | * On some use case that need zero data loss, for example data warehousing 17 | * unknown fields error should be handled properly to prevent missing fields. 18 | */ 19 | public class UnknownFieldsException extends DeserializerException { 20 | 21 | public UnknownFieldsException(DynamicMessage dynamicMessage) { 22 | super(String.format("unknown fields found, message : %s", dynamicMessage)); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/log/LogSink.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.log; 2 | 3 | import org.raystack.depot.Sink; 4 | import org.raystack.depot.config.SinkConfig; 5 | import org.raystack.depot.error.ErrorInfo; 6 | import org.raystack.depot.error.ErrorType; 7 | import org.raystack.depot.exception.SinkException; 8 | import org.raystack.depot.message.Message; 9 | import org.raystack.depot.message.MessageParser; 10 | import org.raystack.depot.message.ParsedMessage; 11 | import org.raystack.depot.message.SinkConnectorSchemaMessageMode; 12 | import org.raystack.depot.metrics.Instrumentation; 13 | import org.raystack.depot.SinkResponse; 14 | 15 | import java.io.IOException; 16 | import java.util.List; 17 | 18 | public class LogSink implements Sink { 19 | private final MessageParser messageParser; 20 | private final Instrumentation instrumentation; 21 | private final SinkConfig config; 22 | 23 | public LogSink(SinkConfig config, MessageParser messageParser, Instrumentation instrumentation) { 24 | this.messageParser = messageParser; 25 | this.instrumentation = instrumentation; 26 | this.config = config; 27 | } 28 | 29 | @Override 30 | public SinkResponse pushToSink(List messages) throws SinkException { 31 | SinkResponse response = new SinkResponse(); 32 | SinkConnectorSchemaMessageMode mode = config.getSinkConnectorSchemaMessageMode(); 33 | String schemaClass = mode == SinkConnectorSchemaMessageMode.LOG_MESSAGE 34 | ? config.getSinkConnectorSchemaProtoMessageClass() 35 | : config.getSinkConnectorSchemaProtoKeyClass(); 36 | for (int ii = 0; ii < messages.size(); ii++) { 37 | Message message = messages.get(ii); 38 | try { 39 | ParsedMessage parsedMessage = messageParser.parse( 40 | message, 41 | mode, 42 | schemaClass); 43 | instrumentation.logInfo("\n================= DATA =======================\n{}" 44 | + "\n================= METADATA =======================\n{}\n", 45 | parsedMessage.toString(), message.getMetadataString()); 46 | } catch (IOException e) { 47 | response.addErrors(ii, new ErrorInfo(e, ErrorType.DESERIALIZATION_ERROR)); 48 | } 49 | } 50 | return response; 51 | } 52 | 53 | @Override 54 | public void close() throws IOException { 55 | 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/log/LogSinkFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.log; 2 | 3 | import org.raystack.depot.config.SinkConfig; 4 | import org.raystack.depot.message.MessageParser; 5 | import org.raystack.depot.message.MessageParserFactory; 6 | import org.raystack.depot.metrics.Instrumentation; 7 | import org.raystack.depot.metrics.StatsDReporter; 8 | import com.timgroup.statsd.NoOpStatsDClient; 9 | import org.raystack.depot.Sink; 10 | import org.aeonbits.owner.ConfigFactory; 11 | 12 | import java.util.Map; 13 | 14 | public class LogSinkFactory { 15 | 16 | private final StatsDReporter statsDReporter; 17 | private MessageParser messageParser; 18 | private final SinkConfig sinkConfig; 19 | 20 | public LogSinkFactory(Map env, StatsDReporter statsDReporter) { 21 | this(ConfigFactory.create(SinkConfig.class, env), statsDReporter); 22 | } 23 | 24 | public LogSinkFactory(SinkConfig sinkConfig, StatsDReporter statsDReporter) { 25 | this.sinkConfig = sinkConfig; 26 | this.statsDReporter = statsDReporter; 27 | } 28 | 29 | public LogSinkFactory(SinkConfig sinkConfig) { 30 | this(sinkConfig, new StatsDReporter(new NoOpStatsDClient())); 31 | } 32 | 33 | public void init() { 34 | this.messageParser = MessageParserFactory.getParser(sinkConfig, statsDReporter); 35 | } 36 | 37 | public Sink create() { 38 | return new LogSink(sinkConfig, messageParser, new Instrumentation(statsDReporter, LogSink.class)); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/Message.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message; 2 | 3 | import org.raystack.depot.common.Tuple; 4 | import org.raystack.depot.common.TupleString; 5 | import lombok.EqualsAndHashCode; 6 | import lombok.Getter; 7 | 8 | import java.util.Arrays; 9 | import java.util.HashMap; 10 | import java.util.List; 11 | import java.util.Map; 12 | import java.util.stream.Collectors; 13 | 14 | @Getter 15 | @EqualsAndHashCode 16 | public class Message { 17 | private final Object logKey; 18 | private final Object logMessage; 19 | private final Map metadata = new HashMap<>(); 20 | 21 | public String getMetadataString() { 22 | return metadata.keySet().stream() 23 | .map(key -> key + "=" + metadata.get(key)) 24 | .collect(Collectors.joining(", ", "{", "}")); 25 | } 26 | 27 | @SafeVarargs 28 | public Message(Object logKey, Object logMessage, Tuple... tuples) { 29 | this.logKey = logKey; 30 | this.logMessage = logMessage; 31 | Arrays.stream(tuples).forEach(t -> metadata.put(t.getFirst(), t.getSecond())); 32 | } 33 | 34 | public Map getMetadata(List metadataColumnsTypes) { 35 | return metadataColumnsTypes.stream() 36 | .collect(Collectors.toMap( 37 | TupleString::getFirst, columnAndType -> metadata.get(columnAndType.getFirst()))); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/MessageParser.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message; 2 | 3 | import java.io.IOException; 4 | 5 | public interface MessageParser { 6 | ParsedMessage parse(Message message, SinkConnectorSchemaMessageMode type, String schemaClass) throws IOException; 7 | 8 | MessageSchema getSchema(String schemaClass) throws IOException; 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/MessageParserFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message; 2 | 3 | import org.raystack.depot.message.json.JsonMessageParser; 4 | import org.raystack.depot.message.proto.ProtoMessageParser; 5 | import org.raystack.depot.metrics.Instrumentation; 6 | import org.raystack.depot.metrics.JsonParserMetrics; 7 | import org.raystack.depot.metrics.StatsDReporter; 8 | import org.raystack.depot.stencil.DepotStencilUpdateListener; 9 | import org.raystack.depot.config.SinkConfig; 10 | 11 | public class MessageParserFactory { 12 | public static MessageParser getParser(SinkConfig config, StatsDReporter statsDReporter, 13 | DepotStencilUpdateListener depotStencilUpdateListener) { 14 | switch (config.getSinkConnectorSchemaDataType()) { 15 | case JSON: 16 | return new JsonMessageParser(config, 17 | new Instrumentation(statsDReporter, JsonMessageParser.class), 18 | new JsonParserMetrics(config)); 19 | case PROTOBUF: 20 | return new ProtoMessageParser(config, statsDReporter, depotStencilUpdateListener); 21 | default: 22 | throw new IllegalArgumentException("Schema Type is not supported"); 23 | } 24 | } 25 | 26 | public static MessageParser getParser(SinkConfig config, StatsDReporter statsDReporter) { 27 | return getParser(config, statsDReporter, null); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/MessageSchema.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message; 2 | 3 | public interface MessageSchema { 4 | 5 | Object getSchema(); 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/MessageUtils.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message; 2 | 3 | import com.jayway.jsonpath.Configuration; 4 | import com.jayway.jsonpath.JsonPath; 5 | import com.jayway.jsonpath.PathNotFoundException; 6 | import org.json.JSONObject; 7 | 8 | import java.io.IOException; 9 | 10 | public class MessageUtils { 11 | 12 | public static Object getFieldFromJsonObject(String name, JSONObject jsonObject, Configuration jsonPathConfig) { 13 | try { 14 | String jsonPathName = "$." + name; 15 | JsonPath jsonPath = JsonPath.compile(jsonPathName); 16 | return jsonPath.read(jsonObject, jsonPathConfig); 17 | } catch (PathNotFoundException e) { 18 | throw new IllegalArgumentException("Invalid field config : " + name, e); 19 | } 20 | } 21 | 22 | public static void validate(Message message, Class validClass) throws IOException { 23 | if ((message.getLogKey() != null && !(validClass.isInstance(message.getLogKey()))) 24 | || (message.getLogMessage() != null && !(validClass.isInstance(message.getLogMessage())))) { 25 | throw new IOException( 26 | String.format("Expected class %s, but found: LogKey class: %s, LogMessage class: %s", 27 | validClass, 28 | message.getLogKey() != null ? message.getLogKey().getClass() : "n/a", 29 | message.getLogMessage() != null ? message.getLogMessage().getClass() : "n/a")); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/ParsedMessage.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message; 2 | 3 | import org.raystack.depot.config.SinkConfig; 4 | 5 | import java.io.IOException; 6 | import java.util.Map; 7 | 8 | public interface ParsedMessage { 9 | Object getRaw(); 10 | 11 | void validate(SinkConfig config); 12 | 13 | Map getMapping(MessageSchema schema) throws IOException; 14 | 15 | Object getFieldByName(String name, MessageSchema messageSchema); 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/SinkConnectorSchemaMessageMode.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message; 2 | 3 | public enum SinkConnectorSchemaMessageMode { 4 | LOG_KEY, 5 | LOG_MESSAGE 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/field/FieldUtils.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.field; 2 | 3 | import com.google.gson.Gson; 4 | import com.google.gson.GsonBuilder; 5 | 6 | import java.util.Collection; 7 | import java.util.function.Function; 8 | import java.util.stream.Collectors; 9 | 10 | public class FieldUtils { 11 | private static final Gson GSON = new GsonBuilder().create(); 12 | 13 | public static String convertToStringForMessageTypes(Object value, Function toStringFunc) { 14 | if (value instanceof Collection) { 15 | return "[" + ((Collection) value) 16 | .stream() 17 | .map(toStringFunc) 18 | .collect(Collectors.joining(",")) + "]"; 19 | } 20 | return toStringFunc.apply(value); 21 | } 22 | 23 | /** 24 | * This method is used to convert default types which string formats are not in 25 | * json. 26 | * for example: a list of doubles, strings, enum etc. 27 | */ 28 | public static String convertToString(Object value) { 29 | if (value instanceof Collection) { 30 | return GSON.toJson(value); 31 | } else { 32 | return value.toString(); 33 | } 34 | } 35 | 36 | public static String convertToStringForSpecialTypes(Object value, Function toStringFunc) { 37 | if (value instanceof Collection) { 38 | return GSON.toJson(((Collection) value) 39 | .stream() 40 | .map(toStringFunc) 41 | .collect(Collectors.toList())); 42 | } 43 | return toStringFunc.apply(value); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/field/GenericField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.field; 2 | 3 | public interface GenericField { 4 | String getString(); 5 | } 6 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/field/GenericFieldFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.field; 2 | 3 | import org.raystack.depot.message.field.json.JsonFieldFactory; 4 | import org.raystack.depot.message.field.proto.ProtoFieldFactory; 5 | import org.raystack.depot.message.proto.converter.fields.ProtoField; 6 | 7 | public class GenericFieldFactory { 8 | 9 | public static GenericField getField(Object field) { 10 | if (field instanceof ProtoField) { 11 | return ProtoFieldFactory.getField((ProtoField) field); 12 | } else { 13 | return JsonFieldFactory.getField(field); 14 | } 15 | 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/field/json/DefaultField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.field.json; 2 | 3 | import org.raystack.depot.message.field.GenericField; 4 | 5 | public class DefaultField implements GenericField { 6 | private final Object value; 7 | 8 | public DefaultField(Object field) { 9 | this.value = field; 10 | } 11 | 12 | @Override 13 | public String getString() { 14 | return value.toString(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/field/json/JsonFieldFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.field.json; 2 | 3 | import org.raystack.depot.message.field.GenericField; 4 | 5 | public class JsonFieldFactory { 6 | public static GenericField getField(Object field) { 7 | return new DefaultField(field); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/field/proto/DefaultField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.field.proto; 2 | 3 | import org.raystack.depot.message.field.FieldUtils; 4 | import org.raystack.depot.message.field.GenericField; 5 | 6 | public class DefaultField implements GenericField { 7 | private final Object value; 8 | 9 | public DefaultField(Object value) { 10 | this.value = value; 11 | } 12 | 13 | @Override 14 | public String getString() { 15 | return FieldUtils.convertToString(value); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/field/proto/DurationField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.field.proto; 2 | 3 | import com.google.protobuf.Descriptors; 4 | import com.google.protobuf.Message; 5 | import org.raystack.depot.message.field.FieldUtils; 6 | import org.raystack.depot.message.field.GenericField; 7 | 8 | public class DurationField implements GenericField { 9 | private static final double NANO = 1e-9; 10 | private final Object value; 11 | 12 | public DurationField(Object value) { 13 | this.value = value; 14 | } 15 | 16 | @Override 17 | public String getString() { 18 | return FieldUtils.convertToStringForSpecialTypes(value, this::getDurationString); 19 | } 20 | 21 | private String getDurationString(Object field) { 22 | Message message = (Message) field; 23 | Descriptors.FieldDescriptor secondsDescriptor = message.getDescriptorForType().findFieldByName("seconds"); 24 | Descriptors.FieldDescriptor nanosDescriptor = message.getDescriptorForType().findFieldByName("nanos"); 25 | long seconds = (Long) message.getField(secondsDescriptor); 26 | int nanos = (Integer) message.getField(nanosDescriptor); 27 | if (nanos != 0) { 28 | return (seconds + (nanos * NANO)) + "s"; 29 | } else { 30 | return seconds + "s"; 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/field/proto/MapField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.field.proto; 2 | 3 | import com.google.protobuf.Descriptors; 4 | import com.google.protobuf.Message; 5 | import org.raystack.depot.message.field.GenericField; 6 | import org.json.JSONObject; 7 | 8 | import java.util.Collection; 9 | 10 | public class MapField implements GenericField { 11 | private final Object value; 12 | 13 | public MapField(Object value) { 14 | this.value = value; 15 | } 16 | 17 | @Override 18 | public String getString() { 19 | JSONObject json = new JSONObject(); 20 | ((Collection) value).forEach(m -> { 21 | Message mapEntry = (Message) m; 22 | String k = mapEntry.getField(mapEntry.getDescriptorForType().findFieldByName("key")).toString(); 23 | Descriptors.FieldDescriptor vDescriptor = mapEntry.getDescriptorForType().findFieldByName("value"); 24 | Object v = mapEntry.getField(vDescriptor); 25 | if (vDescriptor.getType() == Descriptors.FieldDescriptor.Type.MESSAGE 26 | && vDescriptor.getMessageType().getFullName() 27 | .equals(com.google.protobuf.Timestamp.getDescriptor().getFullName())) { 28 | json.put(k, new TimeStampField(TimeStampField.getInstant(v)).getString()); 29 | } else if (vDescriptor.getType() == Descriptors.FieldDescriptor.Type.MESSAGE 30 | && vDescriptor.getMessageType().getFullName() 31 | .equals(com.google.protobuf.Duration.getDescriptor().getFullName())) { 32 | json.put(k, new DurationField(v).getString()); 33 | } else if (vDescriptor.getType() == Descriptors.FieldDescriptor.Type.MESSAGE) { 34 | json.put(k, new JSONObject(new MessageField(v).getString())); 35 | } else { 36 | json.put(k, new DefaultField(v).getString()); 37 | } 38 | }); 39 | return json.toString(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/field/proto/MessageField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.field.proto; 2 | 3 | import com.google.protobuf.InvalidProtocolBufferException; 4 | import com.google.protobuf.Message; 5 | import com.google.protobuf.util.JsonFormat; 6 | import org.raystack.depot.message.field.FieldUtils; 7 | import org.raystack.depot.message.field.GenericField; 8 | 9 | public class MessageField implements GenericField { 10 | private static final JsonFormat.Printer JSON_PRINTER = JsonFormat.printer() 11 | .omittingInsignificantWhitespace() 12 | .preservingProtoFieldNames() 13 | .includingDefaultValueFields(); 14 | 15 | private final Object value; 16 | 17 | public MessageField(Object value) { 18 | this.value = value; 19 | } 20 | 21 | @Override 22 | public String getString() { 23 | return FieldUtils.convertToStringForMessageTypes(value, this::getMessageString); 24 | } 25 | 26 | private String getMessageString(Object ob) { 27 | try { 28 | return JSON_PRINTER.print((Message) ob); 29 | } catch (InvalidProtocolBufferException e) { 30 | throw new IllegalArgumentException(e); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/field/proto/ProtoFieldFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.field.proto; 2 | 3 | import org.raystack.depot.message.field.GenericField; 4 | import org.raystack.depot.message.proto.converter.fields.DurationProtoField; 5 | import org.raystack.depot.message.proto.converter.fields.MapProtoField; 6 | import org.raystack.depot.message.proto.converter.fields.MessageProtoField; 7 | import org.raystack.depot.message.proto.converter.fields.ProtoField; 8 | import org.raystack.depot.message.proto.converter.fields.StructProtoField; 9 | import org.raystack.depot.message.proto.converter.fields.TimestampProtoField; 10 | 11 | public class ProtoFieldFactory { 12 | public static GenericField getField(ProtoField field) { 13 | if (field instanceof DurationProtoField) { 14 | return new DurationField(field.getValue()); 15 | } 16 | if (field instanceof MessageProtoField) { 17 | return new MessageField(field.getValue()); 18 | } 19 | if (field instanceof MapProtoField) { 20 | return new MapField(field.getValue()); 21 | } 22 | if (field instanceof StructProtoField) { 23 | return new StructField(field.getValue()); 24 | } 25 | if (field instanceof TimestampProtoField) { 26 | return new TimeStampField(field.getValue()); 27 | } 28 | return new DefaultField(field.getValue()); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/field/proto/StructField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.field.proto; 2 | 3 | import org.raystack.depot.message.field.FieldUtils; 4 | import org.raystack.depot.message.field.GenericField; 5 | 6 | public class StructField implements GenericField { 7 | private final Object value; 8 | 9 | public StructField(Object value) { 10 | this.value = value; 11 | } 12 | 13 | @Override 14 | public String getString() { 15 | // This Struct is already converted into strings, so we just need to concat it. 16 | return FieldUtils.convertToStringForMessageTypes(value, Object::toString); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/field/proto/TimeStampField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.field.proto; 2 | 3 | import com.google.protobuf.Message; 4 | import org.raystack.depot.message.field.FieldUtils; 5 | import org.raystack.depot.message.field.GenericField; 6 | 7 | import java.time.Instant; 8 | 9 | public class TimeStampField implements GenericField { 10 | private final Object value; 11 | 12 | public TimeStampField(Object value) { 13 | this.value = value; 14 | } 15 | 16 | public static Instant getInstant(Object field) { 17 | Message m = (Message) field; 18 | Object seconds = m.getField(m.getDescriptorForType().findFieldByName("seconds")); 19 | Object nanos = m.getField(m.getDescriptorForType().findFieldByName("nanos")); 20 | return Instant.ofEpochSecond((long) seconds, ((Integer) nanos).longValue()); 21 | } 22 | 23 | @Override 24 | public String getString() { 25 | return FieldUtils.convertToStringForSpecialTypes(value, Object::toString); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/json/JsonParsedMessage.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.json; 2 | 3 | import com.jayway.jsonpath.Configuration; 4 | import org.raystack.depot.config.SinkConfig; 5 | import org.raystack.depot.message.MessageUtils; 6 | import org.raystack.depot.message.MessageSchema; 7 | import org.raystack.depot.message.ParsedMessage; 8 | import org.json.JSONObject; 9 | 10 | import java.util.Collections; 11 | import java.util.Map; 12 | 13 | public class JsonParsedMessage implements ParsedMessage { 14 | private final JSONObject jsonObject; 15 | private final Configuration jsonPathConfig; 16 | 17 | public JsonParsedMessage(JSONObject jsonObject, Configuration jsonPathConfig) { 18 | this.jsonObject = jsonObject; 19 | this.jsonPathConfig = jsonPathConfig; 20 | } 21 | 22 | public String toString() { 23 | return jsonObject.toString(); 24 | } 25 | 26 | @Override 27 | public Object getRaw() { 28 | return jsonObject; 29 | } 30 | 31 | @Override 32 | public void validate(SinkConfig config) { 33 | 34 | } 35 | 36 | @Override 37 | public Map getMapping(MessageSchema schema) { 38 | if (jsonObject == null || jsonObject.isEmpty()) { 39 | return Collections.emptyMap(); 40 | } 41 | return jsonObject.toMap(); 42 | } 43 | 44 | public Object getFieldByName(String name, MessageSchema messageSchema) { 45 | if (name == null || name.isEmpty()) { 46 | throw new IllegalArgumentException("Invalid field config : name can not be empty"); 47 | } 48 | return MessageUtils.getFieldFromJsonObject(name, jsonObject, jsonPathConfig); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/Constants.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto; 2 | 3 | public class Constants { 4 | public static class Config { 5 | public static final String RECORD_NAME = "record_name"; 6 | } 7 | 8 | public static class ProtobufTypeName { 9 | public static final String TIMESTAMP_PROTOBUF_TYPE_NAME = ".google.protobuf.Timestamp"; 10 | public static final String STRUCT_PROTOBUF_TYPE_NAME = ".google.protobuf.Struct"; 11 | public static final String DURATION_PROTOBUF_TYPE_NAME = ".google.protobuf.Duration"; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/DescriptorCache.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto; 2 | 3 | import com.google.protobuf.Descriptors; 4 | 5 | import java.util.Map; 6 | 7 | public class DescriptorCache { 8 | public Descriptors.Descriptor fetch(Map allDescriptors, 9 | Map typeNameToPackageNameMap, String protoName) { 10 | if (allDescriptors.get(protoName) != null) { 11 | return allDescriptors.get(protoName); 12 | } 13 | String packageName = typeNameToPackageNameMap.get(protoName); 14 | if (packageName == null) { 15 | return null; 16 | } 17 | return allDescriptors.get(packageName); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/ProtoFieldParser.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto; 2 | 3 | import com.google.protobuf.Descriptors; 4 | import org.raystack.depot.exception.ProtoNotFoundException; 5 | 6 | import java.util.Map; 7 | 8 | public class ProtoFieldParser { 9 | /** 10 | * We support nested data type of 15 or less level. 11 | * We limit the fields to only contain schema upto 15 levels deep 12 | */ 13 | private static final int MAX_NESTED_SCHEMA_LEVEL = 15; 14 | private final DescriptorCache descriptorCache = new DescriptorCache(); 15 | 16 | public ProtoField parseFields(ProtoField protoField, String protoSchema, 17 | Map allDescriptors, 18 | Map typeNameToPackageNameMap) { 19 | return parseFields(protoField, protoSchema, allDescriptors, typeNameToPackageNameMap, 1); 20 | } 21 | 22 | private ProtoField parseFields(ProtoField protoField, String protoSchema, 23 | Map allDescriptors, 24 | Map typeNameToPackageNameMap, int level) { 25 | 26 | Descriptors.Descriptor currentProto = descriptorCache.fetch(allDescriptors, typeNameToPackageNameMap, 27 | protoSchema); 28 | if (currentProto == null) { 29 | throw new ProtoNotFoundException("No Proto found for class " + protoSchema); 30 | } 31 | for (Descriptors.FieldDescriptor field : currentProto.getFields()) { 32 | ProtoField fieldModel = new ProtoField(field.toProto()); 33 | if (fieldModel.isNested()) { 34 | if (protoSchema.substring(1).equals(currentProto.getFullName())) { 35 | if (level >= MAX_NESTED_SCHEMA_LEVEL) { 36 | continue; 37 | } 38 | } 39 | fieldModel = parseFields(fieldModel, field.toProto().getTypeName(), allDescriptors, 40 | typeNameToPackageNameMap, level + 1); 41 | } 42 | protoField.addField(fieldModel); 43 | } 44 | return protoField; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/ProtoMapper.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.fasterxml.jackson.databind.node.JsonNodeFactory; 5 | import com.fasterxml.jackson.databind.node.ObjectNode; 6 | 7 | import java.io.IOException; 8 | import java.util.List; 9 | 10 | public class ProtoMapper { 11 | 12 | public static String generateColumnMappings(List fields) throws IOException { 13 | ObjectMapper objectMapper = new ObjectMapper(); 14 | ObjectNode objectNode = generateColumnMappingsJson(fields); 15 | return objectMapper.writeValueAsString(objectNode); 16 | } 17 | 18 | private static ObjectNode generateColumnMappingsJson(List fields) { 19 | if (fields.size() == 0) { 20 | return JsonNodeFactory.instance.objectNode(); 21 | } 22 | 23 | ObjectNode objNode = JsonNodeFactory.instance.objectNode(); 24 | for (ProtoField field : fields) { 25 | if (field.isNested()) { 26 | ObjectNode innerJSONValue = generateColumnMappingsJson(field.getFields()); 27 | innerJSONValue.put(Constants.Config.RECORD_NAME, field.getName()); 28 | objNode.set(String.valueOf(field.getIndex()), innerJSONValue); 29 | } else { 30 | objNode.put(String.valueOf(field.getIndex()), field.getName()); 31 | } 32 | } 33 | return objNode; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/ProtoMessageSchema.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto; 2 | 3 | import com.google.gson.Gson; 4 | import com.google.gson.reflect.TypeToken; 5 | import org.raystack.depot.message.MessageSchema; 6 | import lombok.EqualsAndHashCode; 7 | import lombok.Getter; 8 | 9 | import java.io.IOException; 10 | import java.lang.reflect.Type; 11 | import java.util.Map; 12 | import java.util.Properties; 13 | 14 | @EqualsAndHashCode 15 | public class ProtoMessageSchema implements MessageSchema { 16 | 17 | @Getter 18 | private final ProtoField protoField; 19 | private static final Gson GSON = new Gson(); 20 | private final Properties properties; 21 | 22 | public ProtoMessageSchema(ProtoField protoField) throws IOException { 23 | this(protoField, createProperties(protoField)); 24 | } 25 | 26 | public ProtoMessageSchema(ProtoField protoField, Properties properties) { 27 | this.protoField = protoField; 28 | this.properties = properties; 29 | } 30 | 31 | @Override 32 | public Properties getSchema() { 33 | return this.properties; 34 | } 35 | 36 | private static Properties createProperties(ProtoField protoField) throws IOException { 37 | String protoMappingString = ProtoMapper.generateColumnMappings(protoField.getFields()); 38 | Type type = new TypeToken>() { 39 | }.getType(); 40 | Map m = GSON.fromJson(protoMappingString, type); 41 | return mapToProperties(m); 42 | } 43 | 44 | private static Properties mapToProperties(Map inputMap) { 45 | Properties properties = new Properties(); 46 | for (Map.Entry kv : inputMap.entrySet()) { 47 | if (kv.getValue() instanceof String) { 48 | properties.put(kv.getKey(), kv.getValue()); 49 | } else if (kv.getValue() instanceof Map) { 50 | properties.put(kv.getKey(), mapToProperties((Map) kv.getValue())); 51 | } 52 | } 53 | return properties; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/UnknownProtoFields.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto; 2 | 3 | import com.google.protobuf.InvalidProtocolBufferException; 4 | import com.google.protobuf.UnknownFieldSet; 5 | import lombok.extern.slf4j.Slf4j; 6 | 7 | /** 8 | * Try to convert raw proto bytes to some meaningful representation that is good 9 | * enough for debug. 10 | */ 11 | @Slf4j 12 | public class UnknownProtoFields { 13 | public static String toString(byte[] message) { 14 | String convertedFields = ""; 15 | try { 16 | convertedFields = UnknownFieldSet.parseFrom(message).toString(); 17 | } catch (InvalidProtocolBufferException e) { 18 | log.warn("invalid byte representation of a protobuf message: {}", new String(message)); 19 | } 20 | return convertedFields; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/converter/fields/ByteProtoField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | import com.google.protobuf.ByteString; 4 | import com.google.protobuf.Descriptors; 5 | import lombok.AllArgsConstructor; 6 | 7 | import java.util.Base64; 8 | import java.util.Collection; 9 | import java.util.stream.Collectors; 10 | 11 | @AllArgsConstructor 12 | public class ByteProtoField implements ProtoField { 13 | 14 | private final Descriptors.FieldDescriptor descriptor; 15 | private final Object fieldValue; 16 | 17 | @Override 18 | public Object getValue() { 19 | if (fieldValue instanceof Collection) { 20 | return ((Collection) fieldValue).stream().map(this::getByteString).collect(Collectors.toList()); 21 | } 22 | return getByteString(fieldValue); 23 | } 24 | 25 | private Object getByteString(Object field) { 26 | ByteString byteString = (ByteString) field; 27 | byte[] bytes = byteString.toStringUtf8().getBytes(); 28 | return base64Encode(bytes); 29 | } 30 | 31 | private String base64Encode(byte[] bytes) { 32 | return new String(Base64.getEncoder().encode(bytes)); 33 | } 34 | 35 | @Override 36 | public boolean matches() { 37 | return descriptor.getType() == Descriptors.FieldDescriptor.Type.BYTES; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/converter/fields/DefaultProtoField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | import lombok.AllArgsConstructor; 4 | 5 | @AllArgsConstructor 6 | public class DefaultProtoField implements ProtoField { 7 | private final Object fieldValue; 8 | 9 | @Override 10 | public Object getValue() { 11 | return fieldValue; 12 | } 13 | 14 | @Override 15 | public boolean matches() { 16 | return false; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/converter/fields/DurationProtoField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | import com.google.protobuf.Descriptors; 4 | 5 | public class DurationProtoField implements ProtoField { 6 | private final Descriptors.FieldDescriptor descriptor; 7 | private final Object fieldValue; 8 | 9 | public DurationProtoField(Descriptors.FieldDescriptor descriptor, Object fieldValue) { 10 | this.descriptor = descriptor; 11 | this.fieldValue = fieldValue; 12 | } 13 | 14 | @Override 15 | public Object getValue() { 16 | return fieldValue; 17 | } 18 | 19 | @Override 20 | public boolean matches() { 21 | return descriptor.getType() == Descriptors.FieldDescriptor.Type.MESSAGE 22 | && descriptor.getMessageType().getFullName() 23 | .equals(com.google.protobuf.Duration.getDescriptor().getFullName()); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/converter/fields/EnumProtoField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | import com.google.protobuf.Descriptors; 4 | import lombok.AllArgsConstructor; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | @AllArgsConstructor 10 | public class EnumProtoField implements ProtoField { 11 | private final Descriptors.FieldDescriptor descriptor; 12 | private final Object fieldValue; 13 | 14 | @Override 15 | public Object getValue() { 16 | if (descriptor.isRepeated()) { 17 | List enumValues = ((List) (fieldValue)); 18 | List enumStrValues = new ArrayList<>(); 19 | for (Descriptors.EnumValueDescriptor enumVal : enumValues) { 20 | enumStrValues.add(enumVal.toString()); 21 | } 22 | return enumStrValues; 23 | } 24 | return fieldValue.toString(); 25 | } 26 | 27 | @Override 28 | public boolean matches() { 29 | return descriptor.getType() == Descriptors.FieldDescriptor.Type.ENUM; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/converter/fields/FloatProtoField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | import com.google.protobuf.Descriptors; 4 | 5 | import java.util.Collection; 6 | import java.util.stream.Collectors; 7 | 8 | public class FloatProtoField implements ProtoField { 9 | private final Object fieldValue; 10 | private final Descriptors.FieldDescriptor descriptor; 11 | 12 | public FloatProtoField(Descriptors.FieldDescriptor descriptor, Object fieldValue) { 13 | this.descriptor = descriptor; 14 | this.fieldValue = fieldValue; 15 | } 16 | 17 | @Override 18 | public Object getValue() { 19 | if (fieldValue instanceof Collection) { 20 | return ((Collection) fieldValue).stream().map(this::getValue).collect(Collectors.toList()); 21 | } 22 | return getValue(fieldValue); 23 | } 24 | 25 | public Double getValue(Object field) { 26 | double val = Double.parseDouble(field.toString()); 27 | boolean valid = !Double.isInfinite(val) && !Double.isNaN(val); 28 | if (!valid) { 29 | throw new IllegalArgumentException("Float/double value is not valid"); 30 | } 31 | return val; 32 | } 33 | 34 | @Override 35 | public boolean matches() { 36 | return descriptor.getType() == Descriptors.FieldDescriptor.Type.FLOAT 37 | || descriptor.getType() == Descriptors.FieldDescriptor.Type.DOUBLE; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/converter/fields/IntegerProtoField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | import com.google.protobuf.Descriptors; 4 | 5 | import java.util.Collection; 6 | import java.util.stream.Collectors; 7 | 8 | public class IntegerProtoField implements ProtoField { 9 | private final Descriptors.FieldDescriptor descriptor; 10 | private final Object fieldValue; 11 | 12 | public IntegerProtoField(Descriptors.FieldDescriptor descriptor, Object fieldValue) { 13 | this.descriptor = descriptor; 14 | this.fieldValue = fieldValue; 15 | } 16 | 17 | @Override 18 | public Object getValue() { 19 | if (fieldValue instanceof Collection) { 20 | return ((Collection) fieldValue).stream().map(this::getValue).collect(Collectors.toList()); 21 | } 22 | return getValue(fieldValue); 23 | } 24 | 25 | public Long getValue(Object field) { 26 | return Long.valueOf(field.toString()); 27 | } 28 | 29 | @Override 30 | public boolean matches() { 31 | return descriptor.getType() == Descriptors.FieldDescriptor.Type.INT64 32 | || descriptor.getType() == Descriptors.FieldDescriptor.Type.UINT64 33 | || descriptor.getType() == Descriptors.FieldDescriptor.Type.FIXED64 34 | || descriptor.getType() == Descriptors.FieldDescriptor.Type.SFIXED64 35 | || descriptor.getType() == Descriptors.FieldDescriptor.Type.SINT64 36 | || descriptor.getType() == Descriptors.FieldDescriptor.Type.INT32 37 | || descriptor.getType() == Descriptors.FieldDescriptor.Type.UINT32 38 | || descriptor.getType() == Descriptors.FieldDescriptor.Type.FIXED32 39 | || descriptor.getType() == Descriptors.FieldDescriptor.Type.SFIXED32 40 | || descriptor.getType() == Descriptors.FieldDescriptor.Type.SINT32; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/converter/fields/MapProtoField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | import com.google.protobuf.Descriptors; 4 | 5 | public class MapProtoField implements ProtoField { 6 | 7 | private final Descriptors.FieldDescriptor descriptor; 8 | private final Object fieldValue; 9 | 10 | public MapProtoField(Descriptors.FieldDescriptor descriptor, Object fieldValue) { 11 | this.descriptor = descriptor; 12 | this.fieldValue = fieldValue; 13 | } 14 | 15 | @Override 16 | public Object getValue() { 17 | return fieldValue; 18 | } 19 | 20 | @Override 21 | public boolean matches() { 22 | return descriptor.isMapField(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/converter/fields/MessageProtoField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | import com.google.protobuf.Descriptors; 4 | import lombok.AllArgsConstructor; 5 | 6 | @AllArgsConstructor 7 | public class MessageProtoField implements ProtoField { 8 | private final Descriptors.FieldDescriptor descriptor; 9 | private final Object fieldValue; 10 | 11 | @Override 12 | public Object getValue() { 13 | return fieldValue; 14 | } 15 | 16 | @Override 17 | public boolean matches() { 18 | return descriptor.getType() == Descriptors.FieldDescriptor.Type.MESSAGE; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/converter/fields/ProtoField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | public interface ProtoField { 4 | 5 | Object getValue(); 6 | 7 | boolean matches(); 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/converter/fields/ProtoFieldFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | import com.google.protobuf.Descriptors; 4 | 5 | import java.util.Arrays; 6 | import java.util.List; 7 | import java.util.Optional; 8 | 9 | public class ProtoFieldFactory { 10 | 11 | public static ProtoField getField(Descriptors.FieldDescriptor descriptor, Object fieldValue) { 12 | List protoFields = Arrays.asList( 13 | new DurationProtoField(descriptor, fieldValue), 14 | new TimestampProtoField(descriptor, fieldValue), 15 | new EnumProtoField(descriptor, fieldValue), 16 | new StructProtoField(descriptor, fieldValue), 17 | new FloatProtoField(descriptor, fieldValue), 18 | new IntegerProtoField(descriptor, fieldValue), 19 | new MessageProtoField(descriptor, fieldValue)); 20 | Optional first = protoFields 21 | .stream() 22 | .filter(ProtoField::matches) 23 | .findFirst(); 24 | return first.orElseGet(() -> new DefaultProtoField(fieldValue)); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/converter/fields/StructProtoField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | import com.google.protobuf.Descriptors; 4 | import com.google.protobuf.InvalidProtocolBufferException; 5 | import com.google.protobuf.Message; 6 | import com.google.protobuf.util.JsonFormat; 7 | import lombok.AllArgsConstructor; 8 | 9 | import java.util.ArrayList; 10 | import java.util.Collection; 11 | import java.util.List; 12 | 13 | @AllArgsConstructor 14 | public class StructProtoField implements ProtoField { 15 | private static JsonFormat.Printer printer = JsonFormat.printer() 16 | .preservingProtoFieldNames() 17 | .omittingInsignificantWhitespace(); 18 | private final Descriptors.FieldDescriptor descriptor; 19 | private final Object fieldValue; 20 | 21 | @Override 22 | public Object getValue() { 23 | try { 24 | if (fieldValue instanceof Collection) { 25 | List structStrValues = new ArrayList<>(); 26 | for (Object field : (Collection) fieldValue) { 27 | structStrValues.add(getString(field)); 28 | } 29 | return structStrValues; 30 | } 31 | return getString(fieldValue); 32 | } catch (InvalidProtocolBufferException e) { 33 | return ""; 34 | } 35 | } 36 | 37 | private String getString(Object field) throws InvalidProtocolBufferException { 38 | return printer.print((Message) field); 39 | } 40 | 41 | @Override 42 | public boolean matches() { 43 | return descriptor.getType() == Descriptors.FieldDescriptor.Type.MESSAGE 44 | && descriptor.getMessageType().getFullName() 45 | .equals(com.google.protobuf.Struct.getDescriptor().getFullName()); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/message/proto/converter/fields/TimestampProtoField.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | import com.google.protobuf.Descriptors; 4 | import com.google.protobuf.DynamicMessage; 5 | import lombok.AllArgsConstructor; 6 | 7 | import java.time.Instant; 8 | import java.util.ArrayList; 9 | import java.util.Collection; 10 | import java.util.List; 11 | import java.util.stream.Collectors; 12 | 13 | @AllArgsConstructor 14 | public class TimestampProtoField implements ProtoField { 15 | private final Descriptors.FieldDescriptor descriptor; 16 | private final Object fieldValue; 17 | 18 | @Override 19 | public Object getValue() { 20 | if (fieldValue instanceof Collection) { 21 | return ((Collection) fieldValue).stream().map(this::getTime).collect(Collectors.toList()); 22 | } 23 | return getTime(fieldValue); 24 | } 25 | 26 | private Instant getTime(Object field) { 27 | DynamicMessage dynamicField = (DynamicMessage) field; 28 | List descriptors = dynamicField.getDescriptorForType().getFields(); 29 | List timeFields = new ArrayList<>(); 30 | descriptors.forEach(desc -> timeFields.add(dynamicField.getField(desc))); 31 | return Instant.ofEpochSecond((long) timeFields.get(0), ((Integer) timeFields.get(1)).longValue()); 32 | } 33 | 34 | @Override 35 | public boolean matches() { 36 | return descriptor.getType() == Descriptors.FieldDescriptor.Type.MESSAGE 37 | && descriptor.getMessageType().getFullName() 38 | .equals(com.google.protobuf.Timestamp.getDescriptor().getFullName()); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/metrics/BigQueryMetrics.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.metrics; 2 | 3 | import org.raystack.depot.config.SinkConfig; 4 | 5 | public class BigQueryMetrics extends SinkMetrics { 6 | 7 | public BigQueryMetrics(SinkConfig config) { 8 | super(config); 9 | } 10 | 11 | public enum BigQueryAPIType { 12 | TABLE_UPDATE, 13 | TABLE_CREATE, 14 | DATASET_UPDATE, 15 | DATASET_CREATE, 16 | TABLE_INSERT_ALL, 17 | } 18 | 19 | public enum BigQueryStorageAPIType { 20 | STREAM_WRITER_CREATED, 21 | STREAM_WRITER_CLOSED, 22 | STREAM_WRITER_APPEND 23 | } 24 | 25 | public enum BigQueryStorageAPIError { 26 | ROW_APPEND_ERROR 27 | } 28 | 29 | public enum BigQueryErrorType { 30 | UNKNOWN_ERROR, 31 | INVALID_SCHEMA_ERROR, 32 | OOB_ERROR, 33 | STOPPED_ERROR, 34 | } 35 | 36 | public static final String BIGQUERY_SINK_PREFIX = "bigquery_"; 37 | public static final String BIGQUERY_TABLE_TAG = "table=%s"; 38 | public static final String BIGQUERY_DATASET_TAG = "dataset=%s"; 39 | public static final String BIGQUERY_PROJECT_TAG = "project=%s"; 40 | public static final String BIGQUERY_API_TAG = "api=%s"; 41 | public static final String BIGQUERY_ERROR_TAG = "error=%s"; 42 | 43 | public String getBigqueryOperationTotalMetric() { 44 | return getApplicationPrefix() + SINK_PREFIX + BIGQUERY_SINK_PREFIX + "operation_total"; 45 | } 46 | 47 | public String getBigqueryOperationLatencyMetric() { 48 | return getApplicationPrefix() + SINK_PREFIX + BIGQUERY_SINK_PREFIX + "operation_latency_milliseconds"; 49 | } 50 | 51 | public String getBigqueryTotalErrorsMetrics() { 52 | return getApplicationPrefix() + SINK_PREFIX + BIGQUERY_SINK_PREFIX + "errors_total"; 53 | } 54 | 55 | public String getBigqueryPayloadSizeMetrics() { 56 | return getApplicationPrefix() + SINK_PREFIX + BIGQUERY_SINK_PREFIX + "payload_size_bytes"; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/metrics/BigTableMetrics.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.metrics; 2 | 3 | import org.raystack.depot.config.SinkConfig; 4 | 5 | public class BigTableMetrics extends SinkMetrics { 6 | 7 | public static final String BIGTABLE_SINK_PREFIX = "bigtable_"; 8 | public static final String BIGTABLE_INSTANCE_TAG = "instance=%s"; 9 | public static final String BIGTABLE_TABLE_TAG = "table=%s"; 10 | public static final String BIGTABLE_ERROR_TAG = "error=%s"; 11 | 12 | public BigTableMetrics(SinkConfig config) { 13 | super(config); 14 | } 15 | 16 | public enum BigTableErrorType { 17 | QUOTA_FAILURE, // A quota check failed. 18 | PRECONDITION_FAILURE, // Some preconditions have failed. 19 | BAD_REQUEST, // Violations in a client request 20 | RPC_FAILURE, 21 | } 22 | 23 | public String getBigtableOperationLatencyMetric() { 24 | return getApplicationPrefix() + SINK_PREFIX + BIGTABLE_SINK_PREFIX + "operation_latency_milliseconds"; 25 | } 26 | 27 | public String getBigtableOperationTotalMetric() { 28 | return getApplicationPrefix() + SINK_PREFIX + BIGTABLE_SINK_PREFIX + "operation_total"; 29 | } 30 | 31 | public String getBigtableTotalErrorsMetrics() { 32 | return getApplicationPrefix() + SINK_PREFIX + BIGTABLE_SINK_PREFIX + "errors_total"; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/metrics/JsonParserMetrics.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.metrics; 2 | 3 | import org.raystack.depot.config.SinkConfig; 4 | 5 | public class JsonParserMetrics extends SinkMetrics { 6 | public JsonParserMetrics(SinkConfig config) { 7 | super(config); 8 | } 9 | 10 | public static final String JSON_PARSE_PREFIX = "json_parse_"; 11 | 12 | public String getJsonParseTimeTakenMetric() { 13 | return getApplicationPrefix() + SINK_PREFIX + JSON_PARSE_PREFIX + "operation_milliseconds"; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/metrics/SinkMetrics.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.metrics; 2 | 3 | import org.raystack.depot.config.SinkConfig; 4 | import lombok.Getter; 5 | 6 | public class SinkMetrics { 7 | public static final String SINK_PREFIX = "sink_"; 8 | // ERROR TAGS 9 | public static final String ERROR_TYPE_TAG = "error_type=%s"; 10 | public static final String ERROR_PREFIX = "error_"; 11 | public static final String ERROR_MESSAGE_CLASS_TAG = "class"; 12 | public static final String NON_FATAL_ERROR = "nonfatal"; 13 | public static final String FATAL_ERROR = "fatal"; 14 | 15 | @Getter 16 | private final String applicationPrefix; 17 | 18 | public SinkMetrics(SinkConfig config) { 19 | this.applicationPrefix = config.getMetricsApplicationPrefix(); 20 | } 21 | 22 | public String getErrorEventMetric() { 23 | return applicationPrefix + ERROR_PREFIX + "event"; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/metrics/StatsDReporterBuilder.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.metrics; 2 | 3 | import com.timgroup.statsd.NoOpStatsDClient; 4 | import com.timgroup.statsd.NonBlockingStatsDClientBuilder; 5 | import com.timgroup.statsd.StatsDClient; 6 | import org.raystack.depot.config.MetricsConfig; 7 | import lombok.extern.slf4j.Slf4j; 8 | 9 | /** 10 | * StatsDReporterFactory 11 | *

12 | * Create statsDReporter Instance. 13 | */ 14 | @Slf4j 15 | public class StatsDReporterBuilder { 16 | 17 | private MetricsConfig metricsConfig; 18 | private String[] extraTags; 19 | 20 | private static T[] append(T[] arr, T lastElement) { 21 | final int length = arr.length; 22 | arr = java.util.Arrays.copyOf(arr, length + 1); 23 | arr[length] = lastElement; 24 | return arr; 25 | } 26 | 27 | public static StatsDReporterBuilder builder() { 28 | return new StatsDReporterBuilder(); 29 | } 30 | 31 | public StatsDReporterBuilder withMetricConfig(MetricsConfig config) { 32 | this.metricsConfig = config; 33 | return this; 34 | } 35 | 36 | public StatsDReporterBuilder withExtraTags(String... tags) { 37 | this.extraTags = tags; 38 | return this; 39 | } 40 | 41 | private static T[] append(T[] arr, T[] second) { 42 | final int length = arr.length; 43 | arr = java.util.Arrays.copyOf(arr, length + second.length); 44 | System.arraycopy(second, 0, arr, length, second.length); 45 | return arr; 46 | } 47 | 48 | public StatsDReporter build() { 49 | StatsDClient statsDClient = buildStatsDClient(); 50 | return new StatsDReporter(statsDClient, metricsConfig.getMetricStatsDTagsNativeFormatEnabled(), append(metricsConfig.getMetricStatsDTags().split(","), extraTags)); 51 | } 52 | 53 | private StatsDClient buildStatsDClient() { 54 | StatsDClient statsDClient; 55 | try { 56 | statsDClient = new NonBlockingStatsDClientBuilder() 57 | .hostname(metricsConfig.getMetricStatsDHost()) 58 | .port(metricsConfig.getMetricStatsDPort()) 59 | .build(); 60 | log.info("NonBlocking StatsD client connection established"); 61 | } catch (Exception e) { 62 | log.warn("Exception on creating StatsD client, disabling StatsD and Audit client", e); 63 | log.warn("Application is running without collecting any metrics!!!!!!!!"); 64 | statsDClient = new NoOpStatsDClient(); 65 | } 66 | return statsDClient; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/RedisSink.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis; 2 | 3 | import org.raystack.depot.message.Message; 4 | import org.raystack.depot.metrics.Instrumentation; 5 | import org.raystack.depot.redis.client.RedisClient; 6 | import org.raystack.depot.redis.client.response.RedisResponse; 7 | import org.raystack.depot.redis.parsers.RedisParser; 8 | import org.raystack.depot.redis.record.RedisRecord; 9 | import org.raystack.depot.redis.util.RedisSinkUtils; 10 | import org.raystack.depot.Sink; 11 | import org.raystack.depot.SinkResponse; 12 | import org.raystack.depot.error.ErrorInfo; 13 | 14 | import java.io.IOException; 15 | import java.util.List; 16 | import java.util.Map; 17 | import java.util.stream.Collectors; 18 | 19 | public class RedisSink implements Sink { 20 | private final RedisClient redisClient; 21 | private final RedisParser redisParser; 22 | private final Instrumentation instrumentation; 23 | 24 | public RedisSink(RedisClient redisClient, RedisParser redisParser, Instrumentation instrumentation) { 25 | this.redisClient = redisClient; 26 | this.redisParser = redisParser; 27 | this.instrumentation = instrumentation; 28 | } 29 | 30 | @Override 31 | public SinkResponse pushToSink(List messages) { 32 | List records = redisParser.convert(messages); 33 | Map> splitterRecords = records.stream() 34 | .collect(Collectors.partitioningBy(RedisRecord::isValid)); 35 | List invalidRecords = splitterRecords.get(Boolean.FALSE); 36 | List validRecords = splitterRecords.get(Boolean.TRUE); 37 | SinkResponse sinkResponse = new SinkResponse(); 38 | invalidRecords.forEach( 39 | invalidRecord -> sinkResponse.addErrors(invalidRecord.getIndex(), invalidRecord.getErrorInfo())); 40 | if (validRecords.size() > 0) { 41 | List responses = redisClient.send(validRecords); 42 | Map errorInfoMap = RedisSinkUtils.getErrorsFromResponse(validRecords, responses, 43 | instrumentation); 44 | errorInfoMap.forEach(sinkResponse::addErrors); 45 | instrumentation.logInfo("Pushed a batch of {} records to Redis", validRecords.size()); 46 | } 47 | return sinkResponse; 48 | } 49 | 50 | @Override 51 | public void close() throws IOException { 52 | 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/client/RedisClient.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.client; 2 | 3 | import org.raystack.depot.redis.client.response.RedisResponse; 4 | import org.raystack.depot.redis.record.RedisRecord; 5 | 6 | import java.io.Closeable; 7 | import java.util.List; 8 | 9 | /** 10 | * Redis client interface to be used in RedisSink. 11 | */ 12 | public interface RedisClient extends Closeable { 13 | List send(List records); 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/client/RedisClusterClient.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.client; 2 | 3 | import org.raystack.depot.redis.client.response.RedisResponse; 4 | import org.raystack.depot.redis.record.RedisRecord; 5 | import org.raystack.depot.redis.ttl.RedisTtl; 6 | import org.raystack.depot.metrics.Instrumentation; 7 | import lombok.AllArgsConstructor; 8 | import redis.clients.jedis.JedisCluster; 9 | 10 | import java.util.List; 11 | import java.util.stream.Collectors; 12 | 13 | /** 14 | * Redis cluster client. 15 | */ 16 | @AllArgsConstructor 17 | public class RedisClusterClient implements RedisClient { 18 | 19 | private final Instrumentation instrumentation; 20 | private final RedisTtl redisTTL; 21 | private final JedisCluster jedisCluster; 22 | 23 | @Override 24 | public List send(List records) { 25 | return records.stream() 26 | .map(record -> record.send(jedisCluster, redisTTL)) 27 | .collect(Collectors.toList()); 28 | } 29 | 30 | @Override 31 | public void close() { 32 | instrumentation.logInfo("Closing Jedis client"); 33 | jedisCluster.close(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/client/RedisStandaloneClient.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.client; 2 | 3 | import org.raystack.depot.redis.client.response.RedisResponse; 4 | import org.raystack.depot.redis.client.response.RedisStandaloneResponse; 5 | import org.raystack.depot.redis.record.RedisRecord; 6 | import org.raystack.depot.redis.ttl.RedisTtl; 7 | import org.raystack.depot.metrics.Instrumentation; 8 | import lombok.AllArgsConstructor; 9 | import redis.clients.jedis.Jedis; 10 | import redis.clients.jedis.Pipeline; 11 | import redis.clients.jedis.Response; 12 | 13 | import java.util.List; 14 | import java.util.stream.Collectors; 15 | 16 | /** 17 | * Redis standalone client. 18 | */ 19 | @AllArgsConstructor 20 | public class RedisStandaloneClient implements RedisClient { 21 | 22 | private final Instrumentation instrumentation; 23 | private final RedisTtl redisTTL; 24 | private final Jedis jedis; 25 | 26 | /** 27 | * Pushes records in a transaction. 28 | * if the transaction fails, whole batch can be retried. 29 | * 30 | * @param records records to send 31 | * @return Custom response containing status of the API calls. 32 | */ 33 | @Override 34 | public List send(List records) { 35 | Pipeline jedisPipelined = jedis.pipelined(); 36 | jedisPipelined.multi(); 37 | List responses = records.stream() 38 | .map(redisRecord -> redisRecord.send(jedisPipelined, redisTTL)) 39 | .collect(Collectors.toList()); 40 | Response> executeResponse = jedisPipelined.exec(); 41 | jedisPipelined.sync(); 42 | instrumentation.logDebug("jedis responses: {}", executeResponse.get()); 43 | return responses.stream().map(RedisStandaloneResponse::process).collect(Collectors.toList()); 44 | } 45 | 46 | @Override 47 | public void close() { 48 | instrumentation.logInfo("Closing Jedis client"); 49 | jedis.close(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/client/entry/RedisEntry.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.client.entry; 2 | 3 | import org.raystack.depot.redis.client.response.RedisStandaloneResponse; 4 | import org.raystack.depot.redis.client.response.RedisClusterResponse; 5 | import org.raystack.depot.redis.ttl.RedisTtl; 6 | import redis.clients.jedis.JedisCluster; 7 | import redis.clients.jedis.Pipeline; 8 | 9 | /** 10 | * The interface Redis data entry. 11 | */ 12 | public interface RedisEntry { 13 | 14 | /** 15 | * Push messages to jedis pipeline. 16 | * 17 | * @param jedisPipelined the jedis pipelined 18 | * @param redisTTL the redis ttl 19 | */ 20 | RedisStandaloneResponse send(Pipeline jedisPipelined, RedisTtl redisTTL); 21 | 22 | /** 23 | * Push message to jedis cluster. 24 | * 25 | * @param jedisCluster the jedis cluster 26 | * @param redisTTL the redis ttl 27 | */ 28 | RedisClusterResponse send(JedisCluster jedisCluster, RedisTtl redisTTL); 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/client/entry/RedisHashSetFieldEntry.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.client.entry; 2 | 3 | import org.raystack.depot.redis.client.response.RedisClusterResponse; 4 | import org.raystack.depot.redis.client.response.RedisStandaloneResponse; 5 | import org.raystack.depot.metrics.Instrumentation; 6 | import org.raystack.depot.redis.ttl.RedisTtl; 7 | import lombok.AllArgsConstructor; 8 | import lombok.EqualsAndHashCode; 9 | import redis.clients.jedis.JedisCluster; 10 | import redis.clients.jedis.Pipeline; 11 | import redis.clients.jedis.Response; 12 | import redis.clients.jedis.exceptions.JedisException; 13 | 14 | /** 15 | * Class for Redis Hash set entry. 16 | */ 17 | @AllArgsConstructor 18 | @EqualsAndHashCode 19 | public class RedisHashSetFieldEntry implements RedisEntry { 20 | 21 | private final String key; 22 | private final String field; 23 | private final String value; 24 | @EqualsAndHashCode.Exclude 25 | private final Instrumentation instrumentation; 26 | 27 | public String getKey() { 28 | return key; 29 | } 30 | 31 | public String getField() { 32 | return field; 33 | } 34 | 35 | public String getValue() { 36 | return value; 37 | } 38 | 39 | @Override 40 | public RedisStandaloneResponse send(Pipeline jedisPipelined, RedisTtl redisTTL) { 41 | instrumentation.logDebug("key: {}, field: {}, value: {}", key, field, value); 42 | Response response = jedisPipelined.hset(key, field, value); 43 | Response ttlResponse = redisTTL.setTtl(jedisPipelined, key); 44 | return new RedisStandaloneResponse("HSET", response, ttlResponse); 45 | } 46 | 47 | @Override 48 | public RedisClusterResponse send(JedisCluster jedisCluster, RedisTtl redisTTL) { 49 | instrumentation.logDebug("key: {}, field: {}, value: {}", key, field, value); 50 | try { 51 | Long response = jedisCluster.hset(key, field, value); 52 | Long ttlResponse = redisTTL.setTtl(jedisCluster, key); 53 | return new RedisClusterResponse("HSET", response, ttlResponse); 54 | } catch (JedisException e) { 55 | return new RedisClusterResponse(e.getMessage()); 56 | } 57 | } 58 | 59 | @Override 60 | public String toString() { 61 | return String.format("RedisHashSetFieldEntry Key %s, Field %s, Value %s", key, field, value); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/client/entry/RedisKeyValueEntry.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.client.entry; 2 | 3 | import org.raystack.depot.redis.client.response.RedisClusterResponse; 4 | import org.raystack.depot.redis.client.response.RedisStandaloneResponse; 5 | import org.raystack.depot.metrics.Instrumentation; 6 | import org.raystack.depot.redis.ttl.RedisTtl; 7 | import lombok.AllArgsConstructor; 8 | import lombok.EqualsAndHashCode; 9 | import redis.clients.jedis.JedisCluster; 10 | import redis.clients.jedis.Pipeline; 11 | import redis.clients.jedis.Response; 12 | import redis.clients.jedis.exceptions.JedisException; 13 | 14 | @AllArgsConstructor 15 | @EqualsAndHashCode 16 | public class RedisKeyValueEntry implements RedisEntry { 17 | private final String key; 18 | private final String value; 19 | @EqualsAndHashCode.Exclude 20 | private final Instrumentation instrumentation; 21 | 22 | @Override 23 | public RedisStandaloneResponse send(Pipeline jedisPipelined, RedisTtl redisTTL) { 24 | instrumentation.logDebug("key: {}, value: {}", key, value); 25 | Response response = jedisPipelined.set(key, value); 26 | Response ttlResponse = redisTTL.setTtl(jedisPipelined, key); 27 | return new RedisStandaloneResponse("SET", response, ttlResponse); 28 | } 29 | 30 | @Override 31 | public RedisClusterResponse send(JedisCluster jedisCluster, RedisTtl redisTTL) { 32 | instrumentation.logDebug("key: {}, value: {}", key, value); 33 | try { 34 | String response = jedisCluster.set(key, value); 35 | Long ttlResponse = redisTTL.setTtl(jedisCluster, key); 36 | return new RedisClusterResponse("SET", response, ttlResponse); 37 | } catch (JedisException e) { 38 | return new RedisClusterResponse(e.getMessage()); 39 | } 40 | } 41 | 42 | @Override 43 | public String toString() { 44 | return String.format("RedisKeyValueEntry: Key %s, Value %s", key, value); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/client/entry/RedisListEntry.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.client.entry; 2 | 3 | import org.raystack.depot.redis.client.response.RedisClusterResponse; 4 | import org.raystack.depot.redis.client.response.RedisStandaloneResponse; 5 | import org.raystack.depot.metrics.Instrumentation; 6 | import org.raystack.depot.redis.ttl.RedisTtl; 7 | import lombok.AllArgsConstructor; 8 | import lombok.EqualsAndHashCode; 9 | import redis.clients.jedis.JedisCluster; 10 | import redis.clients.jedis.Pipeline; 11 | import redis.clients.jedis.Response; 12 | import redis.clients.jedis.exceptions.JedisException; 13 | 14 | /** 15 | * Class for Redis Hash set entry. 16 | */ 17 | @AllArgsConstructor 18 | @EqualsAndHashCode 19 | public class RedisListEntry implements RedisEntry { 20 | private final String key; 21 | private final String value; 22 | @EqualsAndHashCode.Exclude 23 | private final Instrumentation instrumentation; 24 | 25 | @Override 26 | public RedisStandaloneResponse send(Pipeline jedisPipelined, RedisTtl redisTTL) { 27 | instrumentation.logDebug("key: {}, value: {}", key, value); 28 | Response response = jedisPipelined.lpush(key, value); 29 | Response ttlResponse = redisTTL.setTtl(jedisPipelined, key); 30 | return new RedisStandaloneResponse("LPUSH", response, ttlResponse); 31 | } 32 | 33 | @Override 34 | public RedisClusterResponse send(JedisCluster jedisCluster, RedisTtl redisTTL) { 35 | instrumentation.logDebug("key: {}, value: {}", key, value); 36 | try { 37 | Long response = jedisCluster.lpush(key, value); 38 | Long ttlResponse = redisTTL.setTtl(jedisCluster, key); 39 | return new RedisClusterResponse("LPUSH", response, ttlResponse); 40 | } catch (JedisException e) { 41 | return new RedisClusterResponse(e.getMessage()); 42 | } 43 | } 44 | 45 | @Override 46 | public String toString() { 47 | return String.format("RedisListEntry: Key %s, Value %s", key, value); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/client/response/RedisClusterResponse.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.client.response; 2 | 3 | import lombok.Getter; 4 | 5 | public class RedisClusterResponse implements RedisResponse { 6 | @Getter 7 | private final String message; 8 | @Getter 9 | private final boolean failed; 10 | 11 | public RedisClusterResponse(String command, Object response, Long ttlResponse) { 12 | this.message = String.format( 13 | "%s: %s, TTL: %s", 14 | command, 15 | response, 16 | ttlResponse == null ? "NoOp" : ttlResponse == 0 ? "NOT UPDATED" : "UPDATED"); 17 | this.failed = false; 18 | } 19 | 20 | public RedisClusterResponse(String message) { 21 | this.message = message; 22 | this.failed = true; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/client/response/RedisResponse.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.client.response; 2 | 3 | public interface RedisResponse { 4 | String getMessage(); 5 | 6 | boolean isFailed(); 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/client/response/RedisStandaloneResponse.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.client.response; 2 | 3 | import lombok.Getter; 4 | import redis.clients.jedis.Response; 5 | import redis.clients.jedis.exceptions.JedisException; 6 | 7 | public class RedisStandaloneResponse implements RedisResponse { 8 | private final Response response; 9 | private final Response ttlResponse; 10 | private final String command; 11 | @Getter 12 | private String message; 13 | @Getter 14 | private boolean failed = true; 15 | 16 | public RedisStandaloneResponse(String command, Response response, Response ttlResponse) { 17 | this.command = command; 18 | this.response = response; 19 | this.ttlResponse = ttlResponse; 20 | } 21 | 22 | public RedisStandaloneResponse process() { 23 | try { 24 | Object cmd = response.get(); 25 | Object ttl = ttlResponse != null ? (((long) ttlResponse.get()) == 0L ? "NOT UPDATED" : "UPDATED") : "NoOp"; 26 | message = String.format("%s: %s, TTL: %s", command, cmd, ttl); 27 | failed = false; 28 | } catch (JedisException e) { 29 | message = e.getMessage(); 30 | failed = true; 31 | } 32 | return this; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/enums/RedisSinkDataType.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.enums; 2 | 3 | public enum RedisSinkDataType { 4 | LIST, 5 | HASHSET, 6 | KEYVALUE, 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/enums/RedisSinkDeploymentType.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.enums; 2 | 3 | public enum RedisSinkDeploymentType { 4 | STANDALONE, 5 | CLUSTER 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/enums/RedisSinkTtlType.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.enums; 2 | 3 | public enum RedisSinkTtlType { 4 | EXACT_TIME, 5 | DURATION, 6 | DISABLE 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/parsers/RedisEntryParser.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.parsers; 2 | 3 | import org.raystack.depot.redis.client.entry.RedisEntry; 4 | import org.raystack.depot.message.ParsedMessage; 5 | 6 | import java.util.List; 7 | 8 | public interface RedisEntryParser { 9 | 10 | List getRedisEntry(ParsedMessage parsedMessage); 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/parsers/RedisHashSetEntryParser.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.parsers; 2 | 3 | import org.raystack.depot.message.field.GenericFieldFactory; 4 | import org.raystack.depot.redis.client.entry.RedisEntry; 5 | import org.raystack.depot.redis.client.entry.RedisHashSetFieldEntry; 6 | import org.raystack.depot.common.Template; 7 | import org.raystack.depot.message.MessageSchema; 8 | import org.raystack.depot.message.ParsedMessage; 9 | import org.raystack.depot.metrics.Instrumentation; 10 | import org.raystack.depot.metrics.StatsDReporter; 11 | import lombok.AllArgsConstructor; 12 | 13 | import java.util.List; 14 | import java.util.Map; 15 | import java.util.stream.Collectors; 16 | 17 | /** 18 | * Redis hash set parser. 19 | */ 20 | @AllArgsConstructor 21 | public class RedisHashSetEntryParser implements RedisEntryParser { 22 | private final StatsDReporter statsDReporter; 23 | private final Template keyTemplate; 24 | private final Map fieldTemplates; 25 | private final MessageSchema schema; 26 | 27 | @Override 28 | public List getRedisEntry(ParsedMessage parsedMessage) { 29 | String redisKey = keyTemplate.parse(parsedMessage, schema); 30 | return fieldTemplates 31 | .entrySet() 32 | .stream() 33 | .map(fieldTemplate -> { 34 | String field = fieldTemplate.getValue().parse(parsedMessage, schema); 35 | String redisValue = GenericFieldFactory 36 | .getField(parsedMessage.getFieldByName(fieldTemplate.getKey(), schema)).getString(); 37 | return new RedisHashSetFieldEntry(redisKey, field, redisValue, 38 | new Instrumentation(statsDReporter, RedisHashSetFieldEntry.class)); 39 | }).collect(Collectors.toList()); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/parsers/RedisKeyValueEntryParser.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.parsers; 2 | 3 | import org.raystack.depot.message.field.GenericFieldFactory; 4 | import org.raystack.depot.redis.client.entry.RedisEntry; 5 | import org.raystack.depot.redis.client.entry.RedisKeyValueEntry; 6 | import org.raystack.depot.common.Template; 7 | import org.raystack.depot.message.MessageSchema; 8 | import org.raystack.depot.message.ParsedMessage; 9 | import org.raystack.depot.metrics.Instrumentation; 10 | import org.raystack.depot.metrics.StatsDReporter; 11 | import lombok.AllArgsConstructor; 12 | 13 | import java.util.Collections; 14 | import java.util.List; 15 | 16 | @AllArgsConstructor 17 | public class RedisKeyValueEntryParser implements RedisEntryParser { 18 | private final StatsDReporter statsDReporter; 19 | private final Template keyTemplate; 20 | private final String fieldName; 21 | private final MessageSchema schema; 22 | 23 | @Override 24 | public List getRedisEntry(ParsedMessage parsedMessage) { 25 | String redisKey = keyTemplate.parse(parsedMessage, schema); 26 | String redisValue = GenericFieldFactory.getField(parsedMessage.getFieldByName(fieldName, schema)).getString(); 27 | RedisKeyValueEntry redisKeyValueEntry = new RedisKeyValueEntry(redisKey, redisValue, 28 | new Instrumentation(statsDReporter, RedisKeyValueEntry.class)); 29 | return Collections.singletonList(redisKeyValueEntry); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/parsers/RedisListEntryParser.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.parsers; 2 | 3 | import org.raystack.depot.message.field.GenericFieldFactory; 4 | import org.raystack.depot.redis.client.entry.RedisEntry; 5 | import org.raystack.depot.redis.client.entry.RedisListEntry; 6 | import org.raystack.depot.common.Template; 7 | import org.raystack.depot.message.MessageSchema; 8 | import org.raystack.depot.message.ParsedMessage; 9 | import org.raystack.depot.metrics.Instrumentation; 10 | import org.raystack.depot.metrics.StatsDReporter; 11 | import lombok.AllArgsConstructor; 12 | 13 | import java.util.Collections; 14 | import java.util.List; 15 | 16 | /** 17 | * Redis list parser. 18 | */ 19 | @AllArgsConstructor 20 | public class RedisListEntryParser implements RedisEntryParser { 21 | private final StatsDReporter statsDReporter; 22 | private final Template keyTemplate; 23 | private final String field; 24 | private final MessageSchema schema; 25 | 26 | @Override 27 | public List getRedisEntry(ParsedMessage parsedMessage) { 28 | String redisKey = keyTemplate.parse(parsedMessage, schema); 29 | String redisValue = GenericFieldFactory.getField(parsedMessage.getFieldByName(field, schema)).getString(); 30 | return Collections.singletonList( 31 | new RedisListEntry(redisKey, redisValue, new Instrumentation(statsDReporter, RedisListEntry.class))); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/record/RedisRecord.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.record; 2 | 3 | import org.raystack.depot.redis.client.entry.RedisEntry; 4 | import org.raystack.depot.redis.client.response.RedisClusterResponse; 5 | import org.raystack.depot.redis.client.response.RedisStandaloneResponse; 6 | import org.raystack.depot.error.ErrorInfo; 7 | import org.raystack.depot.redis.ttl.RedisTtl; 8 | import lombok.AllArgsConstructor; 9 | import lombok.Getter; 10 | import redis.clients.jedis.JedisCluster; 11 | import redis.clients.jedis.Pipeline; 12 | 13 | @AllArgsConstructor 14 | public class RedisRecord { 15 | private RedisEntry redisEntry; 16 | @Getter 17 | private final Long index; 18 | @Getter 19 | private final ErrorInfo errorInfo; 20 | @Getter 21 | private final String metadata; 22 | @Getter 23 | private final boolean valid; 24 | 25 | public RedisStandaloneResponse send(Pipeline jedisPipelined, RedisTtl redisTTL) { 26 | return redisEntry.send(jedisPipelined, redisTTL); 27 | } 28 | 29 | public RedisClusterResponse send(JedisCluster jedisCluster, RedisTtl redisTTL) { 30 | return redisEntry.send(jedisCluster, redisTTL); 31 | } 32 | 33 | @Override 34 | public String toString() { 35 | return String.format("Metadata %s %s", metadata, redisEntry != null ? redisEntry.toString() : "NULL"); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/ttl/DurationTtl.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.ttl; 2 | 3 | import lombok.AllArgsConstructor; 4 | import redis.clients.jedis.JedisCluster; 5 | import redis.clients.jedis.Pipeline; 6 | import redis.clients.jedis.Response; 7 | 8 | @AllArgsConstructor 9 | public class DurationTtl implements RedisTtl { 10 | private int ttlInSeconds; 11 | 12 | @Override 13 | public Response setTtl(Pipeline jedisPipelined, String key) { 14 | return jedisPipelined.expire(key, ttlInSeconds); 15 | } 16 | 17 | @Override 18 | public Long setTtl(JedisCluster jedisCluster, String key) { 19 | return jedisCluster.expire(key, ttlInSeconds); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/ttl/ExactTimeTtl.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.ttl; 2 | 3 | import lombok.AllArgsConstructor; 4 | import redis.clients.jedis.JedisCluster; 5 | import redis.clients.jedis.Pipeline; 6 | import redis.clients.jedis.Response; 7 | 8 | @AllArgsConstructor 9 | public class ExactTimeTtl implements RedisTtl { 10 | private long unixTime; 11 | 12 | @Override 13 | public Response setTtl(Pipeline jedisPipelined, String key) { 14 | return jedisPipelined.expireAt(key, unixTime); 15 | } 16 | 17 | @Override 18 | public Long setTtl(JedisCluster jedisCluster, String key) { 19 | return jedisCluster.expireAt(key, unixTime); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/ttl/NoRedisTtl.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.ttl; 2 | 3 | import redis.clients.jedis.JedisCluster; 4 | import redis.clients.jedis.Pipeline; 5 | import redis.clients.jedis.Response; 6 | 7 | public class NoRedisTtl implements RedisTtl { 8 | @Override 9 | public Response setTtl(Pipeline jedisPipelined, String key) { 10 | return null; 11 | } 12 | 13 | @Override 14 | public Long setTtl(JedisCluster jedisCluster, String key) { 15 | return null; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/ttl/RedisTTLFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.ttl; 2 | 3 | import org.raystack.depot.redis.enums.RedisSinkTtlType; 4 | import org.raystack.depot.config.RedisSinkConfig; 5 | import org.raystack.depot.exception.ConfigurationException; 6 | 7 | public class RedisTTLFactory { 8 | 9 | public static RedisTtl getTTl(RedisSinkConfig redisSinkConfig) { 10 | if (redisSinkConfig.getSinkRedisTtlType() == RedisSinkTtlType.DISABLE) { 11 | return new NoRedisTtl(); 12 | } 13 | long redisTTLValue = redisSinkConfig.getSinkRedisTtlValue(); 14 | if (redisTTLValue < 0) { 15 | throw new ConfigurationException("Provide a positive TTL value"); 16 | } 17 | switch (redisSinkConfig.getSinkRedisTtlType()) { 18 | case EXACT_TIME: 19 | return new ExactTimeTtl(redisTTLValue); 20 | case DURATION: 21 | return new DurationTtl((int) redisTTLValue); 22 | default: 23 | throw new ConfigurationException("Not a valid TTL config"); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/ttl/RedisTtl.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.ttl; 2 | 3 | import redis.clients.jedis.JedisCluster; 4 | import redis.clients.jedis.Pipeline; 5 | import redis.clients.jedis.Response; 6 | 7 | /** 8 | * Interface for RedisTTL. 9 | */ 10 | public interface RedisTtl { 11 | Response setTtl(Pipeline jedisPipelined, String key); 12 | 13 | Long setTtl(JedisCluster jedisCluster, String key); 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/redis/util/RedisSinkUtils.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.util; 2 | 3 | import org.raystack.depot.redis.client.response.RedisResponse; 4 | import org.raystack.depot.redis.record.RedisRecord; 5 | import org.raystack.depot.error.ErrorInfo; 6 | import org.raystack.depot.error.ErrorType; 7 | import org.raystack.depot.metrics.Instrumentation; 8 | 9 | import java.util.HashMap; 10 | import java.util.List; 11 | import java.util.Map; 12 | import java.util.stream.IntStream; 13 | 14 | public class RedisSinkUtils { 15 | public static Map getErrorsFromResponse(List redisRecords, 16 | List responses, Instrumentation instrumentation) { 17 | Map errors = new HashMap<>(); 18 | IntStream.range(0, responses.size()).forEach( 19 | index -> { 20 | RedisResponse response = responses.get(index); 21 | if (response.isFailed()) { 22 | RedisRecord record = redisRecords.get(index); 23 | instrumentation.logError("Error while inserting to redis for message. Record: {}, Error: {}", 24 | record.toString(), response.getMessage()); 25 | errors.put(record.getIndex(), 26 | new ErrorInfo(new Exception(response.getMessage()), ErrorType.DEFAULT_ERROR)); 27 | } 28 | }); 29 | return errors; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/stencil/DepotStencilUpdateListener.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.stencil; 2 | 3 | import com.google.protobuf.Descriptors; 4 | import org.raystack.depot.message.MessageParser; 5 | import org.raystack.stencil.SchemaUpdateListener; 6 | import lombok.Getter; 7 | import lombok.Setter; 8 | 9 | import java.util.Map; 10 | 11 | public abstract class DepotStencilUpdateListener implements SchemaUpdateListener { 12 | @Getter 13 | @Setter 14 | private MessageParser messageParser; 15 | 16 | public void onSchemaUpdate(final Map newDescriptor) { 17 | // default implementation is empty 18 | } 19 | 20 | public abstract void updateSchema(); 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/utils/DateUtils.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.utils; 2 | 3 | import java.text.DateFormat; 4 | import java.text.SimpleDateFormat; 5 | import java.util.Date; 6 | import java.util.TimeZone; 7 | 8 | public class DateUtils { 9 | private static final TimeZone TZ = TimeZone.getTimeZone("UTC"); 10 | private static final DateFormat DF = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'"); 11 | static { 12 | DF.setTimeZone(TZ); 13 | } 14 | 15 | public static String formatCurrentTimeAsUTC() { 16 | return formatTimeAsUTC(new Date()); 17 | } 18 | 19 | public static String formatTimeAsUTC(Date date) { 20 | return DF.format(date); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/utils/JsonUtils.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.utils; 2 | 3 | import org.raystack.depot.config.SinkConfig; 4 | import org.json.JSONObject; 5 | 6 | public class JsonUtils { 7 | /** 8 | * Creates a json Object based on the configuration. 9 | * If String mode is enabled, it converts all the fields in string. 10 | * 11 | * @param config Sink Configuration 12 | * @param payload Json Payload in byyes 13 | * @return Json object 14 | */ 15 | public static JSONObject getJsonObject(SinkConfig config, byte[] payload) { 16 | JSONObject jsonObject = new JSONObject(new String(payload)); 17 | if (!config.getSinkConnectorSchemaJsonParserStringModeEnabled()) { 18 | return jsonObject; 19 | } 20 | // convert to all objects to string 21 | JSONObject jsonWithStringValues = new JSONObject(); 22 | jsonObject.keySet() 23 | .forEach(k -> { 24 | Object value = jsonObject.get(k); 25 | if (value instanceof JSONObject) { 26 | throw new UnsupportedOperationException("nested json structure not supported yet"); 27 | } 28 | if (JSONObject.NULL.equals(value)) { 29 | return; 30 | } 31 | jsonWithStringValues.put(k, value.toString()); 32 | }); 33 | 34 | return jsonWithStringValues; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/utils/MessageConfigUtils.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.utils; 2 | 3 | import org.raystack.depot.config.SinkConfig; 4 | import org.raystack.depot.message.SinkConnectorSchemaMessageMode; 5 | import org.raystack.depot.common.Tuple; 6 | 7 | public class MessageConfigUtils { 8 | 9 | public static Tuple getModeAndSchema(SinkConfig sinkConfig) { 10 | SinkConnectorSchemaMessageMode mode = sinkConfig.getSinkConnectorSchemaMessageMode(); 11 | String schemaClass = mode == SinkConnectorSchemaMessageMode.LOG_MESSAGE 12 | ? sinkConfig.getSinkConnectorSchemaProtoMessageClass() 13 | : sinkConfig.getSinkConnectorSchemaProtoKeyClass(); 14 | return new Tuple<>(mode, schemaClass); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/utils/ProtoUtils.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.utils; 2 | 3 | import com.google.protobuf.DynamicMessage; 4 | 5 | import java.util.Collections; 6 | import java.util.LinkedList; 7 | import java.util.List; 8 | import java.util.Queue; 9 | import java.util.stream.Collectors; 10 | 11 | public class ProtoUtils { 12 | public static boolean hasUnknownField(DynamicMessage root) { 13 | List dynamicMessageFields = collectNestedFields(root); 14 | List messageWithUnknownFields = getMessageWithUnknownFields(dynamicMessageFields); 15 | return messageWithUnknownFields.size() > 0; 16 | } 17 | 18 | private static List collectNestedFields(DynamicMessage node) { 19 | List output = new LinkedList<>(); 20 | Queue stack = Collections.asLifoQueue(new LinkedList<>()); 21 | stack.add(node); 22 | while (true) { 23 | DynamicMessage current = stack.poll(); 24 | if (current == null) { 25 | break; 26 | } 27 | List nestedChildNodes = current.getAllFields().values().stream() 28 | .filter(field -> field instanceof DynamicMessage) 29 | .map(field -> (DynamicMessage) field) 30 | .collect(Collectors.toList()); 31 | stack.addAll(nestedChildNodes); 32 | 33 | output.add(current); 34 | } 35 | 36 | return output; 37 | } 38 | 39 | private static List getMessageWithUnknownFields(List messages) { 40 | return messages.stream().filter(message -> message.getUnknownFields().asMap().size() > 0) 41 | .collect(Collectors.toList()); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/utils/StencilUtils.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.utils; 2 | 3 | import org.raystack.depot.config.SinkConfig; 4 | import org.raystack.stencil.SchemaUpdateListener; 5 | import org.raystack.stencil.config.StencilConfig; 6 | import com.timgroup.statsd.StatsDClient; 7 | 8 | public class StencilUtils { 9 | public static StencilConfig getStencilConfig( 10 | SinkConfig sinkConfig, 11 | StatsDClient statsDClient, 12 | SchemaUpdateListener schemaUpdateListener) { 13 | return StencilConfig.builder() 14 | .cacheAutoRefresh(sinkConfig.getSchemaRegistryStencilCacheAutoRefresh()) 15 | .cacheTtlMs(sinkConfig.getSchemaRegistryStencilCacheTtlMs()) 16 | .statsDClient(statsDClient) 17 | .fetchHeaders(sinkConfig.getSchemaRegistryStencilFetchHeaders()) 18 | .fetchBackoffMinMs(sinkConfig.getSchemaRegistryStencilFetchBackoffMinMs()) 19 | .fetchRetries(sinkConfig.getSchemaRegistryStencilFetchRetries()) 20 | .fetchTimeoutMs(sinkConfig.getSchemaRegistryStencilFetchTimeoutMs()) 21 | .refreshStrategy(sinkConfig.getSchemaRegistryStencilRefreshStrategy()) 22 | .updateListener(schemaUpdateListener) 23 | .build(); 24 | } 25 | 26 | public static StencilConfig getStencilConfig(SinkConfig config, StatsDClient statsDClient) { 27 | return getStencilConfig(config, statsDClient, null); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/org/raystack/depot/utils/StringUtils.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.utils; 2 | 3 | import java.util.regex.Matcher; 4 | import java.util.regex.Pattern; 5 | import java.util.stream.IntStream; 6 | 7 | public class StringUtils { 8 | 9 | private static final Pattern PATTERN = Pattern.compile("(?!<%)%" 10 | + "(?:(\\d+)\\$)?" 11 | + "([-#+ 0,(]|<)?" 12 | + "\\d*" 13 | + "(?:\\.\\d+)?" 14 | + "(?:[bBhHsScCdoxXeEfgGaAtT]|" 15 | + "[tT][HIklMSLNpzZsQBbhAaCYyjmdeRTrDFc])"); 16 | 17 | public static int countVariables(String fmt) { 18 | Matcher m = PATTERN.matcher(fmt); 19 | int np = 0; 20 | int maxref = 0; 21 | while (m.find()) { 22 | if (m.group(1) != null) { 23 | String dec = m.group(1); 24 | int ref = Integer.parseInt(dec); 25 | maxref = Math.max(ref, maxref); 26 | } else if (!(m.group(2) != null && "<".equals(m.group(2)))) { 27 | np++; 28 | } 29 | } 30 | return Math.max(np, maxref); 31 | } 32 | 33 | public static int count(String in, char c) { 34 | return IntStream.range(0, in.length()).reduce(0, (x, y) -> x + (in.charAt(y) == c ? 1 : 0)); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/bigquery/TestMetadata.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.EqualsAndHashCode; 5 | import lombok.Getter; 6 | import lombok.ToString; 7 | 8 | @EqualsAndHashCode 9 | @ToString 10 | @Getter 11 | @AllArgsConstructor 12 | public class TestMetadata { 13 | private final String topic; 14 | private final int partition; 15 | private final long offset; 16 | private final long timestamp; 17 | private final long loadTime; 18 | } 19 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/bigquery/client/BigQueryRowWithInsertIdTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.client; 2 | 3 | import com.google.cloud.bigquery.InsertAllRequest; 4 | import org.raystack.depot.bigquery.models.Record; 5 | import org.junit.Test; 6 | 7 | import java.util.HashMap; 8 | 9 | import static org.junit.Assert.assertEquals; 10 | 11 | public class BigQueryRowWithInsertIdTest { 12 | 13 | @Test 14 | public void shouldCreateRowWithInsertID() { 15 | Record record = new Record(new HashMap<>(), new HashMap<>(), 0, null); 16 | 17 | BigQueryRowWithInsertId withInsertId = new BigQueryRowWithInsertId(metadata -> "default_1_1"); 18 | InsertAllRequest.RowToInsert rowToInsert = withInsertId.of(record); 19 | String id = rowToInsert.getId(); 20 | 21 | assertEquals("default_1_1", id); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/bigquery/client/BigQueryRowWithoutInsertIdTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.client; 2 | 3 | import com.google.cloud.bigquery.InsertAllRequest; 4 | import org.raystack.depot.bigquery.models.Record; 5 | import org.junit.Test; 6 | 7 | import java.util.HashMap; 8 | 9 | import static org.junit.Assert.assertNull; 10 | 11 | public class BigQueryRowWithoutInsertIdTest { 12 | 13 | @Test 14 | public void shouldCreateRowWithoutInsertID() { 15 | Record record = new Record(new HashMap<>(), new HashMap<>(), 0, null); 16 | 17 | BigQueryRowWithoutInsertId withoutInsertId = new BigQueryRowWithoutInsertId(); 18 | InsertAllRequest.RowToInsert rowToInsert = withoutInsertId.of(record); 19 | String id = rowToInsert.getId(); 20 | 21 | assertNull(id); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/bigquery/converter/MessageRecordConverterUtilsTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.converter; 2 | 3 | import org.raystack.depot.config.BigQuerySinkConfig; 4 | import org.raystack.depot.message.Message; 5 | import org.aeonbits.owner.ConfigFactory; 6 | import org.junit.Assert; 7 | import org.junit.Test; 8 | import org.mockito.Mockito; 9 | 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | public class MessageRecordConverterUtilsTest { 14 | 15 | @Test 16 | public void shouldAddMetaData() { 17 | Map columns = new HashMap() { 18 | { 19 | put("test", 123); 20 | } 21 | }; 22 | Message message = Mockito.mock(Message.class); 23 | Mockito.when(message.getMetadata(Mockito.any())).thenReturn(new HashMap() { 24 | { 25 | put("test2", "value2"); 26 | put("something", 99L); 27 | put("nvm", "nvm"); 28 | } 29 | }); 30 | BigQuerySinkConfig config = ConfigFactory.create(BigQuerySinkConfig.class, new HashMap() { 31 | { 32 | put("SINK_BIGQUERY_ADD_METADATA_ENABLED", "true"); 33 | put("SINK_BIGQUERY_METADATA_COLUMNS_TYPES", "test2=string,something=long,nvm=string"); 34 | } 35 | }); 36 | MessageRecordConverterUtils.addMetadata(columns, message, config); 37 | Assert.assertEquals(new HashMap() { 38 | { 39 | put("test", 123); 40 | put("test2", "value2"); 41 | put("something", 99L); 42 | put("nvm", "nvm"); 43 | } 44 | }, columns); 45 | } 46 | 47 | @Test 48 | public void shouldAddTimeStampForJson() { 49 | Map columns = new HashMap() { 50 | { 51 | put("test", 123); 52 | } 53 | }; 54 | BigQuerySinkConfig config = ConfigFactory.create(BigQuerySinkConfig.class, new HashMap() { 55 | { 56 | put("SINK_CONNECTOR_SCHEMA_DATA_TYPE", "json"); 57 | put("SINK_BIGQUERY_ADD_EVENT_TIMESTAMP_ENABLE", "true"); 58 | } 59 | }); 60 | MessageRecordConverterUtils.addTimeStampColumnForJson(columns, config); 61 | Assert.assertEquals(2, columns.size()); 62 | Assert.assertNotNull(columns.get("event_timestamp")); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/bigquery/storage/BigQueryWriterUtilsTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.bigquery.storage; 2 | 3 | public class BigQueryWriterUtilsTest { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/config/BigQuerySinkConfigTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config; 2 | 3 | import org.raystack.depot.common.TupleString; 4 | import org.aeonbits.owner.ConfigFactory; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | 11 | public class BigQuerySinkConfigTest { 12 | 13 | @Test 14 | public void testMetadataTypes() { 15 | System.setProperty("SINK_CONNECTOR_SCHEMA_PROTO_MESSAGE_CLASS", "org.raystack.depot.TestKeyBQ"); 16 | System.setProperty("SINK_BIGQUERY_ENABLE_AUTO_SCHEMA_UPDATE", "false"); 17 | System.setProperty("SINK_BIGQUERY_METADATA_COLUMNS_TYPES", "topic=string,partition=integer,offset=integer"); 18 | BigQuerySinkConfig config = ConfigFactory.create(BigQuerySinkConfig.class, System.getProperties()); 19 | List metadataColumnsTypes = config.getMetadataColumnsTypes(); 20 | Assert.assertEquals(new ArrayList() { 21 | { 22 | add(new TupleString("topic", "string")); 23 | add(new TupleString("partition", "integer")); 24 | add(new TupleString("offset", "integer")); 25 | } 26 | }, metadataColumnsTypes); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/config/RedisSinkConfigTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config; 2 | 3 | import org.raystack.depot.redis.enums.RedisSinkDeploymentType; 4 | import org.raystack.depot.redis.enums.RedisSinkTtlType; 5 | import org.aeonbits.owner.ConfigFactory; 6 | import org.junit.Assert; 7 | import org.junit.Test; 8 | 9 | public class RedisSinkConfigTest { 10 | @Test 11 | public void testMetadataTypes() { 12 | System.setProperty("SINK_REDIS_DEPLOYMENT_TYPE", "standalone"); 13 | System.setProperty("SINK_REDIS_TTL_TYPE", "disable"); 14 | System.setProperty("SINK_REDIS_KEY_TEMPLATE", "test-key"); 15 | RedisSinkConfig config = ConfigFactory.create(RedisSinkConfig.class, System.getProperties()); 16 | Assert.assertEquals("test-key", config.getSinkRedisKeyTemplate()); 17 | Assert.assertEquals(RedisSinkDeploymentType.STANDALONE, config.getSinkRedisDeploymentType()); 18 | Assert.assertEquals(RedisSinkTtlType.DISABLE, config.getSinkRedisTtlType()); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/config/converter/ConfToListConverterTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.raystack.depot.common.TupleString; 4 | import org.junit.Assert; 5 | import org.junit.Test; 6 | 7 | public class ConfToListConverterTest { 8 | 9 | @Test 10 | public void shouldReturnNullForEmpty() { 11 | ConfToListConverter converter = new ConfToListConverter(); 12 | Assert.assertNull(converter.convert(null, "")); 13 | } 14 | 15 | @Test 16 | public void shouldCovertToTuple() { 17 | ConfToListConverter converter = new ConfToListConverter(); 18 | Assert.assertEquals(new TupleString("a", "b"), converter.convert(null, "a=b")); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/config/converter/ConverterUtilsTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.raystack.depot.common.Tuple; 4 | import org.junit.Assert; 5 | import org.junit.Test; 6 | 7 | import java.util.List; 8 | 9 | public class ConverterUtilsTest { 10 | 11 | @Test 12 | public void testConvertEmpty() { 13 | List> tuples = ConverterUtils.convertToList(""); 14 | Assert.assertEquals(0, tuples.size()); 15 | } 16 | 17 | @Test 18 | public void testConvert() { 19 | List> tuples = ConverterUtils.convertToList("a=b,c=d"); 20 | Assert.assertEquals(2, tuples.size()); 21 | Assert.assertEquals(new Tuple<>("a", "b"), tuples.get(0)); 22 | Assert.assertEquals(new Tuple<>("c", "d"), tuples.get(1)); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/config/converter/LabelMapConverterTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | import java.util.Collections; 7 | import java.util.HashMap; 8 | 9 | public class LabelMapConverterTest { 10 | 11 | @Test 12 | public void shouldConvertToEmptyMap() { 13 | LabelMapConverter converter = new LabelMapConverter(); 14 | Assert.assertEquals(Collections.emptyMap(), converter.convert(null, "")); 15 | } 16 | 17 | @Test 18 | public void shouldConvertToMap() { 19 | LabelMapConverter converter = new LabelMapConverter(); 20 | Assert.assertEquals(new HashMap() { 21 | { 22 | put("a", "b"); 23 | put("c", "d"); 24 | put("test", "testing"); 25 | } 26 | }, converter.convert(null, "a=b,c=d,test=testing")); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/config/converter/RedisSinkDataTypeConverterTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.raystack.depot.redis.enums.RedisSinkDataType; 4 | import org.gradle.internal.impldep.org.junit.Assert; 5 | import org.junit.Before; 6 | import org.junit.Test; 7 | 8 | public class RedisSinkDataTypeConverterTest { 9 | 10 | private RedisSinkDataTypeConverter redisSinkDataTypeConverter; 11 | 12 | @Before 13 | public void setUp() { 14 | redisSinkDataTypeConverter = new RedisSinkDataTypeConverter(); 15 | } 16 | 17 | @Test 18 | public void shouldReturnListSinkTypeFromLowerCaseInput() { 19 | RedisSinkDataType redisSinkDataType = redisSinkDataTypeConverter.convert(null, "list"); 20 | Assert.assertTrue(redisSinkDataType.equals(RedisSinkDataType.LIST)); 21 | } 22 | 23 | @Test 24 | public void shouldReturnListSinkTypeFromUpperCaseInput() { 25 | RedisSinkDataType redisSinkDataType = redisSinkDataTypeConverter.convert(null, "LIST"); 26 | Assert.assertTrue(redisSinkDataType.equals(RedisSinkDataType.LIST)); 27 | } 28 | 29 | @Test 30 | public void shouldReturnListSinkTypeFromMixedCaseInput() { 31 | RedisSinkDataType redisSinkDataType = redisSinkDataTypeConverter.convert(null, "LiSt"); 32 | Assert.assertTrue(redisSinkDataType.equals(RedisSinkDataType.LIST)); 33 | } 34 | 35 | @Test 36 | public void shouldReturnHashSetSinkTypeFromInput() { 37 | RedisSinkDataType redisSinkDataType = redisSinkDataTypeConverter.convert(null, "hashset"); 38 | Assert.assertTrue(redisSinkDataType.equals(RedisSinkDataType.HASHSET)); 39 | } 40 | 41 | @Test 42 | public void shouldReturnKeyValueSinkTypeFromInput() { 43 | RedisSinkDataType redisSinkDataType = redisSinkDataTypeConverter.convert(null, "keyvalue"); 44 | Assert.assertTrue(redisSinkDataType.equals(RedisSinkDataType.KEYVALUE)); 45 | } 46 | 47 | @Test(expected = IllegalArgumentException.class) 48 | public void shouldThrowOnEmptyArgument() { 49 | redisSinkDataTypeConverter.convert(null, ""); 50 | } 51 | 52 | @Test(expected = IllegalArgumentException.class) 53 | public void shouldThrowOnInvalidArgument() { 54 | redisSinkDataTypeConverter.convert(null, "INVALID"); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/config/converter/RedisSinkDeploymentTypeConverterTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.raystack.depot.redis.enums.RedisSinkDeploymentType; 4 | import org.gradle.internal.impldep.org.junit.Assert; 5 | import org.junit.Before; 6 | import org.junit.Test; 7 | 8 | public class RedisSinkDeploymentTypeConverterTest { 9 | private RedisSinkDeploymentTypeConverter redisSinkDeploymentTypeConverter; 10 | 11 | @Before 12 | public void setup() { 13 | redisSinkDeploymentTypeConverter = new RedisSinkDeploymentTypeConverter(); 14 | } 15 | 16 | @Test 17 | public void shouldReturnStandaloneTypeFromLowerCaseInput() { 18 | RedisSinkDeploymentType redisSinkDeploymentType = redisSinkDeploymentTypeConverter.convert(null, "standalone"); 19 | Assert.assertTrue(redisSinkDeploymentType.equals(RedisSinkDeploymentType.STANDALONE)); 20 | } 21 | 22 | @Test 23 | public void shouldReturnStandaloneTypeFromUpperCaseInput() { 24 | RedisSinkDeploymentType redisSinkDeploymentType = redisSinkDeploymentTypeConverter.convert(null, "STANDALONE"); 25 | Assert.assertTrue(redisSinkDeploymentType.equals(RedisSinkDeploymentType.STANDALONE)); 26 | } 27 | 28 | @Test 29 | public void shouldReturnStandaloneTypeFromMixedCaseInput() { 30 | RedisSinkDeploymentType redisSinkDeploymentType = redisSinkDeploymentTypeConverter.convert(null, "stANdAlOne"); 31 | Assert.assertTrue(redisSinkDeploymentType.equals(RedisSinkDeploymentType.STANDALONE)); 32 | } 33 | 34 | @Test 35 | public void shouldReturnClusterTypeFromUpperCaseInput() { 36 | RedisSinkDeploymentType redisSinkDeploymentType = redisSinkDeploymentTypeConverter.convert(null, "CLUSTER"); 37 | Assert.assertTrue(redisSinkDeploymentType.equals(RedisSinkDeploymentType.CLUSTER)); 38 | } 39 | 40 | @Test(expected = IllegalArgumentException.class) 41 | public void shouldThrowOnEmptyArgument() { 42 | redisSinkDeploymentTypeConverter.convert(null, ""); 43 | } 44 | 45 | @Test(expected = IllegalArgumentException.class) 46 | public void shouldThrowOnInvalidArgument() { 47 | redisSinkDeploymentTypeConverter.convert(null, "INVALID"); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/config/converter/RedisSinkTtlTypeConverterTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.raystack.depot.redis.enums.RedisSinkTtlType; 4 | import org.gradle.internal.impldep.org.junit.Assert; 5 | import org.junit.Before; 6 | import org.junit.Test; 7 | 8 | public class RedisSinkTtlTypeConverterTest { 9 | private RedisSinkTtlTypeConverter redisSinkTtlTypeConverter; 10 | 11 | @Before 12 | public void setUp() { 13 | redisSinkTtlTypeConverter = new RedisSinkTtlTypeConverter(); 14 | } 15 | 16 | @Test 17 | public void shouldReturnExactTimeTypeFromLowerCaseInput() { 18 | RedisSinkTtlType redisSinkTtlType = redisSinkTtlTypeConverter.convert(null, "exact_time"); 19 | Assert.assertTrue(redisSinkTtlType.equals(RedisSinkTtlType.EXACT_TIME)); 20 | } 21 | 22 | @Test 23 | public void shouldReturnExactTimeTypeFromUpperCaseInput() { 24 | RedisSinkTtlType redisSinkTtlType = redisSinkTtlTypeConverter.convert(null, "EXACT_TIME"); 25 | Assert.assertTrue(redisSinkTtlType.equals(RedisSinkTtlType.EXACT_TIME)); 26 | } 27 | 28 | @Test 29 | public void shouldReturnExactTimeTypeFromMixedCaseInput() { 30 | RedisSinkTtlType redisSinkTtlType = redisSinkTtlTypeConverter.convert(null, "eXAct_TiMe"); 31 | Assert.assertTrue(redisSinkTtlType.equals(RedisSinkTtlType.EXACT_TIME)); 32 | } 33 | 34 | @Test 35 | public void shouldReturnDisableTypeFromInput() { 36 | RedisSinkTtlType redisSinkTtlType = redisSinkTtlTypeConverter.convert(null, "DISABLE"); 37 | Assert.assertTrue(redisSinkTtlType.equals(RedisSinkTtlType.DISABLE)); 38 | } 39 | 40 | @Test 41 | public void shouldReturnDurationTypeFromInput() { 42 | RedisSinkTtlType redisSinkTtlType = redisSinkTtlTypeConverter.convert(null, "DURATION"); 43 | Assert.assertTrue(redisSinkTtlType.equals(RedisSinkTtlType.DURATION)); 44 | } 45 | 46 | @Test(expected = IllegalArgumentException.class) 47 | public void shouldThrowOnEmptyArgument() { 48 | redisSinkTtlTypeConverter.convert(null, ""); 49 | } 50 | 51 | @Test(expected = IllegalArgumentException.class) 52 | public void shouldThrowOnInvalidArgument() { 53 | redisSinkTtlTypeConverter.convert(null, "INVALID"); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/config/converter/SchemaRegistryHeadersConverterTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.raystack.depot.config.SinkConfig; 4 | import org.aeonbits.owner.ConfigFactory; 5 | import org.apache.http.message.BasicHeader; 6 | import org.junit.Assert; 7 | import org.junit.Test; 8 | 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | 12 | public class SchemaRegistryHeadersConverterTest { 13 | @Test 14 | public void testConvertIfFetchHeadersValueEmpty() { 15 | Map properties = new HashMap() { 16 | { 17 | put("SCHEMA_REGISTRY_STENCIL_FETCH_HEADERS", ""); 18 | } 19 | }; 20 | SinkConfig config = ConfigFactory.create(SinkConfig.class, properties); 21 | Assert.assertEquals(0, config.getSchemaRegistryStencilFetchHeaders().size()); 22 | } 23 | 24 | @Test 25 | public void shouldReturnZeroIfPropertyNotMentioned() { 26 | Map properties = new HashMap() { 27 | }; 28 | SinkConfig config = ConfigFactory.create(SinkConfig.class, properties); 29 | Assert.assertEquals(0, config.getSchemaRegistryStencilFetchHeaders().size()); 30 | } 31 | 32 | @Test 33 | public void shouldConvertHeaderKeyValuesWithHeaderObject() { 34 | Map properties = new HashMap() { 35 | { 36 | put("SCHEMA_REGISTRY_STENCIL_FETCH_HEADERS", "key1:value1 ,,, key2 : value2,"); 37 | } 38 | }; 39 | SinkConfig config = ConfigFactory.create(SinkConfig.class, properties); 40 | Assert.assertEquals((new BasicHeader("key1", "value1")).toString(), 41 | config.getSchemaRegistryStencilFetchHeaders().get(0).toString()); 42 | Assert.assertEquals((new BasicHeader("key2", "value2")).toString(), 43 | config.getSchemaRegistryStencilFetchHeaders().get(1).toString()); 44 | Assert.assertEquals(2, config.getSchemaRegistryStencilFetchHeaders().size()); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/config/converter/SinkConnectorSchemaMessageModeConverterTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.config.converter; 2 | 3 | import org.raystack.depot.message.SinkConnectorSchemaMessageMode; 4 | import org.junit.Assert; 5 | import org.junit.Test; 6 | import org.junit.jupiter.api.Assertions; 7 | 8 | public class SinkConnectorSchemaMessageModeConverterTest { 9 | 10 | @Test 11 | public void shouldConvertLogKey() { 12 | SinkConnectorSchemaMessageModeConverter converter = new SinkConnectorSchemaMessageModeConverter(); 13 | SinkConnectorSchemaMessageMode mode = converter.convert(null, "LOG_KEY"); 14 | Assert.assertEquals(SinkConnectorSchemaMessageMode.LOG_KEY, mode); 15 | } 16 | 17 | @Test 18 | public void shouldThrowException() { 19 | SinkConnectorSchemaMessageModeConverter converter = new SinkConnectorSchemaMessageModeConverter(); 20 | Exception exception = Assertions.assertThrows(RuntimeException.class, () -> { 21 | converter.convert(null, "Invalid"); 22 | }); 23 | Assert.assertEquals("No enum constant org.raystack.depot.message.SinkConnectorSchemaMessageMode.INVALID", 24 | exception.getMessage()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/message/field/proto/DefaultFieldTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.field.proto; 2 | 3 | import org.raystack.depot.message.field.GenericField; 4 | import org.junit.Assert; 5 | import org.junit.Test; 6 | 7 | import java.time.Instant; 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | 11 | public class DefaultFieldTest { 12 | 13 | @Test 14 | public void shouldReturnDefaultPrimitiveFields() { 15 | GenericField f = new DefaultField("test"); 16 | Assert.assertEquals("test", f.getString()); 17 | List strings = new ArrayList<>(); 18 | strings.add("test1"); 19 | strings.add("test2"); 20 | strings.add("test3"); 21 | f = new DefaultField(strings); 22 | Assert.assertEquals("[\"test1\",\"test2\",\"test3\"]", f.getString()); 23 | List integers = new ArrayList<>(); 24 | integers.add(123); 25 | integers.add(2323); 26 | integers.add(23); 27 | f = new DefaultField(integers); 28 | Assert.assertEquals("[123,2323,23]", f.getString()); 29 | 30 | List tss = new ArrayList<>(); 31 | tss.add(Instant.ofEpochSecond(1000121010)); 32 | tss.add(Instant.ofEpochSecond(1002121010)); 33 | tss.add(Instant.ofEpochSecond(1003121010)); 34 | f = new TimeStampField(tss); 35 | Assert.assertEquals("[\"2001-09-10T11:23:30Z\",\"2001-10-03T14:56:50Z\",\"2001-10-15T04:43:30Z\"]", 36 | f.getString()); 37 | 38 | List booleanList = new ArrayList<>(); 39 | booleanList.add(true); 40 | booleanList.add(false); 41 | booleanList.add(true); 42 | f = new DefaultField(booleanList); 43 | Assert.assertEquals("[true,false,true]", f.getString()); 44 | 45 | List doubles = new ArrayList<>(); 46 | doubles.add(123.93); 47 | doubles.add(13.0); 48 | doubles.add(23.0); 49 | f = new DefaultField(doubles); 50 | Assert.assertEquals("[123.93,13.0,23.0]", f.getString()); 51 | 52 | List enums = new ArrayList<>(); 53 | enums.add(TestEnum.INACTIVE); 54 | enums.add(TestEnum.COMPLETED); 55 | enums.add(TestEnum.RUNNING); 56 | f = new DefaultField(enums); 57 | Assert.assertEquals("[\"INACTIVE\",\"COMPLETED\",\"RUNNING\"]", f.getString()); 58 | 59 | } 60 | 61 | private enum TestEnum { 62 | COMPLETED, 63 | RUNNING, 64 | INACTIVE 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/message/field/proto/DurationFieldTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.field.proto; 2 | 3 | import com.google.protobuf.Duration; 4 | import com.google.protobuf.DynamicMessage; 5 | import com.google.protobuf.InvalidProtocolBufferException; 6 | import org.junit.Assert; 7 | import org.junit.Test; 8 | 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | 12 | public class DurationFieldTest { 13 | 14 | @Test 15 | public void shouldReturnDurationString() throws InvalidProtocolBufferException { 16 | Duration duration = Duration.newBuilder().setSeconds(1000).setNanos(12123).build(); 17 | DynamicMessage message = DynamicMessage.parseFrom(duration.getDescriptorForType(), duration.toByteArray()); 18 | DurationField field = new DurationField(message); 19 | Assert.assertEquals("1000.000012123s", field.getString()); 20 | } 21 | 22 | @Test 23 | public void shouldReturnDurationWithoutNanosString() throws InvalidProtocolBufferException { 24 | Duration duration = Duration.newBuilder().setSeconds(408).build(); 25 | DynamicMessage message = DynamicMessage.parseFrom(duration.getDescriptorForType(), duration.toByteArray()); 26 | DurationField field = new DurationField(message); 27 | Assert.assertEquals("408s", field.getString()); 28 | } 29 | 30 | @Test 31 | public void shouldReturnDurationListString() throws InvalidProtocolBufferException { 32 | Duration duration1 = Duration.newBuilder().setSeconds(1200).setNanos(1232138).build(); 33 | Duration duration2 = Duration.newBuilder().setSeconds(1300).setNanos(3333434).build(); 34 | Duration duration3 = Duration.newBuilder().setSeconds(1400).setNanos(5665656).build(); 35 | Duration duration4 = Duration.newBuilder().setSeconds(1500).setNanos(9089898).build(); 36 | List messages = new ArrayList<>(); 37 | messages.add(DynamicMessage.parseFrom(duration1.getDescriptorForType(), duration1.toByteArray())); 38 | messages.add(DynamicMessage.parseFrom(duration2.getDescriptorForType(), duration2.toByteArray())); 39 | messages.add(DynamicMessage.parseFrom(duration3.getDescriptorForType(), duration3.toByteArray())); 40 | messages.add(DynamicMessage.parseFrom(duration4.getDescriptorForType(), duration4.toByteArray())); 41 | DurationField field = new DurationField(messages); 42 | Assert.assertEquals( 43 | "[\"1200.001232138s\",\"1300.003333434s\",\"1400.005665656s\",\"1500.009089898s\"]", field.getString()); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/message/field/proto/TimeStampFieldTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.field.proto; 2 | 3 | import com.google.protobuf.Timestamp; 4 | import org.raystack.depot.TestDurationMessage; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | public class TimeStampFieldTest { 9 | 10 | @Test 11 | public void shouldReturnTimeStamps() { 12 | TestDurationMessage message = TestDurationMessage 13 | .newBuilder() 14 | .setEventTimestamp(Timestamp.newBuilder().setSeconds(1669962594).build()) 15 | .build(); 16 | TimeStampField field = new TimeStampField( 17 | TimeStampField.getInstant( 18 | message.getField(message.getDescriptorForType().findFieldByName("event_timestamp")))); 19 | Assert.assertEquals("2022-12-02T06:29:54Z", field.getString()); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/message/proto/UnknownProtoFieldsTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto; 2 | 3 | import org.raystack.depot.TestMessage; 4 | import org.junit.Assert; 5 | import org.junit.Test; 6 | 7 | import java.nio.charset.StandardCharsets; 8 | 9 | public class UnknownProtoFieldsTest { 10 | 11 | @Test 12 | public void shouldGetEmptyStringWithWrongMessageBytes() { 13 | String out = UnknownProtoFields.toString("abcd".getBytes(StandardCharsets.UTF_8)); 14 | Assert.assertEquals("", out); 15 | } 16 | 17 | @Test 18 | public void shouldGetUnknownFields() { 19 | TestMessage message = TestMessage.newBuilder().setOrderDetails("test").build(); 20 | String out = UnknownProtoFields.toString(message.toByteArray()); 21 | Assert.assertEquals("3: \"test\"\n", out); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/message/proto/converter/fields/ByteProtoFieldTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | import com.google.protobuf.ByteString; 4 | import com.google.protobuf.Descriptors; 5 | import org.raystack.depot.TestBytesMessage; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | 9 | import java.nio.charset.StandardCharsets; 10 | import java.util.Base64; 11 | 12 | import static org.junit.Assert.*; 13 | 14 | public class ByteProtoFieldTest { 15 | 16 | private ByteProtoField byteProtoField; 17 | private final String content = "downing street"; 18 | 19 | @Before 20 | public void setUp() throws Exception { 21 | TestBytesMessage bytesMessage = TestBytesMessage.newBuilder() 22 | .setContent(ByteString.copyFromUtf8(content)) 23 | .build(); 24 | 25 | Descriptors.FieldDescriptor fieldDescriptor = bytesMessage.getDescriptorForType().findFieldByName("content"); 26 | byteProtoField = new ByteProtoField(fieldDescriptor, bytesMessage.getField(fieldDescriptor)); 27 | } 28 | 29 | @Test 30 | public void shouldConvertBytesToString() { 31 | String parseResult = (String) byteProtoField.getValue(); 32 | String encodedBytes = new String(Base64.getEncoder().encode(content.getBytes(StandardCharsets.UTF_8))); 33 | assertEquals(encodedBytes, parseResult); 34 | } 35 | 36 | @Test 37 | public void shouldMatchByteProtobufField() { 38 | assertTrue(byteProtoField.matches()); 39 | } 40 | 41 | @Test 42 | public void shouldNotMatchFieldOtherThanByteProtobufField() { 43 | TestBytesMessage bytesMessage = TestBytesMessage.newBuilder() 44 | .build(); 45 | Descriptors.FieldDescriptor fieldDescriptor = bytesMessage.getDescriptorForType() 46 | .findFieldByName("order_number"); 47 | byteProtoField = new ByteProtoField(fieldDescriptor, bytesMessage.getField(fieldDescriptor)); 48 | 49 | assertFalse(byteProtoField.matches()); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/message/proto/converter/fields/DefaultProtoFieldTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | import com.google.protobuf.Descriptors; 4 | import com.google.protobuf.DynamicMessage; 5 | import com.google.protobuf.InvalidProtocolBufferException; 6 | import org.raystack.depot.TestMessage; 7 | import org.junit.Test; 8 | 9 | import static org.junit.Assert.assertEquals; 10 | import static org.junit.Assert.assertFalse; 11 | 12 | public class DefaultProtoFieldTest { 13 | 14 | @Test 15 | public void shouldReturnProtobufElementsAsItIs() throws InvalidProtocolBufferException { 16 | String orderNumber = "123X"; 17 | TestMessage testMessage = TestMessage.newBuilder().setOrderNumber(orderNumber).build(); 18 | DynamicMessage dynamicMessage = DynamicMessage.parseFrom(testMessage.getDescriptorForType(), 19 | testMessage.toByteArray()); 20 | Descriptors.FieldDescriptor fieldDescriptor = dynamicMessage.getDescriptorForType() 21 | .findFieldByName("order_number"); 22 | DefaultProtoField defaultProtoField = new DefaultProtoField(dynamicMessage.getField(fieldDescriptor)); 23 | Object value = defaultProtoField.getValue(); 24 | 25 | assertEquals(orderNumber, value); 26 | } 27 | 28 | @Test 29 | public void shouldNotMatchAnyType() { 30 | DefaultProtoField defaultProtoField = new DefaultProtoField(null); 31 | boolean isMatch = defaultProtoField.matches(); 32 | assertFalse(isMatch); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/message/proto/converter/fields/EnumProtoFieldTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | import com.google.protobuf.Descriptors; 4 | import com.google.protobuf.DynamicMessage; 5 | import com.google.protobuf.InvalidProtocolBufferException; 6 | import org.raystack.depot.TestEnumMessage; 7 | import org.raystack.depot.TestStatus; 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | 11 | import java.util.ArrayList; 12 | 13 | import static org.junit.Assert.assertEquals; 14 | import static org.junit.Assert.assertTrue; 15 | 16 | public class EnumProtoFieldTest { 17 | 18 | private EnumProtoField enumProtoField; 19 | 20 | @Before 21 | public void setUp() throws Exception { 22 | TestEnumMessage testEnumMessage = TestEnumMessage.newBuilder().setLastStatus(TestStatus.Enum.CREATED).build(); 23 | DynamicMessage dynamicMessage = DynamicMessage.parseFrom(testEnumMessage.getDescriptorForType(), 24 | testEnumMessage.toByteArray()); 25 | Descriptors.FieldDescriptor fieldDescriptor = dynamicMessage.getDescriptorForType() 26 | .findFieldByName("last_status"); 27 | enumProtoField = new EnumProtoField(fieldDescriptor, dynamicMessage.getField(fieldDescriptor)); 28 | } 29 | 30 | @Test 31 | public void shouldConvertProtobufEnumToString() { 32 | String fieldValue = (String) enumProtoField.getValue(); 33 | assertEquals("CREATED", fieldValue); 34 | } 35 | 36 | @Test 37 | public void shouldConvertRepeatedProtobufEnumToListOfString() throws InvalidProtocolBufferException { 38 | TestEnumMessage testEnumMessage = TestEnumMessage.newBuilder() 39 | .addStatusHistory(TestStatus.Enum.CREATED) 40 | .addStatusHistory(TestStatus.Enum.IN_PROGRESS) 41 | .build(); 42 | DynamicMessage dynamicMessage = DynamicMessage.parseFrom(testEnumMessage.getDescriptorForType(), 43 | testEnumMessage.toByteArray()); 44 | Descriptors.FieldDescriptor fieldDescriptor = dynamicMessage.getDescriptorForType() 45 | .findFieldByName("status_history"); 46 | enumProtoField = new EnumProtoField(fieldDescriptor, dynamicMessage.getField(fieldDescriptor)); 47 | Object fieldValue = enumProtoField.getValue(); 48 | 49 | ArrayList enumValueList = new ArrayList<>(); 50 | enumValueList.add("CREATED"); 51 | enumValueList.add("IN_PROGRESS"); 52 | assertEquals(enumValueList, fieldValue); 53 | } 54 | 55 | @Test 56 | public void shouldMatchEnumProtobufField() { 57 | boolean isMatch = enumProtoField.matches(); 58 | assertTrue(isMatch); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/message/proto/converter/fields/MessageProtoFieldTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | import com.google.protobuf.Descriptors; 4 | import com.google.protobuf.DynamicMessage; 5 | import org.raystack.depot.TestMessage; 6 | import org.raystack.depot.TestNestedMessage; 7 | import org.junit.Before; 8 | import org.junit.Test; 9 | 10 | import static org.junit.Assert.assertEquals; 11 | import static org.junit.Assert.assertTrue; 12 | 13 | public class MessageProtoFieldTest { 14 | 15 | private MessageProtoField messageProtoField; 16 | private TestMessage childField; 17 | 18 | @Before 19 | public void setUp() throws Exception { 20 | childField = TestMessage.newBuilder() 21 | .setOrderNumber("123X") 22 | .build(); 23 | TestNestedMessage nestedMessage = TestNestedMessage.newBuilder() 24 | .setSingleMessage(childField) 25 | .build(); 26 | DynamicMessage dynamicMessage = DynamicMessage.parseFrom(nestedMessage.getDescriptorForType(), 27 | nestedMessage.toByteArray()); 28 | 29 | Descriptors.FieldDescriptor fieldDescriptor = nestedMessage.getDescriptorForType() 30 | .findFieldByName("single_message"); 31 | messageProtoField = new MessageProtoField(fieldDescriptor, dynamicMessage.getField(fieldDescriptor)); 32 | 33 | } 34 | 35 | @Test 36 | public void shouldReturnDynamicMessage() { 37 | Object nestedChild = messageProtoField.getValue(); 38 | assertEquals(childField, nestedChild); 39 | } 40 | 41 | @Test 42 | public void shouldMatchDynamicMessageAsNested() { 43 | boolean isMatch = messageProtoField.matches(); 44 | assertTrue(isMatch); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/message/proto/converter/fields/TimestampProtoFieldTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.message.proto.converter.fields; 2 | 3 | import com.google.protobuf.Descriptors; 4 | import com.google.protobuf.DynamicMessage; 5 | import com.google.protobuf.InvalidProtocolBufferException; 6 | import com.google.protobuf.Timestamp; 7 | import org.raystack.depot.TestDurationMessage; 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | 11 | import java.time.Instant; 12 | 13 | import static org.junit.Assert.assertEquals; 14 | import static org.junit.Assert.assertTrue; 15 | 16 | public class TimestampProtoFieldTest { 17 | private TimestampProtoField timestampProtoField; 18 | private Instant time; 19 | 20 | @Before 21 | public void setUp() throws Exception { 22 | time = Instant.ofEpochSecond(200, 200); 23 | TestDurationMessage message = TestDurationMessage.newBuilder() 24 | .setEventTimestamp(Timestamp.newBuilder() 25 | .setSeconds(time.getEpochSecond()) 26 | .setNanos(time.getNano()) 27 | .build()) 28 | .build(); 29 | DynamicMessage dynamicMessage = DynamicMessage.parseFrom(message.getDescriptorForType(), message.toByteArray()); 30 | Descriptors.FieldDescriptor fieldDescriptor = dynamicMessage.getDescriptorForType() 31 | .findFieldByName("event_timestamp"); 32 | timestampProtoField = new TimestampProtoField(fieldDescriptor, dynamicMessage.getField(fieldDescriptor)); 33 | } 34 | 35 | @Test 36 | public void shouldParseGoogleProtobufTimestampProtoMessageToInstant() throws InvalidProtocolBufferException { 37 | assertEquals(time, timestampProtoField.getValue()); 38 | } 39 | 40 | @Test 41 | public void shouldMatchGoogleProtobufTimestamp() { 42 | assertTrue(timestampProtoField.matches()); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/metrics/MetricsTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.metrics; 2 | 3 | public class MetricsTest { 4 | } 5 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/redis/client/response/RedisClusterResponseTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.client.response; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class RedisClusterResponseTest { 7 | private RedisClusterResponse redisClusterResponse; 8 | 9 | @Test 10 | public void shouldReportWhenSuccess() { 11 | String response = "Success"; 12 | Long ttlResponse = 1L; 13 | redisClusterResponse = new RedisClusterResponse("SET", response, ttlResponse); 14 | Assert.assertFalse(redisClusterResponse.isFailed()); 15 | Assert.assertEquals("SET: Success, TTL: UPDATED", redisClusterResponse.getMessage()); 16 | } 17 | 18 | @Test 19 | public void shouldReportWhenFailed() { 20 | redisClusterResponse = new RedisClusterResponse("Failed"); 21 | Assert.assertTrue(redisClusterResponse.isFailed()); 22 | Assert.assertEquals("Failed", redisClusterResponse.getMessage()); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/redis/client/response/RedisStandaloneResponseTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.client.response; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | import org.junit.runner.RunWith; 6 | import org.mockito.Mock; 7 | import org.mockito.junit.MockitoJUnitRunner; 8 | import redis.clients.jedis.Response; 9 | import redis.clients.jedis.exceptions.JedisException; 10 | 11 | import static org.mockito.Mockito.when; 12 | 13 | @RunWith(MockitoJUnitRunner.class) 14 | public class RedisStandaloneResponseTest { 15 | @Mock 16 | private Response response; 17 | @Mock 18 | private Response ttlResponse; 19 | private RedisStandaloneResponse redisResponse; 20 | 21 | @Test 22 | public void shouldReportNotFailedWhenJedisExceptionNotThrown() { 23 | when(response.get()).thenReturn("Success response"); 24 | when(ttlResponse.get()).thenReturn(1L); 25 | redisResponse = new RedisStandaloneResponse("SET", response, ttlResponse); 26 | Assert.assertFalse(redisResponse.process().isFailed()); 27 | Assert.assertEquals("SET: Success response, TTL: UPDATED", redisResponse.process().getMessage()); 28 | } 29 | 30 | @Test 31 | public void shouldReportFailedWhenJedisExceptionThrown() { 32 | when(response.get()).thenThrow(new JedisException("Failed response")); 33 | redisResponse = new RedisStandaloneResponse("SET", response, ttlResponse); 34 | Assert.assertTrue(redisResponse.process().isFailed()); 35 | Assert.assertEquals("Failed response", redisResponse.process().getMessage()); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/redis/ttl/DurationTTLTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.ttl; 2 | 3 | import org.junit.Before; 4 | import org.junit.Test; 5 | import org.junit.runner.RunWith; 6 | import org.mockito.Mock; 7 | import org.mockito.junit.MockitoJUnitRunner; 8 | import redis.clients.jedis.JedisCluster; 9 | import redis.clients.jedis.Pipeline; 10 | 11 | import static org.mockito.Mockito.times; 12 | import static org.mockito.Mockito.verify; 13 | 14 | @RunWith(MockitoJUnitRunner.class) 15 | public class DurationTTLTest { 16 | 17 | private DurationTtl durationTTL; 18 | 19 | @Mock 20 | private Pipeline pipeline; 21 | 22 | @Mock 23 | private JedisCluster jedisCluster; 24 | 25 | @Before 26 | public void setup() { 27 | durationTTL = new DurationTtl(10); 28 | } 29 | 30 | @Test 31 | public void shouldSetTTLInSecondsForPipeline() { 32 | durationTTL.setTtl(pipeline, "test-key"); 33 | verify(pipeline, times(1)).expire("test-key", 10); 34 | } 35 | 36 | @Test 37 | public void shouldSetTTLInSecondsForCluster() { 38 | durationTTL.setTtl(jedisCluster, "test-key"); 39 | verify(jedisCluster, times(1)).expire("test-key", 10); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/redis/ttl/ExactTimeTTLTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.ttl; 2 | 3 | import org.junit.Before; 4 | import org.junit.Test; 5 | import org.junit.runner.RunWith; 6 | import org.mockito.Mock; 7 | import org.mockito.junit.MockitoJUnitRunner; 8 | import redis.clients.jedis.JedisCluster; 9 | import redis.clients.jedis.Pipeline; 10 | 11 | import static org.mockito.Mockito.times; 12 | import static org.mockito.Mockito.verify; 13 | 14 | @RunWith(MockitoJUnitRunner.class) 15 | public class ExactTimeTTLTest { 16 | 17 | private ExactTimeTtl exactTimeTTL; 18 | @Mock 19 | private Pipeline pipeline; 20 | 21 | @Mock 22 | private JedisCluster jedisCluster; 23 | 24 | @Before 25 | public void setup() { 26 | exactTimeTTL = new ExactTimeTtl(10000000L); 27 | } 28 | 29 | @Test 30 | public void shouldSetUnixTimeStampAsTTLForPipeline() { 31 | exactTimeTTL.setTtl(pipeline, "test-key"); 32 | verify(pipeline, times(1)).expireAt("test-key", 10000000L); 33 | } 34 | 35 | @Test 36 | public void shouldSetUnixTimeStampAsTTLForCluster() { 37 | exactTimeTTL.setTtl(jedisCluster, "test-key"); 38 | verify(jedisCluster, times(1)).expireAt("test-key", 10000000L); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/redis/ttl/RedisTtlFactoryTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.redis.ttl; 2 | 3 | import org.raystack.depot.config.RedisSinkConfig; 4 | import org.raystack.depot.exception.ConfigurationException; 5 | import org.raystack.depot.redis.enums.RedisSinkTtlType; 6 | import org.junit.Assert; 7 | import org.junit.Before; 8 | import org.junit.Rule; 9 | import org.junit.Test; 10 | import org.junit.rules.ExpectedException; 11 | import org.junit.runner.RunWith; 12 | import org.mockito.Mock; 13 | import org.mockito.junit.MockitoJUnitRunner; 14 | 15 | import static org.mockito.Mockito.when; 16 | 17 | @RunWith(MockitoJUnitRunner.class) 18 | public class RedisTtlFactoryTest { 19 | 20 | @Mock 21 | private RedisSinkConfig redisSinkConfig; 22 | 23 | @Rule 24 | public ExpectedException expectedException = ExpectedException.none(); 25 | 26 | @Before 27 | public void setup() { 28 | when(redisSinkConfig.getSinkRedisTtlType()).thenReturn(RedisSinkTtlType.DISABLE); 29 | } 30 | 31 | @Test 32 | public void shouldReturnNoTTLIfNothingGiven() { 33 | RedisTtl redisTTL = RedisTTLFactory.getTTl(redisSinkConfig); 34 | Assert.assertEquals(redisTTL.getClass(), NoRedisTtl.class); 35 | } 36 | 37 | @Test 38 | public void shouldReturnExactTimeTTL() { 39 | when(redisSinkConfig.getSinkRedisTtlType()).thenReturn(RedisSinkTtlType.EXACT_TIME); 40 | when(redisSinkConfig.getSinkRedisTtlValue()).thenReturn(100L); 41 | RedisTtl redisTTL = RedisTTLFactory.getTTl(redisSinkConfig); 42 | Assert.assertEquals(redisTTL.getClass(), ExactTimeTtl.class); 43 | } 44 | 45 | @Test 46 | public void shouldReturnDurationTTL() { 47 | when(redisSinkConfig.getSinkRedisTtlType()).thenReturn(RedisSinkTtlType.DURATION); 48 | when(redisSinkConfig.getSinkRedisTtlValue()).thenReturn(100L); 49 | RedisTtl redisTTL = RedisTTLFactory.getTTl(redisSinkConfig); 50 | Assert.assertEquals(redisTTL.getClass(), DurationTtl.class); 51 | } 52 | 53 | @Test 54 | public void shouldThrowExceptionInCaseOfInvalidConfiguration() { 55 | expectedException.expect(ConfigurationException.class); 56 | expectedException.expectMessage("Provide a positive TTL value"); 57 | when(redisSinkConfig.getSinkRedisTtlValue()).thenReturn(-1L); 58 | when(redisSinkConfig.getSinkRedisTtlType()).thenReturn(RedisSinkTtlType.DURATION); 59 | RedisTTLFactory.getTTl(redisSinkConfig); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/test/java/org/raystack/depot/utils/StringUtilsTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.depot.utils; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class StringUtilsTest { 7 | 8 | @Test 9 | public void shouldReturnValidArgumentsForStringFormat() { 10 | Assert.assertEquals(0, StringUtils.countVariables("test")); 11 | Assert.assertEquals(0, StringUtils.countVariables("")); 12 | Assert.assertEquals(1, StringUtils.countVariables("test%dtest")); 13 | Assert.assertEquals(2, StringUtils.countVariables("test%dtest%ttest")); 14 | Assert.assertEquals(5, StringUtils.countVariables("test%dtest%ttest dskladja %s ds %d sdajk %b")); 15 | } 16 | 17 | @Test 18 | public void shouldReturnCharacterCount() { 19 | Assert.assertEquals(0, StringUtils.count("test", 'i')); 20 | Assert.assertEquals(0, StringUtils.count("", '5')); 21 | Assert.assertEquals(2, StringUtils.count("test", 't')); 22 | Assert.assertEquals(1, StringUtils.count("test", 'e')); 23 | Assert.assertEquals(1, StringUtils.count("test", 's')); 24 | Assert.assertEquals(0, StringUtils.count("test", '%')); 25 | } 26 | 27 | @Test 28 | public void shouldReturnValidArgsAndCharacters() { 29 | String testString = "test%s%ddjaklsjd%%%%s%y%d"; 30 | Assert.assertEquals(8, StringUtils.count(testString, '%')); 31 | Assert.assertEquals(4, StringUtils.countVariables(testString)); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/proto/TestGrpc.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package org.raystack.depot; 4 | 5 | option java_multiple_files = true; 6 | option java_package = "org.raystack.depot"; 7 | option java_outer_classname = "SampleGrpcServerProto"; 8 | 9 | service TestServer { 10 | rpc TestRpcMethod (TestGrpcRequest) returns (TestGrpcResponse) {} 11 | } 12 | 13 | message TestGrpcRequest { 14 | string field1 = 1; 15 | string field2 = 2; 16 | } 17 | 18 | message Error { 19 | string code = 1; 20 | string entity = 2; 21 | } 22 | 23 | message TestGrpcResponse { 24 | bool success = 1; 25 | repeated Error error = 2; 26 | string field3 = 3; 27 | string field4 = 4; 28 | } 29 | -------------------------------------------------------------------------------- /src/test/proto/TestMessageBQ.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package raystack.depot; 4 | 5 | import "google/protobuf/timestamp.proto"; 6 | import "google/protobuf/duration.proto"; 7 | import "google/protobuf/struct.proto"; 8 | import "google/type/date.proto"; 9 | 10 | option java_multiple_files = true; 11 | option java_package = "org.raystack.depot"; 12 | option java_outer_classname = "TestMessageProtoBQ"; 13 | 14 | message TestKeyBQ { 15 | string order_number = 1; 16 | string order_url = 2; 17 | } 18 | 19 | message TestMessageBQ { 20 | string order_number = 1; 21 | string order_url = 2; 22 | string order_details = 3; 23 | google.protobuf.Timestamp created_at = 4; 24 | StatusBQ status = 5; 25 | int64 discount = 6; 26 | bool success = 7; 27 | float price = 8; 28 | map current_state = 9; 29 | bytes user_token = 10; 30 | google.protobuf.Duration trip_duration = 11; 31 | repeated string aliases = 12; 32 | google.protobuf.Struct properties = 13; 33 | google.type.Date order_date = 14; 34 | repeated google.protobuf.Timestamp updated_at = 15; 35 | repeated google.protobuf.Struct attributes = 16; 36 | repeated google.protobuf.Duration intervals = 17; 37 | int32 counter = 18; 38 | string camelCase = 19; 39 | } 40 | 41 | message TestMessageChildBQ { 42 | string order_number = 1; 43 | bool success = 7; 44 | } 45 | 46 | message TestNestedMessageBQ { 47 | string nested_id = 1; 48 | TestMessageBQ single_message = 2; 49 | } 50 | 51 | message TestRecursiveMessageBQ { 52 | string string_value = 1; 53 | float float_value = 2; 54 | TestRecursiveMessageBQ recursive_message = 3; 55 | } 56 | 57 | message TestNestedRepeatedMessageBQ { 58 | TestMessageBQ single_message = 1; 59 | repeated TestMessageBQ repeated_message = 2; 60 | int32 number_field = 3; 61 | repeated int32 repeated_number_field = 4; 62 | } 63 | 64 | enum StatusBQ { 65 | COMPLETED = 0; 66 | CANCELLED = 1; 67 | } 68 | -------------------------------------------------------------------------------- /src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker: -------------------------------------------------------------------------------- 1 | mock-maker-inline --------------------------------------------------------------------------------