├── .github └── workflows │ ├── build.yml │ ├── dependencies-publish.yml │ ├── docs.yml │ ├── integration.yml │ ├── package.yml │ ├── python_release.yml │ └── python_validation.yml ├── .gitignore ├── LICENSE ├── README.md ├── build.gradle ├── config └── checkstyle │ ├── checkstyle.xml │ └── suppressions.xml ├── dagger-common ├── build.gradle └── src │ ├── main │ └── java │ │ └── org │ │ └── raystack │ │ └── dagger │ │ └── common │ │ ├── configuration │ │ └── Configuration.java │ │ ├── core │ │ ├── Constants.java │ │ ├── DaggerContext.java │ │ ├── FieldDescriptorCache.java │ │ ├── StencilClientOrchestrator.java │ │ ├── StreamInfo.java │ │ └── Transformer.java │ │ ├── exceptions │ │ ├── DaggerContextException.java │ │ ├── DescriptorNotFoundException.java │ │ └── serde │ │ │ ├── DaggerDeserializationException.java │ │ │ ├── DaggerSerializationException.java │ │ │ ├── DataTypeNotSupportedException.java │ │ │ ├── EnumFieldNotFoundException.java │ │ │ ├── InvalidColumnMappingException.java │ │ │ ├── InvalidDataTypeException.java │ │ │ ├── InvalidJSONSchemaException.java │ │ │ └── SimpleGroupParsingException.java │ │ ├── metrics │ │ ├── aspects │ │ │ ├── AspectType.java │ │ │ └── Aspects.java │ │ └── managers │ │ │ ├── CounterStatsManager.java │ │ │ ├── GaugeStatsManager.java │ │ │ └── MeterStatsManager.java │ │ ├── serde │ │ ├── DaggerDeserializer.java │ │ ├── DaggerInternalTypeInformation.java │ │ ├── DataTypes.java │ │ ├── json │ │ │ └── deserialization │ │ │ │ ├── JsonDeserializer.java │ │ │ │ └── JsonType.java │ │ ├── parquet │ │ │ ├── SimpleGroupValidation.java │ │ │ └── deserialization │ │ │ │ └── SimpleGroupDeserializer.java │ │ ├── proto │ │ │ ├── deserialization │ │ │ │ ├── ProtoDeserializer.java │ │ │ │ └── ProtoType.java │ │ │ └── serialization │ │ │ │ ├── KafkaProtoSerializer.java │ │ │ │ └── ProtoSerializer.java │ │ └── typehandler │ │ │ ├── PrimitiveTypeHandler.java │ │ │ ├── RowFactory.java │ │ │ ├── TypeHandler.java │ │ │ ├── TypeHandlerFactory.java │ │ │ ├── TypeInformationFactory.java │ │ │ ├── complex │ │ │ ├── EnumHandler.java │ │ │ ├── MapHandler.java │ │ │ ├── MessageHandler.java │ │ │ ├── StructMessageHandler.java │ │ │ └── TimestampHandler.java │ │ │ ├── primitive │ │ │ ├── BooleanHandler.java │ │ │ ├── ByteStringHandler.java │ │ │ ├── DoubleHandler.java │ │ │ ├── FloatHandler.java │ │ │ ├── IntegerHandler.java │ │ │ ├── LongHandler.java │ │ │ ├── PrimitiveHandler.java │ │ │ ├── PrimitiveHandlerFactory.java │ │ │ └── StringHandler.java │ │ │ └── repeated │ │ │ ├── RepeatedEnumHandler.java │ │ │ ├── RepeatedMessageHandler.java │ │ │ ├── RepeatedPrimitiveHandler.java │ │ │ └── RepeatedStructMessageHandler.java │ │ ├── udfs │ │ ├── AggregateUdf.java │ │ ├── ScalarUdf.java │ │ ├── TableUdf.java │ │ └── UdfFactory.java │ │ └── watermark │ │ ├── LastColumnWatermark.java │ │ ├── NoWatermark.java │ │ ├── RowtimeFieldWatermark.java │ │ ├── StreamWatermarkAssigner.java │ │ └── WatermarkStrategyDefinition.java │ └── test │ ├── java │ └── com │ │ └── gotocompany │ │ └── dagger │ │ └── common │ │ ├── configuration │ │ └── ConfigurationTest.java │ │ ├── core │ │ ├── DaggerContextTestBase.java │ │ ├── FieldDescriptorCacheTest.java │ │ └── StencilClientOrchestratorTest.java │ │ ├── metrics │ │ └── managers │ │ │ ├── CounterStatsManagerTest.java │ │ │ ├── GaugeStatsManagerTest.java │ │ │ ├── MeterStatsManagerTest.java │ │ │ └── utils │ │ │ └── TestAspects.java │ │ ├── serde │ │ ├── json │ │ │ └── deserialization │ │ │ │ ├── JsonDeserializerTest.java │ │ │ │ └── JsonTypeTest.java │ │ ├── parquet │ │ │ ├── SimpleGroupValidationTest.java │ │ │ └── deserialization │ │ │ │ └── SimpleGroupDeserializerTest.java │ │ ├── proto │ │ │ ├── deserialization │ │ │ │ ├── ProtoDeserializerTest.java │ │ │ │ └── ProtoTypeTest.java │ │ │ └── serialization │ │ │ │ ├── KafkaProtoSerializerTest.java │ │ │ │ └── ProtoSerializerTest.java │ │ └── typehandler │ │ │ ├── PrimitiveTypeHandlerTest.java │ │ │ ├── RowFactoryTest.java │ │ │ ├── TypeHandlerFactoryTest.java │ │ │ ├── TypeInformationFactoryTest.java │ │ │ ├── complex │ │ │ ├── EnumHandlerTest.java │ │ │ ├── MapHandlerTest.java │ │ │ ├── MessageHandlerTest.java │ │ │ ├── StructMessageHandlerTest.java │ │ │ └── TimestampHandlerTest.java │ │ │ ├── primitive │ │ │ ├── BooleanHandlerTest.java │ │ │ ├── ByteStringHandlerTest.java │ │ │ ├── DoubleHandlerTest.java │ │ │ ├── FloatHandlerTest.java │ │ │ ├── IntegerHandlerTest.java │ │ │ ├── LongHandlerTest.java │ │ │ ├── PrimitiveHandlerFactoryTest.java │ │ │ └── StringHandlerTest.java │ │ │ └── repeated │ │ │ ├── RepeatedEnumHandlerTest.java │ │ │ ├── RepeatedMessageHandlerTest.java │ │ │ ├── RepeatedPrimitiveHandlerTest.java │ │ │ └── RepeatedStructMessageHandlerTest.java │ │ └── watermark │ │ └── StreamWatermarkAssignerTest.java │ └── proto │ ├── TestGrpc.proto │ ├── TestLogMessage.proto │ ├── TestMessage.proto │ └── sample_message.txt ├── dagger-core ├── build.gradle ├── env │ └── local.properties └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── raystack │ │ │ └── dagger │ │ │ └── core │ │ │ ├── KafkaProtoSQLProcessor.java │ │ │ ├── StreamManager.java │ │ │ ├── config │ │ │ ├── CommandlineConfigurationProvider.java │ │ │ ├── ConfigurationProvider.java │ │ │ ├── ConfigurationProviderFactory.java │ │ │ ├── EnvironmentConfigurationProvider.java │ │ │ ├── FileConfigurationProvider.java │ │ │ └── KafkaEnvironmentVariables.java │ │ │ ├── deserializer │ │ │ ├── DaggerDeserializerFactory.java │ │ │ ├── DaggerDeserializerProvider.java │ │ │ ├── JsonDeserializerProvider.java │ │ │ ├── ProtoDeserializerProvider.java │ │ │ └── SimpleGroupDeserializerProvider.java │ │ │ ├── exception │ │ │ ├── BigQueryWriterException.java │ │ │ ├── ChannelNotAvailableException.java │ │ │ ├── ConsumerLagNotZeroException.java │ │ │ ├── DaggerConfigurationException.java │ │ │ ├── GrpcFailureException.java │ │ │ ├── HttpFailureException.java │ │ │ ├── InfluxWriteException.java │ │ │ ├── InputOutputMappingException.java │ │ │ ├── InvalidConfigurationException.java │ │ │ ├── InvalidDaggerSourceException.java │ │ │ ├── InvalidGrpcBodyException.java │ │ │ ├── InvalidHttpVerbException.java │ │ │ ├── InvalidLongbowDurationException.java │ │ │ ├── InvalidTimeRangeException.java │ │ │ ├── ParquetFileSourceReaderInitializationException.java │ │ │ ├── PathParserNotProvidedException.java │ │ │ ├── RecordsNotConsumedException.java │ │ │ ├── ThorCommandFailedException.java │ │ │ ├── TransformClassNotDefinedException.java │ │ │ └── UDFFactoryClassNotDefinedException.java │ │ │ ├── metrics │ │ │ ├── aspects │ │ │ │ ├── ChronologyOrderedSplitAssignerAspects.java │ │ │ │ ├── ExternalSourceAspects.java │ │ │ │ ├── LongbowReaderAspects.java │ │ │ │ ├── LongbowWriterAspects.java │ │ │ │ ├── ParquetReaderAspects.java │ │ │ │ └── TelemetryAspects.java │ │ │ ├── reporters │ │ │ │ ├── ErrorReporter.java │ │ │ │ ├── ErrorReporterFactory.java │ │ │ │ ├── ErrorStatsReporter.java │ │ │ │ ├── NoOpErrorReporter.java │ │ │ │ └── statsd │ │ │ │ │ ├── DaggerMetricsConfig.java │ │ │ │ │ ├── DaggerStatsDReporter.java │ │ │ │ │ ├── SerializedStatsDReporterSupplier.java │ │ │ │ │ ├── StatsDErrorReporter.java │ │ │ │ │ ├── manager │ │ │ │ │ ├── DaggerCounterManager.java │ │ │ │ │ ├── DaggerGaugeManager.java │ │ │ │ │ ├── DaggerHistogramManager.java │ │ │ │ │ └── MeasurementManager.java │ │ │ │ │ ├── measurement │ │ │ │ │ ├── Counter.java │ │ │ │ │ ├── Gauge.java │ │ │ │ │ └── Histogram.java │ │ │ │ │ └── tags │ │ │ │ │ ├── ComponentTags.java │ │ │ │ │ ├── GlobalTags.java │ │ │ │ │ └── StatsDTag.java │ │ │ └── telemetry │ │ │ │ ├── TelemetryPublisher.java │ │ │ │ ├── TelemetrySubscriber.java │ │ │ │ └── TelemetryTypes.java │ │ │ ├── processors │ │ │ ├── ColumnNameManager.java │ │ │ ├── ParentPostProcessor.java │ │ │ ├── PostProcessorConfig.java │ │ │ ├── PostProcessorFactory.java │ │ │ ├── PreProcessorConfig.java │ │ │ ├── PreProcessorFactory.java │ │ │ ├── PreProcessorOrchestrator.java │ │ │ ├── common │ │ │ │ ├── DescriptorManager.java │ │ │ │ ├── EndpointHandler.java │ │ │ │ ├── FetchOutputDecorator.java │ │ │ │ ├── InitializationDecorator.java │ │ │ │ ├── OutputMapping.java │ │ │ │ ├── PostResponseTelemetry.java │ │ │ │ ├── RowManager.java │ │ │ │ ├── SchemaConfig.java │ │ │ │ └── ValidRecordsDecorator.java │ │ │ ├── external │ │ │ │ ├── AsyncConnector.java │ │ │ │ ├── ExternalMetricConfig.java │ │ │ │ ├── ExternalPostProcessor.java │ │ │ │ ├── ExternalSourceConfig.java │ │ │ │ ├── es │ │ │ │ │ ├── EsAsyncConnector.java │ │ │ │ │ ├── EsResponseHandler.java │ │ │ │ │ ├── EsSourceConfig.java │ │ │ │ │ ├── EsSourceConfigBuilder.java │ │ │ │ │ └── EsStreamDecorator.java │ │ │ │ ├── grpc │ │ │ │ │ ├── GrpcAsyncConnector.java │ │ │ │ │ ├── GrpcResponseHandler.java │ │ │ │ │ ├── GrpcSourceConfig.java │ │ │ │ │ ├── GrpcSourceConfigBuilder.java │ │ │ │ │ ├── GrpcStreamDecorator.java │ │ │ │ │ └── client │ │ │ │ │ │ ├── DynamicMessageMarshaller.java │ │ │ │ │ │ ├── GrpcClient.java │ │ │ │ │ │ └── GrpcRequestHandler.java │ │ │ │ ├── http │ │ │ │ │ ├── HttpAsyncConnector.java │ │ │ │ │ ├── HttpResponseHandler.java │ │ │ │ │ ├── HttpSourceConfig.java │ │ │ │ │ ├── HttpStreamDecorator.java │ │ │ │ │ └── request │ │ │ │ │ │ ├── HttpGetRequestHandler.java │ │ │ │ │ │ ├── HttpPostRequestHandler.java │ │ │ │ │ │ ├── HttpPutRequestHandler.java │ │ │ │ │ │ ├── HttpRequestFactory.java │ │ │ │ │ │ └── HttpRequestHandler.java │ │ │ │ └── pg │ │ │ │ │ ├── PgAsyncConnector.java │ │ │ │ │ ├── PgResponseHandler.java │ │ │ │ │ ├── PgSourceConfig.java │ │ │ │ │ ├── PgSourceConfigBuilder.java │ │ │ │ │ └── PgStreamDecorator.java │ │ │ ├── internal │ │ │ │ ├── InternalDecorator.java │ │ │ │ ├── InternalPostProcessor.java │ │ │ │ ├── InternalSourceConfig.java │ │ │ │ └── processor │ │ │ │ │ ├── InternalConfigHandlerFactory.java │ │ │ │ │ ├── InternalConfigProcessor.java │ │ │ │ │ ├── constant │ │ │ │ │ └── ConstantInternalConfigProcessor.java │ │ │ │ │ ├── function │ │ │ │ │ ├── FunctionInternalConfigProcessor.java │ │ │ │ │ ├── FunctionProcessor.java │ │ │ │ │ ├── FunctionProcessorFactory.java │ │ │ │ │ └── functions │ │ │ │ │ │ ├── CurrentTimestampFunction.java │ │ │ │ │ │ ├── InvalidFunction.java │ │ │ │ │ │ └── JsonPayloadFunction.java │ │ │ │ │ ├── invalid │ │ │ │ │ └── InvalidInternalConfigProcessor.java │ │ │ │ │ └── sql │ │ │ │ │ ├── SqlConfigTypePathParser.java │ │ │ │ │ ├── SqlInternalFieldConfig.java │ │ │ │ │ └── fields │ │ │ │ │ ├── SqlInternalAutoFieldImport.java │ │ │ │ │ ├── SqlInternalConfigProcessor.java │ │ │ │ │ ├── SqlInternalFieldFactory.java │ │ │ │ │ └── SqlInternalFieldImport.java │ │ │ ├── longbow │ │ │ │ ├── AsyncProcessor.java │ │ │ │ ├── LongbowFactory.java │ │ │ │ ├── LongbowProcessor.java │ │ │ │ ├── LongbowSchema.java │ │ │ │ ├── columnmodifier │ │ │ │ │ ├── ColumnModifier.java │ │ │ │ │ ├── LongbowReadColumnModifier.java │ │ │ │ │ ├── LongbowWriteColumnModifier.java │ │ │ │ │ └── NoOpColumnModifier.java │ │ │ │ ├── data │ │ │ │ │ ├── LongbowData.java │ │ │ │ │ ├── LongbowDataFactory.java │ │ │ │ │ ├── LongbowProtoData.java │ │ │ │ │ └── LongbowTableData.java │ │ │ │ ├── exceptions │ │ │ │ │ ├── LongbowReaderException.java │ │ │ │ │ └── LongbowWriterException.java │ │ │ │ ├── outputRow │ │ │ │ │ ├── OutputIdentity.java │ │ │ │ │ ├── OutputSynchronizer.java │ │ │ │ │ ├── ReaderOutputLongbowData.java │ │ │ │ │ ├── ReaderOutputProtoData.java │ │ │ │ │ ├── ReaderOutputRow.java │ │ │ │ │ └── WriterOutputRow.java │ │ │ │ ├── processor │ │ │ │ │ ├── LongbowReader.java │ │ │ │ │ └── LongbowWriter.java │ │ │ │ ├── range │ │ │ │ │ ├── LongbowAbsoluteRange.java │ │ │ │ │ ├── LongbowDurationRange.java │ │ │ │ │ ├── LongbowRange.java │ │ │ │ │ └── LongbowRangeFactory.java │ │ │ │ ├── request │ │ │ │ │ ├── ProtoBytePutRequest.java │ │ │ │ │ ├── ProtoByteScanRequest.java │ │ │ │ │ ├── PutRequestFactory.java │ │ │ │ │ ├── ScanRequestFactory.java │ │ │ │ │ ├── TablePutRequest.java │ │ │ │ │ └── TableScanRequest.java │ │ │ │ ├── storage │ │ │ │ │ ├── LongbowStore.java │ │ │ │ │ ├── PutRequest.java │ │ │ │ │ └── ScanRequest.java │ │ │ │ └── validator │ │ │ │ │ ├── LongbowType.java │ │ │ │ │ └── LongbowValidator.java │ │ │ ├── telemetry │ │ │ │ ├── TelemetryProcessor.java │ │ │ │ └── processor │ │ │ │ │ └── MetricsTelemetryExporter.java │ │ │ ├── transformers │ │ │ │ ├── TableTransformConfig.java │ │ │ │ ├── TransformConfig.java │ │ │ │ ├── TransformProcessor.java │ │ │ │ └── TransformerUtils.java │ │ │ └── types │ │ │ │ ├── FilterDecorator.java │ │ │ │ ├── MapDecorator.java │ │ │ │ ├── PostProcessor.java │ │ │ │ ├── Preprocessor.java │ │ │ │ ├── SourceConfig.java │ │ │ │ ├── StreamDecorator.java │ │ │ │ └── Validator.java │ │ │ ├── sink │ │ │ ├── SinkOrchestrator.java │ │ │ ├── bigquery │ │ │ │ ├── BigQuerySink.java │ │ │ │ ├── BigQuerySinkBuilder.java │ │ │ │ └── BigQuerySinkWriter.java │ │ │ ├── influx │ │ │ │ ├── ErrorHandler.java │ │ │ │ ├── InfluxDBFactoryWrapper.java │ │ │ │ ├── InfluxDBSink.java │ │ │ │ ├── InfluxDBWriter.java │ │ │ │ └── errors │ │ │ │ │ ├── InfluxError.java │ │ │ │ │ ├── LateRecordDropError.java │ │ │ │ │ ├── NoError.java │ │ │ │ │ ├── ValidError.java │ │ │ │ │ └── ValidException.java │ │ │ ├── kafka │ │ │ │ ├── KafkaSerializationSchemaFactory.java │ │ │ │ ├── KafkaSerializerBuilder.java │ │ │ │ └── builder │ │ │ │ │ ├── KafkaJsonSerializerBuilder.java │ │ │ │ │ └── KafkaProtoSerializerBuilder.java │ │ │ └── log │ │ │ │ ├── LogSink.java │ │ │ │ └── LogSinkWriter.java │ │ │ ├── source │ │ │ ├── DaggerSource.java │ │ │ ├── DaggerSourceFactory.java │ │ │ ├── Stream.java │ │ │ ├── StreamsFactory.java │ │ │ ├── config │ │ │ │ ├── StreamConfig.java │ │ │ │ ├── StreamConfigValidator.java │ │ │ │ ├── adapter │ │ │ │ │ ├── DaggerSASLMechanismAdaptor.java │ │ │ │ │ ├── DaggerSSLKeyStoreFileTypeAdaptor.java │ │ │ │ │ ├── DaggerSSLProtocolAdaptor.java │ │ │ │ │ ├── DaggerSSLTrustStoreFileTypeAdaptor.java │ │ │ │ │ ├── DaggerSecurityProtocolAdaptor.java │ │ │ │ │ ├── FileDateRangeAdaptor.java │ │ │ │ │ └── SourceParquetFilePathsAdapter.java │ │ │ │ └── models │ │ │ │ │ ├── SourceDetails.java │ │ │ │ │ ├── SourceName.java │ │ │ │ │ ├── SourceType.java │ │ │ │ │ ├── TimeRange.java │ │ │ │ │ └── TimeRangePool.java │ │ │ ├── flinkkafkaconsumer │ │ │ │ ├── FlinkKafkaConsumerCustom.java │ │ │ │ └── FlinkKafkaConsumerDaggerSource.java │ │ │ ├── kafka │ │ │ │ └── KafkaDaggerSource.java │ │ │ └── parquet │ │ │ │ ├── ParquetDaggerSource.java │ │ │ │ ├── ParquetFileRecordFormat.java │ │ │ │ ├── ParquetFileSource.java │ │ │ │ ├── SourceParquetReadOrderStrategy.java │ │ │ │ ├── SourceParquetSchemaMatchStrategy.java │ │ │ │ ├── path │ │ │ │ ├── HourDatePathParser.java │ │ │ │ └── PathParser.java │ │ │ │ ├── reader │ │ │ │ ├── ParquetReader.java │ │ │ │ └── ReaderProvider.java │ │ │ │ └── splitassigner │ │ │ │ ├── ChronologyOrderedSplitAssigner.java │ │ │ │ ├── IndexOrderedSplitAssigner.java │ │ │ │ └── InstantEnrichedSplit.java │ │ │ └── utils │ │ │ └── Constants.java │ └── resources │ │ ├── core-site.xml │ │ ├── log4j.properties │ │ └── logback.xml │ └── test │ ├── java │ └── org │ │ └── raystack │ │ └── dagger │ │ └── core │ │ ├── StreamManagerTest.java │ │ ├── config │ │ ├── ConfigurationProviderFactoryTest.java │ │ ├── KafkaEnvironmentVariablesTest.java │ │ ├── commandline │ │ │ └── CommandlineConfigurationProviderTest.java │ │ └── system │ │ │ ├── EnvironmentConfigurationProviderTest.java │ │ │ └── FileConfigurationProvideTest.java │ │ ├── deserializer │ │ ├── DaggerDeserializerFactoryTest.java │ │ ├── JsonDeserializerProviderTest.java │ │ ├── ProtoDeserializerProviderTest.java │ │ └── SimpleGroupDeserializerProviderTest.java │ │ ├── metrics │ │ └── reporters │ │ │ ├── ErrorReporterFactoryTest.java │ │ │ ├── ErrorStatsReporterTest.java │ │ │ └── statsd │ │ │ ├── DaggerMetricsConfigTest.java │ │ │ ├── DaggerStatsDReporterTest.java │ │ │ ├── StatsDErrorReporterTest.java │ │ │ ├── manager │ │ │ ├── DaggerCounterManagerTest.java │ │ │ ├── DaggerGaugeManagerTest.java │ │ │ └── DaggerHistogramManagerTest.java │ │ │ └── tags │ │ │ └── StatsDTagTest.java │ │ ├── processors │ │ ├── ParentPostProcessorTest.java │ │ ├── PostProcessorConfigTest.java │ │ ├── PostProcessorFactoryTest.java │ │ ├── PreProcessorFactoryTest.java │ │ ├── PreProcessorOrchestratorTest.java │ │ ├── common │ │ │ ├── DescriptorManagerTest.java │ │ │ ├── EndpointHandlerTest.java │ │ │ ├── FetchOutputDecoratorTest.java │ │ │ ├── InitializationDecoratorTest.java │ │ │ ├── OutputMappingTest.java │ │ │ ├── RowManagerTest.java │ │ │ ├── SchemaConfigTest.java │ │ │ └── ValidRecordsDecoratorTest.java │ │ ├── external │ │ │ ├── ExternalPostProcessorTest.java │ │ │ ├── ExternalSourceConfigTest.java │ │ │ ├── es │ │ │ │ ├── EsAsyncConnectorTest.java │ │ │ │ ├── EsResponseHandlerTest.java │ │ │ │ ├── EsSourceConfigTest.java │ │ │ │ └── EsStreamDecoratorTest.java │ │ │ ├── grpc │ │ │ │ ├── GrpcAsyncConnectorTest.java │ │ │ │ ├── GrpcResponseHandlerTest.java │ │ │ │ ├── GrpcSourceConfigTest.java │ │ │ │ ├── GrpcStreamDecoratorTest.java │ │ │ │ └── client │ │ │ │ │ ├── DynamicMessageMarshallerTest.java │ │ │ │ │ ├── GrpcClientTest.java │ │ │ │ │ └── GrpcRequestHandlerTest.java │ │ │ ├── http │ │ │ │ ├── HttpAsyncConnectorTest.java │ │ │ │ ├── HttpResponseHandlerTest.java │ │ │ │ ├── HttpSourceConfigTest.java │ │ │ │ ├── HttpStreamDecoratorTest.java │ │ │ │ └── request │ │ │ │ │ ├── HttpGetRequestHandlerTest.java │ │ │ │ │ ├── HttpPostRequestHandlerTest.java │ │ │ │ │ └── HttpRequestFactoryTest.java │ │ │ └── pg │ │ │ │ ├── PgAsyncConnectorTest.java │ │ │ │ ├── PgResponseHandlerTest.java │ │ │ │ ├── PgSourceConfigTest.java │ │ │ │ └── PgStreamDecoratorTest.java │ │ ├── internal │ │ │ ├── InternalDecoratorTest.java │ │ │ ├── InternalPostProcessorTest.java │ │ │ ├── InternalSourceConfigTest.java │ │ │ └── processor │ │ │ │ ├── InternalConfigHandlerFactoryTest.java │ │ │ │ ├── constant │ │ │ │ └── ConstantInternalConfigProcessorTest.java │ │ │ │ ├── function │ │ │ │ ├── FunctionInternalConfigProcessorTest.java │ │ │ │ ├── FunctionProcessorFactoryTest.java │ │ │ │ └── functions │ │ │ │ │ ├── CurrentTimestampFunctionTest.java │ │ │ │ │ ├── InvalidFunctionTest.java │ │ │ │ │ └── JsonPayloadFunctionTest.java │ │ │ │ ├── invalid │ │ │ │ └── InvalidInternalConfigProcessorTest.java │ │ │ │ └── sql │ │ │ │ ├── SqlConfigTypePathParserTest.java │ │ │ │ └── fields │ │ │ │ ├── SqlInternalAutoFieldImportTest.java │ │ │ │ ├── SqlInternalConfigProcessorTest.java │ │ │ │ ├── SqlInternalFieldFactoryTest.java │ │ │ │ └── SqlInternalFieldImportTest.java │ │ ├── longbow │ │ │ ├── LongbowFactoryTest.java │ │ │ ├── LongbowProcessorTest.java │ │ │ ├── LongbowReadColumnModifierTest.java │ │ │ ├── LongbowSchemaTest.java │ │ │ ├── LongbowWriteColumnModifierTest.java │ │ │ ├── data │ │ │ │ ├── LongbowDataFactoryTest.java │ │ │ │ ├── LongbowProtoDataTest.java │ │ │ │ └── LongbowTableDataTest.java │ │ │ ├── outputRow │ │ │ │ ├── OutputLongbowDataTest.java │ │ │ │ ├── OutputProtoDataTest.java │ │ │ │ └── OutputSynchronizerTest.java │ │ │ ├── processor │ │ │ │ ├── LongbowReaderTest.java │ │ │ │ └── LongbowWriterTest.java │ │ │ ├── range │ │ │ │ ├── LongbowAbsoluteRangeTest.java │ │ │ │ └── LongbowDurationRangeTest.java │ │ │ ├── request │ │ │ │ ├── ProtoBytePutRequestTest.java │ │ │ │ ├── ProtoByteScanRequestTest.java │ │ │ │ ├── PutRequestFactoryTest.java │ │ │ │ ├── ScanRequestFactoryTest.java │ │ │ │ ├── TablePutRequestTest.java │ │ │ │ └── TableScanRequestTest.java │ │ │ └── validator │ │ │ │ └── LongbowValidatorTest.java │ │ ├── telemetry │ │ │ ├── TelemetryProcessorTest.java │ │ │ └── processor │ │ │ │ └── MetricsTelemetryExporterTest.java │ │ └── transformers │ │ │ ├── MockTransformer.java │ │ │ ├── TransformConfigTest.java │ │ │ └── TransformProcessorTest.java │ │ ├── sink │ │ ├── SinkOrchestratorTest.java │ │ ├── bigquery │ │ │ ├── BigQuerySinkBuilderTest.java │ │ │ ├── BigQuerySinkTest.java │ │ │ └── BigQuerySinkWriterTest.java │ │ ├── influx │ │ │ ├── InfluxDBSinkTest.java │ │ │ ├── InfluxDBWriterTest.java │ │ │ └── errors │ │ │ │ ├── LateRecordDropErrorTest.java │ │ │ │ ├── NoErrorTest.java │ │ │ │ ├── ValidErrorTest.java │ │ │ │ └── ValidExceptionTest.java │ │ └── kafka │ │ │ ├── KafkaSerializationSchemaFactoryTest.java │ │ │ └── builder │ │ │ ├── KafkaJsonSerializerBuilderTest.java │ │ │ └── KafkaProtoSerializerBuilderTest.java │ │ └── source │ │ ├── DaggerSourceFactoryTest.java │ │ ├── StreamTest.java │ │ ├── StreamsFactoryTest.java │ │ ├── config │ │ ├── StreamConfigTest.java │ │ ├── adapter │ │ │ ├── DaggerSASLMechanismAdaptorTest.java │ │ │ ├── DaggerSSLKeyStoreFileTypeAdaptorTest.java │ │ │ ├── DaggerSSLProtocolAdaptorTest.java │ │ │ ├── DaggerSSLTrustStoreFileTypeAdaptorTest.java │ │ │ ├── DaggerSecurityProtocolAdaptorTest.java │ │ │ ├── FileDateRangeAdaptorTest.java │ │ │ └── SourceParquetFilePathsAdapterTest.java │ │ └── models │ │ │ ├── TimeRangePoolTest.java │ │ │ └── TimeRangeTest.java │ │ ├── flinkkafkaconsumer │ │ ├── FlinkKafkaConsumerCustomTest.java │ │ └── FlinkKafkaConsumerDaggerSourceTest.java │ │ ├── kafka │ │ └── KafkaDaggerSourceTest.java │ │ └── parquet │ │ ├── ParquetDaggerSourceTest.java │ │ ├── ParquetFileRecordFormatTest.java │ │ ├── ParquetFileSourceTest.java │ │ ├── path │ │ └── HourDatePathParserTest.java │ │ ├── reader │ │ └── ParquetReaderTest.java │ │ └── splitassigner │ │ └── ChronologyOrderedSplitAssignerTest.java │ └── resources │ ├── binary │ └── broken-message │ ├── multiple_row_groups_test_file.parquet │ └── test_file.parquet ├── dagger-functions ├── build.gradle └── src │ ├── main │ └── java │ │ └── org │ │ └── raystack │ │ └── dagger │ │ └── functions │ │ ├── common │ │ └── Constants.java │ │ ├── exceptions │ │ ├── ArrayAggregationException.java │ │ ├── ArrayOperateException.java │ │ ├── BucketDoesNotExistException.java │ │ ├── InvalidHashFieldException.java │ │ ├── InvalidNumberOfArgumentsException.java │ │ ├── KeyDoesNotExistException.java │ │ ├── LongbowException.java │ │ ├── MadZeroException.java │ │ ├── MedianNotFound.java │ │ ├── OddNumberOfArgumentsException.java │ │ ├── PythonFilesEmptyException.java │ │ ├── PythonFilesFormatException.java │ │ ├── RowHashException.java │ │ └── TagDoesNotExistException.java │ │ ├── transformers │ │ ├── ClearColumnTransformer.java │ │ ├── DeDuplicationTransformer.java │ │ ├── FeatureTransformer.java │ │ ├── FeatureWithTypeTransformer.java │ │ ├── HashTransformer.java │ │ ├── InvalidRecordFilterTransformer.java │ │ ├── SQLTransformer.java │ │ ├── feature │ │ │ └── FeatureWithTypeHandler.java │ │ ├── filter │ │ │ └── FilterAspects.java │ │ └── hash │ │ │ ├── FieldHasherFactory.java │ │ │ ├── PathReader.java │ │ │ └── field │ │ │ ├── FieldHasher.java │ │ │ ├── IntegerFieldHasher.java │ │ │ ├── LongFieldHasher.java │ │ │ ├── RowHasher.java │ │ │ ├── StringFieldHasher.java │ │ │ └── UnsupportedDataTypeHasher.java │ │ └── udfs │ │ ├── aggregate │ │ ├── CollectArray.java │ │ ├── DistinctCount.java │ │ ├── Features.java │ │ ├── FeaturesWithType.java │ │ ├── PercentileAggregator.java │ │ ├── accumulator │ │ │ ├── ArrayAccumulator.java │ │ │ ├── DistinctCountAccumulator.java │ │ │ ├── FeatureAccumulator.java │ │ │ ├── FeatureWithTypeAccumulator.java │ │ │ └── PercentileAccumulator.java │ │ └── feast │ │ │ ├── FeatureUtils.java │ │ │ └── handler │ │ │ ├── BigDecimalValueTransformer.java │ │ │ ├── BooleanValueTransformer.java │ │ │ ├── ByteValueTransformer.java │ │ │ ├── DoubleValueTransformer.java │ │ │ ├── FloatValueTransformer.java │ │ │ ├── IntegerValueTransformer.java │ │ │ ├── LongValueTransformer.java │ │ │ ├── NullValueTransformer.java │ │ │ ├── StringValueTransformer.java │ │ │ ├── TimestampValueTransformer.java │ │ │ ├── ValueEnum.java │ │ │ ├── ValueTransformer.java │ │ │ └── ValueTransformerFactory.java │ │ ├── factories │ │ └── FunctionFactory.java │ │ ├── python │ │ ├── PythonUdfConfig.java │ │ ├── PythonUdfManager.java │ │ └── file │ │ │ ├── source │ │ │ ├── FileSource.java │ │ │ ├── FileSourceFactory.java │ │ │ ├── gcs │ │ │ │ ├── GcsClient.java │ │ │ │ └── GcsFileSource.java │ │ │ └── local │ │ │ │ └── LocalFileSource.java │ │ │ └── type │ │ │ ├── FileType.java │ │ │ ├── FileTypeFactory.java │ │ │ ├── PythonFileType.java │ │ │ └── ZipFileType.java │ │ ├── scalar │ │ ├── ArrayAggregate.java │ │ ├── ArrayOperate.java │ │ ├── ByteToString.java │ │ ├── CondEq.java │ │ ├── DartContains.java │ │ ├── DartGet.java │ │ ├── Distance.java │ │ ├── ElementAt.java │ │ ├── EndOfMonth.java │ │ ├── EndOfWeek.java │ │ ├── ExponentialMovingAverage.java │ │ ├── Filters.java │ │ ├── FormatTimeInZone.java │ │ ├── GeoHash.java │ │ ├── JsonDelete.java │ │ ├── JsonQuery.java │ │ ├── JsonUpdate.java │ │ ├── LinearTrend.java │ │ ├── ListContains.java │ │ ├── MapGet.java │ │ ├── S2AreaInKm2.java │ │ ├── S2Id.java │ │ ├── SelectFields.java │ │ ├── SingleFeatureWithType.java │ │ ├── Split.java │ │ ├── StartOfMonth.java │ │ ├── StartOfWeek.java │ │ ├── TimeInDate.java │ │ ├── TimestampFromUnix.java │ │ ├── dart │ │ │ ├── DartAspects.java │ │ │ ├── store │ │ │ │ ├── DataStore.java │ │ │ │ └── gcs │ │ │ │ │ ├── GcsClient.java │ │ │ │ │ └── GcsDataStore.java │ │ │ └── types │ │ │ │ ├── Cache.java │ │ │ │ ├── MapCache.java │ │ │ │ └── SetCache.java │ │ ├── elementAt │ │ │ ├── MessageReader.java │ │ │ ├── descriptor │ │ │ │ └── CustomDescriptor.java │ │ │ └── row │ │ │ │ ├── Element.java │ │ │ │ ├── RowElement.java │ │ │ │ └── ValueElement.java │ │ └── longbow │ │ │ ├── MessageParser.java │ │ │ ├── ProtoToRow.java │ │ │ └── array │ │ │ ├── LongbowArrayType.java │ │ │ ├── expression │ │ │ ├── AggregationExpression.java │ │ │ ├── Expression.java │ │ │ └── OperationExpression.java │ │ │ └── processors │ │ │ ├── ArrayAggregateProcessor.java │ │ │ ├── ArrayOperateProcessor.java │ │ │ └── ArrayProcessor.java │ │ └── table │ │ ├── HistogramBucket.java │ │ ├── OutlierMad.java │ │ └── outlier │ │ └── mad │ │ ├── Mad.java │ │ └── Point.java │ └── test │ ├── java │ └── org │ │ └── raystack │ │ └── dagger │ │ └── functions │ │ ├── transformers │ │ ├── ClearColumnTransformerTest.java │ │ ├── DeDuplicationTransformerTest.java │ │ ├── FeatureTransformerTest.java │ │ ├── FeatureWithTypeTransformerTest.java │ │ ├── HashTransformerTest.java │ │ ├── InvalidRecordFilterTransformerTest.java │ │ ├── SQLTransformerTest.java │ │ ├── feature │ │ │ └── FeatureWithTypeHandlerTest.java │ │ └── hash │ │ │ ├── FieldHasherFactoryTest.java │ │ │ ├── PathReaderTest.java │ │ │ └── field │ │ │ ├── IntegerFieldHasherTest.java │ │ │ ├── LongFieldHasherTest.java │ │ │ ├── RowHasherTest.java │ │ │ ├── StringFieldHasherTest.java │ │ │ └── UnsupportedDataTypeHasherTest.java │ │ └── udfs │ │ ├── aggregate │ │ ├── CollectArrayTest.java │ │ ├── DistinctCountTest.java │ │ ├── FeaturesTest.java │ │ ├── FeaturesWithTypeTest.java │ │ ├── PercentileAggregatorTest.java │ │ ├── accumulator │ │ │ ├── ArrayAccumulatorTest.java │ │ │ ├── DistinctCountAccumulatorTest.java │ │ │ ├── FeatureAccumulatorTest.java │ │ │ ├── FeatureWithTypeAccumulatorTest.java │ │ │ └── PercentileAccumulatorTest.java │ │ └── feast │ │ │ ├── FeatureUtilsTest.java │ │ │ └── handler │ │ │ ├── BigDecimalValueTransformerTest.java │ │ │ ├── BooleanValueTransformerTest.java │ │ │ ├── ByteValueTransformerTest.java │ │ │ ├── DoubleValueTransformerTest.java │ │ │ ├── FloatValueTransformerTest.java │ │ │ ├── IntegerValueTransformerTest.java │ │ │ ├── LongValueTransformerTest.java │ │ │ ├── NullValueTransformerTest.java │ │ │ ├── StringValueTransformerTest.java │ │ │ └── TimestampValueTransformerTest.java │ │ ├── factories │ │ └── FunctionFactoryTest.java │ │ ├── python │ │ ├── PythonUdfConfigTest.java │ │ ├── PythonUdfManagerTest.java │ │ └── file │ │ │ ├── source │ │ │ ├── FileSourceFactoryTest.java │ │ │ ├── gcs │ │ │ │ ├── GcsClientTest.java │ │ │ │ └── GcsFileSourceTest.java │ │ │ └── local │ │ │ │ └── LocalFileSourceTest.java │ │ │ └── type │ │ │ ├── FileTypeFactoryTest.java │ │ │ ├── PythonFileTypeTest.java │ │ │ └── ZipFileTypeTest.java │ │ ├── scalar │ │ ├── ArrayAggregateTest.java │ │ ├── ArrayOperateTest.java │ │ ├── ByteToStringTest.java │ │ ├── CondEqTest.java │ │ ├── DartContainsTest.java │ │ ├── DartGetTest.java │ │ ├── DistanceTest.java │ │ ├── ElementAtTest.java │ │ ├── EndOfMonthTest.java │ │ ├── EndOfWeekTest.java │ │ ├── ExponentialMovingAverageTest.java │ │ ├── FiltersTest.java │ │ ├── FormatTimeInZoneTest.java │ │ ├── GeoHashTest.java │ │ ├── JsonDeleteTest.java │ │ ├── JsonQueryTest.java │ │ ├── JsonUpdateTest.java │ │ ├── LinearTrendTest.java │ │ ├── ListContainsTest.java │ │ ├── MapGetTest.java │ │ ├── S2AreaInKm2Test.java │ │ ├── S2IdTest.java │ │ ├── SelectFieldsTest.java │ │ ├── SingleFeatureWithTypeTest.java │ │ ├── SplitTest.java │ │ ├── StartOfMonthTest.java │ │ ├── StartOfWeekTest.java │ │ ├── TimeInDateTest.java │ │ ├── TimestampFromUnixTest.java │ │ ├── dart │ │ │ ├── store │ │ │ │ └── gcs │ │ │ │ │ └── GcsDataStoreTest.java │ │ │ └── types │ │ │ │ ├── CacheTest.java │ │ │ │ ├── MapCacheTest.java │ │ │ │ └── SetCacheTest.java │ │ ├── elementAt │ │ │ ├── MessageReaderTest.java │ │ │ ├── descriptor │ │ │ │ └── CustomDescriptorTest.java │ │ │ └── row │ │ │ │ ├── ElementTest.java │ │ │ │ ├── RowElementTest.java │ │ │ │ └── ValueElementTest.java │ │ └── longbow │ │ │ ├── MessageParserTest.java │ │ │ └── array │ │ │ ├── expression │ │ │ ├── AggregationExpressionTest.java │ │ │ └── OperationExpressionTest.java │ │ │ └── processors │ │ │ ├── ArrayAggregateProcessorTest.java │ │ │ └── ArrayOperateProcessorTest.java │ │ └── table │ │ ├── HistogramBucketTest.java │ │ ├── OutlierMadTest.java │ │ └── outlier │ │ └── mad │ │ └── MadTest.java │ └── resources │ ├── python_udf.zip │ ├── test_no_py.zip │ └── test_udf.py ├── dagger-py-functions ├── data │ └── sample_data.txt ├── requirements.txt ├── tests │ ├── __init__.py │ └── udfs │ │ ├── __init__.py │ │ └── scalar │ │ ├── __init__.py │ │ ├── multiply_test.py │ │ └── sample_test.py └── udfs │ └── scalar │ ├── multiply.py │ └── sample.py ├── dagger-tests ├── build.gradle └── src │ └── integrationtest │ └── java │ └── org │ └── raystack │ └── dagger │ └── integrationtest │ ├── EsExternalPostProcessorIntegrationTest.java │ ├── GrpcExternalPostProcessorIntegrationTest.java │ ├── HttpExternalPostProcessorIntegrationTest.java │ └── PostGresExternalPostProcessorIntegrationTest.java ├── docs ├── .gitignore ├── README.md ├── babel.config.js ├── blog │ ├── 2023-05-23-intro-dagger-docs.md │ └── authors.yaml ├── docs │ ├── advance │ │ ├── DARTS.md │ │ ├── longbow.md │ │ ├── longbow_plus.md │ │ ├── overview.md │ │ ├── post_processor.md │ │ ├── pre_processor.md │ │ └── security.md │ ├── concepts │ │ ├── architecture.md │ │ ├── basics.md │ │ ├── lifecycle.md │ │ └── overview.md │ ├── contribute │ │ ├── add_transformer.md │ │ ├── add_udf.md │ │ ├── contribution.md │ │ └── development.md │ ├── examples │ │ ├── aggregation_tumble_window.md │ │ ├── deduplication_transformer.md │ │ ├── distance_java_udf.md │ │ ├── elasticsearch_enrichment.md │ │ ├── kafka_inner_join.md │ │ └── overview.md │ ├── guides │ │ ├── choose_source.md │ │ ├── create_dagger.md │ │ ├── deployment.md │ │ ├── monitoring.md │ │ ├── overview.md │ │ ├── query_examples.md │ │ ├── quickstart.md │ │ ├── troubleshooting.md │ │ ├── use_transformer.md │ │ └── use_udf.md │ ├── intro.md │ ├── reference │ │ ├── configuration.md │ │ ├── metrics.md │ │ ├── overview.md │ │ ├── transformers.md │ │ └── udfs.md │ ├── rfcs │ │ └── 20220504_python_udf.md │ ├── roadmap.md │ └── usecase │ │ ├── api_monitoring.md │ │ ├── feature_ingestion.md │ │ ├── overview.md │ │ └── stream_enrichment.md ├── docusaurus.config.js ├── package-lock.json ├── package.json ├── sidebars.js ├── src │ ├── core │ │ ├── Container.js │ │ └── GridBlock.js │ ├── css │ │ ├── custom.css │ │ └── theme.css │ └── pages │ │ ├── help.js │ │ └── index.js └── static │ ├── .nojekyll │ ├── assets │ └── dagger-grafana-dashboard.json │ ├── img │ ├── api-monitoring.png │ ├── api-status.png │ ├── dart-contains.png │ ├── dart-get.png │ ├── enrichment.png │ ├── external-http-post-processor.png │ ├── external-internal-post-processor.png │ ├── external-internal-transformer-post-processor.png │ ├── favicon.ico │ ├── homebanner.svg │ ├── lifecycle │ │ ├── dagger_lifecycle.drawio │ │ └── dagger_lifecycle.png │ ├── logo-n.svg │ ├── longbow.png │ ├── longbowplus-reader.png │ ├── longbowplus-writer.png │ ├── overview │ │ ├── dagger_overview.drawio │ │ └── dagger_overview.png │ ├── pattern.svg │ ├── pre-processor.png │ ├── query.svg │ ├── sliding.png │ ├── system_design │ │ ├── dagger_system_design.drawio │ │ └── dagger_system_design.png │ └── tumble.png │ └── users │ ├── gojek.png │ ├── goto.png │ ├── jago.png │ ├── mapan.png │ ├── midtrans.png │ ├── moka.png │ └── paylater.png ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── quickstart ├── docker-compose │ ├── compose.yaml │ └── resources │ │ ├── TestLogMessage.proto │ │ ├── kafkafeeder.sh │ │ ├── local.properties │ │ └── sample_message.txt └── examples │ ├── aggregation │ └── tumble_window │ │ ├── README.md │ │ ├── compose.yaml │ │ └── resources │ │ ├── TestLogMessage.proto │ │ ├── kafkafeeder.sh │ │ ├── local.properties │ │ └── sample_message.txt │ ├── enrichment │ └── elasticsearch_enrichment │ │ ├── README.md │ │ ├── compose.yaml │ │ └── resources │ │ ├── com │ │ └── gotocompany │ │ │ └── dagger │ │ │ └── consumer │ │ │ ├── EnrichedBookingLogMessage.proto │ │ │ └── TestLogMessage.proto │ │ ├── kafkafeeder.sh │ │ ├── local.properties │ │ ├── sample_message.txt │ │ └── seed1.json │ ├── joins │ └── inner_join │ │ ├── README.md │ │ ├── compose.yaml │ │ └── resources │ │ ├── TestLogMessage.proto │ │ ├── kafkafeeder.sh │ │ ├── local.properties │ │ └── sample_message.txt │ ├── transformer │ └── deduplication_transformer │ │ ├── README.md │ │ ├── compose.yaml │ │ └── resources │ │ ├── TestLogMessage.proto │ │ ├── kafkafeeder.sh │ │ ├── local.properties │ │ └── sample_message.txt │ └── udfs │ └── distance_udf │ ├── README.md │ ├── compose.yaml │ └── resources │ ├── TestLogMessage.proto │ ├── kafkafeeder.sh │ ├── local.properties │ └── sample_message.txt ├── settings.gradle └── version.txt /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: push 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - name: Set up JDK 1.8 11 | uses: actions/setup-java@v1 12 | with: 13 | java-version: 1.8 14 | - name: Cache Gradle packages 15 | uses: actions/cache@v2 16 | with: 17 | path: | 18 | ~/.gradle/caches 19 | ~/.gradle/wrapper 20 | key: ${{ runner.os }}-dagger-cache-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} 21 | restore-keys: | 22 | ${{ runner.os }}-dagger-cache- 23 | - name: Building dagger core. All dependent sub project would be built as part of this 24 | run: ./gradlew build --no-daemon 25 | -------------------------------------------------------------------------------- /.github/workflows/dependencies-publish.yml: -------------------------------------------------------------------------------- 1 | name: dependencies-publish 2 | 3 | on: 4 | workflow_dispatch 5 | 6 | jobs: 7 | 8 | publishDependenciesJar: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v2 12 | - uses: actions/setup-java@v1 13 | with: 14 | java-version: 1.8 15 | - name: Cache Gradle packages 16 | uses: actions/cache@v2 17 | with: 18 | path: | 19 | ~/.gradle/caches 20 | ~/.gradle/wrapper 21 | key: ${{ runner.os }}-dagger-cache-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} 22 | restore-keys: | 23 | ${{ runner.os }}-dagger-cache- 24 | - name: Publish dependencies package of core 25 | run: ./gradlew :dagger-core:dependenciesPublish 26 | env: 27 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 28 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: docs 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | workflow_dispatch: 8 | 9 | jobs: 10 | documentation: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - uses: actions/setup-node@v2 15 | - name: Installation 16 | uses: bahmutov/npm-install@v1 17 | with: 18 | install-command: yarn 19 | working-directory: docs 20 | - name: Build docs 21 | working-directory: docs 22 | run: cd docs && yarn build 23 | - name: Deploy docs 24 | env: 25 | GIT_USER: ravisuhag 26 | GIT_PASS: ${{ secrets.DOCU_RS_TOKEN }} 27 | DEPLOYMENT_BRANCH: gh-pages 28 | CURRENT_BRANCH: main 29 | working-directory: docs 30 | run: | 31 | git config --global user.email "suhag.ravi@gmail.com" 32 | git config --global user.name "ravisuhag" 33 | yarn deploy 34 | -------------------------------------------------------------------------------- /.github/workflows/package.yml: -------------------------------------------------------------------------------- 1 | name: Package 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | publishJar: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v2 12 | - uses: actions/setup-java@v1 13 | with: 14 | java-version: 1.8 15 | - name: Cache Gradle packages 16 | uses: actions/cache@v2 17 | with: 18 | path: | 19 | ~/.gradle/caches 20 | ~/.gradle/wrapper 21 | key: ${{ runner.os }}-dagger-cache-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} 22 | restore-keys: | 23 | ${{ runner.os }}-dagger-cache- 24 | - name: Publish packages of common subprojects 25 | run: ./gradlew :dagger-common:publish 26 | env: 27 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 28 | - name: Publish minimal and shadow packages of core 29 | run: ./gradlew :dagger-core:minimalAndShadowPublish 30 | env: 31 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 32 | -------------------------------------------------------------------------------- /.github/workflows/python_release.yml: -------------------------------------------------------------------------------- 1 | name: Python Package 2 | on: 3 | release: 4 | types: [created] 5 | 6 | jobs: 7 | publishPythonZip: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - name: Zip Python Udf 12 | run: | 13 | cd dagger-py-functions 14 | zip -r python_udfs.zip udfs -x "*/__init__.py" 15 | zip -jr data.zip data 16 | zip -r dagger-py-functions.zip requirements.txt data.zip python_udfs.zip 17 | - name: Upload Release 18 | uses: ncipollo/release-action@v1 19 | with: 20 | artifacts: dagger-py-functions/dagger-py-functions.zip 21 | allowUpdates: true 22 | omitNameDuringUpdate: true 23 | omitBodyDuringUpdate: true 24 | omitPrereleaseDuringUpdate: true 25 | token: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/python_validation.yml: -------------------------------------------------------------------------------- 1 | name: Python Validation 2 | 3 | on: push 4 | 5 | jobs: 6 | pythonValidation: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v3 10 | - name: Set up Python 3.8 11 | uses: actions/setup-python@v3 12 | with: 13 | python-version: '3.8' 14 | - name: Install dependencies 15 | run: | 16 | python -m pip install --upgrade pip 17 | pip install apache-flink==1.14.3 18 | cd dagger-py-functions 19 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 20 | - name: Lint with flake8 21 | run: | 22 | # stop the build if there are Python syntax errors or undefined names 23 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 24 | - name: Test with pytest 25 | run: | 26 | cd dagger-py-functions 27 | pytest --disable-warnings -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .gradle 3 | *.iml 4 | build 5 | out 6 | classes 7 | generated 8 | .pairs 9 | target/ 10 | .classpath 11 | .project 12 | bin 13 | .settings 14 | .gradletasknamecache 15 | .DS_Store 16 | dagger-common/src/generated-sources/ -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | ext { 2 | flinkVersion = System.getenv('flinkVersion') ?: '1.14.3' 3 | } 4 | 5 | subprojects { 6 | repositories { 7 | mavenLocal() 8 | mavenCentral() 9 | jcenter() 10 | maven { url "https://repository.apache.org/content/repositories/snapshots/" } 11 | } 12 | 13 | apply plugin: 'java' 14 | apply plugin: 'jacoco' 15 | apply plugin: 'idea' 16 | apply plugin: 'checkstyle' 17 | 18 | group 'org.raystack' 19 | 20 | checkstyle { 21 | toolVersion '7.6.1' 22 | configFile rootProject.file("config/checkstyle/checkstyle.xml") 23 | } 24 | } -------------------------------------------------------------------------------- /config/checkstyle/suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 10 | 12 | 13 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/configuration/Configuration.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.configuration; 2 | 3 | import org.apache.flink.api.java.utils.ParameterTool; 4 | 5 | import java.io.Serializable; 6 | 7 | public class Configuration implements Serializable { 8 | private final ParameterTool param; 9 | 10 | public Configuration(ParameterTool param) { 11 | this.param = param; 12 | } 13 | 14 | public ParameterTool getParam() { 15 | return param; 16 | } 17 | 18 | public String getString(String configKey, String defaultValue) { 19 | return param.get(configKey, defaultValue); 20 | } 21 | 22 | public Integer getInteger(String configKey, Integer defaultValue) { 23 | return param.getInt(configKey, defaultValue); 24 | } 25 | 26 | public Boolean getBoolean(String configKey, Boolean defaultValue) { 27 | return param.getBoolean(configKey, defaultValue); 28 | } 29 | 30 | public Long getLong(String configKey, Long defaultValue) { 31 | return param.getLong(configKey, defaultValue); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/core/StreamInfo.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.core; 2 | 3 | import org.apache.flink.streaming.api.datastream.DataStream; 4 | import org.apache.flink.types.Row; 5 | 6 | /** 7 | * The class to hold the data stream and column names. 8 | */ 9 | public class StreamInfo { 10 | private DataStream dataStream; 11 | private String[] columnNames; 12 | 13 | /** 14 | * Instantiates a new Stream info. 15 | * 16 | * @param dataStream the data stream 17 | * @param columnNames the column names 18 | */ 19 | public StreamInfo(DataStream dataStream, String[] columnNames) { 20 | this.dataStream = dataStream; 21 | this.columnNames = columnNames; 22 | } 23 | 24 | /** 25 | * Gets data stream. 26 | * 27 | * @return the data stream 28 | */ 29 | public DataStream getDataStream() { 30 | return dataStream; 31 | } 32 | 33 | /** 34 | * Get column names. 35 | * 36 | * @return list of column names 37 | */ 38 | public String[] getColumnNames() { 39 | return columnNames; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/core/Transformer.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.core; 2 | 3 | /** 4 | * The interface for all the transformer. 5 | */ 6 | public interface Transformer { 7 | StreamInfo transform(StreamInfo streamInfo); 8 | } 9 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/exceptions/DaggerContextException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.exceptions; 2 | 3 | /** 4 | * The class Exception if there is something wrong with Dagger context object. 5 | */ 6 | public class DaggerContextException extends RuntimeException { 7 | 8 | /** 9 | * Instantiates a new Dagger context exception with specified error message. 10 | * 11 | * @param message the message 12 | */ 13 | public DaggerContextException(String message) { 14 | super(message); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/exceptions/DescriptorNotFoundException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.exceptions; 2 | 3 | /** 4 | * The class Exception if Descriptor not found. 5 | */ 6 | public class DescriptorNotFoundException extends RuntimeException { 7 | public static final String DESCRIPTOR_NOT_FOUND = "descriptor not found"; 8 | 9 | /** 10 | * Instantiates a new Descriptor not found exception. 11 | */ 12 | public DescriptorNotFoundException() { 13 | this(DESCRIPTOR_NOT_FOUND); 14 | } 15 | 16 | /** 17 | * Instantiates a new Descriptor not found exception with the specified detail message. 18 | * 19 | * @param protoClassMisconfiguredError the proto class misconfigured error 20 | */ 21 | public DescriptorNotFoundException(String protoClassMisconfiguredError) { 22 | super(protoClassMisconfiguredError); 23 | } 24 | 25 | /** 26 | * Instantiates a new Descriptor not found exception with an inner exception as detail message. 27 | * 28 | * @param innerException the inner exception 29 | */ 30 | public DescriptorNotFoundException(Exception innerException) { 31 | super(innerException); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/exceptions/serde/DaggerDeserializationException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.exceptions.serde; 2 | 3 | /** 4 | * The class Exception if failed on Deserialize the protobuf message. 5 | */ 6 | public class DaggerDeserializationException extends RuntimeException { 7 | /** 8 | * Instantiates a new Dagger deserialization exception. 9 | * 10 | * @param innerException the inner exception 11 | */ 12 | public DaggerDeserializationException(Exception innerException) { 13 | super(innerException); 14 | } 15 | 16 | public DaggerDeserializationException(String message) { 17 | super(message); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/exceptions/serde/DaggerSerializationException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.exceptions.serde; 2 | 3 | /** 4 | * The class Exception if failed on Serializing the protobuf message. 5 | */ 6 | public class DaggerSerializationException extends RuntimeException { 7 | /** 8 | * Instantiates a new Dagger serialization exception. 9 | * 10 | * @param protoClassMisconfiguredError the proto class misconfigured error 11 | */ 12 | public DaggerSerializationException(String protoClassMisconfiguredError) { 13 | super(protoClassMisconfiguredError); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/exceptions/serde/DataTypeNotSupportedException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.exceptions.serde; 2 | 3 | /** 4 | * The class Exception for unsupported protobuf data type. 5 | */ 6 | public class DataTypeNotSupportedException extends RuntimeException { 7 | /** 8 | * Instantiates a new Data type not supported exception. 9 | * 10 | * @param message the message 11 | */ 12 | public DataTypeNotSupportedException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/exceptions/serde/EnumFieldNotFoundException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.exceptions.serde; 2 | 3 | /** 4 | * The class Exception if Enum field not found in proto descriptor. 5 | */ 6 | public class EnumFieldNotFoundException extends RuntimeException { 7 | /** 8 | * Instantiates a new Enum field not found exception. 9 | * 10 | * @param message the message 11 | */ 12 | public EnumFieldNotFoundException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/exceptions/serde/InvalidColumnMappingException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.exceptions.serde; 2 | 3 | /** 4 | * The class Exception if there is Invalid Column Mapping. 5 | */ 6 | public class InvalidColumnMappingException extends RuntimeException { 7 | /** 8 | * Instantiates a new Invalid column mapping exception with specified message. 9 | * 10 | * @param message the message 11 | */ 12 | public InvalidColumnMappingException(String message) { 13 | super(message); 14 | } 15 | 16 | /** 17 | * Instantiates a new Invalid column mapping exception with specified message and error message. 18 | * 19 | * @param message the message 20 | * @param err the err 21 | */ 22 | public InvalidColumnMappingException(String message, Throwable err) { 23 | super(message, err); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/exceptions/serde/InvalidDataTypeException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.exceptions.serde; 2 | 3 | /** 4 | * The class Exception if there is an Invalid Data type. 5 | */ 6 | public class InvalidDataTypeException extends RuntimeException { 7 | /** 8 | * Instantiates a new Invalid data type exception. 9 | * 10 | * @param message the message 11 | */ 12 | public InvalidDataTypeException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/exceptions/serde/InvalidJSONSchemaException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.exceptions.serde; 2 | 3 | public class InvalidJSONSchemaException extends RuntimeException { 4 | public InvalidJSONSchemaException(Exception innerException) { 5 | super(innerException); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/exceptions/serde/SimpleGroupParsingException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.exceptions.serde; 2 | 3 | /** 4 | * This runtime exception is thrown when a field cannot be parsed from a Parquet SimpleGroup. 5 | **/ 6 | public class SimpleGroupParsingException extends RuntimeException { 7 | 8 | public SimpleGroupParsingException(String message) { 9 | super(message); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/metrics/aspects/AspectType.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.metrics.aspects; 2 | 3 | /** 4 | * The enum Aspect type. 5 | */ 6 | public enum AspectType { 7 | Gauge, 8 | Histogram, 9 | Metric, 10 | Counter 11 | } 12 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/metrics/aspects/Aspects.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.metrics.aspects; 2 | 3 | /** 4 | * The interface for aspects. 5 | */ 6 | public interface Aspects { 7 | String getValue(); 8 | 9 | AspectType getAspectType(); 10 | } 11 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/serde/DaggerDeserializer.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.serde; 2 | 3 | import org.apache.flink.api.java.typeutils.ResultTypeQueryable; 4 | 5 | import java.io.Serializable; 6 | 7 | public interface DaggerDeserializer extends Serializable, ResultTypeQueryable { 8 | 9 | } 10 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/serde/DaggerInternalTypeInformation.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.serde; 2 | 3 | import org.apache.flink.api.common.typeinfo.TypeInformation; 4 | import org.apache.flink.api.common.typeinfo.Types; 5 | import org.apache.flink.api.java.typeutils.RowTypeInfo; 6 | import org.apache.flink.types.Row; 7 | 8 | import org.raystack.dagger.common.core.Constants; 9 | 10 | import java.util.ArrayList; 11 | import java.util.Arrays; 12 | 13 | public interface DaggerInternalTypeInformation { 14 | TypeInformation getRowType(); 15 | 16 | default TypeInformation addInternalFields(TypeInformation initialTypeInfo, String rowtimeAttributeName) { 17 | RowTypeInfo rowTypeInfo = (RowTypeInfo) initialTypeInfo; 18 | ArrayList fieldNames = new ArrayList<>(Arrays.asList(rowTypeInfo.getFieldNames())); 19 | ArrayList fieldTypes = new ArrayList<>(Arrays.asList(rowTypeInfo.getFieldTypes())); 20 | fieldNames.add(Constants.INTERNAL_VALIDATION_FIELD_KEY); 21 | fieldTypes.add(Types.BOOLEAN); 22 | fieldNames.add(rowtimeAttributeName); 23 | fieldTypes.add(Types.SQL_TIMESTAMP); 24 | return Types.ROW_NAMED(fieldNames.toArray(new String[0]), fieldTypes.toArray(new TypeInformation[0])); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/serde/DataTypes.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.serde; 2 | 3 | public enum DataTypes { 4 | JSON, 5 | PROTO 6 | } 7 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/serde/json/deserialization/JsonType.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.serde.json.deserialization; 2 | 3 | import org.apache.flink.api.common.typeinfo.TypeInformation; 4 | import org.apache.flink.formats.json.JsonRowSchemaConverter; 5 | import org.apache.flink.types.Row; 6 | 7 | import org.raystack.dagger.common.serde.DaggerInternalTypeInformation; 8 | 9 | import java.io.Serializable; 10 | 11 | public class JsonType implements Serializable, DaggerInternalTypeInformation { 12 | private String jsonSchema; 13 | private String rowtimeAttributeName; 14 | 15 | public JsonType(String jsonSchema, String rowtimeAttributeName) { 16 | this.jsonSchema = jsonSchema; 17 | this.rowtimeAttributeName = rowtimeAttributeName; 18 | } 19 | 20 | public TypeInformation getRowType() { 21 | TypeInformation rowNamed = JsonRowSchemaConverter.convert(jsonSchema); 22 | 23 | return addInternalFields(rowNamed, rowtimeAttributeName); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/watermark/LastColumnWatermark.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.watermark; 2 | 3 | import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner; 4 | import org.apache.flink.api.common.eventtime.WatermarkStrategy; 5 | import org.apache.flink.types.Row; 6 | 7 | import java.sql.Timestamp; 8 | import java.time.Duration; 9 | 10 | public class LastColumnWatermark implements WatermarkStrategyDefinition { 11 | 12 | @Override 13 | public WatermarkStrategy getWatermarkStrategy(long waterMarkDelayInMs) { 14 | return WatermarkStrategy. 15 | forBoundedOutOfOrderness(Duration.ofMillis(waterMarkDelayInMs)) 16 | .withTimestampAssigner((SerializableTimestampAssigner) (element, recordTimestamp) -> { 17 | int index = element.getArity() - 1; 18 | return ((Timestamp) element.getField(index)).getTime(); 19 | }); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/watermark/NoWatermark.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.watermark; 2 | 3 | import org.apache.flink.api.common.eventtime.WatermarkStrategy; 4 | import org.apache.flink.types.Row; 5 | 6 | public class NoWatermark implements WatermarkStrategyDefinition { 7 | @Override 8 | public WatermarkStrategy getWatermarkStrategy(long waterMarkDelayInMs) { 9 | return WatermarkStrategy.noWatermarks(); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/watermark/RowtimeFieldWatermark.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.watermark; 2 | 3 | import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner; 4 | import org.apache.flink.api.common.eventtime.WatermarkStrategy; 5 | import org.apache.flink.types.Row; 6 | 7 | import java.sql.Timestamp; 8 | import java.time.Duration; 9 | import java.util.Arrays; 10 | 11 | public class RowtimeFieldWatermark implements WatermarkStrategyDefinition { 12 | private static final String ROWTIME = "rowtime"; 13 | private final String[] columnNames; 14 | 15 | public RowtimeFieldWatermark(String[] columnNames) { 16 | this.columnNames = columnNames; 17 | } 18 | 19 | @Override 20 | public WatermarkStrategy getWatermarkStrategy(long waterMarkDelayInMs) { 21 | return WatermarkStrategy. 22 | forBoundedOutOfOrderness(Duration.ofMillis(waterMarkDelayInMs)) 23 | .withTimestampAssigner((SerializableTimestampAssigner) 24 | (element, recordTimestamp) -> ((Timestamp) element.getField(Arrays.asList(columnNames).indexOf(ROWTIME))).getTime()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/watermark/StreamWatermarkAssigner.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.watermark; 2 | 3 | import org.apache.flink.streaming.api.datastream.DataStream; 4 | import org.apache.flink.types.Row; 5 | 6 | import java.io.Serializable; 7 | 8 | public class StreamWatermarkAssigner implements Serializable { 9 | private WatermarkStrategyDefinition watermarkStrategyDefinition; 10 | 11 | public StreamWatermarkAssigner(WatermarkStrategyDefinition watermarkStrategyDefinition) { 12 | this.watermarkStrategyDefinition = watermarkStrategyDefinition; 13 | } 14 | 15 | public DataStream assignTimeStampAndWatermark(DataStream inputStream, long watermarkDelayMs, boolean enablePerPartitionWatermark) { 16 | return !enablePerPartitionWatermark ? inputStream 17 | .assignTimestampsAndWatermarks(watermarkStrategyDefinition.getWatermarkStrategy(watermarkDelayMs)) : inputStream; 18 | } 19 | 20 | public DataStream assignTimeStampAndWatermark(DataStream inputStream, long watermarkDelayMs) { 21 | return inputStream 22 | .assignTimestampsAndWatermarks(watermarkStrategyDefinition.getWatermarkStrategy(watermarkDelayMs)); 23 | } 24 | 25 | } 26 | 27 | -------------------------------------------------------------------------------- /dagger-common/src/main/java/org/raystack/dagger/common/watermark/WatermarkStrategyDefinition.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.watermark; 2 | 3 | import org.apache.flink.api.common.eventtime.WatermarkStrategy; 4 | import org.apache.flink.types.Row; 5 | 6 | import java.io.Serializable; 7 | 8 | public interface WatermarkStrategyDefinition extends Serializable { 9 | WatermarkStrategy getWatermarkStrategy(long waterMarkDelayInMs); 10 | } 11 | -------------------------------------------------------------------------------- /dagger-common/src/test/java/com/gotocompany/dagger/common/metrics/managers/utils/TestAspects.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.common.metrics.managers.utils; 2 | 3 | import org.raystack.dagger.common.metrics.aspects.AspectType; 4 | import org.raystack.dagger.common.metrics.aspects.Aspects; 5 | 6 | public enum TestAspects implements Aspects { 7 | TEST_ASPECT_ONE("test_aspect1", AspectType.Histogram), 8 | TEST_ASPECT_TWO("test_aspect2", AspectType.Metric), 9 | TEST_ASPECT_THREE("test_aspect3", AspectType.Counter); 10 | 11 | private String value; 12 | private AspectType aspectType; 13 | 14 | TestAspects(String value, AspectType aspectType) { 15 | this.value = value; 16 | this.aspectType = aspectType; 17 | } 18 | 19 | @Override 20 | public String getValue() { 21 | return value; 22 | } 23 | 24 | @Override 25 | public AspectType getAspectType() { 26 | return aspectType; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /dagger-common/src/test/proto/TestGrpc.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package org.raystack.dagger.consumer; 4 | 5 | option java_multiple_files = true; 6 | option java_package = "org.raystack.dagger.consumer"; 7 | option java_outer_classname = "SampleGrpcServerProto"; 8 | 9 | service TestServer { 10 | rpc TestRpcMethod (TestGrpcRequest) returns (TestGrpcResponse) {} 11 | } 12 | 13 | message TestGrpcRequest { 14 | string field1 = 1; 15 | string field2 = 2; 16 | } 17 | 18 | message Error { 19 | string code = 1; 20 | string entity = 2; 21 | } 22 | 23 | message TestGrpcResponse { 24 | bool success = 1; 25 | repeated Error error = 2; 26 | string field3 = 3; 27 | string field4 = 4; 28 | } 29 | -------------------------------------------------------------------------------- /dagger-common/src/test/proto/sample_message.txt: -------------------------------------------------------------------------------- 1 | is_valid: true 2 | order_number: "INV-53535321" 3 | event_timestamp: { 4 | seconds: 1663565509 5 | nanos: 0 6 | } 7 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/config/ConfigurationProvider.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.config; 2 | 3 | import org.raystack.dagger.common.configuration.Configuration; 4 | 5 | /** 6 | * The interface for all Configuration provider class. 7 | */ 8 | public interface ConfigurationProvider { 9 | /** 10 | * Get configuration. 11 | * 12 | * @return the configuration 13 | */ 14 | Configuration get(); 15 | } 16 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/config/EnvironmentConfigurationProvider.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.config; 2 | 3 | import org.apache.flink.api.java.utils.ParameterTool; 4 | 5 | import org.raystack.dagger.common.configuration.Configuration; 6 | 7 | import java.util.Map; 8 | 9 | /** 10 | * The class which handle configuration provided from Environment. 11 | */ 12 | public class EnvironmentConfigurationProvider implements ConfigurationProvider { 13 | 14 | private Map environmentParameters; 15 | 16 | /** 17 | * Instantiates a new Environment configuration provider. 18 | * 19 | * @param environmentParameters the environment parameters 20 | */ 21 | public EnvironmentConfigurationProvider(Map environmentParameters) { 22 | this.environmentParameters = environmentParameters; 23 | } 24 | 25 | @Override 26 | public Configuration get() { 27 | return new Configuration(ParameterTool.fromMap(environmentParameters)); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/config/KafkaEnvironmentVariables.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.config; 2 | 3 | import org.apache.flink.configuration.Configuration; 4 | 5 | import java.util.Properties; 6 | 7 | /** 8 | * The type Kafka environment variables. 9 | */ 10 | public class KafkaEnvironmentVariables { 11 | 12 | private static final String KAFKA_PREFIX = "source_kafka_config_"; 13 | 14 | /** 15 | * Parse properties. 16 | * 17 | * @param configuration the configuration 18 | * @return the properties 19 | */ 20 | public static Properties parse(Configuration configuration) { 21 | Properties props = new Properties(); 22 | 23 | if (configuration == null || configuration.keySet().size() == 0) { 24 | return props; 25 | } 26 | 27 | configuration.toMap().entrySet() 28 | .stream() 29 | .filter(e -> e.getKey().toLowerCase().startsWith(KAFKA_PREFIX)) 30 | .forEach(e -> props.setProperty(parseVarName(e.getKey()), e.getValue())); 31 | return props; 32 | } 33 | 34 | private static String parseVarName(String varName) { 35 | String[] names = varName.toLowerCase().replaceAll(KAFKA_PREFIX, "").split("_"); 36 | return String.join(".", names); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/deserializer/DaggerDeserializerProvider.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.deserializer; 2 | 3 | import org.raystack.dagger.common.serde.DaggerDeserializer; 4 | 5 | public interface DaggerDeserializerProvider { 6 | DaggerDeserializer getDaggerDeserializer(); 7 | boolean canProvide(); 8 | } 9 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/BigQueryWriterException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | import java.io.IOException; 4 | 5 | public class BigQueryWriterException extends IOException { 6 | 7 | public BigQueryWriterException(String message, Throwable cause) { 8 | super(message, cause); 9 | } 10 | 11 | public BigQueryWriterException(String message) { 12 | super(message); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/ChannelNotAvailableException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | /** 4 | * The class Exception if Grpc Channel not available. 5 | */ 6 | public class ChannelNotAvailableException extends Exception { 7 | 8 | /** 9 | * Instantiates a new Channel not available exception. 10 | * 11 | * @param message the message 12 | */ 13 | public ChannelNotAvailableException(String message) { 14 | super(message); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/ConsumerLagNotZeroException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | /** 4 | * The class Exception if kafka consumer lag is not zero. 5 | */ 6 | public class ConsumerLagNotZeroException extends RuntimeException { 7 | /** 8 | * Instantiates a new Consumer lag not zero exception. 9 | * 10 | * @param message the message 11 | */ 12 | public ConsumerLagNotZeroException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/DaggerConfigurationException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | /** 4 | * The class Exception if there is something wrong with Dagger configuration. 5 | */ 6 | public class DaggerConfigurationException extends RuntimeException { 7 | 8 | /** 9 | * Instantiates a new Dagger configuration exception with specified error message. 10 | * 11 | * @param message the message 12 | */ 13 | public DaggerConfigurationException(String message) { 14 | super(message); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/GrpcFailureException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | /** 4 | * The class Exception if there is failure in Grpc. 5 | */ 6 | public class GrpcFailureException extends RuntimeException { 7 | /** 8 | * Instantiates a new Grpc failure exception. 9 | * 10 | * @param message the message 11 | */ 12 | public GrpcFailureException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/HttpFailureException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | /** 4 | * The class Exception if there is failure in Http. 5 | */ 6 | public class HttpFailureException extends RuntimeException { 7 | /** 8 | * Instantiates a new Http failure exception. 9 | * 10 | * @param message the message 11 | */ 12 | public HttpFailureException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/InfluxWriteException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | import java.io.IOException; 4 | 5 | public class InfluxWriteException extends IOException { 6 | public InfluxWriteException(Throwable err) { 7 | super(err); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/InputOutputMappingException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | /** 4 | * The class Exception if error happens on input output mapping. 5 | */ 6 | public class InputOutputMappingException extends RuntimeException { 7 | /** 8 | * Instantiates a new Input output mapping exception. 9 | * 10 | * @param message the message 11 | */ 12 | public InputOutputMappingException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/InvalidConfigurationException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | /** 4 | * The class Exception if there is an Invalid Configuration. 5 | */ 6 | public class InvalidConfigurationException extends RuntimeException { 7 | /** 8 | * Instantiates a new Invalid configuration exception. 9 | * 10 | * @param message the message 11 | */ 12 | public InvalidConfigurationException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/InvalidDaggerSourceException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | public class InvalidDaggerSourceException extends RuntimeException { 4 | public InvalidDaggerSourceException(String message) { 5 | super(message); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/InvalidGrpcBodyException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | /** 4 | * The class Exception if there is an Invalid Grpc body. 5 | */ 6 | public class InvalidGrpcBodyException extends RuntimeException { 7 | 8 | /** 9 | * Instantiates a new Invalid grpc body exception. 10 | * 11 | * @param message the message 12 | */ 13 | public InvalidGrpcBodyException(String message) { 14 | super(message); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/InvalidHttpVerbException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | /** 4 | * The class Exception if there is an Invalid Http verb. 5 | */ 6 | public class InvalidHttpVerbException extends RuntimeException { 7 | /** 8 | * Instantiates a new Invalid http verb exception. 9 | * 10 | * @param message the message 11 | */ 12 | public InvalidHttpVerbException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/InvalidLongbowDurationException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | /** 4 | * The class Exception if there is an Invalid Longbow duration. 5 | */ 6 | public class InvalidLongbowDurationException extends RuntimeException { 7 | /** 8 | * Instantiates a new Invalid longbow duration exception. 9 | * 10 | * @param message the message 11 | */ 12 | public InvalidLongbowDurationException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/InvalidTimeRangeException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | public class InvalidTimeRangeException extends RuntimeException { 4 | public InvalidTimeRangeException(String message) { 5 | super(message); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/ParquetFileSourceReaderInitializationException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | /*** 4 | * This exception is thrown when the reader for Parquet FileSource could not be initialized. 5 | */ 6 | public class ParquetFileSourceReaderInitializationException extends RuntimeException { 7 | public ParquetFileSourceReaderInitializationException(Throwable cause) { 8 | super(cause); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/PathParserNotProvidedException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | public class PathParserNotProvidedException extends RuntimeException { 4 | public PathParserNotProvidedException(String message) { 5 | super(message); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/RecordsNotConsumedException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | /** 4 | * The class Exception if there is records that not consumed. 5 | */ 6 | public class RecordsNotConsumedException extends RuntimeException { 7 | /** 8 | * Instantiates a new Records not consumed exception. 9 | * 10 | * @param message the message 11 | */ 12 | public RecordsNotConsumedException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/ThorCommandFailedException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | /** 4 | * The class Exception if there is failure on Thor command. 5 | */ 6 | public class ThorCommandFailedException extends RuntimeException { 7 | /** 8 | * Instantiates a new Thor command failed exception. 9 | * 10 | * @param message the message 11 | */ 12 | public ThorCommandFailedException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/TransformClassNotDefinedException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | /** 4 | * The class Exception if Transformer class is not defined in transformer configuration. 5 | */ 6 | public class TransformClassNotDefinedException extends RuntimeException { 7 | /** 8 | * Instantiates a new Transform class not defined exception. 9 | * 10 | * @param message the message 11 | */ 12 | public TransformClassNotDefinedException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/exception/UDFFactoryClassNotDefinedException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.exception; 2 | 3 | /** 4 | * The class Exception if Udf factory class is not defined in function factory configuration. 5 | */ 6 | public class UDFFactoryClassNotDefinedException extends RuntimeException { 7 | /** 8 | * Instantiates a new Udf factory class not defined exception. 9 | * 10 | * @param message the message 11 | */ 12 | public UDFFactoryClassNotDefinedException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/metrics/aspects/ChronologyOrderedSplitAssignerAspects.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.metrics.aspects; 2 | 3 | import org.raystack.dagger.common.metrics.aspects.AspectType; 4 | import org.raystack.dagger.common.metrics.aspects.Aspects; 5 | 6 | public enum ChronologyOrderedSplitAssignerAspects implements Aspects { 7 | TOTAL_SPLITS_DISCOVERED("total_splits_discovered", AspectType.Gauge), 8 | TOTAL_SPLITS_RECORDED("total_splits_recorded", AspectType.Gauge), 9 | SPLITS_AWAITING_ASSIGNMENT("splits_awaiting_assignment", AspectType.Counter); 10 | 11 | ChronologyOrderedSplitAssignerAspects(String value, AspectType aspectType) { 12 | this.value = value; 13 | this.aspectType = aspectType; 14 | } 15 | 16 | private final String value; 17 | private final AspectType aspectType; 18 | 19 | @Override 20 | public String getValue() { 21 | return value; 22 | } 23 | 24 | @Override 25 | public AspectType getAspectType() { 26 | return aspectType; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/metrics/aspects/ParquetReaderAspects.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.metrics.aspects; 2 | 3 | import org.raystack.dagger.common.metrics.aspects.AspectType; 4 | import org.raystack.dagger.common.metrics.aspects.Aspects; 5 | 6 | public enum ParquetReaderAspects implements Aspects { 7 | READER_CREATED("reader_created", AspectType.Counter), 8 | READER_CLOSED("reader_closed", AspectType.Counter), 9 | READER_ROWS_EMITTED("reader_rows_emitted", AspectType.Counter), 10 | READER_ROW_DESERIALIZATION_TIME("reader_row_deserialization_time", AspectType.Histogram), 11 | READER_ROW_READ_TIME("reader_row_read_time", AspectType.Histogram); 12 | 13 | private final String value; 14 | private final AspectType aspectType; 15 | 16 | ParquetReaderAspects(String value, AspectType aspectType) { 17 | this.value = value; 18 | this.aspectType = aspectType; 19 | } 20 | 21 | @Override 22 | public String getValue() { 23 | return value; 24 | } 25 | 26 | @Override 27 | public AspectType getAspectType() { 28 | return aspectType; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/metrics/aspects/TelemetryAspects.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.metrics.aspects; 2 | 3 | import org.raystack.dagger.common.metrics.aspects.AspectType; 4 | import org.raystack.dagger.common.metrics.aspects.Aspects; 5 | 6 | /** 7 | * The enum Telemetry aspects. 8 | */ 9 | public enum TelemetryAspects implements Aspects { 10 | /** 11 | * Value telemetry aspects. 12 | */ 13 | VALUE("value", AspectType.Metric); 14 | 15 | private String value; 16 | private AspectType aspectType; 17 | 18 | TelemetryAspects(String value, AspectType aspectType) { 19 | this.value = value; 20 | this.aspectType = aspectType; 21 | } 22 | 23 | @Override 24 | public String getValue() { 25 | return value; 26 | } 27 | 28 | @Override 29 | public AspectType getAspectType() { 30 | return aspectType; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/metrics/reporters/ErrorReporter.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.metrics.reporters; 2 | 3 | import org.apache.flink.metrics.Counter; 4 | import org.apache.flink.metrics.MetricGroup; 5 | 6 | /** 7 | * The interface Error reporter. 8 | */ 9 | public interface ErrorReporter { 10 | /** 11 | * Report fatal exception. 12 | * 13 | * @param exception the exception 14 | */ 15 | void reportFatalException(Exception exception); 16 | /** 17 | * Report non fatal exception. 18 | * 19 | * @param exception the exception 20 | */ 21 | void reportNonFatalException(Exception exception); 22 | 23 | 24 | default Counter addExceptionToCounter(Exception exception, MetricGroup metricGroup, String metricGroupKey) { 25 | return metricGroup.addGroup(metricGroupKey, exception.getClass().getName()).counter("value"); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/metrics/reporters/NoOpErrorReporter.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.metrics.reporters; 2 | 3 | /** 4 | * The No op error reporter. 5 | */ 6 | public class NoOpErrorReporter implements ErrorReporter { 7 | @Override 8 | public void reportFatalException(Exception exception) { 9 | 10 | } 11 | 12 | @Override 13 | public void reportNonFatalException(Exception exception) { 14 | 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/metrics/reporters/statsd/SerializedStatsDReporterSupplier.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.metrics.reporters.statsd; 2 | 3 | import org.raystack.depot.metrics.StatsDReporter; 4 | 5 | import java.io.Serializable; 6 | 7 | /* Flink requires that all objects which are needed to prepare the Job Graph should be serializable along with their 8 | properties/fields. StatsDReporter and its fields are not serializable. Hence, in order to mitigate job graph creation 9 | failure, we create a serializable interface around the reporter as below. This is a common idiom to make un-serializable 10 | fields serializable in Java 8: https://stackoverflow.com/a/22808112 */ 11 | 12 | @FunctionalInterface 13 | public interface SerializedStatsDReporterSupplier extends Serializable { 14 | StatsDReporter buildStatsDReporter(); 15 | } 16 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/metrics/reporters/statsd/manager/MeasurementManager.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.metrics.reporters.statsd.manager; 2 | 3 | import org.raystack.dagger.core.metrics.reporters.statsd.tags.StatsDTag; 4 | 5 | import java.io.Serializable; 6 | 7 | public interface MeasurementManager extends Serializable { 8 | void register(StatsDTag[] tags); 9 | } 10 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/metrics/reporters/statsd/measurement/Counter.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.metrics.reporters.statsd.measurement; 2 | 3 | import org.raystack.dagger.common.metrics.aspects.Aspects; 4 | 5 | import java.io.Serializable; 6 | 7 | public interface Counter extends Serializable { 8 | void increment(Aspects aspect); 9 | 10 | void increment(Aspects aspect, long num); 11 | 12 | void decrement(Aspects aspect); 13 | 14 | void decrement(Aspects aspect, long num); 15 | } 16 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/metrics/reporters/statsd/measurement/Gauge.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.metrics.reporters.statsd.measurement; 2 | 3 | import org.raystack.dagger.common.metrics.aspects.Aspects; 4 | 5 | import java.io.Serializable; 6 | 7 | public interface Gauge extends Serializable { 8 | void markValue(Aspects aspect, int gaugeValue); 9 | } 10 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/metrics/reporters/statsd/measurement/Histogram.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.metrics.reporters.statsd.measurement; 2 | 3 | import org.raystack.dagger.common.metrics.aspects.Aspects; 4 | 5 | import java.io.Serializable; 6 | 7 | public interface Histogram extends Serializable { 8 | void recordValue(Aspects aspect, long value); 9 | } 10 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/metrics/reporters/statsd/tags/ComponentTags.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.metrics.reporters.statsd.tags; 2 | 3 | public class ComponentTags { 4 | private static StatsDTag[] parquetReaderTags; 5 | private static StatsDTag[] splitAssignerTags; 6 | private static final String COMPONENT_TAG_KEY = "component"; 7 | 8 | public static StatsDTag[] getParquetReaderTags() { 9 | if (parquetReaderTags == null) { 10 | parquetReaderTags = new StatsDTag[]{ 11 | new StatsDTag(COMPONENT_TAG_KEY, "parquet_reader"), 12 | }; 13 | } 14 | return parquetReaderTags; 15 | } 16 | 17 | public static StatsDTag[] getSplitAssignerTags() { 18 | if (splitAssignerTags == null) { 19 | splitAssignerTags = new StatsDTag[]{ 20 | new StatsDTag(COMPONENT_TAG_KEY, "split_assigner"), 21 | }; 22 | } 23 | return splitAssignerTags; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/metrics/reporters/statsd/tags/GlobalTags.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.metrics.reporters.statsd.tags; 2 | 3 | public class GlobalTags { 4 | public static final String JOB_ID = "job_id"; 5 | } 6 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/metrics/reporters/statsd/tags/StatsDTag.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.metrics.reporters.statsd.tags; 2 | 3 | import org.apache.flink.util.Preconditions; 4 | 5 | public class StatsDTag { 6 | private final String tagKey; 7 | private final String tagValue; 8 | private static final String NIL_TAG_VALUE = "NIL_TAG_VALUE"; 9 | 10 | public StatsDTag(String key, String value) { 11 | Preconditions.checkArgument(key != null && !key.isEmpty(), "Tag key cannot be null or empty"); 12 | this.tagKey = key; 13 | this.tagValue = (value != null && !value.isEmpty()) ? value : NIL_TAG_VALUE; 14 | } 15 | 16 | public StatsDTag(String tagName) { 17 | this(tagName, NIL_TAG_VALUE); 18 | } 19 | 20 | public String getFormattedTag() { 21 | if (tagValue.equals(NIL_TAG_VALUE)) { 22 | return tagKey; 23 | } else { 24 | return String.format("%s=%s", tagKey, tagValue); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/metrics/telemetry/TelemetrySubscriber.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.metrics.telemetry; 2 | 3 | /** 4 | * The interface Telemetry subscriber. 5 | */ 6 | public interface TelemetrySubscriber { 7 | /** 8 | * Updated telemetry publisher. 9 | * 10 | * @param publisher the publisher 11 | */ 12 | void updated(TelemetryPublisher publisher); 13 | } 14 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/metrics/telemetry/TelemetryTypes.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.metrics.telemetry; 2 | 3 | /** 4 | * The enum Telemetry types. 5 | */ 6 | public enum TelemetryTypes { 7 | INPUT_TOPIC("input_topic"), 8 | INPUT_PROTO("input_proto"), 9 | INPUT_STREAM("input_stream"), 10 | SINK_TYPE("sink_type"), 11 | OUTPUT_TOPIC("output_topic"), 12 | OUTPUT_PROTO("output_proto"), 13 | OUTPUT_STREAM("output_stream"), 14 | POST_PROCESSOR_TYPE("post_processor_type"), 15 | PRE_PROCESSOR_TYPE("pre_processor_type"), 16 | SOURCE_METRIC_ID("source_metricId"); 17 | 18 | /** 19 | * Gets telemetry type value. 20 | * 21 | * @return the value 22 | */ 23 | public String getValue() { 24 | return value; 25 | } 26 | 27 | private String value; 28 | 29 | TelemetryTypes(String value) { 30 | this.value = value; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/PreProcessorConfig.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors; 2 | 3 | import org.raystack.dagger.core.processors.transformers.TableTransformConfig; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * The Preprocessor config. 9 | */ 10 | public class PreProcessorConfig { 11 | /** 12 | * Gets table transformers. 13 | * 14 | * @return the table transformers 15 | */ 16 | public List getTableTransformers() { 17 | return tableTransformers; 18 | } 19 | 20 | /** 21 | * The Table transformers. 22 | */ 23 | protected List tableTransformers; 24 | 25 | /** 26 | * Check if it has transformer configs. 27 | * 28 | * @return the boolean 29 | */ 30 | public boolean hasTransformConfigs() { 31 | return tableTransformers != null && !tableTransformers.isEmpty(); 32 | } 33 | 34 | /** 35 | * Check if transformers config is empty. 36 | * 37 | * @return the boolean 38 | */ 39 | public boolean isEmpty() { 40 | return !hasTransformConfigs(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/PreProcessorFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors; 2 | 3 | import org.raystack.dagger.common.core.DaggerContext; 4 | import org.raystack.dagger.core.processors.types.Preprocessor; 5 | import org.raystack.dagger.core.processors.telemetry.processor.MetricsTelemetryExporter; 6 | 7 | import java.util.Collections; 8 | import java.util.List; 9 | 10 | /** 11 | * The factory class for Preprocessor. 12 | */ 13 | public class PreProcessorFactory { 14 | /** 15 | * Gets preprocessors. 16 | * 17 | * @param daggerContext the daggerContext 18 | * @param tableName the table name 19 | * @param metricsTelemetryExporter the metrics telemetry exporter 20 | * @return the preprocessors 21 | */ 22 | public static List getPreProcessors(DaggerContext daggerContext, String tableName, MetricsTelemetryExporter metricsTelemetryExporter) { 23 | return Collections.singletonList(new PreProcessorOrchestrator(daggerContext, metricsTelemetryExporter, tableName)); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/common/InitializationDecorator.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.common; 2 | 3 | import org.raystack.dagger.core.processors.ColumnNameManager; 4 | import org.raystack.dagger.core.processors.types.MapDecorator; 5 | 6 | import org.apache.flink.types.Row; 7 | 8 | /** 9 | * The Initialization decorator. 10 | */ 11 | public class InitializationDecorator implements MapDecorator { 12 | 13 | private ColumnNameManager columnNameManager; 14 | 15 | /** 16 | * Instantiates a new Initialization decorator. 17 | * 18 | * @param columnNameManager the column name manager 19 | */ 20 | public InitializationDecorator(ColumnNameManager columnNameManager) { 21 | this.columnNameManager = columnNameManager; 22 | } 23 | 24 | @Override 25 | public Boolean canDecorate() { 26 | return false; 27 | } 28 | 29 | @Override 30 | public Row map(Row input) { 31 | RowManager rowManager = new RowManager(input, columnNameManager.getOutputSize()); 32 | return rowManager.getAll(); 33 | } 34 | 35 | 36 | } 37 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/external/http/request/HttpRequestHandler.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.external.http.request; 2 | 3 | import org.asynchttpclient.BoundRequestBuilder; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * The interface Http request handler. 9 | */ 10 | public interface HttpRequestHandler { 11 | /** 12 | * Create bound request builder. 13 | * 14 | * @return the bound request builder 15 | */ 16 | BoundRequestBuilder create(); 17 | 18 | /** 19 | * Check if can create the request. 20 | * 21 | * @return the boolean 22 | */ 23 | boolean canCreate(); 24 | 25 | /** 26 | * Add headers bound request builder. 27 | * 28 | * @param request the request 29 | * @param headerMap the header map 30 | * @return the bound request builder 31 | */ 32 | default BoundRequestBuilder addHeaders(BoundRequestBuilder request, Map headerMap) { 33 | headerMap.keySet().forEach(headerKey -> { 34 | request.addHeader(headerKey, headerMap.get(headerKey)); 35 | }); 36 | return request; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/internal/processor/InternalConfigProcessor.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.internal.processor; 2 | 3 | import org.raystack.dagger.core.processors.common.RowManager; 4 | 5 | /** 6 | * The interface for Internal config processor. 7 | */ 8 | public interface InternalConfigProcessor { 9 | /** 10 | * Check if can process internal post processor. 11 | * 12 | * @param type the type 13 | * @return the boolean 14 | */ 15 | boolean canProcess(String type); 16 | 17 | /** 18 | * Process. 19 | * 20 | * @param rowManager the row manager 21 | */ 22 | void process(RowManager rowManager); 23 | } 24 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/internal/processor/function/FunctionProcessor.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.internal.processor.function; 2 | import org.raystack.dagger.core.processors.common.RowManager; 3 | 4 | public interface FunctionProcessor { 5 | /** 6 | * Check if function can be processed. 7 | * 8 | * @param functionName the function name 9 | * @return the boolean 10 | */ 11 | boolean canProcess(String functionName); 12 | 13 | /** 14 | * Process. 15 | * 16 | * @param rowManager the row manager 17 | */ 18 | Object getResult(RowManager rowManager); 19 | } 20 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/internal/processor/function/functions/CurrentTimestampFunction.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.internal.processor.function.functions; 2 | 3 | import org.raystack.dagger.core.processors.common.RowManager; 4 | import org.raystack.dagger.core.processors.internal.processor.function.FunctionProcessor; 5 | 6 | import java.sql.Timestamp; 7 | import java.io.Serializable; 8 | import java.time.Clock; 9 | 10 | public class CurrentTimestampFunction implements FunctionProcessor, Serializable { 11 | public static final String CURRENT_TIMESTAMP_FUNCTION_KEY = "CURRENT_TIMESTAMP"; 12 | 13 | private Clock clock; 14 | 15 | public CurrentTimestampFunction(Clock clock) { 16 | this.clock = clock; 17 | } 18 | @Override 19 | public boolean canProcess(String functionName) { 20 | return CURRENT_TIMESTAMP_FUNCTION_KEY.equals(functionName); 21 | } 22 | 23 | /** 24 | * Gets current time. 25 | * 26 | * @return the current time 27 | */ 28 | @Override 29 | public Object getResult(RowManager rowManager) { 30 | return new Timestamp(clock.millis()); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/internal/processor/sql/SqlInternalFieldConfig.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.internal.processor.sql; 2 | 3 | import org.raystack.dagger.core.processors.common.RowManager; 4 | 5 | /** 6 | * The interface for Sql internal field config. 7 | */ 8 | public interface SqlInternalFieldConfig { 9 | 10 | /** 11 | * Process input columns. 12 | * 13 | * @param rowManager the row manager 14 | */ 15 | void processInputColumns(RowManager rowManager); 16 | } 17 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/AsyncProcessor.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow; 2 | 3 | import org.apache.flink.streaming.api.datastream.AsyncDataStream; 4 | import org.apache.flink.streaming.api.datastream.DataStream; 5 | import org.apache.flink.streaming.api.functions.async.AsyncFunction; 6 | import org.apache.flink.types.Row; 7 | 8 | import java.util.concurrent.TimeUnit; 9 | 10 | /** 11 | * The Async processor. 12 | */ 13 | public class AsyncProcessor { 14 | /** 15 | * Ordered wait data stream. 16 | * 17 | * @param inputStream the input stream 18 | * @param function the function 19 | * @param timeout the timeout 20 | * @param timeunit the timeunit 21 | * @param capacity the capacity 22 | * @return the data stream 23 | */ 24 | public DataStream orderedWait(DataStream inputStream, AsyncFunction function, long timeout, TimeUnit timeunit, Integer capacity) { 25 | return AsyncDataStream.orderedWait(inputStream, function, timeout, timeunit, capacity); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/columnmodifier/ColumnModifier.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.columnmodifier; 2 | 3 | /** 4 | * The interface Column modifier. 5 | */ 6 | public interface ColumnModifier { 7 | /** 8 | * Modify the column names. 9 | * 10 | * @param inputColumnNames the input column names 11 | * @return modified column names 12 | */ 13 | String[] modifyColumnNames(String[] inputColumnNames); 14 | } 15 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/columnmodifier/LongbowReadColumnModifier.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.columnmodifier; 2 | 3 | import org.raystack.dagger.core.utils.Constants; 4 | 5 | import java.util.ArrayList; 6 | import java.util.Arrays; 7 | 8 | /** 9 | * The Longbow read column modifier. 10 | */ 11 | public class LongbowReadColumnModifier implements ColumnModifier { 12 | 13 | @Override 14 | public String[] modifyColumnNames(String[] inputColumnNames) { 15 | ArrayList inputColumnList = new ArrayList<>(Arrays.asList(inputColumnNames)); 16 | inputColumnList.add(inputColumnList.size(), Constants.LONGBOW_PROTO_DATA_KEY); 17 | 18 | return inputColumnList.toArray(new String[0]); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/columnmodifier/LongbowWriteColumnModifier.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.columnmodifier; 2 | 3 | import org.raystack.dagger.core.utils.Constants; 4 | 5 | import java.util.ArrayList; 6 | import java.util.Arrays; 7 | 8 | /** 9 | * The Longbow write column modifier. 10 | */ 11 | public class LongbowWriteColumnModifier implements ColumnModifier { 12 | 13 | @Override 14 | public String[] modifyColumnNames(String[] inputColumnNames) { 15 | ArrayList outputList = new ArrayList<>(Arrays.asList(inputColumnNames)); 16 | outputList.add(Constants.SYNCHRONIZER_BIGTABLE_TABLE_ID_KEY); 17 | outputList.add(Constants.SYNCHRONIZER_INPUT_CLASSNAME_KEY); 18 | outputList.add(Constants.SYNCHRONIZER_LONGBOW_READ_KEY); 19 | return outputList.toArray(new String[0]); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/columnmodifier/NoOpColumnModifier.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.columnmodifier; 2 | 3 | /** 4 | * The No op column modifier. 5 | */ 6 | public class NoOpColumnModifier implements ColumnModifier { 7 | @Override 8 | public String[] modifyColumnNames(String[] inputColumnNames) { 9 | return inputColumnNames; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/data/LongbowData.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.data; 2 | 3 | import org.apache.hadoop.hbase.client.Result; 4 | 5 | import java.io.Serializable; 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | /** 10 | * The interface Longbow data. 11 | */ 12 | public interface LongbowData extends Serializable { 13 | /** 14 | * Parse the scan result. 15 | * 16 | * @param scanResult the scan result 17 | * @return the map 18 | */ 19 | Map parse(List scanResult); 20 | } 21 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/data/LongbowDataFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.data; 2 | 3 | import org.raystack.dagger.core.processors.longbow.LongbowSchema; 4 | 5 | /** 6 | * The factory class for Longbow data. 7 | */ 8 | public class LongbowDataFactory { 9 | private LongbowSchema longbowSchema; 10 | 11 | /** 12 | * Instantiates a new Longbow data factory. 13 | * 14 | * @param longbowSchema the longbow schema 15 | */ 16 | public LongbowDataFactory(LongbowSchema longbowSchema) { 17 | this.longbowSchema = longbowSchema; 18 | } 19 | 20 | /** 21 | * Gets longbow data. 22 | * 23 | * @return the longbow data 24 | */ 25 | public LongbowData getLongbowData() { 26 | if (!longbowSchema.isLongbowPlus()) { 27 | return new LongbowTableData(longbowSchema); 28 | } 29 | return new LongbowProtoData(); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/data/LongbowProtoData.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.data; 2 | 3 | import org.raystack.dagger.core.utils.Constants; 4 | import org.apache.hadoop.hbase.client.Result; 5 | import org.apache.hadoop.hbase.util.Bytes; 6 | 7 | import java.util.ArrayList; 8 | import java.util.HashMap; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | /** 13 | * The Longbow proto data. 14 | */ 15 | public class LongbowProtoData implements LongbowData { 16 | private static final byte[] COLUMN_FAMILY_NAME = Bytes.toBytes(Constants.LONGBOW_COLUMN_FAMILY_DEFAULT); 17 | 18 | /** 19 | * Instantiates a new Longbow proto data. 20 | */ 21 | public LongbowProtoData() { 22 | } 23 | 24 | @Override 25 | public Map> parse(List scanResult) { 26 | ArrayList data = new ArrayList<>(); 27 | 28 | for (int i = 0; i < scanResult.size(); i++) { 29 | data.add(i, scanResult.get(i).getValue(COLUMN_FAMILY_NAME, Bytes.toBytes(Constants.LONGBOW_QUALIFIER_DEFAULT))); 30 | } 31 | 32 | HashMap> longbowData = new HashMap<>(); 33 | longbowData.put(Constants.LONGBOW_PROTO_DATA_KEY, data); 34 | return longbowData; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/exceptions/LongbowReaderException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.exceptions; 2 | 3 | /** 4 | * The Exception for Longbow reader. 5 | */ 6 | public class LongbowReaderException extends RuntimeException { 7 | /** 8 | * Instantiates a new Longbow reader exception. 9 | * 10 | * @param throwable the throwable 11 | */ 12 | public LongbowReaderException(Throwable throwable) { 13 | super(throwable); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/exceptions/LongbowWriterException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.exceptions; 2 | 3 | /** 4 | * The Exception for Longbow writer. 5 | */ 6 | public class LongbowWriterException extends RuntimeException { 7 | /** 8 | * Instantiates a new Longbow writer exception. 9 | * 10 | * @param throwable the throwable 11 | */ 12 | public LongbowWriterException(Throwable throwable) { 13 | super(throwable); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/outputRow/OutputIdentity.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.outputRow; 2 | 3 | import org.apache.flink.types.Row; 4 | 5 | /** 6 | * The Output identity. 7 | */ 8 | public class OutputIdentity implements WriterOutputRow { 9 | @Override 10 | public Row get(Row input) { 11 | return input; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/outputRow/ReaderOutputRow.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.outputRow; 2 | 3 | import org.apache.flink.types.Row; 4 | 5 | import java.io.Serializable; 6 | import java.util.Map; 7 | 8 | /** 9 | * The interface Reader output row. 10 | */ 11 | public interface ReaderOutputRow extends Serializable { 12 | /** 13 | * Get row. 14 | * 15 | * @param scanResult the scan result 16 | * @param input the input 17 | * @return the row 18 | */ 19 | Row get(Map scanResult, Row input); 20 | } 21 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/outputRow/WriterOutputRow.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.outputRow; 2 | 3 | import org.apache.flink.types.Row; 4 | 5 | import java.io.Serializable; 6 | 7 | /** 8 | * The interface Writer output row. 9 | */ 10 | public interface WriterOutputRow extends Serializable { 11 | /** 12 | * Get row. 13 | * 14 | * @param input the input 15 | * @return the row 16 | */ 17 | Row get(Row input); 18 | } 19 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/range/LongbowAbsoluteRange.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.range; 2 | 3 | import org.raystack.dagger.core.utils.Constants; 4 | import org.raystack.dagger.core.processors.longbow.LongbowSchema; 5 | 6 | import org.apache.flink.types.Row; 7 | 8 | /** 9 | * Absolute range on Longbow. 10 | */ 11 | public class LongbowAbsoluteRange implements LongbowRange { 12 | private LongbowSchema longbowSchema; 13 | 14 | /** 15 | * Instantiates a new Longbow absolute range. 16 | * 17 | * @param longbowSchema the longbow schema 18 | */ 19 | public LongbowAbsoluteRange(LongbowSchema longbowSchema) { 20 | this.longbowSchema = longbowSchema; 21 | } 22 | 23 | @Override 24 | public byte[] getUpperBound(Row input) { 25 | return longbowSchema.getAbsoluteKey(input, (long) longbowSchema.getValue(input, Constants.LONGBOW_LATEST_KEY)); 26 | } 27 | 28 | @Override 29 | public byte[] getLowerBound(Row input) { 30 | return longbowSchema.getAbsoluteKey(input, (long) longbowSchema.getValue(input, Constants.LONGBOW_EARLIEST_KEY)); 31 | } 32 | 33 | @Override 34 | public String[] getInvalidFields() { 35 | return new String[]{Constants.LONGBOW_DURATION_KEY}; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/range/LongbowDurationRange.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.range; 2 | 3 | import org.raystack.dagger.core.utils.Constants; 4 | import org.raystack.dagger.core.processors.longbow.LongbowSchema; 5 | 6 | import org.apache.flink.types.Row; 7 | 8 | /** 9 | * Duration range on Longbow. 10 | */ 11 | public class LongbowDurationRange implements LongbowRange { 12 | private LongbowSchema longbowSchema; 13 | 14 | /** 15 | * Instantiates a new Longbow duration range. 16 | * 17 | * @param longbowSchema the longbow schema 18 | */ 19 | public LongbowDurationRange(LongbowSchema longbowSchema) { 20 | this.longbowSchema = longbowSchema; 21 | } 22 | 23 | @Override 24 | public byte[] getUpperBound(Row input) { 25 | return longbowSchema.getKey(input, 0); 26 | } 27 | 28 | @Override 29 | public byte[] getLowerBound(Row input) { 30 | return longbowSchema.getKey(input, longbowSchema.getDurationInMillis(input)); 31 | } 32 | 33 | @Override 34 | public String[] getInvalidFields() { 35 | return new String[]{Constants.LONGBOW_EARLIEST_KEY, Constants.LONGBOW_LATEST_KEY}; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/range/LongbowRange.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.range; 2 | 3 | import org.apache.flink.types.Row; 4 | 5 | import java.io.Serializable; 6 | 7 | /** 8 | * The interface Longbow range. 9 | */ 10 | public interface LongbowRange extends Serializable { 11 | /** 12 | * Get upper bounds. 13 | * 14 | * @param input the input 15 | * @return the upper bounds 16 | */ 17 | byte[] getUpperBound(Row input); 18 | 19 | /** 20 | * Get lower bounds. 21 | * 22 | * @param input the input 23 | * @return the lower bounds 24 | */ 25 | byte[] getLowerBound(Row input); 26 | 27 | /** 28 | * Get invalid fields. 29 | * 30 | * @return the invalid fields 31 | */ 32 | String[] getInvalidFields(); 33 | } 34 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/range/LongbowRangeFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.range; 2 | 3 | import org.raystack.dagger.core.exception.DaggerConfigurationException; 4 | import org.raystack.dagger.core.utils.Constants; 5 | import org.raystack.dagger.core.processors.longbow.LongbowSchema; 6 | 7 | /** 8 | * The factor class for Longbow range. 9 | */ 10 | public class LongbowRangeFactory { 11 | /** 12 | * Gets longbow range. 13 | * 14 | * @param longbowSchema the longbow schema 15 | * @return the longbow range 16 | */ 17 | public static LongbowRange getLongbowRange(LongbowSchema longbowSchema) { 18 | if (longbowSchema.contains(Constants.LONGBOW_DURATION_KEY)) { 19 | return new LongbowDurationRange(longbowSchema); 20 | } else if (longbowSchema.contains(Constants.LONGBOW_EARLIEST_KEY) && longbowSchema.contains(Constants.LONGBOW_LATEST_KEY)) { 21 | return new LongbowAbsoluteRange(longbowSchema); 22 | } else { 23 | throw new DaggerConfigurationException("Missing required field: Either (" + Constants.LONGBOW_DURATION_KEY + ") or both (" + Constants.LONGBOW_EARLIEST_KEY + " and " + Constants.LONGBOW_LATEST_KEY + ") should be passed"); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/storage/PutRequest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.storage; 2 | 3 | import org.apache.hadoop.hbase.client.Put; 4 | 5 | /** 6 | * The interface Put request. 7 | */ 8 | public interface PutRequest { 9 | /** 10 | * Get put. 11 | * 12 | * @return the put 13 | */ 14 | Put get(); 15 | 16 | /** 17 | * Gets table id. 18 | * 19 | * @return the table id 20 | */ 21 | String getTableId(); 22 | } 23 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/longbow/storage/ScanRequest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.storage; 2 | 3 | import org.apache.hadoop.hbase.client.Scan; 4 | 5 | /** 6 | * The interface Scan request. 7 | */ 8 | public interface ScanRequest { 9 | /** 10 | * Get scan. 11 | * 12 | * @return the scan 13 | */ 14 | Scan get(); 15 | 16 | /** 17 | * Sets scan range. 18 | * 19 | * @param startRow the start row 20 | * @param stopRow the stop row 21 | * @return the scan range 22 | */ 23 | default Scan setScanRange(byte[] startRow, byte[] stopRow) { 24 | Scan scan = new Scan(); 25 | scan.withStartRow(startRow, true); 26 | scan.withStopRow(stopRow, true); 27 | return scan; 28 | } 29 | 30 | /** 31 | * Gets table id. 32 | * 33 | * @return the table id 34 | */ 35 | String getTableId(); 36 | } 37 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/transformers/TransformerUtils.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.transformers; 2 | 3 | /** 4 | * The utils of the Transformer. 5 | */ 6 | public class TransformerUtils { 7 | /** 8 | * The enum Default argument. 9 | */ 10 | enum DefaultArgument { 11 | /** 12 | * Table name default argument. 13 | */ 14 | INPUT_SCHEMA_TABLE("table_name"); 15 | private final String argument; 16 | 17 | DefaultArgument(String argument) { 18 | this.argument = argument; 19 | } 20 | 21 | @Override 22 | public String toString() { 23 | return this.argument; 24 | } 25 | } 26 | 27 | /** 28 | * Populate default arguments. 29 | * 30 | * @param processor the processor 31 | */ 32 | protected static void populateDefaultArguments(TransformProcessor processor) { 33 | for (TransformConfig config : processor.transformConfigs) { 34 | config.validateFields(); 35 | config.getTransformationArguments().put(DefaultArgument.INPUT_SCHEMA_TABLE.toString(), processor.tableName); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/types/FilterDecorator.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.types; 2 | 3 | import org.apache.flink.api.common.functions.FilterFunction; 4 | import org.apache.flink.streaming.api.datastream.DataStream; 5 | import org.apache.flink.types.Row; 6 | 7 | /** 8 | * The interface Filter decorator. 9 | */ 10 | public interface FilterDecorator extends FilterFunction, StreamDecorator { 11 | 12 | @Override 13 | default DataStream decorate(DataStream inputStream) { 14 | return inputStream.filter(this); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/types/MapDecorator.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.types; 2 | 3 | import org.apache.flink.api.common.functions.MapFunction; 4 | import org.apache.flink.streaming.api.datastream.DataStream; 5 | import org.apache.flink.types.Row; 6 | 7 | /** 8 | * The interface Map decorator. 9 | */ 10 | public interface MapDecorator extends MapFunction, StreamDecorator { 11 | 12 | @Override 13 | default DataStream decorate(DataStream inputStream) { 14 | return inputStream.map(this); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/types/PostProcessor.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.types; 2 | 3 | import org.raystack.dagger.common.core.StreamInfo; 4 | import org.raystack.dagger.core.processors.PostProcessorConfig; 5 | 6 | /** 7 | * The interface Post processor. 8 | */ 9 | public interface PostProcessor { 10 | 11 | /** 12 | * Process stream info. 13 | * 14 | * @param streamInfo the stream info 15 | * @return the stream info 16 | */ 17 | StreamInfo process(StreamInfo streamInfo); 18 | 19 | /** 20 | * Check if it can process the post processor with given config. 21 | * 22 | * @param processorConfig the processor config 23 | * @return the boolean 24 | */ 25 | boolean canProcess(PostProcessorConfig processorConfig); 26 | } 27 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/types/Preprocessor.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.types; 2 | 3 | import org.raystack.dagger.common.core.StreamInfo; 4 | import org.raystack.dagger.core.processors.PreProcessorConfig; 5 | 6 | /** 7 | * The interface Preprocessor. 8 | */ 9 | public interface Preprocessor { 10 | /** 11 | * Process stream info. 12 | * 13 | * @param streamInfo the stream info 14 | * @return the stream info 15 | */ 16 | StreamInfo process(StreamInfo streamInfo); 17 | 18 | /** 19 | * Check if it can process the preprocessor with given config. 20 | * 21 | * @param processorConfig the processor config 22 | * @return the boolean 23 | */ 24 | boolean canProcess(PreProcessorConfig processorConfig); 25 | } 26 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/types/SourceConfig.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.types; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * The interface Source config. 7 | */ 8 | public interface SourceConfig extends Validator { 9 | /** 10 | * Gets output columns. 11 | * 12 | * @return the output columns 13 | */ 14 | List getOutputColumns(); 15 | 16 | /** 17 | * Check if the source config is failed on errors. 18 | * 19 | * @return the boolean 20 | */ 21 | boolean isFailOnErrors(); 22 | 23 | /** 24 | * Gets metric id. 25 | * 26 | * @return the metric id 27 | */ 28 | String getMetricId(); 29 | 30 | /** 31 | * Gets pattern. 32 | * 33 | * @return the pattern 34 | */ 35 | String getPattern(); 36 | 37 | /** 38 | * Gets variables. 39 | * 40 | * @return the variables 41 | */ 42 | String getVariables(); 43 | 44 | /** 45 | * Gets type. 46 | * 47 | * @return the type 48 | */ 49 | String getType(); 50 | } 51 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/processors/types/StreamDecorator.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.types; 2 | 3 | import org.apache.flink.streaming.api.datastream.DataStream; 4 | import org.apache.flink.types.Row; 5 | 6 | import java.io.Serializable; 7 | 8 | /** 9 | * The interface Stream decorator. 10 | */ 11 | public interface StreamDecorator extends Serializable { 12 | /** 13 | * Check if can decorate the Stream or not. 14 | * 15 | * @return the boolean 16 | */ 17 | Boolean canDecorate(); 18 | 19 | /** 20 | * Decorate data stream. 21 | * 22 | * @param inputStream the input stream 23 | * @return the data stream 24 | */ 25 | DataStream decorate(DataStream inputStream); 26 | } 27 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/sink/influx/InfluxDBFactoryWrapper.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.sink.influx; 2 | 3 | import org.influxdb.InfluxDB; 4 | import org.influxdb.InfluxDBFactory; 5 | 6 | import java.io.Serializable; 7 | 8 | /** 9 | * The Influx db factory wrapper. 10 | */ 11 | public class InfluxDBFactoryWrapper implements Serializable { 12 | 13 | /** 14 | * Connect influx db. 15 | * 16 | * @param url the url 17 | * @param username the username 18 | * @param password the password 19 | * @return the influx db 20 | */ 21 | public InfluxDB connect(String url, String username, String password) { 22 | return InfluxDBFactory.connect(url, username, password); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/sink/influx/errors/InfluxError.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.sink.influx.errors; 2 | 3 | import org.influxdb.dto.Point; 4 | 5 | import org.slf4j.Logger; 6 | 7 | import java.io.IOException; 8 | 9 | /** 10 | * The interface Influx error. 11 | */ 12 | public interface InfluxError { 13 | 14 | /** 15 | * Check if it has exception. 16 | * 17 | * @return the boolean 18 | */ 19 | boolean hasException(); 20 | 21 | /** 22 | * Gets current exception. 23 | * 24 | * @return the current exception 25 | */ 26 | IOException getCurrentException(); 27 | 28 | /** 29 | * Filter the error. 30 | * 31 | * @param throwable the throwable 32 | * @return the boolean 33 | */ 34 | boolean filterError(Throwable throwable); 35 | 36 | /** 37 | * Handle the error. 38 | * 39 | * @param points the points 40 | * @param throwable the throwable 41 | */ 42 | void handle(Iterable points, Throwable throwable); 43 | 44 | /** 45 | * Log failed points. 46 | * 47 | * @param points the points 48 | * @param logger the logger 49 | */ 50 | default void logFailedPoints(Iterable points, Logger logger) { 51 | points.forEach(point -> logger.warn("Error writing to influx {}", point.toString())); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/sink/influx/errors/NoError.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.sink.influx.errors; 2 | 3 | import org.influxdb.dto.Point; 4 | 5 | import java.io.IOException; 6 | 7 | /** 8 | * No error found on Influx sink. 9 | */ 10 | public class NoError implements InfluxError { 11 | @Override 12 | public boolean hasException() { 13 | return false; 14 | } 15 | 16 | @Override 17 | public IOException getCurrentException() { 18 | return null; 19 | } 20 | 21 | @Override 22 | public boolean filterError(Throwable throwable) { 23 | return false; 24 | } 25 | 26 | @Override 27 | public void handle(Iterable points, Throwable throwable) { 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/sink/influx/errors/ValidError.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.sink.influx.errors; 2 | 3 | import org.raystack.dagger.core.exception.InfluxWriteException; 4 | import org.raystack.dagger.core.sink.influx.InfluxDBSink; 5 | import org.influxdb.dto.Point; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.io.IOException; 10 | 11 | /** 12 | * The Valid error. 13 | */ 14 | public class ValidError implements InfluxError { 15 | 16 | private static final Logger LOGGER = LoggerFactory.getLogger(InfluxDBSink.class.getName()); 17 | private IOException error; 18 | 19 | @Override 20 | public boolean hasException() { 21 | return true; 22 | } 23 | 24 | @Override 25 | public IOException getCurrentException() { 26 | return error; 27 | } 28 | 29 | @Override 30 | public boolean filterError(Throwable throwable) { 31 | return throwable instanceof Error; 32 | } 33 | 34 | @Override 35 | public void handle(Iterable points, Throwable throwable) { 36 | error = new InfluxWriteException(throwable); 37 | logFailedPoints(points, LOGGER); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/sink/influx/errors/ValidException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.sink.influx.errors; 2 | 3 | import org.raystack.dagger.core.exception.InfluxWriteException; 4 | import org.influxdb.dto.Point; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.io.IOException; 9 | 10 | /** 11 | * The Valid exception. 12 | */ 13 | public class ValidException implements InfluxError { 14 | private static final Logger LOGGER = LoggerFactory.getLogger(ValidException.class.getName()); 15 | private IOException exception; 16 | 17 | @Override 18 | public boolean hasException() { 19 | return true; 20 | } 21 | 22 | @Override 23 | public IOException getCurrentException() { 24 | return exception; 25 | } 26 | 27 | @Override 28 | public boolean filterError(Throwable throwable) { 29 | return throwable instanceof Exception; 30 | } 31 | 32 | @Override 33 | public void handle(Iterable points, Throwable throwable) { 34 | exception = new InfluxWriteException(throwable); 35 | logFailedPoints(points, LOGGER); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/sink/kafka/KafkaSerializationSchemaFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.sink.kafka; 2 | 3 | import org.raystack.dagger.common.configuration.Configuration; 4 | import org.raystack.dagger.common.core.StencilClientOrchestrator; 5 | import org.raystack.dagger.common.serde.DataTypes; 6 | import org.raystack.dagger.core.sink.kafka.builder.KafkaJsonSerializerBuilder; 7 | import org.raystack.dagger.core.sink.kafka.builder.KafkaProtoSerializerBuilder; 8 | import org.raystack.dagger.core.utils.Constants; 9 | 10 | public class KafkaSerializationSchemaFactory { 11 | public static KafkaSerializerBuilder getSerializationSchema(Configuration configuration, StencilClientOrchestrator stencilClientOrchestrator, String[] columnNames) { 12 | DataTypes dataTypes = DataTypes.valueOf(configuration.getString(Constants.SINK_KAFKA_DATA_TYPE, "PROTO")); 13 | 14 | if (dataTypes == DataTypes.JSON) { 15 | return new KafkaJsonSerializerBuilder(configuration); 16 | } 17 | return new KafkaProtoSerializerBuilder(configuration, stencilClientOrchestrator, columnNames); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/sink/kafka/KafkaSerializerBuilder.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.sink.kafka; 2 | 3 | import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema; 4 | 5 | public interface KafkaSerializerBuilder { 6 | KafkaRecordSerializationSchema build(); 7 | } 8 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/sink/log/LogSinkWriter.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.sink.log; 2 | 3 | import org.apache.flink.api.connector.sink.SinkWriter; 4 | import org.apache.flink.types.Row; 5 | 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.util.HashMap; 10 | import java.util.List; 11 | import java.util.Map; 12 | 13 | public class LogSinkWriter implements SinkWriter { 14 | private static final Logger LOGGER = LoggerFactory.getLogger(LogSinkWriter.class.getName()); 15 | private final String[] columnNames; 16 | 17 | public LogSinkWriter(String[] columnNames) { 18 | this.columnNames = columnNames; 19 | } 20 | 21 | @Override 22 | public void write(Row row, Context context) { 23 | Map map = new HashMap<>(); 24 | for (int i = 0; i < columnNames.length; i++) { 25 | Object field = row.getField(i); 26 | if (field != null) { 27 | map.put(columnNames[i], field.toString()); 28 | } 29 | } 30 | LOGGER.info(map.toString()); 31 | } 32 | 33 | @Override 34 | public List prepareCommit(boolean flush) { 35 | return null; 36 | } 37 | 38 | @Override 39 | public void close() throws Exception { 40 | 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/source/DaggerSource.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.source; 2 | 3 | import org.apache.flink.api.common.eventtime.WatermarkStrategy; 4 | import org.apache.flink.streaming.api.datastream.DataStream; 5 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 6 | 7 | public interface DaggerSource { 8 | DataStream register(StreamExecutionEnvironment executionEnvironment, WatermarkStrategy watermarkStrategy); 9 | 10 | boolean canBuild(); 11 | } 12 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/source/StreamsFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.source; 2 | 3 | import org.raystack.dagger.common.configuration.Configuration; 4 | import org.raystack.dagger.common.core.StencilClientOrchestrator; 5 | import org.raystack.dagger.core.metrics.reporters.statsd.SerializedStatsDReporterSupplier; 6 | import org.raystack.dagger.core.source.config.StreamConfig; 7 | 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | 11 | public class StreamsFactory { 12 | public static List getStreams(Configuration configuration, StencilClientOrchestrator stencilClientOrchestrator, SerializedStatsDReporterSupplier statsDReporterSupplier) { 13 | StreamConfig[] streamConfigs = StreamConfig.parse(configuration); 14 | ArrayList streams = new ArrayList<>(); 15 | 16 | for (StreamConfig streamConfig : streamConfigs) { 17 | Stream.Builder builder = new Stream.Builder(streamConfig, configuration, stencilClientOrchestrator, statsDReporterSupplier); 18 | streams.add(builder.build()); 19 | } 20 | return streams; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/source/config/adapter/SourceParquetFilePathsAdapter.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.source.config.adapter; 2 | 3 | import com.google.gson.Gson; 4 | import com.google.gson.TypeAdapter; 5 | import com.google.gson.stream.JsonReader; 6 | import com.google.gson.stream.JsonWriter; 7 | 8 | import java.io.IOException; 9 | import java.util.Arrays; 10 | 11 | public class SourceParquetFilePathsAdapter extends TypeAdapter { 12 | @Override 13 | public void write(JsonWriter jsonWriter, String[] strings) { 14 | } 15 | 16 | @Override 17 | public String[] read(JsonReader jsonReader) throws IOException { 18 | Gson gson = new Gson(); 19 | String[] filePathArray = gson.fromJson(jsonReader, String[].class); 20 | return Arrays.stream(filePathArray) 21 | .map(String::valueOf) 22 | .map(String::trim).toArray(String[]::new); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/source/config/models/SourceDetails.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.source.config.models; 2 | 3 | import com.google.gson.annotations.SerializedName; 4 | import lombok.Getter; 5 | 6 | import java.io.Serializable; 7 | 8 | import static org.raystack.dagger.core.utils.Constants.STREAM_SOURCE_DETAILS_SOURCE_NAME_KEY; 9 | import static org.raystack.dagger.core.utils.Constants.STREAM_SOURCE_DETAILS_SOURCE_TYPE_KEY; 10 | 11 | public class SourceDetails implements Serializable { 12 | @SerializedName(STREAM_SOURCE_DETAILS_SOURCE_NAME_KEY) 13 | @Getter 14 | private SourceName sourceName; 15 | 16 | @SerializedName(STREAM_SOURCE_DETAILS_SOURCE_TYPE_KEY) 17 | @Getter 18 | private SourceType sourceType; 19 | 20 | public SourceDetails(SourceName sourceName, SourceType sourceType) { 21 | this.sourceName = sourceName; 22 | this.sourceType = sourceType; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/source/config/models/SourceName.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.source.config.models; 2 | 3 | import com.google.gson.annotations.SerializedName; 4 | 5 | import static org.raystack.dagger.core.utils.Constants.STREAM_SOURCE_DETAILS_SOURCE_NAME_KAFKA_CONSUMER; 6 | import static org.raystack.dagger.core.utils.Constants.STREAM_SOURCE_DETAILS_SOURCE_NAME_KAFKA; 7 | import static org.raystack.dagger.core.utils.Constants.STREAM_SOURCE_DETAILS_SOURCE_NAME_PARQUET; 8 | 9 | public enum SourceName { 10 | @SerializedName(STREAM_SOURCE_DETAILS_SOURCE_NAME_KAFKA) 11 | KAFKA_SOURCE, 12 | @SerializedName(STREAM_SOURCE_DETAILS_SOURCE_NAME_PARQUET) 13 | PARQUET_SOURCE, 14 | @SerializedName(STREAM_SOURCE_DETAILS_SOURCE_NAME_KAFKA_CONSUMER) 15 | KAFKA_CONSUMER 16 | } 17 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/source/config/models/SourceType.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.source.config.models; 2 | 3 | import com.google.gson.annotations.SerializedName; 4 | 5 | import static org.raystack.dagger.core.utils.Constants.STREAM_SOURCE_DETAILS_SOURCE_TYPE_BOUNDED; 6 | import static org.raystack.dagger.core.utils.Constants.STREAM_SOURCE_DETAILS_SOURCE_TYPE_UNBOUNDED; 7 | 8 | public enum SourceType { 9 | @SerializedName(STREAM_SOURCE_DETAILS_SOURCE_TYPE_BOUNDED) 10 | BOUNDED, 11 | @SerializedName(STREAM_SOURCE_DETAILS_SOURCE_TYPE_UNBOUNDED) 12 | UNBOUNDED 13 | } 14 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/source/config/models/TimeRange.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.source.config.models; 2 | 3 | import lombok.Getter; 4 | 5 | import java.io.Serializable; 6 | import java.time.Instant; 7 | 8 | @Getter 9 | public class TimeRange implements Serializable { 10 | public TimeRange(Instant startInstant, Instant endInstant) { 11 | this.startInstant = startInstant; 12 | this.endInstant = endInstant; 13 | } 14 | 15 | private Instant startInstant; 16 | 17 | private Instant endInstant; 18 | 19 | public boolean contains(Instant instant) { 20 | return instant.equals(startInstant) || instant.equals(endInstant) || (instant.isAfter(startInstant) && instant.isBefore(endInstant)); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/source/config/models/TimeRangePool.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.source.config.models; 2 | 3 | import lombok.Getter; 4 | 5 | import java.io.Serializable; 6 | import java.time.Instant; 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | 10 | public class TimeRangePool implements Serializable { 11 | public TimeRangePool() { 12 | this.timeRanges = new ArrayList<>(); 13 | } 14 | 15 | @Getter 16 | private List timeRanges; 17 | 18 | public boolean add(TimeRange timeRange) { 19 | return timeRanges.add(timeRange); 20 | } 21 | 22 | public boolean contains(Instant instant) { 23 | return timeRanges.stream().anyMatch(timeRange -> timeRange.contains(instant)); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/source/parquet/SourceParquetReadOrderStrategy.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.source.parquet; 2 | 3 | import com.google.gson.annotations.SerializedName; 4 | 5 | import static org.raystack.dagger.core.utils.Constants.STREAM_SOURCE_PARQUET_READ_ORDER_STRATEGY_EARLIEST_INDEX_FIRST; 6 | import static org.raystack.dagger.core.utils.Constants.STREAM_SOURCE_PARQUET_READ_ORDER_STRATEGY_EARLIEST_TIME_URL_FIRST; 7 | 8 | public enum SourceParquetReadOrderStrategy { 9 | @SerializedName(STREAM_SOURCE_PARQUET_READ_ORDER_STRATEGY_EARLIEST_TIME_URL_FIRST) 10 | EARLIEST_TIME_URL_FIRST, 11 | @SerializedName(STREAM_SOURCE_PARQUET_READ_ORDER_STRATEGY_EARLIEST_INDEX_FIRST) 12 | EARLIEST_INDEX_FIRST 13 | } 14 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/source/parquet/SourceParquetSchemaMatchStrategy.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.source.parquet; 2 | 3 | import com.google.gson.annotations.SerializedName; 4 | 5 | import static org.raystack.dagger.core.utils.Constants.STREAM_SOURCE_PARQUET_BACKWARD_COMPATIBLE_SCHEMA_MATCH_STRATEGY; 6 | import static org.raystack.dagger.core.utils.Constants.STREAM_SOURCE_PARQUET_SAME_SCHEMA_MATCH_STRATEGY; 7 | 8 | public enum SourceParquetSchemaMatchStrategy { 9 | @SerializedName(STREAM_SOURCE_PARQUET_SAME_SCHEMA_MATCH_STRATEGY) 10 | SAME_SCHEMA_WITH_FAIL_ON_MISMATCH, 11 | @SerializedName(STREAM_SOURCE_PARQUET_BACKWARD_COMPATIBLE_SCHEMA_MATCH_STRATEGY) 12 | BACKWARD_COMPATIBLE_SCHEMA_WITH_FAIL_ON_TYPE_MISMATCH 13 | } 14 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/source/parquet/path/PathParser.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.source.parquet.path; 2 | 3 | import org.apache.flink.core.fs.Path; 4 | 5 | import java.text.ParseException; 6 | import java.time.Instant; 7 | 8 | public interface PathParser { 9 | 10 | Instant instantFromFilePath(Path path) throws ParseException; 11 | } 12 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/source/parquet/reader/ReaderProvider.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.source.parquet.reader; 2 | 3 | import org.apache.flink.connector.file.src.reader.FileRecordFormat; 4 | import org.apache.flink.types.Row; 5 | 6 | import java.io.Serializable; 7 | 8 | @FunctionalInterface 9 | public interface ReaderProvider extends Serializable { 10 | FileRecordFormat.Reader getReader(String filePath); 11 | } 12 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/source/parquet/splitassigner/IndexOrderedSplitAssigner.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.source.parquet.splitassigner; 2 | 3 | import org.apache.flink.connector.file.src.FileSourceSplit; 4 | import org.apache.flink.connector.file.src.assigners.FileSplitAssigner; 5 | 6 | import javax.annotation.Nullable; 7 | import java.util.Collection; 8 | import java.util.Optional; 9 | 10 | /* TODO */ 11 | public class IndexOrderedSplitAssigner implements FileSplitAssigner { 12 | 13 | public IndexOrderedSplitAssigner(Collection fileSourceSplits) { 14 | } 15 | 16 | @Override 17 | public Optional getNext(@Nullable String hostname) { 18 | return Optional.empty(); 19 | } 20 | 21 | @Override 22 | public void addSplits(Collection splits) { 23 | 24 | } 25 | 26 | @Override 27 | public Collection remainingSplits() { 28 | return null; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /dagger-core/src/main/java/org/raystack/dagger/core/source/parquet/splitassigner/InstantEnrichedSplit.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.source.parquet.splitassigner; 2 | 3 | import lombok.Getter; 4 | import org.apache.flink.connector.file.src.FileSourceSplit; 5 | 6 | import java.io.Serializable; 7 | import java.time.Instant; 8 | 9 | public class InstantEnrichedSplit implements Serializable { 10 | @Getter 11 | private final FileSourceSplit fileSourceSplit; 12 | @Getter 13 | private final Instant instant; 14 | 15 | public InstantEnrichedSplit(FileSourceSplit fileSourceSplit, Instant instant) { 16 | this.fileSourceSplit = fileSourceSplit; 17 | this.instant = instant; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /dagger-core/src/main/resources/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | google.cloud.auth.service.account.enable 4 | true 5 | 6 | 7 | google.cloud.auth.service.account.json.keyfile 8 | absolute/path/to/key-file.json 9 | 10 | 11 | fs.gs.requester.pays.mode 12 | CUSTOM 13 | true 14 | 15 | 16 | fs.gs.requester.pays.buckets 17 | sample-bucket 18 | true 19 | 20 | 21 | fs.gs.requester.pays.project.id 22 | sample-project-id 23 | true 24 | 25 | -------------------------------------------------------------------------------- /dagger-core/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, console 2 | log4j.logger.org.apache.flink.runtime.checkpoint.CheckpointCoordinator=WARN 3 | log4j.logger.org.apache.flink.runtime.state.DefaultOperatorStateBackend=WARN 4 | log4j.logger.org.apache.flink.runtime.state.heap.HeapKeyedStateBackend=WARN 5 | log4j.appender.console=org.apache.log4j.ConsoleAppender 6 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n 8 | -------------------------------------------------------------------------------- /dagger-core/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %date{"yyyy-MM-dd'T'HH:mm:ss,SSSXXX", UTC} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /dagger-core/src/test/java/org/raystack/dagger/core/config/system/EnvironmentConfigurationProviderTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.config.system; 2 | 3 | import org.raystack.dagger.common.configuration.Configuration; 4 | import org.raystack.dagger.core.config.EnvironmentConfigurationProvider; 5 | import org.junit.Test; 6 | import org.junit.runner.RunWith; 7 | import org.mockito.junit.MockitoJUnitRunner; 8 | 9 | import java.util.HashMap; 10 | 11 | import static org.junit.Assert.assertEquals; 12 | 13 | @RunWith(MockitoJUnitRunner.class) 14 | public class EnvironmentConfigurationProviderTest { 15 | 16 | @Test 17 | public void shouldProvideSystemConfiguration() { 18 | HashMap environmentParameters = new HashMap() {{ 19 | put("key", "value"); 20 | put("key2", "value2"); 21 | }}; 22 | 23 | Configuration configuration = new EnvironmentConfigurationProvider(environmentParameters).get(); 24 | assertEquals("value", configuration.getString("key", "")); 25 | assertEquals("value2", configuration.getString("key2", "")); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /dagger-core/src/test/java/org/raystack/dagger/core/config/system/FileConfigurationProvideTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.config.system; 2 | 3 | import org.raystack.dagger.common.configuration.Configuration; 4 | import org.raystack.dagger.core.config.FileConfigurationProvider; 5 | import org.raystack.dagger.core.exception.DaggerConfigurationException; 6 | import org.junit.Test; 7 | 8 | import static org.junit.Assert.assertEquals; 9 | import static org.junit.Assert.assertThrows; 10 | 11 | public class FileConfigurationProvideTest { 12 | 13 | @Test 14 | public void readFromAConfigurationFile() { 15 | 16 | System.setProperty("DAGGER_CONFIG_PATH", "env/local.properties"); 17 | Configuration configuration = new FileConfigurationProvider().get(); 18 | assertEquals("1", configuration.getString("FLINK_PARALLELISM", "1")); 19 | } 20 | 21 | @Test 22 | public void shouldThrowExceptionForFileNotfound() { 23 | System.setProperty("DAGGER_CONFIG_PATH", "dd"); 24 | DaggerConfigurationException exception = assertThrows(DaggerConfigurationException.class, 25 | () -> new FileConfigurationProvider().get()); 26 | assertEquals("Config source not provided", exception.getMessage()); 27 | 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /dagger-core/src/test/java/org/raystack/dagger/core/processors/common/InitializationDecoratorTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.common; 2 | 3 | import org.raystack.dagger.core.processors.ColumnNameManager; 4 | import org.apache.flink.types.Row; 5 | import org.junit.Test; 6 | 7 | import java.util.ArrayList; 8 | import java.util.Arrays; 9 | 10 | import static org.junit.Assert.assertEquals; 11 | import static org.junit.Assert.assertFalse; 12 | 13 | public class InitializationDecoratorTest { 14 | 15 | @Test 16 | public void canDecorateShouldBeFalse() { 17 | InitializationDecorator initializationDecorator = new InitializationDecorator(new ColumnNameManager(new String[0], new ArrayList<>())); 18 | assertFalse(initializationDecorator.canDecorate()); 19 | } 20 | 21 | @Test 22 | public void shouldMapInputAndEmptyOutputRowInStream() { 23 | InitializationDecorator initializationDecorator = new InitializationDecorator(new ColumnNameManager(new String[5], Arrays.asList("one", "two", "three"))); 24 | 25 | Row inputData = new Row(5); 26 | Row actualRow = initializationDecorator.map(inputData); 27 | assertEquals(2, actualRow.getArity()); 28 | assertEquals(inputData, actualRow.getField(0)); 29 | Row outputRow = ((Row) actualRow.getField(1)); 30 | assertEquals(3, outputRow.getArity()); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /dagger-core/src/test/java/org/raystack/dagger/core/processors/longbow/LongbowReadColumnModifierTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow; 2 | 3 | import org.raystack.dagger.core.processors.longbow.columnmodifier.LongbowReadColumnModifier; 4 | import org.junit.Test; 5 | 6 | import static org.junit.Assert.*; 7 | 8 | public class LongbowReadColumnModifierTest { 9 | @Test 10 | public void shouldReturnProtoDataWhenEmptyInputColumnNames() { 11 | LongbowReadColumnModifier longbowReadColumnModifier = new LongbowReadColumnModifier(); 12 | String[] inputColumnNames = {}; 13 | String[] outputColumnNames = longbowReadColumnModifier.modifyColumnNames(inputColumnNames); 14 | String[] expected = {"proto_data"}; 15 | assertArrayEquals(expected, outputColumnNames); 16 | } 17 | 18 | @Test 19 | public void shouldAddProtoColumnNames() { 20 | LongbowReadColumnModifier longbowReadColumnModifier = new LongbowReadColumnModifier(); 21 | String[] inputColumnNames = {"hello", "world"}; 22 | String[] outputColumnNames = longbowReadColumnModifier.modifyColumnNames(inputColumnNames); 23 | String[] expected = {"hello", "world", "proto_data"}; 24 | assertArrayEquals(expected, outputColumnNames); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /dagger-core/src/test/java/org/raystack/dagger/core/processors/longbow/request/ProtoByteScanRequestTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.longbow.request; 2 | 3 | import org.apache.hadoop.hbase.client.Scan; 4 | import org.apache.hadoop.hbase.util.Bytes; 5 | import org.junit.Before; 6 | import org.junit.Test; 7 | 8 | import static org.junit.Assert.*; 9 | import static org.mockito.MockitoAnnotations.initMocks; 10 | 11 | public class ProtoByteScanRequestTest { 12 | 13 | private byte[] startRow; 14 | private byte[] endRow; 15 | private String tableId = "tableId"; 16 | 17 | @Before 18 | public void setup() { 19 | initMocks(this); 20 | startRow = Bytes.toBytes("startRow"); 21 | endRow = Bytes.toBytes("endRow"); 22 | } 23 | 24 | @Test 25 | public void shouldCreateProtoScanRequest() { 26 | ProtoByteScanRequest protoByteScanRequest = new ProtoByteScanRequest(startRow, endRow, tableId); 27 | Scan expectedScan = new Scan(); 28 | expectedScan.withStartRow(startRow, true); 29 | expectedScan.withStopRow(endRow, true); 30 | expectedScan.addColumn(Bytes.toBytes("ts"), Bytes.toBytes("proto")); 31 | assertEquals(expectedScan.getFamilyMap(), protoByteScanRequest.get().getFamilyMap()); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /dagger-core/src/test/java/org/raystack/dagger/core/processors/transformers/MockTransformer.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.processors.transformers; 2 | 3 | import org.raystack.dagger.common.core.DaggerContext; 4 | import org.apache.flink.api.common.functions.MapFunction; 5 | import org.apache.flink.streaming.api.datastream.DataStream; 6 | import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 7 | import org.apache.flink.types.Row; 8 | 9 | import org.raystack.dagger.common.core.StreamInfo; 10 | import org.raystack.dagger.common.core.Transformer; 11 | 12 | import java.util.Map; 13 | 14 | public class MockTransformer implements Transformer, MapFunction { 15 | public MockTransformer(Map transformationArguments, String[] columnNames, DaggerContext daggerContext) { 16 | } 17 | 18 | @Override 19 | public StreamInfo transform(StreamInfo inputStreamInfo) { 20 | DataStream inputStream = inputStreamInfo.getDataStream(); 21 | SingleOutputStreamOperator outputStream = inputStream.map(this); 22 | return new StreamInfo(outputStream, inputStreamInfo.getColumnNames()); 23 | } 24 | 25 | @Override 26 | public Row map(Row value) throws Exception { 27 | return value; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /dagger-core/src/test/java/org/raystack/dagger/core/sink/bigquery/BigQuerySinkBuilderTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.sink.bigquery; 2 | 3 | import org.raystack.dagger.common.configuration.Configuration; 4 | import org.raystack.dagger.common.core.StencilClientOrchestrator; 5 | import org.apache.flink.api.java.utils.ParameterTool; 6 | import org.junit.Assert; 7 | import org.junit.Test; 8 | import org.mockito.Mockito; 9 | 10 | import java.util.HashMap; 11 | 12 | 13 | public class BigQuerySinkBuilderTest { 14 | 15 | @Test 16 | public void shouldBuildBigQuerySink() { 17 | StencilClientOrchestrator stencilClientOrchestrator = Mockito.mock(StencilClientOrchestrator.class); 18 | BigQuerySinkBuilder builder = BigQuerySinkBuilder.create(); 19 | builder.setColumnNames(new String[]{"test", "some_column"}); 20 | builder.setConfiguration(new Configuration(ParameterTool.fromMap(new HashMap() {{ 21 | put("SINK_CONNECTOR_SCHEMA_PROTO_MESSAGE_CLASS", "test"); 22 | }}))); 23 | builder.setStencilClientOrchestrator(stencilClientOrchestrator); 24 | Assert.assertNotNull(builder.build()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /dagger-core/src/test/java/org/raystack/dagger/core/sink/influx/errors/NoErrorTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.sink.influx.errors; 2 | 3 | import org.junit.Test; 4 | 5 | import static org.junit.Assert.assertFalse; 6 | import static org.junit.Assert.assertNull; 7 | 8 | public class NoErrorTest { 9 | 10 | @Test 11 | public void shouldNotReturnAnyError() { 12 | NoError noError = new NoError(); 13 | assertNull(noError.getCurrentException()); 14 | } 15 | 16 | @Test 17 | public void shouldHaveNoError() { 18 | NoError noError = new NoError(); 19 | assertFalse(noError.hasException()); 20 | } 21 | 22 | @Test 23 | public void shouldFilterAllErrors() { 24 | NoError noError = new NoError(); 25 | 26 | assertFalse(noError.filterError(new Throwable())); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /dagger-core/src/test/java/org/raystack/dagger/core/sink/influx/errors/ValidErrorTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.core.sink.influx.errors; 2 | 3 | import org.influxdb.dto.Point; 4 | import org.junit.Test; 5 | 6 | import java.util.Collections; 7 | 8 | import static org.junit.Assert.*; 9 | 10 | public class ValidErrorTest { 11 | 12 | @Test 13 | public void shouldHaveError() { 14 | ValidError validError = new ValidError(); 15 | assertTrue(validError.hasException()); 16 | } 17 | 18 | @Test 19 | public void shouldFilterOnlyError() { 20 | ValidError validError = new ValidError(); 21 | 22 | assertTrue(validError.filterError(new Error())); 23 | } 24 | 25 | @Test 26 | public void shouldNotFilterException() { 27 | ValidError validError = new ValidError(); 28 | 29 | assertFalse(validError.filterError(new Exception())); 30 | } 31 | 32 | @Test 33 | public void shouldWrapErrorsInExceptions() { 34 | Iterable points = Collections.emptyList(); 35 | ValidError validError = new ValidError(); 36 | validError.handle(points, new Error("Test")); 37 | Exception currentException = validError.getCurrentException(); 38 | assertTrue(currentException instanceof Exception); 39 | assertEquals("java.lang.Error: Test", currentException.getMessage()); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /dagger-core/src/test/resources/binary/broken-message: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/dagger-core/src/test/resources/binary/broken-message -------------------------------------------------------------------------------- /dagger-core/src/test/resources/multiple_row_groups_test_file.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/dagger-core/src/test/resources/multiple_row_groups_test_file.parquet -------------------------------------------------------------------------------- /dagger-core/src/test/resources/test_file.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/dagger-core/src/test/resources/test_file.parquet -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/exceptions/ArrayAggregationException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.exceptions; 2 | 3 | /** 4 | * The Exception class for ArrayAggregate udf. 5 | */ 6 | public class ArrayAggregationException extends RuntimeException { 7 | /** 8 | * Instantiates a new Array aggregation exception. 9 | * 10 | * @param message the message 11 | */ 12 | public ArrayAggregationException(String message) { 13 | super(message); 14 | } 15 | 16 | /** 17 | * Instantiates a new Array aggregation exception with innerException. 18 | * 19 | * @param innerException the inner exception 20 | */ 21 | public ArrayAggregationException(Exception innerException) { 22 | super(innerException); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/exceptions/ArrayOperateException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.exceptions; 2 | 3 | /** 4 | * The Exception class for ArrayOperate udf. 5 | */ 6 | public class ArrayOperateException extends RuntimeException { 7 | 8 | /** 9 | * Instantiates a new Array operate exception. 10 | * 11 | * @param message the message 12 | */ 13 | public ArrayOperateException(String message) { 14 | super(message); 15 | } 16 | 17 | /** 18 | * Instantiates a new Array operate exception. 19 | * 20 | * @param innerException the inner exception 21 | */ 22 | public ArrayOperateException(Exception innerException) { 23 | super(innerException); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/exceptions/BucketDoesNotExistException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.exceptions; 2 | 3 | /** 4 | * The class Exception if Bucket does not exist. 5 | */ 6 | public class BucketDoesNotExistException extends RuntimeException { 7 | /** 8 | * Instantiates a new Bucket does not exist exception. 9 | * 10 | * @param message the message 11 | */ 12 | public BucketDoesNotExistException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/exceptions/InvalidHashFieldException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.exceptions; 2 | 3 | /** 4 | * The class Exception for Invalid hash field. 5 | */ 6 | public class InvalidHashFieldException extends RuntimeException { 7 | /** 8 | * Instantiates a new Invalid hash field exception. 9 | * 10 | * @param message the message 11 | */ 12 | public InvalidHashFieldException(String message) { 13 | super(message); 14 | } 15 | 16 | /** 17 | * Instantiates a new Invalid hash field exception. 18 | * 19 | * @param innerException the inner exception 20 | */ 21 | public InvalidHashFieldException(Exception innerException) { 22 | super(innerException); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/exceptions/InvalidNumberOfArgumentsException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.exceptions; 2 | 3 | /** 4 | * The class Exception for Invalid number of arguments. 5 | */ 6 | public class InvalidNumberOfArgumentsException extends RuntimeException { 7 | private static final String DEFAULT_ERROR_MESSAGE = "Invalid number of arguments given to Udf. Requires arguments that is divisible by 3."; 8 | 9 | /** 10 | * Instantiates a new Invalid number of arguments exception. 11 | */ 12 | public InvalidNumberOfArgumentsException() { 13 | super(DEFAULT_ERROR_MESSAGE); 14 | } 15 | 16 | /** 17 | * Instantiates a new Invalid number of arguments exception. 18 | * 19 | * @param message the message 20 | */ 21 | public InvalidNumberOfArgumentsException(String message) { 22 | super(message); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/exceptions/KeyDoesNotExistException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.exceptions; 2 | 3 | /** 4 | * The class Exception if Key does not exist. 5 | */ 6 | public class KeyDoesNotExistException extends RuntimeException { 7 | /** 8 | * Instantiates a new Key does not exist exception. 9 | * 10 | * @param message the message 11 | */ 12 | public KeyDoesNotExistException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/exceptions/LongbowException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.exceptions; 2 | 3 | /** 4 | * The Exception class for Longbow. 5 | */ 6 | public class LongbowException extends RuntimeException { 7 | 8 | /** 9 | * Instantiates a new Longbow exception. 10 | * 11 | * @param message the message 12 | */ 13 | public LongbowException(String message) { 14 | super(message); 15 | } 16 | } 17 | 18 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/exceptions/MadZeroException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.exceptions; 2 | 3 | /** 4 | * The class Exception if Mad is zero on OutlierMad Udf. 5 | */ 6 | public class MadZeroException extends RuntimeException { 7 | /** 8 | * Instantiates a new Mad zero exception. 9 | * 10 | * @param message the message 11 | */ 12 | public MadZeroException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/exceptions/MedianNotFound.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.exceptions; 2 | 3 | /** 4 | * The class Exception if Median not found on OutlierMad Udf. 5 | */ 6 | public class MedianNotFound extends RuntimeException { 7 | /** 8 | * Instantiates a new Median not found. 9 | * 10 | * @param message the message 11 | */ 12 | public MedianNotFound(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/exceptions/OddNumberOfArgumentsException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.exceptions; 2 | 3 | /** 4 | * The class Exception for Odd number of arguments on Features Udf. 5 | */ 6 | public class OddNumberOfArgumentsException extends RuntimeException { 7 | 8 | private static final String DEFAULT_ERROR_MESSAGE = "Odd number of arguments given to Udf. Requires even."; 9 | 10 | /** 11 | * Instantiates a new Odd number of arguments exception. 12 | */ 13 | public OddNumberOfArgumentsException() { 14 | super(DEFAULT_ERROR_MESSAGE); 15 | } 16 | 17 | /** 18 | * Instantiates a new Odd number of arguments exception. 19 | * 20 | * @param message the message 21 | */ 22 | public OddNumberOfArgumentsException(String message) { 23 | super(message); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/exceptions/PythonFilesEmptyException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.exceptions; 2 | 3 | /** 4 | * The type Python files empty exception. 5 | */ 6 | public class PythonFilesEmptyException extends RuntimeException { 7 | 8 | /** 9 | * Instantiates a new Python files empty exception. 10 | * 11 | * @param message the message 12 | */ 13 | public PythonFilesEmptyException(String message) { 14 | super(message); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/exceptions/PythonFilesFormatException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.exceptions; 2 | 3 | /** 4 | * The type Python files format exception. 5 | */ 6 | public class PythonFilesFormatException extends RuntimeException { 7 | 8 | /** 9 | * Instantiates a new Python files format exception. 10 | * 11 | * @param message the message 12 | */ 13 | public PythonFilesFormatException(String message) { 14 | super(message); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/exceptions/RowHashException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.exceptions; 2 | 3 | /** 4 | * The class Exception if failed on hashing the Row. 5 | */ 6 | public class RowHashException extends RuntimeException { 7 | /** 8 | * Instantiates a new Row hash exception. 9 | * 10 | * @param message the message 11 | */ 12 | public RowHashException(String message) { 13 | super(message); 14 | } 15 | 16 | /** 17 | * Instantiates a new Row hash exception. 18 | * 19 | * @param innerException the inner exception 20 | */ 21 | public RowHashException(Exception innerException) { 22 | super(innerException); 23 | } 24 | 25 | /** 26 | * Instantiates a new Row hash exception. 27 | * 28 | * @param message the message 29 | * @param innerException the inner exception 30 | */ 31 | public RowHashException(String message, Exception innerException) { 32 | super(message, innerException); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/exceptions/TagDoesNotExistException.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.exceptions; 2 | 3 | /** 4 | * The class Exception if Tag does not exist. 5 | */ 6 | public class TagDoesNotExistException extends RuntimeException { 7 | /** 8 | * Instantiates a new Tag does not exist exception. 9 | * 10 | * @param message the message 11 | */ 12 | public TagDoesNotExistException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/transformers/filter/FilterAspects.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.transformers.filter; 2 | 3 | import org.raystack.dagger.common.metrics.aspects.AspectType; 4 | import org.raystack.dagger.common.metrics.aspects.Aspects; 5 | 6 | /** 7 | * The enum Filter aspects. 8 | */ 9 | public enum FilterAspects implements Aspects { 10 | /** 11 | * Filtered invalid records filter aspects. 12 | */ 13 | FILTERED_INVALID_RECORDS("filtered_invalid_records", AspectType.Counter); 14 | 15 | FilterAspects(String value, AspectType type) { 16 | this.value = value; 17 | this.type = type; 18 | } 19 | 20 | private String value; 21 | private AspectType type; 22 | 23 | @Override 24 | public String getValue() { 25 | return this.value; 26 | } 27 | 28 | @Override 29 | public AspectType getAspectType() { 30 | return this.type; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/aggregate/accumulator/ArrayAccumulator.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.accumulator; 2 | 3 | import org.apache.flink.table.annotation.DataTypeHint; 4 | 5 | import java.io.Serializable; 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | /** 10 | * The accumulator for CollectArray udf. 11 | */ 12 | public class ArrayAccumulator implements Serializable { 13 | 14 | private @DataTypeHint("RAW") List arrayList = new ArrayList<>(); 15 | 16 | /** 17 | * Add object to array list. 18 | * 19 | * @param object the object 20 | */ 21 | public void add(Object object) { 22 | arrayList.add(object); 23 | } 24 | 25 | /** 26 | * Emit array list. 27 | * 28 | * @return the array list 29 | */ 30 | public List emit() { 31 | return arrayList; 32 | } 33 | 34 | public List getArrayList() { 35 | return arrayList; 36 | } 37 | 38 | public void setArrayList(List arrayList) { 39 | this.arrayList = arrayList; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/aggregate/accumulator/DistinctCountAccumulator.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.accumulator; 2 | 3 | import org.apache.flink.table.annotation.DataTypeHint; 4 | 5 | import java.io.Serializable; 6 | import java.util.HashSet; 7 | 8 | /** 9 | * The accumulator for DistinctCount udf. 10 | */ 11 | public class DistinctCountAccumulator implements Serializable { 12 | 13 | private @DataTypeHint("RAW") HashSet distinctItems = new HashSet<>(); 14 | 15 | /** 16 | * Count size of the distinct items. 17 | * 18 | * @return the size of distinct items 19 | */ 20 | public int count() { 21 | return distinctItems.size(); 22 | } 23 | 24 | /** 25 | * Add item. 26 | * 27 | * @param item the item 28 | */ 29 | public void add(String item) { 30 | distinctItems.add(item); 31 | } 32 | 33 | public HashSet getDistinctItems() { 34 | return distinctItems; 35 | } 36 | 37 | public void setDistinctItems(HashSet distinctItems) { 38 | this.distinctItems = distinctItems; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/aggregate/feast/handler/BigDecimalValueTransformer.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.feast.handler; 2 | 3 | import java.math.BigDecimal; 4 | 5 | import static org.raystack.dagger.functions.udfs.aggregate.feast.handler.ValueEnum.DoubleType; 6 | 7 | /** 8 | * The Big decimal value transformer. 9 | */ 10 | public class BigDecimalValueTransformer implements ValueTransformer { 11 | @Override 12 | public boolean canTransform(Object value) { 13 | return value instanceof BigDecimal; 14 | } 15 | 16 | @Override 17 | public boolean canTransformWithTargetType(Object value, ValueEnum targetType) { 18 | return value instanceof BigDecimal && targetType == DoubleType; 19 | } 20 | 21 | @Override 22 | public Integer getIndex() { 23 | return DoubleType.getValue(); 24 | } 25 | 26 | @Override 27 | public Object getValue(Object value) { 28 | if (value == null) { 29 | return 0.0D; 30 | } 31 | BigDecimal bigDecimalValue = (BigDecimal) value; 32 | return bigDecimalValue.doubleValue(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/aggregate/feast/handler/BooleanValueTransformer.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.feast.handler; 2 | 3 | import static org.raystack.dagger.functions.udfs.aggregate.feast.handler.ValueEnum.BooleanType; 4 | 5 | /** 6 | * The Boolean value transformer. 7 | */ 8 | public class BooleanValueTransformer implements ValueTransformer { 9 | @Override 10 | public boolean canTransform(Object value) { 11 | return value instanceof Boolean; 12 | } 13 | 14 | @Override 15 | public boolean canTransformWithTargetType(Object value, ValueEnum targetType) { 16 | return targetType == BooleanType; 17 | } 18 | 19 | @Override 20 | public Integer getIndex() { 21 | return BooleanType.getValue(); 22 | } 23 | 24 | @Override 25 | public Object getValue(Object value) { 26 | return value != null ? value : false; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/aggregate/feast/handler/ByteValueTransformer.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.feast.handler; 2 | 3 | import com.google.protobuf.ByteString; 4 | 5 | import static org.raystack.dagger.functions.udfs.aggregate.feast.handler.ValueEnum.ByteType; 6 | 7 | /** 8 | * The Byte value transformer. 9 | */ 10 | public class ByteValueTransformer implements ValueTransformer { 11 | @Override 12 | public boolean canTransform(Object value) { 13 | return value instanceof ByteString; 14 | } 15 | 16 | @Override 17 | public boolean canTransformWithTargetType(Object value, ValueEnum targetType) { 18 | return targetType == ByteType; 19 | } 20 | 21 | @Override 22 | public Integer getIndex() { 23 | return ByteType.getValue(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/aggregate/feast/handler/DoubleValueTransformer.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.feast.handler; 2 | 3 | import static org.raystack.dagger.functions.udfs.aggregate.feast.handler.ValueEnum.DoubleType; 4 | 5 | /** 6 | * The Double value transformer. 7 | */ 8 | public class DoubleValueTransformer implements ValueTransformer { 9 | 10 | @Override 11 | public boolean canTransform(Object value) { 12 | return value instanceof Double; 13 | } 14 | 15 | @Override 16 | public boolean canTransformWithTargetType(Object value, ValueEnum targetType) { 17 | return targetType == DoubleType; 18 | } 19 | 20 | @Override 21 | public Integer getIndex() { 22 | return DoubleType.getValue(); 23 | } 24 | 25 | @Override 26 | public Object getValue(Object value) { 27 | return value != null ? Double.valueOf(value.toString()) : 0.0d; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/aggregate/feast/handler/FloatValueTransformer.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.feast.handler; 2 | 3 | import static org.raystack.dagger.functions.udfs.aggregate.feast.handler.ValueEnum.FloatType; 4 | 5 | /** 6 | * The Float value transformer. 7 | */ 8 | public class FloatValueTransformer implements ValueTransformer { 9 | @Override 10 | public boolean canTransform(Object value) { 11 | return value instanceof Float; 12 | } 13 | 14 | @Override 15 | public boolean canTransformWithTargetType(Object value, ValueEnum targetType) { 16 | return targetType == FloatType; 17 | } 18 | 19 | @Override 20 | public Integer getIndex() { 21 | return FloatType.getValue(); 22 | } 23 | 24 | @Override 25 | public Object getValue(Object value) { 26 | return value != null ? Float.valueOf(value.toString()) : 0.0f; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/aggregate/feast/handler/IntegerValueTransformer.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.feast.handler; 2 | 3 | import static org.raystack.dagger.functions.udfs.aggregate.feast.handler.ValueEnum.IntegerType; 4 | 5 | /** 6 | * The Integer value transformer. 7 | */ 8 | public class IntegerValueTransformer implements ValueTransformer { 9 | 10 | 11 | @Override 12 | public boolean canTransform(Object value) { 13 | return value instanceof Integer; 14 | } 15 | 16 | @Override 17 | public boolean canTransformWithTargetType(Object value, ValueEnum targetType) { 18 | return targetType == IntegerType; 19 | } 20 | 21 | @Override 22 | public Integer getIndex() { 23 | return IntegerType.getValue(); 24 | } 25 | 26 | @Override 27 | public Object getValue(Object value) { 28 | return value != null ? Integer.valueOf(value.toString()) : 0; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/aggregate/feast/handler/LongValueTransformer.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.feast.handler; 2 | 3 | import static org.raystack.dagger.functions.udfs.aggregate.feast.handler.ValueEnum.LongType; 4 | 5 | /** 6 | * The Long value transformer. 7 | */ 8 | public class LongValueTransformer implements ValueTransformer { 9 | 10 | @Override 11 | public boolean canTransform(Object value) { 12 | return value instanceof Long; 13 | } 14 | 15 | @Override 16 | public boolean canTransformWithTargetType(Object value, ValueEnum targetType) { 17 | return targetType == LongType; 18 | } 19 | 20 | @Override 21 | public Integer getIndex() { 22 | return LongType.getValue(); 23 | } 24 | 25 | @Override 26 | public Object getValue(Object value) { 27 | return value != null ? Long.valueOf(value.toString()) : 0L; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/aggregate/feast/handler/NullValueTransformer.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.feast.handler; 2 | 3 | import org.raystack.dagger.functions.common.Constants; 4 | import org.apache.commons.lang3.NotImplementedException; 5 | import org.apache.flink.types.Row; 6 | 7 | /** 8 | * The Null value transformer. 9 | */ 10 | public class NullValueTransformer implements ValueTransformer { 11 | 12 | @Override 13 | public boolean canTransform(Object value) { 14 | return null == value; 15 | } 16 | 17 | @Override 18 | public boolean canTransformWithTargetType(Object value, ValueEnum targetType) { 19 | return null == value; 20 | } 21 | 22 | @Override 23 | public Integer getIndex() { 24 | throw new NotImplementedException("Index for Null Value shouldn't be used"); 25 | } 26 | 27 | @Override 28 | public Row transform(Object value) { 29 | return new Row(Constants.NUMBER_OF_DATA_TYPES_IN_FEATURE_ROW); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/aggregate/feast/handler/StringValueTransformer.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.feast.handler; 2 | 3 | import static org.raystack.dagger.functions.udfs.aggregate.feast.handler.ValueEnum.StringType; 4 | 5 | /** 6 | * The String value transformer. 7 | */ 8 | public class StringValueTransformer implements ValueTransformer { 9 | 10 | @Override 11 | public boolean canTransform(Object value) { 12 | return value instanceof String; 13 | } 14 | 15 | @Override 16 | public boolean canTransformWithTargetType(Object value, ValueEnum targetType) { 17 | return targetType == StringType; 18 | } 19 | 20 | @Override 21 | public Integer getIndex() { 22 | return StringType.getValue(); 23 | } 24 | 25 | @Override 26 | public Object getValue(Object value) { 27 | return value.toString(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/aggregate/feast/handler/TimestampValueTransformer.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.feast.handler; 2 | 3 | import org.apache.flink.types.Row; 4 | 5 | import static org.raystack.dagger.functions.udfs.aggregate.feast.handler.ValueEnum.TimestampType; 6 | 7 | /** 8 | * The Timestamp value transformer. 9 | */ 10 | public class TimestampValueTransformer implements ValueTransformer { 11 | 12 | @Override 13 | public boolean canTransform(Object value) { 14 | return value instanceof Row && ((Row) value).getArity() == 2; 15 | } 16 | 17 | @Override 18 | public boolean canTransformWithTargetType(Object value, ValueEnum targetType) { 19 | return value instanceof Row && ((Row) value).getArity() == 2 && targetType == ValueEnum.TimestampType; 20 | } 21 | 22 | @Override 23 | public Integer getIndex() { 24 | return TimestampType.getValue(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/aggregate/feast/handler/ValueEnum.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.feast.handler; 2 | 3 | /** 4 | * The enum Value. 5 | */ 6 | public enum ValueEnum { 7 | /** 8 | * Boolean type value enum. 9 | */ 10 | BooleanType(6), 11 | /** 12 | * Byte type value enum. 13 | */ 14 | ByteType(0), 15 | /** 16 | * Double type value enum. 17 | */ 18 | DoubleType(4), 19 | /** 20 | * Float type value enum. 21 | */ 22 | FloatType(5), 23 | /** 24 | * Integer type value enum. 25 | */ 26 | IntegerType(2), 27 | /** 28 | * Long type value enum. 29 | */ 30 | LongType(3), 31 | /** 32 | * String type value enum. 33 | */ 34 | StringType(1), 35 | /** 36 | * Timestamp type value enum. 37 | */ 38 | TimestampType(7); 39 | 40 | private Integer value; 41 | 42 | ValueEnum(Integer value) { 43 | this.value = value; 44 | } 45 | 46 | /** 47 | * Gets value. 48 | * 49 | * @return the value 50 | */ 51 | public Integer getValue() { 52 | return value; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/aggregate/feast/handler/ValueTransformerFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.feast.handler; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | /** 7 | * The factory class for Value transformer. 8 | */ 9 | public class ValueTransformerFactory { 10 | /** 11 | * Gets value transformers. 12 | * 13 | * @return the value transformers 14 | */ 15 | public static List getValueTransformers() { 16 | return Arrays.asList(new BooleanValueTransformer(), 17 | new ByteValueTransformer(), 18 | new DoubleValueTransformer(), 19 | new FloatValueTransformer(), 20 | new IntegerValueTransformer(), 21 | new LongValueTransformer(), 22 | new StringValueTransformer(), 23 | new TimestampValueTransformer(), 24 | new NullValueTransformer(), 25 | new BigDecimalValueTransformer()); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/python/file/source/FileSource.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.python.file.source; 2 | 3 | import java.io.IOException; 4 | 5 | /** 6 | * The interface File source. 7 | */ 8 | public interface FileSource { 9 | 10 | /** 11 | * Get object file byte [ ]. 12 | * 13 | * @return the byte [ ] 14 | */ 15 | byte[] getObjectFile() throws IOException; 16 | } 17 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/python/file/source/FileSourceFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.python.file.source; 2 | 3 | import org.raystack.dagger.functions.udfs.python.file.source.gcs.GcsFileSource; 4 | import org.raystack.dagger.functions.udfs.python.file.source.local.LocalFileSource; 5 | 6 | /** 7 | * The type File source factory. 8 | */ 9 | public class FileSourceFactory { 10 | 11 | /** 12 | * Gets file source. 13 | * 14 | * @param pythonFile the python file 15 | * @return the file source 16 | */ 17 | public static FileSource getFileSource(String pythonFile) { 18 | if ("GS".equals(getFileSourcePrefix(pythonFile))) { 19 | return new GcsFileSource(pythonFile); 20 | } else { 21 | return new LocalFileSource(pythonFile); 22 | } 23 | } 24 | 25 | private static String getFileSourcePrefix(String pythonFile) { 26 | String[] files = pythonFile.split("://"); 27 | return files[0].toUpperCase(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/python/file/source/local/LocalFileSource.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.python.file.source.local; 2 | 3 | import org.raystack.dagger.functions.udfs.python.file.source.FileSource; 4 | 5 | import java.io.IOException; 6 | import java.nio.file.Files; 7 | import java.nio.file.Paths; 8 | 9 | /** 10 | * The type Local file source. 11 | */ 12 | public class LocalFileSource implements FileSource { 13 | 14 | private String pythonFile; 15 | 16 | /** 17 | * Instantiates a new Local file source. 18 | * 19 | * @param pythonFile the python file 20 | */ 21 | public LocalFileSource(String pythonFile) { 22 | this.pythonFile = pythonFile; 23 | } 24 | 25 | @Override 26 | public byte[] getObjectFile() throws IOException { 27 | return Files.readAllBytes(Paths.get(pythonFile)); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/python/file/type/FileType.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.python.file.type; 2 | 3 | import java.io.IOException; 4 | import java.util.List; 5 | 6 | /** 7 | * The interface File type. 8 | */ 9 | public interface FileType { 10 | 11 | /** 12 | * Gets file names. 13 | * 14 | * @return the file names 15 | */ 16 | List getFileNames() throws IOException; 17 | } 18 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/python/file/type/FileTypeFactory.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.python.file.type; 2 | 3 | import org.raystack.dagger.functions.exceptions.PythonFilesFormatException; 4 | import org.raystack.dagger.functions.udfs.python.file.source.FileSource; 5 | import org.raystack.dagger.functions.udfs.python.file.source.FileSourceFactory; 6 | 7 | /** 8 | * The type File type factory. 9 | */ 10 | public class FileTypeFactory { 11 | 12 | /** 13 | * Gets file type. 14 | * 15 | * @param pythonFile the python file 16 | * @return the file type 17 | */ 18 | public static FileType getFileType(String pythonFile) { 19 | FileSource fileSource = FileSourceFactory.getFileSource(pythonFile); 20 | switch (getFileTypeFormat(pythonFile)) { 21 | case "PY": 22 | return new PythonFileType(pythonFile); 23 | case "ZIP": 24 | return new ZipFileType(fileSource); 25 | default: 26 | throw new PythonFilesFormatException("Python files should be in .py or .zip format"); 27 | } 28 | } 29 | 30 | private static String getFileTypeFormat(String pythonFile) { 31 | String[] files = pythonFile.split("\\."); 32 | return files[files.length - 1].toUpperCase(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/python/file/type/PythonFileType.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.python.file.type; 2 | 3 | import org.raystack.dagger.functions.exceptions.PythonFilesEmptyException; 4 | 5 | import java.util.Collections; 6 | import java.util.List; 7 | 8 | /** 9 | * The type Python file type. 10 | */ 11 | public class PythonFileType implements FileType { 12 | 13 | private String pythonFile; 14 | 15 | /** 16 | * Instantiates a new Python file type. 17 | * 18 | * @param pythonFile the python file 19 | */ 20 | public PythonFileType(String pythonFile) { 21 | this.pythonFile = pythonFile; 22 | } 23 | 24 | @Override 25 | public List getFileNames() { 26 | if (pythonFile == null) { 27 | throw new PythonFilesEmptyException("Python files can not be null"); 28 | } 29 | String name = pythonFile.substring(pythonFile.lastIndexOf('/') + 1); 30 | 31 | return Collections.singletonList(name); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/scalar/FormatTimeInZone.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar; 2 | 3 | import org.raystack.dagger.common.udfs.ScalarUdf; 4 | 5 | import java.sql.Timestamp; 6 | import java.text.DateFormat; 7 | import java.text.SimpleDateFormat; 8 | import java.util.Date; 9 | import java.util.TimeZone; 10 | 11 | 12 | /** 13 | * The FormatTimeInZone udf. 14 | */ 15 | public class FormatTimeInZone extends ScalarUdf { 16 | /** 17 | * gets Formatted time from timestamp in given timezone. 18 | * 19 | * @param timestamp the timestamp 20 | * @param timeZone the time zone eg. "Asia/Kolkata" 21 | * @param dateFormat time format in which we want the result 22 | * @return the formatted time as string 23 | * @author oshank 24 | * @team GoFood 25 | */ 26 | public String eval(Timestamp timestamp, String timeZone, String dateFormat) { 27 | Date date = new Date(timestamp.getTime()); 28 | DateFormat format = new SimpleDateFormat(dateFormat); 29 | format.setTimeZone(TimeZone.getTimeZone(timeZone)); 30 | return format.format(date); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/scalar/GeoHash.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar; 2 | 3 | import org.raystack.dagger.common.udfs.ScalarUdf; 4 | 5 | import static com.github.davidmoten.geo.GeoHash.encodeHash; 6 | 7 | /** 8 | * The Geo hash udf. 9 | */ 10 | public class GeoHash extends ScalarUdf { 11 | 12 | /** 13 | * Returns a geohash of given length for the given WGS84 point. 14 | * 15 | * @param latitude the latitude 16 | * @param longitude the longitude 17 | * @param length the length 18 | * @return the geohash string 19 | * @author Sumanth 20 | * @team DE 21 | */ 22 | public String eval(Double latitude, Double longitude, Integer length) { 23 | return encodeHash(latitude, longitude, length); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/scalar/JsonDelete.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar; 2 | 3 | import com.jayway.jsonpath.Configuration; 4 | import com.jayway.jsonpath.JsonPath; 5 | import com.jayway.jsonpath.Option; 6 | import com.jayway.jsonpath.PathNotFoundException; 7 | import org.raystack.dagger.common.udfs.ScalarUdf; 8 | import org.apache.flink.table.annotation.DataTypeHint; 9 | 10 | import java.io.Serializable; 11 | 12 | /** 13 | * The Json delete udf. 14 | */ 15 | public class JsonDelete extends ScalarUdf implements Serializable { 16 | 17 | /** 18 | * Deletes JSON node or value from a JSON path. 19 | * The result is always returned as a JSON STRING. 20 | * 21 | * @param jsonEvent the json String 22 | * @param jPath the jPath 23 | * @return jsonString 24 | */ 25 | public @DataTypeHint("STRING") String eval(String jsonEvent, String jPath) throws PathNotFoundException { 26 | Configuration configuration = Configuration.defaultConfiguration().setOptions(Option.DEFAULT_PATH_LEAF_TO_NULL); 27 | return JsonPath.using(configuration).parse(jsonEvent).delete(JsonPath.compile(jPath)).jsonString(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/scalar/JsonUpdate.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar; 2 | 3 | import com.jayway.jsonpath.Configuration; 4 | import com.jayway.jsonpath.JsonPath; 5 | import com.jayway.jsonpath.Option; 6 | import com.jayway.jsonpath.PathNotFoundException; 7 | import org.raystack.dagger.common.udfs.ScalarUdf; 8 | import org.apache.flink.table.annotation.DataTypeHint; 9 | import org.apache.flink.table.annotation.InputGroup; 10 | 11 | import java.io.Serializable; 12 | 13 | /** 14 | * The Json update udf. 15 | */ 16 | public class JsonUpdate extends ScalarUdf implements Serializable { 17 | 18 | /** 19 | * Updates JSON from a JSON path. 20 | * The result is always returned as a JSON STRING. 21 | * 22 | * @param jsonEvent the json String 23 | * @param jPath the jPath 24 | * @param newValue the new object value 25 | * @return jsonString 26 | */ 27 | public @DataTypeHint("STRING") String eval(String jsonEvent, String jPath, @DataTypeHint(inputGroup = InputGroup.ANY) Object newValue) throws PathNotFoundException { 28 | Configuration configuration = Configuration.defaultConfiguration().setOptions(Option.DEFAULT_PATH_LEAF_TO_NULL); 29 | return JsonPath.using(configuration).parse(jsonEvent).set(JsonPath.compile(jPath), newValue).jsonString(); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/scalar/ListContains.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar; 2 | 3 | import org.raystack.dagger.common.udfs.ScalarUdf; 4 | 5 | import java.util.Arrays; 6 | 7 | /** 8 | * The List contains udf. 9 | */ 10 | public class ListContains extends ScalarUdf { 11 | /** 12 | * checks if a list contains a given item. 13 | * 14 | * @param inputList the input list 15 | * @param item the item 16 | * @return true if list has the desired element or false 17 | * @author prakhar.m 18 | * @team DE 19 | */ 20 | public boolean eval(String[] inputList, String item) { 21 | if (inputList == null) { 22 | return false; 23 | } 24 | return Arrays.asList(inputList).contains(item); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/scalar/S2AreaInKm2.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar; 2 | 3 | import com.google.common.geometry.S2Cell; 4 | import com.google.common.geometry.S2CellId; 5 | import org.raystack.dagger.common.udfs.ScalarUdf; 6 | 7 | /** 8 | * The type S2AreaInKm2 udf. 9 | */ 10 | public class S2AreaInKm2 extends ScalarUdf { 11 | 12 | private static final long TOTAL_EARTH_AREA_KM2 = 510072000; 13 | private static final long FACTOR = 4; 14 | 15 | /** 16 | * compute area in km2 for a specific s2 cell. 17 | * 18 | * @param s2id s2id of the cell 19 | * @return area in km2 20 | * @author leyi.seah 21 | * @team DS Marketplace Pricing 22 | */ 23 | public double eval(String s2id) { 24 | long id = Long.parseLong(s2id); 25 | S2Cell s2Cell = getS2CellfromId(id); 26 | return (s2Cell.exactArea() * TOTAL_EARTH_AREA_KM2) / (FACTOR * Math.PI); 27 | } 28 | 29 | private S2Cell getS2CellfromId(long id) { 30 | return new S2Cell(new S2CellId(id)); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/scalar/S2Id.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar; 2 | 3 | import com.google.common.geometry.S2CellId; 4 | import com.google.common.geometry.S2LatLng; 5 | import org.raystack.dagger.common.udfs.ScalarUdf; 6 | 7 | /** 8 | * The type S2id udf. 9 | */ 10 | public class S2Id extends ScalarUdf { 11 | /** 12 | * computes s2id for a given lat, long and level. 13 | * 14 | * @param latitude the latitude 15 | * @param longitude the longitude 16 | * @param level the level of s2 cells 17 | * @return s2id string 18 | * @author gaurav.s 19 | * @team DE 20 | */ 21 | public String eval(Double latitude, Double longitude, Integer level) { 22 | S2LatLng s2LatLng = S2LatLng.fromDegrees(latitude, longitude); 23 | return Long.toString(S2CellId.fromLatLng(s2LatLng).parent(level).id()); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/scalar/Split.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar; 2 | 3 | import org.raystack.dagger.common.udfs.ScalarUdf; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | /** 8 | * The Split udf. 9 | */ 10 | public class Split extends ScalarUdf { 11 | private static final Logger LOGGER = LoggerFactory.getLogger(Split.class.getName()); 12 | 13 | /** 14 | * This UDF returns array of string that has been split from input string. 15 | * 16 | * @param input input string that want to be split 17 | * @param delimiter separator or delimiter to split the input string 18 | * @return array of string that has been split 19 | * @author jesry.pandawa 20 | * @team DE 21 | */ 22 | //Works fine dont need any type hints 23 | public String[] eval(String input, String delimiter) { 24 | if (input == null || delimiter == null) { 25 | LOGGER.info("Not able to split input string, either input string or delimiter is null"); 26 | return new String[0]; 27 | } 28 | if (delimiter.equals("")) { 29 | delimiter = " "; 30 | } 31 | 32 | return input.split(delimiter); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/scalar/TimestampFromUnix.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar; 2 | 3 | import org.raystack.dagger.common.udfs.ScalarUdf; 4 | 5 | import java.sql.Timestamp; 6 | 7 | 8 | /** 9 | * The Timestamp from unix udf. 10 | */ 11 | public class TimestampFromUnix extends ScalarUdf { 12 | 13 | private static final int SECONDS_TO_MILISECONDS = 1000; 14 | 15 | /** 16 | * gets timestamp from UNIX seconds. 17 | * 18 | * @param seconds the seconds 19 | * @return the timestamp 20 | * @author asfarudin 21 | * @team Fraud 22 | */ 23 | public Timestamp eval(Long seconds) { 24 | return new Timestamp(Math.abs(seconds) * SECONDS_TO_MILISECONDS); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/scalar/dart/store/DataStore.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar.dart.store; 2 | 3 | import org.raystack.dagger.functions.udfs.scalar.dart.types.MapCache; 4 | import org.raystack.dagger.functions.udfs.scalar.dart.types.SetCache; 5 | 6 | /** 7 | * The interface Data store. 8 | */ 9 | public interface DataStore { 10 | /** 11 | * Gets set. 12 | * 13 | * @param setName the set name 14 | * @return the set 15 | */ 16 | SetCache getSet(String setName); 17 | 18 | /** 19 | * Gets map. 20 | * 21 | * @param mapName the map name 22 | * @return the map 23 | */ 24 | MapCache getMap(String mapName); 25 | } 26 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/scalar/dart/types/Cache.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar.dart.types; 2 | 3 | import org.apache.commons.lang3.time.DateUtils; 4 | 5 | import java.io.Serializable; 6 | import java.util.Date; 7 | 8 | /** 9 | * The type Cache. 10 | */ 11 | public abstract class Cache implements Serializable { 12 | 13 | private Date timeOfCaching; 14 | 15 | /** 16 | * Instantiates a new Cache. 17 | * 18 | * @param timeOfCaching the time of caching 19 | */ 20 | Cache(Date timeOfCaching) { 21 | this.timeOfCaching = timeOfCaching; 22 | } 23 | 24 | /** 25 | * Has expired boolean. 26 | * 27 | * @param refreshIntervalInHours the refresh interval in hours 28 | * @return the boolean 29 | */ 30 | public boolean hasExpired(int refreshIntervalInHours) { 31 | if (timeOfCaching == null) { 32 | return true; 33 | } 34 | Date currentTime = new Date(); 35 | Date timeOfExpire = DateUtils.addHours(timeOfCaching, refreshIntervalInHours); 36 | return currentTime.after(timeOfExpire); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/scalar/elementAt/row/RowElement.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar.elementAt.row; 2 | 3 | import org.raystack.dagger.functions.udfs.scalar.elementAt.descriptor.CustomDescriptor; 4 | import org.apache.flink.types.Row; 5 | 6 | import java.util.Optional; 7 | 8 | import static com.google.protobuf.Descriptors.FieldDescriptor; 9 | 10 | /** 11 | * The Row element. 12 | */ 13 | class RowElement extends Element { 14 | 15 | /** 16 | * Instantiates a new Row element. 17 | * 18 | * @param parent the parent 19 | * @param row the row 20 | * @param fieldDescriptor the field descriptor 21 | */ 22 | RowElement(Element parent, Row row, FieldDescriptor fieldDescriptor) { 23 | super(parent, row, fieldDescriptor); 24 | } 25 | 26 | public Optional createNext(String pathElement) { 27 | Optional childElement = initialize(this, null, new CustomDescriptor(getFieldDescriptor().getMessageType()), pathElement); 28 | return childElement; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/scalar/elementAt/row/ValueElement.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar.elementAt.row; 2 | 3 | 4 | import com.google.protobuf.Descriptors; 5 | import org.apache.flink.types.Row; 6 | 7 | import java.util.Optional; 8 | 9 | /** 10 | * The Value element. 11 | */ 12 | class ValueElement extends Element { 13 | 14 | /** 15 | * Instantiates a new Value element. 16 | * 17 | * @param parent the parent 18 | * @param row the row 19 | * @param fieldDescriptor the field descriptor 20 | */ 21 | ValueElement(Element parent, Row row, Descriptors.FieldDescriptor fieldDescriptor) { 22 | super(parent, row, fieldDescriptor); 23 | } 24 | 25 | @Override 26 | public Optional createNext(String pathElement) { 27 | return Optional.empty(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/scalar/longbow/array/expression/AggregationExpression.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar.longbow.array.expression; 2 | 3 | /** 4 | * The Aggregation expression. 5 | */ 6 | public class AggregationExpression implements Expression { 7 | private String expressionString; 8 | 9 | @Override 10 | public String getExpressionString() { 11 | return expressionString; 12 | } 13 | 14 | @Override 15 | public void createExpression(String operationType) { 16 | this.expressionString = BASE_STRING + getOperationExpression(operationType); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/scalar/longbow/array/expression/OperationExpression.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar.longbow.array.expression; 2 | 3 | /** 4 | * The Operation expression. 5 | */ 6 | public class OperationExpression implements Expression { 7 | public static final String CONVERT_TO_ARRAY = ".toArray()"; 8 | private String expressionString; 9 | 10 | @Override 11 | public String getExpressionString() { 12 | return expressionString; 13 | } 14 | 15 | @Override 16 | public void createExpression(String operationType) { 17 | this.expressionString = BASE_STRING + getOperationExpression(operationType) + CONVERT_TO_ARRAY; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /dagger-functions/src/main/java/org/raystack/dagger/functions/udfs/table/HistogramBucket.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.table; 2 | 3 | import org.raystack.dagger.common.udfs.TableUdf; 4 | import org.apache.flink.api.java.tuple.Tuple1; 5 | 6 | import java.util.Arrays; 7 | 8 | 9 | /** 10 | * The class responsible for Histogram bucket udf. 11 | */ 12 | public class HistogramBucket extends TableUdf> { 13 | 14 | /** 15 | * This UDF returns buckets for given value to calculate histograms. 16 | * see https://github.com/influxdata/telegraf/tree/master/plugins/aggregators/histogram#tags 17 | * 18 | * @param dValue Value to be compared 19 | * @param buckets Buckets for Cumulative Histograms 20 | * @author lavkesh.lahngir 21 | */ 22 | public void eval(Double dValue, String buckets) { 23 | // Always emit the bucket for '+Inf' 24 | collect(new Tuple1<>("+Inf")); 25 | Arrays.stream(buckets.split(",")). 26 | filter(bucket -> dValue <= Double.parseDouble(bucket)). 27 | map(Tuple1::new). 28 | forEach(this::collect); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /dagger-functions/src/test/java/org/raystack/dagger/functions/udfs/aggregate/accumulator/ArrayAccumulatorTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.accumulator; 2 | 3 | import org.junit.Test; 4 | 5 | import java.util.List; 6 | 7 | import static org.junit.Assert.assertEquals; 8 | 9 | public class ArrayAccumulatorTest { 10 | @Test 11 | public void shouldGiveAllValuesFromAccumulator() { 12 | ArrayAccumulator arrayAccumulator = new ArrayAccumulator(); 13 | String value1 = "first"; 14 | String value2 = "second"; 15 | 16 | arrayAccumulator.add(value1); 17 | arrayAccumulator.add(value2); 18 | 19 | List values = arrayAccumulator.emit(); 20 | assertEquals(2, values.size()); 21 | assertEquals("first", (String) values.get(0)); 22 | assertEquals("second", (String) values.get(1)); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /dagger-functions/src/test/java/org/raystack/dagger/functions/udfs/aggregate/accumulator/PercentileAccumulatorTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.accumulator; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | public class PercentileAccumulatorTest { 7 | 8 | // Just calling apache commons so not many tests are needed. 9 | @Test 10 | public void shouldReturn90thPercentileValue() { 11 | PercentileAccumulator accumulator = new PercentileAccumulator(); 12 | accumulator.add(90, 10); 13 | accumulator.add(90, 30); 14 | accumulator.add(90, 40); 15 | accumulator.add(90, 20); 16 | accumulator.add(90, 70); 17 | accumulator.add(90, 90); 18 | accumulator.add(90, 100); 19 | accumulator.add(90, 80); 20 | accumulator.add(90, 50); 21 | accumulator.add(90, 60); 22 | Assert.assertEquals(99.0, accumulator.getPercentileValue(), 0.001); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /dagger-functions/src/test/java/org/raystack/dagger/functions/udfs/aggregate/feast/handler/BooleanValueTransformerTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.feast.handler; 2 | 3 | import org.apache.flink.types.Row; 4 | import org.junit.Test; 5 | 6 | import static org.junit.Assert.assertEquals; 7 | 8 | public class BooleanValueTransformerTest { 9 | 10 | @Test 11 | public void shouldReturnTrueForBooleanValues() { 12 | BooleanValueTransformer booleanValueHandler = new BooleanValueTransformer(); 13 | assertEquals(booleanValueHandler.canTransform(true), true); 14 | } 15 | 16 | @Test 17 | public void shouldReturnValue() { 18 | BooleanValueTransformer booleanValueHandler = new BooleanValueTransformer(); 19 | Row rowBoolean = booleanValueHandler.transform(true); 20 | assertEquals(true, rowBoolean.getField(6)); 21 | } 22 | 23 | @Test 24 | public void shouldReturnDefaultValueForNull() { 25 | BooleanValueTransformer booleanValueHandler = new BooleanValueTransformer(); 26 | Row rowBoolean = booleanValueHandler.transform(null); 27 | assertEquals(false, rowBoolean.getField(6)); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /dagger-functions/src/test/java/org/raystack/dagger/functions/udfs/aggregate/feast/handler/ByteValueTransformerTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.feast.handler; 2 | 3 | import com.google.protobuf.ByteString; 4 | import org.apache.flink.types.Row; 5 | import org.junit.Test; 6 | 7 | import static org.junit.Assert.assertEquals; 8 | 9 | public class ByteValueTransformerTest { 10 | 11 | @Test 12 | public void shouldReturnTrueForByteValues() { 13 | ByteValueTransformer byteValueHandler = new ByteValueTransformer(); 14 | assertEquals(byteValueHandler.canTransform(ByteString.copyFrom("value1".getBytes())), true); 15 | } 16 | 17 | @Test 18 | public void shouldReturnValue() { 19 | ByteValueTransformer byteValueHandler = new ByteValueTransformer(); 20 | Row byteRow = byteValueHandler.transform(ByteString.copyFrom("value1".getBytes())); 21 | assertEquals(ByteString.copyFrom("value1".getBytes()), byteRow.getField(0)); 22 | } 23 | 24 | @Test 25 | public void shouldReturnDefaultValueForNull() { 26 | ByteValueTransformer byteValueHandler = new ByteValueTransformer(); 27 | Row byteRow = byteValueHandler.transform(null); 28 | assertEquals(null, byteRow.getField(0)); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /dagger-functions/src/test/java/org/raystack/dagger/functions/udfs/aggregate/feast/handler/NullValueTransformerTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.feast.handler; 2 | 3 | import org.apache.commons.lang3.NotImplementedException; 4 | import org.apache.flink.types.Row; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | import static org.junit.Assert.assertEquals; 9 | 10 | public class NullValueTransformerTest { 11 | 12 | @Test 13 | public void shouldReturnTrueForNullValues() { 14 | NullValueTransformer nullValueTransformer = new NullValueTransformer(); 15 | assertEquals(nullValueTransformer.canTransform(null), true); 16 | } 17 | 18 | @Test 19 | public void shouldReturnEmptyRow() { 20 | NullValueTransformer nullValueTransformer = new NullValueTransformer(); 21 | Row row = nullValueTransformer.transform(null); 22 | Assert.assertEquals(8, row.getArity()); 23 | Assert.assertEquals("+I[null, null, null, null, null, null, null, null]", row.toString()); 24 | } 25 | 26 | @Test(expected = NotImplementedException.class) 27 | public void shouldThrowExceptionForGetIndex() { 28 | NullValueTransformer nullValueTransformer = new NullValueTransformer(); 29 | nullValueTransformer.getIndex(); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /dagger-functions/src/test/java/org/raystack/dagger/functions/udfs/aggregate/feast/handler/StringValueTransformerTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.aggregate.feast.handler; 2 | 3 | import org.junit.Test; 4 | 5 | import static org.junit.Assert.assertEquals; 6 | 7 | public class StringValueTransformerTest { 8 | 9 | @Test 10 | public void shouldReturnTrueForStringValues() { 11 | StringValueTransformer stringValueHandler = new StringValueTransformer(); 12 | assertEquals(stringValueHandler.canTransform("value"), true); 13 | } 14 | 15 | @Test 16 | public void shouldReturnValue() { 17 | StringValueTransformer stringValueHandler = new StringValueTransformer(); 18 | assertEquals("value", stringValueHandler.transform("value").getField(1)); 19 | } 20 | 21 | @Test 22 | public void shouldTransformAnyObjectToString() { 23 | StringValueTransformer stringValueHandler = new StringValueTransformer(); 24 | assertEquals("1", stringValueHandler.transform(1).getField(1)); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /dagger-functions/src/test/java/org/raystack/dagger/functions/udfs/python/file/source/FileSourceFactoryTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.python.file.source; 2 | 3 | import org.raystack.dagger.functions.udfs.python.file.source.gcs.GcsFileSource; 4 | import org.raystack.dagger.functions.udfs.python.file.source.local.LocalFileSource; 5 | import org.junit.Assert; 6 | import org.junit.Test; 7 | 8 | public class FileSourceFactoryTest { 9 | 10 | @Test 11 | public void shouldGetLocalFileSource() { 12 | String pythonFile = "/path/to/file/test_function.py"; 13 | 14 | FileSource fileSource = FileSourceFactory.getFileSource(pythonFile); 15 | 16 | Assert.assertTrue(fileSource instanceof LocalFileSource); 17 | } 18 | 19 | @Test 20 | public void shouldGetGcsFileSource() { 21 | String pythonFile = "gs://bucket-name/path/to/file/test_function.py"; 22 | 23 | FileSource fileSource = FileSourceFactory.getFileSource(pythonFile); 24 | 25 | Assert.assertTrue(fileSource instanceof GcsFileSource); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /dagger-functions/src/test/java/org/raystack/dagger/functions/udfs/python/file/source/gcs/GcsFileSourceTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.python.file.source.gcs; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | import org.mockito.Mock; 7 | 8 | import java.io.IOException; 9 | import java.nio.file.Files; 10 | import java.nio.file.Paths; 11 | 12 | import static org.mockito.Mockito.when; 13 | import static org.mockito.MockitoAnnotations.initMocks; 14 | 15 | public class GcsFileSourceTest { 16 | 17 | @Mock 18 | private GcsClient gcsClient; 19 | 20 | @Before 21 | public void setup() { 22 | initMocks(this); 23 | } 24 | 25 | @Test 26 | public void shouldGetObjectFile() throws IOException { 27 | ClassLoader classLoader = getClass().getClassLoader(); 28 | String pythonFile = classLoader.getResource("python_udf.zip").getFile(); 29 | byte[] expectedObject = Files.readAllBytes(Paths.get(pythonFile)); 30 | 31 | when(gcsClient.getFile(pythonFile)).thenReturn(expectedObject); 32 | GcsFileSource gcsFileSource = new GcsFileSource(pythonFile, gcsClient); 33 | 34 | byte[] actualObject = gcsFileSource.getObjectFile(); 35 | 36 | Assert.assertEquals(expectedObject, actualObject); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /dagger-functions/src/test/java/org/raystack/dagger/functions/udfs/python/file/source/local/LocalFileSourceTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.python.file.source.local; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | import java.io.IOException; 7 | import java.nio.charset.StandardCharsets; 8 | import java.nio.file.Files; 9 | import java.nio.file.Paths; 10 | 11 | public class LocalFileSourceTest { 12 | 13 | @Test 14 | public void shouldGetObjectFile() throws IOException { 15 | ClassLoader classLoader = getClass().getClassLoader(); 16 | 17 | String pythonFile = classLoader.getResource("python_udf.zip").getPath(); 18 | 19 | byte[] object = Files.readAllBytes(Paths.get(pythonFile)); 20 | String stringObject = new String(object, StandardCharsets.UTF_8); 21 | 22 | LocalFileSource localFileSource = new LocalFileSource(pythonFile); 23 | byte[] actualObject = localFileSource.getObjectFile(); 24 | 25 | String actualStringObject = new String(actualObject, StandardCharsets.UTF_8); 26 | Assert.assertEquals(stringObject, actualStringObject); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /dagger-functions/src/test/java/org/raystack/dagger/functions/udfs/scalar/dart/types/CacheTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar.dart.types; 2 | 3 | import org.junit.Before; 4 | import org.junit.Test; 5 | import java.util.Date; 6 | import static org.junit.Assert.assertFalse; 7 | import static org.junit.Assert.assertTrue; 8 | 9 | public class CacheTest { 10 | private Cache cache; 11 | @Before 12 | public void setUp() { 13 | cache = new Cache(new Date()) { }; 14 | } 15 | 16 | @Test 17 | public void shouldNotExpireContentWhenTimeHasNotElapsed() { 18 | 19 | assertFalse(cache.hasExpired(1)); 20 | } 21 | 22 | @Test 23 | public void shouldExpireContentWhenTimeElapsed() { 24 | cache = new Cache(new Date(1535546718115L)) { }; 25 | 26 | assertTrue(cache.hasExpired(1)); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /dagger-functions/src/test/java/org/raystack/dagger/functions/udfs/scalar/longbow/array/expression/AggregationExpressionTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar.longbow.array.expression; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | 7 | import static org.mockito.MockitoAnnotations.initMocks; 8 | 9 | public class AggregationExpressionTest { 10 | 11 | private AggregationExpression aggregationExpression; 12 | 13 | @Before 14 | public void setup() { 15 | initMocks(this); 16 | aggregationExpression = new AggregationExpression(); 17 | } 18 | 19 | @Test 20 | public void shouldCreateExpressionForSingleExpression() { 21 | String operationType = "distinct"; 22 | aggregationExpression.createExpression(operationType); 23 | 24 | Assert.assertEquals(aggregationExpression.getExpressionString(), "stream.distinct()"); 25 | } 26 | 27 | @Test 28 | public void shouldCreateExpressionForMultipleExpression() { 29 | String operationType = "distinct.sorted.sum"; 30 | aggregationExpression.createExpression(operationType); 31 | 32 | Assert.assertEquals(aggregationExpression.getExpressionString(), "stream.distinct().sorted().sum()"); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /dagger-functions/src/test/java/org/raystack/dagger/functions/udfs/scalar/longbow/array/expression/OperationExpressionTest.java: -------------------------------------------------------------------------------- 1 | package org.raystack.dagger.functions.udfs.scalar.longbow.array.expression; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | 7 | import static org.mockito.MockitoAnnotations.initMocks; 8 | 9 | public class OperationExpressionTest { 10 | 11 | private OperationExpression operationExpression; 12 | 13 | @Before 14 | public void setup() { 15 | initMocks(this); 16 | operationExpression = new OperationExpression(); 17 | } 18 | 19 | @Test 20 | public void shouldCreateExpressionForSingleExpression() { 21 | String operationType = "distinct"; 22 | operationExpression.createExpression(operationType); 23 | 24 | Assert.assertEquals(operationExpression.getExpressionString(), "stream.distinct().toArray()"); 25 | } 26 | 27 | @Test 28 | public void shouldCreateExpressionForMultipleExpression() { 29 | String operationType = "distinct.sorted"; 30 | operationExpression.createExpression(operationType); 31 | 32 | Assert.assertEquals(operationExpression.getExpressionString(), "stream.distinct().sorted().toArray()"); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /dagger-functions/src/test/resources/python_udf.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/dagger-functions/src/test/resources/python_udf.zip -------------------------------------------------------------------------------- /dagger-functions/src/test/resources/test_no_py.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/dagger-functions/src/test/resources/test_no_py.zip -------------------------------------------------------------------------------- /dagger-functions/src/test/resources/test_udf.py: -------------------------------------------------------------------------------- 1 | from pyflink.table import DataTypes 2 | from pyflink.table.udf import udf 3 | 4 | 5 | @udf(result_type=DataTypes.STRING()) 6 | def test_udf(text: str): 7 | return text + "_added_text" 8 | -------------------------------------------------------------------------------- /dagger-py-functions/data/sample_data.txt: -------------------------------------------------------------------------------- 1 | sample_text -------------------------------------------------------------------------------- /dagger-py-functions/requirements.txt: -------------------------------------------------------------------------------- 1 | pytest==7.1.2 2 | flake8==4.0.1 -------------------------------------------------------------------------------- /dagger-py-functions/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/dagger-py-functions/tests/__init__.py -------------------------------------------------------------------------------- /dagger-py-functions/tests/udfs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/dagger-py-functions/tests/udfs/__init__.py -------------------------------------------------------------------------------- /dagger-py-functions/tests/udfs/scalar/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/dagger-py-functions/tests/udfs/scalar/__init__.py -------------------------------------------------------------------------------- /dagger-py-functions/tests/udfs/scalar/multiply_test.py: -------------------------------------------------------------------------------- 1 | from udfs.scalar.multiply import multiply 2 | 3 | 4 | def testMultiply(): 5 | value = multiply._func 6 | assert value(5,10) == 50 7 | -------------------------------------------------------------------------------- /dagger-py-functions/tests/udfs/scalar/sample_test.py: -------------------------------------------------------------------------------- 1 | from udfs.scalar.sample import sample 2 | 3 | 4 | def testSample(): 5 | value = sample._func 6 | assert value("input_text_") == "input_text_sample_text" 7 | -------------------------------------------------------------------------------- /dagger-py-functions/udfs/scalar/multiply.py: -------------------------------------------------------------------------------- 1 | from pyflink.table import DataTypes 2 | from pyflink.table.udf import udf 3 | 4 | 5 | @udf(result_type=DataTypes.FLOAT()) 6 | def multiply(i, j): 7 | return i * j -------------------------------------------------------------------------------- /dagger-py-functions/udfs/scalar/sample.py: -------------------------------------------------------------------------------- 1 | from pyflink.table import DataTypes 2 | from pyflink.table.udf import udf 3 | 4 | 5 | @udf(result_type=DataTypes.STRING()) 6 | def sample(text): 7 | file = open("data/sample_data.txt", "r") 8 | data = file.read() 9 | return text + data 10 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | /node_modules 3 | 4 | # Production 5 | /build 6 | 7 | # Generated files 8 | .docusaurus 9 | .cache-loader 10 | 11 | # Misc 12 | .DS_Store 13 | .env.local 14 | .env.development.local 15 | .env.test.local 16 | .env.production.local 17 | 18 | npm-debug.log* 19 | yarn-debug.log* 20 | yarn-error.log* 21 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Website 2 | 3 | This website is built using [Docusaurus 2](https://docusaurus.io/), a modern static website generator. 4 | 5 | ### Installation 6 | 7 | ``` 8 | $ yarn 9 | ``` 10 | 11 | ### Local Development 12 | 13 | ``` 14 | $ yarn start 15 | ``` 16 | 17 | This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server. 18 | 19 | ### Build 20 | 21 | ``` 22 | $ yarn build 23 | ``` 24 | 25 | This command generates static content into the `build` directory and can be served using any static contents hosting service. 26 | 27 | ### Deployment 28 | 29 | ``` 30 | $ GIT_USER= USE_SSH=true yarn deploy 31 | ``` 32 | 33 | If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch. 34 | -------------------------------------------------------------------------------- /docs/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [require.resolve('@docusaurus/core/lib/babel/preset')], 3 | }; 4 | -------------------------------------------------------------------------------- /docs/blog/2023-05-23-intro-dagger-docs.md: -------------------------------------------------------------------------------- 1 | --- 2 | slug: introducing-dagger 3 | title: Introducing Dagger Documentation page 4 | authors: 5 | name: Hari krishna 6 | tags: [dagger] 7 | --- 8 | 9 | Our docs website is live. 10 | -------------------------------------------------------------------------------- /docs/blog/authors.yaml: -------------------------------------------------------------------------------- 1 | harikrishna: 2 | name: Hari krishna 3 | title: Maintainer 4 | url: https://github.com/harikrishnakanchi 5 | image_url: https://github.com/harikrishnakanchi.png 6 | -------------------------------------------------------------------------------- /docs/docs/concepts/overview.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | The following section sheds light on some basic concepts and several components of Dagger. 4 | 5 | ### [Architecture](../concepts/architecture.md) 6 | 7 | Describes some of the internal logical and infrastructural blocks of Dagger and how they are interconnected. 8 | 9 | ### [Basics](../concepts/basics.md) 10 | 11 | A brief introduction to stream processing with some basic terminologies and keywords in Dagger. 12 | 13 | ### [Lifecycle](../concepts/lifecycle.md) 14 | 15 | Explains how unbounded data goes through internal processing pipelines before materializing the results. 16 | -------------------------------------------------------------------------------- /docs/docs/examples/overview.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | The following example tutorials will help you to quickly try out some of Dagger's most useful features with real-world usecases - 4 | 5 | - [Data Aggregation using a Tumble Window](../examples/aggregation_tumble_window.md) 6 | - [Removing duplicate records using Transformers](../examples/deduplication_transformer.md) 7 | - [Distance computation using Java UDF](../examples/distance_java_udf.md) 8 | - [Stream enrichment using ElasticSearch source](../examples/elasticsearch_enrichment.md) 9 | - [Joining two Kafka topics using Inner join](../examples/kafka_inner_join.md) 10 | -------------------------------------------------------------------------------- /docs/docs/reference/overview.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | The following section describes the references of properties on Dagger deployment. 4 | 5 | ### [Configurations](../reference/configuration.md) 6 | 7 | Describes all the configurations needed for deploying Dagger. 8 | 9 | ### [Metrics](../reference/metrics.md) 10 | 11 | Describes all the metrics captured by Dagger deployment. 12 | 13 | ### [Transformers](../reference/transformers.md) 14 | 15 | Describes all the custom transformers available and the configuration needed to be used in Dagger. 16 | 17 | ### [Udfs](../reference/udfs.md) 18 | 19 | Describes all the custom udfs available and the configuration needed to be used on Dagger. 20 | -------------------------------------------------------------------------------- /docs/docs/roadmap.md: -------------------------------------------------------------------------------- 1 | --- 2 | sidebar_position: 2 3 | --- 4 | 5 | # Roadmap 6 | 7 | In the following section you can learn about what features we're working on, what stage they're in, and when we expect to bring them to you. Have any questions or comments about items on the roadmap? Join the [discussions](https://github.com/raystack/dagger/discussions) on Dagger Github forum. 8 | 9 | We’re planning to iterate on the format of the roadmap itself, and we see potential to engage more in discussions about the future of Dagger features. If you have feedback about this roadmap section itself, such as how the issues are presented, let us know through [discussions](https://github.com/raystack/dagger/discussions). 10 | 11 | ### Dagger 0.2 12 | 13 | - Flink upgrade from 1.9 to 1.13 14 | 15 | ### Dagger 0.3 16 | 17 | - Support for Python in custom functions 18 | - Parquet file processing 19 | 20 | ### Dagger 0.4 21 | 22 | - Support for JSON and Avro 23 | - CDC input source 24 | - Support for dead letter queue 25 | -------------------------------------------------------------------------------- /docs/docs/usecase/overview.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | This section talks a bit about some of the use cases that can be solved using Dagger with some examples. 4 | 5 | ### [Stream Enrichment](stream_enrichment.md) 6 | 7 | Enrichment of streaming Data with external more static Data sources is seamless in Dagger and is just writing a few configuration. We will explain this in more details with some example. 8 | 9 | ### [Feature Ingestion](feature_ingestion.md) 10 | 11 | Realtime feature generation is really important for online machine learning pipelines. Dagger can be a great tool for feature ingestion. 12 | 13 | ### [API Monitoring](api_monitoring.md) 14 | 15 | Behind every great API is a reliable uptime monitoring system. But knowing/monitoring health of API metrics in real time is tricky. With Dagger this can be easily solved by writing some simple queries. 16 | -------------------------------------------------------------------------------- /docs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dagger", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "docusaurus": "docusaurus", 7 | "start": "docusaurus start", 8 | "build": "docusaurus build", 9 | "swizzle": "docusaurus swizzle", 10 | "deploy": "docusaurus deploy", 11 | "clear": "docusaurus clear", 12 | "serve": "docusaurus serve", 13 | "write-translations": "docusaurus write-translations", 14 | "write-heading-ids": "docusaurus write-heading-ids" 15 | }, 16 | "dependencies": { 17 | "@docusaurus/core": "2.0.0-beta.6", 18 | "@docusaurus/plugin-google-gtag": "^2.0.0-beta.6", 19 | "@docusaurus/preset-classic": "2.0.0-beta.6", 20 | "@mdx-js/react": "^1.6.21", 21 | "@svgr/webpack": "^5.5.0", 22 | "classnames": "^2.3.1", 23 | "clsx": "^1.1.1", 24 | "file-loader": "^6.2.0", 25 | "prism-react-renderer": "^1.2.1", 26 | "react": "^17.0.1", 27 | "react-dom": "^17.0.1", 28 | "url-loader": "^4.1.1" 29 | }, 30 | "browserslist": { 31 | "production": [ 32 | ">0.5%", 33 | "not dead", 34 | "not op_mini all" 35 | ], 36 | "development": [ 37 | "last 1 chrome version", 38 | "last 1 firefox version", 39 | "last 1 safari version" 40 | ] 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /docs/src/core/Container.js: -------------------------------------------------------------------------------- 1 | import classNames from 'classnames'; 2 | import * as React from 'react'; 3 | 4 | const Container = props => { 5 | const containerClasses = classNames(props.className, { 6 | darkBackground: props.background === 'dark', 7 | highlightBackground: props.background === 'highlight', 8 | lightBackground: props.background === 'light', 9 | paddingAll: props.padding.indexOf('all') >= 0, 10 | paddingBottom: props.padding.indexOf('bottom') >= 0, 11 | paddingLeft: props.padding.indexOf('left') >= 0, 12 | paddingRight: props.padding.indexOf('right') >= 0, 13 | paddingTop: props.padding.indexOf('top') >= 0, 14 | }); 15 | let wrappedChildren; 16 | 17 | if (props.wrapper) { 18 | wrappedChildren =
{props.children}
; 19 | } else { 20 | wrappedChildren = props.children; 21 | } 22 | return ( 23 |
24 | {wrappedChildren} 25 |
26 | ); 27 | }; 28 | 29 | Container.defaultProps = { 30 | background: null, 31 | padding: [], 32 | wrapper: true, 33 | }; 34 | 35 | export default Container; 36 | -------------------------------------------------------------------------------- /docs/src/css/theme.css: -------------------------------------------------------------------------------- 1 | /** 2 | * Any CSS included here will be global. The classic template 3 | * bundles Infima by default. Infima is a CSS framework designed to 4 | * work well for content-centric websites. 5 | */ 6 | 7 | /* You can override the default Infima variables here. */ 8 | :root { 9 | --ifm-color-primary-lightest: #FF923C; 10 | --ifm-color-primary-lighter: #FF7E17; 11 | --ifm-color-primary-light: #FF770B; 12 | --ifm-color-primary: #f26b00; 13 | --ifm-color-primary-dark: #DA6000; 14 | --ifm-color-primary-darker: #CE5B00; 15 | --ifm-color-primary-darkest: #A94B00; 16 | --ifm-code-font-size: 95%; 17 | } 18 | 19 | .docusaurus-highlight-code-line { 20 | background-color: rgba(0, 0, 0, 0.1); 21 | display: block; 22 | margin: 0 calc(-1 * var(--ifm-pre-padding)); 23 | padding: 0 var(--ifm-pre-padding); 24 | } 25 | 26 | html[data-theme='dark'] .docusaurus-highlight-code-line { 27 | background-color: rgba(0, 0, 0, 0.3); 28 | } 29 | 30 | html[data-theme="light"] { 31 | --main-bg-color: var(--ifm-color-primary); 32 | --light-bg-color: transparent; 33 | --dark-bg-color: #F8F6F0; 34 | } 35 | 36 | html[data-theme="dark"] { 37 | --main-bg-color: var(--ifm-color-primary); 38 | --light-bg-color: transparent; 39 | --dark-bg-color:#131313; 40 | } 41 | 42 | -------------------------------------------------------------------------------- /docs/static/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/.nojekyll -------------------------------------------------------------------------------- /docs/static/img/api-monitoring.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/api-monitoring.png -------------------------------------------------------------------------------- /docs/static/img/api-status.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/api-status.png -------------------------------------------------------------------------------- /docs/static/img/dart-contains.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/dart-contains.png -------------------------------------------------------------------------------- /docs/static/img/dart-get.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/dart-get.png -------------------------------------------------------------------------------- /docs/static/img/enrichment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/enrichment.png -------------------------------------------------------------------------------- /docs/static/img/external-http-post-processor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/external-http-post-processor.png -------------------------------------------------------------------------------- /docs/static/img/external-internal-post-processor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/external-internal-post-processor.png -------------------------------------------------------------------------------- /docs/static/img/external-internal-transformer-post-processor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/external-internal-transformer-post-processor.png -------------------------------------------------------------------------------- /docs/static/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/favicon.ico -------------------------------------------------------------------------------- /docs/static/img/lifecycle/dagger_lifecycle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/lifecycle/dagger_lifecycle.png -------------------------------------------------------------------------------- /docs/static/img/longbow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/longbow.png -------------------------------------------------------------------------------- /docs/static/img/longbowplus-reader.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/longbowplus-reader.png -------------------------------------------------------------------------------- /docs/static/img/longbowplus-writer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/longbowplus-writer.png -------------------------------------------------------------------------------- /docs/static/img/overview/dagger_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/overview/dagger_overview.png -------------------------------------------------------------------------------- /docs/static/img/pre-processor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/pre-processor.png -------------------------------------------------------------------------------- /docs/static/img/sliding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/sliding.png -------------------------------------------------------------------------------- /docs/static/img/system_design/dagger_system_design.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/system_design/dagger_system_design.png -------------------------------------------------------------------------------- /docs/static/img/tumble.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/img/tumble.png -------------------------------------------------------------------------------- /docs/static/users/gojek.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/users/gojek.png -------------------------------------------------------------------------------- /docs/static/users/goto.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/users/goto.png -------------------------------------------------------------------------------- /docs/static/users/jago.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/users/jago.png -------------------------------------------------------------------------------- /docs/static/users/mapan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/users/mapan.png -------------------------------------------------------------------------------- /docs/static/users/midtrans.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/users/midtrans.png -------------------------------------------------------------------------------- /docs/static/users/moka.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/users/moka.png -------------------------------------------------------------------------------- /docs/static/users/paylater.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/docs/static/users/paylater.png -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | org.gradle.daemon=true 2 | org.gradle.caching=true 3 | export GRADLE_OPTS=-Xmx2560m -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/raystack/dagger/9aac71e9c2d1b4f0cb1d5dc5a561a312edea37cf/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Wed Mar 31 10:29:20 IST 2021 2 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-all.zip 3 | distributionBase=GRADLE_USER_HOME 4 | distributionPath=wrapper/dists 5 | zipStorePath=wrapper/dists 6 | zipStoreBase=GRADLE_USER_HOME 7 | -------------------------------------------------------------------------------- /quickstart/docker-compose/resources/kafkafeeder.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | timestamp_now=$(date +%s) 3 | random_3char_suffix=$(pwgen 3 1) 4 | random_enum_index=$(($RANDOM %3)) 5 | declare -a myArray=("FLIGHT" "BUS" "TRAIN") 6 | cat sample_message.txt | \ 7 | sed "s/replace_timestamp_here/$timestamp_now/g; s/replace_service_type_here/${myArray[$random_enum_index]}/g; s/replace_customer_suffix_here/$random_3char_suffix/g" | \ 8 | protoc --proto_path=./ --encode=org.raystack.dagger.consumer.TestBookingLogMessage ./TestLogMessage.proto > message.bin 9 | -------------------------------------------------------------------------------- /quickstart/docker-compose/resources/sample_message.txt: -------------------------------------------------------------------------------- 1 | service_type: replace_service_type_here 2 | order_number: "ynnhjv45" 3 | event_timestamp { 4 | seconds: replace_timestamp_here 5 | } 6 | customer_id: "customer-replace_customer_suffix_here" 7 | driver_pickup_location { 8 | latitude: 19.075983 9 | longitude: 72.877655 10 | } 11 | driver_dropoff_location { 12 | latitude: 19.237188 13 | longitude: 72.844139 14 | } -------------------------------------------------------------------------------- /quickstart/examples/aggregation/tumble_window/resources/kafkafeeder.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | timestamp_now=$(date +%s) 3 | random_3char_suffix=$(pwgen 3 1) 4 | random_enum_index=$(($RANDOM %3)) 5 | declare -a myArray=("FLIGHT" "BUS" "TRAIN") 6 | cat sample_message.txt | \ 7 | sed "s/replace_timestamp_here/$timestamp_now/g; s/replace_service_type_here/${myArray[$random_enum_index]}/g; s/replace_customer_suffix_here/$random_3char_suffix/g" | \ 8 | protoc --proto_path=./ --encode=org.raystack.dagger.consumer.TestBookingLogMessage ./TestLogMessage.proto > message.bin 9 | -------------------------------------------------------------------------------- /quickstart/examples/aggregation/tumble_window/resources/sample_message.txt: -------------------------------------------------------------------------------- 1 | service_type: replace_service_type_here 2 | order_number: "ynnhjv45" 3 | event_timestamp { 4 | seconds: replace_timestamp_here 5 | } 6 | customer_id: "customer-replace_customer_suffix_here" 7 | driver_pickup_location { 8 | latitude: 19.075983 9 | longitude: 72.877655 10 | } 11 | driver_dropoff_location { 12 | latitude: 19.237188 13 | longitude: 72.844139 14 | } -------------------------------------------------------------------------------- /quickstart/examples/enrichment/elasticsearch_enrichment/resources/kafkafeeder.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | timestamp_now=$(date +%s) 3 | random_enum_index=$(($RANDOM %3)) 4 | random_customer_id=$(($RANDOM %12)) 5 | declare -a myArray=("FLIGHT" "BUS" "TRAIN") 6 | cat sample_message.txt | \ 7 | sed "s/replace_timestamp_here/$timestamp_now/g; s/replace_service_type_here/${myArray[$random_enum_index]}/g; s/replace_customer_suffix_here/$random_customer_id/g" | \ 8 | protoc --proto_path=org/raystack/dagger/consumer/ --encode=org.raystack.dagger.consumer.TestBookingLogMessage org/raystack/dagger/consumer/TestLogMessage.proto > message.bin 9 | -------------------------------------------------------------------------------- /quickstart/examples/enrichment/elasticsearch_enrichment/resources/sample_message.txt: -------------------------------------------------------------------------------- 1 | service_type: replace_service_type_here 2 | order_number: "ynnhjv45" 3 | event_timestamp { 4 | seconds: replace_timestamp_here 5 | } 6 | customer_id: "replace_customer_suffix_here" 7 | driver_pickup_location { 8 | latitude: 19.075983 9 | longitude: 72.877655 10 | } 11 | driver_dropoff_location { 12 | latitude: 19.237188 13 | longitude: 72.844139 14 | } -------------------------------------------------------------------------------- /quickstart/examples/enrichment/elasticsearch_enrichment/resources/seed1.json: -------------------------------------------------------------------------------- 1 | { 2 | "created_at": "2017-05-18T01:19:54Z", 3 | "customer_id": 11, 4 | "email": "dummy_customer@raystack.org", 5 | "name": "dummy customer", 6 | "phone": "+666666000666666", 7 | "updated_at": "2017-05-18T01:20:00Z", 8 | "type": "customer", 9 | "wallet_id": "171380079590001027" 10 | } 11 | -------------------------------------------------------------------------------- /quickstart/examples/joins/inner_join/resources/kafkafeeder.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | timestamp_now=$(date +%s) 3 | random_3char_suffix=$(pwgen 3 1) 4 | random_enum_index=$(($RANDOM %3)) 5 | declare -a myArray=("FLIGHT" "BUS" "TRAIN") 6 | cat sample_message.txt | \ 7 | sed "s/replace_timestamp_here/$timestamp_now/g; s/replace_service_type_here/${myArray[$random_enum_index]}/g; s/replace_customer_suffix_here/$random_3char_suffix/g" | \ 8 | protoc --proto_path=./ --encode=org.raystack.dagger.consumer.TestBookingLogMessage ./TestLogMessage.proto > message.bin 9 | -------------------------------------------------------------------------------- /quickstart/examples/joins/inner_join/resources/sample_message.txt: -------------------------------------------------------------------------------- 1 | service_type: replace_service_type_here 2 | order_number: "ynnhjv45" 3 | event_timestamp { 4 | seconds: replace_timestamp_here 5 | } 6 | customer_id: "customer-replace_customer_suffix_here" 7 | driver_pickup_location { 8 | latitude: 19.075983 9 | longitude: 72.877655 10 | } 11 | driver_dropoff_location { 12 | latitude: 19.237188 13 | longitude: 72.844139 14 | } -------------------------------------------------------------------------------- /quickstart/examples/transformer/deduplication_transformer/resources/kafkafeeder.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | timestamp_now=$(date +%s) 3 | random_3char_suffix=$(pwgen 3 1) 4 | random_enum_index=$(($RANDOM %3)) 5 | declare -a myArray=("FLIGHT" "BUS" "TRAIN") 6 | cat sample_message.txt | \ 7 | sed "s/replace_timestamp_here/$timestamp_now/g; s/replace_service_type_here/${myArray[$random_enum_index]}/g; s/replace_customer_suffix_here/$random_3char_suffix/g" | \ 8 | protoc --proto_path=./ --encode=org.raystack.dagger.consumer.TestBookingLogMessage ./TestLogMessage.proto > message.bin 9 | -------------------------------------------------------------------------------- /quickstart/examples/transformer/deduplication_transformer/resources/sample_message.txt: -------------------------------------------------------------------------------- 1 | service_type: replace_service_type_here 2 | order_number: "ynnhjv45" 3 | event_timestamp { 4 | seconds: replace_timestamp_here 5 | } 6 | customer_id: "customer-replace_customer_suffix_here" 7 | driver_pickup_location { 8 | latitude: 19.075983 9 | longitude: 72.877655 10 | } 11 | driver_dropoff_location { 12 | latitude: 19.237188 13 | longitude: 72.844139 14 | } -------------------------------------------------------------------------------- /quickstart/examples/udfs/distance_udf/resources/kafkafeeder.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | timestamp_now=$(date +%s) 3 | random_3char_suffix=$(pwgen 3 1) 4 | random_enum_index=$(($RANDOM %3)) 5 | declare -a myArray=("FLIGHT" "BUS" "TRAIN") 6 | cat sample_message.txt | \ 7 | sed "s/replace_timestamp_here/$timestamp_now/g; s/replace_service_type_here/${myArray[$random_enum_index]}/g; s/replace_customer_suffix_here/$random_3char_suffix/g" | \ 8 | protoc --proto_path=./ --encode=org.raystack.dagger.consumer.TestBookingLogMessage ./TestLogMessage.proto > message.bin 9 | -------------------------------------------------------------------------------- /quickstart/examples/udfs/distance_udf/resources/sample_message.txt: -------------------------------------------------------------------------------- 1 | service_type: replace_service_type_here 2 | order_number: "ynnhjv45" 3 | event_timestamp { 4 | seconds: replace_timestamp_here 5 | } 6 | customer_id: "customer-replace_customer_suffix_here" 7 | driver_pickup_location { 8 | latitude: 19.075983 9 | longitude: 72.877655 10 | } 11 | driver_dropoff_location { 12 | latitude: 59.237188 13 | longitude: 23.844139 14 | } -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'dagger' 2 | 3 | include 'dagger-core' 4 | include 'dagger-common' 5 | include 'dagger-functions' 6 | include 'dagger-tests' 7 | -------------------------------------------------------------------------------- /version.txt: -------------------------------------------------------------------------------- 1 | 0.7.0 --------------------------------------------------------------------------------