├── .editorconfig ├── .gitignore ├── LICENSE ├── README.md ├── etl_light.png ├── etlite └── src │ ├── it │ ├── docker-compose.sh │ ├── docker-compose.yml │ ├── resources │ │ ├── json_application.conf │ │ └── proto_application.conf │ └── scala │ │ └── yamrcraft │ │ └── etlite │ │ ├── DockerEnv.scala │ │ ├── KafkaPublisher.scala │ │ ├── json │ │ └── JsonETLIntegrationTest.scala │ │ └── proto │ │ └── ProtobufETLIntegrationTest.scala │ └── main │ ├── resources │ ├── example_hdfs_application.conf │ └── example_s3_application.conf │ └── scala │ └── yamrcraft │ └── etlite │ ├── EtlException.scala │ ├── Main.scala │ ├── Settings.scala │ ├── pipeline │ ├── AbstractProtoPiplineFactory.scala │ ├── GenericProtoPipelineFactory.scala │ ├── JsonToParquetPipelineFactory.scala │ ├── JsonToSimpleFilePipelineFactory.scala │ ├── Pipeline.scala │ └── PipelineFactory.scala │ ├── processors │ ├── EtlProcessor.scala │ └── PartitionProcessor.scala │ ├── state │ ├── KafkaStateManager.scala │ └── StateManager.scala │ ├── transformers │ ├── JsonToAvroTransformer.scala │ ├── JsonTransformer.scala │ ├── Message.scala │ ├── ProtoTransformer.scala │ └── Transformer.scala │ ├── utils │ ├── ConfigConversions.scala │ ├── CustomJsonGenericRecordReader.scala │ ├── DLock.scala │ ├── FileSystemProvider.scala │ ├── FileUtils.scala │ ├── HdfsUtils.scala │ ├── JsonAvroConverter.scala │ └── TimeUtils.scala │ └── writers │ ├── AvroToParquetWriter.scala │ ├── ErrorEventsWriter.scala │ ├── ProtoToParquetWriter.scala │ ├── StringWriter.scala │ ├── TimePartitioningWriter.scala │ ├── Writer.scala │ └── package.scala ├── examples └── protobuf-messages │ ├── proto │ └── user.proto │ └── src │ └── main │ └── java │ └── examples │ └── protobuf │ └── UserOuterClass.java └── project ├── Dependencies.scala ├── build.properties └── plugins.sbt /.editorconfig: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/.editorconfig -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #intellij 2 | .idea/ 3 | *.iml 4 | 5 | #mac finder files 6 | .DS_Store* 7 | 8 | target/ 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/README.md -------------------------------------------------------------------------------- /etl_light.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etl_light.png -------------------------------------------------------------------------------- /etlite/src/it/docker-compose.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/it/docker-compose.sh -------------------------------------------------------------------------------- /etlite/src/it/docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/it/docker-compose.yml -------------------------------------------------------------------------------- /etlite/src/it/resources/json_application.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/it/resources/json_application.conf -------------------------------------------------------------------------------- /etlite/src/it/resources/proto_application.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/it/resources/proto_application.conf -------------------------------------------------------------------------------- /etlite/src/it/scala/yamrcraft/etlite/DockerEnv.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/it/scala/yamrcraft/etlite/DockerEnv.scala -------------------------------------------------------------------------------- /etlite/src/it/scala/yamrcraft/etlite/KafkaPublisher.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/it/scala/yamrcraft/etlite/KafkaPublisher.scala -------------------------------------------------------------------------------- /etlite/src/it/scala/yamrcraft/etlite/json/JsonETLIntegrationTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/it/scala/yamrcraft/etlite/json/JsonETLIntegrationTest.scala -------------------------------------------------------------------------------- /etlite/src/it/scala/yamrcraft/etlite/proto/ProtobufETLIntegrationTest.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/it/scala/yamrcraft/etlite/proto/ProtobufETLIntegrationTest.scala -------------------------------------------------------------------------------- /etlite/src/main/resources/example_hdfs_application.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/resources/example_hdfs_application.conf -------------------------------------------------------------------------------- /etlite/src/main/resources/example_s3_application.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/resources/example_s3_application.conf -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/EtlException.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/EtlException.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/Main.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/Main.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/Settings.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/Settings.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/pipeline/AbstractProtoPiplineFactory.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/pipeline/AbstractProtoPiplineFactory.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/pipeline/GenericProtoPipelineFactory.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/pipeline/GenericProtoPipelineFactory.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/pipeline/JsonToParquetPipelineFactory.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/pipeline/JsonToParquetPipelineFactory.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/pipeline/JsonToSimpleFilePipelineFactory.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/pipeline/JsonToSimpleFilePipelineFactory.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/pipeline/Pipeline.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/pipeline/Pipeline.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/pipeline/PipelineFactory.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/pipeline/PipelineFactory.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/processors/EtlProcessor.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/processors/EtlProcessor.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/processors/PartitionProcessor.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/processors/PartitionProcessor.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/state/KafkaStateManager.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/state/KafkaStateManager.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/state/StateManager.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/state/StateManager.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/transformers/JsonToAvroTransformer.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/transformers/JsonToAvroTransformer.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/transformers/JsonTransformer.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/transformers/JsonTransformer.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/transformers/Message.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/transformers/Message.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/transformers/ProtoTransformer.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/transformers/ProtoTransformer.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/transformers/Transformer.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/transformers/Transformer.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/utils/ConfigConversions.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/utils/ConfigConversions.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/utils/CustomJsonGenericRecordReader.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/utils/CustomJsonGenericRecordReader.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/utils/DLock.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/utils/DLock.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/utils/FileSystemProvider.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/utils/FileSystemProvider.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/utils/FileUtils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/utils/FileUtils.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/utils/HdfsUtils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/utils/HdfsUtils.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/utils/JsonAvroConverter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/utils/JsonAvroConverter.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/utils/TimeUtils.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/utils/TimeUtils.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/writers/AvroToParquetWriter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/writers/AvroToParquetWriter.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/writers/ErrorEventsWriter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/writers/ErrorEventsWriter.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/writers/ProtoToParquetWriter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/writers/ProtoToParquetWriter.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/writers/StringWriter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/writers/StringWriter.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/writers/TimePartitioningWriter.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/writers/TimePartitioningWriter.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/writers/Writer.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/writers/Writer.scala -------------------------------------------------------------------------------- /etlite/src/main/scala/yamrcraft/etlite/writers/package.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/etlite/src/main/scala/yamrcraft/etlite/writers/package.scala -------------------------------------------------------------------------------- /examples/protobuf-messages/proto/user.proto: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/examples/protobuf-messages/proto/user.proto -------------------------------------------------------------------------------- /examples/protobuf-messages/src/main/java/examples/protobuf/UserOuterClass.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/examples/protobuf-messages/src/main/java/examples/protobuf/UserOuterClass.java -------------------------------------------------------------------------------- /project/Dependencies.scala: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/project/Dependencies.scala -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.11 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yamrcraft/etl-light/HEAD/project/plugins.sbt --------------------------------------------------------------------------------