├── images └── diagram.png ├── fds-lambda-api ├── src │ ├── test │ │ └── java │ │ │ └── com │ │ │ └── provectus │ │ │ └── fds │ │ │ └── api │ │ │ └── ApiFunctionHandlerTest.java │ └── main │ │ └── java │ │ └── com │ │ └── provectus │ │ └── fds │ │ └── api │ │ ├── ClickBcnHandler.java │ │ ├── ImpressionBcnHandler.java │ │ ├── BidBcnHandler.java │ │ └── AbstractBcnHandler.java └── pom.xml ├── fds-lambda-ml-integration ├── src │ ├── main │ │ ├── resources │ │ │ ├── model.tar.gz │ │ │ ├── log4j2.xml │ │ │ └── categorized_bids.sql │ │ └── java │ │ │ └── com │ │ │ └── provectus │ │ │ └── fds │ │ │ └── ml │ │ │ ├── utils │ │ │ └── IntegrationModuleHelper.java │ │ │ ├── processor │ │ │ ├── RecordProcessor.java │ │ │ ├── AthenaConfig.java │ │ │ └── AthenaProcessor.java │ │ │ ├── Configuration.java │ │ │ ├── JobDataGenerator.java │ │ │ ├── SagemakerAlgorithmsRegistry.java │ │ │ ├── LambdaS3EventProxy.java │ │ │ ├── InvokeEndpointLambda.java │ │ │ ├── ReplaceEndpointConfigLambda.java │ │ │ ├── PredictRequest.java │ │ │ └── JobRunner.java │ └── test │ │ └── java │ │ └── com │ │ └── provectus │ │ └── fds │ │ └── ml │ │ └── EndpointUpdaterTest.java └── pom.xml ├── fds-it └── src │ └── test │ ├── java │ └── com │ │ └── provectus │ │ └── fds │ │ └── it │ │ ├── Capability.java │ │ ├── ItConfig.java │ │ ├── SampleMain.java │ │ ├── SampleDataResult.java │ │ ├── AbstarctIntegration.java │ │ ├── BucketRemover.java │ │ └── SampleGenerator.java │ └── resources │ └── log4j.properties ├── fds-lambda-compaction └── src │ ├── test │ ├── resources │ │ ├── testbcn1.json │ │ └── testbcn.json │ └── java │ │ └── com │ │ └── provectus │ │ └── fds │ │ └── compaction │ │ ├── utils │ │ ├── PathFormatterTest.java │ │ ├── JsonParquetConverterTest.java │ │ └── ParquetUtilsTest.java │ │ ├── KinesisFireHoseResponseTest.java │ │ └── TestSchemaBuilder.java │ └── main │ └── java │ ├── com │ └── provectus │ │ └── fds │ │ └── compaction │ │ ├── utils │ │ ├── JsonParquetConverter.java │ │ ├── GlueConverter.java │ │ ├── S3Utils.java │ │ ├── JsonNodeVisitor.java │ │ ├── PathFormatter.java │ │ ├── Period.java │ │ ├── ParquetTripletUtils.java │ │ ├── JsonFileReader.java │ │ ├── SchemaNodeVisitor.java │ │ └── BlocksCombiner.java │ │ ├── JsonNewLineAdderLambda.java │ │ ├── KinesisFirehoseResponse.java │ │ └── GlueDAO.java │ └── org │ └── apache │ └── parquet │ └── hadoop │ └── ColumnReadStorePublicImpl.java ├── gatling ├── conf │ ├── gatling-akka.conf │ ├── logback.xml │ └── recorder.conf └── simulations │ └── basic │ └── ApiPerformanceTest.scala ├── gatling.sh ├── fds-lambda-locations-ingestion ├── src │ ├── main │ │ └── resources │ │ │ └── log4j.properties │ └── test │ │ └── java │ │ └── com │ │ └── provectus │ │ └── fds │ │ └── ingestion │ │ └── LocationsHandlerTest.java └── pom.xml ├── fds-models ├── src │ └── main │ │ └── java │ │ └── com │ │ └── provectus │ │ └── fds │ │ └── models │ │ ├── bcns │ │ ├── Partitioned.java │ │ ├── BcnType.java │ │ ├── ClickBcn.java │ │ ├── ImpressionBcn.java │ │ ├── WalkinClick.java │ │ ├── Bcn.java │ │ ├── Walkin.java │ │ └── BidBcn.java │ │ ├── utils │ │ ├── JsonUtils.java │ │ └── DateTimeUtils.java │ │ └── events │ │ ├── Aggregation.java │ │ ├── Location.java │ │ ├── Click.java │ │ └── Impression.java └── pom.xml ├── fds-lambda-reports ├── src │ └── main │ │ └── java │ │ └── com │ │ └── provectus │ │ └── fds │ │ └── reports │ │ ├── AbstractReportHandler.java │ │ ├── ExecutionContext.java │ │ ├── ApiResponse.java │ │ ├── JsonUtils.java │ │ ├── ApiHandler.java │ │ └── AggregationsReportHandler.java └── pom.xml ├── NOTICE.txt ├── fds-flink-streaming ├── src │ └── main │ │ └── java │ │ └── com │ │ └── provectus │ │ └── fds │ │ └── flink │ │ ├── selectors │ │ └── EventSelector.java │ │ ├── aggregators │ │ ├── AggregationKey.java │ │ ├── MetricsAggregator.java │ │ ├── AggregationWindowProcessor.java │ │ ├── MetricsAccumulator.java │ │ ├── AggregationAccumulator.java │ │ └── Metrics.java │ │ ├── config │ │ └── EventTimeExtractor.java │ │ ├── StreamingApp.java │ │ └── schemas │ │ ├── BcnSchema.java │ │ ├── BidBcnSchema.java │ │ ├── WalkinSchema.java │ │ ├── ClickBcnSchema.java │ │ ├── LocationSchema.java │ │ ├── WalkinClickSchema.java │ │ ├── AggregationSchema.java │ │ └── ImpressionBcnSchema.java └── pom.xml ├── fds-common-dynamodb ├── src │ ├── test │ │ └── java │ │ │ └── com │ │ │ └── provectus │ │ │ └── fds │ │ │ └── dynamodb │ │ │ ├── ZoneDateTimeUtilsTest.java │ │ │ ├── repositories │ │ │ └── AggregationRepositoryTest.java │ │ │ └── ItemMapperTest.java │ └── main │ │ └── java │ │ └── com │ │ └── provectus │ │ └── fds │ │ └── dynamodb │ │ ├── ZoneDateTimeUtils.java │ │ ├── DynamoDAO.java │ │ └── models │ │ └── Aggregation.java └── pom.xml ├── taskcat-run.sh ├── .gitignore ├── settings.xml ├── fds-lambda-utils ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── provectus │ └── fds │ └── utils │ └── ApplicationStartLambda.java ├── .circleci └── config.yml ├── fds-lambda-dynamodb-persister ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── provectus │ └── fds │ └── dynamodb │ └── DynamoDBPersisterLambda.java └── stack.sh /images/diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/provectus/streaming-data-platform/HEAD/images/diagram.png -------------------------------------------------------------------------------- /fds-lambda-api/src/test/java/com/provectus/fds/api/ApiFunctionHandlerTest.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.api; 2 | 3 | 4 | public class ApiFunctionHandlerTest { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /fds-lambda-ml-integration/src/main/resources/model.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/provectus/streaming-data-platform/HEAD/fds-lambda-ml-integration/src/main/resources/model.tar.gz -------------------------------------------------------------------------------- /fds-it/src/test/java/com/provectus/fds/it/Capability.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.it; 2 | 3 | public enum Capability { 4 | CAPABILITY_IAM, 5 | CAPABILITY_AUTO_EXPAND 6 | } 7 | -------------------------------------------------------------------------------- /fds-lambda-compaction/src/test/resources/testbcn1.json: -------------------------------------------------------------------------------- 1 | {"type": "imp", "timestamp": 4384732763, "txid": "3y44326423784323223", "field": {"name": "test"}} 2 | {"type": "click", "timestamp": 4384732773, "txid": "3y4323432642378423"} -------------------------------------------------------------------------------- /gatling/conf/gatling-akka.conf: -------------------------------------------------------------------------------- 1 | akka { 2 | #loggers = ["akka.event.slf4j.Slf4jLogger"] 3 | #logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" 4 | #log-dead-letters = off 5 | actor { 6 | default-dispatcher { 7 | #throughput = 20 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /gatling.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | API=$1 3 | docker run -it --rm -e JAVA_OPTS="-Dduration=60 -DbaseUrl=https://$API.execute-api.us-west-2.amazonaws.com -Dgatling.core.simulationClass=basic.ApiPerformanceTest" -v `pwd`/gatling:/opt/gatling/user-files -v `pwd`/gatling/results:/opt/gatling/results -v `pwd`/gatling/conf:/opt/gatling/conf denvazh/gatling -------------------------------------------------------------------------------- /fds-lambda-locations-ingestion/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log = . 2 | log4j.rootLogger = DEBUG, LAMBDA 3 | log4j.appender.LAMBDA=com.amazonaws.services.lambda.runtime.log4j.LambdaAppender 4 | log4j.appender.LAMBDA.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.LAMBDA.layout.conversionPattern=%d{yyyy-MM-dd HH:mm:ss} <%X{AWSRequestId}> %-5p %c{1}:%m%n -------------------------------------------------------------------------------- /fds-models/src/main/java/com/provectus/fds/models/bcns/Partitioned.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.models.bcns; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnore; 4 | 5 | import java.io.IOException; 6 | 7 | public interface Partitioned { 8 | @JsonIgnore 9 | String getPartitionKey(); 10 | 11 | @JsonIgnore 12 | byte[] getBytes() throws IOException; 13 | } -------------------------------------------------------------------------------- /fds-it/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Set root logger level to INFO and its only appender to A1. 2 | log4j.rootLogger=INFO, A1 3 | 4 | # A1 is set to be a ConsoleAppender. 5 | log4j.appender.A1=org.apache.log4j.ConsoleAppender 6 | 7 | # A1 uses PatternLayout. 8 | log4j.appender.A1.layout=org.apache.log4j.PatternLayout 9 | log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n 10 | -------------------------------------------------------------------------------- /fds-models/src/main/java/com/provectus/fds/models/bcns/BcnType.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.models.bcns; 2 | 3 | public enum BcnType { 4 | BID("bid"), 5 | CLICK("click"), 6 | IMPRESSION("imp"); 7 | 8 | private String code; 9 | 10 | BcnType(String code) { 11 | this.code = code; 12 | } 13 | 14 | public String getCode() { 15 | return code; 16 | } 17 | } -------------------------------------------------------------------------------- /fds-lambda-reports/src/main/java/com/provectus/fds/reports/AbstractReportHandler.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.reports; 2 | 3 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; 4 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; 5 | 6 | public abstract class AbstractReportHandler { 7 | 8 | protected final AmazonDynamoDB client; 9 | 10 | public AbstractReportHandler() { 11 | this.client = AmazonDynamoDBClientBuilder.standard().build(); 12 | } 13 | 14 | 15 | } 16 | -------------------------------------------------------------------------------- /fds-it/src/test/java/com/provectus/fds/it/ItConfig.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.it; 2 | 3 | public interface ItConfig { 4 | String STACK_NAME_PREFIX = "integration"; 5 | String REGION = "us-west-2"; 6 | 7 | String BID_TYPE = "bid"; 8 | String IMP_TYPE = "impression"; 9 | String CLICK_TYPE = "click"; 10 | String LOCATION_TYPE = "location"; 11 | String USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8"; 12 | } 13 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2019 Provectus inc. or its affiliates. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | 5 | http://aws.amazon.com/apache2.0/ 6 | 7 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/selectors/EventSelector.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink.selectors; 2 | 3 | import com.provectus.fds.models.bcns.Bcn; 4 | import org.apache.flink.streaming.api.collector.selector.OutputSelector; 5 | 6 | public class EventSelector implements OutputSelector { 7 | private static final long serialVersionUID = -8133060755314810477L; 8 | 9 | @Override 10 | public Iterable select(Bcn event) { 11 | return java.util.Collections.singletonList(event.getType()); 12 | } 13 | } -------------------------------------------------------------------------------- /fds-lambda-ml-integration/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %d{yyyy-MM-dd HH:mm:ss} %X{AWSRequestId} %-5p %c{1}:%L - %m%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/aggregators/AggregationKey.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink.aggregators; 2 | 3 | import com.provectus.fds.models.events.Aggregation; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | @Data 9 | @NoArgsConstructor 10 | @AllArgsConstructor 11 | public class AggregationKey { 12 | private long campaignItemId; 13 | private String period; 14 | 15 | public static AggregationKey of(Aggregation aggregation) { 16 | return new AggregationKey(aggregation.getCampaignItemId(), aggregation.getPeriod()); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /fds-lambda-compaction/src/test/java/com/provectus/fds/compaction/utils/PathFormatterTest.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction.utils; 2 | 3 | import org.junit.Test; 4 | 5 | import static org.junit.Assert.assertEquals; 6 | 7 | public class PathFormatterTest { 8 | @Test 9 | public void testCreate() { 10 | PathFormatter formatter = PathFormatter.fromS3Path("raw/bcns/2018/12/09/11/filenameuuuid.gz"); 11 | assertEquals("parquet/bcns/year=2018/day=17874", formatter.path("parquet")); 12 | assertEquals("parquet/bcns/year=2018/day=17874/data.parquet", formatter.pathWithFile("parquet", "data.parquet")); 13 | } 14 | } -------------------------------------------------------------------------------- /gatling/conf/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %d{HH:mm:ss.SSS} [%-5level] %logger{15} - %msg%n%rEx 7 | 8 | false 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/config/EventTimeExtractor.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink.config; 2 | 3 | import org.apache.flink.streaming.api.functions.AssignerWithPeriodicWatermarks; 4 | import org.apache.flink.streaming.api.watermark.Watermark; 5 | 6 | public class EventTimeExtractor implements AssignerWithPeriodicWatermarks { 7 | @Override 8 | public Watermark getCurrentWatermark() { 9 | return new Watermark(System.currentTimeMillis() - 5000); 10 | } 11 | 12 | @Override 13 | public long extractTimestamp(T element, long previousElementTimestamp) { 14 | return System.currentTimeMillis(); 15 | } 16 | } -------------------------------------------------------------------------------- /fds-common-dynamodb/src/test/java/com/provectus/fds/dynamodb/ZoneDateTimeUtilsTest.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.dynamodb; 2 | 3 | import org.junit.Test; 4 | 5 | import java.time.Instant; 6 | import java.time.ZoneOffset; 7 | import java.time.ZonedDateTime; 8 | import java.time.temporal.ChronoUnit; 9 | 10 | import static org.junit.Assert.assertEquals; 11 | 12 | public class ZoneDateTimeUtilsTest { 13 | @Test 14 | public void convert() { 15 | ZonedDateTime zdt = 16 | ZoneDateTimeUtils.truncatedTo(Instant.ofEpochSecond(1546041600).atZone(ZoneOffset.UTC), ChronoUnit.DAYS); 17 | 18 | assertEquals(1546041600, zdt.toInstant().getEpochSecond()); 19 | } 20 | 21 | } -------------------------------------------------------------------------------- /taskcat-run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | REGIONS="us-east-1 us-east-2 us-west-2 eu-west-1 eu-central-1" 5 | 6 | for region in $REGIONS; do 7 | service_prefix="sdp-$(cat /dev/urandom | tr -dc 'a-z' | fold -w 7 | head -n 1)" 8 | echo Processing $region 9 | export AWS_DEFAULT_REGION=$region 10 | 11 | cat < .taskcat.yml 12 | project: 13 | name: sdp 14 | regions: 15 | - $region 16 | template: ./fds.yaml 17 | tests: 18 | t1: 19 | parameters: 20 | ServicePrefix: $service_prefix 21 | regions: 22 | - $region 23 | EOT 24 | 25 | ./stack.sh -p -t -c -r ${AWS_DEFAULT_REGION} -v tcat -b fdp-itstack-${AWS_DEFAULT_REGION} 26 | taskcat -d test run 27 | done 28 | 29 | -------------------------------------------------------------------------------- /fds-lambda-compaction/src/main/java/com/provectus/fds/compaction/utils/JsonParquetConverter.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction.utils; 2 | 3 | import org.apache.avro.Schema; 4 | import org.apache.avro.generic.GenericData; 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.fs.FileSystem; 7 | import org.apache.hadoop.fs.Path; 8 | import org.apache.parquet.avro.AvroParquetWriter; 9 | import org.apache.parquet.hadoop.ParquetWriter; 10 | import org.apache.parquet.hadoop.metadata.CompressionCodecName; 11 | 12 | import java.io.File; 13 | import java.io.IOException; 14 | import java.util.Optional; 15 | import java.util.UUID; 16 | 17 | public class JsonParquetConverter { 18 | 19 | 20 | } 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.nar 17 | *.ear 18 | *.zip 19 | *.tar.gz 20 | *.rar 21 | tree 22 | 23 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 24 | hs_err_pid* 25 | 26 | .DS_Store 27 | 28 | .idea/ 29 | target/ 30 | *.iml 31 | fds.yaml 32 | ml.yaml 33 | processing.yaml 34 | packaged-template.yaml 35 | gatling/results 36 | fds-template-autogenerated.yaml 37 | 38 | .classpath 39 | .project 40 | .settings 41 | 42 | dependency-reduced-pom.xml 43 | .taskcat/ 44 | taskcat_outputs/ 45 | .envrc 46 | 47 | .taskcat.yml 48 | 49 | -------------------------------------------------------------------------------- /fds-lambda-compaction/src/test/java/com/provectus/fds/compaction/KinesisFireHoseResponseTest.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction; 2 | 3 | import org.junit.Test; 4 | 5 | import java.nio.ByteBuffer; 6 | import java.util.Base64; 7 | 8 | import static org.junit.Assert.assertEquals; 9 | 10 | public class KinesisFireHoseResponseTest { 11 | @Test 12 | public void testNewLineAppender() { 13 | KinesisFirehoseResponse.FirehoseRecord record = 14 | KinesisFirehoseResponse.FirehoseRecord 15 | .appendNewLine("Test", ByteBuffer.wrap("Test".getBytes())); 16 | 17 | String result = new String(Base64.getDecoder().decode(record.getData())); 18 | assertEquals("Test" + '\n', result); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /fds-lambda-ml-integration/src/test/java/com/provectus/fds/ml/EndpointUpdaterTest.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.ml; 2 | 3 | import org.junit.Test; 4 | 5 | import static org.junit.Assert.*; 6 | 7 | public class EndpointUpdaterTest { 8 | 9 | @Test 10 | public void testGeneratedNames() { 11 | 12 | EndpointUpdater.EndpointUpdaterBuilder builder 13 | = new EndpointUpdater.EndpointUpdaterBuilder(new ReplaceEndpointConfigLambda.LambdaConfiguration()); 14 | 15 | builder.withServicePrefix("0123456789"); 16 | String generatedName 17 | = builder.generateName("EndpointConfiguration"); 18 | 19 | assertTrue(generatedName.contains("EndpointConfiguration")); 20 | assertEquals(63, generatedName.length()); 21 | } 22 | } -------------------------------------------------------------------------------- /fds-common-dynamodb/src/main/java/com/provectus/fds/dynamodb/ZoneDateTimeUtils.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.dynamodb; 2 | 3 | import java.time.DayOfWeek; 4 | import java.time.ZonedDateTime; 5 | import java.time.temporal.ChronoUnit; 6 | 7 | public class ZoneDateTimeUtils { 8 | public static ZonedDateTime truncatedTo(ZonedDateTime zonedDateTime, ChronoUnit chronoUnit) { 9 | switch (chronoUnit) { 10 | case WEEKS: return zonedDateTime.truncatedTo(ChronoUnit.DAYS).with(DayOfWeek.MONDAY); 11 | case MONTHS: return zonedDateTime.truncatedTo(ChronoUnit.DAYS).withDayOfMonth(1); 12 | case YEARS: return zonedDateTime.truncatedTo(ChronoUnit.DAYS).withDayOfMonth(1).withMonth(1); 13 | default: return zonedDateTime.truncatedTo(chronoUnit); 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /fds-lambda-api/src/main/java/com/provectus/fds/api/ClickBcnHandler.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.api; 2 | 3 | import com.amazonaws.services.lambda.runtime.Context; 4 | import com.fasterxml.jackson.databind.JsonNode; 5 | import com.provectus.fds.models.bcns.Partitioned; 6 | import com.provectus.fds.models.bcns.ClickBcn; 7 | 8 | import java.io.IOException; 9 | import java.time.Instant; 10 | import java.util.Optional; 11 | 12 | public class ClickBcnHandler extends AbstractBcnHandler { 13 | 14 | @Override 15 | public Optional buildBcn(JsonNode parameters, Context context) throws IOException { 16 | Optional result = Optional.empty(); 17 | if (parameters.has("tx_id")) { 18 | String txid = parameters.get("tx_id").asText(); 19 | 20 | result = Optional.of(new ClickBcn(txid)); 21 | } 22 | return result; 23 | } 24 | } -------------------------------------------------------------------------------- /fds-lambda-ml-integration/src/main/java/com/provectus/fds/ml/utils/IntegrationModuleHelper.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.ml.utils; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | 6 | /** 7 | * Helper utility class for current module 8 | */ 9 | public class IntegrationModuleHelper { 10 | 11 | /** 12 | * Proxy method for ObjectMapper which hides all of the checked 13 | * exception 14 | * 15 | * @param o Object to serialize into String 16 | * @param mapper Configured ObjectMapper 17 | * @return JSON String of the object 18 | */ 19 | public String writeValueAsString(Object o, ObjectMapper mapper) { 20 | try { 21 | return mapper.writeValueAsString(o); 22 | } catch (JsonProcessingException e) { 23 | throw new RuntimeException(e); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /fds-lambda-ml-integration/src/main/resources/categorized_bids.sql: -------------------------------------------------------------------------------- 1 | select 2 | case when i.tx_id is null then 0 else 1 end positive, 3 | bitwise_and(from_big_endian_64(xxhash64(to_utf8(cast(b.campaign_item_id as varchar)))), 9223372036854775807) / 9223372036854775807.0 as campaign_item_id, 4 | bitwise_and(from_big_endian_64(xxhash64(to_utf8(b.domain))), 9223372036854775807) / 9223372036854775807.0 as domain, 5 | bitwise_and(from_big_endian_64(xxhash64(to_utf8(b.creative_id))), 9223372036854775807) / 9223372036854775807.0 as creative_id, 6 | bitwise_and(from_big_endian_64(xxhash64(to_utf8(b.creative_category))), 9223372036854775807) / 9223372036854775807.0 as creative_category, 7 | coalesce(i.win_price, 0) as win_price 8 | from parquet_bcns b TABLESAMPLE BERNOULLI(100) 9 | left join parquet_impressions i on i.tx_id = b.tx_id 10 | where b.type = 'bid' and 11 | b.day >= round((to_unixtime(current_timestamp) - 86400) / 86400) -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/aggregators/MetricsAggregator.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink.aggregators; 2 | 3 | import org.apache.flink.api.common.functions.AggregateFunction; 4 | 5 | public class MetricsAggregator implements AggregateFunction, Metrics> { 6 | @Override 7 | public AggregationAccumulator createAccumulator() { 8 | 9 | return new AggregationAccumulator<>(); 10 | } 11 | 12 | @Override 13 | public AggregationAccumulator add(T value, AggregationAccumulator accumulator) { 14 | return accumulator.add(value); 15 | } 16 | 17 | @Override 18 | public Metrics getResult(AggregationAccumulator accumulator) { 19 | return accumulator.build(); 20 | } 21 | 22 | @Override 23 | public AggregationAccumulator merge(AggregationAccumulator a, AggregationAccumulator b) { 24 | return a.merge(b); 25 | } 26 | } -------------------------------------------------------------------------------- /fds-lambda-ml-integration/src/main/java/com/provectus/fds/ml/processor/RecordProcessor.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.ml.processor; 2 | 3 | import com.amazonaws.services.athena.model.Row; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * Base interface for record processing 9 | */ 10 | public interface RecordProcessor extends AutoCloseable { 11 | 12 | /** 13 | * Initialize all internal structure if needed 14 | * @throws Exception 15 | */ 16 | void initialize() throws Exception; 17 | 18 | /** 19 | * Process single row 20 | * 21 | * @param row 22 | * @throws Exception 23 | */ 24 | void process(Row row) throws Exception; 25 | 26 | /** 27 | * Returns statistics about processed records. Same implementations 28 | * may have reliable statistic only after method comlete() has been 29 | * called 30 | * 31 | * @return Map with implementation specific values 32 | */ 33 | Map getStatistic(); 34 | } 35 | -------------------------------------------------------------------------------- /fds-models/src/main/java/com/provectus/fds/models/bcns/ClickBcn.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.models.bcns; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import com.provectus.fds.models.utils.JsonUtils; 5 | import lombok.*; 6 | 7 | import java.io.IOException; 8 | 9 | @Builder 10 | @Getter 11 | @Setter 12 | @NoArgsConstructor 13 | @ToString 14 | @EqualsAndHashCode 15 | public class ClickBcn implements Partitioned { 16 | @JsonProperty("tx_id") 17 | private String txId; 18 | 19 | public ClickBcn(@JsonProperty("tx_id") String txId) { 20 | this.txId = txId; 21 | } 22 | 23 | @Override 24 | public String getPartitionKey() { 25 | return txId; 26 | } 27 | 28 | @Override 29 | public byte[] getBytes() throws IOException { 30 | return JsonUtils.write(this); 31 | } 32 | 33 | public static ClickBcn from(Bcn bcn) { 34 | return ClickBcn.builder() 35 | .txId(bcn.getTxId()) 36 | .build(); 37 | } 38 | } -------------------------------------------------------------------------------- /fds-lambda-compaction/src/test/resources/testbcn.json: -------------------------------------------------------------------------------- 1 | {"type": "imp", "timestamp": 4384732743, "txid": "3y4432642378423"} 2 | {"type": "click", "timestamp": 4384732743, "txid": "3y4432642378423"} 3 | {"type": "bid", "timestamp": 4384732743, "campaign_item_id": 12121212, "txid": "3y4432642378423"} 4 | {"type": "bid", "timestamp": 4384732743, "campaign_item_id": 12121212, "txid": "3y4432642378423"} 5 | {"type": "bid", "timestamp": 4384732743, "campaign_item_id": 12121212, "txid": "3y4432642378423"} 6 | {"type": "bid", "timestamp": 4384732743, "campaign_item_id": 12121212, "txid": "3y4432642378423"} 7 | {"type": "bid", "timestamp": 4384732743, "campaign_item_id": 12121212, "txid": "3y4432642378423"} 8 | {"type": "bid", "timestamp": 4384732743, "campaign_item_id": 12121212, "txid": "3y4432642378423"} 9 | {"type": "bid", "timestamp": 4384732743, "campaign_item_id": 12121212, "txid": "3y4432642378423"} 10 | {"type": "bid", "timestamp": 4384732743, "campaign_item_id": 12121212, "domain": "google.com", "txid": "3y4432642378423"} 11 | -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/aggregators/AggregationWindowProcessor.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink.aggregators; 2 | 3 | import com.provectus.fds.models.events.Aggregation; 4 | import com.provectus.fds.models.utils.DateTimeUtils; 5 | import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction; 6 | import org.apache.flink.streaming.api.windowing.windows.TimeWindow; 7 | import org.apache.flink.util.Collector; 8 | 9 | public class AggregationWindowProcessor extends ProcessWindowFunction { 10 | @Override 11 | public void process(Long key, Context context, Iterable result, Collector out) { 12 | Metrics metrics = result.iterator().next(); 13 | 14 | out.collect(new Aggregation(key, 15 | DateTimeUtils.format(context.window().getStart()), 16 | metrics.getBids(), 17 | metrics.getImpressions(), 18 | metrics.getClicks())); 19 | } 20 | } -------------------------------------------------------------------------------- /fds-it/src/test/java/com/provectus/fds/it/SampleMain.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.it; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import org.asynchttpclient.AsyncHttpClient; 6 | 7 | import static org.asynchttpclient.Dsl.asyncHttpClient; 8 | import static org.asynchttpclient.Dsl.config; 9 | 10 | public class SampleMain { 11 | 12 | final static AsyncHttpClient httpClient = asyncHttpClient(config()); 13 | final static ObjectMapper objectMapper = new ObjectMapper(); 14 | 15 | public static void main(String[] args) throws JsonProcessingException { 16 | System.out.println("Starting generate sample data"); 17 | SampleDataResult result = new SampleGenerator(50, 60, 18 | "https://x4e3w5eif9.execute-api.us-west-2.amazonaws.com/integration8009b5224f95441ab22", 19 | httpClient, objectMapper).generateSampleData(); 20 | 21 | System.out.println("Data generation finished"); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /fds-lambda-reports/src/main/java/com/provectus/fds/reports/ExecutionContext.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.reports; 2 | 3 | import java.util.Objects; 4 | 5 | public class ExecutionContext { 6 | private final String method; 7 | private final String path; 8 | 9 | public ExecutionContext(String method, String path) { 10 | this.method = method; 11 | this.path = path; 12 | } 13 | 14 | public ExecutionContext(ExecutionValues values) { 15 | this.method = values.getHttpMethod(); 16 | this.path = values.getResource(); 17 | } 18 | 19 | @Override 20 | public boolean equals(Object o) { 21 | if (this == o) return true; 22 | if (o == null || getClass() != o.getClass()) return false; 23 | ExecutionContext that = (ExecutionContext) o; 24 | return Objects.equals(method, that.method) && 25 | Objects.equals(path, that.path); 26 | } 27 | 28 | @Override 29 | public int hashCode() { 30 | return Objects.hash(method, path); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/StreamingApp.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink; 2 | 3 | import com.provectus.fds.flink.config.StreamingProperties; 4 | import org.apache.flink.streaming.api.TimeCharacteristic; 5 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | public class StreamingApp { 10 | private static final Logger log = LoggerFactory.getLogger(StreamingApp.class); 11 | 12 | private static StreamingProperties properties = StreamingProperties.fromRuntime(); 13 | 14 | public static void main(String[] args) throws Exception { 15 | StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment(); 16 | environment.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); 17 | 18 | new StreamingJob(environment, properties); 19 | 20 | environment.execute("Streaming Data Platform"); 21 | 22 | log.info("StreamingApp has started"); 23 | } 24 | } -------------------------------------------------------------------------------- /fds-lambda-reports/src/main/java/com/provectus/fds/reports/ApiResponse.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.reports; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | public class ApiResponse { 7 | private final int statusCode; 8 | private final String body; 9 | private final boolean isBase64Encoded = false; 10 | private final Map headers = new HashMap<>(); 11 | 12 | public ApiResponse(int statusCode, String body) { 13 | this.statusCode = statusCode; 14 | this.body = body; 15 | } 16 | 17 | public ApiResponse addHeader(String key, String value) { 18 | this.headers.put(key, value); 19 | return this; 20 | } 21 | 22 | public int getStatusCode() { 23 | return statusCode; 24 | } 25 | 26 | public String getBody() { 27 | return body; 28 | } 29 | 30 | public boolean isBase64Encoded() { 31 | return isBase64Encoded; 32 | } 33 | 34 | public Map getHeaders() { 35 | return headers; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /fds-lambda-api/src/main/java/com/provectus/fds/api/ImpressionBcnHandler.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.api; 2 | 3 | 4 | import com.amazonaws.services.lambda.runtime.Context; 5 | import com.fasterxml.jackson.databind.JsonNode; 6 | import com.provectus.fds.models.bcns.Partitioned; 7 | import com.provectus.fds.models.bcns.ImpressionBcn; 8 | 9 | import java.io.IOException; 10 | import java.time.Instant; 11 | import java.util.Optional; 12 | 13 | public class ImpressionBcnHandler extends AbstractBcnHandler { 14 | 15 | @Override 16 | public Optional buildBcn(JsonNode parameters, Context context) throws IOException { 17 | 18 | Optional result = Optional.empty(); 19 | 20 | if (parameters.has("tx_id") && parameters.has("win_price")) { 21 | String txid = parameters.get("tx_id").asText(); 22 | long winPrice = parameters.get("win_price").asLong(); 23 | 24 | result = Optional.of(new ImpressionBcn(txid, winPrice)); 25 | } 26 | 27 | return result; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /fds-lambda-compaction/src/test/java/com/provectus/fds/compaction/utils/JsonParquetConverterTest.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction.utils; 2 | 3 | import org.apache.parquet.hadoop.ParquetFileReader; 4 | import org.junit.Test; 5 | 6 | import java.io.File; 7 | import java.io.IOException; 8 | import java.nio.file.Files; 9 | 10 | import static org.junit.Assert.*; 11 | 12 | public class JsonParquetConverterTest { 13 | 14 | private final File tmpDir = Files.createTempDirectory("s3").toFile(); 15 | private final ParquetUtils parquetUtils = new ParquetUtils(); 16 | 17 | public JsonParquetConverterTest() throws IOException { 18 | tmpDir.deleteOnExit(); 19 | } 20 | 21 | @Test 22 | public void testJsonFileConvert() throws IOException { 23 | ClassLoader classLoader = getClass().getClassLoader(); 24 | File testBcnFile = new File(classLoader.getResource("testbcn.json").getFile()); 25 | File result = parquetUtils.convert(tmpDir,testBcnFile, "prefix"); 26 | result.deleteOnExit(); 27 | assertTrue(result.exists()); 28 | } 29 | 30 | } -------------------------------------------------------------------------------- /fds-common-dynamodb/src/test/java/com/provectus/fds/dynamodb/repositories/AggregationRepositoryTest.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.dynamodb.repositories; 2 | 3 | import com.provectus.fds.dynamodb.models.Aggregation; 4 | import org.junit.Test; 5 | 6 | import java.time.ZoneOffset; 7 | import java.time.temporal.ChronoUnit; 8 | import java.util.Arrays; 9 | import java.util.List; 10 | 11 | import static org.junit.Assert.*; 12 | 13 | public class AggregationRepositoryTest { 14 | 15 | @Test 16 | public void getGrouped() { 17 | AggregationRepository repository = new AggregationRepository(); 18 | List aggregations = Arrays.asList( 19 | new Aggregation(-1987217790,1546068600,0L,0L,17986L) 20 | ); 21 | 22 | List expected = Arrays.asList( 23 | new Aggregation(-1987217790,1546041600,0L,0L,17986L) 24 | ); 25 | 26 | List result = repository.getGrouped(aggregations, ChronoUnit.DAYS, ZoneOffset.UTC, false); 27 | assertArrayEquals(expected.toArray(), result.toArray()); 28 | 29 | } 30 | } -------------------------------------------------------------------------------- /fds-it/src/test/java/com/provectus/fds/it/SampleDataResult.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.it; 2 | 3 | public class SampleDataResult { 4 | 5 | long campaignItemId; 6 | int countOfBids; 7 | int countOfImpressions; 8 | int countOfClicks; 9 | 10 | public SampleDataResult(long campaignItemId) { 11 | this.campaignItemId = campaignItemId; 12 | } 13 | 14 | public long getCampaignItemId() { 15 | return campaignItemId; 16 | } 17 | 18 | public int getCountOfBids() { 19 | return countOfBids; 20 | } 21 | 22 | public int getCountOfImpressions() { 23 | return countOfImpressions; 24 | } 25 | 26 | public int getCountOfClicks() { 27 | return countOfClicks; 28 | } 29 | 30 | public void setCountOfBids(int countOfBids) { 31 | this.countOfBids = countOfBids; 32 | } 33 | 34 | public void setCountOfImpressions(int countOfImpressions) { 35 | this.countOfImpressions = countOfImpressions; 36 | } 37 | 38 | public void setCountOfClicks(int countOfClicks) { 39 | this.countOfClicks = countOfClicks; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /fds-it/src/test/java/com/provectus/fds/it/AbstarctIntegration.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.it; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import org.asynchttpclient.AsyncHttpClient; 6 | 7 | import static org.asynchttpclient.Dsl.asyncHttpClient; 8 | import static org.asynchttpclient.Dsl.config; 9 | 10 | abstract class AbstarctIntegration { 11 | 12 | static final String URL_FOR_API = "UrlForAPI"; 13 | static final String URL_FOR_REPORTS = "UrlForReports"; 14 | static final String URL_FOR_PREDICTIONS = "UrlForPredictions"; 15 | 16 | static CloudFormation cloudFormation; 17 | static String reportUrl; 18 | static String apiUrl; 19 | static String predictionsUrl; 20 | 21 | final AsyncHttpClient httpClient = asyncHttpClient(config()); 22 | final ObjectMapper objectMapper = new ObjectMapper(); 23 | 24 | SampleDataResult generateSampleData(int minBids, int maxBids) throws JsonProcessingException { 25 | return new SampleGenerator(minBids, maxBids, apiUrl, httpClient, objectMapper).generateSampleData(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /fds-lambda-locations-ingestion/src/test/java/com/provectus/fds/ingestion/LocationsHandlerTest.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.ingestion; 2 | 3 | import com.provectus.fds.models.events.Location; 4 | import com.provectus.fds.models.utils.JsonUtils; 5 | import org.junit.Test; 6 | 7 | import java.io.IOException; 8 | 9 | import static org.junit.Assert.assertEquals; 10 | 11 | public class LocationsHandlerTest { 12 | @Test 13 | public void shouldParseLocation() { 14 | String row = "app1,1557705600,55.789609,49.124116"; 15 | Location expected = getLocation(); 16 | 17 | assertEquals(expected, Location.from(row.split(","))); 18 | } 19 | 20 | @Test 21 | public void shouldConvertLocation() throws IOException { 22 | Location expected = getLocation(); 23 | 24 | assertEquals(expected, JsonUtils.read(JsonUtils.write(expected), Location.class)); 25 | } 26 | 27 | private Location getLocation() { 28 | return Location.builder() 29 | .appUID("app1") 30 | .timestamp(1557705600) 31 | .longitude(55.789609) 32 | .latitude(49.124116) 33 | .build(); 34 | } 35 | } -------------------------------------------------------------------------------- /fds-lambda-compaction/src/main/java/com/provectus/fds/compaction/JsonNewLineAdderLambda.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction; 2 | 3 | import com.amazonaws.services.lambda.runtime.Context; 4 | import com.amazonaws.services.lambda.runtime.RequestHandler; 5 | import com.amazonaws.services.lambda.runtime.events.KinesisFirehoseEvent; 6 | 7 | import java.util.stream.Collectors; 8 | 9 | public class JsonNewLineAdderLambda implements RequestHandler { 10 | 11 | @Override 12 | public KinesisFirehoseResponse handleRequest(KinesisFirehoseEvent kinesisFirehoseEvent, Context context) { 13 | context.getLogger().log("Received: "+kinesisFirehoseEvent.toString()); 14 | KinesisFirehoseResponse response = new KinesisFirehoseResponse( 15 | kinesisFirehoseEvent.getRecords() 16 | .stream().map( 17 | r -> KinesisFirehoseResponse.FirehoseRecord.appendNewLine(r.getRecordId(), r.getData()) 18 | ).collect(Collectors.toList()) 19 | ); 20 | context.getLogger().log("Added lines to "+response.size()+" events"); 21 | 22 | return response; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /fds-models/src/main/java/com/provectus/fds/models/utils/JsonUtils.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.models.utils; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.JsonNode; 5 | import com.fasterxml.jackson.databind.ObjectMapper; 6 | 7 | import java.io.IOException; 8 | import java.io.Reader; 9 | 10 | public class JsonUtils { 11 | private static final ObjectMapper objectMapper = new ObjectMapper(); 12 | 13 | public static JsonNode readTree(Reader reader) throws IOException { 14 | return objectMapper.readTree(reader); 15 | } 16 | 17 | public static byte[] write(T value) throws JsonProcessingException { 18 | return objectMapper.writeValueAsBytes(value); 19 | } 20 | 21 | public static T read(Reader reader, Class clazz) throws IOException { 22 | return objectMapper.readerFor(clazz).readValue(reader); 23 | } 24 | 25 | public static T read(byte[] bytes, Class clazz) throws IOException { 26 | return objectMapper.readerFor(clazz).readValue(bytes); 27 | } 28 | 29 | public static String writeAsString(T object) throws IOException { 30 | return objectMapper.writeValueAsString(object); 31 | } 32 | } -------------------------------------------------------------------------------- /fds-lambda-reports/src/main/java/com/provectus/fds/reports/JsonUtils.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.reports; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.DeserializationFeature; 5 | import com.fasterxml.jackson.databind.JsonNode; 6 | import com.fasterxml.jackson.databind.ObjectMapper; 7 | 8 | import java.io.IOException; 9 | import java.io.Reader; 10 | 11 | public class JsonUtils { 12 | private static final ObjectMapper objectMapper = new ObjectMapper(); 13 | { 14 | objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); 15 | } 16 | 17 | public static JsonNode readTree(Reader reader) throws IOException { 18 | return objectMapper.readTree(reader); 19 | } 20 | 21 | public static byte[] write(T value) throws JsonProcessingException { 22 | return objectMapper.writeValueAsBytes(value); 23 | } 24 | 25 | public static T read(Reader reader, Class clazz) throws IOException { 26 | return objectMapper.readerFor(clazz).readValue(reader); 27 | } 28 | 29 | 30 | public static String stringify(T object) throws JsonProcessingException { 31 | return objectMapper.writeValueAsString(object); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /fds-models/src/main/java/com/provectus/fds/models/bcns/ImpressionBcn.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.models.bcns; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import com.provectus.fds.models.utils.JsonUtils; 5 | import lombok.*; 6 | 7 | import java.io.IOException; 8 | 9 | @Builder 10 | @Getter 11 | @Setter 12 | @NoArgsConstructor 13 | @ToString 14 | @EqualsAndHashCode 15 | public class ImpressionBcn implements Partitioned { 16 | @JsonProperty("tx_id") 17 | private String txId; 18 | 19 | @JsonProperty("win_price") 20 | private long winPrice; 21 | 22 | public ImpressionBcn( 23 | @JsonProperty("tx_id") String txId, 24 | @JsonProperty("win_price") long winPrice) { 25 | this.txId = txId; 26 | this.winPrice = winPrice; 27 | } 28 | 29 | @Override 30 | public String getPartitionKey() { 31 | return txId; 32 | } 33 | 34 | @Override 35 | public byte[] getBytes() throws IOException { 36 | return JsonUtils.write(this); 37 | } 38 | 39 | public static ImpressionBcn from(Bcn bcn) { 40 | return ImpressionBcn.builder() 41 | .txId(bcn.getTxId()) 42 | .winPrice(bcn.getWinPrice()) 43 | .build(); 44 | } 45 | } -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/schemas/BcnSchema.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink.schemas; 2 | 3 | import com.provectus.fds.models.bcns.Bcn; 4 | import com.provectus.fds.models.utils.JsonUtils; 5 | import org.apache.flink.api.common.serialization.DeserializationSchema; 6 | import org.apache.flink.api.common.serialization.SerializationSchema; 7 | import org.apache.flink.api.common.typeinfo.TypeInformation; 8 | import org.apache.flink.api.java.typeutils.TypeExtractor; 9 | 10 | import java.io.IOException; 11 | 12 | public class BcnSchema implements SerializationSchema, DeserializationSchema { 13 | @Override 14 | public byte[] serialize(Bcn bcn) { 15 | try { 16 | return JsonUtils.write(bcn); 17 | } catch (Exception e) { 18 | throw new RuntimeException("Error during Bcn serialization", e); 19 | } 20 | } 21 | 22 | @Override 23 | public Bcn deserialize(byte[] bytes) throws IOException { 24 | return JsonUtils.read(bytes, Bcn.class); 25 | } 26 | 27 | @Override 28 | public boolean isEndOfStream(Bcn bcn) { 29 | return false; 30 | } 31 | 32 | @Override 33 | public TypeInformation getProducedType() { 34 | return TypeExtractor.getForClass(Bcn.class); 35 | } 36 | } -------------------------------------------------------------------------------- /fds-models/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | fds-parent 6 | com.provectus.fds 7 | 1.0-SNAPSHOT 8 | 9 | 10 | 4.0.0 11 | fds-models 12 | 13 | 14 | 15 | com.fasterxml.jackson.core 16 | jackson-core 17 | ${com.fasterxml.jackson.version} 18 | 19 | 20 | 21 | com.fasterxml.jackson.core 22 | jackson-databind 23 | ${com.fasterxml.jackson.version} 24 | 25 | 26 | 27 | org.projectlombok 28 | lombok 29 | ${lombok.version} 30 | provided 31 | 32 | 33 | -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/schemas/BidBcnSchema.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink.schemas; 2 | 3 | import com.provectus.fds.models.bcns.BidBcn; 4 | import com.provectus.fds.models.utils.JsonUtils; 5 | import org.apache.flink.api.common.serialization.DeserializationSchema; 6 | import org.apache.flink.api.common.serialization.SerializationSchema; 7 | import org.apache.flink.api.common.typeinfo.TypeInformation; 8 | import org.apache.flink.api.java.typeutils.TypeExtractor; 9 | 10 | import java.io.IOException; 11 | 12 | public class BidBcnSchema implements SerializationSchema, DeserializationSchema { 13 | @Override 14 | public byte[] serialize(BidBcn bid) { 15 | try { 16 | return JsonUtils.write(bid); 17 | } catch (Exception e) { 18 | throw new RuntimeException("Error during BidBcn serialization", e); 19 | } 20 | } 21 | 22 | @Override 23 | public BidBcn deserialize(byte[] bytes) throws IOException { 24 | return JsonUtils.read(bytes, BidBcn.class); 25 | } 26 | 27 | @Override 28 | public boolean isEndOfStream(BidBcn bid) { 29 | return false; 30 | } 31 | 32 | @Override 33 | public TypeInformation getProducedType() { 34 | return TypeExtractor.getForClass(BidBcn.class); 35 | } 36 | } -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/schemas/WalkinSchema.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink.schemas; 2 | 3 | import com.provectus.fds.models.bcns.Walkin; 4 | import com.provectus.fds.models.utils.JsonUtils; 5 | import org.apache.flink.api.common.serialization.DeserializationSchema; 6 | import org.apache.flink.api.common.serialization.SerializationSchema; 7 | import org.apache.flink.api.common.typeinfo.TypeInformation; 8 | import org.apache.flink.api.java.typeutils.TypeExtractor; 9 | 10 | import java.io.IOException; 11 | 12 | public class WalkinSchema implements SerializationSchema, DeserializationSchema { 13 | @Override 14 | public byte[] serialize(Walkin walkin) { 15 | try { 16 | return JsonUtils.write(walkin); 17 | } catch (Exception e) { 18 | throw new RuntimeException("Error during Walkin serialization", e); 19 | } 20 | } 21 | 22 | @Override 23 | public Walkin deserialize(byte[] bytes) throws IOException { 24 | return JsonUtils.read(bytes, Walkin.class); 25 | } 26 | 27 | @Override 28 | public boolean isEndOfStream(Walkin walkin) { 29 | return false; 30 | } 31 | 32 | @Override 33 | public TypeInformation getProducedType() { 34 | return TypeExtractor.getForClass(Walkin.class); 35 | } 36 | } -------------------------------------------------------------------------------- /fds-lambda-compaction/src/main/java/com/provectus/fds/compaction/utils/GlueConverter.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction.utils; 2 | 3 | import com.amazonaws.services.glue.model.Column; 4 | import com.amazonaws.services.glue.model.StorageDescriptor; 5 | import org.apache.parquet.hadoop.metadata.FileMetaData; 6 | import org.apache.parquet.schema.MessageType; 7 | 8 | import java.util.List; 9 | import java.util.Map; 10 | import java.util.stream.Collectors; 11 | 12 | public class GlueConverter { 13 | private static GlueTypeConverter glueTypeConverter = new GlueTypeConverter(); 14 | 15 | public static StorageDescriptor convertStorage(FileMetaData metaData, String location, StorageDescriptor insd) { 16 | StorageDescriptor sd = insd.clone(); 17 | sd.setLocation(location); 18 | sd.setColumns(convertColumns(metaData.getSchema())); 19 | return sd; 20 | } 21 | 22 | public static List convertColumns(MessageType messageType) { 23 | return glueTypeConverter.convert(messageType) 24 | .stream() 25 | .map(GlueConverter::convertColumn) 26 | .collect(Collectors.toList()); 27 | } 28 | 29 | public static Column convertColumn(Map.Entry entry) { 30 | return new Column().withName(entry.getKey()).withType(entry.getValue()); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/schemas/ClickBcnSchema.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink.schemas; 2 | 3 | import com.provectus.fds.models.bcns.ClickBcn; 4 | import com.provectus.fds.models.utils.JsonUtils; 5 | import org.apache.flink.api.common.serialization.DeserializationSchema; 6 | import org.apache.flink.api.common.serialization.SerializationSchema; 7 | import org.apache.flink.api.common.typeinfo.TypeInformation; 8 | import org.apache.flink.api.java.typeutils.TypeExtractor; 9 | 10 | import java.io.IOException; 11 | 12 | public class ClickBcnSchema implements SerializationSchema, DeserializationSchema { 13 | @Override 14 | public byte[] serialize(ClickBcn click) { 15 | try { 16 | return JsonUtils.write(click); 17 | } catch (Exception e) { 18 | throw new RuntimeException("Error during ClickBcn serialization", e); 19 | } 20 | } 21 | 22 | @Override 23 | public ClickBcn deserialize(byte[] bytes) throws IOException { 24 | return JsonUtils.read(bytes, ClickBcn.class); 25 | } 26 | 27 | @Override 28 | public boolean isEndOfStream(ClickBcn click) { 29 | return false; 30 | } 31 | 32 | @Override 33 | public TypeInformation getProducedType() { 34 | return TypeExtractor.getForClass(ClickBcn.class); 35 | } 36 | } -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/schemas/LocationSchema.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink.schemas; 2 | 3 | import com.provectus.fds.models.events.Location; 4 | import com.provectus.fds.models.utils.JsonUtils; 5 | import org.apache.flink.api.common.serialization.DeserializationSchema; 6 | import org.apache.flink.api.common.serialization.SerializationSchema; 7 | import org.apache.flink.api.common.typeinfo.TypeInformation; 8 | import org.apache.flink.api.java.typeutils.TypeExtractor; 9 | 10 | import java.io.IOException; 11 | 12 | public class LocationSchema implements SerializationSchema, DeserializationSchema { 13 | @Override 14 | public byte[] serialize(Location location) { 15 | try { 16 | return JsonUtils.write(location); 17 | } catch (Exception e) { 18 | throw new RuntimeException("Error during Location serialization", e); 19 | } 20 | } 21 | 22 | @Override 23 | public Location deserialize(byte[] bytes) throws IOException { 24 | return JsonUtils.read(bytes, Location.class); 25 | } 26 | 27 | @Override 28 | public boolean isEndOfStream(Location location) { 29 | return false; 30 | } 31 | 32 | @Override 33 | public TypeInformation getProducedType() { 34 | return TypeExtractor.getForClass(Location.class); 35 | } 36 | } -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/aggregators/MetricsAccumulator.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink.aggregators; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | 6 | @Getter 7 | @Setter 8 | public class MetricsAccumulator { 9 | private long bids; 10 | private long impressions; 11 | private long clicks; 12 | 13 | public MetricsAccumulator() { 14 | } 15 | 16 | public MetricsAccumulator(MetricsAccumulator acc) { 17 | this.bids = acc.bids; 18 | this.impressions = acc.impressions; 19 | this.clicks = acc.clicks; 20 | } 21 | 22 | public MetricsAccumulator add(Metrics metrics) { 23 | bids += metrics.getBids(); 24 | impressions += metrics.getImpressions(); 25 | clicks += metrics.getClicks(); 26 | 27 | return this; 28 | } 29 | 30 | public MetricsAccumulator merge(MetricsAccumulator other) { 31 | MetricsAccumulator newAcc = new MetricsAccumulator(this); 32 | newAcc.bids += other.bids; 33 | newAcc.impressions += other.impressions; 34 | newAcc.clicks += other.clicks; 35 | 36 | return newAcc; 37 | } 38 | 39 | public Metrics build() { 40 | return new Metrics.MetricsBuilder() 41 | .bids(bids) 42 | .impressions(impressions) 43 | .clicks(clicks) 44 | .build(); 45 | } 46 | } -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/schemas/WalkinClickSchema.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink.schemas; 2 | 3 | import com.provectus.fds.models.bcns.WalkinClick; 4 | import com.provectus.fds.models.utils.JsonUtils; 5 | import org.apache.flink.api.common.serialization.DeserializationSchema; 6 | import org.apache.flink.api.common.serialization.SerializationSchema; 7 | import org.apache.flink.api.common.typeinfo.TypeInformation; 8 | import org.apache.flink.api.java.typeutils.TypeExtractor; 9 | 10 | import java.io.IOException; 11 | 12 | public class WalkinClickSchema implements SerializationSchema, DeserializationSchema { 13 | @Override 14 | public byte[] serialize(WalkinClick walkinClick) { 15 | try { 16 | return JsonUtils.write(walkinClick); 17 | } catch (Exception e) { 18 | throw new RuntimeException("Error during WalkinClick serialization", e); 19 | } 20 | } 21 | 22 | @Override 23 | public WalkinClick deserialize(byte[] bytes) throws IOException { 24 | return JsonUtils.read(bytes, WalkinClick.class); 25 | } 26 | 27 | @Override 28 | public boolean isEndOfStream(WalkinClick walkinClick) { 29 | return false; 30 | } 31 | 32 | @Override 33 | public TypeInformation getProducedType() { 34 | return TypeExtractor.getForClass(WalkinClick.class); 35 | } 36 | } -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/schemas/AggregationSchema.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink.schemas; 2 | 3 | import com.provectus.fds.models.events.Aggregation; 4 | import com.provectus.fds.models.utils.JsonUtils; 5 | import org.apache.flink.api.common.serialization.DeserializationSchema; 6 | import org.apache.flink.api.common.serialization.SerializationSchema; 7 | import org.apache.flink.api.common.typeinfo.TypeInformation; 8 | import org.apache.flink.api.java.typeutils.TypeExtractor; 9 | 10 | import java.io.IOException; 11 | 12 | public class AggregationSchema implements SerializationSchema, DeserializationSchema { 13 | @Override 14 | public byte[] serialize(Aggregation aggregation) { 15 | try { 16 | return JsonUtils.write(aggregation); 17 | } catch (Exception e) { 18 | throw new RuntimeException("Error during Aggregation serialization", e); 19 | } 20 | } 21 | 22 | @Override 23 | public Aggregation deserialize(byte[] bytes) throws IOException { 24 | return JsonUtils.read(bytes, Aggregation.class); 25 | } 26 | 27 | @Override 28 | public boolean isEndOfStream(Aggregation aggregation) { 29 | return false; 30 | } 31 | 32 | @Override 33 | public TypeInformation getProducedType() { 34 | return TypeExtractor.getForClass(Aggregation.class); 35 | } 36 | } -------------------------------------------------------------------------------- /fds-common-dynamodb/src/main/java/com/provectus/fds/dynamodb/DynamoDAO.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.dynamodb; 2 | 3 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; 4 | import com.amazonaws.services.dynamodbv2.document.*; 5 | import com.amazonaws.services.dynamodbv2.model.BatchWriteItemResult; 6 | 7 | import java.util.Collection; 8 | import java.util.List; 9 | 10 | public class DynamoDAO { 11 | private final String tableName; 12 | private final DynamoDB client; 13 | private final Table table; 14 | 15 | public DynamoDAO(String tableName, AmazonDynamoDB amazonDynamoDB) { 16 | this.client = new DynamoDB(amazonDynamoDB); 17 | this.tableName = tableName; 18 | this.table = client.getTable(this.tableName); 19 | } 20 | 21 | public List batchGet(Collection keys) { 22 | BatchGetItemOutcome result = client.batchGetItem( 23 | new TableKeysAndAttributes(tableName).withPrimaryKeys(keys.toArray(new PrimaryKey[keys.size()])) 24 | ); 25 | return result.getTableItems().get(tableName); 26 | } 27 | 28 | public BatchWriteItemResult batchWrite(Collection items) { 29 | TableWriteItems threadTableWriteItems = new TableWriteItems(tableName).withItemsToPut(items); 30 | BatchWriteItemOutcome result = client.batchWriteItem(threadTableWriteItems); 31 | return result.getBatchWriteItemResult(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/schemas/ImpressionBcnSchema.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink.schemas; 2 | 3 | import com.provectus.fds.models.bcns.ImpressionBcn; 4 | import com.provectus.fds.models.utils.JsonUtils; 5 | import org.apache.flink.api.common.serialization.DeserializationSchema; 6 | import org.apache.flink.api.common.serialization.SerializationSchema; 7 | import org.apache.flink.api.common.typeinfo.TypeInformation; 8 | import org.apache.flink.api.java.typeutils.TypeExtractor; 9 | 10 | import java.io.IOException; 11 | 12 | public class ImpressionBcnSchema implements SerializationSchema, DeserializationSchema { 13 | @Override 14 | public byte[] serialize(ImpressionBcn impression) { 15 | try { 16 | return JsonUtils.write(impression); 17 | } catch (Exception e) { 18 | throw new RuntimeException("Error during ImpressionBcn serialization", e); 19 | } 20 | } 21 | 22 | 23 | @Override 24 | public ImpressionBcn deserialize(byte[] bytes) throws IOException { 25 | return JsonUtils.read(bytes, ImpressionBcn.class); 26 | } 27 | 28 | @Override 29 | public boolean isEndOfStream(ImpressionBcn impression) { 30 | return false; 31 | } 32 | 33 | @Override 34 | public TypeInformation getProducedType() { 35 | return TypeExtractor.getForClass(ImpressionBcn.class); 36 | } 37 | } -------------------------------------------------------------------------------- /fds-lambda-compaction/src/main/java/com/provectus/fds/compaction/utils/S3Utils.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction.utils; 2 | 3 | import com.amazonaws.services.s3.model.S3Object; 4 | import com.amazonaws.services.s3.model.S3ObjectInputStream; 5 | import org.apache.hadoop.fs.Path; 6 | 7 | import java.io.File; 8 | import java.io.FileOutputStream; 9 | import java.io.InputStream; 10 | import java.io.IOException; 11 | import java.util.zip.GZIPInputStream; 12 | 13 | public class S3Utils { 14 | public static File downloadFile(S3Object object, String fileName) throws IOException { 15 | File outputFile = File.createTempFile(object.getBucketName(), fileName); 16 | InputStream s3is = null; 17 | 18 | try (S3ObjectInputStream s3gs = object.getObjectContent(); 19 | FileOutputStream fos = new FileOutputStream(outputFile)) { 20 | 21 | InputStream fileStream = s3gs; 22 | 23 | if (fileName.endsWith(".gz")) { 24 | s3is = new GZIPInputStream(s3gs); 25 | fileStream = s3is; 26 | } 27 | 28 | byte[] read_buf = new byte[1024]; 29 | int read_len = 0; 30 | while ((read_len = fileStream.read(read_buf)) > 0) { 31 | fos.write(read_buf, 0, read_len); 32 | } 33 | } finally { 34 | if (s3is!=null) { 35 | s3is.close(); 36 | } 37 | } 38 | return outputFile; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /fds-lambda-compaction/src/main/java/com/provectus/fds/compaction/utils/JsonNodeVisitor.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction.utils; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import com.fasterxml.jackson.databind.node.*; 5 | 6 | import java.util.Optional; 7 | 8 | public interface JsonNodeVisitor { 9 | Optional visitObject(String name, JsonNode node); 10 | 11 | Optional visitArray(String name, ArrayNode node); 12 | 13 | Optional visitBinary(String name, BinaryNode binaryNode); 14 | 15 | Optional visitText(String name, TextNode binaryNode); 16 | 17 | Optional visitNumber(String name, NumericNode numericNode); 18 | 19 | Optional visitBoolean(String name, BooleanNode booleanNode); 20 | 21 | default Optional visit(String name, JsonNode node) { 22 | switch(node.getNodeType()) { 23 | case OBJECT: 24 | return visitObject(name, node); 25 | case ARRAY: 26 | return visitArray(name, (ArrayNode)node); 27 | case BINARY: 28 | return visitBinary(name, (BinaryNode)node); 29 | case STRING: 30 | return visitText(name, (TextNode)node); 31 | case NUMBER: 32 | return visitNumber(name, (NumericNode)node); 33 | case BOOLEAN: 34 | return visitBoolean(name, (BooleanNode)node); 35 | default: 36 | return Optional.empty(); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | ${env.CI_ARTIFACTORY_USER_NAME} 7 | ${env.CI_ARTIFACTORY_USER_PASSWORD} 8 | release 9 | 10 | 11 | ${env.CI_ARTIFACTORY_USER_NAME} 12 | ${env.CI_ARTIFACTORY_USER_PASSWORD} 13 | snapshot 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | false 22 | 23 | release 24 | libs-release 25 | https://artifactory.provectus.com/artifactory/libs-release 26 | 27 | 28 | 29 | snapshot 30 | libs-snapshot 31 | https://artifactory.provectus.com/artifactory/libs-release 32 | 33 | 34 | artifactory 35 | 36 | 37 | 38 | artifactory 39 | 40 | 41 | -------------------------------------------------------------------------------- /fds-lambda-ml-integration/src/main/java/com/provectus/fds/ml/Configuration.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.ml; 2 | 3 | public class Configuration { 4 | 5 | private String region; 6 | private String bucket; 7 | private String endpoint; 8 | private String sagemakerRole; 9 | 10 | private String get(String key) { 11 | return System.getenv(key); 12 | } 13 | 14 | String getOrElse(String key, String defaultValue) { 15 | String result = get(key); 16 | 17 | if (result == null) { 18 | return defaultValue; 19 | } 20 | 21 | return result; 22 | } 23 | 24 | String getOrThrow(String key) { 25 | String result = get(key); 26 | 27 | if (result == null) { 28 | throw new IllegalStateException(String.format("The variable '%s' is not defined with current environment", key)); 29 | } 30 | 31 | return result; 32 | } 33 | 34 | String getRegion() { 35 | if (region == null) 36 | region = getOrThrow("REGION"); 37 | return region; 38 | } 39 | String getBucket() { 40 | if (bucket == null) 41 | bucket = getOrThrow("S3_BUCKET"); 42 | return bucket; 43 | } 44 | 45 | String getEndpoint() { 46 | if (endpoint == null) 47 | endpoint = getOrThrow("ENDPOINT"); 48 | return endpoint; 49 | } 50 | public String getSagemakerRole() { 51 | if (sagemakerRole == null) 52 | sagemakerRole = getOrThrow("SAGEMAKER_ROLE_ARN"); 53 | return sagemakerRole; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /fds-models/src/main/java/com/provectus/fds/models/events/Aggregation.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.models.events; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import lombok.*; 6 | 7 | import java.util.Optional; 8 | 9 | @Builder 10 | @Setter 11 | @NoArgsConstructor 12 | @EqualsAndHashCode 13 | @ToString 14 | public class Aggregation { 15 | @JsonProperty("campaign_item_id") 16 | private long campaignItemId; 17 | private String period; 18 | private Long bids; 19 | private Long imps; 20 | private Long clicks; 21 | 22 | @JsonCreator 23 | public Aggregation( 24 | @JsonProperty("campaign_item_id") long campaignItemId, 25 | @JsonProperty("period") String period, 26 | @JsonProperty("bids") Long bids, 27 | @JsonProperty("imps") Long imps, 28 | @JsonProperty("clicks") Long clicks) { 29 | this.campaignItemId = campaignItemId; 30 | this.period = period; 31 | this.bids = bids; 32 | this.imps = imps; 33 | this.clicks = clicks; 34 | } 35 | 36 | public Long getCampaignItemId() { 37 | return campaignItemId; 38 | } 39 | 40 | public String getPeriod() { 41 | return period; 42 | } 43 | 44 | public Long getBids() { 45 | return Optional.ofNullable(bids).orElse(0L); 46 | } 47 | 48 | public Long getImps() { 49 | return Optional.ofNullable(imps).orElse(0L); 50 | } 51 | 52 | public Long getClicks() { 53 | return Optional.ofNullable(clicks).orElse(0L); 54 | } 55 | } -------------------------------------------------------------------------------- /fds-lambda-compaction/src/main/java/com/provectus/fds/compaction/utils/PathFormatter.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction.utils; 2 | 3 | public class PathFormatter { 4 | private final String source; 5 | private final String msgtype; 6 | private final Period period; 7 | private final String filename; 8 | 9 | public PathFormatter(String source, String msgtype, Period period, String filename) { 10 | this.source = source; 11 | this.msgtype = msgtype; 12 | this.period = period; 13 | this.filename = filename; 14 | } 15 | 16 | public String getSource() { 17 | return source; 18 | } 19 | 20 | public String getMsgtype() { 21 | return msgtype; 22 | } 23 | 24 | public Period getPeriod() { 25 | return period; 26 | } 27 | 28 | public String getFilename() { 29 | return filename; 30 | } 31 | 32 | public static PathFormatter fromS3Path(String path) { 33 | String[] parts = path.split("/"); 34 | String source = parts[0]; 35 | String msgtype = parts[1]; 36 | Period period = Period.fromJsonPath(path.substring(source.length() + msgtype.length() + 1)); 37 | String filename = parts[parts.length-1]; 38 | return new PathFormatter(source, msgtype, period, filename); 39 | } 40 | 41 | public String path(String source) { 42 | return String.format("%s/%s/%s", source, this.msgtype, period.path()); 43 | } 44 | 45 | public String pathWithFile(String source, String filename) { 46 | return String.format("%s/%s/%s/%s", source, this.msgtype, period.path(), filename); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /fds-lambda-ml-integration/src/main/java/com/provectus/fds/ml/processor/AthenaConfig.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.ml.processor; 2 | 3 | import com.amazonaws.services.athena.AmazonAthena; 4 | 5 | public class AthenaConfig { 6 | 7 | private AmazonAthena client; 8 | private String dbName; 9 | private String outputLocation; 10 | private String query; 11 | private long sleepTime; 12 | private RecordProcessor recordProcessor; 13 | 14 | public AmazonAthena getClient() { 15 | return client; 16 | } 17 | 18 | public void setClient(AmazonAthena client) { 19 | this.client = client; 20 | } 21 | 22 | public String getDbName() { 23 | return dbName; 24 | } 25 | 26 | public void setDbName(String dbName) { 27 | this.dbName = dbName; 28 | } 29 | 30 | public String getOutputLocation() { 31 | return outputLocation; 32 | } 33 | 34 | public void setOutputLocation(String outputLocation) { 35 | this.outputLocation = outputLocation; 36 | } 37 | 38 | public String getQuery() { 39 | return query; 40 | } 41 | 42 | public void setQuery(String query) { 43 | this.query = query; 44 | } 45 | 46 | public long getSleepTime() { 47 | return sleepTime; 48 | } 49 | 50 | public void setSleepTime(long sleepTime) { 51 | this.sleepTime = sleepTime; 52 | } 53 | 54 | public RecordProcessor getRecordProcessor() { 55 | return recordProcessor; 56 | } 57 | 58 | public void setRecordProcessor(RecordProcessor recordProcessor) { 59 | this.recordProcessor = recordProcessor; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /fds-lambda-compaction/src/main/java/com/provectus/fds/compaction/utils/Period.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction.utils; 2 | 3 | import avro.shaded.com.google.common.collect.ImmutableMap; 4 | 5 | import java.time.LocalDateTime; 6 | import java.time.ZoneOffset; 7 | import java.util.AbstractMap; 8 | import java.util.Arrays; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | public class Period { 13 | private final int year; 14 | private final int day; 15 | 16 | public Period(int year, int day) { 17 | this.year = year; 18 | this.day = day; 19 | } 20 | 21 | 22 | public String path() { 23 | return String.format("year=%d/day=%d", year,day); 24 | } 25 | 26 | public List> toList() { 27 | return Arrays.asList( 28 | new AbstractMap.SimpleEntry<>("year", Integer.toString(year)), 29 | new AbstractMap.SimpleEntry<>("day", Integer.toString(day)) 30 | ); 31 | } 32 | 33 | public static Period fromJsonPath(String path) { 34 | String[] parts = path.split("/"); 35 | if (parts.length>=4) { 36 | int year = Integer.parseInt(parts[1]); 37 | int month = Integer.parseInt(parts[2]); 38 | int dayOfMonth = Integer.parseInt(parts[3]); 39 | LocalDateTime ldt = LocalDateTime.of(year, month, dayOfMonth, 0, 0); 40 | int unixday = (int)((ldt.toInstant(ZoneOffset.UTC).getEpochSecond() / (24 * 60 * 60))); 41 | return new Period(year, unixday); 42 | } else { 43 | throw new IllegalArgumentException("Invalid path"); 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /fds-common-dynamodb/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | fds-parent 5 | com.provectus.fds 6 | 1.0-SNAPSHOT 7 | 8 | 9 | 4.0.0 10 | fds-common-dynamodb 11 | 12 | 13 | 14 | 15 | com.amazonaws 16 | aws-java-sdk-dynamodb 17 | 18 | 19 | 20 | com.provectus.fds 21 | fds-models 22 | test 23 | 24 | 25 | 26 | 27 | 28 | 29 | org.apache.maven.plugins 30 | maven-shade-plugin 31 | 32 | false 33 | 34 | 35 | 36 | package 37 | 38 | shade 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/aggregators/AggregationAccumulator.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink.aggregators; 2 | 3 | import com.provectus.fds.models.bcns.BidBcn; 4 | import com.provectus.fds.models.events.Click; 5 | import com.provectus.fds.models.events.Impression; 6 | import lombok.Getter; 7 | import lombok.Setter; 8 | 9 | @Getter 10 | @Setter 11 | public class AggregationAccumulator { 12 | private long clicks; 13 | private long impressions; 14 | private long bids; 15 | 16 | public AggregationAccumulator() { 17 | } 18 | 19 | public AggregationAccumulator(AggregationAccumulator acc) { 20 | this.bids = acc.bids; 21 | this.impressions = acc.impressions; 22 | this.clicks = acc.clicks; 23 | } 24 | 25 | public AggregationAccumulator add(T value) { 26 | Class tClass = value.getClass(); 27 | 28 | if (BidBcn.class.isAssignableFrom(tClass)) { 29 | bids++; 30 | } else if (Impression.class.isAssignableFrom(tClass)) { 31 | impressions++; 32 | } else if (Click.class.isAssignableFrom(tClass)) { 33 | clicks++; 34 | } else { 35 | throw new IllegalStateException(String.format("Unsupported metric class: %s", tClass)); 36 | } 37 | 38 | return this; 39 | } 40 | 41 | public AggregationAccumulator merge(AggregationAccumulator other) { 42 | AggregationAccumulator newAcc = new AggregationAccumulator<>(this); 43 | newAcc.bids += other.bids; 44 | newAcc.impressions += other.impressions; 45 | newAcc.clicks += other.clicks; 46 | 47 | return newAcc; 48 | } 49 | 50 | public Metrics build() { 51 | return new Metrics(bids, impressions, clicks); 52 | } 53 | } -------------------------------------------------------------------------------- /fds-models/src/main/java/com/provectus/fds/models/events/Location.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.models.events; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import lombok.*; 6 | 7 | @Builder 8 | @Setter 9 | @Getter 10 | @NoArgsConstructor 11 | @EqualsAndHashCode 12 | @ToString 13 | public class Location { 14 | private static final int FIELDS_COUNT = 4; 15 | 16 | @JsonProperty("app_uid") 17 | private String appUID; 18 | 19 | private long timestamp; 20 | private double longitude; 21 | private double latitude; 22 | 23 | @JsonCreator 24 | public Location( 25 | @JsonProperty("app_uid") String appUID, 26 | @JsonProperty("timestamp") long timestamp, 27 | @JsonProperty("longitude") double longitude, 28 | @JsonProperty("latitude") double latitude) { 29 | this.appUID = appUID; 30 | this.timestamp = timestamp; 31 | this.longitude = longitude; 32 | this.latitude = latitude; 33 | } 34 | 35 | public static Location from(String[] elements) { 36 | if (elements == null || elements.length == 0) { 37 | throw new IllegalArgumentException("No elements"); 38 | } 39 | 40 | if (elements.length != 4) { 41 | throw new IllegalArgumentException(String.format("Wrong format. Fields count: %s. Expected: %s", 42 | elements.length, FIELDS_COUNT)); 43 | } 44 | 45 | return Location.builder() 46 | .appUID(elements[0]) 47 | .timestamp(Long.parseLong(elements[1])) 48 | .longitude(Double.parseDouble(elements[2])) 49 | .latitude(Double.parseDouble(elements[3])) 50 | .build(); 51 | } 52 | } -------------------------------------------------------------------------------- /fds-models/src/main/java/com/provectus/fds/models/utils/DateTimeUtils.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.models.utils; 2 | 3 | import java.time.Instant; 4 | import java.time.ZoneOffset; 5 | import java.time.ZonedDateTime; 6 | import java.time.format.DateTimeFormatter; 7 | import java.time.format.DateTimeFormatterBuilder; 8 | import java.time.format.SignStyle; 9 | import java.util.Locale; 10 | 11 | import static java.time.temporal.ChronoField.*; 12 | 13 | public class DateTimeUtils { 14 | public static final DateTimeFormatter AWS_DATE_TIME = 15 | new DateTimeFormatterBuilder().appendValue(YEAR, 4, 10, SignStyle.EXCEEDS_PAD) 16 | .appendLiteral('-') 17 | .appendValue(MONTH_OF_YEAR, 2) 18 | .appendLiteral('-') 19 | .appendValue(DAY_OF_MONTH, 2) 20 | .appendLiteral(' ') 21 | .appendValue(HOUR_OF_DAY, 2) 22 | .appendLiteral(':') 23 | .appendValue(MINUTE_OF_HOUR, 2) 24 | .appendLiteral(':') 25 | .appendValue(SECOND_OF_MINUTE, 2) 26 | .optionalStart() 27 | .appendLiteral('.') 28 | .appendValue(MILLI_OF_SECOND, 3) 29 | .optionalEnd() 30 | .toFormatter(Locale.ENGLISH) 31 | .withZone(ZoneOffset.UTC); 32 | 33 | public static String format(long epochMillis) { 34 | return ZonedDateTime.ofInstant(Instant.ofEpochMilli(epochMillis), ZoneOffset.UTC).format(AWS_DATE_TIME); 35 | } 36 | 37 | public static long truncate(long epochMillis, long period) { 38 | //return shiftBack * (epochMillis / shiftBack); 39 | return epochMillis - (epochMillis % period); 40 | } 41 | } -------------------------------------------------------------------------------- /fds-lambda-api/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | fds-parent 5 | com.provectus.fds 6 | 1.0-SNAPSHOT 7 | 8 | 9 | 4.0.0 10 | fds-lambda-api 11 | 12 | 13 | 14 | 15 | com.provectus.fds 16 | fds-models 17 | 18 | 19 | 20 | com.amazonaws 21 | aws-lambda-java-core 22 | 23 | 24 | 25 | com.amazonaws 26 | aws-java-sdk-kinesis 27 | 28 | 29 | 30 | 31 | 32 | 33 | org.apache.maven.plugins 34 | maven-shade-plugin 35 | 36 | false 37 | 38 | 39 | 40 | package 41 | 42 | shade 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /fds-lambda-reports/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | fds-parent 6 | com.provectus.fds 7 | 1.0-SNAPSHOT 8 | 9 | 10 | 4.0.0 11 | fds-lambda-reports 12 | 13 | 14 | 15 | 16 | com.amazonaws 17 | aws-lambda-java-core 18 | 19 | 20 | 21 | com.amazonaws 22 | aws-lambda-java-events 23 | 24 | 25 | 26 | com.provectus.fds 27 | fds-common-dynamodb 28 | 29 | 30 | 31 | 32 | 33 | 34 | org.apache.maven.plugins 35 | maven-shade-plugin 36 | 37 | false 38 | 39 | 40 | 41 | package 42 | 43 | shade 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /fds-models/src/main/java/com/provectus/fds/models/events/Click.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.models.events; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import com.provectus.fds.models.bcns.ClickBcn; 5 | import com.provectus.fds.models.bcns.Partitioned; 6 | import com.provectus.fds.models.utils.JsonUtils; 7 | import lombok.Builder; 8 | import lombok.Getter; 9 | import lombok.NoArgsConstructor; 10 | import lombok.Setter; 11 | 12 | import java.io.IOException; 13 | import java.util.StringJoiner; 14 | 15 | 16 | @Builder 17 | @Getter 18 | @Setter 19 | @NoArgsConstructor 20 | public class Click implements Partitioned { 21 | private Impression impression; 22 | 23 | public Click(@JsonProperty("impression") Impression impression) { 24 | this.impression = impression; 25 | } 26 | 27 | @Override 28 | public String getPartitionKey() { 29 | return impression.getBidBcn().getPartitionKey(); 30 | } 31 | 32 | @Override 33 | public byte[] getBytes() throws IOException { 34 | return JsonUtils.write(this); 35 | } 36 | 37 | @Override 38 | public boolean equals(Object o) { 39 | if (this == o) return true; 40 | if (!(o instanceof Click)) return false; 41 | 42 | Click click = (Click) o; 43 | 44 | return getImpression().equals(click.getImpression()); 45 | } 46 | 47 | @Override 48 | public int hashCode() { 49 | return getImpression().hashCode(); 50 | } 51 | 52 | public static Click from(Impression impression, ClickBcn clickBcn) { 53 | return Click.builder().impression(impression).build(); 54 | } 55 | 56 | @Override 57 | public String toString() { 58 | return new StringJoiner(", ", Click.class.getSimpleName() + "[", "]") 59 | .add("impression=" + impression) 60 | .toString(); 61 | } 62 | } -------------------------------------------------------------------------------- /fds-lambda-utils/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | fds-parent 7 | com.provectus.fds 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | fds-lambda-utils 13 | 14 | 15 | 16 | com.amazonaws 17 | aws-lambda-java-core 18 | 19 | 20 | org.json 21 | json 22 | 20180813 23 | 24 | 25 | software.amazon.awssdk 26 | kinesisanalyticsv2 27 | 2.5.32 28 | 29 | 30 | 31 | 32 | 33 | 34 | org.apache.maven.plugins 35 | maven-shade-plugin 36 | 37 | false 38 | 39 | 40 | 41 | package 42 | 43 | shade 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /fds-models/src/main/java/com/provectus/fds/models/bcns/WalkinClick.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.models.bcns; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import com.provectus.fds.models.events.Click; 5 | import com.provectus.fds.models.events.Location; 6 | import com.provectus.fds.models.utils.JsonUtils; 7 | import lombok.*; 8 | 9 | import java.io.IOException; 10 | 11 | @Builder 12 | @Getter 13 | @Setter 14 | @NoArgsConstructor 15 | @ToString 16 | @EqualsAndHashCode 17 | public class WalkinClick implements Partitioned { 18 | @JsonProperty("tx_id") 19 | private String txId; 20 | 21 | @JsonProperty("app_uid") 22 | private String appUID; 23 | 24 | private long timestamp; 25 | private double longitude; 26 | private double latitude; 27 | 28 | public WalkinClick( 29 | @JsonProperty("tx_id") String txId, 30 | @JsonProperty("app_uid") String appUID, 31 | @JsonProperty("timestamp") long timestamp, 32 | @JsonProperty("longitude") double longitude, 33 | @JsonProperty("latitude") double latitude) { 34 | this.txId = txId; 35 | this.appUID = appUID; 36 | this.timestamp = timestamp; 37 | this.longitude = longitude; 38 | this.latitude = latitude; 39 | } 40 | 41 | @Override 42 | public String getPartitionKey() { 43 | return txId; 44 | } 45 | 46 | @Override 47 | public byte[] getBytes() throws IOException { 48 | return JsonUtils.write(this); 49 | } 50 | 51 | public static WalkinClick from(Click click, Location location) { 52 | return builder() 53 | .txId(click.getPartitionKey()) 54 | .appUID(location.getAppUID()) 55 | .timestamp(location.getTimestamp()) 56 | .longitude(location.getLongitude()) 57 | .latitude(location.getLatitude()) 58 | .build(); 59 | } 60 | } -------------------------------------------------------------------------------- /fds-lambda-reports/src/main/java/com/provectus/fds/reports/ApiHandler.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.reports; 2 | 3 | import com.amazonaws.services.lambda.runtime.Context; 4 | import com.amazonaws.services.lambda.runtime.RequestStreamHandler; 5 | 6 | import java.io.*; 7 | import java.util.HashMap; 8 | import java.util.Map; 9 | import java.util.function.BiFunction; 10 | import java.util.function.Function; 11 | 12 | public class ApiHandler implements RequestStreamHandler { 13 | Map> handlerMap = new HashMap<>(); 14 | 15 | public ApiHandler() { 16 | AggregationsReportHandler aggregationsHandler = new AggregationsReportHandler(); 17 | 18 | this.handlerMap.put( 19 | new ExecutionContext("GET", "/reports/campaigns/{campaign_item_id}"), 20 | aggregationsHandler::getTotal 21 | ); 22 | 23 | this.handlerMap.put( 24 | new ExecutionContext("GET", "/reports/campaigns/{campaign_item_id}/period"), 25 | aggregationsHandler::getByPeriod 26 | ); 27 | } 28 | 29 | @Override 30 | public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException { 31 | BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); 32 | ExecutionValues executionValues = JsonUtils.read(reader, ExecutionValues.class); 33 | context.getLogger().log("Received event: "+executionValues.toString()); 34 | ExecutionContext executionContext = new ExecutionContext(executionValues); 35 | BiFunction handler = this.handlerMap.get(executionContext); 36 | if (handler!=null) { 37 | ApiResponse response = new ApiResponse(200, JsonUtils.stringify(handler.apply(executionValues,context))); 38 | outputStream.write(JsonUtils.write(response)); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | jobs/release: 4 | docker: 5 | - image: circleci/openjdk:8-jdk-stretch 6 | working_directory: ~/circleci 7 | 8 | steps: 9 | - checkout 10 | - run: sudo apt-get update && sudo apt-get install -y python3-pip 11 | - run: sudo pip3 install awscli 12 | - run: mvn -B package 13 | - run: 14 | name: Deploy a package to the all regions 15 | command: | 16 | REGIONS="us-east-1 us-east-2 us-west-2 eu-west-1 eu-central-1" 17 | for region in ${REGIONS}; do 18 | export AWS_DEFAULT_REGION=$region 19 | ./stack.sh -p -c \ 20 | -b streaming-data-platform-releases-${AWS_DEFAULT_REGION} \ 21 | -r ${AWS_DEFAULT_REGION} \ 22 | -v $CIRCLE_TAG 23 | done 24 | 25 | jobs/integration-test: 26 | docker: 27 | - image: circleci/openjdk:8-jdk-stretch 28 | working_directory: ~/circleci 29 | 30 | steps: 31 | - checkout 32 | - run: sudo apt-get update && sudo apt-get install -y python3-pip 33 | - run: sudo pip3 install awscli 34 | - restore_cache: 35 | keys: 36 | - v1-dependencies-{{ checksum "pom.xml" }} 37 | - v1-dependencies- 38 | - run: mvn -B -s settings.xml package 39 | - run: mvn -B -s settings.xml verify -DresourceBucket=fdp-itstack-us-west-2 40 | - save_cache: 41 | paths: 42 | - ~/.m2 43 | key: v1-dependencies-{{ checksum "pom.xml" }} 44 | 45 | workflows: 46 | version: 2 47 | build: 48 | jobs: 49 | - jobs/release: 50 | filters: 51 | branches: 52 | ignore: /.*/ 53 | tags: 54 | only: /.*/ 55 | 56 | - jobs/integration-test: 57 | filters: 58 | branches: 59 | only: /.*/ 60 | tags: 61 | ignore: /.*/ 62 | -------------------------------------------------------------------------------- /fds-lambda-api/src/main/java/com/provectus/fds/api/BidBcnHandler.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.api; 2 | 3 | import com.amazonaws.services.lambda.runtime.Context; 4 | import com.fasterxml.jackson.databind.JsonNode; 5 | import com.provectus.fds.models.bcns.Partitioned; 6 | import com.provectus.fds.models.bcns.BidBcn; 7 | 8 | import java.io.IOException; 9 | import java.util.Optional; 10 | 11 | public class BidBcnHandler extends AbstractBcnHandler { 12 | @Override 13 | public Optional buildBcn(JsonNode parameters, Context context) throws IOException { 14 | Optional result = Optional.empty(); 15 | if (parameters.has("tx_id") && parameters.has("app_uid") && parameters.has("campaign_item_id")) { 16 | String txid = parameters.get("tx_id").asText(); 17 | String appuid = parameters.get("app_uid").asText(); 18 | long campaignItemId = parameters.get("campaign_item_id").asLong(); 19 | String domain = ""; 20 | if (parameters.has("domain")) { 21 | domain = parameters.get("domain").asText(); 22 | } 23 | String creativeId = ""; 24 | if (parameters.has("creative_id")) { 25 | creativeId = parameters.get("creative_id").asText(); 26 | } 27 | 28 | String creativeCategory = ""; 29 | if (parameters.has("creative_category")) { 30 | creativeCategory = parameters.get("creative_category").asText(); 31 | } 32 | 33 | result = Optional.of( 34 | new BidBcn( 35 | txid, 36 | campaignItemId, 37 | domain, 38 | creativeId, 39 | creativeCategory, 40 | appuid 41 | ) 42 | ); 43 | } 44 | 45 | return result; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /fds-flink-streaming/src/main/java/com/provectus/fds/flink/aggregators/Metrics.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.flink.aggregators; 2 | 3 | import com.provectus.fds.models.bcns.BidBcn; 4 | import com.provectus.fds.models.events.Click; 5 | import com.provectus.fds.models.events.Impression; 6 | import lombok.AllArgsConstructor; 7 | import lombok.Builder; 8 | import lombok.Data; 9 | import lombok.NoArgsConstructor; 10 | 11 | @Data 12 | @Builder 13 | @AllArgsConstructor 14 | @NoArgsConstructor 15 | public class Metrics { 16 | private long campaignItemId; 17 | private long bids; 18 | private long impressions; 19 | private long clicks; 20 | 21 | public Metrics(long bids, long impressions, long clicks) { 22 | this.bids = bids; 23 | this.impressions = impressions; 24 | this.clicks = clicks; 25 | } 26 | 27 | public static Metrics of(BidBcn bidBcn) { 28 | return Metrics.builder() 29 | .campaignItemId(getCampaignItemId(bidBcn)) 30 | .bids(1) 31 | .build(); 32 | } 33 | 34 | public static Metrics of(Impression impression) { 35 | return Metrics.builder() 36 | .campaignItemId(getCampaignItemId(impression)) 37 | .impressions(1) 38 | .build(); 39 | } 40 | 41 | public static Metrics of(Click click) { 42 | return Metrics.builder() 43 | .campaignItemId(getCampaignItemId(click)) 44 | .clicks(1) 45 | .build(); 46 | } 47 | 48 | private static long getCampaignItemId(BidBcn bidBcn) { 49 | return bidBcn == null ? 0 : bidBcn.getCampaignItemId(); 50 | } 51 | 52 | private static long getCampaignItemId(Impression impression) { 53 | return impression == null ? 0 : getCampaignItemId(impression.getBidBcn()); 54 | } 55 | 56 | private static long getCampaignItemId(Click click) { 57 | return click == null ? 0 : getCampaignItemId(click.getImpression()); 58 | } 59 | } -------------------------------------------------------------------------------- /fds-lambda-compaction/src/main/java/com/provectus/fds/compaction/utils/ParquetTripletUtils.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction.utils; 2 | 3 | import org.apache.parquet.column.ColumnDescriptor; 4 | import org.apache.parquet.column.ColumnReader; 5 | import org.apache.parquet.column.ColumnWriter; 6 | import org.apache.parquet.schema.PrimitiveType; 7 | 8 | public class ParquetTripletUtils { 9 | 10 | public static void consumeTriplet(ColumnWriter columnWriter, ColumnReader columnReader) { 11 | int definitionLevel = columnReader.getCurrentDefinitionLevel(); 12 | int repetitionLevel = columnReader.getCurrentRepetitionLevel(); 13 | ColumnDescriptor column = columnReader.getDescriptor(); 14 | PrimitiveType type = column.getPrimitiveType(); 15 | if (definitionLevel < column.getMaxDefinitionLevel()) { 16 | columnWriter.writeNull(repetitionLevel, definitionLevel); 17 | } else { 18 | switch (type.getPrimitiveTypeName()) { 19 | case INT32: 20 | columnWriter.write(columnReader.getInteger(), repetitionLevel, definitionLevel); 21 | break; 22 | case INT64: 23 | columnWriter.write(columnReader.getLong(), repetitionLevel, definitionLevel); 24 | break; 25 | case BINARY: 26 | case FIXED_LEN_BYTE_ARRAY: 27 | case INT96: 28 | columnWriter.write(columnReader.getBinary(), repetitionLevel, definitionLevel); 29 | break; 30 | case BOOLEAN: 31 | columnWriter.write(columnReader.getBoolean(), repetitionLevel, definitionLevel); 32 | break; 33 | case FLOAT: 34 | columnWriter.write(columnReader.getFloat(), repetitionLevel, definitionLevel); 35 | break; 36 | case DOUBLE: 37 | columnWriter.write(columnReader.getDouble(), repetitionLevel, definitionLevel); 38 | break; 39 | default: 40 | throw new IllegalArgumentException("Unknown primitive type " + type); 41 | } 42 | } 43 | columnReader.consume(); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /fds-models/src/main/java/com/provectus/fds/models/bcns/Bcn.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.models.bcns; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import com.provectus.fds.models.utils.JsonUtils; 6 | import lombok.*; 7 | 8 | import java.io.IOException; 9 | 10 | @Getter 11 | @Builder 12 | @Setter 13 | @NoArgsConstructor 14 | @ToString 15 | @EqualsAndHashCode 16 | public class Bcn implements Partitioned { 17 | @JsonProperty("tx_id") 18 | private String txId; 19 | 20 | @JsonProperty("campaign_item_id") 21 | private long campaignItemId; 22 | 23 | private String domain; 24 | 25 | @JsonProperty("creative_id") 26 | private String creativeId; 27 | 28 | @JsonProperty("creative_category") 29 | private String creativeCategory; 30 | 31 | @JsonProperty("app_uid") 32 | private String appUID; 33 | 34 | @JsonProperty("win_price") 35 | private long winPrice; 36 | 37 | private String type; 38 | 39 | @JsonCreator 40 | public Bcn( 41 | @JsonProperty("tx_id") String txid, 42 | @JsonProperty("campaign_item_id") long campaignItemId, 43 | @JsonProperty("domain") String domain, 44 | @JsonProperty("creative_id") String creativeId, 45 | @JsonProperty("creative_category") String creativeCategory, 46 | @JsonProperty("app_uid") String appUID, 47 | @JsonProperty("win_price") long winPrice, 48 | @JsonProperty("type") String type) { 49 | this.txId = txid; 50 | this.campaignItemId = campaignItemId; 51 | this.domain = domain; 52 | this.creativeId = creativeId; 53 | this.creativeCategory = creativeCategory; 54 | this.appUID = appUID; 55 | this.winPrice = winPrice; 56 | this.type = type; 57 | } 58 | 59 | @Override 60 | public String getPartitionKey() { 61 | return txId; 62 | } 63 | 64 | @Override 65 | public byte[] getBytes() throws IOException { 66 | return JsonUtils.write(this); 67 | } 68 | } -------------------------------------------------------------------------------- /fds-lambda-compaction/src/test/java/com/provectus/fds/compaction/utils/ParquetUtilsTest.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction.utils; 2 | 3 | import org.apache.hadoop.fs.Path; 4 | import org.junit.Test; 5 | 6 | import java.io.File; 7 | import java.io.IOException; 8 | import java.nio.file.Files; 9 | import java.util.Arrays; 10 | 11 | import static org.junit.Assert.assertTrue; 12 | 13 | public class ParquetUtilsTest { 14 | 15 | private final File tmpDir = Files.createTempDirectory("s3").toFile(); 16 | 17 | public ParquetUtilsTest() throws IOException { 18 | tmpDir.deleteOnExit(); 19 | } 20 | 21 | @Test 22 | public void testJsonFileConvert() throws IOException { 23 | File result1 = null; 24 | File result2 = null; 25 | File targetFile = null; 26 | 27 | try { 28 | ClassLoader classLoader = getClass().getClassLoader(); 29 | File testBcnFile = new File(classLoader.getResource("testbcn.json").getFile()); 30 | File testBcn1File = new File(classLoader.getResource("testbcn1.json").getFile()); 31 | 32 | ParquetUtils parquetUtils = new ParquetUtils(); 33 | 34 | result1 = parquetUtils.convert(tmpDir,testBcn1File, "prefix"); 35 | result2 = parquetUtils.convert(tmpDir,testBcnFile, "prefix2"); 36 | 37 | 38 | targetFile = File.createTempFile("prefix", "targetparquetfile"); 39 | targetFile.delete(); 40 | 41 | parquetUtils.mergeFiles( 42 | Arrays.asList( 43 | new Path("file://" + result1.getAbsolutePath()), 44 | new Path("file://" + result2.getAbsolutePath()) 45 | ), 46 | new Path("file://"+targetFile.getAbsolutePath()) 47 | ); 48 | 49 | assertTrue(targetFile.exists()); 50 | 51 | 52 | 53 | } finally { 54 | if (result1!=null) result1.delete(); 55 | if (result2!=null) result2.delete(); 56 | if (targetFile!=null) targetFile.delete(); 57 | 58 | } 59 | } 60 | } -------------------------------------------------------------------------------- /fds-lambda-compaction/src/test/java/com/provectus/fds/compaction/TestSchemaBuilder.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction; 2 | 3 | import com.provectus.fds.compaction.utils.JsonUtils; 4 | import org.apache.avro.Schema; 5 | import org.junit.Test; 6 | 7 | import java.io.File; 8 | import java.io.IOException; 9 | import java.util.Arrays; 10 | import java.util.Optional; 11 | 12 | import static org.junit.Assert.assertEquals; 13 | import static org.junit.Assert.assertTrue; 14 | 15 | public class TestSchemaBuilder { 16 | @Test 17 | public void test() throws IOException { 18 | 19 | ClassLoader classLoader = getClass().getClassLoader(); 20 | File testBcnFile = new File(classLoader.getResource("testbcn.json").getFile()); 21 | 22 | Optional schema = JsonUtils.buildSchema(testBcnFile); 23 | assertTrue(schema.isPresent()); 24 | 25 | assertEquals(Schema.createUnion(Arrays.asList( 26 | Schema.create(Schema.Type.NULL) 27 | ,Schema.create(Schema.Type.STRING)) 28 | ),schema.get().getField("type").schema() 29 | ); 30 | 31 | assertEquals(Schema.createUnion(Arrays.asList( 32 | Schema.create(Schema.Type.NULL) 33 | ,Schema.create(Schema.Type.LONG)) 34 | ),schema.get().getField("timestamp").schema() 35 | ); 36 | 37 | assertEquals(Schema.createUnion(Arrays.asList( 38 | Schema.create(Schema.Type.NULL) 39 | ,Schema.create(Schema.Type.LONG)) 40 | ),schema.get().getField("campaign_item_id").schema() 41 | ); 42 | 43 | assertEquals(Schema.createUnion(Arrays.asList( 44 | Schema.create(Schema.Type.NULL), 45 | Schema.create(Schema.Type.STRING)) 46 | ),schema.get().getField("domain").schema()); 47 | 48 | assertEquals(Schema.createUnion(Arrays.asList( 49 | Schema.create(Schema.Type.NULL) 50 | ,Schema.create(Schema.Type.STRING)) 51 | ),schema.get().getField("txid").schema() 52 | ); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /fds-models/src/main/java/com/provectus/fds/models/bcns/Walkin.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.models.bcns; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import com.provectus.fds.models.events.Impression; 5 | import com.provectus.fds.models.events.Location; 6 | import com.provectus.fds.models.utils.JsonUtils; 7 | import lombok.*; 8 | 9 | import java.io.IOException; 10 | 11 | @Builder 12 | @Getter 13 | @Setter 14 | @NoArgsConstructor 15 | @ToString 16 | @EqualsAndHashCode 17 | public class Walkin implements Partitioned { 18 | @JsonProperty("tx_id") 19 | private String txId; 20 | 21 | @JsonProperty("win_price") 22 | private long winPrice; 23 | 24 | @JsonProperty("app_uid") 25 | private String appUID; 26 | 27 | private long timestamp; 28 | private double longitude; 29 | private double latitude; 30 | 31 | public Walkin( 32 | @JsonProperty("tx_id") String txId, 33 | @JsonProperty("win_price") long winPrice, 34 | @JsonProperty("app_uid") String appUID, 35 | @JsonProperty("timestamp") long timestamp, 36 | @JsonProperty("longitude") double longitude, 37 | @JsonProperty("latitude") double latitude) { 38 | this.txId = txId; 39 | this.winPrice = winPrice; 40 | this.appUID = appUID; 41 | this.timestamp = timestamp; 42 | this.longitude = longitude; 43 | this.latitude = latitude; 44 | } 45 | 46 | @Override 47 | public String getPartitionKey() { 48 | return txId; 49 | } 50 | 51 | @Override 52 | public byte[] getBytes() throws IOException { 53 | return JsonUtils.write(this); 54 | } 55 | 56 | public static Walkin from(Impression impression, Location location) { 57 | return builder() 58 | .txId(impression.getPartitionKey()) 59 | .winPrice(impression.getImpressionBcn() == null ? 0 : impression.getImpressionBcn().getWinPrice()) 60 | .appUID(location.getAppUID()) 61 | .timestamp(location.getTimestamp()) 62 | .longitude(location.getLongitude()) 63 | .latitude(location.getLatitude()) 64 | .build(); 65 | } 66 | } -------------------------------------------------------------------------------- /fds-lambda-dynamodb-persister/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | fds-parent 5 | com.provectus.fds 6 | 1.0-SNAPSHOT 7 | 8 | 9 | 4.0.0 10 | fds-lambda-dynamodb-persister 11 | 12 | 13 | 14 | 15 | com.amazonaws 16 | aws-lambda-java-core 17 | 18 | 19 | 20 | com.amazonaws 21 | aws-lambda-java-events 22 | 23 | 24 | 25 | com.provectus.fds 26 | fds-common-dynamodb 27 | 28 | 29 | 30 | com.amazonaws 31 | aws-java-sdk-kinesis 32 | 33 | 34 | 35 | com.provectus.fds 36 | fds-models 37 | test 38 | 39 | 40 | 41 | 42 | 43 | 44 | org.apache.maven.plugins 45 | maven-shade-plugin 46 | 47 | false 48 | 49 | 50 | 51 | package 52 | 53 | shade 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /gatling/simulations/basic/ApiPerformanceTest.scala: -------------------------------------------------------------------------------- 1 | package basic 2 | import scala.util.Random 3 | import scala.concurrent.duration._ 4 | import com.typesafe.config.{Config, ConfigFactory} 5 | import io.gatling.app.Gatling 6 | import io.gatling.core.Predef._ 7 | import io.gatling.core.config.GatlingPropertiesBuilder 8 | import io.gatling.http.Predef._ 9 | import java.util.concurrent.ThreadLocalRandom 10 | import java.util.UUID.randomUUID 11 | 12 | class ApiPerformanceTest extends Simulation { 13 | 14 | val rnd = ThreadLocalRandom.current 15 | val Appuid = Random.alphanumeric.take(10).mkString 16 | val Creative_category = Random.alphanumeric.take(10).mkString 17 | val Creative_id = Random.alphanumeric.take(10).mkString 18 | val Domain = Random.alphanumeric.take(10).mkString 19 | val Campaign_item_id = rnd.nextInt(1000000, 2000000) 20 | val duration = Integer.valueOf(System.getProperty("duration","60")) 21 | val httpProtocol = http.baseUrl(System.getProperty("baseUrl")) 22 | .acceptHeader("application/json, text/plain, */*") 23 | .acceptEncodingHeader("gzip, deflate") 24 | .acceptLanguageHeader("en-us") 25 | .userAgentHeader("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8") 26 | 27 | 28 | 29 | val scn = scenario("fast-data-solution") 30 | .exec(_.setAll( 31 | ("Appuid" -> Appuid), 32 | ("Campaign_item_id" -> Campaign_item_id), 33 | ("Creative_category"-> Creative_category), 34 | ("Creative_id"-> Creative_id), 35 | ("Domain"-> Domain), 36 | ("Txid", randomUUID().toString) 37 | )) 38 | .exec( 39 | http("Bid request") 40 | .post("/bid") 41 | .body(StringBody("""{"win_price": 0, "app_uid": "${Appuid}", "campaign_item_id": ${Campaign_item_id}, "creative_category": "${Creative_category}", "creative_id":"${Creative_id}", "tx_id":"${Txid}", "domain":"${Domain}"}""")).asJson 42 | .check(status.is(200)) 43 | ) 44 | .exec( 45 | http("Impression request") 46 | .post("/impression") 47 | .body(StringBody("""{"tx_id":"${Txid}", "win_price": 10 }""")).asJson 48 | .check(status.is(200)) 49 | ) 50 | .exec( 51 | http("Click request") 52 | .post("/click") 53 | .body(StringBody("""{"tx_id":"${Txid}"}""")).asJson 54 | .check(status.is(200)) 55 | ) 56 | setUp(scn.inject(rampUsersPerSec(1) to(10) during(duration)).protocols(httpProtocol)) 57 | } 58 | -------------------------------------------------------------------------------- /fds-models/src/main/java/com/provectus/fds/models/events/Impression.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.models.events; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import com.provectus.fds.models.bcns.BidBcn; 5 | import com.provectus.fds.models.bcns.ImpressionBcn; 6 | import com.provectus.fds.models.bcns.Partitioned; 7 | import com.provectus.fds.models.utils.JsonUtils; 8 | import lombok.Builder; 9 | import lombok.Getter; 10 | import lombok.NoArgsConstructor; 11 | import lombok.Setter; 12 | 13 | import java.io.IOException; 14 | import java.util.StringJoiner; 15 | 16 | 17 | @Builder 18 | @Getter 19 | @Setter 20 | @NoArgsConstructor 21 | public class Impression implements Partitioned { 22 | @JsonProperty("bid_bcn") 23 | private BidBcn bidBcn; 24 | 25 | @JsonProperty("impression_bcn") 26 | private ImpressionBcn impressionBcn; 27 | 28 | public Impression( 29 | @JsonProperty("bid_bcn") BidBcn bidBcn, 30 | @JsonProperty("impression_bcn") ImpressionBcn impressionBcn) { 31 | this.bidBcn = bidBcn; 32 | this.impressionBcn = impressionBcn; 33 | } 34 | 35 | @Override 36 | public String getPartitionKey() { 37 | return bidBcn.getPartitionKey(); 38 | } 39 | 40 | @Override 41 | public byte[] getBytes() throws IOException { 42 | return JsonUtils.write(this); 43 | } 44 | 45 | @Override 46 | public boolean equals(Object o) { 47 | if (this == o) return true; 48 | if (!(o instanceof Impression)) return false; 49 | 50 | Impression that = (Impression) o; 51 | 52 | if (!getBidBcn().equals(that.getBidBcn())) return false; 53 | return getImpressionBcn().equals(that.getImpressionBcn()); 54 | } 55 | 56 | @Override 57 | public int hashCode() { 58 | int result = getBidBcn().hashCode(); 59 | result = 31 * result + getImpressionBcn().hashCode(); 60 | return result; 61 | } 62 | 63 | public static Impression from(BidBcn bidBcn, ImpressionBcn impressionBcn) { 64 | return Impression.builder() 65 | .bidBcn(bidBcn) 66 | .impressionBcn(impressionBcn) 67 | .build(); 68 | } 69 | 70 | @Override 71 | public String toString() { 72 | return new StringJoiner(", ", Impression.class.getSimpleName() + "[", "]") 73 | .add("bidBcn=" + bidBcn) 74 | .add("impressionBcn=" + impressionBcn) 75 | .toString(); 76 | } 77 | } -------------------------------------------------------------------------------- /fds-lambda-dynamodb-persister/src/main/java/com/provectus/fds/dynamodb/DynamoDBPersisterLambda.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.dynamodb; 2 | 3 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; 4 | import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; 5 | import com.amazonaws.services.dynamodbv2.document.Item; 6 | import com.amazonaws.services.dynamodbv2.document.PrimaryKey; 7 | import com.amazonaws.services.lambda.runtime.Context; 8 | import com.amazonaws.services.lambda.runtime.LambdaLogger; 9 | import com.amazonaws.services.lambda.runtime.RequestHandler; 10 | import com.amazonaws.services.lambda.runtime.events.KinesisEvent; 11 | 12 | import java.util.Collection; 13 | import java.util.List; 14 | import java.util.Map; 15 | import java.util.stream.Collectors; 16 | 17 | public class DynamoDBPersisterLambda implements RequestHandler { 18 | private final static String DYNAMO_TABLE_ENV = "DYNAMO_TABLE"; 19 | public static final String DYNAMO_TABLE_DEFAULT = "aggregations"; 20 | private final ItemMapper itemMapper = new ItemMapper(); 21 | private final DynamoDAO dynamoDAO; 22 | 23 | 24 | public DynamoDBPersisterLambda() { 25 | AmazonDynamoDB client = AmazonDynamoDBClientBuilder.standard().build(); 26 | this.dynamoDAO = new DynamoDAO(System.getenv().getOrDefault(DYNAMO_TABLE_ENV, DYNAMO_TABLE_DEFAULT), client); 27 | } 28 | 29 | @Override 30 | public Integer handleRequest(KinesisEvent event, Context context) { 31 | LambdaLogger logger = context.getLogger(); 32 | 33 | 34 | if (event == null || event.getRecords() == null) { 35 | logger.log("Event contains no data" + System.lineSeparator()); 36 | return null; 37 | } else { 38 | logger.log("Received " + event.getRecords().size() + 39 | " records from " + event.getRecords().get(0).getEventSourceARN() + System.lineSeparator()); 40 | } 41 | 42 | Map merged = itemMapper.mergeItems( 43 | event.getRecords().stream().map( r -> r.getKinesis().getData()).collect(Collectors.toList()) 44 | ); 45 | 46 | Collection keys = merged.keySet(); 47 | Collection created = merged.values(); 48 | 49 | List old = dynamoDAO.batchGet(keys); 50 | List items = itemMapper.mergeItems(created, old); 51 | 52 | dynamoDAO.batchWrite(items); 53 | return event.getRecords().size(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /fds-lambda-compaction/src/main/java/com/provectus/fds/compaction/KinesisFirehoseResponse.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction; 2 | 3 | import java.nio.ByteBuffer; 4 | import java.util.ArrayList; 5 | import java.util.Arrays; 6 | import java.util.Base64; 7 | import java.util.List; 8 | 9 | public class KinesisFirehoseResponse { 10 | /// The record was transformed successfully. 11 | public final static String TRANSFORMED_STATE_OK = "Ok"; 12 | 13 | /// The record was dropped intentionally by your processing logic. 14 | public final static String TRANSFORMED_STATE_DROPPED = "Dropped"; 15 | 16 | /// The record could not be transformed. 17 | public final static String TRANSFORMED_STATE_PROCESSINGFAILED = "ProcessingFailed"; 18 | 19 | private List records = new ArrayList<>(); 20 | 21 | public List getRecords() { 22 | return records; 23 | } 24 | 25 | public void setRecords(List records) { 26 | this.records = records; 27 | } 28 | 29 | public KinesisFirehoseResponse(List records) { 30 | this.records = records; 31 | } 32 | 33 | public KinesisFirehoseResponse append(FirehoseRecord record) { 34 | this.records.add(record); 35 | return this; 36 | } 37 | 38 | public int size() { 39 | return records.size(); 40 | } 41 | 42 | 43 | public static class FirehoseRecord { 44 | private final String recordId; 45 | private final String Result; 46 | private final String data; 47 | 48 | public FirehoseRecord(String recordId, String data) { 49 | this(recordId, TRANSFORMED_STATE_OK, data); 50 | } 51 | 52 | public FirehoseRecord(String recordId, String result, String data) { 53 | this.recordId = recordId; 54 | Result = result; 55 | this.data = data; 56 | } 57 | 58 | public static FirehoseRecord appendNewLine(String recordId, ByteBuffer data) { 59 | byte[] result = Arrays.copyOf(data.array(), data.array().length+1); 60 | result[data.array().length] = '\n'; 61 | return new FirehoseRecord( 62 | recordId, 63 | Base64.getEncoder().encodeToString(result) 64 | ); 65 | } 66 | 67 | public String getRecordId() { 68 | return recordId; 69 | } 70 | 71 | public String getResult() { 72 | return Result; 73 | } 74 | 75 | public String getData() { 76 | return data; 77 | } 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /fds-lambda-compaction/src/main/java/com/provectus/fds/compaction/utils/JsonFileReader.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction.utils; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.JsonNode; 5 | import com.fasterxml.jackson.databind.ObjectMapper; 6 | import org.apache.avro.Schema; 7 | import org.apache.avro.generic.GenericData; 8 | 9 | import java.io.BufferedReader; 10 | import java.io.File; 11 | import java.io.FileReader; 12 | import java.io.IOException; 13 | import java.util.Iterator; 14 | 15 | public class JsonFileReader implements AutoCloseable,Iterable { 16 | 17 | 18 | private final Schema schema; 19 | private final BufferedReader bufferedReader; 20 | 21 | 22 | public JsonFileReader(Schema schema, File file) throws IOException { 23 | this.schema = schema; 24 | this.bufferedReader = new BufferedReader(new FileReader(file)); 25 | } 26 | 27 | @Override 28 | public void close() throws Exception { 29 | this.bufferedReader.close(); 30 | } 31 | 32 | @Override 33 | public Iterator iterator() { 34 | return new JsonFileReaderIterator(schema, bufferedReader); 35 | } 36 | 37 | private static class JsonFileReaderIterator implements Iterator { 38 | private static final ObjectMapper mapper = new ObjectMapper(); 39 | 40 | private final Schema schema; 41 | private final BufferedReader bufferedReader; 42 | private String currentLine; 43 | 44 | public JsonFileReaderIterator(Schema schema, BufferedReader bufferedReader) { 45 | this.schema = schema; 46 | this.bufferedReader = bufferedReader; 47 | try { 48 | this.currentLine=bufferedReader.readLine(); 49 | } catch (IOException e) { 50 | throw new RuntimeException(e); 51 | } 52 | } 53 | 54 | @Override 55 | public boolean hasNext() { 56 | return this.currentLine!=null; 57 | } 58 | 59 | @Override 60 | public GenericData.Record next() { 61 | try { 62 | JsonNode jsonRecord = mapper.readTree(currentLine); 63 | GenericData.Record record = (GenericData.Record) JsonUtils.convertToAvro(GenericData.get(), jsonRecord, schema); 64 | this.currentLine=this.bufferedReader.readLine(); 65 | return record; 66 | } catch (IOException e) { 67 | throw new RuntimeException(e); 68 | } 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /fds-lambda-reports/src/main/java/com/provectus/fds/reports/AggregationsReportHandler.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.reports; 2 | 3 | import com.amazonaws.services.lambda.runtime.Context; 4 | import com.provectus.fds.dynamodb.DynamoDAO; 5 | import com.provectus.fds.dynamodb.models.Aggregation; 6 | import com.provectus.fds.dynamodb.repositories.AggregationRepository; 7 | 8 | import java.time.Instant; 9 | import java.time.ZoneId; 10 | import java.time.ZoneOffset; 11 | import java.time.ZonedDateTime; 12 | import java.time.temporal.ChronoUnit; 13 | import java.util.List; 14 | 15 | public class AggregationsReportHandler extends AbstractReportHandler { 16 | private final static String DYNAMO_TABLE_ENV = "DYNAMO_TABLE"; 17 | public static final String DYNAMO_TABLE_DEFAULT = "aggregations"; 18 | public static final String CAMPAIGN_ITEM_ID_FIELD = "campaign_item_id"; 19 | 20 | private final AggregationRepository repository; 21 | 22 | public AggregationsReportHandler() { 23 | this.repository = new AggregationRepository( 24 | this.client 25 | , System.getenv().getOrDefault(DYNAMO_TABLE_ENV, DYNAMO_TABLE_DEFAULT) 26 | ); 27 | } 28 | 29 | public Aggregation getTotal(ExecutionValues values, Context context) { 30 | long campaignItemId = values.orThrow(values.getPathLong(CAMPAIGN_ITEM_ID_FIELD), CAMPAIGN_ITEM_ID_FIELD); 31 | return this.repository.total(campaignItemId); 32 | } 33 | 34 | 35 | public List getByPeriod(ExecutionValues values, Context context) { 36 | long campaignItemId = values.orThrow(values.getPathLong(CAMPAIGN_ITEM_ID_FIELD), CAMPAIGN_ITEM_ID_FIELD); 37 | 38 | ZoneId zoneId = values.getQueryZone("timezone").orElse(ZoneOffset.UTC); 39 | ChronoUnit chronoUnit = values.getQueryChronoUnit("period").orElse(ChronoUnit.DAYS); 40 | ZonedDateTime from = values.getQueryZoneDateTime(zoneId, "from").orElse(Instant.ofEpochSecond(0).atZone(zoneId)); 41 | ZonedDateTime to = values.getQueryZoneDateTime(zoneId, "to").orElse(Instant.now().atZone(zoneId)); 42 | boolean desc = values.getQueryBoolean("desc").orElse(true); 43 | 44 | context.getLogger().log("campaignItemId: "+campaignItemId); 45 | context.getLogger().log("zoneid: "+zoneId.toString()); 46 | context.getLogger().log("chronoUnit: "+chronoUnit.toString()); 47 | context.getLogger().log("from: "+from.toInstant().getEpochSecond()); 48 | context.getLogger().log("to: "+to.toInstant().getEpochSecond()); 49 | context.getLogger().log("desc: "+desc); 50 | 51 | return this.repository.getGrouped(campaignItemId,chronoUnit,zoneId,from, to, desc); 52 | } 53 | 54 | 55 | 56 | } 57 | -------------------------------------------------------------------------------- /fds-lambda-compaction/src/main/java/com/provectus/fds/compaction/utils/SchemaNodeVisitor.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction.utils; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import com.fasterxml.jackson.databind.node.*; 5 | import org.apache.avro.Schema; 6 | 7 | import java.util.*; 8 | 9 | public class SchemaNodeVisitor implements JsonNodeVisitor { 10 | public Optional visitObject(String name, JsonNode node) { 11 | List fields = new ArrayList<>(); 12 | Iterator> fieldsIterator = node.fields(); 13 | while (fieldsIterator.hasNext()) { 14 | Map.Entry field = fieldsIterator.next(); 15 | visit(field.getKey(), field.getValue()).ifPresent(fields::add); 16 | } 17 | return Optional.of(new Schema.Field(name, Schema.createRecord(name,"","", false, fields), "", null)); 18 | } 19 | 20 | public Optional visitArray(String name, ArrayNode node) { 21 | if (node.size()>0) { 22 | Optional arrayField = visit("test", node.get(0)); 23 | if (arrayField.isPresent()) { 24 | return Optional.of(new Schema.Field(name, Schema.createArray(arrayField.get().schema()), "", null)); 25 | } 26 | } 27 | return Optional.empty(); 28 | } 29 | 30 | public Optional visitBinary(String name, BinaryNode binaryNode) { 31 | Schema schema = Schema.createUnion( 32 | Schema.create(Schema.Type.NULL), 33 | Schema.create(Schema.Type.BYTES) 34 | ); 35 | 36 | return Optional.of(new Schema.Field(name, schema, "", null)); 37 | } 38 | 39 | public Optional visitText(String name, TextNode binaryNode) { 40 | Schema schema = Schema.createUnion( 41 | Schema.create(Schema.Type.NULL), 42 | Schema.create(Schema.Type.STRING) 43 | ); 44 | 45 | return Optional.of(new Schema.Field(name, schema, "", null)); 46 | } 47 | 48 | 49 | public Optional visitNumber(String name, NumericNode numericNode) { 50 | Schema schema = Schema.createUnion( 51 | Schema.create(Schema.Type.NULL), 52 | Schema.create(Schema.Type.LONG) 53 | ); 54 | 55 | return Optional.of(new Schema.Field(name, schema, "", null)); 56 | } 57 | 58 | public Optional visitBoolean(String name, BooleanNode booleanNode) { 59 | Schema schema = Schema.createUnion( 60 | Schema.create(Schema.Type.NULL), 61 | Schema.create(Schema.Type.BOOLEAN) 62 | ); 63 | 64 | return Optional.of(new Schema.Field(name, schema, "", null)); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /fds-models/src/main/java/com/provectus/fds/models/bcns/BidBcn.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.models.bcns; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import com.provectus.fds.models.utils.JsonUtils; 5 | import lombok.*; 6 | 7 | import java.io.IOException; 8 | 9 | @Builder 10 | @Getter 11 | @Setter 12 | @NoArgsConstructor 13 | @ToString 14 | @EqualsAndHashCode 15 | public class BidBcn implements Partitioned { 16 | @JsonProperty("tx_id") 17 | private String txId; 18 | 19 | @JsonProperty("campaign_item_id") 20 | private long campaignItemId; 21 | 22 | @JsonProperty("creative_id") 23 | private String creativeId; 24 | 25 | @JsonProperty("creative_category") 26 | private String creativeCategory; 27 | 28 | @JsonProperty("app_uid") 29 | private String appUID; 30 | 31 | private String domain; 32 | 33 | public BidBcn( 34 | @JsonProperty("tx_id") String txId, 35 | @JsonProperty("campaign_item_id") long campaignItemId, 36 | @JsonProperty("creative_id") String creativeId, 37 | @JsonProperty("creative_category") String creativeCategory, 38 | @JsonProperty("app_uid") String appUID, 39 | @JsonProperty("domain") String domain) { 40 | this.txId = txId; 41 | this.campaignItemId = campaignItemId; 42 | this.creativeId = creativeId; 43 | this.creativeCategory = creativeCategory; 44 | this.appUID = appUID; 45 | this.domain = domain; 46 | } 47 | 48 | @Override 49 | public String getPartitionKey() { 50 | return txId; 51 | } 52 | 53 | @Override 54 | public byte[] getBytes() throws IOException { 55 | return JsonUtils.write(this); 56 | } 57 | 58 | @Override 59 | public boolean equals(Object o) { 60 | if (this == o) return true; 61 | if (!(o instanceof BidBcn)) return false; 62 | 63 | BidBcn bidBcn = (BidBcn) o; 64 | 65 | if (getCampaignItemId() != bidBcn.getCampaignItemId()) return false; 66 | if (!getTxId().equals(bidBcn.getTxId())) return false; 67 | if (!getCreativeId().equals(bidBcn.getCreativeId())) return false; 68 | if (!getDomain().equals(bidBcn.getDomain())) return false; 69 | if (!getCreativeCategory().equals(bidBcn.getCreativeCategory())) return false; 70 | return getAppUID().equals(bidBcn.getAppUID()); 71 | } 72 | 73 | public static BidBcn from(Bcn bcn) { 74 | return BidBcn.builder() 75 | .txId(bcn.getTxId()) 76 | .campaignItemId(bcn.getCampaignItemId()) 77 | .creativeId(bcn.getCreativeId()) 78 | .domain(bcn.getDomain()) 79 | .creativeCategory(bcn.getCreativeCategory()) 80 | .appUID(bcn.getAppUID()) 81 | .build(); 82 | } 83 | } -------------------------------------------------------------------------------- /fds-lambda-compaction/src/main/java/org/apache/parquet/hadoop/ColumnReadStorePublicImpl.java: -------------------------------------------------------------------------------- 1 | package org.apache.parquet.hadoop; 2 | 3 | import org.apache.parquet.VersionParser; 4 | import org.apache.parquet.column.ColumnDescriptor; 5 | import org.apache.parquet.column.ColumnReadStore; 6 | import org.apache.parquet.column.ColumnReader; 7 | import org.apache.parquet.column.impl.ColumnReaderImpl; 8 | import org.apache.parquet.column.page.PageReadStore; 9 | import org.apache.parquet.column.page.PageReader; 10 | import org.apache.parquet.io.api.Converter; 11 | import org.apache.parquet.io.api.GroupConverter; 12 | import org.apache.parquet.io.api.PrimitiveConverter; 13 | import org.apache.parquet.schema.GroupType; 14 | import org.apache.parquet.schema.MessageType; 15 | import org.apache.parquet.schema.Type; 16 | 17 | public class ColumnReadStorePublicImpl implements ColumnReadStore { 18 | private final PageReadStore pageReadStore; 19 | private final GroupConverter recordConverter; 20 | private final MessageType schema; 21 | private final VersionParser.ParsedVersion writerVersion; 22 | 23 | public ColumnReadStorePublicImpl(PageReadStore pageReadStore, GroupConverter recordConverter, MessageType schema, String createdBy) { 24 | this.pageReadStore = pageReadStore; 25 | this.recordConverter = recordConverter; 26 | this.schema = schema; 27 | 28 | VersionParser.ParsedVersion version; 29 | try { 30 | version = VersionParser.parse(createdBy); 31 | } catch (RuntimeException var7) { 32 | version = null; 33 | } catch (VersionParser.VersionParseException var8) { 34 | version = null; 35 | } 36 | 37 | this.writerVersion = version; 38 | } 39 | 40 | public ColumnReader getColumnReader(ColumnDescriptor path) { 41 | return this.newMemColumnReader(path, this.pageReadStore.getPageReader(path)); 42 | } 43 | 44 | public ColumnReaderImpl newMemColumnReader(ColumnDescriptor path, PageReader pageReader) { 45 | PrimitiveConverter converter = this.getPrimitiveConverter(path); 46 | return new ColumnReaderImpl(path, pageReader, converter, this.writerVersion); 47 | } 48 | 49 | private PrimitiveConverter getPrimitiveConverter(ColumnDescriptor path) { 50 | Type currentType = this.schema; 51 | Converter currentConverter = this.recordConverter; 52 | String[] var4 = path.getPath(); 53 | int var5 = var4.length; 54 | 55 | for(int var6 = 0; var6 < var5; ++var6) { 56 | String fieldName = var4[var6]; 57 | GroupType groupType = ((Type)currentType).asGroupType(); 58 | int fieldIndex = groupType.getFieldIndex(fieldName); 59 | currentType = groupType.getType(fieldName); 60 | currentConverter = ((Converter)currentConverter).asGroupConverter().getConverter(fieldIndex); 61 | } 62 | 63 | PrimitiveConverter converter = ((Converter)currentConverter).asPrimitiveConverter(); 64 | return converter; 65 | } 66 | } -------------------------------------------------------------------------------- /fds-lambda-ml-integration/src/main/java/com/provectus/fds/ml/JobDataGenerator.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.ml; 2 | 3 | import com.amazonaws.ClientConfiguration; 4 | import com.amazonaws.services.athena.AmazonAthena; 5 | import com.amazonaws.services.athena.AmazonAthenaClientBuilder; 6 | import com.provectus.fds.ml.processor.AthenaConfig; 7 | import com.provectus.fds.ml.processor.AthenaProcessor; 8 | import com.provectus.fds.ml.processor.CsvRecordProcessor; 9 | 10 | import java.io.BufferedReader; 11 | import java.io.InputStream; 12 | import java.io.InputStreamReader; 13 | import java.util.stream.Collectors; 14 | 15 | class JobDataGenerator { 16 | 17 | private static final long SLEEP_TIME = 1000L; 18 | 19 | private static final int CLIENT_EXECUTION_TIMEOUT = 0; 20 | private static final int TRAINING_FACTOR = 90; 21 | private static final int VERIFICATION_FACTOR = 10; 22 | 23 | private static final String SQL_RESOURCE = "categorized_bids.sql"; 24 | 25 | CsvRecordProcessor generateTrainingData(PrepareDataForTrainingJobLambda.LambdaConfiguration config) throws Exception { 26 | 27 | ClientConfiguration configuration = new ClientConfiguration() 28 | .withClientExecutionTimeout(CLIENT_EXECUTION_TIMEOUT); 29 | 30 | AmazonAthenaClientBuilder athenaClientBuilder = AmazonAthenaClientBuilder.standard() 31 | .withRegion(config.getRegion()) 32 | .withClientConfiguration(configuration); 33 | 34 | AmazonAthena client = athenaClientBuilder.build(); 35 | 36 | try (CsvRecordProcessor recordProcessor 37 | = new CsvRecordProcessor(config.getBucket(), 38 | config.getAthenaKey(), 39 | TRAINING_FACTOR, VERIFICATION_FACTOR)) { 40 | 41 | AthenaConfig athenaConfig = new AthenaConfig(); 42 | athenaConfig.setClient(client); 43 | athenaConfig.setDbName(config.getAthenaDatabase()); 44 | athenaConfig.setOutputLocation(getOutputLocation(config)); 45 | athenaConfig.setQuery(getSqlString()); 46 | athenaConfig.setSleepTime(SLEEP_TIME); 47 | athenaConfig.setRecordProcessor(recordProcessor); 48 | 49 | AthenaProcessor athenaProcessor = new AthenaProcessor(); 50 | athenaProcessor.process(athenaConfig); 51 | 52 | return recordProcessor; 53 | } 54 | } 55 | 56 | private String getSqlString() { 57 | InputStream is = getClass().getClassLoader().getResourceAsStream(JobDataGenerator.SQL_RESOURCE); 58 | if (is != null) { 59 | BufferedReader reader = new BufferedReader(new InputStreamReader(is)); 60 | return reader.lines().collect(Collectors.joining(System.lineSeparator())); 61 | } 62 | return null; 63 | } 64 | 65 | private String getOutputLocation(PrepareDataForTrainingJobLambda.LambdaConfiguration config) { 66 | return String.format("s3://%s/%s", 67 | config.getBucket(), 68 | config.getAthenaKey()); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /fds-lambda-ml-integration/src/main/java/com/provectus/fds/ml/SagemakerAlgorithmsRegistry.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.ml; 2 | 3 | import java.util.Collections; 4 | import java.util.Map; 5 | import java.util.stream.Collectors; 6 | import java.util.stream.Stream; 7 | 8 | class SagemakerAlgorithmsRegistry { 9 | 10 | // This map intended only for these algorithms 11 | // 'pca', 'kmeans', 'linear-learner', 'factorization-machines', 'ntm', 12 | // 'randomcutforest', 'knn', 'object2vec', 'ipinsights' 13 | // 14 | // If you want some different algos, then visit 15 | // https://github.com/aws/sagemaker-python-sdk/blob/master/src/sagemaker/amazon/amazon_estimator.py#L286 16 | // and implement absent part. 17 | @java.lang.SuppressWarnings("squid:S3878") 18 | private Map commonContainers = Stream.of(new String[][] { 19 | { "us-east-1", "382416733822" }, 20 | { "us-east-2", "404615174143" }, 21 | { "us-west-2", "174872318107" }, 22 | { "eu-west-1", "438346466558" }, 23 | { "eu-central-1", "664544806723" }, 24 | { "ap-northeast-1", "351501993468" }, 25 | { "ap-northeast-2", "835164637446" }, 26 | { "ap-southeast-2", "712309505854" }, 27 | { "us-gov-west-1", "226302683700" }, 28 | { "ap-southeast-1", "475088953585" }, 29 | { "ap-south-1", "991648021394" }, 30 | { "ca-central-1", "469771592824" }, 31 | { "eu-west-2", "644912444149" }, 32 | { "us-west-1", "632365934929" }, 33 | { "us-iso-east-1", "490574956308" }, 34 | }).collect(Collectors.collectingAndThen( 35 | Collectors.toMap(data -> data[0], data -> data[1]), 36 | Collections::unmodifiableMap)); 37 | 38 | public static class RegistryException extends RuntimeException { 39 | RegistryException(String message) { 40 | super(message); 41 | } 42 | } 43 | 44 | public String getImageUri(String regionName, String algorithm) { 45 | switch (algorithm) { 46 | case "pca": 47 | case "kmeans": 48 | case "linear-learner": 49 | case "factorization-machines": 50 | case "ntm": 51 | case "randomcutforest": 52 | case "knn": 53 | case "object2vec": 54 | case "ipinsights": 55 | return getEcrImageUriPrefix(commonContainers.get(regionName), regionName); 56 | default: 57 | throw new RegistryException("Algorithm " + algorithm + " is not supported by this class.\n" + 58 | "Probably, you must extend this class by yourself!"); 59 | } 60 | } 61 | 62 | public String getFullImageUri(String regionName, String algorithm) { 63 | return getImageUri(regionName, algorithm) + "/" + algorithm + ":latest"; 64 | } 65 | 66 | private String getEcrImageUriPrefix(String account, String region) { 67 | String domain = "amazonaws.com"; 68 | if ("us-iso-east-1".equals(region)) { 69 | domain = "c2s.ic.gov"; 70 | } 71 | return String.format("%s.dkr.ecr.%s.%s", account, region, domain); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /fds-lambda-locations-ingestion/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | fds-parent 7 | com.provectus.fds 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | fds-lambda-locations-ingestion 13 | 14 | 15 | 16 | com.provectus.fds 17 | fds-models 18 | 19 | 20 | com.fasterxml.jackson.core 21 | jackson-core 22 | 23 | 24 | com.fasterxml.jackson.core 25 | jackson-databind 26 | 27 | 28 | 29 | 30 | 31 | com.amazonaws 32 | aws-lambda-java-core 33 | 34 | 35 | 36 | com.amazonaws 37 | aws-lambda-java-events 38 | 39 | 40 | 41 | com.amazonaws 42 | aws-java-sdk-s3 43 | 44 | 45 | 46 | com.amazonaws 47 | aws-java-sdk-kinesis 48 | 49 | 50 | 51 | com.amazonaws 52 | amazon-kinesis-producer 53 | 54 | 55 | 56 | 57 | com.amazonaws 58 | aws-lambda-java-log4j 59 | 60 | 61 | 62 | org.slf4j 63 | jcl-over-slf4j 64 | 65 | 66 | 67 | org.slf4j 68 | slf4j-log4j12 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | org.apache.maven.plugins 77 | maven-shade-plugin 78 | 79 | false 80 | 81 | 82 | 83 | package 84 | 85 | shade 86 | 87 | 88 | 89 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /fds-lambda-ml-integration/src/main/java/com/provectus/fds/ml/LambdaS3EventProxy.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.ml; 2 | 3 | import com.amazonaws.services.kinesis.AmazonKinesis; 4 | import com.amazonaws.services.kinesis.AmazonKinesisClientBuilder; 5 | import com.amazonaws.services.kinesis.model.PutRecordsRequest; 6 | import com.amazonaws.services.kinesis.model.PutRecordsRequestEntry; 7 | import com.amazonaws.services.kinesis.model.PutRecordsResult; 8 | import com.amazonaws.services.lambda.runtime.Context; 9 | import com.amazonaws.services.lambda.runtime.RequestHandler; 10 | import com.amazonaws.services.lambda.runtime.events.S3Event; 11 | import com.fasterxml.jackson.core.JsonProcessingException; 12 | import com.fasterxml.jackson.databind.DeserializationFeature; 13 | import com.fasterxml.jackson.databind.ObjectMapper; 14 | import org.apache.logging.log4j.LogManager; 15 | import org.apache.logging.log4j.Logger; 16 | 17 | import java.nio.ByteBuffer; 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | 21 | @SuppressWarnings("unused") 22 | public class LambdaS3EventProxy implements RequestHandler { 23 | private static final Logger logger = LogManager.getLogger(LambdaS3EventProxy.class); 24 | 25 | private final ObjectMapper mapper 26 | = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); 27 | private final LambdaConfiguration config = new LambdaConfiguration(); 28 | 29 | private static final String PARTITION_KEY_PATTERN = "partitionKey-%d"; 30 | 31 | public LambdaS3EventProxy() { 32 | } 33 | 34 | public String handle(S3Event s3event) throws JsonProcessingException { 35 | 36 | logger.debug("Handle S3 event: {}", mapper.writeValueAsString(s3event)); 37 | 38 | AmazonKinesisClientBuilder clientBuilder = AmazonKinesisClientBuilder.standard(); 39 | AmazonKinesis kinesisClient = clientBuilder.build(); 40 | 41 | PutRecordsRequest putRecordsRequest = new PutRecordsRequest(); 42 | putRecordsRequest.setStreamName(config.getStreamName()); 43 | 44 | List entryList = new ArrayList<>(); 45 | 46 | PutRecordsRequestEntry entry = new PutRecordsRequestEntry(); 47 | entry.setData(ByteBuffer.wrap(mapper.writeValueAsBytes(s3event))); 48 | entry.setPartitionKey(String.format(PARTITION_KEY_PATTERN, s3event.hashCode())); 49 | entryList.add(entry); 50 | 51 | putRecordsRequest.setRecords(entryList); 52 | 53 | logger.info("Putting S3Event into stream: '{}' '{}'", 54 | s3event.toString(), config.getStreamName()); 55 | 56 | PutRecordsResult putRecordsResult = kinesisClient.putRecords(putRecordsRequest); 57 | 58 | String message = String.format("kinesisClient.putRecords: %s", putRecordsResult.toString()); 59 | logger.info(message); 60 | 61 | return message; 62 | } 63 | 64 | @Override 65 | public String handleRequest(S3Event s3event, Context context) { 66 | try { 67 | return handle(s3event); 68 | } catch (JsonProcessingException e) { 69 | throw new RuntimeException(logger.throwing(e)); 70 | } 71 | } 72 | 73 | private static class LambdaConfiguration extends Configuration { 74 | private String streamName; 75 | 76 | private String getStreamName() { 77 | if (streamName == null) 78 | streamName = getOrThrow("STREAM_NAME"); 79 | return streamName; 80 | } 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /fds-it/src/test/java/com/provectus/fds/it/BucketRemover.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.it; 2 | 3 | import com.amazonaws.services.s3.AmazonS3; 4 | import com.amazonaws.services.s3.AmazonS3ClientBuilder; 5 | import com.amazonaws.services.s3.model.*; 6 | 7 | import java.util.Iterator; 8 | 9 | /** 10 | * Helper class intended to completely remove a S3 bucket. 11 | * 12 | * Code derived from AWS example: 13 | * https://docs.aws.amazon.com/en_us/AmazonS3/latest/dev/delete-or-empty-bucket.html 14 | */ 15 | class BucketRemover { 16 | 17 | private final AmazonS3 s3Client; 18 | 19 | BucketRemover(String clientRegion) { 20 | s3Client = AmazonS3ClientBuilder.standard() 21 | .withRegion(clientRegion) 22 | .build(); 23 | } 24 | 25 | private boolean checkIfBucketExists(String bucketName) { 26 | return s3Client.doesBucketExistV2(bucketName); 27 | } 28 | 29 | void removeBucketWithRetries(String bucketName, int retries) { 30 | int i = 0; 31 | for ( ; checkIfBucketExists(bucketName) && i < retries; i++) { 32 | removeBucket(bucketName); 33 | } 34 | 35 | if (checkIfBucketExists(bucketName)) { 36 | System.out.printf("WARNING: Oh-oh... Despite of %d tries bucket '%s' was not removed properly\n", retries, bucketName); 37 | } else { 38 | System.out.printf("SUCCESS: Bucket '%s' was removed successfully after %d retries", bucketName, i); 39 | } 40 | } 41 | 42 | private void removeBucket(String bucketName) { 43 | 44 | // Delete all objects from the bucket. This is sufficient 45 | // for unversioned buckets. For versioned buckets, when you attempt to delete objects, Amazon S3 inserts 46 | // delete markers for all objects, but doesn't delete the object versions. 47 | // To delete objects from versioned buckets, delete all of the object versions before deleting 48 | // the bucket (see below for an example). 49 | ObjectListing objectListing = s3Client.listObjects(bucketName); 50 | while (true) { 51 | for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) { 52 | s3Client.deleteObject(bucketName, s3ObjectSummary.getKey()); 53 | } 54 | 55 | // If the bucket contains many objects, the listObjects() call 56 | // might not return all of the objects in the first listing. Check to 57 | // see whether the listing was truncated. If so, retrieve the next page of objects 58 | // and delete them. 59 | if (objectListing.isTruncated()) { 60 | objectListing = s3Client.listNextBatchOfObjects(objectListing); 61 | } else { 62 | break; 63 | } 64 | } 65 | 66 | // Delete all object versions (required for versioned buckets). 67 | VersionListing versionList = s3Client.listVersions(new ListVersionsRequest().withBucketName(bucketName)); 68 | while (true) { 69 | for (S3VersionSummary vs : versionList.getVersionSummaries()) { 70 | s3Client.deleteVersion(bucketName, vs.getKey(), vs.getVersionId()); 71 | } 72 | 73 | if (versionList.isTruncated()) { 74 | versionList = s3Client.listNextBatchOfVersions(versionList); 75 | } else { 76 | break; 77 | } 78 | } 79 | 80 | // After all objects and object versions are deleted, delete the bucket. 81 | s3Client.deleteBucket(bucketName); 82 | } 83 | } -------------------------------------------------------------------------------- /fds-lambda-ml-integration/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | fds-parent 7 | com.provectus.fds 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | fds-lambda-ml-integration 13 | 14 | 15 | com.amazonaws 16 | aws-lambda-java-core 17 | 18 | 19 | com.amazonaws 20 | aws-lambda-java-events 21 | 22 | 23 | com.amazonaws 24 | aws-java-sdk-s3 25 | 26 | 27 | com.amazonaws 28 | aws-java-sdk-athena 29 | 30 | 31 | com.amazonaws 32 | aws-java-sdk-sagemaker 33 | 34 | 35 | com.amazonaws 36 | aws-java-sdk-sagemakerruntime 37 | 38 | 39 | com.amazonaws 40 | aws-java-sdk-kinesis 41 | 42 | 43 | 44 | com.amazonaws 45 | aws-lambda-java-log4j2 46 | 47 | 48 | org.apache.logging.log4j 49 | log4j-core 50 | 51 | 52 | org.apache.logging.log4j 53 | log4j-api 54 | 55 | 56 | 57 | io.airlift 58 | slice 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | org.apache.maven.plugins 67 | maven-shade-plugin 68 | 3.2.0 69 | 70 | false 71 | 72 | 73 | *:* 74 | 75 | **/Log4j2Plugins.dat 76 | 77 | 78 | 79 | 80 | 81 | 82 | package 83 | 84 | shade 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /gatling/conf/recorder.conf: -------------------------------------------------------------------------------- 1 | recorder { 2 | core { 3 | #mode = "Proxy" 4 | #encoding = "utf-8" # The encoding used for reading/writing request bodies and the generated simulation 5 | #simulationsFolder = "" # The folder where generated simulation will be generated 6 | #package = "" # The package's name of the generated simulation 7 | #className = "RecordedSimulation" # The name of the generated Simulation class 8 | #thresholdForPauseCreation = 100 # The minimum time, in milliseconds, that must pass between requests to trigger a pause creation 9 | #saveConfig = false # When set to true, the configuration from the Recorder GUI overwrites this configuration 10 | #headless = false # When set to true, run the Recorder in headless mode instead of the GUI 11 | #harFilePath = "" # The path of the HAR file to convert 12 | } 13 | filters { 14 | #filterStrategy = "Disabled" # The selected filter resources filter strategy (currently supported : "Disabled", "BlackList", "WhiteList") 15 | #whitelist = [] # The list of ressources patterns that are part of the Recorder's whitelist 16 | #blacklist = [] # The list of ressources patterns that are part of the Recorder's blacklist 17 | } 18 | http { 19 | #automaticReferer = true # When set to false, write the referer + enable 'disableAutoReferer' in the generated simulation 20 | #followRedirect = true # When set to false, write redirect requests + enable 'disableFollowRedirect' in the generated simulation 21 | #removeCacheHeaders = true # When set to true, removes from the generated requests headers leading to request caching 22 | #inferHtmlResources = true # When set to true, add inferred resources + set 'inferHtmlResources' with the configured blacklist/whitelist in the generated simulation 23 | #checkResponseBodies = false # When set to true, save response bodies as files and add raw checks in the generated simulation 24 | } 25 | proxy { 26 | #port = 8000 # Local port used by Gatling's Proxy for HTTP/HTTPS 27 | https { 28 | #mode = "SelfSignedCertificate" # The selected "HTTPS mode" (currently supported : "SelfSignedCertificate", "ProvidedKeyStore", "CertificateAuthority") 29 | keyStore { 30 | #path = "" # The path of the custom key store 31 | #password = "" # The password for this key store 32 | #type = "JKS" # The type of the key store (currently supported: "JKS") 33 | } 34 | certificateAuthority { 35 | #certificatePath = "" # The path of the custom certificate 36 | #privateKeyPath = "" # The certificate's private key path 37 | } 38 | } 39 | outgoing { 40 | #host = "" # The outgoing proxy's hostname 41 | #username = "" # The username to use to connect to the outgoing proxy 42 | #password = "" # The password corresponding to the user to use to connect to the outgoing proxy 43 | #port = 0 # The HTTP port to use to connect to the outgoing proxy 44 | #sslPort = 0 # If set, The HTTPS port to use to connect to the outgoing proxy 45 | } 46 | } 47 | netty { 48 | #maxInitialLineLength = 10000 # Maximum length of the initial line of the response (e.g. "HTTP/1.0 200 OK") 49 | #maxHeaderSize = 20000 # Maximum size, in bytes, of each request's headers 50 | #maxChunkSize = 8192 # Maximum length of the content or each chunk 51 | #maxContentLength = 100000000 # Maximum length of the aggregated content of each response 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /fds-lambda-ml-integration/src/main/java/com/provectus/fds/ml/InvokeEndpointLambda.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.ml; 2 | 3 | import com.amazonaws.services.lambda.runtime.Context; 4 | import com.amazonaws.services.lambda.runtime.RequestHandler; 5 | import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyRequestEvent; 6 | import com.amazonaws.services.lambda.runtime.events.APIGatewayProxyResponseEvent; 7 | import com.amazonaws.services.sagemakerruntime.AmazonSageMakerRuntime; 8 | import com.amazonaws.services.sagemakerruntime.AmazonSageMakerRuntimeClientBuilder; 9 | import com.amazonaws.services.sagemakerruntime.model.InvokeEndpointRequest; 10 | import com.amazonaws.services.sagemakerruntime.model.InvokeEndpointResult; 11 | import com.fasterxml.jackson.databind.DeserializationFeature; 12 | import com.fasterxml.jackson.databind.ObjectMapper; 13 | import org.apache.http.HttpStatus; 14 | import org.apache.logging.log4j.LogManager; 15 | import org.apache.logging.log4j.Logger; 16 | 17 | import java.nio.ByteBuffer; 18 | import java.nio.charset.Charset; 19 | import java.util.StringJoiner; 20 | 21 | @SuppressWarnings("unused") 22 | public class InvokeEndpointLambda implements RequestHandler { 23 | private static final Logger logger = LogManager.getLogger(InvokeEndpointLambda.class); 24 | 25 | private final ObjectMapper objectMapper 26 | = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); 27 | 28 | private final Configuration config = new Configuration(); 29 | 30 | @Override 31 | public APIGatewayProxyResponseEvent handleRequest(APIGatewayProxyRequestEvent input, Context context) { 32 | try { 33 | PredictRequest r = objectMapper.readValue(input.getBody(), PredictRequest.class); 34 | 35 | logger.debug("Got a prediction request: {}", objectMapper.writeValueAsString(r)); 36 | 37 | AmazonSageMakerRuntime runtime 38 | = AmazonSageMakerRuntimeClientBuilder 39 | .standard() 40 | .withRegion(config.getRegion()) 41 | .build(); 42 | 43 | StringJoiner joiner = new StringJoiner(","); 44 | joiner.add(String.valueOf(r.getCategorizedCampaignItemId())) 45 | .add(String.valueOf(r.getCategorizedDomain())) 46 | .add(String.valueOf(r.getCategorizedCreativeId())) 47 | .add(String.valueOf(r.getCategorizedCreativeCategory())) 48 | .add(String.valueOf(r.getWinPrice())); 49 | 50 | logger.info("Invoke the request: {}", joiner); 51 | 52 | ByteBuffer bodyBuffer 53 | = ByteBuffer.wrap(joiner.toString().getBytes(Charset.forName("UTF-8"))); 54 | 55 | InvokeEndpointRequest request = new InvokeEndpointRequest() 56 | .withEndpointName(config.getEndpoint()) 57 | .withContentType("text/csv") 58 | .withBody(bodyBuffer); 59 | 60 | InvokeEndpointResult invokeEndpointResult = runtime.invokeEndpoint(request); 61 | 62 | String bodyResponse = new String(invokeEndpointResult.getBody().array()); 63 | 64 | logger.info("Got the prediction answer: {}", bodyResponse); 65 | 66 | APIGatewayProxyResponseEvent responseEvent = new APIGatewayProxyResponseEvent(); 67 | responseEvent.setBody(bodyResponse); 68 | responseEvent.setStatusCode(HttpStatus.SC_OK); 69 | 70 | return responseEvent; 71 | 72 | } catch (Exception e) { 73 | logger.throwing(e); 74 | throw new RuntimeException(e); 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /fds-lambda-compaction/src/main/java/com/provectus/fds/compaction/GlueDAO.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction; 2 | 3 | import com.amazonaws.ClientConfiguration; 4 | import com.amazonaws.services.glue.AWSGlueAsync; 5 | import com.amazonaws.services.glue.AWSGlueAsyncClientBuilder; 6 | import com.amazonaws.services.glue.model.*; 7 | import com.provectus.fds.compaction.utils.GlueConverter; 8 | import org.apache.parquet.hadoop.metadata.FileMetaData; 9 | 10 | import java.util.List; 11 | import java.util.Map; 12 | import java.util.stream.Collectors; 13 | 14 | public class GlueDAO { 15 | private final AWSGlueAsync client; 16 | private final String catalogId; 17 | 18 | 19 | private static AWSGlueAsync createAsyncGlueClient(int maxConnections) 20 | { 21 | ClientConfiguration clientConfig = new ClientConfiguration().withMaxConnections(maxConnections); 22 | AWSGlueAsyncClientBuilder asyncGlueClientBuilder = AWSGlueAsyncClientBuilder.standard() 23 | .withClientConfiguration(clientConfig); 24 | 25 | return asyncGlueClientBuilder.build(); 26 | } 27 | 28 | 29 | public GlueDAO(String catalogId) { 30 | this.catalogId = catalogId; 31 | this.client = createAsyncGlueClient(1); 32 | } 33 | 34 | public void addPartition(String databaseName 35 | , String tableName 36 | , List> values 37 | , FileMetaData metaData) { 38 | 39 | GetTableResult getTableResult = client.getTable(new GetTableRequest() 40 | .withCatalogId(catalogId) 41 | .withDatabaseName(databaseName) 42 | .withName(tableName)); 43 | 44 | List partitionValues = values.stream().map(Map.Entry::getValue).collect(Collectors.toList()); 45 | 46 | 47 | StorageDescriptor sd = getTableResult.getTable().getStorageDescriptor(); 48 | 49 | StringBuilder sb = new StringBuilder(); 50 | sb.append(sd.getLocation()); 51 | if (!sd.getLocation().endsWith("/")) { 52 | sb.append("/"); 53 | } 54 | 55 | for (Map.Entry value : values) { 56 | sb.append(value.getKey()); 57 | sb.append("="); 58 | sb.append(value.getValue()); 59 | sb.append("/"); 60 | } 61 | 62 | CreatePartitionRequest request = new CreatePartitionRequest(); 63 | request.setCatalogId(catalogId); 64 | request.setDatabaseName(databaseName); 65 | request.setTableName(tableName); 66 | PartitionInput input = new PartitionInput(); 67 | input.setValues(partitionValues); 68 | 69 | input.setStorageDescriptor(GlueConverter.convertStorage(metaData, sb.toString(), sd)); 70 | 71 | try { 72 | GetPartitionResult partitionResult = client.getPartition( 73 | new GetPartitionRequest() 74 | .withCatalogId(catalogId) 75 | .withDatabaseName(databaseName) 76 | .withTableName(tableName) 77 | .withPartitionValues(partitionValues) 78 | ); 79 | 80 | client.updatePartition(new UpdatePartitionRequest() 81 | .withPartitionValueList(partitionValues) 82 | .withPartitionInput(input) 83 | .withCatalogId(catalogId) 84 | .withDatabaseName(databaseName) 85 | .withTableName(tableName) 86 | ); 87 | } catch (EntityNotFoundException e){ 88 | client.createPartition(new CreatePartitionRequest() 89 | .withPartitionInput(input) 90 | .withCatalogId(catalogId) 91 | .withDatabaseName(databaseName) 92 | .withTableName(tableName) 93 | ); 94 | } 95 | } 96 | 97 | } 98 | -------------------------------------------------------------------------------- /fds-lambda-api/src/main/java/com/provectus/fds/api/AbstractBcnHandler.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.api; 2 | 3 | import com.amazonaws.services.kinesis.AmazonKinesis; 4 | import com.amazonaws.services.kinesis.AmazonKinesisClientBuilder; 5 | import com.amazonaws.services.kinesis.model.PutRecordRequest; 6 | import com.amazonaws.services.kinesis.model.PutRecordResult; 7 | import com.amazonaws.services.lambda.runtime.Context; 8 | import com.amazonaws.services.lambda.runtime.RequestStreamHandler; 9 | import com.fasterxml.jackson.databind.JsonNode; 10 | import com.provectus.fds.models.bcns.Partitioned; 11 | import com.provectus.fds.models.utils.JsonUtils; 12 | 13 | import java.io.*; 14 | import java.nio.ByteBuffer; 15 | import java.util.Optional; 16 | import java.util.concurrent.atomic.AtomicReference; 17 | 18 | public abstract class AbstractBcnHandler implements RequestStreamHandler { 19 | 20 | public static final String ENV_STREAM_NAME = "STREAM_NAME"; 21 | public static final String STREAM_NAME_DEFUALT_VALUE = "bcns"; 22 | public static final String ENV_DEFAULT_REGION = "AWS_DEFAULT_REGION"; 23 | public static final byte[] RESPONSE_OK = "{\"statusCode\": 200}".getBytes(); 24 | 25 | private final AtomicReference amazonKinesisReference = new AtomicReference<>(); 26 | 27 | private final String streamName; 28 | 29 | 30 | public AbstractBcnHandler() { 31 | this.streamName = System.getenv().getOrDefault(ENV_STREAM_NAME, STREAM_NAME_DEFUALT_VALUE); 32 | } 33 | 34 | @Override 35 | public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException { 36 | context.getLogger().log("Handling request"); 37 | try { 38 | BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); 39 | JsonNode inputNode = JsonUtils.readTree(reader); 40 | if (inputNode.has("queryStringParameters")) { 41 | JsonNode parameters = inputNode.get("queryStringParameters"); 42 | Optional bcn = this.buildBcn(parameters, context); 43 | if (bcn.isPresent()) { 44 | Partitioned rawBcn = bcn.get(); 45 | 46 | this.send( 47 | rawBcn.getPartitionKey(), 48 | rawBcn.getBytes(), 49 | context 50 | ); 51 | } 52 | } else { 53 | context.getLogger().log(String.format("Wrong request: %s", inputNode.toString())); 54 | } 55 | 56 | } catch (Throwable e) { 57 | context.getLogger().log("Error on processing bcn: " + e.getMessage()); 58 | } 59 | outputStream.write(RESPONSE_OK); 60 | context.getLogger().log(RESPONSE_OK); 61 | } 62 | 63 | private void send(String partitionKey, byte[] data, Context context) { 64 | AmazonKinesis client = getKinesisOrBuild(); 65 | 66 | PutRecordRequest putRecordRequest = new PutRecordRequest().withStreamName(streamName) 67 | .withData(ByteBuffer.wrap(data)).withPartitionKey(partitionKey); 68 | 69 | 70 | PutRecordResult response = client.putRecord(putRecordRequest); 71 | context.getLogger().log(String.format("Record was sent to Kenesis %s", streamName)); 72 | } 73 | 74 | private AmazonKinesis getKinesisOrBuild() { 75 | AmazonKinesis result = this.amazonKinesisReference.get(); 76 | if (result == null) { 77 | result = buildKinesis(); 78 | if (!this.amazonKinesisReference.compareAndSet(null, result)) { 79 | return this.amazonKinesisReference.get(); 80 | } 81 | } 82 | return result; 83 | } 84 | 85 | private AmazonKinesis buildKinesis() { 86 | AmazonKinesisClientBuilder clientBuilder = AmazonKinesisClientBuilder.standard(); 87 | clientBuilder.setRegion(System.getenv(ENV_DEFAULT_REGION)); 88 | 89 | return clientBuilder.build(); 90 | } 91 | 92 | 93 | public abstract Optional buildBcn(JsonNode parameters, Context context) throws IOException; 94 | 95 | } 96 | -------------------------------------------------------------------------------- /fds-lambda-ml-integration/src/main/java/com/provectus/fds/ml/ReplaceEndpointConfigLambda.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.ml; 2 | 3 | import com.amazonaws.services.lambda.runtime.Context; 4 | import com.amazonaws.services.lambda.runtime.RequestHandler; 5 | import com.amazonaws.services.lambda.runtime.events.KinesisEvent; 6 | import com.amazonaws.services.lambda.runtime.events.S3Event; 7 | import com.amazonaws.services.s3.event.S3EventNotification; 8 | import com.fasterxml.jackson.databind.DeserializationFeature; 9 | import com.fasterxml.jackson.databind.ObjectMapper; 10 | import com.provectus.fds.ml.utils.IntegrationModuleHelper; 11 | import org.apache.logging.log4j.LogManager; 12 | import org.apache.logging.log4j.Logger; 13 | 14 | import java.io.IOException; 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | 18 | @SuppressWarnings("unused") 19 | public class ReplaceEndpointConfigLambda implements RequestHandler> { 20 | 21 | private static final Logger logger = LogManager.getLogger(ReplaceEndpointConfigLambda.class); 22 | private final ObjectMapper mapper 23 | = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); 24 | private final IntegrationModuleHelper h = new IntegrationModuleHelper(); 25 | 26 | private static final String modelFileName = "model.tar.gz"; 27 | LambdaConfiguration config = new LambdaConfiguration(); 28 | 29 | @Override 30 | public List handleRequest(KinesisEvent input, Context context) { 31 | logger.debug("Processing Kinesis event: {}", h.writeValueAsString(input, mapper)); 32 | 33 | List results = new ArrayList<>(); 34 | 35 | for (KinesisEvent.KinesisEventRecord r : input.getRecords()) { 36 | try { 37 | S3Event s3Event = mapper.readerFor(S3Event.class) 38 | .readValue(r.getKinesis().getData().array()); 39 | results.add(handleRequest(s3Event, context)); 40 | } catch (IOException e) { 41 | throw new RuntimeException(logger.throwing(e)); 42 | } 43 | } 44 | return results; 45 | } 46 | 47 | @SuppressWarnings("unused") 48 | private S3Event handleRequest(S3Event s3Event, Context context) { 49 | logger.debug("Received S3 event: {}", h.writeValueAsString(s3Event, mapper)); 50 | String configBucket = config.getBucket(); 51 | 52 | for (S3EventNotification.S3EventNotificationRecord record : s3Event.getRecords()) { 53 | String eventBucket = record.getS3().getBucket().getName(); 54 | String eventKey = record.getS3().getObject().getKey(); 55 | 56 | logger.info("Got an event with s3://{}/{}, {}", eventBucket, eventKey, record.getEventName()); 57 | 58 | if (eventKey.endsWith(modelFileName) && eventBucket.equals(configBucket)) { 59 | 60 | logger.info("Starting updating endpoint process"); 61 | 62 | EndpointUpdater.EndpointUpdaterBuilder updaterBuilder 63 | = new EndpointUpdater.EndpointUpdaterBuilder(config); 64 | 65 | updaterBuilder 66 | .withEndpointName(config.getEndpoint()) 67 | .withServicePrefix(config.getServicePrefx()) 68 | .withRegionId(config.getRegion()) 69 | .withSageMakerRole(config.getSagemakerRole()) 70 | .withDataUrl(String.format("s3://%s/%s", eventBucket, eventKey)); 71 | 72 | updaterBuilder.build().updateEndpoint(); 73 | } 74 | } 75 | return s3Event; 76 | } 77 | 78 | static class LambdaConfiguration extends Configuration { 79 | private String servicePrefix; 80 | private Integer initialInstanceCount; 81 | 82 | private String getServicePrefx() { 83 | if (servicePrefix == null) 84 | servicePrefix = getOrThrow("SERVICE_PREFIX"); 85 | return servicePrefix; 86 | } 87 | public int getInitialInstanceCount() { 88 | if (initialInstanceCount == null) 89 | initialInstanceCount = Integer.valueOf(getOrElse("PRODUCTION_VARIANT_INITIAL_INSTANCE_COUNT", "1")); 90 | 91 | return initialInstanceCount; 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /fds-lambda-utils/src/main/java/com/provectus/fds/utils/ApplicationStartLambda.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.utils; 2 | 3 | import com.amazonaws.services.lambda.runtime.Context; 4 | import com.amazonaws.services.lambda.runtime.LambdaLogger; 5 | import com.amazonaws.services.lambda.runtime.RequestHandler; 6 | import org.json.JSONObject; 7 | import software.amazon.awssdk.services.kinesisanalyticsv2.KinesisAnalyticsV2Client; 8 | import software.amazon.awssdk.services.kinesisanalyticsv2.model.ApplicationRestoreConfiguration; 9 | import software.amazon.awssdk.services.kinesisanalyticsv2.model.ApplicationRestoreType; 10 | import software.amazon.awssdk.services.kinesisanalyticsv2.model.RunConfiguration; 11 | import software.amazon.awssdk.services.kinesisanalyticsv2.model.StartApplicationRequest; 12 | 13 | import java.io.IOException; 14 | import java.io.OutputStreamWriter; 15 | import java.net.HttpURLConnection; 16 | import java.net.URL; 17 | import java.util.LinkedHashMap; 18 | import java.util.Map; 19 | 20 | public class ApplicationStartLambda implements RequestHandler, Object> { 21 | 22 | @Override 23 | public Object handleRequest(Map input, Context context) { 24 | LambdaLogger logger = context.getLogger(); 25 | logger.log("Input: " + input); 26 | LinkedHashMap properties = (LinkedHashMap) input.get("ResourceProperties"); 27 | String appName = (String) properties.get("ApplicationName"); 28 | String requestType = (String) input.get("RequestType"); 29 | JSONObject responseData = new JSONObject(); 30 | 31 | 32 | if (requestType.equalsIgnoreCase("Create") || requestType.equalsIgnoreCase("Update")) { 33 | KinesisAnalyticsV2Client kac = KinesisAnalyticsV2Client.create(); 34 | kac.startApplication(StartApplicationRequest.builder() 35 | .applicationName(appName) 36 | .runConfiguration(RunConfiguration.builder() 37 | .applicationRestoreConfiguration( 38 | ApplicationRestoreConfiguration.builder() 39 | .applicationRestoreType( 40 | ApplicationRestoreType.RESTORE_FROM_LATEST_SNAPSHOT) 41 | .build()) 42 | .build()) 43 | .build()); 44 | logger.log(String.format("The %s was started", appName)); 45 | sendResponse(input, context, "SUCCESS", responseData); 46 | } else if (requestType.equalsIgnoreCase("DELETE")) { 47 | logger.log("Resource delete action"); 48 | sendResponse(input, context, "SUCCESS", responseData); 49 | } else { 50 | sendResponse(input, context, "SUCCESS", responseData); 51 | } 52 | 53 | return null; 54 | } 55 | 56 | private Object sendResponse( 57 | Map input, 58 | Context context, 59 | String responseStatus, 60 | JSONObject responseData) { 61 | 62 | String responseUrl = (String) input.get("ResponseURL"); 63 | context.getLogger().log("ResponseURL: " + responseUrl); 64 | 65 | URL url; 66 | try { 67 | url = new URL(responseUrl); 68 | HttpURLConnection connection = (HttpURLConnection) url.openConnection(); 69 | connection.setDoOutput(true); 70 | connection.setRequestMethod("PUT"); 71 | 72 | JSONObject responseBody = new JSONObject(); 73 | responseBody.put("Status", responseStatus); 74 | responseBody.put("PhysicalResourceId", context.getLogStreamName()); 75 | responseBody.put("StackId", input.get("StackId")); 76 | responseBody.put("RequestId", input.get("RequestId")); 77 | responseBody.put("LogicalResourceId", input.get("LogicalResourceId")); 78 | responseBody.put("Data", responseData); 79 | 80 | OutputStreamWriter response = new OutputStreamWriter(connection.getOutputStream()); 81 | response.write(responseBody.toString()); 82 | response.close(); 83 | context.getLogger().log("Response Code: " + connection.getResponseCode()); 84 | 85 | } catch (IOException e) { 86 | e.printStackTrace(); 87 | } 88 | 89 | return null; 90 | } 91 | 92 | } 93 | -------------------------------------------------------------------------------- /fds-lambda-compaction/src/main/java/com/provectus/fds/compaction/utils/BlocksCombiner.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.compaction.utils; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.fs.Path; 5 | import org.apache.parquet.hadoop.ParquetFileReader; 6 | import org.apache.parquet.hadoop.metadata.BlockMetaData; 7 | import org.apache.parquet.hadoop.metadata.ParquetMetadata; 8 | 9 | import java.io.IOException; 10 | import java.util.ArrayList; 11 | import java.util.HashMap; 12 | import java.util.List; 13 | import java.util.Map; 14 | 15 | import static java.util.Collections.unmodifiableList; 16 | 17 | public class BlocksCombiner { 18 | 19 | private final Map footers = new HashMap<>(); 20 | private final List inputFiles; 21 | private final long maxBlockSize; 22 | private final Configuration configuration; 23 | 24 | public BlocksCombiner(List inputFiles, long maxBlockSize, Configuration configuration) { 25 | this.inputFiles = inputFiles; 26 | this.maxBlockSize = maxBlockSize; 27 | this.configuration = configuration; 28 | } 29 | 30 | public List combineLargeBlocks() { 31 | List blocks = new ArrayList<>(); 32 | long largeBlockSize = 0; 33 | long largeBlockRecords = 0; 34 | List smallBlocks = new ArrayList<>(); 35 | for (Path inputFile : inputFiles) { 36 | try (ParquetFileReader reader = getReader(inputFile, configuration, footers)) { 37 | for (int blockIndex = 0; blockIndex < reader.blocksCount(); blockIndex++) { 38 | BlockMetaData block = reader.getBlockMetaData(blockIndex); 39 | if (!smallBlocks.isEmpty() && largeBlockSize + block.getTotalByteSize() > maxBlockSize) { 40 | blocks.add(new SmallBlocksUnion(smallBlocks, largeBlockRecords)); 41 | smallBlocks = new ArrayList<>(); 42 | largeBlockSize = 0; 43 | largeBlockRecords = 0; 44 | } 45 | largeBlockSize += block.getTotalByteSize(); 46 | largeBlockRecords += block.getRowCount(); 47 | smallBlocks.add(new SmallBlock(inputFile, blockIndex,configuration, footers)); 48 | } 49 | } catch (Exception e) { 50 | e.printStackTrace(); 51 | } 52 | } 53 | if (!smallBlocks.isEmpty()) { 54 | blocks.add(new SmallBlocksUnion(smallBlocks, largeBlockRecords)); 55 | } 56 | return unmodifiableList(blocks); 57 | } 58 | 59 | 60 | public static ParquetFileReader getReader(Path inputFile, Configuration configuration, Map footers) throws IOException { 61 | ParquetMetadata footer = footers.computeIfAbsent(inputFile, (path) -> readFooter(path, configuration)); 62 | return new ParquetFileReader(configuration, inputFile, footer); 63 | } 64 | 65 | public static ParquetMetadata readFooter(Path inputFile, Configuration configuration) { 66 | try { 67 | return ParquetFileReader.readFooter(configuration, inputFile); 68 | } catch (IOException e) { 69 | throw new RuntimeException(e); 70 | } 71 | } 72 | 73 | public static class SmallBlocksUnion { 74 | private final List blocks; 75 | private final long rowCount; 76 | 77 | public SmallBlocksUnion(List blocks, long rowCount) { 78 | this.blocks = blocks; 79 | this.rowCount = rowCount; 80 | } 81 | 82 | public List getBlocks() { 83 | return blocks; 84 | } 85 | 86 | public long getRowCount() { 87 | return rowCount; 88 | } 89 | } 90 | 91 | public static class SmallBlock { 92 | private final Path path; 93 | private final int blockIndex; 94 | private final Configuration configuration; 95 | private final Map footers; 96 | 97 | public SmallBlock(Path path, int blockIndex, Configuration configuration, Map footers) { 98 | this.path = path; 99 | this.blockIndex = blockIndex; 100 | this.configuration = configuration; 101 | this.footers = footers; 102 | } 103 | 104 | public ParquetFileReader getReader() throws IOException { 105 | return BlocksCombiner.getReader(this.path, this.configuration, this.footers); 106 | } 107 | 108 | public int getBlockIndex() { 109 | return blockIndex; 110 | } 111 | 112 | public Path getPath() { 113 | return path; 114 | } 115 | } 116 | } -------------------------------------------------------------------------------- /fds-common-dynamodb/src/main/java/com/provectus/fds/dynamodb/models/Aggregation.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.dynamodb.models; 2 | 3 | import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBHashKey; 4 | import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBRangeKey; 5 | import com.fasterxml.jackson.annotation.JsonIgnore; 6 | import com.fasterxml.jackson.annotation.JsonIgnoreProperties; 7 | import com.fasterxml.jackson.annotation.JsonProperty; 8 | import com.provectus.fds.dynamodb.ZoneDateTimeUtils; 9 | 10 | import java.time.Instant; 11 | import java.time.ZoneId; 12 | import java.time.ZonedDateTime; 13 | import java.time.temporal.ChronoField; 14 | import java.time.temporal.ChronoUnit; 15 | import java.util.Objects; 16 | import java.util.Optional; 17 | 18 | @JsonIgnoreProperties(ignoreUnknown = true) 19 | public class Aggregation { 20 | private long campaignItemId = 0L; 21 | private long period = 0L; 22 | private Long clicks = 0L; 23 | private Long imps = 0L; 24 | private Long bids = 0L; 25 | 26 | public Aggregation() { 27 | } 28 | 29 | public Aggregation(long campaignItemId) { 30 | this.campaignItemId = campaignItemId; 31 | } 32 | 33 | public Aggregation(long campaignItemId, long period, Long clicks, Long imps, Long bids) { 34 | this.campaignItemId = campaignItemId; 35 | this.period = period; 36 | this.clicks = clicks; 37 | this.imps = imps; 38 | this.bids = bids; 39 | } 40 | 41 | @DynamoDBHashKey(attributeName="campaign_item_id") 42 | public long getCampaignItemId() { 43 | return campaignItemId; 44 | } 45 | 46 | public void setCampaignItemId(long campaignItemId) { 47 | this.campaignItemId = campaignItemId; 48 | } 49 | 50 | public long getClicks() { 51 | return Optional.ofNullable(clicks).orElse(0L); 52 | } 53 | 54 | public void setClicks(long clicks) { 55 | this.clicks = clicks; 56 | } 57 | 58 | public long getImps() { 59 | return Optional.ofNullable(imps).orElse(0L); 60 | } 61 | 62 | public void setImps(long imps) { 63 | this.imps = imps; 64 | } 65 | 66 | public long getBids() { 67 | return Optional.ofNullable(bids).orElse(0L); 68 | } 69 | 70 | public void setBids(long bids) { 71 | this.bids = bids; 72 | } 73 | 74 | @DynamoDBRangeKey(attributeName = "period") 75 | public long getPeriod() { 76 | return period; 77 | } 78 | 79 | public void setPeriod(long period) { 80 | this.period = period; 81 | } 82 | 83 | public ZonedDateTime getDateTime(ZoneId zoneId) { 84 | return Instant.ofEpochSecond(this.period).atZone(zoneId); 85 | } 86 | 87 | public ZonedDateTime getDateTime(ZoneId zoneId, ChronoUnit unit) { 88 | return ZoneDateTimeUtils.truncatedTo(Instant.ofEpochSecond(this.period).atZone(zoneId), unit); 89 | } 90 | 91 | 92 | public Aggregation addAggregation(Aggregation other) { 93 | this.bids+=other.bids; 94 | this.clicks+=other.clicks; 95 | this.imps+=other.imps; 96 | return this; 97 | } 98 | 99 | @Override 100 | public boolean equals(Object o) { 101 | if (this == o) return true; 102 | if (o == null || getClass() != o.getClass()) return false; 103 | Aggregation that = (Aggregation) o; 104 | return campaignItemId == that.campaignItemId && 105 | period == that.period && 106 | Objects.equals(clicks, that.clicks) && 107 | Objects.equals(imps, that.imps) && 108 | Objects.equals(bids, that.bids); 109 | } 110 | 111 | @Override 112 | public int hashCode() { 113 | return Objects.hash(campaignItemId, period, clicks, imps, bids); 114 | } 115 | 116 | public Aggregation clone() { 117 | return new Aggregation(campaignItemId, period, clicks, imps, bids); 118 | } 119 | 120 | public Aggregation withPeriod(long period) { 121 | Aggregation that = this.clone(); 122 | that.setPeriod(period); 123 | return that; 124 | } 125 | 126 | @Override 127 | public String toString() { 128 | return "Aggregation{" + 129 | "campaignItemId=" + campaignItemId + 130 | ", period=" + period + 131 | ", clicks=" + clicks + 132 | ", imps=" + imps + 133 | ", bids=" + bids + 134 | '}'; 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /fds-lambda-ml-integration/src/main/java/com/provectus/fds/ml/PredictRequest.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.ml; 2 | 3 | import io.airlift.slice.Slices; 4 | import io.airlift.slice.XxHash64; 5 | 6 | import java.io.IOException; 7 | 8 | public class PredictRequest { 9 | private String campaignItemId; 10 | private String domain; 11 | private String creativeId; 12 | private String creativeCategory; 13 | private Double winPrice; 14 | 15 | public PredictRequest() { 16 | } 17 | 18 | public PredictRequest(String campaignItemId, String domain, String creativeId, String creativeCategory, Double winPrice) { 19 | this.campaignItemId = campaignItemId; 20 | this.domain = domain; 21 | this.creativeId = creativeId; 22 | this.creativeCategory = creativeCategory; 23 | this.winPrice = winPrice; 24 | } 25 | 26 | public String getCampaignItemId() { 27 | return campaignItemId; 28 | } 29 | 30 | public Double getCategorizedCampaignItemId() throws IOException { 31 | return calculateCategory(campaignItemId); 32 | } 33 | 34 | public void setCampaignItemId(String campaignItemId) { 35 | this.campaignItemId = campaignItemId; 36 | } 37 | 38 | public String getDomain() { 39 | return domain; 40 | } 41 | 42 | public Double getCategorizedDomain() throws IOException { 43 | return calculateCategory(domain); 44 | } 45 | 46 | public void setDomain(String domain) { 47 | this.domain = domain; 48 | } 49 | 50 | public String getCreativeId() { 51 | return creativeId; 52 | } 53 | 54 | public Double getCategorizedCreativeId() throws IOException { 55 | return calculateCategory(creativeId); 56 | } 57 | 58 | public void setCreativeId(String creativeId) { 59 | this.creativeId = creativeId; 60 | } 61 | 62 | public String getCreativeCategory() { 63 | return creativeCategory; 64 | } 65 | 66 | public Double getCategorizedCreativeCategory() throws IOException { 67 | return calculateCategory(creativeCategory); 68 | } 69 | 70 | public void setCreativeCategory(String creativeCategory) { 71 | this.creativeCategory = creativeCategory; 72 | } 73 | 74 | public Double getWinPrice() { 75 | return winPrice; 76 | } 77 | 78 | public void setWinPrice(Double winPrice) { 79 | this.winPrice = winPrice; 80 | } 81 | 82 | @Override 83 | public String toString() { 84 | return "PredictRequest{" + 85 | "campaignItemId='" + campaignItemId + '\'' + 86 | ", domain='" + domain + '\'' + 87 | ", creativeId='" + creativeId + '\'' + 88 | ", creativeCategory='" + creativeCategory + '\'' + 89 | ", winPrice=" + winPrice + 90 | '}'; 91 | } 92 | 93 | @Override 94 | public boolean equals(Object o) { 95 | if (this == o) return true; 96 | if (o == null || getClass() != o.getClass()) return false; 97 | 98 | PredictRequest request = (PredictRequest) o; 99 | 100 | if (campaignItemId != null ? !campaignItemId.equals(request.campaignItemId) : request.campaignItemId != null) 101 | return false; 102 | if (domain != null ? !domain.equals(request.domain) : request.domain != null) return false; 103 | if (creativeId != null ? !creativeId.equals(request.creativeId) : request.creativeId != null) return false; 104 | if (creativeCategory != null ? !creativeCategory.equals(request.creativeCategory) : request.creativeCategory != null) 105 | return false; 106 | return winPrice != null ? winPrice.equals(request.winPrice) : request.winPrice == null; 107 | 108 | } 109 | 110 | @Override 111 | public int hashCode() { 112 | int result = campaignItemId != null ? campaignItemId.hashCode() : 0; 113 | result = 31 * result + (domain != null ? domain.hashCode() : 0); 114 | result = 31 * result + (creativeId != null ? creativeId.hashCode() : 0); 115 | result = 31 * result + (creativeCategory != null ? creativeCategory.hashCode() : 0); 116 | result = 31 * result + (winPrice != null ? winPrice.hashCode() : 0); 117 | return result; 118 | } 119 | 120 | private double calculateCategory(String string) throws IOException { 121 | return (calculateXXHash64(string) & 9223372036854775807L) / 9223372036854775807D; 122 | } 123 | 124 | private long calculateXXHash64(String arg) throws IOException { 125 | return XxHash64.hash(Slices.utf8Slice(arg)); 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /fds-common-dynamodb/src/test/java/com/provectus/fds/dynamodb/ItemMapperTest.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.dynamodb; 2 | 3 | 4 | import com.amazonaws.services.dynamodbv2.document.Item; 5 | import com.amazonaws.services.dynamodbv2.document.PrimaryKey; 6 | import com.fasterxml.jackson.databind.ObjectMapper; 7 | import com.provectus.fds.models.events.Aggregation; 8 | import org.junit.Test; 9 | 10 | import java.nio.ByteBuffer; 11 | import java.util.Arrays; 12 | import java.util.List; 13 | import java.util.Map; 14 | import java.util.stream.Collectors; 15 | 16 | import static org.junit.Assert.assertEquals; 17 | import static org.junit.Assert.assertTrue; 18 | 19 | public class ItemMapperTest { 20 | private static final ObjectMapper mapper = new ObjectMapper(); 21 | 22 | private List aggregations = Arrays.asList( 23 | new Aggregation(1000L, "2008-02-20 10:15:00", 1L, null, 1L), 24 | new Aggregation(1000L, "2008-02-20 10:15:00.000", 1L, 1L, null) 25 | ); 26 | 27 | 28 | private ItemMapper itemMapper = new ItemMapper(); 29 | 30 | @Test 31 | public void testMapper() throws Exception { 32 | 33 | for (Aggregation aggregation : aggregations) { 34 | Item item = itemMapper.fromByteBuffer(serialize(aggregation)); 35 | assertEquals((long) aggregation.getCampaignItemId(), item.getLong("campaign_item_id")); 36 | //assertEquals(aggregation.getPeriod(), item.getString("period")); 37 | assertEquals((long) aggregation.getClicks(), item.getLong("clicks")); 38 | assertEquals((long) aggregation.getImps(), item.getLong("imps")); 39 | assertEquals((long) aggregation.getBids(), item.getLong("bids")); 40 | 41 | } 42 | } 43 | 44 | @Test 45 | public void key() throws Exception { 46 | for (Aggregation aggregation : aggregations) { 47 | byte[] bytes = mapper.writeValueAsBytes(aggregation); 48 | PrimaryKey primaryKey = itemMapper.key(ByteBuffer.wrap(bytes)); 49 | 50 | assertTrue(primaryKey.hasComponent(ItemMapper.CAMPAIGN_TABLE_HASH_KEY)); 51 | assertTrue(primaryKey.hasComponent(ItemMapper.PERIOD_TABLE_RANGE_KEY)); 52 | } 53 | } 54 | 55 | 56 | @Test 57 | public void primaryKey() throws Exception { 58 | for (Aggregation aggregation : aggregations) { 59 | byte[] bytes = mapper.writeValueAsBytes(aggregation); 60 | Item item = itemMapper.fromByteBuffer(ByteBuffer.wrap(bytes)); 61 | assertEquals(itemMapper.primaryKey(item), itemMapper.key(ByteBuffer.wrap(bytes))); 62 | } 63 | 64 | } 65 | 66 | private ByteBuffer serialize(Aggregation aggregation) { 67 | try { 68 | return ByteBuffer.wrap(mapper.writeValueAsBytes(aggregation)); 69 | } catch (Exception e) { 70 | throw new RuntimeException(e); 71 | } 72 | } 73 | 74 | @Test 75 | public void mergeItems() { 76 | Map merged = itemMapper.mergeItems( 77 | aggregations.stream() 78 | .map(this::serialize) 79 | .collect(Collectors.toList()) 80 | ); 81 | assertEquals(1, merged.size()); 82 | Item mergedItem = merged.values().iterator().next(); 83 | Aggregation aggregation1 = aggregations.get(0); 84 | Aggregation aggregation2 = aggregations.get(1); 85 | 86 | assertEquals(aggregation1.getClicks() + aggregation2.getClicks(), mergedItem.getLong("clicks")); 87 | assertEquals(aggregation1.getBids() + aggregation2.getBids(), mergedItem.getLong("bids")); 88 | assertEquals(aggregation1.getImps() + aggregation2.getImps(), mergedItem.getLong("imps")); 89 | } 90 | 91 | @Test 92 | public void mergeItem() throws Exception { 93 | Aggregation aggregation1 = aggregations.get(0); 94 | Aggregation aggregation2 = aggregations.get(1); 95 | Item aggregation1_item = itemMapper.fromByteBuffer(serialize(aggregation1)); 96 | Item aggregation2_item = itemMapper.fromByteBuffer(serialize(aggregation2)); 97 | Item resultItem = itemMapper.mergeItem(itemMapper.primaryKey(aggregation1_item), aggregation1_item, aggregation2_item); 98 | assertEquals(itemMapper.primaryKey(aggregation2_item), itemMapper.primaryKey(resultItem)); 99 | assertEquals(aggregation1.getClicks() + aggregation2.getClicks(), resultItem.getLong("clicks")); 100 | assertEquals(aggregation1.getBids() + aggregation2.getBids(), resultItem.getLong("bids")); 101 | assertEquals(aggregation1.getImps() + aggregation2.getImps(), resultItem.getLong("imps")); 102 | } 103 | 104 | @Test 105 | public void mergeValue() { 106 | } 107 | 108 | } -------------------------------------------------------------------------------- /fds-it/src/test/java/com/provectus/fds/it/SampleGenerator.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.it; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.provectus.fds.models.bcns.BidBcn; 6 | import com.provectus.fds.models.bcns.ClickBcn; 7 | import com.provectus.fds.models.bcns.ImpressionBcn; 8 | import com.provectus.fds.models.events.Location; 9 | import org.asynchttpclient.AsyncHttpClient; 10 | import org.asynchttpclient.ListenableFuture; 11 | import org.asynchttpclient.Response; 12 | 13 | import java.util.*; 14 | import java.util.concurrent.ThreadLocalRandom; 15 | 16 | import static com.provectus.fds.it.ItConfig.*; 17 | 18 | public class SampleGenerator { 19 | private int minBids; 20 | private int maxBids; 21 | private String apiUrl; 22 | 23 | private AsyncHttpClient httpClient; 24 | private ObjectMapper objectMapper; 25 | 26 | public SampleGenerator(int minBids, int maxBids, String apiUrl, AsyncHttpClient httpClient, ObjectMapper objectMapper) { 27 | this.minBids = minBids; 28 | this.maxBids = maxBids; 29 | this.apiUrl = apiUrl; 30 | 31 | this.httpClient = httpClient; 32 | this.objectMapper = objectMapper; 33 | } 34 | 35 | ListenableFuture sendRequest(String type, T model) throws JsonProcessingException { 36 | ListenableFuture future = httpClient.preparePost(String.format("%s/%s", apiUrl, type)) 37 | .addHeader("Content-Type", "application/json") 38 | .addHeader("User-Agent", USER_AGENT) 39 | .setBody(objectMapper.writeValueAsBytes(model)) 40 | .execute(); 41 | return future; 42 | } 43 | 44 | int awaitSuccessfull(List> futures) { 45 | int n = 0; 46 | for (ListenableFuture future : futures) { 47 | try { 48 | Response response = future.get(); 49 | //System.out.println(response.toString()); 50 | if (response.getStatusCode() == 200) { 51 | n++; 52 | } 53 | } catch (Throwable e) { 54 | System.out.println("ERROR: " + e.getMessage()); 55 | } 56 | } 57 | return n; 58 | } 59 | 60 | SampleDataResult generateSampleData() throws JsonProcessingException { 61 | ThreadLocalRandom random = ThreadLocalRandom.current(); 62 | 63 | int numberOfBids = random.nextInt(minBids, maxBids); 64 | int numberOfImps = random.nextInt(numberOfBids / 4, numberOfBids / 2); 65 | int numberOfClicks = random.nextInt(numberOfImps / 4, numberOfImps / 2); 66 | 67 | String domain = "www.google.com"; 68 | String creativeCategory = "testCreativeCategory"; 69 | String creativeId = UUID.randomUUID().toString(); 70 | long campaignItemId = random.nextLong(1_000_000L, 2_000_000L); 71 | SampleDataResult result = new SampleDataResult(campaignItemId); 72 | 73 | Map>> futuresByType = new HashMap<>(); 74 | 75 | for (int i = 0; i < numberOfBids; i++) { 76 | 77 | String txid = UUID.randomUUID().toString(); 78 | String appuid = UUID.randomUUID().toString(); 79 | long winPrice = random.nextInt(1_000, 2_000); 80 | 81 | BidBcn bid = BidBcn.builder() 82 | .txId(txid) 83 | .campaignItemId(campaignItemId) 84 | .domain(domain) 85 | .creativeId(creativeId) 86 | .creativeCategory(creativeCategory) 87 | .appUID(appuid) 88 | .build(); 89 | ImpressionBcn impressionBcn = new ImpressionBcn(txid, winPrice); 90 | ClickBcn clickBcn = new ClickBcn(txid); 91 | Location location = new Location(appuid, System.currentTimeMillis(), 55.796506, 49.108451); 92 | 93 | futuresByType.computeIfAbsent(BID_TYPE, (k) -> new ArrayList<>()) 94 | .add(sendRequest(BID_TYPE, bid)); 95 | 96 | futuresByType.computeIfAbsent(LOCATION_TYPE, (k) -> new ArrayList<>()) 97 | .add(sendRequest(LOCATION_TYPE, location)); 98 | 99 | if (numberOfImps > 0) { 100 | futuresByType.computeIfAbsent(IMP_TYPE, (k) -> new ArrayList<>()) 101 | .add(sendRequest(IMP_TYPE, impressionBcn)); 102 | numberOfImps--; 103 | } 104 | 105 | if (numberOfClicks > 0) { 106 | futuresByType.computeIfAbsent(CLICK_TYPE, (k) -> new ArrayList<>()) 107 | .add(sendRequest(CLICK_TYPE, clickBcn)); 108 | numberOfClicks--; 109 | } 110 | } 111 | 112 | result.setCountOfBids(awaitSuccessfull(futuresByType.get(BID_TYPE))); 113 | result.setCountOfImpressions(awaitSuccessfull(futuresByType.get(IMP_TYPE))); 114 | result.setCountOfClicks(awaitSuccessfull(futuresByType.get(CLICK_TYPE))); 115 | 116 | awaitSuccessfull(futuresByType.get(LOCATION_TYPE)); 117 | 118 | return result; 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /fds-lambda-ml-integration/src/main/java/com/provectus/fds/ml/JobRunner.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.ml; 2 | 3 | import com.amazonaws.services.sagemaker.AmazonSageMakerAsync; 4 | import com.amazonaws.services.sagemaker.AmazonSageMakerAsyncClient; 5 | import com.amazonaws.services.sagemaker.AmazonSageMakerAsyncClientBuilder; 6 | import com.amazonaws.services.sagemaker.model.*; 7 | 8 | import java.io.IOException; 9 | import java.time.Instant; 10 | import java.util.Arrays; 11 | import java.util.List; 12 | 13 | public class JobRunner { 14 | 15 | private static final String JOB_PREFIX = "fds-training-job-"; 16 | 17 | private static final String HYPER_PARAMETER_FEATURE_DIM = "5"; 18 | private static final String HYPER_PARAMETER_PREDICTOR_TYPE = "binary_classifier"; 19 | private static final String HYPER_PARAMETER_MINI_BATCH_SIZE = "128"; 20 | private static final String HYPER_PARAMETER_EPOCHS = "1"; 21 | 22 | private static final String TRAINING_ALGORITHM = "linear-learner"; 23 | private static final String TRAINING_INPUT_MODE = "File"; 24 | 25 | private static final int RESOURCE_INSTANCE_COUNT = 1; 26 | private static final String RESOURCE_INSTANCE_TYPE = "ml.m5.large"; 27 | private static final int RESOURCE_VOLUME_SIZE_IN_GB = 30; 28 | 29 | public static final String TRAIN = "train"; 30 | public static final String VALIDATION = "validation"; 31 | 32 | 33 | CreateTrainingJobResult createJob(PrepareDataForTrainingJobLambda.LambdaConfiguration config, 34 | String trainSource, String validationSource) throws IOException { 35 | AmazonSageMakerAsyncClientBuilder sageMakerBuilder 36 | = AmazonSageMakerAsyncClient.asyncBuilder(); 37 | 38 | AmazonSageMakerAsync sage = sageMakerBuilder.build(); 39 | CreateTrainingJobRequest req = new CreateTrainingJobRequest(); 40 | req.setTrainingJobName(JOB_PREFIX + Instant.now().getEpochSecond()); 41 | req.setRoleArn(config.getSagemakerRole()); 42 | 43 | setHyperParameters(req); 44 | setAlgorithm(config, req); 45 | setDataConfig(config, trainSource, validationSource, req); 46 | setStoppingConditions(req); 47 | setResources(req); 48 | 49 | return sage.createTrainingJob(req); 50 | } 51 | 52 | private void setResources(CreateTrainingJobRequest jobRequest) { 53 | ResourceConfig resourceConfig = new ResourceConfig(); 54 | resourceConfig.setInstanceCount(RESOURCE_INSTANCE_COUNT); 55 | resourceConfig.setInstanceType(RESOURCE_INSTANCE_TYPE); 56 | resourceConfig.setVolumeSizeInGB(RESOURCE_VOLUME_SIZE_IN_GB); 57 | 58 | jobRequest.setResourceConfig(resourceConfig); 59 | } 60 | 61 | private void setStoppingConditions(CreateTrainingJobRequest jobRequest) { 62 | StoppingCondition stoppingCondition = new StoppingCondition(); 63 | stoppingCondition.setMaxRuntimeInSeconds(86400); 64 | 65 | jobRequest.setStoppingCondition(stoppingCondition); 66 | } 67 | 68 | private void setDataConfig(PrepareDataForTrainingJobLambda.LambdaConfiguration config, String trainSource, String validationSource, CreateTrainingJobRequest jobRequest) { 69 | Channel trainChannel = createChannel(TRAIN, trainSource); 70 | Channel validationChannel = createChannel(VALIDATION, validationSource); 71 | 72 | List channels = Arrays.asList(trainChannel, validationChannel); 73 | jobRequest.setInputDataConfig(channels); 74 | 75 | OutputDataConfig outputDataConfig = new OutputDataConfig(); 76 | outputDataConfig.setS3OutputPath(config.getModelOutputPath()); 77 | jobRequest.setOutputDataConfig(outputDataConfig); 78 | } 79 | 80 | private void setAlgorithm(PrepareDataForTrainingJobLambda.LambdaConfiguration config, CreateTrainingJobRequest jobRequest) { 81 | AlgorithmSpecification specification = new AlgorithmSpecification(); 82 | SagemakerAlgorithmsRegistry registry = new SagemakerAlgorithmsRegistry(); 83 | 84 | specification.setTrainingImage(registry.getFullImageUri(config.getRegion(), TRAINING_ALGORITHM)); 85 | specification.setTrainingInputMode(TRAINING_INPUT_MODE); 86 | 87 | jobRequest.setAlgorithmSpecification(specification); 88 | } 89 | 90 | private void setHyperParameters(CreateTrainingJobRequest jobRequest) { 91 | jobRequest.addHyperParametersEntry("feature_dim", HYPER_PARAMETER_FEATURE_DIM); 92 | jobRequest.addHyperParametersEntry("predictor_type", HYPER_PARAMETER_PREDICTOR_TYPE); 93 | jobRequest.addHyperParametersEntry("mini_batch_size", HYPER_PARAMETER_MINI_BATCH_SIZE); 94 | jobRequest.addHyperParametersEntry("epochs", HYPER_PARAMETER_EPOCHS); 95 | } 96 | 97 | private Channel createChannel(String name, String uri) { 98 | Channel channel = new Channel(); 99 | channel.setChannelName(name); 100 | channel.setContentType("text/csv"); 101 | 102 | S3DataSource s3ds = new S3DataSource(); 103 | s3ds.setS3Uri(uri); 104 | s3ds.setS3DataType("S3Prefix"); 105 | 106 | DataSource ds = new DataSource(); 107 | ds.setS3DataSource(s3ds); 108 | 109 | channel.setDataSource(ds); 110 | return channel; 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /fds-lambda-ml-integration/src/main/java/com/provectus/fds/ml/processor/AthenaProcessor.java: -------------------------------------------------------------------------------- 1 | package com.provectus.fds.ml.processor; 2 | 3 | import com.amazonaws.services.athena.AmazonAthena; 4 | import com.amazonaws.services.athena.model.*; 5 | 6 | import java.util.List; 7 | 8 | public class AthenaProcessor { 9 | 10 | public void process(AthenaConfig config) throws Exception { 11 | String queryExecutionId 12 | = submitAthenaQuery(config.getClient(), 13 | config.getDbName(), 14 | config.getOutputLocation(), 15 | config.getQuery() 16 | ); 17 | 18 | waitForQueryToComplete(config.getClient(), queryExecutionId, config.getSleepTime()); 19 | processResultRows(config.getClient(), queryExecutionId, config.getRecordProcessor()); 20 | } 21 | 22 | /** 23 | * Submits a query to Athena and returns the execution ID of the query. 24 | */ 25 | protected String submitAthenaQuery(AmazonAthena client, String defaultDatabase, 26 | String outputLocation, String athenaQuery) { 27 | 28 | QueryExecutionContext queryExecutionContext 29 | = new QueryExecutionContext().withDatabase(defaultDatabase); 30 | 31 | ResultConfiguration resultConfiguration = new ResultConfiguration() 32 | .withOutputLocation(outputLocation); 33 | 34 | StartQueryExecutionRequest startQueryExecutionRequest = new StartQueryExecutionRequest() 35 | .withQueryString(athenaQuery) 36 | .withQueryExecutionContext(queryExecutionContext) 37 | .withResultConfiguration(resultConfiguration); 38 | 39 | StartQueryExecutionResult startQueryExecutionResult = client.startQueryExecution(startQueryExecutionRequest); 40 | return startQueryExecutionResult.getQueryExecutionId(); 41 | } 42 | 43 | public static class AthenaProcessorException extends RuntimeException { 44 | AthenaProcessorException(String message) { 45 | super(message); 46 | } 47 | } 48 | 49 | /** 50 | * Wait for an Athena query to complete, fail or to be cancelled. This is done by polling Athena over an 51 | * interval of time. If a query fails or is cancelled, then it will throw an exception. 52 | */ 53 | protected void waitForQueryToComplete(AmazonAthena client, String queryExecutionId, long sleepTime) throws InterruptedException { 54 | GetQueryExecutionRequest getQueryExecutionRequest = new GetQueryExecutionRequest() 55 | .withQueryExecutionId(queryExecutionId); 56 | 57 | GetQueryExecutionResult getQueryExecutionResult; 58 | boolean isQueryStillRunning = true; 59 | while (isQueryStillRunning) { 60 | getQueryExecutionResult = client.getQueryExecution(getQueryExecutionRequest); 61 | String queryState = getQueryExecutionResult.getQueryExecution().getStatus().getState(); 62 | if (queryState.equals(QueryExecutionState.FAILED.toString())) { 63 | throw new AthenaProcessorException("Query Failed to run with Error Message: " + getQueryExecutionResult.getQueryExecution().getStatus().getStateChangeReason()); 64 | } else if (queryState.equals(QueryExecutionState.CANCELLED.toString())) { 65 | throw new AthenaProcessorException("Query was cancelled."); 66 | } else if (queryState.equals(QueryExecutionState.SUCCEEDED.toString())) { 67 | isQueryStillRunning = false; 68 | } else { 69 | // Sleep an amount of time before retrying again. 70 | Thread.sleep(sleepTime); 71 | } 72 | } 73 | } 74 | 75 | /** 76 | * This code calls Athena and retrieves the results of a query. 77 | * The query must be in a completed state before the results can be retrieved and 78 | * paginated. 79 | */ 80 | protected void processResultRows(AmazonAthena client, String queryExecutionId, RecordProcessor recordProcessor) throws Exception { 81 | recordProcessor.initialize(); 82 | 83 | GetQueryResultsRequest getQueryResultsRequest = new GetQueryResultsRequest() 84 | .withQueryExecutionId(queryExecutionId); 85 | 86 | GetQueryResultsResult getQueryResultsResult = client.getQueryResults(getQueryResultsRequest); 87 | 88 | boolean firstPage = true; 89 | while (true) { 90 | List results = getQueryResultsResult.getResultSet().getRows(); 91 | 92 | for (int i = 0; i < results.size(); i++) { 93 | // Process the row. The first row of the first page holds the column names. 94 | // so skip it. 95 | if (i == 0 && firstPage) { 96 | firstPage = false; 97 | } else { 98 | recordProcessor.process(results.get(i)); 99 | } 100 | } 101 | 102 | // If nextToken is null, there are no more pages to read. Break out of the loop. 103 | if (getQueryResultsResult.getNextToken() == null) { 104 | break; 105 | } 106 | getQueryResultsResult = client.getQueryResults( 107 | getQueryResultsRequest.withNextToken(getQueryResultsResult.getNextToken())); 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /stack.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | usage() { 6 | cat </dev/null 2>&1 && pwd )" 26 | AWS_CLI=$(which aws) 27 | 28 | cd "$PROJECT_DIR" || exit 29 | 30 | if [[ $? != 0 ]]; then 31 | echo "AWS CLI not found. Exiting." 32 | exit 1 33 | fi 34 | 35 | while getopts "mb:s:pcthdr:v:" o; do 36 | case "${o}" in 37 | m) 38 | maven=1 39 | ;; 40 | b) 41 | resourceBucket=${OPTARG} 42 | ;; 43 | r) 44 | regionName=${OPTARG} 45 | ;; 46 | s) 47 | stackname=${OPTARG} 48 | ;; 49 | p) 50 | onlyPackage=1 51 | ;; 52 | c) 53 | copyResources=1 54 | ;; 55 | t) 56 | addTimestamp=1 57 | ;; 58 | d) 59 | dropCreateResourceBucket=1 60 | ;; 61 | v) 62 | platformVersion=${OPTARG} 63 | ;; 64 | *) 65 | usage 66 | ;; 67 | esac 68 | done 69 | shift $((OPTIND-1)) 70 | 71 | if [[ -z "${resourceBucket}" ]]; then 72 | echo "ERROR: -b is required parameter" 73 | usage 74 | fi 75 | 76 | if [[ -z "${platformVersion}" ]]; then 77 | echo "ERROR: -v is required parameter" 78 | usage 79 | fi 80 | 81 | if [[ ${addTimestamp} -eq 1 ]]; then 82 | platformVersion=${platformVersion}-$(date +%s) 83 | if [[ ! -z $CIRCLE_SHA1 ]]; then 84 | platformVersion=${platformVersion}-$CIRCLE_SHA1 85 | fi 86 | fi 87 | 88 | if [[ ! -z "${maven}" ]]; then 89 | echo "Running mvn clean package" 90 | mvn clean package 91 | fi 92 | 93 | if [[ -n "${dropCreateResourceBucket}" ]]; then 94 | echo "Dropping bucket ${resourceBucket}" 95 | ${AWS_CLI} s3 rb s3://${resourceBucket} --force 96 | 97 | echo "Creating bucket ${resourceBucket} " 98 | ${AWS_CLI} s3 mb s3://${resourceBucket} --region ${regionName} 99 | fi 100 | 101 | echo Preprocess templates to set valid resource bucket values 102 | 103 | function preprocess() { 104 | source="$1-template.yaml" 105 | target="$1.yaml" 106 | 107 | sed \ 108 | -e "s/@S3ResourceBucket@/${resourceBucket}/" \ 109 | -e "s/@PlatformVersion@/${platformVersion}/" \ 110 | $PROJECT_DIR/$source > $PROJECT_DIR/$target 111 | } 112 | 113 | preprocess ml 114 | preprocess processing 115 | 116 | echo Packaging fds-template.yaml to fds.yaml with bucket ${resourceBucket} 117 | 118 | ${AWS_CLI} cloudformation package \ 119 | --template-file ${PROJECT_DIR}/fds-template.yaml \ 120 | --s3-bucket ${resourceBucket} \ 121 | --s3-prefix ${platformVersion} \ 122 | --output-template-file ${PROJECT_DIR}/fds.yaml 123 | 124 | echo "Copying fds.yaml to s3://${resourceBucket}/${platformVersion}/fds.template" 125 | ${AWS_CLI} s3 cp ${PROJECT_DIR}/fds.yaml \ 126 | s3://${resourceBucket}/${platformVersion}/fds.template 127 | 128 | if [[ ! -z "${copyResources}" ]]; then 129 | echo "Copying model.tar.gz to s3://${resourceBucket}/${platformVersion}/model.tar.gz" 130 | ${AWS_CLI} s3 cp ${PROJECT_DIR}/fds-lambda-ml-integration/src/main/resources/model.tar.gz \ 131 | s3://${resourceBucket}/${platformVersion}/model.tar.gz 132 | 133 | echo "Copying fds-flink-streaming-1.0-SNAPSHOT.jar to s3://${resourceBucket}/${platformVersion}/fds-flink-streaming.jar" 134 | ${AWS_CLI} s3 cp ${PROJECT_DIR}/fds-flink-streaming/target/fds-flink-streaming-1.0-SNAPSHOT.jar \ 135 | s3://${resourceBucket}/${platformVersion}/fds-flink-streaming.jar 136 | fi 137 | 138 | if [[ -z "${onlyPackage}" ]]; then 139 | echo "Deploying to the stack ${stackname} ..." 140 | 141 | if [[ -z ${stackname} ]]; then 142 | echo 143 | echo " **** ERROR: stackname is mandatory when package deployed ****" 2>&1 144 | echo 145 | usage 146 | exit 1 147 | fi 148 | 149 | service_prefix="sdp-$(cat /dev/urandom | tr -dc 'a-z' | fold -w 7 | head -n 1)" 150 | 151 | ${AWS_CLI} --region ${regionName} cloudformation deploy \ 152 | --s3-bucket ${resourceBucket} \ 153 | --s3-prefix ${platformVersion} \ 154 | --template-file ${PROJECT_DIR}/fds.yaml \ 155 | --capabilities CAPABILITY_IAM CAPABILITY_AUTO_EXPAND \ 156 | --stack-name ${stackname} \ 157 | --parameter-overrides ServicePrefix=${service_prefix} 158 | 159 | ${AWS_CLI} --region ${regionName} cloudformation \ 160 | describe-stacks --stack-name ${stackname} 161 | fi 162 | 163 | cd - || exit 1 164 | -------------------------------------------------------------------------------- /fds-flink-streaming/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | fds-parent 7 | com.provectus.fds 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | fds-flink-streaming 13 | 14 | 15 | 1.0.1 16 | 17 | 18 | 19 | 20 | com.provectus.fds 21 | fds-models 22 | 23 | 24 | com.fasterxml.jackson.core 25 | jackson-core 26 | 27 | 28 | com.fasterxml.jackson.core 29 | jackson-databind 30 | 31 | 32 | com.fasterxml.jackson.core 33 | jackson-annotations 34 | 35 | 36 | 37 | 38 | 39 | com.amazonaws 40 | aws-kinesisanalytics-runtime 41 | ${kda.version} 42 | 43 | 44 | 45 | org.apache.flink 46 | flink-connector-kinesis_2.11 47 | ${flink.version} 48 | 49 | 50 | 51 | org.apache.flink 52 | flink-streaming-java_2.11 53 | ${flink.version} 54 | provided 55 | 56 | 57 | 58 | org.projectlombok 59 | lombok 60 | ${lombok.version} 61 | provided 62 | 63 | 64 | 65 | 66 | org.awaitility 67 | awaitility 68 | 69 | 70 | 71 | org.hamcrest 72 | hamcrest 73 | 74 | 75 | 76 | io.flinkspector 77 | flinkspector-datastream_2.11 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | org.apache.maven.plugins 86 | maven-compiler-plugin 87 | 88 | 89 | 90 | org.apache.maven.plugins 91 | maven-shade-plugin 92 | 93 | 94 | package 95 | 96 | shade 97 | 98 | 99 | false 100 | 101 | 102 | org.apache.flink:force-shading 103 | com.google.code.findbugs:jsr305 104 | org.slf4j:* 105 | log4j:* 106 | 107 | 108 | 109 | 110 | *:* 111 | 112 | META-INF/*.SF 113 | META-INF/*.DSA 114 | META-INF/*.RSA 115 | 116 | 117 | 118 | 119 | 121 | com.provectus.fds.flink.StreamingApp 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | --------------------------------------------------------------------------------