{
29 |
30 | private static final Logger LOG = LoggerFactory.getLogger(PrintMutation.class);
31 | String message;
32 |
33 | public Print(String message) {
34 | this.message = message;
35 | }
36 |
37 | @ProcessElement
38 | public void process(@Element T row) throws IOException {
39 |
40 | LOG.info(message + row.toString());
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/data-engineering-dept/business-logic/src/main/java/com/google/dataflow/sample/retail/businesslogic/externalservices/RetailCompanyServices.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.retail.businesslogic.externalservices;
19 |
20 | import com.google.dataflow.sample.retail.dataobjects.Item;
21 | import java.util.HashMap;
22 | import java.util.List;
23 | import java.util.Map;
24 | import org.apache.beam.sdk.annotations.Experimental;
25 | import org.apache.beam.sdk.schemas.Schema;
26 |
27 | /**
28 | * This class represents a mock client to a microservice implemented by the Demo Retail company.
29 | *
30 | * The class emulates communication between the Dataflow pipeline, and a hypothetical internal
31 | * microservice.
32 | *
33 | *
Real services will often take 10-100's of ms to respond, which cause back pressure within a
34 | * pipeline. This version of this mock does not cause push back.
35 | *
36 | *
TODO convert to a service which requires a few hundred ms to respond.
37 | */
38 | @Experimental
39 | public class RetailCompanyServices {
40 |
41 | public Map convertItemIdsToFullText(List itemIds, Schema itemSchema) {
42 |
43 | Map map = new HashMap<>();
44 |
45 | Item item =
46 | Item.builder()
47 | .setItemBrand("item_brand")
48 | .setItemCat01("foo_category")
49 | .setItemName("foo_name")
50 | .build();
51 |
52 | itemIds.forEach(x -> map.put(x, item));
53 | return map;
54 | }
55 | }
56 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/data-engineering-dept/business-logic/src/test/java/com/google/dataflow/sample/retail/businesslogic/core/utils/test/avrotestobjects/InventoryAVRO.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.retail.businesslogic.core.utils.test.avrotestobjects;
19 |
20 | import org.apache.avro.reflect.Nullable;
21 | import org.apache.beam.sdk.coders.AvroCoder;
22 | import org.apache.beam.sdk.coders.DefaultCoder;
23 |
24 | @DefaultCoder(AvroCoder.class)
25 | /**
26 | * Used as part of utility for creation of JSON with {@link Gson}. TODO Remove in favour of raw
27 | * String for the JSON.
28 | */
29 | public class InventoryAVRO {
30 |
31 | public @Nullable long timestamp;
32 | public @Nullable int count;
33 | public @Nullable int sku;
34 | public @Nullable int product_id;
35 | public @Nullable int store_id;
36 | public @Nullable int aisleId;
37 | public @Nullable String product_name;
38 | public @Nullable int departmentId;
39 | public @Nullable Float price;
40 | public @Nullable String recipeId;
41 | public @Nullable String image;
42 | }
43 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/data-engineering-dept/business-logic/src/test/java/com/google/dataflow/sample/retail/businesslogic/core/utils/test/clickstream/WriterawJSONMessagesToBigQureyTest.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.retail.businesslogic.core.utils.test.clickstream;
19 |
20 | import com.google.dataflow.sample.retail.businesslogic.core.options.RetailPipelineOptions;
21 | import org.apache.beam.sdk.options.PipelineOptionsFactory;
22 | import org.apache.beam.sdk.testing.TestPipeline;
23 | import org.junit.Rule;
24 | import org.junit.Test;
25 | import org.junit.runner.RunWith;
26 | import org.junit.runners.JUnit4;
27 |
28 | @RunWith(JUnit4.class)
29 | /** Unit tests for {@link ClickstreamProcessing}. */
30 | public class WriterawJSONMessagesToBigQureyTest {
31 |
32 | RetailPipelineOptions options = PipelineOptionsFactory.as(RetailPipelineOptions.class);
33 |
34 | {
35 | options.setTestModeEnabled(true);
36 | }
37 |
38 | @Rule public transient TestPipeline pipeline = TestPipeline.create();
39 |
40 | @Test
41 | public void testCountViews() {
42 |
43 | pipeline.run();
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/data-engineering-dept/data-objects/README.MD:
--------------------------------------------------------------------------------
1 |
19 |
20 | #Overview
21 | This module holds all the schema information for the objects used in the pipelines.
22 | Other schema information, for example BigTable column names which are stored in this module.
23 |
24 |
25 |
26 |
27 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/data-engineering-dept/data-objects/build.gradle:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * License); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an AS IS BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | plugins {
20 | id 'java'
21 | }
22 |
23 |
24 | description = "Retail Sample Application :: Java :: Data Objects"
25 | ext.summary = "Data Objects used by all modules, within the pipeline and for I/O"
26 |
27 | dependencies {
28 | // Main
29 | compile group: 'org.apache.beam', name: 'beam-sdks-java-core', version: "${beamVersion}"
30 |
31 | // -- AutoValue
32 | annotationProcessor "com.google.auto.value:auto-value:${autoValueVersion}"
33 | compile "com.google.auto.value:auto-value-annotations:${autoValueVersion}"
34 |
35 | // Test
36 | testCompile group: 'junit', name: 'junit', version: '4.12'
37 | }
38 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/data-engineering-dept/data-objects/src/main/java/com/google/dataflow/sample/retail/dataobjects/Ecommerce.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.retail.dataobjects;
19 |
20 | import com.google.auto.value.AutoValue;
21 | import java.util.List;
22 | import javax.annotation.Nullable;
23 | import org.apache.beam.sdk.schemas.AutoValueSchema;
24 | import org.apache.beam.sdk.schemas.annotations.DefaultSchema;
25 | import org.apache.beam.sdk.schemas.annotations.SchemaFieldName;
26 |
27 | @AutoValue
28 | @DefaultSchema(AutoValueSchema.class)
29 | public abstract class Ecommerce {
30 |
31 | @SchemaFieldName("items")
32 | public @Nullable abstract List- getItems();
33 |
34 | @SchemaFieldName("purchase")
35 | public @Nullable abstract Purchase getPurchase();
36 |
37 | public static Builder builder() {
38 |
39 | return new AutoValue_Ecommerce.Builder();
40 | }
41 |
42 | @AutoValue.Builder
43 | public abstract static class Builder {
44 | public abstract Builder setItems(List
- value);
45 |
46 | public abstract Builder setPurchase(Purchase purchase);
47 |
48 | public abstract Ecommerce build();
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/data-engineering-dept/data-objects/src/main/java/com/google/dataflow/sample/retail/dataobjects/StockAggregation.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.retail.dataobjects;
19 |
20 | import com.google.auto.value.AutoValue;
21 | import javax.annotation.Nullable;
22 | import org.apache.beam.sdk.annotations.Experimental;
23 | import org.apache.beam.sdk.schemas.AutoValueSchema;
24 | import org.apache.beam.sdk.schemas.annotations.DefaultSchema;
25 | import org.apache.beam.sdk.schemas.annotations.SchemaFieldName;
26 |
27 | @AutoValue
28 | @DefaultSchema(AutoValueSchema.class)
29 | @Experimental
30 | public abstract class StockAggregation {
31 |
32 | @Nullable
33 | public abstract Long getDurationMS();
34 |
35 | @Nullable
36 | public abstract Long getStartTime();
37 |
38 | @Nullable
39 | @SchemaFieldName("product_id")
40 | public abstract Integer getProductId();
41 |
42 | @Nullable
43 | @SchemaFieldName("store_id")
44 | public abstract Integer getStoreId();
45 |
46 | @Nullable
47 | public abstract Long getCount();
48 |
49 | public abstract StockAggregation.Builder toBuilder();
50 |
51 | public static StockAggregation.Builder builder() {
52 | return new AutoValue_StockAggregation.Builder();
53 | }
54 |
55 | @AutoValue.Builder
56 | public abstract static class Builder {
57 |
58 | public abstract Builder setDurationMS(Long value);
59 |
60 | public abstract Builder setStartTime(Long value);
61 |
62 | public abstract Builder setProductId(Integer value);
63 |
64 | public abstract Builder setStoreId(Integer value);
65 |
66 | public abstract Builder setCount(Long value);
67 |
68 | public abstract StockAggregation build();
69 | }
70 | }
71 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/data-engineering-dept/pipelines/README.MD:
--------------------------------------------------------------------------------
1 |
19 | # Overview
20 |
21 |
22 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/data-engineering-dept/pipelines/build.gradle:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * License); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an AS IS BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | plugins {
19 | id 'java'
20 | }
21 |
22 | description = "Retail Sample Application :: Java :: Pipelines"
23 | ext.summary = "Pipelines composition."
24 |
25 | dependencies {
26 | // Main
27 | compile project(':data-engineering-dept:data-objects')
28 | compile project(':data-engineering-dept:business-logic')
29 |
30 | // Test
31 | compile group: 'org.slf4j', name: 'slf4j-jdk14', version: '1.7.7'
32 | testCompile group: 'junit', name: 'junit', version: '4.12'
33 | testCompile project(':data-engineering-dept:business-logic').sourceSets.test.output
34 | }
35 |
36 | task executeOnDataflow(type: JavaExec){
37 | classpath=sourceSets.main.runtimeClasspath
38 | main = "com.google.dataflow.sample.retail.pipeline.RetailDataProcessingPipeline"
39 | systemProperties System.getProperties()
40 | args System.getProperty("exec.args", "").split()
41 | }
--------------------------------------------------------------------------------
/retail/retail-java-applications/gradle/wrapper/gradle-wrapper.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/dataflow-sample-applications/beb6e20133939d7f7024c338ec2dc02e84e17475/retail/retail-java-applications/gradle/wrapper/gradle-wrapper.jar
--------------------------------------------------------------------------------
/retail/retail-java-applications/gradle/wrapper/gradle-wrapper.properties:
--------------------------------------------------------------------------------
1 | ################################################################################
2 | # Licensed to the Apache Software Foundation (ASF) under one
3 | # or more contributor license agreements. See the NOTICE file
4 | # distributed with this work for additional information
5 | # regarding copyright ownership. The ASF licenses this file
6 | # to you under the Apache License, Version 2.0 (the
7 | # "License"); you may not use this file except in compliance
8 | # with the License. You may obtain a copy of the License at
9 | #
10 | # http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | ################################################################################
18 | distributionBase=GRADLE_USER_HOME
19 | distributionPath=wrapper/dists
20 | distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.1-bin.zip
21 | zipStoreBase=GRADLE_USER_HOME
22 | zipStorePath=wrapper/dists
23 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/settings.gradle:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * License); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an AS IS BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | rootProject.name = 'retail-java-applications'
20 |
21 | include ':data-engineering-dept:data-objects'
22 | include ':data-engineering-dept:business-logic'
23 | include ':data-engineering-dept:pipelines'
24 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/spotless/Apache_Licence_For_Spotless:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
--------------------------------------------------------------------------------
/retail/retail-java-applications/terraform/examples/pubsub/terraform.tfvars:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | region = "us-central1"
17 | topic_clickstream_inbound = "Clickstream-inbound"
18 | topic_transactions_inbound = "Transactions-inbound"
19 | topic_inventory_inbound = "Inventory-inbound"
20 | topic_inventory_outbound = "Inventory-outbound"
21 | clickstream_inbound_sub = "Clickstream-inbound-sub"
22 | transactions_inbound_sub = "Transactions-inbound-sub"
23 | inventory_inbound_sub = "Inventory-inbound-sub"
24 |
25 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/terraform/examples/pubsub/variables.tf:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | variable "project_id" {
18 | type = string
19 | description = "Project ID in GCP"
20 | }
21 |
22 | variable "region" {
23 | type = string
24 | description = "Name of the selected region"
25 | }
26 |
27 | variable "topic_clickstream_inbound" {
28 | type = string
29 | description = "Topic name for clickstream inbound"
30 | }
31 |
32 | variable "topic_transactions_inbound" {
33 | type = string
34 | description = "Topic name for transactions inbound"
35 | }
36 |
37 | variable "topic_inventory_inbound" {
38 | type = string
39 | description = "Topic name for inventory inbound"
40 | }
41 |
42 | variable "topic_inventory_outbound" {
43 | type = string
44 | description = "Topic name for inventory outbound"
45 | }
46 |
47 | variable "clickstream_inbound_sub" {
48 | type = string
49 | description = "Subscription for clickstream inbound"
50 | }
51 |
52 | variable "transactions_inbound_sub" {
53 | type = string
54 | description = "Subscription for transactions inbound"
55 | }
56 |
57 | variable "inventory_inbound_sub" {
58 | type = string
59 | description = "Subscription for inventory inbound"
60 | }
61 |
62 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/terraform/main.tf:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | terraform {
18 | required_providers {
19 | google = {
20 | source = "hashicorp/google"
21 | version = ">= 3.48.0"
22 | }
23 | }
24 | }
25 |
26 | provider "google" {
27 | project = var.project_id
28 | region = var.region
29 | }
30 |
31 | module "pubsub" {
32 | source = "./modules/pubsub"
33 | topic_clickstream_inbound = var.topic_clickstream_inbound
34 | topic_transactions_inbound = var.topic_transactions_inbound
35 | topic_inventory_inbound = var.topic_inventory_inbound
36 | topic_inventory_outbound = var.topic_inventory_outbound
37 | clickstream_inbound_sub = var.clickstream_inbound_sub
38 | transactions_inbound_sub = var.transactions_inbound_sub
39 | inventory_inbound_sub = var.inventory_inbound_sub
40 | }
41 |
42 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/terraform/modules/pubsub/variables.tf:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | variable "topic_clickstream_inbound" {
17 | type = string
18 | description = "Topic name for clickstream inbound"
19 | }
20 |
21 | variable "topic_transactions_inbound" {
22 | type = string
23 | description = "Topic name for transactions inbound"
24 | }
25 |
26 | variable "topic_inventory_inbound" {
27 | type = string
28 | description = "Topic name for inventory inbound"
29 | }
30 |
31 | variable "topic_inventory_outbound" {
32 | type = string
33 | description = "Topic name for inventory outbound"
34 | }
35 |
36 | variable "clickstream_inbound_sub" {
37 | type = string
38 | description = "Subscription name for clickstream inbound"
39 | }
40 |
41 | variable "transactions_inbound_sub" {
42 | type = string
43 | description = "Subscription name for transactions inbound"
44 | }
45 |
46 | variable "inventory_inbound_sub" {
47 | type = string
48 | description = "Subscription name for inventory inbound"
49 | }
50 |
51 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/terraform/terraform.tfvars:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | region = "us-central1"
17 | topic_clickstream_inbound = "Clickstream-inbound"
18 | topic_transactions_inbound = "Transactions-inbound"
19 | topic_inventory_inbound = "Inventory-inbound"
20 | topic_inventory_outbound = "Inventory-outbound"
21 | clickstream_inbound_sub = "Clickstream-inbound-sub"
22 | transactions_inbound_sub = "Transactions-inbound-sub"
23 | inventory_inbound_sub = "Inventory-inbound-sub"
24 |
25 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/terraform/test/pubsub/go.mod:
--------------------------------------------------------------------------------
1 | module dataflow-sample-applications/retail/retail-java-applications/terraform/test
2 |
3 | go 1.16
4 |
5 | require (
6 | cloud.google.com/go/pubsub v1.3.1
7 | github.com/gruntwork-io/terratest v0.40.22
8 | github.com/stretchr/testify v1.7.0
9 | )
10 |
--------------------------------------------------------------------------------
/retail/retail-java-applications/terraform/variables.tf:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | variable "project_id" {
18 | type = string
19 | description = "Project ID in GCP"
20 | }
21 |
22 | variable "region" {
23 | type = string
24 | description = "Name of the selected region"
25 | }
26 |
27 | variable "topic_clickstream_inbound" {
28 | type = string
29 | description = "Topic name for clickstream inbound"
30 | }
31 |
32 | variable "topic_transactions_inbound" {
33 | type = string
34 | description = "Topic name for transactions inbound"
35 | }
36 |
37 | variable "topic_inventory_inbound" {
38 | type = string
39 | description = "Topic name for inventory inbound"
40 | }
41 |
42 | variable "topic_inventory_outbound" {
43 | type = string
44 | description = "Topic name for inventory outbound"
45 | }
46 |
47 | variable "clickstream_inbound_sub" {
48 | type = string
49 | description = "Subscription for clickstream inbound"
50 | }
51 |
52 | variable "transactions_inbound_sub" {
53 | type = string
54 | description = "Subscription for transactions inbound"
55 | }
56 |
57 | variable "inventory_inbound_sub" {
58 | type = string
59 | description = "Subscription for inventory inbound"
60 | }
61 |
62 |
--------------------------------------------------------------------------------
/timeseries-streaming/docs/Draft_Dataflow_streaming_time_series_sample_library_user_guide_v0.4.0.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/dataflow-sample-applications/beb6e20133939d7f7024c338ec2dc02e84e17475/timeseries-streaming/docs/Draft_Dataflow_streaming_time_series_sample_library_user_guide_v0.4.0.pdf
--------------------------------------------------------------------------------
/timeseries-streaming/images/timeseries_metrics.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/dataflow-sample-applications/beb6e20133939d7f7024c338ec2dc02e84e17475/timeseries-streaming/images/timeseries_metrics.png
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/Adapters/build.gradle:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * License); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an AS IS BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | plugins {
19 | id 'java'
20 | id 'idea'
21 | id "com.diffplug.gradle.spotless" version "3.24.2"
22 | }
23 |
24 | sourceCompatibility = 1.8
25 |
26 | dependencies {
27 | compile project(':TimeSeriesPipeline')
28 | compile project(':TimeSeriesMetricsLibrary')
29 |
30 | compile "com.google.auto.value:auto-value-annotations:${autoValueVersion}"
31 | annotationProcessor "com.google.auto.value:auto-value:${autoValueVersion}"
32 |
33 | testCompile project(path: ":TimeSeriesPipeline", configuration: 'testArtifacts')
34 | testCompile group: 'junit', name: 'junit', version: '4.12'
35 | }
36 |
37 |
38 | // Enable code formatting
39 | spotless {
40 | java {
41 | licenseHeaderFile '../Apache_Licence_For_Spotless'
42 | googleJavaFormat('1.7')
43 | }
44 | }
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/Adapters/src/main/java/com/google/dataflow/sample/timeseriesflow/adaptors/fsi/data/cme/DeadLetterSink.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.adaptors.fsi.data.cme;
19 |
20 | import org.apache.beam.sdk.annotations.Experimental;
21 |
22 | @Experimental
23 | public enum DeadLetterSink {
24 | LOG,
25 | BIGQUERY
26 | }
27 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/Adapters/src/main/java/com/google/dataflow/sample/timeseriesflow/adaptors/fsi/data/cme/LogElements.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.adaptors.fsi.data.cme;
19 |
20 | import org.apache.beam.sdk.annotations.Experimental;
21 | import org.apache.beam.sdk.transforms.DoFn;
22 | import org.apache.beam.sdk.transforms.PTransform;
23 | import org.apache.beam.sdk.transforms.ParDo;
24 | import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
25 | import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
26 | import org.apache.beam.sdk.values.PCollection;
27 | import org.slf4j.Logger;
28 | import org.slf4j.LoggerFactory;
29 |
30 | @Experimental
31 | class LogElements extends PTransform, PCollection> {
32 |
33 | private static final Logger LOG = LoggerFactory.getLogger(LogElements.class);
34 |
35 | @Override
36 | public PCollection expand(PCollection input) {
37 |
38 | return input.apply(
39 | "Logging Elements",
40 | ParDo.of(
41 | new DoFn() {
42 |
43 | @ProcessElement
44 | public void processElement(
45 | @Element T element, OutputReceiver out, BoundedWindow window) {
46 |
47 | String message = element.toString();
48 |
49 | if (!(window instanceof GlobalWindow)) {
50 | message = message + " Window: " + window.toString();
51 | }
52 |
53 | LOG.info(message);
54 |
55 | out.output(element);
56 | }
57 | }));
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/Adapters/src/test/resources/TSTestBlendedIndexNoGaps.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "key": {
4 | "major_key": "Key-A",
5 | "minor_key_string": "MKey-a"
6 | },
7 | "data": {"double_val": 10.0
8 | }, "timestamp": "2000-01-01T00:00:00Z"
9 | },
10 | {
11 | "key": {
12 | "major_key": "Key-B",
13 | "minor_key_string": "MKey-a"
14 | },
15 | "data": {"double_val": 110.0
16 | }, "timestamp": "2000-01-01T00:00:00Z"
17 | },
18 | {
19 | "key": {
20 | "major_key": "Key-C",
21 | "minor_key_string": "MKey-a"
22 | },
23 | "data": {"double_val": 210.0
24 | }, "timestamp": "2000-01-01T00:00:00Z",
25 | "time": {
26 | "advance_watermark_seconds": 1
27 | }
28 | },
29 | {
30 | "key": {
31 | "major_key": "Key-A",
32 | "minor_key_string": "MKey-a"
33 | },
34 | "data": {"double_val": 20.0
35 | }, "timestamp": "2000-01-01T00:00:01Z"
36 | },
37 | {
38 | "key": {
39 | "major_key": "Key-B",
40 | "minor_key_string": "MKey-a"
41 | },
42 | "data": {"double_val": 120.0
43 | }, "timestamp": "2000-01-01T00:00:01Z"
44 | },
45 | {
46 | "key": {
47 | "major_key": "Key-C",
48 | "minor_key_string": "MKey-a"
49 | },
50 | "data": {"double_val": 220.0
51 | }, "timestamp": "2000-01-01T00:00:01Z",
52 | "time": {
53 | "advance_watermark_seconds": 2
54 | }
55 | },
56 | {
57 | "key": {
58 | "major_key": "Key-A",
59 | "minor_key_string": "MKey-a"
60 | },
61 | "data": {"double_val": 30.0
62 | }, "timestamp": "2000-01-01T00:00:02Z"
63 | },
64 | {
65 | "key": {
66 | "major_key": "Key-B",
67 | "minor_key_string": "MKey-a"
68 | },
69 | "data": {"double_val": 130.0
70 | }, "timestamp": "2000-01-01T00:00:02Z"
71 | },
72 | {
73 | "key": {
74 | "major_key": "Key-C",
75 | "minor_key_string": "MKey-a"
76 | },
77 | "data": {"double_val": 230.0
78 | }, "timestamp": "2000-01-01T00:00:02Z",
79 | "time": {
80 | "advance_watermark_expression": "INFINITY"
81 | }
82 | }
83 | ]
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/Adapters/src/test/resources/TSTestBlendedIndexWithGaps.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "key": {
4 | "major_key": "Key-A",
5 | "minor_key_string": "MKey-a"
6 | },
7 | "data": {"double_val": 10.0
8 | }, "timestamp": "2000-01-01T00:00:00Z"
9 | },
10 | {
11 | "key": {
12 | "major_key": "Key-B",
13 | "minor_key_string": "MKey-a"
14 | },
15 | "data": {"double_val": 110.0
16 | }, "timestamp": "2000-01-01T00:00:00Z"
17 | },
18 | {
19 | "key": {
20 | "major_key": "Key-C",
21 | "minor_key_string": "MKey-a"
22 | },
23 | "data": {"double_val": 210.0
24 | }, "timestamp": "2000-01-01T00:00:00Z",
25 | "time": {
26 | "advance_watermark_seconds": 1
27 | }
28 | },
29 | {
30 | "key": {
31 | "major_key": "Key-A",
32 | "minor_key_string": "MKey-a"
33 | },
34 | "data": {"double_val": 20.0
35 | }, "timestamp": "2000-01-01T00:00:01Z"
36 | },
37 | {
38 | "key": {
39 | "major_key": "Key-C",
40 | "minor_key_string": "MKey-a"
41 | },
42 | "data": {"double_val": 220.0
43 | }, "timestamp": "2000-01-01T00:00:01Z",
44 | "time": {
45 | "advance_watermark_seconds": 2
46 | }
47 | },
48 | {
49 | "key": {
50 | "major_key": "Key-A",
51 | "minor_key_string": "MKey-a"
52 | },
53 | "data": {"double_val": 30.0
54 | }, "timestamp": "2000-01-01T00:00:02Z"
55 | },
56 | {
57 | "key": {
58 | "major_key": "Key-B",
59 | "minor_key_string": "MKey-a"
60 | },
61 | "data": {
62 | "double_val": 130.0
63 | }, "timestamp": "2000-01-01T00:00:02Z",
64 | "time": {
65 | "advance_watermark_seconds": 3
66 | }
67 | },
68 | {
69 | "time": {
70 | "advance_watermark_expression": "INFINITY"
71 | }
72 | }
73 | ]
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/Adapters/src/test/resources/TSTestData.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "key": {
4 | "major_key": "Key-A",
5 | "minor_key_string": "MKey-a"
6 | },
7 | "data": {"double_val": 10.0
8 | }
9 | },
10 | {
11 | "key": {
12 | "major_key": "Key-A",
13 | "minor_key_string": "MKey-a"
14 | },
15 | "data": {"double_val": 20.0
16 | }
17 | },
18 | {
19 | "key": {
20 | "major_key": "Key-A",
21 | "minor_key_string": "MKey-a"
22 | },
23 | "data": {"double_val": 30.0
24 | }
25 | },
26 | {
27 | "key": {
28 | "major_key": "Key-B",
29 | "minor_key_string": "MKey-a"
30 | },
31 | "data": {"double_val": 110.0
32 | }
33 | },
34 | {
35 | "key": {
36 | "major_key": "Key-B",
37 | "minor_key_string": "MKey-a"
38 | },
39 | "data": {"double_val": 120.0
40 | }
41 | },
42 | {
43 | "key": {
44 | "major_key": "Key-C",
45 | "minor_key_string": "MKey-a"
46 | },
47 | "data": {"double_val": 130.0
48 | }
49 | },
50 | {
51 | "key": {
52 | "major_key": "Key-C",
53 | "minor_key_string": "MKey-a"
54 | },
55 | "data": {"double_val": 210.0
56 | }
57 | },
58 | {
59 | "key": {
60 | "major_key": "Key-C",
61 | "minor_key_string": "MKey-a"
62 | },
63 | "data": {"double_val": 220.0
64 | }
65 | },
66 | {
67 | "key": {
68 | "major_key": "Key-C",
69 | "minor_key_string": "MKey-a"
70 | },
71 | "data": {"double_val": 230.0
72 | }
73 | }
74 | ]
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/Apache_Licence_For_Spotless:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/Examples/src/main/java/com/google/dataflow/sample/timeseriesflow/examples/fsi/forex/ExampleForexPipelineOptions.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.examples.fsi.forex;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.options.TFXOptions;
21 | import com.google.dataflow.sample.timeseriesflow.options.TSFlowOptions;
22 | import org.apache.beam.sdk.annotations.Experimental;
23 | import org.apache.beam.sdk.options.Description;
24 |
25 | @Experimental
26 | public interface ExampleForexPipelineOptions extends TSFlowOptions, TFXOptions {
27 |
28 | @Description("Option to specify BigQuery target table to push metrics")
29 | String getBigQueryTableForTSAccumOutputLocation();
30 |
31 | void setBigQueryTableForTSAccumOutputLocation(String bigQueryTableForTSAccumOutputLocation);
32 |
33 | @Description("Option to specify absolute path for input dataset")
34 | String getInputPath();
35 |
36 | void setInputPath(String inputPath);
37 |
38 | @Description(
39 | "Option to specify the end timestamp to stop filling gaps, e.g., end of time series dataset")
40 | String getEndTimestamp();
41 |
42 | void setEndTimestamp(String endTimestamp);
43 |
44 | @Description("Option to specify sampling period in seconds")
45 | Integer getResampleSec();
46 |
47 | void setResampleSec(Integer resampleSec);
48 |
49 | @Description("Option to specify rolling window to calculate metrics in seconds")
50 | Integer getWindowSec();
51 |
52 | void setWindowSec(Integer resampleSec);
53 | }
54 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/Examples/src/main/java/com/google/dataflow/sample/timeseriesflow/examples/fsi/forex/HistoryForexReader.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.examples.fsi.forex;
19 |
20 | import com.google.auto.value.AutoValue;
21 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData;
22 | import java.util.Set;
23 | import org.apache.beam.sdk.io.TextIO;
24 | import org.apache.beam.sdk.transforms.PTransform;
25 | import org.apache.beam.sdk.values.PBegin;
26 | import org.apache.beam.sdk.values.PCollectionTuple;
27 | import org.apache.beam.sdk.values.TupleTag;
28 |
29 | @AutoValue
30 | public abstract class HistoryForexReader extends PTransform {
31 |
32 | public abstract String getSourceFilesURI();
33 |
34 | public abstract Set getTickers();
35 |
36 | public static Builder builder() {
37 | return new AutoValue_HistoryForexReader.Builder();
38 | }
39 |
40 | @AutoValue.Builder
41 | public abstract static class Builder {
42 |
43 | public abstract Builder setSourceFilesURI(String newSourceFilesURI);
44 |
45 | public abstract Builder setTickers(Set newTickers);
46 |
47 | public abstract HistoryForexReader build();
48 | }
49 |
50 | // Tags to implement basic example of deadletter queue pattern
51 | static final TupleTag successfulParse =
52 | new TupleTag();
53 | static final TupleTag deadLetterTag = new TupleTag();
54 |
55 | @Override
56 | public PCollectionTuple expand(PBegin input) {
57 | return input
58 | .apply(TextIO.read().from(getSourceFilesURI()))
59 | .apply(new ForexCSVAdaptor.ConvertCSVForex(getTickers()));
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/Examples/src/main/resources/[EMPTY]-EURUSD-2020-05-11_2020-05-11.csv:
--------------------------------------------------------------------------------
1 | PLEASE DOWNLOAD ONE DAY WORTH OF FOREX TICK DATA FROM YOUR FAVORITE PROVIDER.
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/SyntheticExamples/build.gradle:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * License); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an AS IS BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | plugins {
19 | id 'java'
20 | id 'idea'
21 | id "com.diffplug.gradle.spotless" version "3.24.2"
22 | }
23 |
24 | sourceCompatibility = 1.8
25 |
26 | dependencies {
27 | compile project(':TimeSeriesPipeline')
28 | compile project(':TimeSeriesMetricsLibrary')
29 | testCompile group: 'junit', name: 'junit', version: '4.12'
30 | }
31 |
32 | // Enable code formatting
33 | spotless {
34 | java {
35 | licenseHeaderFile '../Apache_Licence_For_Spotless'
36 | googleJavaFormat('1.7')
37 | }
38 | }
39 |
40 | task run_example(type: JavaExec) {
41 | classpath sourceSets.main.runtimeClasspath
42 | main = "com.google.dataflow.sample.timeseriesflow.examples.simpledata.transforms.SinWaveExample"
43 | group = "samples"
44 | }
45 |
46 | task generate_bootstrap_data(type: JavaExec) {
47 | classpath sourceSets.main.runtimeClasspath
48 | main = "com.google.dataflow.sample.timeseriesflow.examples.simpledata.transforms.SimpleDataBootstrapGenerator"
49 | group = "samples"
50 | }
51 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/SyntheticExamples/src/main/java/com/google/dataflow/sample/timeseriesflow/examples/simpledata/transforms/SinWaveExampleOptions.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.examples.simpledata.transforms;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.metrics.core.TSMetricsOptions;
21 | import com.google.dataflow.sample.timeseriesflow.options.TFXOptions;
22 | import com.google.dataflow.sample.timeseriesflow.options.TSOutputPipelineOptions;
23 | import org.apache.beam.sdk.options.Default;
24 | import org.apache.beam.sdk.options.Description;
25 |
26 | public interface SinWaveExampleOptions
27 | extends TSOutputPipelineOptions, TSMetricsOptions, TFXOptions {
28 |
29 | @Description(
30 | "In order to see easy output of metrics for demos set this to true. This will result in all values being 'printed' to logs.")
31 | @Default.Boolean(false)
32 | Boolean getEnablePrintMetricsToLogs();
33 |
34 | void setEnablePrintMetricsToLogs(Boolean value);
35 |
36 | @Description(
37 | "In order to see easy output of TF.Examples for demos set this to true. This will result in all values being 'printed' to logs.")
38 | @Default.Boolean(false)
39 | Boolean getEnablePrintTFExamplesToLogs();
40 |
41 | void setEnablePrintTFExamplesToLogs(Boolean value);
42 |
43 | @Description("Enable sending outliers with the stream of synthetic data.")
44 | Boolean getWithOutliers();
45 |
46 | void setWithOutliers(Boolean value);
47 | }
48 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesMetricsLibrary/src/main/java/com/google/dataflow/sample/timeseriesflow/metrics/README.MD:
--------------------------------------------------------------------------------
1 | #Style guide
2 | -
3 | ## All Metrics
4 | * All metrics should have an option interface even if its empty.
5 | * Options should be placed at the top of the class below LOG.
6 | * Options interface should be added to [MetricsOptions](MetricsOptions.java)
7 |
8 | ## Basic Type Two
9 | * Name of basic metrics should be in the format XXFn, for example StdDevFn.
10 |
11 | ## Complex Type Two
12 | * Name of complex metrics should be in the format XXGFn, for example RSIGFn. (G denotes graph as complex metrics mutate the graph)
13 |
14 |
15 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesMetricsLibrary/src/main/java/com/google/dataflow/sample/timeseriesflow/metrics/core/typeone/Max.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.metrics.core.typeone;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccum;
21 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSDataPoint;
22 | import com.google.dataflow.sample.timeseriesflow.combiners.BTypeOne;
23 | import com.google.dataflow.sample.timeseriesflow.common.TSDataUtils;
24 | import com.google.dataflow.sample.timeseriesflow.datamap.AccumCoreNumericBuilder;
25 | import org.apache.beam.sdk.options.PipelineOptions;
26 |
27 | public class Max extends BTypeOne {
28 |
29 | public interface MaxOptions extends PipelineOptions {};
30 |
31 | @Override
32 | public TSAccum addInput(TSAccum accumulator, TSDataPoint dataPoint) {
33 | AccumCoreNumericBuilder coreNumeric = new AccumCoreNumericBuilder(accumulator);
34 | coreNumeric.setMax(TSDataUtils.findMaxValue(coreNumeric.getMaxOrNull(), dataPoint.getData()));
35 | return coreNumeric.build();
36 | }
37 |
38 | @Override
39 | public TSAccum mergeDataAccums(TSAccum a, TSAccum b) {
40 | AccumCoreNumericBuilder aBuilder = new AccumCoreNumericBuilder(a);
41 | AccumCoreNumericBuilder bBuilder = new AccumCoreNumericBuilder(b);
42 | aBuilder.setMax(TSDataUtils.findMaxValue(aBuilder.getMaxOrNull(), bBuilder.getMaxOrNull()));
43 |
44 | return aBuilder.build();
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesMetricsLibrary/src/main/java/com/google/dataflow/sample/timeseriesflow/metrics/core/typeone/Min.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.metrics.core.typeone;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccum;
21 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSDataPoint;
22 | import com.google.dataflow.sample.timeseriesflow.combiners.BTypeOne;
23 | import com.google.dataflow.sample.timeseriesflow.common.TSDataUtils;
24 | import com.google.dataflow.sample.timeseriesflow.datamap.AccumCoreNumericBuilder;
25 | import org.apache.beam.sdk.options.PipelineOptions;
26 |
27 | public class Min extends BTypeOne {
28 |
29 | public interface MinOptions extends PipelineOptions {}
30 |
31 | @Override
32 | public TSAccum addInput(TSAccum accumulator, TSDataPoint dataPoint) {
33 | AccumCoreNumericBuilder coreNumeric = new AccumCoreNumericBuilder(accumulator);
34 | coreNumeric.setMin(TSDataUtils.findMinData(coreNumeric.getMinOrNull(), dataPoint.getData()));
35 | return coreNumeric.build();
36 | }
37 |
38 | @Override
39 | public TSAccum mergeDataAccums(TSAccum a, TSAccum b) {
40 | AccumCoreNumericBuilder aBuilder = new AccumCoreNumericBuilder(a);
41 | AccumCoreNumericBuilder bBuilder = new AccumCoreNumericBuilder(b);
42 | aBuilder.setMin(TSDataUtils.findMinData(aBuilder.getMinOrNull(), bBuilder.getMinOrNull()));
43 |
44 | return aBuilder.build();
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesMetricsLibrary/src/main/java/com/google/dataflow/sample/timeseriesflow/metrics/core/typeone/Sum.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.metrics.core.typeone;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccum;
21 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSDataPoint;
22 | import com.google.dataflow.sample.timeseriesflow.combiners.BTypeOne;
23 | import com.google.dataflow.sample.timeseriesflow.common.CommonUtils;
24 | import com.google.dataflow.sample.timeseriesflow.datamap.AccumCoreNumericBuilder;
25 | import org.apache.beam.sdk.options.PipelineOptions;
26 |
27 | public class Sum extends BTypeOne {
28 |
29 | public interface SumOptions extends PipelineOptions {}
30 |
31 | @Override
32 | public TSAccum addInput(TSAccum accumulator, TSDataPoint dataPoint) {
33 | AccumCoreNumericBuilder coreNumeric = new AccumCoreNumericBuilder(accumulator);
34 | coreNumeric.setSum(
35 | CommonUtils.sumNumericDataNullAsZero(coreNumeric.getSumOrNull(), dataPoint.getData()));
36 | return coreNumeric.build();
37 | }
38 |
39 | @Override
40 | public TSAccum mergeDataAccums(TSAccum a, TSAccum b) {
41 | AccumCoreNumericBuilder aBuilder = new AccumCoreNumericBuilder(a);
42 | AccumCoreNumericBuilder bBuilder = new AccumCoreNumericBuilder(b);
43 | aBuilder.setSum(
44 | CommonUtils.sumNumericDataNullAsZero(aBuilder.getSumOrNull(), bBuilder.getSumOrNull()));
45 |
46 | return aBuilder.build();
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesMetricsLibrary/src/main/proto/TSFSITechKeys.proto:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | syntax = "proto3";
19 |
20 | package protos;
21 |
22 | option java_outer_classname = "FSITechnicalDerivedAggregations";
23 | option java_multiple_files = false;
24 | option java_package = "com.google.dataflow.sample.timeseriesflow";
25 |
26 | /**
27 | This proto stores all of the keys used in the accum map.
28 |
29 | */
30 | enum FsiTechnicalIndicators{
31 | SUM_UP_MOVEMENT = 0;
32 | SUM_DOWN_MOVEMENT = 1;
33 | SUM_MOVEMENT_COUNT = 12;
34 |
35 | // Always ABS but here to keep consistent with LOSS naming
36 | ABS_MOVING_AVERAGE_GAIN = 2;
37 | ABS_MOVING_AVERAGE_LOSS = 3;
38 | RELATIVE_STRENGTH = 4;
39 | RELATIVE_STRENGTH_INDICATOR = 5;
40 | SUM_MOVEMENT=6;
41 | SIMPLE_MOVING_AVERAGE=7;
42 | EXPONENTIAL_MOVING_AVERAGE=8;
43 | WEIGHTED_MOVING_AVERAGE=9;
44 | BB_MIDDLE_BAND_SMA=10;
45 | BB_UPPER_BAND_SMA=11;
46 | BB_BOTTOM_BAND_SMA=13;
47 | BB_MIDDLE_BAND_EMA=14;
48 | BB_UPPER_BAND_EMA=15;
49 | BB_BOTTOM_BAND_EMA=16;
50 | STANDARD_DEVIATION=17;
51 | LOG_RTN=18;
52 |
53 | }
54 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesMetricsLibrary/src/test/resources/TSAccumVWAPTest.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "key": {
4 | "major_key": "Key-A",
5 | "minor_key_string": "PRICE"
6 | },
7 | "data": {"double_val": 10.0
8 | },
9 | "extendedData": {
10 | "Data::VOL": {
11 | "doubleVal": 2.0
12 | }
13 | },
14 | "timestamp": "2000-01-01T00:00:00Z"
15 | },
16 |
17 | {
18 | "key": {
19 | "major_key": "Key-B",
20 | "minor_key_string": "PRICE"
21 | },
22 | "data": {"double_val": 20.0
23 | }, "extendedData": {
24 | "Data::VOL": {
25 | "doubleVal": 2.0
26 | }
27 | },
28 | "timestamp": "2000-01-01T00:00:00Z"
29 | },
30 |
31 | {
32 | "key": {
33 | "major_key": "Key-B",
34 | "minor_key_string": "PRICE"
35 | },
36 | "data": {"double_val": 40.0
37 | },
38 | "extendedData": {
39 | "Data::VOL": {
40 | "doubleVal": 8.0
41 | }
42 | },
43 | "timestamp": "2000-01-01T00:00:01Z",
44 | "time": {
45 | "advance_watermark_seconds": 5
46 | }
47 | },
48 |
49 | {
50 | "key": {
51 | "major_key": "Key-A",
52 | "minor_key_string": "PRICE"
53 | },
54 | "data": {"double_val": 10.0
55 | },
56 | "extendedData": {
57 | "Data::VOL": {
58 | "doubleVal": 2.0
59 | }
60 | },
61 |
62 | "timestamp": "2000-01-01T00:00:05Z",
63 | "time": {
64 | "advance_watermark_expression": "INFINITY"
65 | }
66 | }
67 | ]
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesMetricsLibrary/src/test/resources/VWAPTestGap.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "key": {
4 | "major_key": "Key-A",
5 | "minor_key_string": "PRICE"
6 | },
7 | "data": {"double_val": 10.0
8 | },
9 | "extendedData": {
10 | "Data::VOL": {
11 | "doubleVal": 2.0
12 | }
13 | },
14 | "timestamp": "2000-01-01T00:00:00Z"
15 | },
16 | {
17 | "key": {
18 | "major_key": "Key-B",
19 | "minor_key_string": "PRICE"
20 | },
21 | "data": {"double_val": 20.0
22 | }, "extendedData": {
23 | "Data::VOL": {
24 | "doubleVal": 2.0
25 | }
26 | },
27 | "timestamp": "2000-01-01T00:00:00Z",
28 | "time": {
29 | "advance_watermark_seconds": 5
30 | }
31 | },
32 | {
33 | "key": {
34 | "major_key": "Key-B",
35 | "minor_key_string": "PRICE"
36 | },
37 | "data": {"double_val": 40.0
38 | },
39 | "extendedData": {
40 | "Data::VOL": {
41 | "doubleVal": 8.0
42 | }
43 | },
44 | "timestamp": "2000-01-01T00:00:11Z",
45 | "time": {
46 | "advance_watermark_seconds": 5
47 | }
48 | },
49 |
50 | {
51 | "key": {
52 | "major_key": "Key-A",
53 | "minor_key_string": "PRICE"
54 | },
55 | "data": {"double_val": 10.0
56 | },
57 | "extendedData": {
58 | "Data::VOL": {
59 | "doubleVal": 2.0
60 | }
61 | },
62 |
63 | "timestamp": "2000-01-01T00:00:11Z",
64 | "time": {
65 | "advance_watermark_expression": "INFINITY"
66 | }
67 | }
68 | ]
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesMetricsLibrary/src/test/resources/VWAPWithinBoundTest.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "key": {
4 | "major_key": "Key-A",
5 | "minor_key_string": "PRICE"
6 | },
7 | "data": {"double_val": 10.0
8 | },
9 | "extendedData": {
10 | "Data::VOL": {
11 | "doubleVal": 2.0
12 | }
13 | },
14 | "timestamp": "2000-01-01T00:00:00Z",
15 | "time": {
16 | "advance_watermark_seconds": 5
17 | }
18 | },
19 | {
20 | "key": {
21 | "major_key": "Key-A",
22 | "minor_key_string": "ASK"
23 | },
24 | "data": {"double_val": 8.0
25 | },
26 | "timestamp": "2000-01-01T00:00:06Z"
27 | },
28 | {
29 | "key": {
30 | "major_key": "Key-A",
31 | "minor_key_string": "BID"
32 | },
33 | "data": {"double_val": 6.0
34 | },
35 |
36 | "timestamp": "2000-01-01T00:00:06Z",
37 | "time": {
38 | "advance_watermark_seconds": 5
39 | }
40 | },
41 |
42 | {
43 | "key": {
44 | "major_key": "Key-A",
45 | "minor_key_string": "PRICE"
46 | },
47 | "data": {"double_val": 10.0
48 | },
49 | "extendedData": {
50 | "Data::VOL": {
51 | "doubleVal": 2.0
52 | }
53 | },
54 |
55 | "timestamp": "2000-01-01T00:00:11Z",
56 | "time": {
57 | "advance_watermark_expression": "INFINITY"
58 | }
59 | }
60 | ]
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/combiners/BTypeOne.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.combiners;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccum;
21 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSDataPoint;
22 | import java.io.Serializable;
23 |
24 | /** Allows for type one metrics to be added to the system. */
25 | public abstract class BTypeOne implements Serializable {
26 |
27 | /** Define how a datapoints values should be added to a TSAccum for a type one computation */
28 | public abstract TSAccum addInput(TSAccum accumulator, TSDataPoint dataPoint);
29 |
30 | /** Define how two accums should be merged for the type one computation */
31 | public abstract TSAccum mergeDataAccums(TSAccum a, TSAccum b);
32 | }
33 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/combiners/TSCombiner.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.combiners;
19 |
20 | import org.apache.beam.sdk.annotations.Experimental;
21 |
22 | @Experimental
23 | /** Place holder, will be used in future as way to optimise the DAG for type 2 computations. */
24 | public interface TSCombiner {}
25 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/combiners/typeone/TSCategoricalCombiner.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.combiners.typeone;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccum;
21 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSDataPoint;
22 | import com.google.dataflow.sample.timeseriesflow.combiners.TSCombiner;
23 | import org.apache.beam.sdk.annotations.Experimental;
24 |
25 | /** Not implemented. */
26 | @Experimental
27 | public class TSCategoricalCombiner extends TSBaseCombiner implements TSCombiner {
28 |
29 | // TODO implement in next cycle.
30 | private TSCategoricalCombiner() {}
31 |
32 | public static TSCategoricalCombiner combine() {
33 | return new TSCategoricalCombiner();
34 | }
35 |
36 | @Override
37 | public TSAccum mergeTypedDataAccum(TSAccum a, TSAccum b) {
38 |
39 | return null;
40 | }
41 |
42 | @Override
43 | public TSAccum addTypeSpecificInput(TSAccum accumulator, TSDataPoint dataPoint) {
44 | // AccumCategoricalBuilder accumStoreCoreCategorical = new AccumCategoricalBuilder(accum);
45 | // accumStoreCoreCategorical.setDOW(createNumData(time.toDateTime().dayOfWeek().get()));
46 | // accumStoreCoreCategorical.setDOM(createNumData(time.toDateTime().dayOfMonth().get()));
47 | // accumStoreCoreCategorical.setYY(createNumData(time.toDateTime().year().get()));
48 |
49 | return null;
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/common/TupleTypes.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.common;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSDataPoint;
21 | import org.apache.beam.sdk.annotations.Experimental;
22 | import org.apache.beam.sdk.values.TupleTag;
23 |
24 | @Experimental
25 | /** Used for filter operations within aggregations. */
26 | public class TupleTypes {
27 |
28 | public static TupleTag t_str = new TupleTag() {};
29 | public static TupleTag t_int = new TupleTag() {};
30 | public static TupleTag t_double = new TupleTag() {};
31 | public static TupleTag t_long = new TupleTag() {};
32 | public static TupleTag t_float = new TupleTag() {};
33 | }
34 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/datamap/AccumCategoricalBuilder.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.datamap;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.DerivedAggregations.Indicators;
21 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.Data;
22 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccum;
23 | import org.apache.beam.sdk.annotations.Experimental;
24 |
25 | @Experimental
26 | /** Builder for the as yet un- implemented Categorical values. */
27 | public class AccumCategoricalBuilder extends AccumCoreMetadataBuilder {
28 |
29 | public AccumCategoricalBuilder(TSAccum tsAccum) {
30 | super(tsAccum);
31 | }
32 |
33 | public Data getDOWOrNull() {
34 | return getValueOrNull(Indicators.DOW.name());
35 | }
36 |
37 | public Data getDOMOrNull() {
38 | return getValueOrNull(Indicators.DOM.name());
39 | }
40 |
41 | public Data getYYOrNull() {
42 | return getValueOrNull(Indicators.YY.name());
43 | }
44 |
45 | public void setDOW(Data data) {
46 | setValue(Indicators.DOW.name(), data);
47 | }
48 |
49 | public void setDOM(Data data) {
50 | setValue(Indicators.DOM.name(), data);
51 | }
52 |
53 | public void setYY(Data data) {
54 | setValue(Indicators.YY.name(), data);
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/datamap/AccumCoreNumericBuilder.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.datamap;
19 |
20 | import com.google.common.base.Preconditions;
21 | import com.google.dataflow.sample.timeseriesflow.DerivedAggregations.Indicators;
22 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.Data;
23 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccum;
24 | import org.apache.beam.sdk.annotations.Experimental;
25 |
26 | @Experimental
27 | /** Accum Numeric Builder, dealing with common numeric aggregations Sum, Min, Max, First, Last. */
28 | public class AccumCoreNumericBuilder extends AccumCoreMetadataBuilder {
29 |
30 | public AccumCoreNumericBuilder(TSAccum tsAccum) {
31 | super(tsAccum);
32 | }
33 |
34 | public Data getSumOrNull() {
35 | return getValueOrNull(Indicators.SUM.name());
36 | }
37 |
38 | public Data getMaxOrNull() {
39 | return getValueOrNull(Indicators.MAX.name());
40 | }
41 |
42 | public Data getMinOrNull() {
43 | return getValueOrNull(Indicators.MIN.name());
44 | }
45 |
46 | public void setSum(Data data) {
47 | Preconditions.checkNotNull(data);
48 | setValue(Indicators.SUM.name(), data);
49 | }
50 |
51 | public void setMax(Data data) {
52 |
53 | Preconditions.checkNotNull(data);
54 | setValue(Indicators.MAX.name(), data);
55 | }
56 |
57 | public void setMin(Data data) {
58 | Preconditions.checkNotNull(data);
59 | setValue(Indicators.MIN.name(), data);
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/metrics/BTypeTwoFn.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.metrics;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccum;
21 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccumSequence;
22 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSKey;
23 | import org.apache.beam.sdk.options.PipelineOptions;
24 | import org.apache.beam.sdk.transforms.Contextful;
25 | import org.apache.beam.sdk.transforms.Contextful.Fn;
26 | import org.apache.beam.sdk.transforms.SerializableFunction;
27 | import org.apache.beam.sdk.values.KV;
28 |
29 | public abstract class BTypeTwoFn {
30 |
31 | public abstract SerializableFunction, KV> getFunction(
32 | PipelineOptions options);
33 |
34 | public Contextful, KV>> getContextualFn(
35 | PipelineOptions options) {
36 | return Contextful.fn(getFunction(options));
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/metrics/CTypeTwo.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.metrics;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccum;
21 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccumSequence;
22 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSKey;
23 | import com.google.dataflow.sample.timeseriesflow.combiners.BTypeOne;
24 | import java.util.List;
25 | import org.apache.beam.sdk.transforms.PTransform;
26 | import org.apache.beam.sdk.values.KV;
27 | import org.apache.beam.sdk.values.PCollection;
28 |
29 | /**
30 | * Complex Type 2 metrics require access to the TSFlow graph to create multi stage computations. If
31 | * a computation is a simple f(KV) -> KV make use of {@link
32 | * BTypeTwo}
33 | */
34 | public abstract class CTypeTwo
35 | extends PTransform>, PCollection>> {
36 |
37 | /**
38 | * Some type 2 computations may need specialized type 1 computations. {@link
39 | * com.google.dataflow.sample.timeseriesflow.graph.GenerateComputations} class will attach any
40 | * provided by this list. If none required return an empty list.
41 | */
42 | public abstract List> requiredTypeOne();
43 |
44 | public abstract List excludeFromOutput();
45 | }
46 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/metrics/core/typeone/TestMax.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.metrics.core.typeone;
19 |
20 | import com.google.common.annotations.VisibleForTesting;
21 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccum;
22 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSDataPoint;
23 | import com.google.dataflow.sample.timeseriesflow.combiners.BTypeOne;
24 | import com.google.dataflow.sample.timeseriesflow.common.TSDataUtils;
25 | import com.google.dataflow.sample.timeseriesflow.datamap.AccumCoreNumericBuilder;
26 | import org.apache.beam.sdk.annotations.Experimental;
27 |
28 | @VisibleForTesting
29 | @Experimental
30 | /** This is a dummy class used as a test artifact only. */
31 | public class TestMax extends BTypeOne {
32 | @Override
33 | public TSAccum addInput(TSAccum accumulator, TSDataPoint dataPoint) {
34 | AccumCoreNumericBuilder coreNumeric = new AccumCoreNumericBuilder(accumulator);
35 | coreNumeric.setMax(TSDataUtils.findMaxValue(coreNumeric.getMaxOrNull(), dataPoint.getData()));
36 | return coreNumeric.build();
37 | }
38 |
39 | @Override
40 | public TSAccum mergeDataAccums(TSAccum a, TSAccum b) {
41 | AccumCoreNumericBuilder aBuilder = new AccumCoreNumericBuilder(a);
42 | AccumCoreNumericBuilder bBuilder = new AccumCoreNumericBuilder(b);
43 | aBuilder.setMax(TSDataUtils.findMaxValue(aBuilder.getMaxOrNull(), bBuilder.getMaxOrNull()));
44 |
45 | return aBuilder.build();
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/metrics/core/typeone/TestMin.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.metrics.core.typeone;
19 |
20 | import com.google.common.annotations.VisibleForTesting;
21 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccum;
22 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSDataPoint;
23 | import com.google.dataflow.sample.timeseriesflow.combiners.BTypeOne;
24 | import com.google.dataflow.sample.timeseriesflow.common.TSDataUtils;
25 | import com.google.dataflow.sample.timeseriesflow.datamap.AccumCoreNumericBuilder;
26 | import org.apache.beam.sdk.annotations.Experimental;
27 |
28 | @VisibleForTesting
29 | @Experimental
30 | /** This is a dummy class used as a test artifact only. */
31 | public class TestMin extends BTypeOne {
32 | @Override
33 | public TSAccum addInput(TSAccum accumulator, TSDataPoint dataPoint) {
34 | AccumCoreNumericBuilder coreNumeric = new AccumCoreNumericBuilder(accumulator);
35 | coreNumeric.setMin(TSDataUtils.findMinData(coreNumeric.getMinOrNull(), dataPoint.getData()));
36 | return coreNumeric.build();
37 | }
38 |
39 | @Override
40 | public TSAccum mergeDataAccums(TSAccum a, TSAccum b) {
41 | AccumCoreNumericBuilder aBuilder = new AccumCoreNumericBuilder(a);
42 | AccumCoreNumericBuilder bBuilder = new AccumCoreNumericBuilder(b);
43 | aBuilder.setMin(TSDataUtils.findMinData(aBuilder.getMinOrNull(), bBuilder.getMinOrNull()));
44 |
45 | return aBuilder.build();
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/metrics/core/typeone/TestSum.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.metrics.core.typeone;
19 |
20 | import com.google.common.annotations.VisibleForTesting;
21 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccum;
22 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSDataPoint;
23 | import com.google.dataflow.sample.timeseriesflow.combiners.BTypeOne;
24 | import com.google.dataflow.sample.timeseriesflow.common.CommonUtils;
25 | import com.google.dataflow.sample.timeseriesflow.datamap.AccumCoreNumericBuilder;
26 | import org.apache.beam.sdk.annotations.Experimental;
27 |
28 | @VisibleForTesting
29 | @Experimental
30 | /** This is a dummy class used as a test artifact only. */
31 | public class TestSum extends BTypeOne {
32 | @Override
33 | public TSAccum addInput(TSAccum accumulator, TSDataPoint dataPoint) {
34 | AccumCoreNumericBuilder coreNumeric = new AccumCoreNumericBuilder(accumulator);
35 | coreNumeric.setSum(
36 | CommonUtils.sumNumericDataNullAsZero(coreNumeric.getSumOrNull(), dataPoint.getData()));
37 | return coreNumeric.build();
38 | }
39 |
40 | @Override
41 | public TSAccum mergeDataAccums(TSAccum a, TSAccum b) {
42 | AccumCoreNumericBuilder aBuilder = new AccumCoreNumericBuilder(a);
43 | AccumCoreNumericBuilder bBuilder = new AccumCoreNumericBuilder(b);
44 | aBuilder.setSum(
45 | CommonUtils.sumNumericDataNullAsZero(aBuilder.getSumOrNull(), bBuilder.getSumOrNull()));
46 |
47 | return aBuilder.build();
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/metrics/core/typetwo/Test1Fn.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.metrics.core.typetwo;
19 |
20 | import com.google.common.annotations.VisibleForTesting;
21 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccum;
22 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccumSequence;
23 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSKey;
24 | import com.google.dataflow.sample.timeseriesflow.metrics.BTypeTwoFn;
25 | import org.apache.beam.sdk.annotations.Experimental;
26 | import org.apache.beam.sdk.options.PipelineOptions;
27 | import org.apache.beam.sdk.transforms.SerializableFunction;
28 | import org.apache.beam.sdk.values.KV;
29 |
30 | @VisibleForTesting
31 | @Experimental
32 | /** This is a dummy class used as a test artifact only. */
33 | public class Test1Fn extends BTypeTwoFn {
34 |
35 | public SerializableFunction, KV> getFunction(
36 | PipelineOptions options) {
37 |
38 | return (SerializableFunction, KV>)
39 | element -> {
40 | return KV.of(element.getKey(), TSAccum.newBuilder().build());
41 | };
42 | }
43 |
44 | /** Options for {@link Test1Fn} fn. */
45 | public static interface TestFnOptions extends PipelineOptions {}
46 | }
47 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/metrics/core/typetwo/Test2Fn.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.metrics.core.typetwo;
19 |
20 | import com.google.common.annotations.VisibleForTesting;
21 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccum;
22 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccumSequence;
23 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSKey;
24 | import com.google.dataflow.sample.timeseriesflow.metrics.BTypeTwoFn;
25 | import org.apache.beam.sdk.annotations.Experimental;
26 | import org.apache.beam.sdk.options.PipelineOptions;
27 | import org.apache.beam.sdk.transforms.SerializableFunction;
28 | import org.apache.beam.sdk.values.KV;
29 |
30 | @VisibleForTesting
31 | @Experimental
32 | /** This is a dummy class used as a test artifact only. */
33 | public class Test2Fn extends BTypeTwoFn {
34 |
35 | public SerializableFunction, KV> getFunction(
36 | PipelineOptions options) {
37 |
38 | return (SerializableFunction, KV>)
39 | element -> {
40 | return KV.of(element.getKey(), TSAccum.newBuilder().build());
41 | };
42 | }
43 |
44 | /** Options for {@link Test2Fn} fn. */
45 | public static interface TestFnOptions extends PipelineOptions {}
46 | }
47 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/options/GapFillOptions.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.options;
19 |
20 | import org.apache.beam.sdk.options.Default;
21 | import org.apache.beam.sdk.options.Description;
22 | import org.apache.beam.sdk.options.PipelineOptions;
23 |
24 | public interface GapFillOptions extends PipelineOptions {
25 |
26 | @Description("Is Gap filling enabled.")
27 | Boolean isGapFillEnabled();
28 |
29 | void setGapFillEnabled(Boolean ttlDurationSecs);
30 |
31 | @Description(
32 | "The time to live value for how long after a key does not output values we continue to gap fill.")
33 | Integer getTTLDurationSecs();
34 |
35 | void setTTLDurationSecs(Integer ttlDurationSecs);
36 |
37 | @Description(
38 | "The absolute time EpocMilli to stop gap filling, used in bootstrap pipelines when working with Bounded sources")
39 | Long getAbsoluteStopTimeMSTimestamp();
40 |
41 | void setAbsoluteStopTimeMSTimestamp(Long absoluteStopTimeMSTimestamp);
42 |
43 | @Description(
44 | "Enable hold and propagate of last known value for key, within the TTLDurationSec to gaps.")
45 | @Default.Boolean(false)
46 | Boolean getEnableHoldAndPropogateLastValue();
47 |
48 | void setEnableHoldAndPropogateLastValue(Boolean enableHoldAndPropogate);
49 | }
50 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/options/GenerateComputationsOptions.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.options;
19 |
20 | import java.util.List;
21 | import org.apache.beam.sdk.annotations.Experimental;
22 | import org.apache.beam.sdk.options.Description;
23 | import org.apache.beam.sdk.options.PipelineOptions;
24 |
25 | @Experimental
26 | /**
27 | * PipelineOptions to allow the Out of the box metrics to be called by name. For custom metrics use
28 | * {@link com.google.dataflow.sample.timeseriesflow.graph.GenerateComputations.Builder} .
29 | */
30 | public interface GenerateComputationsOptions extends PipelineOptions {
31 |
32 | @Description("Type one computations, for example typeone.Sum")
33 | List getTypeOneBasicMetrics();
34 |
35 | @Description("Type one computations, for example typeone.Sum")
36 | void setTypeOneBasicMetrics(List value);
37 |
38 | @Description("Type two basic computations (order is preserved), for example typetwo.basic.BB")
39 | List getTypeTwoBasicMetrics();
40 |
41 | @Description("Type two basic computations (order is preserved), for example typetwo.basic.BB")
42 | void setTypeTwoBasicMetrics(List typeTwoBasicMetrics);
43 |
44 | @Description(
45 | "Type two basic computations (order is preserved), for example typetwo.complex.fsi.RSIGFn")
46 | List getTypeTwoComplexMetrics();
47 |
48 | @Description(
49 | "Type two basic computations (order is preserved), for example typetwo.complex.fsi.RSIGFn")
50 | void setTypeTwoComplexMetrics(List typeTwoBasicMetrics);
51 | }
52 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/options/TFXOptions.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.options;
19 |
20 | import org.apache.beam.sdk.annotations.Experimental;
21 | import org.apache.beam.sdk.options.Description;
22 |
23 | @Experimental
24 | public interface TFXOptions extends TSFlowOptions {
25 |
26 | @Description("Metadata interchange location")
27 | public String getInterchangeLocation();
28 |
29 | @Description("Metadata interchange location")
30 | public void setInterchangeLocation(String interchangeLocation);
31 | }
32 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/options/TSFlowOptions.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.options;
19 |
20 | import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
21 | import org.apache.beam.sdk.annotations.Experimental;
22 | import org.apache.beam.sdk.options.Description;
23 |
24 | @Experimental
25 | public interface TSFlowOptions
26 | extends DataflowPipelineOptions,
27 | GapFillOptions,
28 | GenerateComputationsOptions,
29 | TSOutputPipelineOptions {
30 |
31 | @Description("Fixed window length for type 1 computations in seconds")
32 | Integer getTypeOneComputationsLengthInSecs();
33 |
34 | void setTypeOneComputationsLengthInSecs(Integer value);
35 |
36 | @Description("Sliding window length for type 2 computations in seconds")
37 | Integer getTypeTwoComputationsLengthInSecs();
38 |
39 | void setTypeTwoComputationsLengthInSecs(Integer value);
40 |
41 | @Description("Sliding window offset length for type 2 computations in seconds")
42 | Integer getTypeTwoComputationsOffsetLengthInSecs();
43 |
44 | void setTypeTwoComputationsOffsetLengthInSecs(Integer value);
45 |
46 | @Description(
47 | "The number of timesteps that will be output is based on this value divided by the type 1 fixed window length.")
48 | Integer getOutputTimestepLengthInSecs();
49 |
50 | void setOutputTimestepLengthInSecs(Integer value);
51 | }
52 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/options/TSOutputPipelineOptions.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.options;
19 |
20 | import org.apache.beam.sdk.annotations.Experimental;
21 | import org.apache.beam.sdk.options.PipelineOptions;
22 |
23 | @Experimental
24 | public interface TSOutputPipelineOptions extends PipelineOptions {
25 |
26 | String getBigQueryTableForTSAccumOutputLocation();
27 |
28 | void setBigQueryTableForTSAccumOutputLocation(String bigQueryTableForTSAccumOutputLocation);
29 |
30 | String getPubSubTopicForTSAccumOutputLocation();
31 |
32 | void setPubSubTopicForTSAccumOutputLocation(String pubSubTopicForTSAccumOutputLocation);
33 | }
34 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/transforms/ParseTSDataPointFromBytes.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.transforms;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSDataPoint;
21 | import com.google.protobuf.InvalidProtocolBufferException;
22 | import com.google.protobuf.util.Timestamps;
23 | import org.apache.beam.sdk.annotations.Experimental;
24 | import org.apache.beam.sdk.transforms.DoFn;
25 | import org.joda.time.Instant;
26 | import org.slf4j.Logger;
27 | import org.slf4j.LoggerFactory;
28 |
29 | /** Return a {@link TSDataPoint} from bytes. */
30 | @Experimental
31 | public class ParseTSDataPointFromBytes extends DoFn {
32 |
33 | private static final Logger LOG = LoggerFactory.getLogger(ParseTSDataPointFromBytes.class);
34 |
35 | public static ParseTSDataPointFromBytes create() {
36 | return new ParseTSDataPointFromBytes();
37 | }
38 |
39 | @ProcessElement
40 | public void process(@Element byte[] input, OutputReceiver o) {
41 | try {
42 | TSDataPoint dataPoint = TSDataPoint.parseFrom(input);
43 | o.outputWithTimestamp(
44 | dataPoint, Instant.ofEpochMilli(Timestamps.toMillis(dataPoint.getTimestamp())));
45 | } catch (InvalidProtocolBufferException e) {
46 | LOG.error(e.getMessage());
47 | }
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/transforms/TSAccumSequenceToRow.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.transforms;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccumSequence;
21 | import org.apache.beam.sdk.annotations.Experimental;
22 | import org.apache.beam.sdk.extensions.protobuf.ProtoMessageSchema;
23 | import org.apache.beam.sdk.schemas.transforms.Convert;
24 | import org.apache.beam.sdk.transforms.PTransform;
25 | import org.apache.beam.sdk.values.PCollection;
26 | import org.apache.beam.sdk.values.Row;
27 |
28 | @Experimental
29 | /** Convert a {@link TSAccumSequence} to a {@link Row}. */
30 | public class TSAccumSequenceToRow
31 | extends PTransform, PCollection
> {
32 | @Override
33 | public PCollection expand(PCollection input) {
34 |
35 | input
36 | .getPipeline()
37 | .getSchemaRegistry()
38 | .registerSchemaProvider(TSAccumSequence.class, new ProtoMessageSchema());
39 |
40 | return input.apply(Convert.toRows());
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/transforms/TSAccumToJson.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.transforms;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccum;
21 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSKey;
22 | import com.google.protobuf.InvalidProtocolBufferException;
23 | import com.google.protobuf.util.JsonFormat;
24 | import org.apache.beam.sdk.annotations.Experimental;
25 | import org.apache.beam.sdk.transforms.MapElements;
26 | import org.apache.beam.sdk.transforms.PTransform;
27 | import org.apache.beam.sdk.values.KV;
28 | import org.apache.beam.sdk.values.PCollection;
29 | import org.apache.beam.sdk.values.TypeDescriptors;
30 |
31 | @Experimental
32 | /** Return a Json representation of the TSAccum object. */
33 | public class TSAccumToJson
34 | extends PTransform>, PCollection> {
35 |
36 | public static TSAccumToJson create() {
37 | return new TSAccumToJson();
38 | }
39 |
40 | private TSAccumToJson() {};
41 |
42 | @Override
43 | public PCollection expand(PCollection> input) {
44 | return input.apply(
45 | MapElements.into(TypeDescriptors.strings())
46 | .via(
47 | x -> {
48 | String json = null;
49 | try {
50 | json = JsonFormat.printer().print(x.getValue().toBuilder().setKey(x.getKey()));
51 | } catch (InvalidProtocolBufferException e) {
52 | json = String.format("{error: %s}", e.getMessage());
53 | }
54 | return json;
55 | }));
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/transforms/TSDataPointToRow.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.transforms;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSDataPoint;
21 | import org.apache.beam.sdk.annotations.Experimental;
22 | import org.apache.beam.sdk.extensions.protobuf.ProtoMessageSchema;
23 | import org.apache.beam.sdk.schemas.transforms.Convert;
24 | import org.apache.beam.sdk.transforms.PTransform;
25 | import org.apache.beam.sdk.values.PCollection;
26 | import org.apache.beam.sdk.values.Row;
27 |
28 | @Experimental
29 | /** Convert {@link TSDataPoint} to {@link Row} */
30 | public class TSDataPointToRow extends PTransform, PCollection> {
31 | @Override
32 | public PCollection expand(PCollection input) {
33 |
34 | input
35 | .getPipeline()
36 | .getSchemaRegistry()
37 | .registerSchemaProvider(TSDataPoint.class, new ProtoMessageSchema());
38 |
39 | return input.apply(Convert.toRows());
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/java/com/google/dataflow/sample/timeseriesflow/transforms/TypeTwoComputation.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.transforms;
19 |
20 | import java.lang.annotation.Retention;
21 | import java.lang.annotation.RetentionPolicy;
22 |
23 | @Retention(RetentionPolicy.RUNTIME)
24 | public @interface TypeTwoComputation {
25 | ComputeType computeType();
26 |
27 | enum ComputeType {
28 | SINGLE_KEY,
29 | COMPOSITE_KEY
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/proto/TFExampleKeys.proto:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
3 | * agreements. See the NOTICE file distributed with this work for additional information regarding
4 | * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
5 | * "License"); you may not use this file except in compliance with the License. You may obtain a
6 | * copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software distributed under the License
11 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12 | * or implied. See the License for the specific language governing permissions and limitations under
13 | * the License.
14 | */
15 | syntax = "proto3";
16 |
17 | package protos;
18 |
19 | option java_outer_classname = "TimeSeriesTFExampleKeys";
20 | option java_multiple_files = false;
21 | option java_package = "com.google.dataflow.sample.timeseriesflow";
22 |
23 | enum ExampleMetadata {
24 | METADATA_SPAN_START_TS = 0;
25 | METADATA_SPAN_END_TS = 1;
26 | METADATA_MAJOR_KEY = 3;
27 | METADATA_MINOR_KEY = 4;
28 | }
29 |
30 | enum ExampleTypes{
31 | BYTE = 0;
32 | INT64 = 1;
33 | FLOAT = 2;
34 | }
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/main/proto/TSBaseKeys.proto:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
3 | * agreements. See the NOTICE file distributed with this work for additional information regarding
4 | * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
5 | * "License"); you may not use this file except in compliance with the License. You may obtain a
6 | * copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software distributed under the License
11 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12 | * or implied. See the License for the specific language governing permissions and limitations under
13 | * the License.
14 | */
15 | syntax = "proto3";
16 |
17 | package protos;
18 |
19 | option java_outer_classname = "DerivedAggregations";
20 | option java_multiple_files = false;
21 | option java_package = "com.google.dataflow.sample.timeseriesflow";
22 |
23 | /**
24 | This proto stores all of the keys used in the accum map.
25 |
26 | */
27 | enum Indicators{
28 |
29 | DATA_POINT_COUNT = 0;
30 |
31 | SUM = 1;
32 | MIN = 2;
33 | MAX = 3;
34 | FIRST = 4;
35 | LAST = 5;
36 |
37 | // First timestamp within a aggregation
38 | FIRST_TIMESTAMP = 6;
39 |
40 | // Last timestamp within a aggregation
41 | LAST_TIMESTAMP = 7;
42 |
43 | // Day Of Week . In TSAccum this will be from First Value seen. UTC
44 | DOW = 8;
45 | // Day of Month. In TSAccum this will be from First Value seen. UTC
46 | DOM = 9;
47 | // Year. In TSAccum this will be from First Value seen. UTC
48 | YY = 10;
49 | // Indication if a value is generated via GapFill routine
50 | HB = 11;
51 |
52 | }
53 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/test/java/com/google/dataflow/sample/timeseriesflow/test/PerfectRectanglesScalability_5Days_5Keys_EvenGaps.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.test;
19 |
20 | import org.apache.beam.runners.dataflow.DataflowRunner;
21 | import org.apache.beam.sdk.Pipeline;
22 | import org.apache.beam.sdk.options.PipelineOptionsFactory;
23 |
24 | public class PerfectRectanglesScalability_5Days_5Keys_EvenGaps {
25 |
26 | public static void main(String args[]) {
27 | System.out.println("Running 1 Day 86400 with 1 Key");
28 |
29 | ScaleTestingOptions options =
30 | PipelineOptionsFactory.fromArgs(args).as(ScaleTestingOptions.class);
31 |
32 | options.setAppName("TestPerfectRectangles_86400S_NoGaps");
33 | options.setTypeOneComputationsLengthInSecs(1);
34 | options.setTypeTwoComputationsLengthInSecs(60);
35 | options.setOutputTimestepLengthInSecs(60);
36 | options.setTTLDurationSecs(2);
37 | options.setNumKeys(5);
38 | options.setPerfectRecNumberDataSecs(86400 * 5);
39 | options.setRunner(DataflowRunner.class);
40 | options.setSkipEvens(true);
41 |
42 | Pipeline p = Pipeline.create(options);
43 |
44 | PerfectRectangleUtils.testPerfectRecScalability(p);
45 |
46 | long time = System.currentTimeMillis();
47 | p.run();
48 | System.out.println(System.currentTimeMillis() - time);
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/test/java/com/google/dataflow/sample/timeseriesflow/test/PerfectRectanglesScalability_86400S_AllGaps.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.test;
19 |
20 | import org.apache.beam.sdk.Pipeline;
21 | import org.apache.beam.sdk.options.PipelineOptionsFactory;
22 |
23 | public class PerfectRectanglesScalability_86400S_AllGaps {
24 |
25 | public static void main(String args[]) {
26 | System.out.println("Running 1 Day 86400 with 1 Key");
27 |
28 | ScaleTestingOptions options =
29 | PipelineOptionsFactory.fromArgs(args).as(ScaleTestingOptions.class);
30 |
31 | options.setAppName("TestPerfectRectangles_86400S_NoGaps");
32 | options.setTypeOneComputationsLengthInSecs(1);
33 | options.setTypeTwoComputationsLengthInSecs(60);
34 | options.setOutputTimestepLengthInSecs(60);
35 | options.setTTLDurationSecs(86400);
36 | options.setNumKeys(1);
37 | options.setPerfectRecNumberDataSecs(1);
38 | options.setSkipEvens(false);
39 |
40 | Pipeline p = Pipeline.create(options);
41 |
42 | PerfectRectangleUtils.testPerfectRecScalability(p);
43 |
44 | long time = System.currentTimeMillis();
45 | p.run();
46 | System.out.println(System.currentTimeMillis() - time);
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/test/java/com/google/dataflow/sample/timeseriesflow/test/PerfectRectanglesScalability_86400S_EvenGaps.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.test;
19 |
20 | import org.apache.beam.runners.dataflow.DataflowRunner;
21 | import org.apache.beam.sdk.Pipeline;
22 | import org.apache.beam.sdk.options.PipelineOptionsFactory;
23 |
24 | public class PerfectRectanglesScalability_86400S_EvenGaps {
25 |
26 | public static void main(String args[]) {
27 | System.out.println("Running 1 Day 86400 with 1 Key");
28 |
29 | ScaleTestingOptions options =
30 | PipelineOptionsFactory.fromArgs(args).as(ScaleTestingOptions.class);
31 |
32 | options.setAppName("TestPerfectRectangles_86400S_NoGaps");
33 | options.setTypeOneComputationsLengthInSecs(1);
34 | options.setTypeTwoComputationsLengthInSecs(60);
35 | options.setOutputTimestepLengthInSecs(60);
36 | options.setTTLDurationSecs(2);
37 | options.setNumKeys(1);
38 | options.setPerfectRecNumberDataSecs(86400);
39 | options.setRunner(DataflowRunner.class);
40 | options.setNumFeatures(1);
41 | options.setSkipEvens(true);
42 |
43 | Pipeline p = Pipeline.create(options);
44 |
45 | PerfectRectangleUtils.testPerfectRecScalability(p);
46 |
47 | long time = System.currentTimeMillis();
48 | p.run();
49 | System.out.println(System.currentTimeMillis() - time);
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/test/java/com/google/dataflow/sample/timeseriesflow/test/PerfectRectanglesScalability_86400S_NoGaps.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.test;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSDataPoint;
21 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSKey;
22 | import org.apache.beam.sdk.Pipeline;
23 | import org.apache.beam.sdk.options.PipelineOptionsFactory;
24 | import org.apache.beam.sdk.values.KV;
25 | import org.apache.beam.sdk.values.PCollection;
26 |
27 | public class PerfectRectanglesScalability_86400S_NoGaps {
28 |
29 | public static void main(String args[]) {
30 | System.out.println("Running 1 Day 86400 with 1 Key");
31 |
32 | ScaleTestingOptions options =
33 | PipelineOptionsFactory.fromArgs(args).as(ScaleTestingOptions.class);
34 |
35 | options.setAppName("TestPerfectRectangles_86400S_NoGaps");
36 | options.setTypeOneComputationsLengthInSecs(1);
37 | options.setTypeTwoComputationsLengthInSecs(60);
38 | options.setOutputTimestepLengthInSecs(60);
39 | options.setTTLDurationSecs(0);
40 | options.setNumKeys(1);
41 | options.setPerfectRecNumberDataSecs(86400);
42 | options.setSkipEvens(false);
43 |
44 | Pipeline p = Pipeline.create(options);
45 |
46 | PCollection> results =
47 | PerfectRectangleUtils.testPerfectRecScalability(p);
48 |
49 | long time = System.currentTimeMillis();
50 | p.run();
51 | System.out.println(System.currentTimeMillis() - time);
52 | }
53 | }
54 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/test/java/com/google/dataflow/sample/timeseriesflow/test/ScaleTestingOptions.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.test;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.options.TSFlowOptions;
21 |
22 | public interface ScaleTestingOptions extends TSFlowOptions {
23 |
24 | public Integer getNumSecs();
25 |
26 | public void setNumSecs(Integer typeOneComputationsLengthInSecs);
27 |
28 | public Integer getNumFeatures();
29 |
30 | public void setNumFeatures(Integer typeOneComputationsLengthInSecs);
31 |
32 | public Integer getNumKeys();
33 |
34 | public void setNumKeys(Integer numKeys);
35 |
36 | public Boolean getWithFileOutput();
37 |
38 | public void setWithFileOutput(Boolean withFileOutput);
39 |
40 | public Integer getPerfectRecNumberDataSecs();
41 |
42 | public void setPerfectRecNumberDataSecs(Integer endTimestamp);
43 |
44 | public Boolean getSkipEvens();
45 |
46 | public void setSkipEvens(Boolean skipEvens);
47 | }
48 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/test/java/com/google/dataflow/sample/timeseriesflow/test/SnapShotScalability_1D_100M_100K_1FW_60SW_withTFExampleSerlization.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.test;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccumSequence;
21 | import com.google.dataflow.sample.timeseriesflow.common.CommonUtils;
22 | import com.google.dataflow.sample.timeseriesflow.io.tfexample.FeaturesFromIterableAccumSequence;
23 | import com.google.dataflow.sample.timeseriesflow.options.TFXOptions;
24 | import org.apache.beam.sdk.Pipeline;
25 | import org.apache.beam.sdk.options.PipelineOptionsFactory;
26 | import org.apache.beam.sdk.values.PCollection;
27 |
28 | public class SnapShotScalability_1D_100M_100K_1FW_60SW_withTFExampleSerlization {
29 |
30 | public static void main(String args[]) {
31 | System.out.println("Running 1 Day with 1 Key and 100 features @ Type 1 1 sec Type 2 60 sec");
32 |
33 | ScaleTestingOptions options =
34 | PipelineOptionsFactory.fromArgs(args).as(ScaleTestingOptions.class);
35 |
36 | options.setAppName("SimpleDataStreamTSDataPoints");
37 | options.setTypeOneComputationsLengthInSecs(1);
38 | options.setTypeTwoComputationsLengthInSecs(60);
39 | options.setOutputTimestepLengthInSecs(60);
40 | options.setNumKeys(100);
41 | options.setNumSecs(86400);
42 | options.setNumFeatures(100);
43 |
44 | Pipeline p = Pipeline.create(options);
45 |
46 | PCollection> examples = SnapShotUtils.testSnapShotScalability(p);
47 |
48 | Integer timesteps = CommonUtils.getNumOfSequenceTimesteps(p.getOptions().as(TFXOptions.class));
49 |
50 | examples.apply(new FeaturesFromIterableAccumSequence(timesteps, true));
51 |
52 | p.run();
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/test/java/com/google/dataflow/sample/timeseriesflow/test/SnapShotScalability_1D_100M_1K_1FW_60SW.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.test;
19 |
20 | import org.apache.beam.sdk.Pipeline;
21 | import org.apache.beam.sdk.options.PipelineOptionsFactory;
22 |
23 | public class SnapShotScalability_1D_100M_1K_1FW_60SW {
24 |
25 | public static void main(String args[]) {
26 | System.out.println("Running 1 Day with 1 Key and 100 features @ Type 1 1 sec Type 2 60 sec");
27 |
28 | ScaleTestingOptions options =
29 | PipelineOptionsFactory.fromArgs(args).as(ScaleTestingOptions.class);
30 |
31 | options.setAppName("SimpleDataStreamTSDataPoints");
32 | options.setTypeOneComputationsLengthInSecs(1);
33 | options.setTypeTwoComputationsLengthInSecs(60);
34 | options.setOutputTimestepLengthInSecs(60);
35 | options.setNumKeys(1);
36 | options.setNumSecs(86400);
37 | options.setNumFeatures(1);
38 |
39 | Pipeline p = Pipeline.create(options);
40 |
41 | SnapShotUtils.testSnapShotScalability(p);
42 |
43 | p.run();
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/test/java/com/google/dataflow/sample/timeseriesflow/test/SnapShotScalability_1D_100M_1K_1FW_60SW_withTFExampleSerlization.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.test;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSAccumSequence;
21 | import com.google.dataflow.sample.timeseriesflow.common.CommonUtils;
22 | import com.google.dataflow.sample.timeseriesflow.io.tfexample.FeaturesFromIterableAccumSequence;
23 | import com.google.dataflow.sample.timeseriesflow.options.TFXOptions;
24 | import org.apache.beam.sdk.Pipeline;
25 | import org.apache.beam.sdk.options.PipelineOptionsFactory;
26 | import org.apache.beam.sdk.values.PCollection;
27 |
28 | public class SnapShotScalability_1D_100M_1K_1FW_60SW_withTFExampleSerlization {
29 |
30 | public static void main(String args[]) {
31 | System.out.println("Running 1 Day with 1 Key and 100 features @ Type 1 1 sec Type 2 60 sec");
32 |
33 | ScaleTestingOptions options =
34 | PipelineOptionsFactory.fromArgs(args).as(ScaleTestingOptions.class);
35 |
36 | options.setAppName("SimpleDataStreamTSDataPoints");
37 | options.setTypeOneComputationsLengthInSecs(1);
38 | options.setTypeTwoComputationsLengthInSecs(60);
39 | options.setOutputTimestepLengthInSecs(60);
40 | options.setNumKeys(1);
41 | options.setNumSecs(86400);
42 | options.setNumFeatures(100);
43 |
44 | Pipeline p = Pipeline.create(options);
45 |
46 | PCollection> examples = SnapShotUtils.testSnapShotScalability(p);
47 |
48 | Integer timesteps = CommonUtils.getNumOfSequenceTimesteps(p.getOptions().as(TFXOptions.class));
49 |
50 | examples.apply(new FeaturesFromIterableAccumSequence(timesteps, true));
51 |
52 | p.run();
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/test/java/com/google/dataflow/sample/timeseriesflow/test/TestUtils.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package com.google.dataflow.sample.timeseriesflow.test;
19 |
20 | import com.google.dataflow.sample.timeseriesflow.TimeSeriesData.TSDataPoint;
21 | import com.google.protobuf.util.Timestamps;
22 | import org.apache.beam.sdk.values.TimestampedValue;
23 | import org.joda.time.Instant;
24 |
25 | public class TestUtils {
26 |
27 | public static TimestampedValue timestampedValueFromTSDataPoint(
28 | TSDataPoint tsDataPoint) {
29 |
30 | return TimestampedValue.of(
31 | tsDataPoint, Instant.ofEpochMilli(Timestamps.toMillis(tsDataPoint.getTimestamp())));
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/test/proto/TSTest.proto:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
3 | * agreements. See the NOTICE file distributed with this work for additional information regarding
4 | * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
5 | * "License"); you may not use this file except in compliance with the License. You may obtain a
6 | * copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software distributed under the License
11 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
12 | * or implied. See the License for the specific language governing permissions and limitations under
13 | * the License.
14 | */
15 | syntax = "proto3";
16 |
17 | package protos;
18 |
19 | import "google/protobuf/duration.proto";
20 | import "google/protobuf/timestamp.proto";
21 |
22 | option java_outer_classname = "TimeSeriesDataTest";
23 | option java_multiple_files = false;
24 | option java_package = "com.google.dataflow.sample.timeseriesflow";
25 |
26 | message Time {
27 | oneof time_point {
28 |
29 | AdvanceWatermarkExpression advance_watermark_expression = 1;
30 |
31 | int32 advance_watermark_seconds = 2;
32 |
33 | int32 advance_processing_timeSeconds = 3;
34 | }
35 | }
36 |
37 | message TSTimePointTest {
38 | Time time = 1;
39 |
40 | }
41 |
42 | enum AdvanceWatermarkExpression {
43 | LATE_ALLOWED = 0;
44 | TOO_LATE = 1;
45 | INFINITY = 2;
46 | }
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/TimeSeriesPipeline/src/test/resources/CreateCompositeTSAccumTest.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "key": {
4 | "major_key": "Key-A",
5 | "minor_key_string": "MKey-a"
6 | },
7 | "data": {"double_val": 1.0
8 | }
9 | },
10 | {
11 | "key": {
12 | "major_key": "Key-A",
13 | "minor_key_string": "MKey-b"
14 | },
15 | "data": {
16 | "double_val": 10.0
17 | },
18 | "time": {
19 | "advance_watermark_seconds": 5
20 | },
21 | "time": {
22 | "advance_watermark_expression": "INFINITY"
23 | }
24 | }
25 | ]
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/build.gradle:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * License); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an AS IS BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | // Needed for spotless apply
20 | buildscript { repositories { mavenCentral() }}
21 |
22 | // All common configuration
23 | allprojects {
24 |
25 | ext {
26 | beamVersion = '2.41.0'
27 | protobuf_version = '3.19.4'
28 | autoValueVersion = '1.6.3'
29 | }
30 |
31 | repositories {
32 | mavenCentral()
33 | maven {
34 | url "https://packages.confluent.io/maven"
35 | }
36 | }
37 | wrapper {
38 | gradleVersion = '6.8' //version required
39 | }
40 | }
41 |
42 |
43 |
44 |
45 |
46 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/gradle/wrapper/gradle-wrapper.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/dataflow-sample-applications/beb6e20133939d7f7024c338ec2dc02e84e17475/timeseries-streaming/timeseries-java-applications/gradle/wrapper/gradle-wrapper.jar
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/gradle/wrapper/gradle-wrapper.properties:
--------------------------------------------------------------------------------
1 | distributionBase=GRADLE_USER_HOME
2 | distributionPath=wrapper/dists
3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.8-bin.zip
4 | zipStoreBase=GRADLE_USER_HOME
5 | zipStorePath=wrapper/dists
6 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-java-applications/settings.gradle:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * License); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an AS IS BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | rootProject.name = 'timeseries-java-applications'
19 |
20 | include 'TimeSeriesPipeline'
21 | include 'TimeSeriesMetricsLibrary'
22 | include 'SyntheticExamples'
23 | include 'Examples'
24 |
25 | include 'Adapters'
26 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/__init__.py:
--------------------------------------------------------------------------------
1 | #
2 | # Licensed to the Apache Software Foundation (ASF) under one or more
3 | # contributor license agreements. See the NOTICE file distributed with
4 | # this work for additional information regarding copyright ownership.
5 | # The ASF licenses this file to You under the Apache License, Version 2.0
6 | # (the "License"); you may not use this file except in compliance with
7 | # the License. You may obtain a copy of the License at
8 | #
9 | # http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing, software
12 | # distributed under the License is distributed on an "AS IS" BASIS,
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | # See the License for the specific language governing permissions and
15 | # limitations under the License.
16 | #
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/ml_pipeline/__init__.py:
--------------------------------------------------------------------------------
1 | #
2 | # Licensed to the Apache Software Foundation (ASF) under one or more
3 | # contributor license agreements. See the NOTICE file distributed with
4 | # this work for additional information regarding copyright ownership.
5 | # The ASF licenses this file to You under the Apache License, Version 2.0
6 | # (the "License"); you may not use this file except in compliance with
7 | # the License. You may obtain a copy of the License at
8 | #
9 | # http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing, software
12 | # distributed under the License is distributed on an "AS IS" BASIS,
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | # See the License for the specific language governing permissions and
15 | # limitations under the License.
16 | #
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/ml_pipeline/timeseries/__init__.py:
--------------------------------------------------------------------------------
1 | #
2 | # Licensed to the Apache Software Foundation (ASF) under one or more
3 | # contributor license agreements. See the NOTICE file distributed with
4 | # this work for additional information regarding copyright ownership.
5 | # The ASF licenses this file to You under the Apache License, Version 2.0
6 | # (the "License"); you may not use this file except in compliance with
7 | # the License. You may obtain a copy of the License at
8 | #
9 | # http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing, software
12 | # distributed under the License is distributed on an "AS IS" BASIS,
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | # See the License for the specific language governing permissions and
15 | # limitations under the License.
16 | #
17 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/ml_pipeline/timeseries/encoder_decoder/__init__.py:
--------------------------------------------------------------------------------
1 | # Lint as: python2, python3
2 | # Copyright 2020 Google LLC. All Rights Reserved.
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/ml_pipeline/timeseries/encoder_decoder/encoder_decoder_preprocessing.py:
--------------------------------------------------------------------------------
1 | # Lint as: python2, python3
2 | # Copyright 2020 Google LLC. All Rights Reserved.
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | from typing import Dict, Text, Any
17 | import tensorflow as tf
18 | import tensorflow_transform as tft
19 | from ml_pipeline.timeseries.utils import timeseries_transform_utils
20 |
21 |
22 | def preprocessing_fn(inputs: Dict[Text, Any],
23 | custom_config: Dict[Text, Any]) -> Dict[Text, Any]:
24 | """tf.transform's callback function for preprocessing inputs.
25 |
26 | Args:
27 | inputs: map from feature keys to raw not-yet-transformed features.
28 | custom_config:
29 | timesteps: The number of timesteps in the look back window
30 | features: Which of the features from the TF.Example to use in the model.
31 |
32 | Returns:
33 | Map from string feature key to transformed feature operations.
34 | """
35 | timesteps = custom_config['timesteps']
36 |
37 | outputs = inputs.copy()
38 |
39 | # Generate features to be used in the model
40 | train_x_tensors = timeseries_transform_utils.create_feature_list_from_dict(
41 | outputs, custom_config)
42 |
43 | # Scale the inputs with the exception of TIMESTAMPS
44 |
45 | for key in train_x_tensors:
46 | # TODO provide option for user to enable / disable -Timestamp scale
47 | # if not str(key).endswith('-TIMESTAMP'):
48 | train_x_tensors[key] = tft.scale_to_z_score(train_x_tensors[key])
49 |
50 | train_x_values = [train_x_tensors[k] for k in sorted(train_x_tensors)]
51 |
52 | float32 = tf.reshape(
53 | tf.stack(train_x_values, axis=-1),
54 | [-1, timesteps, len(train_x_values)])
55 |
56 | # Auto Encoder / Decoder requires label == data
57 | outputs = {'Float32': float32, 'LABEL': float32}
58 | return outputs
59 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/ml_pipeline/timeseries/encoder_decoder/transforms/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/dataflow-sample-applications/beb6e20133939d7f7024c338ec2dc02e84e17475/timeseries-streaming/timeseries-python-applications/ml_pipeline/timeseries/encoder_decoder/transforms/__init__.py
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/ml_pipeline/timeseries/pipeline_templates/__init__.py:
--------------------------------------------------------------------------------
1 | # Lint as: python2, python3
2 | # Copyright 2020 Google LLC. All Rights Reserved.
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/ml_pipeline/timeseries/utils/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/dataflow-sample-applications/beb6e20133939d7f7024c338ec2dc02e84e17475/timeseries-streaming/timeseries-python-applications/ml_pipeline/timeseries/utils/__init__.py
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/ml_pipeline_examples/__init__.py:
--------------------------------------------------------------------------------
1 | # Lint as: python2, python3
2 | # Copyright 2020 Google LLC. All Rights Reserved.
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/ml_pipeline_examples/sin_wave_example/__init__.py:
--------------------------------------------------------------------------------
1 | # Lint as: python2, python3
2 | # Copyright 2020 Google LLC. All Rights Reserved.
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/ml_pipeline_examples/sin_wave_example/inference/__init__.py:
--------------------------------------------------------------------------------
1 | # Lint as: python2, python3
2 | # Copyright 2020 Google LLC. All Rights Reserved.
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/ml_pipeline_examples/sin_wave_example/saved_model_example/serving_model_dir/saved_model.pb:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/dataflow-sample-applications/beb6e20133939d7f7024c338ec2dc02e84e17475/timeseries-streaming/timeseries-python-applications/ml_pipeline_examples/sin_wave_example/saved_model_example/serving_model_dir/saved_model.pb
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/ml_pipeline_examples/sin_wave_example/saved_model_example/serving_model_dir/variables/variables.data-00000-of-00001:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/dataflow-sample-applications/beb6e20133939d7f7024c338ec2dc02e84e17475/timeseries-streaming/timeseries-python-applications/ml_pipeline_examples/sin_wave_example/saved_model_example/serving_model_dir/variables/variables.data-00000-of-00001
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/ml_pipeline_examples/sin_wave_example/saved_model_example/serving_model_dir/variables/variables.index:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/dataflow-sample-applications/beb6e20133939d7f7024c338ec2dc02e84e17475/timeseries-streaming/timeseries-python-applications/ml_pipeline_examples/sin_wave_example/saved_model_example/serving_model_dir/variables/variables.index
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/ml_pipeline_examples/sin_wave_example/tf_transform_graph_dir/transform_fn/saved_model.pb:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/dataflow-sample-applications/beb6e20133939d7f7024c338ec2dc02e84e17475/timeseries-streaming/timeseries-python-applications/ml_pipeline_examples/sin_wave_example/tf_transform_graph_dir/transform_fn/saved_model.pb
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/ml_pipeline_examples/sin_wave_example/tf_transform_graph_dir/transformed_metadata/schema.pbtxt:
--------------------------------------------------------------------------------
1 | feature {
2 | name: "Float32"
3 | type: FLOAT
4 | presence {
5 | min_fraction: 1.0
6 | }
7 | shape {
8 | dim {
9 | size: 5
10 | }
11 | dim {
12 | size: 3
13 | }
14 | }
15 | }
16 | feature {
17 | name: "LABEL"
18 | type: FLOAT
19 | presence {
20 | min_fraction: 1.0
21 | }
22 | shape {
23 | dim {
24 | size: 5
25 | }
26 | dim {
27 | size: 3
28 | }
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/ml_pipeline_examples/sin_wave_example/training/__init__.py:
--------------------------------------------------------------------------------
1 | # Lint as: python2, python3
2 | # Copyright 2020 Google LLC. All Rights Reserved.
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/notebooks/img/FILLINGBQ.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/dataflow-sample-applications/beb6e20133939d7f7024c338ec2dc02e84e17475/timeseries-streaming/timeseries-python-applications/notebooks/img/FILLINGBQ.png
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/notebooks/img/FILLINGPANDAS.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/dataflow-sample-applications/beb6e20133939d7f7024c338ec2dc02e84e17475/timeseries-streaming/timeseries-python-applications/notebooks/img/FILLINGPANDAS.png
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/notebooks/img/MA60.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/dataflow-sample-applications/beb6e20133939d7f7024c338ec2dc02e84e17475/timeseries-streaming/timeseries-python-applications/notebooks/img/MA60.png
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/notebooks/img/STDDEV.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GoogleCloudPlatform/dataflow-sample-applications/beb6e20133939d7f7024c338ec2dc02e84e17475/timeseries-streaming/timeseries-python-applications/notebooks/img/STDDEV.png
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/setup.cfg:
--------------------------------------------------------------------------------
1 | #
2 | # Licensed to the Apache Software Foundation (ASF) under one or more
3 | # contributor license agreements. See the NOTICE file distributed with
4 | # this work for additional information regarding copyright ownership.
5 | # The ASF licenses this file to You under the Apache License, Version 2.0
6 | # (the "License"); you may not use this file except in compliance with
7 | # the License. You may obtain a copy of the License at
8 | #
9 | # http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing, software
12 | # distributed under the License is distributed on an "AS IS" BASIS,
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | # See the License for the specific language governing permissions and
15 | # limitations under the License.
16 | #
17 | [tox:tox]
18 | envlist = yapf-check, yapf, lint
19 | toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox}
20 |
21 | # Configuration applicable to all tox environments.
22 | [testenv]
23 | commands_pre =
24 | python --version
25 | pip --version
26 | # TODO: Reenable pip check after dependency conflicts are resolved.
27 | # pip check
28 |
29 | # Don't warn that these commands aren't installed.
30 | whitelist_externals =
31 | echo
32 |
33 | # Autoformatter (yet-another-python-formatter) configuration
34 | [yapf]
35 | indent_width = 4
36 | continuation_indent_width = 8
37 | column_limit = 80
38 | allow_split_before_dict_value = False
39 | blank_line_before_module_docstring = True
40 | coalesce_brackets = True
41 | each_dict_entry_on_separate_line = True
42 | split_all_top_level_comma_separated_values = True
43 | split_arguments_when_comma_terminated = True
44 | split_before_expression_after_opening_paren = True
45 | split_before_first_argument = True
46 | split_before_logical_operator = False
47 |
48 | # Run autoformatter in dry-run mode.
49 | [testenv:yapf-check]
50 | deps =
51 | yapf
52 | commands =
53 | yapf --diff --recursive ml_pipeline/timeseries, ml_pipeline_examples/sin_wave_example
54 | skip_install = True
55 |
56 | [testenv:yapf]
57 | deps =
58 | yapf
59 | commands =
60 | yapf --parallel --recursive --in-place ml_pipeline/timeseries, ml_pipeline_examples/sin_wave_example
61 | skip_install = True
62 |
63 | [testenv:lint]
64 | deps =
65 | flake8
66 | pylint
67 |
68 | commands =
69 | echo "Running pylint..."
70 | pylint ml_pipeline/timeseries
71 | echo "Running flake8..."
72 | flake8 ml_pipeline/timeseries --show-source --statistics
73 |
--------------------------------------------------------------------------------
/timeseries-streaming/timeseries-python-applications/setup.py:
--------------------------------------------------------------------------------
1 | # Lint as: python2, python3
2 | # Copyright 2020 Google LLC. All Rights Reserved.
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | import setuptools
17 |
18 | setuptools.setup(
19 | name='tsflow',
20 | version='0.3.2-sample',
21 | python_requires='>=3.7',
22 | install_requires=[
23 | 'tfx==0.24.0', 'kfp==1.0.3'
24 | ],
25 | packages=setuptools.find_packages()
26 | )
27 |
--------------------------------------------------------------------------------