outputHeaders = new ArrayList<>();
138 | outputHeaders.add(H_ROW_TYPE);
139 | outputHeaders.add(H_ACTION);
140 | outputHeaders.add(H_STATUS);
141 | outputHeaders.addAll(Arrays.asList(headers));
142 | outputHeaders.forEach(header -> values.add(element.get(header)));
143 | String valuesString = String.join(",", values);
144 | c.output(valuesString);
145 | }
146 | }
147 | })).apply("MozartWriteKeywords",
148 | TextIO.write().to(options.getOutputKeywordsFile()).withoutSharding());
149 | }
150 |
151 | }
152 |
--------------------------------------------------------------------------------
/dataflow/src/main/java/com/google/cse/mozart/examples/FirebaseInput.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2018 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
5 | * in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * https://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the License
10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
11 | * or implied. See the License for the specific language governing permissions and limitations under
12 | * the License.
13 | */
14 | package com.google.cse.mozart.examples;
15 |
16 | import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
17 | import com.google.cse.mozart.Mozart;
18 | import com.google.cse.mozart.Mozart.MozartOptions;
19 | import com.mashape.unirest.http.HttpResponse;
20 | import com.mashape.unirest.http.Unirest;
21 | import com.mashape.unirest.http.exceptions.UnirestException;
22 | import java.io.IOException;
23 | import java.io.InputStream;
24 | import java.util.Arrays;
25 | import java.util.Map;
26 | import org.apache.beam.sdk.Pipeline;
27 | import org.apache.beam.sdk.options.Description;
28 | import org.apache.beam.sdk.options.PipelineOptionsFactory;
29 | import org.apache.beam.sdk.options.ValueProvider;
30 | import org.apache.beam.sdk.transforms.DoFn;
31 | import org.apache.beam.sdk.transforms.PTransform;
32 | import org.apache.beam.sdk.transforms.ParDo;
33 | import org.apache.beam.sdk.values.PCollection;
34 | import org.slf4j.Logger;
35 | import org.slf4j.LoggerFactory;
36 |
37 | /**
38 | * Mozart example pipeline.
39 | *
40 | * This is an example of a Beam pipeline that uses Mozart. This example pulls data from Firebase
41 | * realtime database.
42 | */
43 | public class FirebaseInput {
44 |
45 | // Extend MozartOptions to add your custom options to the pipeline
46 | interface MozartExampleOptions extends MozartOptions {
47 | @Description("Path to the custom input file (customer data table)")
48 | ValueProvider getInputCustomDataFile();
49 |
50 | void setInputCustomDataFile(ValueProvider value);
51 |
52 | @Description("Custom data column names")
53 | ValueProvider getCustomDataColumnNames();
54 |
55 | void setCustomDataColumnNames(ValueProvider value);
56 | }
57 |
58 | private static final Logger LOG = LoggerFactory.getLogger(FirebaseInput.class);
59 |
60 | public static String getFirebaseData(String path) throws UnirestException, IOException {
61 | // Use the Google credential to generate an access token
62 | InputStream credentialsStream = FirebaseInput.class.getResourceAsStream("service_account.json");
63 | GoogleCredential credentials = GoogleCredential.fromStream(credentialsStream);
64 | credentialsStream.close();
65 | final GoogleCredential scopedCredentials =
66 | credentials.createScoped(
67 | Arrays.asList(
68 | "https://www.googleapis.com/auth/firebase.database",
69 | "https://www.googleapis.com/auth/userinfo.email"));
70 | scopedCredentials.refreshToken();
71 | String token = scopedCredentials.getAccessToken();
72 | HttpResponse jsonResponse =
73 | Unirest.get("https://fir-test-a5923.firebaseio.com" + path)
74 | .queryString("access_token", token)
75 | .asString();
76 | return jsonResponse.getBody();
77 | }
78 |
79 | public static void main(String[] args) throws IOException {
80 |
81 | // Build your Pipeline as usual
82 | PipelineOptionsFactory.register(MozartOptions.class);
83 | MozartExampleOptions options =
84 | PipelineOptionsFactory.fromArgs(args).withValidation().as(MozartExampleOptions.class);
85 | options.getCustomDataColumnNames().isAccessible();
86 | options.getInputCustomDataFile().isAccessible();
87 | Pipeline p = Pipeline.create(options);
88 |
89 | // Use the Google credential to generate an access token
90 |
91 | // Define your PTransform. Here is where your business logic is implemented
92 | PTransform>, PCollection>>
93 | businessTransform =
94 | ParDo.of(
95 | new DoFn, Map>() {
96 | @ProcessElement
97 | public void processElement(ProcessContext c) {
98 | final Map element = c.element();
99 | // Skip if element is empty
100 | if (element.size() > 0) {
101 | try {
102 | String newMaxCPC = getFirebaseData("/maxCPC.json");
103 | LOG.info("newMaxCPC from Firebase: {}", newMaxCPC);
104 | element.put("Keyword max CPC", newMaxCPC);
105 | } catch (Exception e) {
106 | LOG.error("Error while processing element", e);
107 | }
108 | c.output(element);
109 | }
110 | }
111 | });
112 |
113 | // Use Mozart to complete the pipeline
114 | // First, get the PCollection with all the keywords
115 | PCollection> keywordsBeforeLogic = Mozart.getKeywords(options, p);
116 |
117 | // Then, apply your business logic over the keywords
118 | PCollection> keywordsAfterLogic =
119 | keywordsBeforeLogic.apply(businessTransform);
120 |
121 | // Lastly, use Mozart to write the keywords output
122 | Mozart.writeKeywordsOutput(options, keywordsAfterLogic);
123 |
124 | // Run your pipeline as usual
125 | p.run();
126 | }
127 | }
128 |
--------------------------------------------------------------------------------
/dataflow/src/main/java/com/google/cse/mozart/examples/GCSInput.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2018 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
5 | * in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * https://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the License
10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
11 | * or implied. See the License for the specific language governing permissions and limitations under
12 | * the License.
13 | */
14 | package com.google.cse.mozart.examples;
15 |
16 | import com.google.cse.mozart.Mozart;
17 | import com.google.cse.mozart.Mozart.MozartOptions;
18 | import java.util.HashMap;
19 | import java.util.List;
20 | import java.util.Map;
21 | import org.apache.beam.sdk.Pipeline;
22 | import org.apache.beam.sdk.io.TextIO;
23 | import org.apache.beam.sdk.options.Description;
24 | import org.apache.beam.sdk.options.PipelineOptionsFactory;
25 | import org.apache.beam.sdk.options.ValueProvider;
26 | import org.apache.beam.sdk.transforms.DoFn;
27 | import org.apache.beam.sdk.transforms.PTransform;
28 | import org.apache.beam.sdk.transforms.ParDo;
29 | import org.apache.beam.sdk.transforms.View;
30 | import org.apache.beam.sdk.values.PCollection;
31 | import org.apache.beam.sdk.values.PCollectionView;
32 | import org.slf4j.Logger;
33 | import org.slf4j.LoggerFactory;
34 |
35 | /**
36 | * Mozart example pipeline.
37 | *
38 | * This is an example of a Beam pipeline that uses Mozart
39 | */
40 | public class GCSInput {
41 |
42 | // Extend MozartOptions to add your custom options to the pipeline
43 | interface MozartExampleOptions extends MozartOptions {
44 | @Description("Path to the custom input file (customer data table)")
45 | ValueProvider getInputCustomDataFile();
46 |
47 | void setInputCustomDataFile(ValueProvider value);
48 |
49 | @Description("Custom data column names")
50 | ValueProvider getCustomDataColumnNames();
51 |
52 | void setCustomDataColumnNames(ValueProvider value);
53 | }
54 |
55 | private static final Logger LOG = LoggerFactory.getLogger(GCSInput.class);
56 |
57 | public static void main(String[] args) {
58 |
59 | // Build your Pipeline as usual
60 | PipelineOptionsFactory.register(MozartOptions.class);
61 | MozartExampleOptions options =
62 | PipelineOptionsFactory.fromArgs(args).withValidation().as(MozartExampleOptions.class);
63 | options.getCustomDataColumnNames().isAccessible();
64 | Pipeline p = Pipeline.create(options);
65 |
66 | // Load additional custom (non-SA360) data. For example: a file containing information about
67 | // whether a certain brand is in promotion state
68 | PCollection> customData =
69 | p.apply("ReadCustomData", TextIO.read().from(options.getInputCustomDataFile()))
70 | // Create dictionary
71 | .apply(
72 | "CreateCustomDataDict",
73 | ParDo.of(
74 | new DoFn>() {
75 | @ProcessElement
76 | public void processElement(ProcessContext c) {
77 | String[] element = c.element().split(",");
78 | String[] headers =
79 | c.getPipelineOptions()
80 | .as(MozartExampleOptions.class)
81 | .getCustomDataColumnNames()
82 | .get()
83 | .split(",");
84 | Map newOutput = new HashMap<>();
85 | if (element.length == headers.length) {
86 | for (int i = 0; i < headers.length; i++) {
87 | newOutput.put(headers[i], element[i]);
88 | }
89 | c.output(newOutput);
90 | } else {
91 | LOG.warn(
92 | "Different length for headers and element. header: {}. element: {}",
93 | headers,
94 | element);
95 | }
96 | }
97 | }));
98 |
99 | // Create a view of your custom data (this is necessary to pass it as side input to your
100 | // main PTransform
101 | PCollectionView>> customDataView =
102 | customData.apply("CreateCustomDataView", View.asList());
103 |
104 | final String maxCPCPromo = "5";
105 | final String maxCPCNormal = "1";
106 |
107 | // Define your PTransform. Here is where your business logic is implemented
108 | PTransform>, PCollection>>
109 | businessTransform =
110 | ParDo.of(
111 | new DoFn, Map>() {
112 | @ProcessElement
113 | public void processElement(ProcessContext c) {
114 | List> customData = c.sideInput(customDataView);
115 | final Map element = c.element();
116 | // Skip if element is empty
117 | if (element.size() > 0) {
118 | String newMaxCPC = maxCPCNormal;
119 | for (Map customEntry : customData) {
120 | if (element.get("Keyword").contains(customEntry.get("brand"))) {
121 | if (customEntry.get("status").equals("promotion")) {
122 | newMaxCPC = maxCPCPromo;
123 | }
124 | }
125 | }
126 | element.put("Keyword max CPC", newMaxCPC);
127 | c.output(element);
128 | }
129 | }
130 | })
131 | .withSideInputs(customDataView);
132 |
133 | // Use Mozart to complete the pipeline
134 | // First, get the PCollection with all the keywords
135 | PCollection> keywordsBeforeLogic = Mozart.getKeywords(options, p);
136 |
137 | // Then, apply your business logic over the keywords
138 | PCollection> keywordsAfterLogic =
139 | keywordsBeforeLogic.apply(businessTransform);
140 |
141 | // Lastly, use Mozart to write the keywords output
142 | Mozart.writeKeywordsOutput(options, keywordsAfterLogic);
143 |
144 | // Run your pipeline as usual
145 | p.run();
146 | }
147 | }
148 |
--------------------------------------------------------------------------------
/doc/images/environment-set0.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/mozart/21cdfef70a8acb8814d17575f458c53285f7f51a/doc/images/environment-set0.png
--------------------------------------------------------------------------------
/doc/images/environment-set1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/mozart/21cdfef70a8acb8814d17575f458c53285f7f51a/doc/images/environment-set1.png
--------------------------------------------------------------------------------
/doc/images/environment-set2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/mozart/21cdfef70a8acb8814d17575f458c53285f7f51a/doc/images/environment-set2.png
--------------------------------------------------------------------------------
/doc/images/environment-set3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/mozart/21cdfef70a8acb8814d17575f458c53285f7f51a/doc/images/environment-set3.png
--------------------------------------------------------------------------------
/doc/images/environment-set4.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/mozart/21cdfef70a8acb8814d17575f458c53285f7f51a/doc/images/environment-set4.png
--------------------------------------------------------------------------------
/doc/images/environment-set5.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/mozart/21cdfef70a8acb8814d17575f458c53285f7f51a/doc/images/environment-set5.png
--------------------------------------------------------------------------------
/doc/images/environment-set6.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/mozart/21cdfef70a8acb8814d17575f458c53285f7f51a/doc/images/environment-set6.png
--------------------------------------------------------------------------------