outputHeaders = new ArrayList<>();
138 | outputHeaders.add(H_ROW_TYPE);
139 | outputHeaders.add(H_ACTION);
140 | outputHeaders.add(H_STATUS);
141 | outputHeaders.addAll(Arrays.asList(headers));
142 | outputHeaders.forEach(header -> values.add(element.get(header)));
143 | String valuesString = String.join(",", values);
144 | c.output(valuesString);
145 | }
146 | }
147 | })).apply("MozartWriteKeywords",
148 | TextIO.write().to(options.getOutputKeywordsFile()).withoutSharding());
149 | }
150 |
151 | }
152 |
--------------------------------------------------------------------------------
/dataflow/src/main/java/com/google/cse/mozart/examples/FirebaseInput.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2018 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
5 | * in compliance with the License. You may obtain a copy of the License at
6 | *
7 | * https://www.apache.org/licenses/LICENSE-2.0
8 | *
9 | * Unless required by applicable law or agreed to in writing, software distributed under the License
10 | * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
11 | * or implied. See the License for the specific language governing permissions and limitations under
12 | * the License.
13 | */
14 | package com.google.cse.mozart.examples;
15 |
16 | import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
17 | import com.google.cse.mozart.Mozart;
18 | import com.google.cse.mozart.Mozart.MozartOptions;
19 | import com.mashape.unirest.http.HttpResponse;
20 | import com.mashape.unirest.http.Unirest;
21 | import com.mashape.unirest.http.exceptions.UnirestException;
22 | import java.io.IOException;
23 | import java.io.InputStream;
24 | import java.util.Arrays;
25 | import java.util.Map;
26 | import org.apache.beam.sdk.Pipeline;
27 | import org.apache.beam.sdk.options.Description;
28 | import org.apache.beam.sdk.options.PipelineOptionsFactory;
29 | import org.apache.beam.sdk.options.ValueProvider;
30 | import org.apache.beam.sdk.transforms.DoFn;
31 | import org.apache.beam.sdk.transforms.PTransform;
32 | import org.apache.beam.sdk.transforms.ParDo;
33 | import org.apache.beam.sdk.values.PCollection;
34 | import org.slf4j.Logger;
35 | import org.slf4j.LoggerFactory;
36 |
37 | /**
38 | * Mozart example pipeline.
39 | *
40 | * This is an example of a Beam pipeline that uses Mozart. This example pulls data from Firebase
41 | * realtime database.
42 | */
43 | public class FirebaseInput {
44 |
45 | // Extend MozartOptions to add your custom options to the pipeline
46 | interface MozartExampleOptions extends MozartOptions {
47 | @Description("Path to the custom input file (customer data table)")
48 | ValueProvider getInputCustomDataFile();
49 |
50 | void setInputCustomDataFile(ValueProvider value);
51 |
52 | @Description("Custom data column names")
53 | ValueProvider getCustomDataColumnNames();
54 |
55 | void setCustomDataColumnNames(ValueProvider value);
56 | }
57 |
58 | private static final Logger LOG = LoggerFactory.getLogger(FirebaseInput.class);
59 |
60 | public static String getFirebaseData(String path) throws UnirestException, IOException {
61 | // Use the Google credential to generate an access token
62 | InputStream credentialsStream = FirebaseInput.class.getResourceAsStream("service_account.json");
63 | GoogleCredential credentials = GoogleCredential.fromStream(credentialsStream);
64 | credentialsStream.close();
65 | final GoogleCredential scopedCredentials =
66 | credentials.createScoped(
67 | Arrays.asList(
68 | "https://www.googleapis.com/auth/firebase.database",
69 | "https://www.googleapis.com/auth/userinfo.email"));
70 | scopedCredentials.refreshToken();
71 | String token = scopedCredentials.getAccessToken();
72 | HttpResponse jsonResponse =
73 | Unirest.get("https://fir-test-a5923.firebaseio.com" + path)
74 | .queryString("access_token", token)
75 | .asString();
76 | return jsonResponse.getBody();
77 | }
78 |
79 | public static void main(String[] args) throws IOException {
80 |
81 | // Build your Pipeline as usual
82 | PipelineOptionsFactory.register(MozartOptions.class);
83 | MozartExampleOptions options =
84 | PipelineOptionsFactory.fromArgs(args).withValidation().as(MozartExampleOptions.class);
85 | options.getCustomDataColumnNames().isAccessible();
86 | options.getInputCustomDataFile().isAccessible();
87 | Pipeline p = Pipeline.create(options);
88 |
89 | // Use the Google credential to generate an access token
90 |
91 | // Define your PTransform. Here is where your business logic is implemented
92 | PTransform super PCollection