) options.valuesOf(EMAIL_RECIPIENTS);
114 | strategy = TransportStrategy.valueOf((String) options.valueOf(EMAIL_SMTP_STRATEGY));
115 | return true;
116 | }
117 |
118 | /**
119 | * {@inheritDoc}
120 | *
121 | * Render the report HTML using Jtwig and send the result to the recipient emails.
122 | */
123 | @Override
124 | public void writeReport(List testSuites) throws IOException {
125 | if (testSuites == null) {
126 | testSuites = Collections.emptyList();
127 | }
128 | log.info("Sending report email for {} test suites", testSuites.size());
129 | List testList = new ArrayList<>(testSuites.size());
130 | boolean hasError = false;
131 | for (TestSuite testSuite : testSuites) {
132 | TestSuiteModel testSuiteModel = new TestSuiteModel(testSuite);
133 | hasError = hasError || !testSuiteModel.allPassed();
134 | testList.add(testSuiteModel);
135 | }
136 | JtwigTemplate template = JtwigTemplate.classpathTemplate(TWIG_TEMPLATE);
137 | JtwigModel model = JtwigModel.newModel()
138 | .with(TWIG_ERROR_PARAM, hasError)
139 | .with(TWIG_TEST_LIST_PARAM, testList);
140 | String reportHtml = template.render(model);
141 | EmailBuilder emailBuilder =
142 | new EmailBuilder().from(senderName, fromEmail)
143 | .replyTo(senderName, replyTo)
144 | .subject(subjectPrefix + (hasError ? SUBJECT_FAILURE_SUFFIX : SUBJECT_SUCCESS_SUFFIX))
145 | .addHeader(X_PRIORITY, PRIORITY) .textHTML(reportHtml);
146 | for (String recipientEmail : recipientEmails) {
147 | log.info("Emailing {}", recipientEmail);
148 | emailBuilder.to(recipientEmail);
149 | }
150 | Email reportEmail = emailBuilder.build();
151 | ServerConfig mailServerConfig = new ServerConfig(smtpHost, smtpPort);
152 | Mailer reportMailer = new Mailer(mailServerConfig, strategy);
153 | sendEmail(reportMailer, reportEmail);
154 | log.info("Finished sending report to recipients");
155 | }
156 |
157 | /**
158 | * Method uses the provided mailer to send the email report.
159 | *
160 | * @param mailer mailer to use
161 | * @param email report email
162 | */
163 | protected void sendEmail(Mailer mailer, Email email) {
164 | mailer.sendMail(email);
165 | }
166 |
167 | @Override
168 | public String getName() {
169 | return EMAIL_FORMATTER;
170 | }
171 |
172 | @Override
173 | public void printHelp() {
174 | Helpable.printHelp("Email report options", PARSER);
175 | }
176 | }
177 |
--------------------------------------------------------------------------------
/src/main/java/com/yahoo/validatar/execution/fixed/DSV.java:
--------------------------------------------------------------------------------
1 | package com.yahoo.validatar.execution.fixed;
2 |
3 | import com.yahoo.validatar.common.Helpable;
4 | import com.yahoo.validatar.common.Query;
5 | import com.yahoo.validatar.common.Result;
6 | import com.yahoo.validatar.common.TypeSystem;
7 | import com.yahoo.validatar.common.TypeSystem.Type;
8 | import com.yahoo.validatar.common.TypedObject;
9 | import com.yahoo.validatar.execution.Engine;
10 | import joptsimple.OptionParser;
11 | import joptsimple.OptionSet;
12 | import lombok.extern.slf4j.Slf4j;
13 | import org.apache.commons.csv.CSVFormat;
14 | import org.apache.commons.csv.CSVParser;
15 | import org.apache.commons.csv.CSVRecord;
16 | import org.apache.commons.lang3.StringEscapeUtils;
17 |
18 | import java.io.ByteArrayInputStream;
19 | import java.io.File;
20 | import java.io.FileInputStream;
21 | import java.io.InputStream;
22 | import java.io.InputStreamReader;
23 | import java.io.Reader;
24 | import java.nio.charset.StandardCharsets;
25 | import java.util.HashMap;
26 | import java.util.Map;
27 |
28 | @Slf4j
29 | public class DSV implements Engine {
30 | public static final String ENGINE_NAME = "csv";
31 |
32 | public static final String CSV_DELIMITER = "csv-delimiter";
33 | public static final String METADATA_DELIMITER_KEY = "delimiter";
34 |
35 | public static final String DEFAULT_TYPE = Type.STRING.name();
36 | public static final String DEFAULT_DELIMITER = ",";
37 |
38 | private String defaultDelimiter;
39 |
40 | private final OptionParser parser = new OptionParser() {
41 | {
42 | accepts(CSV_DELIMITER, "The delimiter to use while parsing fields within a record. Defaults to ',' or CSV")
43 | .withRequiredArg()
44 | .describedAs("The field delimiter")
45 | .defaultsTo(DEFAULT_DELIMITER);
46 | allowsUnrecognizedOptions();
47 | }
48 | };
49 |
50 | @Override
51 | public boolean setup(String[] arguments) {
52 | OptionSet options = parser.parse(arguments);
53 | defaultDelimiter = (String) options.valueOf(CSV_DELIMITER);
54 | return true;
55 | }
56 |
57 | @Override
58 | public void execute(Query query) {
59 | String queryName = query.name;
60 | String queryValue = query.value.trim();
61 | log.info("Running {}", queryName);
62 |
63 | Map metadata = query.getMetadata();
64 |
65 | String delimiter = Query.getKey(metadata, METADATA_DELIMITER_KEY).orElse(defaultDelimiter);
66 | char character = delimiter.charAt(0);
67 | log.info("Using delimiter as a character: {}", character);
68 |
69 | boolean isFile = isPath(queryValue);
70 |
71 | log.info("Running or loading data from \n{}", queryValue);
72 |
73 | try (InputStream stream = isFile ? new FileInputStream(queryValue) : new ByteArrayInputStream(getBytes(queryValue));
74 | Reader reader = new InputStreamReader(stream)) {
75 |
76 | CSVFormat format = CSVFormat.RFC4180.withDelimiter(character).withFirstRecordAsHeader().withIgnoreSurroundingSpaces();
77 | CSVParser parser = new CSVParser(reader, format);
78 |
79 | Map headerIndices = parser.getHeaderMap();
80 | Map typeMap = getTypeMapping(headerIndices, metadata);
81 | Result result = query.createResults();
82 | parser.iterator().forEachRemaining(r -> addRow(r, result, typeMap));
83 | } catch (Exception e) {
84 | log.error("Error while parsing data for {} with {}", queryName, queryValue);
85 | log.error("Error", e);
86 | query.setFailure(e.toString());
87 | }
88 | }
89 |
90 | private static void addRow(CSVRecord record, Result result, Map header) {
91 | if (!record.isConsistent()) {
92 | log.warn("Record does not have the same number of fields as the header mapping. Skipping record: {}", record);
93 | return;
94 | }
95 | for (Map.Entry field : header.entrySet()) {
96 | String fieldName = field.getKey();
97 | TypedObject fieldValue = getTyped(field.getValue(), record.get(fieldName));
98 | result.addColumnRow(fieldName, fieldValue);
99 | }
100 | }
101 |
102 | private static TypedObject getTyped(Type type, String field) {
103 | // Can't be null
104 | if (type != Type.STRING && field.isEmpty()) {
105 | log.warn("Found an empty value for a non-string field of type {}. Nulled it. Asserts that use this may fail.", type);
106 | return null;
107 | }
108 | return TypeSystem.cast(type, new TypedObject(field, Type.STRING));
109 | }
110 |
111 | /**
112 | * Gets a mapping of the {@link Type} for the columns in the data.
113 | *
114 | * @param headerIndices The {@link Map} of header field names to their positions.
115 | * @param metadata The metadata of the query viewed as a {@link Map}.
116 | * @return The {@link Map} of column names to their types.
117 | */
118 | static Map getTypeMapping(Map headerIndices, Map metadata) {
119 | if (headerIndices == null) {
120 | log.error("No header row found. The first row in your data needs to be a header row.");
121 | throw new RuntimeException("Header row not found for data. First row in data needs to be a header");
122 | }
123 |
124 | Map typeMap = new HashMap<>();
125 | for (String column : headerIndices.keySet()) {
126 | String typeMapping = Query.getKey(metadata, column).orElse(DEFAULT_TYPE);
127 | try {
128 | Type type = Type.valueOf(typeMapping);
129 | typeMap.put(column, type);
130 | } catch (IllegalArgumentException iae) {
131 | log.error("Unable to find type {}. Using STRING instead. Valid values are {}", typeMapping, Type.values());
132 | typeMap.put(column, Type.STRING);
133 | }
134 | }
135 | return typeMap;
136 | }
137 |
138 | private static byte[] getBytes(String string) {
139 | String unescaped = StringEscapeUtils.unescapeJava(string);
140 | return unescaped.getBytes(StandardCharsets.UTF_8);
141 | }
142 |
143 | private static boolean isPath(String string) {
144 | File file = new File(string);
145 | return file.exists() && file.isFile();
146 | }
147 |
148 | @Override
149 | public String getName() {
150 | return ENGINE_NAME;
151 | }
152 |
153 | @Override
154 | public void printHelp() {
155 | Helpable.printHelp("CSV Engine options", parser);
156 | System.out.println("This Engine lets you load delimited text data from files or to specify it directly as a query.");
157 | System.out.println("It follows the RFC 4180 CSV specification: https://tools.ietf.org/html/rfc4180\n");
158 | System.out.println("Your data MUST contain a header row naming your columns.");
159 | System.out.println("The types of all fields will be inferred as STRINGS. However, you can provide mappings ");
160 | System.out.println("for each column name by adding entries to the metadata section of the query, where");
161 | System.out.println("the key is the name of your column and the value is the type of the column.");
162 | System.out.println("The values can be BOOLEAN, STRING, LONG, DECIMAL, DOUBLE, and TIMESTAMP.");
163 | System.out.println("DECIMAL is used for really large numbers that cannot fit inside a long (2^63). TIMESTAMP is");
164 | System.out.println("used to interpret a whole number as a timestamp field - millis from epoch. Use to load dates.");
165 | System.out.println("This engine primarily exists to let you easily load expected data in as a dataset. You can");
166 | System.out.println("then use the data by joining it with some other data and performing asserts on the joined");
167 | System.out.println("dataset.");
168 | }
169 |
170 | }
171 |
--------------------------------------------------------------------------------
/src/test/java/com/yahoo/validatar/execution/fixed/DSVTest.java:
--------------------------------------------------------------------------------
1 | package com.yahoo.validatar.execution.fixed;
2 |
3 | import com.yahoo.validatar.common.Column;
4 | import com.yahoo.validatar.common.Metadata;
5 | import com.yahoo.validatar.common.Query;
6 | import com.yahoo.validatar.common.Result;
7 | import com.yahoo.validatar.common.TypeSystem.Type;
8 | import org.testng.Assert;
9 | import org.testng.annotations.BeforeMethod;
10 | import org.testng.annotations.Test;
11 |
12 | import java.io.IOException;
13 | import java.math.BigDecimal;
14 | import java.util.ArrayList;
15 | import java.util.HashMap;
16 | import java.util.Map;
17 |
18 | import static com.yahoo.validatar.OutputCaptor.runWithoutOutput;
19 | import static com.yahoo.validatar.TestHelpers.asColumn;
20 | import static com.yahoo.validatar.TestHelpers.getQueryFrom;
21 | import static com.yahoo.validatar.TestHelpers.isEqual;
22 | import static java.util.Collections.singletonMap;
23 |
24 | public class DSVTest {
25 | private final String[] defaults = {"--csv-delimiter", ","};
26 | private DSV dsv;
27 |
28 | @BeforeMethod
29 | public void setup() {
30 | dsv = new DSV();
31 | dsv.setup(defaults);
32 | }
33 |
34 | @Test
35 | public void testDefaults() {
36 | Assert.assertTrue(dsv.setup(new String[0]));
37 | Assert.assertEquals(dsv.getName(), DSV.ENGINE_NAME);
38 | runWithoutOutput(dsv::printHelp);
39 | }
40 |
41 | @Test
42 | public void testEmptyQuery() {
43 | Query query = new Query();
44 | query.value = "";
45 | dsv.execute(query);
46 |
47 | Assert.assertFalse(query.failed());
48 | Assert.assertEquals(query.getResult().numberOfRows(), 0);
49 | Assert.assertEquals(query.getResult().getColumns().size(), 0);
50 | }
51 |
52 | @Test
53 | public void testMissingValuesForStrings() {
54 | Query query = new Query();
55 | query.value = "foo,bar\n0,1\n,3";
56 | dsv.execute(query);
57 |
58 | Assert.assertFalse(query.failed());
59 | Column foo = query.getResult().getColumn("foo");
60 | Column bar = query.getResult().getColumn("bar");
61 | Assert.assertEquals(foo.get(0).data, "0");
62 | Assert.assertEquals(foo.get(1).data, "");
63 | Assert.assertEquals(bar.get(0).data, "1");
64 | Assert.assertEquals(bar.get(1).data, "3");
65 | }
66 |
67 | @Test
68 | public void testMissingValuesForNonStrings() {
69 | Query query = new Query();
70 |
71 | query.metadata = new ArrayList<>();
72 | Metadata metadata = new Metadata();
73 | metadata.key = "foo";
74 | metadata.value = Type.LONG.name();
75 | query.metadata.add(metadata);
76 |
77 | query.value = "foo,bar\n0,1\n,3";
78 |
79 | dsv.execute(query);
80 |
81 | Assert.assertFalse(query.failed());
82 | Column foo = query.getResult().getColumn("foo");
83 | Column bar = query.getResult().getColumn("bar");
84 | Assert.assertEquals(foo.get(0).data, 0L);
85 | Assert.assertNull(foo.get(1));
86 | Assert.assertEquals(bar.get(0).data, "1");
87 | Assert.assertEquals(bar.get(1).data, "3");
88 | }
89 |
90 | @Test
91 | public void testDuplicateHeader() {
92 | Query query = new Query();
93 | query.value = ",";
94 | dsv.execute(query);
95 |
96 | Assert.assertTrue(query.failed());
97 | Assert.assertTrue(query.getMessages().get(0).contains("header contains a duplicate"));
98 | }
99 |
100 | @Test
101 | public void testBadPath() {
102 | Query query = new Query();
103 | query.value = "src/test/resources/csv-tests";
104 | dsv.execute(query);
105 |
106 | Assert.assertFalse(query.failed());
107 | Assert.assertEquals(query.getResult().numberOfRows(), 0);
108 | Assert.assertEquals(query.getResult().getColumns().size(), 0);
109 | }
110 |
111 | @Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = ".*Header row not found.*")
112 | public void testNoHeader() {
113 | DSV.getTypeMapping(null, null);
114 | }
115 |
116 | @Test
117 | public void testTypeMapping() {
118 | Map headers = new HashMap<>();
119 | headers.put("foo", 0);
120 | headers.put("bar", 1);
121 | headers.put("baz", 2);
122 | headers.put("qux", 3);
123 |
124 | Map mapping = new HashMap<>();
125 | mapping.put("foo", "DECIMAL");
126 | mapping.put("baz", "TIMESTAMP");
127 |
128 | Map result = DSV.getTypeMapping(headers, mapping);
129 | Assert.assertEquals(result.get("foo"), Type.DECIMAL);
130 | Assert.assertEquals(result.get("bar"), Type.STRING);
131 | Assert.assertEquals(result.get("baz"), Type.TIMESTAMP);
132 | Assert.assertEquals(result.get("qux"), Type.STRING);
133 | }
134 |
135 | @Test
136 | public void testForcedDefaultTypeMapping() {
137 | Map result = DSV.getTypeMapping(singletonMap("foo", 0), singletonMap("foo", "GARBAGE"));
138 | Assert.assertEquals(result.get("foo"), Type.STRING);
139 | }
140 |
141 | @Test
142 | public void testStringLoading() throws IOException {
143 | Query query = getQueryFrom("csv-tests/sample.yaml", "StringTest");
144 | dsv.execute(query);
145 |
146 | Result actual = query.getResult();
147 | Result expected = new Result("StringTest");
148 | expected.addColumn("A", asColumn(Type.STRING, "foo", "baz", "foo"));
149 | expected.addColumn("B", asColumn(Type.DOUBLE, 234.3, 9.0, 42.0));
150 | expected.addColumn("C", asColumn(Type.STRING, "bar", "qux", "norf"));
151 |
152 | Assert.assertTrue(isEqual(actual, expected));
153 | }
154 |
155 | @Test
156 | public void testFileLoading() throws IOException {
157 | Query query = getQueryFrom("csv-tests/sample.yaml", "FileLoadingTest");
158 | dsv.execute(query);
159 |
160 | Result actual = query.getResult();
161 | Result expected = new Result("FileLoadingTest");
162 | // We should have skipped a row that contained more fields than the header
163 | // We should have nulled out a field that was missing a value but was not a STRING.
164 | // We should have used an empty STRING for a missing field that was of type STRING
165 | expected.addColumn("fieldA", asColumn(Type.STRING, "foo", "baz", "qux", "bar"));
166 | expected.addColumn("fieldB", asColumn(Type.LONG, 15L, 42L, 0L, null));
167 | expected.addColumn("fieldC", asColumn(Type.DECIMAL, new BigDecimal("34.2"), new BigDecimal("4.2"),
168 | new BigDecimal("8.4"), new BigDecimal("0.2")));
169 | expected.addColumn("fieldD", asColumn(Type.STRING, "bar", "bar", "norf", ""));
170 |
171 | Assert.assertTrue(isEqual(actual, expected));
172 | }
173 |
174 | @Test
175 | public void testCustomDelimiterASCII() throws IOException {
176 | Query query = getQueryFrom("csv-tests/sample.yaml", "CustomDelimASCII");
177 | dsv.execute(query);
178 |
179 | Result actual = query.getResult();
180 | Result expected = new Result("CustomDelimASCII");
181 | expected.addColumn("A", asColumn(Type.STRING, "foo", "baz", "foo"));
182 | expected.addColumn("B", asColumn(Type.STRING, "234.3", "9", "42"));
183 | expected.addColumn("C", asColumn(Type.STRING, "bar", "qux", "norf"));
184 |
185 | Assert.assertTrue(isEqual(actual, expected));
186 | }
187 |
188 | @Test
189 | public void testCustomDelimiterUnicode() throws IOException {
190 | Query query = getQueryFrom("csv-tests/sample.yaml", "CustomDelimUnicode");
191 | dsv.execute(query);
192 |
193 | Result actual = query.getResult();
194 | Result expected = new Result("CustomDelimUnicode");
195 | expected.addColumn("A", asColumn(Type.STRING, "foo", "baz", "foo"));
196 | expected.addColumn("B", asColumn(Type.STRING, "234.3", "9", "42"));
197 | expected.addColumn("C", asColumn(Type.STRING, "bar", "qux", "norf"));
198 |
199 | Assert.assertTrue(isEqual(actual, expected));
200 | }
201 | }
202 |
--------------------------------------------------------------------------------
/src/main/java/com/yahoo/validatar/execution/EngineManager.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2015 Yahoo Inc.
3 | * Licensed under the terms of the Apache 2 license. Please see LICENSE file in the project root for terms.
4 | */
5 | package com.yahoo.validatar.execution;
6 |
7 | import com.yahoo.validatar.common.Helpable;
8 | import com.yahoo.validatar.common.Pluggable;
9 | import com.yahoo.validatar.common.Query;
10 | import com.yahoo.validatar.execution.fixed.DSV;
11 | import com.yahoo.validatar.execution.hive.Apiary;
12 | import com.yahoo.validatar.execution.pig.Sty;
13 | import com.yahoo.validatar.execution.rest.JSON;
14 | import joptsimple.OptionParser;
15 | import joptsimple.OptionSet;
16 | import lombok.extern.slf4j.Slf4j;
17 |
18 | import java.util.Arrays;
19 | import java.util.Collections;
20 | import java.util.HashMap;
21 | import java.util.List;
22 | import java.util.Map;
23 | import java.util.TreeMap;
24 | import java.util.concurrent.ForkJoinPool;
25 | import java.util.stream.Collectors;
26 |
27 | /**
28 | * Manages the creation and execution of execution engines.
29 | */
30 | @Slf4j
31 | public class EngineManager extends Pluggable implements Helpable {
32 | public static final String CUSTOM_ENGINE = "custom-engine";
33 | public static final String CUSTOM_ENGINE_DESCRIPTION = "Additional custom engine to load.";
34 | public static final String QUERY_PARALLEL_ENABLE = "query-parallel-enable";
35 | public static final String QUERY_PARALLEL_MAX = "query-parallel-max";
36 | private static final int QUERY_PARALLEL_MIN = 1;
37 |
38 | protected boolean queryParallelEnable;
39 | protected int queryParallelMax;
40 |
41 | private static final OptionParser PARSER = new OptionParser() {
42 | {
43 | accepts(QUERY_PARALLEL_ENABLE, "Whether or not queries should run in parallel.")
44 | .withRequiredArg()
45 | .describedAs("Query parallelism option")
46 | .ofType(Boolean.class)
47 | .defaultsTo(false);
48 | accepts(QUERY_PARALLEL_MAX, "The max number of queries that will run concurrently. If non-positive or " +
49 | "unspecified, all queries will run at once.")
50 | .withRequiredArg()
51 | .describedAs("Max query parallelism")
52 | .ofType(Integer.class)
53 | .defaultsTo(0);
54 | allowsUnrecognizedOptions();
55 | }
56 | };
57 |
58 | /**
59 | * The Engine classes to manage.
60 | */
61 | public static final List> MANAGED_ENGINES = Arrays.asList(Apiary.class, Sty.class, JSON.class, DSV.class);
62 |
63 | /**
64 | * Stores the CLI arguments.
65 | */
66 | protected String[] arguments;
67 |
68 | /**
69 | * Stores engine names to engine references.
70 | */
71 | protected Map engines;
72 |
73 | /**
74 | * A simple wrapper to mark an engine as started.
75 | */
76 | protected class WorkingEngine {
77 | public boolean isStarted = false;
78 | private Engine engine = null;
79 |
80 | /**
81 | * Constructor.
82 | *
83 | * @param engine The engine to wrap.
84 | */
85 | public WorkingEngine(Engine engine) {
86 | this.engine = engine;
87 | }
88 |
89 | /**
90 | * Getter.
91 | *
92 | * @return The wrapped Engine.
93 | */
94 | public Engine getEngine() {
95 | return this.engine;
96 | }
97 | }
98 |
99 | /**
100 | * Store arguments and create the engine map.
101 | *
102 | * @param arguments CLI arguments.
103 | */
104 | public EngineManager(String[] arguments) {
105 | super(MANAGED_ENGINES, CUSTOM_ENGINE, CUSTOM_ENGINE_DESCRIPTION);
106 |
107 | this.arguments = arguments;
108 |
109 | // Create the engines map, engine name -> engine
110 | engines = new HashMap<>();
111 | for (Engine engine : getPlugins(arguments)) {
112 | engines.put(engine.getName(), new WorkingEngine(engine));
113 | log.info("Added engine {} to list of engines.", engine.getName());
114 | }
115 |
116 | OptionSet parser = PARSER.parse(arguments);
117 | queryParallelEnable = (Boolean) parser.valueOf(QUERY_PARALLEL_ENABLE);
118 | queryParallelMax = (Integer) parser.valueOf(QUERY_PARALLEL_MAX);
119 | }
120 |
121 | /**
122 | * For testing purposes to inject engines. Will always override existing engines.
123 | *
124 | * @param engines A list of engines to use as the engines to work with.
125 | */
126 | void setEngines(List engines) {
127 | List all = engines == null ? Collections.emptyList() : engines;
128 | this.engines = all.stream().collect(Collectors.toMap(Engine::getName, WorkingEngine::new));
129 | }
130 |
131 | /**
132 | * For a list of queries, start corresponding engines.
133 | *
134 | * @param queries Queries to check for engine support.
135 | * @return true iff the required engines were loaded.
136 | */
137 | protected boolean startEngines(List queries) {
138 | List all = queries == null ? Collections.emptyList() : queries;
139 | // Queries -> engine name Set -> start engine -> verify all started
140 | return all.stream().map(q -> q.engine).distinct().allMatch(this::startEngine);
141 | }
142 |
143 | private boolean startEngine(String engine) {
144 | WorkingEngine working = engines.get(engine);
145 | if (working == null) {
146 | log.error("Engine {} not loaded but required by query.", engine);
147 | return false;
148 | }
149 | // Already started?
150 | if (working.isStarted) {
151 | return true;
152 | }
153 | working.isStarted = working.getEngine().setup(arguments);
154 | if (!working.isStarted) {
155 | log.error("Required engine {} could not be setup.", engine);
156 | working.getEngine().printHelp();
157 | return false;
158 | }
159 | return true;
160 | }
161 |
162 | @Override
163 | public void printHelp() {
164 | Helpable.printHelp("Engine Options", PARSER);
165 | engines.values().stream().map(WorkingEngine::getEngine).forEach(Engine::printHelp);
166 | Helpable.printHelp("Advanced Engine Options", getPluginOptionsParser());
167 | }
168 |
169 | private void run(Query query) {
170 | try {
171 | engines.get(query.engine).getEngine().execute(query);
172 | } catch (Exception e) {
173 | query.setFailure(e.toString());
174 | }
175 | }
176 |
177 | /**
178 | * Run a query and store the results in the query object.
179 | *
180 | * @param queries Queries to execute and store.
181 | * @return true iff the required engines were loaded and the queries were able to run.
182 | */
183 | public boolean run(List queries) {
184 | if (queries.isEmpty()) {
185 | return true;
186 | }
187 | if (!startEngines(queries)) {
188 | return false;
189 | }
190 | // Run each query.
191 | if (!queryParallelEnable) {
192 | queries.forEach(this::run);
193 | } else {
194 | // Split queries into groups by priority where lowers value correspond to higher priority and run first
195 | Map> queryGroups = queries.stream().collect(Collectors.groupingBy(Query::getPriority, TreeMap::new, Collectors.toList()));
196 | int maxGroupSize = queryGroups.values().stream().mapToInt(List::size).max().getAsInt();
197 | int poolSize = Math.max(queryParallelMax > 0 ? queryParallelMax : maxGroupSize, QUERY_PARALLEL_MIN);
198 | log.info("Creating a ForkJoinPool with size {}", poolSize);
199 | ForkJoinPool forkJoinPool = new ForkJoinPool(poolSize);
200 | queryGroups.values().forEach(queryGroup -> {
201 | try {
202 | forkJoinPool.submit(() -> queryGroup.parallelStream().forEach(this::run)).get();
203 | } catch (Exception e) {
204 | log.error("Caught exception", e);
205 | }
206 | });
207 | forkJoinPool.shutdown();
208 | }
209 | return true;
210 | }
211 | }
212 |
--------------------------------------------------------------------------------
/src/main/java/com/yahoo/validatar/execution/pig/Sty.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2015 Yahoo Inc.
3 | * Licensed under the terms of the Apache 2 license. Please see LICENSE file in the project root for terms.
4 | */
5 | package com.yahoo.validatar.execution.pig;
6 |
7 | import com.yahoo.validatar.common.Helpable;
8 | import com.yahoo.validatar.common.Query;
9 | import com.yahoo.validatar.common.Result;
10 | import com.yahoo.validatar.common.TypeSystem;
11 | import com.yahoo.validatar.common.TypedObject;
12 | import com.yahoo.validatar.execution.Engine;
13 | import joptsimple.OptionParser;
14 | import joptsimple.OptionSet;
15 | import lombok.extern.slf4j.Slf4j;
16 | import org.apache.pig.PigServer;
17 | import org.apache.pig.backend.executionengine.ExecException;
18 | import org.apache.pig.data.DataType;
19 | import org.apache.pig.data.Tuple;
20 | import org.apache.pig.impl.logicalLayer.schema.Schema;
21 |
22 | import java.io.ByteArrayInputStream;
23 | import java.io.IOException;
24 | import java.sql.Timestamp;
25 | import java.util.Collections;
26 | import java.util.Iterator;
27 | import java.util.List;
28 | import java.util.Map;
29 | import java.util.Properties;
30 | import java.util.stream.Collectors;
31 |
32 | @Slf4j
33 | public class Sty implements Engine {
34 | public static final String PIG_EXEC_TYPE = "pig-exec-type";
35 | public static final String PIG_OUTPUT_ALIAS = "pig-output-alias";
36 | public static final String PIG_SETTING = "pig-setting";
37 |
38 | /** Engine name. */
39 | public static final String ENGINE_NAME = "pig";
40 |
41 | public static final String DEFAULT_EXEC_TYPE = "mr";
42 | public static final String DEFAULT_OUTPUT_ALIAS = "validatar_results";
43 | public static final String SETTING_DELIMITER = "=";
44 |
45 | public static final String METADATA_EXEC_TYPE_KEY = "exec-type";
46 | public static final String METADATA_ALIAS_KEY = "output-alias";
47 |
48 | private String defaultExecType;
49 | private String defaultOutputAlias;
50 | private Properties properties;
51 |
52 | private class FieldDetail {
53 | public final String alias;
54 | public final byte type;
55 |
56 | public FieldDetail(String alias, byte type) {
57 | this.alias = alias;
58 | this.type = type;
59 | }
60 | }
61 |
62 | private final OptionParser parser = new OptionParser() {
63 | {
64 | accepts(PIG_EXEC_TYPE, "The exec-type for Pig to use (the -x argument used when running Pig. Ex: local, mr, tez)")
65 | .withRequiredArg()
66 | .describedAs("Pig execution type")
67 | .defaultsTo(DEFAULT_EXEC_TYPE);
68 | accepts(PIG_OUTPUT_ALIAS, "The default name of the alias where the result is. This should contain the data that will be collected")
69 | .withRequiredArg()
70 | .describedAs("Pig default output alias")
71 | .defaultsTo(DEFAULT_OUTPUT_ALIAS);
72 | accepts(PIG_SETTING, "Settings and their values. The -D params that would have been sent to Pig. Ex: 'mapreduce.job.acl-view-job=*'")
73 | .withRequiredArg()
74 | .describedAs("Pig generic settings to use.");
75 | allowsUnrecognizedOptions();
76 | }
77 | };
78 |
79 | @Override
80 | public String getName() {
81 | return ENGINE_NAME;
82 | }
83 |
84 | @Override
85 | public boolean setup(String[] arguments) {
86 | OptionSet options = parser.parse(arguments);
87 | defaultExecType = (String) options.valueOf(PIG_EXEC_TYPE);
88 | defaultOutputAlias = (String) options.valueOf(PIG_OUTPUT_ALIAS);
89 | properties = getProperties(options);
90 | // We will boot up a PigServer per query, so nothing else to do...
91 | return true;
92 | }
93 |
94 | @Override
95 | public void printHelp() {
96 | Helpable.printHelp("Pig engine options", parser);
97 | }
98 |
99 | @Override
100 | public void execute(Query query) {
101 | String queryName = query.name;
102 | String queryValue = query.value;
103 | Map queryMetadata = query.getMetadata();
104 | String execType = Query.getKey(queryMetadata, METADATA_EXEC_TYPE_KEY).orElse(defaultExecType);
105 | String alias = Query.getKey(queryMetadata, METADATA_ALIAS_KEY).orElse(defaultOutputAlias);
106 | log.info("Running {} for alias {}: {}", queryName, alias, queryValue);
107 | try {
108 | PigServer server = getPigServer(execType);
109 | server.registerScript(new ByteArrayInputStream(queryValue.getBytes()));
110 | Iterator queryResults = server.openIterator(alias);
111 | Result result = query.createResults();
112 | // dumpSchema will also, unfortunately, print the schema to stdout.
113 | List metadata = getFieldDetails(server.dumpSchema(alias));
114 | populateColumns(metadata, result);
115 | while (queryResults.hasNext()) {
116 | populateRow(queryResults.next(), metadata, result);
117 | }
118 | server.shutdown();
119 | } catch (IOException ioe) {
120 | log.error("Problem with Pig query: {}\n{}", queryValue, ioe);
121 | query.setFailure(ioe.toString());
122 | } catch (Exception e) {
123 | log.error("Error occurred while processing Pig query: {}\n{}", queryValue, e);
124 | query.setFailure(e.toString());
125 | }
126 | }
127 |
128 | private void populateColumns(List metadata, Result result) throws IOException {
129 | if (metadata.isEmpty()) {
130 | throw new IOException("No metadata of columns found for Pig query");
131 | }
132 | metadata.forEach(m -> result.addColumn(m.alias));
133 | }
134 |
135 | private void populateRow(Tuple row, List metadata, Result result) throws ExecException {
136 | if (row == null) {
137 | log.info("Skipping null row in results...");
138 | return;
139 | }
140 | for (int i = 0; i < metadata.size(); ++i) {
141 | FieldDetail column = metadata.get(i);
142 | TypedObject value = getTypedObject(row.get(i), column);
143 | log.info("Column: {}\tType: {}\tValue: {}", column.alias, column.type, (value == null ? "null" : value.data));
144 | result.addColumnRow(column.alias, value);
145 | }
146 | }
147 |
148 | private TypedObject getTypedObject(Object data, FieldDetail detail) throws ExecException {
149 | if (data == null) {
150 | return null;
151 | }
152 | byte type = detail.type;
153 | switch (type) {
154 | case DataType.BOOLEAN:
155 | return TypeSystem.asTypedObject(DataType.toBoolean(data, type));
156 | case DataType.INTEGER:
157 | case DataType.LONG:
158 | return TypeSystem.asTypedObject(DataType.toLong(data, type));
159 | case DataType.FLOAT:
160 | case DataType.DOUBLE:
161 | return TypeSystem.asTypedObject(DataType.toDouble(data, type));
162 | case DataType.DATETIME:
163 | return TypeSystem.asTypedObject(new Timestamp(DataType.toDateTime(data, type).getMillis()));
164 | case DataType.BYTE:
165 | case DataType.BYTEARRAY:
166 | case DataType.CHARARRAY:
167 | return TypeSystem.asTypedObject(DataType.toString(data, type));
168 | case DataType.BIGINTEGER:
169 | case DataType.BIGDECIMAL:
170 | return TypeSystem.asTypedObject(DataType.toBigDecimal(data, type));
171 | default:
172 | //TUPLE, BAG, MAP, INTERNALMAP, GENERIC_WRITABLECOMPARABLE, ERROR, UNKNOWN, NULL and anything else
173 | return null;
174 | }
175 | }
176 |
177 | private List getFieldDetails(Schema schema) {
178 | if (schema == null) {
179 | return Collections.emptyList();
180 | }
181 | return schema.getFields().stream().map(f -> new FieldDetail(f.alias, f.type)).collect(Collectors.toList());
182 | }
183 |
184 | private Properties getProperties(OptionSet options) {
185 | List settings = (List) options.valuesOf(PIG_SETTING);
186 | Properties properties = new Properties();
187 | for (String setting : settings) {
188 | String[] tokens = setting.split(SETTING_DELIMITER);
189 | if (tokens.length != 2) {
190 | log.error("Ignoring unknown Pig setting provided: {}", setting);
191 | continue;
192 | }
193 | properties.put(tokens[0], tokens[1]);
194 | }
195 | return properties;
196 | }
197 |
198 | PigServer getPigServer(String execType) throws IOException {
199 | return new PigServer(execType, properties);
200 | }
201 | }
202 |
--------------------------------------------------------------------------------
/src/main/java/com/yahoo/validatar/common/Operations.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2017 Yahoo Inc.
3 | * Licensed under the terms of the Apache 2 license. Please see LICENSE file in the project root for terms.
4 | */
5 | package com.yahoo.validatar.common;
6 |
7 | import java.util.Objects;
8 | import java.util.function.BinaryOperator;
9 | import java.util.function.UnaryOperator;
10 |
11 | import static com.yahoo.validatar.common.TypeSystem.asTypedObject;
12 | import static com.yahoo.validatar.common.TypeSystem.compare;
13 |
14 | /**
15 | * This defines the various operations we will support and provides default implementations for all of them. A particular
16 | * type specific implementation can implement this to provide its own specific logic.
17 | */
18 | public interface Operations {
19 | /**
20 | * These are the unary operations we will support.
21 | */
22 | enum UnaryOperation {
23 | CAST, NOT
24 | }
25 |
26 | /**
27 | * These are the binary operations we will support.
28 | */
29 | enum BinaryOperation {
30 | ADD, SUBTRACT, MULTIPLY, DIVIDE, MODULUS, EQUAL, NOT_EQUAL, GREATER, LESS, GREATER_EQUAL, LESS_EQUAL, OR, AND
31 | }
32 |
33 | /**
34 | * Adds two TypedObjects.
35 | *
36 | * @param first The first object.
37 | * @param second The second object.
38 | * @return The result object.
39 | */
40 | default TypedObject add(TypedObject first, TypedObject second) {
41 | return null;
42 | }
43 |
44 | /**
45 | * Subtracts two TypedObjects.
46 | *
47 | * @param first The first object.
48 | * @param second The second object.
49 | * @return The result object.
50 | */
51 | default TypedObject subtract(TypedObject first, TypedObject second) {
52 | return null;
53 | }
54 |
55 | /**
56 | * Multiplies two TypedObjects.
57 | *
58 | * @param first The first object.
59 | * @param second The second object.
60 | * @return The result object.
61 | */
62 | default TypedObject multiply(TypedObject first, TypedObject second) {
63 | return null;
64 | }
65 |
66 | /**
67 | * Divides two TypedObjects.
68 | *
69 | * @param first The first object.
70 | * @param second The second object.
71 | * @return The result object.
72 | */
73 | default TypedObject divide(TypedObject first, TypedObject second) {
74 | return null;
75 | }
76 |
77 | /**
78 | * Finds the integer remainder after division two TypedObjects.
79 | *
80 | * @param first The first object.
81 | * @param second The second object.
82 | * @return The result object.
83 | */
84 | default TypedObject modulus(TypedObject first, TypedObject second) {
85 | return null;
86 | }
87 |
88 | /**
89 | * Checks to see if the two {@link TypedObject} are equal.
90 | *
91 | * @param first The first object.
92 | * @param second The second object.
93 | * @return The result TypedObject of type {@link TypeSystem.Type#BOOLEAN}.
94 | */
95 | default TypedObject equal(TypedObject first, TypedObject second) {
96 | return asTypedObject(compare(first, second) == 0);
97 | }
98 |
99 | /**
100 | * Checks to see if the two {@link TypedObject} are not equal.
101 | *
102 | * @param first The first object.
103 | * @param second The second object.
104 | * @return The result TypedObject of type {@link TypeSystem.Type#BOOLEAN}.
105 | */
106 | default TypedObject notEqual(TypedObject first, TypedObject second) {
107 | return asTypedObject(compare(first, second) != 0);
108 | }
109 |
110 | /**
111 | * Checks to see if the first {@link TypedObject} is greater than the second.
112 | *
113 | * @param first The first object.
114 | * @param second The second object.
115 | * @return The result TypedObject of type {@link TypeSystem.Type#BOOLEAN}.
116 | */
117 | default TypedObject greater(TypedObject first, TypedObject second) {
118 | return asTypedObject(compare(first, second) > 0);
119 | }
120 |
121 | /**
122 | * Checks to see if the first {@link TypedObject} is less than the second.
123 | *
124 | * @param first The first object.
125 | * @param second The second object.
126 | * @return The result TypedObject of type {@link TypeSystem.Type#BOOLEAN}.
127 | */
128 | default TypedObject less(TypedObject first, TypedObject second) {
129 | return asTypedObject(compare(first, second) < 0);
130 | }
131 |
132 | /**
133 | * Checks to see if the first {@link TypedObject} is greater than or equal to the second.
134 | *
135 | * @param first The first object.
136 | * @param second The second object.
137 | * @return The result TypedObject of type {@link TypeSystem.Type#BOOLEAN}.
138 | */
139 | default TypedObject greaterEqual(TypedObject first, TypedObject second) {
140 | return asTypedObject(compare(first, second) >= 0);
141 | }
142 |
143 | /**
144 | * Checks to see if the first {@link TypedObject} is less than or equal to the second.
145 | *
146 | * @param first The first object.
147 | * @param second The second object.
148 | * @return The result TypedObject of type {@link TypeSystem.Type#BOOLEAN}.
149 | */
150 | default TypedObject lessEqual(TypedObject first, TypedObject second) {
151 | return asTypedObject(compare(first, second) <= 0);
152 | }
153 |
154 | /**
155 | * Logical ors two TypedObjects.
156 | *
157 | * @param first The first object.
158 | * @param second The second object.
159 | * @return The result object.
160 | */
161 | default TypedObject or(TypedObject first, TypedObject second) {
162 | return null;
163 | }
164 |
165 | /**
166 | * Logical ands two TypedObjects.
167 | *
168 | * @param first The first object.
169 | * @param second The second object.
170 | * @return The result object.
171 | */
172 | default TypedObject and(TypedObject first, TypedObject second) {
173 | return null;
174 | }
175 |
176 | /**
177 | * Logical negates a TypedObject.
178 | *
179 | * @param object The object.
180 | * @return The result object.
181 | */
182 | default TypedObject not(TypedObject object) {
183 | return null;
184 | }
185 |
186 | /**
187 | * Casts a TypedObject into its given type.
188 | *
189 | * @param object The object.
190 | * @return The result object.
191 | */
192 | default TypedObject cast(TypedObject object) {
193 | return null;
194 | }
195 |
196 | /**
197 | * Given a BinaryOperation, finds the operator for it. Null if it cannot.
198 | *
199 | * @param operation The operation
200 | * @return The result binary operator that can be applied.
201 | */
202 | default BinaryOperator dispatch(BinaryOperation operation) {
203 | // Can assign to a return value and return it, getting rid of the unreachable default...
204 | Objects.requireNonNull(operation);
205 | BinaryOperator operator = null;
206 | switch (operation) {
207 | case ADD:
208 | operator = this::add;
209 | break;
210 | case SUBTRACT:
211 | operator = this::subtract;
212 | break;
213 | case MULTIPLY:
214 | operator = this::multiply;
215 | break;
216 | case DIVIDE:
217 | operator = this::divide;
218 | break;
219 | case MODULUS:
220 | operator = this::modulus;
221 | break;
222 | case EQUAL:
223 | operator = this::equal;
224 | break;
225 | case NOT_EQUAL:
226 | operator = this::notEqual;
227 | break;
228 | case GREATER:
229 | operator = this::greater;
230 | break;
231 | case LESS:
232 | operator = this::less;
233 | break;
234 | case GREATER_EQUAL:
235 | operator = this::greaterEqual;
236 | break;
237 | case LESS_EQUAL:
238 | operator = this::lessEqual;
239 | break;
240 | case OR:
241 | operator = this::or;
242 | break;
243 | case AND:
244 | operator = this::and;
245 | break;
246 | }
247 | return operator;
248 | }
249 |
250 | /**
251 | * Given a UnaryOperation, finds the operator for it. Null if it cannot.
252 | *
253 | * @param operation The operation.
254 | * @return The result unary operator that can be applied.
255 | */
256 | default UnaryOperator dispatch(UnaryOperation operation) {
257 | Objects.requireNonNull(operation);
258 | UnaryOperator operator = null;
259 | switch (operation) {
260 | case NOT:
261 | operator = this::not;
262 | break;
263 | case CAST:
264 | operator = this::cast;
265 | break;
266 | }
267 | return operator;
268 | }
269 | }
270 |
--------------------------------------------------------------------------------
/src/main/java/com/yahoo/validatar/execution/hive/Apiary.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2015 Yahoo Inc.
3 | * Licensed under the terms of the Apache 2 license. Please see LICENSE file in the project root for terms.
4 | */
5 | package com.yahoo.validatar.execution.hive;
6 |
7 | import com.yahoo.validatar.common.Helpable;
8 | import com.yahoo.validatar.common.Query;
9 | import com.yahoo.validatar.common.Result;
10 | import com.yahoo.validatar.common.TypeSystem;
11 | import com.yahoo.validatar.common.TypedObject;
12 | import com.yahoo.validatar.execution.Engine;
13 | import joptsimple.OptionParser;
14 | import joptsimple.OptionSet;
15 | import lombok.extern.slf4j.Slf4j;
16 |
17 | import java.sql.Connection;
18 | import java.sql.DriverManager;
19 | import java.sql.ResultSet;
20 | import java.sql.ResultSetMetaData;
21 | import java.sql.SQLException;
22 | import java.sql.Statement;
23 | import java.sql.Types;
24 | import java.util.List;
25 |
26 | @Slf4j
27 | public class Apiary implements Engine {
28 | public static final String HIVE_JDBC = "hive-jdbc";
29 | public static final String HIVE_DRIVER = "hive-driver";
30 | public static final String HIVE_USERNAME = "hive-username";
31 | public static final String HIVE_PASSWORD = "hive-password";
32 | public static final String HIVE_SETTING = "hive-setting";
33 |
34 | public static final String ENGINE_NAME = "hive";
35 |
36 | public static final String DRIVER_NAME = "org.apache.hive.jdbc.HiveDriver";
37 | public static final String SETTING_PREFIX = "set ";
38 |
39 | protected Connection connection;
40 | protected OptionSet options;
41 |
42 | private final OptionParser parser = new OptionParser() {
43 | {
44 | accepts(HIVE_JDBC, "JDBC string to the HiveServer2 with an optional database. " +
45 | "If the database is provided, the queries must NOT have one. " +
46 | "Ex: 'jdbc:hive2://HIVE_SERVER:PORT/[DATABASE_FOR_ALL_QUERIES]' ")
47 | .withRequiredArg()
48 | .required()
49 | .describedAs("Hive JDBC connector");
50 | accepts(HIVE_DRIVER, "Fully qualified package name to the hive driver.")
51 | .withRequiredArg()
52 | .describedAs("Hive driver")
53 | .defaultsTo(DRIVER_NAME);
54 | accepts(HIVE_USERNAME, "Hive server username.")
55 | .withRequiredArg()
56 | .describedAs("Hive server username")
57 | .defaultsTo("anon");
58 | accepts(HIVE_PASSWORD, "Hive server password.")
59 | .withRequiredArg()
60 | .describedAs("Hive server password")
61 | .defaultsTo("anon");
62 | accepts(HIVE_SETTING, "Settings and their values. Ex: 'hive.execution.engine=mr'")
63 | .withRequiredArg()
64 | .describedAs("Hive generic settings to use.");
65 | allowsUnrecognizedOptions();
66 | }
67 | };
68 |
69 | @Override
70 | public boolean setup(String[] arguments) {
71 | options = parser.parse(arguments);
72 | try {
73 | connection = setupConnection();
74 | } catch (ClassNotFoundException | SQLException e) {
75 | log.error("Could not set up the Hive engine", e);
76 | return false;
77 | }
78 | return true;
79 | }
80 |
81 | @Override
82 | public void printHelp() {
83 | Helpable.printHelp("Hive engine options", parser);
84 | }
85 |
86 | @Override
87 | public void execute(Query query) {
88 | String queryName = query.name;
89 | String queryValue = query.value;
90 | log.info("Running {}: {}", queryName, queryValue);
91 | try (Statement statement = connection.createStatement()) {
92 | setHiveSettings(statement);
93 | ResultSet result = statement.executeQuery(queryValue);
94 | ResultSetMetaData metadata = result.getMetaData();
95 | int columns = metadata.getColumnCount();
96 |
97 | Result queryResult = query.createResults();
98 |
99 | addHeader(metadata, columns, queryResult);
100 | while (result.next()) {
101 | addRow(result, metadata, columns, queryResult);
102 | }
103 | result.close();
104 | } catch (SQLException e) {
105 | log.error("SQL problem with Hive query: {}\n{}\n{}", queryName, queryValue, e);
106 | query.setFailure(e.getMessage());
107 | }
108 | }
109 |
110 | private void addHeader(ResultSetMetaData metadata, int columns, Result queryResult) throws SQLException {
111 | for (int i = 1; i < columns + 1; i++) {
112 | String name = metadata.getColumnName(i);
113 | queryResult.addColumn(name);
114 | }
115 | }
116 |
117 | private void addRow(ResultSet result, ResultSetMetaData metadata, int columns, Result storage) throws SQLException {
118 | for (int i = 1; i < columns + 1; i++) {
119 | // The name and type getting is being done per row. We should fix it even though Hive gets it only once.
120 | String name = metadata.getColumnName(i);
121 | int type = metadata.getColumnType(i);
122 | TypedObject value = getAsTypedObject(result, i, type);
123 | storage.addColumnRow(name, value);
124 | log.info("Column: {}\tType: {}\tValue: {}", name, type, (value == null ? "null" : value.data));
125 | }
126 | }
127 |
128 | @Override
129 | public String getName() {
130 | return ENGINE_NAME;
131 | }
132 |
133 | /**
134 | * Takes a value and its type and returns it as the appropriate TypedObject.
135 | *
136 | * @param results The ResultSet that has a confirmed value for reading by its iterator.
137 | * @param index The index of the column in the results to get.
138 | * @param type The java.sql.TypesSQL type of the value.
139 | * @return A non-null TypedObject representation of the value or null if the result was null.
140 | * @throws java.sql.SQLException if any.
141 | */
142 | TypedObject getAsTypedObject(ResultSet results, int index, int type) throws SQLException {
143 | if (results.getObject(index) == null || results.wasNull()) {
144 | return null;
145 | }
146 |
147 | TypedObject toReturn;
148 | switch (type) {
149 | case (Types.DATE):
150 | case (Types.CHAR):
151 | case (Types.VARCHAR):
152 | toReturn = TypeSystem.asTypedObject(results.getString(index));
153 | break;
154 | case (Types.FLOAT):
155 | case (Types.DOUBLE):
156 | toReturn = TypeSystem.asTypedObject(results.getDouble(index));
157 | break;
158 | case (Types.BOOLEAN):
159 | toReturn = TypeSystem.asTypedObject(results.getBoolean(index));
160 | break;
161 | case (Types.TINYINT):
162 | case (Types.SMALLINT):
163 | case (Types.INTEGER):
164 | case (Types.BIGINT):
165 | toReturn = TypeSystem.asTypedObject(results.getLong(index));
166 | break;
167 | case (Types.DECIMAL):
168 | toReturn = TypeSystem.asTypedObject(results.getBigDecimal(index));
169 | break;
170 | case (Types.TIMESTAMP):
171 | toReturn = TypeSystem.asTypedObject(results.getTimestamp(index));
172 | break;
173 | case (Types.NULL):
174 | toReturn = null;
175 | break;
176 | default:
177 | throw new UnsupportedOperationException("Unknown SQL type encountered from Hive: " + type);
178 | }
179 | return toReturn;
180 | }
181 |
182 | /**
183 | * Sets up the connection using JDBC.
184 | *
185 | * @return The created {@link java.sql.Statement} object.
186 | * @throws java.lang.ClassNotFoundException if any.
187 | * @throws java.sql.SQLException if any.
188 | */
189 | Connection setupConnection() throws ClassNotFoundException, SQLException {
190 | // Load the JDBC driver
191 | String driver = (String) options.valueOf(HIVE_DRIVER);
192 | log.info("Loading JDBC driver: {}", driver);
193 | Class.forName(driver);
194 |
195 | // Get the JDBC connector
196 | String jdbcConnector = (String) options.valueOf(HIVE_JDBC);
197 |
198 | log.info("Connecting to: {}", jdbcConnector);
199 | String username = (String) options.valueOf(HIVE_USERNAME);
200 | String password = (String) options.valueOf(HIVE_PASSWORD);
201 |
202 | // Start the connection
203 | return DriverManager.getConnection(jdbcConnector, username, password);
204 | }
205 |
206 | /**
207 | * Applies any settings if provided.
208 | *
209 | * @param statement A {@link java.sql.Statement} to execute the setting updates to.
210 | * @throws java.sql.SQLException if any.
211 | */
212 | void setHiveSettings(Statement statement) throws SQLException {
213 | for (String setting : (List) options.valuesOf(HIVE_SETTING)) {
214 | log.info("Applying setting {}", setting);
215 | statement.executeUpdate(SETTING_PREFIX + setting);
216 | }
217 | }
218 | }
219 |
--------------------------------------------------------------------------------
/src/test/java/com/yahoo/validatar/report/email/EmailFormatterTest.java:
--------------------------------------------------------------------------------
1 | package com.yahoo.validatar.report.email;
2 |
3 | import com.yahoo.validatar.OutputCaptor;
4 | import com.yahoo.validatar.common.Query;
5 | import com.yahoo.validatar.common.TestSuite;
6 | import org.simplejavamail.email.Email;
7 | import org.simplejavamail.mailer.Mailer;
8 | import org.simplejavamail.mailer.config.TransportStrategy;
9 | import org.testng.annotations.Test;
10 |
11 | import java.io.IOException;
12 | import java.lang.reflect.Field;
13 | import java.util.Arrays;
14 | import java.util.Collections;
15 | import java.util.List;
16 |
17 | import static org.junit.Assert.assertTrue;
18 | import static org.junit.Assert.fail;
19 | import static org.mockito.Matchers.any;
20 | import static org.mockito.Mockito.doAnswer;
21 | import static org.mockito.Mockito.doCallRealMethod;
22 | import static org.mockito.Mockito.mock;
23 | import static org.mockito.Mockito.verify;
24 | import static org.testng.Assert.assertEquals;
25 | import static org.testng.Assert.assertFalse;
26 |
27 | public class EmailFormatterTest {
28 | private static T get(Object target, String name, Class clazz) {
29 | try {
30 | Field f = target.getClass().getDeclaredField(name);
31 | f.setAccessible(true);
32 | Object o = f.get(target);
33 | return clazz.cast(o);
34 | } catch (Exception e) {
35 | throw new RuntimeException(e);
36 | }
37 | }
38 |
39 | private static void set(EmailFormatter target, String name, Object value) {
40 | Class cls = EmailFormatter.class;
41 | try {
42 | Field f = cls.getDeclaredField(name);
43 | f.setAccessible(true);
44 | f.set(target, value);
45 | } catch (Exception e) {
46 | throw new RuntimeException(e);
47 | }
48 | }
49 |
50 | @Test
51 | public void testSetup() {
52 | String[] args = {
53 | "--" + EmailFormatter.EMAIL_RECIPIENTS, "email@email.com",
54 | "--" + EmailFormatter.EMAIL_SENDER_NAME, "Validatar",
55 | "--" + EmailFormatter.EMAIL_FROM, "validatar@validatar.com",
56 | "--" + EmailFormatter.EMAIL_REPLY_TO, "validatar@validatar.com",
57 | "--" + EmailFormatter.EMAIL_SMTP_HOST, "host.host.com",
58 | "--" + EmailFormatter.EMAIL_SMTP_PORT, "25",
59 | "--" + EmailFormatter.EMAIL_SMTP_STRATEGY, "SMTP_PLAIN"
60 | };
61 | EmailFormatter formatter = new EmailFormatter();
62 | formatter.setup(args);
63 | List recipientEmails = get(formatter, "recipientEmails", List.class);
64 | assertEquals(recipientEmails.size(), 1);
65 | assertEquals(recipientEmails.get(0), "email@email.com");
66 | assertEquals("Validatar", get(formatter, "senderName", String.class));
67 | assertEquals("validatar@validatar.com", get(formatter, "fromEmail", String.class));
68 | assertEquals("validatar@validatar.com", get(formatter, "replyTo", String.class));
69 | assertEquals("host.host.com", get(formatter, "smtpHost", String.class));
70 | assertEquals((Integer) 25, get(formatter, "smtpPort", Integer.class));
71 | assertEquals(TransportStrategy.SMTP_PLAIN, get(formatter, "strategy", TransportStrategy.class));
72 | }
73 |
74 | @Test
75 | public void testWriteReportShowsFailures() throws IOException {
76 | com.yahoo.validatar.common.Test test = new com.yahoo.validatar.common.Test();
77 | com.yahoo.validatar.common.Test skipped = new com.yahoo.validatar.common.Test();
78 | skipped.name = "SkippedTest";
79 | skipped.warnOnly = true;
80 | skipped.addMessage("SkippedTestMessage");
81 | Query query = new Query();
82 | TestSuite ts = new TestSuite();
83 | ts.name = "testSuiteName1";
84 | test.name = "testName1";
85 | query.name = "queryName1";
86 | test.addMessage("testMessage1");
87 | test.addMessage("testMessage2");
88 | test.addMessage("testMessage3");
89 | test.setFailed();
90 | query.addMessage("queryMessage");
91 | query.setFailed();
92 | ts.queries = Collections.singletonList(query);
93 | ts.tests = Arrays.asList(test, skipped);
94 | EmailFormatter formatter = mock(EmailFormatter.class);
95 | doCallRealMethod().when(formatter).writeReport(any());
96 | set(formatter, "recipientEmails", Collections.singletonList("email@email.com"));
97 | set(formatter, "senderName", "Validatar");
98 | set(formatter, "fromEmail", "from@mail.com");
99 | set(formatter, "replyTo", "reply@mail.com");
100 | set(formatter, "smtpHost", "host.host.com");
101 | set(formatter, "smtpPort", 25);
102 | doAnswer(iom -> {
103 | Email email = (Email) iom.getArguments()[1];
104 | String html = email.getTextHTML();
105 | String[] containsAllOf = {
106 | "testSuiteName1", "testName1", "queryName1", "testMessage1", "SkippedTestMessage",
107 | "testMessage2", "testMessage3", "queryMessage", "SKIPPED", "SkippedTest"
108 | };
109 | for (String str : containsAllOf) {
110 | assertTrue(html.contains(str));
111 | }
112 | return null;
113 | }
114 | ).when(formatter).sendEmail(any(), any());
115 | formatter.writeReport(Collections.singletonList(ts));
116 | verify(formatter).sendEmail(any(), any());
117 | }
118 |
119 | @Test
120 | public void testWriteReportPassesAndShowsMessagesWhenOnlyWarnings() throws IOException {
121 | com.yahoo.validatar.common.Test test = new com.yahoo.validatar.common.Test();
122 | com.yahoo.validatar.common.Test skipped = new com.yahoo.validatar.common.Test();
123 | skipped.name = "SkippedTest";
124 | skipped.warnOnly = true;
125 | skipped.addMessage("SkippedTestMessage");
126 | Query query = new Query();
127 | TestSuite ts = new TestSuite();
128 | ts.name = "testSuiteName1";
129 | test.name = "testName1";
130 | query.name = "queryName1";
131 | test.addMessage("testMessage1");
132 | test.addMessage("testMessage2");
133 | test.addMessage("testMessage3");
134 | query.addMessage("queryMessage");
135 | ts.queries = Collections.singletonList(query);
136 | ts.tests = Arrays.asList(test, skipped);
137 | EmailFormatter formatter = mock(EmailFormatter.class);
138 | doCallRealMethod().when(formatter).writeReport(any());
139 | set(formatter, "recipientEmails", Collections.singletonList("email@email.com"));
140 | set(formatter, "senderName", "Validatar");
141 | set(formatter, "fromEmail", "from@mail.com");
142 | set(formatter, "replyTo", "reply@mail.com");
143 | set(formatter, "smtpHost", "host.host.com");
144 | set(formatter, "smtpPort", 25);
145 | doAnswer(iom -> {
146 | Email email = (Email) iom.getArguments()[1];
147 | String html = email.getTextHTML();
148 | String[] containsAllOf = {
149 | "SkippedTest", "SkippedTestMessage", "testSuiteName1"
150 | };
151 | String[] containsNoneOf = {
152 | "testMessage1", "testMessage2", "testMessage3", "queryMessage",
153 | "testName1", "queryName1"
154 | };
155 | for (String str : containsAllOf) {
156 | assertTrue(html.contains(str));
157 | }
158 | for (String str : containsNoneOf) {
159 | assertFalse(html.contains(str));
160 | }
161 | return null;
162 | }
163 | ).when(formatter).sendEmail(any(), any());
164 | formatter.writeReport(Collections.singletonList(ts));
165 | verify(formatter).sendEmail(any(), any());
166 | }
167 |
168 | @Test
169 | public void testSetupReturnsFailMissingParams() {
170 | EmailFormatter formatter = new EmailFormatter();
171 | assertFalse(formatter.setup(new String[]{}));
172 | }
173 |
174 | @Test
175 | public void testWriteReportEmptyTestSuites() throws IOException {
176 | EmailFormatter formatter = mock(EmailFormatter.class);
177 | doCallRealMethod().when(formatter).writeReport(any());
178 | set(formatter, "recipientEmails", Collections.singletonList("email@email.com"));
179 | set(formatter, "senderName", "Validatar");
180 | set(formatter, "fromEmail", "from@mail.com");
181 | set(formatter, "replyTo", "reply@mail.com");
182 | set(formatter, "smtpHost", "host.host.com");
183 | set(formatter, "smtpPort", 25);
184 | doAnswer(iom -> {
185 | Email email = (Email) iom.getArguments()[1];
186 | String html = email.getTextHTML();
187 | assertTrue(html.contains("Nice!"));
188 | return null;
189 | }
190 | ).when(formatter).sendEmail(any(), any());
191 | formatter.writeReport(null);
192 | }
193 |
194 | @Test
195 | public void testSendEmail() {
196 | Mailer mailer = mock(Mailer.class);
197 | Email email = mock(Email.class);
198 | EmailFormatter formatter = new EmailFormatter();
199 | formatter.sendEmail(mailer, email);
200 | verify(mailer).sendMail(email);
201 | }
202 |
203 | @Test
204 | public void testGetName() {
205 | EmailFormatter formatter = new EmailFormatter();
206 | assertEquals(EmailFormatter.EMAIL_FORMATTER, formatter.getName());
207 | OutputCaptor.redirectToDevNull();
208 | try {
209 | formatter.printHelp();
210 | } catch (Exception e) {
211 | OutputCaptor.redirectToStandard();
212 | fail();
213 | }
214 | OutputCaptor.redirectToStandard();
215 | }
216 | }
217 |
--------------------------------------------------------------------------------
/src/main/java/com/yahoo/validatar/common/Operators.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2017 Yahoo Inc.
3 | * Licensed under the terms of the Apache 2 license. Please see LICENSE file in the project root for terms.
4 | */
5 | package com.yahoo.validatar.common;
6 |
7 | import java.math.BigDecimal;
8 | import java.sql.Timestamp;
9 |
10 | import static com.yahoo.validatar.common.TypeSystem.asTypedObject;
11 |
12 | /**
13 | * Contains the various type specific {@link Operations}.
14 | *
15 | * In general, we don't want lossy casting, or strange casting like a boolean to a short etc.
16 | * But we will follow the basic Java widening primitive rules.
17 | * https://docs.oracle.com/javase/specs/jls/se7/html/jls-5.html
18 | *
19 | * Exceptions:
20 | * Timestamp to and from Long will do a millis since epoch
21 | */
22 | public class Operators {
23 | public static class BooleanOperator implements Operations {
24 | @Override
25 | public TypedObject or(TypedObject first, TypedObject second) {
26 | return asTypedObject((Boolean) first.data || (Boolean) second.data);
27 | }
28 |
29 | @Override
30 | public TypedObject and(TypedObject first, TypedObject second) {
31 | return asTypedObject((Boolean) first.data && (Boolean) second.data);
32 | }
33 |
34 | @Override
35 | public TypedObject not(TypedObject object) {
36 | return asTypedObject(!(Boolean) object.data);
37 | }
38 |
39 | @Override
40 | public TypedObject cast(TypedObject object) {
41 | switch (object.type) {
42 | case STRING:
43 | object.data = Boolean.valueOf((String) object.data);
44 | break;
45 | case BOOLEAN:
46 | break;
47 | case LONG:
48 | case DOUBLE:
49 | case DECIMAL:
50 | case TIMESTAMP:
51 | return null;
52 | }
53 | object.type = TypeSystem.Type.BOOLEAN;
54 | return object;
55 | }
56 | }
57 |
58 | public static class LongOperator implements Operations {
59 | @Override
60 | public TypedObject add(TypedObject first, TypedObject second) {
61 | return asTypedObject((Long) first.data + (Long) second.data);
62 | }
63 |
64 | @Override
65 | public TypedObject subtract(TypedObject first, TypedObject second) {
66 | return asTypedObject((Long) first.data - (Long) second.data);
67 | }
68 |
69 | @Override
70 | public TypedObject multiply(TypedObject first, TypedObject second) {
71 | return asTypedObject((Long) first.data * (Long) second.data);
72 | }
73 |
74 | @Override
75 | public TypedObject divide(TypedObject first, TypedObject second) {
76 | return asTypedObject((Long) first.data / (Long) second.data);
77 | }
78 |
79 | @Override
80 | public TypedObject modulus(TypedObject first, TypedObject second) {
81 | return asTypedObject((Long) first.data % (Long) second.data);
82 | }
83 |
84 | @Override
85 | public TypedObject cast(TypedObject object) {
86 | switch (object.type) {
87 | case STRING:
88 | object.data = Long.valueOf((String) object.data);
89 | break;
90 | case LONG:
91 | break;
92 | case TIMESTAMP:
93 | object.data = ((Timestamp) object.data).getTime();
94 | break;
95 | case DOUBLE:
96 | case DECIMAL:
97 | case BOOLEAN:
98 | return null;
99 | }
100 | object.type = TypeSystem.Type.LONG;
101 | return object;
102 | }
103 | }
104 | public static class DoubleOperator implements Operations {
105 | @Override
106 | public TypedObject add(TypedObject first, TypedObject second) {
107 | return asTypedObject((Double) first.data + (Double) second.data);
108 | }
109 |
110 | @Override
111 | public TypedObject subtract(TypedObject first, TypedObject second) {
112 | return asTypedObject((Double) first.data - (Double) second.data);
113 | }
114 |
115 | @Override
116 | public TypedObject multiply(TypedObject first, TypedObject second) {
117 | return asTypedObject((Double) first.data * (Double) second.data);
118 | }
119 |
120 | @Override
121 | public TypedObject divide(TypedObject first, TypedObject second) {
122 | return asTypedObject((Double) first.data / (Double) second.data);
123 | }
124 |
125 | @Override
126 | public TypedObject cast(TypedObject object) {
127 | switch (object.type) {
128 | case STRING:
129 | object.data = Double.valueOf((String) object.data);
130 | break;
131 | case DOUBLE:
132 | break;
133 | case LONG:
134 | object.data = ((Long) object.data).doubleValue();
135 | break;
136 | case DECIMAL:
137 | case BOOLEAN:
138 | case TIMESTAMP:
139 | return null;
140 | }
141 | object.type = TypeSystem.Type.DOUBLE;
142 | return object;
143 | }
144 | }
145 |
146 | public static class StringOperator implements Operations {
147 | @Override
148 | public TypedObject add(TypedObject first, TypedObject second) {
149 | return asTypedObject((String) first.data + (String) second.data);
150 | }
151 |
152 | @Override
153 | public TypedObject cast(TypedObject object) {
154 | switch (object.type) {
155 | case STRING:
156 | break;
157 | case LONG:
158 | object.data = ((Long) object.data).toString();
159 | break;
160 | case DOUBLE:
161 | object.data = ((Double) object.data).toString();
162 | break;
163 | case DECIMAL:
164 | object.data = ((BigDecimal) object.data).toString();
165 | break;
166 | case BOOLEAN:
167 | object.data = ((Boolean) object.data).toString();
168 | break;
169 | case TIMESTAMP:
170 | return null;
171 | }
172 | object.type = TypeSystem.Type.STRING;
173 | return object;
174 | }
175 | }
176 |
177 | public static class DecimalOperator implements Operations {
178 | @Override
179 | public TypedObject add(TypedObject first, TypedObject second) {
180 | return asTypedObject(((BigDecimal) first.data).add((BigDecimal) second.data));
181 | }
182 |
183 | @Override
184 | public TypedObject subtract(TypedObject first, TypedObject second) {
185 | return asTypedObject(((BigDecimal) first.data).subtract((BigDecimal) second.data));
186 | }
187 |
188 | @Override
189 | public TypedObject multiply(TypedObject first, TypedObject second) {
190 | return asTypedObject(((BigDecimal) first.data).multiply((BigDecimal) second.data));
191 | }
192 |
193 | @Override
194 | public TypedObject divide(TypedObject first, TypedObject second) {
195 | return asTypedObject(((BigDecimal) first.data).divide((BigDecimal) second.data));
196 | }
197 |
198 | @Override
199 | public TypedObject modulus(TypedObject first, TypedObject second) {
200 | return asTypedObject(((BigDecimal) first.data).divideAndRemainder((BigDecimal) second.data)[1]);
201 | }
202 |
203 | @Override
204 | public TypedObject cast(TypedObject object) {
205 | switch (object.type) {
206 | case STRING:
207 | object.data = new BigDecimal((String) object.data);
208 | break;
209 | case LONG:
210 | object.data = BigDecimal.valueOf((Long) object.data);
211 | break;
212 | case DOUBLE:
213 | object.data = BigDecimal.valueOf((Double) object.data);
214 | break;
215 | case DECIMAL:
216 | break;
217 | case TIMESTAMP:
218 | object.data = BigDecimal.valueOf(((Timestamp) object.data).getTime());
219 | break;
220 | case BOOLEAN:
221 | return null;
222 | }
223 | object.type = TypeSystem.Type.DECIMAL;
224 | return object;
225 | }
226 | }
227 |
228 | public static class TimestampOperator implements Operations {
229 | @Override
230 | public TypedObject add(TypedObject first, TypedObject second) {
231 | return asTypedObject(new Timestamp(((Timestamp) first.data).getTime() + ((Timestamp) second.data).getTime()));
232 | }
233 |
234 | @Override
235 | public TypedObject subtract(TypedObject first, TypedObject second) {
236 | return asTypedObject(new Timestamp(((Timestamp) first.data).getTime() - ((Timestamp) second.data).getTime()));
237 | }
238 |
239 | @Override
240 | public TypedObject multiply(TypedObject first, TypedObject second) {
241 | return asTypedObject(new Timestamp(((Timestamp) first.data).getTime() * ((Timestamp) second.data).getTime()));
242 | }
243 |
244 | @Override
245 | public TypedObject divide(TypedObject first, TypedObject second) {
246 | return asTypedObject(new Timestamp(((Timestamp) first.data).getTime() / ((Timestamp) second.data).getTime()));
247 | }
248 |
249 | @Override
250 | public TypedObject modulus(TypedObject first, TypedObject second) {
251 | return asTypedObject(new Timestamp(((Timestamp) first.data).getTime() % ((Timestamp) second.data).getTime()));
252 | }
253 |
254 | @Override
255 | public TypedObject cast(TypedObject object) {
256 | switch (object.type) {
257 | case LONG:
258 | object.data = new Timestamp((Long) object.data);
259 | break;
260 | case TIMESTAMP:
261 | break;
262 | case STRING:
263 | case DOUBLE:
264 | case DECIMAL:
265 | case BOOLEAN:
266 | return null;
267 | }
268 | object.type = TypeSystem.Type.TIMESTAMP;
269 | return object;
270 | }
271 | }
272 | }
273 |
--------------------------------------------------------------------------------