tableEvents = eventsPerTable.get(qName);
52 | if (tableEvents == null) {
53 | tableEvents = new ArrayList<>();
54 | eventsPerTable.put(qName, tableEvents);
55 | }
56 | tableEvents.add(e);
57 | }
58 | return eventsPerTable;
59 | }
60 |
61 | }
62 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/main/java/com/expediagroup/shuntingyard/replicator/exec/event/aggregation/DefaultMetaStoreEventCompactor.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.event.aggregation;
17 |
18 | import java.util.LinkedList;
19 | import java.util.List;
20 | import java.util.ListIterator;
21 | import java.util.function.Predicate;
22 |
23 | import org.slf4j.Logger;
24 | import org.slf4j.LoggerFactory;
25 |
26 | import com.expediagroup.shuntingyard.replicator.exec.event.MetaStoreEvent;
27 |
28 | class DefaultMetaStoreEventCompactor {
29 | private static final Logger log = LoggerFactory.getLogger(DefaultMetaStoreEventCompactor.class);
30 |
31 | private final EventMerger eventMerger;
32 |
33 | DefaultMetaStoreEventCompactor() {
34 | this(new EventMerger());
35 | }
36 |
37 | DefaultMetaStoreEventCompactor(EventMerger eventMerger) {
38 | this.eventMerger = eventMerger;
39 | }
40 |
41 | /**
42 | * Reduces the number of events to the minimum necessary to execute Circus Train as few times as possible.
43 | *
44 | * This method assumes all the events passed in are for the same table and are sorted in chronological order, i.e.
45 | * they come from the same source table.
46 | *
47 | * Basic rules:
48 | *
49 | * - Drop events are always kept untouched
50 | * - Any create/alter event before a drop event is removed from the final list of events
51 | * - Create and alter events are aggregated together
52 | * - If an alter table has been issue with the CASCADE the CASCADE option of the aggregated event will on as
53 | * well
54 | * - Parameters from previous events which are also in later events will be overwritten with the most recent value.
55 | * This statement is also true for the environment context
56 | *
57 | *
58 | * @param tableEvents a chronological ordered list of events on a single table
59 | * @return compacted set of events
60 | */
61 | public List compact(List tableEvents) {
62 | LinkedList finalEvents = new LinkedList<>();
63 | for (MetaStoreEvent e : tableEvents) {
64 | switch (e.getEventType()) {
65 | case DROP_TABLE:
66 | // Keep only previous drop events
67 | processDropTable(finalEvents, e);
68 | break;
69 | case DROP_PARTITION:
70 | mergeOrAdd(finalEvents, e, evt -> evt.isDropEvent());
71 | break;
72 | case CREATE_TABLE:
73 | case ADD_PARTITION:
74 | case ALTER_PARTITION:
75 | case INSERT:
76 | mergeOrAdd(finalEvents, e, evt -> !evt.isDropEvent());
77 | break;
78 | default:
79 | log.debug("Unknown event type {}: adding to list of event", e.getEventType());
80 | finalEvents.add(e);
81 | break;
82 | }
83 | }
84 | return finalEvents;
85 | }
86 |
87 | private void processDropTable(LinkedList finalEvents, MetaStoreEvent event) {
88 | ListIterator it = finalEvents.listIterator();
89 | while (it.hasNext()) {
90 | MetaStoreEvent e = it.next();
91 | if (e.isDropEvent()) {
92 | continue;
93 | }
94 | it.remove();
95 | }
96 | finalEvents.add(event);
97 | }
98 |
99 | private void mergeOrAdd(
100 | LinkedList events,
101 | MetaStoreEvent event,
102 | Predicate matchPredicate) {
103 | ListIterator it = events.listIterator(events.size());
104 | MetaStoreEvent previousEvent = null;
105 | boolean found = false;
106 | while (!found && it.hasPrevious()) {
107 | previousEvent = it.previous();
108 | found = matchPredicate.test(previousEvent);
109 | }
110 | if (found && eventMerger.canMerge(previousEvent, event)) {
111 | it.set(eventMerger.merge(previousEvent, event));
112 | } else {
113 | events.add(event);
114 | }
115 | }
116 |
117 | }
118 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/main/java/com/expediagroup/shuntingyard/replicator/exec/event/aggregation/EventMerger.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.event.aggregation;
17 |
18 | import static java.lang.Boolean.TRUE;
19 |
20 | import static org.apache.hadoop.hive.common.StatsSetupConst.CASCADE;
21 |
22 | import static com.google.common.base.Preconditions.checkArgument;
23 |
24 | import java.util.Objects;
25 |
26 | import com.expediagroup.shuntingyard.replicator.exec.event.MetaStoreEvent;
27 | import com.expediagroup.shuntingyard.replicator.exec.event.MetaStoreEvent.Builder;
28 | import com.google.common.collect.ImmutableMap;
29 |
30 | class EventMerger {
31 |
32 | EventMerger() {}
33 |
34 | public boolean canMerge(MetaStoreEvent a, MetaStoreEvent b) {
35 | return (Objects.equals(a.getEventType(), b.getEventType()) || (!a.isDropEvent() && !b.isDropEvent()))
36 | && Objects.equals(a.getQualifiedTableName(), b.getQualifiedTableName())
37 | && Objects.equals(a.getPartitionColumns(), b.getPartitionColumns());
38 | }
39 |
40 | // Note this method creates a new object each time it's invoked and this may end-up generating a lot
41 | // of garbage. However the number of expected events to be merged is low so this should not be an issue. Keep an eye
42 | // on this anyway.
43 | public MetaStoreEvent merge(MetaStoreEvent a, MetaStoreEvent b) {
44 | checkArgument(canMerge(a, b), "Events cannot be merged");
45 | // Event type of the first event is kept for the new event
46 | Builder builder = MetaStoreEvent
47 | .builder(a.getEventType(), a.getDatabaseName(), a.getTableName(), a.getReplicaDatabaseName(),
48 | a.getReplicaTableName())
49 | .parameters(a.getParameters())
50 | .parameters(b.getParameters())
51 | .environmentContext(a.getEnvironmentContext())
52 | .environmentContext(b.getEnvironmentContext());
53 | if (a.getPartitionColumns() != null) {
54 | builder.partitionColumns(a.getPartitionColumns());
55 | }
56 | if (a.getPartitionValues() != null) {
57 | a.getPartitionValues().stream().forEach(builder::partitionValues);
58 | }
59 | if (b.getPartitionValues() != null) {
60 | b.getPartitionValues().stream().forEach(builder::partitionValues);
61 | }
62 | // Ensure that cascade is preserved
63 | if (a.isCascade() || b.isCascade()) {
64 | builder.environmentContext(ImmutableMap.of(CASCADE, TRUE.toString()));
65 | }
66 | return builder.build();
67 | }
68 |
69 | }
70 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/main/java/com/expediagroup/shuntingyard/replicator/exec/event/aggregation/MetaStoreEventAggregator.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.event.aggregation;
17 |
18 | import java.util.List;
19 |
20 | import com.expediagroup.shuntingyard.replicator.exec.event.MetaStoreEvent;
21 |
22 | public interface MetaStoreEventAggregator {
23 |
24 | List aggregate(List events);
25 |
26 | }
27 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/main/java/com/expediagroup/shuntingyard/replicator/exec/external/Marshaller.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.external;
17 |
18 | import java.io.IOException;
19 | import java.io.OutputStreamWriter;
20 | import java.io.Writer;
21 |
22 | import org.apache.commons.vfs2.FileObject;
23 | import org.apache.commons.vfs2.FileSystemException;
24 | import org.apache.commons.vfs2.FileSystemManager;
25 | import org.apache.commons.vfs2.VFS;
26 | import org.yaml.snakeyaml.Yaml;
27 |
28 | import com.expediagroup.shuntingyard.replicator.yaml.YamlFactory;
29 | import com.google.common.base.Charsets;
30 |
31 | public class Marshaller {
32 |
33 | private final FileSystemManager fsManager;
34 | private final Yaml yaml;
35 |
36 | public Marshaller() {
37 | try {
38 | fsManager = VFS.getManager();
39 | } catch (FileSystemException e) {
40 | throw new RuntimeException("Unable to initialize Virtual File System", e);
41 | }
42 | yaml = YamlFactory.newYaml();
43 | }
44 |
45 | public void marshall(String configLocation, CircusTrainConfig config) {
46 | try (FileObject target = fsManager.resolveFile(configLocation);
47 | Writer writer = new OutputStreamWriter(target.getContent().getOutputStream(), Charsets.UTF_8)) {
48 | yaml.dump(config, writer);
49 | } catch (IOException e) {
50 | throw new RuntimeException("Unable to write Circus Train config to '" + configLocation + "'", e);
51 | }
52 | }
53 |
54 | }
55 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/main/java/com/expediagroup/shuntingyard/replicator/exec/launcher/CircusTrainRunner.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.launcher;
17 |
18 | import static org.apache.commons.exec.environment.EnvironmentUtils.getProcEnvironment;
19 |
20 | import static com.expediagroup.shuntingyard.replicator.exec.Constants.CIRCUS_TRAIN_HOME_ENV_VAR;
21 | import static com.expediagroup.shuntingyard.replicator.exec.Constants.CIRCUS_TRAIN_HOME_SCRIPT;
22 |
23 | import java.io.File;
24 | import java.io.FileOutputStream;
25 | import java.io.IOException;
26 | import java.io.OutputStream;
27 |
28 | import org.apache.commons.exec.CommandLine;
29 | import org.apache.commons.exec.DefaultExecutor;
30 | import org.apache.commons.exec.Executor;
31 | import org.apache.commons.exec.LogOutputStream;
32 | import org.apache.commons.exec.PumpStreamHandler;
33 | import org.apache.commons.io.output.TeeOutputStream;
34 | import org.apache.commons.lang3.StringUtils;
35 | import org.slf4j.Logger;
36 | import org.slf4j.LoggerFactory;
37 |
38 | import com.expediagroup.shuntingyard.replicator.exec.receiver.Context;
39 |
40 | import com.hotels.bdp.circustrain.api.CircusTrainException;
41 | import com.hotels.bdp.circustrain.api.conf.OrphanedDataStrategy;
42 |
43 | public class CircusTrainRunner {
44 | private static final Logger log = LoggerFactory.getLogger(CircusTrainRunner.class);
45 |
46 | public void run(Context context) {
47 | try (OutputStream out = outStream(context); OutputStream err = errStream(context)) {
48 | CommandLine cli = CommandLine
49 | .parse(String.format("%s/%s", getProcEnvironment().get(CIRCUS_TRAIN_HOME_ENV_VAR), CIRCUS_TRAIN_HOME_SCRIPT));
50 | cli.addArgument("--config=" + context.getConfigLocation());
51 |
52 | if (!StringUtils.isEmpty(context.getCircusTrainConfigLocation())) {
53 | cli.addArgument("--config=" + context.getCircusTrainConfigLocation());
54 | }
55 |
56 | String modules = "--modules=replication";
57 | if (context.getOrphanedDataStrategy() == OrphanedDataStrategy.HOUSEKEEPING) {
58 | modules = modules + ", housekeeping";
59 | }
60 | cli.addArgument(modules);
61 | log.info("Running Circus Train with orphaned data strategy: {}", context.getOrphanedDataStrategy());
62 | Executor executor = new DefaultExecutor();
63 | executor.setWorkingDirectory(new File(context.getWorkspace()));
64 | executor.setStreamHandler(new PumpStreamHandler(out, err));
65 |
66 | log.debug("Executing {} with environment {}", cli, getProcEnvironment());
67 | int returnValue = executor.execute(cli, getProcEnvironment());
68 | log.debug("Command exited with value {} ", returnValue);
69 | if (returnValue != 0) {
70 | throw new CircusTrainException("Circus Train exited with error value " + returnValue);
71 | }
72 | } catch (Throwable e) {
73 | log.error("Unable to execute Circus Train", e);
74 | }
75 | }
76 |
77 | private OutputStream errStream(Context context) throws IOException {
78 | OutputStream outputStream = new FileOutputStream(new File(context.getWorkspace(), "stderr.log"));
79 | return new TeeOutputStream(outputStream, logStream());
80 | }
81 |
82 | private OutputStream outStream(Context context) throws IOException {
83 | OutputStream log = new FileOutputStream(new File(context.getWorkspace(), "stdout.log"));
84 | return new TeeOutputStream(log, logStream());
85 | }
86 |
87 | private static LogOutputStream logStream() {
88 | return new LogOutputStream() {
89 | @Override
90 | protected void processLine(String line, int level) {
91 | // TODO
92 | System.out.println(line);
93 | }
94 | };
95 | }
96 |
97 | }
98 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/main/java/com/expediagroup/shuntingyard/replicator/exec/messaging/FilteringMessageReader.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.messaging;
17 |
18 | import java.io.IOException;
19 | import java.util.Optional;
20 |
21 | import org.slf4j.Logger;
22 | import org.slf4j.LoggerFactory;
23 |
24 | import com.expediagroup.shuntingyard.replicator.exec.receiver.TableSelector;
25 |
26 | import com.expedia.apiary.extensions.receiver.common.messaging.MessageEvent;
27 | import com.expedia.apiary.extensions.receiver.common.messaging.MessageReader;
28 |
29 | public class FilteringMessageReader implements MessageReader {
30 |
31 | private static final Logger log = LoggerFactory.getLogger(FilteringMessageReader.class);
32 |
33 | private final MessageReader delegate;
34 | private final TableSelector tableSelector;
35 |
36 | public FilteringMessageReader(MessageReader delegate, TableSelector tableSelector) {
37 | this.delegate = delegate;
38 | this.tableSelector = tableSelector;
39 | }
40 |
41 | @Override
42 | public Optional read() {
43 | Optional event = delegate.read();
44 | if (!event.isPresent()) {
45 | return Optional.empty();
46 | }
47 | MessageEvent messageEvent = event.get();
48 | if (tableSelector.canProcess(messageEvent.getEvent())) {
49 | return event;
50 | } else {
51 | delete(messageEvent);
52 | return Optional.empty();
53 | }
54 | }
55 |
56 | @Override
57 | public void delete(MessageEvent event) {
58 | try {
59 | delegate.delete(event);
60 | log.debug("Message deleted successfully");
61 | } catch (Exception e) {
62 | log.error("Could not delete message from queue: ", e);
63 | }
64 | }
65 |
66 | @Override
67 | public void close() throws IOException {
68 | delegate.close();
69 | }
70 |
71 | }
72 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/main/java/com/expediagroup/shuntingyard/replicator/exec/messaging/MetaStoreEventReader.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.messaging;
17 |
18 | import java.io.Closeable;
19 | import java.io.IOException;
20 | import java.util.Optional;
21 |
22 | import com.expediagroup.shuntingyard.replicator.exec.event.MetaStoreEvent;
23 |
24 | public interface MetaStoreEventReader extends Closeable {
25 |
26 | Optional read();
27 |
28 | }
29 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/main/java/com/expediagroup/shuntingyard/replicator/exec/receiver/Context.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.receiver;
17 |
18 | import com.hotels.bdp.circustrain.api.conf.OrphanedDataStrategy;
19 |
20 | public class Context {
21 |
22 | private final String workspace;
23 | private final String configLocation;
24 | private final String circusTrainConfigLocation;
25 | private final OrphanedDataStrategy orphanedDataStrategy;
26 |
27 | Context(String workspace, String configLocation, String circusTrainConfigLocation,
28 | OrphanedDataStrategy orphanedDataStrategy) {
29 | this.workspace = workspace;
30 | this.configLocation = configLocation;
31 | this.circusTrainConfigLocation = circusTrainConfigLocation;
32 | this.orphanedDataStrategy = orphanedDataStrategy;
33 | }
34 |
35 | public String getWorkspace() {
36 | return workspace;
37 | }
38 |
39 | public String getConfigLocation() {
40 | return configLocation;
41 | }
42 |
43 | public String getCircusTrainConfigLocation() {
44 | return circusTrainConfigLocation;
45 | }
46 |
47 | public OrphanedDataStrategy getOrphanedDataStrategy() {
48 | return orphanedDataStrategy;
49 | }
50 |
51 | }
52 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/main/java/com/expediagroup/shuntingyard/replicator/exec/receiver/ReplicationMetaStoreEventListener.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.receiver;
17 |
18 | import com.expediagroup.shuntingyard.replicator.exec.event.MetaStoreEvent;
19 |
20 | /**
21 | * A listener interface for processing {@link MetaStoreEvent}s
22 | */
23 | public interface ReplicationMetaStoreEventListener {
24 |
25 | void onEvent(MetaStoreEvent event);
26 |
27 | }
28 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/main/java/com/expediagroup/shuntingyard/replicator/exec/receiver/TableSelector.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.receiver;
17 |
18 | import java.util.List;
19 |
20 | import com.expediagroup.shuntingyard.replicator.exec.conf.SourceTableFilter;
21 |
22 | import com.expedia.apiary.extensions.receiver.common.event.ListenerEvent;
23 |
24 | public class TableSelector {
25 |
26 | private final List tableNames;
27 |
28 | public TableSelector(SourceTableFilter targetReplication) {
29 | this.tableNames = targetReplication.getTableNames();
30 | }
31 |
32 | public boolean canProcess(ListenerEvent listenerEvent) {
33 | String tableNameToBeProcessed = listenerEvent.getDbName() + "." + listenerEvent.getTableName();
34 | return tableNames.contains(tableNameToBeProcessed);
35 | }
36 |
37 | }
38 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/main/java/com/expediagroup/shuntingyard/replicator/metastore/DefaultMetaStoreClientSupplier.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.metastore;
17 |
18 | import org.apache.hadoop.hive.conf.HiveConf;
19 |
20 | import com.google.common.base.Supplier;
21 |
22 | import com.hotels.hcommon.hive.metastore.client.api.CloseableMetaStoreClient;
23 | import com.hotels.hcommon.hive.metastore.client.api.MetaStoreClientFactory;
24 |
25 | /*
26 | * Based on Circus Train
27 | */
28 | public class DefaultMetaStoreClientSupplier implements Supplier {
29 |
30 | private final MetaStoreClientFactory metaStoreClientFactory;
31 | private final HiveConf hiveConf;
32 |
33 | public DefaultMetaStoreClientSupplier(HiveConf hiveConf, MetaStoreClientFactory metaStoreClientFactory) {
34 | this.hiveConf = hiveConf;
35 | this.metaStoreClientFactory = metaStoreClientFactory;
36 | }
37 |
38 | @Override
39 | public CloseableMetaStoreClient get() {
40 | return metaStoreClientFactory.newInstance(hiveConf, "shunting-yard");
41 | }
42 |
43 | }
44 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/main/java/com/expediagroup/shuntingyard/replicator/util/TableDatabaseNameJoiner.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.util;
17 |
18 | public class TableDatabaseNameJoiner {
19 |
20 | private TableDatabaseNameJoiner() {}
21 |
22 | public static String dotJoin(String x, String y) {
23 | return String.join(".", x, y);
24 | }
25 |
26 | }
27 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/main/java/com/expediagroup/shuntingyard/replicator/yaml/AdvancedPropertyUtils.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.yaml;
17 |
18 | import java.beans.IntrospectionException;
19 | import java.beans.PropertyDescriptor;
20 | import java.beans.Transient;
21 | import java.lang.reflect.Field;
22 | import java.util.Collection;
23 | import java.util.Set;
24 | import java.util.TreeSet;
25 |
26 | import org.yaml.snakeyaml.introspector.BeanAccess;
27 | import org.yaml.snakeyaml.introspector.MethodProperty;
28 | import org.yaml.snakeyaml.introspector.Property;
29 | import org.yaml.snakeyaml.introspector.PropertyUtils;
30 |
31 | import com.google.common.base.CaseFormat;
32 |
33 | public class AdvancedPropertyUtils extends PropertyUtils {
34 |
35 | private boolean allowReadOnlyProperties = false;
36 |
37 | @Override
38 | public Property getProperty(Class type, String name) throws IntrospectionException {
39 | if (name.indexOf('-') > -1) {
40 | name = CaseFormat.LOWER_HYPHEN.to(CaseFormat.LOWER_CAMEL, name);
41 | }
42 | return super.getProperty(type, name);
43 | }
44 |
45 | @Override
46 | protected Set createPropertySet(Class type, BeanAccess beanAccess)
47 | throws IntrospectionException {
48 | Set properties = new TreeSet<>();
49 | Collection props = getPropertiesMap(type, beanAccess).values();
50 | for (Property property : props) {
51 | if (include(property)) {
52 | properties.add(property);
53 | }
54 | }
55 | return properties;
56 | }
57 |
58 | private synchronized boolean include(Property property) {
59 | boolean eligible = property.isReadable() && (allowReadOnlyProperties || property.isWritable());
60 | if (!eligible) {
61 | return false;
62 | }
63 | if (MethodProperty.class.isAssignableFrom(property.getClass())) {
64 | PropertyDescriptor propertyDescriptor = getPropertyDescriptor((MethodProperty) property);
65 | if (propertyDescriptor != null && isTransient(propertyDescriptor)) {
66 | return false;
67 | }
68 | }
69 | return true;
70 | }
71 |
72 | private synchronized PropertyDescriptor getPropertyDescriptor(MethodProperty methodProperty) {
73 | Field propertyField = null;
74 | try {
75 | propertyField = MethodProperty.class.getDeclaredField("property");
76 | propertyField.setAccessible(true);
77 | return (PropertyDescriptor) propertyField.get(methodProperty);
78 | } catch (Exception e) {
79 | return null;
80 | } finally {
81 | if (propertyField != null) {
82 | propertyField.setAccessible(false);
83 | }
84 | }
85 | }
86 |
87 | private boolean isTransient(PropertyDescriptor propertyDescriptor) {
88 | return (propertyDescriptor.getReadMethod() != null
89 | && propertyDescriptor.getReadMethod().getAnnotation(Transient.class) != null)
90 | || (propertyDescriptor.getWriteMethod() != null
91 | && propertyDescriptor.getWriteMethod().getAnnotation(Transient.class) != null);
92 | }
93 |
94 | @Override
95 | public void setAllowReadOnlyProperties(boolean allowReadOnlyProperties) {
96 | this.allowReadOnlyProperties = allowReadOnlyProperties;
97 | super.setAllowReadOnlyProperties(allowReadOnlyProperties);
98 | }
99 |
100 | }
101 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/main/java/com/expediagroup/shuntingyard/replicator/yaml/AdvancedRepresenter.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.yaml;
17 |
18 | import org.yaml.snakeyaml.introspector.Property;
19 | import org.yaml.snakeyaml.nodes.CollectionNode;
20 | import org.yaml.snakeyaml.nodes.MappingNode;
21 | import org.yaml.snakeyaml.nodes.Node;
22 | import org.yaml.snakeyaml.nodes.NodeTuple;
23 | import org.yaml.snakeyaml.nodes.SequenceNode;
24 | import org.yaml.snakeyaml.nodes.Tag;
25 | import org.yaml.snakeyaml.representer.Representer;
26 |
27 | import com.google.common.base.CaseFormat;
28 |
29 | public class AdvancedRepresenter extends Representer {
30 |
31 | @Override
32 | protected NodeTuple representJavaBeanProperty(
33 | Object javaBean,
34 | Property property,
35 | Object propertyValue,
36 | Tag customTag) {
37 | NodeTuple nodeTuple = super.representJavaBeanProperty(javaBean, property, propertyValue, customTag);
38 | Node valueNode = nodeTuple.getValueNode();
39 | if (Tag.NULL.equals(valueNode.getTag())) {
40 | return null; // skip 'null' values
41 | }
42 | if (valueNode instanceof CollectionNode) {
43 | if (Tag.SEQ.equals(valueNode.getTag())) {
44 | SequenceNode seq = (SequenceNode) valueNode;
45 | if (seq.getValue().isEmpty()) {
46 | return null; // skip empty lists
47 | }
48 | }
49 | if (Tag.MAP.equals(valueNode.getTag())) {
50 | MappingNode seq = (MappingNode) valueNode;
51 | if (seq.getValue().isEmpty()) {
52 | return null; // skip empty maps
53 | }
54 | }
55 | }
56 |
57 | Object name = CaseFormat.LOWER_CAMEL.to(CaseFormat.LOWER_HYPHEN, property.getName());
58 | return new NodeTuple(representData(name), valueNode);
59 | }
60 |
61 | }
62 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/main/java/com/expediagroup/shuntingyard/replicator/yaml/YamlFactory.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.yaml;
17 |
18 | import org.yaml.snakeyaml.DumperOptions;
19 | import org.yaml.snakeyaml.DumperOptions.FlowStyle;
20 | import org.yaml.snakeyaml.Yaml;
21 | import org.yaml.snakeyaml.constructor.Constructor;
22 | import org.yaml.snakeyaml.introspector.PropertyUtils;
23 | import org.yaml.snakeyaml.nodes.Tag;
24 | import org.yaml.snakeyaml.representer.Representer;
25 |
26 | import com.expediagroup.shuntingyard.replicator.exec.external.CircusTrainConfig;
27 |
28 | public final class YamlFactory {
29 |
30 | private YamlFactory() {}
31 |
32 | public static Yaml newYaml() {
33 | PropertyUtils propertyUtils = new AdvancedPropertyUtils();
34 | propertyUtils.setSkipMissingProperties(true);
35 | propertyUtils.setAllowReadOnlyProperties(false);
36 |
37 | Constructor constructor = new Constructor();
38 |
39 | Representer representer = new AdvancedRepresenter();
40 | representer.setPropertyUtils(propertyUtils);
41 | representer.addClassTag(CircusTrainConfig.class, Tag.MAP);
42 |
43 | DumperOptions dumperOptions = new DumperOptions();
44 | dumperOptions.setIndent(2);
45 | dumperOptions.setDefaultFlowStyle(FlowStyle.BLOCK);
46 | dumperOptions.setAllowReadOnlyProperties(true);
47 |
48 | return new Yaml(constructor, representer, dumperOptions);
49 | }
50 |
51 | }
52 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/test/java/com/expediagroup/shuntingyard/replicator/exec/ConfigFileValidationApplicationListenerTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec;
17 |
18 | import static org.mockito.Mockito.when;
19 |
20 | import java.io.File;
21 |
22 | import org.junit.Before;
23 | import org.junit.Rule;
24 | import org.junit.Test;
25 | import org.junit.rules.TemporaryFolder;
26 | import org.junit.runner.RunWith;
27 | import org.mockito.Mock;
28 | import org.mockito.junit.MockitoJUnitRunner;
29 | import org.springframework.boot.context.event.ApplicationEnvironmentPreparedEvent;
30 | import org.springframework.core.env.ConfigurableEnvironment;
31 |
32 | import com.expediagroup.shuntingyard.replicator.exec.ConfigFileValidationApplicationListener;
33 | import com.expediagroup.shuntingyard.replicator.exec.ConfigFileValidationException;
34 | import com.google.common.base.Joiner;
35 |
36 | @RunWith(MockitoJUnitRunner.class)
37 | public class ConfigFileValidationApplicationListenerTest {
38 |
39 | private static final String SPRING_CONFIG_LOCATION = "spring.config.location";
40 | private static final Joiner COMMA_JOINER = Joiner.on(",");
41 |
42 | public @Rule TemporaryFolder tmp = new TemporaryFolder();
43 |
44 | private @Mock ApplicationEnvironmentPreparedEvent event;
45 | private @Mock ConfigurableEnvironment environment;
46 |
47 | private final ConfigFileValidationApplicationListener listener = new ConfigFileValidationApplicationListener();
48 |
49 | @Before
50 | public void init() {
51 | when(event.getEnvironment()).thenReturn(environment);
52 | }
53 |
54 | @Test(expected = ConfigFileValidationException.class)
55 | public void configFileLocationIsNotSet() throws Exception {
56 | listener.onApplicationEvent(event);
57 | }
58 |
59 | @Test(expected = ConfigFileValidationException.class)
60 | public void configFileLocationIsBlank() throws Exception {
61 | when(environment.getProperty(SPRING_CONFIG_LOCATION)).thenReturn(" ");
62 | listener.onApplicationEvent(event);
63 | }
64 |
65 | @Test
66 | public void singleFileExists() throws Exception {
67 | File location = tmp.newFile();
68 | when(environment.getProperty(SPRING_CONFIG_LOCATION)).thenReturn(location.getAbsolutePath());
69 | listener.onApplicationEvent(event);
70 | }
71 |
72 | @Test(expected = ConfigFileValidationException.class)
73 | public void singleFileDoesNotExist() throws Exception {
74 | File location = new File(tmp.getRoot(), "trick");
75 | when(environment.getProperty(SPRING_CONFIG_LOCATION)).thenReturn(location.getAbsolutePath());
76 | listener.onApplicationEvent(event);
77 | }
78 |
79 | @Test(expected = ConfigFileValidationException.class)
80 | public void singleFileIsADirectory() throws Exception {
81 | when(environment.getProperty(SPRING_CONFIG_LOCATION)).thenReturn(tmp.getRoot().getAbsolutePath());
82 | listener.onApplicationEvent(event);
83 | }
84 |
85 | @Test
86 | public void multipleFilesExist() throws Exception {
87 | File location1 = tmp.newFile();
88 | File location2 = tmp.newFile();
89 | when(environment.getProperty(SPRING_CONFIG_LOCATION))
90 | .thenReturn(COMMA_JOINER.join(location1.getAbsolutePath(), location2.getAbsolutePath()));
91 | listener.onApplicationEvent(event);
92 | }
93 |
94 | @Test(expected = ConfigFileValidationException.class)
95 | public void multipleFilesAndOneDoesNotExist() throws Exception {
96 | File location1 = tmp.newFile();
97 | File location2 = new File(tmp.getRoot(), "trick");
98 | when(environment.getProperty(SPRING_CONFIG_LOCATION))
99 | .thenReturn(COMMA_JOINER.join(location1.getAbsolutePath(), location2.getAbsolutePath()));
100 | listener.onApplicationEvent(event);
101 | }
102 |
103 | @Test(expected = ConfigFileValidationException.class)
104 | public void multipleFilesAndOneIsADirectory() throws Exception {
105 | File location1 = tmp.newFile();
106 | File location2 = tmp.getRoot();
107 | when(environment.getProperty(SPRING_CONFIG_LOCATION))
108 | .thenReturn(COMMA_JOINER.join(location1.getAbsolutePath(), location2.getAbsolutePath()));
109 | listener.onApplicationEvent(event);
110 | }
111 |
112 | }
113 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/test/java/com/expediagroup/shuntingyard/replicator/exec/ConfigFileValidatorTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec;
17 |
18 | import java.io.File;
19 |
20 | import org.junit.Rule;
21 | import org.junit.Test;
22 | import org.junit.rules.TemporaryFolder;
23 |
24 | import com.expediagroup.shuntingyard.replicator.exec.ConfigFileValidationException;
25 | import com.expediagroup.shuntingyard.replicator.exec.ConfigFileValidator;
26 | import com.google.common.base.Joiner;
27 |
28 | public class ConfigFileValidatorTest {
29 |
30 | private static final Joiner COMMA_JOINER = Joiner.on(",");
31 |
32 | public @Rule TemporaryFolder tmp = new TemporaryFolder();
33 |
34 | @Test(expected = ConfigFileValidationException.class)
35 | public void configFileLocationIsNotSet() throws Exception {
36 | ConfigFileValidator.validate(null);
37 | }
38 |
39 | @Test(expected = ConfigFileValidationException.class)
40 | public void configFileLocationIsBlank() throws Exception {
41 | ConfigFileValidator.validate(" ");
42 | }
43 |
44 | @Test
45 | public void singleFileExists() throws Exception {
46 | File location = tmp.newFile();
47 | ConfigFileValidator.validate(location.getAbsolutePath());
48 | }
49 |
50 | @Test(expected = ConfigFileValidationException.class)
51 | public void singleFileDoesNotExist() throws Exception {
52 | File location = new File(tmp.getRoot(), "trick");
53 | ConfigFileValidator.validate(location.getAbsolutePath());
54 | }
55 |
56 | @Test(expected = ConfigFileValidationException.class)
57 | public void singleFileIsADirectory() throws Exception {
58 | ConfigFileValidator.validate(tmp.getRoot().getAbsolutePath());
59 | }
60 |
61 | @Test
62 | public void multipleFilesExist() throws Exception {
63 | File location1 = tmp.newFile();
64 | File location2 = tmp.newFile();
65 | ConfigFileValidator.validate(COMMA_JOINER.join(location1.getAbsolutePath(), location2.getAbsolutePath()));
66 | }
67 |
68 | @Test(expected = ConfigFileValidationException.class)
69 | public void multipleFilesAndOneDoesNotExist() throws Exception {
70 | File location1 = tmp.newFile();
71 | File location2 = new File(tmp.getRoot(), "trick");
72 | ConfigFileValidator.validate(COMMA_JOINER.join(location1.getAbsolutePath(), location2.getAbsolutePath()));
73 | }
74 |
75 | @Test(expected = ConfigFileValidationException.class)
76 | public void multipleFilesAndOneIsADirectory() throws Exception {
77 | File location1 = tmp.newFile();
78 | File location2 = tmp.getRoot();
79 | ConfigFileValidator.validate(COMMA_JOINER.join(location1.getAbsolutePath(), location2.getAbsolutePath()));
80 | }
81 |
82 | }
83 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/test/java/com/expediagroup/shuntingyard/replicator/exec/app/ConfigurationVariablesTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.app;
17 |
18 | import static org.assertj.core.api.Assertions.assertThat;
19 |
20 | import static com.expediagroup.shuntingyard.replicator.exec.app.ConfigurationVariables.CT_CONFIG;
21 | import static com.expediagroup.shuntingyard.replicator.exec.app.ConfigurationVariables.WORKSPACE;
22 |
23 | import org.junit.Test;
24 |
25 | import com.expediagroup.shuntingyard.replicator.exec.app.ConfigurationVariables;
26 |
27 | public class ConfigurationVariablesTest {
28 |
29 | private static String prefixedKey(String key) {
30 | return "com.hotels.shunting.yard.replicator.exec.app." + key;
31 | }
32 |
33 | @Test
34 | public void numberOfProperties() {
35 | assertThat(ConfigurationVariables.values().length).isEqualTo(2);
36 | }
37 |
38 | @Test
39 | public void workspace() {
40 | assertThat(WORKSPACE.unPrefixedKey()).isEqualTo("workspace");
41 | assertThat(WORKSPACE.key()).isEqualTo(prefixedKey("workspace"));
42 | assertThat(WORKSPACE.defaultValue()).isNull();
43 | }
44 |
45 | @Test
46 | public void circusTrainConfig() {
47 | assertThat(CT_CONFIG.unPrefixedKey()).isEqualTo("ct-config");
48 | assertThat(CT_CONFIG.key()).isEqualTo(prefixedKey("ct-config"));
49 | assertThat(CT_CONFIG.defaultValue()).isNull();
50 | }
51 |
52 | }
53 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/test/java/com/expediagroup/shuntingyard/replicator/exec/app/ReplicationRunnerTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.app;
17 |
18 | import static org.assertj.core.api.Assertions.assertThat;
19 | import static org.mockito.Mockito.atLeastOnce;
20 | import static org.mockito.Mockito.verify;
21 | import static org.mockito.Mockito.verifyZeroInteractions;
22 | import static org.mockito.Mockito.when;
23 |
24 | import java.util.Optional;
25 | import java.util.concurrent.ExecutorService;
26 | import java.util.concurrent.Executors;
27 | import java.util.concurrent.TimeUnit;
28 |
29 | import org.junit.Before;
30 | import org.junit.Test;
31 | import org.junit.runner.RunWith;
32 | import org.mockito.Mock;
33 | import org.mockito.junit.MockitoJUnitRunner;
34 | import org.springframework.boot.ApplicationArguments;
35 |
36 | import com.expediagroup.shuntingyard.replicator.exec.app.ReplicationRunner;
37 | import com.expediagroup.shuntingyard.replicator.exec.event.MetaStoreEvent;
38 | import com.expediagroup.shuntingyard.replicator.exec.messaging.MetaStoreEventReader;
39 | import com.expediagroup.shuntingyard.replicator.exec.receiver.ReplicationMetaStoreEventListener;
40 |
41 | @RunWith(MockitoJUnitRunner.class)
42 | public class ReplicationRunnerTest {
43 |
44 | private @Mock ApplicationArguments args;
45 | private @Mock MetaStoreEventReader eventReader;
46 | private @Mock ReplicationMetaStoreEventListener listener;
47 | private @Mock MetaStoreEvent event;
48 |
49 | private final ExecutorService executor = Executors.newFixedThreadPool(1);
50 | private ReplicationRunner runner;
51 |
52 | @Before
53 | public void init() {
54 | runner = new ReplicationRunner(eventReader, listener);
55 | }
56 |
57 | private class Runner implements Runnable {
58 |
59 | @Override
60 | public void run() {
61 | runner.run(args);
62 | }
63 |
64 | }
65 |
66 | private void runRunner() throws InterruptedException {
67 | executor.execute(new Runner());
68 | Thread.sleep(500);
69 | runner.stop();
70 | executor.awaitTermination(1, TimeUnit.SECONDS);
71 | }
72 |
73 | @Test
74 | public void exitCode() {
75 | assertThat(runner.getExitCode()).isEqualTo(0);
76 | }
77 |
78 | @Test
79 | public void onEvent() throws InterruptedException {
80 | when(eventReader.read()).thenReturn(Optional.of(event));
81 | runRunner();
82 | verify(listener, atLeastOnce()).onEvent(event);
83 | }
84 |
85 | @Test
86 | public void onEmptyEvent() throws InterruptedException {
87 | when(eventReader.read()).thenReturn(Optional.empty());
88 | runRunner();
89 | verifyZeroInteractions(listener);
90 | }
91 |
92 | @Test
93 | public void onEventProcessingFailure() throws InterruptedException {
94 | when(eventReader.read()).thenThrow(RuntimeException.class);
95 | runRunner();
96 | verifyZeroInteractions(listener);
97 | }
98 |
99 | }
100 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/test/java/com/expediagroup/shuntingyard/replicator/exec/conf/ShuntingYardReplicationsMapTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.conf;
17 |
18 | import static org.assertj.core.api.Assertions.assertThat;
19 |
20 | import java.util.ArrayList;
21 | import java.util.List;
22 |
23 | import org.junit.Before;
24 | import org.junit.Test;
25 |
26 | import com.expediagroup.shuntingyard.replicator.exec.conf.ShuntingYardTableReplicationsMap;
27 | import com.expediagroup.shuntingyard.replicator.exec.conf.ct.ShuntingYardTableReplication;
28 | import com.expediagroup.shuntingyard.replicator.exec.conf.ct.ShuntingYardTableReplications;
29 |
30 | import com.hotels.bdp.circustrain.api.conf.ReplicaTable;
31 | import com.hotels.bdp.circustrain.api.conf.SourceTable;
32 |
33 | public class ShuntingYardReplicationsMapTest {
34 |
35 | private static final String SOURCE_DATABASE = "DATABASE";
36 | private static final String SOURCE_TABLE = "TABLE";
37 | private static final String REPLICA_DATABASE = "replica_db";
38 | private static final String REPLICA_TABLE = "replica_tbl";
39 |
40 | private ShuntingYardTableReplicationsMap syTableReplications;
41 |
42 | @Before
43 | public void init() {
44 | ShuntingYardTableReplication tableReplication = new ShuntingYardTableReplication();
45 |
46 | SourceTable sourceTable = new SourceTable();
47 | sourceTable.setDatabaseName(SOURCE_DATABASE);
48 | sourceTable.setTableName(SOURCE_TABLE);
49 | tableReplication.setSourceTable(sourceTable);
50 |
51 | ReplicaTable replicaTable = new ReplicaTable();
52 | replicaTable.setDatabaseName(REPLICA_DATABASE);
53 | replicaTable.setTableName(REPLICA_TABLE);
54 | tableReplication.setReplicaTable(replicaTable);
55 |
56 | List tableReplications = new ArrayList<>();
57 | tableReplications.add(tableReplication);
58 |
59 | ShuntingYardTableReplications tableReplicationsWrapper = new ShuntingYardTableReplications();
60 | tableReplicationsWrapper.setTableReplications(tableReplications);
61 | syTableReplications = new ShuntingYardTableReplicationsMap(tableReplicationsWrapper);
62 | }
63 |
64 | @Test
65 | public void typical() {
66 | ShuntingYardTableReplication tableReplication = syTableReplications
67 | .getTableReplication(SOURCE_DATABASE, SOURCE_TABLE);
68 |
69 | assertThat(tableReplication.getReplicaDatabaseName()).isEqualTo(REPLICA_DATABASE);
70 | assertThat(tableReplication.getReplicaTableName()).isEqualTo(REPLICA_TABLE);
71 | }
72 |
73 | @Test
74 | public void queryMapWithLowerCaseSourceDatabaseAndTable() {
75 | ShuntingYardTableReplication tableReplication = syTableReplications
76 | .getTableReplication(SOURCE_DATABASE.toLowerCase(), SOURCE_TABLE.toLowerCase());
77 |
78 | assertThat(tableReplication.getReplicaDatabaseName()).isEqualTo(REPLICA_DATABASE);
79 | assertThat(tableReplication.getReplicaTableName()).isEqualTo(REPLICA_TABLE);
80 | }
81 |
82 | @Test
83 | public void emptyTableReplications() {
84 | syTableReplications = new ShuntingYardTableReplicationsMap(new ShuntingYardTableReplications());
85 | assertThat(syTableReplications.getTableReplication(SOURCE_DATABASE, SOURCE_TABLE)).isNull();
86 | }
87 |
88 | @Test
89 | public void nullTableReplications() {
90 | syTableReplications = new ShuntingYardTableReplicationsMap(null);
91 | assertThat(syTableReplications.getTableReplication(SOURCE_DATABASE, SOURCE_TABLE)).isNull();
92 | }
93 |
94 | }
95 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/test/java/com/expediagroup/shuntingyard/replicator/exec/event/aggregation/DefaultMetaStoreEventAggregatorTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.event.aggregation;
17 |
18 | import static org.assertj.core.api.Assertions.assertThat;
19 | import static org.mockito.Mockito.mock;
20 | import static org.mockito.Mockito.verify;
21 | import static org.mockito.Mockito.when;
22 |
23 | import java.util.ArrayList;
24 | import java.util.Arrays;
25 | import java.util.List;
26 |
27 | import org.junit.Before;
28 | import org.junit.Test;
29 | import org.junit.runner.RunWith;
30 | import org.mockito.Mock;
31 | import org.mockito.junit.MockitoJUnitRunner;
32 |
33 | import com.expediagroup.shuntingyard.replicator.exec.event.MetaStoreEvent;
34 | import com.expediagroup.shuntingyard.replicator.exec.event.aggregation.DefaultMetaStoreEventAggregator;
35 | import com.expediagroup.shuntingyard.replicator.exec.event.aggregation.DefaultMetaStoreEventCompactor;
36 |
37 | @RunWith(MockitoJUnitRunner.class)
38 | public class DefaultMetaStoreEventAggregatorTest {
39 |
40 | private static final String DATABASE = "database";
41 | private static final String TABLE1 = "table1";
42 | private static final String TABLE2 = "table2";
43 |
44 | private @Mock DefaultMetaStoreEventCompactor compactor;
45 |
46 | private DefaultMetaStoreEventAggregator aggregator;
47 |
48 | @Before
49 | public void init() {
50 | aggregator = new DefaultMetaStoreEventAggregator(compactor);
51 | }
52 |
53 | @Test
54 | public void eventsFromSingleTable() {
55 | List events = Arrays.asList(mockEvent(DATABASE, TABLE1), mockEvent(DATABASE, TABLE1));
56 | when(compactor.compact(events)).thenReturn(events);
57 | List aggregatedEvents = aggregator.aggregate(events);
58 | assertThat(aggregatedEvents).isEqualTo(events);
59 | verify(compactor).compact(events);
60 | }
61 |
62 | @Test
63 | public void eventsFromMultipleTables() {
64 | List table1Events = Arrays.asList(mockEvent(DATABASE, TABLE1));
65 | List table2Events = Arrays.asList(mockEvent(DATABASE, TABLE2));
66 | List events = new ArrayList<>();
67 | events.addAll(table1Events);
68 | events.addAll(table2Events);
69 | when(compactor.compact(table1Events)).thenReturn(table1Events);
70 | when(compactor.compact(table2Events)).thenReturn(table2Events);
71 | List aggregatedEvents = aggregator.aggregate(events);
72 | assertThat(aggregatedEvents).isEqualTo(events);
73 | verify(compactor).compact(table1Events);
74 | verify(compactor).compact(table2Events);
75 | }
76 |
77 | private static MetaStoreEvent mockEvent(String database, String table) {
78 | MetaStoreEvent event = mock(MetaStoreEvent.class);
79 | when(event.getQualifiedTableName()).thenReturn(database + "." + table);
80 | return event;
81 | }
82 | }
83 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/test/java/com/expediagroup/shuntingyard/replicator/exec/event/aggregation/DefaultMetaStoreEventCompactorTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.event.aggregation;
17 |
18 | import static java.util.Arrays.asList;
19 |
20 | import static org.assertj.core.api.Assertions.assertThat;
21 | import static org.mockito.Mockito.mock;
22 | import static org.powermock.api.mockito.PowerMockito.when;
23 |
24 | import static com.expedia.apiary.extensions.receiver.common.event.EventType.ADD_PARTITION;
25 | import static com.expedia.apiary.extensions.receiver.common.event.EventType.CREATE_TABLE;
26 | import static com.expedia.apiary.extensions.receiver.common.event.EventType.DROP_PARTITION;
27 | import static com.expedia.apiary.extensions.receiver.common.event.EventType.DROP_TABLE;
28 |
29 | import java.util.List;
30 |
31 | import org.junit.Before;
32 | import org.junit.Test;
33 | import org.junit.runner.RunWith;
34 | import org.mockito.Mock;
35 | import org.mockito.junit.MockitoJUnitRunner;
36 |
37 | import com.expediagroup.shuntingyard.replicator.exec.event.MetaStoreEvent;
38 | import com.expediagroup.shuntingyard.replicator.exec.event.aggregation.DefaultMetaStoreEventCompactor;
39 | import com.expediagroup.shuntingyard.replicator.exec.event.aggregation.EventMerger;
40 |
41 | import com.expedia.apiary.extensions.receiver.common.event.EventType;
42 |
43 | @RunWith(MockitoJUnitRunner.class)
44 | public class DefaultMetaStoreEventCompactorTest {
45 |
46 | private @Mock EventMerger merger;
47 |
48 | private DefaultMetaStoreEventCompactor compactor;
49 |
50 | @Before
51 | public void init() {
52 | compactor = new DefaultMetaStoreEventCompactor(merger);
53 | }
54 |
55 | @Test
56 | public void shouldReturnSameEventsIfEventsCannotBeMerged() {
57 | List events = asList(mockEvent(DROP_PARTITION), mockEvent(DROP_TABLE),
58 | mockEvent(CREATE_TABLE));
59 | List compactEvents = compactor.compact(events);
60 | assertThat(compactEvents).isEqualTo(events);
61 | }
62 |
63 | @Test
64 | public void mergeEventsIfEventsCanBeMerged() {
65 | MetaStoreEvent eventA = mockEvent(DROP_PARTITION);
66 | MetaStoreEvent eventB = mockEvent(DROP_TABLE);
67 | MetaStoreEvent eventC = mockEvent(CREATE_TABLE);
68 | MetaStoreEvent eventD = mockEvent(ADD_PARTITION);
69 | MetaStoreEvent eventE = mockEvent(ADD_PARTITION);
70 |
71 | when(merger.canMerge(eventC, eventD)).thenReturn(true);
72 | MetaStoreEvent eventCD = mockEvent(CREATE_TABLE);
73 | when(merger.merge(eventC, eventD)).thenReturn(eventCD);
74 | when(merger.canMerge(eventCD, eventE)).thenReturn(true);
75 | MetaStoreEvent eventCDE = mockEvent(CREATE_TABLE);
76 | when(merger.merge(eventCD, eventE)).thenReturn(eventCDE);
77 |
78 | List compactEvents = compactor.compact(asList(eventA, eventB, eventC, eventD, eventE));
79 |
80 | assertThat(compactEvents).isEqualTo(asList(eventA, eventB, eventCDE));
81 | }
82 |
83 | @Test
84 | public void mergeDiscardPreviousEventsIfADropTableIsFoundInBetween() {
85 | MetaStoreEvent eventA = mockEvent(CREATE_TABLE);
86 | MetaStoreEvent eventB = mockEvent(ADD_PARTITION);
87 | MetaStoreEvent eventC = mockEvent(DROP_TABLE);
88 | MetaStoreEvent eventD = mockEvent(CREATE_TABLE);
89 | MetaStoreEvent eventE = mockEvent(ADD_PARTITION);
90 |
91 | when(merger.canMerge(eventA, eventB)).thenReturn(true);
92 | MetaStoreEvent eventAB = mockEvent(CREATE_TABLE);
93 | when(merger.merge(eventA, eventB)).thenReturn(eventAB);
94 |
95 | when(merger.canMerge(eventD, eventE)).thenReturn(true);
96 | MetaStoreEvent eventDE = mockEvent(CREATE_TABLE);
97 | when(merger.merge(eventD, eventE)).thenReturn(eventDE);
98 |
99 | List compactEvents = compactor.compact(asList(eventA, eventB, eventC, eventD, eventE));
100 |
101 | assertThat(compactEvents).isEqualTo(asList(eventC, eventDE));
102 | }
103 |
104 | private static MetaStoreEvent mockEvent(EventType eventType) {
105 | MetaStoreEvent event = mock(MetaStoreEvent.class);
106 | when(event.getEventType()).thenReturn(eventType);
107 | if (eventType == DROP_PARTITION || eventType == DROP_TABLE) {
108 | when(event.isDropEvent()).thenReturn(true);
109 | }
110 | return event;
111 | }
112 |
113 | }
114 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/test/java/com/expediagroup/shuntingyard/replicator/exec/external/MarshallerTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.external;
17 |
18 | import static org.assertj.core.api.Assertions.assertThat;
19 |
20 | import java.io.File;
21 | import java.nio.charset.StandardCharsets;
22 | import java.util.Arrays;
23 |
24 | import org.junit.Rule;
25 | import org.junit.Test;
26 | import org.junit.rules.TemporaryFolder;
27 |
28 | import com.google.common.io.Files;
29 |
30 | import com.hotels.bdp.circustrain.api.conf.OrphanedDataStrategy;
31 | import com.hotels.bdp.circustrain.api.conf.ReplicationMode;
32 |
33 | public class MarshallerTest {
34 |
35 | public @Rule TemporaryFolder tmp = new TemporaryFolder();
36 |
37 | private final Marshaller marshaller = new Marshaller();
38 |
39 | @Test
40 | public void typical() throws Exception {
41 | CircusTrainConfig config = CircusTrainConfig
42 | .builder()
43 | .sourceName("sourceName")
44 | .sourceMetaStoreUri("sourceMetaStoreUri")
45 | .replicaName("replicaName")
46 | .replicaMetaStoreUri("replicaMetaStoreUri")
47 | .copierOption("p1", "val1")
48 | .copierOption("p2", "val2")
49 | .replication(ReplicationMode.FULL, "databaseName", "tableName", "replicaDatabaseName", "replicaTableName",
50 | "replicaTableLocation", Arrays.asList("part"), Arrays.asList(Arrays.asList("partval")),
51 | OrphanedDataStrategy.NONE)
52 | .build();
53 |
54 | File file = tmp.newFile("conif.yml");
55 | marshaller.marshall(file.getAbsolutePath(), config);
56 | assertThat(Files.readLines(file, StandardCharsets.UTF_8))
57 | .contains("source-catalog:")
58 | .contains(" disable-snapshots: false")
59 | .contains(" name: sourceName")
60 | .contains(" hive-metastore-uris: sourceMetaStoreUri")
61 | .contains("replica-catalog:")
62 | .contains(" name: replicaName")
63 | .contains(" hive-metastore-uris: replicaMetaStoreUri")
64 | .contains("copier-options:")
65 | .contains(" p1: val1")
66 | .contains(" p2: val2")
67 | .contains("table-replications:")
68 | .contains("- orphaned-data-strategy: NONE")
69 | .contains(" replication-mode: FULL")
70 | .contains(" source-table:")
71 | .contains(" database-name: databaseName")
72 | .contains(" table-name: tableName")
73 | .contains(" partition-filter: (part='partval')")
74 | .contains(" partition-limit: 32767")
75 | .contains(" replica-table:")
76 | .contains(" database-name: replicaDatabaseName")
77 | .contains(" table-name: replicaTableName")
78 | .contains(" table-location: replicaTableLocation");
79 | }
80 |
81 | @Test
82 | public void generatePartitionFilter() throws Exception {
83 | CircusTrainConfig config = CircusTrainConfig
84 | .builder()
85 | .sourceName("sourceName")
86 | .sourceMetaStoreUri("sourceMetaStoreUri")
87 | .replicaName("replicaName")
88 | .replicaMetaStoreUri("replicaMetaStoreUri")
89 | .copierOption("p1", "val1")
90 | .copierOption("p2", "val2")
91 | .replication(ReplicationMode.FULL, "databaseName", "tableName", "replicaDatabaseName", "replicaTableName",
92 | "replicaTableLocation", OrphanedDataStrategy.HOUSEKEEPING)
93 | .build();
94 |
95 | File file = tmp.newFile("conif.yml");
96 | marshaller.marshall(file.getAbsolutePath(), config);
97 | assertThat(Files.readLines(file, StandardCharsets.UTF_8))
98 | .contains("source-catalog:")
99 | .contains(" disable-snapshots: false")
100 | .contains(" name: sourceName")
101 | .contains(" hive-metastore-uris: sourceMetaStoreUri")
102 | .contains("replica-catalog:")
103 | .contains(" name: replicaName")
104 | .contains(" hive-metastore-uris: replicaMetaStoreUri")
105 | .contains("copier-options:")
106 | .contains(" p1: val1")
107 | .contains(" p2: val2")
108 | .contains("table-replications:")
109 | .contains("- orphaned-data-strategy: HOUSEKEEPING")
110 | .contains(" replication-mode: FULL")
111 | .contains(" source-table:")
112 | .contains(" database-name: databaseName")
113 | .contains(" table-name: tableName")
114 | .contains(" generate-partition-filter: true")
115 | .contains(" partition-limit: 32767")
116 | .contains(" replica-table:")
117 | .contains(" database-name: replicaDatabaseName")
118 | .contains(" table-name: replicaTableName")
119 | .contains(" table-location: replicaTableLocation");
120 | }
121 |
122 | }
123 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/test/java/com/expediagroup/shuntingyard/replicator/exec/receiver/TableSelectorTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.exec.receiver;
17 |
18 | import static org.assertj.core.api.Assertions.assertThat;
19 | import static org.mockito.Mockito.when;
20 |
21 | import java.util.Arrays;
22 |
23 | import org.junit.Before;
24 | import org.junit.Test;
25 | import org.junit.runner.RunWith;
26 | import org.mockito.Mock;
27 | import org.mockito.junit.MockitoJUnitRunner;
28 |
29 | import com.expediagroup.shuntingyard.replicator.exec.conf.SourceTableFilter;
30 | import com.expediagroup.shuntingyard.replicator.exec.receiver.TableSelector;
31 |
32 | import com.expedia.apiary.extensions.receiver.common.event.ListenerEvent;
33 |
34 | @RunWith(MockitoJUnitRunner.class)
35 | public class TableSelectorTest {
36 |
37 | private static final String DB_NAME = "test_db";
38 | private static final String TABLE_NAME = "test_table";
39 |
40 | private TableSelector tableSelector;
41 | private @Mock ListenerEvent listenerEvent;
42 |
43 | @Before
44 | public void init() {
45 | when(listenerEvent.getDbName()).thenReturn(DB_NAME);
46 | when(listenerEvent.getTableName()).thenReturn(TABLE_NAME);
47 |
48 | SourceTableFilter targetReplication = new SourceTableFilter();
49 | targetReplication.setTableNames(Arrays.asList(DB_NAME + "." + TABLE_NAME, "db1.table1"));
50 |
51 | tableSelector = new TableSelector(targetReplication);
52 | }
53 |
54 | @Test
55 | public void returnsTrueWhenTableSelected() {
56 | assertThat(tableSelector.canProcess(listenerEvent)).isTrue();
57 | }
58 |
59 | @Test
60 | public void returnsFalseEventWhenTableNotSelected() {
61 | when(listenerEvent.getTableName()).thenReturn(TABLE_NAME + "1");
62 | assertThat(tableSelector.canProcess(listenerEvent)).isFalse();
63 | }
64 |
65 | }
66 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/test/java/com/expediagroup/shuntingyard/replicator/metastore/DefaultMetaStoreClientSupplierTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.metastore;
17 |
18 | import static org.assertj.core.api.Assertions.assertThat;
19 | import static org.mockito.ArgumentMatchers.anyString;
20 | import static org.mockito.ArgumentMatchers.eq;
21 | import static org.mockito.Mockito.when;
22 |
23 | import org.apache.hadoop.hive.conf.HiveConf;
24 | import org.junit.Before;
25 | import org.junit.Test;
26 | import org.junit.runner.RunWith;
27 | import org.mockito.Mock;
28 | import org.mockito.junit.MockitoJUnitRunner;
29 |
30 | import com.expediagroup.shuntingyard.replicator.metastore.DefaultMetaStoreClientSupplier;
31 |
32 | import com.hotels.hcommon.hive.metastore.client.api.CloseableMetaStoreClient;
33 | import com.hotels.hcommon.hive.metastore.client.api.MetaStoreClientFactory;
34 |
35 | @RunWith(MockitoJUnitRunner.class)
36 | public class DefaultMetaStoreClientSupplierTest {
37 |
38 | private @Mock MetaStoreClientFactory metaStoreClientFactory;
39 | private @Mock CloseableMetaStoreClient metaStoreClient;
40 |
41 | private final HiveConf hiveConf = new HiveConf();
42 | private DefaultMetaStoreClientSupplier supplier;
43 |
44 | @Before
45 | public void init() {
46 | supplier = new DefaultMetaStoreClientSupplier(hiveConf, metaStoreClientFactory);
47 | }
48 |
49 | @Test
50 | public void get() {
51 | when(metaStoreClientFactory.newInstance(eq(hiveConf), anyString())).thenReturn(metaStoreClient);
52 | assertThat(supplier.get()).isSameAs(metaStoreClient);
53 | }
54 |
55 | @Test(expected = NullPointerException.class)
56 | public void bubbleUpExceptions() {
57 | when(metaStoreClientFactory.newInstance(eq(hiveConf), anyString())).thenThrow(NullPointerException.class);
58 | supplier.get();
59 | }
60 |
61 | }
62 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/test/java/com/expediagroup/shuntingyard/replicator/util/TableDatabaseNameJoinerTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.util;
17 |
18 | import static org.assertj.core.api.Assertions.assertThat;
19 |
20 | import org.junit.Test;
21 |
22 | import com.expediagroup.shuntingyard.replicator.util.TableDatabaseNameJoiner;
23 |
24 | public class TableDatabaseNameJoinerTest {
25 |
26 | @Test
27 | public void typical() {
28 | String databaseName = "database";
29 | String tableName = "table";
30 |
31 | assertThat(TableDatabaseNameJoiner.dotJoin(databaseName, tableName)).isEqualTo("database.table");
32 | }
33 |
34 | @Test
35 | public void nullInput() {
36 | String databaseName = "database";
37 | String tableName = null;
38 |
39 | assertThat(TableDatabaseNameJoiner.dotJoin(databaseName, tableName)).isEqualTo("database.null");
40 | }
41 |
42 | @Test
43 | public void blankInput() {
44 | String databaseName = "database";
45 | String tableName = "";
46 |
47 | assertThat(TableDatabaseNameJoiner.dotJoin(databaseName, tableName)).isEqualTo("database.");
48 | }
49 |
50 | }
51 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/test/java/com/expediagroup/shuntingyard/replicator/yaml/AdvancedPropertyUtilsTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.yaml;
17 |
18 | import static org.assertj.core.api.Assertions.assertThat;
19 |
20 | import java.beans.Transient;
21 | import java.util.Iterator;
22 | import java.util.Set;
23 |
24 | import org.junit.Test;
25 | import org.yaml.snakeyaml.error.YAMLException;
26 | import org.yaml.snakeyaml.introspector.BeanAccess;
27 | import org.yaml.snakeyaml.introspector.Property;
28 |
29 | import com.expediagroup.shuntingyard.replicator.yaml.AdvancedPropertyUtils;
30 |
31 | public class AdvancedPropertyUtilsTest {
32 |
33 | static class TestBean {
34 | private String longPropertyName;
35 | public transient int transientField;
36 | private char transientProperty;
37 |
38 | public String getLongPropertyName() {
39 | return longPropertyName;
40 | }
41 |
42 | public void setLongPropertyName(String longPropertyName) {
43 | this.longPropertyName = longPropertyName;
44 | }
45 |
46 | @Transient
47 | public char getTransientProperty() {
48 | return transientProperty;
49 | }
50 |
51 | @Transient
52 | public void setTransientProperty(char transientProperty) {
53 | this.transientProperty = transientProperty;
54 | }
55 | }
56 |
57 | private final AdvancedPropertyUtils properyUtils = new AdvancedPropertyUtils();
58 |
59 | @Test
60 | public void regularPropertyName() throws Exception {
61 | Property property = properyUtils.getProperty(TestBean.class, "longPropertyName");
62 | assertThat(property).isNotNull();
63 | assertThat(property.getName()).isEqualTo("longPropertyName");
64 | }
65 |
66 | @Test
67 | public void lowerHyphenPropertyName() throws Exception {
68 | Property property = properyUtils.getProperty(TestBean.class, "long-property-name");
69 | assertThat(property).isNotNull();
70 | assertThat(property.getName()).isEqualTo("longPropertyName");
71 | }
72 |
73 | @Test(expected = YAMLException.class)
74 | public void illegalPropertyName() throws Exception {
75 | properyUtils.getProperty(TestBean.class, "unknown");
76 | }
77 |
78 | @Test
79 | public void createPropertySetWithDefaultBeanAccess() throws Exception {
80 | Set properties = properyUtils.createPropertySet(TestBean.class, BeanAccess.DEFAULT);
81 | assertThat(properties.size()).isEqualTo(1);
82 | assertThat(properties.iterator().next().getName()).isEqualTo("longPropertyName");
83 | }
84 |
85 | @Test
86 | public void createPropertySetWithFieldBeanAccess() throws Exception {
87 | Set properties = properyUtils.createPropertySet(TestBean.class, BeanAccess.FIELD);
88 | assertThat(properties.size()).isEqualTo(2);
89 | Iterator iterator = properties.iterator();
90 | assertThat(iterator.next().getName()).isEqualTo("longPropertyName");
91 | assertThat(iterator.next().getName()).isEqualTo("transientProperty");
92 | }
93 |
94 | @Test
95 | public void createPropertySetWithPropertyBeanAccess() throws Exception {
96 | Set properties = properyUtils.createPropertySet(TestBean.class, BeanAccess.PROPERTY);
97 | assertThat(properties.size()).isEqualTo(1);
98 | assertThat(properties.iterator().next().getName()).isEqualTo("longPropertyName");
99 | }
100 |
101 | }
102 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/test/java/com/expediagroup/shuntingyard/replicator/yaml/YamlFactoryTest.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (C) 2016-2019 Expedia, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.expediagroup.shuntingyard.replicator.yaml;
17 |
18 | import static org.assertj.core.api.Assertions.assertThat;
19 |
20 | import java.io.StringWriter;
21 |
22 | import org.junit.Test;
23 | import org.yaml.snakeyaml.Yaml;
24 |
25 | import com.expediagroup.shuntingyard.replicator.exec.external.CircusTrainConfig;
26 |
27 | import com.hotels.bdp.circustrain.api.conf.OrphanedDataStrategy;
28 | import com.hotels.bdp.circustrain.api.conf.ReplicationMode;
29 |
30 | public class YamlFactoryTest {
31 |
32 | @Test
33 | public void typical() {
34 | String expectedYaml = new StringBuffer()
35 | .append("replica-catalog:\n")
36 | .append(" hive-metastore-uris: replicaMetaStoreUri\n")
37 | .append(" name: replica\n")
38 | .append("source-catalog:\n")
39 | .append(" disable-snapshots: false\n")
40 | .append(" hive-metastore-uris: sourceMetaStoreUri\n")
41 | .append(" name: source\n")
42 | .append("table-replications:\n")
43 | .append("- orphaned-data-strategy: HOUSEKEEPING\n")
44 | .append(" partition-fetcher-buffer-size: 1000\n")
45 | .append(" partition-iterator-batch-size: 1000\n")
46 | .append(" qualified-replica-name: replicadatabasename.replicatablename\n")
47 | .append(" replica-database-name: replicadatabasename\n")
48 | .append(" replica-table:\n")
49 | .append(" database-name: replicaDatabaseName\n")
50 | .append(" table-location: replicaTableLocation\n")
51 | .append(" table-name: replicaTableName\n")
52 | .append(" replica-table-name: replicatablename\n")
53 | .append(" replication-mode: FULL\n")
54 | .append(" replication-strategy: UPSERT\n")
55 | .append(" source-table:\n")
56 | .append(" database-name: databaseName\n")
57 | .append(" generate-partition-filter: true\n")
58 | .append(" partition-limit: 32767\n")
59 | .append(" qualified-name: databasename.tablename\n")
60 | .append(" table-name: tableName\n")
61 | .toString();
62 | StringWriter sw = new StringWriter();
63 | CircusTrainConfig circusTrainConfig = CircusTrainConfig
64 | .builder()
65 | .sourceMetaStoreUri("sourceMetaStoreUri")
66 | .replicaMetaStoreUri("replicaMetaStoreUri")
67 | .replication(ReplicationMode.FULL, "databaseName", "tableName", "replicaDatabaseName", "replicaTableName",
68 | "replicaTableLocation", OrphanedDataStrategy.HOUSEKEEPING)
69 | .build();
70 | Yaml yaml = YamlFactory.newYaml();
71 | yaml.dump(circusTrainConfig, sw);
72 | assertThat(sw.toString()).isEqualTo(expectedYaml);
73 | }
74 |
75 | }
76 |
--------------------------------------------------------------------------------
/shunting-yard-replicator/src/test/resources/log4j.xml:
--------------------------------------------------------------------------------
1 |
2 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
--------------------------------------------------------------------------------
/shunting-yard.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ExpediaGroup/shunting-yard/369ac242640b24121fd661d40ae94a0196ced0fd/shunting-yard.png
--------------------------------------------------------------------------------