hooks = new ArrayList<>(prePanicHooks);
46 | Collections.reverse(hooks);
47 | hooks.forEach(it -> {
48 | try {
49 | it.run();
50 | } catch (Throwable t) {
51 | log.error("Pre-panic hook threw exception.", t);
52 | }
53 | });
54 | } catch (Throwable t) {
55 | log.error("Failed to invoke pre-panic hooks.", t);
56 | }
57 | }
58 |
59 | public void mildPanic(String message) {
60 | try {
61 | runPrePanicHooks();
62 |
63 | if (runningInKubernetes()) {
64 | writeTerminationMessage(message);
65 | }
66 |
67 | log.warn("*** TERMINATING: {}", message);
68 |
69 | } finally {
70 | System.exit(1);
71 | }
72 | }
73 |
74 | @Override
75 | public void panic(String message, Throwable t) {
76 | //noinspection finally
77 | try {
78 | runPrePanicHooks();
79 |
80 | if (runningInKubernetes()) {
81 | writeTerminationMessage(t == null ? message : message + "\n" + getStackTraceAsString(t));
82 | }
83 |
84 | log.error("PANIC: {}", message, t);
85 |
86 | } finally {
87 | System.exit(1);
88 | }
89 | }
90 |
91 | private static boolean runningInKubernetes() {
92 | return System.getenv("KUBERNETES_SERVICE_HOST") != null;
93 | }
94 |
95 | private static void writeTerminationMessage(String msg) {
96 | String path = "/dev/termination-log";
97 | try (Writer w = new OutputStreamWriter(new FileOutputStream(path))) {
98 | w.write(msg + "\n");
99 | } catch (Exception e) {
100 | log.warn("Failed to write termination message to {}", path, e);
101 | }
102 | }
103 | }
104 |
--------------------------------------------------------------------------------
/src/main/java/com/couchbase/connector/util/RuntimeHelper.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Couchbase, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.couchbase.connector.util;
18 |
19 | import java.time.Duration;
20 | import java.time.Instant;
21 | import java.util.ArrayList;
22 | import java.util.List;
23 |
24 | import static java.util.Objects.requireNonNull;
25 | import static java.util.concurrent.TimeUnit.NANOSECONDS;
26 |
27 | /**
28 | * Runs shutdown hooks with a timeout.
29 | *
30 | * Prevents stuck hooks from delaying or preventing JVM termination.
31 | */
32 | public class RuntimeHelper {
33 | private RuntimeHelper() {
34 | throw new AssertionError("not instantiable");
35 | }
36 |
37 | private static final Duration shutdownHookTimeout = Duration.ofSeconds(3);
38 |
39 | private static final List managedShutdownHooks = new ArrayList<>();
40 |
41 | static {
42 | // Register an "umbrella" hook that starts the other hooks
43 | // and halts the JVM if they take too long.
44 | Runtime.getRuntime().addShutdownHook(new Thread(() -> {
45 | try {
46 | synchronized (managedShutdownHooks) {
47 | for (Thread t : managedShutdownHooks) {
48 | t.start();
49 | }
50 |
51 | final long deadlineNanos = System.nanoTime() + shutdownHookTimeout.toNanos();
52 |
53 | for (Thread t : managedShutdownHooks) {
54 | // prevent joining for 0 milliseconds, which would mean "wait forever."
55 | final long remainingMillis = Math.max(1, NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()));
56 | t.join(remainingMillis);
57 | if (t.isAlive()) {
58 | // logging has likely been shut down by this point, so use stderr
59 | System.err.println(Instant.now() + " Shutdown hook failed to terminate within " + shutdownHookTimeout + " : " + t);
60 | halt();
61 | }
62 | }
63 | }
64 | } catch (Throwable t) {
65 | // logging has likely been shut down by this point, so use stderr
66 | t.printStackTrace();
67 | halt();
68 | }
69 | }));
70 | }
71 |
72 | /**
73 | * Immediately terminate the JVM process, without running any shutdown hooks or finalizers.
74 | */
75 | private static void halt() {
76 | // logging has likely been shut down by this point, so use stderr
77 | System.err.println("Halting.");
78 | Runtime.getRuntime().halt(-1);
79 | }
80 |
81 | public static void addShutdownHook(Thread hook) {
82 | requireNonNull(hook);
83 | synchronized (managedShutdownHooks) {
84 | managedShutdownHooks.add(hook);
85 | }
86 | }
87 |
88 | public static boolean removeShutdownHook(Thread hook) {
89 | requireNonNull(hook);
90 | synchronized (managedShutdownHooks) {
91 | return managedShutdownHooks.remove(hook);
92 | }
93 | }
94 | }
95 |
--------------------------------------------------------------------------------
/src/main/java/com/couchbase/connector/cluster/consul/ConsulDocumentWatcher.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2019 Couchbase, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.couchbase.connector.cluster.consul;
18 |
19 | import com.couchbase.consul.ConsulOps;
20 | import com.couchbase.consul.KvReadResult;
21 | import reactor.core.publisher.Flux;
22 |
23 | import java.time.Duration;
24 | import java.util.Optional;
25 | import java.util.concurrent.TimeoutException;
26 | import java.util.function.Function;
27 | import java.util.function.Predicate;
28 |
29 | import static com.couchbase.connector.cluster.consul.ReactorHelper.await;
30 | import static java.util.Objects.requireNonNull;
31 |
32 | public class ConsulDocumentWatcher {
33 | private final ConsulOps consul;
34 | private final ConsulResourceWatcher resourceWatcher;
35 |
36 | public ConsulDocumentWatcher(ConsulOps consul) {
37 | this(consul, Duration.ofMinutes(5));
38 | }
39 |
40 | private ConsulDocumentWatcher(ConsulOps consul, Duration pollingInterval) {
41 | this.consul = requireNonNull(consul);
42 | this.resourceWatcher = new ConsulResourceWatcher().withPollingInterval(pollingInterval);
43 | }
44 |
45 | public ConsulDocumentWatcher withPollingInterval(Duration pollingInterval) {
46 | return new ConsulDocumentWatcher(this.consul, pollingInterval);
47 | }
48 |
49 | public Optional awaitCondition(String key, Predicate> valueCondition) throws InterruptedException {
50 | return await(watch(key), valueCondition);
51 | }
52 |
53 | public Optional awaitCondition(String key, Function valueMapper, Predicate> condition)
54 | throws InterruptedException {
55 | return await(watch(key).map(s -> s.map(valueMapper)), condition);
56 | }
57 |
58 | public Optional awaitCondition(String key, Function valueMapper, Predicate> condition, Duration timeout)
59 | throws InterruptedException, TimeoutException {
60 | return await(watch(key).map(s -> s.map(valueMapper)), condition, timeout);
61 | }
62 |
63 | /**
64 | * Blocks until the document does not exist or the current thread is interrupted.
65 | *
66 | * @param key the document to watch
67 | * @throws InterruptedException if the current thread is interrupted while waiting for document state to change.
68 | */
69 | public void awaitAbsence(String key) throws InterruptedException {
70 | awaitCondition(key, Optional::isEmpty);
71 | }
72 |
73 | public Optional awaitValueChange(String key, String valueBeforeChange) throws InterruptedException {
74 | return awaitCondition(key, doc -> !doc.equals(Optional.ofNullable(valueBeforeChange)));
75 | }
76 |
77 | public Flux> watch(String key) {
78 | requireNonNull(key);
79 | return resourceWatcher.watch(opts -> consul.kv().readOneKey(key, opts))
80 | .map(it -> it.body().map(KvReadResult::valueAsString))
81 | .distinctUntilChanged();
82 | }
83 | }
84 |
--------------------------------------------------------------------------------
/README.adoc:
--------------------------------------------------------------------------------
1 | = Couchbase Elasticsearch Connector
2 |
3 | https://docs.couchbase.com/elasticsearch-connector/4.4/release-notes.html[*Download*]
4 | | https://docs.couchbase.com/elasticsearch-connector/4.4/index.html[*Documentation*]
5 | | https://issues.couchbase.com/projects/CBES[*Issues*]
6 | | https://forums.couchbase.com/c/elasticsearch-connector[*Discussion*]
7 |
8 | The Couchbase Elasticsearch Connector replicates your documents from Couchbase Server to Elasticsearch in near real time.
9 | The connector uses the high-performance Database Change Protocol (DCP) to receive notifications when documents change in Couchbase.
10 |
11 | NOTE: If you're looking for the Elasticsearch Plug-in flavor of the connector, that's in a https://github.com/couchbase/couchbase-elasticsearch-connector/tree/release/cypress[different branch].
12 |
13 | [small]_This product is neither affiliated with nor endorsed by Elastic.
14 | Elasticsearch is a trademark of Elasticsearch BV, registered in the U.S. and in other countries._
15 |
16 | == Building the connector from source
17 |
18 | The connector distribution may be built from source with the command:
19 |
20 | ./gradlew build
21 |
22 | The distribution archive will be generated under `build/distributions`.
23 | During development, it might be more convenient to run:
24 |
25 | ./gradlew installDist
26 |
27 | which creates `build/install/couchbase-elasticsearch-connector` as a `$CBES_HOME` directory.
28 |
29 |
30 | === Running the integration tests
31 |
32 | A local Docker installation is required for these tests.
33 | To quickly test using only the latest Couchbase and Elasticsearch:
34 |
35 | ./gradlew integrationTest
36 |
37 |
38 | To test against _all_ supported versions of Couchbase and Elasticsearch:
39 |
40 | ./gradlew exhaustiveTest
41 |
42 |
43 | === IntelliJ IDEA setup
44 | Because the project uses annotation processors, some link:INTELLIJ-SETUP.md[fiddly setup] is required when importing the project into IntelliJ IDEA.
45 |
46 |
47 | === Building a Docker image
48 |
49 | Use `Dockerfile` to build a Docker image from source using Gradle.
50 | The version should be set in `build.gradle` before running.
51 |
52 | docker build -t imagename:tag .
53 |
54 | Use `Dockerfile.download` to build a Docker image from released binaries hosted at packages.couchbase.com.
55 |
56 | docker build -f Dockerfile.download -t imagename:tag --build-arg VERSION=
57 |
58 | where `` is the latest https://github.com/couchbase/couchbase-elasticsearch-connector/tags[tag] from the connector's GitHub repo.
59 |
60 | === Running a Docker image
61 |
62 | The built docker image can be configured using volume mounts.
63 | The `/opt/couchbase-elasticsearch-connector/config` directory should contain the configuration files, and the `/opt/couchbase-elasticsearch-connector/secrets` directory should contain the secrets.
64 |
65 | Find example configuration files in the `src/dist` directory.
66 | Be sure to rename `example-connector.toml` to `default-connector.toml`.
67 |
68 | docker run -p 31415:31415 -v ./config:/opt/couchbase-elasticsearch-connector/config -v ./secrets:/opt/couchbase-elasticsearch-connector/secrets -e CBES_GROUPNAME=groupname image:tag
69 |
70 | It is also valid to pass environment variables in via the Docker command line, which can then be used to substitute values in `default-connector.toml`.
71 | Port 31415 can be accessed via HTTP to get metrics.
72 |
73 | === Running in Kubernetes
74 |
75 | The connector can run in Kubernetes.
76 | See the examples in the `examples/kubernetes` directory, and https://docs.couchbase.com/elasticsearch-connector/current/kubernetes.html[the documentation] for more details.
77 |
--------------------------------------------------------------------------------
/src/main/java/com/couchbase/connector/cluster/consul/rpc/Broadcaster.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2019 Couchbase, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.couchbase.connector.cluster.consul.rpc;
18 |
19 | import com.google.common.base.Stopwatch;
20 | import org.slf4j.Logger;
21 | import org.slf4j.LoggerFactory;
22 |
23 | import java.io.Closeable;
24 | import java.util.ArrayList;
25 | import java.util.HashMap;
26 | import java.util.List;
27 | import java.util.Map;
28 | import java.util.concurrent.ExecutionException;
29 | import java.util.concurrent.ExecutorService;
30 | import java.util.concurrent.Executors;
31 | import java.util.concurrent.Future;
32 | import java.util.function.Consumer;
33 | import java.util.function.Function;
34 |
35 | import static com.couchbase.client.core.logging.RedactableArgument.redactSystem;
36 |
37 | public class Broadcaster implements Closeable {
38 | private static final Logger LOGGER = LoggerFactory.getLogger(Broadcaster.class);
39 |
40 | private final ExecutorService executor = Executors.newCachedThreadPool();
41 |
42 | public Map> broadcast(String description, List endpoints, Class serviceInterface, Consumer endpointCallback) {
43 | return broadcast(description, endpoints, serviceInterface, s -> {
44 | endpointCallback.accept(s);
45 | return null;
46 | });
47 | }
48 |
49 | public Map> broadcast(String description, List endpoints, Class serviceInterface, Function endpointCallback) {
50 | LOGGER.info("Broadcasting '{}' request to {} endpoints", description, endpoints.size());
51 | LOGGER.debug("Endpoints: {}", endpoints);
52 |
53 | final Stopwatch timer = Stopwatch.createStarted();
54 |
55 | final List>> futures = new ArrayList<>();
56 | for (RpcEndpoint endpoint : endpoints) {
57 | futures.add(executor.submit(
58 | () -> RpcResult.newSuccess(endpointCallback.apply(endpoint.service(serviceInterface)))));
59 | }
60 |
61 | LOGGER.info("Scheduled all '{}' requests for broadcast. Awaiting responses...", description);
62 |
63 | final Map> results = new HashMap<>();
64 | for (int i = 0; i < endpoints.size(); i++) {
65 | final RpcEndpoint endpoint = endpoints.get(i);
66 | final Future> f = futures.get(i);
67 |
68 | try {
69 | results.put(endpoint, f.get());
70 |
71 | } catch (Throwable e) {
72 | if (e instanceof ExecutionException) {
73 | e = e.getCause();
74 | }
75 | results.put(endpoint, RpcResult.newFailure(e));
76 | LOGGER.error("Failed to apply '{}' callback for endpoint {}", description, redactSystem(endpoint), e);
77 | }
78 | }
79 |
80 | LOGGER.info("Finished collecting '{}' broadcast responses. Broadcasting to {} endpoints took {}", description, endpoints.size(), timer);
81 | LOGGER.debug("Broadcast results: {}", results);
82 | return results;
83 | }
84 |
85 | @Override
86 | public void close() {
87 | executor.shutdownNow();
88 | }
89 | }
90 |
--------------------------------------------------------------------------------
/src/main/java/com/couchbase/consul/KvReadResult.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2022 Couchbase, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * https://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.couchbase.consul;
18 |
19 | import com.fasterxml.jackson.annotation.JsonCreator;
20 | import com.fasterxml.jackson.annotation.JsonProperty;
21 |
22 | import java.util.Arrays;
23 | import java.util.Objects;
24 | import java.util.Optional;
25 |
26 | import static java.nio.charset.StandardCharsets.UTF_8;
27 |
28 | public class KvReadResult {
29 | private static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
30 |
31 | private final String key;
32 | private final long flags;
33 | private final byte[] value;
34 | private final String session;
35 | private final long createIndex;
36 | private final long modifyIndex;
37 | private final long lockIndex;
38 |
39 | @JsonCreator
40 | public KvReadResult(
41 | @JsonProperty("Key") String key,
42 | @JsonProperty("Flags") long flags,
43 | @JsonProperty("Value") byte[] value,
44 | @JsonProperty("Session") String session,
45 | @JsonProperty("CreateIndex") long createIndex,
46 | @JsonProperty("ModifyIndex") long modifyIndex,
47 | @JsonProperty("LockIndex") long lockIndex
48 | ) {
49 | this.key = key;
50 | this.flags = flags;
51 | this.value = value == null ? EMPTY_BYTE_ARRAY : value;
52 | this.session = session;
53 | this.createIndex = createIndex;
54 | this.modifyIndex = modifyIndex;
55 | this.lockIndex = lockIndex;
56 | }
57 |
58 | public String key() {
59 | return key;
60 | }
61 |
62 | public long flags() {
63 | return flags;
64 | }
65 |
66 | public byte[] value() {
67 | return value;
68 | }
69 |
70 | public String valueAsString() {
71 | return new String(value, UTF_8);
72 | }
73 |
74 | public Optional session() {
75 | return Optional.ofNullable(session);
76 | }
77 |
78 | public long createIndex() {
79 | return createIndex;
80 | }
81 |
82 | public long modifyIndex() {
83 | return modifyIndex;
84 | }
85 |
86 | public long lockIndex() {
87 | return lockIndex;
88 | }
89 |
90 | @Override
91 | public String toString() {
92 | return "KvReadResult{" +
93 | "key='" + key + '\'' +
94 | ", flags=" + flags +
95 | ", session='" + session + '\'' +
96 | ", createIndex=" + createIndex +
97 | ", modifyIndex=" + modifyIndex +
98 | ", lockIndex=" + lockIndex +
99 | ", value=" + valueAsString() +
100 | '}';
101 | }
102 |
103 | @Override
104 | public boolean equals(Object o) {
105 | if (this == o) {
106 | return true;
107 | }
108 | if (o == null || getClass() != o.getClass()) {
109 | return false;
110 | }
111 | KvReadResult that = (KvReadResult) o;
112 | return flags == that.flags && createIndex == that.createIndex && modifyIndex == that.modifyIndex && lockIndex == that.lockIndex && key.equals(that.key) && Arrays.equals(value, that.value) && Objects.equals(session, that.session);
113 | }
114 |
115 | @Override
116 | public int hashCode() {
117 | return Objects.hash(key, modifyIndex, lockIndex);
118 | }
119 | }
120 |
--------------------------------------------------------------------------------
/src/main/java/com/couchbase/connector/config/StorageSize.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2022 Couchbase, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * https://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.couchbase.connector.config;
18 |
19 | import java.util.Locale;
20 | import java.util.regex.Matcher;
21 | import java.util.regex.Pattern;
22 |
23 | public class StorageSize {
24 | private static final Pattern DIGITS = Pattern.compile("\\d+");
25 |
26 | @Override
27 | public String toString() {
28 | return bytes + " bytes";
29 | }
30 |
31 | private final long bytes;
32 |
33 | private StorageSize(long bytes) {
34 | this.bytes = bytes;
35 | }
36 |
37 | public static StorageSize ofBytes(long value) {
38 | return new StorageSize(value);
39 | }
40 |
41 | public static StorageSize ofMebibytes(long value) {
42 | return new StorageSize(value * 1024 * 1024);
43 | }
44 |
45 | public long getBytes() {
46 | return bytes;
47 | }
48 |
49 | public static StorageSize parse(String value) {
50 | try {
51 | String normalized = value
52 | .trim()
53 | .toLowerCase(Locale.ROOT);
54 |
55 | Matcher m = DIGITS.matcher(normalized);
56 | if (!m.find()) {
57 | throw new IllegalArgumentException("Value does not start with a number.");
58 | }
59 | String number = m.group();
60 |
61 | long num = Long.parseLong(number);
62 |
63 | String unit = normalized.substring(number.length()).trim();
64 | if (unit.isEmpty()) {
65 | throw new IllegalArgumentException("Missing size unit.");
66 | }
67 |
68 | final long scale;
69 | switch (unit) {
70 | case "b":
71 | scale = 1;
72 | break;
73 | case "k":
74 | case "kb":
75 | scale = 1024L;
76 | break;
77 | case "m":
78 | case "mb":
79 | scale = 1024L * 1024;
80 | break;
81 | case "g":
82 | case "gb":
83 | scale = 1024L * 1024 * 1024;
84 | break;
85 | case "t":
86 | case "tb":
87 | scale = 1024L * 1024 * 1024 * 1024;
88 | break;
89 | case "p":
90 | case "pb":
91 | scale = 1024L * 1024 * 1024 * 1024 * 1024;
92 | break;
93 | default:
94 | throw new IllegalArgumentException("Unrecognized size unit: '" + unit + "'");
95 | }
96 |
97 | return new StorageSize(Math.multiplyExact(num, scale));
98 |
99 | } catch (Exception e) {
100 | throw new IllegalArgumentException(
101 | "Failed to parse storage size value '" + value + "'; " + e.getMessage() +
102 | " ; A valid value is a number followed by a unit (for example, '10mb')." +
103 | " Valid units are" +
104 | " 'b' for bytes" +
105 | ", 'k' or 'kb' for kilobytes" +
106 | ", 'm' or 'mb' for megabytes" +
107 | ", 'g' or 'gb' for gigabytes" +
108 | ", 't' or 'tb' for terabytes" +
109 | ", 'p' or 'pb' for petabytes" +
110 | "." +
111 | " Values must be >= 0 bytes and <= " + Long.MAX_VALUE + " bytes.");
112 | }
113 | }
114 | }
115 |
--------------------------------------------------------------------------------
/src/main/java/com/couchbase/connector/cluster/consul/ConsulResourceWatcher.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2019 Couchbase, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.couchbase.connector.cluster.consul;
18 |
19 | import com.couchbase.consul.ConsulResponse;
20 | import com.google.common.primitives.Longs;
21 | import org.slf4j.Logger;
22 | import org.slf4j.LoggerFactory;
23 | import reactor.core.publisher.Flux;
24 | import reactor.core.publisher.Mono;
25 |
26 | import java.time.Duration;
27 | import java.util.Map;
28 | import java.util.concurrent.atomic.AtomicLong;
29 | import java.util.function.Function;
30 |
31 | import static java.util.Objects.requireNonNull;
32 | import static java.util.concurrent.TimeUnit.MILLISECONDS;
33 | import static java.util.concurrent.TimeUnit.MINUTES;
34 |
35 | public class ConsulResourceWatcher {
36 | private static final Logger log = LoggerFactory.getLogger(ConsulResourceWatcher.class);
37 |
38 | private final String wait;
39 |
40 | public ConsulResourceWatcher() {
41 | this(Duration.ofMinutes(5));
42 | }
43 |
44 | private ConsulResourceWatcher(Duration pollingInterval) {
45 | final long requestedPollingIntervalSeconds = MILLISECONDS.toSeconds(pollingInterval.toMillis());
46 | this.wait = Longs.constrainToRange(requestedPollingIntervalSeconds, 1, MINUTES.toSeconds(5)) + "s";
47 | }
48 |
49 | public ConsulResourceWatcher withPollingInterval(Duration pollingInterval) {
50 | return new ConsulResourceWatcher(pollingInterval);
51 | }
52 |
53 | public Flux> watch(
54 | Function, Mono>> queryParametersToRequest
55 | ) {
56 | requireNonNull(queryParametersToRequest);
57 |
58 | return Flux.defer(() -> {
59 | AtomicLong prevIndex = new AtomicLong();
60 |
61 | return queryParametersToRequest.apply((Map.of()))
62 | .expand(item -> {
63 | // Consul docs advise resetting index to zero if it goes backwards,
64 | // and sanity checking the response index is at least 1. Reference:
65 | // https://www.consul.io/api-docs/features/blocking#implementation-details
66 | long index = item.index().orElse(0);
67 | if (index < 1) {
68 | log.warn("Consul resource index was <= 0; this is probably a bug in Consul. Recovering by assuming an index of 1.");
69 | index = 1;
70 | } else if (index < prevIndex.get()) {
71 | log.debug("Consul resource index went backwards; resetting to 0.");
72 | index = 0;
73 | }
74 | prevIndex.set(index);
75 |
76 | Mono> expansion = queryParametersToRequest.apply(Map.of(
77 | "index", index,
78 | "wait", wait
79 | ));
80 |
81 | // If not found, delay to prevent busy loop while waiting for resource to appear.
82 | // Especially important when watching KV entries.
83 | return item.httpStatusCode() == 404
84 | ? Mono.delay(withFullJitter(Duration.ofSeconds(1))).then(expansion)
85 | : expansion;
86 | }).onBackpressureLatest();
87 | });
88 | }
89 |
90 | private static Duration withFullJitter(Duration d) {
91 | return Duration.ofMillis((long) (Math.random() * d.toMillis()));
92 | }
93 |
94 | }
95 |
--------------------------------------------------------------------------------
/src/integrationTest/java/com/couchbase/connector/elasticsearch/TestCouchbaseClient.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2019 Couchbase, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.couchbase.connector.elasticsearch;
18 |
19 | import com.couchbase.client.core.service.ServiceType;
20 | import com.couchbase.client.java.Bucket;
21 | import com.couchbase.client.java.Cluster;
22 | import com.couchbase.connector.config.es.ConnectorConfig;
23 | import com.couchbase.connector.config.es.ImmutableConnectorConfig;
24 | import com.couchbase.connector.dcp.CouchbaseHelper;
25 | import com.couchbase.connector.testcontainers.CustomCouchbaseContainer;
26 | import com.google.common.io.Closer;
27 |
28 | import java.io.Closeable;
29 | import java.io.IOException;
30 | import java.time.Duration;
31 |
32 | import static com.couchbase.client.dcp.core.utils.CbCollections.setOf;
33 | import static com.couchbase.client.java.diagnostics.WaitUntilReadyOptions.waitUntilReadyOptions;
34 | import static com.couchbase.connector.dcp.CouchbaseHelper.environmentConfigurator;
35 |
36 | class TestCouchbaseClient implements Closeable {
37 | private final Cluster cluster;
38 | private final Closer closer = Closer.create();
39 |
40 | public TestCouchbaseClient(String config) {
41 | this(ConnectorConfig.from(config));
42 | }
43 |
44 | public TestCouchbaseClient(PatchableConfig patcher) {
45 | this(patcher.toConfig());
46 | }
47 |
48 | public TestCouchbaseClient(ImmutableConnectorConfig config) {
49 | this.cluster = CouchbaseHelper.createCluster(
50 | config.couchbase(),
51 | environmentConfigurator(config).andThen(env -> env
52 | .ioConfig(io -> io.enableMutationTokens(true))
53 | .timeoutConfig(timeout -> timeout
54 | .connectTimeout(Duration.ofSeconds(15))
55 | .kvTimeout(Duration.ofSeconds(10))
56 | )
57 | )
58 | );
59 | }
60 |
61 | public Cluster cluster() {
62 | return cluster;
63 | }
64 |
65 | /**
66 | * Create a new bucket with a unique name. The bucket will be deleted
67 | * when this client is closed.
68 | */
69 | public Bucket createTempBucket(CustomCouchbaseContainer couchbase) {
70 | final TempBucket temp = closer.register(new TempBucket(couchbase));
71 | Bucket bucket = cluster().bucket(temp.name());
72 | Duration timeout = bucket.environment().timeoutConfig().connectTimeout();
73 |
74 | // Multiplying timeout by 2 as a temporary workaround for JVMCBC-817
75 | // (giving the config loader time for the KV attempt to timeout and still leaving
76 | // time for loading the config from the manager).
77 | timeout = timeout.multipliedBy(2);
78 |
79 | bucket.waitUntilReady(timeout, waitUntilReadyOptions()
80 | .serviceTypes(setOf(ServiceType.KV)));
81 |
82 | return bucket;
83 | }
84 |
85 | /**
86 | * Create a new bucket with the given name. The bucket will be deleted
87 | * when this client is closed.
88 | */
89 | public Bucket createTempBucket(CustomCouchbaseContainer couchbase, String bucketName) {
90 | final TempBucket temp = closer.register(new TempBucket(couchbase, bucketName));
91 | return cluster().bucket(temp.name());
92 | }
93 |
94 | @Override
95 | public void close() throws IOException {
96 | cluster.disconnect();
97 | closer.close();
98 | }
99 | }
100 |
--------------------------------------------------------------------------------
/src/main/java/com/couchbase/connector/cluster/consul/LeaderElectionTask.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2018 Couchbase, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.couchbase.connector.cluster.consul;
18 |
19 | import org.slf4j.Logger;
20 | import org.slf4j.LoggerFactory;
21 |
22 | import java.util.Optional;
23 | import java.util.function.Consumer;
24 |
25 | import static java.util.Objects.requireNonNull;
26 | import static java.util.concurrent.TimeUnit.SECONDS;
27 |
28 | public class LeaderElectionTask extends AbstractLongPollTask {
29 | private static final Logger LOGGER = LoggerFactory.getLogger(LeaderElectionTask.class);
30 |
31 | private final String candidateUuid;
32 | private final Consumer fatalErrorConsumer;
33 | private final LeaderController leaderController;
34 |
35 | public LeaderElectionTask(ConsulContext ctx, String sessionId, String endpointId, Consumer fatalErrorConsumer, LeaderController leaderController) {
36 | super(ctx, "leader-election-", sessionId);
37 | this.candidateUuid = requireNonNull(endpointId);
38 | this.fatalErrorConsumer = requireNonNull(fatalErrorConsumer);
39 | this.leaderController = requireNonNull(leaderController);
40 | }
41 |
42 | protected void doRun(ConsulContext ctx, String sessionId) {
43 | final String leaderKey = ctx.keys().leader();
44 | LOGGER.info("Leader key: {}", leaderKey);
45 |
46 | try {
47 | while (!closed()) {
48 | LOGGER.info("Racing to become new leader.");
49 | final boolean acquired = ctx.acquireLock(leaderKey, candidateUuid, sessionId);
50 |
51 | if (acquired) {
52 | LOGGER.info("Won the leader election! {}", candidateUuid);
53 | leaderController.startLeading();
54 |
55 | final Optional newLeader = ctx.documentWatcher().awaitValueChange(leaderKey, candidateUuid);
56 |
57 | LOGGER.info("No longer the leader; new leader is {}", newLeader);
58 | leaderController.stopLeading();
59 |
60 | } else {
61 | LOGGER.info("Not the leader");
62 |
63 | // Sleeping because the attempt may have failed due to Consul's lock delay;
64 | // see https://www.consul.io/docs/internals/sessions.html
65 | SECONDS.sleep(1);
66 |
67 | ctx.documentWatcher().awaitAbsence(leaderKey);
68 | }
69 | }
70 |
71 | } catch (Throwable t) {
72 | if (closed()) {
73 | // Closing the task is likely to result in an InterruptedException
74 | LOGGER.debug("Caught exception in leader election loop after closing. Don't panic; this is expected.", t);
75 | } else {
76 | // Something went horribly, terribly, unrecoverably wrong.
77 | fatalErrorConsumer.accept(t);
78 | }
79 |
80 | } finally {
81 | try {
82 | leaderController.stopLeading();
83 |
84 | // Abdicate (delete leadership document); only succeeds if we own the lock. This lets another node acquire
85 | // the lock immediately. If we don't do this, the lock will be auto-released when the session ends,
86 | // but the lock won't be eligible for acquisition until the Consul lock delay has elapsed.
87 | ctx.runCleanup(() -> ctx.unlockAndDelete(leaderKey, sessionId));
88 |
89 | } catch (Exception e) {
90 | LOGGER.warn("Failed to abdicate", e);
91 | }
92 |
93 | LOGGER.info("Exiting leader election thread for session {}", sessionId);
94 | }
95 | }
96 |
97 | }
98 |
--------------------------------------------------------------------------------
/src/main/java/com/couchbase/consul/ReadTimeoutSetter.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2022 Couchbase, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * https://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.couchbase.consul;
18 |
19 | import com.couchbase.client.core.util.Golang;
20 | import com.couchbase.consul.internal.OkHttpHelper;
21 | import com.google.common.annotations.VisibleForTesting;
22 | import okhttp3.Interceptor;
23 | import okhttp3.Request;
24 | import okhttp3.Response;
25 | import org.jspecify.annotations.NonNull;
26 |
27 | import java.io.IOException;
28 | import java.time.Duration;
29 | import java.util.Optional;
30 | import java.util.concurrent.TimeUnit;
31 |
32 | import static com.couchbase.client.core.util.CbStrings.emptyToNull;
33 |
34 | /**
35 | * Sets the read timeout to match the "wait" query parameter of
36 | * Consul blocking requests.
37 | */
38 | public class ReadTimeoutSetter implements Interceptor {
39 | private static final Duration MAX_WAIT = Duration.ofMinutes(10);
40 |
41 | /**
42 | * Maximum amount of time any Consul request is expected to take.
43 | *
44 | * (There's no easy way to set the call timeout on a per-request basis, unfortunately;
45 | * see Retrofit issue 3434)
46 | */
47 | public static final Duration CALL_TIMEOUT = addConsulJitter(MAX_WAIT)
48 | .plus(Duration.ofSeconds(10)); // *extra* grace period for response processing, etc.
49 |
50 | @Override
51 | public @NonNull Response intercept(@NonNull Chain chain) throws IOException {
52 | Request request = chain.request();
53 | boolean hasIndex = request.url().queryParameterNames().contains("index");
54 |
55 | if (!hasIndex) {
56 | // It's not a blocking request, so honor the client's configured timeout.
57 | return chain.proceed(request);
58 | }
59 |
60 | Duration wait = getWait(request).orElse(null);
61 | if (wait == null) {
62 | // Make the default wait time explicit, in case it changes in a future Consul version.
63 | request = OkHttpHelper.withQueryParam(request, "wait", "5m");
64 | wait = getWait(request).orElseThrow();
65 | }
66 |
67 | if (wait.compareTo(MAX_WAIT) > 0) {
68 | throw new IllegalArgumentException("Specified wait " + wait + " is greater than max wait " + MAX_WAIT);
69 | }
70 |
71 | wait = addConsulJitter(wait);
72 |
73 | return chain
74 | .withReadTimeout((int) wait.toMillis(), TimeUnit.MILLISECONDS)
75 | .proceed(request);
76 | }
77 |
78 | /**
79 | * Returns a Duration slightly longer than the given duration.
80 | * Accounts for the 6.25% (1/16) jitter added by Consul, plus some grace time.
81 | *
82 | * See Consul Blocking Queries.
83 | */
84 | @VisibleForTesting
85 | static Duration addConsulJitter(Duration wait) {
86 | int gracePeriodSeconds = 10;
87 | int waitSecondsPlusJitter = Math.toIntExact((long) Math.ceil(wait.toSeconds() * 1.0625));
88 | return Duration.ofSeconds(gracePeriodSeconds + waitSecondsPlusJitter);
89 | }
90 |
91 | @VisibleForTesting
92 | static Optional getWait(Request r) {
93 | String wait = r.url().queryParameter("wait");
94 | return Optional.ofNullable(emptyToNull(wait))
95 | .map(ReadTimeoutSetter::parseWait);
96 | }
97 |
98 | private static Duration parseWait(String s) {
99 | // Consul's duration format is a subset of Go's duration format.
100 | return Golang.parseDuration(s);
101 | }
102 |
103 | }
104 |
--------------------------------------------------------------------------------
/src/main/java/com/couchbase/connector/elasticsearch/cli/AbstractCliCommand.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2018 Couchbase, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.couchbase.connector.elasticsearch.cli;
18 |
19 | import joptsimple.OptionException;
20 | import joptsimple.OptionParser;
21 | import joptsimple.OptionSet;
22 | import joptsimple.OptionSpec;
23 |
24 | import java.io.File;
25 | import java.io.IOException;
26 |
27 | import static com.couchbase.connector.VersionHelper.getVersionString;
28 | import static com.couchbase.connector.util.EnvironmentHelper.getInstallDir;
29 |
30 | public abstract class AbstractCliCommand {
31 |
32 | // Init Log4J. This must happen first before logger is declared.
33 | static {
34 | // Expect the launch script generated by Gradle to set this environment variable.
35 | final String appHome = System.getenv("APP_HOME");
36 | if (appHome == null) {
37 | // Use the default log4j2.xml file in the class path.
38 | System.err.println("WARNING: Environment variable 'APP_HOME' not set (launch script is responsible for this). " +
39 | "Using embedded logging config.");
40 | } else {
41 | System.setProperty("log4j.configurationFile", appHome + "/config/log4j2.xml");
42 | }
43 | }
44 |
45 | public static class CommonParser {
46 | protected final OptionParser parser = new OptionParser();
47 |
48 | public final OptionSpec configFile = parser.accepts("config", "Optionally specify a config file other than the default. See the 'config' directory for examples.")
49 | .withRequiredArg().ofType(File.class).describedAs("path to config file")
50 | .defaultsTo(defaultConfigFile());
51 |
52 | public OptionSet parse(String... args) throws IOException {
53 |
54 | try {
55 | parser.accepts("version", "Display connector version information.");
56 | handleVersionAndHelp(parser, args);
57 | final OptionSet options = parser.parse(args);
58 |
59 | final File configFile = this.configFile.value(options);
60 | if (!configFile.exists()) {
61 | System.err.println("ERROR: Config file not found at " + configFile.getAbsoluteFile());
62 | System.err.println(" Please create this file, or use the --config option to specify a different file.");
63 | System.err.println(" Examples may be found in the 'config' directory.");
64 | System.exit(1);
65 | }
66 |
67 | return options;
68 |
69 | } catch (OptionException e) {
70 | System.err.println(e.getMessage());
71 | System.err.println();
72 | parser.printHelpOn(System.err);
73 | System.exit(1);
74 | throw new AssertionError("unreachable");
75 | }
76 |
77 | }
78 |
79 | void handleVersionAndHelp(OptionParser parser, String[] args) throws IOException {
80 | // parse version separately, since the "real" parser might have required arguments
81 | final OptionParser versionParser = new OptionParser();
82 | versionParser.allowsUnrecognizedOptions();
83 | versionParser.accepts("version");
84 | versionParser.accepts("help");
85 | if (versionParser.parse(args).has("version")) {
86 | System.out.println(getVersionString());
87 | System.exit(0);
88 | }
89 |
90 | if (versionParser.parse(args).has("help")) {
91 | parser.printHelpOn(System.out);
92 | System.exit(0);
93 | }
94 | }
95 | }
96 |
97 | static File defaultConfigFile() {
98 | return new File(getInstallDir(), "config/default-connector.toml");
99 | }
100 | }
101 |
--------------------------------------------------------------------------------
/src/main/java/com/couchbase/connector/cluster/consul/AsyncTask.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2018 Couchbase, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.couchbase.connector.cluster.consul;
18 |
19 | import com.google.common.base.Throwables;
20 | import org.slf4j.Logger;
21 | import org.slf4j.LoggerFactory;
22 |
23 | import java.io.Closeable;
24 | import java.io.IOException;
25 | import java.time.Duration;
26 | import java.util.concurrent.TimeoutException;
27 | import java.util.concurrent.atomic.AtomicInteger;
28 | import java.util.concurrent.atomic.AtomicReference;
29 | import java.util.function.Consumer;
30 |
31 | import static com.google.common.util.concurrent.Uninterruptibles.joinUninterruptibly;
32 | import static java.util.Objects.requireNonNull;
33 |
34 | /**
35 | * Helper for running interruptible tasks in a separate thread.
36 | */
37 | public class AsyncTask implements Closeable {
38 | private static final Logger LOGGER = LoggerFactory.getLogger(AsyncTask.class);
39 |
40 | private static final AtomicInteger threadCounter = new AtomicInteger();
41 |
42 | @FunctionalInterface
43 | public interface Interruptible {
44 | /**
45 | * When the thread executing this method is interrupted,
46 | * the method should throw InterruptedException to indicate clean termination.
47 | */
48 | void run() throws Throwable;
49 | }
50 |
51 | private final Thread thread;
52 | private final AtomicReference connectorException = new AtomicReference<>();
53 |
54 | public static AsyncTask run(Interruptible task, Consumer fatalErrorListener) {
55 | return new AsyncTask(task, fatalErrorListener).start();
56 | }
57 |
58 | public static AsyncTask run(Interruptible task) {
59 | return run(task, t -> {
60 | });
61 | }
62 |
63 | private AsyncTask(Interruptible task, Consumer fatalErrorListener) {
64 | requireNonNull(task);
65 | requireNonNull(fatalErrorListener);
66 |
67 | thread = new Thread(() -> {
68 | try {
69 | task.run();
70 | } catch (Throwable t) {
71 | connectorException.set(t);
72 | if (t instanceof InterruptedException) {
73 | LOGGER.debug("Connector task interrupted");
74 | } else {
75 | LOGGER.error("Connector task exited early due to failure", t);
76 | fatalErrorListener.accept(t);
77 | }
78 | }
79 | }, "connector-main-" + threadCounter.getAndIncrement());
80 | }
81 |
82 | private AsyncTask start() {
83 | try {
84 | thread.start();
85 | LOGGER.info("Thread {} started.", thread.getName());
86 | return this;
87 |
88 | } catch (IllegalThreadStateException e) {
89 | throw new IllegalStateException("May only be started once", e);
90 | }
91 | }
92 |
93 | public void stop() throws Throwable {
94 | thread.interrupt();
95 |
96 | Duration timeout = Duration.ofSeconds(30);
97 | joinUninterruptibly(thread, timeout);
98 | if (thread.isAlive()) {
99 | throw new TimeoutException("Connector didn't exit within " + timeout);
100 | }
101 |
102 | Throwable t = connectorException.get();
103 | if (t == null) {
104 | throw new IllegalStateException("Connector didn't exit by throwing exception");
105 | }
106 | if (!(t instanceof InterruptedException)) {
107 | // The connector failed before we asked it to exit!
108 | throw t;
109 | }
110 | }
111 |
112 | @Override
113 | public void close() throws IOException {
114 | try {
115 | stop();
116 | } catch (Throwable t) {
117 | Throwables.propagateIfPossible(t, IOException.class);
118 | throw new RuntimeException(t);
119 | }
120 | }
121 | }
122 |
--------------------------------------------------------------------------------