├── .dockerignore ├── .editorconfig ├── .gitignore ├── CONTRIBUTING.adoc ├── DIST_README.txt ├── Dockerfile ├── Dockerfile.download ├── HOWTO-DOCKERIZE.md ├── HOWTO-RELEASE.md ├── INTELLIJ-SETUP.md ├── LICENSE ├── Makefile ├── README.adoc ├── build.gradle ├── docs └── README.adoc ├── examples └── kubernetes │ ├── README.adoc │ ├── elasticsearch-connector-configuration.yaml │ ├── elasticsearch-connector-service-account.yaml │ └── elasticsearch-connector.yaml ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── imgs └── intellij-annotation-proc-config.png ├── settings.gradle └── src ├── dist ├── config │ ├── consul.toml │ ├── example-connector.toml │ └── log4j2.xml └── secrets │ ├── couchbase-client-cert-password.toml │ ├── couchbase-password.toml │ ├── elasticsearch-client-cert-password.toml │ └── elasticsearch-password.toml ├── integrationTest └── java │ └── com │ └── couchbase │ └── connector │ ├── elasticsearch │ ├── AutonomousOpsTest.java │ ├── BasicReplicationTest.java │ ├── ConsulCluster.java │ ├── ConsulDocumentWatcherTest.java │ ├── DockerHelper.java │ ├── ElasticsearchSinkContainer.java │ ├── IntegrationTestHelper.java │ ├── JsonDocument.java │ ├── OpensearchSinkContainer.java │ ├── PatchableConfig.java │ ├── SinkContainer.java │ ├── TempBucket.java │ ├── TestConfigHelper.java │ ├── TestConnectorGroup.java │ ├── TestCouchbaseClient.java │ └── TestEsClient.java │ └── testcontainers │ ├── CouchbaseOps.java │ ├── CustomCouchbaseContainer.java │ ├── ExecUtils.java │ └── Poller.java ├── main ├── java │ └── com │ │ └── couchbase │ │ ├── connector │ │ ├── VersionHelper.java │ │ ├── cluster │ │ │ ├── DefaultPanicButton.java │ │ │ ├── KillSwitch.java │ │ │ ├── Membership.java │ │ │ ├── PanicButton.java │ │ │ ├── consul │ │ │ │ ├── AbstractLongPollTask.java │ │ │ │ ├── AsyncTask.java │ │ │ │ ├── ConsulConnector.java │ │ │ │ ├── ConsulContext.java │ │ │ │ ├── ConsulDocumentWatcher.java │ │ │ │ ├── ConsulHelper.java │ │ │ │ ├── ConsulResourceWatcher.java │ │ │ │ ├── DocumentKeys.java │ │ │ │ ├── LeaderController.java │ │ │ │ ├── LeaderElectionTask.java │ │ │ │ ├── LeaderEvent.java │ │ │ │ ├── LeaderTask.java │ │ │ │ ├── ReactorHelper.java │ │ │ │ ├── SessionTask.java │ │ │ │ ├── TimeoutEnforcer.java │ │ │ │ ├── WorkerService.java │ │ │ │ ├── WorkerServiceImpl.java │ │ │ │ └── rpc │ │ │ │ │ ├── Broadcaster.java │ │ │ │ │ ├── ConsulRpcTransport.java │ │ │ │ │ ├── EndpointDocument.java │ │ │ │ │ ├── RpcEndpoint.java │ │ │ │ │ ├── RpcResult.java │ │ │ │ │ └── RpcServerTask.java │ │ │ └── k8s │ │ │ │ ├── ReplicaChangeWatcher.java │ │ │ │ └── StatefulSetInfo.java │ │ ├── config │ │ │ ├── ConfigException.java │ │ │ ├── ConfigHelper.java │ │ │ ├── ScopeAndCollection.java │ │ │ ├── StorageSize.java │ │ │ ├── common │ │ │ │ ├── ClientCertConfig.java │ │ │ │ ├── ConsulConfig.java │ │ │ │ ├── CouchbaseConfig.java │ │ │ │ ├── DcpConfig.java │ │ │ │ ├── GroupConfig.java │ │ │ │ ├── LoggingConfig.java │ │ │ │ ├── MetricsConfig.java │ │ │ │ └── TrustStoreConfig.java │ │ │ ├── es │ │ │ │ ├── AwsConfig.java │ │ │ │ ├── BulkRequestConfig.java │ │ │ │ ├── ConnectorConfig.java │ │ │ │ ├── DocStructureConfig.java │ │ │ │ ├── ElasticCloudConfig.java │ │ │ │ ├── ElasticsearchConfig.java │ │ │ │ ├── RejectLogConfig.java │ │ │ │ └── TypeConfig.java │ │ │ └── toml │ │ │ │ ├── ConfigArray.java │ │ │ │ ├── ConfigPosition.java │ │ │ │ ├── ConfigTable.java │ │ │ │ ├── ParseResult.java │ │ │ │ └── Toml.java │ │ ├── dcp │ │ │ ├── Checkpoint.java │ │ │ ├── CheckpointDao.java │ │ │ ├── CheckpointService.java │ │ │ ├── CouchbaseCheckpointDao.java │ │ │ ├── CouchbaseHelper.java │ │ │ ├── DcpHelper.java │ │ │ ├── Event.java │ │ │ ├── MarkableCrc32.java │ │ │ ├── ResolvedBucketConfig.java │ │ │ └── SnapshotMarker.java │ │ ├── elasticsearch │ │ │ ├── BucketMismatchException.java │ │ │ ├── DocumentLifecycle.java │ │ │ ├── ElasticsearchBulkRequestBuilder.java │ │ │ ├── ElasticsearchConnector.java │ │ │ ├── ElasticsearchHelper.java │ │ │ ├── ElasticsearchSinkOps.java │ │ │ ├── ElasticsearchVersionSniffer.java │ │ │ ├── ErrorListener.java │ │ │ ├── Metrics.java │ │ │ ├── OpenSearchBulkRequestBuilder.java │ │ │ ├── OpenSearchHelper.java │ │ │ ├── OpenSearchSinkOps.java │ │ │ ├── cli │ │ │ │ ├── AbstractCliCommand.java │ │ │ │ ├── CheckpointBackup.java │ │ │ │ ├── CheckpointClear.java │ │ │ │ ├── CheckpointRestore.java │ │ │ │ └── ConsulCli.java │ │ │ ├── io │ │ │ │ ├── BackoffPolicy.java │ │ │ │ ├── BackoffPolicyBuilder.java │ │ │ │ ├── DefaultDocumentTransformer.java │ │ │ │ ├── DocumentTransformer.java │ │ │ │ ├── MoreBackoffPolicies.java │ │ │ │ ├── PreserializedJson.java │ │ │ │ └── RequestFactory.java │ │ │ └── sink │ │ │ │ ├── BaseOperation.java │ │ │ │ ├── DeleteOperation.java │ │ │ │ ├── IndexOperation.java │ │ │ │ ├── Operation.java │ │ │ │ ├── RejectOperation.java │ │ │ │ ├── RetryReporter.java │ │ │ │ ├── SinkBulkRequestBuilder.java │ │ │ │ ├── SinkBulkResponse.java │ │ │ │ ├── SinkBulkResponseItem.java │ │ │ │ ├── SinkErrorCause.java │ │ │ │ ├── SinkOps.java │ │ │ │ ├── SinkTestOps.java │ │ │ │ ├── SinkWorker.java │ │ │ │ ├── SinkWorkerGroup.java │ │ │ │ └── SinkWriter.java │ │ └── util │ │ │ ├── EnvironmentHelper.java │ │ │ ├── HttpServer.java │ │ │ ├── KeyStoreHelper.java │ │ │ ├── ListHelper.java │ │ │ ├── RuntimeHelper.java │ │ │ ├── SeedNodeHelper.java │ │ │ └── ThrowableHelper.java │ │ └── consul │ │ ├── ConsulHttpClient.java │ │ ├── ConsulOps.java │ │ ├── ConsulResponse.java │ │ ├── KvReadResult.java │ │ ├── ReadTimeoutSetter.java │ │ └── internal │ │ ├── ConsulJacksonHelper.java │ │ └── OkHttpHelper.java └── resources │ ├── com │ └── couchbase │ │ └── connector │ │ └── version.properties │ └── log4j2.xml └── test └── java └── com └── couchbase ├── connector ├── config │ └── ConfigHelperTest.java ├── dcp │ └── CouchbaseHelperTest.java ├── elasticsearch │ └── io │ │ └── DefaultDocumentTransformerTest.java └── util │ └── ListHelperTest.java └── consul └── ReadTimeoutSetterTest.java /.dockerignore: -------------------------------------------------------------------------------- 1 | .gradle 2 | .idea 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | hs_err_pid* 2 | 3 | .DS_Store 4 | .AppleDouble 5 | .LSOverride 6 | 7 | ._* 8 | 9 | # Files that might appear in the root of a volume 10 | .DocumentRevisions-V100 11 | .fseventsd 12 | .Spotlight-V100 13 | .TemporaryItems 14 | .Trashes 15 | .VolumeIcon.icns 16 | .com.apple.timemachine.donotpresent 17 | 18 | .AppleDB 19 | .AppleDesktop 20 | Network Trash Folder 21 | Temporary Items 22 | .apdisk 23 | 24 | .gradle/ 25 | build/ 26 | /dist/ 27 | .idea/ 28 | *.iml 29 | *.ipr 30 | *.iws 31 | *.log 32 | /out/ 33 | 34 | # Generated by annotation processors 35 | src/*/generated/ 36 | src/*/generated_tests/ 37 | 38 | 39 | -------------------------------------------------------------------------------- /CONTRIBUTING.adoc: -------------------------------------------------------------------------------- 1 | = Contributing 2 | 3 | == Bug reports and feature requests 4 | 5 | This project uses https://issues.couchbase.com[Couchbase Jira] for issue tracking. 6 | 7 | If you come across a bug or find something unintuitive, let us know and we’ll work to fix it in an upcoming release. 8 | 9 | When filing a bug report, please include: 10 | 11 | * Couchbase Server version 12 | * Elasticsearch version 13 | * Connector version 14 | * Operating system 15 | * A summary of the problem 16 | * Detailed steps to reproduce the problem 17 | 18 | == Contributing code 19 | 20 | Whether you have a fix for a typo in a comment, a bugfix, or a new feature, we love to receive code from the community! 21 | 22 | It takes a lot of work to get from a potential new bug fix or feature idea to well-tested shipping code. 23 | Our engineers want to help you get there. 24 | 25 | === One-time setup 26 | The first step is to sign our Contributor License Agreement. 27 | 28 | NOTE: Contributor License Agreements (CLAs) are common for projects under the Apache license, and typically serve to grant control of the code to a central entity. 29 | Because our code is available under the Apache License, signing the CLA doesn’t prevent you from using your code however you like, but it does give Couchbase the ability to defend the source legally and build and maintain a business around the technology. 30 | 31 | Create an account on our https://review.couchbase.org/[Gerrit code review site]. 32 | Make sure the email address you register with matches the email address on your git commits. 33 | You can associate additional email addresses with your Gerrit account later if needed. 34 | 35 | Fill out the agreement under **Settings > Agreements**. 36 | 37 | === Submitting code 38 | 39 | We encourage you to submit patch sets directly to the Gerrit server. 40 | That makes things easier for us, but if you're new to Gerrit it might be intimidating. 41 | Alternatively, free to submit a pull request on GitHub. 42 | A bot will automatically import your change into Gerrit. 43 | -------------------------------------------------------------------------------- /DIST_README.txt: -------------------------------------------------------------------------------- 1 | Documentation: 2 | https://docs.couchbase.com/elasticsearch-connector/current/ 3 | 4 | Discussion: 5 | https://forums.couchbase.com/c/elasticsearch-connector/36 6 | 7 | Issue tracker: 8 | https://github.com/couchbase/couchbase-elasticsearch-connector/issues 9 | 10 | Source code: 11 | https://github.com/couchbase/couchbase-elasticsearch-connector 12 | 13 | Disclaimer: 14 | This product is neither affiliated with nor endorsed by Elastic. 15 | Elasticsearch is a trademark of Elasticsearch BV, registered in 16 | the U.S. and in other countries. 17 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Builds an image from the output of the `gradle install` command. 2 | # To build from a pre-built connector distribution, see Dockerfile.download 3 | 4 | # Use Red Hat Universal Base Image (UBI) for compatibility with OpenShift 5 | FROM registry.access.redhat.com/ubi8/openjdk-17-runtime:1.21 6 | 7 | ARG CBES_HOME=/opt/couchbase-elasticsearch-connector 8 | 9 | # Set owner to jboss to appease the base image. 10 | COPY --chown=jboss:root build/install/couchbase-elasticsearch-connector $CBES_HOME 11 | VOLUME [ "$CBES_HOME/config", "$CBES_HOME/secrets" ] 12 | 13 | ENV PATH="$CBES_HOME/bin:$PATH" 14 | WORKDIR $CBES_HOME 15 | 16 | EXPOSE 31415 17 | 18 | ENTRYPOINT [ "cbes" ] 19 | -------------------------------------------------------------------------------- /Dockerfile.download: -------------------------------------------------------------------------------- 1 | # Builds an image using a pre-built connector distribution archive from couchbase.com. 2 | 3 | # First stage downloads and unzips the connector distribution archive 4 | FROM redhat/ubi8-minimal:latest AS build 5 | 6 | # Version of the connector to download. See release notes for valid versions 7 | # https://docs.couchbase.com/elasticsearch-connector/current/release-notes.html 8 | ARG VERSION 9 | RUN [ -z "$VERSION" ] && echo "Missing VERSION argument. Must specify like: --build-arg VERSION=" && exit 1 || true 10 | 11 | RUN curl -s https://packages.couchbase.com/clients/connectors/elasticsearch/${VERSION}/couchbase-elasticsearch-connector-${VERSION}.zip -o /couchbase-elasticsearch-connector.zip 12 | RUN microdnf install unzip 13 | RUN unzip -o -d / /couchbase-elasticsearch-connector.zip 14 | RUN mv /couchbase-elasticsearch-connector-${VERSION} /couchbase-elasticsearch-connector 15 | 16 | 17 | # Second stage uses the unzipped connector distribution to build the final image 18 | FROM registry.access.redhat.com/ubi8/openjdk-17-runtime:1.21 19 | ARG CBES_HOME=/opt/couchbase-elasticsearch-connector 20 | 21 | # Set owner to jboss to appease the base image. 22 | COPY --chown=jboss:root --from=build /couchbase-elasticsearch-connector $CBES_HOME 23 | VOLUME [ "$CBES_HOME/config", "$CBES_HOME/secrets" ] 24 | 25 | ENV PATH="$CBES_HOME/bin:$PATH" 26 | WORKDIR $CBES_HOME 27 | 28 | EXPOSE 31415 29 | 30 | ENTRYPOINT [ "cbes" ] 31 | -------------------------------------------------------------------------------- /HOWTO-DOCKERIZE.md: -------------------------------------------------------------------------------- 1 | # Creating a Docker Image 2 | 3 | _A guide for creating a new Docker image with the Elasticsearch connector._ 4 | 5 | ## Base image 6 | 7 | The connector requires a Linux base image with glibc (not Alpine). 8 | 9 | ## Java 10 | 11 | The connector requires Java 11 or later, preferably the latest Long-Term Support (LTS) 12 | version. 13 | 14 | Only the Java Runtime is required, not the JDK. 15 | 16 | ## Ports 17 | 18 | The connector runs an HTTP server that listens on port 31415 by default. 19 | 20 | ## Permissions 21 | 22 | The connector must have read+write access to its installation directory. 23 | -------------------------------------------------------------------------------- /HOWTO-RELEASE.md: -------------------------------------------------------------------------------- 1 | # Release Instructions 2 | 3 | This is a guide for Couchbase employees. It describes how to cut a release of this project 4 | and publish it to S3. 5 | 6 | 7 | ## Prerequisites 8 | 9 | You will need: 10 | * AWS credentials with write access to the `packages.couchbase.com` S3 bucket. 11 | * The [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/), for uploading the distribution archive to S3. 12 | * A local Docker installation, if you wish to run the integration tests. 13 | 14 | All set? In that case... 15 | 16 | 17 | ## Let's do this! 18 | 19 | Start by running `./gradlew clean integrationTest` to make sure the project builds successfully, 20 | and the unit and integration tests pass. 21 | When you're satisfied with the test results, it's time to... 22 | 23 | 24 | ## Bump the project version number 25 | 26 | 1. Edit `build.gradle` and remove the `-SNAPSHOT` suffix from the version string. 27 | 2. Edit `docs/modules/ROOT/pages/_attributes` and bump the `:version:` attribute. 28 | 3. Edit `README.adoc` and bump the version numbers if appropriate. 29 | 4. Check whether `compatibility.adoc` needs to be updated to include the new version. 30 | 5. Commit these changes, with message "Prepare x.y.z release" 31 | (where x.y.z is the version you're releasing). 32 | 33 | # Check for Docker base image updates 34 | 35 | Look for a new version of the Docker base image, and update the Dockerfiles if necessary. 36 | 37 | ## Tag the release 38 | 39 | Run the command `git tag -s x.y.z` (where x.y.z is the release version number). 40 | 41 | Suggested tag message is "Release x.y.z". 42 | 43 | Don't push the tag right away, though. 44 | Wait until the release is successful and you're sure there will be no more changes. 45 | Otherwise it can be a pain to remove an unwanted tag from Gerrit. 46 | 47 | 48 | ## Go! Go! Go! 49 | 50 | Here it is, the moment of truth. 51 | When you're ready to build the distribution archive: 52 | 53 | ./gradlew clean build 54 | 55 | If the build is successful, you're ready to publish the distribution archive to S3 with this shell command: 56 | 57 | VERS=x.y.z 58 | aws s3 cp build/distributions/couchbase-elasticsearch-connector-${VERS}.zip \ 59 | s3://packages.couchbase.com/clients/connectors/elasticsearch/${VERS}/couchbase-elasticsearch-connector-${VERS}.zip \ 60 | --acl public-read 61 | 62 | 63 | Whew, you did it! 64 | Or building or publishing failed and you're looking at a cryptic error message, in which case you might want to check out the Troubleshooting section below. 65 | 66 | If the release succeeded, now's the time to publish the tag: 67 | 68 | git push origin x.y.z 69 | 70 | ## Publish Docker image 71 | 72 | A daily job builds a Docker image from the current `master` branch. 73 | After pushing the tag, and **before making any other changes to the repo,** sit on your butt until the new image appears here: 74 | 75 | https://github.com/orgs/cb-vanilla/packages/container/package/elasticsearch-connector 76 | 77 | When you've identified the image built from the release tag, file a CBD issue in Jira. 78 | 79 | * Subject: Release Docker image for Elasticsearch connector x.y.z 80 | * Component: build 81 | * Description: link to the image to release 82 | 83 | ## Prepare for next dev cycle 84 | 85 | **STOP!** Did you wait for the Docker image to be built? Yes? Okay. Go! 86 | 87 | Increment the version number in `build.gradle` and restore the `-SNAPSHOT` suffix. 88 | Commit and push to Gerrit. 89 | 90 | ## Update the build manifest 91 | 92 | Clone the manifest repository https://review.couchbase.org/admin/repos/manifest 93 | 94 | Edit `manifest/couchbase-elasticsearch-connector/master.xml` 95 | 96 | Inside the `project` element, look for an `annotation` element where `name="VERSION"`. 97 | Update this element's `value` property to refer to the version under development, without the SNAPSHOT suffix. 98 | For example, if you just bumped the version to 4.3.2-SNAPSHOT, the version you're specifying in the manifest should be "4.3.2". 99 | 100 | Commit the change. 101 | 102 | ## Troubleshooting 103 | 104 | * Take another look at the Prerequisites section. 105 | Did you miss anything? 106 | -------------------------------------------------------------------------------- /INTELLIJ-SETUP.md: -------------------------------------------------------------------------------- 1 | When importing the Gradle project, make sure to enable: 2 | 3 | - [x] "Create separate module per source set". 4 | 5 | Then go to `Preferences > Project Settings > Compiler > Annotation Processors` and make it look like this: 6 | 7 | ![Annotation Processor Settings](imgs/intellij-annotation-proc-config.png) 8 | 9 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # This Makefile is used by our internal automation, its purpose is to generate 2 | # a tarball containing all the files needed to create a container image. The 3 | # resulting tarball is ingested in subsequent automation workflows. 4 | 5 | product = couchbase-elasticsearch-connector 6 | bldNum = $(if $(BLD_NUM),$(BLD_NUM),9999) 7 | version = $(if $(VERSION),$(VERSION),1.0.0) 8 | 9 | appDir = build/install/$(product) 10 | artifact = $(product)-image_$(version)-$(bldNum) 11 | 12 | dist: package-artifact 13 | 14 | package-artifact: create-tarball 15 | gzip -9 dist/$(artifact).tar 16 | mv dist/$(artifact).tar.gz dist/$(artifact).tgz 17 | 18 | create-tarball: compile 19 | mkdir -p dist/$(product)/$(appDir) 20 | cp Dockerfile dist/$(product) 21 | cp -a $(appDir)/* dist/$(product)/$(appDir) 22 | cd dist && tar -cf $(artifact).tar $(product) 23 | rm -rf dist/$(product) 24 | 25 | 26 | compile: clean 27 | ./gradlew clean installDist 28 | 29 | clean: 30 | rm -rf dist 31 | 32 | .PHONY: clean compile create-tarball package-artifact dist 33 | -------------------------------------------------------------------------------- /README.adoc: -------------------------------------------------------------------------------- 1 | = Couchbase Elasticsearch Connector 2 | 3 | https://docs.couchbase.com/elasticsearch-connector/4.4/release-notes.html[*Download*] 4 | | https://docs.couchbase.com/elasticsearch-connector/4.4/index.html[*Documentation*] 5 | | https://issues.couchbase.com/projects/CBES[*Issues*] 6 | | https://forums.couchbase.com/c/elasticsearch-connector[*Discussion*] 7 | 8 | The Couchbase Elasticsearch Connector replicates your documents from Couchbase Server to Elasticsearch in near real time. 9 | The connector uses the high-performance Database Change Protocol (DCP) to receive notifications when documents change in Couchbase. 10 | 11 | NOTE: If you're looking for the Elasticsearch Plug-in flavor of the connector, that's in a https://github.com/couchbase/couchbase-elasticsearch-connector/tree/release/cypress[different branch]. 12 | 13 | [small]_This product is neither affiliated with nor endorsed by Elastic. 14 | Elasticsearch is a trademark of Elasticsearch BV, registered in the U.S. and in other countries._ 15 | 16 | == Building the connector from source 17 | 18 | The connector distribution may be built from source with the command: 19 | 20 | ./gradlew build 21 | 22 | The distribution archive will be generated under `build/distributions`. 23 | During development, it might be more convenient to run: 24 | 25 | ./gradlew installDist 26 | 27 | which creates `build/install/couchbase-elasticsearch-connector` as a `$CBES_HOME` directory. 28 | 29 | 30 | === Running the integration tests 31 | 32 | A local Docker installation is required for these tests. 33 | To quickly test using only the latest Couchbase and Elasticsearch: 34 | 35 | ./gradlew integrationTest 36 | 37 | 38 | To test against _all_ supported versions of Couchbase and Elasticsearch: 39 | 40 | ./gradlew exhaustiveTest 41 | 42 | 43 | === IntelliJ IDEA setup 44 | Because the project uses annotation processors, some link:INTELLIJ-SETUP.md[fiddly setup] is required when importing the project into IntelliJ IDEA. 45 | 46 | 47 | === Building a Docker image 48 | 49 | Use `Dockerfile` to build a Docker image from source using Gradle. 50 | The version should be set in `build.gradle` before running. 51 | 52 | docker build -t imagename:tag . 53 | 54 | Use `Dockerfile.download` to build a Docker image from released binaries hosted at packages.couchbase.com. 55 | 56 | docker build -f Dockerfile.download -t imagename:tag --build-arg VERSION= 57 | 58 | where `` is the latest https://github.com/couchbase/couchbase-elasticsearch-connector/tags[tag] from the connector's GitHub repo. 59 | 60 | === Running a Docker image 61 | 62 | The built docker image can be configured using volume mounts. 63 | The `/opt/couchbase-elasticsearch-connector/config` directory should contain the configuration files, and the `/opt/couchbase-elasticsearch-connector/secrets` directory should contain the secrets. 64 | 65 | Find example configuration files in the `src/dist` directory. 66 | Be sure to rename `example-connector.toml` to `default-connector.toml`. 67 | 68 | docker run -p 31415:31415 -v ./config:/opt/couchbase-elasticsearch-connector/config -v ./secrets:/opt/couchbase-elasticsearch-connector/secrets -e CBES_GROUPNAME=groupname image:tag 69 | 70 | It is also valid to pass environment variables in via the Docker command line, which can then be used to substitute values in `default-connector.toml`. 71 | Port 31415 can be accessed via HTTP to get metrics. 72 | 73 | === Running in Kubernetes 74 | 75 | The connector can run in Kubernetes. 76 | See the examples in the `examples/kubernetes` directory, and https://docs.couchbase.com/elasticsearch-connector/current/kubernetes.html[the documentation] for more details. 77 | -------------------------------------------------------------------------------- /docs/README.adoc: -------------------------------------------------------------------------------- 1 | Documentation repository: https://github.com/couchbase/docs-elastic-search 2 | -------------------------------------------------------------------------------- /examples/kubernetes/README.adoc: -------------------------------------------------------------------------------- 1 | = Running the Couchbase Elasticsearch Connector in Kubernetes 2 | 3 | == Prerequisites 4 | 5 | - A Kubernetes cluster (minikube is fine) 6 | 7 | == Quickstart 8 | 9 | 1. Update `elasticsearch-connector-configuration.yaml` with your configuration. 10 | If using secure connections, add `truststore.jks` to the ConfigMap as a binary file with the certificates to be trusted. 11 | 2. Update `elasticsearch-connector.yaml` with your resource limits. 12 | 3. `kubectl apply -f .` 13 | 14 | == Group Name 15 | 16 | Each instance of the Elasticsearch connector MUST have a unique group name, 17 | otherwise the instances will interfere with each other. 18 | 19 | This group name is used to persist state information so the stream may resume gracefully. 20 | To change the group name, search for "example-group" in `elasticsearch-connector.yaml`. 21 | This name must be changed at each location it appears. 22 | 23 | == Additional Resources 24 | 25 | https://docs.couchbase.com/elasticsearch-connector/current/kubernetes.html[More documentation] is available. 26 | -------------------------------------------------------------------------------- /examples/kubernetes/elasticsearch-connector-service-account.yaml: -------------------------------------------------------------------------------- 1 | # Create the Kubernetes service account used by the connector 2 | # and grant it read-only access to StatefulSet resources. 3 | # This is only required when deploying the connector with 4 | # CBES_K8S_WATCH_REPLICAS=true 5 | apiVersion: v1 6 | kind: ServiceAccount 7 | metadata: 8 | name: couchbase-elasticsearch-connector 9 | namespace: default # adjust as necessary 10 | --- 11 | apiVersion: rbac.authorization.k8s.io/v1 12 | kind: ClusterRole 13 | metadata: 14 | name: couchbase-elasticsearch-conector-statefulset-reader 15 | rules: 16 | - apiGroups: ["apps"] 17 | resources: ["statefulsets"] 18 | verbs: ["get", "watch", "list"] 19 | --- 20 | apiVersion: rbac.authorization.k8s.io/v1 21 | kind: ClusterRoleBinding 22 | metadata: 23 | name: couchbase-elasticsearch-conector-read-statefulset 24 | subjects: 25 | - kind: ServiceAccount 26 | name: couchbase-elasticsearch-connector 27 | namespace: default # adjust as necessary 28 | roleRef: 29 | kind: ClusterRole 30 | name: couchbase-elasticsearch-conector-statefulset-reader 31 | apiGroup: rbac.authorization.k8s.io 32 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/couchbase/couchbase-elasticsearch-connector/8f86c0062565291e3f17bd9911c1790d83632d30/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%"=="" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%"=="" set DIRNAME=. 29 | @rem This is normally unused 30 | set APP_BASE_NAME=%~n0 31 | set APP_HOME=%DIRNAME% 32 | 33 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 34 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 35 | 36 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 37 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 38 | 39 | @rem Find java.exe 40 | if defined JAVA_HOME goto findJavaFromJavaHome 41 | 42 | set JAVA_EXE=java.exe 43 | %JAVA_EXE% -version >NUL 2>&1 44 | if %ERRORLEVEL% equ 0 goto execute 45 | 46 | echo. 47 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 48 | echo. 49 | echo Please set the JAVA_HOME variable in your environment to match the 50 | echo location of your Java installation. 51 | 52 | goto fail 53 | 54 | :findJavaFromJavaHome 55 | set JAVA_HOME=%JAVA_HOME:"=% 56 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 57 | 58 | if exist "%JAVA_EXE%" goto execute 59 | 60 | echo. 61 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 62 | echo. 63 | echo Please set the JAVA_HOME variable in your environment to match the 64 | echo location of your Java installation. 65 | 66 | goto fail 67 | 68 | :execute 69 | @rem Setup the command line 70 | 71 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 72 | 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if %ERRORLEVEL% equ 0 goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | set EXIT_CODE=%ERRORLEVEL% 85 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 86 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 87 | exit /b %EXIT_CODE% 88 | 89 | :mainEnd 90 | if "%OS%"=="Windows_NT" endlocal 91 | 92 | :omega 93 | -------------------------------------------------------------------------------- /imgs/intellij-annotation-proc-config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/couchbase/couchbase-elasticsearch-connector/8f86c0062565291e3f17bd9911c1790d83632d30/imgs/intellij-annotation-proc-config.png -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'org.gradle.toolchains.foojay-resolver-convention' version '0.7.0' 3 | } 4 | 5 | rootProject.name = 'couchbase-elasticsearch-connector' 6 | -------------------------------------------------------------------------------- /src/dist/config/consul.toml: -------------------------------------------------------------------------------- 1 | # This config file contains options specific to Autonomous Operations 2 | # mode with Consul. In order for these options to take effect, the path to 3 | # this file must be specified as the `--consul-config` argument of the 4 | # `cbes-consul` command. 5 | 6 | [consul] 7 | # Optional ACL Token to include in all Consul requests. If the value is an 8 | # empty string, the token is determined by the Consul agent configuration. 9 | # 10 | # You should not typically need to set this value. Instead, configure the 11 | # local Consul agent to use a token when talking to the Consul cluster. 12 | # 13 | # The pre-populated value gets the token from the "CBES_CONSUL_ACL_TOKEN" 14 | # environment variable, falling back to empty string if the environment 15 | # variable is not present. 16 | aclToken = '${CBES_CONSUL_ACL_TOKEN:}' 17 | 18 | # If true, the connector removes its Consul service entry prior to 19 | # shutting down in response to an interrupt signal. 20 | # 21 | # If not specified, defaults to true. 22 | deregisterServiceOnGracefulShutdown = true 23 | 24 | # When the connector terminates unexpectedly, or fails its health check, 25 | # Consul waits this long (plus a bit longer) before automatically removing 26 | # the connector's service entry. 27 | # 28 | # If not specified, defaults to 168 hours (7 days). 29 | # 30 | # Value is a number followed by a unit; use "m" for minutes, "h" for hours. 31 | # 32 | # Note from https://www.consul.io/api-docs/agent/check#register-check -- 33 | # The minimum timeout is 1 minute, and the process that reaps critical 34 | # services runs every 30 seconds, so it may take slightly longer than the 35 | # configured timeout to trigger the deregistration. This should generally 36 | # be configured with a timeout that's much, much longer than any expected 37 | # recoverable outage for the given service. 38 | deregisterCriticalServiceAfter = '168h' 39 | -------------------------------------------------------------------------------- /src/dist/config/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /src/dist/secrets/couchbase-client-cert-password.toml: -------------------------------------------------------------------------------- 1 | # Password for the PKCS12 bundle holding the client certificate for Couchbase. 2 | # Don't worry about this if you're authenticating with username & password. 3 | password = 'password' 4 | -------------------------------------------------------------------------------- /src/dist/secrets/couchbase-password.toml: -------------------------------------------------------------------------------- 1 | # Password for the Couchbase user. 2 | password = 'password' 3 | -------------------------------------------------------------------------------- /src/dist/secrets/elasticsearch-client-cert-password.toml: -------------------------------------------------------------------------------- 1 | # Password for the PKCS12 bundle holding the client certificate for Elasticsearch. 2 | # Don't worry about this if you're authenticating with username & password. 3 | password = 'password' 4 | -------------------------------------------------------------------------------- /src/dist/secrets/elasticsearch-password.toml: -------------------------------------------------------------------------------- 1 | # Password for the Elasticsearch user. 2 | # If connecting to Elastic Cloud, use your API key as the password. 3 | password = 'changeme' 4 | -------------------------------------------------------------------------------- /src/integrationTest/java/com/couchbase/connector/elasticsearch/DockerHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch; 18 | 19 | import com.google.common.base.Strings; 20 | 21 | import java.net.URI; 22 | import java.net.URISyntaxException; 23 | 24 | public class DockerHelper { 25 | private DockerHelper() { 26 | throw new AssertionError("not instantiable"); 27 | } 28 | 29 | public static String getDockerHost() { 30 | final String env = System.getenv("DOCKER_HOST"); 31 | if (Strings.isNullOrEmpty(env)) { 32 | return "localhost"; 33 | } 34 | 35 | try { 36 | return env.contains("://") ? new URI(env).getHost() : env; 37 | } catch (URISyntaxException e) { 38 | throw new RuntimeException(e); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/integrationTest/java/com/couchbase/connector/elasticsearch/ElasticsearchSinkContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch; 18 | 19 | import org.testcontainers.elasticsearch.ElasticsearchContainer; 20 | 21 | import java.io.IOException; 22 | import java.util.Optional; 23 | 24 | public class ElasticsearchSinkContainer implements SinkContainer { 25 | private final ElasticsearchContainer wrapped; 26 | 27 | public ElasticsearchSinkContainer(ElasticsearchContainer wrapped) { 28 | this.wrapped = wrapped; 29 | } 30 | 31 | @Override 32 | public boolean isRunning() { 33 | return wrapped.isRunning(); 34 | } 35 | 36 | @Override 37 | public String getHttpHostAddress() { 38 | return wrapped.getHttpHostAddress(); 39 | } 40 | 41 | @Override 42 | public String username() { 43 | return "elastic"; 44 | } 45 | 46 | @Override 47 | public String password() { 48 | return "changeme"; 49 | } 50 | 51 | @Override 52 | public Optional caCertAsBytes() { 53 | return wrapped.caCertAsBytes(); 54 | } 55 | 56 | @Override 57 | public void start() { 58 | wrapped.start(); 59 | } 60 | 61 | @Override 62 | public String getDockerImageName() { 63 | return wrapped.getDockerImageName(); 64 | } 65 | 66 | @Override 67 | public void close() throws IOException { 68 | wrapped.close(); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/integrationTest/java/com/couchbase/connector/elasticsearch/JsonDocument.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch; 18 | 19 | import static java.util.Objects.requireNonNull; 20 | 21 | public class JsonDocument { 22 | private final String id; 23 | private final Object content; 24 | 25 | public static JsonDocument create(String id, Object content) { 26 | return new JsonDocument(id, content); 27 | } 28 | 29 | public JsonDocument(String id, Object content) { 30 | this.id = requireNonNull(id); 31 | this.content = content; 32 | } 33 | 34 | public String id() { 35 | return id; 36 | } 37 | 38 | public Object content() { 39 | return content; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/integrationTest/java/com/couchbase/connector/elasticsearch/OpensearchSinkContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch; 18 | 19 | import org.opensearch.testcontainers.OpensearchContainer; 20 | 21 | import java.io.IOException; 22 | import java.util.Optional; 23 | 24 | import static java.util.Objects.requireNonNull; 25 | 26 | public class OpensearchSinkContainer implements SinkContainer { 27 | private final OpensearchContainer wrapped; 28 | 29 | public OpensearchSinkContainer(OpensearchContainer opensearchContainer) { 30 | this.wrapped = requireNonNull(opensearchContainer); 31 | } 32 | 33 | @Override 34 | public boolean isRunning() { 35 | return wrapped.isRunning(); 36 | } 37 | 38 | @Override 39 | public String getHttpHostAddress() { 40 | return wrapped.getHttpHostAddress(); 41 | } 42 | 43 | @Override 44 | public String username() { 45 | return wrapped.getUsername(); 46 | } 47 | 48 | @Override 49 | public String password() { 50 | return wrapped.getPassword(); 51 | } 52 | 53 | @Override 54 | public Optional caCertAsBytes() { 55 | return Optional.empty(); 56 | } 57 | 58 | @Override 59 | public void start() { 60 | wrapped.start(); 61 | } 62 | 63 | @Override 64 | public String getDockerImageName() { 65 | return wrapped.getDockerImageName(); 66 | } 67 | 68 | @Override 69 | public void close() throws IOException { 70 | wrapped.close(); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/integrationTest/java/com/couchbase/connector/elasticsearch/PatchableConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2025 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch; 18 | 19 | import com.couchbase.client.java.Bucket; 20 | import com.couchbase.connector.config.common.ImmutableCouchbaseConfig; 21 | import com.couchbase.connector.config.common.ImmutableGroupConfig; 22 | import com.couchbase.connector.config.es.ConnectorConfig; 23 | import com.couchbase.connector.config.es.ImmutableConnectorConfig; 24 | 25 | import java.util.function.Function; 26 | 27 | /** 28 | * A wrapper around an immutable connector config, with helper methods 29 | * for transforming nested properties. 30 | */ 31 | public class PatchableConfig { 32 | private final ImmutableConnectorConfig config; 33 | 34 | private PatchableConfig(ConnectorConfig config) { 35 | this.config = ImmutableConnectorConfig.copyOf(config); 36 | } 37 | 38 | public static PatchableConfig from(ConnectorConfig config) { 39 | return new PatchableConfig(config); 40 | } 41 | 42 | public PatchableConfig withBucket(String bucketName) { 43 | return withCouchbase(couchbase -> couchbase 44 | .withBucket(bucketName) 45 | .withMetadataBucket(bucketName) 46 | ); 47 | } 48 | 49 | public PatchableConfig withBucket(Bucket bucket) { 50 | return withBucket(bucket.name()); 51 | } 52 | 53 | public PatchableConfig withCouchbase(Function transformer) { 54 | return new PatchableConfig( 55 | config.withCouchbase(transformer.apply(ImmutableCouchbaseConfig.copyOf(config.couchbase()))) 56 | ); 57 | } 58 | 59 | public PatchableConfig withGroup(Function transformer) { 60 | return new PatchableConfig( 61 | config.withGroup(transformer.apply(ImmutableGroupConfig.copyOf(config.group()))) 62 | ); 63 | } 64 | 65 | public ImmutableConnectorConfig toConfig() { 66 | return config; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/integrationTest/java/com/couchbase/connector/elasticsearch/SinkContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch; 18 | 19 | import org.opensearch.testcontainers.OpensearchContainer; 20 | import org.slf4j.LoggerFactory; 21 | import org.testcontainers.containers.output.Slf4jLogConsumer; 22 | import org.testcontainers.elasticsearch.ElasticsearchContainer; 23 | 24 | import java.io.Closeable; 25 | import java.time.Duration; 26 | import java.util.Optional; 27 | 28 | public interface SinkContainer extends Closeable { 29 | 30 | static SinkContainer create(ElasticsearchVersionSniffer.Flavor flavor, String version) { 31 | 32 | if (flavor == ElasticsearchVersionSniffer.Flavor.ELASTICSEARCH) { 33 | return new ElasticsearchSinkContainer( 34 | new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:" + version) 35 | .withLogConsumer(new Slf4jLogConsumer(LoggerFactory.getLogger("container.elasticsearch"))) 36 | .withStartupTimeout(Duration.ofMinutes(5)) 37 | ); // CI Docker host is sloooooowwwwwwww 38 | } 39 | 40 | return new OpensearchSinkContainer( 41 | new OpensearchContainer<>("opensearchproject/opensearch:" + version) 42 | .withLogConsumer(new Slf4jLogConsumer(LoggerFactory.getLogger("container.opensearch"))) 43 | .withStartupTimeout(Duration.ofMinutes(5)) 44 | ); 45 | } 46 | 47 | boolean isRunning(); 48 | 49 | String getHttpHostAddress(); 50 | 51 | String username(); 52 | 53 | String password(); 54 | 55 | Optional caCertAsBytes(); 56 | 57 | void start(); 58 | 59 | String getDockerImageName(); 60 | } 61 | -------------------------------------------------------------------------------- /src/integrationTest/java/com/couchbase/connector/elasticsearch/TempBucket.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch; 18 | 19 | import com.couchbase.connector.testcontainers.CustomCouchbaseContainer; 20 | 21 | import java.io.Closeable; 22 | import java.util.concurrent.atomic.AtomicInteger; 23 | 24 | class TempBucket implements Closeable { 25 | private static final AtomicInteger counter = new AtomicInteger(); 26 | 27 | private final CustomCouchbaseContainer couchbase; 28 | private final String bucketName; 29 | 30 | public TempBucket(CustomCouchbaseContainer couchbase) { 31 | this(couchbase, nextName()); 32 | } 33 | 34 | public TempBucket(CustomCouchbaseContainer couchbase, String name) { 35 | this.couchbase = couchbase; 36 | this.bucketName = name; 37 | couchbase.createBucket(bucketName); 38 | } 39 | 40 | public static String nextName() { 41 | return "temp-" + counter.getAndIncrement(); 42 | } 43 | 44 | @Override 45 | public void close() { 46 | couchbase.deleteBucket(bucketName); 47 | } 48 | 49 | public String name() { 50 | return bucketName; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/integrationTest/java/com/couchbase/connector/elasticsearch/TestCouchbaseClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch; 18 | 19 | import com.couchbase.client.core.service.ServiceType; 20 | import com.couchbase.client.java.Bucket; 21 | import com.couchbase.client.java.Cluster; 22 | import com.couchbase.connector.config.es.ConnectorConfig; 23 | import com.couchbase.connector.config.es.ImmutableConnectorConfig; 24 | import com.couchbase.connector.dcp.CouchbaseHelper; 25 | import com.couchbase.connector.testcontainers.CustomCouchbaseContainer; 26 | import com.google.common.io.Closer; 27 | 28 | import java.io.Closeable; 29 | import java.io.IOException; 30 | import java.time.Duration; 31 | 32 | import static com.couchbase.client.dcp.core.utils.CbCollections.setOf; 33 | import static com.couchbase.client.java.diagnostics.WaitUntilReadyOptions.waitUntilReadyOptions; 34 | import static com.couchbase.connector.dcp.CouchbaseHelper.environmentConfigurator; 35 | 36 | class TestCouchbaseClient implements Closeable { 37 | private final Cluster cluster; 38 | private final Closer closer = Closer.create(); 39 | 40 | public TestCouchbaseClient(String config) { 41 | this(ConnectorConfig.from(config)); 42 | } 43 | 44 | public TestCouchbaseClient(PatchableConfig patcher) { 45 | this(patcher.toConfig()); 46 | } 47 | 48 | public TestCouchbaseClient(ImmutableConnectorConfig config) { 49 | this.cluster = CouchbaseHelper.createCluster( 50 | config.couchbase(), 51 | environmentConfigurator(config).andThen(env -> env 52 | .ioConfig(io -> io.enableMutationTokens(true)) 53 | .timeoutConfig(timeout -> timeout 54 | .connectTimeout(Duration.ofSeconds(15)) 55 | .kvTimeout(Duration.ofSeconds(10)) 56 | ) 57 | ) 58 | ); 59 | } 60 | 61 | public Cluster cluster() { 62 | return cluster; 63 | } 64 | 65 | /** 66 | * Create a new bucket with a unique name. The bucket will be deleted 67 | * when this client is closed. 68 | */ 69 | public Bucket createTempBucket(CustomCouchbaseContainer couchbase) { 70 | final TempBucket temp = closer.register(new TempBucket(couchbase)); 71 | Bucket bucket = cluster().bucket(temp.name()); 72 | Duration timeout = bucket.environment().timeoutConfig().connectTimeout(); 73 | 74 | // Multiplying timeout by 2 as a temporary workaround for JVMCBC-817 75 | // (giving the config loader time for the KV attempt to timeout and still leaving 76 | // time for loading the config from the manager). 77 | timeout = timeout.multipliedBy(2); 78 | 79 | bucket.waitUntilReady(timeout, waitUntilReadyOptions() 80 | .serviceTypes(setOf(ServiceType.KV))); 81 | 82 | return bucket; 83 | } 84 | 85 | /** 86 | * Create a new bucket with the given name. The bucket will be deleted 87 | * when this client is closed. 88 | */ 89 | public Bucket createTempBucket(CustomCouchbaseContainer couchbase, String bucketName) { 90 | final TempBucket temp = closer.register(new TempBucket(couchbase, bucketName)); 91 | return cluster().bucket(temp.name()); 92 | } 93 | 94 | @Override 95 | public void close() throws IOException { 96 | cluster.disconnect(); 97 | closer.close(); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/integrationTest/java/com/couchbase/connector/testcontainers/CustomCouchbaseContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.testcontainers; 18 | 19 | import com.couchbase.client.dcp.util.Version; 20 | import org.testcontainers.couchbase.CouchbaseContainer; 21 | import org.testcontainers.couchbase.CouchbaseService; 22 | import org.testcontainers.utility.DockerImageName; 23 | 24 | import java.time.Duration; 25 | import java.util.Optional; 26 | import java.util.Set; 27 | 28 | public class CustomCouchbaseContainer extends CouchbaseContainer { 29 | private final CouchbaseOps ops; 30 | 31 | public CustomCouchbaseContainer(String containerName) { 32 | super(DockerImageName.parse(containerName).asCompatibleSubstituteFor("couchbase/server")); 33 | this.ops = new CouchbaseOps(this, "localhost"); 34 | } 35 | 36 | public static CustomCouchbaseContainer newCouchbaseCluster(String dockerImageName) { 37 | @SuppressWarnings("resource") 38 | CouchbaseContainer couchbase = new CustomCouchbaseContainer(dockerImageName) 39 | .withEnabledServices(CouchbaseService.KV, CouchbaseService.QUERY, CouchbaseService.INDEX) 40 | .withStartupTimeout(Duration.ofMinutes(5)); // CI Docker host is sloooooowwwwwwww 41 | 42 | couchbase.start(); 43 | 44 | return (CustomCouchbaseContainer) couchbase; 45 | } 46 | 47 | public void loadSampleBucket(String bucketName, int bucketQuotaMb) { 48 | ops.loadSampleBucket(bucketName, bucketQuotaMb); 49 | } 50 | 51 | public void createBucket(String bucketName) { 52 | createBucket(bucketName, 100, 0, Set.of("kv", "n1ql", "index")); 53 | } 54 | 55 | public void createBucket(String bucketName, int bucketQuotaMb, int replicas, Set servicesToWaitFor) { 56 | ops.createBucket(bucketName, bucketQuotaMb, replicas, servicesToWaitFor); 57 | } 58 | 59 | public void deleteBucket(String bucketName) { 60 | ops.deleteBucket(bucketName); 61 | } 62 | 63 | public String getVersionString() { 64 | return getVersion().map(Version::toString).orElse("unknown"); 65 | } 66 | 67 | public Optional getVersion() { 68 | return ops.getVersion(); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/integrationTest/java/com/couchbase/connector/testcontainers/ExecUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.testcontainers; 18 | 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | import org.testcontainers.containers.Container; 22 | 23 | import java.io.IOException; 24 | import java.io.InterruptedIOException; 25 | import java.io.UncheckedIOException; 26 | import java.util.Arrays; 27 | 28 | public class ExecUtils { 29 | private static final Logger log = LoggerFactory.getLogger(ExecUtils.class); 30 | 31 | private ExecUtils() { 32 | throw new AssertionError("not instantiable"); 33 | } 34 | 35 | public static Container.ExecResult execOrDie(Container container, String... command) { 36 | return checkExitCode(execInContainerUnchecked(container, command)); 37 | } 38 | 39 | private static Container.ExecResult checkExitCode(Container.ExecResult result) { 40 | if (result.getExitCode() != 0) { 41 | throw new UncheckedIOException(new IOException(result.toString())); 42 | } 43 | return result; 44 | } 45 | 46 | public static Container.ExecResult execInContainerUnchecked(Container container, String... command) { 47 | try { 48 | log.info("Executing command: " + Arrays.toString(command)); 49 | 50 | return container.execInContainer(command); 51 | } catch (IOException e) { 52 | throw new UncheckedIOException(e); 53 | } catch (InterruptedException e) { 54 | throw new UncheckedIOException(new InterruptedIOException(e.getMessage())); 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/integrationTest/java/com/couchbase/connector/testcontainers/Poller.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.testcontainers; 18 | 19 | import com.couchbase.client.core.util.NanoTimestamp; 20 | 21 | import java.time.Duration; 22 | import java.util.concurrent.TimeoutException; 23 | import java.util.function.Supplier; 24 | 25 | import static java.util.Objects.requireNonNull; 26 | import static java.util.concurrent.TimeUnit.MILLISECONDS; 27 | 28 | /** 29 | * Useful for blocking the current thread until some condition is met. 30 | */ 31 | public class Poller { 32 | private Duration timeout = Duration.ofMinutes(2); 33 | private Duration interval = Duration.ofSeconds(2); 34 | private boolean timeoutIsFatal = true; 35 | 36 | public static Poller poll() { 37 | return new Poller(); 38 | } 39 | 40 | private Poller() { 41 | } 42 | 43 | public Poller atInterval(Duration interval) { 44 | this.interval = requireNonNull(interval); 45 | return this; 46 | } 47 | 48 | public Poller withTimeout(Duration timeout) { 49 | this.timeout = requireNonNull(timeout); 50 | return this; 51 | } 52 | 53 | public void untilTimeExpiresOr(Supplier condition) throws TimeoutException, InterruptedException { 54 | timeoutIsFatal = false; 55 | until(condition); 56 | } 57 | 58 | public void until(Supplier condition) throws TimeoutException, InterruptedException { 59 | NanoTimestamp start = NanoTimestamp.now(); 60 | Throwable suppressed = null; 61 | 62 | boolean first = true; 63 | 64 | do { 65 | if (first) { 66 | first = false; 67 | } else { 68 | MILLISECONDS.sleep(interval.toMillis()); 69 | } 70 | 71 | try { 72 | if (condition.get()) { 73 | return; 74 | } 75 | 76 | } catch (Throwable t) { 77 | suppressed = t; 78 | System.out.println("Polling condition threw exception: " + t.getMessage()); 79 | } 80 | } while (!start.hasElapsed(timeout)); 81 | 82 | if (timeoutIsFatal) { 83 | TimeoutException t = new TimeoutException(); 84 | if (suppressed != null) { 85 | t.addSuppressed(suppressed); 86 | } 87 | throw t; 88 | } 89 | } 90 | } 91 | 92 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/VersionHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector; 18 | 19 | import java.io.IOException; 20 | import java.io.InputStream; 21 | import java.io.InputStreamReader; 22 | import java.io.Reader; 23 | import java.util.Optional; 24 | import java.util.Properties; 25 | 26 | import static java.nio.charset.StandardCharsets.UTF_8; 27 | 28 | public class VersionHelper { 29 | private VersionHelper() { 30 | throw new AssertionError("not instantiable"); 31 | } 32 | 33 | private static final String version; 34 | private static final String gitInfo; 35 | 36 | static { 37 | String tempVersion = "unknown"; 38 | String tempGitInfo = "unknown"; 39 | 40 | try { 41 | final String versionResource = "version.properties"; 42 | try (InputStream is = VersionHelper.class.getResourceAsStream(versionResource)) { 43 | if (is == null) { 44 | throw new IOException("missing version properties resource: " + versionResource); 45 | } 46 | try (Reader r = new InputStreamReader(is, UTF_8)) { 47 | final Properties props = new Properties(); 48 | props.load(r); 49 | tempVersion = (String) props.getOrDefault("version", "unknown"); 50 | tempGitInfo = (String) props.getOrDefault("git", "unknown"); 51 | } 52 | } 53 | } catch (Exception e) { 54 | e.printStackTrace(); 55 | } 56 | 57 | version = tempVersion; 58 | gitInfo = tempGitInfo; 59 | } 60 | 61 | public static String getVersionString() { 62 | final String gitInfo = getGitInfo().orElse(null); 63 | return gitInfo == null ? version : version + "(" + gitInfo + ")"; 64 | } 65 | 66 | public static String getVersion() { 67 | return version; 68 | } 69 | 70 | public static Optional getGitInfo() { 71 | return version.equals(gitInfo) ? Optional.empty() : Optional.of(gitInfo); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/DefaultPanicButton.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster; 18 | 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import java.io.FileOutputStream; 23 | import java.io.OutputStreamWriter; 24 | import java.io.Writer; 25 | import java.util.ArrayList; 26 | import java.util.Collections; 27 | import java.util.List; 28 | import java.util.concurrent.CopyOnWriteArrayList; 29 | 30 | import static com.google.common.base.Throwables.getStackTraceAsString; 31 | import static java.util.Objects.requireNonNull; 32 | 33 | public class DefaultPanicButton implements PanicButton { 34 | private static final Logger log = LoggerFactory.getLogger(DefaultPanicButton.class); 35 | 36 | private final List prePanicHooks = new CopyOnWriteArrayList<>(); 37 | 38 | @Override 39 | public void addPrePanicHook(Runnable hook) { 40 | prePanicHooks.add(requireNonNull(hook)); 41 | } 42 | 43 | private void runPrePanicHooks() { 44 | try { 45 | List hooks = new ArrayList<>(prePanicHooks); 46 | Collections.reverse(hooks); 47 | hooks.forEach(it -> { 48 | try { 49 | it.run(); 50 | } catch (Throwable t) { 51 | log.error("Pre-panic hook threw exception.", t); 52 | } 53 | }); 54 | } catch (Throwable t) { 55 | log.error("Failed to invoke pre-panic hooks.", t); 56 | } 57 | } 58 | 59 | public void mildPanic(String message) { 60 | try { 61 | runPrePanicHooks(); 62 | 63 | if (runningInKubernetes()) { 64 | writeTerminationMessage(message); 65 | } 66 | 67 | log.warn("*** TERMINATING: {}", message); 68 | 69 | } finally { 70 | System.exit(1); 71 | } 72 | } 73 | 74 | @Override 75 | public void panic(String message, Throwable t) { 76 | //noinspection finally 77 | try { 78 | runPrePanicHooks(); 79 | 80 | if (runningInKubernetes()) { 81 | writeTerminationMessage(t == null ? message : message + "\n" + getStackTraceAsString(t)); 82 | } 83 | 84 | log.error("PANIC: {}", message, t); 85 | 86 | } finally { 87 | System.exit(1); 88 | } 89 | } 90 | 91 | private static boolean runningInKubernetes() { 92 | return System.getenv("KUBERNETES_SERVICE_HOST") != null; 93 | } 94 | 95 | private static void writeTerminationMessage(String msg) { 96 | String path = "/dev/termination-log"; 97 | try (Writer w = new OutputStreamWriter(new FileOutputStream(path))) { 98 | w.write(msg + "\n"); 99 | } catch (Exception e) { 100 | log.warn("Failed to write termination message to {}", path, e); 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/KillSwitch.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster; 18 | 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import java.time.Duration; 23 | import java.util.concurrent.ScheduledExecutorService; 24 | import java.util.concurrent.ScheduledFuture; 25 | import java.util.concurrent.TimeUnit; 26 | 27 | import static java.util.Objects.requireNonNull; 28 | import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; 29 | 30 | public class KillSwitch { 31 | private static final Logger log = LoggerFactory.getLogger(KillSwitch.class); 32 | 33 | private static final ScheduledExecutorService executor = newSingleThreadScheduledExecutor(r -> { 34 | Thread t = new Thread(r); 35 | t.setDaemon(true); 36 | t.setName("kill-switch-scheduler"); 37 | return t; 38 | } 39 | ); 40 | 41 | private ScheduledFuture future; 42 | private final Runnable deathRattle; 43 | private final Duration duration; 44 | 45 | private KillSwitch(Duration duration, Runnable deathRattle) { 46 | this.deathRattle = requireNonNull(deathRattle); 47 | this.duration = requireNonNull(duration); 48 | reset(); 49 | } 50 | 51 | public static KillSwitch start(Duration duration, Runnable deathRattle) { 52 | return new KillSwitch(duration, deathRattle); 53 | } 54 | 55 | public synchronized void cancel() { 56 | if (future != null) { 57 | boolean cancelledSuccessfully = future.cancel(false); 58 | if (!cancelledSuccessfully) { 59 | log.error("Kill switch cancellation failed. Oh dear."); 60 | } 61 | future = null; 62 | } 63 | } 64 | 65 | public synchronized void reset() { 66 | cancel(); 67 | 68 | future = executor.schedule(() -> { 69 | log.info("Kill switch activated."); 70 | 71 | //noinspection finally 72 | try { 73 | deathRattle.run(); 74 | 75 | } catch (Throwable t) { 76 | log.error("Kill switch death rattle threw exception.", t); 77 | 78 | } finally { 79 | System.exit(1); 80 | } 81 | 82 | return null; // unreachable, but lets compiler infer Void 83 | 84 | }, duration.toNanos(), TimeUnit.NANOSECONDS); 85 | 86 | log.debug("Kill switch reset; will activate in {} unless reset or cancelled.", duration); 87 | } 88 | 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/Membership.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster; 18 | 19 | import com.fasterxml.jackson.annotation.JsonCreator; 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | 22 | import java.util.LinkedHashSet; 23 | import java.util.Set; 24 | 25 | import static com.couchbase.connector.dcp.DcpHelper.allPartitions; 26 | import static com.couchbase.connector.util.ListHelper.chunks; 27 | 28 | public class Membership { 29 | private final int memberNumber; // valid rage = from 1 to clusterSize, inclusive 30 | private final int clusterSize; 31 | 32 | @JsonCreator 33 | public static Membership of(@JsonProperty("memberNumber") int memberNumber, @JsonProperty("clusterSize") int clusterSize) { 34 | return new Membership(memberNumber, clusterSize); 35 | } 36 | 37 | private Membership(int memberNumber, int clusterSize) { 38 | if (memberNumber <= 0 || memberNumber > clusterSize) { 39 | throw new IllegalArgumentException("Invalid static group membership number, must be between 1 and cluster size (" + clusterSize + ") inclusive."); 40 | } 41 | this.memberNumber = memberNumber; 42 | this.clusterSize = clusterSize; 43 | } 44 | 45 | public int getMemberNumber() { 46 | return memberNumber; 47 | } 48 | 49 | public int getClusterSize() { 50 | return clusterSize; 51 | } 52 | 53 | public Set getPartitions(int numPartitions) { 54 | return new LinkedHashSet<>(chunks(allPartitions(numPartitions), clusterSize).get(memberNumber - 1)); 55 | } 56 | 57 | @Override 58 | public String toString() { 59 | return memberNumber + "/" + clusterSize; 60 | } 61 | 62 | @Override 63 | public boolean equals(Object o) { 64 | if (this == o) { 65 | return true; 66 | } 67 | if (o == null || getClass() != o.getClass()) { 68 | return false; 69 | } 70 | 71 | Membership that = (Membership) o; 72 | 73 | if (memberNumber != that.memberNumber) { 74 | return false; 75 | } 76 | return clusterSize == that.clusterSize; 77 | } 78 | 79 | @Override 80 | public int hashCode() { 81 | int result = memberNumber; 82 | result = 31 * result + clusterSize; 83 | return result; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/PanicButton.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster; 18 | 19 | /** 20 | * The big red button that brings everything to a crashing halt. 21 | */ 22 | public interface PanicButton { 23 | default void panic(String message) { 24 | panic(message, null); 25 | } 26 | 27 | /** 28 | * Same as a panic, but log as WARN instead of ERROR and 29 | * don't say anything about panicking in the log message. 30 | */ 31 | void mildPanic(String message); 32 | 33 | void panic(String message, Throwable t); 34 | 35 | void addPrePanicHook(Runnable hook); 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/consul/AbstractLongPollTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster.consul; 18 | 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import java.time.Duration; 23 | import java.util.concurrent.atomic.AtomicInteger; 24 | 25 | import static com.google.common.util.concurrent.Uninterruptibles.joinUninterruptibly; 26 | import static java.util.Objects.requireNonNull; 27 | 28 | public abstract class AbstractLongPollTask implements AutoCloseable { 29 | private static final AtomicInteger threadCounter = new AtomicInteger(); 30 | private final Logger LOGGER = LoggerFactory.getLogger(getClass()); 31 | private final Thread thread; 32 | private volatile boolean closed; 33 | 34 | public AbstractLongPollTask(ConsulContext ctx, String threadNamePrefix, String sessionId) { 35 | requireNonNull(ctx); 36 | requireNonNull(sessionId); 37 | this.thread = new Thread(() -> doRun(ctx, sessionId), threadNamePrefix + threadCounter.getAndIncrement()); 38 | } 39 | 40 | protected abstract void doRun(ConsulContext ctx, String sessionId); 41 | 42 | public void awaitTermination() { 43 | if (!closed) { 44 | throw new IllegalStateException("must call close() first"); 45 | } 46 | 47 | Duration timeout = Duration.ofMinutes(1); 48 | LOGGER.info("Waiting for thread {} to terminate.", thread); 49 | joinUninterruptibly(thread, timeout); 50 | if (thread.isAlive()) { 51 | LOGGER.error("Thread {} did not terminate within {}.", thread, timeout); 52 | } else { 53 | LOGGER.info("Thread {} terminated.", thread.getName()); 54 | } 55 | } 56 | 57 | /** 58 | * @throws IllegalStateException if already started 59 | */ 60 | @SuppressWarnings("unchecked") 61 | public SELF start() { 62 | try { 63 | thread.start(); 64 | LOGGER.info("Thread {} started.", thread.getName()); 65 | return (SELF) this; 66 | 67 | } catch (IllegalThreadStateException e) { 68 | throw new IllegalStateException("May only be started once", e); 69 | } 70 | } 71 | 72 | protected boolean closed() { 73 | return closed; 74 | } 75 | 76 | public void close(boolean awaitTermination) { 77 | closed = true; 78 | 79 | LOGGER.info("Asking thread {} to terminate.", thread.getName()); 80 | thread.interrupt(); 81 | 82 | if (awaitTermination) { 83 | awaitTermination(); 84 | } 85 | } 86 | 87 | @Override 88 | public void close() { 89 | close(true); 90 | } 91 | 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/consul/AsyncTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster.consul; 18 | 19 | import com.google.common.base.Throwables; 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | import java.io.Closeable; 24 | import java.io.IOException; 25 | import java.time.Duration; 26 | import java.util.concurrent.TimeoutException; 27 | import java.util.concurrent.atomic.AtomicInteger; 28 | import java.util.concurrent.atomic.AtomicReference; 29 | import java.util.function.Consumer; 30 | 31 | import static com.google.common.util.concurrent.Uninterruptibles.joinUninterruptibly; 32 | import static java.util.Objects.requireNonNull; 33 | 34 | /** 35 | * Helper for running interruptible tasks in a separate thread. 36 | */ 37 | public class AsyncTask implements Closeable { 38 | private static final Logger LOGGER = LoggerFactory.getLogger(AsyncTask.class); 39 | 40 | private static final AtomicInteger threadCounter = new AtomicInteger(); 41 | 42 | @FunctionalInterface 43 | public interface Interruptible { 44 | /** 45 | * When the thread executing this method is interrupted, 46 | * the method should throw InterruptedException to indicate clean termination. 47 | */ 48 | void run() throws Throwable; 49 | } 50 | 51 | private final Thread thread; 52 | private final AtomicReference connectorException = new AtomicReference<>(); 53 | 54 | public static AsyncTask run(Interruptible task, Consumer fatalErrorListener) { 55 | return new AsyncTask(task, fatalErrorListener).start(); 56 | } 57 | 58 | public static AsyncTask run(Interruptible task) { 59 | return run(task, t -> { 60 | }); 61 | } 62 | 63 | private AsyncTask(Interruptible task, Consumer fatalErrorListener) { 64 | requireNonNull(task); 65 | requireNonNull(fatalErrorListener); 66 | 67 | thread = new Thread(() -> { 68 | try { 69 | task.run(); 70 | } catch (Throwable t) { 71 | connectorException.set(t); 72 | if (t instanceof InterruptedException) { 73 | LOGGER.debug("Connector task interrupted"); 74 | } else { 75 | LOGGER.error("Connector task exited early due to failure", t); 76 | fatalErrorListener.accept(t); 77 | } 78 | } 79 | }, "connector-main-" + threadCounter.getAndIncrement()); 80 | } 81 | 82 | private AsyncTask start() { 83 | try { 84 | thread.start(); 85 | LOGGER.info("Thread {} started.", thread.getName()); 86 | return this; 87 | 88 | } catch (IllegalThreadStateException e) { 89 | throw new IllegalStateException("May only be started once", e); 90 | } 91 | } 92 | 93 | public void stop() throws Throwable { 94 | thread.interrupt(); 95 | 96 | Duration timeout = Duration.ofSeconds(30); 97 | joinUninterruptibly(thread, timeout); 98 | if (thread.isAlive()) { 99 | throw new TimeoutException("Connector didn't exit within " + timeout); 100 | } 101 | 102 | Throwable t = connectorException.get(); 103 | if (t == null) { 104 | throw new IllegalStateException("Connector didn't exit by throwing exception"); 105 | } 106 | if (!(t instanceof InterruptedException)) { 107 | // The connector failed before we asked it to exit! 108 | throw t; 109 | } 110 | } 111 | 112 | @Override 113 | public void close() throws IOException { 114 | try { 115 | stop(); 116 | } catch (Throwable t) { 117 | Throwables.propagateIfPossible(t, IOException.class); 118 | throw new RuntimeException(t); 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/consul/ConsulDocumentWatcher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster.consul; 18 | 19 | import com.couchbase.consul.ConsulOps; 20 | import com.couchbase.consul.KvReadResult; 21 | import reactor.core.publisher.Flux; 22 | 23 | import java.time.Duration; 24 | import java.util.Optional; 25 | import java.util.concurrent.TimeoutException; 26 | import java.util.function.Function; 27 | import java.util.function.Predicate; 28 | 29 | import static com.couchbase.connector.cluster.consul.ReactorHelper.await; 30 | import static java.util.Objects.requireNonNull; 31 | 32 | public class ConsulDocumentWatcher { 33 | private final ConsulOps consul; 34 | private final ConsulResourceWatcher resourceWatcher; 35 | 36 | public ConsulDocumentWatcher(ConsulOps consul) { 37 | this(consul, Duration.ofMinutes(5)); 38 | } 39 | 40 | private ConsulDocumentWatcher(ConsulOps consul, Duration pollingInterval) { 41 | this.consul = requireNonNull(consul); 42 | this.resourceWatcher = new ConsulResourceWatcher().withPollingInterval(pollingInterval); 43 | } 44 | 45 | public ConsulDocumentWatcher withPollingInterval(Duration pollingInterval) { 46 | return new ConsulDocumentWatcher(this.consul, pollingInterval); 47 | } 48 | 49 | public Optional awaitCondition(String key, Predicate> valueCondition) throws InterruptedException { 50 | return await(watch(key), valueCondition); 51 | } 52 | 53 | public Optional awaitCondition(String key, Function valueMapper, Predicate> condition) 54 | throws InterruptedException { 55 | return await(watch(key).map(s -> s.map(valueMapper)), condition); 56 | } 57 | 58 | public Optional awaitCondition(String key, Function valueMapper, Predicate> condition, Duration timeout) 59 | throws InterruptedException, TimeoutException { 60 | return await(watch(key).map(s -> s.map(valueMapper)), condition, timeout); 61 | } 62 | 63 | /** 64 | * Blocks until the document does not exist or the current thread is interrupted. 65 | * 66 | * @param key the document to watch 67 | * @throws InterruptedException if the current thread is interrupted while waiting for document state to change. 68 | */ 69 | public void awaitAbsence(String key) throws InterruptedException { 70 | awaitCondition(key, Optional::isEmpty); 71 | } 72 | 73 | public Optional awaitValueChange(String key, String valueBeforeChange) throws InterruptedException { 74 | return awaitCondition(key, doc -> !doc.equals(Optional.ofNullable(valueBeforeChange))); 75 | } 76 | 77 | public Flux> watch(String key) { 78 | requireNonNull(key); 79 | return resourceWatcher.watch(opts -> consul.kv().readOneKey(key, opts)) 80 | .map(it -> it.body().map(KvReadResult::valueAsString)) 81 | .distinctUntilChanged(); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/consul/ConsulResourceWatcher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster.consul; 18 | 19 | import com.couchbase.consul.ConsulResponse; 20 | import com.google.common.primitives.Longs; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | import reactor.core.publisher.Flux; 24 | import reactor.core.publisher.Mono; 25 | 26 | import java.time.Duration; 27 | import java.util.Map; 28 | import java.util.concurrent.atomic.AtomicLong; 29 | import java.util.function.Function; 30 | 31 | import static java.util.Objects.requireNonNull; 32 | import static java.util.concurrent.TimeUnit.MILLISECONDS; 33 | import static java.util.concurrent.TimeUnit.MINUTES; 34 | 35 | public class ConsulResourceWatcher { 36 | private static final Logger log = LoggerFactory.getLogger(ConsulResourceWatcher.class); 37 | 38 | private final String wait; 39 | 40 | public ConsulResourceWatcher() { 41 | this(Duration.ofMinutes(5)); 42 | } 43 | 44 | private ConsulResourceWatcher(Duration pollingInterval) { 45 | final long requestedPollingIntervalSeconds = MILLISECONDS.toSeconds(pollingInterval.toMillis()); 46 | this.wait = Longs.constrainToRange(requestedPollingIntervalSeconds, 1, MINUTES.toSeconds(5)) + "s"; 47 | } 48 | 49 | public ConsulResourceWatcher withPollingInterval(Duration pollingInterval) { 50 | return new ConsulResourceWatcher(pollingInterval); 51 | } 52 | 53 | public Flux> watch( 54 | Function, Mono>> queryParametersToRequest 55 | ) { 56 | requireNonNull(queryParametersToRequest); 57 | 58 | return Flux.defer(() -> { 59 | AtomicLong prevIndex = new AtomicLong(); 60 | 61 | return queryParametersToRequest.apply((Map.of())) 62 | .expand(item -> { 63 | // Consul docs advise resetting index to zero if it goes backwards, 64 | // and sanity checking the response index is at least 1. Reference: 65 | // https://www.consul.io/api-docs/features/blocking#implementation-details 66 | long index = item.index().orElse(0); 67 | if (index < 1) { 68 | log.warn("Consul resource index was <= 0; this is probably a bug in Consul. Recovering by assuming an index of 1."); 69 | index = 1; 70 | } else if (index < prevIndex.get()) { 71 | log.debug("Consul resource index went backwards; resetting to 0."); 72 | index = 0; 73 | } 74 | prevIndex.set(index); 75 | 76 | Mono> expansion = queryParametersToRequest.apply(Map.of( 77 | "index", index, 78 | "wait", wait 79 | )); 80 | 81 | // If not found, delay to prevent busy loop while waiting for resource to appear. 82 | // Especially important when watching KV entries. 83 | return item.httpStatusCode() == 404 84 | ? Mono.delay(withFullJitter(Duration.ofSeconds(1))).then(expansion) 85 | : expansion; 86 | }).onBackpressureLatest(); 87 | }); 88 | } 89 | 90 | private static Duration withFullJitter(Duration d) { 91 | return Duration.ofMillis((long) (Math.random() * d.toMillis())); 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/consul/LeaderController.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster.consul; 18 | 19 | public interface LeaderController { 20 | void startLeading(); 21 | 22 | void stopLeading(); 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/consul/LeaderElectionTask.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster.consul; 18 | 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import java.util.Optional; 23 | import java.util.function.Consumer; 24 | 25 | import static java.util.Objects.requireNonNull; 26 | import static java.util.concurrent.TimeUnit.SECONDS; 27 | 28 | public class LeaderElectionTask extends AbstractLongPollTask { 29 | private static final Logger LOGGER = LoggerFactory.getLogger(LeaderElectionTask.class); 30 | 31 | private final String candidateUuid; 32 | private final Consumer fatalErrorConsumer; 33 | private final LeaderController leaderController; 34 | 35 | public LeaderElectionTask(ConsulContext ctx, String sessionId, String endpointId, Consumer fatalErrorConsumer, LeaderController leaderController) { 36 | super(ctx, "leader-election-", sessionId); 37 | this.candidateUuid = requireNonNull(endpointId); 38 | this.fatalErrorConsumer = requireNonNull(fatalErrorConsumer); 39 | this.leaderController = requireNonNull(leaderController); 40 | } 41 | 42 | protected void doRun(ConsulContext ctx, String sessionId) { 43 | final String leaderKey = ctx.keys().leader(); 44 | LOGGER.info("Leader key: {}", leaderKey); 45 | 46 | try { 47 | while (!closed()) { 48 | LOGGER.info("Racing to become new leader."); 49 | final boolean acquired = ctx.acquireLock(leaderKey, candidateUuid, sessionId); 50 | 51 | if (acquired) { 52 | LOGGER.info("Won the leader election! {}", candidateUuid); 53 | leaderController.startLeading(); 54 | 55 | final Optional newLeader = ctx.documentWatcher().awaitValueChange(leaderKey, candidateUuid); 56 | 57 | LOGGER.info("No longer the leader; new leader is {}", newLeader); 58 | leaderController.stopLeading(); 59 | 60 | } else { 61 | LOGGER.info("Not the leader"); 62 | 63 | // Sleeping because the attempt may have failed due to Consul's lock delay; 64 | // see https://www.consul.io/docs/internals/sessions.html 65 | SECONDS.sleep(1); 66 | 67 | ctx.documentWatcher().awaitAbsence(leaderKey); 68 | } 69 | } 70 | 71 | } catch (Throwable t) { 72 | if (closed()) { 73 | // Closing the task is likely to result in an InterruptedException 74 | LOGGER.debug("Caught exception in leader election loop after closing. Don't panic; this is expected.", t); 75 | } else { 76 | // Something went horribly, terribly, unrecoverably wrong. 77 | fatalErrorConsumer.accept(t); 78 | } 79 | 80 | } finally { 81 | try { 82 | leaderController.stopLeading(); 83 | 84 | // Abdicate (delete leadership document); only succeeds if we own the lock. This lets another node acquire 85 | // the lock immediately. If we don't do this, the lock will be auto-released when the session ends, 86 | // but the lock won't be eligible for acquisition until the Consul lock delay has elapsed. 87 | ctx.runCleanup(() -> ctx.unlockAndDelete(leaderKey, sessionId)); 88 | 89 | } catch (Exception e) { 90 | LOGGER.warn("Failed to abdicate", e); 91 | } 92 | 93 | LOGGER.info("Exiting leader election thread for session {}", sessionId); 94 | } 95 | } 96 | 97 | } 98 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/consul/LeaderEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster.consul; 18 | 19 | public enum LeaderEvent { 20 | CONFIG_CHANGE, 21 | MEMBERSHIP_CHANGE, 22 | PAUSE, 23 | RESUME, 24 | FATAL_ERROR 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/consul/TimeoutEnforcer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster.consul; 18 | 19 | import java.time.Duration; 20 | import java.util.concurrent.TimeUnit; 21 | import java.util.concurrent.TimeoutException; 22 | 23 | import static com.google.common.base.Preconditions.checkArgument; 24 | import static java.util.Objects.requireNonNull; 25 | import static java.util.concurrent.TimeUnit.NANOSECONDS; 26 | 27 | class TimeoutEnforcer { 28 | private final long startNanos = System.nanoTime(); 29 | private final long timeoutNanos; 30 | private final String description; 31 | 32 | private TimeoutEnforcer(String description, long timeoutNanos) { 33 | checkArgument(timeoutNanos >= 0, "timeout must be positive"); 34 | this.timeoutNanos = timeoutNanos; 35 | this.description = requireNonNull(description); 36 | } 37 | 38 | /** 39 | * @param timeout nullable (for no limit) 40 | */ 41 | public TimeoutEnforcer(String description, Duration timeout) { 42 | this(description, toNanos(timeout, Long.MAX_VALUE)); 43 | } 44 | 45 | public long remaining(TimeUnit timeUnit) throws TimeoutException { 46 | final long elapsed = System.nanoTime() - startNanos; 47 | final long nanosLeft = timeoutNanos - elapsed; 48 | 49 | if (nanosLeft <= 0) { 50 | throw new TimeoutException(description + " timed out after " + timeoutNanos + "ns"); 51 | } 52 | 53 | return convertRoundUp(nanosLeft, NANOSECONDS, timeUnit); 54 | } 55 | 56 | public void throwIfExpired() throws TimeoutException { 57 | remaining(TimeUnit.SECONDS); 58 | } 59 | 60 | public void throwIfExpiredUnchecked() { 61 | try { 62 | remaining(TimeUnit.SECONDS); 63 | } catch (TimeoutException e) { 64 | throw new RuntimeException(e); 65 | } 66 | } 67 | 68 | private static long convertRoundUp(long sourceDuration, TimeUnit sourceUnit, TimeUnit destUnit) { 69 | checkArgument(sourceDuration >= 0, "Duration must be non-negative"); 70 | long nanos = sourceUnit.toNanos(sourceDuration); 71 | nanos += destUnit.toNanos(1) - 1; 72 | if (nanos < 0) { 73 | nanos = Long.MAX_VALUE; 74 | } 75 | return destUnit.convert(nanos, NANOSECONDS); 76 | } 77 | 78 | private static long toNanos(Duration d) { 79 | try { 80 | return d.toNanos(); 81 | } catch (ArithmeticException e) { 82 | return d.isNegative() ? Long.MIN_VALUE : Long.MAX_VALUE; 83 | } 84 | } 85 | 86 | private static long toNanos(Duration d, long defaultValue) { 87 | return d == null ? defaultValue : toNanos(d); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/consul/WorkerService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster.consul; 18 | 19 | import com.couchbase.connector.cluster.Membership; 20 | import com.couchbase.connector.elasticsearch.Metrics; 21 | import com.fasterxml.jackson.annotation.JsonProperty; 22 | import com.fasterxml.jackson.databind.JsonNode; 23 | import com.github.therapi.core.annotation.Remotable; 24 | 25 | import static java.util.concurrent.TimeUnit.SECONDS; 26 | 27 | @Remotable("worker") 28 | public interface WorkerService { 29 | 30 | class Status { 31 | public static Status IDLE = new Status(null); 32 | 33 | private final Membership membership; 34 | 35 | /** 36 | * @param membership nullable (null means not streaming) 37 | */ 38 | public Status(@JsonProperty("membership") Membership membership) { 39 | this.membership = membership; 40 | } 41 | 42 | public Membership getMembership() { 43 | return membership; 44 | } 45 | 46 | @Override 47 | public String toString() { 48 | return "Status{" + 49 | "membership=" + membership + 50 | '}'; 51 | } 52 | } 53 | 54 | void stopStreaming(); 55 | 56 | void startStreaming(Membership membership, String config); 57 | 58 | Status status(); 59 | 60 | default void ping() { 61 | } 62 | 63 | default boolean ready() { 64 | return true; 65 | } 66 | 67 | default JsonNode metrics() { 68 | return Metrics.toJsonNode(); 69 | } 70 | 71 | default void sleep(long seconds) { 72 | try { 73 | SECONDS.sleep(seconds); 74 | } catch (InterruptedException e) { 75 | Thread.currentThread().interrupt(); 76 | } 77 | } 78 | 79 | default boolean stopped() { 80 | return status().getMembership() == null; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/consul/rpc/Broadcaster.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster.consul.rpc; 18 | 19 | import com.google.common.base.Stopwatch; 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | import java.io.Closeable; 24 | import java.util.ArrayList; 25 | import java.util.HashMap; 26 | import java.util.List; 27 | import java.util.Map; 28 | import java.util.concurrent.ExecutionException; 29 | import java.util.concurrent.ExecutorService; 30 | import java.util.concurrent.Executors; 31 | import java.util.concurrent.Future; 32 | import java.util.function.Consumer; 33 | import java.util.function.Function; 34 | 35 | import static com.couchbase.client.core.logging.RedactableArgument.redactSystem; 36 | 37 | public class Broadcaster implements Closeable { 38 | private static final Logger LOGGER = LoggerFactory.getLogger(Broadcaster.class); 39 | 40 | private final ExecutorService executor = Executors.newCachedThreadPool(); 41 | 42 | public Map> broadcast(String description, List endpoints, Class serviceInterface, Consumer endpointCallback) { 43 | return broadcast(description, endpoints, serviceInterface, s -> { 44 | endpointCallback.accept(s); 45 | return null; 46 | }); 47 | } 48 | 49 | public Map> broadcast(String description, List endpoints, Class serviceInterface, Function endpointCallback) { 50 | LOGGER.info("Broadcasting '{}' request to {} endpoints", description, endpoints.size()); 51 | LOGGER.debug("Endpoints: {}", endpoints); 52 | 53 | final Stopwatch timer = Stopwatch.createStarted(); 54 | 55 | final List>> futures = new ArrayList<>(); 56 | for (RpcEndpoint endpoint : endpoints) { 57 | futures.add(executor.submit( 58 | () -> RpcResult.newSuccess(endpointCallback.apply(endpoint.service(serviceInterface))))); 59 | } 60 | 61 | LOGGER.info("Scheduled all '{}' requests for broadcast. Awaiting responses...", description); 62 | 63 | final Map> results = new HashMap<>(); 64 | for (int i = 0; i < endpoints.size(); i++) { 65 | final RpcEndpoint endpoint = endpoints.get(i); 66 | final Future> f = futures.get(i); 67 | 68 | try { 69 | results.put(endpoint, f.get()); 70 | 71 | } catch (Throwable e) { 72 | if (e instanceof ExecutionException) { 73 | e = e.getCause(); 74 | } 75 | results.put(endpoint, RpcResult.newFailure(e)); 76 | LOGGER.error("Failed to apply '{}' callback for endpoint {}", description, redactSystem(endpoint), e); 77 | } 78 | } 79 | 80 | LOGGER.info("Finished collecting '{}' broadcast responses. Broadcasting to {} endpoints took {}", description, endpoints.size(), timer); 81 | LOGGER.debug("Broadcast results: {}", results); 82 | return results; 83 | } 84 | 85 | @Override 86 | public void close() { 87 | executor.shutdownNow(); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/consul/rpc/RpcEndpoint.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster.consul.rpc; 18 | 19 | import com.couchbase.connector.cluster.consul.ConsulDocumentWatcher; 20 | import com.couchbase.consul.ConsulOps; 21 | import com.fasterxml.jackson.databind.ObjectMapper; 22 | import com.github.therapi.jsonrpc.client.ServiceFactory; 23 | 24 | import java.time.Duration; 25 | 26 | import static com.github.therapi.jackson.ObjectMappers.newLenientObjectMapper; 27 | import static java.util.Objects.requireNonNull; 28 | 29 | /** 30 | * Used by RPC client to create service instances that talk to remote server. 31 | */ 32 | public class RpcEndpoint { 33 | private static final ObjectMapper rpcObjectMapper = newLenientObjectMapper(); 34 | 35 | private final ConsulRpcTransport transport; 36 | private final ServiceFactory serviceFactory; 37 | 38 | public RpcEndpoint(ConsulOps.KvOps kv, ConsulDocumentWatcher watcher, String endpointKey, Duration timeout) { 39 | this(new ConsulRpcTransport(kv, watcher, endpointKey, timeout)); 40 | } 41 | 42 | public RpcEndpoint(ConsulRpcTransport transport) { 43 | this.transport = requireNonNull(transport); 44 | this.serviceFactory = new ServiceFactory(rpcObjectMapper, transport); 45 | } 46 | 47 | public T service(Class remotableServiceInterface) { 48 | return serviceFactory.createService(remotableServiceInterface); 49 | } 50 | 51 | public RpcEndpoint withTimeout(Duration timeout) { 52 | return new RpcEndpoint(transport.withTimeout(timeout)); 53 | } 54 | 55 | @Override 56 | public String toString() { 57 | return transport.toString(); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/consul/rpc/RpcResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster.consul.rpc; 18 | 19 | import com.google.common.base.Throwables; 20 | 21 | public class RpcResult { 22 | private final T result; // nullable 23 | private final Throwable failure; // nullable 24 | 25 | public static RpcResult newSuccess(T value) { 26 | return new RpcResult<>(value, null); 27 | } 28 | 29 | public static RpcResult newSuccess() { 30 | return new RpcResult<>(null, null); 31 | } 32 | 33 | public static RpcResult newFailure(Throwable t) { 34 | return new RpcResult<>(null, t); 35 | } 36 | 37 | private RpcResult(T result, Throwable failure) { 38 | this.result = result; 39 | this.failure = failure; 40 | } 41 | 42 | public T get() { 43 | if (failure != null) { 44 | Throwables.throwIfUnchecked(failure); 45 | throw new RuntimeException(failure); 46 | } 47 | return result; 48 | } 49 | 50 | public boolean isFailed() { 51 | return failure != null; 52 | } 53 | 54 | @Override 55 | public String toString() { 56 | return "RpcResult{" + 57 | "result=" + result + 58 | ", failure=" + failure + 59 | '}'; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/cluster/k8s/StatefulSetInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.cluster.k8s; 18 | 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import static java.util.Objects.requireNonNull; 23 | 24 | public class StatefulSetInfo { 25 | private static final Logger log = LoggerFactory.getLogger(StatefulSetInfo.class); 26 | 27 | public final String name; 28 | public final int podOrdinal; 29 | 30 | public StatefulSetInfo(String name, int podOrdinal) { 31 | this.name = requireNonNull(name); 32 | this.podOrdinal = podOrdinal; 33 | } 34 | 35 | @Override 36 | public String toString() { 37 | return "StatefulSetInfo{" + 38 | "name='" + name + '\'' + 39 | ", podOrdinal=" + podOrdinal + 40 | '}'; 41 | } 42 | 43 | /** 44 | * The hostname of a pod running in a stateful set is the name of the stateful set 45 | * followed by a hyphen (-) and finally the pod number (zero-based). 46 | * This method parses the hostname into those parts. 47 | */ 48 | public static StatefulSetInfo fromHostname() { 49 | String hostname = System.getenv("HOSTNAME"); 50 | requireNonNull(hostname, "HOSTNAME environment variable not set."); 51 | try { 52 | log.debug("HOSTNAME = {}", hostname); 53 | int separatorIndex = hostname.lastIndexOf("-"); 54 | int ordinal = Integer.parseInt(hostname.substring(separatorIndex + 1)); 55 | String statefulSetName = hostname.substring(0, separatorIndex); 56 | return new StatefulSetInfo(statefulSetName, ordinal); 57 | } catch (Exception e) { 58 | throw new RuntimeException( 59 | "HOSTNAME environment variable '" + hostname + "' doesn't match -." + 60 | " Make sure to deploy as StatefulSet."); 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/ConfigException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config; 18 | 19 | public class ConfigException extends RuntimeException { 20 | public ConfigException(String message) { 21 | super(message); 22 | } 23 | 24 | public ConfigException(String message, Throwable cause) { 25 | super(message, cause); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/ScopeAndCollection.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config; 18 | 19 | import java.util.Objects; 20 | 21 | import static java.util.Objects.requireNonNull; 22 | 23 | public class ScopeAndCollection { 24 | public static final ScopeAndCollection DEFAULT = new ScopeAndCollection("_default", "_default"); 25 | 26 | private final String scope; 27 | private final String collection; 28 | 29 | public static ScopeAndCollection parse(String scopeAndCollection) { 30 | String[] split = scopeAndCollection.split("\\.", -1); 31 | if (split.length != 2) { 32 | throw new IllegalArgumentException( 33 | "Expected qualified collection name (scope.collection) but got: " + scopeAndCollection); 34 | } 35 | return new ScopeAndCollection(split[0], split[1]); 36 | } 37 | 38 | public ScopeAndCollection(String scope, String collection) { 39 | this.scope = requireNonNull(scope); 40 | this.collection = requireNonNull(collection); 41 | } 42 | 43 | public String getScope() { 44 | return scope; 45 | } 46 | 47 | public String getCollection() { 48 | return collection; 49 | } 50 | 51 | public String format() { 52 | return scope + "." + collection; 53 | } 54 | 55 | @Override 56 | public String toString() { 57 | return format(); 58 | } 59 | 60 | @Override 61 | public boolean equals(Object o) { 62 | if (this == o) return true; 63 | if (o == null || getClass() != o.getClass()) return false; 64 | ScopeAndCollection that = (ScopeAndCollection) o; 65 | return scope.equals(that.scope) && 66 | collection.equals(that.collection); 67 | } 68 | 69 | @Override 70 | public int hashCode() { 71 | return Objects.hash(scope, collection); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/StorageSize.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config; 18 | 19 | import java.util.Locale; 20 | import java.util.regex.Matcher; 21 | import java.util.regex.Pattern; 22 | 23 | public class StorageSize { 24 | private static final Pattern DIGITS = Pattern.compile("\\d+"); 25 | 26 | @Override 27 | public String toString() { 28 | return bytes + " bytes"; 29 | } 30 | 31 | private final long bytes; 32 | 33 | private StorageSize(long bytes) { 34 | this.bytes = bytes; 35 | } 36 | 37 | public static StorageSize ofBytes(long value) { 38 | return new StorageSize(value); 39 | } 40 | 41 | public static StorageSize ofMebibytes(long value) { 42 | return new StorageSize(value * 1024 * 1024); 43 | } 44 | 45 | public long getBytes() { 46 | return bytes; 47 | } 48 | 49 | public static StorageSize parse(String value) { 50 | try { 51 | String normalized = value 52 | .trim() 53 | .toLowerCase(Locale.ROOT); 54 | 55 | Matcher m = DIGITS.matcher(normalized); 56 | if (!m.find()) { 57 | throw new IllegalArgumentException("Value does not start with a number."); 58 | } 59 | String number = m.group(); 60 | 61 | long num = Long.parseLong(number); 62 | 63 | String unit = normalized.substring(number.length()).trim(); 64 | if (unit.isEmpty()) { 65 | throw new IllegalArgumentException("Missing size unit."); 66 | } 67 | 68 | final long scale; 69 | switch (unit) { 70 | case "b": 71 | scale = 1; 72 | break; 73 | case "k": 74 | case "kb": 75 | scale = 1024L; 76 | break; 77 | case "m": 78 | case "mb": 79 | scale = 1024L * 1024; 80 | break; 81 | case "g": 82 | case "gb": 83 | scale = 1024L * 1024 * 1024; 84 | break; 85 | case "t": 86 | case "tb": 87 | scale = 1024L * 1024 * 1024 * 1024; 88 | break; 89 | case "p": 90 | case "pb": 91 | scale = 1024L * 1024 * 1024 * 1024 * 1024; 92 | break; 93 | default: 94 | throw new IllegalArgumentException("Unrecognized size unit: '" + unit + "'"); 95 | } 96 | 97 | return new StorageSize(Math.multiplyExact(num, scale)); 98 | 99 | } catch (Exception e) { 100 | throw new IllegalArgumentException( 101 | "Failed to parse storage size value '" + value + "'; " + e.getMessage() + 102 | " ; A valid value is a number followed by a unit (for example, '10mb')." + 103 | " Valid units are" + 104 | " 'b' for bytes" + 105 | ", 'k' or 'kb' for kilobytes" + 106 | ", 'm' or 'mb' for megabytes" + 107 | ", 'g' or 'gb' for gigabytes" + 108 | ", 't' or 'tb' for terabytes" + 109 | ", 'p' or 'pb' for petabytes" + 110 | "." + 111 | " Values must be >= 0 bytes and <= " + Long.MAX_VALUE + " bytes."); 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/common/ClientCertConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.common; 18 | 19 | import com.couchbase.connector.config.toml.ConfigPosition; 20 | import com.couchbase.connector.config.toml.ConfigTable; 21 | import com.couchbase.connector.util.KeyStoreHelper; 22 | import org.immutables.value.Value; 23 | import org.jspecify.annotations.Nullable; 24 | 25 | import java.security.KeyStore; 26 | 27 | import static com.couchbase.connector.config.ConfigHelper.readPassword; 28 | import static com.google.common.base.Strings.emptyToNull; 29 | 30 | @Value.Immutable 31 | public interface ClientCertConfig { 32 | 33 | boolean use(); 34 | 35 | String path(); 36 | 37 | @Nullable 38 | @Value.Redacted 39 | String password(); 40 | 41 | @Nullable 42 | @Value.Auxiliary 43 | ConfigPosition position(); 44 | 45 | @Value.Lazy 46 | default KeyStore getKeyStore() { 47 | return KeyStoreHelper.get(path(), position(), password()); 48 | } 49 | 50 | static ImmutableClientCertConfig from(ConfigTable config, String parent) { 51 | if (config.isEmpty()) { 52 | return ClientCertConfig.disabled(); 53 | } 54 | 55 | String[] configProps = {"use", "path", "pathToPassword"}; 56 | config.expectOnly(configProps); 57 | config.require(parent, configProps); 58 | 59 | if (!config.getRequiredBoolean("use")) { 60 | return ClientCertConfig.disabled(); 61 | } 62 | 63 | return ImmutableClientCertConfig.builder() 64 | .use(true) 65 | .path(config.getRequiredString("path")) 66 | .password(emptyToNull(readPassword(config, parent, "pathToPassword"))) 67 | .position(config.inputPositionOf("path")) 68 | .build(); 69 | } 70 | 71 | static ImmutableClientCertConfig disabled() { 72 | return ImmutableClientCertConfig.builder() 73 | .use(false) 74 | .password("") 75 | .path("") 76 | .build(); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/common/ConsulConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.common; 18 | 19 | import com.couchbase.client.core.util.CbStrings; 20 | import com.couchbase.client.core.util.Golang; 21 | import com.couchbase.connector.config.ConfigException; 22 | import com.couchbase.connector.config.toml.ConfigTable; 23 | import com.couchbase.connector.config.toml.Toml; 24 | import org.immutables.value.Value; 25 | 26 | import java.io.File; 27 | import java.io.FileInputStream; 28 | import java.io.IOException; 29 | import java.io.InputStream; 30 | import java.time.Duration; 31 | import java.util.Optional; 32 | 33 | import static com.couchbase.connector.config.ConfigHelper.resolveVariables; 34 | 35 | @Value.Immutable 36 | @Value.Style(defaultAsDefault = true) 37 | public interface ConsulConfig { 38 | @Value.Redacted 39 | Optional aclToken(); 40 | 41 | default boolean deregisterServiceOnGracefulShutdown() { 42 | return true; 43 | } 44 | 45 | default Duration deregisterCriticalServiceAfter() { 46 | return Duration.ofDays(3); 47 | } 48 | 49 | static ImmutableConsulConfig from(ConfigTable root) { 50 | root.expectOnly("consul"); 51 | ConfigTable config = root.getTableOrEmpty("consul"); 52 | 53 | config.expectOnly( 54 | "aclToken", 55 | "deregisterServiceOnGracefulShutdown", 56 | "deregisterCriticalServiceAfter" 57 | ); 58 | 59 | ImmutableConsulConfig.Builder builder = ImmutableConsulConfig.builder() 60 | .aclToken(config.getString("aclToken").map(CbStrings::emptyToNull)); 61 | 62 | config.getString("deregisterCriticalServiceAfter") 63 | .map(Golang::parseDuration) 64 | .ifPresent(it -> { 65 | if (it.compareTo(Duration.ofMinutes(1)) < 0) { 66 | throw new ConfigException("Consul config property `deregisterCriticalServiceAfter` must be at least one minute, but got " + it); 67 | } 68 | builder.deregisterCriticalServiceAfter(it); 69 | }); 70 | 71 | config.getBoolean("deregisterServiceOnGracefulShutdown") 72 | .ifPresent(builder::deregisterServiceOnGracefulShutdown); 73 | 74 | return builder.build(); 75 | } 76 | 77 | static ImmutableConsulConfig from(String toml) { 78 | return ConsulConfig.from(Toml.parse(resolveVariables(toml))); 79 | } 80 | 81 | static ImmutableConsulConfig from(InputStream toml) throws IOException { 82 | return ConsulConfig.from(Toml.parse(resolveVariables(toml))); 83 | } 84 | 85 | static ImmutableConsulConfig from(File toml) throws IOException { 86 | try (InputStream is = new FileInputStream(toml)) { 87 | return ConsulConfig.from(is); 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/common/DcpConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.common; 18 | 19 | import com.couchbase.client.core.annotation.Stability; 20 | import com.couchbase.client.dcp.config.CompressionMode; 21 | import com.couchbase.connector.config.StorageSize; 22 | import com.couchbase.connector.config.toml.ConfigTable; 23 | import org.immutables.value.Value; 24 | 25 | import java.time.Duration; 26 | 27 | import static com.couchbase.connector.config.ConfigHelper.getSize; 28 | import static com.couchbase.connector.config.ConfigHelper.getTime; 29 | 30 | @Value.Immutable 31 | public interface DcpConfig { 32 | CompressionMode compression(); 33 | 34 | Duration persistencePollingInterval(); 35 | 36 | StorageSize flowControlBuffer(); 37 | 38 | @Stability.Volatile 39 | Duration connectTimeout(); 40 | 41 | static ImmutableDcpConfig from(ConfigTable config) { 42 | return ImmutableDcpConfig.builder() 43 | .compression(config.getBoolean("compression").orElse(true) ? CompressionMode.ENABLED : CompressionMode.DISABLED) 44 | .persistencePollingInterval(getTime(config, "persistencePollingInterval").orElse(Duration.ofMillis(100))) 45 | .flowControlBuffer(getSize(config, "flowControlBuffer").orElse(StorageSize.ofMebibytes(16))) 46 | .connectTimeout(getTime(config, "connectTimeout").orElse(Duration.ofSeconds(10))) 47 | .build(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/common/GroupConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.common; 18 | 19 | import com.couchbase.connector.cluster.Membership; 20 | import com.couchbase.connector.config.ConfigException; 21 | import com.couchbase.connector.config.toml.ConfigTable; 22 | import org.immutables.value.Value; 23 | 24 | @Value.Immutable 25 | public interface GroupConfig { 26 | String name(); 27 | 28 | Membership staticMembership(); 29 | 30 | static ImmutableGroupConfig from(ConfigTable config) { 31 | config.expectOnly("name", "static"); 32 | 33 | final ConfigTable staticGroup = config.getTableOrEmpty("static"); 34 | staticGroup.expectOnly("memberNumber", "totalMembers"); 35 | 36 | final int totalMembers = staticGroup.getIntInRange("totalMembers", 1, 1024).orElseThrow((() -> 37 | new ConfigException("missing 'static.totalMembers' property"))); 38 | 39 | final int memberNumber = staticGroup.getInt("memberNumber").orElseThrow(() -> 40 | new ConfigException("missing 'static.memberNumber' property")); 41 | 42 | try { 43 | return ImmutableGroupConfig.builder() 44 | .name(config.getString("name").orElseThrow(() -> new ConfigException("missing 'name' property"))) 45 | .staticMembership(Membership.of(memberNumber, totalMembers)) 46 | .build(); 47 | } catch (IllegalArgumentException e) { 48 | throw new ConfigException(e.getMessage()); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/common/LoggingConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.common; 18 | 19 | import com.couchbase.client.core.logging.RedactionLevel; 20 | import com.couchbase.connector.config.ConfigException; 21 | import com.couchbase.connector.config.toml.ConfigTable; 22 | import org.immutables.value.Value; 23 | 24 | @Value.Immutable 25 | public interface LoggingConfig { 26 | 27 | boolean logDocumentLifecycle(); 28 | 29 | RedactionLevel redactionLevel(); 30 | 31 | static ImmutableLoggingConfig from(ConfigTable config) { 32 | config.expectOnly("logDocumentLifecycle", "redactionLevel"); 33 | 34 | try { 35 | return ImmutableLoggingConfig.builder() 36 | .logDocumentLifecycle(config.getBoolean("logDocumentLifecycle").orElse(false)) 37 | .redactionLevel(config.getEnum("redactionLevel", RedactionLevel.class).orElse(RedactionLevel.NONE)) 38 | .build(); 39 | } catch (IllegalArgumentException e) { 40 | throw new ConfigException(e.getMessage()); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/common/MetricsConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.common; 18 | 19 | import com.couchbase.connector.config.toml.ConfigTable; 20 | import org.immutables.value.Value; 21 | 22 | import java.time.Duration; 23 | 24 | import static com.couchbase.connector.config.ConfigHelper.getTime; 25 | 26 | @Value.Immutable 27 | public interface MetricsConfig { 28 | Duration logInterval(); 29 | 30 | int httpPort(); 31 | 32 | static ImmutableMetricsConfig from(ConfigTable config) { 33 | config.expectOnly("logInterval", "httpPort"); 34 | 35 | return ImmutableMetricsConfig.builder() 36 | .logInterval(getTime(config, "logInterval").orElse(Duration.ofMinutes(1))) 37 | .httpPort(config.getIntInRange("httpPort", -1, 65535).orElse(-1)) 38 | .build(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/common/TrustStoreConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.common; 18 | 19 | import com.couchbase.connector.config.toml.ConfigPosition; 20 | import com.couchbase.connector.config.toml.ConfigTable; 21 | import com.couchbase.connector.util.KeyStoreHelper; 22 | import com.google.common.base.Supplier; 23 | import org.immutables.value.Value; 24 | import org.jspecify.annotations.Nullable; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | import java.security.KeyStore; 29 | import java.util.Optional; 30 | 31 | import static com.couchbase.connector.config.ConfigHelper.readPassword; 32 | import static com.google.common.base.Strings.emptyToNull; 33 | import static org.apache.commons.lang3.StringUtils.isBlank; 34 | 35 | @Value.Immutable 36 | public interface TrustStoreConfig extends Supplier { 37 | Logger log = LoggerFactory.getLogger(TrustStoreConfig.class); 38 | 39 | String path(); 40 | 41 | @Nullable 42 | @Value.Redacted 43 | String password(); 44 | 45 | @Nullable 46 | @Value.Auxiliary 47 | ConfigPosition position(); 48 | 49 | @Value.Lazy 50 | @Override 51 | default KeyStore get() { 52 | return KeyStoreHelper.get(path(), position(), password()); 53 | } 54 | 55 | static Optional from(ConfigTable config) { 56 | config.expectOnly("path", "pathToPassword"); 57 | 58 | String path = config.getString("path").orElse(null); 59 | if (isBlank(path) || path.equals("path/to/truststore")) { 60 | return Optional.empty(); 61 | } 62 | 63 | log.warn("The [truststore] config section is deprecated. Please use the 'pathToCaCertificate' properties instead."); 64 | 65 | final String password = readPassword(config, "truststore", "pathToPassword"); 66 | 67 | return Optional.of( 68 | ImmutableTrustStoreConfig.builder() 69 | .path(config.getRequiredString("path")) 70 | .password(emptyToNull(password)) 71 | .position(config.inputPositionOf("path")) 72 | .build() 73 | ); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/es/AwsConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.es; 18 | 19 | import com.couchbase.connector.config.toml.ConfigTable; 20 | import org.immutables.value.Value; 21 | 22 | @Value.Immutable 23 | public interface AwsConfig { 24 | String region(); 25 | 26 | static ImmutableAwsConfig from(ConfigTable config) { 27 | config.expectOnly("region"); 28 | return ImmutableAwsConfig.builder() 29 | .region(config.getString("region").orElse("")) 30 | .build(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/es/BulkRequestConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.es; 18 | 19 | import com.couchbase.connector.config.StorageSize; 20 | import com.couchbase.connector.config.toml.ConfigTable; 21 | import org.immutables.value.Value; 22 | 23 | import java.time.Duration; 24 | 25 | import static com.couchbase.connector.config.ConfigHelper.getSize; 26 | import static com.couchbase.connector.config.ConfigHelper.getTime; 27 | 28 | @Value.Immutable 29 | public interface BulkRequestConfig { 30 | int maxActions(); 31 | 32 | StorageSize maxBytes(); 33 | 34 | int concurrentRequests(); 35 | 36 | Duration timeout(); 37 | 38 | @Value.Check 39 | default void check() { 40 | if (concurrentRequests() <= 0) { 41 | throw new IllegalArgumentException("concurrentRequests must be > 0"); 42 | } 43 | } 44 | 45 | static ImmutableBulkRequestConfig from(ConfigTable config) { 46 | config.expectOnly("actions", "bytes", "timeout", "concurrentRequests"); 47 | return ImmutableBulkRequestConfig.builder() 48 | .maxActions(config.getInt("actions").orElse(1000)) 49 | .maxBytes(getSize(config, "bytes").orElse(StorageSize.ofMebibytes(10))) 50 | .timeout(getTime(config, "timeout").orElse(Duration.ofMinutes(1))) 51 | .concurrentRequests(config.getIntInRange("concurrentRequests", 1, 16).orElse(2)) 52 | .build(); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/es/ConnectorConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.es; 18 | 19 | import com.couchbase.connector.config.common.CouchbaseConfig; 20 | import com.couchbase.connector.config.common.GroupConfig; 21 | import com.couchbase.connector.config.common.LoggingConfig; 22 | import com.couchbase.connector.config.common.MetricsConfig; 23 | import com.couchbase.connector.config.common.TrustStoreConfig; 24 | import com.couchbase.connector.config.toml.ConfigTable; 25 | import com.couchbase.connector.config.toml.Toml; 26 | import org.immutables.value.Value; 27 | 28 | import java.io.File; 29 | import java.io.FileInputStream; 30 | import java.io.IOException; 31 | import java.io.InputStream; 32 | import java.util.Optional; 33 | 34 | import static com.couchbase.connector.config.ConfigHelper.resolveVariables; 35 | 36 | @Value.Immutable 37 | public interface ConnectorConfig { 38 | 39 | CouchbaseConfig couchbase(); 40 | 41 | ElasticsearchConfig elasticsearch(); 42 | 43 | LoggingConfig logging(); 44 | 45 | MetricsConfig metrics(); 46 | 47 | GroupConfig group(); 48 | 49 | Optional trustStore(); 50 | 51 | static ImmutableConnectorConfig from(ConfigTable config) { 52 | config.expectOnly("couchbase", "elasticsearch", "logging", "metrics", "group", "truststore"); 53 | 54 | return ImmutableConnectorConfig.builder() 55 | .couchbase(CouchbaseConfig.from(config.getTableOrEmpty("couchbase"))) 56 | .elasticsearch(ElasticsearchConfig.from(config.getTableOrEmpty("elasticsearch"))) 57 | .logging(LoggingConfig.from(config.getTableOrEmpty("logging"))) 58 | .metrics(MetricsConfig.from(config.getTableOrEmpty("metrics"))) 59 | .group(GroupConfig.from(config.getTableOrEmpty("group"))) 60 | .trustStore(TrustStoreConfig.from(config.getTableOrEmpty("truststore"))) 61 | .build(); 62 | } 63 | 64 | static ImmutableConnectorConfig from(String toml) { 65 | return ConnectorConfig.from(Toml.parse(resolveVariables(toml))); 66 | } 67 | 68 | static ImmutableConnectorConfig from(InputStream toml) throws IOException { 69 | return ConnectorConfig.from(Toml.parse(resolveVariables(toml))); 70 | } 71 | 72 | static ImmutableConnectorConfig from(File toml) throws IOException { 73 | try (InputStream is = new FileInputStream(toml)) { 74 | return ConnectorConfig.from(is); 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/es/DocStructureConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.es; 18 | 19 | import com.couchbase.connector.config.toml.ConfigTable; 20 | import com.google.common.base.Strings; 21 | import org.immutables.value.Value; 22 | import org.jspecify.annotations.Nullable; 23 | 24 | @Value.Immutable 25 | public interface DocStructureConfig { 26 | 27 | @Nullable 28 | String metadataFieldName(); 29 | 30 | boolean documentContentAtTopLevel(); 31 | 32 | boolean wrapCounters(); 33 | 34 | static ImmutableDocStructureConfig from(ConfigTable config) { 35 | config.expectOnly("metadataFieldName", "documentContentAtTopLevel", "wrapCounters"); 36 | 37 | return ImmutableDocStructureConfig.builder() 38 | .metadataFieldName(Strings.emptyToNull(config.getString("metadataFieldName").orElse(null))) 39 | .documentContentAtTopLevel(config.getBoolean("documentContentAtTopLevel").orElse(false)) 40 | .wrapCounters(config.getBoolean("wrapCounters").orElse(false)) 41 | .build(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/es/ElasticCloudConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.es; 18 | 19 | import com.couchbase.connector.config.toml.ConfigTable; 20 | import org.immutables.value.Value; 21 | 22 | @Value.Immutable 23 | public interface ElasticCloudConfig { 24 | boolean enabled(); 25 | 26 | static ImmutableElasticCloudConfig from(ConfigTable config) { 27 | config.expectOnly("enabled"); 28 | 29 | return ImmutableElasticCloudConfig.builder() 30 | .enabled(config.getBoolean("enabled").orElse(false)) 31 | .build(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/es/RejectLogConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.es; 18 | 19 | import com.couchbase.connector.config.toml.ConfigTable; 20 | import com.google.common.base.Strings; 21 | import org.immutables.value.Value; 22 | import org.jspecify.annotations.Nullable; 23 | 24 | import static com.couchbase.connector.config.ConfigHelper.warnIfDeprecatedTypeNameIsPresent; 25 | 26 | @Value.Immutable 27 | public interface RejectLogConfig { 28 | @Nullable 29 | String index(); 30 | 31 | static ImmutableRejectLogConfig from(ConfigTable config) { 32 | config.expectOnly("index", "typeName"); 33 | 34 | warnIfDeprecatedTypeNameIsPresent(config); 35 | 36 | return ImmutableRejectLogConfig.builder() 37 | .index(Strings.emptyToNull(config.getString("index").orElse(null))) 38 | .build(); 39 | } 40 | } 41 | 42 | 43 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/toml/ConfigArray.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.toml; 18 | 19 | import org.apache.tuweni.toml.TomlArray; 20 | import org.apache.tuweni.toml.TomlPosition; 21 | 22 | import static java.util.Objects.requireNonNull; 23 | 24 | public class ConfigArray { 25 | private final TomlArray wrapped; 26 | 27 | public ConfigArray(TomlArray wrapped) { 28 | this.wrapped = requireNonNull(wrapped); 29 | } 30 | 31 | public int size() { 32 | return wrapped.size(); 33 | } 34 | 35 | public ConfigTable getTable(int i) { 36 | return new ConfigTable(wrapped.getTable(i)); 37 | } 38 | 39 | public ConfigPosition inputPositionOf(int i) { 40 | TomlPosition pos = wrapped.inputPositionOf(i); 41 | return pos == null ? null : new ConfigPosition(pos); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/toml/ConfigPosition.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.toml; 18 | 19 | import org.apache.tuweni.toml.TomlPosition; 20 | 21 | import static java.util.Objects.requireNonNull; 22 | 23 | public class ConfigPosition { 24 | private final TomlPosition wrapped; 25 | 26 | public ConfigPosition(TomlPosition wrapped) { 27 | this.wrapped = requireNonNull(wrapped); 28 | } 29 | 30 | @Override 31 | public String toString() { 32 | return wrapped.toString(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/toml/ParseResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.toml; 18 | 19 | import com.couchbase.connector.config.ConfigException; 20 | import org.apache.tuweni.toml.TomlParseResult; 21 | 22 | public class ParseResult extends ConfigTable { 23 | public ParseResult(TomlParseResult wrapped) { 24 | super(wrapped); 25 | 26 | if (wrapped.hasErrors()) { 27 | throw new ConfigException("Config syntax error: " + wrapped.errors()); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/config/toml/Toml.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config.toml; 18 | 19 | public class Toml { 20 | public static ParseResult parse(String toml) { 21 | return new ParseResult(org.apache.tuweni.toml.Toml.parse(toml)); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/dcp/Checkpoint.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.dcp; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | 21 | import static java.util.Objects.requireNonNull; 22 | 23 | /** 24 | * Position in a DCP stream. 25 | */ 26 | public class Checkpoint { 27 | private final long vbuuid; // vbucket uuid for which the sequence number is valid 28 | private final long seqno; // sequence number of a DCP event 29 | private final SnapshotMarker snapshot; 30 | 31 | public static final Checkpoint ZERO = new Checkpoint(0, 0, new SnapshotMarker(0, 0)); 32 | 33 | public Checkpoint(@JsonProperty("vbuuid") long vbuuid, 34 | @JsonProperty("seqno") long seqno, 35 | @JsonProperty("snapshot") SnapshotMarker snapshot) { 36 | this.vbuuid = vbuuid; 37 | this.seqno = seqno; 38 | this.snapshot = requireNonNull(snapshot); 39 | } 40 | 41 | public long getVbuuid() { 42 | return vbuuid; 43 | } 44 | 45 | public long getSeqno() { 46 | return seqno; 47 | } 48 | 49 | public SnapshotMarker getSnapshot() { 50 | return snapshot; 51 | } 52 | 53 | @Override 54 | public String toString() { 55 | return vbuuid + "@" + seqno + snapshot; 56 | } 57 | 58 | @Override 59 | public boolean equals(Object o) { 60 | if (this == o) { 61 | return true; 62 | } 63 | if (o == null || getClass() != o.getClass()) { 64 | return false; 65 | } 66 | 67 | Checkpoint that = (Checkpoint) o; 68 | 69 | if (vbuuid != that.vbuuid) { 70 | return false; 71 | } 72 | if (seqno != that.seqno) { 73 | return false; 74 | } 75 | return snapshot.equals(that.snapshot); 76 | } 77 | 78 | @Override 79 | public int hashCode() { 80 | int result = (int) (vbuuid ^ (vbuuid >>> 32)); 81 | result = 31 * result + (int) (seqno ^ (seqno >>> 32)); 82 | return result; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/dcp/CheckpointDao.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.dcp; 18 | 19 | import java.util.Map; 20 | import java.util.Set; 21 | 22 | public interface CheckpointDao { 23 | 24 | void save(String bucketUuid, Map vbucketToCheckpoint); 25 | 26 | Map loadExisting(String bucketUuid, Set vbuckets); 27 | 28 | default Map loadOrDefaultToZero(String bucketUuid, Set vbuckets) { 29 | Map result = loadExisting(bucketUuid, vbuckets); 30 | for (Integer vbucket : vbuckets) { 31 | result.putIfAbsent(vbucket, Checkpoint.ZERO); 32 | } 33 | return result; 34 | } 35 | 36 | void clear(String bucketUuid, Set vbuckets); 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/dcp/Event.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.dcp; 18 | 19 | import com.couchbase.client.dcp.highlevel.DocumentChange; 20 | import com.couchbase.client.dcp.highlevel.Mutation; 21 | import com.couchbase.client.dcp.highlevel.StreamOffset; 22 | 23 | import static java.util.Objects.requireNonNull; 24 | 25 | public class Event { 26 | private final DocumentChange change; 27 | private final boolean mutation; 28 | private final long receivedNanos = System.nanoTime(); 29 | 30 | public Event(DocumentChange change) { 31 | this.change = requireNonNull(change); 32 | this.mutation = change instanceof Mutation; 33 | } 34 | 35 | /** 36 | * Must be called when the connector is finished processing the event. 37 | */ 38 | public void release() { 39 | change.flowControlAck(); 40 | } 41 | 42 | public int getVbucket() { 43 | return change.getVbucket(); 44 | } 45 | 46 | public String getKey() { 47 | return change.getKey(); 48 | } 49 | 50 | public String getKey(boolean qualifiedWithScopeAndCollection) { 51 | return qualifiedWithScopeAndCollection ? change.getQualifiedKey() : change.getKey(); 52 | } 53 | 54 | public boolean isMutation() { 55 | return mutation; 56 | } 57 | 58 | public long getReceivedNanos() { 59 | return receivedNanos; 60 | } 61 | 62 | /** 63 | * Returns a tracing token for correlating lifecycle log messages. 64 | */ 65 | public long getTracingToken() { 66 | return change.getTracingToken(); 67 | } 68 | 69 | /** 70 | * Be aware that sequence numbers are unsigned, and must be compared using 71 | * {@link Long#compareUnsigned(long, long)} 72 | */ 73 | public long getSeqno() { 74 | return change.getOffset().getSeqno(); 75 | } 76 | 77 | public Checkpoint getCheckpoint() { 78 | return toCheckpoint(change.getOffset()); 79 | } 80 | 81 | private static Checkpoint toCheckpoint(StreamOffset offset) { 82 | return new Checkpoint(offset.getVbuuid(), offset.getSeqno(), 83 | new SnapshotMarker( 84 | offset.getSnapshot().getStartSeqno(), 85 | offset.getSnapshot().getEndSeqno())); 86 | } 87 | 88 | public DocumentChange getChange() { 89 | return change; 90 | } 91 | 92 | public byte[] getContent() { 93 | return change.getContent(); 94 | } 95 | 96 | @Override 97 | public String toString() { 98 | return change.toString(); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/dcp/ResolvedBucketConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.dcp; 18 | 19 | 20 | import com.couchbase.client.core.config.CouchbaseBucketConfig; 21 | import com.couchbase.client.core.config.NodeInfo; 22 | import com.couchbase.client.core.service.ServiceType; 23 | import com.couchbase.client.core.util.HostAndPort; 24 | 25 | import java.util.List; 26 | import java.util.Map; 27 | import java.util.stream.Collectors; 28 | 29 | import static java.util.Objects.requireNonNull; 30 | 31 | /** 32 | * A wrapper around a {@link CouchbaseBucketConfig} that automatically resolves alternate addresses. 33 | * 34 | * @implNote This is a trimmed down version of DcpBucketConfig from the DCP client, 35 | * modified to work with the core-io version of BucketConfig. 36 | */ 37 | public class ResolvedBucketConfig { 38 | private final boolean sslEnabled; 39 | private final CouchbaseBucketConfig config; 40 | private final List allNodes; 41 | 42 | public ResolvedBucketConfig(final CouchbaseBucketConfig config, final boolean sslEnabled) { 43 | this.config = requireNonNull(config); 44 | this.sslEnabled = sslEnabled; 45 | this.allNodes = config.nodes(); 46 | } 47 | 48 | public long rev() { 49 | return config.rev(); 50 | } 51 | 52 | public String uuid() { 53 | return config.uuid(); 54 | } 55 | 56 | public int numberOfPartitions() { 57 | return config.numberOfPartitions(); 58 | } 59 | 60 | public int numberOfReplicas() { 61 | return config.numberOfReplicas(); 62 | } 63 | 64 | public List nodes() { 65 | return allNodes; 66 | } 67 | 68 | public List getKvAddresses() { 69 | return allNodes.stream() 70 | .filter(this::hasKvService) 71 | .map(this::getKvAddress) 72 | .collect(Collectors.toList()); 73 | } 74 | 75 | private HostAndPort getKvAddress(final NodeInfo node) { 76 | int port = getServicePortMap(node).get(ServiceType.KV); 77 | return new HostAndPort(node.hostname(), port); 78 | } 79 | 80 | private Map getServicePortMap(final NodeInfo node) { 81 | return sslEnabled ? node.sslServices() : node.services(); 82 | } 83 | 84 | private boolean hasKvService(final NodeInfo node) { 85 | return getServicePortMap(node).containsKey(ServiceType.KV); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/dcp/SnapshotMarker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.dcp; 18 | 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | 21 | public class SnapshotMarker { 22 | private final long startSeqno; 23 | private final long endSeqno; 24 | 25 | public SnapshotMarker(@JsonProperty("startSeqno") long startSeqno, 26 | @JsonProperty("endSeqno") long endSeqno) { 27 | this.startSeqno = startSeqno; 28 | this.endSeqno = endSeqno; 29 | } 30 | 31 | public long getStartSeqno() { 32 | return startSeqno; 33 | } 34 | 35 | public long getEndSeqno() { 36 | return endSeqno; 37 | } 38 | 39 | @Override 40 | public String toString() { 41 | return "[" + startSeqno + "-" + endSeqno + "]"; 42 | } 43 | 44 | @Override 45 | public boolean equals(Object o) { 46 | if (this == o) { 47 | return true; 48 | } 49 | if (o == null || getClass() != o.getClass()) { 50 | return false; 51 | } 52 | 53 | SnapshotMarker snapshot = (SnapshotMarker) o; 54 | 55 | if (startSeqno != snapshot.startSeqno) { 56 | return false; 57 | } 58 | return endSeqno == snapshot.endSeqno; 59 | } 60 | 61 | @Override 62 | public int hashCode() { 63 | int result = (int) (startSeqno ^ (startSeqno >>> 32)); 64 | result = 31 * result + (int) (endSeqno ^ (endSeqno >>> 32)); 65 | return result; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/BucketMismatchException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch; 18 | 19 | import java.io.IOException; 20 | 21 | public class BucketMismatchException extends IOException { 22 | public BucketMismatchException(String message) { 23 | super(message); 24 | } 25 | 26 | public BucketMismatchException() { 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/ElasticsearchBulkRequestBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch; 18 | 19 | import co.elastic.clients.elasticsearch.core.BulkRequest; 20 | import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; 21 | import com.couchbase.connector.elasticsearch.sink.DeleteOperation; 22 | import com.couchbase.connector.elasticsearch.sink.IndexOperation; 23 | import com.couchbase.connector.elasticsearch.sink.SinkBulkRequestBuilder; 24 | 25 | import java.util.ArrayList; 26 | import java.util.List; 27 | 28 | public class ElasticsearchBulkRequestBuilder implements SinkBulkRequestBuilder { 29 | private final List operations = new ArrayList<>(); 30 | 31 | public BulkRequest build(String timeout) { 32 | return new BulkRequest.Builder() 33 | .operations(operations) 34 | .timeout(time -> time.time(timeout)) 35 | .build(); 36 | } 37 | 38 | @Override 39 | public void add(DeleteOperation operation) { 40 | operations.add( 41 | new BulkOperation.Builder() 42 | .delete(op -> op 43 | .index(operation.getIndex()) 44 | .id(operation.getEvent().getKey()) 45 | ) 46 | .build() 47 | ); 48 | } 49 | 50 | @Override 51 | public void add(IndexOperation operation) { 52 | operations.add( 53 | new BulkOperation.Builder() 54 | .index(op -> op 55 | .index(operation.getIndex()) 56 | .id(operation.getEvent().getKey()) 57 | .document(operation.document()) 58 | .pipeline(operation.pipeline()) 59 | .routing(operation.routing()) 60 | ) 61 | .build() 62 | ); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/ErrorListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch; 18 | 19 | import com.couchbase.connector.dcp.Event; 20 | import com.couchbase.connector.elasticsearch.sink.SinkBulkResponseItem; 21 | 22 | public interface ErrorListener { 23 | 24 | void onFailureToCreateIndexRequest(Event event, Throwable error); 25 | 26 | void onError(Event event, Throwable error); 27 | 28 | void onFailedIndexResponse(Event event, SinkBulkResponseItem response); 29 | 30 | 31 | ErrorListener NOOP = new ErrorListener() { 32 | @Override 33 | public void onFailureToCreateIndexRequest(Event event, Throwable error) { 34 | 35 | } 36 | 37 | @Override 38 | public void onFailedIndexResponse(Event event, SinkBulkResponseItem response) { 39 | 40 | } 41 | 42 | @Override 43 | public void onError(Event event, Throwable error) { 44 | 45 | } 46 | }; 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/OpenSearchBulkRequestBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch; 18 | 19 | import com.couchbase.connector.elasticsearch.sink.DeleteOperation; 20 | import com.couchbase.connector.elasticsearch.sink.IndexOperation; 21 | import com.couchbase.connector.elasticsearch.sink.SinkBulkRequestBuilder; 22 | import org.opensearch.client.opensearch.core.BulkRequest; 23 | import org.opensearch.client.opensearch.core.bulk.BulkOperation; 24 | 25 | import java.util.ArrayList; 26 | import java.util.List; 27 | 28 | public class OpenSearchBulkRequestBuilder implements SinkBulkRequestBuilder { 29 | private final List operations = new ArrayList<>(); 30 | 31 | public BulkRequest build(String timeout) { 32 | return new BulkRequest.Builder() 33 | .operations(operations) 34 | .timeout(time -> time.time(timeout)) 35 | .build(); 36 | } 37 | 38 | @Override 39 | public void add(DeleteOperation operation) { 40 | operations.add( 41 | new BulkOperation.Builder() 42 | .delete(op -> op 43 | .index(operation.getIndex()) 44 | .id(operation.getEvent().getKey()) 45 | ) 46 | .build() 47 | ); 48 | } 49 | 50 | @Override 51 | public void add(IndexOperation operation) { 52 | operations.add( 53 | new BulkOperation.Builder() 54 | .index(op -> op 55 | .index(operation.getIndex()) 56 | .id(operation.getEvent().getKey()) 57 | .document(operation.document()) 58 | .pipeline(operation.pipeline()) 59 | .routing(operation.routing()) 60 | ) 61 | .build() 62 | ); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/cli/AbstractCliCommand.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch.cli; 18 | 19 | import joptsimple.OptionException; 20 | import joptsimple.OptionParser; 21 | import joptsimple.OptionSet; 22 | import joptsimple.OptionSpec; 23 | 24 | import java.io.File; 25 | import java.io.IOException; 26 | 27 | import static com.couchbase.connector.VersionHelper.getVersionString; 28 | import static com.couchbase.connector.util.EnvironmentHelper.getInstallDir; 29 | 30 | public abstract class AbstractCliCommand { 31 | 32 | // Init Log4J. This must happen first before logger is declared. 33 | static { 34 | // Expect the launch script generated by Gradle to set this environment variable. 35 | final String appHome = System.getenv("APP_HOME"); 36 | if (appHome == null) { 37 | // Use the default log4j2.xml file in the class path. 38 | System.err.println("WARNING: Environment variable 'APP_HOME' not set (launch script is responsible for this). " + 39 | "Using embedded logging config."); 40 | } else { 41 | System.setProperty("log4j.configurationFile", appHome + "/config/log4j2.xml"); 42 | } 43 | } 44 | 45 | public static class CommonParser { 46 | protected final OptionParser parser = new OptionParser(); 47 | 48 | public final OptionSpec configFile = parser.accepts("config", "Optionally specify a config file other than the default. See the 'config' directory for examples.") 49 | .withRequiredArg().ofType(File.class).describedAs("path to config file") 50 | .defaultsTo(defaultConfigFile()); 51 | 52 | public OptionSet parse(String... args) throws IOException { 53 | 54 | try { 55 | parser.accepts("version", "Display connector version information."); 56 | handleVersionAndHelp(parser, args); 57 | final OptionSet options = parser.parse(args); 58 | 59 | final File configFile = this.configFile.value(options); 60 | if (!configFile.exists()) { 61 | System.err.println("ERROR: Config file not found at " + configFile.getAbsoluteFile()); 62 | System.err.println(" Please create this file, or use the --config option to specify a different file."); 63 | System.err.println(" Examples may be found in the 'config' directory."); 64 | System.exit(1); 65 | } 66 | 67 | return options; 68 | 69 | } catch (OptionException e) { 70 | System.err.println(e.getMessage()); 71 | System.err.println(); 72 | parser.printHelpOn(System.err); 73 | System.exit(1); 74 | throw new AssertionError("unreachable"); 75 | } 76 | 77 | } 78 | 79 | void handleVersionAndHelp(OptionParser parser, String[] args) throws IOException { 80 | // parse version separately, since the "real" parser might have required arguments 81 | final OptionParser versionParser = new OptionParser(); 82 | versionParser.allowsUnrecognizedOptions(); 83 | versionParser.accepts("version"); 84 | versionParser.accepts("help"); 85 | if (versionParser.parse(args).has("version")) { 86 | System.out.println(getVersionString()); 87 | System.exit(0); 88 | } 89 | 90 | if (versionParser.parse(args).has("help")) { 91 | parser.printHelpOn(System.out); 92 | System.exit(0); 93 | } 94 | } 95 | } 96 | 97 | static File defaultConfigFile() { 98 | return new File(getInstallDir(), "config/default-connector.toml"); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/io/BackoffPolicy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch.io; 18 | 19 | import java.time.Duration; 20 | 21 | public interface BackoffPolicy extends Iterable { 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/io/BackoffPolicyBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch.io; 18 | 19 | import com.google.common.collect.Iterables; 20 | 21 | import java.time.Duration; 22 | import java.util.Iterator; 23 | 24 | import static java.util.Objects.requireNonNull; 25 | 26 | public class BackoffPolicyBuilder { 27 | private BackoffPolicy policy; 28 | 29 | public static BackoffPolicyBuilder truncatedExponentialBackoff(Duration seed, Duration cap) { 30 | return new BackoffPolicyBuilder(MoreBackoffPolicies.truncatedExponentialBackoff(seed, cap)); 31 | } 32 | 33 | public static BackoffPolicyBuilder constantBackoff(Duration delay) { 34 | BackoffPolicy policy = new BackoffPolicy() { 35 | public String toString() { 36 | return "constantBackoff(delay: " + delay + ")"; 37 | } 38 | 39 | @Override 40 | public Iterator iterator() { 41 | return Iterables.cycle(delay).iterator(); 42 | } 43 | }; 44 | 45 | return new BackoffPolicyBuilder(policy); 46 | } 47 | 48 | public BackoffPolicyBuilder(BackoffPolicy policy) { 49 | this.policy = requireNonNull(policy); 50 | } 51 | 52 | public BackoffPolicyBuilder fullJitter() { 53 | policy = MoreBackoffPolicies.withFullJitter(policy); 54 | return this; 55 | } 56 | 57 | public BackoffPolicyBuilder timeout(Duration timeout) { 58 | policy = MoreBackoffPolicies.withTimeout(timeout, policy); 59 | return this; 60 | } 61 | 62 | public BackoffPolicyBuilder limit(int maxRetries) { 63 | policy = MoreBackoffPolicies.limit(maxRetries, policy); 64 | return this; 65 | } 66 | 67 | public BackoffPolicy build() { 68 | return policy; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/io/DocumentTransformer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch.io; 18 | 19 | import com.couchbase.connector.dcp.Event; 20 | import org.jspecify.annotations.Nullable; 21 | 22 | public interface DocumentTransformer { 23 | /** 24 | * Returns the `document` to use for an index request if the 25 | * given event is eligible for replication to Elasticsearch, 26 | * otherwise null. 27 | */ 28 | @Nullable 29 | Object getElasticsearchDocument(Event mutationEvent); 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/io/PreserializedJson.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch.io; 18 | 19 | import com.fasterxml.jackson.core.JsonGenerator; 20 | import com.fasterxml.jackson.databind.SerializerProvider; 21 | import com.fasterxml.jackson.databind.annotation.JsonSerialize; 22 | import com.fasterxml.jackson.databind.ser.std.StdSerializer; 23 | 24 | import java.io.IOException; 25 | import java.nio.charset.StandardCharsets; 26 | 27 | import static java.util.Objects.requireNonNull; 28 | 29 | @JsonSerialize(using = PreserializedJson.Serializer.class) 30 | public class PreserializedJson { 31 | private final String value; 32 | 33 | public PreserializedJson(String value) { 34 | this.value = requireNonNull(value); 35 | } 36 | 37 | public PreserializedJson(byte[] value) { 38 | this(new String(value, StandardCharsets.UTF_8)); 39 | } 40 | 41 | public static class Serializer extends StdSerializer { 42 | public Serializer() { 43 | super(PreserializedJson.class); 44 | } 45 | 46 | @Override 47 | public void serialize(PreserializedJson value, JsonGenerator gen, SerializerProvider provider) throws IOException { 48 | gen.writeRaw(value.value); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/sink/BaseOperation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch.sink; 18 | 19 | import com.couchbase.connector.dcp.Event; 20 | 21 | import static java.util.Objects.requireNonNull; 22 | 23 | public abstract class BaseOperation implements Operation { 24 | private final String index; 25 | private final Event event; 26 | 27 | public BaseOperation(String index, Event event) { 28 | this.index = requireNonNull(index); 29 | this.event = requireNonNull(event); 30 | } 31 | 32 | @Override 33 | public Event getEvent() { 34 | return event; 35 | } 36 | 37 | public String getIndex() { 38 | return index; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/sink/DeleteOperation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch.sink; 18 | 19 | import com.couchbase.connector.dcp.Event; 20 | 21 | public class DeleteOperation extends BaseOperation { 22 | public DeleteOperation(String index, Event event) { 23 | super(index, event); 24 | } 25 | 26 | @Override 27 | public int estimatedSizeInBytes() { 28 | return REQUEST_OVERHEAD; 29 | } 30 | 31 | @Override 32 | public void addTo(SinkBulkRequestBuilder bulkRequestBuilder) { 33 | bulkRequestBuilder.add(this); 34 | } 35 | 36 | @Override 37 | public Type type() { 38 | return Type.DELETE; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/sink/IndexOperation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch.sink; 18 | 19 | import com.couchbase.connector.dcp.Event; 20 | import org.jspecify.annotations.Nullable; 21 | 22 | import static java.util.Objects.requireNonNull; 23 | 24 | public class IndexOperation extends BaseOperation { 25 | private final @Nullable String pipeline; 26 | private final @Nullable String routing; 27 | private final Object document; 28 | 29 | public IndexOperation(String index, Event event, Object document, @Nullable String pipeline, @Nullable String routing) { 30 | super(index, event); 31 | this.pipeline = pipeline; 32 | this.routing = routing; 33 | this.document = requireNonNull(document); 34 | } 35 | 36 | @Override 37 | public int estimatedSizeInBytes() { 38 | return REQUEST_OVERHEAD + getEvent().getContent().length; 39 | } 40 | 41 | @Override 42 | public void addTo(SinkBulkRequestBuilder bulkRequestBuilder) { 43 | bulkRequestBuilder.add(this); 44 | } 45 | 46 | @Override 47 | public Type type() { 48 | return Type.INDEX; 49 | } 50 | 51 | public @Nullable String pipeline() { 52 | return pipeline; 53 | } 54 | 55 | public @Nullable String routing() { 56 | return routing; 57 | } 58 | 59 | public Object document() { 60 | return document; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/sink/Operation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch.sink; 18 | 19 | import com.couchbase.connector.dcp.Event; 20 | 21 | /** 22 | * An Elasticsearch operation, with an attached DCP event. 23 | */ 24 | public interface Operation { 25 | 26 | enum Type { 27 | INDEX, DELETE 28 | } 29 | 30 | /** 31 | * Estimate of the overhead associated with each item in a bulk request. 32 | */ 33 | int REQUEST_OVERHEAD = 50; 34 | 35 | String getIndex(); 36 | 37 | Event getEvent(); 38 | 39 | /** 40 | * Returns {@link #REQUEST_OVERHEAD} plus the size of the request content. 41 | */ 42 | default int estimatedSizeInBytes() { 43 | return REQUEST_OVERHEAD; 44 | } 45 | 46 | void addTo(SinkBulkRequestBuilder bulkRequestBuilder); 47 | 48 | Type type(); 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/sink/RejectOperation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch.sink; 18 | 19 | import com.couchbase.connector.dcp.Event; 20 | import com.google.common.collect.ImmutableMap; 21 | 22 | import static java.util.Objects.requireNonNull; 23 | 24 | public class RejectOperation extends IndexOperation { 25 | 26 | public RejectOperation(String index, Operation origRequest, SinkBulkResponseItem failure) { 27 | this(index, 28 | origRequest.getEvent(), 29 | origRequest.getIndex(), 30 | origRequest.type(), 31 | requireNonNull(failure.error(), "bulk response item did not fail").toString()); 32 | } 33 | 34 | public RejectOperation(String index, Event event, String eventIndex, Operation.Type action, String failureMessage) { 35 | super(index, 36 | event, 37 | ImmutableMap.of( 38 | "index", eventIndex, 39 | //"type", eventType, 40 | "action", action, 41 | "error", failureMessage), 42 | null, 43 | null 44 | ); 45 | } 46 | 47 | @Override 48 | public int estimatedSizeInBytes() { 49 | int wildGuess = 64; 50 | return Operation.REQUEST_OVERHEAD + wildGuess; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/sink/RetryReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch.sink; 18 | 19 | import com.couchbase.connector.dcp.Event; 20 | import com.google.common.collect.ArrayListMultimap; 21 | import com.google.common.collect.ListMultimap; 22 | import org.slf4j.Logger; 23 | 24 | import java.util.List; 25 | 26 | import static com.couchbase.client.core.logging.RedactableArgument.redactUser; 27 | import static java.util.Objects.requireNonNull; 28 | 29 | /** 30 | * Aggregates and summarizes failures we intend to retry. 31 | */ 32 | class RetryReporter { 33 | private final ListMultimap errorMessageToEvents = ArrayListMultimap.create(); 34 | private final Logger logger; 35 | 36 | private RetryReporter(Logger logger) { 37 | this.logger = requireNonNull(logger); 38 | } 39 | 40 | static RetryReporter forLogger(Logger logger) { 41 | return new RetryReporter(logger); 42 | } 43 | 44 | void add(Event e, SinkBulkResponseItem failure) { 45 | if (!logger.isInfoEnabled()) { 46 | return; 47 | } 48 | SinkErrorCause cause = failure.error(); 49 | if (cause == null) { 50 | throw new IllegalArgumentException("bulk response item did not fail"); 51 | } 52 | 53 | final String message = "status=" + failure.status() + " message=" + cause.reason(); 54 | errorMessageToEvents.put(message, redactUser(e).toString()); 55 | } 56 | 57 | void report() { 58 | if (!logger.isInfoEnabled()) { 59 | return; 60 | } 61 | 62 | for (String errorMessage : errorMessageToEvents.keySet()) { 63 | final List events = errorMessageToEvents.get(errorMessage); 64 | String message = "Retrying " + events.get(0); 65 | if (events.size() > 1) { 66 | message += " (and " + (events.size() - 1) + " others)"; 67 | } 68 | message += " due to: " + errorMessage; 69 | 70 | logger.info(message); 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/sink/SinkBulkRequestBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch.sink; 18 | 19 | public interface SinkBulkRequestBuilder { 20 | void add(DeleteOperation operation); 21 | 22 | void add(IndexOperation operation); 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/sink/SinkBulkResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch.sink; 18 | 19 | import java.time.Duration; 20 | import java.util.List; 21 | 22 | public interface SinkBulkResponse { 23 | Duration ingestTook(); 24 | 25 | List items(); 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/sink/SinkBulkResponseItem.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch.sink; 18 | 19 | import org.jspecify.annotations.Nullable; 20 | 21 | public interface SinkBulkResponseItem { 22 | int status(); 23 | 24 | @Nullable 25 | SinkErrorCause error(); 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/sink/SinkErrorCause.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch.sink; 18 | 19 | import org.jspecify.annotations.Nullable; 20 | 21 | public interface SinkErrorCause { 22 | @Nullable 23 | String reason(); 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/elasticsearch/sink/SinkTestOps.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.elasticsearch.sink; 18 | 19 | import com.fasterxml.jackson.databind.JsonNode; 20 | import org.jspecify.annotations.Nullable; 21 | 22 | import java.io.Closeable; 23 | import java.util.Collection; 24 | import java.util.List; 25 | import java.util.Optional; 26 | 27 | import static java.util.Objects.requireNonNull; 28 | 29 | public interface SinkTestOps extends Closeable { 30 | Optional getDocument(String index, String id, @Nullable String routing); 31 | 32 | long countDocuments(String index); 33 | 34 | List multiGet(String index, Collection ids); 35 | 36 | class MultiGetItem { 37 | public String id; 38 | public @Nullable String error; 39 | public @Nullable JsonNode document; 40 | 41 | public MultiGetItem(String id, @Nullable String error, @Nullable JsonNode document) { 42 | this.id = requireNonNull(id); 43 | this.error = error; 44 | this.document = document; 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/util/EnvironmentHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.util; 18 | 19 | import java.io.File; 20 | 21 | public class EnvironmentHelper { 22 | private static final String appHome = System.getenv("APP_HOME"); 23 | 24 | // 25 | // MUST NOT DECLARE A LOGGER! -- Otherwise Log4J won't get read from the correct location 26 | // ($APP_HOME/config/log4j2.xml). See AbstractCliCommand's static init block. 27 | // 28 | 29 | static { 30 | // Expect the launch script generated by Gradle to set this environment variable. 31 | if (appHome == null) { 32 | System.err.println("WARNING: Environment variable 'APP_HOME' not set (launch script is responsible for this). " + 33 | "Assuming development environment (will resolve paths relative to src/dist)"); 34 | } 35 | } 36 | 37 | private EnvironmentHelper() { 38 | throw new AssertionError("not instantiable"); 39 | } 40 | 41 | public static File getInstallDir() { 42 | return appHome == null ? new File("src/dist") : new File(appHome); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/util/ListHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.util; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | import static com.google.common.base.Preconditions.checkArgument; 23 | import static java.util.Objects.requireNonNull; 24 | 25 | public class ListHelper { 26 | private ListHelper() { 27 | throw new AssertionError("not instantiable"); 28 | } 29 | 30 | /** 31 | * Splits the given list into the requested number of chunks. 32 | * The smallest and largest chunks are guaranteed to differ in size by no more than 1. 33 | * If the requested number of chunks is greater than the number of items, 34 | * some chunks will be empty. 35 | */ 36 | public static List> chunks(List items, int chunks) { 37 | checkArgument(chunks > 0, "chunks must be > 0"); 38 | requireNonNull(items); 39 | 40 | final int maxChunkSize = ((items.size() - 1) / chunks) + 1; // size / chunks, rounded up 41 | final int numFullChunks = chunks - (maxChunkSize * chunks - items.size()); 42 | 43 | final List> result = new ArrayList<>(chunks); 44 | 45 | int startIndex = 0; 46 | for (int i = 0; i < chunks; i++) { 47 | int endIndex = startIndex + maxChunkSize; 48 | if (i >= numFullChunks) { 49 | endIndex--; 50 | } 51 | result.add(items.subList(startIndex, endIndex)); 52 | startIndex = endIndex; 53 | } 54 | return result; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/util/RuntimeHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.util; 18 | 19 | import java.time.Duration; 20 | import java.time.Instant; 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import static java.util.Objects.requireNonNull; 25 | import static java.util.concurrent.TimeUnit.NANOSECONDS; 26 | 27 | /** 28 | * Runs shutdown hooks with a timeout. 29 | *

30 | * Prevents stuck hooks from delaying or preventing JVM termination. 31 | */ 32 | public class RuntimeHelper { 33 | private RuntimeHelper() { 34 | throw new AssertionError("not instantiable"); 35 | } 36 | 37 | private static final Duration shutdownHookTimeout = Duration.ofSeconds(3); 38 | 39 | private static final List managedShutdownHooks = new ArrayList<>(); 40 | 41 | static { 42 | // Register an "umbrella" hook that starts the other hooks 43 | // and halts the JVM if they take too long. 44 | Runtime.getRuntime().addShutdownHook(new Thread(() -> { 45 | try { 46 | synchronized (managedShutdownHooks) { 47 | for (Thread t : managedShutdownHooks) { 48 | t.start(); 49 | } 50 | 51 | final long deadlineNanos = System.nanoTime() + shutdownHookTimeout.toNanos(); 52 | 53 | for (Thread t : managedShutdownHooks) { 54 | // prevent joining for 0 milliseconds, which would mean "wait forever." 55 | final long remainingMillis = Math.max(1, NANOSECONDS.toMillis(deadlineNanos - System.nanoTime())); 56 | t.join(remainingMillis); 57 | if (t.isAlive()) { 58 | // logging has likely been shut down by this point, so use stderr 59 | System.err.println(Instant.now() + " Shutdown hook failed to terminate within " + shutdownHookTimeout + " : " + t); 60 | halt(); 61 | } 62 | } 63 | } 64 | } catch (Throwable t) { 65 | // logging has likely been shut down by this point, so use stderr 66 | t.printStackTrace(); 67 | halt(); 68 | } 69 | })); 70 | } 71 | 72 | /** 73 | * Immediately terminate the JVM process, without running any shutdown hooks or finalizers. 74 | */ 75 | private static void halt() { 76 | // logging has likely been shut down by this point, so use stderr 77 | System.err.println("Halting."); 78 | Runtime.getRuntime().halt(-1); 79 | } 80 | 81 | public static void addShutdownHook(Thread hook) { 82 | requireNonNull(hook); 83 | synchronized (managedShutdownHooks) { 84 | managedShutdownHooks.add(hook); 85 | } 86 | } 87 | 88 | public static boolean removeShutdownHook(Thread hook) { 89 | requireNonNull(hook); 90 | synchronized (managedShutdownHooks) { 91 | return managedShutdownHooks.remove(hook); 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/connector/util/ThrowableHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.util; 18 | 19 | import com.google.common.base.Throwables; 20 | 21 | import java.util.Optional; 22 | 23 | public class ThrowableHelper { 24 | /** 25 | * If the given Throwable's causal chain includes an instance of the given type, 26 | * throw the matching instance. 27 | */ 28 | public static void propagateCauseIfPossible(Throwable t, Class type) throws T { 29 | propagateIfPresent(findCause(t, type)); 30 | } 31 | 32 | /** 33 | * Returns true if the given Throwable's causal chain includes an instance of any of the given types. 34 | */ 35 | @SafeVarargs 36 | public static boolean hasCause(Throwable t, Class type, Class... otherTypes) { 37 | if (findCause(t, type).isPresent()) { 38 | return true; 39 | } 40 | for (Class other : otherTypes) { 41 | if (findCause(t, other).isPresent()) { 42 | return true; 43 | } 44 | } 45 | return false; 46 | } 47 | 48 | private static void propagateIfPresent(Optional t) throws T { 49 | if (t.isPresent()) { 50 | throw t.get(); 51 | } 52 | } 53 | 54 | private static Optional findCause(Throwable t, Class type) { 55 | for (Throwable cause : Throwables.getCausalChain(t)) { 56 | if (type.isAssignableFrom(cause.getClass())) { 57 | return Optional.of(type.cast(cause)); 58 | } 59 | } 60 | return Optional.empty(); 61 | } 62 | 63 | public static String formatMessageWithStackTrace(String message, Throwable t) { 64 | return message + System.lineSeparator() + Throwables.getStackTraceAsString(t); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/consul/KvReadResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.consul; 18 | 19 | import com.fasterxml.jackson.annotation.JsonCreator; 20 | import com.fasterxml.jackson.annotation.JsonProperty; 21 | 22 | import java.util.Arrays; 23 | import java.util.Objects; 24 | import java.util.Optional; 25 | 26 | import static java.nio.charset.StandardCharsets.UTF_8; 27 | 28 | public class KvReadResult { 29 | private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; 30 | 31 | private final String key; 32 | private final long flags; 33 | private final byte[] value; 34 | private final String session; 35 | private final long createIndex; 36 | private final long modifyIndex; 37 | private final long lockIndex; 38 | 39 | @JsonCreator 40 | public KvReadResult( 41 | @JsonProperty("Key") String key, 42 | @JsonProperty("Flags") long flags, 43 | @JsonProperty("Value") byte[] value, 44 | @JsonProperty("Session") String session, 45 | @JsonProperty("CreateIndex") long createIndex, 46 | @JsonProperty("ModifyIndex") long modifyIndex, 47 | @JsonProperty("LockIndex") long lockIndex 48 | ) { 49 | this.key = key; 50 | this.flags = flags; 51 | this.value = value == null ? EMPTY_BYTE_ARRAY : value; 52 | this.session = session; 53 | this.createIndex = createIndex; 54 | this.modifyIndex = modifyIndex; 55 | this.lockIndex = lockIndex; 56 | } 57 | 58 | public String key() { 59 | return key; 60 | } 61 | 62 | public long flags() { 63 | return flags; 64 | } 65 | 66 | public byte[] value() { 67 | return value; 68 | } 69 | 70 | public String valueAsString() { 71 | return new String(value, UTF_8); 72 | } 73 | 74 | public Optional session() { 75 | return Optional.ofNullable(session); 76 | } 77 | 78 | public long createIndex() { 79 | return createIndex; 80 | } 81 | 82 | public long modifyIndex() { 83 | return modifyIndex; 84 | } 85 | 86 | public long lockIndex() { 87 | return lockIndex; 88 | } 89 | 90 | @Override 91 | public String toString() { 92 | return "KvReadResult{" + 93 | "key='" + key + '\'' + 94 | ", flags=" + flags + 95 | ", session='" + session + '\'' + 96 | ", createIndex=" + createIndex + 97 | ", modifyIndex=" + modifyIndex + 98 | ", lockIndex=" + lockIndex + 99 | ", value=" + valueAsString() + 100 | '}'; 101 | } 102 | 103 | @Override 104 | public boolean equals(Object o) { 105 | if (this == o) { 106 | return true; 107 | } 108 | if (o == null || getClass() != o.getClass()) { 109 | return false; 110 | } 111 | KvReadResult that = (KvReadResult) o; 112 | return flags == that.flags && createIndex == that.createIndex && modifyIndex == that.modifyIndex && lockIndex == that.lockIndex && key.equals(that.key) && Arrays.equals(value, that.value) && Objects.equals(session, that.session); 113 | } 114 | 115 | @Override 116 | public int hashCode() { 117 | return Objects.hash(key, modifyIndex, lockIndex); 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/consul/ReadTimeoutSetter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.consul; 18 | 19 | import com.couchbase.client.core.util.Golang; 20 | import com.couchbase.consul.internal.OkHttpHelper; 21 | import com.google.common.annotations.VisibleForTesting; 22 | import okhttp3.Interceptor; 23 | import okhttp3.Request; 24 | import okhttp3.Response; 25 | import org.jspecify.annotations.NonNull; 26 | 27 | import java.io.IOException; 28 | import java.time.Duration; 29 | import java.util.Optional; 30 | import java.util.concurrent.TimeUnit; 31 | 32 | import static com.couchbase.client.core.util.CbStrings.emptyToNull; 33 | 34 | /** 35 | * Sets the read timeout to match the "wait" query parameter of 36 | * Consul blocking requests. 37 | */ 38 | public class ReadTimeoutSetter implements Interceptor { 39 | private static final Duration MAX_WAIT = Duration.ofMinutes(10); 40 | 41 | /** 42 | * Maximum amount of time any Consul request is expected to take. 43 | *

44 | * (There's no easy way to set the call timeout on a per-request basis, unfortunately; 45 | * see Retrofit issue 3434) 46 | */ 47 | public static final Duration CALL_TIMEOUT = addConsulJitter(MAX_WAIT) 48 | .plus(Duration.ofSeconds(10)); // *extra* grace period for response processing, etc. 49 | 50 | @Override 51 | public @NonNull Response intercept(@NonNull Chain chain) throws IOException { 52 | Request request = chain.request(); 53 | boolean hasIndex = request.url().queryParameterNames().contains("index"); 54 | 55 | if (!hasIndex) { 56 | // It's not a blocking request, so honor the client's configured timeout. 57 | return chain.proceed(request); 58 | } 59 | 60 | Duration wait = getWait(request).orElse(null); 61 | if (wait == null) { 62 | // Make the default wait time explicit, in case it changes in a future Consul version. 63 | request = OkHttpHelper.withQueryParam(request, "wait", "5m"); 64 | wait = getWait(request).orElseThrow(); 65 | } 66 | 67 | if (wait.compareTo(MAX_WAIT) > 0) { 68 | throw new IllegalArgumentException("Specified wait " + wait + " is greater than max wait " + MAX_WAIT); 69 | } 70 | 71 | wait = addConsulJitter(wait); 72 | 73 | return chain 74 | .withReadTimeout((int) wait.toMillis(), TimeUnit.MILLISECONDS) 75 | .proceed(request); 76 | } 77 | 78 | /** 79 | * Returns a Duration slightly longer than the given duration. 80 | * Accounts for the 6.25% (1/16) jitter added by Consul, plus some grace time. 81 | *

82 | * See Consul Blocking Queries. 83 | */ 84 | @VisibleForTesting 85 | static Duration addConsulJitter(Duration wait) { 86 | int gracePeriodSeconds = 10; 87 | int waitSecondsPlusJitter = Math.toIntExact((long) Math.ceil(wait.toSeconds() * 1.0625)); 88 | return Duration.ofSeconds(gracePeriodSeconds + waitSecondsPlusJitter); 89 | } 90 | 91 | @VisibleForTesting 92 | static Optional getWait(Request r) { 93 | String wait = r.url().queryParameter("wait"); 94 | return Optional.ofNullable(emptyToNull(wait)) 95 | .map(ReadTimeoutSetter::parseWait); 96 | } 97 | 98 | private static Duration parseWait(String s) { 99 | // Consul's duration format is a subset of Go's duration format. 100 | return Golang.parseDuration(s); 101 | } 102 | 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/consul/internal/ConsulJacksonHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.consul.internal; 18 | 19 | import com.fasterxml.jackson.databind.DeserializationFeature; 20 | import com.fasterxml.jackson.databind.PropertyNamingStrategies; 21 | import com.fasterxml.jackson.databind.json.JsonMapper; 22 | import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; 23 | import com.fasterxml.jackson.module.paramnames.ParameterNamesModule; 24 | 25 | public class ConsulJacksonHelper { 26 | private ConsulJacksonHelper() { 27 | throw new AssertionError("not instantiable"); 28 | } 29 | 30 | private static final JsonMapper mapper = new JsonMapper(); 31 | 32 | static { 33 | mapper.registerModule(new ParameterNamesModule()); 34 | mapper.registerModule(new Jdk8Module()); 35 | mapper.setPropertyNamingStrategy(PropertyNamingStrategies.UPPER_CAMEL_CASE); 36 | mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); 37 | } 38 | 39 | public static JsonMapper consulJsonMapper() { 40 | return mapper; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/couchbase/consul/internal/OkHttpHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.consul.internal; 18 | 19 | import okhttp3.Call; 20 | import okhttp3.Callback; 21 | import okhttp3.OkHttpClient; 22 | import okhttp3.Request; 23 | import okhttp3.Response; 24 | import org.jspecify.annotations.NonNull; 25 | import reactor.core.publisher.Mono; 26 | 27 | import java.io.IOException; 28 | import java.util.function.Function; 29 | 30 | public class OkHttpHelper { 31 | private OkHttpHelper() { 32 | throw new AssertionError("not instantiable"); 33 | } 34 | 35 | public static Mono toMono(OkHttpClient client, Request request, Function responseTransformer) { 36 | return Mono.create(sink -> { 37 | Call call = client.newCall(request); 38 | 39 | sink.onCancel(call::cancel); 40 | 41 | call.enqueue(new Callback() { 42 | @Override 43 | public void onFailure(@NonNull Call call, @NonNull IOException e) { 44 | sink.error(e); 45 | } 46 | 47 | @Override 48 | public void onResponse(@NonNull Call call, okhttp3.@NonNull Response response) throws IOException { 49 | try (response) { 50 | sink.success(responseTransformer.apply(response)); 51 | } catch (Throwable t) { 52 | sink.error(t); 53 | } 54 | } 55 | }); 56 | }); 57 | } 58 | 59 | public static Request withQueryParam(Request original, String paramName, String paramValue) { 60 | return original.newBuilder() 61 | .url( 62 | original.url().newBuilder() 63 | .setQueryParameter(paramName, paramValue) 64 | .build() 65 | ) 66 | .build(); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/resources/com/couchbase/connector/version.properties: -------------------------------------------------------------------------------- 1 | version=@application.version@ 2 | git=@git.info@ 3 | -------------------------------------------------------------------------------- /src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/test/java/com/couchbase/connector/config/ConfigHelperTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.config; 18 | 19 | import org.junit.Test; 20 | 21 | import java.time.Duration; 22 | import java.time.temporal.ChronoUnit; 23 | 24 | import static com.couchbase.connector.config.ConfigHelper.createHttpHost; 25 | import static com.couchbase.connector.config.ConfigHelper.parseTime; 26 | import static org.junit.Assert.assertEquals; 27 | 28 | public class ConfigHelperTest { 29 | 30 | @Test 31 | public void canCreateHttpHost() throws Exception { 32 | assertEquals("http://localhost:9200", createHttpHost("localhost", 9200, false).toURI()); 33 | assertEquals("http://localhost:9201", createHttpHost("localhost:9201", 9200, false).toURI()); 34 | assertEquals("https://localhost:9200", createHttpHost("localhost", 9200, true).toURI()); 35 | assertEquals("https://localhost:9201", createHttpHost("localhost:9201", 9200, true).toURI()); 36 | 37 | assertEquals("http://localhost:9200", createHttpHost("http://localhost", 9200, false).toURI()); 38 | assertEquals("http://localhost:9201", createHttpHost("http://localhost:9201", 9200, false).toURI()); 39 | assertEquals("https://localhost:9200", createHttpHost("https://localhost", 9200, true).toURI()); 40 | assertEquals("https://localhost:9201", createHttpHost("https://localhost:9201", 9200, true).toURI()); 41 | } 42 | 43 | @Test(expected = ConfigException.class) 44 | public void insecureHttpsDefaultPort() throws Exception { 45 | createHttpHost("https://localhost", 9200, false); 46 | } 47 | 48 | @Test(expected = ConfigException.class) 49 | public void insecureHttpsCustomPort() throws Exception { 50 | createHttpHost("https://localhost:1234", 9200, false); 51 | } 52 | 53 | @Test(expected = ConfigException.class) 54 | public void secureHttpCustomPort() throws Exception { 55 | createHttpHost("http://localhost:1234", 9200, true); 56 | } 57 | 58 | @Test(expected = ConfigException.class) 59 | public void secureHttpDefaultPort() throws Exception { 60 | createHttpHost("http://localhost", 9200, true); 61 | } 62 | 63 | @Test 64 | public void canParseElasticsearchDurations() { 65 | assertEquals(Duration.ofNanos(10), parseTime("10ns")); 66 | assertEquals(Duration.ofNanos(10), parseTime("10nanos")); 67 | assertEquals(Duration.ofNanos(10), parseTime(" 10 nanos ")); 68 | assertEquals(Duration.of(10, ChronoUnit.MICROS), parseTime(" 10 micros ")); 69 | 70 | assertEquals(Duration.ofNanos(10), parseTime("10NANOS")); 71 | assertEquals(Duration.ofHours(10), parseTime("10H")); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/test/java/com/couchbase/connector/dcp/CouchbaseHelperTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.connector.dcp; 18 | 19 | import com.couchbase.client.core.util.ConnectionString; 20 | import org.junit.Test; 21 | 22 | import java.lang.reflect.Method; 23 | 24 | import static com.couchbase.client.core.util.CbCollections.listOf; 25 | import static org.junit.Assert.assertEquals; 26 | 27 | public class CouchbaseHelperTest { 28 | @Test 29 | public void getAlias() throws Exception { 30 | Method fromString = ConnectionString.PortType.class.getDeclaredMethod("fromString", String.class); 31 | fromString.setAccessible(true); 32 | 33 | for (ConnectionString.PortType type : ConnectionString.PortType.values()) { 34 | if (type == ConnectionString.PortType.PROTOSTELLAR) { 35 | continue; 36 | } 37 | String alias = CouchbaseHelper.getAlias(type); 38 | ConnectionString.PortType roundTrip = (ConnectionString.PortType) fromString.invoke(null, alias); 39 | assertEquals(type, roundTrip); 40 | } 41 | } 42 | 43 | @Test 44 | public void qualifyPorts() throws Exception { 45 | assertEquals(listOf("foo:123=manager", "bar", "zot:123=kv", "moo:123=manager"), 46 | CouchbaseHelper.qualifyPorts( 47 | listOf("foo:123", "bar", "zot:123=kv", "moo:123=manager"), 48 | ConnectionString.PortType.MANAGER)); 49 | 50 | assertEquals(listOf("foo:123=kv", "bar", "zot:123=kv", "moo:123=manager"), 51 | CouchbaseHelper.qualifyPorts( 52 | listOf("foo:123", "bar", "zot:123=kv", "moo:123=manager"), 53 | ConnectionString.PortType.KV)); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/com/couchbase/connector/elasticsearch/io/DefaultDocumentTransformerTest.java: -------------------------------------------------------------------------------- 1 | package com.couchbase.connector.elasticsearch.io; 2 | 3 | import org.junit.Test; 4 | 5 | import java.math.BigInteger; 6 | 7 | import static java.math.BigInteger.ONE; 8 | import static java.nio.charset.StandardCharsets.UTF_8; 9 | import static org.junit.Assert.assertEquals; 10 | 11 | public class DefaultDocumentTransformerTest { 12 | private static final BigInteger MAX_UNSIGNED_LONG = new BigInteger("2").pow(64).subtract(ONE); 13 | private static final BigInteger MAX_SIGNED_LONG = new BigInteger("2").pow(63).subtract(ONE); 14 | 15 | @Test 16 | public void getCounterValue() { 17 | assertCounter(0L, "0"); 18 | assertCounter(1L, "1"); 19 | assertCounter(Long.MAX_VALUE, MAX_SIGNED_LONG); 20 | assertCounter(Long.MIN_VALUE, MAX_SIGNED_LONG.add(ONE)); 21 | assertCounter(-1L, MAX_UNSIGNED_LONG); 22 | } 23 | 24 | @Test 25 | public void getCounterValueOutsideRange() { 26 | assertCounter(null, MAX_UNSIGNED_LONG.add(ONE)); 27 | assertCounter(null, "-1"); 28 | } 29 | 30 | @Test 31 | public void getCounterValueNotIntegral() { 32 | assertCounter(null, "1.0"); 33 | assertCounter(null, "-1.0"); 34 | } 35 | 36 | @Test 37 | public void getCounterValueInvalidJson() { 38 | assertCounter(null, "1 true"); 39 | assertCounter(null, "xyz"); 40 | assertCounter(null, ""); 41 | assertCounter(null, " "); 42 | assertCounter(null, "{}"); 43 | assertCounter(null, "1 2"); 44 | assertCounter(null, "\"abc\""); 45 | assertCounter(null, "\"1\""); 46 | assertCounter(null, "\"1\""); 47 | assertCounter(null, "1a"); 48 | } 49 | 50 | private static void assertCounter(Long expected, String json) { 51 | assertEquals(expected, DefaultDocumentTransformer.getCounterValue(json.getBytes(UTF_8))); 52 | } 53 | 54 | private static void assertCounter(Long expected, BigInteger value) { 55 | assertCounter(expected, value.toString()); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/test/java/com/couchbase/connector/util/ListHelperTest.java: -------------------------------------------------------------------------------- 1 | package com.couchbase.connector.util; 2 | 3 | import com.couchbase.client.core.deps.com.fasterxml.jackson.core.type.TypeReference; 4 | import com.couchbase.client.dcp.core.utils.DefaultObjectMapper; 5 | import com.google.common.collect.ImmutableList; 6 | import org.junit.Test; 7 | 8 | import java.io.IOException; 9 | import java.util.List; 10 | import java.util.stream.IntStream; 11 | 12 | import static com.couchbase.connector.util.ListHelper.chunks; 13 | import static java.util.stream.Collectors.toList; 14 | import static org.junit.Assert.assertEquals; 15 | 16 | public class ListHelperTest { 17 | @Test(expected = IllegalArgumentException.class) 18 | public void chunkIntoZero() throws Exception { 19 | chunks(ImmutableList.of(1, 2, 3, 4, 5), 0); 20 | } 21 | 22 | @Test 23 | public void chunkIntoVarious() throws Exception { 24 | check(0, 1, "[[]]"); 25 | check(0, 2, "[[],[]]"); 26 | check(4, 2, "[[1,2],[3,4]]"); 27 | check(3, 1, "[[1,2,3]]"); 28 | check(3, 2, "[[1,2],[3]]"); 29 | check(3, 3, "[[1],[2],[3]]"); 30 | check(3, 5, "[[1],[2],[3],[],[]]"); 31 | check(5, 3, "[[1,2],[3,4],[5]]"); 32 | } 33 | 34 | private static void check(int listSize, int numChunks, String expectedJson) throws IOException { 35 | final List list = IntStream.range(1, listSize + 1).boxed().collect(toList()); 36 | List> expected = DefaultObjectMapper.readValue(expectedJson, new TypeReference>>() { 37 | }); 38 | assertEquals(expected, chunks(list, numChunks)); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/com/couchbase/consul/ReadTimeoutSetterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Couchbase, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.couchbase.consul; 18 | 19 | import com.couchbase.consul.internal.OkHttpHelper; 20 | import okhttp3.Request; 21 | import org.junit.Test; 22 | 23 | import java.time.Duration; 24 | import java.util.Optional; 25 | 26 | import static org.junit.Assert.assertEquals; 27 | 28 | public class ReadTimeoutSetterTest { 29 | 30 | @Test 31 | public void getWait() throws Exception { 32 | Request r = new Request.Builder() 33 | .url("http://127.0.0.1") 34 | .build(); 35 | assertEquals( 36 | Optional.empty(), 37 | ReadTimeoutSetter.getWait(r) 38 | ); 39 | 40 | r = OkHttpHelper.withQueryParam(r, "wait", ""); 41 | assertEquals( 42 | Optional.empty(), 43 | ReadTimeoutSetter.getWait(r) 44 | ); 45 | 46 | r = OkHttpHelper.withQueryParam(r, "wait", "7m"); 47 | assertEquals( 48 | Optional.of(Duration.ofMinutes(7)), 49 | ReadTimeoutSetter.getWait(r) 50 | ); 51 | 52 | r = OkHttpHelper.withQueryParam(r, "wait", "20s"); 53 | assertEquals( 54 | Optional.of(Duration.ofSeconds(20)), 55 | ReadTimeoutSetter.getWait(r) 56 | ); 57 | } 58 | 59 | @Test 60 | public void accountsForConsulJitter() throws Exception { 61 | Duration d = Duration.ofSeconds(100); 62 | assertEquals( 63 | Duration.ofSeconds(117), // original, + 6.25% (rounded up to 7) + 10 seconds grace period 64 | ReadTimeoutSetter.addConsulJitter(d) 65 | ); 66 | } 67 | } 68 | --------------------------------------------------------------------------------