├── src ├── main │ ├── resources │ │ └── es-plugin.properties │ ├── assemblies │ │ └── plugin.xml │ └── java │ │ └── org │ │ └── elasticsearch │ │ ├── river │ │ └── couchdb │ │ │ ├── CouchdbRiverModule.java │ │ │ └── CouchdbRiver.java │ │ └── plugin │ │ └── river │ │ └── couchdb │ │ └── CouchdbRiverPlugin.java └── test │ └── java │ └── org │ └── elasticsearch │ └── river │ └── couchdb │ ├── AbstractCouchdbTest.java │ ├── helper │ ├── HttpClientResponse.java │ ├── CouchDBClient.java │ └── HttpClient.java │ └── CouchdbRiverIntegrationTest.java ├── .gitignore ├── pom.xml ├── dev-tools └── release.py ├── CONTRIBUTING.md ├── LICENSE.txt └── README.md /src/main/resources/es-plugin.properties: -------------------------------------------------------------------------------- 1 | plugin=org.elasticsearch.plugin.river.couchdb.CouchdbRiverPlugin 2 | version=${project.version} 3 | 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /data 2 | /work 3 | /logs 4 | /.idea 5 | /target 6 | .DS_Store 7 | *.iml 8 | /.project 9 | /.settings 10 | /.classpath 11 | /plugin_tools 12 | /.local-execution-hints.log 13 | /.local-*-execution-hints.log 14 | -------------------------------------------------------------------------------- /src/main/assemblies/plugin.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | plugin 4 | 5 | zip 6 | 7 | false 8 | 9 | 10 | / 11 | true 12 | true 13 | 14 | org.elasticsearch:elasticsearch 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/river/couchdb/CouchdbRiverModule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.river.couchdb; 21 | 22 | import org.elasticsearch.common.inject.AbstractModule; 23 | import org.elasticsearch.river.River; 24 | 25 | /** 26 | * 27 | */ 28 | public class CouchdbRiverModule extends AbstractModule { 29 | 30 | @Override 31 | protected void configure() { 32 | bind(River.class).to(CouchdbRiver.class).asEagerSingleton(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/test/java/org/elasticsearch/river/couchdb/AbstractCouchdbTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.river.couchdb; 21 | 22 | import org.elasticsearch.test.ElasticsearchIntegrationTest; 23 | import org.elasticsearch.test.ElasticsearchIntegrationTest.ThirdParty; 24 | 25 | /** 26 | * Base class for tests that require CouchDB to run. CouchDB tests are disabled by default. 27 | *

28 | * To enable test add -Dtests.thirdparty=true 29 | *

30 | */ 31 | @ThirdParty 32 | public abstract class AbstractCouchdbTest extends ElasticsearchIntegrationTest { 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/plugin/river/couchdb/CouchdbRiverPlugin.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.plugin.river.couchdb; 21 | 22 | import org.elasticsearch.common.inject.Inject; 23 | import org.elasticsearch.plugins.AbstractPlugin; 24 | import org.elasticsearch.river.RiversModule; 25 | import org.elasticsearch.river.couchdb.CouchdbRiverModule; 26 | 27 | /** 28 | * 29 | */ 30 | public class CouchdbRiverPlugin extends AbstractPlugin { 31 | 32 | @Inject 33 | public CouchdbRiverPlugin() { 34 | } 35 | 36 | @Override 37 | public String name() { 38 | return "river-couchdb"; 39 | } 40 | 41 | @Override 42 | public String description() { 43 | return "River CouchDB Plugin"; 44 | } 45 | 46 | public void onModule(RiversModule module) { 47 | module.registerRiver("couchdb", CouchdbRiverModule.class); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/test/java/org/elasticsearch/river/couchdb/helper/HttpClientResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.river.couchdb.helper; 20 | 21 | import java.util.List; 22 | import java.util.Map; 23 | 24 | public class HttpClientResponse { 25 | private final String response; 26 | private final int errorCode; 27 | private Map> headers; 28 | private final Throwable e; 29 | 30 | public HttpClientResponse(String response, int errorCode, Map> headers, Throwable e) { 31 | this.response = response; 32 | this.errorCode = errorCode; 33 | this.headers = headers; 34 | this.e = e; 35 | } 36 | 37 | public String response() { 38 | return response; 39 | } 40 | 41 | public int errorCode() { 42 | return errorCode; 43 | } 44 | 45 | public Throwable cause() { 46 | return e; 47 | } 48 | 49 | public Map> getHeaders() { 50 | return headers; 51 | } 52 | 53 | public String getHeader(String name) { 54 | if (headers == null) { 55 | return null; 56 | } 57 | List vals = headers.get(name); 58 | if (vals == null || vals.size() == 0) { 59 | return null; 60 | } 61 | return vals.iterator().next(); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.elasticsearch 8 | elasticsearch-river-couchdb 9 | 3.0.0-SNAPSHOT 10 | jar 11 | Elasticsearch CouchDB River plugin 12 | The CouchDB River plugin allows to hook into couchdb _changes feed and automatically index it into elasticsearch. 13 | https://github.com/elastic/elasticsearch-river-couchdb/ 14 | 2009 15 | 16 | 17 | The Apache Software License, Version 2.0 18 | http://www.apache.org/licenses/LICENSE-2.0.txt 19 | repo 20 | 21 | 22 | 23 | scm:git:git@github.com:elastic/elasticsearch-river-couchdb.git 24 | scm:git:git@github.com:elastic/elasticsearch-river-couchdb.git 25 | http://github.com/elastic/elasticsearch-river-couchdb 26 | 27 | 28 | 29 | org.elasticsearch 30 | elasticsearch-plugin 31 | 2.0.0-SNAPSHOT 32 | 33 | 34 | 35 | 1 36 | 37 | warn 38 | 39 | 40 | 41 | 42 | org.codehaus.groovy 43 | groovy-all 44 | indy 45 | true 46 | test 47 | 48 | 49 | 50 | 51 | 52 | 53 | org.apache.maven.plugins 54 | maven-assembly-plugin 55 | 56 | 57 | 58 | 59 | 60 | 61 | oss-snapshots 62 | Sonatype OSS Snapshots 63 | https://oss.sonatype.org/content/repositories/snapshots/ 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /src/test/java/org/elasticsearch/river/couchdb/helper/CouchDBClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.river.couchdb.helper; 21 | 22 | import org.elasticsearch.common.base.Predicate; 23 | import org.elasticsearch.common.xcontent.XContentBuilder; 24 | import org.elasticsearch.test.ElasticsearchTestCase; 25 | import org.junit.Assert; 26 | 27 | import java.io.IOException; 28 | import java.nio.charset.StandardCharsets; 29 | import java.util.Iterator; 30 | import java.util.List; 31 | import java.util.concurrent.TimeUnit; 32 | 33 | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; 34 | import static org.hamcrest.CoreMatchers.is; 35 | 36 | public class CouchDBClient { 37 | public static final String host = "localhost"; 38 | public static final Integer port = 5984; 39 | public static final HttpClient httpClient = new HttpClient(host, port); 40 | 41 | public static void checkCouchDbRunning() { 42 | HttpClientResponse response = httpClient.request("/"); 43 | Assert.assertThat("Couchdb is not running on [" + host + "][" + port + "]", response.errorCode(), is(200)); 44 | } 45 | 46 | public static void dropTestDatabase(final String dbName) throws InterruptedException { 47 | // Remove the database 48 | httpClient.request("DELETE", "/" + dbName); 49 | 50 | // We wait fo the deletion to happen 51 | ElasticsearchTestCase.awaitBusy(new Predicate() { 52 | public boolean apply(Object obj) { 53 | HttpClientResponse response = httpClient.request("/" + dbName); 54 | return response.errorCode() == 404; 55 | } 56 | }, 30, TimeUnit.SECONDS); 57 | } 58 | 59 | public static void createTestDatabase(final String dbName) throws InterruptedException { 60 | // Create the database 61 | HttpClientResponse response = httpClient.request("PUT", "/" + dbName); 62 | 63 | Assert.assertThat("can not create database " + dbName, response.errorCode(), is(201)); 64 | } 65 | 66 | public static void dropAndCreateTestDatabase(final String dbName) throws InterruptedException { 67 | dropTestDatabase(dbName); 68 | createTestDatabase(dbName); 69 | } 70 | 71 | public static void putDocument(String dbName, String id, String json) { 72 | HttpClientResponse response = httpClient.request("PUT", "/" + dbName + "/" + id, json); 73 | 74 | Assert.assertThat("can not create document " + id, response.errorCode(), is(201)); 75 | } 76 | 77 | public static void putDocument(String dbName, String id, String... fielddata) throws IOException { 78 | XContentBuilder xContentBuilder = jsonBuilder().startObject(); 79 | for (int i = 0; i < fielddata.length; i++) { 80 | xContentBuilder.field(fielddata[i], fielddata[++i]); 81 | } 82 | xContentBuilder.endObject(); 83 | putDocument(dbName, id, xContentBuilder.string()); 84 | } 85 | 86 | public static void putDocumentWithAttachments(String dbName, String id, List doc, String... attachments) throws IOException { 87 | XContentBuilder xContentBuilder = jsonBuilder() 88 | .startObject() 89 | .field("_id", id); 90 | 91 | if (doc != null) { 92 | Iterator iterator = doc.iterator(); 93 | while (iterator.hasNext()) { 94 | xContentBuilder.field(iterator.next(), iterator.next()); 95 | } 96 | } 97 | 98 | xContentBuilder.startObject("_attachments"); 99 | 100 | for (int i = 0; i < attachments.length; i++) { 101 | String filename = attachments[i]; 102 | byte[] content = attachments[++i].getBytes(StandardCharsets.UTF_8); 103 | 104 | xContentBuilder.startObject(filename) 105 | .field("content_type", "text/plain") 106 | .field("data", content) 107 | .endObject(); 108 | } 109 | 110 | xContentBuilder.endObject().endObject(); 111 | 112 | CouchDBClient.putDocument(dbName, id, xContentBuilder.string()); 113 | } 114 | 115 | } 116 | -------------------------------------------------------------------------------- /src/test/java/org/elasticsearch/river/couchdb/helper/HttpClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | package org.elasticsearch.river.couchdb.helper; 20 | 21 | import org.elasticsearch.ElasticsearchException; 22 | import org.elasticsearch.common.base.Charsets; 23 | import org.elasticsearch.common.io.Streams; 24 | 25 | import java.io.IOException; 26 | import java.io.InputStream; 27 | import java.io.InputStreamReader; 28 | import java.io.OutputStreamWriter; 29 | import java.net.HttpURLConnection; 30 | import java.net.MalformedURLException; 31 | import java.net.URL; 32 | import java.nio.charset.StandardCharsets; 33 | import java.util.List; 34 | import java.util.Map; 35 | 36 | public class HttpClient { 37 | 38 | private final URL baseUrl; 39 | 40 | public HttpClient(String hostname, Integer port) { 41 | try { 42 | baseUrl = new URL("http", hostname, port, "/"); 43 | } catch (MalformedURLException e) { 44 | throw new ElasticsearchException("", e); 45 | } 46 | } 47 | 48 | public HttpClientResponse request(String path) { 49 | return request("GET", path, null, null); 50 | } 51 | 52 | public HttpClientResponse request(String method, String path) { 53 | return request(method, path, null, null); 54 | } 55 | 56 | public HttpClientResponse request(String method, String path, String payload) { 57 | return request(method, path, null, payload); 58 | } 59 | 60 | public HttpClientResponse request(String method, String path, Map headers, String payload) { 61 | URL url; 62 | try { 63 | url = new URL(baseUrl, path); 64 | } catch (MalformedURLException e) { 65 | throw new ElasticsearchException("Cannot parse " + path, e); 66 | } 67 | 68 | HttpURLConnection urlConnection; 69 | try { 70 | urlConnection = (HttpURLConnection) url.openConnection(); 71 | urlConnection.setRequestMethod(method); 72 | if (headers != null) { 73 | for (Map.Entry headerEntry : headers.entrySet()) { 74 | urlConnection.setRequestProperty(headerEntry.getKey(), headerEntry.getValue()); 75 | } 76 | } 77 | 78 | if (payload != null) { 79 | urlConnection.setDoOutput(true); 80 | urlConnection.setRequestProperty("Content-Type", "application/json"); 81 | urlConnection.setRequestProperty("Accept", "application/json"); 82 | OutputStreamWriter osw = new OutputStreamWriter(urlConnection.getOutputStream(), StandardCharsets.UTF_8); 83 | osw.write(payload); 84 | osw.flush(); 85 | osw.close(); 86 | } 87 | 88 | urlConnection.connect(); 89 | } catch (IOException e) { 90 | throw new ElasticsearchException("", e); 91 | } 92 | 93 | int errorCode = -1; 94 | Map> respHeaders = null; 95 | try { 96 | errorCode = urlConnection.getResponseCode(); 97 | respHeaders = urlConnection.getHeaderFields(); 98 | InputStream inputStream = urlConnection.getInputStream(); 99 | String body = null; 100 | try { 101 | body = Streams.copyToString(new InputStreamReader(inputStream, Charsets.UTF_8)); 102 | } catch (IOException e1) { 103 | throw new ElasticsearchException("problem reading error stream", e1); 104 | } 105 | return new HttpClientResponse(body, errorCode, respHeaders, null); 106 | } catch (IOException e) { 107 | InputStream errStream = urlConnection.getErrorStream(); 108 | String body = null; 109 | if (errStream != null) { 110 | try { 111 | body = Streams.copyToString(new InputStreamReader(errStream, Charsets.UTF_8)); 112 | } catch (IOException e1) { 113 | throw new ElasticsearchException("problem reading error stream", e1); 114 | } 115 | } 116 | return new HttpClientResponse(body, errorCode, respHeaders, e); 117 | } finally { 118 | urlConnection.disconnect(); 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /dev-tools/release.py: -------------------------------------------------------------------------------- 1 | # Licensed to Elasticsearch under one or more contributor 2 | # license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright 4 | # ownership. Elasticsearch licenses this file to you under 5 | # the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on 13 | # an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 14 | # either express or implied. See the License for the specific 15 | # language governing permissions and limitations under the License. 16 | 17 | import datetime 18 | import os 19 | import shutil 20 | import sys 21 | import time 22 | import urllib 23 | import urllib.request 24 | import zipfile 25 | 26 | from os.path import dirname, abspath 27 | 28 | """ 29 | This tool builds a release from the a given elasticsearch plugin branch. 30 | 31 | It is basically a wrapper on top of launch_release.py which: 32 | 33 | - tries to get a more recent version of launch_release.py in ... 34 | - download it if needed 35 | - launch it passing all arguments to it, like: 36 | 37 | $ python3 dev_tools/release.py --branch master --publish --remote origin 38 | 39 | Important options: 40 | 41 | # Dry run 42 | $ python3 dev_tools/release.py 43 | 44 | # Dry run without tests 45 | python3 dev_tools/release.py --skiptests 46 | 47 | # Release, publish artifacts and announce 48 | $ python3 dev_tools/release.py --publish 49 | 50 | See full documentation in launch_release.py 51 | """ 52 | env = os.environ 53 | 54 | # Change this if the source repository for your scripts is at a different location 55 | SOURCE_REPO = 'elasticsearch/elasticsearch-plugins-script' 56 | # We define that we should download again the script after 1 days 57 | SCRIPT_OBSOLETE_DAYS = 1 58 | # We ignore in master.zip file the following files 59 | IGNORED_FILES = ['.gitignore', 'README.md'] 60 | 61 | 62 | ROOT_DIR = abspath(os.path.join(abspath(dirname(__file__)), '../')) 63 | TARGET_TOOLS_DIR = ROOT_DIR + '/plugin_tools' 64 | DEV_TOOLS_DIR = ROOT_DIR + '/dev-tools' 65 | BUILD_RELEASE_FILENAME = 'release.zip' 66 | BUILD_RELEASE_FILE = TARGET_TOOLS_DIR + '/' + BUILD_RELEASE_FILENAME 67 | SOURCE_URL = 'https://github.com/%s/archive/master.zip' % SOURCE_REPO 68 | 69 | # Download a recent version of the release plugin tool 70 | try: 71 | os.mkdir(TARGET_TOOLS_DIR) 72 | print('directory %s created' % TARGET_TOOLS_DIR) 73 | except FileExistsError: 74 | pass 75 | 76 | 77 | try: 78 | # we check latest update. If we ran an update recently, we 79 | # are not going to check it again 80 | download = True 81 | 82 | try: 83 | last_download_time = datetime.datetime.fromtimestamp(os.path.getmtime(BUILD_RELEASE_FILE)) 84 | if (datetime.datetime.now()-last_download_time).days < SCRIPT_OBSOLETE_DAYS: 85 | download = False 86 | except FileNotFoundError: 87 | pass 88 | 89 | if download: 90 | urllib.request.urlretrieve(SOURCE_URL, BUILD_RELEASE_FILE) 91 | with zipfile.ZipFile(BUILD_RELEASE_FILE) as myzip: 92 | for member in myzip.infolist(): 93 | filename = os.path.basename(member.filename) 94 | # skip directories 95 | if not filename: 96 | continue 97 | if filename in IGNORED_FILES: 98 | continue 99 | 100 | # copy file (taken from zipfile's extract) 101 | source = myzip.open(member.filename) 102 | target = open(os.path.join(TARGET_TOOLS_DIR, filename), "wb") 103 | with source, target: 104 | shutil.copyfileobj(source, target) 105 | # We keep the original date 106 | date_time = time.mktime(member.date_time + (0, 0, -1)) 107 | os.utime(os.path.join(TARGET_TOOLS_DIR, filename), (date_time, date_time)) 108 | print('plugin-tools updated from %s' % SOURCE_URL) 109 | except urllib.error.HTTPError: 110 | pass 111 | 112 | 113 | # Let see if we need to update the release.py script itself 114 | source_time = os.path.getmtime(TARGET_TOOLS_DIR + '/release.py') 115 | repo_time = os.path.getmtime(DEV_TOOLS_DIR + '/release.py') 116 | if source_time > repo_time: 117 | input('release.py needs an update. Press a key to update it...') 118 | shutil.copyfile(TARGET_TOOLS_DIR + '/release.py', DEV_TOOLS_DIR + '/release.py') 119 | 120 | # We can launch the build process 121 | try: 122 | PYTHON = 'python' 123 | # make sure python3 is used if python3 is available 124 | # some systems use python 2 as default 125 | os.system('python3 --version > /dev/null 2>&1') 126 | PYTHON = 'python3' 127 | except RuntimeError: 128 | pass 129 | 130 | release_args = '' 131 | for x in range(1, len(sys.argv)): 132 | release_args += ' ' + sys.argv[x] 133 | 134 | os.system('%s %s/build_release.py %s' % (PYTHON, TARGET_TOOLS_DIR, release_args)) 135 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing to elasticsearch 2 | ============================= 3 | 4 | Elasticsearch is an open source project and we love to receive contributions from our community — you! There are many ways to contribute, from writing tutorials or blog posts, improving the documentation, submitting bug reports and feature requests or writing code which can be incorporated into Elasticsearch itself. 5 | 6 | Bug reports 7 | ----------- 8 | 9 | If you think you have found a bug in Elasticsearch, first make sure that you are testing against the [latest version of Elasticsearch](http://www.elasticsearch.org/download/) - your issue may already have been fixed. If not, search our [issues list](https://github.com/elasticsearch/elasticsearch/issues) on GitHub in case a similar issue has already been opened. 10 | 11 | It is very helpful if you can prepare a reproduction of the bug. In other words, provide a small test case which we can run to confirm your bug. It makes it easier to find the problem and to fix it. Test cases should be provided as `curl` commands which we can copy and paste into a terminal to run it locally, for example: 12 | 13 | ```sh 14 | # delete the index 15 | curl -XDELETE localhost:9200/test 16 | 17 | # insert a document 18 | curl -XPUT localhost:9200/test/test/1 -d '{ 19 | "title": "test document" 20 | }' 21 | 22 | # this should return XXXX but instead returns YYY 23 | curl .... 24 | ``` 25 | 26 | Provide as much information as you can. You may think that the problem lies with your query, when actually it depends on how your data is indexed. The easier it is for us to recreate your problem, the faster it is likely to be fixed. 27 | 28 | Feature requests 29 | ---------------- 30 | 31 | If you find yourself wishing for a feature that doesn't exist in Elasticsearch, you are probably not alone. There are bound to be others out there with similar needs. Many of the features that Elasticsearch has today have been added because our users saw the need. 32 | Open an issue on our [issues list](https://github.com/elasticsearch/elasticsearch/issues) on GitHub which describes the feature you would like to see, why you need it, and how it should work. 33 | 34 | Contributing code and documentation changes 35 | ------------------------------------------- 36 | 37 | If you have a bugfix or new feature that you would like to contribute to Elasticsearch, please find or open an issue about it first. Talk about what you would like to do. It may be that somebody is already working on it, or that there are particular issues that you should know about before implementing the change. 38 | 39 | We enjoy working with contributors to get their code accepted. There are many approaches to fixing a problem and it is important to find the best approach before writing too much code. 40 | 41 | The process for contributing to any of the [Elasticsearch repositories](https://github.com/elasticsearch/) is similar. Details for individual projects can be found below. 42 | 43 | ### Fork and clone the repository 44 | 45 | You will need to fork the main Elasticsearch code or documentation repository and clone it to your local machine. See 46 | [github help page](https://help.github.com/articles/fork-a-repo) for help. 47 | 48 | Further instructions for specific projects are given below. 49 | 50 | ### Submitting your changes 51 | 52 | Once your changes and tests are ready to submit for review: 53 | 54 | 1. Test your changes 55 | Run the test suite to make sure that nothing is broken. 56 | 57 | 2. Sign the Contributor License Agreement 58 | Please make sure you have signed our [Contributor License Agreement](http://www.elasticsearch.org/contributor-agreement/). We are not asking you to assign copyright to us, but to give us the right to distribute your code without restriction. We ask this of all contributors in order to assure our users of the origin and continuing existence of the code. You only need to sign the CLA once. 59 | 60 | 3. Rebase your changes 61 | Update your local repository with the most recent code from the main Elasticsearch repository, and rebase your branch on top of the latest master branch. We prefer your changes to be squashed into a single commit. 62 | 63 | 4. Submit a pull request 64 | Push your local changes to your forked copy of the repository and [submit a pull request](https://help.github.com/articles/using-pull-requests). In the pull request, describe what your changes do and mention the number of the issue where discussion has taken place, eg "Closes #123". 65 | 66 | Then sit back and wait. There will probably be discussion about the pull request and, if any changes are needed, we would love to work with you to get your pull request merged into Elasticsearch. 67 | 68 | 69 | Contributing to the Elasticsearch plugin 70 | ---------------------------------------- 71 | 72 | **Repository:** [https://github.com/elasticsearch/elasticsearch-river-couchdb](https://github.com/elasticsearch/elasticsearch-river-couchdb) 73 | 74 | Make sure you have [Maven](http://maven.apache.org) installed, as Elasticsearch uses it as its build system. Integration with IntelliJ and Eclipse should work out of the box. Eclipse users can automatically configure their IDE by running `mvn eclipse:eclipse` and then importing the project into their workspace: `File > Import > Existing project into workspace`. 75 | 76 | Please follow these formatting guidelines: 77 | 78 | * Java indent is 4 spaces 79 | * Line width is 140 characters 80 | * The rest is left to Java coding standards 81 | * Disable “auto-format on save” to prevent unnecessary format changes. This makes reviews much harder as it generates unnecessary formatting changes. If your IDE supports formatting only modified chunks that is fine to do. 82 | 83 | To create a distribution from the source, simply run: 84 | 85 | ```sh 86 | cd elasticsearch-river-couchdb/ 87 | mvn clean package -DskipTests 88 | ``` 89 | 90 | You will find the newly built packages under: `./target/releases/`. 91 | 92 | Before submitting your changes, run the test suite to make sure that nothing is broken, with: 93 | 94 | ```sh 95 | mvn clean test -Dtests.couchdb=true 96 | ``` 97 | 98 | Source: [Contributing to elasticsearch](http://www.elasticsearch.org/contributing-to-elasticsearch/) 99 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **Important**: This project has been stopped since elasticsearch 2.0. 2 | 3 | ---- 4 | 5 | CouchDB River Plugin for Elasticsearch 6 | ================================== 7 | 8 | The CouchDB River plugin allows to automatically index couchdb and make it searchable using the excellent 9 | [_changes](http://guide.couchdb.org/draft/notifications.html) stream couchdb provides. 10 | 11 | **Rivers are [deprecated](https://www.elastic.co/blog/deprecating_rivers) and will be removed in the future.** 12 | Have a look at [logstash couchdb changes input](http://www.elastic.co/guide/en/logstash/current/plugins-inputs-couchdb_changes.html). 13 | 14 | In order to install the plugin, run: 15 | 16 | ```sh 17 | bin/plugin install elasticsearch/elasticsearch-river-couchdb/2.6.0 18 | ``` 19 | 20 | You need to install a version matching your Elasticsearch version: 21 | 22 | | Elasticsearch |CouchDB River Plugin| Docs | 23 | |------------------------|--------------------|------------------------------------------------------------------------------------------------------------------------------------| 24 | | master | Build from source | See below | 25 | | es-1.x | Build from source | [2.7.0-SNAPSHOT](https://github.com/elasticsearch/elasticsearch-river-couchdb/tree/es-1.x/#version-270-snapshot-for-elasticsearch-1x)| 26 | | es-1.6 | 2.6.0 | [2.6.0](https://github.com/elastic/elasticsearch-river-couchdb/tree/v2.6.0/#version-260-for-elasticsearch-16) | 27 | | es-1.5 | 2.5.0 | [2.5.0](https://github.com/elastic/elasticsearch-river-couchdb/tree/v2.5.0/#version-250-for-elasticsearch-15) | 28 | | es-1.4 | 2.4.2 | [2.4.2](https://github.com/elasticsearch/elasticsearch-river-couchdb/tree/v2.4.2/#version-242-for-elasticsearch-14) | 29 | | es-1.3 | 2.3.0 | [2.3.0](https://github.com/elasticsearch/elasticsearch-river-couchdb/tree/v2.3.0/#version-230-for-elasticsearch-13) | 30 | | es-1.2 | 2.2.0 | [2.2.0](https://github.com/elasticsearch/elasticsearch-river-couchdb/tree/v2.2.0/#couchdb-river-plugin-for-elasticsearch) | 31 | | es-1.0 | 2.0.0 | [2.0.0](https://github.com/elasticsearch/elasticsearch-river-couchdb/tree/v2.0.0/#couchdb-river-plugin-for-elasticsearch) | 32 | | es-0.90 | 1.3.0 | [1.3.0](https://github.com/elasticsearch/elasticsearch-river-couchdb/tree/v1.3.0/#couchdb-river-plugin-for-elasticsearch) | 33 | 34 | To build a `SNAPSHOT` version, you need to build it with Maven: 35 | 36 | ```bash 37 | mvn clean install 38 | plugin --install river-couchdb \ 39 | --url file:target/releases/elasticsearch-river-couchdb-X.X.X-SNAPSHOT.zip 40 | ``` 41 | 42 | Create river 43 | ------------ 44 | 45 | Setting it up is as simple as executing the following against elasticsearch: 46 | 47 | ```sh 48 | curl -XPUT 'localhost:9200/_river/my_db/_meta' -d '{ 49 | "type" : "couchdb", 50 | "couchdb" : { 51 | "host" : "localhost", 52 | "port" : 5984, 53 | "db" : "my_db", 54 | "filter" : null 55 | }, 56 | "index" : { 57 | "index" : "my_db", 58 | "type" : "my_db", 59 | "bulk_size" : "100", 60 | "bulk_timeout" : "10ms" 61 | } 62 | }' 63 | ``` 64 | 65 | This call will create a river that uses the `_changes` stream to index all data within couchdb. Moreover, any "future" changes will automatically be indexed as well, making your search index and couchdb synchronized at all times. 66 | 67 | The couchdb river is provided as a [plugin](https://github.com/elasticsearch/elasticsearch-river-couchdb) (including explanation on how to install it). 68 | 69 | On top of that, in case of a failover, the couchdb river will automatically be started on another elasticsearch node, and continue indexing from the last indexed seq. 70 | 71 | Bulking 72 | ====== 73 | 74 | Bulking is automatically done in order to speed up the indexing process. If within the specified `bulk_timeout` more changes are detected, 75 | changes will be bulked up to `bulk_size` before they are indexed. 76 | 77 | Since 1.3.0, by default, `bulk` size is `100`. A bulk is flushed every `5s`. Number of concurrent requests allowed to be executed is 1. 78 | You can modify those settings within index section: 79 | 80 | ```javascript 81 | { 82 | "type" : "couchdb", 83 | "index" : { 84 | "index" : "my_index", 85 | "type" : "my_type", 86 | "bulk_size" : 1000, 87 | "flush_interval" : "1s", 88 | "max_concurrent_bulk" : 3 89 | } 90 | } 91 | ``` 92 | 93 | Filtering 94 | ====== 95 | 96 | The `changes` stream allows to provide a filter with parameters that will be used by couchdb to filter the stream of changes. Here is how it can be configured: 97 | 98 | ```javascript 99 | { 100 | "couchdb" : { 101 | "filter" : "test", 102 | "filter_params" : { 103 | "param1" : "value1", 104 | "param2" : "value2" 105 | } 106 | } 107 | } 108 | ``` 109 | 110 | Script Filters 111 | ========= 112 | 113 | Filtering can also be performed by providing a script that will further process each changed item 114 | within the changes stream. The json provided to the script is under a var called **ctx** with the relevant seq stream 115 | change (for example, **ctx.doc** will refer to the document, or **ctx.deleted** is the flag if its deleted or not). 116 | 117 | Any other [script language supported by Elasticsearch](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-plugins.html#scripting) 118 | may be used by setting the `script_type` parameter to the appropriate value. 119 | 120 | If unspecified, the default is `groovy`. 121 | See [Scripting documentation](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-scripting.html) for details. 122 | 123 | The **ctx.doc** can be changed and its value can will be indexed (assuming its not a deleted change). 124 | Also, if **ctx.ignore** is set to true, the change seq will be ignore and not applied. 125 | 126 | Other possible values that can be set are **ctx.index** to control the index name to index the doc into, **ctx.type** 127 | to control the (mapping) type to index into, **ctx._parent** and **ctx._routing**. 128 | 129 | Here is an example setting that adds `field1` with value `value1` to all docs: 130 | 131 | ```javascript 132 | { 133 | "type" : "couchdb", 134 | "couchdb" : { 135 | "script" : "ctx.doc.field1 = 'value1'" 136 | } 137 | } 138 | ``` 139 | 140 | Basic Authentication 141 | =============== 142 | 143 | Basic Authentication can be used by passing the **user** and **password** attributes. 144 | 145 | ```javascript 146 | { 147 | "type" : "couchdb", 148 | "couchdb" : { 149 | "user" : "alice", 150 | "password" : "secret" 151 | } 152 | } 153 | ``` 154 | 155 | HTTPS 156 | ===== 157 | 158 | To use HTTPS, pass the **protocol** field. Most likely, you will also have to change the **port**. If you have unfixable problems with the servers certificates for any reason, you can disable hostname verification by passing **no_verify**. 159 | 160 | ```javascript 161 | { 162 | "type" : "couchdb", 163 | "couchdb" : { 164 | "protocol" : "https", 165 | "port" : 443, 166 | "no_verify" : "true" 167 | } 168 | } 169 | ``` 170 | 171 | 172 | Ignoring Attachments 173 | ==================== 174 | 175 | You can ignore attachments as provided by couchDb for each document (`_attachments` field). 176 | 177 | Here is an example setting that disable *attachments* for all docs: 178 | 179 | ```javascript 180 | { 181 | "type":"couchdb", 182 | "couchdb": { 183 | "ignore_attachments":true 184 | } 185 | } 186 | ``` 187 | 188 | 189 | Note, by default, attachments are not ignored (**false**) 190 | 191 | 192 | Heartbeat 193 | ========= 194 | 195 | By default, couchdb river set _changes API heartbeat to `10s`. 196 | Since 1.3.0, an additional option has been added to control the HTTP connection timeout (default to `30s`). 197 | you can control both settings using `heartbeat` and `read_timeout` options: 198 | 199 | ```sh 200 | curl -XPUT 'localhost:9200/_river/my_db/_meta' -d '{ 201 | "type" : "couchdb", 202 | "couchdb" : { 203 | "host" : "localhost", 204 | "port" : 5984, 205 | "db" : "my_db", 206 | "heartbeat" : "5s", 207 | "read_timeout" : "15s" 208 | } 209 | }' 210 | ``` 211 | 212 | Starting at a Specific Sequence 213 | ========== 214 | 215 | The CouchDB river stores the `last_seq` value in a document called `_seq` in the `_river` index. You can use this fact to start or resume rivers at a particular sequence. 216 | 217 | To have the CouchDB river start at a particular `last_seq`, create a document with contents like this: 218 | 219 | ````sh 220 | curl -XPUT 'localhost:9200/_river/my_db/_seq' -d ' 221 | { 222 | "couchdb": { 223 | "last_seq": "100" 224 | } 225 | }' 226 | ```` 227 | 228 | where 100 is the sequence number you want the river to start from. Then create the `_meta` document as before. The CouchDB river will startup and read the last sequence value and start indexing from there. 229 | 230 | 231 | Examples 232 | ======== 233 | 234 | 235 | Indexing Databases with Multiple Types 236 | ------------------------------------- 237 | 238 | A common pattern in CouchDB is to have a single database hold documents 239 | of multiple types. Typically each document will have a field containing 240 | the type of the document. 241 | 242 | For example, a database of products from Amazon might have a book type: 243 | 244 | ```json 245 | { 246 | "_id": 1, 247 | "type" : "book", 248 | "author" : "Michael McCandless", 249 | "title" : "Lucene in Action" 250 | } 251 | ``` 252 | 253 | and a CD type: 254 | 255 | ```json 256 | { 257 | "_id": 2, 258 | "type" : "cd", 259 | "artist" : "Tool", 260 | "title" : "Undertow" 261 | } 262 | ``` 263 | 264 | Elasticsearch also supports multiple types in the same index and we need 265 | to tell ES where each type from CouchDB goes. You do this with a river 266 | definition like this: 267 | 268 | ```json 269 | { 270 | "type" : "couchdb", 271 | "couchdb" : { 272 | "host" : "localhost", 273 | "port" : 5984, 274 | "db" : "amazon", 275 | "script" : "ctx._type = ctx.doc.type" 276 | }, 277 | "index" : { 278 | "index" : "amazon" 279 | } 280 | } 281 | ``` 282 | 283 | Setting `ctx._type` tells Elasticsearch what type in the index to use. 284 | So if doc.type for a CouchDB changeset is "book" then Elasticsearch will 285 | index it as the "book" type. 286 | 287 | The script block can be expanded to handle more complicated cases if 288 | your types don't map one-to-one between the systems. 289 | 290 | If you need to also handle deleting documents from the right type in 291 | Elasticsearch, be aware that the above setup requires you to delete 292 | documents from CouchDB in a special way. If you use the `DELETE` `HTTP` 293 | verb with CouchDB you will break the above river as ES will be unable 294 | to determine what type you're trying to delete. Instead you must 295 | preserve some information about the document you deleted by using the 296 | CouchDB bulk document interface. 297 | 298 | For example, to delete the above "cd" document, you must post a document 299 | like this to the CouchDB server, replacing the `_rev` field with the 300 | revision of the document you want to delete: 301 | 302 | ```sh 303 | curl -XPOST http://localhost:5984/amazon/_bulk_docs -d ' 304 | { 305 | "docs" : [ 306 | { 307 | "_id: 2, 308 | "_rev" : "rev", 309 | "_deleted" : true, 310 | "type" : "cd" 311 | } 312 | ] 313 | }' 314 | ``` 315 | 316 | This deletes the document while preserving the type information for ES. 317 | You can extend this technique to store more data in deleted documents 318 | but be aware of the disk space usage. 319 | 320 | Indexing parent/child documents 321 | ------------------------------- 322 | 323 | If you need to index relational documents using the parent/child feature, you could 324 | do it using a script filter. 325 | 326 | For example, let's say you have two types of document in CouchDB: Regions and Campuses. 327 | 328 | ``` 329 | // Region 1 330 | { 331 | "type": "region", 332 | "name": "bretagne" 333 | } 334 | ``` 335 | 336 | ``` 337 | // Campus 2 338 | { 339 | "type": "campus", 340 | "name": "enib", 341 | "parent_id": "1" 342 | } 343 | ``` 344 | 345 | You can use the following mapping for `campus`: 346 | 347 | ``` 348 | { 349 | "campus" : { 350 | "_parent" : { 351 | "type" : "region" 352 | } 353 | } 354 | } 355 | ``` 356 | 357 | And the launch the river with the following script: 358 | 359 | ``` 360 | { 361 | "type": "couchdb", 362 | "couchdb": { 363 | "script": "ctx._type = ctx.doc.type; if (ctx._type == 'campus') { ctx._parent = ctx.doc.parent_id; }" 364 | } 365 | } 366 | ``` 367 | 368 | Removing fields using a script 369 | ------------------------------ 370 | 371 | You can remove fields using a script like this: 372 | 373 | ``` 374 | { 375 | "type": "couchdb", 376 | "couchdb": { 377 | "script": "var oblitertron = function(x) { var things = [\"foo\"]; var toberemoved = new java.util.ArrayList(); foreach (i : x.keySet()) { if(things.indexOf(i) == -1) { toberemoved.add(i); } } foreach (i : toberemoved) { x.remove(i); } return x; }; ctx.doc = oblitertron(ctx.doc);" 378 | } 379 | } 380 | ``` 381 | 382 | A more readable version of the script (with comments) is: 383 | 384 | ```js 385 | var oblitertron = function (x) { 386 | // List of fields we want to keep. Others will be removed, such as _id, _rev... 387 | var things = ["foo"]; 388 | var toberemoved = new java.util.ArrayList(); 389 | foreach(i : x.keySet()) { 390 | if (things.indexOf(i) == -1) { 391 | // If we find a field to be removed, we store its name in an array 392 | toberemoved.add(i); 393 | } 394 | } 395 | // We remove useless fields 396 | foreach(i : toberemoved) { 397 | x.remove(i); 398 | } 399 | // We return the new document which will be indexed. 400 | return x; 401 | }; 402 | ctx.doc = oblitertron(ctx.doc); 403 | ``` 404 | 405 | 406 | Tests 407 | ===== 408 | 409 | To run couchdb integration tests, you need to have couchdb running on `localhost` using default `5984` port. 410 | Then you can run tests using `tests.couchdb` option: 411 | 412 | ```sh 413 | mvn clean test -Dtests.couchdb=true 414 | ``` 415 | 416 | 417 | License 418 | ======= 419 | 420 | This software is licensed under the Apache 2 license, quoted below. 421 | 422 | Copyright 2009-2014 Elasticsearch 423 | 424 | Licensed under the Apache License, Version 2.0 (the "License"); you may not 425 | use this file except in compliance with the License. You may obtain a copy of 426 | the License at 427 | 428 | http://www.apache.org/licenses/LICENSE-2.0 429 | 430 | Unless required by applicable law or agreed to in writing, software 431 | distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 432 | WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 433 | License for the specific language governing permissions and limitations under 434 | the License. 435 | -------------------------------------------------------------------------------- /src/test/java/org/elasticsearch/river/couchdb/CouchdbRiverIntegrationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.river.couchdb; 21 | 22 | import org.elasticsearch.action.get.GetResponse; 23 | import org.elasticsearch.action.search.SearchResponse; 24 | import org.elasticsearch.common.Strings; 25 | import org.elasticsearch.common.base.Predicate; 26 | import org.elasticsearch.common.collect.ImmutableList; 27 | import org.elasticsearch.common.settings.Settings; 28 | import org.elasticsearch.common.xcontent.XContentBuilder; 29 | import org.elasticsearch.index.query.QueryBuilders; 30 | import org.elasticsearch.indices.IndexAlreadyExistsException; 31 | import org.elasticsearch.indices.IndexMissingException; 32 | import org.elasticsearch.plugins.PluginsService; 33 | import org.elasticsearch.river.couchdb.helper.CouchDBClient; 34 | import org.elasticsearch.search.SearchHit; 35 | import org.elasticsearch.test.ElasticsearchIntegrationTest; 36 | import org.junit.After; 37 | import org.junit.Before; 38 | import org.junit.Test; 39 | 40 | import java.io.IOException; 41 | import java.util.concurrent.ExecutionException; 42 | import java.util.concurrent.TimeUnit; 43 | 44 | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; 45 | import static org.elasticsearch.river.couchdb.helper.CouchDBClient.putDocument; 46 | import static org.elasticsearch.river.couchdb.helper.CouchDBClient.putDocumentWithAttachments; 47 | import static org.hamcrest.Matchers.*; 48 | 49 | /** 50 | * Integration tests for CouchDb river
51 | * You may have a couchdb instance running on localhost:5984 with a mytest database. 52 | */ 53 | @ElasticsearchIntegrationTest.ClusterScope( 54 | scope = ElasticsearchIntegrationTest.Scope.SUITE, 55 | numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0) 56 | public class CouchdbRiverIntegrationTest extends AbstractCouchdbTest { 57 | 58 | @Override 59 | protected Settings nodeSettings(int nodeOrdinal) { 60 | return Settings.builder() 61 | .put(super.nodeSettings(nodeOrdinal)) 62 | .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) 63 | .build(); 64 | } 65 | 66 | private interface InjectorHook { 67 | public void inject(); 68 | } 69 | 70 | private static final String testDbPrefix = "elasticsearch_couch_test_"; 71 | 72 | private String suffix; 73 | 74 | @Before 75 | public final void dbSuffixName() { 76 | suffix = String.valueOf(System.nanoTime()) + "_" + randomInt(); 77 | } 78 | 79 | @After 80 | public final void wipeDbAfterTest() { 81 | logger.info(" -> Removing test database [{}]", getDbName()); 82 | try { 83 | CouchDBClient.dropTestDatabase(getDbName()); 84 | } catch (Exception e) { 85 | // Let's ignore it 86 | } 87 | } 88 | 89 | private String getDbName() { 90 | return testDbPrefix.concat(Strings.toUnderscoreCase(getTestName())).concat(suffix); 91 | } 92 | 93 | private void launchTest(XContentBuilder river, final Integer numDocs, InjectorHook injectorHook) 94 | throws IOException, InterruptedException { 95 | logger.info(" -> Checking couchdb running"); 96 | CouchDBClient.checkCouchDbRunning(); 97 | logger.info(" -> Creating test database [{}]", getDbName()); 98 | CouchDBClient.dropAndCreateTestDatabase(getDbName()); 99 | logger.info(" -> Put [{}] documents", numDocs); 100 | for (int i = 0; i < numDocs; i++) { 101 | CouchDBClient.putDocument(getDbName(), "" + i, "foo", "bar", "content", "" + i); 102 | } 103 | logger.info(" -> Put [{}] documents done", numDocs); 104 | 105 | if (injectorHook != null) { 106 | logger.info(" -> Injecting extra data"); 107 | injectorHook.inject(); 108 | } 109 | 110 | logger.info(" -> Create river"); 111 | try { 112 | createIndex(getDbName()); 113 | } catch (IndexAlreadyExistsException e) { 114 | // No worries. We already created the index before 115 | } 116 | index("_river", getDbName(), "_meta", river); 117 | 118 | logger.info(" -> Wait for some docs"); 119 | assertThat(awaitBusy(new Predicate() { 120 | public boolean apply(Object obj) { 121 | try { 122 | refresh(); 123 | SearchResponse response = client().prepareSearch(getDbName()).get(); 124 | logger.info(" -> got {} docs in {} index", response.getHits().totalHits(), getDbName()); 125 | return response.getHits().totalHits() == numDocs; 126 | } catch (IndexMissingException e) { 127 | return false; 128 | } 129 | } 130 | }, 1, TimeUnit.MINUTES), equalTo(true)); 131 | } 132 | 133 | /** 134 | * This is a simple test case for testing attachments removing. 135 | */ 136 | @Test 137 | public void testAttachmentEnabled() throws IOException, InterruptedException { 138 | runAttachmentTest(false); 139 | } 140 | 141 | /** 142 | * This is a simple test case for testing attachments removing. 143 | */ 144 | @Test 145 | public void testAttachmentDisabled() throws IOException, InterruptedException { 146 | runAttachmentTest(true); 147 | } 148 | 149 | private void runAttachmentTest(boolean disabled) throws IOException, InterruptedException { 150 | // Create the river 151 | launchTest(jsonBuilder() 152 | .startObject() 153 | .field("type", "couchdb") 154 | .startObject("couchdb") 155 | .field("host", CouchDBClient.host) 156 | .field("port", CouchDBClient.port) 157 | .field("db", getDbName()) 158 | .field("ignore_attachments", disabled) 159 | .endObject() 160 | .endObject(), 0, new InjectorHook() { 161 | @Override 162 | public void inject() { 163 | try { 164 | putDocumentWithAttachments(getDbName(), "1", 165 | new ImmutableList.Builder().add("foo", "bar").build(), 166 | "text-in-english.txt", "God save the queen!", 167 | "text-in-french.txt", "Allons enfants !"); 168 | } catch (IOException e) { 169 | logger.error("Error while injecting attachments"); 170 | } 171 | 172 | } 173 | }); 174 | 175 | // Check that docs are indexed by the river 176 | assertThat(awaitBusy(new Predicate() { 177 | public boolean apply(Object obj) { 178 | try { 179 | refresh(); 180 | SearchResponse response = client().prepareSearch(getDbName()).get(); 181 | logger.info(" -> got {} docs in {} index", response.getHits().totalHits(), getDbName()); 182 | return response.getHits().totalHits() == 1; 183 | } catch (IndexMissingException e) { 184 | return false; 185 | } 186 | } 187 | }, 1, TimeUnit.MINUTES), equalTo(true)); 188 | 189 | SearchResponse response = client().prepareSearch(getDbName()) 190 | .addField("_attachments.text-in-english.txt.content_type") 191 | .addField("_attachments.text-in-french.txt.content_type") 192 | .get(); 193 | 194 | assertThat(response.getHits().getAt(0).field("_attachments.text-in-english.txt.content_type"), disabled ? nullValue() : notNullValue()); 195 | assertThat(response.getHits().getAt(0).field("_attachments.text-in-french.txt.content_type"), disabled ? nullValue() : notNullValue()); 196 | if (!disabled) { 197 | assertThat(response.getHits().getAt(0).field("_attachments.text-in-english.txt.content_type").getValue().toString(), is("text/plain")); 198 | assertThat(response.getHits().getAt(0).field("_attachments.text-in-french.txt.content_type").getValue().toString(), is("text/plain")); 199 | } 200 | } 201 | 202 | @Test 203 | public void testParameters() throws IOException, InterruptedException { 204 | launchTest(jsonBuilder() 205 | .startObject() 206 | .field("type", "couchdb") 207 | .startObject("couchdb") 208 | .field("heartbeat", "5s") 209 | .field("read_timeout", "15s") 210 | .endObject() 211 | .endObject(), randomIntBetween(5, 1000), null); 212 | } 213 | 214 | @Test 215 | public void testSimple() throws IOException, InterruptedException { 216 | launchTest(jsonBuilder() 217 | .startObject() 218 | .field("type", "couchdb") 219 | .endObject(), randomIntBetween(5, 1000), null); 220 | } 221 | 222 | @Test 223 | public void testScriptingDefaultEngine() throws IOException, InterruptedException { 224 | launchTest(jsonBuilder() 225 | .startObject() 226 | .field("type", "couchdb") 227 | .startObject("couchdb") 228 | .field("script", "ctx.doc.newfield = ctx.doc.foo") 229 | .endObject() 230 | .endObject(), randomIntBetween(5, 1000), null); 231 | 232 | SearchResponse response = client().prepareSearch(getDbName()) 233 | .addField("newfield") 234 | .get(); 235 | 236 | assertThat(response.getHits().getAt(0).field("newfield"), notNullValue()); 237 | assertThat(response.getHits().getAt(0).field("newfield").getValue().toString(), is("bar")); 238 | } 239 | 240 | /** 241 | * Test case for #44: https://github.com/elasticsearch/elasticsearch-river-couchdb/issues/44 242 | */ 243 | @Test 244 | public void testScriptingQuote_44() throws IOException, InterruptedException { 245 | launchTest(jsonBuilder() 246 | .startObject() 247 | .field("type", "couchdb") 248 | .startObject("couchdb") 249 | .field("script", "ctx.doc.newfield = 'value1'") 250 | .endObject() 251 | .endObject(), randomIntBetween(5, 1000), null); 252 | 253 | SearchResponse response = client().prepareSearch(getDbName()) 254 | .addField("newfield") 255 | .get(); 256 | 257 | assertThat(response.getHits().getAt(0).field("newfield"), notNullValue()); 258 | assertThat(response.getHits().getAt(0).field("newfield").getValue().toString(), is("value1")); 259 | } 260 | 261 | /** 262 | * Test case for #51: https://github.com/elasticsearch/elasticsearch-river-couchdb/issues/51 263 | */ 264 | @Test 265 | public void testScriptingParentChild_51() throws IOException, InterruptedException, ExecutionException { 266 | prepareCreate(getDbName()) 267 | .addMapping("region", jsonBuilder() 268 | .startObject() 269 | .startObject("region") 270 | .endObject() 271 | .endObject().string()) 272 | .addMapping("campus", jsonBuilder() 273 | .startObject() 274 | .startObject("campus") 275 | .startObject("_parent") 276 | .field("type", "region") 277 | .endObject() 278 | .endObject() 279 | .endObject().string()) 280 | .get(); 281 | 282 | launchTest(jsonBuilder() 283 | .startObject() 284 | .field("type", "couchdb") 285 | .startObject("couchdb") 286 | .field("script", "ctx._type = ctx.doc.type; if (ctx._type == 'campus') { ctx._parent = ctx.doc.parent_id; }") 287 | .endObject() 288 | .endObject(), 0, new InjectorHook() { 289 | @Override 290 | public void inject() { 291 | try { 292 | putDocument(getDbName(), "1", 293 | "type", "region", 294 | "name", "bretagne"); 295 | putDocument(getDbName(), "2", 296 | "type", "campus", 297 | "name", "enib", 298 | "parent_id", "1"); 299 | } catch (IOException e) { 300 | logger.error("Error while injecting documents"); 301 | } 302 | } 303 | }); 304 | 305 | // Check that docs are indexed by the river 306 | assertThat(awaitBusy(new Predicate() { 307 | public boolean apply(Object obj) { 308 | try { 309 | refresh(); 310 | SearchResponse response = client().prepareSearch(getDbName()).get(); 311 | logger.info(" -> got {} docs in {} index", response.getHits().totalHits(), getDbName()); 312 | return response.getHits().totalHits() == 2; 313 | } catch (IndexMissingException e) { 314 | return false; 315 | } 316 | } 317 | }, 1, TimeUnit.MINUTES), equalTo(true)); 318 | 319 | SearchResponse response = client().prepareSearch(getDbName()) 320 | .setQuery( 321 | QueryBuilders.hasChildQuery("campus", 322 | QueryBuilders.matchQuery("name", "enib")) 323 | ) 324 | .get(); 325 | 326 | assertThat(response.getHits().getTotalHits(), is(1L)); 327 | assertThat(response.getHits().getAt(0).getType(), is("region")); 328 | assertThat(response.getHits().getAt(0).getId(), is("1")); 329 | } 330 | 331 | /** 332 | * Test case for #45: https://github.com/elasticsearch/elasticsearch-river-couchdb/issues/45 333 | */ 334 | @Test 335 | public void testScriptingTypeOf_45() throws IOException, InterruptedException { 336 | launchTest(jsonBuilder() 337 | .startObject() 338 | .field("type", "couchdb") 339 | .startObject("couchdb") 340 | .field("script_type", "groovy") 341 | // This groovy script removes all "_id" fields and 50% of "content" fields 342 | .field("script", " def docId = Integer.parseInt(ctx.doc[\"_id\"]);\n" + 343 | " def removals = [\"content\", \"_id\"]; \n" + 344 | " for(i in removals) { \n" + 345 | "\tif (ctx.doc.containsKey(i)) { \n" + 346 | "\t\tif (\"content\".equals(i)) {\n" + 347 | "\t\t\tif ((docId % 2) == 0) {\n" + 348 | "\t\t\t\tctx.doc.remove(i)\n" + 349 | "\t\t\t} \n" + 350 | "\t\t} else {\n" + 351 | "\t\t\tctx.doc.remove(i)\n" + 352 | "\t\t}\n" + 353 | "\t} \n" + 354 | "}") 355 | .endObject() 356 | .endObject(), randomIntBetween(5, 1000), null); 357 | 358 | int nbOfResultsToCheck = 100; 359 | 360 | SearchResponse response = client().prepareSearch(getDbName()) 361 | .addField("foo") 362 | .addField("content") 363 | .addField("_id") 364 | .setSize(nbOfResultsToCheck) 365 | .get(); 366 | 367 | for (int i=0; i < Math.min(response.getHits().getTotalHits(), nbOfResultsToCheck); i++) { 368 | SearchHit hit = response.getHits().getAt(i); 369 | int docId = Integer.parseInt(hit.getId()); 370 | 371 | assertThat(hit.field("foo"), notNullValue()); 372 | assertThat(hit.field("foo").getValue().toString(), is("bar")); 373 | assertThat(hit.field("_id"), nullValue()); 374 | if ((docId % 2) == 0) { 375 | assertThat(hit.field("content"), nullValue()); 376 | } else { 377 | assertThat(hit.field("content").getValue().toString(), is(hit.getId())); 378 | } 379 | } 380 | } 381 | 382 | /** 383 | * Test case for #66: https://github.com/elasticsearch/elasticsearch-river-couchdb/issues/66 384 | */ 385 | @Test 386 | public void testClosingWhileIndexing_66() throws IOException, InterruptedException { 387 | final int nbDocs = 10; 388 | logger.info(" -> Checking couchdb running"); 389 | CouchDBClient.checkCouchDbRunning(); 390 | logger.info(" -> Creating test database [{}]", getDbName()); 391 | CouchDBClient.dropAndCreateTestDatabase(getDbName()); 392 | 393 | logger.info(" -> Inserting [{}] docs in couchdb", nbDocs); 394 | for (int i = 0; i < nbDocs; i++) { 395 | CouchDBClient.putDocument(getDbName(), "" + i, "foo", "bar", "content", "" + i); 396 | } 397 | 398 | logger.info(" -> Create index"); 399 | try { 400 | createIndex(getDbName()); 401 | } catch (IndexAlreadyExistsException e) { 402 | // No worries. We already created the index before 403 | } 404 | 405 | logger.info(" -> Create river"); 406 | index("_river", getDbName(), "_meta", jsonBuilder() 407 | .startObject() 408 | .field("type", "couchdb") 409 | .startObject("couchdb") 410 | // We use here a script to have a chance to slow down the process and close the river while processing it 411 | .field("script", "for (int x = 0; x < 10000000; x++) { x*x*x } ;") 412 | .endObject() 413 | .startObject("index") 414 | .field("flush_interval", "100ms") 415 | .endObject() 416 | .endObject()); 417 | 418 | // Check that docs are indexed by the river 419 | assertThat(awaitBusy(new Predicate() { 420 | public boolean apply(Object obj) { 421 | try { 422 | refresh(); 423 | SearchResponse response = client().prepareSearch(getDbName()).get(); 424 | logger.info(" -> got {} docs in {} index", response.getHits().totalHits(), getDbName()); 425 | return response.getHits().totalHits() == nbDocs; 426 | } catch (IndexMissingException e) { 427 | return false; 428 | } 429 | } 430 | }, 1, TimeUnit.MINUTES), equalTo(true)); 431 | 432 | 433 | logger.info(" -> Inserting [{}] docs in couchdb", nbDocs); 434 | for (int i = nbDocs; i < 2*nbDocs; i++) { 435 | CouchDBClient.putDocument(getDbName(), "" + i, "foo", "bar", "content", "" + i); 436 | } 437 | 438 | // Check that docs are still processed by the river 439 | assertThat(awaitBusy(new Predicate() { 440 | public boolean apply(Object obj) { 441 | try { 442 | refresh(); 443 | SearchResponse response = client().prepareSearch(getDbName()).get(); 444 | logger.info(" -> got {} docs in {} index", response.getHits().totalHits(), getDbName()); 445 | return response.getHits().totalHits() > nbDocs; 446 | } catch (IndexMissingException e) { 447 | return false; 448 | } 449 | } 450 | }, 10, TimeUnit.SECONDS), equalTo(true)); 451 | 452 | logger.info(" -> Remove river while injecting"); 453 | client().prepareDelete("_river", getDbName(), "_meta").get(); 454 | 455 | logger.info(" -> Inserting [{}] docs in couchdb", nbDocs); 456 | for (int i = 2*nbDocs; i < 3*nbDocs; i++) { 457 | CouchDBClient.putDocument(getDbName(), "" + i, "foo", "bar", "content", "" + i); 458 | } 459 | 460 | // Check that docs are indexed by the river 461 | boolean foundAllDocs = awaitBusy(new Predicate() { 462 | public boolean apply(Object obj) { 463 | try { 464 | refresh(); 465 | SearchResponse response = client().prepareSearch(getDbName()).get(); 466 | logger.info(" -> got {} docs in {} index", response.getHits().totalHits(), getDbName()); 467 | return response.getHits().totalHits() == 3 * nbDocs; 468 | } catch (IndexMissingException e) { 469 | return false; 470 | } 471 | } 472 | }, 10, TimeUnit.SECONDS); 473 | 474 | // We should not have 30 documents at the end as we removed the river immediately after having 475 | // injecting 10 more docs in couchdb 476 | assertThat("We should not have 30 documents as the river is supposed to have been stopped!", foundAllDocs, is(false)); 477 | 478 | // We expect seeing a line in logs like: 479 | // [WARN ][org.elasticsearch.river.couchdb] [node_0] [couchdb][elasticsearch_couch_test_test_closing_while_indexing_66] river was closing while trying to index document [elasticsearch_couch_test_test_closing_while_indexing_66/elasticsearch_couch_test_test_closing_while_indexing_66/11]. Operation skipped. 480 | } 481 | 482 | /** 483 | * Test case for #17: https://github.com/elasticsearch/elasticsearch-river-couchdb/issues/17 484 | */ 485 | @Test 486 | public void testCreateCouchdbDatabaseWhileRunning_17() throws IOException, InterruptedException { 487 | final int nbDocs = between(50, 300); 488 | logger.info(" -> Checking couchdb running"); 489 | CouchDBClient.checkCouchDbRunning(); 490 | 491 | logger.info(" -> Create index"); 492 | try { 493 | createIndex(getDbName()); 494 | } catch (IndexAlreadyExistsException e) { 495 | // No worries. We already created the index before 496 | } 497 | 498 | logger.info(" -> Create river"); 499 | index("_river", getDbName(), "_meta", jsonBuilder() 500 | .startObject() 501 | .field("type", "couchdb") 502 | .endObject()); 503 | 504 | // Check that the river is started 505 | assertThat(awaitBusy(new Predicate() { 506 | public boolean apply(Object obj) { 507 | try { 508 | refresh(); 509 | GetResponse response = get("_river", getDbName(), "_status"); 510 | return response.isExists(); 511 | } catch (IndexMissingException e) { 512 | return false; 513 | } 514 | } 515 | }, 5, TimeUnit.SECONDS), equalTo(true)); 516 | 517 | logger.info(" -> Creating test database [{}]", getDbName()); 518 | CouchDBClient.dropAndCreateTestDatabase(getDbName()); 519 | 520 | logger.info(" -> Inserting [{}] docs in couchdb", nbDocs); 521 | for (int i = 0; i < nbDocs; i++) { 522 | CouchDBClient.putDocument(getDbName(), "" + i, "foo", "bar", "content", "" + i); 523 | } 524 | 525 | // Check that docs are still processed by the river 526 | assertThat(awaitBusy(new Predicate() { 527 | public boolean apply(Object obj) { 528 | try { 529 | refresh(); 530 | SearchResponse response = client().prepareSearch(getDbName()).get(); 531 | logger.info(" -> got {} docs in {} index", response.getHits().totalHits(), getDbName()); 532 | return response.getHits().totalHits() == nbDocs; 533 | } catch (IndexMissingException e) { 534 | return false; 535 | } 536 | } 537 | }, 1, TimeUnit.MINUTES), equalTo(true)); 538 | } 539 | 540 | /** 541 | * Test case for #71: https://github.com/elasticsearch/elasticsearch-river-couchdb/issues/71 542 | */ 543 | @Test 544 | public void testDropCouchdbDatabaseWhileRunning_71() throws IOException, InterruptedException { 545 | final int nbDocs = between(50, 300); 546 | launchTest(jsonBuilder() 547 | .startObject() 548 | .field("type", "couchdb") 549 | .endObject(), nbDocs, null); 550 | 551 | logger.info(" -> Removing test database [{}]", getDbName()); 552 | CouchDBClient.dropTestDatabase(getDbName()); 553 | 554 | // We wait for 10 seconds 555 | awaitBusy(new Predicate() { 556 | public boolean apply(Object obj) { 557 | return false; 558 | } 559 | }, 10, TimeUnit.SECONDS); 560 | } 561 | } 562 | -------------------------------------------------------------------------------- /src/main/java/org/elasticsearch/river/couchdb/CouchdbRiver.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.elasticsearch.river.couchdb; 21 | 22 | import org.apache.lucene.util.IOUtils; 23 | import org.elasticsearch.action.bulk.BulkItemResponse; 24 | import org.elasticsearch.action.bulk.BulkProcessor; 25 | import org.elasticsearch.action.bulk.BulkRequest; 26 | import org.elasticsearch.action.bulk.BulkResponse; 27 | import org.elasticsearch.action.delete.DeleteRequest; 28 | import org.elasticsearch.action.get.GetResponse; 29 | import org.elasticsearch.action.index.IndexRequest; 30 | import org.elasticsearch.client.Client; 31 | import org.elasticsearch.common.Base64; 32 | import org.elasticsearch.common.collect.Maps; 33 | import org.elasticsearch.common.inject.Inject; 34 | import org.elasticsearch.common.unit.TimeValue; 35 | import org.elasticsearch.common.util.concurrent.EsExecutors; 36 | import org.elasticsearch.common.xcontent.XContentBuilder; 37 | import org.elasticsearch.common.xcontent.XContentFactory; 38 | import org.elasticsearch.common.xcontent.XContentType; 39 | import org.elasticsearch.common.xcontent.support.XContentMapValues; 40 | import org.elasticsearch.river.*; 41 | import org.elasticsearch.script.ExecutableScript; 42 | import org.elasticsearch.script.Script; 43 | import org.elasticsearch.script.ScriptContext; 44 | import org.elasticsearch.script.ScriptService; 45 | 46 | import javax.net.ssl.HostnameVerifier; 47 | import javax.net.ssl.HttpsURLConnection; 48 | import javax.net.ssl.SSLSession; 49 | import java.io.*; 50 | import java.net.HttpURLConnection; 51 | import java.net.URL; 52 | import java.net.URLEncoder; 53 | import java.nio.charset.StandardCharsets; 54 | import java.util.List; 55 | import java.util.Map; 56 | import java.util.concurrent.ArrayBlockingQueue; 57 | import java.util.concurrent.BlockingQueue; 58 | import java.util.concurrent.LinkedTransferQueue; 59 | import java.util.concurrent.TimeUnit; 60 | 61 | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; 62 | 63 | /** 64 | * 65 | */ 66 | public class CouchdbRiver extends AbstractRiverComponent implements River { 67 | 68 | private final Client client; 69 | 70 | private final String riverIndexName; 71 | 72 | private final String couchProtocol; 73 | private final String couchHost; 74 | private final int couchPort; 75 | private final String couchDb; 76 | private final String couchFilter; 77 | private final String couchFilterParamsUrl; 78 | private final String basicAuth; 79 | private final boolean noVerify; 80 | private final boolean couchIgnoreAttachments; 81 | private final TimeValue heartbeat; 82 | private final TimeValue readTimeout; 83 | 84 | private final String indexName; 85 | private final String typeName; 86 | private final int bulkSize; 87 | private final TimeValue bulkTimeout; 88 | private final int throttleSize; 89 | 90 | private final ExecutableScript script; 91 | 92 | private volatile Thread slurperThread; 93 | private volatile Thread indexerThread; 94 | private volatile boolean closed; 95 | 96 | private final BlockingQueue stream; 97 | 98 | private final TimeValue bulkFlushInterval; 99 | private volatile BulkProcessor bulkProcessor; 100 | private final int maxConcurrentBulk; 101 | 102 | @SuppressWarnings({"unchecked"}) 103 | @Inject 104 | public CouchdbRiver(RiverName riverName, RiverSettings settings, @RiverIndexName String riverIndexName, Client client, ScriptService scriptService) { 105 | super(riverName, settings); 106 | this.riverIndexName = riverIndexName; 107 | this.client = client; 108 | 109 | if (settings.settings().containsKey("couchdb")) { 110 | Map couchSettings = (Map) settings.settings().get("couchdb"); 111 | couchProtocol = XContentMapValues.nodeStringValue(couchSettings.get("protocol"), "http"); 112 | noVerify = XContentMapValues.nodeBooleanValue(couchSettings.get("no_verify"), false); 113 | couchHost = XContentMapValues.nodeStringValue(couchSettings.get("host"), "localhost"); 114 | couchPort = XContentMapValues.nodeIntegerValue(couchSettings.get("port"), 5984); 115 | couchDb = XContentMapValues.nodeStringValue(couchSettings.get("db"), riverName.name()); 116 | couchFilter = XContentMapValues.nodeStringValue(couchSettings.get("filter"), null); 117 | if (couchSettings.containsKey("filter_params")) { 118 | Map filterParams = (Map) couchSettings.get("filter_params"); 119 | StringBuilder sb = new StringBuilder(); 120 | for (Map.Entry entry : filterParams.entrySet()) { 121 | try { 122 | sb.append("&").append(URLEncoder.encode(entry.getKey(), "UTF-8")).append("=").append(URLEncoder.encode(entry.getValue().toString(), "UTF-8")); 123 | } catch (UnsupportedEncodingException e) { 124 | // should not happen... 125 | } 126 | } 127 | couchFilterParamsUrl = sb.toString(); 128 | } else { 129 | couchFilterParamsUrl = null; 130 | } 131 | heartbeat = XContentMapValues.nodeTimeValue(couchSettings.get("heartbeat"), TimeValue.timeValueSeconds(10)); 132 | readTimeout = XContentMapValues.nodeTimeValue(couchSettings.get("read_timeout"), TimeValue.timeValueSeconds(heartbeat.getSeconds()*3)); 133 | couchIgnoreAttachments = XContentMapValues.nodeBooleanValue(couchSettings.get("ignore_attachments"), false); 134 | if (couchSettings.containsKey("user") && couchSettings.containsKey("password")) { 135 | String user = couchSettings.get("user").toString(); 136 | String password = couchSettings.get("password").toString(); 137 | basicAuth = "Basic " + Base64.encodeBytes((user + ":" + password).getBytes(StandardCharsets.UTF_8)); 138 | } else { 139 | basicAuth = null; 140 | } 141 | 142 | if (couchSettings.containsKey("script")) { 143 | String scriptType = "groovy"; 144 | if(couchSettings.containsKey("script_type")) { 145 | scriptType = couchSettings.get("script_type").toString(); 146 | } 147 | 148 | script = scriptService.executable( 149 | new Script(scriptType, couchSettings.get("script").toString(), ScriptService.ScriptType.INLINE, Maps.newHashMap()), 150 | ScriptContext.Standard.UPDATE); 151 | } else { 152 | script = null; 153 | } 154 | } else { 155 | couchProtocol = "http"; 156 | couchHost = "localhost"; 157 | couchPort = 5984; 158 | couchDb = riverName.name(); 159 | couchFilter = null; 160 | couchFilterParamsUrl = null; 161 | couchIgnoreAttachments = false; 162 | heartbeat = TimeValue.timeValueSeconds(10); 163 | readTimeout = TimeValue.timeValueSeconds(heartbeat.getSeconds()*3); 164 | noVerify = false; 165 | basicAuth = null; 166 | script = null; 167 | } 168 | 169 | if (settings.settings().containsKey("index")) { 170 | Map indexSettings = (Map) settings.settings().get("index"); 171 | indexName = XContentMapValues.nodeStringValue(indexSettings.get("index"), couchDb); 172 | typeName = XContentMapValues.nodeStringValue(indexSettings.get("type"), couchDb); 173 | bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size"), 100); 174 | if (indexSettings.containsKey("bulk_timeout")) { 175 | bulkTimeout = TimeValue.parseTimeValue(XContentMapValues.nodeStringValue(indexSettings.get("bulk_timeout"), "10ms"), TimeValue.timeValueMillis(10)); 176 | } else { 177 | bulkTimeout = TimeValue.timeValueMillis(10); 178 | } 179 | this.bulkFlushInterval = TimeValue.parseTimeValue(XContentMapValues.nodeStringValue( 180 | indexSettings.get("flush_interval"), "5s"), TimeValue.timeValueSeconds(5)); 181 | this.maxConcurrentBulk = XContentMapValues.nodeIntegerValue(indexSettings.get("max_concurrent_bulk"), 1); 182 | throttleSize = XContentMapValues.nodeIntegerValue(indexSettings.get("throttle_size"), bulkSize * 5); 183 | } else { 184 | indexName = couchDb; 185 | typeName = couchDb; 186 | bulkSize = 100; 187 | bulkTimeout = TimeValue.timeValueMillis(10); 188 | throttleSize = bulkSize * 5; 189 | this.maxConcurrentBulk = 1; 190 | this.bulkFlushInterval = TimeValue.timeValueSeconds(5); 191 | } 192 | if (throttleSize == -1) { 193 | stream = new LinkedTransferQueue(); 194 | } else { 195 | stream = new ArrayBlockingQueue(throttleSize); 196 | } 197 | } 198 | 199 | @Override 200 | public void start() { 201 | logger.info("starting couchdb stream: host [{}], port [{}], filter [{}], db [{}], indexing to [{}]/[{}]", couchHost, couchPort, couchFilter, couchDb, indexName, typeName); 202 | 203 | // Creating bulk processor 204 | this.bulkProcessor = BulkProcessor.builder(client, new BulkProcessor.Listener() { 205 | @Override 206 | public void beforeBulk(long executionId, BulkRequest request) { 207 | logger.debug("Going to execute new bulk composed of {} actions", request.numberOfActions()); 208 | } 209 | 210 | @Override 211 | public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { 212 | logger.debug("Executed bulk composed of {} actions", request.numberOfActions()); 213 | if (response.hasFailures()) { 214 | logger.warn("There was failures while executing bulk", response.buildFailureMessage()); 215 | if (logger.isDebugEnabled()) { 216 | for (BulkItemResponse item : response.getItems()) { 217 | if (item.isFailed()) { 218 | logger.debug("Error for {}/{}/{} for {} operation: {}", item.getIndex(), 219 | item.getType(), item.getId(), item.getOpType(), item.getFailureMessage()); 220 | } 221 | } 222 | } 223 | } 224 | } 225 | 226 | @Override 227 | public void afterBulk(long executionId, BulkRequest request, Throwable failure) { 228 | logger.warn("Error executing bulk", failure); 229 | } 230 | }) 231 | .setBulkActions(bulkSize) 232 | .setConcurrentRequests(maxConcurrentBulk) 233 | .setFlushInterval(bulkFlushInterval) 234 | .build(); 235 | 236 | 237 | slurperThread = EsExecutors.daemonThreadFactory(settings.globalSettings(), "couchdb_river_slurper").newThread(new Slurper()); 238 | indexerThread = EsExecutors.daemonThreadFactory(settings.globalSettings(), "couchdb_river_indexer").newThread(new Indexer()); 239 | indexerThread.start(); 240 | slurperThread.start(); 241 | } 242 | 243 | @Override 244 | public void close() { 245 | if (closed) { 246 | return; 247 | } 248 | logger.info("closing couchdb stream river"); 249 | if (slurperThread != null) { 250 | slurperThread.interrupt(); 251 | } 252 | if (indexerThread != null) { 253 | indexerThread.interrupt(); 254 | } 255 | 256 | closed = true; 257 | 258 | if (this.bulkProcessor != null) { 259 | this.bulkProcessor.close(); 260 | } 261 | } 262 | 263 | @SuppressWarnings({"unchecked"}) 264 | private Object processLine(String s) { 265 | Map ctx; 266 | try { 267 | ctx = XContentFactory.xContent(XContentType.JSON).createParser(s).mapAndClose(); 268 | } catch (IOException e) { 269 | logger.warn("failed to parse {}", e, s); 270 | return null; 271 | } 272 | if (ctx.containsKey("error")) { 273 | logger.warn("received error {}", s); 274 | return null; 275 | } 276 | Object seq = ctx.get("seq"); 277 | Object oId = ctx.get("id"); 278 | if (oId == null) { 279 | return null; 280 | } 281 | 282 | String id = oId.toString(); 283 | 284 | if (closed) { 285 | logger.warn("river was closing while processing couchdb doc [{}]. Operation skipped.", id); 286 | return null; 287 | } 288 | 289 | // Ignore design documents 290 | if (id.startsWith("_design/")) { 291 | if (logger.isTraceEnabled()) { 292 | logger.trace("ignoring design document {}", id); 293 | } 294 | return seq; 295 | } 296 | 297 | if (script != null) { 298 | script.setNextVar("ctx", ctx); 299 | try { 300 | script.run(); 301 | // we need to unwrap the ctx... 302 | ctx = (Map) script.unwrap(ctx); 303 | } catch (Exception e) { 304 | logger.warn("failed to script process {}, ignoring", e, ctx); 305 | return seq; 306 | } 307 | } 308 | 309 | if (ctx.containsKey("ignore") && ctx.get("ignore").equals(Boolean.TRUE)) { 310 | // ignore dock 311 | } else if (ctx.containsKey("deleted") && ctx.get("deleted").equals(Boolean.TRUE)) { 312 | String index = extractIndex(ctx); 313 | String type = extractType(ctx); 314 | if (logger.isTraceEnabled()) { 315 | logger.trace("processing [delete]: [{}]/[{}]/[{}]", index, type, id); 316 | } 317 | if (closed) { 318 | logger.warn("river was closing while trying to delete document [{}/{}/{}]. Operation skipped.", index, type, id); 319 | return null; 320 | } 321 | bulkProcessor.add(new DeleteRequest(index, type, id).routing(extractRouting(ctx)).parent(extractParent(ctx))); 322 | } else if (ctx.containsKey("doc")) { 323 | String index = extractIndex(ctx); 324 | String type = extractType(ctx); 325 | Map doc = (Map) ctx.get("doc"); 326 | 327 | // Remove _attachment from doc if needed 328 | if (couchIgnoreAttachments) { 329 | // no need to log that we removed it, the doc indexed will be shown without it 330 | doc.remove("_attachments"); 331 | } else { 332 | // TODO by now, couchDB river does not really store attachments but only attachments meta infomration 333 | // So we perhaps need to fully support attachments 334 | } 335 | 336 | if (logger.isTraceEnabled()) { 337 | logger.trace("processing [index ]: [{}]/[{}]/[{}], source {}", index, type, id, doc); 338 | } 339 | if (closed) { 340 | logger.warn("river was closing while trying to index document [{}/{}/{}]. Operation skipped.", index, type, id); 341 | return null; 342 | } 343 | bulkProcessor.add(new IndexRequest(index, type, id).source(doc).routing(extractRouting(ctx)).parent(extractParent(ctx))); 344 | } else { 345 | logger.warn("ignoring unknown change {}", s); 346 | } 347 | return seq; 348 | } 349 | 350 | private String extractParent(Map ctx) { 351 | return (String) ctx.get("_parent"); 352 | } 353 | 354 | private String extractRouting(Map ctx) { 355 | return (String) ctx.get("_routing"); 356 | } 357 | 358 | private String extractType(Map ctx) { 359 | String type = (String) ctx.get("_type"); 360 | if (type == null) { 361 | type = typeName; 362 | } 363 | return type; 364 | } 365 | 366 | private String extractIndex(Map ctx) { 367 | String index = (String) ctx.get("_index"); 368 | if (index == null) { 369 | index = indexName; 370 | } 371 | return index; 372 | } 373 | 374 | private class Indexer implements Runnable { 375 | @Override 376 | public void run() { 377 | while (true) { 378 | if (closed) { 379 | return; 380 | } 381 | String s; 382 | try { 383 | s = stream.take(); 384 | } catch (InterruptedException e) { 385 | if (closed) { 386 | return; 387 | } 388 | continue; 389 | } 390 | 391 | Object lastSeq = null; 392 | Object lineSeq = processLine(s); 393 | if (lineSeq != null) { 394 | lastSeq = lineSeq; 395 | } 396 | 397 | // spin a bit to see if we can get some more changes 398 | try { 399 | while ((s = stream.poll(bulkTimeout.millis(), TimeUnit.MILLISECONDS)) != null) { 400 | lineSeq = processLine(s); 401 | if (lineSeq != null) { 402 | lastSeq = lineSeq; 403 | } 404 | } 405 | } catch (InterruptedException e) { 406 | if (closed) { 407 | return; 408 | } 409 | } 410 | 411 | if (lastSeq != null) { 412 | try { 413 | // we always store it as a string 414 | String lastSeqAsString = null; 415 | if (lastSeq instanceof List) { 416 | // bigcouch uses array for the seq 417 | try { 418 | XContentBuilder builder = XContentFactory.jsonBuilder(); 419 | //builder.startObject(); 420 | builder.startArray(); 421 | for (Object value : ((List) lastSeq)) { 422 | builder.value(value); 423 | } 424 | builder.endArray(); 425 | //builder.endObject(); 426 | lastSeqAsString = builder.string(); 427 | } catch (Exception e) { 428 | logger.error("failed to convert last_seq to a json string", e); 429 | } 430 | } else { 431 | lastSeqAsString = lastSeq.toString(); 432 | } 433 | if (logger.isTraceEnabled()) { 434 | logger.trace("processing [_seq ]: [{}]/[{}]/[{}], last_seq [{}]", riverIndexName, riverName.name(), "_seq", lastSeqAsString); 435 | } 436 | if (closed) { 437 | logger.warn("river was closing while trying to update sequence [{}/{}/{}]. Operation skipped.", riverIndexName, riverName.name(), "_seq"); 438 | return; 439 | } 440 | bulkProcessor.add(new IndexRequest(riverIndexName, riverName.name(), "_seq") 441 | .source(jsonBuilder().startObject().startObject("couchdb").field("last_seq", lastSeqAsString).endObject().endObject())); 442 | } catch (IOException e) { 443 | logger.warn("failed to add last_seq entry to bulk indexing"); 444 | } 445 | } 446 | } 447 | } 448 | } 449 | 450 | 451 | private class Slurper implements Runnable { 452 | @SuppressWarnings({"unchecked"}) 453 | @Override 454 | public void run() { 455 | 456 | while (true) { 457 | if (closed) { 458 | return; 459 | } 460 | 461 | String lastSeq = null; 462 | try { 463 | client.admin().indices().prepareRefresh(riverIndexName).execute().actionGet(); 464 | GetResponse lastSeqGetResponse = client.prepareGet(riverIndexName, riverName().name(), "_seq").execute().actionGet(); 465 | if (lastSeqGetResponse.isExists()) { 466 | Map couchdbState = (Map) lastSeqGetResponse.getSourceAsMap().get("couchdb"); 467 | if (couchdbState != null) { 468 | lastSeq = couchdbState.get("last_seq").toString(); // we know its always a string 469 | } 470 | } 471 | } catch (Exception e) { 472 | logger.warn("failed to get last_seq, throttling....", e); 473 | try { 474 | Thread.sleep(5000); 475 | continue; 476 | } catch (InterruptedException e1) { 477 | if (closed) { 478 | return; 479 | } 480 | } 481 | } 482 | 483 | String file = "/" + couchDb + "/_changes?feed=continuous&include_docs=true&heartbeat=" + heartbeat.getMillis(); 484 | if (couchFilter != null) { 485 | try { 486 | file = file + "&filter=" + URLEncoder.encode(couchFilter, "UTF-8"); 487 | } catch (UnsupportedEncodingException e) { 488 | // should not happen! 489 | } 490 | if (couchFilterParamsUrl != null) { 491 | file = file + couchFilterParamsUrl; 492 | } 493 | } 494 | 495 | if (lastSeq != null) { 496 | try { 497 | file = file + "&since=" + URLEncoder.encode(lastSeq, "UTF-8"); 498 | } catch (UnsupportedEncodingException e) { 499 | // should not happen, but in any case... 500 | file = file + "&since=" + lastSeq; 501 | } 502 | } 503 | 504 | if (logger.isDebugEnabled()) { 505 | logger.debug("using host [{}], port [{}], path [{}]", couchHost, couchPort, file); 506 | } 507 | 508 | HttpURLConnection connection = null; 509 | InputStream is = null; 510 | try { 511 | URL url = new URL(couchProtocol, couchHost, couchPort, file); 512 | connection = (HttpURLConnection) url.openConnection(); 513 | if (basicAuth != null) { 514 | connection.addRequestProperty("Authorization", basicAuth); 515 | } 516 | connection.setDoInput(true); 517 | connection.setReadTimeout((int) readTimeout.getMillis()); 518 | connection.setUseCaches(false); 519 | 520 | if (noVerify) { 521 | ((HttpsURLConnection) connection).setHostnameVerifier( 522 | new HostnameVerifier() { 523 | public boolean verify(String string, SSLSession ssls) { 524 | return true; 525 | } 526 | } 527 | ); 528 | } 529 | 530 | is = connection.getInputStream(); 531 | 532 | final BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); 533 | String line; 534 | while ((line = reader.readLine()) != null) { 535 | if (closed) { 536 | return; 537 | } 538 | if (line.length() == 0) { 539 | logger.trace("[couchdb] heartbeat"); 540 | continue; 541 | } 542 | if (logger.isTraceEnabled()) { 543 | logger.trace("[couchdb] {}", line); 544 | } 545 | // we put here, so we block if there is no space to add 546 | stream.put(line); 547 | } 548 | } catch (Exception e) { 549 | IOUtils.closeWhileHandlingException(is); 550 | if (connection != null) { 551 | try { 552 | connection.disconnect(); 553 | } catch (Exception e1) { 554 | // ignore 555 | } finally { 556 | connection = null; 557 | } 558 | } 559 | if (closed) { 560 | return; 561 | } 562 | logger.warn("failed to read from _changes, throttling....", e); 563 | try { 564 | Thread.sleep(5000); 565 | } catch (InterruptedException e1) { 566 | if (closed) { 567 | return; 568 | } 569 | } 570 | } finally { 571 | IOUtils.closeWhileHandlingException(is); 572 | if (connection != null) { 573 | try { 574 | connection.disconnect(); 575 | } catch (Exception e1) { 576 | // ignore 577 | } finally { 578 | connection = null; 579 | } 580 | } 581 | } 582 | } 583 | } 584 | } 585 | } 586 | --------------------------------------------------------------------------------