├── .gitignore ├── LICENSE ├── README.md ├── pom.xml └── src ├── main └── java │ └── ru │ └── yandex │ └── qatools │ └── embed │ └── service │ ├── AbstractElasticEmbeddedService.java │ ├── AbstractEmbeddedService.java │ ├── ElasticMongoIndexingService.java │ ├── ElasticPostgresIndexingService.java │ ├── EmbeddedService.java │ ├── IndexingService.java │ ├── LogWatchStreamProcessor.java │ ├── MongoEmbeddedService.java │ ├── PostgresEmbeddedService.java │ └── beans │ └── IndexingResult.java └── test ├── java └── ru │ └── yandex │ └── qatools │ └── embed │ └── service │ ├── ElasticMongoExcludingFieldsTest.java │ ├── ElasticMongoIndexingServiceTest.java │ ├── ElasticPostgresIndexingServiceTest.java │ ├── IndexingServiceMatcher.java │ ├── MongoDBEmbeddedDoubleStartTest.java │ ├── MongoDBEmbeddedWithoutAuthTest.java │ ├── db │ ├── Database.java │ ├── MorphiaDBService.java │ ├── PostDAO.java │ ├── PostMongo.java │ ├── PostPostgres.java │ ├── UserDAO.java │ ├── UserDetailMongo.java │ ├── UserDetailMongoLong.java │ ├── UserDetailMongoString.java │ └── UserMongo.java │ └── util │ └── SocketUtil.java └── resources ├── db └── migration │ └── V1__create_tables.sql └── log4j.properties /.gitignore: -------------------------------------------------------------------------------- 1 | # Package Files # 2 | *.jar 3 | *.war 4 | *.ear 5 | 6 | target 7 | .idea 8 | *.iml 9 | *.ipr 10 | *.iws 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2015 YANDEX 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Embedded services 2 | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/ru.yandex.qatools.embed/embedded-services/badge.svg?style=flat)](https://maven-badges.herokuapp.com/maven-central/ru.yandex.qatools.embed/embedded-services) [![covarage](https://img.shields.io/sonar/http/sonar.qatools.ru/ru.yandex.qatools.embed:embedded-services/coverage.svg?style=flat)](http://sonar.qatools.ru/dashboard/index/887) 3 | 4 | This project allows you to easily start your project with the embedded database (PostgreSQL, MongoDB) services and connect 5 | them with the embedded ElasticSearch instance for full text indexing and search. 6 | 7 | ## Why? 8 | 9 | It's very easy to incorporate the embedded MongoDB/PostgreSQL within your test process. 10 | 11 | ### Maven 12 | 13 | Add the following dependency to your pom.xml: 14 | ```xml 15 | 16 | ru.yandex.qatools.embed 17 | embedded-services 18 | 1.21 19 | 20 | ``` 21 | ## How to run embedded MongoDB with ElasticSearch 22 | 23 | ```java 24 | // Starting the embedded services within temporary dir 25 | MongoEmbeddedService mongo = new MongoEmbeddedService( 26 | "localhost:27017", "dbname", "username", "password", "localreplica" 27 | ); 28 | mongo.start(); 29 | ElasticMongoIndexingService elastic = new ElasticMongoIndexingService( 30 | "localhost:27017", "dbname", "username", "password" 31 | ); 32 | elastic.start(); 33 | 34 | // Indexing collection `posts` 35 | elastic.addToIndex("posts"); 36 | 37 | // Searching within collection `posts` using Elastic (IndexingResult contains id of each post) 38 | List posts = elastic.search("posts", "body:(lorem AND NOT ipsum)") 39 | ``` 40 | 41 | ## How to run embedded PostgreSQL with ElasticSearch 42 | 43 | ```java 44 | // Starting the embedded services within temporary dir 45 | PostgresEmbeddedService postgres = new PostgresEmbeddedService( 46 | "localhost", 5429, "username", "password", "dbname" 47 | ); 48 | postgres.start(); 49 | ElasticPostgresIndexingService elastic = new ElasticPostgresIndexingService( 50 | Driver.class, "postgresql", "", "localhost", 5429, "username", "password", "dbname" 51 | ); 52 | elastic.start(); 53 | 54 | // Indexing table `posts` 55 | elastic.addToIndex("posts"); 56 | 57 | // Searching within table `posts` using Elastic (IndexingResult contains id of each post) 58 | List posts = elastic.search("posts", "body:(lorem AND NOT ipsum)") 59 | ``` 60 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | ru.yandex.qatools.embed 6 | embedded-services 7 | 1.22-SNAPSHOT 8 | jar 9 | Embedded Services 10 | 11 | 12 | org.sonatype.oss 13 | oss-parent 14 | 9 15 | 16 | 17 | 18 | Yandex 19 | http://company.yandex.com 20 | 21 | 22 | 23 | The Apache Software License, Version 2.0 24 | http://www.apache.org/licenses/LICENSE-2.0.txt 25 | repo 26 | 27 | 28 | 29 | GitHub Issues 30 | https://github.com/yandex-qatools/embedded-services/issues 31 | 32 | 33 | 34 | scm:git:git@github.com:yandex-qatools/embedded-services.git 35 | scm:git:git@github.com:yandex-qatools/embedded-services.git 36 | git@github.com:yandex-qatools/embedded-services.git 37 | HEAD 38 | 39 | 40 | 41 | 42 | xbib.org 43 | http://xbib.org/repository 44 | 45 | 46 | 47 | 48 | 1.7 49 | UTF-8 50 | 1.7.2 51 | 2.13.2 52 | 1.7.7 53 | 2.0.9 54 | 55 | 56 | 57 | 58 | 59 | commons-io 60 | commons-io 61 | 2.4 62 | 63 | 64 | org.jodd 65 | jodd 66 | 3.3.8 67 | 68 | 69 | org.slf4j 70 | slf4j-api 71 | ${sl4j.version} 72 | 73 | 74 | org.slf4j 75 | slf4j-log4j12 76 | ${sl4j.version} 77 | 78 | 79 | 80 | 81 | de.flapdoodle.embed 82 | de.flapdoodle.embed.mongo 83 | 1.50.2 84 | 85 | 86 | slf4j-api 87 | org.slf4j 88 | 89 | 90 | 91 | 92 | org.mongodb 93 | mongo-java-driver 94 | ${mongo-java-driver.version} 95 | 96 | 97 | 98 | 99 | ru.yandex.qatools.embed 100 | postgresql-embedded 101 | 1.11 102 | 103 | 104 | 105 | 106 | 107 | org.elasticsearch 108 | elasticsearch 109 | ${elasticsearch.version} 110 | 111 | 112 | org.codehaus.groovy 113 | groovy-all 114 | 2.4.3 115 | 116 | 117 | org.apache.lucene 118 | lucene-expressions 119 | 5.2.1 120 | 121 | 122 | com.github.richardwilly98.elasticsearch 123 | elasticsearch-river-mongodb 124 | ${elasticsearch-river-mongodb.version} 125 | 126 | 127 | elasticsearch 128 | org.elasticsearch 129 | 130 | 131 | mongo-java-driver 132 | org.mongodb 133 | 134 | 135 | 136 | 137 | org.xbib.elasticsearch.plugin 138 | elasticsearch-river-jdbc 139 | 1.5.0.0 140 | 141 | 142 | elasticsearch 143 | org.elasticsearch 144 | 145 | 146 | 147 | 148 | 149 | 150 | junit 151 | junit 152 | 4.11 153 | test 154 | 155 | 156 | org.mockito 157 | mockito-all 158 | 1.9.5 159 | test 160 | 161 | 162 | org.hamcrest 163 | hamcrest-all 164 | 1.3 165 | test 166 | 167 | 168 | matcher-decorators 169 | ru.yandex.qatools.matchers 170 | 1.0 171 | test 172 | 173 | 174 | org.mongodb.morphia 175 | morphia 176 | 0.106 177 | 178 | 179 | org.mongodb 180 | mongo-java-driver 181 | 182 | 183 | test 184 | 185 | 186 | 187 | org.javalite 188 | activejdbc 189 | 1.4.9 190 | test 191 | 192 | 193 | slf4j-api 194 | org.slf4j 195 | 196 | 197 | 198 | 199 | org.postgresql 200 | postgresql 201 | 9.3-1100-jdbc41 202 | test 203 | 204 | 205 | org.flywaydb 206 | flyway-core 207 | 3.0 208 | test 209 | 210 | 211 | com.googlecode.lambdaj 212 | lambdaj 213 | 2.3.3 214 | test 215 | 216 | 217 | 218 | com.zaxxer 219 | HikariCP-java6 220 | 2.2.5 221 | test 222 | 223 | 224 | slf4j-api 225 | org.slf4j 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | org.apache.maven.wagon 235 | wagon-ssh 236 | 2.5 237 | 238 | 239 | 240 | 241 | ${project.basedir}/src/test/resources 242 | true 243 | 244 | 245 | 246 | 247 | org.apache.maven.plugins 248 | maven-source-plugin 249 | 2.2.1 250 | 251 | 252 | 253 | jar 254 | 255 | 256 | 257 | 258 | 259 | org.apache.maven.plugins 260 | maven-compiler-plugin 261 | 2.3.2 262 | 263 | ${compiler.version} 264 | ${compiler.version} 265 | 266 | 267 | 268 | org.apache.maven.plugins 269 | maven-release-plugin 270 | 2.5 271 | 272 | deploy -DskipTests=true 273 | 274 | 275 | 276 | org.apache.maven.scm 277 | maven-scm-api 278 | 1.9.1 279 | 280 | 281 | org.apache.maven.scm 282 | maven-scm-provider-gitexe 283 | 1.9.1 284 | 285 | 286 | 287 | 288 | org.javalite 289 | activejdbc-instrumentation 290 | 1.4.9 291 | 292 | 293 | process-test-classes 294 | 295 | instrument 296 | 297 | 298 | 299 | ${project.basedir}/target/test-classes 300 | 301 | 302 | 303 | 304 | 305 | 306 | org.javalite 307 | activejdbc 308 | 1.4.9 309 | 310 | 311 | 312 | 313 | 314 | 315 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/qatools/embed/service/AbstractElasticEmbeddedService.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service; 2 | 3 | import org.elasticsearch.action.ActionListener; 4 | import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; 5 | import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; 6 | import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; 7 | import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; 8 | import org.elasticsearch.action.count.CountResponse; 9 | import org.elasticsearch.action.search.SearchResponse; 10 | import org.elasticsearch.client.Client; 11 | import org.elasticsearch.common.settings.ImmutableSettings; 12 | import org.elasticsearch.common.xcontent.XContentBuilder; 13 | import org.elasticsearch.index.query.QueryBuilder; 14 | import org.elasticsearch.node.Node; 15 | import org.elasticsearch.search.SearchHit; 16 | import ru.yandex.qatools.embed.service.beans.IndexingResult; 17 | 18 | import java.io.IOException; 19 | import java.util.*; 20 | import java.util.concurrent.ConcurrentHashMap; 21 | 22 | import static java.lang.String.format; 23 | import static java.util.Collections.newSetFromMap; 24 | import static org.apache.commons.lang3.StringUtils.join; 25 | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; 26 | import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; 27 | import static org.elasticsearch.node.NodeBuilder.nodeBuilder; 28 | 29 | /** 30 | * @author Ilya Sadykov 31 | */ 32 | public abstract class AbstractElasticEmbeddedService extends AbstractEmbeddedService implements IndexingService { 33 | public static final int MAP_REQ_DELAY_MS = 100; 34 | protected final String dbName; 35 | protected final Set indexedCollections = newSetFromMap(new ConcurrentHashMap()); 36 | protected final Map settings = new HashMap<>(); 37 | protected volatile Node node; 38 | 39 | public AbstractElasticEmbeddedService(String dbName, String dataDirectory, boolean enabled, int initTimeout) throws IOException { 40 | super(dataDirectory, enabled, initTimeout); 41 | this.dbName = dbName; 42 | 43 | // Initializing the defaults 44 | settings.put("http.enabled", "false"); 45 | settings.put("path.home", this.dataDirectory); 46 | settings.put("threadpool.bulk.queue_size", 5000); 47 | settings.put("path.data", this.dataDirectory + "/data"); 48 | settings.put("path.logs", this.dataDirectory + "/logs"); 49 | } 50 | 51 | protected abstract void indexAllCollections() throws IOException; 52 | 53 | protected abstract void indexCollection(String collectionName) throws IOException; 54 | 55 | public void setBindHost(String host) { 56 | settings.put("network.bind_host", host); 57 | settings.put("network.publish_host", host); 58 | settings.put("network.host", host); 59 | } 60 | 61 | @Override 62 | public void doStart() { 63 | ImmutableSettings.Builder elasticsearchSettings = ImmutableSettings.settingsBuilder(); 64 | for (String key : settings.keySet()) { 65 | elasticsearchSettings.put(key, String.valueOf(settings.get(key))); 66 | } 67 | this.node = nodeBuilder().local(true).settings(elasticsearchSettings.build()).node(); 68 | node.client().admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(initTimeout); 69 | } 70 | 71 | @Override 72 | public void doStop() { 73 | if (node != null) { 74 | node.close(); 75 | node = null; 76 | } 77 | } 78 | 79 | @Override 80 | public List search(Class modelClass, String value) { 81 | return search(collectionName(modelClass), value); 82 | } 83 | 84 | @Override 85 | public List search(String collectionName, String value) { 86 | final List results = new ArrayList<>(); 87 | if (enabled) { 88 | logger.debug(format("Searching for '%s' in collection '%s' ...", value, collectionName)); 89 | final SearchResponse resp = search(collectionName, queryStringQuery(value)); 90 | for (SearchHit hit : resp.getHits()) { 91 | results.add(new IndexingResult(hit.getId(), hit.score(), hit.getSource())); 92 | } 93 | logger.debug(format("Search for '%s' in collection '%s' gave %d results...", 94 | value, collectionName, results.size())); 95 | } 96 | return results; 97 | } 98 | 99 | @Override 100 | public void indexAll() { 101 | try { 102 | indexAllCollections(); 103 | } catch (IOException e) { 104 | throw new RuntimeException("Failed to index all collections", e); 105 | } 106 | } 107 | 108 | @Override 109 | public void addToIndex(Class modelClass) { 110 | addToIndex(collectionName(modelClass)); 111 | } 112 | 113 | @Override 114 | public void addToIndex(String collectionName) { 115 | try { 116 | if (indexedCollections.contains(collectionName)) { 117 | logger.debug(format("Skipping collection '%s' indexing: it is already added to index!", collectionName)); 118 | return; 119 | } 120 | indexedCollections.add(collectionName); 121 | logger.debug(format("Adding collection '%s' to the embedded ElasticSearch index...", collectionName)); 122 | indexCollection(collectionName); 123 | } catch (IOException e) { 124 | throw new RuntimeException("Failed to index collection", e); 125 | } 126 | } 127 | 128 | @Override 129 | public void initSettings(final Map settings, 130 | final Map> typedFields) { 131 | initSettings(settings, typedFields, null); 132 | } 133 | 134 | @Override 135 | public void initSettings(final Map settings, 136 | final Map> typedFields, final Runnable callback) { 137 | this.settings.putAll(settings); 138 | getClient().admin().indices().exists(new IndicesExistsRequest(dbName), new ActionListener() { 139 | @Override 140 | public void onResponse(IndicesExistsResponse response) { 141 | if (!response.isExists()) { 142 | createIndex(settings); 143 | } 144 | updateMappings(typedFields, callback); 145 | } 146 | 147 | @Override 148 | public void onFailure(Throwable e) { 149 | logger.error("Failed to check index existence {}", dbName, e); 150 | } 151 | }); 152 | } 153 | 154 | protected void createIndex(final Map settings) { 155 | if (enabled) { 156 | try { 157 | if (settings.isEmpty()) { 158 | logger.info("Database {} skipping settings configuration: empty", dbName); 159 | return; 160 | } 161 | try { 162 | try { 163 | IndicesExistsResponse existsResp = getClient().admin().indices().prepareExists(dbName) 164 | .execute().actionGet(initTimeout); 165 | if (existsResp.isExists()) { 166 | logger.info("Index exists {}, removing...", dbName); 167 | getClient().admin().indices().prepareDelete(dbName).execute().actionGet(initTimeout); 168 | } else { 169 | logger.info("Index does not exists {}, skipping remove...", dbName); 170 | } 171 | } catch (Exception e) { 172 | logger.error("Failed to recreate index {}", dbName, e); 173 | } 174 | 175 | logger.info("Creating settings index {}...", dbName); 176 | final Map config = new HashMap<>(); 177 | Map currentMap = config; 178 | for (String fieldPath : settings.keySet()) { 179 | String[] parts = fieldPath.split("\\."); 180 | for (int i = 0; i < parts.length; ++i) { 181 | if ((i < parts.length - 1)) { 182 | currentMap.put(parts[i], (currentMap = new HashMap<>())); 183 | } else { 184 | currentMap.put(parts[i], settings.get(fieldPath)); 185 | } 186 | } 187 | } 188 | getClient().admin().indices().prepareCreate(dbName).setSettings(config).execute().actionGet(initTimeout); 189 | logger.info("Settings index {} creation sent to ES", dbName); 190 | } catch (Exception e) { 191 | logger.error("Failed to create settings index {}", dbName, e); 192 | } 193 | logger.info("Database {} settings requests sent to ES", dbName); 194 | } catch (Exception e) { 195 | logger.error("Failed to setup settings for db {}", dbName, e); 196 | } 197 | } 198 | 199 | } 200 | 201 | @Override 202 | public void updateMappings(final Map> typedFields, final Runnable callback) { 203 | if (enabled) { 204 | try { 205 | if (typedFields.isEmpty()) { 206 | logger.info("Database {} skipping mapping configuration", dbName); 207 | if (callback != null) { 208 | callback.run(); 209 | } 210 | return; 211 | } 212 | sleepBetweenRequests(); 213 | IndicesExistsResponse existsResp = getClient().admin().indices().prepareExists(dbName).execute().actionGet(initTimeout); 214 | if (existsResp.isExists()) { 215 | logger.info("Getting existing mappings {}...", dbName); 216 | GetMappingsResponse mappingsResp = getClient().admin().indices().prepareGetMappings(dbName) 217 | .execute().actionGet(initTimeout); 218 | sleepBetweenRequests(); 219 | for (String fieldPath : typedFields.keySet()) { 220 | String[] parts = fieldPath.split("\\."); 221 | String type = parts[0]; 222 | logger.info("Checking index mapping existence {}/{}...", dbName, fieldPath); 223 | if (mappingsResp.getMappings().containsKey(type)) { 224 | logger.info("Mapping index already exists {}/{}...", dbName, fieldPath); 225 | try { 226 | logger.info("Deleting mapping index {}/{}...", dbName, fieldPath); 227 | getClient().admin().indices().prepareDeleteMapping(dbName).setType(type).execute().actionGet(initTimeout); 228 | } catch (Exception e) { 229 | logger.error("Failed to delete mapping index {}/{}", dbName, fieldPath, e); 230 | } 231 | sleepBetweenRequests(); 232 | } 233 | try { 234 | logger.info("Creating mapping index {}/{}...", dbName, fieldPath); 235 | final PutMappingRequestBuilder putBuilder = getClient().admin().indices().preparePutMapping(dbName); 236 | final XContentBuilder config = jsonBuilder() 237 | .startObject(). 238 | startObject(type); 239 | boolean idMapping = (parts.length == 2 && parts[1].equals("_id")); 240 | for (int i = 1; i < parts.length; ++i) { 241 | if (idMapping) { 242 | config 243 | .startObject("_id"); 244 | } else { 245 | config 246 | .startObject("properties") 247 | .startObject(parts[i]); 248 | } 249 | final Map opts = typedFields.get(fieldPath); 250 | if (i < parts.length - 1) { 251 | config.field("type", "nested"); 252 | } else { 253 | for (String opt : opts.keySet()) { 254 | config.field(opt, opts.get(opt)); 255 | } 256 | } 257 | } 258 | for (String part : parts) { 259 | config.endObject(); 260 | if (!idMapping) { 261 | config.endObject(); 262 | } 263 | } 264 | 265 | putBuilder.setType(type).setSource(config); 266 | putBuilder.execute().actionGet(initTimeout); 267 | logger.info("Mapping index {}/{}: {} creation sent to ES", dbName, fieldPath, join(typedFields.get(fieldPath))); 268 | } catch (Exception e) { 269 | logger.error("Failed to create mapping index {}/{}", dbName, fieldPath, e); 270 | } 271 | 272 | sleepBetweenRequests(); 273 | } 274 | logger.info("Database {} mapping requests sent to ES", dbName); 275 | if (callback != null) { 276 | callback.run(); 277 | } 278 | } else { 279 | logger.info("Index does not exists {}, skipping mappings...", dbName); 280 | } 281 | } catch (Exception e) { 282 | logger.error("Failed to setup mappings for db {}", dbName, e); 283 | } 284 | } 285 | } 286 | 287 | private void sleepBetweenRequests() { 288 | try { 289 | logger.debug("Sleeping for a while to not stress the ES..."); 290 | Thread.sleep(MAP_REQ_DELAY_MS); 291 | } catch (InterruptedException e) { 292 | logger.warn("Failed to sleep before the next ES request...", e); 293 | } 294 | } 295 | 296 | public Client getClient() { 297 | return node.client(); 298 | } 299 | 300 | protected SearchResponse search(String collectionName, QueryBuilder query) { 301 | final CountResponse count = count(collectionName, query); 302 | return getClient().prepareSearch().setTypes(collectionName) 303 | .setQuery(query) 304 | .setSize((int) count.getCount()) 305 | .addFields("id") 306 | .execute() 307 | .actionGet(initTimeout); 308 | } 309 | 310 | protected CountResponse count(String collectionName, QueryBuilder query) { 311 | return getClient().prepareCount() 312 | .setTypes(collectionName) 313 | .setQuery(query) 314 | .execute() 315 | .actionGet(initTimeout); 316 | } 317 | 318 | @Override 319 | public String collectionName(Class modelClass) { 320 | return modelClass.getSimpleName().toLowerCase(); 321 | } 322 | 323 | 324 | } 325 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/qatools/embed/service/AbstractEmbeddedService.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.io.File; 7 | import java.io.IOException; 8 | import java.nio.file.DirectoryStream; 9 | import java.nio.file.Files; 10 | import java.nio.file.Path; 11 | import java.nio.file.Paths; 12 | 13 | import static jodd.io.FileUtil.createTempDirectory; 14 | import static jodd.io.FileUtil.deleteDir; 15 | import static org.apache.commons.lang3.StringUtils.isEmpty; 16 | 17 | /** 18 | * @author Ilya Sadykov 19 | */ 20 | public abstract class AbstractEmbeddedService implements EmbeddedService { 21 | protected final Logger logger = LoggerFactory.getLogger(getClass()); 22 | protected final String dataDirectory; 23 | protected final int initTimeout; 24 | protected final boolean removeDataDir; 25 | protected final boolean enabled; 26 | protected final boolean newDirectory; 27 | protected volatile boolean stopped = false; 28 | protected volatile boolean started = false; 29 | 30 | public AbstractEmbeddedService(String dataDirectory, boolean enabled, int initTimeout) throws IOException { 31 | this.enabled = enabled; 32 | this.initTimeout = initTimeout; 33 | 34 | if (isEmpty(dataDirectory) || dataDirectory.equals("TMP")) { 35 | this.removeDataDir = true; 36 | final String prefix = getClass().getName().substring( 37 | getClass().getName().lastIndexOf(".") + 1 38 | ); 39 | this.dataDirectory = createTempDirectory(prefix, "data").getPath(); 40 | this.newDirectory = true; 41 | } else { 42 | this.dataDirectory = dataDirectory; 43 | this.removeDataDir = false; 44 | this.newDirectory = !new File(dataDirectory).exists() || isDatadirEmpty(); 45 | } 46 | } 47 | 48 | 49 | protected abstract void doStart() throws Exception; 50 | 51 | protected abstract void doStop() throws Exception; 52 | 53 | public boolean isDatadirEmpty() { 54 | try (DirectoryStream dirStream = Files.newDirectoryStream(Paths.get(dataDirectory))) { 55 | return !dirStream.iterator().hasNext(); 56 | } catch (IOException e) { 57 | logger.error("Failed to check for data directory emptiness!", e); 58 | return false; 59 | } 60 | } 61 | 62 | @Override 63 | public boolean isStarted() { 64 | return started; 65 | } 66 | 67 | @Override 68 | public void start() { 69 | if (this.enabled) { 70 | try { 71 | logger.info("Starting the embedded service..."); 72 | doStart(); 73 | started = true; 74 | } catch (Exception e) { 75 | logger.error("Failed to start embedded service", e); 76 | } 77 | } 78 | } 79 | 80 | @Override 81 | public void stop() { 82 | if (!stopped) { 83 | logger.info("Shutting down the embedded service..."); 84 | started = false; 85 | try { 86 | doStop(); 87 | stopped = true; 88 | if (removeDataDir) { 89 | try { 90 | deleteDir(new File(dataDirectory)); 91 | } catch (Exception e) { 92 | logger.error("Failed to remove data dir", e); 93 | } 94 | } 95 | } catch (Exception e) { 96 | logger.error("Failed to stop service", e); 97 | } 98 | } 99 | 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/qatools/embed/service/ElasticMongoIndexingService.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service; 2 | 3 | import org.elasticsearch.common.xcontent.XContentBuilder; 4 | 5 | import java.io.IOException; 6 | 7 | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; 8 | 9 | public class ElasticMongoIndexingService extends AbstractElasticEmbeddedService { 10 | 11 | private final String mongoReplicaSet; 12 | private final String username; 13 | private final String password; 14 | private final boolean enabled; 15 | 16 | public ElasticMongoIndexingService(String mongoReplicaSet, String mongoDatabaseName, 17 | String mongoUsername, String mongoPassword) throws IOException { 18 | this(mongoReplicaSet, mongoDatabaseName, mongoUsername, mongoPassword, null, true, 10000); 19 | } 20 | 21 | public ElasticMongoIndexingService( 22 | String mongoReplicaSet, 23 | String mongoDatabaseName, 24 | String mongoUsername, 25 | String mongoPassword, 26 | String dataDirectory, 27 | boolean enabled, 28 | int initTimeout 29 | ) throws IOException { 30 | super(mongoDatabaseName, dataDirectory, enabled, initTimeout); 31 | this.mongoReplicaSet = mongoReplicaSet; 32 | this.enabled = enabled; 33 | this.username = mongoUsername; 34 | this.password = mongoPassword; 35 | } 36 | 37 | 38 | protected void addIndexAllCollectionsOptions(XContentBuilder config) { 39 | } 40 | 41 | @Override 42 | protected void indexAllCollections() throws IOException { 43 | if (enabled) { 44 | final XContentBuilder config =jsonBuilder() 45 | .startObject() 46 | .field("type", "mongodb") 47 | .startObject("mongodb"); 48 | config 49 | .startArray("servers"); 50 | for (String replSetEl : mongoReplicaSet.split(",")) { 51 | final String[] hostPort = replSetEl.split(":"); 52 | config 53 | .startObject() 54 | .field("host", hostPort[0]) 55 | .field("port", Integer.parseInt(hostPort[1])) 56 | .endObject(); 57 | } 58 | config 59 | .endArray(); 60 | config 61 | .startArray("credentials") 62 | .startObject() 63 | .field("db", "local") 64 | .field("auth", dbName) 65 | .field("user", username) 66 | .field("password", password) 67 | .endObject() 68 | .startObject() 69 | .field("db", dbName) 70 | .field("auth", dbName) 71 | .field("user", username) 72 | .field("password", password) 73 | .endObject() 74 | .endArray(); 75 | config 76 | .field("db", dbName) 77 | .field("gridfs", false) 78 | .startObject("options") 79 | .field("secondary_read_preference", false) 80 | .field("drop_collection", true) 81 | .field("is_mongos", false) 82 | .field("import_all_collections", true); 83 | addIndexAllCollectionsOptions(config); 84 | config 85 | .endObject() 86 | .endObject() 87 | .startObject("index") 88 | .field("name", dbName) 89 | .field("bulk_size", "1000") 90 | .field("bulk_timeout", "30") 91 | .endObject() 92 | .endObject(); 93 | getClient().prepareIndex("_river", dbName, "_meta").setSource(config) 94 | .execute().actionGet(initTimeout); 95 | logger.info("Collection {}.* indexing request sent to ES", dbName); 96 | } 97 | 98 | } 99 | 100 | @Override 101 | protected synchronized void indexCollection(String collectionName) throws IOException { 102 | if (enabled) { 103 | final XContentBuilder config = 104 | jsonBuilder() 105 | .startObject() 106 | .field("type", "mongodb") 107 | .startObject("mongodb"); 108 | config 109 | .startArray("servers"); 110 | for (String replSetEl : mongoReplicaSet.split(",")) { 111 | final String[] hostPort = replSetEl.split(":"); 112 | config 113 | .startObject() 114 | .field("host", hostPort[0]) 115 | .field("port", Integer.parseInt(hostPort[1])) 116 | .endObject(); 117 | } 118 | config 119 | .endArray(); 120 | config 121 | .startArray("credentials") 122 | .startObject() 123 | .field("db", "local") 124 | .field("auth", dbName) 125 | .field("user", username) 126 | .field("password", password) 127 | .endObject() 128 | .startObject() 129 | .field("db", dbName) 130 | .field("auth", dbName) 131 | .field("user", username) 132 | .field("password", password) 133 | .endObject() 134 | .endArray(); 135 | config 136 | .field("db", dbName) 137 | .field("collection", collectionName) 138 | .field("gridfs", false) 139 | .startObject("options") 140 | .field("secondary_read_preference", "false") 141 | .field("drop_collection", "true") 142 | .field("is_mongos", "false") 143 | .endObject() 144 | .endObject() 145 | .startObject("index") 146 | .field("name", dbName) 147 | .field("type", collectionName) 148 | .field("bulk_size", "1000") 149 | .field("bulk_timeout", "30") 150 | .endObject() 151 | .endObject(); 152 | getClient().prepareIndex("_river", collectionName, "_meta").setSource(config) 153 | .execute().actionGet(initTimeout); 154 | logger.info("Collection {}.{} indexing request sent to ES", dbName, collectionName); 155 | } 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/qatools/embed/service/ElasticPostgresIndexingService.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service; 2 | 3 | import org.elasticsearch.common.xcontent.XContentBuilder; 4 | import sun.reflect.generics.reflectiveObjects.NotImplementedException; 5 | 6 | import java.io.IOException; 7 | import java.sql.Driver; 8 | 9 | import static java.lang.String.format; 10 | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; 11 | 12 | public class ElasticPostgresIndexingService extends AbstractElasticEmbeddedService implements IndexingService { 13 | protected final String host; 14 | protected final int port; 15 | protected final String username; 16 | protected final String password; 17 | protected final Class driverClass; 18 | protected final String driverProto; 19 | protected final String driverOpts; 20 | 21 | public ElasticPostgresIndexingService(Class driverClass, 22 | String driverProto, String driverOpts, 23 | String host, int port, String username, String password, String dbName) throws IOException { 24 | this(driverClass, driverProto, driverOpts, host, port, username, password, dbName, null, true, 10000); 25 | } 26 | 27 | public ElasticPostgresIndexingService(Class driverClass, 28 | String driverProto, String driverOpts, 29 | String host, int port, String username, String password, String dbName, 30 | String dataDirectory, boolean enabled, int initTimeout) throws IOException { 31 | super(dbName, dataDirectory, enabled, initTimeout); 32 | this.username = username; 33 | this.password = password; 34 | this.host = host; 35 | this.port = port; 36 | this.driverClass = driverClass; 37 | this.driverOpts = driverOpts; 38 | this.driverProto = driverProto; 39 | } 40 | 41 | @Override 42 | protected void indexAllCollections() throws IOException { 43 | throw new NotImplementedException(); 44 | } 45 | 46 | @Override 47 | protected synchronized void indexCollection(String tableName) throws IOException { 48 | if (enabled) { 49 | final XContentBuilder config = jsonBuilder() 50 | .startObject() 51 | .field("type", "jdbc") 52 | .startObject("jdbc") 53 | .field("driver", driverClass.getName()) 54 | .field("url", formatConnectionUrl()) 55 | .field("user", username) 56 | .field("password", password) 57 | .field("index", "index") 58 | .field("type", tableName) 59 | .field("strategy", "simple") 60 | .field("sql", formatIndexQuery(tableName)) 61 | .endObject() 62 | .startObject("index") 63 | .field("index", tableName) 64 | .field("type", tableName) 65 | .field("bulk_size", "1000") 66 | .field("bulk_timeout", "30") 67 | .endObject() 68 | .endObject(); 69 | getClient().prepareDelete("_river", tableName, "_meta").execute().actionGet(initTimeout); 70 | getClient().prepareIndex("_river", tableName, "_meta").setSource(config) 71 | .execute().actionGet(initTimeout); 72 | logger.info("Table {}.{} indexing request sent to ES", dbName, tableName); 73 | } 74 | } 75 | 76 | protected String formatConnectionUrl() { 77 | return format("jdbc:%s://%s:%s/%s%s", driverProto, host, port, dbName, driverOpts); 78 | } 79 | 80 | protected String formatIndexQuery(String collectionName) { 81 | return format("select id as _id, * from %s ", collectionName); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/qatools/embed/service/EmbeddedService.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service; 2 | 3 | /** 4 | * @author smecsia 5 | */ 6 | public interface EmbeddedService { 7 | void start(); 8 | 9 | void stop(); 10 | 11 | boolean isStarted(); 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/qatools/embed/service/IndexingService.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service; 2 | 3 | import ru.yandex.qatools.embed.service.beans.IndexingResult; 4 | 5 | import java.util.List; 6 | import java.util.Map; 7 | 8 | /** 9 | * @author Ilya Sadykov 10 | */ 11 | public interface IndexingService extends EmbeddedService { 12 | List search(Class modelClass, String value); 13 | 14 | List search(String collectionName, String value); 15 | 16 | void addToIndex(Class modelClass); 17 | 18 | void indexAll(); 19 | 20 | void addToIndex(String collectionName); 21 | 22 | void initSettings(Map settings, 23 | Map> typedFields); 24 | 25 | void initSettings(Map settings, Map> typedFields, Runnable callback); 26 | 27 | void updateMappings(Map> typedFields, Runnable callback); 28 | 29 | String collectionName(Class modelClass); 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/qatools/embed/service/LogWatchStreamProcessor.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service; 2 | 3 | import de.flapdoodle.embed.process.io.IStreamProcessor; 4 | 5 | import java.util.Set; 6 | 7 | /** 8 | * @author Ilya Sadykov 9 | */ 10 | public class LogWatchStreamProcessor extends de.flapdoodle.embed.process.io.LogWatchStreamProcessor { 11 | private final Object mutex = new Object(); 12 | private final String success; 13 | private final Set failures; 14 | private volatile boolean found = false; 15 | 16 | public LogWatchStreamProcessor(String success, Set failures, IStreamProcessor destination) { 17 | super(success, failures, destination); 18 | this.success = success; 19 | this.failures = failures; 20 | } 21 | 22 | @Override 23 | public void process(String block) { 24 | if (containsSuccess(block) || containsFailure(block)) { 25 | synchronized (mutex) { 26 | found = true; 27 | mutex.notifyAll(); 28 | } 29 | } else { 30 | super.process(block); 31 | } 32 | } 33 | 34 | private boolean containsSuccess(String block) { 35 | return block.contains(success); 36 | } 37 | 38 | private boolean containsFailure(String block) { 39 | for (String failure : failures) { 40 | if (block.contains(failure)) { 41 | return true; 42 | } 43 | } 44 | return false; 45 | } 46 | 47 | public void waitForResult(long timeout) { 48 | synchronized (mutex) { 49 | try { 50 | while (!found) { 51 | mutex.wait(timeout); 52 | } 53 | } catch (InterruptedException e) { 54 | e.printStackTrace(); 55 | } 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/qatools/embed/service/MongoEmbeddedService.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service; 2 | 3 | import de.flapdoodle.embed.mongo.*; 4 | import de.flapdoodle.embed.mongo.config.*; 5 | import de.flapdoodle.embed.mongo.distribution.Version; 6 | import de.flapdoodle.embed.process.config.IRuntimeConfig; 7 | import de.flapdoodle.embed.process.config.io.ProcessOutput; 8 | import de.flapdoodle.embed.process.io.IStreamProcessor; 9 | import de.flapdoodle.embed.process.io.NamedOutputStreamProcessor; 10 | import de.flapdoodle.embed.process.runtime.Network; 11 | import org.apache.commons.lang3.StringUtils; 12 | 13 | import java.io.BufferedWriter; 14 | import java.io.File; 15 | import java.io.FileWriter; 16 | import java.io.IOException; 17 | import java.nio.file.Paths; 18 | import java.util.Collections; 19 | import java.util.HashSet; 20 | 21 | import static de.flapdoodle.embed.mongo.distribution.Version.Main.*; 22 | import static de.flapdoodle.embed.process.io.Processors.console; 23 | import static de.flapdoodle.embed.process.io.Processors.namedConsole; 24 | import static java.lang.Integer.parseInt; 25 | import static java.lang.Runtime.getRuntime; 26 | import static java.lang.String.format; 27 | import static java.util.Arrays.asList; 28 | import static jodd.io.FileUtil.delete; 29 | import static org.apache.commons.lang3.StringUtils.isEmpty; 30 | import static org.apache.commons.lang3.StringUtils.join; 31 | 32 | /** 33 | * Embedded MongoDB service 34 | * 35 | * @author smecsia 36 | */ 37 | public class MongoEmbeddedService extends AbstractEmbeddedService { 38 | private static final String HOST_PORT_SPLIT_PATTERN = "(?emptySet(), 127 | namedConsole("[mongod output]")); 128 | runtimeConfig = new RuntimeConfigBuilder() 129 | .defaults(Command.MongoD) 130 | .processOutput(new ProcessOutput( 131 | mongodOutput, 132 | namedConsole("[mongod error]"), 133 | console())) 134 | .build(); 135 | runtime = MongodStarter.getInstance(runtimeConfig); 136 | 137 | try { 138 | final MongoEmbeddedService self = this; 139 | getRuntime().addShutdownHook(new Thread() { 140 | public void run() { 141 | self.stop(); 142 | } 143 | }); 144 | 145 | startWithAuth(); 146 | if (newDirectory) { 147 | addAdmin(); 148 | } 149 | if (useAuth) { 150 | addUser(); 151 | } 152 | } catch (Exception e) { 153 | logger.error("Failed to startup embedded MongoDB", e); 154 | } 155 | } 156 | 157 | private boolean isMongo3() { 158 | return asList(V3_0, V3_1, PRODUCTION, DEVELOPMENT).contains(useVersion); 159 | } 160 | 161 | private void startWithAuth() throws IOException { 162 | prepareExecutable(useAuth); 163 | startMongoProcess(); 164 | } 165 | 166 | private void startMongoProcess() throws IOException { 167 | mongod = executable.start(); 168 | if (newDirectory && replSetName != null) { 169 | try { 170 | initiateReplicaSet(); 171 | } catch (Exception e) { 172 | logger.error("Failed to initialize replica set", e); 173 | } 174 | } 175 | } 176 | 177 | private void prepareExecutable(boolean authEnabled) throws IOException { 178 | final MongoCmdOptionsBuilder cmdBuilder = new MongoCmdOptionsBuilder(); 179 | cmdBuilder.enableAuth(authEnabled); 180 | if (useWiredTiger && isMongo3()) { 181 | cmdBuilder.useStorageEngine(WIRED_TIGER); 182 | } 183 | final IMongoCmdOptions cmdOptions = cmdBuilder.build(); 184 | MongodConfigBuilder builder = new MongodConfigBuilder() 185 | .version(useVersion) 186 | .cmdOptions(cmdOptions) 187 | .net(new Net(host, port, Network.localhostIsIPv6())); 188 | if (authEnabled && isMongo3()) { 189 | builder.setParameter("authenticationMechanisms", authMechanisms); 190 | } 191 | if (replSetName != null) { 192 | removeLockFile(builder); 193 | builder.replication(new Storage(dataDirectory, replSetName, oplogSizeMb)); 194 | } 195 | mongodConfig = builder.build(); 196 | executable = null; 197 | executable = runtime.prepare(mongodConfig); 198 | } 199 | 200 | private void removeLockFile(MongodConfigBuilder builder) { 201 | final File lockFile = Paths.get(dataDirectory, "mongod.lock").toFile(); 202 | try { 203 | delete(lockFile); 204 | } catch (Exception e) { 205 | logger.warn("No lock file found for embedded mongodb or removal failed: " + e.getMessage()); 206 | } 207 | } 208 | 209 | @Override 210 | public void doStop() { 211 | if (executable != null) { 212 | executable.stop(); 213 | } 214 | } 215 | 216 | public void initiateReplicaSet() throws IOException, InterruptedException { 217 | final String scriptText = join(asList( 218 | format("rs.initiate({\"_id\":\"%s\",\"members\":[{\"_id\":1,\"host\":\"%s:%s\"}]});", 219 | replSetName, host, port), 220 | "rs.slaveOk();rs.status();"), ""); 221 | runScriptAndWait(scriptText, null, null, null, null, null); 222 | mongodOutput.waitForResult(INIT_TIMEOUT_MS); 223 | } 224 | 225 | private void addAdmin() throws IOException { 226 | final String scriptText = join( 227 | format("db.createUser(" + 228 | "{\"user\":\"%s\",\"pwd\":\"%s\"," + 229 | "\"roles\":[" + 230 | "\"root\"," + 231 | "{\"role\":\"userAdmin\",\"db\":\"admin\"}," + 232 | "{\"role\":\"dbAdmin\",\"db\":\"admin\"}," + 233 | "{\"role\":\"userAdminAnyDatabase\",\"db\":\"admin\"}," + 234 | "{\"role\":\"dbAdminAnyDatabase\",\"db\":\"admin\"}," + 235 | "{\"role\":\"clusterAdmin\",\"db\":\"admin\"}," + 236 | "{\"role\":\"dbOwner\",\"db\":\"admin\"}," + 237 | "]});\n", 238 | adminUsername, adminPassword)); 239 | runScriptAndWait(scriptText, USER_ADDED_TOKEN, new String[]{"couldn't add user", "failed to load", "login failed"}, "admin", null, null); 240 | } 241 | 242 | private void addUser() throws IOException { 243 | final String scriptText = join(format("db = db.getSiblingDB('%s'); " + 244 | "db.createUser({\"user\":\"%s\",\"pwd\":\"%s\",\"roles\":[%s]});\n" + 245 | "db.getUser('%s');", 246 | mongoDBName, username, password, StringUtils.join(roles, ","), username), ""); 247 | runScriptAndWait(scriptText, USER_ADDED_TOKEN, new String[]{"already exists", "failed to load", "login failed"}, "admin", adminUsername, adminPassword); 248 | } 249 | 250 | private void runScriptAndWait(String scriptText, String token, String[] failures, String dbName, String username, String password) throws IOException { 251 | IStreamProcessor mongoOutput; 252 | if (!isEmpty(token)) { 253 | mongoOutput = new LogWatchStreamProcessor( 254 | format(token), 255 | (failures != null) ? new HashSet<>(asList(failures)) : Collections.emptySet(), 256 | namedConsole("[mongo shell output]")); 257 | } else { 258 | mongoOutput = new NamedOutputStreamProcessor("[mongo shell output]", console()); 259 | } 260 | IRuntimeConfig runtimeConfig = new RuntimeConfigBuilder() 261 | .defaults(Command.Mongo) 262 | .processOutput(new ProcessOutput( 263 | mongoOutput, 264 | namedConsole("[mongo shell error]"), 265 | console())) 266 | .build(); 267 | MongoShellStarter starter = MongoShellStarter.getInstance(runtimeConfig); 268 | final File scriptFile = writeTmpScriptFile(scriptText); 269 | final MongoShellConfigBuilder builder = new MongoShellConfigBuilder(); 270 | if (!isEmpty(dbName)) { 271 | builder.dbName(dbName); 272 | } 273 | if (!isEmpty(username)) { 274 | builder.username(username); 275 | } 276 | if (!isEmpty(password)) { 277 | builder.password(password); 278 | } 279 | starter.prepare(builder 280 | .scriptName(scriptFile.getAbsolutePath()) 281 | .version(mongodConfig.version()) 282 | .net(mongodConfig.net()) 283 | .build()).start(); 284 | if (mongoOutput instanceof LogWatchStreamProcessor) { 285 | ((LogWatchStreamProcessor) mongoOutput).waitForResult(INIT_TIMEOUT_MS); 286 | } 287 | } 288 | 289 | private File writeTmpScriptFile(String scriptText) throws IOException { 290 | File scriptFile = File.createTempFile("tempfile", ".js"); 291 | scriptFile.deleteOnExit(); 292 | BufferedWriter bw = new BufferedWriter(new FileWriter(scriptFile)); 293 | bw.write(scriptText); 294 | bw.close(); 295 | return scriptFile; 296 | } 297 | 298 | public Net net() { 299 | return mongodConfig.net(); 300 | } 301 | 302 | 303 | public String getHost() { 304 | return host; 305 | } 306 | 307 | public int getPort() { 308 | return port; 309 | } 310 | 311 | public String[] getRoles() { 312 | return roles; 313 | } 314 | 315 | public void setRoles(String... roles) { 316 | this.roles = roles; 317 | } 318 | 319 | public String getAdminUsername() { 320 | return adminUsername; 321 | } 322 | 323 | public void setAdminUsername(String adminUsername) { 324 | this.adminUsername = adminUsername; 325 | } 326 | 327 | public String getAdminPassword() { 328 | return adminPassword; 329 | } 330 | 331 | public void setAdminPassword(String adminPassword) { 332 | this.adminPassword = adminPassword; 333 | } 334 | } 335 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/qatools/embed/service/PostgresEmbeddedService.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service; 2 | 3 | import de.flapdoodle.embed.process.config.IRuntimeConfig; 4 | import ru.yandex.qatools.embed.postgresql.Command; 5 | import ru.yandex.qatools.embed.postgresql.PostgresExecutable; 6 | import ru.yandex.qatools.embed.postgresql.PostgresProcess; 7 | import ru.yandex.qatools.embed.postgresql.PostgresStarter; 8 | import ru.yandex.qatools.embed.postgresql.config.DownloadConfigBuilder; 9 | import ru.yandex.qatools.embed.postgresql.config.PostgresConfig; 10 | import ru.yandex.qatools.embed.postgresql.config.RuntimeConfigBuilder; 11 | import ru.yandex.qatools.embed.postgresql.ext.ArtifactStoreBuilder; 12 | 13 | import java.io.IOException; 14 | 15 | import static ru.yandex.qatools.embed.postgresql.config.AbstractPostgresConfig.*; 16 | import static ru.yandex.qatools.embed.postgresql.distribution.Version.Main.PRODUCTION; 17 | 18 | /** 19 | * Embedded Postgres server 20 | */ 21 | public class PostgresEmbeddedService extends AbstractEmbeddedService { 22 | 23 | protected final String host; 24 | protected final int port; 25 | protected final String dbName; 26 | protected final String username; 27 | protected final String password; 28 | PostgresProcess process; 29 | 30 | public PostgresEmbeddedService(String host, int port, String username, String password, String dbName) throws IOException { 31 | this(host, port, username, password, dbName, null, true, 10000); 32 | } 33 | 34 | public PostgresEmbeddedService(String host, int port, 35 | String username, String password, String dbName, 36 | String dataDirectory, boolean enabled, int initTimeout) throws IOException { 37 | super(dataDirectory, enabled, initTimeout); 38 | this.username = username; 39 | this.password = password; 40 | this.host = host; 41 | this.port = port; 42 | this.dbName = dbName; 43 | } 44 | 45 | @Override 46 | public void doStart() throws Exception { 47 | final IRuntimeConfig runtimeConfig = new RuntimeConfigBuilder() 48 | .defaults(Command.Postgres) 49 | .artifactStore(new ArtifactStoreBuilder() 50 | .defaults(Command.Postgres) 51 | .download(new DownloadConfigBuilder() 52 | .defaultsForCommand(Command.Postgres) 53 | .build() 54 | ) 55 | ).build(); 56 | final PostgresStarter runtime = PostgresStarter.getInstance(runtimeConfig); 57 | final PostgresConfig configDb = new PostgresConfig( 58 | PRODUCTION, new Net(host, port), new Storage(dbName, dataDirectory), 59 | new Timeout(initTimeout), new Credentials(username, password)); 60 | PostgresExecutable exec = runtime.prepare(configDb); 61 | process = exec.start(); 62 | } 63 | 64 | @Override 65 | public void doStop() throws Exception { 66 | if (process != null) { 67 | process.stop(); 68 | process = null; 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/ru/yandex/qatools/embed/service/beans/IndexingResult.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service.beans; 2 | 3 | import java.util.Map; 4 | 5 | /** 6 | * @author smecsia 7 | */ 8 | public class IndexingResult { 9 | 10 | final String id; 11 | final float score; 12 | final Map attrs; 13 | 14 | public IndexingResult(String id, float score, Map attrs) { 15 | this.id = id; 16 | this.score = score; 17 | this.attrs = attrs; 18 | } 19 | 20 | public String getId() { 21 | return id; 22 | } 23 | 24 | public float getScore() { 25 | return score; 26 | } 27 | 28 | public Map getAttrs() { 29 | return attrs; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/ElasticMongoExcludingFieldsTest.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service; 2 | 3 | import de.flapdoodle.embed.mongo.distribution.Version; 4 | import org.elasticsearch.common.xcontent.XContentBuilder; 5 | import org.junit.Before; 6 | import ru.yandex.qatools.embed.service.db.MorphiaDBService; 7 | import ru.yandex.qatools.embed.service.db.PostDAO; 8 | import ru.yandex.qatools.embed.service.db.UserDAO; 9 | 10 | import java.io.IOException; 11 | 12 | import static com.mongodb.ReadPreference.nearest; 13 | import static com.mongodb.WriteConcern.ACKNOWLEDGED; 14 | import static ru.yandex.qatools.embed.service.util.SocketUtil.findFreePort; 15 | 16 | public class ElasticMongoExcludingFieldsTest extends ElasticMongoIndexingServiceTest { 17 | 18 | @Before 19 | @Override 20 | public void startEmbeddedServers() throws IOException, InterruptedException { 21 | final String RS = "localhost:" + findFreePort(); 22 | mongo = new MongoEmbeddedService(RS, DB, USER, PASS, RS_NAME, null, true, INIT_TIMEOUT) 23 | .useVersion(Version.Main.PRODUCTION).useWiredTiger(); 24 | mongo.start(); 25 | final MorphiaDBService dbService = new MorphiaDBService(RS, DB, USER, PASS); 26 | dbService.getDatastore().getDB().getMongo().setReadPreference(nearest()); 27 | dbService.getDatastore().setDefaultWriteConcern(ACKNOWLEDGED); 28 | postDAO = new PostDAO(dbService); 29 | userDAO = new UserDAO(dbService); 30 | 31 | es = new ElasticMongoIndexingService(RS, DB, USER, PASS, null, true, INIT_TIMEOUT) { 32 | @Override 33 | protected void addIndexAllCollectionsOptions(XContentBuilder config) { 34 | try { 35 | config.array("exclude_fields", "detail", "user.detail"); 36 | } catch (IOException e) { 37 | throw new RuntimeException(e); 38 | } 39 | } 40 | }; 41 | es.setBindHost("localhost"); 42 | es.start(); 43 | es.indexAll(); 44 | } 45 | } -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/ElasticMongoIndexingServiceTest.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service; 2 | 3 | import de.flapdoodle.embed.mongo.distribution.Version; 4 | import org.javalite.common.Collections; 5 | import org.junit.After; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | import ru.yandex.qatools.embed.service.beans.IndexingResult; 9 | import ru.yandex.qatools.embed.service.db.*; 10 | 11 | import java.io.IOException; 12 | import java.net.UnknownHostException; 13 | import java.util.HashSet; 14 | import java.util.List; 15 | import java.util.Map; 16 | import java.util.Set; 17 | 18 | import static ch.lambdaj.Lambda.collect; 19 | import static ch.lambdaj.Lambda.on; 20 | import static com.mongodb.ReadPreference.nearest; 21 | import static com.mongodb.WriteConcern.ACKNOWLEDGED; 22 | import static java.lang.Thread.sleep; 23 | import static org.hamcrest.Matchers.containsInAnyOrder; 24 | import static org.hamcrest.Matchers.hasSize; 25 | import static org.hamcrest.core.Is.is; 26 | import static org.junit.Assert.assertThat; 27 | import static ru.yandex.qatools.embed.service.IndexingServiceMatcher.findIndexedAtLeast; 28 | import static ru.yandex.qatools.embed.service.util.SocketUtil.findFreePort; 29 | import static ru.yandex.qatools.matchers.decorators.MatcherDecorators.should; 30 | import static ru.yandex.qatools.matchers.decorators.MatcherDecorators.timeoutHasExpired; 31 | 32 | public class ElasticMongoIndexingServiceTest { 33 | public static final String RS_NAME = "local"; 34 | public static final String DB = "mongolastic"; 35 | public static final String USER = "user"; 36 | public static final String PASS = "password"; 37 | public static final int INIT_TIMEOUT = 10000; 38 | ElasticMongoIndexingService es; 39 | MongoEmbeddedService mongo; 40 | PostDAO postDAO; 41 | UserDAO userDAO; 42 | 43 | @Before 44 | public void startEmbeddedServers() throws IOException, InterruptedException { 45 | final String RS = "localhost:" + findFreePort(); 46 | mongo = new MongoEmbeddedService(RS, DB, USER, PASS, RS_NAME, null, true, INIT_TIMEOUT) 47 | .useVersion(Version.Main.PRODUCTION).useWiredTiger(); 48 | mongo.start(); 49 | final MorphiaDBService dbService = new MorphiaDBService(RS, DB, USER, PASS); 50 | dbService.getDatastore().getDB().getMongo().setReadPreference(nearest()); 51 | dbService.getDatastore().setDefaultWriteConcern(ACKNOWLEDGED); 52 | postDAO = new PostDAO(dbService); 53 | userDAO = new UserDAO(dbService); 54 | 55 | es = new ElasticMongoIndexingService(RS, DB, USER, PASS, null, true, INIT_TIMEOUT); 56 | es.setBindHost("localhost"); 57 | es.start(); 58 | 59 | es.initSettings(Collections.map( 60 | "index.mapping.ignore_malformed", true, 61 | "index.fail_on_merge_failure", false 62 | ), Collections.>map( 63 | "posts._id", Collections.map( 64 | "type", "string", "index", "not_analyzed", "store", true), 65 | "posts.user.detail", Collections.map( 66 | "type", "object", "enabled", false), 67 | "users.detail", Collections.map( 68 | "type", "object", "enabled", false) 69 | ), new Runnable() { 70 | @Override 71 | public void run() { 72 | try { 73 | es.indexAllCollections(); 74 | } catch (IOException e) { 75 | throw new RuntimeException(e); 76 | } 77 | } 78 | }); 79 | } 80 | 81 | @After 82 | public void shutdownEmbeddedServers() throws IOException { 83 | mongo.stop(); 84 | es.stop(); 85 | } 86 | 87 | @Test 88 | public void testElasticFullTextSearch() throws IOException, InterruptedException { 89 | // create some test data 90 | UserMongo user1 = createUser(1L, "Ivan Petrov", new UserDetailMongoLong(1L)); 91 | sleep(1000); 92 | UserMongo user2 = createUser(2L, "Ivan Ivanov", new UserDetailMongoString("some-string")); 93 | sleep(1000); 94 | PostMongo post1 = createPost(1L, user1, "Some title", "Some post with keyword among other words"); 95 | sleep(1000); 96 | PostMongo post2 = createPost(2L, user2, "Some another title", "Some post without the required word"); 97 | sleep(1000); 98 | PostMongo post3 = createPost(3L, user2, "Some third title", "Some post with the required keyword among other words"); 99 | 100 | assertThat("At least two users must be found by query", 101 | es, should(findIndexedAtLeast(UserMongo.class, "users", "name:Ivan", 2)) 102 | .whileWaitingUntil(timeoutHasExpired(20000))); 103 | final List users = es.search("users", "name:(Ivan AND Petrov)"); 104 | assertThat(users, hasSize(1)); 105 | assertThat(users.get(0).getId(), is(String.valueOf(user1.getId()))); 106 | 107 | assertThat("User must be found by id", 108 | es, should(findIndexedAtLeast(UserMongo.class, "users", "_id:\"" + user1.getId() + "\"", 1)) 109 | .whileWaitingUntil(timeoutHasExpired(20000))); 110 | 111 | assertThat("Post must be found by id", 112 | es, should(findIndexedAtLeast(PostMongo.class, "posts", "_id:\"" + post1.getId() + "\"", 1)) 113 | .whileWaitingUntil(timeoutHasExpired(20000))); 114 | 115 | assertThat("At least two posts must be found by query", 116 | es, should(findIndexedAtLeast(PostMongo.class, "posts", "body:keyword", 2)) 117 | .whileWaitingUntil(timeoutHasExpired(20000))); 118 | 119 | // perform the search 120 | final List posts = es.search("posts", "body:keyword"); 121 | assertThat(posts, hasSize(2)); 122 | Set postIds = new HashSet<>(collect(posts, on(IndexingResult.class).getId())); 123 | assertThat(postIds, containsInAnyOrder(String.valueOf(post1.getId()), String.valueOf(post3.getId()))); 124 | } 125 | 126 | 127 | private UserMongo createUser(Long id, String name, UserDetailMongo detail) throws UnknownHostException { 128 | final UserMongo user = new UserMongo(); 129 | user.setId(id); 130 | user.setName(name); 131 | user.setDetail(detail); 132 | userDAO.save(user); 133 | return user; 134 | } 135 | 136 | private PostMongo createPost(long id, UserMongo user, String title, String description) throws UnknownHostException { 137 | final PostMongo post = new PostMongo(); 138 | post.setId(id); 139 | post.setTitle(title); 140 | post.setUser(user); 141 | post.setBody(description); 142 | postDAO.save(post); 143 | return post; 144 | } 145 | 146 | } -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/ElasticPostgresIndexingServiceTest.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service; 2 | 3 | import org.junit.After; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | import org.postgresql.Driver; 7 | import ru.yandex.qatools.embed.service.db.Database; 8 | import ru.yandex.qatools.embed.service.db.PostPostgres; 9 | 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | 13 | import static jodd.io.FileUtil.createTempDirectory; 14 | import static org.hamcrest.CoreMatchers.not; 15 | import static org.hamcrest.CoreMatchers.nullValue; 16 | import static org.hamcrest.core.Is.is; 17 | import static org.junit.Assert.assertThat; 18 | import static ru.yandex.qatools.embed.service.IndexingServiceMatcher.findIndexedAtLeast; 19 | import static ru.yandex.qatools.matchers.decorators.MatcherDecorators.should; 20 | import static ru.yandex.qatools.matchers.decorators.TimeoutWaiter.timeoutHasExpired; 21 | 22 | public class ElasticPostgresIndexingServiceTest { 23 | public static final int INIT_TIMEOUT = 10000; 24 | public static final String DB_NAME = "test"; 25 | public static final String HOST = "localhost"; 26 | public static final int PORT = 5429; 27 | public static final String USERNAME = "sa"; 28 | public static final String PASSWORD = "123"; 29 | public static final int POOL_SIZE = 10; 30 | PostgresEmbeddedService postgres; 31 | ElasticPostgresIndexingService elastic; 32 | Database database; 33 | 34 | @Before 35 | public void setUp() throws Exception { 36 | final String dbDir = createTempDirectory("postgresdb", "").getAbsolutePath(); 37 | final String idxDir = createTempDirectory("elastic", "").getAbsolutePath(); 38 | postgres = new PostgresEmbeddedService(HOST, PORT, USERNAME, PASSWORD, DB_NAME, dbDir, true, INIT_TIMEOUT); 39 | postgres.start(); 40 | elastic = new ElasticPostgresIndexingService(Driver.class, "postgresql", "", 41 | HOST, PORT, USERNAME, PASSWORD, DB_NAME, idxDir, true, INIT_TIMEOUT); 42 | elastic.setBindHost("localhost"); 43 | elastic.start(); 44 | database = new Database(HOST, PORT, DB_NAME, USERNAME, PASSWORD, POOL_SIZE); 45 | database.connect(); 46 | elastic.addToIndex("posts"); 47 | } 48 | 49 | @Test 50 | public void testSaveFindPost() throws Exception { 51 | List posts = new ArrayList<>(); 52 | for (int i = 0; i < 100; ++i) { 53 | PostPostgres post = new PostPostgres(); 54 | post.setTitle("Post title" + i); 55 | post.setText("Lorem ipsum dolor sit amet " + i); 56 | assertThat(post.saveIt(), is(true)); 57 | posts.add(post); 58 | } 59 | assertThat(PostPostgres.findById(posts.get(0).getId()), not(nullValue())); 60 | assertThat("post must be found by full text query", 61 | elastic, should(findIndexedAtLeast(PostPostgres.class, "posts", "dolor sit", 10)) 62 | .whileWaitingUntil(timeoutHasExpired(10000))); 63 | } 64 | 65 | @After 66 | public void tearDown() throws Exception { 67 | database.disconnect(); 68 | postgres.stop(); 69 | elastic.stop(); 70 | } 71 | 72 | } -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/IndexingServiceMatcher.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service; 2 | 3 | import org.hamcrest.Description; 4 | import org.hamcrest.TypeSafeMatcher; 5 | 6 | public class IndexingServiceMatcher extends TypeSafeMatcher { 7 | 8 | private final Class entityClass; 9 | private final String query; 10 | private final MatchFunction callable; 11 | 12 | private IndexingServiceMatcher(Class entityClass, String query, 13 | MatchFunction callable) { 14 | this.entityClass = entityClass; 15 | this.query = query; 16 | this.callable = callable; 17 | } 18 | 19 | @Override 20 | public void describeTo(Description description) { 21 | description.appendText(String.format( 22 | "Repository containing some %s found by query '%s'", 23 | entityClass.getSimpleName(), query)); 24 | } 25 | 26 | @Override 27 | protected boolean matchesSafely(IndexingService repo) { 28 | try { 29 | return callable.call(repo); 30 | } catch (Exception e) { 31 | throw new RuntimeException(e); 32 | } 33 | } 34 | 35 | public static interface MatchFunction { 36 | public R call(P arg); 37 | } 38 | 39 | public static IndexingServiceMatcher findIndexedAny( 40 | final Class entityClass, final String query) { 41 | return new IndexingServiceMatcher<>(entityClass, query, new MatchFunction() { 42 | @Override 43 | public Boolean call(IndexingService service) { 44 | return service.search(entityClass, query).size() > 0; 45 | } 46 | }); 47 | } 48 | 49 | public static IndexingServiceMatcher findIndexedAtLeast(final Class entityClass, 50 | final String query, final int count) { 51 | return new IndexingServiceMatcher<>(entityClass, query, new MatchFunction() { 52 | @Override 53 | public Boolean call(IndexingService service) { 54 | return service.search(entityClass, query).size() >= count; 55 | } 56 | }); 57 | } 58 | 59 | public static IndexingServiceMatcher findIndexedAtLeast(final Class entityClass, final String collection, 60 | final String query, final int count) { 61 | return new IndexingServiceMatcher<>(entityClass, query, new MatchFunction() { 62 | @Override 63 | public Boolean call(IndexingService service) { 64 | return service.search(collection, query).size() >= count; 65 | } 66 | }); 67 | } 68 | 69 | } -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/MongoDBEmbeddedDoubleStartTest.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service; 2 | 3 | import de.flapdoodle.embed.mongo.distribution.Version; 4 | import org.junit.After; 5 | import org.junit.Before; 6 | import org.junit.Test; 7 | import ru.yandex.qatools.embed.service.db.*; 8 | 9 | import java.io.IOException; 10 | import java.net.UnknownHostException; 11 | 12 | import static ch.lambdaj.Lambda.collect; 13 | import static ch.lambdaj.Lambda.on; 14 | import static com.mongodb.ReadPreference.nearest; 15 | import static com.mongodb.WriteConcern.ACKNOWLEDGED; 16 | import static jodd.io.FileUtil.createTempDirectory; 17 | import static org.hamcrest.Matchers.contains; 18 | import static org.junit.Assert.assertThat; 19 | import static ru.yandex.qatools.embed.service.util.SocketUtil.findFreePort; 20 | 21 | public class MongoDBEmbeddedDoubleStartTest { 22 | public static final String DB = "mongolastic"; 23 | public static final String RS_NAME = "local"; 24 | public static final String USER = "user"; 25 | public static final String PASS = "password"; 26 | private String tmpDir; 27 | MongoEmbeddedService mongo; 28 | UserDAO userDAO; 29 | 30 | @Before 31 | public void startEmbeddedServers() throws IOException, InterruptedException { 32 | final String RS = "localhost:" + findFreePort(); 33 | tmpDir = createTempDirectory(getClass().getSimpleName().toLowerCase(), "data").getPath(); 34 | mongo = new MongoEmbeddedService(RS, DB, USER, PASS, RS_NAME, tmpDir, true, 10000) 35 | .useVersion(Version.Main.PRODUCTION).useWiredTiger(); 36 | mongo.setAdminUsername("admin-username"); 37 | mongo.setAdminPassword("admin-password"); 38 | mongo.start(); 39 | mongo.stop(); 40 | mongo.start(); 41 | final MorphiaDBService dbService = new MorphiaDBService(RS, DB, USER, PASS); 42 | dbService.getDatastore().getDB().getMongo().setReadPreference(nearest()); 43 | dbService.getDatastore().setDefaultWriteConcern(ACKNOWLEDGED); 44 | userDAO = new UserDAO(dbService); 45 | } 46 | 47 | @After 48 | public void shutdownEmbeddedServers() throws IOException { 49 | mongo.stop(); 50 | } 51 | 52 | @Test 53 | public void testElasticFullTextSearch() throws IOException, InterruptedException { 54 | // create some test data 55 | UserMongo user1 = createUser(1L, "Ivan Petrov", new UserDetailMongoLong(1L)); 56 | assertThat(collect(userDAO.find().asList(), on(UserMongo.class).getId()), contains(user1.getId())); 57 | } 58 | 59 | 60 | private UserMongo createUser(Long id, String name, UserDetailMongo detail) throws UnknownHostException { 61 | final UserMongo user = new UserMongo(); 62 | user.setId(id); 63 | user.setName(name); 64 | user.setDetail(detail); 65 | userDAO.save(user); 66 | return user; 67 | } 68 | } -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/MongoDBEmbeddedWithoutAuthTest.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service; 2 | 3 | import de.flapdoodle.embed.mongo.distribution.Version; 4 | import org.junit.After; 5 | import org.junit.Before; 6 | import org.junit.Test; 7 | import ru.yandex.qatools.embed.service.db.*; 8 | 9 | import java.io.IOException; 10 | import java.net.UnknownHostException; 11 | 12 | import static ch.lambdaj.Lambda.collect; 13 | import static ch.lambdaj.Lambda.on; 14 | import static com.mongodb.ReadPreference.nearest; 15 | import static com.mongodb.WriteConcern.ACKNOWLEDGED; 16 | import static org.hamcrest.Matchers.contains; 17 | import static org.junit.Assert.assertThat; 18 | import static ru.yandex.qatools.embed.service.util.SocketUtil.findFreePort; 19 | 20 | public class MongoDBEmbeddedWithoutAuthTest { 21 | public static final String DB = "mongolastic"; 22 | MongoEmbeddedService mongo; 23 | UserDAO userDAO; 24 | 25 | @Before 26 | public void startEmbeddedServers() throws IOException, InterruptedException { 27 | final String RS = "localhost:" + findFreePort(); 28 | mongo = new MongoEmbeddedService(RS, DB) 29 | .useVersion(Version.Main.PRODUCTION).useWiredTiger(); 30 | mongo.start(); 31 | final MorphiaDBService dbService = new MorphiaDBService(RS, DB); 32 | dbService.getDatastore().getDB().getMongo().setReadPreference(nearest()); 33 | dbService.getDatastore().setDefaultWriteConcern(ACKNOWLEDGED); 34 | userDAO = new UserDAO(dbService); 35 | } 36 | 37 | @After 38 | public void shutdownEmbeddedServers() throws IOException { 39 | mongo.stop(); 40 | } 41 | 42 | @Test 43 | public void testElasticFullTextSearch() throws IOException, InterruptedException { 44 | // create some test data 45 | UserMongo user1 = createUser(1L, "Ivan Petrov", new UserDetailMongoLong(1L)); 46 | assertThat(collect(userDAO.find().asList(), on(UserMongo.class).getId()), contains(user1.getId())); 47 | } 48 | 49 | 50 | private UserMongo createUser(Long id, String name, UserDetailMongo detail) throws UnknownHostException { 51 | final UserMongo user = new UserMongo(); 52 | user.setId(id); 53 | user.setName(name); 54 | user.setDetail(detail); 55 | userDAO.save(user); 56 | return user; 57 | } 58 | } -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/db/Database.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service.db; 2 | 3 | import com.zaxxer.hikari.HikariDataSource; 4 | import org.flywaydb.core.Flyway; 5 | import org.javalite.activejdbc.Base; 6 | import org.postgresql.ds.PGPoolingDataSource; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import javax.sql.DataSource; 11 | 12 | import static java.lang.String.format; 13 | 14 | public class Database { 15 | private final static Logger logger = LoggerFactory.getLogger(Database.class); 16 | private final String dbUrl; 17 | private final String username; 18 | private final String password; 19 | private int maxPoolSize; 20 | 21 | public Database(String host, int port, String dbName, String username, String password, int maxPoolSize) { 22 | this.dbUrl = format("jdbc:postgresql://%s:%s/%s?user=%s&password=%s", host, port, dbName, username, password); 23 | this.username = username; 24 | this.password = password; 25 | this.maxPoolSize = maxPoolSize; 26 | } 27 | 28 | public void connect() { 29 | try { 30 | logger.info(format("Connecting to database with url '%s' ...", dbUrl)); 31 | openConnection(); 32 | Flyway flyway = new Flyway(); 33 | flyway.setDataSource(dataSource()); 34 | flyway.migrate(); 35 | } catch (Exception e) { 36 | throw new RuntimeException("Failed to start database", e); 37 | } 38 | } 39 | 40 | private DataSource dataSource() { 41 | final HikariDataSource ds = new HikariDataSource(); 42 | ds.setMaximumPoolSize(maxPoolSize); 43 | ds.setDataSourceClassName(PGPoolingDataSource.class.getName()); 44 | ds.addDataSourceProperty("url", dbUrl); 45 | ds.addDataSourceProperty("user", username); 46 | ds.addDataSourceProperty("password", password); 47 | return ds; 48 | } 49 | 50 | public void disconnect() { 51 | closeConnection(); 52 | } 53 | 54 | public void openConnection() { 55 | if (!Base.hasConnection()) { 56 | Base.open(dataSource()); 57 | } 58 | } 59 | 60 | public void closeConnection() { 61 | if (Base.hasConnection()) { 62 | Base.close(); 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/db/MorphiaDBService.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service.db; 2 | 3 | import com.mongodb.MongoClient; 4 | import com.mongodb.MongoCredential; 5 | import com.mongodb.ServerAddress; 6 | import org.mongodb.morphia.Datastore; 7 | import org.mongodb.morphia.Morphia; 8 | 9 | import java.net.UnknownHostException; 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | 13 | import static java.util.Arrays.asList; 14 | import static org.apache.commons.lang3.StringUtils.isEmpty; 15 | 16 | /** 17 | * @author smecsia 18 | */ 19 | public class MorphiaDBService { 20 | 21 | private static final String HOST_PORT_SPLIT_PATTERN = "(? addresses = new ArrayList<>(); 33 | for (String host : replicaSet.split(",")) { 34 | String[] hostPort = host.split(HOST_PORT_SPLIT_PATTERN); 35 | addresses.add(new ServerAddress(hostPort[0], Integer.valueOf(hostPort[1]))); 36 | } 37 | mongoClient = ((!isEmpty(username) && !isEmpty(password))) ? 38 | new MongoClient(addresses, asList(MongoCredential.createCredential(username, dbName, password.toCharArray()))) : 39 | new MongoClient(addresses); 40 | datastore = new Morphia().createDatastore(mongoClient, dbName); 41 | } 42 | 43 | public Datastore getDatastore() { 44 | return datastore; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/db/PostDAO.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service.db; 2 | 3 | import org.mongodb.morphia.dao.BasicDAO; 4 | 5 | /** 6 | * @author smecsia 7 | */ 8 | public class PostDAO extends BasicDAO { 9 | 10 | public PostDAO(MorphiaDBService dbService) { 11 | super(dbService.getDatastore()); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/db/PostMongo.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service.db; 2 | 3 | import org.mongodb.morphia.annotations.Embedded; 4 | import org.mongodb.morphia.annotations.Entity; 5 | import org.mongodb.morphia.annotations.Id; 6 | 7 | /** 8 | * @author smecsia 9 | */ 10 | @Entity("posts") 11 | public class PostMongo { 12 | 13 | @Id 14 | private long id; 15 | 16 | @Embedded 17 | UserMongo user; 18 | 19 | private String title; 20 | 21 | private String body; 22 | 23 | public long getId() { 24 | return id; 25 | } 26 | 27 | public void setId(long id) { 28 | this.id = id; 29 | } 30 | 31 | public String getTitle() { 32 | return title; 33 | } 34 | 35 | public void setTitle(String title) { 36 | this.title = title; 37 | } 38 | 39 | public String getBody() { 40 | return body; 41 | } 42 | 43 | public void setBody(String body) { 44 | this.body = body; 45 | } 46 | 47 | public UserMongo getUser() { 48 | return user; 49 | } 50 | 51 | public void setUser(UserMongo user) { 52 | this.user = user; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/db/PostPostgres.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service.db; 2 | 3 | import org.javalite.activejdbc.Model; 4 | import org.javalite.activejdbc.annotations.Table; 5 | 6 | import java.sql.Timestamp; 7 | 8 | @Table("posts") 9 | public class PostPostgres extends Model { 10 | public String getText() { 11 | return getString("text"); 12 | } 13 | 14 | public void setText(String text) { 15 | set("text", text); 16 | } 17 | 18 | public Timestamp getUpdatedAt() { 19 | return getTimestamp("updated_at"); 20 | } 21 | 22 | public Timestamp getCreatedAt() { 23 | return getTimestamp("created_at"); 24 | } 25 | 26 | public String getTitle() { 27 | return getString("title"); 28 | } 29 | 30 | public void setTitle(String title) { 31 | set("title", title); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/db/UserDAO.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service.db; 2 | 3 | import org.mongodb.morphia.dao.BasicDAO; 4 | 5 | /** 6 | * @author smecsia 7 | */ 8 | public class UserDAO extends BasicDAO { 9 | 10 | public UserDAO(MorphiaDBService dbService) { 11 | super(dbService.getDatastore()); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/db/UserDetailMongo.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service.db; 2 | 3 | /** 4 | * @author Ilya Sadykov 5 | */ 6 | public interface UserDetailMongo { 7 | } 8 | -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/db/UserDetailMongoLong.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service.db; 2 | 3 | import org.mongodb.morphia.annotations.Embedded; 4 | 5 | /** 6 | * @author smecsia 7 | */ 8 | @Embedded 9 | public class UserDetailMongoLong implements UserDetailMongo { 10 | 11 | private Long _id; 12 | 13 | public UserDetailMongoLong(Long _id) { 14 | this._id = _id; 15 | } 16 | 17 | public UserDetailMongoLong() { 18 | } 19 | 20 | public Long get_id() { 21 | return _id; 22 | } 23 | 24 | public void set_id(Long _id) { 25 | this._id = _id; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/db/UserDetailMongoString.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service.db; 2 | 3 | import org.mongodb.morphia.annotations.Embedded; 4 | 5 | /** 6 | * @author smecsia 7 | */ 8 | @Embedded 9 | public class UserDetailMongoString implements UserDetailMongo { 10 | 11 | private String _id; 12 | 13 | public UserDetailMongoString(String _id) { 14 | this._id = _id; 15 | } 16 | 17 | public UserDetailMongoString() { 18 | } 19 | 20 | public String get_id() { 21 | return _id; 22 | } 23 | 24 | public void set_id(String _id) { 25 | this._id = _id; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/db/UserMongo.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service.db; 2 | 3 | import org.mongodb.morphia.annotations.Embedded; 4 | import org.mongodb.morphia.annotations.Entity; 5 | import org.mongodb.morphia.annotations.Id; 6 | 7 | /** 8 | * @author smecsia 9 | */ 10 | @Entity("users") 11 | public class UserMongo { 12 | 13 | @Id 14 | private Long id; 15 | 16 | @Embedded 17 | private UserDetailMongo detail; 18 | 19 | private String name; 20 | 21 | public UserDetailMongo getDetail() { 22 | return detail; 23 | } 24 | 25 | public void setDetail(UserDetailMongo detail) { 26 | this.detail = detail; 27 | } 28 | 29 | public long getId() { 30 | return id; 31 | } 32 | 33 | public void setId(Long id) { 34 | this.id = id; 35 | } 36 | 37 | public void setName(String name) { 38 | this.name = name; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/ru/yandex/qatools/embed/service/util/SocketUtil.java: -------------------------------------------------------------------------------- 1 | package ru.yandex.qatools.embed.service.util; 2 | 3 | import java.io.IOException; 4 | import java.net.ServerSocket; 5 | 6 | public abstract class SocketUtil { 7 | /** 8 | * Returns a free port number on localhost. 9 | *

10 | * Heavily inspired from org.eclipse.jdt.launching.SocketUtil (to avoid a dependency to JDT just because of this). 11 | * Slightly improved with close() missing in JDT. And throws exception instead of returning -1. 12 | * 13 | * @return a free port number on localhost 14 | * @throws IllegalStateException if unable to find a free port 15 | */ 16 | public static int findFreePort() { 17 | ServerSocket socket = null; 18 | try { 19 | socket = new ServerSocket(0); 20 | socket.setReuseAddress(true); 21 | int port = socket.getLocalPort(); 22 | try { 23 | socket.close(); 24 | } catch (IOException ignored) { 25 | // Ignore IOException on close() 26 | } 27 | return port; 28 | } catch (IOException ignored) { 29 | // Ignore IOException on open 30 | } finally { 31 | if (socket != null) { 32 | try { 33 | socket.close(); 34 | } catch (IOException ignored) { 35 | // Ignore IOException on close() 36 | } 37 | } 38 | } 39 | throw new IllegalStateException("Could not find a free TCP/IP port to start embedded Jetty HTTP Server on"); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/test/resources/db/migration/V1__create_tables.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE posts ( 2 | id SERIAL PRIMARY KEY, 3 | title VARCHAR(100) NOT NULL, 4 | text VARCHAR(1000000) NOT NULL, 5 | created_at TIMESTAMP, 6 | updated_at TIMESTAMP, 7 | extra_details json 8 | ); 9 | -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # The logging properties used 3 | # 4 | log4j.rootLogger=DEBUG, out 5 | 6 | log4j.logger.org.elasticsearch=TRACE 7 | 8 | # uncomment the following line to turn on Camel debugging 9 | log4j.logger.org.apache.camel=INFO 10 | 11 | log4j.logger.org.springframework=INFO 12 | 13 | log4j.logger.org.apache.activemq=INFO 14 | 15 | log4j.logger.ro.isdc.wro=INFO 16 | 17 | log4j.logger.org.glassfish.jersey=DEBUG 18 | 19 | 20 | # CONSOLE appender not used by default 21 | log4j.appender.out=org.apache.log4j.ConsoleAppender 22 | log4j.appender.out.layout=org.apache.log4j.PatternLayout 23 | #log4j.appender.out.layout.ConversionPattern=[%30.30t] %-30.30c{1} %-5p %m%n 24 | log4j.appender.out.layout.ConversionPattern=%d [%-30.30t] %-5p %-30.30c{1} - %m%n 25 | 26 | log4j.throwableRenderer=org.apache.log4j.EnhancedThrowableRenderer 27 | --------------------------------------------------------------------------------