├── .gitignore ├── LICENSE ├── README.md ├── elasticsearch-core ├── pom.xml └── src │ ├── main │ ├── assembly │ │ └── distribution.xml │ ├── bin │ │ └── ctl.sh │ ├── java │ │ └── zx │ │ │ └── soft │ │ │ └── elasticsearch_core │ │ │ └── App.java │ └── resources │ │ └── logback.xml │ └── test │ ├── java │ └── zx │ │ └── soft │ │ └── elasticsearch_core │ │ └── AppTest.java │ └── resources │ └── logback-test.xml ├── elasticsearch-dao ├── pom.xml └── src │ ├── main │ └── resources │ │ └── logback.xml │ └── test │ └── resources │ └── logback-test.xml ├── elasticsearch-jdbc ├── .travis.yml ├── README.md ├── bin │ ├── feeder.in.sh │ ├── feeder │ │ ├── h2 │ │ │ └── create.sh │ │ ├── mysql │ │ │ ├── create.sh │ │ │ ├── geo.dump │ │ │ └── geo.sh │ │ └── oracle │ │ │ └── create.sh │ ├── river │ │ ├── mysql │ │ │ ├── create.sh │ │ │ ├── delete.sh │ │ │ ├── geo.sh │ │ │ └── schedule.sh │ │ ├── oracle │ │ │ └── create.sh │ │ └── postgresql │ │ │ └── create.sh │ └── travis │ │ └── run-tests.sh ├── pom.xml └── src │ ├── main │ ├── assemblies │ │ └── plugin.xml │ ├── java │ │ └── org │ │ │ └── xbib │ │ │ ├── cron │ │ │ ├── CronExecutorService.java │ │ │ ├── CronExpression.java │ │ │ └── CronThreadPoolExecutor.java │ │ │ ├── elasticsearch │ │ │ ├── action │ │ │ │ └── river │ │ │ │ │ └── jdbc │ │ │ │ │ ├── execute │ │ │ │ │ ├── NodeRiverExecuteRequest.java │ │ │ │ │ ├── NodeRiverExecuteResponse.java │ │ │ │ │ ├── RiverExecuteAction.java │ │ │ │ │ ├── RiverExecuteRequest.java │ │ │ │ │ ├── RiverExecuteRequestBuilder.java │ │ │ │ │ ├── RiverExecuteResponse.java │ │ │ │ │ ├── RunnableRiver.java │ │ │ │ │ └── TransportRiverExecuteAction.java │ │ │ │ │ └── state │ │ │ │ │ ├── RiverState.java │ │ │ │ │ ├── RiverStateModule.java │ │ │ │ │ ├── RiverStateService.java │ │ │ │ │ ├── RiverStatesMetaData.java │ │ │ │ │ ├── StatefulRiver.java │ │ │ │ │ ├── delete │ │ │ │ │ ├── DeleteRiverStateAction.java │ │ │ │ │ ├── DeleteRiverStateRequest.java │ │ │ │ │ ├── DeleteRiverStateRequestBuilder.java │ │ │ │ │ ├── DeleteRiverStateResponse.java │ │ │ │ │ └── TransportDeleteRiverStateAction.java │ │ │ │ │ ├── get │ │ │ │ │ ├── GetRiverStateAction.java │ │ │ │ │ ├── GetRiverStateRequest.java │ │ │ │ │ ├── GetRiverStateRequestBuilder.java │ │ │ │ │ ├── GetRiverStateResponse.java │ │ │ │ │ └── TransportGetRiverStateAction.java │ │ │ │ │ └── put │ │ │ │ │ ├── PutRiverStateAction.java │ │ │ │ │ ├── PutRiverStateRequest.java │ │ │ │ │ ├── PutRiverStateRequestBuilder.java │ │ │ │ │ ├── PutRiverStateResponse.java │ │ │ │ │ └── TransportPutRiverStateAction.java │ │ │ ├── plugin │ │ │ │ ├── feeder │ │ │ │ │ ├── AbstractFeeder.java │ │ │ │ │ ├── CommandLineInterpreter.java │ │ │ │ │ ├── Feeder.java │ │ │ │ │ ├── Runner.java │ │ │ │ │ └── jdbc │ │ │ │ │ │ └── JDBCFeeder.java │ │ │ │ ├── jdbc │ │ │ │ │ ├── ControlKeys.java │ │ │ │ │ ├── IndexableObject.java │ │ │ │ │ ├── LocaleUtil.java │ │ │ │ │ ├── PlainIndexableObject.java │ │ │ │ │ ├── PlainKeyValueStreamListener.java │ │ │ │ │ ├── RiverContext.java │ │ │ │ │ ├── RiverMouthKeyValueStreamListener.java │ │ │ │ │ ├── RiverServiceLoader.java │ │ │ │ │ ├── SQLCommand.java │ │ │ │ │ └── Values.java │ │ │ │ └── river │ │ │ │ │ └── jdbc │ │ │ │ │ ├── Build.java │ │ │ │ │ ├── JDBCRiver.java │ │ │ │ │ ├── JDBCRiverModule.java │ │ │ │ │ └── JDBCRiverPlugin.java │ │ │ ├── rest │ │ │ │ └── action │ │ │ │ │ └── river │ │ │ │ │ ├── execute │ │ │ │ │ └── RestRiverExecuteAction.java │ │ │ │ │ └── state │ │ │ │ │ └── get │ │ │ │ │ └── RestRiverGetStateAction.java │ │ │ ├── river │ │ │ │ └── jdbc │ │ │ │ │ ├── RiverFlow.java │ │ │ │ │ ├── RiverMouth.java │ │ │ │ │ ├── RiverSource.java │ │ │ │ │ └── strategy │ │ │ │ │ ├── column │ │ │ │ │ ├── ColumnRiverFeeder.java │ │ │ │ │ ├── ColumnRiverFlow.java │ │ │ │ │ ├── ColumnRiverMouth.java │ │ │ │ │ └── ColumnRiverSource.java │ │ │ │ │ └── simple │ │ │ │ │ ├── SimpleRiverFlow.java │ │ │ │ │ ├── SimpleRiverMouth.java │ │ │ │ │ └── SimpleRiverSource.java │ │ │ └── support │ │ │ │ ├── client │ │ │ │ ├── BaseIngestTransportClient.java │ │ │ │ ├── BaseTransportClient.java │ │ │ │ ├── ClientHelper.java │ │ │ │ ├── ConfigHelper.java │ │ │ │ ├── Feed.java │ │ │ │ ├── Ingest.java │ │ │ │ ├── State.java │ │ │ │ ├── bulk │ │ │ │ │ ├── BulkProcessorHelper.java │ │ │ │ │ └── BulkTransportClient.java │ │ │ │ └── node │ │ │ │ │ └── BulkNodeClient.java │ │ │ │ └── river │ │ │ │ └── RiverHelper.java │ │ │ ├── io │ │ │ └── URIUtil.java │ │ │ ├── keyvalue │ │ │ ├── KeyValue.java │ │ │ ├── KeyValueReader.java │ │ │ ├── KeyValueStreamAdapter.java │ │ │ └── KeyValueStreamListener.java │ │ │ └── pipeline │ │ │ ├── AbstractPipeline.java │ │ │ ├── Pipeline.java │ │ │ ├── PipelineErrorListener.java │ │ │ ├── PipelineException.java │ │ │ ├── PipelineExecutor.java │ │ │ ├── PipelineProvider.java │ │ │ ├── PipelineRequest.java │ │ │ ├── PipelineRequestListener.java │ │ │ ├── PipelineSink.java │ │ │ ├── element │ │ │ ├── LongPipelineElement.java │ │ │ ├── MapPipelineElement.java │ │ │ ├── PipelineElement.java │ │ │ └── URIPipelineElement.java │ │ │ ├── package-info.java │ │ │ ├── queue │ │ │ └── QueuePipelineExecutor.java │ │ │ └── simple │ │ │ └── SimplePipelineExecutor.java │ └── resources │ │ ├── META-INF │ │ └── services │ │ │ ├── org.xbib.elasticsearch.river.jdbc.RiverFlow │ │ │ ├── org.xbib.elasticsearch.river.jdbc.RiverMouth │ │ │ └── org.xbib.elasticsearch.river.jdbc.RiverSource │ │ └── es-plugin.properties │ ├── site │ ├── resources │ │ ├── database-128.png │ │ ├── jdbc-river-feeder-architecture.png │ │ ├── simple-tabular-json-data.png │ │ └── tabular-json-data.png │ ├── site.xml │ └── wiki │ │ └── cron.html │ └── test │ ├── java │ └── org │ │ └── xbib │ │ ├── cron │ │ └── CronTest.java │ │ └── elasticsearch │ │ ├── river │ │ ├── jdbc │ │ │ ├── Listener.java │ │ │ ├── strategy │ │ │ │ ├── column │ │ │ │ │ ├── ColumnRiverFlowTests.java │ │ │ │ │ └── ColumnRiverSourceTests.java │ │ │ │ ├── mock │ │ │ │ │ ├── MockRiverMouth.java │ │ │ │ │ ├── MockRiverSource.java │ │ │ │ │ └── RiverMockTests.java │ │ │ │ └── simple │ │ │ │ │ ├── RiverDataTests.java │ │ │ │ │ ├── RiverJobTests.java │ │ │ │ │ ├── RiverScheduleTests.java │ │ │ │ │ ├── RiverScriptTests.java │ │ │ │ │ ├── RiverSourceTests.java │ │ │ │ │ ├── RiverStoredProcedureTests.java │ │ │ │ │ └── storedprocedure │ │ │ │ │ └── StoredProcedureJavaDBSample.java │ │ │ └── support │ │ │ │ ├── StringKeyValueStreamListener.java │ │ │ │ ├── TimeWindowedTests.java │ │ │ │ ├── ValueListenerTests.java │ │ │ │ └── ValuesTests.java │ │ └── plugin │ │ │ └── SQLCommandTests.java │ │ └── support │ │ └── helper │ │ ├── AbstractNodeTestHelper.java │ │ └── AbstractRiverNodeTest.java │ └── resources │ ├── log4j.properties │ ├── log4j2.xml │ ├── org │ └── xbib │ │ └── elasticsearch │ │ └── river │ │ └── jdbc │ │ └── strategy │ │ ├── column │ │ ├── derby │ │ │ ├── create-producttables.sql │ │ │ ├── delete-producttables.sql │ │ │ ├── river-existedWhereClause.json │ │ │ ├── river-existedWhereClauseWithOverlap.json │ │ │ ├── river-sqlForTestDeletions.json │ │ │ ├── river-sqlForTestDeletionsAndWherePlaceholder.json │ │ │ ├── river-sqlparams.json │ │ │ └── river-whereClausePlaceholder.json │ │ ├── h2 │ │ │ ├── create-producttables.sql │ │ │ ├── delete-producttables.sql │ │ │ ├── river-existedWhereClause.json │ │ │ ├── river-existedWhereClauseWithOverlap.json │ │ │ ├── river-sqlForTestDeletions.json │ │ │ ├── river-sqlForTestDeletionsAndWherePlaceholder.json │ │ │ ├── river-sqlparams.json │ │ │ └── river-whereClausePlaceholder.json │ │ ├── hsqldb │ │ │ ├── create-producttables.sql │ │ │ ├── delete-producttables.sql │ │ │ ├── river-existedWhereClause.json │ │ │ ├── river-existedWhereClauseWithOverlap.json │ │ │ ├── river-sqlForTestDeletions.json │ │ │ ├── river-sqlForTestDeletionsAndWherePlaceholder.json │ │ │ ├── river-sqlparams.json │ │ │ └── river-whereClausePlaceholder.json │ │ ├── mysql │ │ │ ├── create-producttables.sql │ │ │ ├── delete-producttables.sql │ │ │ ├── river-existedWhereClause.json │ │ │ ├── river-existedWhereClauseWithOverlap.json │ │ │ ├── river-sqlForTestDeletions.json │ │ │ ├── river-sqlForTestDeletionsAndWherePlaceholder.json │ │ │ ├── river-sqlparams.json │ │ │ └── river-whereClausePlaceholder.json │ │ └── postgresql │ │ │ ├── create-producttables.sql │ │ │ ├── delete-producttables.sql │ │ │ ├── river-existedWhereClause.json │ │ │ ├── river-existedWhereClauseWithOverlap.json │ │ │ ├── river-sqlForTestDeletions.json │ │ │ ├── river-sqlForTestDeletionsAndWherePlaceholder.json │ │ │ ├── river-sqlparams.json │ │ │ └── river-whereClausePlaceholder.json │ │ └── simple │ │ ├── derby │ │ ├── create-jobtables.sql │ │ ├── create-ordertables.sql │ │ ├── delete-jobtables.sql │ │ ├── delete-ordertables.sql │ │ ├── jobriver-1.json │ │ ├── river-1.json │ │ ├── river-2.json │ │ ├── river-3.json │ │ ├── river-5.json │ │ ├── river-6.json │ │ └── river-7.json │ │ ├── h2 │ │ ├── create-jobtables.sql │ │ ├── create-ordertables.sql │ │ ├── delete-jobtables.sql │ │ ├── delete-ordertables.sql │ │ ├── jobriver-1.json │ │ ├── river-1.json │ │ ├── river-2.json │ │ ├── river-3.json │ │ ├── river-6.json │ │ └── river-7.json │ │ ├── hsqldb │ │ ├── create-jobtables.sql │ │ ├── create-ordertables.sql │ │ ├── delete-jobtables.sql │ │ ├── delete-ordertables.sql │ │ ├── jobriver-1.json │ │ ├── river-1.json │ │ ├── river-2.json │ │ ├── river-3.json │ │ ├── river-6.json │ │ └── river-7.json │ │ ├── mysql │ │ ├── create-jobtables.sql │ │ ├── create-ordertables.sql │ │ ├── create-suppliertables.sql │ │ ├── delete-jobtables.sql │ │ ├── delete-ordertables.sql │ │ ├── delete-suppliertables.sql │ │ ├── jobriver-1.json │ │ ├── river-1.json │ │ ├── river-2.json │ │ ├── river-3.json │ │ ├── river-6.json │ │ ├── river-7.json │ │ ├── river-8.json │ │ └── river-9.json │ │ ├── oracle │ │ ├── create-jobtables.sql │ │ ├── create-ordertables.sql │ │ ├── delete-jobtables.sql │ │ ├── delete-ordertables.sql │ │ ├── jobriver-1.json │ │ ├── river-1.json │ │ ├── river-2.json │ │ └── river-3.json │ │ ├── postgresql │ │ ├── create-jobtables.sql │ │ ├── create-ordertables.sql │ │ ├── delete-jobtables.sql │ │ ├── delete-ordertables.sql │ │ ├── jobriver-1.json │ │ ├── river-1.json │ │ ├── river-2.json │ │ └── river-3.json │ │ └── sqlite │ │ ├── create-jobtables.sql │ │ ├── create-ordertables.sql │ │ ├── delete-jobtables.sql │ │ ├── delete-ordertables.sql │ │ ├── jobriver-1.json │ │ ├── river-1.json │ │ ├── river-2.json │ │ └── river-3.json │ └── testsuite │ ├── column │ ├── derby.xml │ ├── h2.xml │ ├── hsqldb.xml │ ├── mysql.xml │ └── postgresql.xml │ ├── cron │ └── cron.xml │ ├── mock │ ├── derby.xml │ ├── h2.xml │ ├── hsqldb.xml │ ├── mysql.xml │ ├── oracle.xml │ └── postgresql.xml │ ├── plugin │ └── plugin.xml │ ├── simple │ ├── derby.xml │ ├── h2.xml │ ├── hsqldb.xml │ ├── mysql.xml │ ├── oracle.xml │ ├── postgresql.xml │ └── sqlite.xml │ └── support │ └── support.xml ├── elasticsearch-parent ├── README.md └── pom.xml ├── elasticsearch-web ├── README.md ├── pom.xml └── src │ ├── main │ ├── assembly │ │ └── distribution.xml │ ├── bin │ │ └── ctl.sh │ ├── java │ │ └── zx │ │ │ └── soft │ │ │ └── es │ │ │ ├── client │ │ │ └── BuildClient.java │ │ │ ├── demo │ │ │ └── WeiboData.java │ │ │ ├── driver │ │ │ └── ElasticSearchDriver.java │ │ │ ├── model │ │ │ ├── PostData.java │ │ │ ├── RequestParams.java │ │ │ ├── SearchParams.java │ │ │ └── Weibo.java │ │ │ ├── search │ │ │ ├── Index.java │ │ │ └── Search.java │ │ │ ├── utils │ │ │ ├── ConfigUtil.java │ │ │ ├── IndexResponse.java │ │ │ ├── JsonUtils.java │ │ │ └── URLUtils.java │ │ │ └── web │ │ │ ├── application │ │ │ ├── IndexApplication.java │ │ │ └── SearchApplication.java │ │ │ ├── jackson │ │ │ └── ReplaceConvert.java │ │ │ ├── resource │ │ │ ├── IndexServerResource.java │ │ │ └── SearchServerResource.java │ │ │ └── server │ │ │ ├── ESIndexServer.java │ │ │ └── ESSearchServer.java │ └── resources │ │ ├── elastic_search.properties │ │ ├── logback.xml │ │ └── web-server.properties │ └── test │ └── resources │ └── logback-test.xml └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | data/ 2 | work/ 3 | logs/ 4 | .idea 5 | target/ 6 | .DS_Store 7 | *.iml 8 | .settings/ 9 | .classpath 10 | .project 11 | *~ 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # 舆情搜索统计服务项目 3 | 4 | > 基于Java实现。 5 | 6 | ## 项目内容 7 | 8 | - [项目简介](#项目简介) 9 | - [项目架构](#项目架构) 10 | - [开发人员](#开发人员) 11 | 12 | ---- 13 | 14 | ## 项目简介 15 | 16 | ### 项目起因 17 | 18 | 本项目主要服务于舆情实时数据的搜索统计分析。 19 | 20 | ### 项目框架 21 | 22 | `elasticsearch-parent`: jar和插件依赖工程 23 | 24 | `elasticsearch-dao`: 数据接口层 25 | 26 | `elasticsearch-redis`: 数据缓存层或消息队列层 27 | 28 | `elasticsearch-web`: API接口服务层 29 | 30 | `elasticsearch-core`: 核心业务层 31 | 32 | > **备注:** 框架持续更新中。 33 | 34 | ### API文档 35 | [项目wiki](http://192.168.3.23/wiki) 36 | 37 | > **备注:** API文档统一放在公司的wiki上。 38 | 39 | ---- 40 | 41 | ## 项目架构 42 | 43 | 1. MySQL： 用于存储基本爬虫数据。 44 | 2. Redis： 用于数据去重，基本思想是存储每条数据的md5(key)，根据该值进行数据插入更新判断。 45 | 3. ElasticSearch： 用于提供分布式实时统计计算服务。 46 | 47 | ### 常见约束词 48 | Item | Value 49 | --------- | ----- 50 | Dao | Interface接口层 51 | Domain | 数据 52 | Constant | 常量 53 | Util | 工具 54 | 55 | ### 示例代码 56 | 57 | ```java 58 | ** Redis层调用 ** 59 | // 对象申明 60 | RedisCache redisCache = new RedisCache("hdp321", 6379, "zxsoft"); 61 | String key = "record_key_md5"; 62 | String[] members = { "v1", "v2", "v3", "v4", "v5", "v3" }; 63 | redisCache.sadd(key, members); 64 | System.out.println(redisCache.scard(key)); 65 | System.out.println(redisCache.sismember(key, "v3")); 66 | System.out.println(redisCache.sismember(key, "v6")); 67 | redisCache.sadd(key, "v5", "v7"); 68 | System.out.println(redisCache.scard(key)); 69 | System.out.println(redisCache.smembers(key)); 70 | 71 | ** 启动脚本 ** 72 | 73 | ``` 74 | 75 | ### 开发人员 76 | 77 | WeChat: wgybzb 78 | 79 | QQ: 1010437118 80 | 81 | E-mail: wgybzb@sina.cn 82 | 83 | -------------------------------------------------------------------------------- /elasticsearch-core/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | zx.soft 7 | elasticsearch-parent 8 | 1.0.0 9 | ../elasticsearch-parent/pom.xml 10 | 11 | 12 | elasticsearch-core 13 | ElasticSearch Core 14 | 15 | 16 | UTF-8 17 | 18 | 19 | 20 | 21 | junit 22 | junit 23 | test 24 | 25 | 26 | 27 | 28 | 29 | 30 | org.apache.maven.plugins 31 | maven-compiler-plugin 32 | 33 | 34 | org.apache.maven.plugins 35 | maven-source-plugin 36 | 37 | 38 | org.apache.maven.plugins 39 | maven-resources-plugin 40 | 41 | 42 | org.apache.maven.plugins 43 | maven-surefire-plugin 44 | 45 | 46 | org.apache.maven.plugins 47 | maven-jar-plugin 48 | 2.4 49 | 50 | 51 | elastic_search.properties 52 | logback.xml 53 | web-server.properties 54 | 55 | 56 | 57 | 58 | maven-assembly-plugin 59 | 60 | 61 | src/main/assembly/distribution.xml 62 | 63 | 64 | 65 | 66 | make-assembly 67 | package 68 | 69 | single 70 | 71 | 72 | 73 | 74 | 75 | org.codehaus.mojo 76 | exec-maven-plugin 77 | 78 | 79 | 80 | ${project.artifactId}-${project.version} 81 | 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /elasticsearch-core/src/main/assembly/distribution.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | distribution 7 | 8 | tar.gz 9 | 10 | ${project.artifactId} 11 | 12 | 13 | src/main/resources 14 | 15 | elastic_search.properties 16 | logback.xml 17 | web-server.properties 18 | 19 | /conf 20 | true 21 | 22 | 23 | src/main/bin 24 | 25 | * 26 | 27 | /bin 28 | 0755 29 | 30 | 31 | 32 | 33 | /lib 34 | 35 | 36 | -------------------------------------------------------------------------------- /elasticsearch-core/src/main/bin/ctl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mainClass=zx.soft.es.driver.ElasticSearchDriver 4 | 5 | # resolve links - $0 may be a softlink 6 | PRG="$0" 7 | 8 | while [ -h "$PRG" ]; do 9 | ls=`ls -ld "$PRG"` 10 | link=`expr "$ls" : '.*-> \(.*\)$'` 11 | if expr "$link" : '/.*' > /dev/null; then 12 | PRG="$link" 13 | else 14 | PRG=`dirname "$PRG"`/"$link" 15 | fi 16 | done 17 | 18 | # Get standard environment variables 19 | PRGDIR=`dirname "$PRG"` 20 | 21 | PROJECT_DIR=`cd "$PRGDIR/.." >/dev/null; pwd` 22 | echo PROJECT_DIR=$PROJECT_DIR 23 | 24 | CLASSPATH="$CLASSHPATH:$PROJECT_DIR/conf" 25 | 26 | for jar in "$PROJECT_DIR/lib"/*.jar; do 27 | CLASSPATH="$CLASSPATH:$jar" 28 | done 29 | echo CLASSPATH=$CLASSPATH 30 | 31 | JVMARGS="${JVMARGS} -Dproject_dir=${PROJECT_DIR}" 32 | echo JVMARGS=$JVMARGS 33 | 34 | usage() { 35 | echo >&2 "usage: $PRG [args]" 36 | echo 'Valid commands: start, stop' 37 | exit 1 38 | } 39 | 40 | start() { 41 | JAVA=${JAVA-'java'} 42 | exec $JAVA $JVMARGS -classpath "$CLASSPATH" $mainClass "$@" & 43 | } 44 | 45 | case $1 in 46 | (start) 47 | shift 48 | start $@ 49 | ;; 50 | (stop) 51 | echo "stop" 52 | ;; 53 | (restart) 54 | echo "restart" 55 | ;; 56 | (*) 57 | echo >&2 "$PRG: error: unknown command '$1'" 58 | usage 59 | ;; 60 | esac 61 | -------------------------------------------------------------------------------- /elasticsearch-core/src/main/java/zx/soft/elasticsearch_core/App.java: -------------------------------------------------------------------------------- 1 | package zx.soft.elasticsearch_core; 2 | 3 | /** 4 | * Hello world! 5 | * 6 | */ 7 | public class App 8 | { 9 | public static void main( String[] args ) 10 | { 11 | System.out.println( "Hello World!" ); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /elasticsearch-core/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 8 | 9 | 10 | 11 | INFO 12 | ACCEPT 13 | DENY 14 | 15 | 16 | 17 | 19 | logs/storm-analysis.log 20 | 21 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 22 | 23 | 24 | 25 | INFO 26 | 27 | 28 | logs/storm-analysis.log.%d{yyyy-MM-dd}.tar.gz 29 | 30 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /elasticsearch-core/src/test/java/zx/soft/elasticsearch_core/AppTest.java: -------------------------------------------------------------------------------- 1 | package zx.soft.elasticsearch_core; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /elasticsearch-core/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{MMdd.HHmmss.SSS} [%-20t] [%-5p] [%-20c] [L:%-3L] - %m%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /elasticsearch-dao/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | zx.soft 7 | elasticsearch-parent 8 | 1.0.0 9 | ../elasticsearch-parent/pom.xml 10 | 11 | 12 | elasticsearch-dao 13 | ElasticSearch Dao 14 | 15 | 16 | UTF-8 17 | 18 | 19 | 20 | 21 | junit 22 | junit 23 | test 24 | 25 | 26 | 27 | 28 | 29 | 30 | org.apache.maven.plugins 31 | maven-compiler-plugin 32 | 33 | 34 | org.apache.maven.plugins 35 | maven-source-plugin 36 | 37 | 38 | org.apache.maven.plugins 39 | maven-resources-plugin 40 | 41 | 42 | org.apache.maven.plugins 43 | maven-surefire-plugin 44 | 45 | 46 | org.codehaus.mojo 47 | exec-maven-plugin 48 | 49 | 50 | 51 | ${project.artifactId}-${project.version} 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /elasticsearch-dao/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 8 | 9 | 10 | 11 | INFO 12 | ACCEPT 13 | DENY 14 | 15 | 16 | 17 | 19 | logs/storm-analysis.log 20 | 21 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 22 | 23 | 24 | 25 | INFO 26 | 27 | 28 | logs/storm-analysis.log.%d{yyyy-MM-dd}.tar.gz 29 | 30 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /elasticsearch-dao/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{MMdd.HHmmss.SSS} [%-20t] [%-5p] [%-20c] [L:%-3L] - %m%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | script: ./bin/travis/run-tests.sh 3 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/bin/feeder.in.sh: -------------------------------------------------------------------------------- 1 | 2 | # Configuration for Elasticsearch JDBC feeder mechanism 3 | 4 | # Java home 5 | 6 | # for Mac OS X 7 | #JAVA_HOME=$(/usr/libexec/java_home -v 1.7*) 8 | JAVA_HOME=$(/usr/libexec/java_home -v 1.8*) 9 | # for Linux 10 | #JAVA_HOME"/etc/alternatives/java" 11 | 12 | # Elasticsearch home 13 | ES_HOME=~es/elasticsearch-1.3.0 14 | 15 | # Elasticsearch plugins folder where "jdbc" plugin is installed 16 | ES_PATH_PLUGINS=${ES_HOME}/plugins 17 | 18 | # Classpath for loading JDBC plugin from external Java execution, without other plugins. 19 | # 20 | # The classpath is very similar to Elasticsearch classpath, but it must follow these rules: 21 | # - first, the elasticsearch*.jar in elasticsearch "lib" folder 22 | # - the other jars in elasticsearch "lib" folder 23 | # - the plugins/jdbc folder for log4j.properties (or log4j2.xml) 24 | # - the plugins/jdbc jars (plugin jar and JDBC driver jars) 25 | # - no more, no other (server-side) plugins etc. ! 26 | ES_JDBC_CLASSPATH=${ES_HOME}/lib/elasticsearch\*:${ES_HOME}/lib/\*:${ES_PATH_PLUGINS}/jdbc:${ES_PATH_PLUGINS}/jdbc/\* 27 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/bin/feeder/h2/create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | java="/usr/bin/java" 4 | #java="/Library/Java/JavaVirtualMachines/jdk1.8.0.jdk/Contents/Home/bin/java" 5 | #java="/usr/java/jdk1.8.0/bin/java" 6 | 7 | echo ' 8 | { 9 | "concurrency" : 1, 10 | "elasticsearch" : "es://localhost:9300?es.cluster.name=elasticsearch", 11 | "client" : "ingest", 12 | "index" : "myh2", 13 | "type" : "myh2", 14 | "jdbc" : [ 15 | { 16 | "url" : "jdbc:h2:test", 17 | "user" : "", 18 | "password" : "", 19 | "sql" : [ 20 | { 21 | "statement" : "select *, created as _id, \"myjdbc\" as _index, \"mytype\" as _type from \"orders\"" 22 | } 23 | ] 24 | } 25 | ] 26 | } 27 | ' | ${java} \ 28 | -cp $(pwd):$(pwd)/\*:$(pwd)/../../lib/\* \ 29 | org.xbib.elasticsearch.plugin.feeder.Runner \ 30 | org.xbib.elasticsearch.plugin.feeder.jdbc.JDBCFeeder 31 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/bin/feeder/mysql/create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # This example shows two concurrent feeds from a MySQL database (conncurreny = 2) 4 | # It is possible to connect to many databases in parallel and fetch data for Elasticsearch. 5 | 6 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 7 | . ${DIR}/../../feeder.in.sh 8 | 9 | echo ' 10 | { 11 | "concurrency" : 2, 12 | "elasticsearch" : "es://localhost:9300?es.cluster.name=elasticsearch", 13 | "client" : "bulk", 14 | "jdbc" : [ 15 | { 16 | "url" : "jdbc:mysql://localhost:3306/test", 17 | "user" : "", 18 | "password" : "", 19 | "sql" : [ 20 | { 21 | "statement" : "select *, created as _id, \"myjdbc\" as _index, \"mytype\" as _type from orders" 22 | } 23 | ], 24 | "index" : "myjdbc", 25 | "type" : "mytype", 26 | "index_settings" : { 27 | "index" : { 28 | "number_of_shards" : 1 29 | } 30 | } 31 | }, 32 | { 33 | "url" : "jdbc:mysql://localhost:3306/test", 34 | "user" : "", 35 | "password" : "", 36 | "sql" : [ 37 | { 38 | "statement" : "select *, name as _id, \"myproducts\" as _index, \"myproducts\" as _type from products" 39 | } 40 | ] 41 | } 42 | ] 43 | } 44 | ' | ${JAVA_HOME}/bin/java \ 45 | -cp ${ES_JDBC_CLASSPATH} \ 46 | org.xbib.elasticsearch.plugin.feeder.Runner \ 47 | org.xbib.elasticsearch.plugin.feeder.jdbc.JDBCFeeder 48 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/bin/feeder/mysql/geo.dump: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 5.6.16, for osx10.7 (x86_64) 2 | -- 3 | -- Host: localhost Database: test 4 | -- ------------------------------------------------------ 5 | -- Server version 5.6.16 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Table structure for table `geo` 20 | -- 21 | 22 | DROP TABLE IF EXISTS `geo`; 23 | /*!40101 SET @saved_cs_client = @@character_set_client */; 24 | /*!40101 SET character_set_client = utf8 */; 25 | CREATE TABLE `geo` ( 26 | `lat` double DEFAULT NULL, 27 | `lon` double DEFAULT NULL, 28 | `id` int(11) DEFAULT NULL, 29 | `zip` varchar(255) DEFAULT NULL, 30 | `name` varchar(255) DEFAULT NULL, 31 | `address` varchar(255) DEFAULT NULL, 32 | `city` varchar(255) DEFAULT NULL 33 | ) ENGINE=InnoDB DEFAULT CHARSET=latin1; 34 | /*!40101 SET character_set_client = @saved_cs_client */; 35 | 36 | -- 37 | -- Dumping data for table `geo` 38 | -- 39 | 40 | LOCK TABLES `geo` WRITE; 41 | /*!40000 ALTER TABLE `geo` DISABLE KEYS */; 42 | INSERT INTO `geo` VALUES (50.9406645,6.9599115,NULL,'50667','Dom','Domkloster 4','Köln'); 43 | /*!40000 ALTER TABLE `geo` ENABLE KEYS */; 44 | UNLOCK TABLES; 45 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 46 | 47 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 48 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 49 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 50 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 51 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 52 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 53 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 54 | 55 | -- Dump completed on 2014-05-10 11:35:27 56 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/bin/feeder/oracle/create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # This example is a template to connect to Oracle in feeder mode. 4 | # The JDBC URL and SQL must be replaced by working ones. 5 | 6 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 7 | . ${DIR}/../../feeder.in.sh 8 | 9 | echo ' 10 | { 11 | "concurrency" : 1, 12 | "elasticsearch" : "es://localhost:9300?es.cluster.name=elasticsearch", 13 | "client" : "bulk", 14 | "jdbc" : { 15 | "url" : "jdbc:oracle:thin:@//host:1521/sid", 16 | "user" : "user", 17 | "password" : "password", 18 | "sql" : "select or_id as \"_id\", or_tan as \"tan\" from orders", 19 | "index" : "myoracle", 20 | "type" : "myoracle", 21 | "index_settings" : { 22 | "index" : { 23 | "number_of_shards" : 1, 24 | "number_of_replica" : 0 25 | } 26 | } 27 | } 28 | } 29 | ' | ${JAVA_HOME}/bin/java \ 30 | -cp ${ES_JDBC_CLASSPATH} \ 31 | org.xbib.elasticsearch.plugin.feeder.Runner \ 32 | org.xbib.elasticsearch.plugin.feeder.jdbc.JDBCFeeder 33 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/bin/river/mysql/create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | curl -XPUT '0:9200/_river/my_mysql_river/_meta' -d '{ 4 | "type" : "jdbc", 5 | "jdbc" : { 6 | "url" : "jdbc:mysql://localhost:3306/test", 7 | "user" : "", 8 | "password" : "", 9 | "sql" : "select *, created as _id from orders", 10 | "maxbulkactions" : 10 11 | } 12 | }' 13 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/bin/river/mysql/delete.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | curl -XPUT '0:9200/_river/my_mysql_river/_meta' -d '{ 4 | "type" : "jdbc", 5 | "jdbc" : { 6 | "url" : "jdbc:mysql://localhost:3306/test", 7 | "user" : "", 8 | "password" : "", 9 | "sql" : "select deletethisdoc as _id, delete as _optype from orders" 10 | } 11 | }' 12 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/bin/river/mysql/geo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # a complete minimalistic river example for MySQL geo -> Elasticsearch geo search 4 | 5 | # - install MySQL e.g. in /usr/local/mysql 6 | # - start MySQL on localhost:3306 (default) 7 | # - prepare a 'test' database in MySQL 8 | # - create empty user '' with empty password '' 9 | # - execute SQL in "geo.dump" /usr/local/mysql/bin/mysql test < src/test/resources/geo.dump 10 | # - then run this script: bash bin/river/mysql/geo.sh 11 | 12 | curl -XDELETE 'localhost:9200/_river/my_geo_river/' 13 | 14 | curl -XGET 'localhost:9200/_river/_refresh' 15 | 16 | curl -XDELETE 'localhost:9200/myjdbc' 17 | 18 | curl -XPOST 'localhost:9200/_river/my_geo_river/_meta' -d ' 19 | { 20 | "type" : "jdbc", 21 | "jdbc" : { 22 | "url" : "jdbc:mysql://localhost:3306/test", 23 | "user" : "", 24 | "password" : "", 25 | "locale" : "en_US", 26 | "sql" : [ 27 | { 28 | "statement" : "select \"myjdbc\" as _index, \"mytype\" as _type, name as _id, city, zip, address, lat as \"location.lat\", lon as \"location.lon\" from geo" 29 | } 30 | ], 31 | "index" : "myjdbc", 32 | "type" : "mytype", 33 | "index_settings" : { 34 | "index" : { 35 | "number_of_shards" : 1 36 | } 37 | }, 38 | "type_mapping": { 39 | "mytype" : { 40 | "properties" : { 41 | "location" : { 42 | "type" : "geo_point" 43 | } 44 | } 45 | } 46 | } 47 | } 48 | } 49 | ' 50 | 51 | echo "sleeping while river should run..." 52 | 53 | sleep 15 54 | 55 | curl -XDELETE 'localhost:9200/_river/my_geo_river/' 56 | 57 | #curl -XGET 'localhost:9200/myjdbc/_refresh' 58 | 59 | curl -XPOST 'localhost:9200/myjdbc/_search?pretty' -d ' 60 | { 61 | "query": { 62 | "filtered": { 63 | "query": { 64 | "match_all": { 65 | } 66 | }, 67 | "filter": { 68 | "geo_distance" : { 69 | "distance" : "20km", 70 | "location" : { 71 | "lat" : 51.0, 72 | "lon" : 7.0 73 | } 74 | } 75 | } 76 | } 77 | } 78 | }' 79 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/bin/river/mysql/schedule.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | curl -XPUT '0:9200/_river/my_mysql_river/_meta' -d '{ 4 | "type" : "jdbc", 5 | "jdbc" : { 6 | "strategy" : "simple", 7 | "url" : "jdbc:mysql://localhost:3306/test", 8 | "user" : "", 9 | "password" : "", 10 | "schedule" : "0 0-59 0-23 ? * *", 11 | "sql" : [ 12 | { 13 | "statement" : "select *, created as _id, \"myjdbc\" as _index, \"mytype\" as _type from orders" 14 | } 15 | ], 16 | "index" : "myjdbc", 17 | "type" : "mytype", 18 | "index_settings" : { 19 | "index" : { 20 | "number_of_shards" : 1 21 | } 22 | } 23 | } 24 | }' 25 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/bin/river/oracle/create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | curl -XPUT 'localhost:9200/_river/my_oracle_river/_meta' -d '{ 4 | "type" : "jdbc", 5 | "jdbc" : { 6 | "url" : "jdbc:oracle:thin:@//localhost:1521/DB", 7 | "user" : "user", 8 | "password" : "password", 9 | "sql" : "select * from orders" 10 | } 11 | }' 12 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/bin/river/postgresql/create.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | curl -XPUT 'localhost:9200/_river/my_postgresql_river/_meta' -d '{ 4 | "type" : "jdbc", 5 | "jdbc" : { 6 | "url" : "jdbc:postgresql://localhost:5432/test?loglevel=0", 7 | "user" : "test", 8 | "password" : "test", 9 | "sql" : "select * from large_table", 10 | "index_settings" : { 11 | "index" : { 12 | "number_of_shards" : 10 13 | } 14 | } 15 | } 16 | }' 17 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/bin/travis/run-tests.sh: -------------------------------------------------------------------------------- 1 | mvn clean test -P test-derby 2 | mvn clean test -P test-h2 3 | mvn clean test -P test-hsqldb 4 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/assemblies/plugin.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | plugin 4 | 5 | zip 6 | 7 | false 8 | 9 | 10 | / 11 | true 12 | false 13 | runtime 14 | 15 | org.xbib.elasticsearch.plugin 16 | 17 | 18 | 19 | 20 | 21 | ${project.build.directory}/releases 22 | / 23 | 24 | *-uberjar.jar 25 | 26 | 27 | 28 | ${project.basedir} 29 | / 30 | 31 | bin/*.sh 32 | bin/**/*.sh 33 | 34 | 35 | 36 | ${project.basedir}/src/test/resources 37 | / 38 | 39 | log4j.properties 40 | log4j2.xml 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/cron/CronExecutorService.java: -------------------------------------------------------------------------------- 1 | package org.xbib.cron; 2 | 3 | import java.util.concurrent.ExecutorService; 4 | 5 | /** 6 | * Executor service that schedules a task for execution via a cron expression. 7 | */ 8 | public interface CronExecutorService extends ExecutorService { 9 | /** 10 | * Schedules the specified task to execute according to the specified cron expression. 11 | * 12 | * @param task the Runnable task to schedule 13 | * @param expression a cron expression 14 | */ 15 | void schedule(Runnable task, CronExpression expression); 16 | } 17 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/execute/NodeRiverExecuteRequest.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.execute; 2 | 3 | import org.elasticsearch.action.support.nodes.NodeOperationRequest; 4 | import org.elasticsearch.common.io.stream.StreamInput; 5 | import org.elasticsearch.common.io.stream.StreamOutput; 6 | 7 | import java.io.IOException; 8 | 9 | public class NodeRiverExecuteRequest extends NodeOperationRequest { 10 | 11 | private String riverType; 12 | 13 | private String riverName; 14 | 15 | NodeRiverExecuteRequest() { 16 | } 17 | 18 | public NodeRiverExecuteRequest(String nodeId, RiverExecuteRequest request) { 19 | super(request, nodeId); 20 | this.riverName = request.getRiverName(); 21 | this.riverType = request.getRiverType(); 22 | } 23 | 24 | public NodeRiverExecuteRequest setRiverType(String riverType) { 25 | this.riverType = riverType; 26 | return this; 27 | } 28 | 29 | public String getRiverType() { 30 | return riverType; 31 | } 32 | 33 | public NodeRiverExecuteRequest setRiverName(String riverName) { 34 | this.riverName = riverName; 35 | return this; 36 | } 37 | 38 | public String getRiverName() { 39 | return riverName; 40 | } 41 | 42 | @Override 43 | public void readFrom(StreamInput in) throws IOException { 44 | super.readFrom(in); 45 | this.riverName = in.readString(); 46 | this.riverType = in.readString(); 47 | } 48 | 49 | @Override 50 | public void writeTo(StreamOutput out) throws IOException { 51 | super.writeTo(out); 52 | out.writeString(riverName); 53 | out.writeString(riverType); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/execute/NodeRiverExecuteResponse.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.execute; 2 | 3 | import org.elasticsearch.action.support.nodes.NodeOperationResponse; 4 | import org.elasticsearch.cluster.node.DiscoveryNode; 5 | import org.elasticsearch.common.io.stream.StreamInput; 6 | import org.elasticsearch.common.io.stream.StreamOutput; 7 | 8 | import java.io.IOException; 9 | 10 | public class NodeRiverExecuteResponse extends NodeOperationResponse { 11 | 12 | private boolean executed; 13 | 14 | NodeRiverExecuteResponse() { 15 | } 16 | 17 | public NodeRiverExecuteResponse(DiscoveryNode node) { 18 | super(node); 19 | } 20 | 21 | public NodeRiverExecuteResponse setExecuted(boolean b) { 22 | this.executed = b; 23 | return this; 24 | } 25 | 26 | public boolean isExecuted() { 27 | return executed; 28 | } 29 | 30 | @Override 31 | public void readFrom(StreamInput in) throws IOException { 32 | super.readFrom(in); 33 | executed = in.readBoolean(); 34 | } 35 | 36 | @Override 37 | public void writeTo(StreamOutput out) throws IOException { 38 | super.writeTo(out); 39 | out.writeBoolean(executed); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/execute/RiverExecuteAction.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.execute; 2 | 3 | import org.elasticsearch.action.admin.cluster.ClusterAction; 4 | import org.elasticsearch.client.ClusterAdminClient; 5 | 6 | public class RiverExecuteAction extends ClusterAction { 7 | 8 | public static final RiverExecuteAction INSTANCE = new RiverExecuteAction(); 9 | 10 | public static final String NAME = "org.xbib.elasticsearch.action.river.execute.jdbc"; 11 | 12 | private RiverExecuteAction() { 13 | super(NAME); 14 | } 15 | 16 | @Override 17 | public RiverExecuteRequestBuilder newRequestBuilder(ClusterAdminClient client) { 18 | return new RiverExecuteRequestBuilder(client); 19 | } 20 | 21 | @Override 22 | public RiverExecuteResponse newResponse() { 23 | return new RiverExecuteResponse(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/execute/RiverExecuteRequest.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.execute; 2 | 3 | import org.elasticsearch.action.support.nodes.NodesOperationRequest; 4 | import org.elasticsearch.common.io.stream.StreamInput; 5 | import org.elasticsearch.common.io.stream.StreamOutput; 6 | 7 | import java.io.IOException; 8 | 9 | public class RiverExecuteRequest extends NodesOperationRequest { 10 | 11 | private String riverType; 12 | 13 | private String riverName; 14 | 15 | public RiverExecuteRequest setRiverType(String riverType) { 16 | this.riverType = riverType; 17 | return this; 18 | } 19 | 20 | public String getRiverType() { 21 | return riverType; 22 | } 23 | 24 | public RiverExecuteRequest setRiverName(String riverName) { 25 | this.riverName = riverName; 26 | return this; 27 | } 28 | 29 | public String getRiverName() { 30 | return riverName; 31 | } 32 | 33 | @Override 34 | public void readFrom(StreamInput in) throws IOException { 35 | super.readFrom(in); 36 | this.riverName = in.readString(); 37 | this.riverType = in.readString(); 38 | } 39 | 40 | @Override 41 | public void writeTo(StreamOutput out) throws IOException { 42 | super.writeTo(out); 43 | out.writeString(riverName); 44 | out.writeString(riverType); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/execute/RiverExecuteRequestBuilder.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.execute; 2 | 3 | import org.elasticsearch.action.ActionListener; 4 | import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder; 5 | import org.elasticsearch.client.ClusterAdminClient; 6 | 7 | public class RiverExecuteRequestBuilder extends NodesOperationRequestBuilder { 8 | 9 | public RiverExecuteRequestBuilder(ClusterAdminClient client) { 10 | super(client, new RiverExecuteRequest()); 11 | } 12 | 13 | public RiverExecuteRequestBuilder setRiverType(String riverType) { 14 | request.setRiverType(riverType); 15 | return this; 16 | } 17 | 18 | public RiverExecuteRequestBuilder setRiverName(String riverName) { 19 | request.setRiverName(riverName); 20 | return this; 21 | } 22 | 23 | @Override 24 | protected void doExecute(ActionListener listener) { 25 | client.execute(RiverExecuteAction.INSTANCE, request, listener); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/execute/RiverExecuteResponse.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.execute; 2 | 3 | import org.elasticsearch.action.support.nodes.NodesOperationResponse; 4 | import org.elasticsearch.common.io.stream.StreamInput; 5 | import org.elasticsearch.common.io.stream.StreamOutput; 6 | import org.elasticsearch.common.xcontent.ToXContent; 7 | import org.elasticsearch.common.xcontent.XContentBuilder; 8 | 9 | import java.io.IOException; 10 | 11 | public class RiverExecuteResponse extends NodesOperationResponse implements ToXContent { 12 | 13 | private boolean[] executed; 14 | 15 | public RiverExecuteResponse() { 16 | } 17 | 18 | public RiverExecuteResponse setExecuted(boolean[] b) { 19 | this.executed = b; 20 | return this; 21 | } 22 | 23 | public boolean[] isExecuted() { 24 | return executed; 25 | } 26 | 27 | @Override 28 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 29 | builder.field("executed", executed); 30 | return builder; 31 | } 32 | 33 | @Override 34 | public void readFrom(StreamInput in) throws IOException { 35 | super.readFrom(in); 36 | int len = in.readInt(); 37 | executed = new boolean[len]; 38 | for (int i = 0; i < len; i++) { 39 | executed[i] = in.readBoolean(); 40 | } 41 | } 42 | 43 | @Override 44 | public void writeTo(StreamOutput out) throws IOException { 45 | super.writeTo(out); 46 | out.writeInt(executed.length); 47 | for (boolean b : executed) { 48 | out.writeBoolean(b); 49 | } 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/execute/RunnableRiver.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.execute; 2 | 3 | import org.elasticsearch.river.River; 4 | 5 | public interface RunnableRiver extends River, Runnable { 6 | } 7 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/state/RiverStateModule.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.state; 2 | 3 | import org.elasticsearch.cluster.metadata.MetaData; 4 | import org.elasticsearch.common.inject.AbstractModule; 5 | 6 | public class RiverStateModule extends AbstractModule { 7 | 8 | /** 9 | * Register metadata factory in Elasticsearch 10 | */ 11 | static { 12 | MetaData.registerFactory(RiverStatesMetaData.TYPE, RiverStatesMetaData.FACTORY); 13 | } 14 | 15 | /** 16 | * Only one RiverStateService instance is allowed 17 | */ 18 | @Override 19 | protected void configure() { 20 | bind(RiverStateService.class).asEagerSingleton(); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/state/StatefulRiver.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.state; 2 | 3 | import org.elasticsearch.river.River; 4 | 5 | /** 6 | * A stateful river is an extension of a river with a river state 7 | */ 8 | public interface StatefulRiver extends River { 9 | 10 | RiverState getRiverState(); 11 | } 12 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/state/delete/DeleteRiverStateAction.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.state.delete; 2 | 3 | import org.elasticsearch.action.admin.cluster.ClusterAction; 4 | import org.elasticsearch.client.ClusterAdminClient; 5 | 6 | public class DeleteRiverStateAction extends ClusterAction { 7 | 8 | public static final DeleteRiverStateAction INSTANCE = new DeleteRiverStateAction(); 9 | 10 | public static final String NAME = "org.xbib.elasticsearch.action.river.state.delete.jdbc"; 11 | 12 | private DeleteRiverStateAction() { 13 | super(NAME); 14 | } 15 | 16 | @Override 17 | public DeleteRiverStateRequestBuilder newRequestBuilder(ClusterAdminClient client) { 18 | return new DeleteRiverStateRequestBuilder(client); 19 | } 20 | 21 | @Override 22 | public DeleteRiverStateResponse newResponse() { 23 | return new DeleteRiverStateResponse(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/state/delete/DeleteRiverStateRequest.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.state.delete; 2 | 3 | import org.elasticsearch.action.ActionRequestValidationException; 4 | import org.elasticsearch.action.support.master.AcknowledgedRequest; 5 | import org.elasticsearch.common.io.stream.StreamInput; 6 | import org.elasticsearch.common.io.stream.StreamOutput; 7 | 8 | import java.io.IOException; 9 | 10 | import static org.elasticsearch.action.ValidateActions.addValidationError; 11 | 12 | public class DeleteRiverStateRequest extends AcknowledgedRequest { 13 | 14 | private String riverName; 15 | 16 | public DeleteRiverStateRequest setRiverName(String riverName) { 17 | this.riverName = riverName; 18 | return this; 19 | } 20 | 21 | public String getRiverName() { 22 | return riverName; 23 | } 24 | 25 | @Override 26 | public ActionRequestValidationException validate() { 27 | ActionRequestValidationException validationException = null; 28 | if (riverName == null) { 29 | validationException = addValidationError("name is missing", null); 30 | } 31 | return validationException; 32 | } 33 | 34 | @Override 35 | public void readFrom(StreamInput in) throws IOException { 36 | super.readFrom(in); 37 | readTimeout(in); 38 | this.riverName = in.readString(); 39 | } 40 | 41 | @Override 42 | public void writeTo(StreamOutput out) throws IOException { 43 | super.writeTo(out); 44 | writeTimeout(out); 45 | out.writeString(riverName); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/state/delete/DeleteRiverStateRequestBuilder.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.state.delete; 2 | 3 | import org.elasticsearch.action.ActionListener; 4 | import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; 5 | import org.elasticsearch.client.ClusterAdminClient; 6 | 7 | public class DeleteRiverStateRequestBuilder extends AcknowledgedRequestBuilder { 8 | 9 | public DeleteRiverStateRequestBuilder(ClusterAdminClient client) { 10 | super(client, new DeleteRiverStateRequest()); 11 | } 12 | 13 | public DeleteRiverStateRequestBuilder setRiverName(String riverName) { 14 | request.setRiverName(riverName); 15 | return this; 16 | } 17 | 18 | @Override 19 | protected void doExecute(ActionListener listener) { 20 | client.execute(DeleteRiverStateAction.INSTANCE, request, listener); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/state/delete/DeleteRiverStateResponse.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.state.delete; 2 | 3 | import org.elasticsearch.action.support.master.AcknowledgedResponse; 4 | import org.elasticsearch.common.io.stream.StreamInput; 5 | import org.elasticsearch.common.io.stream.StreamOutput; 6 | 7 | import java.io.IOException; 8 | 9 | public class DeleteRiverStateResponse extends AcknowledgedResponse { 10 | 11 | 12 | public DeleteRiverStateResponse() { 13 | } 14 | 15 | public DeleteRiverStateResponse(boolean acknowledged) { 16 | super(acknowledged); 17 | } 18 | 19 | @Override 20 | public void readFrom(StreamInput in) throws IOException { 21 | super.readFrom(in); 22 | readAcknowledged(in); 23 | } 24 | 25 | @Override 26 | public void writeTo(StreamOutput out) throws IOException { 27 | super.writeTo(out); 28 | writeAcknowledged(out); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/state/get/GetRiverStateAction.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.state.get; 2 | 3 | import org.elasticsearch.action.admin.cluster.ClusterAction; 4 | import org.elasticsearch.client.ClusterAdminClient; 5 | 6 | public class GetRiverStateAction extends ClusterAction { 7 | 8 | public static final GetRiverStateAction INSTANCE = new GetRiverStateAction(); 9 | 10 | public static final String NAME = "org.xbib.elasticsearch.action.river.state.get.jdbc"; 11 | 12 | private GetRiverStateAction() { 13 | super(NAME); 14 | } 15 | 16 | @Override 17 | public GetRiverStateRequestBuilder newRequestBuilder(ClusterAdminClient client) { 18 | return new GetRiverStateRequestBuilder(client); 19 | } 20 | 21 | @Override 22 | public GetRiverStateResponse newResponse() { 23 | return new GetRiverStateResponse(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/state/get/GetRiverStateRequest.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.state.get; 2 | 3 | import org.elasticsearch.action.ActionRequestValidationException; 4 | import org.elasticsearch.action.support.master.MasterNodeReadOperationRequest; 5 | import org.elasticsearch.common.io.stream.StreamInput; 6 | import org.elasticsearch.common.io.stream.StreamOutput; 7 | 8 | import java.io.IOException; 9 | 10 | public class GetRiverStateRequest extends MasterNodeReadOperationRequest { 11 | 12 | private String riverName; 13 | 14 | private String riverType; 15 | 16 | public GetRiverStateRequest setRiverName(String riverName) { 17 | this.riverName = riverName; 18 | return this; 19 | } 20 | 21 | public String getRiverName() { 22 | return riverName; 23 | } 24 | 25 | public GetRiverStateRequest setRiverType(String riverType) { 26 | this.riverType = riverType; 27 | return this; 28 | } 29 | 30 | public String getRiverType() { 31 | return riverType; 32 | } 33 | 34 | @Override 35 | public ActionRequestValidationException validate() { 36 | return null; 37 | } 38 | 39 | @Override 40 | public void readFrom(StreamInput in) throws IOException { 41 | super.readFrom(in); 42 | this.riverName = in.readString(); 43 | this.riverType = in.readString(); 44 | } 45 | 46 | @Override 47 | public void writeTo(StreamOutput out) throws IOException { 48 | super.writeTo(out); 49 | out.writeString(riverName); 50 | out.writeString(riverType); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/state/get/GetRiverStateRequestBuilder.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.state.get; 2 | 3 | import org.elasticsearch.action.ActionListener; 4 | import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; 5 | import org.elasticsearch.client.ClusterAdminClient; 6 | 7 | public class GetRiverStateRequestBuilder extends MasterNodeReadOperationRequestBuilder { 8 | 9 | public GetRiverStateRequestBuilder(ClusterAdminClient client) { 10 | super(client, new GetRiverStateRequest()); 11 | } 12 | 13 | public GetRiverStateRequestBuilder setRiverName(String riverName) { 14 | request.setRiverName(riverName); 15 | return this; 16 | } 17 | 18 | public GetRiverStateRequestBuilder setRiverType(String riverType) { 19 | request.setRiverType(riverType); 20 | return this; 21 | } 22 | 23 | @Override 24 | protected void doExecute(ActionListener listener) { 25 | client.execute(GetRiverStateAction.INSTANCE, request, listener); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/state/get/GetRiverStateResponse.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.state.get; 2 | 3 | import org.elasticsearch.action.ActionResponse; 4 | import org.elasticsearch.common.collect.ImmutableList; 5 | import org.elasticsearch.common.io.stream.StreamInput; 6 | import org.elasticsearch.common.io.stream.StreamOutput; 7 | import org.elasticsearch.common.xcontent.ToXContent; 8 | import org.elasticsearch.common.xcontent.XContentBuilder; 9 | import org.xbib.elasticsearch.action.river.jdbc.state.RiverState; 10 | 11 | import java.io.IOException; 12 | 13 | public class GetRiverStateResponse extends ActionResponse implements ToXContent { 14 | 15 | private GetRiverStateRequest getRiverStateRequest; 16 | 17 | private ImmutableList states; 18 | 19 | public GetRiverStateResponse() { 20 | states = ImmutableList.of(); 21 | } 22 | 23 | public GetRiverStateResponse(GetRiverStateRequest request, ImmutableList riverStates) { 24 | getRiverStateRequest = request; 25 | states = riverStates; 26 | } 27 | 28 | public RiverState getState() { 29 | if (states == null || states.isEmpty()) { 30 | return new RiverState(getRiverStateRequest.getRiverName(), getRiverStateRequest.getRiverType()); 31 | } else { 32 | return states.get(0); 33 | } 34 | } 35 | 36 | public ImmutableList getStates() { 37 | return states; 38 | } 39 | 40 | @Override 41 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 42 | builder.field("state", states); 43 | return builder; 44 | } 45 | 46 | @Override 47 | public void readFrom(StreamInput in) throws IOException { 48 | super.readFrom(in); 49 | getRiverStateRequest = new GetRiverStateRequest(); 50 | getRiverStateRequest.readFrom(in); 51 | int len = in.readInt(); 52 | ImmutableList.Builder builder = ImmutableList.builder(); 53 | for (int i = 0; i < len; i++) { 54 | RiverState rs = new RiverState(); 55 | rs.readFrom(in); 56 | builder.add(rs); 57 | } 58 | states = builder.build(); 59 | } 60 | 61 | @Override 62 | public void writeTo(StreamOutput out) throws IOException { 63 | super.writeTo(out); 64 | getRiverStateRequest.writeTo(out); 65 | out.writeInt(states.size()); 66 | for (RiverState rs : states) { 67 | rs.writeTo(out); 68 | } 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/state/put/PutRiverStateAction.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.state.put; 2 | 3 | import org.elasticsearch.action.admin.cluster.ClusterAction; 4 | import org.elasticsearch.client.ClusterAdminClient; 5 | 6 | public class PutRiverStateAction extends ClusterAction { 7 | 8 | public static final PutRiverStateAction INSTANCE = new PutRiverStateAction(); 9 | 10 | public static final String NAME = "org.xbib.elasticsearch.action.river.state.put.jdbc"; 11 | 12 | private PutRiverStateAction() { 13 | super(NAME); 14 | } 15 | 16 | @Override 17 | public PutRiverStateRequestBuilder newRequestBuilder(ClusterAdminClient client) { 18 | return new PutRiverStateRequestBuilder(client); 19 | } 20 | 21 | @Override 22 | public PutRiverStateResponse newResponse() { 23 | return new PutRiverStateResponse(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/state/put/PutRiverStateRequestBuilder.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.state.put; 2 | 3 | import org.elasticsearch.action.ActionListener; 4 | import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; 5 | import org.elasticsearch.client.ClusterAdminClient; 6 | import org.xbib.elasticsearch.action.river.jdbc.state.RiverState; 7 | 8 | public class PutRiverStateRequestBuilder extends AcknowledgedRequestBuilder { 9 | 10 | public PutRiverStateRequestBuilder(ClusterAdminClient client) { 11 | super(client, new PutRiverStateRequest()); 12 | } 13 | 14 | public PutRiverStateRequestBuilder setRiverName(String riverName) { 15 | request.setRiverName(riverName); 16 | return this; 17 | } 18 | 19 | public PutRiverStateRequestBuilder setRiverType(String riverType) { 20 | request.setRiverType(riverType); 21 | return this; 22 | } 23 | 24 | public PutRiverStateRequestBuilder setRiverState(RiverState riverState) { 25 | request.setRiverState(riverState); 26 | return this; 27 | } 28 | 29 | @Override 30 | protected void doExecute(ActionListener listener) { 31 | client.execute(PutRiverStateAction.INSTANCE, request, listener); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/action/river/jdbc/state/put/PutRiverStateResponse.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.action.river.jdbc.state.put; 2 | 3 | import org.elasticsearch.action.support.master.AcknowledgedResponse; 4 | import org.elasticsearch.common.io.stream.StreamInput; 5 | import org.elasticsearch.common.io.stream.StreamOutput; 6 | 7 | import java.io.IOException; 8 | 9 | public class PutRiverStateResponse extends AcknowledgedResponse { 10 | 11 | 12 | public PutRiverStateResponse() { 13 | } 14 | 15 | public PutRiverStateResponse(boolean acknowledged) { 16 | super(acknowledged); 17 | } 18 | 19 | @Override 20 | public void readFrom(StreamInput in) throws IOException { 21 | super.readFrom(in); 22 | readAcknowledged(in); 23 | } 24 | 25 | @Override 26 | public void writeTo(StreamOutput out) throws IOException { 27 | super.writeTo(out); 28 | writeAcknowledged(out); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/plugin/feeder/CommandLineInterpreter.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.plugin.feeder; 2 | 3 | import java.io.Reader; 4 | 5 | public interface CommandLineInterpreter extends Runnable { 6 | 7 | CommandLineInterpreter readFrom(Reader reader); 8 | 9 | void schedule(Thread thread); 10 | 11 | void shutdown(); 12 | 13 | Thread shutdownHook(); 14 | 15 | } 16 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/plugin/feeder/Runner.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.plugin.feeder; 2 | 3 | import java.io.InputStreamReader; 4 | 5 | /** 6 | * Stub for loading a class (tool) and execute it. Isolating the main method from other methods is of advantage 7 | * for JVMs that thoroughly check if this class ca be executed. 8 | */ 9 | public class Runner { 10 | 11 | public static void main(String[] args) { 12 | try { 13 | Class clazz = Class.forName(args[0]); 14 | CommandLineInterpreter commandLineInterpreter = (CommandLineInterpreter) clazz.newInstance(); 15 | Runtime.getRuntime().addShutdownHook(commandLineInterpreter.shutdownHook()); 16 | commandLineInterpreter.readFrom(new InputStreamReader(System.in, "UTF-8")).run(); 17 | } catch (Throwable e) { 18 | e.printStackTrace(); 19 | System.exit(1); 20 | } 21 | System.exit(0); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/plugin/jdbc/ControlKeys.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.plugin.jdbc; 2 | 3 | import java.util.EnumSet; 4 | import java.util.Set; 5 | 6 | import static org.elasticsearch.common.collect.Sets.newHashSet; 7 | 8 | /** 9 | * The names of keys with a special meaning for controlling Elasticsearch indexing. 10 | * Mostly, they map to the Elasticsearch bulk item control keys. 11 | * The _job column denotes an ID for the event of a fetch execution. 12 | */ 13 | public enum ControlKeys { 14 | 15 | _optype, _index, _type, _id, _version, _timestamp, _ttl, _routing, _parent, _source, _job; 16 | 17 | public static Set makeSet() { 18 | Set set = newHashSet(); 19 | for (ControlKeys k : EnumSet.allOf(ControlKeys.class)) { 20 | set.add(k.name()); 21 | } 22 | return set; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/plugin/jdbc/IndexableObject.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.plugin.jdbc; 2 | 3 | import java.io.IOException; 4 | import java.util.Map; 5 | 6 | /** 7 | * A structured object is composed by an object data source together with 8 | * meta data about the object. 9 | */ 10 | public interface IndexableObject { 11 | 12 | /** 13 | * Set the operation type, either "index", "create", or "delete" 14 | * 15 | * @param optype the operaion type 16 | * @return this indexable object 17 | */ 18 | IndexableObject optype(String optype); 19 | 20 | /** 21 | * Get the operation type 22 | * 23 | * @return the operation type 24 | */ 25 | String optype(); 26 | 27 | /** 28 | * Set the index 29 | * 30 | * @param index the index 31 | * @return this object 32 | */ 33 | IndexableObject index(String index); 34 | 35 | /** 36 | * Get the index 37 | * 38 | * @return the index 39 | */ 40 | String index(); 41 | 42 | /** 43 | * Set the type 44 | * 45 | * @param type the type 46 | * @return this object 47 | */ 48 | IndexableObject type(String type); 49 | 50 | /** 51 | * Get the type 52 | * 53 | * @return the type 54 | */ 55 | String type(); 56 | 57 | /** 58 | * Set the ID 59 | * 60 | * @param id the ID 61 | * @return this object 62 | */ 63 | IndexableObject id(String id); 64 | 65 | /** 66 | * Get the ID 67 | * 68 | * @return the ID 69 | */ 70 | String id(); 71 | 72 | /** 73 | * Set meta data of this indexable object 74 | * 75 | * @param key the meta data key 76 | * @param value the meta data value 77 | * @return this object 78 | */ 79 | IndexableObject meta(String key, String value); 80 | 81 | String meta(String key); 82 | 83 | IndexableObject source(Map source); 84 | 85 | Map source(); 86 | 87 | String build() throws IOException; 88 | 89 | boolean isEmpty(); 90 | 91 | IndexableObject ignoreNull(boolean ignorenull); 92 | 93 | } 94 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/plugin/jdbc/LocaleUtil.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.plugin.jdbc; 2 | 3 | import org.elasticsearch.common.base.Strings; 4 | 5 | import java.util.Locale; 6 | 7 | public class LocaleUtil { 8 | 9 | public static Locale toLocale(String localeString) { 10 | if (Strings.isNullOrEmpty(localeString)) { 11 | return Locale.getDefault(); 12 | } 13 | int separatorCountry = localeString.indexOf('_'); 14 | char separator; 15 | if (separatorCountry >= 0) { 16 | separator = '_'; 17 | } else { 18 | separatorCountry = localeString.indexOf('-'); 19 | separator = '-'; 20 | } 21 | String language; 22 | String country; 23 | String variant; 24 | if (separatorCountry < 0) { 25 | language = localeString; 26 | country = ""; 27 | variant = ""; 28 | } else { 29 | language = localeString.substring(0, separatorCountry); 30 | int separatorVariant = localeString.indexOf(separator, separatorCountry + 1); 31 | if (separatorVariant < 0) { 32 | country = localeString.substring(separatorCountry + 1); 33 | variant = ""; 34 | } else { 35 | country = localeString.substring(separatorCountry + 1, separatorVariant); 36 | variant = localeString.substring(separatorVariant + 1); 37 | } 38 | } 39 | return new Locale(language, country, variant); 40 | } 41 | 42 | public static String fromLocale(Locale locale) { 43 | return locale.getLanguage() + 44 | (Strings.isNullOrEmpty(locale.getCountry()) ? "_" + locale.getCountry() : "") + 45 | (Strings.isNullOrEmpty(locale.getVariant()) ? "_" + locale.getVariant() : ""); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/plugin/jdbc/RiverMouthKeyValueStreamListener.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.plugin.jdbc; 2 | 3 | import org.xbib.elasticsearch.river.jdbc.RiverMouth; 4 | 5 | import java.io.IOException; 6 | 7 | /** 8 | * This class consumes pairs from a key/value stream 9 | * and transports them to the river mouth. 10 | */ 11 | public class RiverMouthKeyValueStreamListener extends PlainKeyValueStreamListener { 12 | 13 | private RiverMouth output; 14 | 15 | public RiverMouthKeyValueStreamListener output(RiverMouth output) { 16 | this.output = output; 17 | return this; 18 | } 19 | 20 | public RiverMouthKeyValueStreamListener shouldIgnoreNull(boolean shouldIgnoreNull) { 21 | super.shouldIgnoreNull(shouldIgnoreNull); 22 | return this; 23 | } 24 | 25 | /** 26 | * The object is complete. Push it to the river mouth. 27 | * 28 | * @param object the object 29 | * @return this value listener 30 | * @throws java.io.IOException 31 | */ 32 | public RiverMouthKeyValueStreamListener end(IndexableObject object) throws IOException { 33 | if (object.isEmpty()) { 34 | return this; 35 | } 36 | if (output != null) { 37 | if (object.optype() == null) { 38 | output.index(object, false); 39 | } else if ("index".equals(object.optype())) { 40 | output.index(object, false); 41 | } else if ("create".equals(object.optype())) { 42 | output.index(object, true); 43 | } else if ("delete".equals(object.optype())) { 44 | output.delete(object); 45 | } else { 46 | throw new IllegalArgumentException("unknown optype: " + object.optype()); 47 | } 48 | } 49 | return this; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/plugin/jdbc/RiverServiceLoader.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.plugin.jdbc; 2 | 3 | import org.xbib.elasticsearch.river.jdbc.RiverFlow; 4 | import org.xbib.elasticsearch.river.jdbc.RiverMouth; 5 | import org.xbib.elasticsearch.river.jdbc.RiverSource; 6 | import org.xbib.elasticsearch.river.jdbc.strategy.simple.SimpleRiverFlow; 7 | import org.xbib.elasticsearch.river.jdbc.strategy.simple.SimpleRiverMouth; 8 | import org.xbib.elasticsearch.river.jdbc.strategy.simple.SimpleRiverSource; 9 | 10 | import java.util.ServiceLoader; 11 | 12 | /** 13 | * The river service loader 14 | */ 15 | public class RiverServiceLoader { 16 | 17 | /** 18 | * A river source is the origin, the data producing side 19 | * 20 | * @param strategy the strategy 21 | * @return a river source, or the SimpleRiverSource 22 | */ 23 | public static RiverSource findRiverSource(String strategy) { 24 | ServiceLoader sourceLoader = ServiceLoader.load(RiverSource.class); 25 | for (RiverSource rs : sourceLoader) { 26 | if (strategy.equals(rs.strategy())) { 27 | return rs; 28 | } 29 | } 30 | return new SimpleRiverSource(); 31 | } 32 | 33 | /** 34 | * A river mouth is the Elasticsearch side of the river, where the bulk processor lives 35 | * 36 | * @param strategy the strategy 37 | * @return a river mouth, or the SimpleRiverMouth 38 | */ 39 | public static RiverMouth findRiverMouth(String strategy) { 40 | ServiceLoader riverMouthLoader = ServiceLoader.load(RiverMouth.class); 41 | for (RiverMouth rt : riverMouthLoader) { 42 | if (strategy.equals(rt.strategy())) { 43 | return rt; 44 | } 45 | } 46 | return new SimpleRiverMouth(); 47 | } 48 | 49 | /** 50 | * A river flow encapsulates the thread that moves the data from source to mouth 51 | * 52 | * @param strategy the strategy 53 | * @return a river flow, or the SimpleRiverFlow 54 | */ 55 | public static RiverFlow findRiverFlow(String strategy) { 56 | ServiceLoader riverFlowServiceLoader = ServiceLoader.load(RiverFlow.class); 57 | for (RiverFlow rc : riverFlowServiceLoader) { 58 | if (strategy.equals(rc.strategy())) { 59 | return rc; 60 | } 61 | } 62 | return new SimpleRiverFlow(); 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/plugin/river/jdbc/JDBCRiverModule.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.plugin.river.jdbc; 2 | 3 | import org.elasticsearch.common.inject.AbstractModule; 4 | import org.elasticsearch.river.River; 5 | 6 | /** 7 | * The JDBC river module 8 | */ 9 | public class JDBCRiverModule extends AbstractModule { 10 | 11 | @Override 12 | protected void configure() { 13 | bind(River.class).to(JDBCRiver.class).asEagerSingleton(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/rest/action/river/execute/RestRiverExecuteAction.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.rest.action.river.execute; 2 | 3 | import org.elasticsearch.client.Client; 4 | import org.elasticsearch.common.inject.Inject; 5 | import org.elasticsearch.common.settings.Settings; 6 | import org.elasticsearch.common.xcontent.XContentBuilder; 7 | import org.elasticsearch.rest.BaseRestHandler; 8 | import org.elasticsearch.rest.BytesRestResponse; 9 | import org.elasticsearch.rest.RestChannel; 10 | import org.elasticsearch.rest.RestController; 11 | import org.elasticsearch.rest.RestRequest; 12 | import org.elasticsearch.rest.RestResponse; 13 | import org.elasticsearch.rest.action.support.RestBuilderListener; 14 | import org.xbib.elasticsearch.action.river.jdbc.execute.RiverExecuteAction; 15 | import org.xbib.elasticsearch.action.river.jdbc.execute.RiverExecuteRequest; 16 | import org.xbib.elasticsearch.action.river.jdbc.execute.RiverExecuteResponse; 17 | 18 | import static org.elasticsearch.rest.RestStatus.OK; 19 | 20 | /** 21 | * Run a river. The river can be executed once with such a call. Example: 22 | *

23 | * curl -XPOST 'localhost:9200/_river/my_jdbc_river/_execute' 24 | */ 25 | public class RestRiverExecuteAction extends BaseRestHandler { 26 | 27 | @Inject 28 | public RestRiverExecuteAction(Settings settings, Client client, RestController controller) { 29 | super(settings, client); 30 | controller.registerHandler(RestRequest.Method.POST, "/_river/jdbc/{riverName}/_execute", this); 31 | } 32 | 33 | @Override 34 | public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) { 35 | String riverName = request.param("riverName"); 36 | String riverType = "jdbc"; 37 | RiverExecuteRequest riverExecuteRequest = new RiverExecuteRequest() 38 | .setRiverName(riverName) 39 | .setRiverType(riverType); 40 | client.admin().cluster().execute(RiverExecuteAction.INSTANCE, riverExecuteRequest, new RestBuilderListener(channel) { 41 | @Override 42 | public RestResponse buildResponse(RiverExecuteResponse riverExecuteResponse, XContentBuilder builder) throws Exception { 43 | boolean isExecuted = false; 44 | for (int i = 0; i < riverExecuteResponse.isExecuted().length; i++) { 45 | isExecuted = isExecuted || riverExecuteResponse.isExecuted()[i]; 46 | } 47 | builder.field("executed", isExecuted); 48 | return new BytesRestResponse(OK, builder); 49 | } 50 | }); 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/river/jdbc/RiverFlow.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.river.jdbc; 2 | 3 | import org.xbib.elasticsearch.plugin.feeder.jdbc.JDBCFeeder; 4 | import org.xbib.elasticsearch.plugin.jdbc.RiverContext; 5 | 6 | public interface RiverFlow { 7 | /** 8 | * The river strategy 9 | * 10 | * @return the strategy 11 | */ 12 | String strategy(); 13 | 14 | RiverFlow setRiverContext(RiverContext riverContext); 15 | 16 | /** 17 | * Set the feeder 18 | * 19 | * @param feeder the feeder 20 | * @return this river flow 21 | */ 22 | RiverFlow setFeeder(JDBCFeeder feeder); 23 | 24 | /** 25 | * Return the feeder 26 | * 27 | * @return the feeder 28 | */ 29 | JDBCFeeder getFeeder(); 30 | } 31 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/river/jdbc/RiverMouth.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.river.jdbc; 2 | 3 | import org.xbib.elasticsearch.plugin.jdbc.IndexableObject; 4 | import org.xbib.elasticsearch.plugin.jdbc.RiverContext; 5 | import org.xbib.elasticsearch.support.client.Ingest; 6 | 7 | import java.io.IOException; 8 | 9 | /** 10 | * The river mouth is the abstraction of the destination where all the data 11 | * is flowing from the river source. 12 | */ 13 | public interface RiverMouth { 14 | 15 | /** 16 | * The river strategy 17 | * 18 | * @return the strategy 19 | */ 20 | String strategy(); 21 | 22 | /** 23 | * The river context 24 | * 25 | * @param context the river context 26 | * @return this river mouth 27 | */ 28 | RiverMouth setRiverContext(RiverContext context); 29 | 30 | /** 31 | * Set the Elasticsearch ingester 32 | * 33 | * @param ingester the ingester 34 | * @return this river mouth 35 | */ 36 | RiverMouth setIngest(Ingest ingester); 37 | 38 | RiverMouth setIndex(String index); 39 | 40 | RiverMouth setType(String type); 41 | 42 | RiverMouth setTimeWindowed(boolean timeWindowed); 43 | 44 | RiverMouth setId(String id); 45 | 46 | String getId(); 47 | 48 | /** 49 | * Index. Indicating that an object has been built and is ready to be indexed. 50 | * The source of the document is held by the XContentBuilder. 51 | * 52 | * @param object the indexable object 53 | * @param create true if the document should be created 54 | * @throws IOException when indexing fails 55 | */ 56 | void index(IndexableObject object, boolean create) throws IOException; 57 | 58 | /** 59 | * Delete. Indicating that an object should be deleted from the index. 60 | * 61 | * @param object the structured object 62 | * @throws IOException when delete fails 63 | */ 64 | void delete(IndexableObject object) throws IOException; 65 | 66 | /** 67 | * Flush data to the river mouth 68 | * 69 | * @throws IOException when flush fails 70 | */ 71 | void flush() throws IOException; 72 | 73 | /** 74 | * Close this river mouth 75 | */ 76 | void close() throws IOException; 77 | 78 | } 79 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/river/jdbc/strategy/column/ColumnRiverFlow.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.river.jdbc.strategy.column; 2 | 3 | import org.elasticsearch.common.logging.ESLogger; 4 | import org.elasticsearch.common.logging.ESLoggerFactory; 5 | import org.xbib.elasticsearch.plugin.feeder.jdbc.JDBCFeeder; 6 | import org.xbib.elasticsearch.river.jdbc.RiverFlow; 7 | import org.xbib.elasticsearch.river.jdbc.strategy.simple.SimpleRiverFlow; 8 | 9 | /** 10 | * River flow implementation for the 'column' strategy 11 | * 12 | * @author Piotr Śliwa 13 | */ 14 | public class ColumnRiverFlow extends SimpleRiverFlow { 15 | 16 | private final ESLogger logger = ESLoggerFactory.getLogger(ColumnRiverFlow.class.getName()); 17 | 18 | public static final String DOCUMENT = "_custom"; 19 | 20 | public static final String LAST_RUN_TIME = "last_run_time"; 21 | 22 | public static final String CURRENT_RUN_STARTED_TIME = "current_run_started_time"; 23 | 24 | protected ESLogger logger() { 25 | return logger; 26 | } 27 | 28 | @Override 29 | public String strategy() { 30 | return "column"; 31 | } 32 | 33 | @Override 34 | public RiverFlow setFeeder(JDBCFeeder feeder) { 35 | this.feeder = feeder; 36 | return this; 37 | } 38 | 39 | @Override 40 | public JDBCFeeder getFeeder() { 41 | if (feeder == null) { 42 | this.feeder = new ColumnRiverFeeder(); 43 | } 44 | return feeder; 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/river/jdbc/strategy/column/ColumnRiverMouth.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.river.jdbc.strategy.column; 2 | 3 | import org.elasticsearch.common.logging.ESLogger; 4 | import org.elasticsearch.common.logging.ESLoggerFactory; 5 | import org.xbib.elasticsearch.river.jdbc.strategy.simple.SimpleRiverMouth; 6 | 7 | /** 8 | * River mouth implementation for the 'column' strategy 9 | * 10 | * @author Piotr Śliwa 11 | */ 12 | public class ColumnRiverMouth extends SimpleRiverMouth { 13 | 14 | private final ESLogger logger = ESLoggerFactory.getLogger(ColumnRiverMouth.class.getName()); 15 | 16 | protected ESLogger logger() { 17 | return logger; 18 | } 19 | 20 | @Override 21 | public String strategy() { 22 | return "column"; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/river/jdbc/strategy/simple/SimpleRiverFlow.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.river.jdbc.strategy.simple; 2 | 3 | import org.xbib.elasticsearch.plugin.feeder.jdbc.JDBCFeeder; 4 | import org.xbib.elasticsearch.plugin.jdbc.RiverContext; 5 | import org.xbib.elasticsearch.river.jdbc.RiverFlow; 6 | 7 | public class SimpleRiverFlow implements RiverFlow { 8 | 9 | protected RiverContext context; 10 | 11 | protected JDBCFeeder feeder; 12 | 13 | @Override 14 | public String strategy() { 15 | return "simple"; 16 | } 17 | 18 | @Override 19 | public RiverFlow setRiverContext(RiverContext context) { 20 | this.context = context; 21 | return this; 22 | } 23 | 24 | @Override 25 | public RiverFlow setFeeder(JDBCFeeder feeder) { 26 | this.feeder = feeder; 27 | return this; 28 | } 29 | 30 | @Override 31 | public JDBCFeeder getFeeder() { 32 | if (feeder == null) { 33 | this.feeder = new JDBCFeeder(); 34 | } 35 | return feeder; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/support/client/Feed.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.support.client; 2 | 3 | import org.elasticsearch.client.Client; 4 | 5 | /** 6 | * Minimal API for feed 7 | */ 8 | public interface Feed { 9 | 10 | Client client(); 11 | 12 | /** 13 | * Index document 14 | * 15 | * @param index the index 16 | * @param type the type 17 | * @param id the id 18 | * @param source the source 19 | * @return this 20 | */ 21 | Feed index(String index, String type, String id, String source); 22 | 23 | /** 24 | * Delete document 25 | * 26 | * @param index the index 27 | * @param type the type 28 | * @param id the id 29 | * @return this 30 | */ 31 | Feed delete(String index, String type, String id); 32 | 33 | } 34 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/elasticsearch/support/client/State.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.support.client; 2 | 3 | import org.elasticsearch.common.metrics.CounterMetric; 4 | import org.elasticsearch.common.metrics.MeanMetric; 5 | 6 | import java.util.HashSet; 7 | import java.util.Set; 8 | 9 | public class State { 10 | 11 | private final Set indexNames = new HashSet(); 12 | 13 | private final MeanMetric totalIngest = new MeanMetric(); 14 | 15 | private final CounterMetric totalIngestSizeInBytes = new CounterMetric(); 16 | 17 | private final CounterMetric currentIngest = new CounterMetric(); 18 | 19 | private final CounterMetric currentIngestNumDocs = new CounterMetric(); 20 | 21 | private final CounterMetric submitted = new CounterMetric(); 22 | 23 | private final CounterMetric succeeded = new CounterMetric(); 24 | 25 | private final CounterMetric failed = new CounterMetric(); 26 | 27 | public MeanMetric getTotalIngest() { 28 | return totalIngest; 29 | } 30 | 31 | public CounterMetric getTotalIngestSizeInBytes() { 32 | return totalIngestSizeInBytes; 33 | } 34 | 35 | public CounterMetric getCurrentIngest() { 36 | return currentIngest; 37 | } 38 | 39 | public CounterMetric getCurrentIngestNumDocs() { 40 | return currentIngestNumDocs; 41 | } 42 | 43 | public CounterMetric getSubmitted() { 44 | return submitted; 45 | } 46 | 47 | public CounterMetric getSucceeded() { 48 | return succeeded; 49 | } 50 | 51 | public CounterMetric getFailed() { 52 | return failed; 53 | } 54 | 55 | public State startBulk(String indexName) { 56 | synchronized (indexNames) { 57 | indexNames.add(indexName); 58 | } 59 | return this; 60 | } 61 | 62 | public boolean isBulk(String indexName) { 63 | return indexNames.contains(indexName); 64 | } 65 | 66 | public State stopBulk(String indexName) { 67 | synchronized (indexNames) { 68 | indexNames.remove(indexName); 69 | } 70 | return this; 71 | } 72 | 73 | public Set indices() { 74 | return indexNames; 75 | } 76 | 77 | } 78 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/keyvalue/KeyValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Jörg Prante and xbib under one or more contributor 3 | * license agreements. See the NOTICE.txt file distributed with this work 4 | * for additional information regarding copyright ownership. 5 | * 6 | * Copyright (C) 2012 Jörg Prante and xbib 7 | * 8 | * This program is free software; you can redistribute it and/or modify 9 | * it under the terms of the GNU Affero General Public License as published 10 | * by the Free Software Foundation; either version 3 of the License, or 11 | * (at your option) any later version. 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Affero General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Affero General Public License 18 | * along with this program; if not, see http://www.gnu.org/licenses 19 | * or write to the Free Software Foundation, Inc., 51 Franklin Street, 20 | * Fifth Floor, Boston, MA 02110-1301 USA. 21 | * 22 | * The interactive user interfaces in modified source and object code 23 | * versions of this program must display Appropriate Legal Notices, 24 | * as required under Section 5 of the GNU Affero General Public License. 25 | * 26 | * In accordance with Section 7(b) of the GNU Affero General Public 27 | * License, these Appropriate Legal Notices must retain the display of the 28 | * "Powered by xbib" logo. If the display of the logo is not reasonably 29 | * feasible for technical reasons, the Appropriate Legal Notices must display 30 | * the words "Powered by xbib". 31 | */ 32 | package org.xbib.keyvalue; 33 | 34 | public class KeyValue { 35 | 36 | private final K key; 37 | 38 | private final V value; 39 | 40 | public KeyValue(K key, V value) { 41 | this.key = key; 42 | this.value = value; 43 | } 44 | 45 | public K key() { 46 | return key; 47 | } 48 | 49 | public V value() { 50 | return value; 51 | } 52 | 53 | public String toString() { 54 | return String.valueOf(key) + "=" + value; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/pipeline/Pipeline.java: -------------------------------------------------------------------------------- 1 | package org.xbib.pipeline; 2 | 3 | import java.io.Closeable; 4 | import java.util.Iterator; 5 | import java.util.concurrent.Callable; 6 | 7 | /** 8 | * A pipeline. 9 | * 10 | * @param the pipeline result type 11 | * @param the pipeline request type 12 | */ 13 | public interface Pipeline extends Callable, Closeable, Iterator { 14 | 15 | boolean isRunning(); 16 | } 17 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/pipeline/PipelineErrorListener.java: -------------------------------------------------------------------------------- 1 | package org.xbib.pipeline; 2 | 3 | public interface PipelineErrorListener { 4 | 5 | /** 6 | * Receive an error from processing a pipeline request. 7 | * 8 | * @param pipeline the pipeline 9 | * @param request the pipeline request 10 | * @param error the pipeline error 11 | */ 12 | void error(Pipeline pipeline, R request, E error) throws PipelineException; 13 | } 14 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/pipeline/PipelineException.java: -------------------------------------------------------------------------------- 1 | package org.xbib.pipeline; 2 | 3 | public class PipelineException extends Exception { 4 | 5 | public PipelineException(String msg) { 6 | super(msg); 7 | } 8 | 9 | public PipelineException(String msg, Throwable cause) { 10 | super(msg, cause); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/pipeline/PipelineExecutor.java: -------------------------------------------------------------------------------- 1 | package org.xbib.pipeline; 2 | 3 | import java.io.IOException; 4 | import java.util.Collection; 5 | import java.util.concurrent.ExecutionException; 6 | 7 | /** 8 | * The Pipeline Executor can execute provided pipelines. 9 | * If the concurrency level is set to higher than one, more than one pipeline is executed in parallel. 10 | * 11 | * @param the result type 12 | * @param the request type 13 | * @param

the pipeline type 14 | */ 15 | public interface PipelineExecutor> { 16 | 17 | /** 18 | * Set the concurrency of this pipeline setExecutor 19 | * 20 | * @param concurrency the concurrency, must be a positive integer 21 | * @return this setExecutor 22 | */ 23 | PipelineExecutor setConcurrency(int concurrency); 24 | 25 | /** 26 | * Set the provider of this pipeline setExecutor 27 | * 28 | * @param provider the pipeline provider 29 | * @return this setExecutor 30 | */ 31 | PipelineExecutor setPipelineProvider(PipelineProvider

provider); 32 | 33 | /** 34 | * Set pipeline sink 35 | * 36 | * @param sink the pipeline sink 37 | * @return this setExecutor 38 | */ 39 | PipelineExecutor setSink(PipelineSink sink); 40 | 41 | /** 42 | * Prepare the pipeline execution. 43 | * 44 | * @return this setExecutor 45 | */ 46 | PipelineExecutor prepare(); 47 | 48 | /** 49 | * Execute the pipelines. 50 | * 51 | * @return this setExecutor 52 | */ 53 | PipelineExecutor execute(); 54 | 55 | /** 56 | * Execute the pipelines. 57 | * 58 | * @return this setExecutor 59 | * @throws InterruptedException 60 | * @throws java.util.concurrent.ExecutionException 61 | * @throws java.io.IOException 62 | */ 63 | PipelineExecutor waitFor() throws InterruptedException, ExecutionException, IOException; 64 | 65 | /** 66 | * Shut down this pipeline executor. 67 | * 68 | * @throws InterruptedException 69 | * @throws java.util.concurrent.ExecutionException 70 | * @throws java.io.IOException 71 | */ 72 | void shutdown() throws InterruptedException, ExecutionException, IOException; 73 | 74 | /** 75 | * Return pipelines 76 | * 77 | * @return the pipelines 78 | */ 79 | Collection

getPipelines(); 80 | } 81 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/pipeline/PipelineProvider.java: -------------------------------------------------------------------------------- 1 | package org.xbib.pipeline; 2 | 3 | public interface PipelineProvider

{ 4 | 5 | /** 6 | * Get a new instance of a pipeline 7 | * 8 | * @return a new pipeline instance 9 | */ 10 | P get(); 11 | 12 | } 13 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/pipeline/PipelineRequest.java: -------------------------------------------------------------------------------- 1 | package org.xbib.pipeline; 2 | 3 | public interface PipelineRequest { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/pipeline/PipelineRequestListener.java: -------------------------------------------------------------------------------- 1 | package org.xbib.pipeline; 2 | 3 | public interface PipelineRequestListener { 4 | 5 | /** 6 | * Receive a new request in this pipeline. 7 | * 8 | * @param pipeline the pipeline 9 | * @param request the pipeline request 10 | */ 11 | void newRequest(Pipeline pipeline, R request) throws PipelineException; 12 | } 13 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/pipeline/PipelineSink.java: -------------------------------------------------------------------------------- 1 | package org.xbib.pipeline; 2 | 3 | import java.io.IOException; 4 | 5 | public interface PipelineSink { 6 | 7 | void write(T t) throws IOException; 8 | } 9 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/pipeline/element/LongPipelineElement.java: -------------------------------------------------------------------------------- 1 | package org.xbib.pipeline.element; 2 | 3 | import org.xbib.pipeline.PipelineRequest; 4 | 5 | import java.util.concurrent.atomic.AtomicLong; 6 | 7 | public class LongPipelineElement implements PipelineElement, PipelineRequest { 8 | 9 | private AtomicLong n; 10 | 11 | @Override 12 | public AtomicLong get() { 13 | return n; 14 | } 15 | 16 | @Override 17 | public LongPipelineElement set(AtomicLong n) { 18 | this.n = n; 19 | return this; 20 | } 21 | 22 | @Override 23 | public String toString() { 24 | return n.toString(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/pipeline/element/MapPipelineElement.java: -------------------------------------------------------------------------------- 1 | package org.xbib.pipeline.element; 2 | 3 | import org.xbib.pipeline.PipelineRequest; 4 | 5 | import java.util.Map; 6 | 7 | public class MapPipelineElement implements PipelineElement>, PipelineRequest { 8 | 9 | private Map map; 10 | 11 | @Override 12 | public Map get() { 13 | return map; 14 | } 15 | 16 | @Override 17 | public MapPipelineElement set(Map map) { 18 | this.map = map; 19 | return this; 20 | } 21 | 22 | @Override 23 | public String toString() { 24 | return map.toString(); 25 | } 26 | } -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/pipeline/element/PipelineElement.java: -------------------------------------------------------------------------------- 1 | package org.xbib.pipeline.element; 2 | 3 | public interface PipelineElement { 4 | 5 | E get(); 6 | 7 | PipelineElement set(E e); 8 | 9 | } 10 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/pipeline/element/URIPipelineElement.java: -------------------------------------------------------------------------------- 1 | package org.xbib.pipeline.element; 2 | 3 | import org.xbib.pipeline.PipelineRequest; 4 | 5 | import java.net.URI; 6 | 7 | public class URIPipelineElement implements PipelineElement, PipelineRequest { 8 | 9 | private URI uri; 10 | 11 | @Override 12 | public URI get() { 13 | return uri; 14 | } 15 | 16 | @Override 17 | public URIPipelineElement set(URI uri) { 18 | this.uri = uri; 19 | return this; 20 | } 21 | 22 | @Override 23 | public String toString() { 24 | return uri.toString(); 25 | } 26 | } -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/java/org/xbib/pipeline/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Pipeline service for multi-threaded importing 3 | */ 4 | package org.xbib.pipeline; 5 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/resources/META-INF/services/org.xbib.elasticsearch.river.jdbc.RiverFlow: -------------------------------------------------------------------------------- 1 | org.xbib.elasticsearch.river.jdbc.strategy.simple.SimpleRiverFlow 2 | org.xbib.elasticsearch.river.jdbc.strategy.column.ColumnRiverFlow 3 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/resources/META-INF/services/org.xbib.elasticsearch.river.jdbc.RiverMouth: -------------------------------------------------------------------------------- 1 | org.xbib.elasticsearch.river.jdbc.strategy.simple.SimpleRiverMouth 2 | org.xbib.elasticsearch.river.jdbc.strategy.column.ColumnRiverMouth -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/resources/META-INF/services/org.xbib.elasticsearch.river.jdbc.RiverSource: -------------------------------------------------------------------------------- 1 | org.xbib.elasticsearch.river.jdbc.strategy.simple.SimpleRiverSource 2 | org.xbib.elasticsearch.river.jdbc.strategy.column.ColumnRiverSource 3 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/main/resources/es-plugin.properties: -------------------------------------------------------------------------------- 1 | plugin=org.xbib.elasticsearch.plugin.river.jdbc.JDBCRiverPlugin 2 | version=${project.version} 3 | hash=${buildNumber} 4 | timestamp=${timestamp} 5 | date=${tstamp} 6 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/site/resources/database-128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szwork2013/elasticsearch-sentiment/6c37784a1c783afd196719841eaa4269fc1f27b8/elasticsearch-jdbc/src/site/resources/database-128.png -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/site/resources/jdbc-river-feeder-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szwork2013/elasticsearch-sentiment/6c37784a1c783afd196719841eaa4269fc1f27b8/elasticsearch-jdbc/src/site/resources/jdbc-river-feeder-architecture.png -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/site/resources/simple-tabular-json-data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szwork2013/elasticsearch-sentiment/6c37784a1c783afd196719841eaa4269fc1f27b8/elasticsearch-jdbc/src/site/resources/simple-tabular-json-data.png -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/site/resources/tabular-json-data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/szwork2013/elasticsearch-sentiment/6c37784a1c783afd196719841eaa4269fc1f27b8/elasticsearch-jdbc/src/site/resources/tabular-json-data.png -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/site/site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | org.apache.maven.skins 5 | maven-fluido-skin 6 | 1.2.1 7 | 8 | 9 | 10 | true 11 | true 12 | 13 | jprante/elasticsearch-river-jdbc 14 | right 15 | black 16 | 17 | 18 | xbib 19 | true 20 | true 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 |

29 | 30 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/java/org/xbib/elasticsearch/river/jdbc/Listener.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.river.jdbc; 2 | 3 | import org.elasticsearch.common.logging.ESLogger; 4 | import org.elasticsearch.common.logging.ESLoggerFactory; 5 | import org.testng.ITestContext; 6 | import org.testng.ITestListener; 7 | import org.testng.ITestResult; 8 | 9 | public class Listener implements ITestListener { 10 | 11 | private final ESLogger logger = ESLoggerFactory.getLogger(Listener.class.getSimpleName()); 12 | 13 | @Override 14 | public void onTestStart(ITestResult result) { 15 | logger.info("----------------------------------------------------------"); 16 | logger.info("starting {}", result.getMethod()); 17 | logger.info("----------------------------------------------------------"); 18 | } 19 | 20 | @Override 21 | public void onTestSuccess(ITestResult result) { 22 | logger.info("----------------------------------------------------------"); 23 | logger.info("success {}", result.getMethod()); 24 | logger.info("----------------------------------------------------------"); 25 | } 26 | 27 | @Override 28 | public void onTestFailure(ITestResult result) { 29 | logger.info("----------------------------------------------------------"); 30 | logger.info("failure of {}", result.getMethod()); 31 | logger.info("----------------------------------------------------------"); 32 | } 33 | 34 | @Override 35 | public void onTestSkipped(ITestResult result) { 36 | logger.info("skipped test {}", result.getMethod()); 37 | result.setStatus(ITestResult.FAILURE); 38 | } 39 | 40 | @Override 41 | public void onTestFailedButWithinSuccessPercentage(ITestResult result) { 42 | } 43 | 44 | @Override 45 | public void onStart(ITestContext context) { 46 | logger.info("----------------------------------------------------------"); 47 | logger.info("starting test {}", context.getName()); 48 | logger.info("----------------------------------------------------------"); 49 | } 50 | 51 | @Override 52 | public void onFinish(ITestContext context) { 53 | logger.info("----------------------------------------------------------"); 54 | logger.info("finished test {}", context.getName()); 55 | logger.info("----------------------------------------------------------"); 56 | } 57 | 58 | } 59 | 60 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/java/org/xbib/elasticsearch/river/jdbc/strategy/simple/RiverDataTests.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.river.jdbc.strategy.simple; 2 | 3 | import org.testng.annotations.Parameters; 4 | import org.testng.annotations.Test; 5 | import org.xbib.elasticsearch.plugin.jdbc.RiverContext; 6 | import org.xbib.elasticsearch.river.jdbc.RiverSource; 7 | import org.xbib.elasticsearch.support.helper.AbstractRiverNodeTest; 8 | 9 | public class RiverDataTests extends AbstractRiverNodeTest { 10 | 11 | @Override 12 | public RiverSource getRiverSource() { 13 | return new SimpleRiverSource(); 14 | } 15 | 16 | @Override 17 | public RiverContext getRiverContext() { 18 | return new RiverContext(); 19 | } 20 | 21 | /** 22 | * Start the river and execute a simple star query 23 | * @throws Exception if test fails 24 | */ 25 | @Test 26 | @Parameters({"river1", "sql1"}) 27 | public void testSimpleRiverOnce(String riverResource, String sql) throws Exception { 28 | createRandomProducts(sql, 100); 29 | createRiver(riverResource); 30 | waitForInactiveRiver(); 31 | } 32 | 33 | /** 34 | * Product table (star query) 35 | * 36 | * @param riverResource the river resource 37 | * @throws Exception if test fails 38 | */ 39 | @Test 40 | @Parameters({"river2", "sql1"}) 41 | public void testSimpleRiverRandom(String riverResource, String sql) throws Exception { 42 | createRandomProducts(sql, 100); 43 | createRiver(riverResource); 44 | waitForInactiveRiver(); 45 | assertHits("1", 104); 46 | logger.info("success"); 47 | } 48 | 49 | /** 50 | * Product table 51 | * 52 | * @param riverResource the river 53 | * @param sql the SQL statement 54 | * @throws Exception if test fails 55 | */ 56 | @Test 57 | @Parameters({"river3", "sql1"}) 58 | public void testSimpleRiverMaxrows(String riverResource, String sql) throws Exception { 59 | createRandomProducts(sql, 100); 60 | createRiver(riverResource); 61 | waitForInactiveRiver(); 62 | assertHits("1", 104); 63 | logger.info("success"); 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/java/org/xbib/elasticsearch/river/jdbc/strategy/simple/RiverJobTests.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.river.jdbc.strategy.simple; 2 | 3 | import org.testng.annotations.Parameters; 4 | import org.testng.annotations.Test; 5 | import org.xbib.elasticsearch.plugin.jdbc.RiverContext; 6 | import org.xbib.elasticsearch.river.jdbc.RiverSource; 7 | import org.xbib.elasticsearch.support.helper.AbstractRiverNodeTest; 8 | 9 | import java.io.IOException; 10 | import java.sql.Connection; 11 | import java.sql.ResultSet; 12 | import java.sql.SQLException; 13 | 14 | public class RiverJobTests extends AbstractRiverNodeTest { 15 | 16 | @Override 17 | public RiverSource getRiverSource() { 18 | return new SimpleRiverSource(); 19 | } 20 | 21 | @Override 22 | public RiverContext getRiverContext() { 23 | return new RiverContext(); 24 | } 25 | 26 | @Test 27 | @Parameters({"river1", "sql1", "sql2"}) 28 | public void testSimpleRiverJob(String riverResource, String sql1, String sql2) 29 | throws IOException, InterruptedException, SQLException { 30 | createRandomProductsJob(sql2, 100); 31 | Connection connection = source.getConnectionForReading(); 32 | ResultSet results = connection.createStatement().executeQuery(sql1); 33 | if (!connection.getAutoCommit()) { 34 | connection.commit(); 35 | } 36 | int count = results.next() ? results.getInt(1) : -1; 37 | source.close(results); 38 | source.closeReading(); 39 | assertEquals(count, 100); 40 | //assertHits("1", 0); 41 | createRiver(riverResource); 42 | waitForInactiveRiver(); 43 | assertHits("1", 100); 44 | connection = source.getConnectionForReading(); 45 | // sql1 = select count(*) 46 | results = connection.createStatement().executeQuery(sql1); 47 | if (!connection.getAutoCommit()) { 48 | connection.commit(); 49 | } 50 | count = results.next() ? results.getInt(1) : -1; 51 | results.close(); 52 | assertEquals(count, 0); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/java/org/xbib/elasticsearch/river/jdbc/strategy/simple/RiverScheduleTests.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.river.jdbc.strategy.simple; 2 | 3 | import org.testng.annotations.Parameters; 4 | import org.testng.annotations.Test; 5 | import org.xbib.elasticsearch.plugin.jdbc.RiverContext; 6 | import org.xbib.elasticsearch.river.jdbc.RiverSource; 7 | import org.xbib.elasticsearch.support.helper.AbstractRiverNodeTest; 8 | 9 | public class RiverScheduleTests extends AbstractRiverNodeTest { 10 | 11 | @Override 12 | public RiverSource getRiverSource() { 13 | return new SimpleRiverSource(); 14 | } 15 | 16 | @Override 17 | public RiverContext getRiverContext() { 18 | return new RiverContext(); 19 | } 20 | 21 | /** 22 | * Product table star select, scheduled for more than two runs 23 | * @param riverResource the river resource 24 | * @param sql the SQL statement 25 | * @throws Exception if test fails 26 | */ 27 | @Test 28 | @Parameters({"river6", "sql1"}) 29 | public void testSimpleSchedule(String riverResource, String sql) throws Exception { 30 | createRandomProducts(sql, 100); 31 | createRiver(riverResource); 32 | Thread.sleep(12500L); // run more than twice 33 | client("1").admin().indices().prepareRefresh(index).execute().actionGet(); 34 | long hits = client("1").prepareSearch(index).execute().actionGet().getHits().getTotalHits(); 35 | assertTrue(hits > 100L); 36 | } 37 | 38 | /** 39 | * Test read and write of timestamps in a table. We create 100 timestamps over hour interval, 40 | * current timestamp $now is in the center. 41 | * Selecting timestamps from $now, there should be at least 50 rows/hits per run, if $now works. 42 | * 43 | * @param riverResource the river JSON resource 44 | * @param sql the sql statement to select timestamps 45 | * @throws Exception 46 | */ 47 | @Test 48 | @Parameters({"river7", "sql2"}) 49 | public void testTimestamps(String riverResource, String sql) throws Exception { 50 | createTimestampedLogs(sql, 100, "iw_IL", "Asia/Jerusalem"); // TODO make timezone/locale configurable 51 | createRiver(riverResource); 52 | Thread.sleep(12500L); // ensure at least two runs 53 | client("1").admin().indices().prepareRefresh(index).execute().actionGet(); 54 | long hits = client("1").prepareSearch(index).execute().actionGet().getHits().getTotalHits(); 55 | // just an estimation, at least two runs should deliver 50 hits each. 56 | assertTrue(hits > 99L); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/java/org/xbib/elasticsearch/river/jdbc/strategy/simple/RiverScriptTests.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.river.jdbc.strategy.simple; 2 | 3 | import org.testng.annotations.Parameters; 4 | import org.testng.annotations.Test; 5 | import org.xbib.elasticsearch.plugin.jdbc.RiverContext; 6 | import org.xbib.elasticsearch.river.jdbc.RiverSource; 7 | import org.xbib.elasticsearch.support.helper.AbstractRiverNodeTest; 8 | 9 | public class RiverScriptTests extends AbstractRiverNodeTest { 10 | 11 | @Override 12 | public RiverSource getRiverSource() { 13 | return new SimpleRiverSource(); 14 | } 15 | 16 | @Override 17 | public RiverContext getRiverContext() { 18 | return new RiverContext(); 19 | } 20 | 21 | /** 22 | * Orders table (star query) 23 | * 24 | * @param riverResource the river definition 25 | * @throws Exception if test fails 26 | */ 27 | @Test 28 | @Parameters({"river1"}) 29 | public void testSimpleRiverOnce(String riverResource) throws Exception { 30 | createRiver(riverResource); 31 | waitForInactiveRiver(); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/java/org/xbib/elasticsearch/river/jdbc/strategy/simple/RiverStoredProcedureTests.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.river.jdbc.strategy.simple; 2 | 3 | import org.elasticsearch.action.search.SearchResponse; 4 | import org.elasticsearch.index.query.QueryBuilders; 5 | import org.testng.annotations.Parameters; 6 | import org.testng.annotations.Test; 7 | import org.xbib.elasticsearch.plugin.jdbc.RiverContext; 8 | import org.xbib.elasticsearch.river.jdbc.RiverSource; 9 | import org.xbib.elasticsearch.support.helper.AbstractRiverNodeTest; 10 | 11 | public class RiverStoredProcedureTests extends AbstractRiverNodeTest { 12 | 13 | @Override 14 | public RiverSource getRiverSource() { 15 | return new SimpleRiverSource(); 16 | } 17 | 18 | @Override 19 | public RiverContext getRiverContext() { 20 | return new RiverContext(); 21 | } 22 | 23 | @Test 24 | @Parameters({"river8"}) 25 | public void testSimpleStoredProcedure(String riverResource) 26 | throws Exception { 27 | createRiver(riverResource); 28 | waitForInactiveRiver(); 29 | assertHits("1", 5); 30 | logger.info("got the five hits"); 31 | } 32 | 33 | @Test 34 | @Parameters({"river9"}) 35 | public void testRegisterStoredProcedure(String riverResource) throws Exception { 36 | createRiver(riverResource); 37 | waitForInactiveRiver(); 38 | assertHits("1", 1); 39 | logger.info("got the hit"); 40 | SearchResponse response = client("1").prepareSearch("my_jdbc_river_index") 41 | .setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(); 42 | String resp = response.getHits().getHits()[0].getSource().toString(); 43 | logger.info("resp={}", resp); 44 | assertEquals("{mySupplierName=Acme, Inc.}", resp); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/java/org/xbib/elasticsearch/river/jdbc/strategy/simple/storedprocedure/StoredProcedureJavaDBSample.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.river.jdbc.strategy.simple.storedprocedure; 2 | 3 | import java.sql.Connection; 4 | import java.sql.DriverManager; 5 | import java.sql.PreparedStatement; 6 | import java.sql.ResultSet; 7 | import java.sql.SQLException; 8 | import java.sql.Statement; 9 | 10 | public class StoredProcedureJavaDBSample { 11 | 12 | public static void showSuppliers(ResultSet[] rs) 13 | throws SQLException { 14 | 15 | Connection con = DriverManager.getConnection("jdbc:default:connection"); 16 | Statement stmt = null; 17 | 18 | String query = 19 | "select SUPPLIERS.SUP_NAME, " + 20 | "COFFEES.COF_NAME " + 21 | "from SUPPLIERS, COFFEES " + 22 | "where SUPPLIERS.SUP_ID = " + 23 | "COFFEES.SUP_ID " + 24 | "order by SUP_NAME"; 25 | 26 | stmt = con.createStatement(); 27 | rs[0] = stmt.executeQuery(query); 28 | } 29 | 30 | public static void getSupplierOfCoffee(String coffeeName, String[] supplierName) 31 | throws SQLException { 32 | 33 | Connection con = DriverManager.getConnection("jdbc:default:connection"); 34 | PreparedStatement pstmt = null; 35 | ResultSet rs = null; 36 | 37 | String query = 38 | "select SUPPLIERS.SUP_NAME " + 39 | "from SUPPLIERS, COFFEES " + 40 | "where " + 41 | "SUPPLIERS.SUP_ID = COFFEES.SUP_ID " + 42 | "and ? = COFFEES.COF_NAME"; 43 | 44 | pstmt = con.prepareStatement(query); 45 | pstmt.setString(1, coffeeName); 46 | rs = pstmt.executeQuery(); 47 | 48 | if (rs.next()) { 49 | supplierName[0] = rs.getString(1); 50 | } else { 51 | supplierName[0] = null; 52 | } 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/java/org/xbib/elasticsearch/river/jdbc/support/StringKeyValueStreamListener.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.river.jdbc.support; 2 | 3 | import org.xbib.elasticsearch.plugin.jdbc.IndexableObject; 4 | import org.xbib.elasticsearch.plugin.jdbc.PlainKeyValueStreamListener; 5 | import org.xbib.elasticsearch.river.jdbc.RiverMouth; 6 | 7 | import java.io.IOException; 8 | 9 | /** 10 | * This class consumes pairs from a key/value stream 11 | * and transports them to the river mouth. 12 | */ 13 | public class StringKeyValueStreamListener extends PlainKeyValueStreamListener { 14 | 15 | private RiverMouth output; 16 | 17 | public StringKeyValueStreamListener output(RiverMouth output) { 18 | this.output = output; 19 | return this; 20 | } 21 | 22 | /** 23 | * The object is complete. Push it to the river mouth. 24 | * 25 | * @param object the object 26 | * @return this value listener 27 | * @throws IOException 28 | */ 29 | public StringKeyValueStreamListener end(IndexableObject object) throws IOException { 30 | if (object.isEmpty()) { 31 | return this; 32 | } 33 | if (output != null) { 34 | if (object.optype() == null) { 35 | output.index(object, false); 36 | } else if ("index".equals(object.optype())) { 37 | output.index(object, false); 38 | } else if ("create".equals(object.optype())) { 39 | output.index(object, true); 40 | } else if ("delete".equals(object.optype())) { 41 | output.delete(object); 42 | } else { 43 | throw new IllegalArgumentException("unknown optype: " + object.optype()); 44 | } 45 | } 46 | return this; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/java/org/xbib/elasticsearch/river/jdbc/support/TimeWindowedTests.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.river.jdbc.support; 2 | 3 | import org.elasticsearch.common.joda.time.DateTime; 4 | import org.elasticsearch.common.joda.time.format.DateTimeFormat; 5 | import org.testng.annotations.Test; 6 | import org.xbib.elasticsearch.plugin.jdbc.PlainIndexableObject; 7 | import org.xbib.elasticsearch.river.jdbc.strategy.simple.SimpleRiverMouth; 8 | 9 | import java.io.IOException; 10 | 11 | import static org.testng.Assert.assertEquals; 12 | 13 | public class TimeWindowedTests { 14 | 15 | @Test 16 | public void testTimeWindow() throws IOException { 17 | SimpleRiverMouth mouth = new SimpleRiverMouth(); 18 | // daily index format 19 | String index = "'test-'YYYY.MM.dd"; 20 | mouth.setTimeWindowed(true).setIndex(index); 21 | mouth.index(new PlainIndexableObject(), false); 22 | String dayIndex = DateTimeFormat.forPattern(index).print(new DateTime()); 23 | assertEquals(mouth.getIndex(), dayIndex); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/java/org/xbib/elasticsearch/river/plugin/SQLCommandTests.java: -------------------------------------------------------------------------------- 1 | package org.xbib.elasticsearch.river.plugin; 2 | 3 | import java.io.IOException; 4 | 5 | import org.testng.Assert; 6 | import org.testng.annotations.Test; 7 | 8 | import org.xbib.elasticsearch.plugin.jdbc.SQLCommand; 9 | 10 | public class SQLCommandTests extends Assert { 11 | @Test 12 | public void simpleQuery() throws IOException { 13 | SQLCommand sc = new SQLCommand().setSQL("select * from table"); 14 | assertTrue(sc.isQuery()); 15 | } 16 | 17 | @Test 18 | public void updateQueryType() throws IOException { 19 | SQLCommand sc = new SQLCommand().setSQL("update foo"); 20 | assertFalse(sc.isQuery()); 21 | } 22 | 23 | @Test 24 | public void updateWithSubselect() throws IOException { 25 | SQLCommand sc = new SQLCommand().setSQL("update foo set thingie = select"); 26 | assertFalse(sc.isQuery()); 27 | } 28 | 29 | @Test 30 | public void updateWithSubselectAndLeadingWhitespace() throws IOException { 31 | SQLCommand sc = new SQLCommand().setSQL(" update foo set thingie = select"); 32 | assertFalse(sc.isQuery()); 33 | } 34 | 35 | @Test 36 | public void updateUpperCaseWithSelect() throws IOException { 37 | SQLCommand sc = new SQLCommand().setSQL("UPDATE foo set thingie = SELECT"); 38 | assertFalse(sc.isQuery()); 39 | } 40 | 41 | @Test 42 | public void insertWithSelect() throws IOException { 43 | SQLCommand sc = new SQLCommand().setSQL("insert into foo values select * from bar"); 44 | assertFalse(sc.isQuery()); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, out 2 | 3 | log4j.logger.BulkTransportClient=DEBUG 4 | log4j.logger.IngestTransportClient=DEBUG 5 | log4j.logger.NodeClient=DEBUG 6 | 7 | log4j.appender.out=org.apache.log4j.ConsoleAppender 8 | log4j.appender.out.layout=org.apache.log4j.PatternLayout 9 | log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c][%t] %m%n 10 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 17 | 18 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/derby/create-producttables.sql: -------------------------------------------------------------------------------- 1 | create table "products" ("id" int not null, "name" varchar(32), "amount" integer, "price" decimal(22,4), "created_at" timestamp, "updated_at" timestamp, "deleted_at" timestamp, primary key ("id")) 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/derby/delete-producttables.sql: -------------------------------------------------------------------------------- 1 | drop table "products" 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/derby/river-existedWhereClause.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:derby:memory:myDB", 5 | "user" : "", 6 | "password" : "", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/derby/river-existedWhereClauseWithOverlap.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:derby:memory:myDB", 5 | "user" : "", 6 | "password" : "", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "last_run_timestamp_overlap" : "2s", 12 | "index" : "my_jdbc_river_index", 13 | "type" : "my_jdbc_river_type" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/derby/river-sqlForTestDeletions.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:derby:memory:myDB", 5 | "user" : "", 6 | "password" : "", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/derby/river-sqlForTestDeletionsAndWherePlaceholder.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:derby:memory:myDB", 5 | "user" : "", 6 | "password" : "", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE $where AND 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/derby/river-sqlparams.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:derby:memory:myDB", 5 | "user" : "", 6 | "password" : "", 7 | "sql": [ 8 | { 9 | "statement" : "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=?", 10 | "parameter" : [ 1 ] 11 | } 12 | ], 13 | "column_updated_at": "updated_at", 14 | "column_created_at": "created_at", 15 | "column_deleted_at": "deleted_at", 16 | "index" : "my_jdbc_river_index", 17 | "type" : "my_jdbc_river_type" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/derby/river-whereClausePlaceholder.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:derby:memory:myDB", 5 | "user" : "", 6 | "password" : "", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=1 AND $where", 8 | "poll": "1h", 9 | "column_updated_at": "updated_at", 10 | "column_created_at": "created_at", 11 | "column_deleted_at": "deleted_at", 12 | "index" : "my_jdbc_river", 13 | "type" : "my_jdbc_river" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/h2/create-producttables.sql: -------------------------------------------------------------------------------- 1 | create table "products" ("id" int not null, "name" varchar(32), "amount" integer, "price" decimal(22,4), "created_at" timestamp, "updated_at" timestamp, "deleted_at" timestamp, primary key ("id")) 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/h2/delete-producttables.sql: -------------------------------------------------------------------------------- 1 | drop table "products" 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/h2/river-existedWhereClause.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:h2:./target/test", 5 | "user" : "", 6 | "password" : "", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/h2/river-existedWhereClauseWithOverlap.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:h2:./target/test", 5 | "user" : "", 6 | "password" : "", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "last_run_timestamp_overlap" : "2s", 12 | "index" : "my_jdbc_river_index", 13 | "type" : "my_jdbc_river_type" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/h2/river-sqlForTestDeletions.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:h2:./target/test", 5 | "user" : "", 6 | "password" : "", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/h2/river-sqlForTestDeletionsAndWherePlaceholder.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:h2:./target/test", 5 | "user" : "", 6 | "password" : "", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE $where AND 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/h2/river-sqlparams.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" :{ 3 | "strategy" : "column", 4 | "url" : "jdbc:h2:./target/test", 5 | "user" : "", 6 | "password" : "", 7 | "sql": [ 8 | { 9 | "statement" : "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=?", 10 | "parameter" : [ 1 ] 11 | } 12 | ], 13 | "column_updated_at": "updated_at", 14 | "column_created_at": "created_at", 15 | "column_deleted_at": "deleted_at", 16 | "index" : "my_jdbc_river_index", 17 | "type" : "my_jdbc_river_type" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/h2/river-whereClausePlaceholder.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:h2:./target/test", 5 | "user" : "", 6 | "password" : "", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=1 AND $where", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/hsqldb/create-producttables.sql: -------------------------------------------------------------------------------- 1 | create table "products" ("id" int not null, "name" varchar(32), "amount" integer, "price" decimal(22,4), "created_at" timestamp, "updated_at" timestamp, "deleted_at" timestamp, primary key ("id")) 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/hsqldb/delete-producttables.sql: -------------------------------------------------------------------------------- 1 | drop table "products" 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/hsqldb/river-existedWhereClause.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:hsqldb:mem:test", 5 | "user" : "sa", 6 | "password" : "", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/hsqldb/river-existedWhereClauseWithOverlap.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:hsqldb:mem:test", 5 | "user" : "sa", 6 | "password" : "", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "last_run_timestamp_overlap" : "2s", 12 | "index" : "my_jdbc_river_index", 13 | "type" : "my_jdbc_river_type" 14 | } 15 | } -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/hsqldb/river-sqlForTestDeletions.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:hsqldb:mem:test", 5 | "user" : "sa", 6 | "password" : "", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/hsqldb/river-sqlForTestDeletionsAndWherePlaceholder.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:hsqldb:mem:test", 5 | "user" : "sa", 6 | "password" : "", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE $where AND 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/hsqldb/river-sqlparams.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "driver" : "org.hsqldb.jdbcDriver", 5 | "url" : "jdbc:hsqldb:mem:test", 6 | "user" : "sa", 7 | "password" : "", 8 | "sql": [ 9 | { 10 | "statement" : "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=?", 11 | "parameter" : [ 1 ] 12 | } 13 | ], 14 | "column_updated_at": "updated_at", 15 | "column_created_at": "created_at", 16 | "column_deleted_at": "deleted_at", 17 | "index" : "my_jdbc_river_index", 18 | "type" : "my_jdbc_river_type" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/hsqldb/river-whereClausePlaceholder.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:hsqldb:mem:test", 5 | "user" : "sa", 6 | "password" : "", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=1 AND $where", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/mysql/create-producttables.sql: -------------------------------------------------------------------------------- 1 | create table products (id int not null, name varchar(32), amount integer, price decimal(22,4), created_at datetime, updated_at datetime, deleted_at datetime, primary key (id)) 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/mysql/delete-producttables.sql: -------------------------------------------------------------------------------- 1 | drop table products 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/mysql/river-existedWhereClause.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:mysql://localhost:3306/test", 5 | "user" : "", 6 | "password" : "", 7 | "sql": "SELECT id AS _id, id, name FROM products WHERE 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/mysql/river-existedWhereClauseWithOverlap.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:mysql://localhost:3306/test", 5 | "user" : "", 6 | "password" : "", 7 | "sql": "SELECT id AS _id, id, name FROM products WHERE 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "last_run_timestamp_overlap" : "2s", 12 | "index" : "my_jdbc_river_index", 13 | "type" : "my_jdbc_river_type" 14 | } 15 | } -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/mysql/river-sqlForTestDeletions.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:mysql://localhost:3306/test", 5 | "user" : "", 6 | "password" : "", 7 | "sql": "SELECT id AS _id, id, name FROM products WHERE 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/mysql/river-sqlForTestDeletionsAndWherePlaceholder.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:mysql://localhost:3306/test", 5 | "user" : "", 6 | "password" : "", 7 | "sql": "SELECT id AS _id, id, name FROM products WHERE $where AND 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/mysql/river-sqlparams.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:mysql://localhost:3306/test", 5 | "user" : "", 6 | "password" : "", 7 | "sql": [ 8 | { 9 | "statement" : "SELECT id as _id, id, name FROM products WHERE 1=?", 10 | "parameter" : [ 1 ] 11 | } 12 | ], 13 | "column_updated_at": "updated_at", 14 | "column_created_at": "created_at", 15 | "column_deleted_at": "deleted_at", 16 | "index" : "my_jdbc_river_index", 17 | "type" : "my_jdbc_river_type" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/mysql/river-whereClausePlaceholder.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:mysql://localhost:3306/test", 5 | "user" : "", 6 | "password" : "", 7 | "sql": "SELECT id as _id, id, name FROM products WHERE 1=1 AND $where", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/postgresql/create-producttables.sql: -------------------------------------------------------------------------------- 1 | create table "products" ("id" int not null, "name" varchar(32), "amount" integer, "price" decimal(22,4), "created_at" timestamp, "updated_at" timestamp, "deleted_at" timestamp, primary key ("id")) 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/postgresql/delete-producttables.sql: -------------------------------------------------------------------------------- 1 | drop table "products" 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/postgresql/river-existedWhereClause.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:postgresql://localhost:5432/test", 5 | "user" : "test", 6 | "password" : "test", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/postgresql/river-existedWhereClauseWithOverlap.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:postgresql://localhost:5432/test", 5 | "user" : "test", 6 | "password" : "test", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "last_run_timestamp_overlap" : "2s", 12 | "index" : "my_jdbc_river_index", 13 | "type" : "my_jdbc_river_type" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/postgresql/river-sqlForTestDeletions.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:postgresql://localhost:5432/test", 5 | "user" : "test", 6 | "password" : "test", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/postgresql/river-sqlForTestDeletionsAndWherePlaceholder.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:postgresql://localhost:5432/test", 5 | "user" : "test", 6 | "password" : "test", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE $where AND 1=1", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/postgresql/river-sqlparams.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" :{ 3 | "strategy" : "column", 4 | "url" : "jdbc:postgresql://localhost:5432/test", 5 | "user" : "test", 6 | "password" : "test", 7 | "sql": [ 8 | { 9 | "statement" : "SELECT \"id\" AS \"_id\", \"id", \"name\" FROM \"products\" WHERE 1=?", 10 | "parameter" : [ 1 ] 11 | } 12 | ], 13 | "column_updated_at": "updated_at", 14 | "column_created_at": "created_at", 15 | "column_deleted_at": "deleted_at", 16 | "index" : "my_jdbc_river_index", 17 | "type" : "my_jdbc_river_type" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/column/postgresql/river-whereClausePlaceholder.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "strategy" : "column", 4 | "url" : "jdbc:postgresql://localhost:5432/test", 5 | "user" : "test", 6 | "password" : "test", 7 | "sql": "SELECT \"id\" AS \"_id\", \"id\", \"name\" FROM \"products\" WHERE 1=1 AND $where", 8 | "column_updated_at": "updated_at", 9 | "column_created_at": "created_at", 10 | "column_deleted_at": "deleted_at", 11 | "index" : "my_jdbc_river_index", 12 | "type" : "my_jdbc_river_type" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/derby/create-jobtables.sql: -------------------------------------------------------------------------------- 1 | create table "products" ("_job" varchar(32), "name" varchar(32), "amount" integer, "price" decimal(22,4)) 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/derby/delete-jobtables.sql: -------------------------------------------------------------------------------- 1 | drop table "products" 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/derby/delete-ordertables.sql: -------------------------------------------------------------------------------- 1 | drop table "employees" 2 | drop table "departments" 3 | drop table "customers" 4 | drop table "products" 5 | drop table "orders" 6 | drop table "logs" -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/derby/jobriver-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:derby:memory:myDB", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : [ 7 | { 8 | "statement" : "select * from \"products\"" 9 | }, 10 | { 11 | "statement" : "delete from \"products\" where \"_job\" = ?", 12 | "parameter" : [ "$job" ] 13 | } 14 | ], 15 | "index" : "my_jdbc_river_index", 16 | "type" : "my_jdbc_river_type" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/derby/river-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:derby:memory:myDB", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : "select * from \"orders\"", 7 | "index" : "my_jdbc_river_index", 8 | "type" : "my_jdbc_river_type" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/derby/river-2.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" :{ 3 | "url" : "jdbc:derby:memory:myDB", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : "select * from \"products\"", 7 | "index" : "my_jdbc_river_index", 8 | "type" : "my_jdbc_river_type" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/derby/river-3.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:derby:memory:myDB", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : "select * from \"products\"", 7 | "maxrows" : 50, 8 | "index" : "my_jdbc_river_index", 9 | "type" : "my_jdbc_river_type" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/derby/river-5.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:derby:memory:myDB", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : [ 7 | { 8 | "statement" : "{call count_products(?)}", 9 | "parameter" : [ 1 ], 10 | "callable" : true 11 | } 12 | ], 13 | "index" : "my_jdbc_river_index", 14 | "type" : "my_jdbc_river_type" 15 | } 16 | } -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/derby/river-6.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:derby:memory:myDB", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : "select * from \"products\"", 7 | "schedule" : "0/5 0-59 0-23 ? * *", 8 | "bulk_flush_interval" : "1s", 9 | "index" : "my_jdbc_river_index", 10 | "type" : "my_jdbc_river_type" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/derby/river-7.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:derby:memory:myDB", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : [ 7 | { 8 | "statement" : "select \"message\" from \"logs\" where {fn timestampdiff(SQL_TSI_HOUR, \"modified\" ,?)} > 0", 9 | "parameter" : [ "$now" ] 10 | } 11 | ], 12 | "schedule" : "0/5 0-59 0-23 ? * *", 13 | "index" : "my_jdbc_river_index", 14 | "type" : "my_jdbc_river_type", 15 | "timezone" : "Asia/Jerusalem", 16 | "locale" : "iw_IL" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/h2/create-jobtables.sql: -------------------------------------------------------------------------------- 1 | create table "products" ("_job" varchar(32), "name" varchar(32), "amount" integer, "price" decimal(22,4)) 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/h2/delete-jobtables.sql: -------------------------------------------------------------------------------- 1 | drop table "products" 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/h2/delete-ordertables.sql: -------------------------------------------------------------------------------- 1 | drop table "employees" 2 | drop table "departments" 3 | drop table "customers" 4 | drop table "products" 5 | drop table "orders" 6 | drop table "logs" -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/h2/jobriver-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:h2:./target/test", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : [ 7 | { 8 | "statement" : "select * from \"products\"" 9 | }, 10 | { 11 | "statement" : "delete from \"products\" where \"_job\" = ?", 12 | "parameter" : [ "$job" ] 13 | } 14 | ], 15 | "index" : "my_jdbc_river_index", 16 | "type" : "my_jdbc_river_type" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/h2/river-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:h2:./target/test", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : "select * from \"orders\"", 7 | "index" : "my_jdbc_river_index", 8 | "type" : "my_jdbc_river_type" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/h2/river-2.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:h2:./target/test", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : "select * from \"products\"", 7 | "index" : "my_jdbc_river_index", 8 | "type" : "my_jdbc_river_type" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/h2/river-3.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:h2:./target/test", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : "select * from \"products\"", 7 | "maxrows" : 50, 8 | "index" : "my_jdbc_river_index", 9 | "type" : "my_jdbc_river_type" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/h2/river-6.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:h2:./target/test", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : "select * from \"products\"", 7 | "schedule" : "0/5 0-59 0-23 ? * *", 8 | "bulk_flush_interval" : "1s", 9 | "index" : "my_jdbc_river_index", 10 | "type" : "my_jdbc_river_type" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/h2/river-7.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:h2:./target/test", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : [ 7 | { 8 | "statement" : "select \"message\" from \"logs\" where {fn timestampdiff(SQL_TSI_HOUR, \"modified\" ,?)} > 0", 9 | "parameter" : [ "$now" ] 10 | } 11 | ], 12 | "schedule" : "0/5 0-59 0-23 ? * *", 13 | "index" : "my_jdbc_river_index", 14 | "type" : "my_jdbc_river_type", 15 | "timezone" : "Asia/Jerusalem", 16 | "locale" : "iw_IL" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/hsqldb/create-jobtables.sql: -------------------------------------------------------------------------------- 1 | create table products ("_job" varchar(32), "name" varchar(32), "amount" integer, "price" decimal(22,4)) 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/hsqldb/delete-jobtables.sql: -------------------------------------------------------------------------------- 1 | drop table "products" 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/hsqldb/delete-ordertables.sql: -------------------------------------------------------------------------------- 1 | drop table employees 2 | drop table departments 3 | drop table customers 4 | drop table products 5 | drop table orders 6 | drop table logs 7 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/hsqldb/jobriver-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:hsqldb:mem:test", 4 | "user" : "sa", 5 | "password" : "", 6 | "sql" : [ 7 | { 8 | "statement" : "select * from products" 9 | }, 10 | { 11 | "statement" : "delete from products where \"_job\" = ?", 12 | "parameter" : [ "$job" ] 13 | } 14 | ], 15 | "index" : "my_jdbc_river_index", 16 | "type" : "my_jdbc_river_type" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/hsqldb/river-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:hsqldb:mem:test", 4 | "user" : "sa", 5 | "password" : "", 6 | "sql" : "select * from orders", 7 | "index" : "my_jdbc_river_index", 8 | "type" : "my_jdbc_river_type" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/hsqldb/river-2.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:hsqldb:mem:test", 4 | "user" : "sa", 5 | "password" : "", 6 | "sql" : "select * from products", 7 | "index" : "my_jdbc_river_index", 8 | "type" : "my_jdbc_river_type" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/hsqldb/river-3.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:hsqldb:mem:test", 4 | "user" : "sa", 5 | "password" : "", 6 | "sql" : "select * from products", 7 | "maxrows" : 50, 8 | "index" : "my_jdbc_river_index", 9 | "type" : "my_jdbc_river_type" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/hsqldb/river-6.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:hsqldb:mem:test", 4 | "user" : "sa", 5 | "password" : "", 6 | "sql" : "select * from products", 7 | "schedule" : "0/5 0-59 0-23 ? * *", 8 | "bulk_flush_interval" : "1s", 9 | "index" : "my_jdbc_river_index", 10 | "type" : "my_jdbc_river_type" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/hsqldb/river-7.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:hsqldb:mem:test", 4 | "user" : "sa", 5 | "password" : "", 6 | "sql" : [ 7 | { 8 | "statement" : "select message from logs where {fn TIMESTAMPDIFF(SQL_TSI_HOUR, modified ,?)} > 0", 9 | "parameter" : [ "$now" ] 10 | } 11 | ], 12 | "schedule" : "0/5 0-59 0-23 ? * *", 13 | "index" : "my_jdbc_river_index", 14 | "type" : "my_jdbc_river_type", 15 | "timezone" : "Asia/Jerusalem", 16 | "locale" : "iw_IL" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/mysql/create-jobtables.sql: -------------------------------------------------------------------------------- 1 | create table products (`_job` varchar(32), `name` varchar(32), `amount` integer, `price` decimal(22,4)) 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/mysql/create-suppliertables.sql: -------------------------------------------------------------------------------- 1 | drop table IF EXISTS SUPPLIERS 2 | drop table IF EXISTS COFFEES 3 | drop procedure IF EXISTS SHOW_SUPPLIERS 4 | drop procedure IF EXISTS GET_SUPPLIER_OF_COFFEE 5 | create table SUPPLIERS (SUP_ID integer NOT NULL, SUP_NAME varchar(40) NOT NULL, STREET varchar(40) NOT NULL, CITY varchar(20) NOT NULL, STATE char(2) NOT NULL, ZIP char(5), PRIMARY KEY (SUP_ID)); 6 | create table COFFEES (COF_NAME varchar(32) NOT NULL, SUP_ID int NOT NULL, PRICE numeric(10,2) NOT NULL, SALES integer NOT NULL, TOTAL integer NOT NULL, PRIMARY KEY (COF_NAME), FOREIGN KEY (SUP_ID) REFERENCES SUPPLIERS (SUP_ID)); 7 | insert into SUPPLIERS values(49, 'Superior Coffee', '1 Party Place', 'Mendocino', 'CA', '95460') 8 | insert into SUPPLIERS values(101, 'Acme, Inc.', '99 Market Street', 'Groundsville', 'CA', '95199') 9 | insert into SUPPLIERS values(150, 'The High Ground', '100 Coffee Lane', 'Meadows', 'CA', '93966') 10 | insert into COFFEES values('Colombian', 00101, 7.99, 0, 0) 11 | insert into COFFEES values('French_Roast', 00049, 8.99, 0, 0) 12 | insert into COFFEES values('Espresso', 00150, 9.99, 0, 0) 13 | insert into COFFEES values('Colombian_Decaf', 00101, 8.99, 0, 0) 14 | insert into COFFEES values('French_Roast_Decaf', 00049, 9.99, 0, 0) 15 | create procedure SHOW_SUPPLIERS() begin select SUPPLIERS.SUP_NAME, COFFEES.COF_NAME from SUPPLIERS, COFFEES where SUPPLIERS.SUP_ID = COFFEES.SUP_ID order by SUP_NAME; end 16 | create procedure GET_SUPPLIER_OF_COFFEE(IN coffeeName varchar(32), OUT supplierName varchar(40)) begin select SUPPLIERS.SUP_NAME into supplierName from SUPPLIERS, COFFEES where SUPPLIERS.SUP_ID = COFFEES.SUP_ID and coffeeName = COFFEES.COF_NAME; select supplierName; end -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/mysql/delete-jobtables.sql: -------------------------------------------------------------------------------- 1 | drop table products 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/mysql/delete-ordertables.sql: -------------------------------------------------------------------------------- 1 | drop table employees 2 | drop table departments 3 | drop table customers 4 | drop table products 5 | drop table orders 6 | drop table logs -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/mysql/delete-suppliertables.sql: -------------------------------------------------------------------------------- 1 | drop table COFFEES 2 | drop table SUPPLIERS 3 | drop procedure SHOW_SUPPLIERS 4 | drop procedure GET_SUPPLIER_OF_COFFEE -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/mysql/jobriver-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:mysql://localhost:3306/test", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : [ 7 | { 8 | "statement" : "select * from products" 9 | }, 10 | { 11 | "statement" : "delete from products where `_job` = ?", 12 | "parameter" : [ "$job" ] 13 | } 14 | ], 15 | "index" : "my_jdbc_river_index", 16 | "type" : "my_jdbc_river_type" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/mysql/river-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:mysql://localhost:3306/test", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : "select * from orders", 7 | "index" : "my_jdbc_river_index", 8 | "type" : "my_jdbc_river_type" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/mysql/river-2.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:mysql://localhost:3306/test", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : "select * from products", 7 | "index" : "my_jdbc_river_index", 8 | "type" : "my_jdbc_river_type" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/mysql/river-3.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:mysql://localhost:3306/test", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : "select * from products", 7 | "maxrows" : 50, 8 | "index" : "my_jdbc_river_index", 9 | "type" : "my_jdbc_river_type" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/mysql/river-6.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:mysql://localhost:3306/test", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : "select * from products", 7 | "schedule" : "0/5 0-59 0-23 ? * *", 8 | "bulk_flush_interval" : "1s", 9 | "index" : "my_jdbc_river_index", 10 | "type" : "my_jdbc_river_type" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/mysql/river-7.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:mysql://localhost:3306/test", 4 | "user" : "", 5 | "password" : "", 6 | 7 | "sql" : [ 8 | { 9 | "statement" : "select message from logs where {fn timestampdiff(SQL_TSI_HOUR, modified ,?)} > 0", 10 | "parameter" : [ "$now" ] 11 | } 12 | ], 13 | "schedule" : "0/5 0-59 0-23 ? * *", 14 | "index" : "my_jdbc_river_index", 15 | "type" : "my_jdbc_river_type", 16 | "timezone" : "Asia/Jerusalem", 17 | "locale" : "iw_IL" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/mysql/river-8.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:mysql://localhost:3306/test", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : [ 7 | { 8 | "callable" : true, 9 | "statement" : "{call SHOW_SUPPLIERS()}" 10 | } 11 | ], 12 | "index" : "my_jdbc_river_index", 13 | "type" : "my_jdbc_river_type" 14 | } 15 | } -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/mysql/river-9.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:mysql://localhost:3306/test", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : [ 7 | { 8 | "callable" : true, 9 | "statement" : "{call GET_SUPPLIER_OF_COFFEE(?,?)}", 10 | "parameter" : [ 11 | "Colombian" 12 | ], 13 | "register" : { 14 | "mySupplierName" : { "pos" : 2, "type" : "varchar" } 15 | } 16 | } 17 | ], 18 | "index" : "my_jdbc_river_index", 19 | "type" : "my_jdbc_river_type" 20 | } 21 | } -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/oracle/create-jobtables.sql: -------------------------------------------------------------------------------- 1 | create table "products" ("_job" varchar(32), "name" varchar(32), "amount" integer, "price" decimal(22,4)) 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/oracle/delete-jobtables.sql: -------------------------------------------------------------------------------- 1 | drop table "products" 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/oracle/delete-ordertables.sql: -------------------------------------------------------------------------------- 1 | drop table "employees" 2 | drop table "departments" 3 | drop table "customers" 4 | drop table "products" 5 | drop table "orders" 6 | drop table "logs" -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/oracle/jobriver-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" :{ 3 | "url" : "jdbc:oracle:thin:@//localhost:1521/test?autoReconnect=true", 4 | "user" : "test", 5 | "password" : "test", 6 | "sql" : [ 7 | { 8 | "statement" : "select * from \"products\"" 9 | }, 10 | { 11 | "statement" : "delete from \"products\" where \"_job\" = ?", 12 | "parameter" : [ "$job" ] 13 | } 14 | ], 15 | "index" : "my_jdbc_river_index", 16 | "type" : "my_jdbc_river_type" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/oracle/river-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:oracle:thin:@//localhost:1521/test?autoReconnect=true", 4 | "user" : "test", 5 | "password" : "test", 6 | "sql" : "select * from \"orders\"", 7 | "index" : "my_jdbc_river_index", 8 | "type" : "my_jdbc_river_type" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/oracle/river-2.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:oracle:thin:@//localhost:1521/test?autoReconnect=true", 4 | "user" : "test", 5 | "password" : "test", 6 | "sql" : "select * from \"products\"", 7 | "index" : "my_jdbc_river_index", 8 | "type" : "my_jdbc_river_type" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/oracle/river-3.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:oracle:thin:@//localhost:1521/test?autoReconnect=true", 4 | "user" : "test", 5 | "password" : "test", 6 | "sql" : "select * from \"products\"", 7 | "maxrows" : 50, 8 | "autocommit" : true, 9 | "index" : "my_jdbc_river_index", 10 | "type" : "my_jdbc_river_type" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/postgresql/create-jobtables.sql: -------------------------------------------------------------------------------- 1 | create table "products" ("_job" varchar(32), "name" varchar(32), "amount" integer, "price" decimal(22,4)) 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/postgresql/delete-jobtables.sql: -------------------------------------------------------------------------------- 1 | drop table "products" 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/postgresql/delete-ordertables.sql: -------------------------------------------------------------------------------- 1 | drop table "employees" 2 | drop table "departments" 3 | drop table "customers" 4 | drop table "products" 5 | drop table "orders" 6 | drop table "logs" 7 | drop table "sal_emp" 8 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/postgresql/jobriver-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:postgresql://localhost:5432/test", 4 | "user" : "test", 5 | "password" : "test", 6 | "sql" : [ 7 | { 8 | "statement" : "select * from \"products\"" 9 | }, 10 | { 11 | "statement" : "delete from \"products\" where \"_job\" = ?", 12 | "parameter" : [ "$job" ] 13 | } 14 | ], 15 | "index" : "my_jdbc_river_index", 16 | "type" : "my_jdbc_river_type" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/postgresql/river-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:postgresql://localhost:5432/test", 4 | "user" : "test", 5 | "password" : "test", 6 | "sql" : "select * from \"orders\"", 7 | "index" : "my_jdbc_river_index", 8 | "type" : "my_jdbc_river_type" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/postgresql/river-2.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:postgresql://localhost:5432/test", 4 | "user" : "test", 5 | "password" : "test", 6 | "sql" : "select * from \"products\"", 7 | "index" : "my_jdbc_river_index", 8 | "type" : "my_jdbc_river_type" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/postgresql/river-3.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:postgresql://localhost:5432/test", 4 | "user" : "test", 5 | "password" : "test", 6 | "sql" : "select * from \"products\"", 7 | "maxrows" : 50, 8 | "autocommit" : true, 9 | "index" : "my_jdbc_river_index", 10 | "type" : "my_jdbc_river_type" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/sqlite/create-jobtables.sql: -------------------------------------------------------------------------------- 1 | create table "products" ("_job" varchar(32), "name" varchar(32), "amount" integer, "price" decimal(22,4)) 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/sqlite/delete-jobtables.sql: -------------------------------------------------------------------------------- 1 | drop table "products" 2 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/sqlite/delete-ordertables.sql: -------------------------------------------------------------------------------- 1 | drop table "employees" 2 | drop table "departments" 3 | drop table "customers" 4 | drop table "products" 5 | drop table "orders" 6 | drop table "sal_emp" 7 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/sqlite/jobriver-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:sqlite:file::memory:?cache=shared", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : [ 7 | { 8 | "statement" : "select * from \"products\"" 9 | }, 10 | { 11 | "statement" : "delete from \"products\" where \"_job\" = ?", 12 | "parameter" : [ "$job" ] 13 | } 14 | ], 15 | "index" : "my_jdbc_river_index", 16 | "type" : "my_jdbc_river_type" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/sqlite/river-1.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:sqlite:file::memory:?cache=shared", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : "select * from \"orders\"", 7 | "index" : "my_jdbc_river_index", 8 | "type" : "my_jdbc_river_type" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/sqlite/river-2.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:sqlite:file::memory:?cache=shared", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : "select * from \"products\"", 7 | "index" : "my_jdbc_river_index", 8 | "type" : "my_jdbc_river_type" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/org/xbib/elasticsearch/river/jdbc/strategy/simple/sqlite/river-3.json: -------------------------------------------------------------------------------- 1 | { 2 | "jdbc" : { 3 | "url" : "jdbc:sqlite:file::memory:?cache=shared", 4 | "user" : "", 5 | "password" : "", 6 | "sql" : "select * from \"products\"", 7 | "maxrows" : 50, 8 | "autocommit" : true, 9 | "index" : "my_jdbc_river_index", 10 | "type" : "my_jdbc_river_type" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/testsuite/column/derby.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 21 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/testsuite/column/h2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 21 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/testsuite/column/hsqldb.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 21 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/testsuite/column/mysql.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 21 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/testsuite/column/postgresql.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 21 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/testsuite/cron/cron.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/testsuite/mock/hsqldb.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 16 | 18 | 20 | 22 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/testsuite/mock/mysql.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 16 | 18 | 20 | 22 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/testsuite/plugin/plugin.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /elasticsearch-jdbc/src/test/resources/testsuite/support/support.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /elasticsearch-parent/README.md: -------------------------------------------------------------------------------- 1 | 项目简介 2 | ----- 3 | Maven父pom项目，将pom公共部分提取出来汇总在父pom中，降低各项目编写pom的复杂度。 4 | 5 | 包含的内容 6 | ----- 7 | * 指定各插件（plugin）的版本，基本参数（如编码） 8 | * 指定常用依赖的具体版本，希望通过父pom能控制所有二方和三方包的依赖，降低项目pom依赖冲突的可能性。如果需要，项目可以自己特殊指定。 9 | * 指定公司私有Maven仓库的地址。 10 | * 指定公司私有仓库的发布地址，发布需要有帐号密码。 11 | 12 | 使用方式 13 | ----- 14 | * 各项目在pom.xml中增加以下内容： 15 | 16 | ```java 17 | 18 | ... 19 | 20 | 21 | 22 | zx.soft 23 | parent 24 | 1.2.0 25 | 26 | 27 | ... 28 | 29 | 30 | 31 | 32 | zxsoft-public 33 | Nexus Release Repository 34 | http://192.168.3.23:18081/nexus/content/groups/public/ 35 | 36 | 37 | 38 | ... 39 | 40 | ``` 41 | 42 | * 如果pom中依赖或者插件的版本处提示“Overriding managed ...”，说明父pom中已经指定，此处不需要再指定。如果有特殊需要可以覆盖。 43 | 44 | > 注意：父pom中依赖的版本或者插件的版本，可能会在不通知的情况下变更。项目需要自己进行回归测试。 -------------------------------------------------------------------------------- /elasticsearch-web/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## ElasticSearch搜索的舆情Web项目 3 | 4 | > 基于Java开发。 -------------------------------------------------------------------------------- /elasticsearch-web/src/main/assembly/distribution.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | distribution 7 | 8 | tar.gz 9 | 10 | ${project.artifactId} 11 | 12 | 13 | src/main/resources 14 | 15 | elastic_search.properties 16 | logback.xml 17 | web-server.properties 18 | 19 | /conf 20 | true 21 | 22 | 23 | src/main/bin 24 | 25 | * 26 | 27 | /bin 28 | 0755 29 | 30 | 31 | 32 | 33 | /lib 34 | 35 | 36 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/bin/ctl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mainClass=zx.soft.es.driver.ElasticSearchDriver 4 | 5 | # resolve links - $0 may be a softlink 6 | PRG="$0" 7 | 8 | while [ -h "$PRG" ]; do 9 | ls=`ls -ld "$PRG"` 10 | link=`expr "$ls" : '.*-> \(.*\)$'` 11 | if expr "$link" : '/.*' > /dev/null; then 12 | PRG="$link" 13 | else 14 | PRG=`dirname "$PRG"`/"$link" 15 | fi 16 | done 17 | 18 | # Get standard environment variables 19 | PRGDIR=`dirname "$PRG"` 20 | 21 | PROJECT_DIR=`cd "$PRGDIR/.." >/dev/null; pwd` 22 | echo PROJECT_DIR=$PROJECT_DIR 23 | 24 | CLASSPATH="$CLASSHPATH:$PROJECT_DIR/conf" 25 | 26 | for jar in "$PROJECT_DIR/lib"/*.jar; do 27 | CLASSPATH="$CLASSPATH:$jar" 28 | done 29 | echo CLASSPATH=$CLASSPATH 30 | 31 | JVMARGS="${JVMARGS} -Dproject_dir=${PROJECT_DIR}" 32 | echo JVMARGS=$JVMARGS 33 | 34 | usage() { 35 | echo >&2 "usage: $PRG [args]" 36 | echo 'Valid commands: start, stop' 37 | exit 1 38 | } 39 | 40 | start() { 41 | JAVA=${JAVA-'java'} 42 | exec $JAVA $JVMARGS -classpath "$CLASSPATH" $mainClass "$@" & 43 | } 44 | 45 | case $1 in 46 | (start) 47 | shift 48 | start $@ 49 | ;; 50 | (stop) 51 | echo "stop" 52 | ;; 53 | (restart) 54 | echo "restart" 55 | ;; 56 | (*) 57 | echo >&2 "$PRG: error: unknown command '$1'" 58 | usage 59 | ;; 60 | esac 61 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/java/zx/soft/es/client/BuildClient.java: -------------------------------------------------------------------------------- 1 | package zx.soft.es.client; 2 | 3 | import java.util.Properties; 4 | 5 | import org.elasticsearch.client.transport.TransportClient; 6 | import org.elasticsearch.common.settings.ImmutableSettings; 7 | import org.elasticsearch.common.settings.Settings; 8 | import org.elasticsearch.common.transport.InetSocketTransportAddress; 9 | 10 | import zx.soft.es.utils.ConfigUtil; 11 | 12 | /** 13 | * 建立Client 与集群建立连接 14 | * @author fgq 15 | * 16 | */ 17 | public class BuildClient { 18 | 19 | public static TransportClient buildClient() { 20 | TransportClient client = null; 21 | try { 22 | Properties props = ConfigUtil.getProps("elastic_search.properties"); 23 | Settings settings = ImmutableSettings.settingsBuilder().put("cluster.name", props.getProperty("es.name")) 24 | .build(); 25 | client = new TransportClient(settings); 26 | client.addTransportAddress(new InetSocketTransportAddress(props.getProperty("es.ip"), Integer 27 | .parseInt(props.getProperty("es.port")))) 28 | /*.addTransportAddress( 29 | new InetSocketTransportAddress(props.getProperty("es.ip2"), Integer.parseInt(props 30 | .getProperty("es.port")))) 31 | .addTransportAddress( 32 | new InetSocketTransportAddress(props.getProperty("es.ip3"), Integer.parseInt(props 33 | .getProperty("es.port"))))*/; 34 | 35 | return client; 36 | } catch (Exception e) { 37 | throw new RuntimeException(e); 38 | } 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/java/zx/soft/es/driver/ElasticSearchDriver.java: -------------------------------------------------------------------------------- 1 | package zx.soft.es.driver; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import zx.soft.es.web.server.ESIndexServer; 7 | import zx.soft.es.web.server.ESSearchServer; 8 | 9 | /** 10 | * 驱动类 11 | * 12 | * @author fgq 13 | * 14 | */ 15 | public class ElasticSearchDriver { 16 | 17 | private static Logger logger = LoggerFactory.getLogger(ElasticSearchDriver.class); 18 | 19 | /** 20 | * 主函数 21 | */ 22 | public static void main(String[] args) { 23 | 24 | if (args.length == 0) { 25 | System.err.println("Usage: Driver "); 26 | System.exit(-1); 27 | } 28 | String[] leftArgs = new String[args.length - 1]; 29 | System.arraycopy(args, 1, leftArgs, 0, leftArgs.length); 30 | 31 | switch (args[0]) { 32 | case "esIndexServer": 33 | logger.info("索引接口： "); 34 | ESIndexServer.main(leftArgs); 35 | break; 36 | case "esSearchServer": 37 | logger.info("搜索接口："); 38 | ESSearchServer.main(leftArgs); 39 | default: 40 | return; 41 | } 42 | 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/java/zx/soft/es/model/PostData.java: -------------------------------------------------------------------------------- 1 | package zx.soft.es.model; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * POST 的索引数据：int num,Listrecords 7 | * @author fgq 8 | * 9 | */ 10 | public class PostData { 11 | 12 | private List records; 13 | private int num; 14 | 15 | public List getRecords() { 16 | return records; 17 | } 18 | 19 | public void setRecords(List records) { 20 | this.records = records; 21 | } 22 | 23 | public int getNum() { 24 | return num; 25 | } 26 | 27 | public void setNum(int num) { 28 | this.num = num; 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/java/zx/soft/es/model/RequestParams.java: -------------------------------------------------------------------------------- 1 | package zx.soft.es.model; 2 | 3 | public class RequestParams { 4 | 5 | private String index = ""; 6 | private String type = ""; 7 | private int from = 0; 8 | private int size = 10; 9 | private boolean explain = Boolean.FALSE; 10 | 11 | @Override 12 | public String toString() { 13 | return "requestParams:{index=" + index + ",type=" + type + ",from=" + from + ",size=" + size + ",explain=" 14 | + explain + "}"; 15 | } 16 | 17 | public String getIndex() { 18 | return index; 19 | } 20 | 21 | public void setIndex(String index) { 22 | this.index = index; 23 | } 24 | 25 | public String getType() { 26 | return type; 27 | } 28 | 29 | public void setType(String type) { 30 | this.type = type; 31 | } 32 | 33 | public int getFrom() { 34 | return from; 35 | } 36 | 37 | public void setFrom(int from) { 38 | this.from = from; 39 | } 40 | 41 | public int getSize() { 42 | return size; 43 | } 44 | 45 | public void setSize(int size) { 46 | this.size = size; 47 | } 48 | 49 | public boolean isExplain() { 50 | return explain; 51 | } 52 | 53 | public void setExplain(boolean explain) { 54 | this.explain = explain; 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/java/zx/soft/es/search/Search.java: -------------------------------------------------------------------------------- 1 | package zx.soft.es.search; 2 | 3 | import org.elasticsearch.action.search.SearchRequestBuilder; 4 | import org.elasticsearch.action.search.SearchResponse; 5 | import org.elasticsearch.index.query.QueryBuilder; 6 | import org.elasticsearch.index.query.QueryBuilders; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import zx.soft.es.client.BuildClient; 11 | import zx.soft.es.model.RequestParams; 12 | import zx.soft.es.model.SearchParams; 13 | 14 | /** 15 | * 检索，查询 16 | * @author fgq 17 | * 18 | */ 19 | public class Search { 20 | 21 | private static Logger logger = LoggerFactory.getLogger(Search.class); 22 | 23 | public SearchResponse doSearch(SearchParams searchParams, RequestParams requestParams) { 24 | 25 | //设置查询index，默认spiderindextest 26 | SearchRequestBuilder searchrequestbuilder = BuildClient.buildClient().prepareSearch(requestParams.getIndex()); 27 | QueryBuilder query = QueryBuilders.matchAllQuery(); 28 | //termQuery 29 | if (searchParams.getField() != "" && searchParams.getKeyword() != "") { 30 | query = QueryBuilders.termQuery(searchParams.getField(), searchParams.getKeyword()); 31 | } 32 | //termsQuery 33 | if (searchParams.getField() != "" && searchParams.getKeywords() != "") { 34 | String[] keys = searchParams.getKeywords().split(","); 35 | query = QueryBuilders.termsQuery(searchParams.getField(), keys); 36 | } 37 | 38 | //Fuzzy like this Query 39 | if (searchParams.getField() != "" && searchParams.getLikethis() != "") { 40 | query = QueryBuilders.fuzzyQuery(searchParams.getField(), searchParams.getLikethis()); 41 | } 42 | 43 | //prefix Query 44 | if (searchParams.getField() != "" && searchParams.getPrefixword() != "") { 45 | query = QueryBuilders.prefixQuery(searchParams.getField(), searchParams.getPrefixword()); 46 | } 47 | 48 | //wildcard Query，？匹配一个字符，*匹配字符序列 49 | if (searchParams.getField() != "" && searchParams.getWildcard() != "") { 50 | query = QueryBuilders.wildcardQuery(searchParams.getField(), searchParams.getWildcard()); 51 | } 52 | searchrequestbuilder.setQuery(query); 53 | 54 | if (requestParams.getType() != "")//设置查询类型，默认weibodatatype 55 | searchrequestbuilder.setTypes(requestParams.getType()); 56 | if (requestParams.getFrom() != 0)//设置返回结果开始显示位置 57 | searchrequestbuilder.setFrom(requestParams.getFrom()); 58 | if (requestParams.getSize() != 0)//应当考虑想要查看获得记录的默认数量，默认10; 59 | searchrequestbuilder.setSize(requestParams.getSize()); 60 | if (requestParams.isExplain())//是否显示详细查询信息，默认不显示; 61 | searchrequestbuilder.setExplain(requestParams.isExplain()); 62 | 63 | return searchrequestbuilder.execute().actionGet(); 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/java/zx/soft/es/utils/ConfigUtil.java: -------------------------------------------------------------------------------- 1 | package zx.soft.es.utils; 2 | 3 | import java.io.InputStream; 4 | import java.util.Properties; 5 | 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | /** 10 | * 配置文件读取类 11 | * 12 | * @author wanggang 13 | * 14 | */ 15 | public class ConfigUtil { 16 | 17 | private static Logger logger = LoggerFactory.getLogger(ConfigUtil.class); 18 | 19 | public static Properties getProps(String confFileName) { 20 | 21 | Properties result = new Properties(); 22 | logger.info("Load resource: " + confFileName); 23 | 24 | try (InputStream in = ConfigUtil.class.getClassLoader().getResourceAsStream(confFileName);) { 25 | result.load(in); 26 | return result; 27 | } catch (Exception e) { 28 | throw new RuntimeException(e); 29 | } 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/java/zx/soft/es/utils/IndexResponse.java: -------------------------------------------------------------------------------- 1 | package zx.soft.es.utils; 2 | 3 | /** 4 | * POST 索引响应类 5 | * @author fgq 6 | * 7 | */ 8 | public class IndexResponse { 9 | 10 | private final int Code; 11 | private final String Info; 12 | 13 | public IndexResponse(Builder builder) { 14 | this.Code = builder.Code; 15 | this.Info = builder.Info; 16 | } 17 | 18 | public int getCode() { 19 | return Code; 20 | } 21 | 22 | public String getInfo() { 23 | return Info; 24 | } 25 | 26 | public static class Builder { 27 | 28 | private final int Code; 29 | private final String Info; 30 | 31 | public Builder(int Code, String Info) { 32 | super(); 33 | this.Code = Code; 34 | this.Info = Info; 35 | } 36 | 37 | public IndexResponse build() { 38 | return new IndexResponse(this); 39 | } 40 | 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/java/zx/soft/es/utils/JsonUtils.java: -------------------------------------------------------------------------------- 1 | package zx.soft.es.utils; 2 | 3 | import java.io.IOException; 4 | import java.text.DateFormat; 5 | import java.text.ParseException; 6 | import java.text.SimpleDateFormat; 7 | import java.util.Date; 8 | import java.util.Locale; 9 | 10 | import org.codehaus.jackson.map.ObjectMapper; 11 | 12 | /** 13 | * JSON工具类 14 | * 15 | * @author fgq 16 | * 17 | */ 18 | public class JsonUtils { 19 | 20 | private static final ObjectMapper mapper = new ObjectMapper(); 21 | 22 | // private static DateFormat dateFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss Z yyyy", Locale.ENGLISH); 23 | private static DateFormat dateFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy", Locale.US); 24 | 25 | static { 26 | mapper.setDateFormat(dateFormat); 27 | } 28 | 29 | public static ObjectMapper getObjectMapper() { 30 | return mapper; 31 | } 32 | 33 | public static String toJson(Object object) { 34 | 35 | try { 36 | return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(object); 37 | } catch (IOException e) { 38 | throw new RuntimeException(e); 39 | } 40 | 41 | } 42 | 43 | public static String toJsonWithoutPretty(Object object) { 44 | 45 | try { 46 | return mapper.writeValueAsString(object); 47 | } catch (IOException e) { 48 | throw new RuntimeException(e); 49 | } 50 | 51 | } 52 | 53 | /** 54 | * 测试函数 55 | */ 56 | public static void main(String[] args) throws ParseException { 57 | 58 | // System.out.println(dateFormat.format(new Date())); 59 | // Wed Oct 23 16:58:17 +0800 2013 60 | // Wed Oct 23 16:58:17 CST 2013 61 | Date parse = dateFormat.parse("Thu Apr 10 11:40:56 CST 2014"); 62 | System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(parse)); 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/java/zx/soft/es/utils/URLUtils.java: -------------------------------------------------------------------------------- 1 | package zx.soft.es.utils; 2 | 3 | import java.io.UnsupportedEncodingException; 4 | import java.net.URLDecoder; 5 | 6 | public class URLUtils { 7 | 8 | public static String getDecoderURL(String url, String code) { 9 | try { 10 | String result = URLDecoder.decode(url, code); 11 | return result; 12 | } catch (UnsupportedEncodingException e) { 13 | return url; 14 | } 15 | 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/java/zx/soft/es/web/application/IndexApplication.java: -------------------------------------------------------------------------------- 1 | package zx.soft.es.web.application; 2 | 3 | import java.util.List; 4 | 5 | import org.restlet.Application; 6 | import org.restlet.Restlet; 7 | import org.restlet.routing.Router; 8 | 9 | import zx.soft.es.model.Weibo; 10 | import zx.soft.es.search.Index; 11 | import zx.soft.es.web.resource.IndexServerResource; 12 | 13 | /** 14 | * 微博数据索引应用类 15 | * @author fgq 16 | * 17 | */ 18 | public class IndexApplication extends Application { 19 | 20 | private final Index in; 21 | 22 | public IndexApplication() { 23 | in = new Index(); 24 | } 25 | 26 | @Override 27 | public Restlet createInboundRoot() { 28 | Router router = new Router(getContext()); 29 | router.attach("/index", IndexServerResource.class); 30 | return router; 31 | } 32 | 33 | public void createIndex(List weibos) { 34 | in.createIndex(weibos); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/java/zx/soft/es/web/application/SearchApplication.java: -------------------------------------------------------------------------------- 1 | package zx.soft.es.web.application; 2 | 3 | import org.elasticsearch.action.search.SearchResponse; 4 | import org.restlet.Application; 5 | import org.restlet.Restlet; 6 | import org.restlet.routing.Router; 7 | 8 | import zx.soft.es.model.RequestParams; 9 | import zx.soft.es.model.SearchParams; 10 | import zx.soft.es.search.Search; 11 | import zx.soft.es.web.resource.SearchServerResource; 12 | 13 | public class SearchApplication extends Application { 14 | 15 | private final Search search; 16 | 17 | public SearchApplication() { 18 | search = new Search(); 19 | } 20 | 21 | @Override 22 | public Restlet createInboundRoot() { 23 | Router router = new Router(); 24 | router.attach("/search", SearchServerResource.class); 25 | return router; 26 | } 27 | 28 | public SearchResponse doSearch(SearchParams searchParams, RequestParams requestParams) { 29 | SearchResponse response = search.doSearch(searchParams, requestParams); 30 | return response; 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/java/zx/soft/es/web/jackson/ReplaceConvert.java: -------------------------------------------------------------------------------- 1 | package zx.soft.es.web.jackson; 2 | 3 | import java.text.DateFormat; 4 | import java.text.SimpleDateFormat; 5 | import java.util.List; 6 | import java.util.Locale; 7 | 8 | import org.codehaus.jackson.map.ObjectMapper; 9 | import org.codehaus.jackson.map.SerializationConfig; 10 | import org.restlet.engine.Engine; 11 | import org.restlet.engine.converter.ConverterHelper; 12 | 13 | /** 14 | * 替代转换工具 15 | * 16 | * @author wanggang 17 | * 18 | */ 19 | public class ReplaceConvert { 20 | 21 | private static DateFormat sinaDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ENGLISH); 22 | 23 | public static void configureJacksonConverter() { 24 | ObjectMapper objectMapper = new ObjectMapper(); 25 | objectMapper.setDateFormat(sinaDateFormat); 26 | objectMapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true); 27 | } 28 | 29 | /** 30 | * 移除给定类的第一个注册的converter后，使用Restlet引擎注册一个converter类 31 | * 参考:http://restlet.tigris.org/ds/viewMessage.do?dsForumId=4447&dsMessageId=2716118 32 | */ 33 | static void replaceConverter(Class converterClass, ConverterHelper newConverter) { 34 | 35 | ConverterHelper oldConverter = null; 36 | List converters = Engine.getInstance().getRegisteredConverters(); 37 | for (ConverterHelper converter : converters) { 38 | if (converter.getClass().equals(converterClass)) { 39 | converters.remove(converter); 40 | oldConverter = converter; 41 | break; 42 | } 43 | } 44 | 45 | converters.add(newConverter); 46 | if (oldConverter == null) { 47 | System.err.println("Added Converter to Restlet Engine: " + newConverter.getClass().getName()); 48 | } else { 49 | System.err.println("Replaced Converter " + oldConverter.getClass().getName() + " with " 50 | + newConverter.getClass().getName() + " in Restlet Engine"); 51 | } 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/java/zx/soft/es/web/resource/IndexServerResource.java: -------------------------------------------------------------------------------- 1 | package zx.soft.es.web.resource; 2 | 3 | import org.restlet.resource.Get; 4 | import org.restlet.resource.Post; 5 | import org.restlet.resource.ServerResource; 6 | 7 | import zx.soft.es.model.PostData; 8 | import zx.soft.es.utils.IndexResponse; 9 | import zx.soft.es.web.application.IndexApplication; 10 | 11 | /** 12 | * 微博数据索引资源类 13 | * @author fgq 14 | * 15 | */ 16 | public class IndexServerResource extends ServerResource { 17 | private IndexApplication application; 18 | 19 | @Override 20 | public void doInit() { 21 | application = (IndexApplication) getApplication(); 22 | } 23 | 24 | @Post("json") 25 | public Object doIndex(final PostData data) { 26 | 27 | if (getReference().getRemainingPart().length() != 0) 28 | return new IndexResponse.Builder(1000, "error url!").build(); 29 | else if (data == null) 30 | return new IndexResponse.Builder(1001, "no data!").build(); 31 | else { 32 | application.createIndex(data.getRecords()); 33 | return new IndexResponse.Builder(0, "index succeed!").build(); 34 | } 35 | } 36 | 37 | @Get("json") 38 | public String getData() { 39 | return "ok "; 40 | } 41 | /*@Get 42 | public Representation toJson() { 43 | Weibo weibo = new Weibo(); 44 | weibo.setContent("hello every body!"); 45 | return new JacksonRepresentation(weibo); 46 | }*/ 47 | 48 | } 49 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/java/zx/soft/es/web/server/ESIndexServer.java: -------------------------------------------------------------------------------- 1 | package zx.soft.es.web.server; 2 | 3 | import java.util.Properties; 4 | 5 | import org.restlet.Component; 6 | import org.restlet.data.Protocol; 7 | 8 | import zx.soft.es.utils.ConfigUtil; 9 | import zx.soft.es.web.application.IndexApplication; 10 | import zx.soft.es.web.jackson.ReplaceConvert; 11 | 12 | /** 13 | * 微博数据服务类 14 | * 索引示例： 15 | * http://192.168.3.21:8084/spider/index POST 16 | * @author fgq 17 | * 18 | */ 19 | public class ESIndexServer { 20 | private final Component component; 21 | private final IndexApplication indexApplication; 22 | private final int PORT; 23 | 24 | public ESIndexServer() { 25 | Properties pros = ConfigUtil.getProps("web-server.properties"); 26 | component = new Component(); 27 | indexApplication = new IndexApplication(); 28 | PORT = Integer.parseInt(pros.getProperty("indexapi.port")); 29 | } 30 | 31 | public static void main(String[] args) { 32 | ESIndexServer server = new ESIndexServer(); 33 | server.start(); 34 | } 35 | 36 | protected void start() { 37 | component.getServers().add(Protocol.HTTP, PORT); 38 | try { 39 | component.getDefaultHost().attach("/spider", indexApplication); 40 | ReplaceConvert.configureJacksonConverter(); 41 | component.start(); 42 | } catch (Exception e) { 43 | e.printStackTrace(); 44 | } 45 | } 46 | 47 | @SuppressWarnings("unused") 48 | private void stop() { 49 | try { 50 | component.stop(); 51 | } catch (Exception e) { 52 | e.printStackTrace(); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/java/zx/soft/es/web/server/ESSearchServer.java: -------------------------------------------------------------------------------- 1 | package zx.soft.es.web.server; 2 | 3 | import java.util.Properties; 4 | 5 | import org.restlet.Component; 6 | import org.restlet.data.Protocol; 7 | 8 | import zx.soft.es.utils.ConfigUtil; 9 | import zx.soft.es.web.application.SearchApplication; 10 | import zx.soft.es.web.jackson.ReplaceConvert; 11 | 12 | /** 13 | * 搜索服务类 14 | * 搜索示例： 15 | *http://192.168.3.22:8085/spider/search?field=content&keywords=hello,大家&size=5 16 | * @author fgq 17 | * 18 | */ 19 | public class ESSearchServer { 20 | 21 | private final Component component; 22 | private final SearchApplication searchApplication; 23 | private final int PORT; 24 | 25 | public ESSearchServer() { 26 | Properties props = ConfigUtil.getProps("web-server.properties"); 27 | component = new Component(); 28 | searchApplication = new SearchApplication(); 29 | PORT = Integer.parseInt(props.getProperty("searchapi.port")); 30 | } 31 | 32 | public static void main(String[] leftArgs) { 33 | ESSearchServer server = new ESSearchServer(); 34 | server.start(); 35 | } 36 | 37 | private void start() { 38 | 39 | try { 40 | component.getServers().add(Protocol.HTTP, PORT); 41 | component.getDefaultHost().attach("/spider", searchApplication); 42 | ReplaceConvert.configureJacksonConverter(); 43 | component.start(); 44 | } catch (Exception e) { 45 | throw new RuntimeException(e); 46 | } 47 | } 48 | 49 | public void stop() { 50 | try { 51 | component.stop(); 52 | } catch (Exception e) { 53 | throw new RuntimeException(e); 54 | } 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/resources/elastic_search.properties: -------------------------------------------------------------------------------- 1 | 2 | es.name=ES 3 | es.ip=192.168.5.201 4 | es.port=9300 -------------------------------------------------------------------------------- /elasticsearch-web/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 8 | 9 | 10 | 11 | INFO 12 | ACCEPT 13 | DENY 14 | 15 | 16 | 17 | 19 | logs/storm-analysis.log 20 | 21 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 22 | 23 | 24 | 25 | INFO 26 | 27 | 28 | logs/storm-analysis.log.%d{yyyy-MM-dd}.tar.gz 29 | 30 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /elasticsearch-web/src/main/resources/web-server.properties: -------------------------------------------------------------------------------- 1 | 2 | indexapi.port=8084 3 | searchapi.port=8085 4 | -------------------------------------------------------------------------------- /elasticsearch-web/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{MMdd.HHmmss.SSS} [%-20t] [%-5p] [%-20c] [L:%-3L] - %m%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | zx.soft 6 | elasticsearch-sentiment 7 | 1.0.0 8 | pom 9 | ElasticSearch Sentiment[Main POM] 10 | Development on Sentiment Search Service Based on ElasticSearch. 11 | 12 | 13 | elasticsearch-parent 14 | elasticsearch-dao 15 | 16 | elasticsearch-core 17 | elasticsearch-web 18 | 19 | 20 | 21 | 22 | 23 | 24 | org.apache.maven.plugins 25 | maven-source-plugin 26 | 27 | 28 | attach-sources 29 | verify 30 | 31 | jar 32 | 33 | 34 | 35 | 36 | 37 | 38 | org.apache.maven.plugins 39 | maven-compiler-plugin 40 | 41 | 1.7 42 | 1.7 43 | 44 | 45 | 46 | 47 | 48 | 2014 49 | 50 | 51 | 52 | Apache 2 53 | http://www.apache.org/licenses/LICENSE-2.0.txt 54 | all 55 | Copyright 2014 (c) Gang Wang - All Right Reserved 56 | 57 | 58 | 59 | 60 | 61 | wgybzb 62 | Gang Wang 63 | wanggang@zxisl.com 64 | http://github.com/wgybzbrobot 65 | 66 | ReleaseManager 67 | Designer 68 | Developer 69 | 70 | UTC-8 71 | 72 | 73 | xwjxwj30abc 74 | Wenjuan Xu 75 | 76 | 77 | wuqinghefei 78 | Qing Wu 79 | 80 | 81 | 82 | --------------------------------------------------------------------------------