├── .gitignore ├── README.md ├── pom.xml ├── sentiment-api ├── .gitignore ├── README.md ├── pom.xml └── src │ ├── main │ ├── assembly │ │ └── distribution.xml │ ├── bin │ │ └── ctl.sh │ ├── java │ │ └── zx │ │ │ └── soft │ │ │ └── sent │ │ │ └── api │ │ │ ├── controller │ │ │ ├── StatisticController.java │ │ │ ├── TaskController.java │ │ │ ├── TaskKeywordsController.java │ │ │ └── TaskQueryController.java │ │ │ ├── dao │ │ │ ├── KeywordTypeMapper.java │ │ │ ├── TaskKeywordMapper.java │ │ │ ├── TaskMapper.java │ │ │ ├── TaskQueryMapper.java │ │ │ ├── all_internet_tasks.sql │ │ │ ├── task_keywords.sql │ │ │ └── task_keywords_type.sql │ │ │ ├── domain │ │ │ ├── ErrorResponse.java │ │ │ ├── IndexErrResponse.java │ │ │ ├── KeywordType.java │ │ │ ├── Task.java │ │ │ ├── TaskKeyword.java │ │ │ ├── TaskStatistic.java │ │ │ └── TaskStatisticResult.java │ │ │ ├── driver │ │ │ └── SentimentApiDriver.java │ │ │ ├── server │ │ │ └── QueryApiServer.java │ │ │ ├── service │ │ │ ├── KeywordService.java │ │ │ └── TaskService.java │ │ │ ├── timer │ │ │ └── OATimer.java │ │ │ └── utils │ │ │ └── CustomObjectMapper.java │ ├── resources │ │ ├── db.properties │ │ ├── logback.xml │ │ ├── web-server.properties │ │ └── zx │ │ │ └── soft │ │ │ └── sent │ │ │ └── api │ │ │ └── dao │ │ │ └── TaskKeywordsMapper.xml │ └── webapp │ │ ├── WEB-INF │ │ └── applicationContext.xml │ │ └── images │ │ └── spring_logo.png │ └── test │ ├── java │ └── zx │ │ └── soft │ │ └── sent │ │ └── api │ │ └── dao │ │ └── AllInternetMapperTest.java │ └── resources │ ├── logback-test.xml │ └── task-curl ├── sentiment-core ├── pom.xml └── src │ ├── main │ ├── java │ │ └── zx │ │ │ └── soft │ │ │ └── sent │ │ │ └── core │ │ │ ├── constant │ │ │ └── RecordsConstant.java │ │ │ ├── persist │ │ │ ├── PersistCoreOld.java │ │ │ └── PersistRunnable.java │ │ │ ├── redis │ │ │ └── RedisReplication.java │ │ │ ├── solr │ │ │ └── SolrCloudCore.java │ │ │ └── tsdb │ │ │ ├── GatherQueueReport.java │ │ │ └── SentimentReportCore.java │ └── resources │ │ └── logback.xml │ └── test │ ├── java │ └── zx │ │ └── soft │ │ └── sent │ │ └── core │ │ └── tsdb │ │ └── GatherQueueReportTest.java │ └── resources │ ├── cache-config.properties │ ├── logback-test.xml │ └── solr_params.properties ├── sentiment-dao ├── pom.xml └── src │ ├── main │ ├── java │ │ └── zx │ │ │ └── soft │ │ │ └── sent │ │ │ └── dao │ │ │ ├── allinternet │ │ │ ├── AllInternet.java │ │ │ ├── AllInternetMapper.java │ │ │ └── oa_all_internet_task_query.sql │ │ │ ├── bdb │ │ │ ├── BasicDatabase.java │ │ │ └── ObjectDatabase.java │ │ │ ├── common │ │ │ ├── Config.java │ │ │ ├── MybatisConfig.java │ │ │ └── SentimentConstant.java │ │ │ ├── demo │ │ │ ├── AllInternetDemo.java │ │ │ ├── BerkeleyDemo.java │ │ │ ├── CacheQueryDemo.java │ │ │ ├── FirstPageDemo.java │ │ │ ├── MybatisConfigDemo.java │ │ │ ├── OracleJDBCDemo.java │ │ │ ├── SentimentRecordDemo.java │ │ │ ├── SpecialQueryDemo.java │ │ │ └── SpiderSiteDemo.java │ │ │ ├── domain │ │ │ ├── allinternet │ │ │ │ ├── InternetTask.java │ │ │ │ └── TaskResult.java │ │ │ ├── firstpage │ │ │ │ └── FirstPageInfo.java │ │ │ ├── platform │ │ │ │ ├── AutmSearch.java │ │ │ │ ├── Blog.java │ │ │ │ ├── Email.java │ │ │ │ ├── Forum.java │ │ │ │ ├── Information.java │ │ │ │ ├── Picture.java │ │ │ │ ├── QQGroup.java │ │ │ │ ├── Record.java │ │ │ │ ├── RecordInfo.java │ │ │ │ ├── Reply.java │ │ │ │ ├── SiteInfo.java │ │ │ │ └── Weibo.java │ │ │ ├── sentiment │ │ │ │ ├── InsertCacheQuery.java │ │ │ │ ├── RecordInsert.java │ │ │ │ ├── RecordSelect.java │ │ │ │ ├── SelectCacheQuery.java │ │ │ │ ├── SelectParamsById.java │ │ │ │ ├── SelectParamsByTime.java │ │ │ │ └── SentTablename.java │ │ │ ├── special │ │ │ │ ├── SpecialResult.java │ │ │ │ └── SpecialTopic.java │ │ │ └── weibo │ │ │ │ ├── SinaWeibo.java │ │ │ │ ├── WeiboInfo.java │ │ │ │ ├── WeiboOld.java │ │ │ │ ├── WeiboOldInfo.java │ │ │ │ └── WeiboSinaParams.java │ │ │ ├── firstpage │ │ │ ├── FirstPage.java │ │ │ ├── FirstPageHarmful.java │ │ │ ├── FirstPageHarmfulMapper.java │ │ │ └── FirstPageMapper.java │ │ │ ├── oracle │ │ │ └── OracleJDBC.java │ │ │ ├── sentiment │ │ │ ├── SentimentRecord.java │ │ │ └── SentimentRecordDao.java │ │ │ ├── special │ │ │ ├── SpecialQuery.java │ │ │ └── SpecialQueryMapper.java │ │ │ ├── spidersite │ │ │ ├── SpiderSite.java │ │ │ └── SpiderSiteMapper.java │ │ │ └── sql │ │ │ ├── CreateTables.java │ │ │ └── SentDbcp.java │ └── resources │ │ ├── SentimentRecordDaoMapper.xml │ │ ├── SpecialQueryMapper.xml │ │ └── mybatis-config.xml │ └── test │ ├── java │ └── zx │ │ └── soft │ │ └── sent │ │ └── dao │ │ ├── common │ │ └── MybatisConfigTest.java │ │ ├── domain │ │ └── platform │ │ │ └── RecordInfoTest.java │ │ └── oracle │ │ └── OracleJDBCTest.java │ ├── resources │ └── logback-test.xml │ └── sql │ └── bozhu.sql ├── sentiment-hdfs ├── pom.xml └── src │ ├── main │ ├── java │ │ └── zx │ │ │ └── soft │ │ │ └── sent │ │ │ └── hdfs │ │ │ └── io │ │ │ └── impl │ │ │ ├── FileExistsException.java │ │ │ ├── HdfsWriter.java │ │ │ ├── HdfsWriterQueueImpl.java │ │ │ └── HdfsWriterSimpleImpl.java │ └── resources │ │ ├── core-site.xml │ │ └── logback.xml │ └── test │ ├── java │ └── zx │ │ └── soft │ │ └── sent │ │ └── hdfs │ │ └── io │ │ └── impl │ │ ├── HdfsWriterDummy.java │ │ └── HdfsWriterTest.java │ └── resources │ └── logback-test.xml ├── sentiment-solr ├── doc │ └── platform-mapping ├── ik_dic │ ├── ext_ik.dic │ └── stopwords_ik.dic ├── pom.xml └── src │ ├── main │ ├── assembly │ │ └── distribution.xml │ ├── bin │ │ ├── ctl.sh │ │ ├── firstpage_timer.sh │ │ ├── special_timer.sh │ │ ├── timer_redis2solr.sh │ │ └── timer_remove_expired.sh │ ├── java │ │ └── zx │ │ │ └── soft │ │ │ └── sent │ │ │ └── solr │ │ │ ├── allinternet │ │ │ ├── TaskUpdate.java │ │ │ └── TaskUpdateRunnable.java │ │ │ ├── demo │ │ │ ├── CacheDemo.java │ │ │ ├── FirstPageDemo.java │ │ │ ├── InsertSortDemo.java │ │ │ ├── LogerDemo.java │ │ │ └── NegativeClassifyDemo.java │ │ │ ├── domain │ │ │ ├── FacetDateParams.java │ │ │ ├── FacetDateResult.java │ │ │ ├── QueryParams.java │ │ │ ├── QueryResult.java │ │ │ ├── SimpleFacetInfo.java │ │ │ └── SolrDocumentResult.java │ │ │ ├── driver │ │ │ └── SentSolrDriver.java │ │ │ ├── ecxception │ │ │ └── SpiderSearchException.java │ │ │ ├── firstpage │ │ │ ├── FirstPageHarmfulRun.java │ │ │ ├── FirstPageRun.java │ │ │ └── OAFirstPage.java │ │ │ ├── index │ │ │ ├── ImportRedisToSC.java │ │ │ └── IndexCloudSolr.java │ │ │ ├── query │ │ │ ├── FacetSearch.java │ │ │ ├── OracleToRedis.java │ │ │ ├── QueryCore.java │ │ │ ├── RemoveExpiredData.java │ │ │ └── SearchingData.java │ │ │ ├── special │ │ │ ├── PieChart.java │ │ │ ├── SpecialInfo.java │ │ │ ├── SpecialTopicRun.java │ │ │ └── TrendChart.java │ │ │ └── utils │ │ │ ├── ConvertToRecord.java │ │ │ ├── HarmfulInfoUtil.java │ │ │ ├── JedisClient.java │ │ │ ├── RedisCacheExpired.java │ │ │ ├── RedisMQ.java │ │ │ ├── RedisMQTest.java │ │ │ ├── Site2Json.java │ │ │ └── curl-site.sh │ └── resources │ │ ├── README.md │ │ ├── cache-config.properties │ │ ├── cate_dict │ │ ├── negative │ │ └── negative_bc │ │ ├── data_db.properties │ │ ├── logback.xml │ │ ├── oracle_db.properties │ │ └── solr_params.properties │ └── test │ ├── java │ └── zx │ │ └── soft │ │ └── sent │ │ └── solr │ │ ├── index │ │ └── IndexCloudSolrTest.java │ │ ├── search │ │ ├── FacetSearchTest.java │ │ └── SearchingDataTest.java │ │ └── utils │ │ └── RedisCacheExpiredTest.java │ └── resources │ ├── cache-config.properties │ ├── data_db.properties │ ├── logback-test.xml │ ├── solr_params.properties │ └── weibo │ ├── WBLR1 │ ├── WBLR2 │ ├── WBLR3 │ └── WBLR4 ├── sentiment-spider ├── README.md ├── pom.xml ├── proxy │ └── proxy-ips.txt └── src │ ├── main │ ├── assembly │ │ └── distribution.xml │ ├── bin │ │ ├── ctl.sh │ │ └── timer_spider.sh │ ├── java │ │ └── zx │ │ │ └── soft │ │ │ └── sent │ │ │ └── spider │ │ │ ├── dns │ │ │ └── CacheDNS.java │ │ │ ├── driver │ │ │ └── SpiderDriver.java │ │ │ ├── http │ │ │ ├── HttpInfoUtils.java │ │ │ ├── RetrivePage.java │ │ │ └── SpiderHttpURLConnection.java │ │ │ ├── proxy │ │ │ └── SocketProxy.java │ │ │ ├── sina │ │ │ ├── SinaPublicWeibosSpider.java │ │ │ └── public-weibo │ │ │ ├── utils │ │ │ ├── DataReader.java │ │ │ └── RedisMQ.java │ │ │ └── xml │ │ │ ├── DOM4JTest.java │ │ │ ├── SinaPublicWeibosXML.java │ │ │ ├── SinaXMLInfo.java │ │ │ └── WriteXML.java │ └── resources │ │ ├── Sentiment-Data │ │ ├── 138-340000-1423211396-10001-WACODE_0080-0.bcp │ │ └── GAB_ZIP_INDEX.xml │ │ ├── cache-config.properties │ │ ├── logback.xml │ │ ├── sina-api.properties │ │ └── super.properties │ └── test │ ├── java │ └── zx │ │ └── soft │ │ └── sent │ │ └── spider │ │ └── dns │ │ └── CacheDNSTest.java │ └── resources │ └── logback-test.xml ├── sentiment-spring ├── README.md ├── ik_dic │ ├── ext_ik.dic │ └── stopwords_ik.dic ├── pom.xml └── src │ ├── main │ ├── assembly │ │ └── distribution.xml │ ├── bin │ │ ├── ctl.sh │ │ └── timer-index.sh │ ├── java │ │ └── zx │ │ │ └── soft │ │ │ └── sent │ │ │ └── spring │ │ │ ├── controller │ │ │ └── IndexController.java │ │ │ ├── domain │ │ │ ├── ErrorResponse.java │ │ │ ├── IndexErrResponse.java │ │ │ └── PostData.java │ │ │ ├── driver │ │ │ └── SentimentSpringDriver.java │ │ │ ├── server │ │ │ └── IndexApiServer.java │ │ │ ├── service │ │ │ └── IndexService.java │ │ │ └── utils │ │ │ ├── CustomObjectMapper.java │ │ │ ├── Demo.java │ │ │ └── PersistCore.java │ ├── resources │ │ ├── cache-config.properties │ │ ├── data_db.properties │ │ ├── logback.xml │ │ ├── solr_params.properties │ │ └── web-server.properties │ └── webapp │ │ ├── WEB-INF │ │ └── applicationContext.xml │ │ └── images │ │ └── spring_logo.png │ └── test │ └── resources │ ├── index-test.sh │ ├── index │ └── curl-index │ └── logback-test.xml └── sentiment-web ├── README.md ├── bin └── timer-index.sh ├── ik_dic ├── ext_ik.dic └── stopwords_ik.dic ├── pom.xml └── src ├── main ├── assembly │ └── distribution.xml ├── bin │ └── ctl.sh ├── java │ └── zx │ │ └── soft │ │ └── sent │ │ └── web │ │ ├── application │ │ ├── FirstPageApplication.java │ │ ├── FirstPageHarmfulApplication.java │ │ ├── HarmfulInfoApplication.java │ │ ├── InternetTaskApplication.java │ │ ├── NicknameGroupApplication.java │ │ ├── PullRecordApplication.java │ │ ├── SentiSearchApplication.java │ │ ├── SiteApplication.java │ │ ├── SpecialApplication.java │ │ └── SpecialSpeedApplication.java │ │ ├── common │ │ ├── LogstashDemo.java │ │ └── NginxLog.java │ │ ├── demo │ │ ├── FirstPageDemo.java │ │ └── SpecialTopicDemo.java │ │ ├── domain │ │ ├── ErrorResponse.java │ │ ├── IndexErrResponse.java │ │ ├── InternetTask.java │ │ └── Task.java │ │ ├── driver │ │ └── SentWebDriver.java │ │ ├── resource │ │ ├── FirstPageHarmfulResource.java │ │ ├── FirstPageResource.java │ │ ├── HarmfulInfoResource.java │ │ ├── InternetTaskResource.java │ │ ├── NicknameGroupResource.java │ │ ├── PullRecordResource.java │ │ ├── SentGetResource.java │ │ ├── SentSearchResource.java │ │ ├── SiteResource.java │ │ ├── SpecialResource.java │ │ └── SpecialSpeedResource.java │ │ └── server │ │ ├── FirstPageHarmfulServer.java │ │ ├── FirstPageServer.java │ │ ├── HarmfulInfoServer.java │ │ ├── InternetTaskServer.java │ │ ├── NicknameGroupServer.java │ │ ├── PullRecordServer.java │ │ ├── SentimentSearchServer.java │ │ ├── SiteServer.java │ │ ├── SpecialServer.java │ │ └── SpecialSpeedServer.java └── resources │ ├── cache-config.properties │ ├── data_db.properties │ ├── logback.xml │ ├── oracle_db.properties │ ├── solr_params.properties │ └── web-server.properties └── test ├── java └── zx │ └── soft │ └── sent │ └── web │ └── json │ └── JsonTest.java └── resources ├── index_data ├── autmsearch ├── blog ├── email ├── forum ├── information ├── postData ├── qqgroup ├── record ├── reply └── weibo ├── logback-test.xml ├── test-indexdata └── test-specialdata /.gitignore: -------------------------------------------------------------------------------- 1 | .classpath 2 | .project 3 | .settings/ 4 | 5 | *.iml 6 | .idea/ 7 | 8 | *~ 9 | target/ 10 | logs/ 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # 舆情监控服务项目 3 | 4 | > 基于Java实现。 5 | 6 | ## 项目内容 7 | 8 | - [项目简介](项目简介) 9 | - [项目架构](项目架构) 10 | - [开发人员](开发人员) 11 | 12 | ---- 13 | 14 | ## 项目简介 15 | 16 | ### 项目起因 17 | 18 | 本项目主要服务于舆情监控底层架构。 19 | 20 | ### 项目框架 21 | 22 | `sentiment-parent`: jar和插件依赖工程 23 | 24 | `sentiment-algorithm`: 基础数据结构和数据挖掘算法 25 | 26 | `sentiment-utils`: 工具类 27 | 28 | `sentiment-dao`: 数据接口层 29 | 30 | `sentiment-cache`: 数据缓存层 31 | 32 | `sentiment-hdfs`: HDFS服务层 33 | 34 | `sentiment-solr`: solr索引搜索工程 35 | 36 | `sentiment-control`: 多服务控制层 37 | 38 | `sentiment-web`: API接口服务层 39 | 40 | 41 | > **备注:** 框架持续更新中。 42 | 43 | ### API文档 44 | [项目wiki](http://192.168.3.23/wiki) 45 | 46 | > **备注:** API文档统一放在公司的wiki上。 47 | 48 | ---- 49 | 50 | ## 项目架构 51 | 52 | 1. MySQL： 用于存储基本爬虫数据。 53 | 2. Redis： 用于数据去重，基本思想是存储每条数据的md5(key)，根据该值进行数据插入更新判断。 54 | 3. SolrCloud： 用于提供分布式搜索服务。 55 | 56 | ### 常见约束词 57 | Item | Value 58 | --------- | ----- 59 | Dao | Interface接口层 60 | Domain | 数据 61 | Constant | 常量 62 | Util | 工具 63 | 64 | ### 示例代码 65 | 66 | ```java 67 | ** Redis层调用 ** 68 | // 对象申明 69 | RedisCache redisCache = new RedisCache("hdp321", 6379, "zxsoft"); 70 | String key = "record_key_md5"; 71 | String[] members = { "v1", "v2", "v3", "v4", "v5", "v3" }; 72 | redisCache.sadd(key, members); 73 | System.out.println(redisCache.scard(key)); 74 | System.out.println(redisCache.sismember(key, "v3")); 75 | System.out.println(redisCache.sismember(key, "v6")); 76 | redisCache.sadd(key, "v5", "v7"); 77 | System.out.println(redisCache.scard(key)); 78 | System.out.println(redisCache.smembers(key)); 79 | 80 | ** 启动脚本 ** 81 | 82 | ``` 83 | 84 | ### 注意事项 85 | 86 | > 将與情数据写入索引时，同样的查询条件，在不同时刻查询不一样（短期内），是由于爬虫在段时间内（1～3天）写入了相同的数据，导致先写入的数据被覆盖，随后lasttime更新，因而根据时间段查询会变化的。 87 | 88 | ## 开发人员 89 | 90 | WeChat: wgybzb 91 | 92 | QQ: 1010437118 93 | 94 | E-mail: wgybzb@sina.cn 95 | 96 | -------------------------------------------------------------------------------- /sentiment-api/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | -------------------------------------------------------------------------------- /sentiment-api/README.md: -------------------------------------------------------------------------------- 1 | 2 | # 與情搜索服务统一接口 3 | 4 | > 基于Spring MVC实现，替代原先的Restlet架构，提高并发性能。 5 | 6 | ## 全网任务后台自动分发 7 | 8 | > 该模块包含`关键词维护接口`和`任务管理接口`两类接口服务。 9 | 10 | ### 关键词维护接口 11 | 12 | 13 | ### 任务管理接口 14 | 15 | 16 | 17 | ## 全网任务实时查询统计接口 18 | 19 | 20 | 21 | ## 舆情系统后台数据实时统计接口 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /sentiment-api/src/main/assembly/distribution.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | distribution 7 | 8 | tar.gz 9 | 10 | ${project.artifactId} 11 | 12 | 13 | src/main/resources 14 | 15 | logback.xml 16 | data_db.properties 17 | web-server.properties 18 | 19 | /conf 20 | true 21 | 22 | 23 | src/main/bin 24 | 25 | * 26 | 27 | /bin 28 | 0755 29 | 30 | 31 | 32 | 33 | /lib 34 | 35 | 36 | -------------------------------------------------------------------------------- /sentiment-api/src/main/bin/ctl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mainClass=zx.soft.sent.api.driver.SentimentApiDriver 4 | 5 | # resolve links - $0 may be a softlink 6 | PRG="$0" 7 | 8 | while [ -h "$PRG" ]; do 9 | ls=`ls -ld "$PRG"` 10 | link=`expr "$ls" : '.*-> \(.*\)$'` 11 | if expr "$link" : '/.*' > /dev/null; then 12 | PRG="$link" 13 | else 14 | PRG=`dirname "$PRG"`/"$link" 15 | fi 16 | done 17 | 18 | # Get standard environment variables 19 | PRGDIR=`dirname "$PRG"` 20 | 21 | PROJECT_DIR=`cd "$PRGDIR/.." >/dev/null; pwd` 22 | echo PROJECT_DIR=$PROJECT_DIR 23 | 24 | CLASSPATH="$CLASSHPATH:$PROJECT_DIR/conf" 25 | 26 | for jar in "$PROJECT_DIR/lib"/*.jar; do 27 | CLASSPATH="$CLASSPATH:$jar" 28 | done 29 | echo CLASSPATH=$CLASSPATH 30 | 31 | JVMARGS="${JVMARGS} -Dproject_dir=${PROJECT_DIR}" 32 | echo JVMARGS=$JVMARGS 33 | 34 | usage() { 35 | echo >&2 "usage: $PRG [args]" 36 | echo 'Valid commands: start, stop' 37 | exit 1 38 | } 39 | 40 | start() { 41 | JAVA=${JAVA-'java'} 42 | exec $JAVA $JVMARGS -classpath "$CLASSPATH" $mainClass "$@" & 43 | echo $! > main.pid 44 | } 45 | 46 | stop() { 47 | kill `cat main.pid` > /dev/null 48 | } 49 | 50 | case $1 in 51 | (start) 52 | shift 53 | start $@ 54 | ;; 55 | (stop) 56 | stop 57 | ;; 58 | (restart) 59 | stop 60 | shift 61 | start $@ 62 | ;; 63 | (*) 64 | echo >&2 "$PRG: error: unknown command '$1'" 65 | usage 66 | ;; 67 | esac 68 | -------------------------------------------------------------------------------- /sentiment-api/src/main/java/zx/soft/sent/api/controller/StatisticController.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.api.controller; 2 | 3 | /** 4 | * 舆情后台数据统计控制类 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | public class StatisticController { 10 | 11 | } 12 | -------------------------------------------------------------------------------- /sentiment-api/src/main/java/zx/soft/sent/api/controller/TaskKeywordsController.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.api.controller; 2 | 3 | /** 4 | * 全网自动任务关键词（热词、区域词、负面词）控制类 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | public class TaskKeywordsController { 10 | 11 | } 12 | -------------------------------------------------------------------------------- /sentiment-api/src/main/java/zx/soft/sent/api/controller/TaskQueryController.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.api.controller; 2 | 3 | import java.util.HashMap; 4 | import java.util.List; 5 | 6 | import javax.inject.Inject; 7 | 8 | import org.springframework.http.HttpStatus; 9 | import org.springframework.stereotype.Controller; 10 | import org.springframework.web.bind.annotation.PathVariable; 11 | import org.springframework.web.bind.annotation.RequestBody; 12 | import org.springframework.web.bind.annotation.RequestMapping; 13 | import org.springframework.web.bind.annotation.RequestMethod; 14 | import org.springframework.web.bind.annotation.ResponseBody; 15 | import org.springframework.web.bind.annotation.ResponseStatus; 16 | 17 | import zx.soft.sent.api.domain.TaskStatisticResult; 18 | import zx.soft.sent.api.service.TaskService; 19 | 20 | /** 21 | * OA全网任务缓存查询控制类 22 | * 23 | * 接口说明： 24 | * 1、POST: http://192.168.32.17:2900/sentiment/oa/tasks 返回缓存数据 25 | * ["CE9402701D796B22E15D433EB17FA4FE","4E4E7BF57E3B57B9718549646D364964","76AC38F439C31812D71251EE024B316A"] 26 | * 2、DELETE: http://192.168.32.17:2900/sentiment/oa/tasks/{identify} 不返回数据 27 | * 28 | * @author wanggang 29 | * 30 | */ 31 | @Controller 32 | @RequestMapping("/sentiment/oa") 33 | public class TaskQueryController { 34 | 35 | @Inject 36 | private TaskService taskService; 37 | 38 | /** 39 | * 根据多个identify查询缓存结果 40 | */ 41 | @RequestMapping(value = "/tasks", method = RequestMethod.POST) 42 | @ResponseStatus(HttpStatus.OK) 43 | public @ResponseBody HashMap queryGenderByUid(@RequestBody List identifys) { 44 | return taskService.getQueryResults(identifys); 45 | } 46 | 47 | /** 48 | * 删除单个任务缓存结果 49 | */ 50 | @RequestMapping(value = "/tasks/{identify}", method = RequestMethod.DELETE) 51 | @ResponseStatus(HttpStatus.NO_CONTENT) 52 | public void delete(@PathVariable String identify) { 53 | taskService.deleteQueryResult(identify); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /sentiment-api/src/main/java/zx/soft/sent/api/dao/TaskKeywordMapper.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.api.dao; 2 | 3 | import java.util.List; 4 | 5 | import zx.soft.sent.api.domain.TaskKeyword; 6 | 7 | /** 8 | * 全网后台自动任务：关键词接口 9 | * 10 | * @author wanggang 11 | * 12 | */ 13 | public interface TaskKeywordMapper { 14 | 15 | /** 16 | * 插入关键词 17 | */ 18 | public void insertTaskKeyword(TaskKeyword taskKeyword); 19 | 20 | /** 21 | * 删除关键词 22 | */ 23 | public void deleteTaskKeyword(String keyword_id); 24 | 25 | public void deleteTaskKeywords(int first_type, int second_type, int third_type); 26 | 27 | public void deleteTaskKeywords(int first_type, int second_type); 28 | 29 | public void deleteTaskKeywords(int first_type); 30 | 31 | /** 32 | * 查询关键词 33 | */ 34 | public List selectTaskKeywords(int first_type, int second_type, int third_type); 35 | 36 | public List selectTaskKeywords(int first_type, int second_type); 37 | 38 | public List selectTaskKeywords(int first_type); 39 | 40 | } 41 | -------------------------------------------------------------------------------- /sentiment-api/src/main/java/zx/soft/sent/api/dao/TaskMapper.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.api.dao; 2 | 3 | import org.apache.ibatis.annotations.Delete; 4 | import org.apache.ibatis.annotations.Insert; 5 | import org.apache.ibatis.annotations.Options; 6 | import org.apache.ibatis.annotations.Select; 7 | import org.apache.ibatis.annotations.Update; 8 | 9 | import zx.soft.sent.api.domain.Task; 10 | 11 | /** 12 | * 任务数据接口 13 | * 14 | * @author wanggang 15 | * 16 | */ 17 | public interface TaskMapper { 18 | 19 | /** 20 | * 插入任务 21 | */ 22 | @Insert("INSERT INTO `all_internet_tasks`(`identify`,`keywords`,`source_ids`,`start_time`,`end_time`," 23 | + "`lasttime`) VALUES (#{identify},#{keywords},#{source_ids},#{start_time},#{end_time},NOW())") 24 | @Options(useGeneratedKeys = true, keyProperty = "tid") 25 | public void insertTask(Task task); 26 | 27 | /** 28 | * 删除任务 29 | */ 30 | @Delete("DELETE FROM `all_internet_tasks` WHERE `identify` = #{0}") 31 | public void deleteTask(String identify); 32 | 33 | /** 34 | * 查询任务 35 | */ 36 | @Select("SELECT `identify`,`keywords`,`source_ids`,`start_time`,`end_time` FROM `all_nternet_tasks` " 37 | + "WHERE `identify` = #{0}") 38 | public Task selectTask(String identify); 39 | 40 | /** 41 | * 修改任务 42 | */ 43 | @Update("UPDATE `all_internet_tasks` SET `keywords` = #{keywords},`source_ids` = #{source_ids},`start_time`" 44 | + " = #{start_time},`end_time` = #{end_time},`lasttime` = NOW() WHERE `identify` = #{identify}") 45 | public void updateTask(Task task); 46 | 47 | } 48 | -------------------------------------------------------------------------------- /sentiment-api/src/main/java/zx/soft/sent/api/dao/all_internet_tasks.sql: -------------------------------------------------------------------------------- 1 | -- phpMyAdmin SQL Dump 2 | -- version 4.2.2 3 | -- http://www.phpmyadmin.net 4 | -- 5 | -- Host: localhost 6 | -- Generation Time: 2015-07-05 10:17:36 7 | -- 服务器版本： 5.5.37-MariaDB-log 8 | -- PHP Version: 5.5.12 9 | 10 | SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO"; 11 | SET time_zone = "+00:00"; 12 | 13 | 14 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 15 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 16 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 17 | /*!40101 SET NAMES utf8 */; 18 | 19 | -- 20 | -- Database: `sentiment_records` 21 | -- 22 | 23 | -- -------------------------------------------------------- 24 | 25 | -- 26 | -- 表的结构 `all_internet_tasks` 27 | -- 28 | 29 | CREATE TABLE IF NOT EXISTS `all_internet_tasks` ( 30 | `tid` int(11) NOT NULL COMMENT '自增ID', 31 | `identify` char(32) NOT NULL COMMENT '任务MD5唯一ID', 32 | `keywords` char(250) NOT NULL COMMENT '关键词组', 33 | `start_time` char(20) NOT NULL COMMENT '监测开始时间', 34 | `end_time` char(20) NOT NULL COMMENT '监测结束时间', 35 | `source_ids` varchar(1000) NOT NULL COMMENT '来源ID列表', 36 | `lasttime` datetime NOT NULL COMMENT '最新更新时间' 37 | ) ENGINE=MyISAM DEFAULT CHARSET=utf8 COMMENT='全网搜索任务数据表' AUTO_INCREMENT=1 ; 38 | 39 | -- 40 | -- Indexes for dumped tables 41 | -- 42 | 43 | -- 44 | -- Indexes for table `all_internet_tasks` 45 | -- 46 | ALTER TABLE `all_internet_tasks` 47 | ADD PRIMARY KEY (`tid`), ADD UNIQUE KEY `identify` (`identify`); 48 | 49 | -- 50 | -- AUTO_INCREMENT for dumped tables 51 | -- 52 | 53 | -- 54 | -- AUTO_INCREMENT for table `all_internet_tasks` 55 | -- 56 | ALTER TABLE `all_internet_tasks` 57 | MODIFY `tid` int(11) NOT NULL AUTO_INCREMENT COMMENT '自增ID'; 58 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 59 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 60 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 61 | -------------------------------------------------------------------------------- /sentiment-api/src/main/java/zx/soft/sent/api/dao/task_keywords.sql: -------------------------------------------------------------------------------- 1 | -- phpMyAdmin SQL Dump 2 | -- version 4.2.2 3 | -- http://www.phpmyadmin.net 4 | -- 5 | -- Host: localhost 6 | -- Generation Time: 2015-07-07 07:31:09 7 | -- 服务器版本： 5.5.37-MariaDB-log 8 | -- PHP Version: 5.5.12 9 | 10 | SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO"; 11 | SET time_zone = "+00:00"; 12 | 13 | 14 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 15 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 16 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 17 | /*!40101 SET NAMES utf8 */; 18 | 19 | -- 20 | -- Database: `sentiment_records` 21 | -- 22 | 23 | -- -------------------------------------------------------- 24 | 25 | -- 26 | -- 表的结构 `task_keywords` 27 | -- 28 | 29 | CREATE TABLE IF NOT EXISTS `task_keywords` ( 30 | `kid` int(10) unsigned NOT NULL COMMENT '自增ID', 31 | `keyword_id` char(50) NOT NULL COMMENT '关键词ID', 32 | `first_type` tinyint(3) unsigned NOT NULL COMMENT '一级分类类型', 33 | `second_type` tinyint(3) unsigned NOT NULL COMMENT '二级分类类型', 34 | `third_type` tinyint(3) unsigned NOT NULL COMMENT '三级分类类型', 35 | `keyword_name` char(30) NOT NULL COMMENT '关键词名称', 36 | `lasttime` datetime NOT NULL COMMENT '最后插入时间' 37 | ) ENGINE=MyISAM DEFAULT CHARSET=utf8 COMMENT='全网自动任务关键词数据表' AUTO_INCREMENT=1 ; 38 | 39 | -- 40 | -- Indexes for dumped tables 41 | -- 42 | 43 | -- 44 | -- Indexes for table `task_keywords` 45 | -- 46 | ALTER TABLE `task_keywords` 47 | ADD PRIMARY KEY (`kid`), ADD UNIQUE KEY `keyword_type` (`first_type`,`second_type`,`third_type`) COMMENT '关键词类型联合索引', ADD UNIQUE KEY `keyword_id` (`keyword_id`), ADD KEY `third_type` (`third_type`), ADD KEY `second_type` (`second_type`), ADD KEY `first_type` (`first_type`); 48 | 49 | -- 50 | -- AUTO_INCREMENT for dumped tables 51 | -- 52 | 53 | -- 54 | -- AUTO_INCREMENT for table `task_keywords` 55 | -- 56 | ALTER TABLE `task_keywords` 57 | MODIFY `kid` int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增ID'; 58 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 59 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 60 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 61 | -------------------------------------------------------------------------------- /sentiment-api/src/main/java/zx/soft/sent/api/domain/ErrorResponse.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.api.domain; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * 错误相应类 7 | * 8 | * @author wanggang 9 | * 10 | */ 11 | public class ErrorResponse implements Serializable { 12 | 13 | private static final long serialVersionUID = -3408598774435409654L; 14 | 15 | private final int errorCode; 16 | private final String errorMessage; 17 | 18 | public ErrorResponse(Builder builder) { 19 | this.errorCode = builder.errorCode; 20 | this.errorMessage = builder.errorMessage; 21 | } 22 | 23 | public static class Builder { 24 | 25 | private final int errorCode; 26 | private final String errorMessage; 27 | 28 | public Builder(int errorCode, String errorMessage) { 29 | super(); 30 | this.errorCode = errorCode; 31 | this.errorMessage = errorMessage; 32 | } 33 | 34 | public ErrorResponse build() { 35 | return new ErrorResponse(this); 36 | } 37 | 38 | } 39 | 40 | public int getErrorCode() { 41 | return errorCode; 42 | } 43 | 44 | public String getErrorMessage() { 45 | return errorMessage; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /sentiment-api/src/main/java/zx/soft/sent/api/domain/IndexErrResponse.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.api.domain; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import zx.soft.utils.json.JsonUtils; 8 | 9 | public class IndexErrResponse implements Serializable { 10 | 11 | private static final long serialVersionUID = -5183178204181065266L; 12 | 13 | private int errorCode; 14 | private List errorMessage; 15 | 16 | public IndexErrResponse(int errorCode, List errorMessage) { 17 | this.errorCode = errorCode; 18 | this.errorMessage = errorMessage; 19 | } 20 | 21 | public int getErrorCode() { 22 | return errorCode; 23 | } 24 | 25 | public void setErrorCode(int errorCode) { 26 | this.errorCode = errorCode; 27 | } 28 | 29 | public List getErrorMessage() { 30 | return errorMessage; 31 | } 32 | 33 | public void setErrorMessage(List errorMessage) { 34 | this.errorMessage = errorMessage; 35 | } 36 | 37 | public static void main(String[] args) { 38 | List errorMessage = new ArrayList<>(); 39 | errorMessage.add("wwfffrrrrr"); 40 | errorMessage.add("1233jjffioo"); 41 | IndexErrResponse err = new IndexErrResponse(-1, errorMessage); 42 | System.out.println(JsonUtils.toJson(err)); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /sentiment-api/src/main/java/zx/soft/sent/api/domain/TaskStatisticResult.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.api.domain; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * OA全网搜索任务缓存结果数据 7 | * 8 | * @author wanggang 9 | * 10 | */ 11 | public class TaskStatisticResult implements Serializable { 12 | 13 | private static final long serialVersionUID = 4224722992054268127L; 14 | 15 | private int local_count; // 本地库搜索结果总数：关键词，时间段：2000.1.1-创建时间 16 | private int autm_count; // 元搜索搜索结果总数：关键词，时间段：创建时间-结束时间/当前时间，source_ids 17 | private int all_count; // 总和：local_count+autm_count 18 | 19 | public int getLocal_count() { 20 | return local_count; 21 | } 22 | 23 | public void setLocal_count(int local_count) { 24 | this.local_count = local_count; 25 | } 26 | 27 | public int getAutm_count() { 28 | return autm_count; 29 | } 30 | 31 | public void setAutm_count(int autm_count) { 32 | this.autm_count = autm_count; 33 | } 34 | 35 | public int getAll_count() { 36 | return all_count; 37 | } 38 | 39 | public void setAll_count(int all_count) { 40 | this.all_count = all_count; 41 | } 42 | 43 | public static long getSerialversionuid() { 44 | return serialVersionUID; 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /sentiment-api/src/main/java/zx/soft/sent/api/driver/SentimentApiDriver.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.api.driver; 2 | 3 | import zx.soft.sent.api.server.QueryApiServer; 4 | import zx.soft.utils.driver.ProgramDriver; 5 | 6 | /** 7 | * 驱动类 8 | * 9 | * @author wanggang 10 | * 11 | */ 12 | public class SentimentApiDriver { 13 | 14 | /** 15 | * 主函数 16 | */ 17 | public static void main(String[] args) { 18 | 19 | int exitCode = -1; 20 | ProgramDriver pgd = new ProgramDriver(); 21 | try { 22 | // 运行在hefei07机器上 23 | pgd.addClass("queryApiServer", QueryApiServer.class, "搜索查询接口"); 24 | pgd.driver(args); 25 | // Success 26 | exitCode = 0; 27 | } catch (Throwable e) { 28 | throw new RuntimeException(e); 29 | } 30 | 31 | System.exit(exitCode); 32 | 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /sentiment-api/src/main/java/zx/soft/sent/api/timer/OATimer.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.api.timer; 2 | 3 | public class OATimer { 4 | 5 | public static void main(String[] args) { 6 | // TODO Auto-generated method stub 7 | 8 | } 9 | 10 | } 11 | -------------------------------------------------------------------------------- /sentiment-api/src/main/java/zx/soft/sent/api/utils/CustomObjectMapper.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.api.utils; 2 | 3 | import java.text.DateFormat; 4 | import java.text.ParseException; 5 | import java.text.SimpleDateFormat; 6 | import java.util.Date; 7 | import java.util.Locale; 8 | 9 | import com.fasterxml.jackson.databind.ObjectMapper; 10 | 11 | /** 12 | * 自定义的ObjectMapper类，用于处理时间格式 13 | * 14 | * @author wanggang 15 | * 16 | */ 17 | public class CustomObjectMapper extends ObjectMapper { 18 | 19 | private static final long serialVersionUID = -3341366863221844412L; 20 | 21 | public static DateFormat sinaDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ENGLISH); 22 | 23 | public CustomObjectMapper() { 24 | super(); 25 | setDateFormat(sinaDateFormat); 26 | } 27 | 28 | /** 29 | * 测试函数 30 | */ 31 | public static void main(String[] args) throws ParseException { 32 | System.out.println(sinaDateFormat.format(new Date())); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /sentiment-api/src/main/resources/db.properties: -------------------------------------------------------------------------------- 1 | 2 | ##### Database ##### 3 | # Product 4 | #sent.db.url=jdbc:mysql://192.168.32.17:3306/sentiment_records?useUnicode=true&characterEncoding=utf-8 5 | # Testing 6 | #sent.db.url=jdbc:mysql://192.168.31.23:3306/sentiment_records?useUnicode=true&characterEncoding=utf-8 7 | #sent.db.oa.url=jdbc:mysql://192.168.31.23:3306/sentiment_oa?useUnicode=true&characterEncoding=utf-8 8 | 9 | ##### Password ##### 10 | #sent.db.username=sentimentrecords 11 | #sent.db.password=zn/iACQ/Xh19+W6Ot2MghQ== 12 | 13 | ##### Localhost ##### 14 | sent.db.url=jdbc:mysql://localhost:3306/sentiment_records?useUnicode=true&characterEncoding=utf-8 15 | sent.db.username=root 16 | sent.db.password=root -------------------------------------------------------------------------------- /sentiment-api/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 8 | 9 | 10 | 11 | INFO 12 | ACCEPT 13 | DENY 14 | 15 | 16 | 17 | 19 | logs/sentiment-api.log 20 | 21 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 22 | 23 | 24 | 25 | INFO 26 | 27 | 28 | logs/sentiment-api.log.%d{yyyy-MM-dd}.gz 29 | 30 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /sentiment-api/src/main/resources/web-server.properties: -------------------------------------------------------------------------------- 1 | httpConn.maxTotal=10000 2 | httpConn.maxPerRoute=5000 3 | api.port=2900 4 | -------------------------------------------------------------------------------- /sentiment-api/src/main/resources/zx/soft/sent/api/dao/TaskKeywordsMapper.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 9 | INSERT INTO `internet_tasks` 10 | (`identify`,`keywords`,`source_ids`,`start_time`,`end_time`,`lasttime`) 11 | VALUES 12 | (#{identify},#{keywords},#{source_ids},#{start_time},#{end_time},NOW()); 13 | 14 | 15 | -------------------------------------------------------------------------------- /sentiment-api/src/main/webapp/images/spring_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wgybzbrobot/sentiment-search/df50eef8f82fbdc4549d1cfffb0d0b393a04f7b5/sentiment-api/src/main/webapp/images/spring_logo.png -------------------------------------------------------------------------------- /sentiment-api/src/test/java/zx/soft/sent/api/dao/AllInternetMapperTest.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.api.dao; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import org.junit.Test; 7 | 8 | public class AllInternetMapperTest { 9 | 10 | @Test 11 | public void test() { 12 | List identifys = new ArrayList<>(); 13 | identifys.add("4E4E7BF57E3B57B9718549646D364964"); 14 | identifys.add("CE9402701D796B22E15D433EB17FA4FE"); 15 | identifys.add("76AC38F439C31812D71251EE024B316A"); 16 | identifys.add("76AC38F439C31812D71251EE024B316B"); 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /sentiment-api/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 8 | 9 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /sentiment-api/src/test/resources/task-curl: -------------------------------------------------------------------------------- 1 | 2 | 1. POST 3 | 4 | curl -X POST http://localhost:8888/tasks --header 'Content-Type:application/json' --data '[{"identify":"671CC22D8FB5669A848B3E9CBE006D1F","keywords":" 维族人","source_ids":"8,220,590,216,7,583","start_time":"2014-06-10T15:31:28Z","end_time":"2015-03-12T19:15:51Z"},{"identify":"C714698D31BADD2921403693093E095A","keywords":"打死","source_ids":"216,8,220,7,583,590","start_time":"2014-07-04T08:57:02Z","end_time":"2015-03-12T19:15:51Z"}]' 5 | 6 | 2. DELETE 7 | 8 | curl -X DELETE http://localhost:8888/tasks/671CC22D8FB5669A848B3E9CBE006D1F,C714698D31BADD2921403693093E095A 9 | 10 | 3. GET 11 | 12 | curl -X GET http://localhost:8888/tasks/671CC22D8FB5669A848B3E9CBE006D1F,C714698D31BADD2921403693093E095A 13 | 14 | 4. PUT 15 | 16 | curl -X PUT http://localhost:8888/tasks --header 'Content-Type:application/json' --data '[{"identify":"671CC22D8FB5669A848B3E9CBE006D1F","keywords":"维族人qqqq","source_ids":"qqqq","start_time":"2014-06-10T15:31:11Z","end_time":"2015-03-12T19:15:11Z"},{"identify":"C714698D31BADD2921403693093E095A","keywords":"打死1111","source_ids":"111","start_time":"2014-07-04T08:57:11Z","end_time":"2015-03-12T19:11:11Z"}]' 17 | 18 | -------------------------------------------------------------------------------- /sentiment-core/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | zx.soft 7 | sentiment-search 8 | 2.2.0 9 | ../pom.xml 10 | 11 | 12 | sentiment-core 13 | Sentiment Core 14 | 15 | 16 | 17 | ${project.groupId} 18 | sentiment-dao 19 | ${project.version} 20 | 21 | 22 | zx.soft 23 | redis-client 24 | 1.2.0 25 | 26 | 27 | org.apache.solr 28 | solr-solrj 29 | 30 | 31 | org.slf4j 32 | slf4j-api 33 | 34 | 35 | junit 36 | junit 37 | test 38 | 39 | 40 | 41 | 42 | ${project.artifactId}-${project.version} 43 | 44 | 45 | -------------------------------------------------------------------------------- /sentiment-core/src/main/java/zx/soft/sent/core/constant/RecordsConstant.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.core.constant; 2 | 3 | /** 4 | * 数据记录相关常量 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | public class RecordsConstant { 10 | 11 | /** 12 | * Redis中所有记录md5(key)所在的key 13 | */ 14 | // 元搜索类、博客类、邮件类、论坛类、资讯类、QQ群类、回复类、微博类 15 | public static final String[] REDIS_RECORDS_KEY = { "autm_search", "blog", "email", "forum", "information", 16 | "qqgroup", "reply", "weibo" }; 17 | 18 | /** 19 | * MySQL数据统计相关表名 20 | */ 21 | public static final String STATISTIC_BY_DAY = "statistic_per_day"; 22 | 23 | } 24 | -------------------------------------------------------------------------------- /sentiment-core/src/main/java/zx/soft/sent/core/tsdb/GatherQueueReport.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.core.tsdb; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import zx.soft.redis.client.cache.Cache; 7 | import zx.soft.redis.client.tsdb.Reportable; 8 | import zx.soft.redis.client.tsdb.Tsdb; 9 | import zx.soft.sent.core.constant.RecordsConstant; 10 | 11 | /** 12 | * 统计Redis中的数据条数等实时数据。 13 | * 14 | * @author wanggang 15 | * 16 | * TODO 17 | * 18 | */ 19 | public class GatherQueueReport implements Reportable { 20 | 21 | private final Cache cache; 22 | 23 | public GatherQueueReport(final Cache cache) { 24 | this.cache = cache; 25 | } 26 | 27 | @Override 28 | public List report() { 29 | List result = new ArrayList(); 30 | long count; 31 | for (String key : RecordsConstant.REDIS_RECORDS_KEY) { 32 | count = cache.scard(key); 33 | result.add(new Tsdb("gather.spider." + key, count, "type", key + "-count")); 34 | } 35 | // 其他数据根据需求添加 36 | // ...... 37 | // ...... 38 | return result; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /sentiment-core/src/main/java/zx/soft/sent/core/tsdb/SentimentReportCore.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.core.tsdb; 2 | 3 | import java.util.Properties; 4 | 5 | import zx.soft.redis.client.cache.Cache; 6 | import zx.soft.redis.client.cache.CacheFactory; 7 | import zx.soft.redis.client.tsdb.TsdbReporter; 8 | import zx.soft.utils.config.ConfigUtil; 9 | 10 | /** 11 | * 舆情数据统计控制类 12 | * 13 | * @author wanggang 14 | * 15 | * TODO 16 | * 17 | */ 18 | public class SentimentReportCore { 19 | 20 | /** 21 | * 主函数 22 | */ 23 | public static void main(String[] args) { 24 | 25 | Cache cache = CacheFactory.getInstance(); 26 | 27 | Properties prop = ConfigUtil.getProps("cache-config.properties"); 28 | 29 | TsdbReporter reporter = new TsdbReporter(prop.getProperty("tsdb.host"), Integer.parseInt(prop 30 | .getProperty("tsdb.port"))); 31 | reporter.addReport(new GatherQueueReport(cache)); 32 | 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /sentiment-core/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 8 | 9 | 10 | 11 | INFO 12 | ACCEPT 13 | DENY 14 | 15 | 16 | 17 | 19 | logs/sentiment-core.log 20 | 21 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 22 | 23 | 24 | 25 | INFO 26 | 27 | 28 | logs/sentiment-core.log.%d{yyyy-MM-dd}.gz 29 | 30 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /sentiment-core/src/test/java/zx/soft/sent/core/tsdb/GatherQueueReportTest.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.core.tsdb; 2 | 3 | import org.junit.Test; 4 | 5 | public class GatherQueueReportTest { 6 | 7 | @Test 8 | public void test() { 9 | // 10 | } 11 | 12 | } 13 | -------------------------------------------------------------------------------- /sentiment-core/src/test/resources/cache-config.properties: -------------------------------------------------------------------------------- 1 | #redis.servers=hdp321,hdp322 2 | redis.servers=zxsofthadoop09,zxsofthadoop10 3 | redis.master=192.168.32.19 4 | redis.slave=192.168.32.20 5 | redis.port=16379 6 | redis.password=zxsoft 7 | 8 | tsdb.host=hdp321 9 | tsdb.port=4242 -------------------------------------------------------------------------------- /sentiment-core/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{MMdd.HHmmss.SSS} [%-20t] [%-5p] [%-20c] [L:%-3L] - %m%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /sentiment-core/src/test/resources/solr_params.properties: -------------------------------------------------------------------------------- 1 | collection=sentiment 2 | zookeeper_cloud=hdp322:2181 3 | #zookeeper_cloud=zxsofthadoop07:2181 4 | zookeeper_connect_timeout=5000 5 | zookeeper_client_timeout=5000 6 | fetch_size=20000 -------------------------------------------------------------------------------- /sentiment-dao/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | zx.soft 7 | sentiment-search 8 | 2.2.0 9 | ../pom.xml 10 | 11 | 12 | sentiment-dao 13 | Sentiment Dao 14 | 15 | 16 | 17 | 18 | org.mybatis 19 | mybatis 20 | 21 | 22 | commons-dbcp 23 | commons-dbcp 24 | 25 | 26 | mysql 27 | mysql-connector-java 28 | 29 | 30 | 31 | com.sleepycat 32 | je 33 | 5.0.73 34 | 35 | 36 | 37 | oracle.jdbc 38 | ojdbc 39 | 40 | 41 | 42 | org.slf4j 43 | slf4j-api 44 | 45 | 46 | 47 | junit 48 | junit 49 | test 50 | 51 | 52 | 53 | 54 | ${project.artifactId}-${project.version} 55 | 56 | 57 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/common/Config.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.common; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.util.Properties; 6 | 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import zx.soft.utils.log.LogbackUtil; 11 | 12 | /** 13 | * 配置文件信息读取类 14 | * 15 | * @author wanggang 16 | * 17 | */ 18 | public class Config { 19 | 20 | private static Logger logger = LoggerFactory.getLogger(Config.class); 21 | 22 | static Properties conf = new Properties() { 23 | 24 | private static final long serialVersionUID = -2493912332089543431L; 25 | 26 | { 27 | try (InputStream in = Config.class.getClassLoader().getResourceAsStream("oracle_db.properties");) { 28 | this.load(in); 29 | } catch (IOException e) { 30 | logger.error("Exception:{}", LogbackUtil.expection2Str(e)); 31 | throw new RuntimeException(e); 32 | } 33 | } 34 | 35 | }; 36 | 37 | public static String get(String key) { 38 | return conf.getProperty(key); 39 | } 40 | 41 | public static Properties getProps(String confFileName) { 42 | Properties result = new Properties(); 43 | logger.info("Load resource: " + confFileName); 44 | try (InputStream in = Config.class.getClassLoader().getResourceAsStream(confFileName);) { 45 | result.load(in); 46 | return result; 47 | } catch (Exception e) { 48 | logger.error("Exception:{}, StackTrace:{}", e.getMessage(), e.getStackTrace()); 49 | throw new RuntimeException(e); 50 | } 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/common/MybatisConfig.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.common; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | 8 | import org.apache.ibatis.io.Resources; 9 | import org.apache.ibatis.session.SqlSessionFactory; 10 | import org.apache.ibatis.session.SqlSessionFactoryBuilder; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import zx.soft.utils.log.LogbackUtil; 15 | 16 | /** 17 | * Mybatis配置类 18 | * 19 | * @author wanggang 20 | * 21 | */ 22 | public class MybatisConfig { 23 | 24 | private static Logger logger = LoggerFactory.getLogger(MybatisConfig.class); 25 | 26 | public enum ServerEnum { 27 | sentiment 28 | } 29 | 30 | private static Map sessionFactorys = new HashMap<>(); 31 | 32 | static { 33 | for (ServerEnum server : ServerEnum.values()) { 34 | try (InputStream inputStream = Resources.getResourceAsStream("mybatis-config.xml");) { 35 | sessionFactorys.put(server, new SqlSessionFactoryBuilder().build(inputStream, server.name())); 36 | } catch (IOException e) { 37 | logger.error("Exception:{}", LogbackUtil.expection2Str(e)); 38 | throw new RuntimeException(e); 39 | } 40 | } 41 | } 42 | 43 | public static SqlSessionFactory getSqlSessionFactory(ServerEnum server) { 44 | return sessionFactorys.get(server); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/common/SentimentConstant.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.common; 2 | 3 | /** 4 | * 與情信息常量值 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | public class SentimentConstant { 10 | 11 | /** 12 | * 與情平台类型 13 | */ 14 | public static final String[] PLATFORM_ARRAY = { "其他类", "资讯类", "论坛类", "微博类", "博客类", "QQ类", "搜索类", "回复类", "邮件类", 15 | "图片类", "微信类" }; 16 | 17 | // 专题模块站点信息键名，用于多个站点Md5-站点列表存储 18 | public static final String SITE_GROUPS = "sent:site:groups"; 19 | 20 | // 所有站点id和站点名称对应的列表名称 21 | public static final String SITE_MAP = "sent:site:map"; 22 | 23 | // 與情数据消息队列 24 | public static final String SENTIMENT_CACHE_KEY = "sent.cache.records"; 25 | 26 | // 存储记录的id，主要用于数据库写入去重 27 | public static final String SENT_KEY_INSERTED = "sent:key:inserted"; 28 | 29 | } 30 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/demo/BerkeleyDemo.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.demo; 2 | 3 | import java.io.File; 4 | 5 | import com.sleepycat.je.Database; 6 | import com.sleepycat.je.DatabaseConfig; 7 | import com.sleepycat.je.Environment; 8 | import com.sleepycat.je.EnvironmentConfig; 9 | 10 | public class BerkeleyDemo { 11 | 12 | public static void main(String[] args) { 13 | 14 | // 打开database环境 15 | EnvironmentConfig envConfig = new EnvironmentConfig(); 16 | //当环境不存在的时候自动创建环境 17 | envConfig.setAllowCreate(true); 18 | //设置支持事务 19 | envConfig.setTransactional(true); 20 | Environment myDbEnvironment = new Environment(new File("bdb/dbEnv"), envConfig); 21 | 22 | // 打开一个数据库，如果数据库不存在则创建一个 23 | DatabaseConfig dbConfig = new DatabaseConfig(); 24 | dbConfig.setAllowCreate(true); 25 | //打开一个数据库，数据库名为sampleDatabase,数据库的配置为dbConfig 26 | Database myDatabase = myDbEnvironment.openDatabase(null, "sampleDatabase", dbConfig); 27 | 28 | // 删除当前环境中指定的数据库 29 | String dbName = myDatabase.getDatabaseName(); 30 | myDbEnvironment.removeDatabase(null, dbName); 31 | // 给当前环境下的数据库改名 32 | String oldName = myDatabase.getDatabaseName(); 33 | myDbEnvironment.renameDatabase(null, oldName, oldName + ".new"); 34 | // 清空database内的所有数据，返回清空了多少条记录 35 | long numDiscarded = myDbEnvironment.truncateDatabase(null, myDatabase.getDatabaseName(), true); 36 | System.out.println("一共删除了 " + numDiscarded + " 条记录 从数据库 " + myDatabase.getDatabaseName()); 37 | 38 | // 关闭database环境 39 | if (myDatabase != null) { 40 | myDatabase.close(); 41 | } 42 | if (myDbEnvironment != null) { 43 | myDbEnvironment.cleanLog(); // 在关闭环境前清理下日志 44 | myDbEnvironment.close(); 45 | } 46 | 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/demo/CacheQueryDemo.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.demo; 2 | 3 | import zx.soft.sent.dao.common.MybatisConfig; 4 | import zx.soft.sent.dao.sentiment.SentimentRecord; 5 | 6 | public class CacheQueryDemo { 7 | 8 | public static void main(String[] args) { 9 | 10 | SentimentRecord sr = new SentimentRecord(MybatisConfig.ServerEnum.sentiment); 11 | /** 12 | * 插入查询缓存数据 13 | */ 14 | sr.insertCacheQuery("cache_query", "123546987ddfeff", "http://www.baidu.com", "result"); 15 | System.out.println(sr.selectCacheQuery("cache_query", "123546987ddfeff")); 16 | 17 | /** 18 | * 更新查询缓存数据 19 | */ 20 | sr.updateCacheQuery("cache_query", "123546987ddfeff", "http://www.baidu.com", "update_result"); 21 | 22 | /** 23 | * 查询查询缓存数据 24 | */ 25 | System.out.println(sr.selectCacheQuery("cache_query", "123546987ddfeff")); 26 | 27 | /** 28 | * 删除查询缓存数据 29 | */ 30 | sr.deleteCacheQuery("cache_query", "123546987ddfeff"); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/demo/FirstPageDemo.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.demo; 2 | 3 | import zx.soft.sent.dao.common.MybatisConfig; 4 | import zx.soft.sent.dao.firstpage.FirstPage; 5 | 6 | public class FirstPageDemo { 7 | 8 | public static void main(String[] args) { 9 | 10 | FirstPage firstPage = new FirstPage(MybatisConfig.ServerEnum.sentiment); 11 | int type = 4; 12 | String timestr = "2012-01-07,05"; 13 | String result = "first page result"; 14 | /** 15 | * 插入OA首页查询数据 16 | */ 17 | firstPage.insertFirstPage(type, timestr, result); 18 | /** 19 | * 查询OA首页查询数据 20 | */ 21 | System.out.println(firstPage.selectFirstPage(type, timestr)); 22 | /** 23 | * 更新OA首页查询数据 24 | */ 25 | firstPage.updateFirstPage(type, timestr, "frist page result update"); 26 | System.out.println(firstPage.selectFirstPage(type, timestr)); 27 | /** 28 | * 删除OA首页查询数据 29 | */ 30 | firstPage.deleteFirstPage(type, timestr); 31 | 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/demo/MybatisConfigDemo.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.demo; 2 | 3 | import java.sql.SQLException; 4 | 5 | import org.apache.ibatis.session.SqlSessionFactory; 6 | 7 | import zx.soft.sent.dao.common.MybatisConfig; 8 | 9 | public class MybatisConfigDemo { 10 | 11 | public static void main(String[] args) throws SQLException { 12 | 13 | SqlSessionFactory sqlSessionFactory = MybatisConfig.getSqlSessionFactory(MybatisConfig.ServerEnum.sentiment); 14 | System.out.println(sqlSessionFactory.getConfiguration().getEnvironment().getId()); 15 | 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/demo/OracleJDBCDemo.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.demo; 2 | 3 | import java.sql.ResultSet; 4 | import java.sql.SQLException; 5 | 6 | import zx.soft.sent.dao.oracle.OracleJDBC; 7 | 8 | public class OracleJDBCDemo { 9 | 10 | public static void main(String[] args) throws SQLException { 11 | OracleJDBC oracleJDBC = new OracleJDBC(); 12 | String sql = "SELECT COUNT(*) as ct FROM SJCJ_WBL"; 13 | ResultSet rs = oracleJDBC.query(sql); 14 | if (rs.next()) { 15 | System.out.println(rs.getInt("ct")); 16 | } else { 17 | System.err.println("Error!"); 18 | } 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/demo/SpiderSiteDemo.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.demo; 2 | 3 | import java.util.Date; 4 | 5 | import zx.soft.sent.dao.common.MybatisConfig; 6 | import zx.soft.sent.dao.domain.platform.SiteInfo; 7 | import zx.soft.sent.dao.spidersite.SpiderSite; 8 | 9 | public class SpiderSiteDemo { 10 | 11 | public static void main(String[] args) { 12 | 13 | SpiderSite spiderSite = new SpiderSite(MybatisConfig.ServerEnum.sentiment); 14 | 15 | SiteInfo siteInfo = new SiteInfo(); 16 | siteInfo.setUrl("http://www.baidu.com"); 17 | siteInfo.setZone(2); 18 | siteInfo.setDescription("站点描述"); 19 | siteInfo.setSource_name("站点名称"); 20 | siteInfo.setPlatform(8); 21 | siteInfo.setStatus(200); 22 | siteInfo.setSpider_id(123); 23 | siteInfo.setSpider_type("爬虫类型"); 24 | siteInfo.setSource_id(456); 25 | siteInfo.setSource_type(4); 26 | siteInfo.setSource_code(10101); 27 | siteInfo.setContact("站点联系方式"); 28 | siteInfo.setAdmin("站点负责人"); 29 | siteInfo.setRoot(123456789L); 30 | siteInfo.setParams("参数名键值对列表"); 31 | siteInfo.setUid(987654321L); 32 | siteInfo.setTimestamp(new Date(1419755627695L)); 33 | siteInfo.setIdentify(111111); 34 | 35 | /** 36 | * 插入站点信息 37 | */ 38 | spiderSite.insertSiteInfo(siteInfo); 39 | 40 | /** 41 | * 根据站点Id返回站点名称 42 | */ 43 | System.out.println(spiderSite.getSourceName(456)); 44 | 45 | /** 46 | * 根据站点名称返回站点Id 47 | */ 48 | System.out.println(spiderSite.getSourceId("站点名称")); 49 | 50 | /** 51 | * 根据站点Id删除站点信息 52 | */ 53 | spiderSite.deleteSiteById(456); 54 | 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/domain/allinternet/TaskResult.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.domain.allinternet; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * OA全网搜索任务缓存结果数据 7 | * 8 | * @author wanggang 9 | * 10 | */ 11 | @Deprecated 12 | public class TaskResult implements Serializable { 13 | 14 | private static final long serialVersionUID = 4224722992054268127L; 15 | 16 | private int local_count; // 本地库搜索结果总数：关键词，时间段：2000.1.1-创建时间 17 | private int autm_count; // 元搜索搜索结果总数：关键词，时间段：创建时间-结束时间/当前时间，source_ids 18 | private int all_count; // 总和：local_count+autm_count 19 | 20 | public int getLocal_count() { 21 | return local_count; 22 | } 23 | 24 | public void setLocal_count(int local_count) { 25 | this.local_count = local_count; 26 | } 27 | 28 | public int getAutm_count() { 29 | return autm_count; 30 | } 31 | 32 | public void setAutm_count(int autm_count) { 33 | this.autm_count = autm_count; 34 | } 35 | 36 | public int getAll_count() { 37 | return all_count; 38 | } 39 | 40 | public void setAll_count(int all_count) { 41 | this.all_count = all_count; 42 | } 43 | 44 | public static long getSerialversionuid() { 45 | return serialVersionUID; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/domain/firstpage/FirstPageInfo.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.domain.firstpage; 2 | 3 | /** 4 | * OA首页查询信息 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | public class FirstPageInfo { 10 | 11 | private int type;// 首页展现类型，如：1-今日概况、2-饼状图、3-舆情聚焦、4-微博当天数据趋势、5-重点关注 12 | private String timestr; // 记录时间字符串，如：2014-09-03 12:23:01; 13 | private String result; // 查询结果 14 | 15 | public FirstPageInfo() { 16 | // 17 | } 18 | 19 | public FirstPageInfo(int type, String timestr, String result) { 20 | this.type = type; 21 | this.timestr = timestr; 22 | this.result = result; 23 | } 24 | 25 | public int getType() { 26 | return type; 27 | } 28 | 29 | public void setType(int type) { 30 | this.type = type; 31 | } 32 | 33 | public String getTimestr() { 34 | return timestr; 35 | } 36 | 37 | public void setTimestr(String timestr) { 38 | this.timestr = timestr; 39 | } 40 | 41 | public String getResult() { 42 | return result; 43 | } 44 | 45 | public void setResult(String result) { 46 | this.result = result; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/domain/sentiment/InsertCacheQuery.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.domain.sentiment; 2 | 3 | /** 4 | * 查询缓存结果 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | public class InsertCacheQuery { 10 | 11 | private String tablename; 12 | private String query_id; // 查询ID 13 | private String query_url; // 查询URL 14 | private String query_result; // 查询结果 15 | 16 | public InsertCacheQuery() { 17 | // 18 | } 19 | 20 | public InsertCacheQuery(String tablename, String query_id, String query_url, String query_result) { 21 | this.tablename = tablename; 22 | this.query_id = query_id; 23 | this.query_url = query_url; 24 | this.query_result = query_result; 25 | } 26 | 27 | public String getTablename() { 28 | return tablename; 29 | } 30 | 31 | public void setTablename(String tablename) { 32 | this.tablename = tablename; 33 | } 34 | 35 | public String getQuery_id() { 36 | return query_id; 37 | } 38 | 39 | public void setQuery_id(String query_id) { 40 | this.query_id = query_id; 41 | } 42 | 43 | public String getQuery_url() { 44 | return query_url; 45 | } 46 | 47 | public void setQuery_url(String query_url) { 48 | this.query_url = query_url; 49 | } 50 | 51 | public String getQuery_result() { 52 | return query_result; 53 | } 54 | 55 | public void setQuery_result(String query_result) { 56 | this.query_result = query_result; 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/domain/sentiment/SelectCacheQuery.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.domain.sentiment; 2 | 3 | /** 4 | * 查询缓存类 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | public class SelectCacheQuery { 10 | 11 | private String query_id; // 查询ID 12 | private String query_url; // 查询URL 13 | private String query_result; // 查询结果 14 | 15 | public SelectCacheQuery() { 16 | // 17 | } 18 | 19 | public SelectCacheQuery(String query_id, String query_url, String query_result) { 20 | this.query_id = query_id; 21 | this.query_url = query_url; 22 | this.query_result = query_result; 23 | } 24 | 25 | public String getQuery_id() { 26 | return query_id; 27 | } 28 | 29 | public void setQuery_id(String query_id) { 30 | this.query_id = query_id; 31 | } 32 | 33 | public String getQuery_url() { 34 | return query_url; 35 | } 36 | 37 | public void setQuery_url(String query_url) { 38 | this.query_url = query_url; 39 | } 40 | 41 | public String getQuery_result() { 42 | return query_result; 43 | } 44 | 45 | public void setQuery_result(String query_result) { 46 | this.query_result = query_result; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/domain/sentiment/SelectParamsById.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.domain.sentiment; 2 | 3 | /** 4 | * 根据ID查询类 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | public class SelectParamsById { 10 | 11 | private String tablename; 12 | private String id; 13 | 14 | public SelectParamsById() { 15 | // 16 | } 17 | 18 | public SelectParamsById(String tablename, String id) { 19 | this.tablename = tablename; 20 | this.id = id; 21 | } 22 | 23 | public String getTablename() { 24 | return tablename; 25 | } 26 | 27 | public void setTablename(String tablename) { 28 | this.tablename = tablename; 29 | } 30 | 31 | public String getId() { 32 | return id; 33 | } 34 | 35 | public void setId(String id) { 36 | this.id = id; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/domain/sentiment/SelectParamsByTime.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.domain.sentiment; 2 | 3 | import java.util.Date; 4 | 5 | /** 6 | * 根据时间查询参数类 7 | * 8 | * @author wanggang 9 | * 10 | */ 11 | public class SelectParamsByTime { 12 | 13 | private String tablename; 14 | private Date low; 15 | private Date high; 16 | 17 | public SelectParamsByTime() { 18 | // 19 | } 20 | 21 | public SelectParamsByTime(String tablename, Date low, Date high) { 22 | this.tablename = tablename; 23 | this.low = low; 24 | this.high = high; 25 | } 26 | 27 | public String getTablename() { 28 | return tablename; 29 | } 30 | 31 | public void setTablename(String tablename) { 32 | this.tablename = tablename; 33 | } 34 | 35 | public Date getLow() { 36 | return low; 37 | } 38 | 39 | public void setLow(Date low) { 40 | this.low = low; 41 | } 42 | 43 | public Date getHigh() { 44 | return high; 45 | } 46 | 47 | public void setHigh(Date high) { 48 | this.high = high; 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/domain/sentiment/SentTablename.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.domain.sentiment; 2 | 3 | /** 4 | * 舆情数据表 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | public class SentTablename { 10 | 11 | private String tablename; 12 | private String name; 13 | private long lasttime; 14 | 15 | public SentTablename() { 16 | // 17 | } 18 | 19 | public SentTablename(String tablename, String name) { 20 | this.tablename = tablename; 21 | this.name = name; 22 | } 23 | 24 | public SentTablename(String tablename, String name, long lasttime) { 25 | this.tablename = tablename; 26 | this.name = name; 27 | this.lasttime = lasttime; 28 | } 29 | 30 | public long getLasttime() { 31 | return lasttime; 32 | } 33 | 34 | public void setLasttime(long lasttime) { 35 | this.lasttime = lasttime; 36 | } 37 | 38 | public String getTablename() { 39 | return tablename; 40 | } 41 | 42 | public void setTablename(String tablename) { 43 | this.tablename = tablename; 44 | } 45 | 46 | public String getName() { 47 | return name; 48 | } 49 | 50 | public void setName(String name) { 51 | this.name = name; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/domain/special/SpecialResult.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.domain.special; 2 | 3 | /** 4 | * 插入专题的查询结果数据 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | public class SpecialResult { 10 | 11 | private String identify; // 唯一标识 12 | private String type; // 专题类型，如：饼状图、趋势图 13 | private String result; // 查询结果 14 | 15 | public SpecialResult() { 16 | // 17 | } 18 | 19 | public SpecialResult(String identify, String type, String result) { 20 | this.identify = identify; 21 | this.type = type; 22 | this.result = result; 23 | } 24 | 25 | @Override 26 | public String toString() { 27 | return "SpecialResult:[identify=" + identify + ",type=" + type + ",result=" + result + "]"; 28 | } 29 | 30 | public String getIdentify() { 31 | return identify; 32 | } 33 | 34 | public void setIdentify(String identify) { 35 | this.identify = identify; 36 | } 37 | 38 | public String getType() { 39 | return type; 40 | } 41 | 42 | public void setType(String type) { 43 | this.type = type; 44 | } 45 | 46 | public String getResult() { 47 | return result; 48 | } 49 | 50 | public void setResult(String result) { 51 | this.result = result; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/domain/special/SpecialTopic.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.domain.special; 2 | 3 | /** 4 | * 插入的专题参数数据 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | 10 | public class SpecialTopic { 11 | 12 | private String identify; // 专题唯一标识 13 | private String name; // 专题名称 14 | private String keywords; // （合肥 and 警察） not 打人 15 | private String start; // '2014-08-25 00:00:00' 16 | private String end; // '2014-08-25 23:59:59' 17 | private int hometype; // 0代表全部 1代表境内 2代表境外 18 | 19 | public SpecialTopic() { 20 | // 21 | } 22 | 23 | public SpecialTopic(String identify, String name, String keywords, String start, String end, int hometype) { 24 | this.name = name; 25 | this.identify = identify; 26 | this.keywords = keywords; 27 | this.start = start; 28 | this.end = end; 29 | this.hometype = hometype; 30 | } 31 | 32 | @Override 33 | public String toString() { 34 | return "SpecialTopic:[identify=" + identify + ",name=" + name + ",keywords=" + keywords + ",start=" 35 | + start + ",end=" + end + ",hometype=" + hometype + "]"; 36 | } 37 | 38 | public String getName() { 39 | return name; 40 | } 41 | 42 | public void setName(String name) { 43 | this.name = name; 44 | } 45 | 46 | public String getIdentify() { 47 | return identify; 48 | } 49 | 50 | public void setIdentify(String identify) { 51 | this.identify = identify; 52 | } 53 | 54 | public String getKeywords() { 55 | return keywords; 56 | } 57 | 58 | public void setKeywords(String keywords) { 59 | this.keywords = keywords; 60 | } 61 | 62 | public String getStart() { 63 | return start; 64 | } 65 | 66 | public void setStart(String start) { 67 | this.start = start; 68 | } 69 | 70 | public String getEnd() { 71 | return end; 72 | } 73 | 74 | public void setEnd(String end) { 75 | this.end = end; 76 | } 77 | 78 | public int getHometype() { 79 | return hometype; 80 | } 81 | 82 | public void setHometype(int hometype) { 83 | this.hometype = hometype; 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/domain/weibo/WeiboSinaParams.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.domain.weibo; 2 | 3 | /** 4 | * 新浪微博查询参数类 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | public class WeiboSinaParams { 10 | 11 | private String tablename; 12 | private int low; 13 | private int high; 14 | 15 | public WeiboSinaParams() { 16 | // 17 | } 18 | 19 | public WeiboSinaParams(String tablename, int low, int high) { 20 | this.tablename = tablename; 21 | this.low = low; 22 | this.high = high; 23 | } 24 | 25 | public String getTablename() { 26 | return tablename; 27 | } 28 | 29 | public void setTablename(String tablename) { 30 | this.tablename = tablename; 31 | } 32 | 33 | public int getLow() { 34 | return low; 35 | } 36 | 37 | public void setLow(int low) { 38 | this.low = low; 39 | } 40 | 41 | public int getHigh() { 42 | return high; 43 | } 44 | 45 | public void setHigh(int high) { 46 | this.high = high; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/firstpage/FirstPageHarmfulMapper.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.firstpage; 2 | 3 | import org.apache.ibatis.annotations.Arg; 4 | import org.apache.ibatis.annotations.ConstructorArgs; 5 | import org.apache.ibatis.annotations.Delete; 6 | import org.apache.ibatis.annotations.Insert; 7 | import org.apache.ibatis.annotations.Param; 8 | import org.apache.ibatis.annotations.Select; 9 | import org.apache.ibatis.annotations.Update; 10 | 11 | import zx.soft.sent.dao.domain.firstpage.FirstPageInfo; 12 | 13 | /** 14 | * OA首页信息接口 15 | * 16 | * @author wanggang 17 | * 18 | */ 19 | public interface FirstPageHarmfulMapper { 20 | 21 | /** 22 | * 插入OA首页查询数据 23 | */ 24 | @Insert("INSERT INTO `oa_firstpage_harmful_cache` (`type`,`timestr`,`result`,`lasttime`) " 25 | + "VALUES (#{type},#{timestr},#{result},NOW())") 26 | public void insertFirstPage(FirstPageInfo firstPageInfo); 27 | 28 | /** 29 | * 更新OA首页查询数据 30 | */ 31 | @Update("UPDATE `oa_firstpage_harmful_cache` SET `result` = #{result} " 32 | + "WHERE `type` = #{type} AND `timestr` = #{timestr}") 33 | public void updateFirstPage(FirstPageInfo firstPageInfo); 34 | 35 | /** 36 | * 查询OA首页查询数据 37 | */ 38 | @Select("SELECT `result` FROM `oa_firstpage_harmful_cache` WHERE `type` = #{type} AND `timestr` = #{timestr}") 39 | @ConstructorArgs(value = { @Arg(column = "result", javaType = String.class) }) 40 | public String selectFirstPage(@Param("type") int p1, @Param("timestr") String p2); 41 | 42 | /** 43 | * 删除OA首页查询数据 44 | */ 45 | @Delete("DELETE FROM `oa_firstpage_harmful_cache` WHERE `type` = #{type} AND `timestr` = #{timestr}") 46 | public void deleteFirstPage(@Param("type") int p1, @Param("timestr") String p2); 47 | 48 | } 49 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/firstpage/FirstPageMapper.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.firstpage; 2 | 3 | import org.apache.ibatis.annotations.Arg; 4 | import org.apache.ibatis.annotations.ConstructorArgs; 5 | import org.apache.ibatis.annotations.Delete; 6 | import org.apache.ibatis.annotations.Insert; 7 | import org.apache.ibatis.annotations.Param; 8 | import org.apache.ibatis.annotations.Select; 9 | import org.apache.ibatis.annotations.Update; 10 | 11 | import zx.soft.sent.dao.domain.firstpage.FirstPageInfo; 12 | 13 | /** 14 | * OA首页信息接口 15 | * 16 | * @author wanggang 17 | * 18 | */ 19 | public interface FirstPageMapper { 20 | 21 | /** 22 | * 插入OA首页查询数据 23 | */ 24 | @Insert("INSERT INTO `oa_firstpage_query_cache` (`type`,`timestr`,`result`,`lasttime`) " 25 | + "VALUES (#{type},#{timestr},#{result},NOW())") 26 | public void insertFirstPage(FirstPageInfo firstPageInfo); 27 | 28 | /** 29 | * 更新OA首页查询数据 30 | */ 31 | @Update("UPDATE `oa_firstpage_query_cache` SET `result` = #{result} " 32 | + "WHERE `type` = #{type} AND `timestr` = #{timestr}") 33 | public void updateFirstPage(FirstPageInfo firstPageInfo); 34 | 35 | /** 36 | * 查询OA首页查询数据 37 | */ 38 | @Select("SELECT `result` FROM `oa_firstpage_query_cache` WHERE `type` = #{type} AND `timestr` = #{timestr}") 39 | @ConstructorArgs(value = { @Arg(column = "result", javaType = String.class) }) 40 | public String selectFirstPage(@Param("type") int p1, @Param("timestr") String p2); 41 | 42 | /** 43 | * 删除OA首页查询数据 44 | */ 45 | @Delete("DELETE FROM `oa_firstpage_query_cache` WHERE `type` = #{type} AND `timestr` = #{timestr}") 46 | public void deleteFirstPage(@Param("type") int p1, @Param("timestr") String p2); 47 | 48 | } 49 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/oracle/OracleJDBC.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.oracle; 2 | 3 | import java.sql.Connection; 4 | import java.sql.DriverManager; 5 | import java.sql.ResultSet; 6 | import java.sql.SQLException; 7 | import java.sql.Statement; 8 | 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import zx.soft.sent.dao.common.Config; 13 | import zx.soft.utils.log.LogbackUtil; 14 | 15 | /** 16 | * Oracle数据库访问类 17 | * 18 | * @author wanggang 19 | * 20 | */ 21 | public class OracleJDBC { 22 | 23 | private static Logger logger = LoggerFactory.getLogger(OracleJDBC.class); 24 | 25 | private String db_url; 26 | private String db_username; 27 | private String db_password; 28 | 29 | private Connection conn; 30 | private Statement statement; 31 | 32 | public OracleJDBC() { 33 | doInit(); 34 | try { 35 | Class.forName("oracle.jdbc.driver.OracleDriver").newInstance(); 36 | conn = DriverManager.getConnection(db_url, db_username, db_password); 37 | statement = conn.createStatement(); 38 | } catch (Exception e) { 39 | logger.error("Exception:{}", LogbackUtil.expection2Str(e)); 40 | throw new RuntimeException(e); 41 | } 42 | } 43 | 44 | private void doInit() { 45 | db_url = Config.get("db_url"); 46 | db_username = Config.get("db_username"); 47 | db_password = Config.get("db_password"); 48 | } 49 | 50 | /** 51 | * 更新数据 52 | */ 53 | public boolean update(String sql) { 54 | try { 55 | statement.executeUpdate(sql); 56 | return true; 57 | } catch (Exception e) { 58 | logger.error("Exception:{}, StackTrace:{}", e.getMessage(), e.getStackTrace()); 59 | return Boolean.FALSE; 60 | } 61 | } 62 | 63 | /** 64 | * 查询数据 65 | */ 66 | public ResultSet query(String sql) { 67 | try { 68 | return statement.executeQuery(sql); 69 | } catch (Exception e) { 70 | logger.error("Exception:{}, StackTrace:{}", e.getMessage(), e.getStackTrace()); 71 | return null; 72 | } 73 | } 74 | 75 | public void close() { 76 | try { 77 | conn.close(); 78 | statement.close(); 79 | } catch (SQLException e) { 80 | logger.error("Exception:{}, StackTrace:{}", e.getMessage(), e.getStackTrace()); 81 | } 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/sentiment/SentimentRecordDao.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.sentiment; 2 | 3 | import java.util.List; 4 | 5 | import zx.soft.sent.dao.domain.sentiment.InsertCacheQuery; 6 | import zx.soft.sent.dao.domain.sentiment.RecordInsert; 7 | import zx.soft.sent.dao.domain.sentiment.RecordSelect; 8 | import zx.soft.sent.dao.domain.sentiment.SelectParamsById; 9 | import zx.soft.sent.dao.domain.sentiment.SelectParamsByTime; 10 | import zx.soft.sent.dao.domain.sentiment.SentTablename; 11 | 12 | /** 13 | * 舆情数据接口 14 | * 15 | * @author wanggang 16 | * 17 | */ 18 | public interface SentimentRecordDao { 19 | 20 | /** 21 | * 获取最大Id 22 | */ 23 | public int getMaxId(SentTablename sentTablename); 24 | 25 | /** 26 | * 插入数据表名 27 | */ 28 | public void insertTablename(SentTablename sentTablename); 29 | 30 | /** 31 | * 插入Record数据 32 | */ 33 | public void insertRecord(RecordInsert recordInsert); 34 | 35 | /** 36 | * 获取Record数据，根据md5的id 37 | */ 38 | public RecordSelect selectRecordById(SelectParamsById selectParamsById); 39 | 40 | /** 41 | * 获取Records数据，根据lasttime 42 | */ 43 | public List selectRecordsByLasttime(SelectParamsByTime selectParamsByTime); 44 | 45 | /** 46 | * 更新Record数据 47 | * 注意：更新数据的时候id要一样 48 | */ 49 | public void updateRecord(RecordInsert recordInsert); 50 | 51 | /** 52 | * 删除Record数据，根据md5的id 53 | */ 54 | public void deleteRecordById(SelectParamsById selectParamsById); 55 | 56 | /** 57 | * 插入查询缓存数据 58 | */ 59 | public void insertCacheQuery(InsertCacheQuery insertCacheQuery); 60 | 61 | /** 62 | * 更新查询缓存数据 63 | */ 64 | public void updateCacheQuery(InsertCacheQuery insertCacheQuery); 65 | 66 | /** 67 | * 查询查询缓存数据 68 | */ 69 | public String selectCacheQuery(InsertCacheQuery insertCacheQuery); 70 | 71 | /** 72 | * 删除查询缓存数据 73 | */ 74 | public void deleteCacheQuery(InsertCacheQuery insertCacheQuery); 75 | 76 | } 77 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/spidersite/SpiderSite.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.spidersite; 2 | 3 | import org.apache.ibatis.session.SqlSession; 4 | import org.apache.ibatis.session.SqlSessionFactory; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import zx.soft.sent.dao.common.MybatisConfig; 9 | import zx.soft.sent.dao.domain.platform.SiteInfo; 10 | import zx.soft.utils.log.LogbackUtil; 11 | 12 | public class SpiderSite { 13 | 14 | private static Logger logger = LoggerFactory.getLogger(SpiderSite.class); 15 | 16 | private static SqlSessionFactory sqlSessionFactory; 17 | 18 | public SpiderSite(MybatisConfig.ServerEnum server) { 19 | try { 20 | sqlSessionFactory = MybatisConfig.getSqlSessionFactory(server); 21 | } catch (RuntimeException e) { 22 | logger.error("Exception:{}", LogbackUtil.expection2Str(e)); 23 | throw new RuntimeException(e); 24 | } 25 | } 26 | 27 | /** 28 | * 插入站点信息 29 | */ 30 | public void insertSiteInfo(SiteInfo siteInfo) { 31 | try (SqlSession session = sqlSessionFactory.openSession();) { 32 | SpiderSiteMapper spiderSiteMapper = session.getMapper(SpiderSiteMapper.class); 33 | spiderSiteMapper.insertSiteInfo(siteInfo); 34 | } 35 | } 36 | 37 | /** 38 | * 根据站点Id返回站点名称 39 | */ 40 | public String getSourceName(int source_id) { 41 | try (SqlSession session = sqlSessionFactory.openSession();) { 42 | SpiderSiteMapper spiderSiteMapper = session.getMapper(SpiderSiteMapper.class); 43 | return spiderSiteMapper.getSourceName(source_id); 44 | } 45 | } 46 | 47 | /** 48 | * 根据站点名称返回站点Id 49 | */ 50 | public int getSourceId(String source_name) { 51 | try (SqlSession session = sqlSessionFactory.openSession();) { 52 | SpiderSiteMapper spiderSiteMapper = session.getMapper(SpiderSiteMapper.class); 53 | return spiderSiteMapper.getSourceId(source_name); 54 | } 55 | } 56 | 57 | /** 58 | * 根据站点Id删除站点信息 59 | */ 60 | public void deleteSiteById(int source_id) { 61 | try (SqlSession session = sqlSessionFactory.openSession();) { 62 | SpiderSiteMapper spiderSiteMapper = session.getMapper(SpiderSiteMapper.class); 63 | spiderSiteMapper.deleteSiteById(source_id); 64 | } 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/spidersite/SpiderSiteMapper.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.spidersite; 2 | 3 | import org.apache.ibatis.annotations.Arg; 4 | import org.apache.ibatis.annotations.ConstructorArgs; 5 | import org.apache.ibatis.annotations.Delete; 6 | import org.apache.ibatis.annotations.Insert; 7 | import org.apache.ibatis.annotations.Select; 8 | 9 | import zx.soft.sent.dao.domain.platform.SiteInfo; 10 | 11 | /** 12 | * 站点信息Mapper 13 | * 14 | * @author wanggang 15 | * 16 | */ 17 | public interface SpiderSiteMapper { 18 | 19 | /** 20 | * 插入站点信息 21 | */ 22 | @Insert("INSERT INTO `sent_site_info` (`url`,`zone`,`description`,`source_name`,`platform`,`status`," 23 | + "`spider_id`,`spider_type`,`source_id`,`source_type`,`source_code`,`contact`,`admin`,`root`," 24 | + "`params`,`uid`,`timestamp`,`identify`,`lasttime`) VALUES (#{url},#{zone},#{description}," 25 | + "#{source_name},#{platform},#{status},#{spider_id},#{spider_type},#{source_id},#{source_type}," 26 | + "#{source_code},#{contact},#{admin},#{root},#{params},#{uid},#{timestamp},#{identify},NOW())") 27 | public void insertSiteInfo(SiteInfo siteInfo); 28 | 29 | /** 30 | * 根据站点Id返回站点名称 31 | */ 32 | @Select("SELECT `source_name` FROM `sent_site_info` WHERE `source_id` = #{source_id}") 33 | @ConstructorArgs(value = { @Arg(column = "source_name", javaType = String.class) }) 34 | public String getSourceName(int source_id); 35 | 36 | /** 37 | * 根据站点名称返回站点Id 38 | */ 39 | @Select("SELECT `source_id` FROM `sent_site_info` WHERE `source_name` = #{source_name}") 40 | @ConstructorArgs(value = { @Arg(column = "source_id", javaType = int.class) }) 41 | public int getSourceId(String source_name); 42 | 43 | /** 44 | * 根据站点Id删除站点信息 45 | */ 46 | @Delete("DELETE FROM `sent_site_info` WHERE `source_id` = #{source_id}") 47 | public void deleteSiteById(int source_id); 48 | 49 | } -------------------------------------------------------------------------------- /sentiment-dao/src/main/java/zx/soft/sent/dao/sql/CreateTables.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.sql; 2 | 3 | /** 4 | * 创建舆情相关数据表，根据crc32校验码hash分表。 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | public class CreateTables { 10 | 11 | public static final int MAX_TABLE_NUM = 128; 12 | 13 | public static final String SENT_TABLE_TABLE = "sentiment_tablenames"; 14 | 15 | public static final String CACHE_QUERY_TABLE = "cache_query"; 16 | 17 | public static final String SENT_TABLE = "sent_records_"; 18 | 19 | public static final String SENT_SITE_TABLE = "sent_site_info"; 20 | 21 | // OA专题信息表 22 | public static final String SPECIAL_INFO_TABLE = "oa_special_info"; 23 | 24 | // OA专题缓存数据表 25 | public static final String SPECIAL_CACHE_TABLE = "oa_special_query_cache"; 26 | 27 | // OA首页数据表 28 | public static final String FIRST_PAGE_TABLE = "oa_firstpage_query_cache"; 29 | 30 | /** 31 | * 主函数 32 | */ 33 | public static void main(String[] args) { 34 | 35 | SentDbcp sentDbcp = new SentDbcp(); 36 | // OA首页数据表 37 | // sentDbcp.createFirstPageTable(FIRST_PAGE_TABLE); 38 | // OA专题信息表 39 | // sentDbcp.createSpecialInfoTable(SPECIAL_INFO_TABLE); 40 | // OA专题缓存数据表 41 | // sentDbcp.createSpecialQueryCacheTable(SPECIAL_CACHE_TABLE); 42 | // 创建站点数据表 43 | // sentDbcp.createSiteTable(SENT_SITE_TABLE); 44 | // 创建查询缓存数据表 45 | // sentDbcp.createQueryCacheTable(CreateTables.CACHE_QUERY_TABLE); 46 | // 创建舆情数据表表名表 47 | // sentDbcp.createTablenameTable(CreateTables.SENT_TABLE_TABLE); 48 | // 创建舆情数据表 49 | for (int i = 0; i < MAX_TABLE_NUM; i++) { 50 | sentDbcp.createSentimentTable(CreateTables.SENT_TABLE + i); 51 | } 52 | sentDbcp.close(); 53 | System.out.println("Finish!"); 54 | 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /sentiment-dao/src/main/resources/SpecialQueryMapper.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 9 | 10 | -------------------------------------------------------------------------------- /sentiment-dao/src/test/java/zx/soft/sent/dao/common/MybatisConfigTest.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.common; 2 | 3 | import static org.junit.Assert.assertNotNull; 4 | import static org.junit.Assert.assertTrue; 5 | 6 | import org.apache.ibatis.session.SqlSessionFactory; 7 | import org.junit.Ignore; 8 | 9 | public class MybatisConfigTest { 10 | 11 | @Ignore 12 | public void testGetConfig() { 13 | SqlSessionFactory sqlSessionFactory = MybatisConfig.getSqlSessionFactory(MybatisConfig.ServerEnum.sentiment); 14 | assertNotNull(sqlSessionFactory); 15 | assertTrue(sqlSessionFactory.getConfiguration().getEnvironment().getId().length() > 0); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /sentiment-dao/src/test/java/zx/soft/sent/dao/oracle/OracleJDBCTest.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.dao.oracle; 2 | 3 | import static org.junit.Assert.assertTrue; 4 | 5 | import java.sql.ResultSet; 6 | 7 | import org.junit.AfterClass; 8 | import org.junit.BeforeClass; 9 | import org.junit.Ignore; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | public class OracleJDBCTest { 14 | 15 | private static Logger logger = LoggerFactory.getLogger(OracleJDBCTest.class); 16 | 17 | private static OracleJDBC dataOJDBC; 18 | 19 | @BeforeClass 20 | public void doInit() { 21 | logger.info("Start DataOJDBC ..."); 22 | dataOJDBC = new OracleJDBC(); 23 | } 24 | 25 | @AfterClass 26 | public void doClose() { 27 | logger.info("Close DataOJDBC ..."); 28 | dataOJDBC.close(); 29 | } 30 | 31 | @Ignore 32 | public void testQueryOne() throws Exception { 33 | String sql = "SELECT COUNT(*) AS data_count FROM SJCJ_WBL"; 34 | ResultSet rs = dataOJDBC.query(sql); 35 | assertTrue(rs.next()); 36 | assertTrue(rs.getInt("data_count") > 0); 37 | } 38 | 39 | @Ignore 40 | public void testQueryTwo() throws Exception { 41 | String sql = "select * from (select rownum as no1,t.* from SJCJ_WBL t where rownum <200) where no1>100"; 42 | ResultSet rs = dataOJDBC.query(sql); 43 | assertTrue(rs.getFetchSize() > 0); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /sentiment-dao/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{MMdd.HHmmss.SSS} [%-20t] [%-5p] [%-20c] [L:%-3L] - %m%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /sentiment-hdfs/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | 6 | zx.soft 7 | sentiment-search 8 | 2.2.0 9 | ../pom.xml 10 | 11 | 12 | sentiment-hdfs 13 | Sentiment Hdfs 14 | 15 | 16 | 17 | org.apache.hadoop 18 | hadoop-core 19 | 20 | 21 | 22 | ch.qos.logback 23 | logback-classic 24 | 25 | 26 | ch.qos.logback 27 | logback-core 28 | 29 | 30 | ch.qos.logback 31 | logback-access 32 | 33 | 34 | org.slf4j 35 | slf4j-api 36 | 37 | 38 | junit 39 | junit 40 | test 41 | 42 | 43 | 44 | 45 | ${project.artifactId}-${project.version} 46 | 47 | 48 | -------------------------------------------------------------------------------- /sentiment-hdfs/src/main/java/zx/soft/sent/hdfs/io/impl/FileExistsException.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.hdfs.io.impl; 2 | 3 | /** 4 | * 文件存在异常 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | public class FileExistsException extends RuntimeException { 10 | 11 | private static final long serialVersionUID = 1L; 12 | 13 | public FileExistsException(String file) { 14 | super("File " + file + " exists"); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /sentiment-hdfs/src/main/java/zx/soft/sent/hdfs/io/impl/HdfsWriter.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.hdfs.io.impl; 2 | 3 | import java.io.Closeable; 4 | 5 | /** 6 | * HDFS写数据接口 7 | * 8 | * @author wanggang 9 | * 10 | */ 11 | public interface HdfsWriter extends Closeable { 12 | 13 | void write(String key, String value); 14 | 15 | } 16 | -------------------------------------------------------------------------------- /sentiment-hdfs/src/main/resources/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | fs.default.name 8 | hdfs://192.168.3.21:19000 9 | 10 | 11 | -------------------------------------------------------------------------------- /sentiment-hdfs/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 8 | 9 | 10 | 11 | INFO 12 | ACCEPT 13 | DENY 14 | 15 | 16 | 17 | 19 | logs/sentiment-hdfs.log 20 | 21 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 22 | 23 | 24 | 25 | INFO 26 | 27 | 28 | logs/sentiment-hdfs.log.%d{yyyy-MM-dd}.gz 29 | 30 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /sentiment-hdfs/src/test/java/zx/soft/sent/hdfs/io/impl/HdfsWriterDummy.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.hdfs.io.impl; 2 | 3 | import java.io.IOException; 4 | import java.util.List; 5 | 6 | public class HdfsWriterDummy implements HdfsWriter { 7 | 8 | private final List result; 9 | 10 | public HdfsWriterDummy(List result) { 11 | this.result = result; 12 | } 13 | 14 | @Override 15 | public void close() throws IOException { 16 | } 17 | 18 | @Override 19 | public void write(String key, String value) { 20 | result.add(new String[] { key, value }); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /sentiment-hdfs/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{MMdd.HHmmss.SSS} [%-20t] [%-5p] [%-20c] [L:%-3L] - %m%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /sentiment-solr/doc/platform-mapping: -------------------------------------------------------------------------------- 1 | 0：other 2 | 1：Information 3 | 2：Forum 4 | 3：Weibo 5 | 4：Blog 6 | 5：QQGroup 7 | 6：AutmSearch 8 | 7：Reply 9 | 8：Email 10 | 9: Picture -------------------------------------------------------------------------------- /sentiment-solr/ik_dic/ext_ik.dic: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wgybzbrobot/sentiment-search/df50eef8f82fbdc4549d1cfffb0d0b393a04f7b5/sentiment-solr/ik_dic/ext_ik.dic -------------------------------------------------------------------------------- /sentiment-solr/ik_dic/stopwords_ik.dic: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wgybzbrobot/sentiment-search/df50eef8f82fbdc4549d1cfffb0d0b393a04f7b5/sentiment-solr/ik_dic/stopwords_ik.dic -------------------------------------------------------------------------------- /sentiment-solr/src/main/assembly/distribution.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | distribution 7 | 8 | tar.gz 9 | 10 | ${project.artifactId} 11 | 12 | 13 | src/main/resources 14 | 15 | logback.xml 16 | data_db.properties 17 | oracle_db.properties 18 | solr_params.properties 19 | cache-config.properties 20 | 21 | /conf 22 | true 23 | 24 | 25 | ik_dic 26 | 27 | * 28 | 29 | /ik_dic 30 | 31 | 32 | src/main/bin 33 | 34 | * 35 | 36 | /bin 37 | 0755 38 | 39 | 40 | 41 | 42 | /lib 43 | 44 | 45 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/bin/ctl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mainClass=zx.soft.sent.solr.driver.SentSolrDriver 4 | 5 | # resolve links - $0 may be a softlink 6 | PRG="$0" 7 | 8 | while [ -h "$PRG" ]; do 9 | ls=`ls -ld "$PRG"` 10 | link=`expr "$ls" : '.*-> \(.*\)$'` 11 | if expr "$link" : '/.*' > /dev/null; then 12 | PRG="$link" 13 | else 14 | PRG=`dirname "$PRG"`/"$link" 15 | fi 16 | done 17 | 18 | # Get standard environment variables 19 | PRGDIR=`dirname "$PRG"` 20 | 21 | PROJECT_DIR=`cd "$PRGDIR/.." >/dev/null; pwd` 22 | echo PROJECT_DIR=$PROJECT_DIR 23 | 24 | CLASSPATH="$CLASSHPATH:$PROJECT_DIR/conf" 25 | 26 | for jar in "$PROJECT_DIR/lib"/*.jar; do 27 | CLASSPATH="$CLASSPATH:$jar" 28 | done 29 | echo CLASSPATH=$CLASSPATH 30 | 31 | JVMARGS="${JVMARGS} -Dproject_dir=${PROJECT_DIR}" 32 | echo JVMARGS=$JVMARGS 33 | 34 | usage() { 35 | echo >&2 "usage: $PRG [args]" 36 | echo 'Valid commands: start, stop' 37 | exit 1 38 | } 39 | 40 | start() { 41 | JAVA=${JAVA-'java'} 42 | exec $JAVA $JVMARGS -classpath "$CLASSPATH" $mainClass "$@" & 43 | echo $! > main.pid 44 | } 45 | 46 | stop() { 47 | kill `cat main.pid` > /dev/null 48 | } 49 | 50 | case $1 in 51 | (start) 52 | shift 53 | start $@ 54 | ;; 55 | (stop) 56 | stop 57 | ;; 58 | (restart) 59 | stop 60 | shift 61 | start $@ 62 | ;; 63 | (*) 64 | echo >&2 "$PRG: error: unknown command '$1'" 65 | usage 66 | ;; 67 | esac 68 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/bin/firstpage_timer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Starting ..." 4 | 5 | bin/ctl.sh start firstPageRun 6 | 7 | while (true) 8 | do 9 | sleep 5m 10 | bin/ctl.sh start firstPageRun 11 | done 12 | 13 | echo "Finishing ..." 14 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/bin/special_timer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Starting ..." 4 | 5 | bin/ctl.sh start specialTopicRun 6 | 7 | while (true) 8 | do 9 | sleep 30m 10 | bin/ctl.sh start specialTopicRun 11 | done 12 | 13 | echo "Finishing ..." -------------------------------------------------------------------------------- /sentiment-solr/src/main/bin/timer_redis2solr.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Starting ..." 4 | 5 | bin/ctl.sh start importRedisToSC 6 | 7 | while (true) 8 | do 9 | sleep 1h 10 | bin/ctl.sh restart importRedisToSC 11 | done 12 | 13 | echo "Finishing ..." 14 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/bin/timer_remove_expired.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Starting ..." 4 | 5 | bin/ctl.sh start removeExpiredData 6 | 7 | while (true) 8 | do 9 | sleep 1d 10 | bin/ctl.sh start removeExpiredData 11 | done 12 | 13 | echo "Finishing ..." 14 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/java/zx/soft/sent/solr/demo/CacheDemo.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.demo; 2 | 3 | import java.util.Map; 4 | import java.util.Map.Entry; 5 | 6 | import zx.soft.redis.client.cache.Cache; 7 | import zx.soft.redis.client.cache.RedisCache; 8 | import zx.soft.sent.dao.common.SentimentConstant; 9 | 10 | public class CacheDemo { 11 | 12 | public static void main(String[] args) { 13 | 14 | Cache cache = new RedisCache("192.168.32.19,192.168.32.20", 6379, "zxsoft"); 15 | Map sourceIdAndNames = cache.hgetAll(SentimentConstant.SITE_MAP); 16 | cache.close(); 17 | System.out.println(sourceIdAndNames.size()); 18 | 19 | Cache cache1 = new RedisCache("192.168.32.19", 16379, "zxsoft"); 20 | for (Entry sourceIdAndName : sourceIdAndNames.entrySet()) { 21 | System.out.println(sourceIdAndName.getKey() + ":" + sourceIdAndName.getValue()); 22 | cache1.hset(SentimentConstant.SITE_MAP, sourceIdAndName.getKey(), sourceIdAndName.getValue()); 23 | } 24 | cache1.close(); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/java/zx/soft/sent/solr/demo/InsertSortDemo.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.demo; 2 | 3 | import zx.soft.utils.sort.InsertSort; 4 | 5 | public class InsertSortDemo { 6 | 7 | public static void main(String[] args) { 8 | 9 | String[] IN_STRS_ARR = { "111=5", "222=1", "333=9", "444=2", "555=6" }; 10 | 11 | String[] table = new String[IN_STRS_ARR.length]; 12 | for (int i = 0; i < table.length; i++) { 13 | table[i] = "0=0"; 14 | } 15 | for (int i = 0; i < table.length; i++) { 16 | table = InsertSort.toptable(table, IN_STRS_ARR[i]); 17 | } 18 | String[] keys = InsertSort.trans(table); 19 | for (int i = 0; i < table.length; i++) { 20 | System.out.println(table[i]); 21 | System.out.println(keys[i]); 22 | } 23 | 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/java/zx/soft/sent/solr/demo/LogerDemo.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.demo; 2 | 3 | import java.io.UnsupportedEncodingException; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | public class LogerDemo { 9 | 10 | private static Logger logger = LoggerFactory.getLogger(LogerDemo.class); 11 | 12 | public static void main(String[] args) throws UnsupportedEncodingException { 13 | 14 | logger.info("Start logging ..."); 15 | logger.info("Finish logging ..."); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/java/zx/soft/sent/solr/demo/NegativeClassifyDemo.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.demo; 2 | 3 | import zx.soft.negative.sentiment.core.NegativeClassify; 4 | 5 | public class NegativeClassifyDemo { 6 | 7 | public static void main(String[] args) { 8 | 9 | NegativeClassify negativeClassify = new NegativeClassify(); 10 | String text = "综合消息,近日,湖南湘潭县妇幼保健院一产妇死亡,经媒体报道引发关注。据“央视新闻”消息,湘潭县卫生局称,胎儿出生后产妇出现呕吐呛咳," 11 | + "院方立即抢救,但产妇因羊水栓塞引发多器官功能衰竭,抢救无效死亡。目前湘潭县卫生局、司法局已介入调查。据此前媒体报道,10日上午11点,医生" 12 | + "给张女士做了产前检查,胎位正常,由于胎儿较重,医生建议家属做剖腹产。随后,张女士被推进五楼的手术室。12点05分,手术室护士告诉家属,产妇顺" 13 | + "利产下宝宝。半个小时后,护士通知守在手术室外的刘先生,称产妇出现大出血情况,刘先生说,1个小时后,护士又通知他,称出血情况没止住,要其赶紧" 14 | + "去买止血药。下午5点左右,一名护士通知刘先生,称产妇大出血,现在必须切除子宫,需要签字。晚上9点,刘先生家人听到有人在议论,说产妇已经死了。" 15 | + "情急之下的刘先生上前使劲拍打手术室的大门,这时,出来一个自称是代理院长的人,说产妇仍在抢救,有脉搏。刘先生只得在外继续焦急地等待。直到" 16 | + "晚上11点,刘先生再也按捺不住,再次敲门,询问护士情况,可此时,手术室内没有任何人回答他,因为手术室的门被反锁,刘先生不得不撬开手术室的大门" 17 | + "。可进去之后,刘先生看到的了让他难以置信的一幕:妻子赤身裸体躺在手术台上,满口鲜血,眼睛里还含着泪水,可却再也没有了呼吸。而本应该在抢救" 18 | + "的医生和护士,却全体失踪了,房间里只有一些不明身份的男子在吃着槟榔,抽着烟。针对此事,湘潭县卫生局称,8月10日12点05分胎儿出生后,产妇出现" 19 | + "呕吐呛咳,院方立即抢救,但产妇因羊水栓塞引发多器官功能衰竭,抢救无效于21点30分死亡。目前湘潭县卫生局、司法局已介入调查。"; 20 | System.out.println(negativeClassify.getTextScore(text)); 21 | negativeClassify.cleanup(); 22 | 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/java/zx/soft/sent/solr/domain/FacetDateParams.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.domain; 2 | 3 | /** 4 | * 按日期分类查询参数 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | public class FacetDateParams { 10 | 11 | private String q = "*:*"; 12 | private String facetDate; 13 | private String facetDateStart; 14 | private String facetDateEnd; 15 | private String facetDateGap = "%2B1DAY"; // 默认是一天 16 | 17 | public String getQ() { 18 | return q; 19 | } 20 | 21 | public void setQ(String q) { 22 | this.q = q; 23 | } 24 | 25 | public String getFacetDate() { 26 | return facetDate; 27 | } 28 | 29 | public void setFacetDate(String facetDate) { 30 | this.facetDate = facetDate; 31 | } 32 | 33 | public String getFacetDateStart() { 34 | return facetDateStart; 35 | } 36 | 37 | public void setFacetDateStart(String facetDateStart) { 38 | this.facetDateStart = facetDateStart; 39 | } 40 | 41 | public String getFacetDateEnd() { 42 | return facetDateEnd; 43 | } 44 | 45 | public void setFacetDateEnd(String facetDateEnd) { 46 | this.facetDateEnd = facetDateEnd; 47 | } 48 | 49 | public String getFacetDateGap() { 50 | return facetDateGap; 51 | } 52 | 53 | public void setFacetDateGap(String facetDateGap) { 54 | this.facetDateGap = facetDateGap; 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/java/zx/soft/sent/solr/domain/FacetDateResult.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.domain; 2 | 3 | import java.util.HashMap; 4 | 5 | /** 6 | * 按时间分类查询结果 7 | * 8 | * @author wanggang 9 | * 10 | */ 11 | public class FacetDateResult { 12 | 13 | private String start; 14 | private String end; 15 | private String gap; 16 | private HashMap dateCounts = new HashMap<>(); 17 | 18 | public String getStart() { 19 | return start; 20 | } 21 | 22 | public void setStart(String start) { 23 | this.start = start; 24 | } 25 | 26 | public String getEnd() { 27 | return end; 28 | } 29 | 30 | public void setEnd(String end) { 31 | this.end = end; 32 | } 33 | 34 | public String getGap() { 35 | return gap; 36 | } 37 | 38 | public void setGap(String gap) { 39 | this.gap = gap; 40 | } 41 | 42 | public HashMap getDateCounts() { 43 | return dateCounts; 44 | } 45 | 46 | public void setDateCounts(HashMap dateCounts) { 47 | this.dateCounts = dateCounts; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/java/zx/soft/sent/solr/domain/SimpleFacetInfo.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.domain; 2 | 3 | import java.util.HashMap; 4 | 5 | /** 6 | * 简单的分片信息 7 | * 8 | * @author wanggang 9 | * 10 | */ 11 | public class SimpleFacetInfo { 12 | 13 | private String name; 14 | private HashMap values = new HashMap<>(); 15 | 16 | public String getName() { 17 | return name; 18 | } 19 | 20 | public void setName(String name) { 21 | this.name = name; 22 | } 23 | 24 | public HashMap getValues() { 25 | return values; 26 | } 27 | 28 | public void setValues(HashMap values) { 29 | this.values = values; 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/java/zx/soft/sent/solr/driver/SentSolrDriver.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.driver; 2 | 3 | import zx.soft.sent.solr.allinternet.TaskUpdate; 4 | import zx.soft.sent.solr.firstpage.FirstPageHarmfulRun; 5 | import zx.soft.sent.solr.firstpage.FirstPageRun; 6 | import zx.soft.sent.solr.index.ImportRedisToSC; 7 | import zx.soft.sent.solr.query.OracleToRedis; 8 | import zx.soft.sent.solr.query.RemoveExpiredData; 9 | import zx.soft.sent.solr.special.SpecialTopicRun; 10 | import zx.soft.sent.solr.utils.RedisMQTest; 11 | import zx.soft.utils.driver.ProgramDriver; 12 | 13 | /** 14 | * 驱动类 15 | * 16 | * @author wanggang 17 | * 18 | */ 19 | public class SentSolrDriver { 20 | 21 | /** 22 | * 主函数 23 | */ 24 | public static void main(String[] args) { 25 | 26 | int exitCode = -1; 27 | ProgramDriver pgd = new ProgramDriver(); 28 | try { 29 | // 在hefei09机器上运行 30 | pgd.addClass("oracleToRedis", OracleToRedis.class, "将站点数据定时导入Redis中（默认是每小时）"); 31 | // 在hefei07机器上运行 32 | pgd.addClass("specialTopicRun", SpecialTopicRun.class, "OA专题数据统计——临时分析"); 33 | // 在hefei07机器上运行 34 | pgd.addClass("firstPageRun", FirstPageRun.class, "OA首页数据统计——临时分析"); 35 | // 在hefei07机器上运行 36 | pgd.addClass("firstPageHarmfulRun", FirstPageHarmfulRun.class, "OA首页负面信息数据定时统计"); 37 | // 在hefei10机器上运行多个进程 38 | pgd.addClass("importRedisToSC", ImportRedisToSC.class, "将Redis中的数据所引到SolrCloud"); 39 | // 在hefei08机器上运行 40 | pgd.addClass("removeExpiredData", RemoveExpiredData.class, "定时每周删除过期舆情数据和Redis缓存数据"); 41 | // 在hefei10机器上运行 42 | pgd.addClass("taskUpdate", TaskUpdate.class, "全网任务信息查询结果存储缓存信息"); 43 | // 测试 44 | pgd.addClass("redisMQTest", RedisMQTest.class, "测试Redis"); 45 | pgd.driver(args); 46 | // Success 47 | exitCode = 0; 48 | } catch (Throwable e) { 49 | throw new RuntimeException(e); 50 | } 51 | 52 | System.exit(exitCode); 53 | 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/java/zx/soft/sent/solr/ecxception/SpiderSearchException.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.ecxception; 2 | 3 | /** 4 | * 舆情搜索异常处理 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | public class SpiderSearchException extends RuntimeException { 10 | 11 | private static final long serialVersionUID = 4719608449546004181L; 12 | 13 | public SpiderSearchException(String message, Throwable cause) { 14 | super(message, cause); 15 | } 16 | 17 | public SpiderSearchException(String message) { 18 | super(message); 19 | } 20 | 21 | public SpiderSearchException(Throwable cause) { 22 | super(cause); 23 | } 24 | 25 | public Throwable getRootCause() { 26 | Throwable t = this; 27 | while (true) { 28 | Throwable cause = t.getCause(); 29 | if (cause != null) { 30 | t = cause; 31 | } else { 32 | break; 33 | } 34 | } 35 | return t; 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/java/zx/soft/sent/solr/special/PieChart.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.special; 2 | 3 | import java.util.HashMap; 4 | 5 | import zx.soft.utils.json.JsonUtils; 6 | 7 | /** 8 | * 與请专题统计数据——饼状图 9 | * 10 | * @author wanggang 11 | * 12 | */ 13 | public class PieChart { 14 | 15 | /* 16 | * 专题信息 17 | */ 18 | private SpecialInfo specialInfo; 19 | 20 | /* 21 | * 各类数据统计: 22 | * 0-其他,1-资讯类,2-论坛类,3-微博类,4-博客类,5-QQ群类, 23 | * 6-元搜索类,7-回复信息（其他），8-邮件类,9-图片识别类 24 | */ 25 | private HashMap platformCount = new HashMap<>(); 26 | 27 | public SpecialInfo getSpecialInfo() { 28 | return specialInfo; 29 | } 30 | 31 | public void setSpecialInfo(SpecialInfo specialInfo) { 32 | this.specialInfo = specialInfo; 33 | } 34 | 35 | public HashMap getPlatformCount() { 36 | return platformCount; 37 | } 38 | 39 | public void setPlatformCount(HashMap platformCount) { 40 | this.platformCount = platformCount; 41 | } 42 | 43 | public static void main(String[] args) { 44 | PieChart pieChart = new PieChart(); 45 | pieChart.setSpecialInfo(new SpecialInfo("405", "冰桶挑战")); 46 | pieChart.getPlatformCount().put("0", 3200L); 47 | pieChart.getPlatformCount().put("1", 12265L); 48 | pieChart.getPlatformCount().put("2", 568L); 49 | pieChart.getPlatformCount().put("3", 3866L); 50 | pieChart.getPlatformCount().put("4", 1235L); 51 | pieChart.getPlatformCount().put("5", 86L); 52 | pieChart.getPlatformCount().put("6", 793L); 53 | pieChart.getPlatformCount().put("7", 17889L); 54 | pieChart.getPlatformCount().put("8", 3556L); 55 | pieChart.getPlatformCount().put("9", 30L); 56 | System.out.println(JsonUtils.toJsonWithoutPretty(pieChart)); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/java/zx/soft/sent/solr/special/SpecialInfo.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.special; 2 | 3 | /** 4 | * 专题数据统计——专题信息 5 | * 6 | * @author wanggang 7 | * 8 | */ 9 | public class SpecialInfo { 10 | 11 | private String identify; // 专题ID 12 | private String specialName; // 专题名称 13 | 14 | public SpecialInfo() { 15 | // 16 | } 17 | 18 | public SpecialInfo(String identify, String specialName) { 19 | this.identify = identify; 20 | this.specialName = specialName; 21 | } 22 | 23 | public String getIdentify() { 24 | return identify; 25 | } 26 | 27 | public void setIdentify(String identify) { 28 | this.identify = identify; 29 | } 30 | 31 | public String getSpecialName() { 32 | return specialName; 33 | } 34 | 35 | public void setSpecialName(String specialName) { 36 | this.specialName = specialName; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/java/zx/soft/sent/solr/special/TrendChart.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.special; 2 | 3 | import java.util.HashMap; 4 | 5 | import zx.soft.utils.json.JsonUtils; 6 | 7 | /** 8 | * 专题数据统计——趋势图 9 | * 10 | * @author wanggang 11 | * 12 | */ 13 | public class TrendChart { 14 | 15 | /* 16 | * 专题信息 17 | */ 18 | private SpecialInfo specialInfo; 19 | 20 | /* 21 | * 每天对应的数据量 22 | */ 23 | private HashMap countByDay = new HashMap<>(); 24 | 25 | public SpecialInfo getSpecialInfo() { 26 | return specialInfo; 27 | } 28 | 29 | public void setSpecialInfo(SpecialInfo specialInfo) { 30 | this.specialInfo = specialInfo; 31 | } 32 | 33 | public HashMap getCountByDay() { 34 | return countByDay; 35 | } 36 | 37 | public void setCountByDay(HashMap countByDay) { 38 | this.countByDay = countByDay; 39 | } 40 | 41 | public static void main(String[] args) { 42 | TrendChart trandChart = new TrendChart(); 43 | trandChart.setSpecialInfo(new SpecialInfo("405", "冰桶挑战")); 44 | trandChart.getCountByDay().put("2014-04-10", 1292728L); 45 | trandChart.getCountByDay().put("2014-04-11", 1512505L); 46 | trandChart.getCountByDay().put("2014-04-12", 80221L); 47 | trandChart.getCountByDay().put("2014-04-13", 1430429L); 48 | System.out.println(JsonUtils.toJsonWithoutPretty(trandChart)); 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/java/zx/soft/sent/solr/utils/HarmfulInfoUtil.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.utils; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashSet; 5 | import java.util.List; 6 | 7 | import org.apache.solr.common.SolrDocument; 8 | 9 | import zx.soft.negative.sentiment.core.NegativeClassify; 10 | import zx.soft.utils.checksum.CheckSumUtils; 11 | import zx.soft.utils.sort.InsertSort; 12 | 13 | public class HarmfulInfoUtil { 14 | 15 | private static final NegativeClassify negativeClassify = new NegativeClassify(); 16 | 17 | /** 18 | * 排序计算，得到前N负面信息 19 | */ 20 | public static List getTopNNegativeRecords(List records, int N) { 21 | List result = new ArrayList<>(); 22 | HashSet md5s = new HashSet<>(); 23 | String[] insertTables = new String[records.size()]; 24 | for (int i = 0; i < records.size(); i++) { 25 | String str = ""; 26 | if (records.get(i).get("title") != null) { 27 | str += records.get(i).get("title").toString(); 28 | } 29 | if (records.get(i).get("content") != null) { 30 | str += records.get(i).get("content").toString(); 31 | } 32 | insertTables[i] = i + "=" + (int) (negativeClassify.getTextScore(str)); 33 | } 34 | String[] table = new String[records.size()]; 35 | for (int i = 0; i < table.length; i++) { 36 | table[i] = "0=0"; 37 | } 38 | for (int i = 0; i < table.length; i++) { 39 | table = InsertSort.toptable(table, insertTables[i]); 40 | } 41 | String[] keyValue = null; 42 | for (int i = 0; result.size() < Math.min(table.length, N) && i < table.length; i++) { 43 | keyValue = table[i].split("="); 44 | SolrDocument doc = records.get(Integer.parseInt(keyValue[0])); 45 | if (md5s.contains(CheckSumUtils.getMD5(doc.getFieldValue("content").toString()))) { 46 | continue; 47 | } 48 | md5s.add(CheckSumUtils.getMD5(doc.getFieldValue("content").toString())); 49 | result.add(doc); 50 | } 51 | 52 | return result; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/java/zx/soft/sent/solr/utils/Site2Json.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.utils; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.File; 5 | import java.io.FileReader; 6 | import java.io.IOException; 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | 10 | import zx.soft.utils.chars.JavaPattern; 11 | import zx.soft.utils.json.JsonUtils; 12 | 13 | public class Site2Json { 14 | 15 | public static void main(String[] args) { 16 | 17 | List data = new ArrayList<>(); 18 | 19 | try (BufferedReader br = new BufferedReader(new FileReader(new File("st.csv")));) { 20 | String str; 21 | String[] strs; 22 | while ((str = br.readLine()) != null) { 23 | str = str.substring(1, str.length() - 1); 24 | strs = str.split(":"); 25 | String ids = ""; 26 | for (int i = 1; i < strs.length - 1; i++) { 27 | // System.out.print(strs[i].split("#")[0] + " "); 28 | if (strs[i].split("#")[0].length() != 0) { 29 | data.add(strs[i].split("#")[0]); 30 | ids += strs[i].split("#")[0] + ","; 31 | } 32 | } 33 | // System.out.print(strs[strs.length - 1] + " "); 34 | if (strs[strs.length - 1].length() != 0) { 35 | data.add(strs[strs.length - 1]); 36 | ids += strs[strs.length - 1] + ","; 37 | } 38 | ids = ids.substring(0, ids.length() - 1); 39 | data.add(ids); 40 | } 41 | 42 | } catch (IOException e) { 43 | e.printStackTrace(); 44 | } 45 | 46 | System.out.println(JsonUtils.toJson(data)); 47 | 48 | for (String d : data) { 49 | if (!JavaPattern.isAllNum(d.replaceAll(",", ""))) { 50 | System.out.println(d); 51 | } 52 | } 53 | 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/resources/README.md: -------------------------------------------------------------------------------- 1 | 2 | ### Redis作用说明 3 | 4 | #### 主从配置集群 5 | 6 | > 两台机器，其中，redis.rp.master和redis.rp.slave 7 | 8 | `master`:hefei09,用于读数据 9 | 10 | `slave`: hefei10,用于读数据 11 | 12 | > 主要用途： 13 | 14 | `用于站点信息存储K-V`: sent.site.groups 15 | 16 | `用于站点Id和站点名存储K-V`: sent:site:map 17 | 18 | #### 消息队列节点 19 | 20 | > 一台机器，hefei10，redis.mq.server 21 | 22 | `索引缓存数据消息队列List`: sent.cache.records 23 | 24 | `用于数据库写入去重`: sent:key:inserted 25 | 26 | #### 有效期缓存节点 27 | 28 | > 一台机器，hefei08，redis.ft.server 29 | 30 | `用于缓存首次抓取时间firsttime`：时效性为7天 31 | 32 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/resources/cache-config.properties: -------------------------------------------------------------------------------- 1 | 2 | ##### Mater-Slave (For Replicating Record-Key,Index-Data Caching) ##### 3 | # Product 4 | redis.rp.master=192.168.32.19 5 | redis.rp.slave=192.168.32.20 6 | # Testing 7 | #redis.rp.master=192.168.31.20 8 | #redis.rp.slave=192.168.31.21 9 | 10 | ##### Message-Queue (For Index-Data Caching) ##### 11 | # Product 12 | redis.mq.server=192.168.32.20 13 | # Testing 14 | #redis.mq.server=192.168.31.XX 15 | 16 | ##### Expired-Queue (For Firsttime Caching) ##### 17 | # Product 18 | redis.ft.server=192.168.32.18 19 | # Testing 20 | #redis.ft.server=192.168.31.XX 21 | 22 | ##### Port ##### 23 | redis.rp.port=16379 24 | redis.mq.port=6379 25 | redis.ft.port=6379 26 | 27 | ##### Password ##### 28 | redis.password=zxsoft 29 | 30 | ## Batch Count to Solr 31 | batch.count=10000 -------------------------------------------------------------------------------- /sentiment-solr/src/main/resources/cate_dict/negative_bc: -------------------------------------------------------------------------------- 1 | 造谣窒息事故暴行毁容致死诬陷猥亵砍人诈骗案反动被害逃逸违法罪犯刺死爆炸物施暴开枪迷幻药毒手炸通缉令砍伤毒打砍杀辱骂偷窃窒息而死违法行为灭口杀灭非法滋事下毒手交警刑事案件贪污腐败恐怖行动伸冤害人强奸淫秽强暴炸药事件起火毒死跳楼尸体恶毒暴打变态新疆人毒害受害人草菅人命威吓通缉侵占霸占打架斗殴砸死报案恶名拆迁房死了假释暴力冤枉灾区嫌疑人暴动腐败分子禁毒仇恨死亡淫笑恐怖主义撞死死人侵害犯罪猥亵作乱审判打劫得罪奸商强拆淫荡申冤虐杀放火勒死追捕刑拘焚烧报复骚扰杀人狂流血暴卒刑事拘留逃犯毒品虐待反腐败冤假冤案歹徒惨杀纵火案纵火瘟疫诈骗罪恩仇世仇奸淫死打死大爆炸骗子行窃禽流感心狠手辣投毒暴民毒行骗出事妖言惑众草芥人命死者流感伤亡分赃杀人犯厌世凶杀判决戒毒陷害包庇潜逃举报杀死作案工具尸抢劫行政拘留肇事者自焚性侵作案贪污杀害残暴谋杀扣押监禁粗暴砍死兽性大发获刑糟蹋身亡医疗事故拘留所报警交通事故内乱出血拘留炸弹肢解赃物毒杀残忍拆迁恐吓侮辱网警举报人监狱行贿吸毒毒性病毒越狱杀人罪堕落车祸噩耗打人扫毒逼供洗脑色狼威胁寻死肇事藏独警备区逃亡流氓炸裂冲突骗人下毒色魔镇压暴徒非法拘禁恐怖分子禽兽惨死烧毁乱砍突发事件轮奸骂我诈骗贿赂受贿恐怖袭击假币假钱假钞闹事制服冰毒擒获抢劫罪偷盗被盗失窃纠纷女尸失踪线索涉嫌色情持刀抓获归案同案赃款被害人击毙击伤搜查围捕血案圣战爆炸声爆炸枪走私涉枪涉赌绑架寻衅枪支子弹枪弹举报线索上当骗走被骗谎称受骗骗取被打因涉嫌故意伤害案发后 -------------------------------------------------------------------------------- /sentiment-solr/src/main/resources/data_db.properties: -------------------------------------------------------------------------------- 1 | 2 | ##### Database ##### 3 | # Product 4 | sent.db.url=jdbc:mysql://192.168.32.17:3306/sentiment_records?useUnicode=true&characterEncoding=utf-8 5 | # Testing 6 | #sent.db.url=jdbc:mysql://192.168.31.23:3306/sentiment_records?useUnicode=true&characterEncoding=utf-8 7 | #sent.db.url=jdbc:mysql://192.168.31.23:3306/sentiment_oa?useUnicode=true&characterEncoding=utf-8 8 | 9 | ##### Password ##### 10 | sent.db.username=sentimentrecords 11 | sent.db.password=zn/iACQ/Xh19+W6Ot2MghQ== 12 | 13 | ##### Localhost ##### 14 | #sent.db.url=jdbc:mysql://localhost:3306/sentiment_records?useUnicode=true&characterEncoding=utf-8 15 | #sent.db.username=root 16 | #sent.db.password=root -------------------------------------------------------------------------------- /sentiment-solr/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 8 | 9 | 10 | 11 | INFO 12 | ACCEPT 13 | DENY 14 | 15 | 16 | 17 | 19 | logs/sentiment-solr.log 20 | 21 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 22 | 23 | 24 | 25 | INFO 26 | 27 | 28 | logs/sentiment-solr.log.%d{yyyy-MM-dd}.gz 29 | 30 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /sentiment-solr/src/main/resources/oracle_db.properties: -------------------------------------------------------------------------------- 1 | 2 | ##### Product ##### 3 | ## HuaiNan 192.168.32.200 4 | ## Shengting 192.168.32.36 5 | db_url=jdbc:oracle:thin:@192.168.32.36:1521:yqjk 6 | ##### Testing ##### 7 | #db_url=jdbc:oracle:thin:@192.168.31.22:1521:yqjk 8 | db_username=yqjk 9 | db_password=yqjk#olap#2014 -------------------------------------------------------------------------------- /sentiment-solr/src/main/resources/solr_params.properties: -------------------------------------------------------------------------------- 1 | 2 | ##### SolrCloud ##### 3 | collection=sentiment 4 | zookeeper_connect_timeout=50000 5 | zookeeper_client_timeout=50000 6 | #zookeeper_cloud=leader:2181 7 | # Product 8 | zookeeper_cloud=192.168.32.11:2181,192.168.32.12:2181,192.168.32.13:2181,192.168.32.14:2181,192.168.32.15:2181,192.168.32.16:2181 9 | # Testing 10 | #zookeeper_cloud=192.168.31.11:2181,192.168.31.12:2181,192.168.31.20:2181,192.168.31.21:2181,192.168.31.26:2181,192.168.31.27:2181 11 | 12 | ##### Fetch Size ##### 13 | fetch_size=20000 -------------------------------------------------------------------------------- /sentiment-solr/src/test/java/zx/soft/sent/solr/search/FacetSearchTest.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.search; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | 5 | import org.junit.Test; 6 | 7 | import zx.soft.sent.solr.domain.FacetDateParams; 8 | import zx.soft.sent.solr.query.FacetSearch; 9 | 10 | public class FacetSearchTest { 11 | 12 | @Test 13 | public void testGetURL() { 14 | FacetDateParams fdp = new FacetDateParams(); 15 | fdp.setQ("*:*"); 16 | fdp.setFacetDate("timestamp"); 17 | fdp.setFacetDateStart("NOW-7DAYS"); 18 | fdp.setFacetDateEnd("NOW"); 19 | fdp.setFacetDateGap("%2B1DAY"); 20 | assertEquals("http://192.168.32.11:8983/solr/sentiment/select?wt=json&indent=true&facet=true&rows=0&q=*:*&" 21 | + "q.op=AND&facet.date=timestamp&facet.date.start=NOW-7DAYS&facet.date.end=NOW&facet.date.gap=%2B1DAY", 22 | FacetSearch.getURL(fdp)); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /sentiment-solr/src/test/java/zx/soft/sent/solr/search/SearchingDataTest.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.search; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | 5 | import org.junit.Test; 6 | 7 | import zx.soft.sent.solr.query.QueryCore; 8 | 9 | public class SearchingDataTest { 10 | 11 | @Test 12 | public void testTransFq() { 13 | 14 | String fqs = QueryCore.transFq("nickname:罗永浩;content:锤子"); 15 | assertEquals(fqs, "nickname:罗永浩;content:锤子"); 16 | 17 | fqs = QueryCore.transFq("nickname:罗永浩;-content:锤子"); 18 | assertEquals(fqs, "nickname:罗永浩;-content:锤子"); 19 | 20 | fqs = QueryCore.transFq("nickname:罗永浩,北京"); 21 | assertEquals(fqs, "nickname:罗永浩 OR nickname:北京"); 22 | 23 | fqs = QueryCore.transFq("-content:锤子,手机"); 24 | assertEquals(fqs, "-content:锤子 AND -content:手机"); 25 | 26 | fqs = QueryCore.transFq("a:b,c,d"); 27 | assertEquals(fqs, "a:b OR a:c OR a:d"); 28 | 29 | fqs = QueryCore.transFq("-a:b,c,d"); 30 | assertEquals(fqs, "-a:b AND -a:c AND -a:d"); 31 | 32 | fqs = QueryCore.transFq("timestamp:[2014-4-22T13:39:26Z TO 2014-4-29T13:39:26Z]"); 33 | assertEquals(fqs, "timestamp:[2014-4-22T13:39:26Z TO 2014-4-29T13:39:26Z]"); 34 | 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /sentiment-solr/src/test/java/zx/soft/sent/solr/utils/RedisCacheExpiredTest.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.solr.utils; 2 | 3 | import org.junit.Ignore; 4 | 5 | public class RedisCacheExpiredTest { 6 | 7 | @Ignore 8 | public void testExpired() { 9 | RedisCacheExpired pool = new RedisCacheExpired(60); 10 | for (int i = 0; i < 30; i++) { 11 | pool.addRecord("test" + i, i + ""); 12 | try { 13 | Thread.sleep(1_000); 14 | } catch (InterruptedException e) { 15 | e.printStackTrace(); 16 | } 17 | } 18 | 19 | pool.close(); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /sentiment-solr/src/test/resources/cache-config.properties: -------------------------------------------------------------------------------- 1 | 2 | ##### Mater-Slave (For Replicating Record-Key) ##### 3 | # Product 4 | #redis.rp.master=192.168.32.19 5 | #redis.rp.slave=192.168.32.20 6 | # Testing 7 | redis.rp.master=192.168.31.20 8 | redis.rp.slave=192.168.31.21 9 | 10 | ##### Message-Queue (for Index-Data and Firsttime Caching) ##### 11 | # Product 12 | #redis.mq.server=192.168.32.18 13 | # Testing 14 | redis.mq.server=192.168.31.12 15 | 16 | ##### Port ##### 17 | redis.rp.port=16379 18 | redis.mq.port=6379 19 | 20 | ##### Password ##### 21 | redis.password=zxsoft -------------------------------------------------------------------------------- /sentiment-solr/src/test/resources/data_db.properties: -------------------------------------------------------------------------------- 1 | 2 | ##### Database ##### 3 | # Product 4 | #sent.db.url=jdbc:mysql://192.168.32.17:3306/sentiment_records?useUnicode=true&characterEncoding=utf-8 5 | # Testing 6 | sent.db.url=jdbc:mysql://192.168.31.23:3306/sentiment_records?useUnicode=true&characterEncoding=utf-8 7 | 8 | ##### Password ##### 9 | sent.db.username=sentimentrecords 10 | sent.db.password=zn/iACQ/Xh19+W6Ot2MghQ== 11 | 12 | ##### Localhost ##### 13 | #sent.db.url=jdbc:mysql://localhost:3306/sentiment_records?useUnicode=true&characterEncoding=utf-8 14 | #sent.db.username=root 15 | #sent.db.password=root -------------------------------------------------------------------------------- /sentiment-solr/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{MMdd.HHmmss.SSS} [%-20t] [%-5p] [%-20c] [L:%-3L] - %m%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /sentiment-solr/src/test/resources/solr_params.properties: -------------------------------------------------------------------------------- 1 | 2 | ##### SolrCloud ##### 3 | collection=sentiment 4 | zookeeper_connect_timeout=50000 5 | zookeeper_client_timeout=50000 6 | #zookeeper_cloud=leader:2181 7 | # Product 8 | #zookeeper_cloud=192.168.32.11:2181,192.168.32.12:2181,192.168.32.13:2181,192.168.32.14:2181,192.168.32.15:2181,192.168.32.16:2181 9 | # Testing 10 | zookeeper_cloud=192.168.31.11:2181,192.168.31.12:2181,192.168.31.20:2181,192.168.31.21:2181,192.168.31.26:2181,192.168.31.27:2181 11 | 12 | ##### Fetch Size ##### 13 | fetch_size=20000 -------------------------------------------------------------------------------- /sentiment-solr/src/test/resources/weibo/WBLR1: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 |

一个连街头小偷都不敢呵斥的民族，却有勇气高呼灭了小日本；一个连活着的人都漠不关心的民族，却让我们不要忘记死去的人；一个连农民的生死都置之不理的国度，却说现在是和谐社会……

5 |
6 | 搜狐新闻| 辽宁抚顺洪灾63人遇难 今日全市哀悼(图)(13108人参与) 7 |
-------------------------------------------------------------------------------- /sentiment-solr/src/test/resources/weibo/WBLR2: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 |

好吧，我们知道了。

5 |
6 | 搜狐新闻| 辽宁抚顺洪灾63人遇难 今日全市哀悼(图)(13107人参与) 7 |
-------------------------------------------------------------------------------- /sentiment-solr/src/test/resources/weibo/WBLR3: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 | 5 |

【辽宁抚顺洪灾死亡人数达76人】失踪101人找到13人，全部遇难；目前仍有88人失踪。 http://t.itc.cn/mjaGW @搜狐新闻客户端

6 | 7 | 8 | -------------------------------------------------------------------------------- /sentiment-solr/src/test/resources/weibo/WBLR4: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 |

以前比较相信网络，现在也不敢了，哀哉！

5 |
6 | 搜狐新闻| 辽宁抚顺洪灾63人遇难 今日全市哀悼(图)(11110人参与) 7 |
8 | -------------------------------------------------------------------------------- /sentiment-spider/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## 使用说明 3 | 4 | ### 使用域名 5 | 6 | > api.weibo.com:80 7 | 8 | ### 使用IP 9 | 10 | > 180.149.135.176:80 11 | 12 | > 180.149.135.230:80 -------------------------------------------------------------------------------- /sentiment-spider/proxy/proxy-ips.txt: -------------------------------------------------------------------------------- 1 | 219.93.178.162:3128 2 | 219.93.178.162:3128 3 | 219.93.178.162:3128 4 | 219.93.178.162:3128 -------------------------------------------------------------------------------- /sentiment-spider/src/main/assembly/distribution.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | distribution 7 | 8 | tar.gz 9 | 10 | ${project.artifactId} 11 | 12 | 13 | src/main/resources 14 | 15 | cache-config.properties 16 | super.properties 17 | sina-api.properties 18 | logback.xml 19 | 20 | /conf 21 | true 22 | 23 | 24 | src/main/bin 25 | 26 | * 27 | 28 | /bin 29 | 0755 30 | 31 | 32 | 33 | 34 | /lib 35 | 36 | 37 | -------------------------------------------------------------------------------- /sentiment-spider/src/main/bin/ctl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mainClass=zx.soft.sent.spider.driver.SpiderDriver 4 | 5 | # resolve links - $0 may be a softlink 6 | PRG="$0" 7 | 8 | while [ -h "$PRG" ]; do 9 | ls=`ls -ld "$PRG"` 10 | link=`expr "$ls" : '.*-> \(.*\)$'` 11 | if expr "$link" : '/.*' > /dev/null; then 12 | PRG="$link" 13 | else 14 | PRG=`dirname "$PRG"`/"$link" 15 | fi 16 | done 17 | 18 | # Get standard environment variables 19 | PRGDIR=`dirname "$PRG"` 20 | 21 | PROJECT_DIR=`cd "$PRGDIR/.." >/dev/null; pwd` 22 | echo PROJECT_DIR=$PROJECT_DIR 23 | 24 | CLASSPATH="$CLASSHPATH:$PROJECT_DIR/conf" 25 | 26 | for jar in "$PROJECT_DIR/lib"/*.jar; do 27 | CLASSPATH="$CLASSPATH:$jar" 28 | done 29 | echo CLASSPATH=$CLASSPATH 30 | 31 | JVMARGS="${JVMARGS} -Dproject_dir=${PROJECT_DIR} -Djava.net.preferIPv4Stack=true" 32 | echo JVMARGS=$JVMARGS 33 | 34 | usage() { 35 | echo >&2 "usage: $PRG [args]" 36 | echo 'Valid commands: start, stop' 37 | exit 1 38 | } 39 | 40 | start() { 41 | JAVA=${JAVA-'java'} 42 | exec $JAVA $JVMARGS -classpath "$CLASSPATH" $mainClass "$@" & 43 | echo $! > main.pid 44 | } 45 | 46 | stop() { 47 | kill `cat main.pid` > /dev/null 48 | } 49 | 50 | case $1 in 51 | (start) 52 | shift 53 | start $@ 54 | ;; 55 | (stop) 56 | stop 57 | ;; 58 | (restart) 59 | stop 60 | shift 61 | start $@ 62 | ;; 63 | (*) 64 | echo >&2 "$PRG: error: unknown command '$1'" 65 | usage 66 | ;; 67 | esac 68 | -------------------------------------------------------------------------------- /sentiment-spider/src/main/bin/timer_spider.sh: -------------------------------------------------------------------------------- 1 | # !/bin/bash 2 | 3 | echo "Starting ..." 4 | 5 | bin/ctl.sh start sinaPublicWeibosSpider 6 | 7 | while (true) 8 | do 9 | sleep 30m 10 | bin/ctl.sh restart sinaPublicWeibosSpider 11 | done 12 | 13 | echo "Finishing ..." 14 | -------------------------------------------------------------------------------- /sentiment-spider/src/main/java/zx/soft/sent/spider/dns/CacheDNS.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.spider.dns; 2 | 3 | import java.net.InetAddress; 4 | import java.net.UnknownHostException; 5 | 6 | /** 7 | * DNS缓存处理 8 | * 9 | * @author wanggang 10 | * 11 | */ 12 | public class CacheDNS { 13 | 14 | public static void main(String[] args) throws UnknownHostException { 15 | InetAddress[] ips = InetAddress.getAllByName("www.baidu.com"); 16 | for (InetAddress ip : ips) { 17 | System.out.println(ip); 18 | } 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /sentiment-spider/src/main/java/zx/soft/sent/spider/driver/SpiderDriver.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.spider.driver; 2 | 3 | import zx.soft.sent.spider.sina.SinaPublicWeibosSpider; 4 | import zx.soft.utils.driver.ProgramDriver; 5 | 6 | /** 7 | * 驱动类 8 | * 9 | * @author wanggang 10 | * 11 | */ 12 | public class SpiderDriver { 13 | 14 | /** 15 | * 主函数 16 | */ 17 | public static void main(String[] args) { 18 | 19 | int exitCode = -1; 20 | ProgramDriver pgd = new ProgramDriver(); 21 | try { 22 | // 运行在hefei05机器上 23 | pgd.addClass("sinaPublicWeibosSpider", SinaPublicWeibosSpider.class, "新浪公共微博抓取"); 24 | pgd.driver(args); 25 | // Success 26 | exitCode = 0; 27 | } catch (Throwable e) { 28 | throw new RuntimeException(e); 29 | } 30 | 31 | System.exit(exitCode); 32 | 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /sentiment-spider/src/main/java/zx/soft/sent/spider/http/SpiderHttpURLConnection.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.spider.http; 2 | 3 | import java.io.IOException; 4 | import java.net.HttpURLConnection; 5 | import java.net.URL; 6 | import java.util.Iterator; 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | /** 11 | * getContentType等方法的错误指出在于只识别头信息中小写的字符串，需要修改成大小写不敏感。 12 | * 13 | * @author wanggang 14 | * 15 | */ 16 | public class SpiderHttpURLConnection extends HttpURLConnection { 17 | 18 | protected SpiderHttpURLConnection(URL u) { 19 | super(u); 20 | // TODO Auto-generated constructor stub 21 | } 22 | 23 | @Override 24 | public String getHeaderField(String fieldKey) { 25 | Map> header = getHeaderFields(); 26 | Iterator iter = header.keySet().iterator(); 27 | String key = null; 28 | while (iter.hasNext()) { 29 | key = iter.next(); 30 | if (key == null) { 31 | if (fieldKey == null) { 32 | return (String) ((List) (header.get(null))).get(0); 33 | } 34 | } else { 35 | if (key.equalsIgnoreCase(fieldKey)) { 36 | return (String) ((List) (header.get(key))).get(0); 37 | } 38 | } 39 | } 40 | return null; 41 | } 42 | 43 | @Override 44 | public void disconnect() { 45 | // TODO Auto-generated method stub 46 | } 47 | 48 | @Override 49 | public boolean usingProxy() { 50 | // TODO Auto-generated method stub 51 | return false; 52 | } 53 | 54 | @Override 55 | public void connect() throws IOException { 56 | // TODO Auto-generated method stub 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /sentiment-spider/src/main/java/zx/soft/sent/spider/proxy/SocketProxy.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.spider.proxy; 2 | 3 | import java.net.InetSocketAddress; 4 | import java.net.Proxy; 5 | import java.net.SocketAddress; 6 | import java.util.List; 7 | import java.util.Random; 8 | 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import zx.soft.sent.spider.utils.DataReader; 13 | 14 | /** 15 | * Socket代理设置 16 | * 17 | * @author wanggang 18 | * 19 | */ 20 | public class SocketProxy { 21 | 22 | private static Logger logger = LoggerFactory.getLogger(SocketProxy.class); 23 | 24 | private static List proxyList; 25 | 26 | private static final Random RANDOM = new Random(); 27 | 28 | static { 29 | proxyList = DataReader.getProxyIPs("proxy/proxy-ips.txt"); 30 | logger.info("Loading proxy ips successful!"); 31 | } 32 | 33 | public static Proxy getProxy() { 34 | String ip = proxyList.get(RANDOM.nextInt(proxyList.size())); 35 | String[] ipAndPort = ip.split(":"); 36 | if (ipAndPort.length == 2) { 37 | SocketAddress socketAddress = new InetSocketAddress(ipAndPort[0], Integer.parseInt(ipAndPort[1])); 38 | return new Proxy(Proxy.Type.HTTP, socketAddress); 39 | } 40 | return null; 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /sentiment-spider/src/main/java/zx/soft/sent/spider/utils/DataReader.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.spider.utils; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.File; 5 | import java.io.FileNotFoundException; 6 | import java.io.FileReader; 7 | import java.io.IOException; 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import zx.soft.utils.log.LogbackUtil; 15 | 16 | /** 17 | * 代理数据读取 18 | * 19 | * @author wanggang 20 | * 21 | */ 22 | public class DataReader { 23 | 24 | private static Logger logger = LoggerFactory.getLogger(DataReader.class); 25 | 26 | public static void main(String[] args) { 27 | List proxy = DataReader.getProxyIPs("proxy/proxy-ips.txt"); 28 | for (String ip : proxy) { 29 | System.out.println(ip); 30 | } 31 | } 32 | 33 | public static List getProxyIPs(String file) { 34 | List result = new ArrayList<>(); 35 | try (BufferedReader br = new BufferedReader(new FileReader(new File(file)));) { 36 | String str; 37 | while ((str = br.readLine()) != null) { 38 | result.add(str); 39 | } 40 | return result; 41 | } catch (FileNotFoundException e) { 42 | logger.error("Exception:{}", LogbackUtil.expection2Str(e)); 43 | throw new RuntimeException(e); 44 | } catch (IOException e) { 45 | logger.error("Exception:{}", LogbackUtil.expection2Str(e)); 46 | throw new RuntimeException(e); 47 | } 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /sentiment-spider/src/main/resources/Sentiment-Data/GAB_ZIP_INDEX.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /sentiment-spider/src/main/resources/cache-config.properties: -------------------------------------------------------------------------------- 1 | 2 | ##### Mater-Slave (For Replicating Record-Key,Index-Data Caching) ##### 3 | # Product 4 | redis.rp.master=192.168.32.19 5 | redis.rp.slave=192.168.32.20 6 | # Testing 7 | #redis.rp.master=192.168.31.20 8 | #redis.rp.slave=192.168.31.21 9 | 10 | ##### Message-Queue (For Index-Data Caching) ##### 11 | # Product 12 | redis.mq.server=192.168.32.20 13 | # Testing 14 | #redis.mq.server=192.168.31.XX 15 | 16 | ##### Expired-Queue (For Firsttime Caching) ##### 17 | # Product 18 | redis.ft.server=192.168.32.18 19 | # Testing 20 | #redis.ft.server=192.168.31.XX 21 | 22 | ##### Port ##### 23 | redis.rp.port=16379 24 | redis.mq.port=6379 25 | redis.ft.port=6379 26 | 27 | ##### Password ##### 28 | redis.password=zxsoft -------------------------------------------------------------------------------- /sentiment-spider/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 8 | 9 | 10 | 11 | INFO 12 | ACCEPT 13 | DENY 14 | 15 | 16 | 17 | 19 | logs/sentiment-spider.log 20 | 21 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 22 | 23 | 24 | 25 | INFO 26 | 27 | 28 | logs/sentiment-spider.log.%d{yyyy-MM-dd}.gz 29 | 30 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /sentiment-spider/src/main/resources/sina-api.properties: -------------------------------------------------------------------------------- 1 | 2 | ## IP 3 | # api.weibo.com:80 4 | # 180.149.135.176 5 | # 180.149.135.230 6 | sina.api.proxy.ip=180.149.135.176:80 -------------------------------------------------------------------------------- /sentiment-spider/src/main/resources/super.properties: -------------------------------------------------------------------------------- 1 | 2 | # super info 3 | super= -------------------------------------------------------------------------------- /sentiment-spider/src/test/java/zx/soft/sent/spider/dns/CacheDNSTest.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.spider.dns; 2 | 3 | import org.junit.Test; 4 | 5 | public class CacheDNSTest { 6 | 7 | @Test 8 | public void test() { 9 | System.out.println("Not yet implemented"); 10 | } 11 | 12 | } 13 | -------------------------------------------------------------------------------- /sentiment-spider/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{MMdd.HHmmss.SSS} [%-20t] [%-5p] [%-20c] [L:%-3L] - %m%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /sentiment-spring/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## 注意事项 3 | 4 | > 该接口运行长时间运行会出现连接释放不了问题，猜测应该是Redis层使用不当造成的，目前的解决方法是定时重启该接口。 5 | 6 | > 长时间运行后，curl测试该接口时出现如下错误： 7 | 8 | ``` 9 | curl: (56) Recv failure: Connection reset by peer 10 | ``` -------------------------------------------------------------------------------- /sentiment-spring/ik_dic/ext_ik.dic: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wgybzbrobot/sentiment-search/df50eef8f82fbdc4549d1cfffb0d0b393a04f7b5/sentiment-spring/ik_dic/ext_ik.dic -------------------------------------------------------------------------------- /sentiment-spring/ik_dic/stopwords_ik.dic: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wgybzbrobot/sentiment-search/df50eef8f82fbdc4549d1cfffb0d0b393a04f7b5/sentiment-spring/ik_dic/stopwords_ik.dic -------------------------------------------------------------------------------- /sentiment-spring/src/main/assembly/distribution.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | distribution 7 | 8 | tar.gz 9 | 10 | ${project.artifactId} 11 | 12 | 13 | src/main/resources 14 | 15 | logback.xml 16 | cache-config.properties 17 | data_db.properties 18 | solr_params.properties 19 | web-server.properties 20 | 21 | /conf 22 | true 23 | 24 | 25 | ik_dic 26 | 27 | * 28 | 29 | /ik_dic 30 | 31 | 32 | src/main/bin 33 | 34 | * 35 | 36 | /bin 37 | 0755 38 | 39 | 40 | 41 | 42 | /lib 43 | 44 | 45 | -------------------------------------------------------------------------------- /sentiment-spring/src/main/bin/ctl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mainClass=zx.soft.sent.spring.driver.SentimentSpringDriver 4 | 5 | # resolve links - $0 may be a softlink 6 | PRG="$0" 7 | 8 | while [ -h "$PRG" ]; do 9 | ls=`ls -ld "$PRG"` 10 | link=`expr "$ls" : '.*-> \(.*\)$'` 11 | if expr "$link" : '/.*' > /dev/null; then 12 | PRG="$link" 13 | else 14 | PRG=`dirname "$PRG"`/"$link" 15 | fi 16 | done 17 | 18 | # Get standard environment variables 19 | PRGDIR=`dirname "$PRG"` 20 | 21 | PROJECT_DIR=`cd "$PRGDIR/.." >/dev/null; pwd` 22 | echo PROJECT_DIR=$PROJECT_DIR 23 | 24 | CLASSPATH="$CLASSHPATH:$PROJECT_DIR/conf" 25 | 26 | for jar in "$PROJECT_DIR/lib"/*.jar; do 27 | CLASSPATH="$CLASSPATH:$jar" 28 | done 29 | echo CLASSPATH=$CLASSPATH 30 | 31 | JVMARGS="${JVMARGS} -Dproject_dir=${PROJECT_DIR}" 32 | echo JVMARGS=$JVMARGS 33 | 34 | usage() { 35 | echo >&2 "usage: $PRG [args]" 36 | echo 'Valid commands: start, stop' 37 | exit 1 38 | } 39 | 40 | start() { 41 | JAVA=${JAVA-'java'} 42 | exec $JAVA $JVMARGS -classpath "$CLASSPATH" $mainClass "$@" & 43 | echo $! > main.pid 44 | } 45 | 46 | stop() { 47 | kill `cat main.pid` > /dev/null 48 | } 49 | 50 | case $1 in 51 | (start) 52 | shift 53 | start $@ 54 | ;; 55 | (stop) 56 | stop 57 | ;; 58 | (restart) 59 | stop 60 | shift 61 | start $@ 62 | ;; 63 | (*) 64 | echo >&2 "$PRG: error: unknown command '$1'" 65 | usage 66 | ;; 67 | esac 68 | -------------------------------------------------------------------------------- /sentiment-spring/src/main/bin/timer-index.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Starting ..." 4 | 5 | bin/ctl.sh start indexApiServer 6 | 7 | while (true) 8 | do 9 | sleep 30m 10 | bin/ctl.sh restart indexApiServer 11 | done 12 | 13 | echo "Finishing ..." 14 | -------------------------------------------------------------------------------- /sentiment-spring/src/main/java/zx/soft/sent/spring/controller/IndexController.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.spring.controller; 2 | 3 | import javax.inject.Inject; 4 | 5 | import org.springframework.http.HttpStatus; 6 | import org.springframework.stereotype.Controller; 7 | import org.springframework.web.bind.annotation.RequestBody; 8 | import org.springframework.web.bind.annotation.RequestMapping; 9 | import org.springframework.web.bind.annotation.RequestMethod; 10 | import org.springframework.web.bind.annotation.ResponseBody; 11 | import org.springframework.web.bind.annotation.ResponseStatus; 12 | 13 | import zx.soft.sent.spring.domain.ErrorResponse; 14 | import zx.soft.sent.spring.domain.PostData; 15 | import zx.soft.sent.spring.service.IndexService; 16 | 17 | /** 18 | * 索引控制类 19 | * 20 | * @author wanggang 21 | * 22 | */ 23 | @Controller 24 | @RequestMapping("/sentiment/index") 25 | public class IndexController { 26 | 27 | @Inject 28 | private IndexService indexService; 29 | 30 | @RequestMapping(method = RequestMethod.POST) 31 | @ResponseStatus(HttpStatus.CREATED) 32 | public @ResponseBody 33 | ErrorResponse add(@RequestBody PostData postData) { 34 | return indexService.addIndexData(postData); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /sentiment-spring/src/main/java/zx/soft/sent/spring/domain/ErrorResponse.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.spring.domain; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * 错误相应类 7 | * 8 | * @author wanggang 9 | * 10 | */ 11 | public class ErrorResponse implements Serializable { 12 | 13 | private static final long serialVersionUID = -3408598774435409654L; 14 | 15 | private final int errorCode; 16 | private final String errorMessage; 17 | 18 | public ErrorResponse(Builder builder) { 19 | this.errorCode = builder.errorCode; 20 | this.errorMessage = builder.errorMessage; 21 | } 22 | 23 | public static class Builder { 24 | 25 | private final int errorCode; 26 | private final String errorMessage; 27 | 28 | public Builder(int errorCode, String errorMessage) { 29 | super(); 30 | this.errorCode = errorCode; 31 | this.errorMessage = errorMessage; 32 | } 33 | 34 | public ErrorResponse build() { 35 | return new ErrorResponse(this); 36 | } 37 | 38 | } 39 | 40 | public int getErrorCode() { 41 | return errorCode; 42 | } 43 | 44 | public String getErrorMessage() { 45 | return errorMessage; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /sentiment-spring/src/main/java/zx/soft/sent/spring/domain/IndexErrResponse.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.spring.domain; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import zx.soft.utils.json.JsonUtils; 8 | 9 | public class IndexErrResponse implements Serializable { 10 | 11 | private static final long serialVersionUID = -5183178204181065266L; 12 | 13 | private int errorCode; 14 | private List errorMessage; 15 | 16 | public IndexErrResponse(int errorCode, List errorMessage) { 17 | this.errorCode = errorCode; 18 | this.errorMessage = errorMessage; 19 | } 20 | 21 | public int getErrorCode() { 22 | return errorCode; 23 | } 24 | 25 | public void setErrorCode(int errorCode) { 26 | this.errorCode = errorCode; 27 | } 28 | 29 | public List getErrorMessage() { 30 | return errorMessage; 31 | } 32 | 33 | public void setErrorMessage(List errorMessage) { 34 | this.errorMessage = errorMessage; 35 | } 36 | 37 | public static void main(String[] args) { 38 | List errorMessage = new ArrayList<>(); 39 | errorMessage.add("wwfffrrrrr"); 40 | errorMessage.add("1233jjffioo"); 41 | IndexErrResponse err = new IndexErrResponse(-1, errorMessage); 42 | System.out.println(JsonUtils.toJson(err)); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /sentiment-spring/src/main/java/zx/soft/sent/spring/domain/PostData.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.spring.domain; 2 | 3 | import java.io.Serializable; 4 | import java.util.List; 5 | 6 | import zx.soft.sent.dao.domain.platform.RecordInfo; 7 | 8 | /** 9 | * POST的索引数据 10 | * 11 | * @author wanggang 12 | * 13 | */ 14 | public class PostData implements Serializable { 15 | 16 | private static final long serialVersionUID = 3183580989697121542L; 17 | 18 | private int num; 19 | private List records; 20 | 21 | public int getNum() { 22 | return num; 23 | } 24 | 25 | public void setNum(int num) { 26 | this.num = num; 27 | } 28 | 29 | public List getRecords() { 30 | return records; 31 | } 32 | 33 | public void setRecords(List records) { 34 | this.records = records; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /sentiment-spring/src/main/java/zx/soft/sent/spring/driver/SentimentSpringDriver.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.spring.driver; 2 | 3 | import zx.soft.sent.spring.server.IndexApiServer; 4 | import zx.soft.utils.driver.ProgramDriver; 5 | 6 | /** 7 | * 驱动类 8 | * 9 | * @author wanggang 10 | * 11 | */ 12 | public class SentimentSpringDriver { 13 | 14 | /** 15 | * 主函数 16 | */ 17 | public static void main(String[] args) { 18 | 19 | int exitCode = -1; 20 | ProgramDriver pgd = new ProgramDriver(); 21 | try { 22 | // 运行在hefei01~hefei06,hefei08~hefei10机器上 23 | // 测试环境：hefei11:192.168.31.11 目录：/home/solr/run-work/api/index 端口：8900 24 | pgd.addClass("indexApiServer", IndexApiServer.class, "舆情数据索引接口"); 25 | pgd.driver(args); 26 | // Success 27 | exitCode = 0; 28 | } catch (Throwable e) { 29 | throw new RuntimeException(e); 30 | } 31 | 32 | System.exit(exitCode); 33 | 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /sentiment-spring/src/main/java/zx/soft/sent/spring/utils/CustomObjectMapper.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.spring.utils; 2 | 3 | import java.text.DateFormat; 4 | import java.text.ParseException; 5 | import java.text.SimpleDateFormat; 6 | import java.util.Date; 7 | import java.util.Locale; 8 | 9 | import com.fasterxml.jackson.databind.ObjectMapper; 10 | 11 | /** 12 | * 自定义的ObjectMapper类，用于处理时间格式 13 | * 14 | * @author wanggang 15 | * 16 | */ 17 | public class CustomObjectMapper extends ObjectMapper { 18 | 19 | private static final long serialVersionUID = -3341366863221844412L; 20 | 21 | public static DateFormat sinaDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ENGLISH); 22 | 23 | public CustomObjectMapper() { 24 | super(); 25 | setDateFormat(sinaDateFormat); 26 | } 27 | 28 | /** 29 | * 测试函数 30 | */ 31 | public static void main(String[] args) throws ParseException { 32 | System.out.println(sinaDateFormat.format(new Date())); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /sentiment-spring/src/main/java/zx/soft/sent/spring/utils/Demo.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.spring.utils; 2 | 3 | import zx.soft.sent.dao.sql.CreateTables; 4 | import zx.soft.utils.checksum.CheckSumUtils; 5 | 6 | public class Demo { 7 | 8 | public static void main(String[] args) { 9 | 10 | System.out 11 | .println(CreateTables.SENT_TABLE + CheckSumUtils.getCRC32("sentiment79") % CreateTables.MAX_TABLE_NUM); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /sentiment-spring/src/main/resources/cache-config.properties: -------------------------------------------------------------------------------- 1 | 2 | ##### Mater-Slave (For Replicating Record-Key,Index-Data Caching) ##### 3 | # Product 4 | redis.rp.master=192.168.32.19 5 | redis.rp.slave=192.168.32.20 6 | # Testing 7 | #redis.rp.master=192.168.31.20 8 | #redis.rp.slave=192.168.31.21 9 | 10 | ##### Message-Queue (For Index-Data Caching) ##### 11 | # Product 12 | redis.mq.server=192.168.32.20 13 | # Testing 14 | #redis.mq.server=192.168.31.XX 15 | 16 | ##### Expired-Queue (For Firsttime Caching) ##### 17 | # Product 18 | redis.ft.server=192.168.32.18 19 | # Testing 20 | #redis.ft.server=192.168.31.XX 21 | 22 | ##### Port ##### 23 | redis.rp.port=16379 24 | redis.mq.port=6379 25 | redis.ft.port=6379 26 | 27 | ##### Password ##### 28 | redis.password=zxsoft -------------------------------------------------------------------------------- /sentiment-spring/src/main/resources/data_db.properties: -------------------------------------------------------------------------------- 1 | 2 | ##### Database ##### 3 | # Product 4 | # OA 5 | #sent.db.url=jdbc:mysql://192.168.32.17:3306/sentiment_records?useUnicode=true&characterEncoding=utf-8 6 | # Records 7 | sent.db.url=jdbc:mysql://192.168.32.18:3306/sentiment_records?useUnicode=true&characterEncoding=utf-8 8 | # Testing 9 | #sent.db.url=jdbc:mysql://192.168.31.23:3306/sentiment_records?useUnicode=true&characterEncoding=utf-8 10 | 11 | ##### Password ##### 12 | sent.db.username=sentimentrecords 13 | sent.db.password=zn/iACQ/Xh19+W6Ot2MghQ== 14 | 15 | ##### Localhost ##### 16 | #sent.db.url=jdbc:mysql://localhost:3306/sentiment_records?useUnicode=true&characterEncoding=utf-8 17 | #sent.db.username=root 18 | #sent.db.password=root -------------------------------------------------------------------------------- /sentiment-spring/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 8 | 9 | 10 | 11 | INFO 12 | ACCEPT 13 | DENY 14 | 15 | 16 | 17 | 19 | logs/sentiment-spring.log 20 | 21 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 22 | 23 | 24 | 25 | INFO 26 | 27 | 28 | logs/sentiment-spring.log.%d{yyyy-MM-dd}.gz 29 | 30 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /sentiment-spring/src/main/resources/solr_params.properties: -------------------------------------------------------------------------------- 1 | 2 | ##### SolrCloud ##### 3 | collection=sentiment 4 | zookeeper_connect_timeout=50000 5 | zookeeper_client_timeout=50000 6 | #zookeeper_cloud=leader:2181 7 | # Product 8 | zookeeper_cloud=192.168.32.11:2181,192.168.32.12:2181,192.168.32.13:2181,192.168.32.14:2181,192.168.32.15:2181,192.168.32.16:2181 9 | # Testing 10 | #zookeeper_cloud=192.168.31.11:2181,192.168.31.12:2181,192.168.31.20:2181,192.168.31.21:2181,192.168.31.26:2181,192.168.31.27:2181 11 | 12 | ##### Fetch Size ##### 13 | fetch_size=20000 -------------------------------------------------------------------------------- /sentiment-spring/src/main/resources/web-server.properties: -------------------------------------------------------------------------------- 1 | httpConn.maxTotal=10000 2 | httpConn.maxPerRoute=5000 3 | api.port=8900 4 | -------------------------------------------------------------------------------- /sentiment-spring/src/main/webapp/WEB-INF/applicationContext.xml: -------------------------------------------------------------------------------- 1 | 2 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 28 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /sentiment-spring/src/main/webapp/images/spring_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wgybzbrobot/sentiment-search/df50eef8f82fbdc4549d1cfffb0d0b393a04f7b5/sentiment-spring/src/main/webapp/images/spring_logo.png -------------------------------------------------------------------------------- /sentiment-spring/src/test/resources/index-test.sh: -------------------------------------------------------------------------------- 1 | # /bin/bash 2 | 3 | for i in $(seq 1 10000) 4 | do echo '"sentiment'$i'"'; 5 | # 192.168.32.17:20000, 36.7.150.150:28094 6 | curl -X POST -H 'Content-Type:application/json' http://192.168.32.17:20000/sentiment/index --data '{"num":1,"records":[{"id":"sentiment'$i'","platform":9,"mid":"123456789987654321","username":"zxsoft","nickname":"中新舆情","original_id":"original_sentiment","original_uid":"original_zxsoft","original_name":"original_中新软件","original_title":"original_标题","original_url":"http://www.orignal_url.com","url":"http://www.url.com","home_url":"http://www.home_url.com","title":"标题","type":"所属类型","isharmful":true,"content":"测试内容","comment_count":10,"read_count":20,"favorite_count":30,"attitude_count":40,"repost_count":50,"video_url":"http://www.video_url.com","pic_url":"htpp://www.pic_url.com","voice_url":"http://www.voice_url.com","timestamp":1419755627695,"source_id":70,"lasttime":1419842027695,"server_id":90,"identify_id":100,"identify_md5":"abcdefg123456789","keyword":"关键词","first_time":1419928427695,"update_time":1420014827695,"ip":"192.168.32.45","location":"安徽省合肥市","geo":"经纬度信息","receive_addr":"receive@gmail.com","append_addr":"append@gmail.com","send_addr":"send@gmail.com","source_name":"新浪微博","source_type":121,"country_code":122,"location_code":123,"province_code":124,"city_code":125}]}'; 7 | done; 8 | -------------------------------------------------------------------------------- /sentiment-spring/src/test/resources/index/curl-index: -------------------------------------------------------------------------------- 1 | curl -X POST --header 'Content-Type:application/json' http://192.168.31.11:8900/sentiment/index --data '{"num":1,"records":[{"id":"sentiment","platform":10,"mid":"123456789987654321","username":"zxsoft","nickname":"中新舆情","original_id":"original_sentiment","original_uid":"original_zxsoft","original_name":"original_中新软件","original_title":"original_标题","original_url":"http://www.orignal_url.com","url":"http://www.url.com","home_url":"http://www.home_url.com","title":"标题","type":"所属类型","isharmful":true,"content":"测试内容","comment_count":10,"read_count":20,"favorite_count":30,"attitude_count":40,"repost_count":50,"video_url":"http://www.video_url.com","pic_url":"htpp://www.pic_url.com","voice_url":"http://www.voice_url.com","timestamp":1416038921,"source_id":70,"lasttime":1416125321,"server_id":90,"identify_id":100,"identify_md5":"abcdefg123456789","keyword":"关键词","first_time":1416211721,"update_time":1416298121,"ip":"192.168.32.45","location":"安徽省合肥市","geo":"经纬度信息","receive_addr":"receive@gmail.com","append_addr":"append@gmail.com","send_addr":"send@gmail.com","source_name":"新浪微博","source_type":121,"country_code":122,"location_code":123,"province_code":124,"city_code":125}]}' -------------------------------------------------------------------------------- /sentiment-spring/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 8 | 9 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /sentiment-web/README.md: -------------------------------------------------------------------------------- 1 | 2 | API接口层 3 | ========== 4 | 5 | `SolrCloud相关` 6 | 1、索引和搜索接口 7 | 8 | `Mysql相关` 9 | 1、读库接口 10 | 11 | `Redis相关` 12 | 1、查询key接口 13 | -------------------------------------------------------------------------------- /sentiment-web/bin/timer-index.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Starting ..." 4 | 5 | bin/ctl.sh start sentimentIndexServer 6 | 7 | while (true) 8 | do 9 | sleep 1h 10 | bin/ctl.sh restart sentimentIndexServer 11 | done 12 | 13 | echo "Finishing ..." 14 | -------------------------------------------------------------------------------- /sentiment-web/ik_dic/ext_ik.dic: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wgybzbrobot/sentiment-search/df50eef8f82fbdc4549d1cfffb0d0b393a04f7b5/sentiment-web/ik_dic/ext_ik.dic -------------------------------------------------------------------------------- /sentiment-web/ik_dic/stopwords_ik.dic: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wgybzbrobot/sentiment-search/df50eef8f82fbdc4549d1cfffb0d0b393a04f7b5/sentiment-web/ik_dic/stopwords_ik.dic -------------------------------------------------------------------------------- /sentiment-web/src/main/assembly/distribution.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | distribution 7 | 8 | tar.gz 9 | 10 | ${project.artifactId} 11 | 12 | 13 | src/main/resources 14 | 15 | logback.xml 16 | cache-config.properties 17 | data_db.properties 18 | solr_params.properties 19 | web-server.properties 20 | 21 | /conf 22 | true 23 | 24 | 25 | ik_dic 26 | 27 | * 28 | 29 | /ik_dic 30 | 31 | 32 | src/main/bin 33 | 34 | * 35 | 36 | /bin 37 | 0755 38 | 39 | 40 | 41 | 42 | /lib 43 | 44 | 45 | -------------------------------------------------------------------------------- /sentiment-web/src/main/bin/ctl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mainClass=zx.soft.sent.web.driver.SentWebDriver 4 | 5 | # resolve links - $0 may be a softlink 6 | PRG="$0" 7 | 8 | while [ -h "$PRG" ]; do 9 | ls=`ls -ld "$PRG"` 10 | link=`expr "$ls" : '.*-> \(.*\)$'` 11 | if expr "$link" : '/.*' > /dev/null; then 12 | PRG="$link" 13 | else 14 | PRG=`dirname "$PRG"`/"$link" 15 | fi 16 | done 17 | 18 | # Get standard environment variables 19 | PRGDIR=`dirname "$PRG"` 20 | 21 | PROJECT_DIR=`cd "$PRGDIR/.." >/dev/null; pwd` 22 | echo PROJECT_DIR=$PROJECT_DIR 23 | 24 | CLASSPATH="$CLASSHPATH:$PROJECT_DIR/conf" 25 | 26 | for jar in "$PROJECT_DIR/lib"/*.jar; do 27 | CLASSPATH="$CLASSPATH:$jar" 28 | done 29 | echo CLASSPATH=$CLASSPATH 30 | 31 | JVMARGS="${JVMARGS} -Dproject_dir=${PROJECT_DIR}" 32 | echo JVMARGS=$JVMARGS 33 | 34 | usage() { 35 | echo >&2 "usage: $PRG [args]" 36 | echo 'Valid commands: start, stop' 37 | exit 1 38 | } 39 | 40 | start() { 41 | JAVA=${JAVA-'java'} 42 | exec $JAVA $JVMARGS -classpath "$CLASSPATH" $mainClass "$@" & 43 | echo $! > main.pid 44 | } 45 | 46 | stop() { 47 | kill `cat main.pid` > /dev/null 48 | } 49 | 50 | case $1 in 51 | (start) 52 | shift 53 | start $@ 54 | ;; 55 | (stop) 56 | stop 57 | ;; 58 | (restart) 59 | stop 60 | shift 61 | start $@ 62 | ;; 63 | (*) 64 | echo >&2 "$PRG: error: unknown command '$1'" 65 | usage 66 | ;; 67 | esac 68 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/application/HarmfulInfoApplication.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.application; 2 | 3 | import java.util.List; 4 | 5 | import org.apache.solr.common.SolrDocument; 6 | import org.restlet.Application; 7 | import org.restlet.Restlet; 8 | import org.restlet.routing.Router; 9 | 10 | import zx.soft.sent.solr.firstpage.OAFirstPage; 11 | import zx.soft.sent.solr.utils.HarmfulInfoUtil; 12 | import zx.soft.sent.web.resource.HarmfulInfoResource; 13 | 14 | public class HarmfulInfoApplication extends Application { 15 | 16 | private final OAFirstPage firstPage; 17 | 18 | public HarmfulInfoApplication() { 19 | firstPage = new OAFirstPage(); 20 | } 21 | 22 | @Override 23 | public Restlet createInboundRoot() { 24 | Router router = new Router(getContext()); 25 | router.attach("/{keywords}/{num}", HarmfulInfoResource.class); 26 | return router; 27 | } 28 | 29 | /** 30 | * 当天有害信息 31 | * @param num：返回条数 32 | * @param q：查询关键词 33 | * @return 34 | */ 35 | public List getTodayNegativeRecords(int num, String q) { 36 | List result = firstPage.getTodayNegativeRecords(7, 100, q); 37 | result = HarmfulInfoUtil.getTopNNegativeRecords(result, num); 38 | return result; 39 | } 40 | 41 | @Override 42 | public void stop() throws Exception { 43 | super.stop(); 44 | firstPage.close(); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/application/InternetTaskApplication.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.application; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import org.restlet.Application; 7 | import org.restlet.Restlet; 8 | import org.restlet.routing.Router; 9 | 10 | import zx.soft.sent.solr.domain.QueryParams; 11 | import zx.soft.sent.solr.domain.QueryResult; 12 | import zx.soft.sent.solr.query.QueryCore; 13 | import zx.soft.sent.web.domain.Task; 14 | import zx.soft.sent.web.resource.InternetTaskResource; 15 | 16 | public class InternetTaskApplication extends Application { 17 | 18 | private final QueryCore queryCore; 19 | 20 | public InternetTaskApplication() { 21 | queryCore = new QueryCore(); 22 | } 23 | 24 | @Override 25 | public Restlet createInboundRoot() { 26 | Router router = new Router(getContext()); 27 | // POST专题信息 28 | router.attach("", InternetTaskResource.class); 29 | return router; 30 | } 31 | 32 | /** 33 | * 查询OA首页查询数据 34 | */ 35 | public List taskResult(List tasks) { 36 | List result = new ArrayList<>(); 37 | QueryParams queryParams = null; 38 | QueryResult queryResult = null; 39 | for (Task task : tasks) { 40 | queryParams = new QueryParams(); 41 | queryParams.setQ(task.getKeywords()); 42 | if (task.getFq().length() > 0) { 43 | queryParams.setFq(task.getFq()); 44 | } 45 | queryResult = queryCore.queryData(queryParams, Boolean.FALSE); 46 | result.add(queryResult.getNumFound()); 47 | } 48 | return result; 49 | } 50 | 51 | @Override 52 | public void stop() throws Exception { 53 | super.stop(); 54 | queryCore.close(); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/application/PullRecordApplication.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.application; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import org.restlet.Application; 7 | import org.restlet.Restlet; 8 | import org.restlet.routing.Router; 9 | 10 | import zx.soft.sent.dao.common.MybatisConfig; 11 | import zx.soft.sent.dao.domain.sentiment.RecordSelect; 12 | import zx.soft.sent.dao.sentiment.SentimentRecord; 13 | import zx.soft.sent.dao.sql.CreateTables; 14 | import zx.soft.sent.web.resource.PullRecordResource; 15 | import zx.soft.utils.checksum.CheckSumUtils; 16 | 17 | /** 18 | * MySQL数据提取应用类 19 | * 20 | * @author wanggang 21 | * 22 | */ 23 | public class PullRecordApplication extends Application { 24 | 25 | // private static Logger logger = LoggerFactory.getLogger(PullRecordApplication.class); 26 | 27 | private final SentimentRecord sentRecord; 28 | 29 | public PullRecordApplication() { 30 | sentRecord = new SentimentRecord(MybatisConfig.ServerEnum.sentiment); 31 | } 32 | 33 | @Override 34 | public Restlet createInboundRoot() { 35 | Router router = new Router(getContext()); 36 | // 1、根据多个id查询记录： 37 | router.attach("/pull/ids/{ids}", PullRecordResource.class); 38 | // 2、根据lasttime时间段查询记录： 39 | // router.attach("/get", PullRecordResource.class); 40 | return router; 41 | } 42 | 43 | public List getRecords(String ids) { 44 | List result = new ArrayList<>(); 45 | for (String id : ids.split(",")) { 46 | result.add(getRecord(id)); 47 | } 48 | return result; 49 | } 50 | 51 | private RecordSelect getRecord(String id) { 52 | return sentRecord.selectRecordById(CreateTables.SENT_TABLE + CheckSumUtils.getCRC32(id) 53 | % CreateTables.MAX_TABLE_NUM, id); 54 | } 55 | 56 | public void close() { 57 | // 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/application/SentiSearchApplication.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.application; 2 | 3 | import org.restlet.Application; 4 | import org.restlet.Restlet; 5 | import org.restlet.routing.Router; 6 | 7 | import zx.soft.sent.solr.domain.QueryParams; 8 | import zx.soft.sent.solr.domain.QueryResult; 9 | import zx.soft.sent.solr.query.QueryCore; 10 | import zx.soft.sent.web.resource.SentSearchResource; 11 | 12 | /** 13 | * 舆情搜索应用类 14 | * 15 | * @author wanggang 16 | * 17 | */ 18 | public class SentiSearchApplication extends Application { 19 | 20 | private final QueryCore queryCore; 21 | 22 | public SentiSearchApplication() { 23 | queryCore = new QueryCore(); 24 | } 25 | 26 | @Override 27 | public Restlet createInboundRoot() { 28 | Router router = new Router(getContext()); 29 | getContext().getParameters().add("maxThreads", "512"); 30 | getContext().getParameters().add("minThreads", "100"); 31 | getContext().getParameters().add("lowThreads", "200"); 32 | getContext().getParameters().add("maxConnectionsPerHost", "128"); 33 | getContext().getParameters().add("initialConnections", "255"); 34 | getContext().getParameters().add("maxTotalConnections", "1024"); 35 | getContext().getParameters().add("maxIoIdleTimeMs", "100"); 36 | router.attach("/search", SentSearchResource.class); 37 | // router.attach("/get", SentGetResource.class); 38 | return router; 39 | } 40 | 41 | public QueryResult queryData(QueryParams queryParams) { 42 | return queryCore.queryData(queryParams, Boolean.TRUE); 43 | } 44 | 45 | public void close() { 46 | queryCore.close(); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/application/SiteApplication.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.application; 2 | 3 | import java.util.List; 4 | 5 | import org.restlet.Application; 6 | import org.restlet.Restlet; 7 | import org.restlet.routing.Router; 8 | 9 | import zx.soft.redis.client.cache.Cache; 10 | import zx.soft.redis.client.cache.RedisCache; 11 | import zx.soft.redis.client.common.Config; 12 | import zx.soft.sent.dao.common.SentimentConstant; 13 | import zx.soft.sent.web.resource.SiteResource; 14 | import zx.soft.utils.checksum.CheckSumUtils; 15 | 16 | /** 17 | * 站点应用类 18 | * 19 | * @author wanggang 20 | * 21 | */ 22 | public class SiteApplication extends Application { 23 | 24 | private static Cache cache; 25 | 26 | public SiteApplication() { 27 | // 写入数据用master节点 28 | cache = new RedisCache(Config.get("redis.rp.master"), Integer.parseInt(Config.get("redis.rp.port")), 29 | Config.get("redis.password")); 30 | } 31 | 32 | @Override 33 | public Restlet createInboundRoot() { 34 | Router router = new Router(getContext()); 35 | router.attach("", SiteResource.class); 36 | return router; 37 | } 38 | 39 | /** 40 | * 插入站点组合数据 41 | */ 42 | public void insertSiteGroups(List data) { 43 | for (String sites : data) { 44 | // 设置hash表 45 | cache.hset(SentimentConstant.SITE_GROUPS, CheckSumUtils.getMD5(sites), sites); 46 | } 47 | } 48 | 49 | public void close() { 50 | cache.close(); 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/common/LogstashDemo.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.common; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.File; 5 | import java.io.FileReader; 6 | import java.io.IOException; 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | 10 | import redis.clients.jedis.Jedis; 11 | import zx.soft.utils.json.JsonUtils; 12 | 13 | public class LogstashDemo { 14 | 15 | public static void main(String[] args) { 16 | Jedis jedis = new Jedis("192.168.31.160", 7000); 17 | String key = "nginx-logs"; 18 | List logs = new ArrayList<>(); 19 | try (BufferedReader br = new BufferedReader(new FileReader(new File("access.log")));) { 20 | String line; 21 | String[] strs; 22 | while ((line = br.readLine()) != null) { 23 | strs = line.split("-"); 24 | NginxLog log = new NginxLog(); 25 | log.setRemote_addr(strs[0]); 26 | log.setTime_local(strs[1]); 27 | log.setRequest_length(strs[2]); 28 | log.setStatus(strs[3]); 29 | log.setRequest_time(strs[4]); 30 | log.setBody_bytes_sent(strs[5]); 31 | logs.add(JsonUtils.toJsonWithoutPretty(log)); 32 | } 33 | } catch (IOException e) { 34 | e.printStackTrace(); 35 | } 36 | jedis.lpush(key, logs.toArray(new String[logs.size()])); 37 | jedis.close(); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/common/NginxLog.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.common; 2 | 3 | public class NginxLog { 4 | 5 | private String remote_addr; 6 | private String time_local; 7 | private String request_length; 8 | private String status; 9 | private String request_time; 10 | private String body_bytes_sent; 11 | 12 | public String getRemote_addr() { 13 | return remote_addr; 14 | } 15 | 16 | public void setRemote_addr(String remote_addr) { 17 | this.remote_addr = remote_addr; 18 | } 19 | 20 | public String getTime_local() { 21 | return time_local; 22 | } 23 | 24 | public void setTime_local(String time_local) { 25 | this.time_local = time_local; 26 | } 27 | 28 | public String getRequest_length() { 29 | return request_length; 30 | } 31 | 32 | public void setRequest_length(String request_length) { 33 | this.request_length = request_length; 34 | } 35 | 36 | public String getStatus() { 37 | return status; 38 | } 39 | 40 | public void setStatus(String status) { 41 | this.status = status; 42 | } 43 | 44 | public String getRequest_time() { 45 | return request_time; 46 | } 47 | 48 | public void setRequest_time(String request_time) { 49 | this.request_time = request_time; 50 | } 51 | 52 | public String getBody_bytes_sent() { 53 | return body_bytes_sent; 54 | } 55 | 56 | public void setBody_bytes_sent(String body_bytes_sent) { 57 | this.body_bytes_sent = body_bytes_sent; 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/demo/SpecialTopicDemo.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.demo; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import zx.soft.sent.dao.domain.special.SpecialTopic; 7 | import zx.soft.utils.json.JsonUtils; 8 | 9 | public class SpecialTopicDemo { 10 | 11 | public static void main(String[] args) { 12 | 13 | List specialTopics = new ArrayList<>(); 14 | SpecialTopic st1 = new SpecialTopic(); 15 | st1.setIdentify("absbhdfhfjfi"); 16 | st1.setName("专题1"); 17 | st1.setKeywords("（合肥 and 警察） not 打人"); 18 | st1.setStart("2014-08-25 00:00:00"); 19 | st1.setEnd("2014-08-25 23:59:59"); 20 | st1.setHometype(0); 21 | SpecialTopic st2 = new SpecialTopic(); 22 | st2.setIdentify("djdjfkfklglg"); 23 | st2.setName("专题2"); 24 | st2.setKeywords("（安徽 and 城管） not 暴力执法"); 25 | st2.setStart("2013-08-25 00:00:00"); 26 | st2.setEnd("2013-08-25 23:59:59"); 27 | st2.setHometype(2); 28 | specialTopics.add(st1); 29 | specialTopics.add(st2); 30 | System.out.println(JsonUtils.toJsonWithoutPretty(specialTopics)); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/domain/ErrorResponse.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.domain; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * 错误相应类 7 | * 8 | * @author wanggang 9 | * 10 | */ 11 | public class ErrorResponse implements Serializable { 12 | 13 | private static final long serialVersionUID = -3408598774435409654L; 14 | 15 | private final int errorCode; 16 | private final String errorMessage; 17 | 18 | public ErrorResponse(Builder builder) { 19 | this.errorCode = builder.errorCode; 20 | this.errorMessage = builder.errorMessage; 21 | } 22 | 23 | public static class Builder { 24 | 25 | private final int errorCode; 26 | private final String errorMessage; 27 | 28 | public Builder(int errorCode, String errorMessage) { 29 | super(); 30 | this.errorCode = errorCode; 31 | this.errorMessage = errorMessage; 32 | } 33 | 34 | public ErrorResponse build() { 35 | return new ErrorResponse(this); 36 | } 37 | 38 | } 39 | 40 | public int getErrorCode() { 41 | return errorCode; 42 | } 43 | 44 | public String getErrorMessage() { 45 | return errorMessage; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/domain/IndexErrResponse.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.domain; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import zx.soft.utils.json.JsonUtils; 8 | 9 | public class IndexErrResponse implements Serializable { 10 | 11 | private static final long serialVersionUID = -5183178204181065266L; 12 | 13 | private int errorCode; 14 | private List errorMessage; 15 | 16 | public IndexErrResponse(int errorCode, List errorMessage) { 17 | this.errorCode = errorCode; 18 | this.errorMessage = errorMessage; 19 | } 20 | 21 | public int getErrorCode() { 22 | return errorCode; 23 | } 24 | 25 | public void setErrorCode(int errorCode) { 26 | this.errorCode = errorCode; 27 | } 28 | 29 | public List getErrorMessage() { 30 | return errorMessage; 31 | } 32 | 33 | public void setErrorMessage(List errorMessage) { 34 | this.errorMessage = errorMessage; 35 | } 36 | 37 | public static void main(String[] args) { 38 | List errorMessage = new ArrayList<>(); 39 | errorMessage.add("wwfffrrrrr"); 40 | errorMessage.add("1233jjffioo"); 41 | IndexErrResponse err = new IndexErrResponse(-1, errorMessage); 42 | System.out.println(JsonUtils.toJson(err)); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/domain/InternetTask.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.domain; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import zx.soft.utils.json.JsonUtils; 8 | 9 | /** 10 | * 全网搜索任务请求参数刘表 11 | * 12 | * @author wanggang 13 | * 14 | */ 15 | public class InternetTask implements Serializable { 16 | 17 | private static final long serialVersionUID = 5887938203759551452L; 18 | 19 | // 任务数 20 | private int num; 21 | // 任务列表 22 | private List tasks; 23 | 24 | public int getNum() { 25 | return num; 26 | } 27 | 28 | public void setNum(int num) { 29 | this.num = num; 30 | } 31 | 32 | public List getTasks() { 33 | return tasks; 34 | } 35 | 36 | public void setTasks(List tasks) { 37 | this.tasks = tasks; 38 | } 39 | 40 | public static void main(String[] args) { 41 | InternetTask internetTask = new InternetTask(); 42 | List tasks = new ArrayList<>(); 43 | Task task1 = new Task(); 44 | task1.setKeywords("穿越火线"); 45 | task1.setFq("lasttime:[2014-12-25T15:09:01Z TO 2014-12-26T15:09:01Z];source_name:腾讯微博;platform:3"); 46 | tasks.add(task1); 47 | Task task2 = new Task(); 48 | task2.setKeywords("圣诞节"); 49 | task2.setFq("lasttime:[2014-12-25T15:09:01Z TO 2014-12-26T15:09:01Z];source_name:新浪微博;platform:3"); 50 | tasks.add(task2); 51 | internetTask.setNum(2); 52 | internetTask.setTasks(tasks); 53 | System.out.println(JsonUtils.toJsonWithoutPretty(internetTask)); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/domain/Task.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.domain; 2 | 3 | import java.io.Serializable; 4 | 5 | public class Task implements Serializable { 6 | 7 | private static final long serialVersionUID = -1391283603740904033L; 8 | 9 | // 关键词 10 | private String keywords; 11 | // 过滤条件 12 | private String fq; 13 | 14 | public String getKeywords() { 15 | return keywords; 16 | } 17 | 18 | public void setKeywords(String keywords) { 19 | this.keywords = keywords; 20 | } 21 | 22 | public String getFq() { 23 | return fq; 24 | } 25 | 26 | public void setFq(String fq) { 27 | this.fq = fq; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/resource/FirstPageHarmfulResource.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.resource; 2 | 3 | import org.restlet.resource.Get; 4 | import org.restlet.resource.ServerResource; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import zx.soft.sent.web.application.FirstPageHarmfulApplication; 9 | import zx.soft.sent.web.domain.ErrorResponse; 10 | import zx.soft.utils.chars.JavaPattern; 11 | import zx.soft.utils.codec.URLCodecUtils; 12 | 13 | public class FirstPageHarmfulResource extends ServerResource { 14 | 15 | private static Logger logger = LoggerFactory.getLogger(FirstPageHarmfulResource.class); 16 | 17 | private FirstPageHarmfulApplication application; 18 | 19 | private String type = ""; 20 | private String datestr = ""; 21 | 22 | @Override 23 | public void doInit() { 24 | logger.info("Request Url: " + URLCodecUtils.decoder(getReference().toString(), "utf-8") + "."); 25 | application = (FirstPageHarmfulApplication) getApplication(); 26 | type = (String) this.getRequest().getAttributes().get("type"); 27 | datestr = (String) this.getRequest().getAttributes().get("datestr"); 28 | } 29 | 30 | @Get("json") 31 | public Object getSpecialResult() { 32 | if (type == null || type.length() == 0 || datestr == null || datestr.length() == 0 33 | || !JavaPattern.isAllNum(type)) { 34 | logger.error("Params `type` or `datestr` is null."); 35 | return new ErrorResponse.Builder(-1, "params error!").build(); 36 | } 37 | Object result = application.selectFirstPageType(Integer.parseInt(type), datestr); 38 | if (result == null) { 39 | return new ErrorResponse.Builder(-1, "params error!").build(); 40 | } else { 41 | return result; 42 | } 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/resource/FirstPageResource.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.resource; 2 | 3 | import org.restlet.resource.Get; 4 | import org.restlet.resource.ServerResource; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import zx.soft.sent.web.application.FirstPageApplication; 9 | import zx.soft.sent.web.domain.ErrorResponse; 10 | import zx.soft.utils.chars.JavaPattern; 11 | import zx.soft.utils.codec.URLCodecUtils; 12 | 13 | public class FirstPageResource extends ServerResource { 14 | 15 | private static Logger logger = LoggerFactory.getLogger(FirstPageResource.class); 16 | 17 | private FirstPageApplication application; 18 | 19 | private String type = ""; 20 | private String datestr = ""; 21 | 22 | @Override 23 | public void doInit() { 24 | logger.info("Request Url: " + URLCodecUtils.decoder(getReference().toString(), "utf-8") + "."); 25 | application = (FirstPageApplication) getApplication(); 26 | type = (String) this.getRequest().getAttributes().get("type"); 27 | datestr = (String) this.getRequest().getAttributes().get("datestr"); 28 | } 29 | 30 | @Get("json") 31 | public Object getSpecialResult() { 32 | if (type == null || type.length() == 0 || datestr == null || datestr.length() == 0 33 | || !JavaPattern.isAllNum(type)) { 34 | logger.error("Params `type` or `datestr` is null."); 35 | return new ErrorResponse.Builder(-1, "params error!").build(); 36 | } 37 | int t = Integer.parseInt(type); 38 | Object result = null; 39 | if (t == 1 || t == 2) { 40 | result = application.selectFirstPageType12(t, datestr); 41 | } else if (t == 4) { 42 | result = application.selectFirstPageType4(datestr); 43 | } else if (t == 52 || t == 53) { 44 | result = application.selectFirstPageType5(t, datestr); 45 | } 46 | if (result == null) { 47 | return new ErrorResponse.Builder(-1, "params error!").build(); 48 | } else { 49 | return result; 50 | } 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/resource/HarmfulInfoResource.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.resource; 2 | 3 | import org.restlet.resource.Get; 4 | import org.restlet.resource.ServerResource; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import zx.soft.sent.web.application.HarmfulInfoApplication; 9 | import zx.soft.sent.web.domain.ErrorResponse; 10 | import zx.soft.utils.chars.JavaPattern; 11 | import zx.soft.utils.codec.URLCodecUtils; 12 | import zx.soft.utils.log.LogbackUtil; 13 | 14 | public class HarmfulInfoResource extends ServerResource { 15 | 16 | private static Logger logger = LoggerFactory.getLogger(HarmfulInfoResource.class); 17 | 18 | private HarmfulInfoApplication application; 19 | 20 | private String keywords = ""; 21 | private String num = ""; 22 | 23 | @Override 24 | public void doInit() { 25 | logger.info("Request Url: " + URLCodecUtils.decoder(getReference().toString(), "utf-8") + "."); 26 | application = (HarmfulInfoApplication) getApplication(); 27 | keywords = (String) this.getRequest().getAttributes().get("keywords"); 28 | num = (String) this.getRequest().getAttributes().get("num"); 29 | } 30 | 31 | @Get("json") 32 | public Object getSpecialResult() { 33 | if (keywords == null || keywords.length() == 0 || num == null || num.length() == 0 34 | || !JavaPattern.isAllNum(num)) { 35 | logger.error("Params `keywords` or `num` is null."); 36 | return new ErrorResponse.Builder(-1, "params error!").build(); 37 | } 38 | try { 39 | return application.getTodayNegativeRecords(Integer.parseInt(num), URLCodecUtils.decoder(keywords, "UTF-8")); 40 | } catch (Exception e) { 41 | logger.error("Exception:{}", LogbackUtil.expection2Str(e)); 42 | return new ErrorResponse.Builder(-1, "params error!").build(); 43 | } 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/resource/InternetTaskResource.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.resource; 2 | 3 | import org.restlet.resource.Post; 4 | import org.restlet.resource.ServerResource; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import zx.soft.sent.web.application.InternetTaskApplication; 9 | import zx.soft.sent.web.domain.ErrorResponse; 10 | import zx.soft.sent.web.domain.InternetTask; 11 | import zx.soft.utils.codec.URLCodecUtils; 12 | import zx.soft.utils.log.LogbackUtil; 13 | 14 | public class InternetTaskResource extends ServerResource { 15 | 16 | private static Logger logger = LoggerFactory.getLogger(InternetTaskResource.class); 17 | 18 | private InternetTaskApplication application; 19 | 20 | @Override 21 | public void doInit() { 22 | logger.info("Request Url: " + URLCodecUtils.decoder(getReference().toString(), "utf-8") + "."); 23 | application = (InternetTaskApplication) getApplication(); 24 | } 25 | 26 | @Post("json") 27 | public Object taskResult(InternetTask internetTask) { 28 | logger.info("Tasks'size={}", internetTask.getNum()); 29 | try { 30 | return application.taskResult(internetTask.getTasks()); 31 | } catch (Exception e) { 32 | logger.error("Exception:{}", LogbackUtil.expection2Str(e)); 33 | return new ErrorResponse.Builder(-1, "params error!").build(); 34 | } 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/resource/SentGetResource.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.resource; 2 | 3 | import java.util.HashMap; 4 | 5 | import org.restlet.data.Form; 6 | import org.restlet.data.Parameter; 7 | import org.restlet.resource.Get; 8 | import org.restlet.resource.ServerResource; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import zx.soft.sent.web.domain.ErrorResponse; 13 | import zx.soft.utils.codec.URLCodecUtils; 14 | import zx.soft.utils.http.HttpUtils; 15 | 16 | /** 17 | * 舆情数据获取资源类 18 | * 19 | * @author wanggang 20 | * 21 | */ 22 | public class SentGetResource extends ServerResource { 23 | 24 | private static Logger logger = LoggerFactory.getLogger(SentGetResource.class); 25 | 26 | private static final String BASE_URL = "http://192.168.3.21:8983/solr/sentiment/get?id="; 27 | 28 | private String id = ""; 29 | 30 | @Override 31 | public void doInit() { 32 | logger.info("Request Url: " + URLCodecUtils.decoder(getReference().toString(), "utf-8") + "."); 33 | HashMap params = new HashMap<>(); 34 | Form form = getRequest().getResourceRef().getQueryAsForm(); 35 | for (Parameter p : form) { 36 | if (params.get(p.getName()) == null) { 37 | params.put(p.getName(), p.getValue()); 38 | } else { // 重复参数以最后一个为准 39 | params.put(p.getName(), p.getValue()); 40 | } 41 | } 42 | if (params.get("id") != null) { 43 | id = params.get("id"); 44 | } else { 45 | logger.error("id is null."); 46 | } 47 | logger.info("id=" + id); 48 | } 49 | 50 | @Get("json") 51 | public Object getQueryResult() { 52 | if (id.length() == 0) { 53 | return new ErrorResponse.Builder(20003, "your query params is illegal.").build(); 54 | } 55 | return HttpUtils.doGet(BASE_URL + id, "utf-8"); 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/resource/SiteResource.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.resource; 2 | 3 | import java.util.List; 4 | 5 | import org.restlet.resource.Post; 6 | import org.restlet.resource.ServerResource; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import zx.soft.sent.web.application.SiteApplication; 11 | import zx.soft.sent.web.domain.ErrorResponse; 12 | import zx.soft.utils.codec.URLCodecUtils; 13 | import zx.soft.utils.json.JsonUtils; 14 | 15 | /** 16 | * 站点数据 17 | * 18 | * @author wanggang 19 | * 20 | */ 21 | public class SiteResource extends ServerResource { 22 | 23 | private static Logger logger = LoggerFactory.getLogger(SiteResource.class); 24 | 25 | private SiteApplication application; 26 | 27 | @Override 28 | public void doInit() { 29 | logger.info("Request Url: " + URLCodecUtils.decoder(getReference().toString(), "utf-8") + "."); 30 | application = (SiteApplication) getApplication(); 31 | } 32 | 33 | @Post("json") 34 | public Object siteDate(List data) { 35 | logger.info("Site data is {}.", JsonUtils.toJsonWithoutPretty(data)); 36 | application.insertSiteGroups(data); 37 | return new ErrorResponse.Builder(0, "ok").build(); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/resource/SpecialSpeedResource.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.resource; 2 | 3 | import java.util.List; 4 | 5 | import org.restlet.resource.Get; 6 | import org.restlet.resource.ServerResource; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import zx.soft.sent.web.application.SpecialSpeedApplication; 11 | import zx.soft.sent.web.application.SpecialSpeedApplication.SpecialResult; 12 | import zx.soft.sent.web.domain.ErrorResponse; 13 | import zx.soft.utils.codec.URLCodecUtils; 14 | 15 | /** 16 | * 专题资源类 17 | * 18 | * @author wanggang 19 | * 20 | */ 21 | public class SpecialSpeedResource extends ServerResource { 22 | 23 | private static Logger logger = LoggerFactory.getLogger(SpecialSpeedResource.class); 24 | 25 | private SpecialSpeedApplication application; 26 | 27 | private String identifys = ""; 28 | 29 | @Override 30 | public void doInit() { 31 | logger.info("Request Url: " + URLCodecUtils.decoder(getReference().toString(), "utf-8") + "."); 32 | application = (SpecialSpeedApplication) getApplication(); 33 | identifys = (String) this.getRequest().getAttributes().get("identifys"); 34 | } 35 | 36 | @Get("json") 37 | public Object getSpecialResult() { 38 | if (identifys == null || identifys.length() == 0) { 39 | logger.error("Params `identifys` is null."); 40 | return new ErrorResponse.Builder(-1, "params error!").build(); 41 | } 42 | List queryResult = application.selectSpecialResult(identifys); 43 | return queryResult; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/server/FirstPageHarmfulServer.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.server; 2 | 3 | import java.util.Properties; 4 | 5 | import org.restlet.Component; 6 | import org.restlet.data.Protocol; 7 | 8 | import zx.soft.sent.web.application.FirstPageHarmfulApplication; 9 | import zx.soft.utils.config.ConfigUtil; 10 | import zx.soft.utils.jackson.ReplaceConvert; 11 | 12 | /** 13 | * OA首页负面信息查询缓存服务：hefei07 14 | * 示例：http://localhost:5902/firstpage/{type}/{datestr} 15 | * 16 | * type是平台类型，1～10 17 | * 18 | * 运行目录：/home/zxdfs/run-work/api/oa-firstpage-harmful 19 | * 运行命令：cd sentiment-web 20 | * bin/ctl.sh start firstPageHarmfulServer 21 | * 22 | * @author wanggang 23 | * 24 | */ 25 | public class FirstPageHarmfulServer { 26 | 27 | private final Component component; 28 | private final FirstPageHarmfulApplication firstPageApplication; 29 | 30 | private final int PORT; 31 | 32 | public FirstPageHarmfulServer() { 33 | Properties props = ConfigUtil.getProps("web-server.properties"); 34 | PORT = Integer.parseInt(props.getProperty("api.port")); 35 | component = new Component(); 36 | firstPageApplication = new FirstPageHarmfulApplication(); 37 | } 38 | 39 | /** 40 | * 主函数 41 | */ 42 | public static void main(String[] args) { 43 | 44 | FirstPageHarmfulServer specialServer = new FirstPageHarmfulServer(); 45 | specialServer.start(); 46 | } 47 | 48 | public void start() { 49 | component.getServers().add(Protocol.HTTP, PORT); 50 | try { 51 | component.getDefaultHost().attach("/firstpage", firstPageApplication); 52 | ReplaceConvert.configureJacksonConverter(); 53 | component.start(); 54 | } catch (Exception e) { 55 | throw new RuntimeException(e); 56 | } 57 | } 58 | 59 | public void stop() { 60 | try { 61 | component.stop(); 62 | firstPageApplication.stop(); 63 | } catch (Exception e) { 64 | throw new RuntimeException(e); 65 | } 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/server/FirstPageServer.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.server; 2 | 3 | import java.util.Properties; 4 | 5 | import org.restlet.Component; 6 | import org.restlet.data.Protocol; 7 | 8 | import zx.soft.sent.web.application.FirstPageApplication; 9 | import zx.soft.utils.config.ConfigUtil; 10 | import zx.soft.utils.jackson.ReplaceConvert; 11 | 12 | /** 13 | * OA首页查询缓存服务：hefei07 14 | * 示例：http://localhost:5901/firstpage/{type}/{datestr} 15 | * 16 | * 运行目录：/home/zxdfs/run-work/api/oa-firstpage 17 | * 运行命令：cd sentiment-web 18 | * bin/ctl.sh start firstPageServer 19 | * 20 | * @author wanggang 21 | * 22 | */ 23 | public class FirstPageServer { 24 | 25 | private final Component component; 26 | private final FirstPageApplication firstPageApplication; 27 | 28 | private final int PORT; 29 | 30 | public FirstPageServer() { 31 | Properties props = ConfigUtil.getProps("web-server.properties"); 32 | PORT = Integer.parseInt(props.getProperty("api.port")); 33 | component = new Component(); 34 | firstPageApplication = new FirstPageApplication(); 35 | } 36 | 37 | /** 38 | * 主函数 39 | */ 40 | public static void main(String[] args) { 41 | 42 | FirstPageServer specialServer = new FirstPageServer(); 43 | specialServer.start(); 44 | } 45 | 46 | public void start() { 47 | component.getServers().add(Protocol.HTTP, PORT); 48 | try { 49 | component.getDefaultHost().attach("/firstpage", firstPageApplication); 50 | ReplaceConvert.configureJacksonConverter(); 51 | component.start(); 52 | } catch (Exception e) { 53 | throw new RuntimeException(e); 54 | } 55 | } 56 | 57 | public void stop() { 58 | try { 59 | component.stop(); 60 | firstPageApplication.stop(); 61 | } catch (Exception e) { 62 | throw new RuntimeException(e); 63 | } 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/server/HarmfulInfoServer.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.server; 2 | 3 | import java.util.Properties; 4 | 5 | import org.restlet.Component; 6 | import org.restlet.data.Protocol; 7 | 8 | import zx.soft.sent.web.application.HarmfulInfoApplication; 9 | import zx.soft.utils.config.ConfigUtil; 10 | import zx.soft.utils.jackson.ReplaceConvert; 11 | 12 | /** 13 | * OA有害信息查询：hefei07 14 | * 根据关键词keywords查询最近7天的数据，并根据评分排序，返回num条结果。 15 | * 示例：http://localhost:4900/harmful/{keywords}/{num} 16 | * 17 | * 运行目录：/home/zxdfs/run-work/api/oa-harmful 18 | * 运行命令：cd sentiment-web 19 | * bin/ctl.sh start harmfulInfoServer 20 | * 21 | * @author wanggang 22 | * 23 | */ 24 | public class HarmfulInfoServer { 25 | 26 | private final Component component; 27 | private final HarmfulInfoApplication harmfulInfoApplication; 28 | 29 | private final int PORT; 30 | 31 | public HarmfulInfoServer() { 32 | Properties props = ConfigUtil.getProps("web-server.properties"); 33 | PORT = Integer.parseInt(props.getProperty("api.port")); 34 | component = new Component(); 35 | harmfulInfoApplication = new HarmfulInfoApplication(); 36 | } 37 | 38 | /** 39 | * 主函数 40 | */ 41 | public static void main(String[] args) { 42 | 43 | HarmfulInfoServer specialServer = new HarmfulInfoServer(); 44 | specialServer.start(); 45 | } 46 | 47 | public void start() { 48 | component.getServers().add(Protocol.HTTP, PORT); 49 | try { 50 | component.getDefaultHost().attach("/harmful", harmfulInfoApplication); 51 | ReplaceConvert.configureJacksonConverter(); 52 | component.start(); 53 | } catch (Exception e) { 54 | throw new RuntimeException(e); 55 | } 56 | } 57 | 58 | public void stop() { 59 | try { 60 | component.stop(); 61 | harmfulInfoApplication.stop(); 62 | } catch (Exception e) { 63 | throw new RuntimeException(e); 64 | } 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/server/InternetTaskServer.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.server; 2 | 3 | import java.util.Properties; 4 | 5 | import org.restlet.Component; 6 | import org.restlet.data.Protocol; 7 | 8 | import zx.soft.sent.web.application.InternetTaskApplication; 9 | import zx.soft.utils.config.ConfigUtil; 10 | import zx.soft.utils.jackson.ReplaceConvert; 11 | 12 | /** 13 | * 全网搜索任务联合接口服务：hefei05 14 | * 示例：http://192.168.32.15:3900/task 15 | * 16 | * 运行目录：/home/zxdfs/run-work/api/internettask 17 | * 运行命令：cd sentiment-web 18 | * bin/ctl.sh start internetTaskServer 19 | * 20 | * @author wanggang 21 | * 22 | */ 23 | public class InternetTaskServer { 24 | 25 | private final Component component; 26 | private final InternetTaskApplication internetTaskApplication; 27 | 28 | private final int PORT; 29 | 30 | public InternetTaskServer() { 31 | Properties props = ConfigUtil.getProps("web-server.properties"); 32 | PORT = Integer.parseInt(props.getProperty("api.port")); 33 | component = new Component(); 34 | internetTaskApplication = new InternetTaskApplication(); 35 | } 36 | 37 | /** 38 | * 主函数 39 | */ 40 | public static void main(String[] args) { 41 | 42 | InternetTaskServer specialServer = new InternetTaskServer(); 43 | specialServer.start(); 44 | } 45 | 46 | public void start() { 47 | component.getServers().add(Protocol.HTTP, PORT); 48 | try { 49 | component.getDefaultHost().attach("/task", internetTaskApplication); 50 | ReplaceConvert.configureJacksonConverter(); 51 | component.start(); 52 | } catch (Exception e) { 53 | throw new RuntimeException(e); 54 | } 55 | } 56 | 57 | public void stop() { 58 | try { 59 | component.stop(); 60 | internetTaskApplication.stop(); 61 | } catch (Exception e) { 62 | throw new RuntimeException(e); 63 | } 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/server/NicknameGroupServer.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.server; 2 | 3 | import java.util.Properties; 4 | 5 | import org.restlet.Component; 6 | import org.restlet.data.Protocol; 7 | 8 | import zx.soft.sent.web.application.NicknameGroupApplication; 9 | import zx.soft.utils.config.ConfigUtil; 10 | import zx.soft.utils.jackson.ReplaceConvert; 11 | 12 | /** 13 | * 舆情搜索Server，针对nickname：hefei02 14 | * 搜索示例：http://192.168.32.12:8901/sentiment/search?q=合肥&fq=timestamp:[2014-12-23T00:00:00Z TO 2014-12-29T23:59:59Z] 15 | * http://192.168.31.11:8902/sentiment/search?q="人"&fq=timestamp:[2015-05-06T00:00:00Z TO 2015-06-04T23:59:59Z]; 16 | * 17 | * 运行目录：/home/zxdfs/run-work/api/nicknamegroup 18 | * 运行命令： cd sentiment-web 19 | * bin/ctl.sh start nicknameGroupServer 20 | * 21 | * @author wanggang 22 | * 23 | */ 24 | public class NicknameGroupServer { 25 | 26 | private final Component component; 27 | private final NicknameGroupApplication nicknameGroupApplication; 28 | 29 | private final int PORT; 30 | 31 | public NicknameGroupServer() { 32 | Properties props = ConfigUtil.getProps("web-server.properties"); 33 | PORT = Integer.parseInt(props.getProperty("api.port")); 34 | component = new Component(); 35 | nicknameGroupApplication = new NicknameGroupApplication(); 36 | } 37 | 38 | /** 39 | * 主函数 40 | */ 41 | public static void main(String[] args) { 42 | 43 | NicknameGroupServer server = new NicknameGroupServer(); 44 | server.start(); 45 | } 46 | 47 | public void start() { 48 | component.getServers().add(Protocol.HTTP, PORT); 49 | try { 50 | component.getDefaultHost().attach("/sentiment", nicknameGroupApplication); 51 | ReplaceConvert.configureJacksonConverter(); 52 | component.start(); 53 | } catch (Exception e) { 54 | throw new RuntimeException(e); 55 | } 56 | } 57 | 58 | public void stop() { 59 | try { 60 | component.stop(); 61 | nicknameGroupApplication.close(); 62 | } catch (Exception e) { 63 | throw new RuntimeException(e); 64 | } 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/server/PullRecordServer.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.server; 2 | 3 | import java.util.Properties; 4 | 5 | import org.restlet.Component; 6 | import org.restlet.data.Protocol; 7 | 8 | import zx.soft.sent.web.application.PullRecordApplication; 9 | import zx.soft.utils.config.ConfigUtil; 10 | import zx.soft.utils.jackson.ReplaceConvert; 11 | 12 | /** 13 | * 从Mysql中根据id获取记录数据：hefei01 14 | * 示例： 15 | * 根据多个id、关键词、高亮标签查询数据：http://192.168.32.11:7900/sentiment/pull/ids/{ids}?keyword=XXXX&hlsimple=red 16 | * 17 | * 运行目录：/home/zxdfs/run-work/api/mysql 18 | * 运行命令：cd sentiment-web 19 | * bin/ctl.sh start pullRecordServer 20 | * 21 | * @author wanggang 22 | * 23 | */ 24 | public class PullRecordServer { 25 | 26 | private final Component component; 27 | private final PullRecordApplication retriveRecordApplication; 28 | 29 | private final int PORT; 30 | 31 | public PullRecordServer() { 32 | Properties props = ConfigUtil.getProps("web-server.properties"); 33 | PORT = Integer.parseInt(props.getProperty("api.port")); 34 | component = new Component(); 35 | retriveRecordApplication = new PullRecordApplication(); 36 | } 37 | 38 | /** 39 | * 主函数 40 | */ 41 | public static void main(String[] args) { 42 | 43 | PullRecordServer server = new PullRecordServer(); 44 | server.start(); 45 | 46 | } 47 | 48 | public void start() { 49 | component.getServers().add(Protocol.HTTP, PORT); 50 | try { 51 | component.getDefaultHost().attach("/sentiment", retriveRecordApplication); 52 | ReplaceConvert.configureJacksonConverter(); 53 | component.start(); 54 | } catch (Exception e) { 55 | throw new RuntimeException(e); 56 | } 57 | } 58 | 59 | public void stop() { 60 | try { 61 | component.stop(); 62 | retriveRecordApplication.close(); 63 | } catch (Exception e) { 64 | throw new RuntimeException(e); 65 | } 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/server/SentimentSearchServer.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.server; 2 | 3 | import java.util.Properties; 4 | 5 | import org.restlet.Component; 6 | import org.restlet.data.Protocol; 7 | 8 | import zx.soft.sent.web.application.SentiSearchApplication; 9 | import zx.soft.utils.config.ConfigUtil; 10 | import zx.soft.utils.jackson.ReplaceConvert; 11 | 12 | /** 13 | * 舆情搜索服务：hefei01和hefei03 14 | * 搜索示例： 15 | * http://localhost:8901/sentiment/search?q=美食&fq=platform:7&sort=lasttime:desc&start=10&rows=50&fl=username,nickname&wt=xml&hlfl=content&facetQuery=abc&facetField=nickname 16 | * 17 | * 运行目录：/home/zxdfs/run-work/api/search 18 | * 运行命令： cd sentiment-web 19 | * bin/ctl.sh start sentimentSearchServer 20 | * 21 | * @author wanggang 22 | * 23 | */ 24 | public class SentimentSearchServer { 25 | 26 | private final Component component; 27 | private final SentiSearchApplication sentiSearchApplication; 28 | 29 | private final int PORT; 30 | 31 | public SentimentSearchServer() { 32 | Properties props = ConfigUtil.getProps("web-server.properties"); 33 | PORT = Integer.parseInt(props.getProperty("api.port")); 34 | component = new Component(); 35 | sentiSearchApplication = new SentiSearchApplication(); 36 | } 37 | 38 | /** 39 | * 主函数 40 | */ 41 | public static void main(String[] args) { 42 | 43 | SentimentSearchServer server = new SentimentSearchServer(); 44 | server.start(); 45 | } 46 | 47 | public void start() { 48 | component.getServers().add(Protocol.HTTP, PORT); 49 | try { 50 | component.getDefaultHost().attach("/sentiment", sentiSearchApplication); 51 | ReplaceConvert.configureJacksonConverter(); 52 | component.start(); 53 | } catch (Exception e) { 54 | throw new RuntimeException(e); 55 | } 56 | } 57 | 58 | public void stop() { 59 | try { 60 | component.stop(); 61 | sentiSearchApplication.close(); 62 | } catch (Exception e) { 63 | throw new RuntimeException(e); 64 | } 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/server/SiteServer.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.server; 2 | 3 | import java.util.Properties; 4 | 5 | import org.restlet.Component; 6 | import org.restlet.data.Protocol; 7 | 8 | import zx.soft.sent.web.application.SiteApplication; 9 | import zx.soft.utils.config.ConfigUtil; 10 | import zx.soft.utils.jackson.ReplaceConvert; 11 | 12 | /** 13 | * 站点组合数据服务：hefei01 14 | * 示例： http://192.168.32.11:6900/site POST: [ "2344,543,23,1355", "667,99,145,987" ] 15 | * 16 | * 运行目录：/home/zxdfs/run-work/api/site 17 | * 运行命令：cd sentiment-web 18 | * bin/ctl.sh start siteServer 19 | * 20 | * @author wanggang 21 | */ 22 | public class SiteServer { 23 | 24 | private final Component component; 25 | private final SiteApplication siteApplication; 26 | 27 | private final int PORT; 28 | 29 | public SiteServer() { 30 | Properties props = ConfigUtil.getProps("web-server.properties"); 31 | PORT = Integer.parseInt(props.getProperty("api.port")); 32 | component = new Component(); 33 | siteApplication = new SiteApplication(); 34 | } 35 | 36 | /** 37 | * 主函数 38 | */ 39 | public static void main(String[] args) { 40 | 41 | SiteServer server = new SiteServer(); 42 | server.start(); 43 | 44 | } 45 | 46 | public void start() { 47 | component.getServers().add(Protocol.HTTP, PORT); 48 | try { 49 | component.getDefaultHost().attach("/site", siteApplication); 50 | ReplaceConvert.configureJacksonConverter(); 51 | component.start(); 52 | } catch (Exception e) { 53 | throw new RuntimeException(e); 54 | } 55 | } 56 | 57 | public void stop() { 58 | try { 59 | component.stop(); 60 | siteApplication.close(); 61 | } catch (Exception e) { 62 | throw new RuntimeException(e); 63 | } 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /sentiment-web/src/main/java/zx/soft/sent/web/server/SpecialSpeedServer.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.server; 2 | 3 | import java.util.Properties; 4 | 5 | import org.restlet.Component; 6 | import org.restlet.data.Protocol; 7 | 8 | import zx.soft.sent.web.application.SpecialSpeedApplication; 9 | import zx.soft.utils.config.ConfigUtil; 10 | import zx.soft.utils.jackson.ReplaceConvert; 11 | 12 | /** 13 | * OA专题查询缓存服务：hefei03 14 | * 示例：GET: http://192.168.32.13:5900/special/{identifys} 15 | * 408,409,410,412,413,414,415,416,417,419,427,450 16 | * 17 | * 运行目录：/home/zxdfs/run-work/api/special 18 | * 运行命令：cd sentiment-web 19 | * bin/ctl.sh start specialSpeedServer 20 | * 21 | * @author wanggang 22 | * 23 | */ 24 | public class SpecialSpeedServer { 25 | 26 | private final Component component; 27 | private final SpecialSpeedApplication specialApplication; 28 | 29 | private final int PORT; 30 | 31 | public SpecialSpeedServer() { 32 | Properties props = ConfigUtil.getProps("web-server.properties"); 33 | PORT = Integer.parseInt(props.getProperty("api.port")); 34 | component = new Component(); 35 | specialApplication = new SpecialSpeedApplication(); 36 | } 37 | 38 | /** 39 | * 主函数 40 | */ 41 | public static void main(String[] args) { 42 | 43 | SpecialSpeedServer specialServer = new SpecialSpeedServer(); 44 | specialServer.start(); 45 | } 46 | 47 | public void start() { 48 | component.getServers().add(Protocol.HTTP, PORT); 49 | try { 50 | component.getDefaultHost().attach("/special", specialApplication); 51 | ReplaceConvert.configureJacksonConverter(); 52 | component.start(); 53 | } catch (Exception e) { 54 | throw new RuntimeException(e); 55 | } 56 | } 57 | 58 | public void stop() { 59 | try { 60 | component.stop(); 61 | specialApplication.stop(); 62 | } catch (Exception e) { 63 | throw new RuntimeException(e); 64 | } 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /sentiment-web/src/main/resources/cache-config.properties: -------------------------------------------------------------------------------- 1 | 2 | ##### Mater-Slave (For Replicating Record-Key,Index-Data Caching) ##### 3 | # Product 4 | redis.rp.master=192.168.32.19 5 | redis.rp.slave=192.168.32.20 6 | # Testing 7 | #redis.rp.master=192.168.31.20 8 | #redis.rp.slave=192.168.31.21 9 | 10 | ##### Message-Queue (For Index-Data Caching) ##### 11 | # Product 12 | redis.mq.server=192.168.32.20 13 | # Testing 14 | #redis.mq.server=192.168.31.XX 15 | 16 | ##### Expired-Queue (For Firsttime Caching) ##### 17 | # Product 18 | redis.ft.server=192.168.32.18 19 | # Testing 20 | #redis.ft.server=192.168.31.XX 21 | 22 | ##### Port ##### 23 | redis.rp.port=16379 24 | redis.mq.port=6379 25 | redis.ft.port=6379 26 | 27 | ##### Password ##### 28 | redis.password=zxsoft -------------------------------------------------------------------------------- /sentiment-web/src/main/resources/data_db.properties: -------------------------------------------------------------------------------- 1 | 2 | ##### Database ##### 3 | # Product 4 | sent.db.url=jdbc:mysql://192.168.32.17:3306/sentiment_records?useUnicode=true&characterEncoding=utf-8 5 | # Testing 6 | #sent.db.url=jdbc:mysql://192.168.31.23:3306/sentiment_records?useUnicode=true&characterEncoding=utf-8 7 | #sent.db.url=jdbc:mysql://192.168.31.23:3306/sentiment_oa?useUnicode=true&characterEncoding=utf-8 8 | 9 | ##### Password ##### 10 | sent.db.username=sentimentrecords 11 | sent.db.password=zn/iACQ/Xh19+W6Ot2MghQ== 12 | 13 | ##### Localhost ##### 14 | #sent.db.url=jdbc:mysql://localhost:3306/sentiment_records?useUnicode=true&characterEncoding=utf-8 15 | #sent.db.username=root 16 | #sent.db.password=root -------------------------------------------------------------------------------- /sentiment-web/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 8 | 9 | 10 | 11 | INFO 12 | ACCEPT 13 | DENY 14 | 15 | 16 | 17 | 19 | logs/sentiment-web.log 20 | 21 | %d{ISO8601} [%thread] %-5level %logger{36} [Line:%-3L] - %msg%n 22 | 23 | 24 | 25 | INFO 26 | 27 | 28 | logs/sentiment-web.log.%d{yyyy-MM-dd}.gz 29 | 30 30 | 31 | 32 | 33 | 34 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /sentiment-web/src/main/resources/oracle_db.properties: -------------------------------------------------------------------------------- 1 | # db params of localhost 2 | db_url=jdbc:oracle:thin:@192.168.3.10:1521:yqjk 3 | db_username=yqjk 4 | db_password=yqjk -------------------------------------------------------------------------------- /sentiment-web/src/main/resources/solr_params.properties: -------------------------------------------------------------------------------- 1 | 2 | ##### SolrCloud ##### 3 | collection=sentiment 4 | zookeeper_connect_timeout=50000 5 | zookeeper_client_timeout=50000 6 | #zookeeper_cloud=leader:2181 7 | # Product 8 | zookeeper_cloud=192.168.32.11:2181,192.168.32.12:2181,192.168.32.13:2181,192.168.32.14:2181,192.168.32.15:2181,192.168.32.16:2181 9 | # Testing 10 | #zookeeper_cloud=192.168.31.11:2181,192.168.31.12:2181,192.168.31.20:2181,192.168.31.21:2181,192.168.31.26:2181,192.168.31.27:2181 11 | 12 | ##### Fetch Size ##### 13 | fetch_size=20000 -------------------------------------------------------------------------------- /sentiment-web/src/main/resources/web-server.properties: -------------------------------------------------------------------------------- 1 | httpConn.maxTotal=10000 2 | httpConn.maxPerRoute=5000 3 | api.port=8900 4 | -------------------------------------------------------------------------------- /sentiment-web/src/test/java/zx/soft/sent/web/json/JsonTest.java: -------------------------------------------------------------------------------- 1 | package zx.soft.sent.web.json; 2 | 3 | import org.codehaus.jackson.JsonParser; 4 | import org.codehaus.jackson.map.ObjectMapper; 5 | import org.junit.Before; 6 | import org.junit.Test; 7 | 8 | public class JsonTest { 9 | 10 | private ObjectMapper om; 11 | 12 | @Before 13 | public void initOM() { 14 | om = new ObjectMapper(); 15 | om.configure(JsonParser.Feature.ALLOW_COMMENTS, true); 16 | } 17 | 18 | @Test 19 | public void test1() throws java.io.IOException { 20 | om.readTree("/* (C) 2099 Yoyodyne Inc. */\n{ \"foo\": \"bar©\" }\n".getBytes("UTF-8")); 21 | } 22 | 23 | @Test 24 | public void test2() throws java.io.IOException { 25 | om.readTree("/* © 2099 Yoyodyne Inc. */\n{ \"foo\": \"bar©\" }\n".getBytes("UTF-8")); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /sentiment-web/src/test/resources/index_data/autmsearch: -------------------------------------------------------------------------------- 1 | {"num":10,"records":[{"id":"123456","platform":2,"mid":"","username":0,"nickname":"","original_uid":0,"original_name":"","original_title":"","original_url":"","url":"","home_url":"","title":"","type":"","content":"合山论坛发图片教程第一点击发帖按钮，如下图下载 (20.77 KB)2010-10-20 14:57然后点击浏览按钮，如下图所示点击浏览按钮之后，就可以通系统下拉按钮找到你所要发布的图片所在电脑中的位置，然后选中要发布的图片，最后点击打开，具体操作如下图下载 (28.64 KB)2010-10-20 14:57当你点击打开按钮后就会进入下图下载 (19.6 KB)2010-10-20 14:57然后，你就可以点击上传按钮，之后会有个上传等待的过程，如下图所示下载 (15.56 KB)2010-10-20 14:57过了一会儿，你所上传的图片完成后会有下图所示下载 (18.13 KB)2010-10-20 14:57你点击确定后，即完成了图片的上传，但是图片还没有发布到论坛你还要进行如下操作，图片描述你可以不写，如下所示下载 (21.1 KB)2010-10-20 14:57然后，将鼠标在空白处点击，会有个光标闪东，然后你就点击你刚才上传的图片，如下所示下载 (19.98 KB)2010-10-20 14:57之后点击发布按钮就可以完成图片的发布了，如下图下载 (26.45 KB)2010-10-20 14:57","comment_count":0,"read_count":0,"favorite_count":0,"attitude_count":0,"repost_count":0,"video_url":"","pic_url":"","voice_url":"","timestamp":1397470718,"source_id":0,"lasttime":0,"server_id":0,"identify_id":0,"identify_md5":"","keyword":"","first_time":0,"update_time":0,"ip":"","location":"","geo":"","receive_addr":"","append_addr":"","send_addr":"","source_name":"","country_code":0,"location_code":0},{"id":"456","platform":4,"mid":"","username":0,"nickname":"","original_uid":0,"original_name":"","original_title":"","original_url":"","url":"","home_url":"","title":"","type":"","content":"","comment_count":0,"read_count":0,"favorite_count":0,"attitude_count":0,"repost_count":0,"video_url":"","pic_url":"","voice_url":"","timestamp":1397384318,"source_id":0,"lasttime":0,"server_id":0,"identify_id":0,"identify_md5":"","keyword":"","first_time":0,"update_time":0,"ip":"","location":"","geo":"","receive_addr":"","append_addr":"","send_addr":"","source_name":"","country_code":0,"location_code":0}]} -------------------------------------------------------------------------------- /sentiment-web/src/test/resources/index_data/email: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wgybzbrobot/sentiment-search/df50eef8f82fbdc4549d1cfffb0d0b393a04f7b5/sentiment-web/src/test/resources/index_data/email -------------------------------------------------------------------------------- /sentiment-web/src/test/resources/index_data/postData: -------------------------------------------------------------------------------- 1 | {"num":2,"records":[{"id":"123456789","platform":7,"mid":"123abcdef","username":"987654321","nickname":"wgybzb","original_id":"5648333","original_uid":"256339988","original_name":"owgybzb","original_title":"原创标题","original_url":"http://www.baidu.com","url":"http://www.pp.cc","home_url":"http://www.google.com","title":"标题","type":"美容美食","isharmful":true,"content":"这是一条测试数据","comment_count":100,"read_count":200,"favorite_count":300,"attitude_count":400,"repost_count":500,"video_url":"http://www.video.vom","pic_url":"http://www.pic.com","voice_url":"http://www.voice.com","timestamp":129856473,"source_id":153,"lasttime":985647213,"server_id":5,"identify_id":125633,"identify_md5":"identifymd5value","keyword":"美食娱乐","first_time":1566423587,"update_time":1426879123,"ip":"192.168.1.100","location":"安徽省合肥市","geo":"经度:120.2366554,纬度:50.122599","receive_addr":"wanggang@zxils.com","append_addr":"wanggang@pp.cc","send_addr":"wgybzb@sina.cn","source_name":"新浪微博","source_type":5,"country_code":1,"location_code":2130123,"province_code":34,"city_code":12},{"id":"987654321","platform":7,"mid":"456jdjdkff","username":"2564788","nickname":"wgybzb1","original_id":"5648333","original_uid":"256339988","original_name":"owgybzb1","original_title":"原创标题","original_url":"http://www.baidu.com","url":"http://www.pp.cc","home_url":"http://www.google.com","title":"标题","type":"美容美食","isharmful":false,"content":"这是一条测试数据","comment_count":100,"read_count":200,"favorite_count":300,"attitude_count":400,"repost_count":500,"video_url":"http://www.video.vom","pic_url":"http://www.pic.com","voice_url":"http://www.voice.com","timestamp":129856473,"source_id":153,"lasttime":985647213,"server_id":5,"identify_id":125633,"identify_md5":"identifymd5value","keyword":"美食娱乐","first_time":1566423587,"update_time":1426879123,"ip":"192.168.1.100","location":"安徽省合肥市","geo":"经度:120.2366554,纬度:50.122599","receive_addr":"wanggang@zxils.com","append_addr":"wanggang@pp.cc","send_addr":"wgybzb@sina.cn","source_name":"新浪微博","source_type":5,"country_code":1,"location_code":2130123,"province_code":34,"city_code":12}]} -------------------------------------------------------------------------------- /sentiment-web/src/test/resources/index_data/qqgroup: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wgybzbrobot/sentiment-search/df50eef8f82fbdc4549d1cfffb0d0b393a04f7b5/sentiment-web/src/test/resources/index_data/qqgroup -------------------------------------------------------------------------------- /sentiment-web/src/test/resources/index_data/record: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wgybzbrobot/sentiment-search/df50eef8f82fbdc4549d1cfffb0d0b393a04f7b5/sentiment-web/src/test/resources/index_data/record -------------------------------------------------------------------------------- /sentiment-web/src/test/resources/index_data/weibo: -------------------------------------------------------------------------------- 1 | {"id":"testweibo1","platform":3,"mid":"","username":0,"nickname":"王可心521","original_uid":0,"original_name":"","original_title":"","original_url":"","url":"","home_url":"","title":"","type":"","content":"看我的发型肿么样？@王可心521","comment_count":0,"read_count":0,"favorite_count":0,"attitude_count":0,"repost_count":0,"video_url":"","pic_url":"","voice_url":"","timestamp":"Mon Aug 26 16:18:00 +0800 2013","source_id":54,"lasttime":"Fri Aug 30 16:50:07 +0800 2013","server_id":454,"identify_id":1,"identify_md5":"","keyword":"肿么","first_time":null,"update_time":"Tue Sep 03 10:35:23 +0800 2013","ip":"","location":"","geo":"","receive_addr":"","append_addr":"","send_addr":"","source_name":"","country_code":0,"location_code":0} -------------------------------------------------------------------------------- /sentiment-web/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | %d{MMdd.HHmmss.SSS} [%-20t] [%-5p] [%-20c] [L:%-3L] - %m%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /sentiment-web/src/test/resources/test-indexdata: -------------------------------------------------------------------------------- 1 | {"num":1,"records":[{"id":"sentiment","platform":10,"mid":"123456789987654321","username":"zxsoft","nickname":"中新舆情","original_id":"original_sentiment","original_uid":"original_zxsoft","original_name":"original_中新软件","original_title":"original_标题","original_url":"http://www.orignal_url.com","url":"http://www.url.com","home_url":"http://www.home_url.com","title":"标题","type":"所属类型","isharmful":true,"content":"测试内容","comment_count":10,"read_count":20,"favorite_count":30,"attitude_count":40,"repost_count":50,"video_url":"http://www.video_url.com","pic_url":"htpp://www.pic_url.com","voice_url":"http://www.voice_url.com","timestamp":1416038921,"source_id":70,"lasttime":1416125321,"server_id":90,"identify_id":100,"identify_md5":"abcdefg123456789","keyword":"关键词","first_time":1416211721,"update_time":1416298121,"ip":"192.168.32.45","location":"安徽省合肥市","geo":"经纬度信息","receive_addr":"receive@gmail.com","append_addr":"append@gmail.com","send_addr":"send@gmail.com","source_name":"新浪微博","source_type":121,"country_code":122,"location_code":123,"province_code":124,"city_code":125}]} -------------------------------------------------------------------------------- /sentiment-web/src/test/resources/test-specialdata: -------------------------------------------------------------------------------- 1 | [{"identify":"absbhdfhfjfi","name":"专题1","keywords":"（合肥 and 警察） not 打人","start":"2014-08-25 00:00:00","end":"2014-08-25 23:59:59","hometype":0},{"identify":"djdjfkfklglg","name":"专题2","keywords":"（安徽 and 城管） not 暴力执法","start":"2013-08-25 00:00:00","end":"2013-08-25 23:59:59","hometype":2}] --------------------------------------------------------------------------------