├── .gitignore ├── img └── work.png ├── lib ├── rexdb-1.0.4.jar └── javassist-3.20.0-GA.jar ├── .settings ├── org.eclipse.m2e.core.prefs └── org.eclipse.jdt.core.prefs ├── resource ├── log4j2.properties ├── banner.txt ├── config.yml └── spam.txt ├── .project ├── src └── org │ ├── youseed │ ├── spider │ │ ├── saver │ │ │ ├── MQBasic.java │ │ │ ├── SpamAnalyzer.java │ │ │ ├── mongo │ │ │ │ ├── UpdateHash.java │ │ │ │ ├── SaveStat.java │ │ │ │ └── NewHash.java │ │ │ ├── es │ │ │ │ ├── UpdateHash.java │ │ │ │ ├── UpdateHashOnTime.java │ │ │ │ └── NewHash.java │ │ │ ├── zsky │ │ │ │ ├── SaveStat.java │ │ │ │ ├── UpdateHash.java │ │ │ │ └── NewHash.java │ │ │ ├── MysqlBasic.java │ │ │ ├── ESBasic.java │ │ │ └── MongoBasic.java │ │ ├── MysqlConn.java │ │ ├── SpiderConfig.java │ │ ├── RabbitMQConn.java │ │ ├── ESConn.java │ │ ├── ConfigUtil.java │ │ └── MongoConn.java │ └── Main.java │ └── rex │ └── db │ └── configuration │ ├── R.java │ └── Configuration.java ├── .classpath ├── youseed-spider-saver.yml ├── pom.xml └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | -------------------------------------------------------------------------------- /img/work.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DHT-open/youseed-spider-saver-public/HEAD/img/work.png -------------------------------------------------------------------------------- /lib/rexdb-1.0.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DHT-open/youseed-spider-saver-public/HEAD/lib/rexdb-1.0.4.jar -------------------------------------------------------------------------------- /lib/javassist-3.20.0-GA.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DHT-open/youseed-spider-saver-public/HEAD/lib/javassist-3.20.0-GA.jar -------------------------------------------------------------------------------- /.settings/org.eclipse.m2e.core.prefs: -------------------------------------------------------------------------------- 1 | activeProfiles= 2 | eclipse.preferences.version=1 3 | resolveWorkspaceProjects=true 4 | version=1 5 | -------------------------------------------------------------------------------- /resource/log4j2.properties: -------------------------------------------------------------------------------- 1 | name=PropertiesConfig 2 | property.filename = logs 3 | appenders = console 4 | 5 | appender.console.type = Console 6 | appender.console.name = STDOUT 7 | appender.console.layout.type = PatternLayout 8 | appender.console.layout.pattern = [%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n 9 | 10 | 11 | rootLogger.level = info 12 | rootLogger.appenderRef.stdout.ref = STDOUT 13 | 14 | -------------------------------------------------------------------------------- /resource/banner.txt: -------------------------------------------------------------------------------- 1 | __ __ __ _ __ 2 | \ \/ /___ __ __________ ___ ____/ / _________ (_)___/ /__ _____ _________ __ _____ _____ 3 | \ / __ \/ / / / ___/ _ \/ _ \/ __ / / ___/ __ \/ / __ / _ \/ ___/ / ___/ __ `/ | / / _ \/ ___/ 4 | / / /_/ / /_/ (__ ) __/ __/ /_/ / (__ ) /_/ / / /_/ / __/ / (__ ) /_/ /| |/ / __/ / 5 | /_/\____/\__,_/____/\___/\___/\__,_/ /____/ .___/_/\__,_/\___/_/ /____/\__,_/ |___/\___/_/ 6 | /_/ 7 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | youseed-spider-saver-public 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.m2e.core.maven2Nature 21 | org.eclipse.jdt.core.javanature 22 | 23 | 24 | -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled 3 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 4 | org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve 5 | org.eclipse.jdt.core.compiler.compliance=1.8 6 | org.eclipse.jdt.core.compiler.debug.lineNumber=generate 7 | org.eclipse.jdt.core.compiler.debug.localVariable=generate 8 | org.eclipse.jdt.core.compiler.debug.sourceFile=generate 9 | org.eclipse.jdt.core.compiler.problem.assertIdentifier=error 10 | org.eclipse.jdt.core.compiler.problem.enumIdentifier=error 11 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning 12 | org.eclipse.jdt.core.compiler.release=disabled 13 | org.eclipse.jdt.core.compiler.source=1.8 14 | -------------------------------------------------------------------------------- /src/org/youseed/spider/saver/MQBasic.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider.saver; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.logging.log4j.LogManager; 6 | import org.apache.logging.log4j.Logger; 7 | 8 | import com.rabbitmq.client.Channel; 9 | 10 | /** 11 | * 操作MQ 12 | */ 13 | public class MQBasic { 14 | 15 | private static Logger logger = LogManager.getLogger(MQBasic.class); 16 | 17 | /** 18 | * 确认消息 19 | */ 20 | public void confirmMsg(Channel channel, long deliveryTag) { 21 | try { 22 | channel.basicAck(deliveryTag, false); 23 | } catch (IOException e) { 24 | logger.error("消息确认失败：" + e.getMessage(), e); 25 | } 26 | } 27 | 28 | /** 29 | * 退回消息 30 | */ 31 | public void rejectMsg(Channel channel, long deliveryTag) { 32 | try { 33 | channel.basicReject(deliveryTag, true); 34 | } catch (IOException e1) { 35 | logger.error("消息回退失败：" + e1.getMessage(), e1); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /src/org/rex/db/configuration/R.java: -------------------------------------------------------------------------------- 1 | package org.rex.db.configuration; 2 | 3 | import java.util.List; 4 | import java.util.Properties; 5 | 6 | import org.rex.DB; 7 | import org.rex.RMap; 8 | import org.rex.db.datasource.DataSourceFactory; 9 | import org.rex.db.datasource.SimpleDataSourceFactory; 10 | import org.rex.db.exception.DBException; 11 | 12 | public class R { 13 | 14 | public static void main(String[] args) throws DBException { 15 | Properties props = new Properties(); 16 | props.put("driverClassName", "com.mysql.jdbc.Driver"); 17 | props.put("url", "jdbc:mysql://localhost:3306/zsky?serverTimezone=GMT%2B8"); 18 | props.put("username", "root"); 19 | props.put("password", "activezz1983"); 20 | 21 | DataSourceFactory factory = new SimpleDataSourceFactory(props); 22 | Configuration conf = new Configuration(); 23 | conf.setDefaultDataSource(factory.getDataSource()); 24 | 25 | Configuration.setInstance(conf); 26 | 27 | List ml = DB.getMapList("select * from complaint"); 28 | System.out.println(ml); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/org/youseed/spider/MysqlConn.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider; 2 | 3 | import java.util.Properties; 4 | 5 | import org.apache.logging.log4j.LogManager; 6 | import org.apache.logging.log4j.Logger; 7 | import org.rex.db.configuration.Configuration; 8 | import org.rex.db.datasource.DataSourceFactory; 9 | import org.rex.db.datasource.SimpleDataSourceFactory; 10 | import org.rex.db.exception.DBException; 11 | 12 | import com.alibaba.fastjson.JSONObject; 13 | 14 | public class MysqlConn { 15 | 16 | private static Logger logger = LogManager.getLogger(MysqlConn.class); 17 | 18 | /** 19 | * 数据库已初始化？ 20 | */ 21 | public static boolean inited = false; 22 | 23 | /** 24 | * 初始化数据库 25 | */ 26 | public static synchronized void initDB() { 27 | if(inited) return; 28 | 29 | JSONObject mysql = ConfigUtil.getConfig().getJSONObject("mysql"); 30 | 31 | Properties props = new Properties(); 32 | props.put("driverClassName", "com.mysql.jdbc.Driver"); 33 | props.put("url", mysql.getString("url")); 34 | props.put("username", mysql.getString("user")); 35 | props.put("password", mysql.getString("psw")); 36 | 37 | try { 38 | DataSourceFactory factory = new SimpleDataSourceFactory(props); 39 | Configuration conf = new Configuration(); 40 | conf.setDefaultDataSource(factory.getDataSource()); 41 | Configuration.setInstance(conf); 42 | inited = true; 43 | 44 | logger.error("已初始化Mysql连接"); 45 | } catch (DBException e) { 46 | logger.error("初始化Mysql连接失败：" + e.getMessage(), e); 47 | } 48 | 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/org/youseed/spider/SpiderConfig.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider; 2 | 3 | public class SpiderConfig { 4 | 5 | //==================MQ配置 6 | public static final String MQ_VIRTUAL_HOSTS = "/"; 7 | 8 | //----用于Mongodb存储 9 | //入库的数据 10 | public static final String MQ_STORE_EXCHANGE = "store"; 11 | 12 | //新hash 13 | public static final String MQ_STORE_HASH_QUEUE = "store.new"; 14 | public static final String MQ_STORE_HASH_ROUTING = "store.new"; 15 | 16 | //待更新热度的hash 17 | public static final String MQ_STORE_UPDATE_QUEUE = "store.update"; 18 | public static final String MQ_STORE_UPDATE_ROUTING = "store.update"; 19 | 20 | //爬虫统计信息 21 | public static final String MQ_STORE_SPIDER_QUEUE = "store.stat"; 22 | public static final String MQ_STORE_SPIDER_ROUTING = "store.stat"; 23 | 24 | //----用于写搜索引擎 25 | //搜索引擎的数据 26 | public static final String MQ_ES_EXCHANGE = "es"; 27 | 28 | //新hash 29 | public static final String MQ_ES_HASH_QUEUE = "es_movie.new"; 30 | public static final String MQ_ES_HASH_ROUTING = "es_movie.new"; 31 | 32 | //待更新热度的hash 33 | public static final String MQ_ES_UPDATE_QUEUE = "es_movie.update"; 34 | public static final String MQ_ES_UPDATE_ROUTING = "es_movie.update"; 35 | 36 | 37 | //==================Mongo配置 38 | public static final String COLL_HASH = "seed_hash"; 39 | public static final String COLL_FILE = "seed_filelist"; 40 | public static final String COLL_STATE = "seed_stat"; 41 | 42 | 43 | //==================ES配置 44 | public static final String ES_INDEX_SEED="seed"; 45 | public static final String ES_TYPE_SEED = "seed"; 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/org/youseed/spider/RabbitMQConn.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider; 2 | 3 | import java.io.IOException; 4 | import java.util.concurrent.TimeoutException; 5 | 6 | import org.apache.logging.log4j.LogManager; 7 | import org.apache.logging.log4j.Logger; 8 | 9 | import com.alibaba.fastjson.JSONObject; 10 | import com.rabbitmq.client.Channel; 11 | import com.rabbitmq.client.Connection; 12 | import com.rabbitmq.client.ConnectionFactory; 13 | 14 | /** 15 | * MQ连接 16 | */ 17 | public class RabbitMQConn { 18 | 19 | private static Logger logger = LogManager.getLogger(RabbitMQConn.class); 20 | 21 | public Channel getChannel() { 22 | JSONObject config = ConfigUtil.getConfig().getJSONObject("mq"); 23 | 24 | String host = config.getString("host"); 25 | int port = config.getIntValue("port"); 26 | String username = config.getString("username"); 27 | String password = config.getString("password"); 28 | String virtualHost = config.getString("virtualHost"); 29 | 30 | logger.info("---------RabbitMQ配置------------"); 31 | logger.info("地址|mq.url: " + host); 32 | logger.info("端口|mq.port: " + port); 33 | logger.info("账户|mq.username: " + username); 34 | logger.info("虚拟目录|mq.virtualHost: " + virtualHost); 35 | logger.info("---------------------------------------------"); 36 | 37 | 38 | ConnectionFactory factory = new ConnectionFactory(); 39 | factory.setHost(host); 40 | factory.setPort(port); 41 | factory.setUsername(username); 42 | factory.setPassword(password); 43 | factory.setVirtualHost(virtualHost); 44 | 45 | Channel channel = null; 46 | try { 47 | Connection connection = factory.newConnection(); 48 | channel = connection.createChannel(); 49 | } catch (IOException | TimeoutException e) { 50 | 51 | } 52 | 53 | return channel; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /resource/config.yml: -------------------------------------------------------------------------------- 1 | #LOGO 2 | banner: banner.txt 3 | 4 | ###########写入配置############ 5 | #设置更新数据到ES搜索引擎的时间，不设置时为长驻内存并随时更新 6 | #设置时每天固定更新一次，格式为“hh:mm:ss”，例如04:00:00为每天4点更新 7 | esUpdateTime: '00:04:00' 8 | 9 | includeCategories: 10 | 11 | ###########连接配置############ 12 | #RabbitMQ连接配置 13 | mq: 14 | host: 127.0.0.1 15 | port: 5672 16 | 17 | username: youseed 18 | password: youseed 19 | virtualHost: / 20 | 21 | #MongoDB连接配置 22 | mongo: 23 | url: 127.0.0.1 24 | port: 27017 25 | db: seed 26 | admindb: 27 | user: 28 | psw: 29 | 30 | #ES搜索引擎连接配置 31 | es: 32 | url: 127.0.0.1 33 | port: 9300 34 | 35 | #mysql连接配置 36 | mysql: 37 | url: jdbc:mysql://localhost:3306/zsky?useUnicode=true&characterEncoding=utf-8&serverTimezone=GMT%2B8 38 | user: root 39 | psw: 40 | 41 | 42 | #########全局配置######### 43 | #消息队列声明和消费者绑定配置 44 | binding: 45 | es: 46 | exchage: search 47 | new: 48 | queue: search.new.1 49 | routing: 'search.new.*' 50 | update: 51 | queue: search.update.1 52 | routing: 'search.update.*' 53 | mongo: 54 | exchage: store 55 | new: 56 | queue: store.new.0 57 | routing: 'store.new.*' 58 | update: 59 | queue: store.update.0 60 | routing: 'store.update.*' 61 | stat: 62 | queue: store.stat.0 63 | routing: 'store.stat.*' 64 | mysql: 65 | exchage: store 66 | new: 67 | queue: store.new.0 68 | routing: 'store.new.*' 69 | update: 70 | queue: store.update.0 71 | routing: 'store.update.*' 72 | stat: 73 | queue: store.stat.0 74 | routing: 'store.stat.*' 75 | 76 | #存储数据的表名、索引名称等配置 77 | store: 78 | es: 79 | index: seed 80 | type: seed 81 | mongo: 82 | hash: seed_hash 83 | filelist: seed_filelist 84 | stat: seed_stat 85 | mysql: 86 | hash: search_hash 87 | filelist: search_filelist 88 | stat: search_statusreport 89 | 90 | -------------------------------------------------------------------------------- /youseed-spider-saver.yml: -------------------------------------------------------------------------------- 1 | #LOGO 2 | banner: banner.txt 3 | 4 | ###########写入配置############ 5 | #设置更新数据到ES搜索引擎的时间，不设置时为长驻内存并随时更新 6 | #设置时每天固定更新一次，格式为“hh:mm:ss”，例如04:00:00为每天4点更新 7 | esUpdateTime: '00:04:00' 8 | 9 | includeCategories: 10 | 11 | ###########连接配置############ 12 | #RabbitMQ连接配置 13 | mq: 14 | host: 127.0.0.1 15 | port: 5672 16 | 17 | username: youseed 18 | password: youseed 19 | virtualHost: / 20 | 21 | #MongoDB连接配置 22 | mongo: 23 | url: 127.0.0.1 24 | port: 27017 25 | db: seed 26 | admindb: 27 | user: 28 | psw: 29 | 30 | #ES搜索引擎连接配置 31 | es: 32 | url: 127.0.0.1 33 | port: 9300 34 | 35 | #mysql连接配置 36 | mysql: 37 | url: jdbc:mysql://localhost:3306/zsky?useUnicode=true&characterEncoding=utf-8&serverTimezone=GMT%2B8 38 | user: root 39 | psw: 40 | 41 | 42 | #########全局配置######### 43 | #消息队列声明和消费者绑定配置 44 | binding: 45 | es: 46 | exchage: search 47 | new: 48 | queue: search.new.1 49 | routing: 'search.new.*' 50 | update: 51 | queue: search.update.1 52 | routing: 'search.update.*' 53 | mongo: 54 | exchage: store 55 | new: 56 | queue: store.new.0 57 | routing: 'store.new.*' 58 | update: 59 | queue: store.update.0 60 | routing: 'store.update.*' 61 | stat: 62 | queue: store.stat.0 63 | routing: 'store.stat.*' 64 | mysql: 65 | exchage: store 66 | new: 67 | queue: store.new.0 68 | routing: 'store.new.*' 69 | update: 70 | queue: store.update.0 71 | routing: 'store.update.*' 72 | stat: 73 | queue: store.stat.0 74 | routing: 'store.stat.*' 75 | 76 | #存储数据的表名、索引名称等配置 77 | store: 78 | es: 79 | index: seed 80 | type: seed 81 | mongo: 82 | hash: seed_hash 83 | filelist: seed_filelist 84 | stat: seed_stat 85 | mysql: 86 | hash: search_hash 87 | filelist: search_filelist 88 | stat: search_statusreport 89 | 90 | -------------------------------------------------------------------------------- /src/org/youseed/spider/saver/SpamAnalyzer.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider.saver; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.IOException; 5 | import java.io.InputStreamReader; 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | import org.apache.logging.log4j.LogManager; 10 | import org.apache.logging.log4j.Logger; 11 | 12 | import com.alibaba.fastjson.JSONArray; 13 | import com.alibaba.fastjson.JSONObject; 14 | 15 | /** 16 | * 分析有无敏感词 17 | */ 18 | public class SpamAnalyzer { 19 | 20 | private static Logger logger = LogManager.getLogger(SpamAnalyzer.class); 21 | 22 | static final String SPAM = "spam.txt"; 23 | 24 | static List spams = readSpam(); 25 | 26 | /** 27 | * 判断是否敏感词，名称和文件列表（前5）都判断 28 | */ 29 | public static boolean isSpam(String name, JSONArray filelist5) { 30 | StringBuffer sb = new StringBuffer(name).append(" "); 31 | 32 | if(filelist5 != null) { 33 | for (int i = 0; i < filelist5.size(); i++) { 34 | JSONObject file = filelist5.getJSONObject(i); 35 | sb.append(file.getString("path")).append(" "); 36 | } 37 | } 38 | 39 | for(String spam : spams) { 40 | if(sb.toString().contains(spam)) { 41 | return true; 42 | } 43 | } 44 | return false; 45 | } 46 | 47 | /** 48 | * 读取关键字 49 | */ 50 | private static List readSpam() { 51 | InputStreamReader ir = new InputStreamReader(SpamAnalyzer.class.getClassLoader().getResourceAsStream(SPAM)); 52 | BufferedReader bf = new BufferedReader(ir); 53 | 54 | List list = new ArrayList(); 55 | String str; 56 | try { 57 | // 按行读取字符串 58 | while ((str = bf.readLine()) != null) { 59 | list.add(str); 60 | } 61 | 62 | bf.close(); 63 | ir.close(); 64 | 65 | } catch (IOException e) { 66 | } 67 | 68 | //加载关键词 69 | logger.info("敏感关键词数量：" + list.size()); 70 | return list; 71 | } 72 | 73 | //test 74 | public static void main(String[] args) { 75 | System.out.println(isSpam("porn", null)); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | youseed-spider-saver-public 6 | youseed-spider-saver-public 7 | 1.0.0 8 | 9 | 10 | org.apache.logging.log4j 11 | log4j-api 12 | 2.11.1 13 | 14 | 15 | org.apache.logging.log4j 16 | log4j-core 17 | 2.11.1 18 | 19 | 20 | org.slf4j 21 | slf4j-nop 22 | 1.7.2 23 | 24 | 25 | org.yaml 26 | snakeyaml 27 | 1.23 28 | 29 | 30 | com.rabbitmq 31 | amqp-client 32 | 5.5.0 33 | 34 | 35 | com.alibaba 36 | fastjson 37 | 1.2.49 38 | 39 | 40 | org.mongodb 41 | mongo-java-driver 42 | 3.8.1 43 | 44 | 45 | org.elasticsearch.client 46 | transport 47 | 6.4.0 48 | 49 | 50 | mysql 51 | mysql-connector-java 52 | 6.0.6 53 | 54 | 55 | 56 | 57 | src 58 | 59 | 60 | maven-compiler-plugin 61 | 3.7.0 62 | 63 | 1.8 64 | 1.8 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /src/org/youseed/spider/ESConn.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider; 2 | 3 | import java.net.InetAddress; 4 | import java.net.UnknownHostException; 5 | 6 | import org.apache.logging.log4j.LogManager; 7 | import org.apache.logging.log4j.Logger; 8 | import org.elasticsearch.action.index.IndexRequestBuilder; 9 | import org.elasticsearch.action.index.IndexResponse; 10 | import org.elasticsearch.client.transport.TransportClient; 11 | import org.elasticsearch.common.settings.Settings; 12 | import org.elasticsearch.common.transport.TransportAddress; 13 | import org.elasticsearch.common.xcontent.XContentType; 14 | import org.elasticsearch.transport.client.PreBuiltTransportClient; 15 | 16 | import com.alibaba.fastjson.JSONObject; 17 | 18 | /** 19 | * Elasticsearch 20 | */ 21 | public class ESConn { 22 | 23 | private static Logger logger = LogManager.getLogger(ESConn.class); 24 | 25 | private TransportClient client = null; 26 | 27 | public ESConn() { 28 | JSONObject config = ConfigUtil.getConfig().getJSONObject("es"); 29 | String url = config.getString("url"); 30 | int port = config.getIntValue("port"); 31 | 32 | logger.info("---------Elasticsearch配置------------"); 33 | logger.info("地址|es.url: " + url); 34 | logger.info("端口|es.port: " + port); 35 | logger.info("---------------------------------------------"); 36 | 37 | try { 38 | client = new PreBuiltTransportClient(Settings.EMPTY) 39 | .addTransportAddress(new TransportAddress(InetAddress.getByName(url), port)); 40 | } catch (UnknownHostException e) { 41 | throw new RuntimeException(e); 42 | } 43 | } 44 | 45 | 46 | public TransportClient getClient() { 47 | return client; 48 | } 49 | 50 | public IndexRequestBuilder getindexBuilder(String indexName) { 51 | return client.prepareIndex(indexName, "main"); 52 | } 53 | 54 | /** 55 | */ 56 | public void indexData(String indexName, String json) { 57 | IndexResponse response = client.prepareIndex(indexName, "main").setSource(json, XContentType.JSON).execute() 58 | .actionGet(); 59 | } 60 | 61 | /** 62 | */ 63 | public void indexData(String indexName, String[] json) { 64 | IndexRequestBuilder builder = client.prepareIndex(indexName, "main"); 65 | for (int i = 0; i < json.length; i++) { 66 | builder.setSource(json[i], XContentType.JSON); 67 | } 68 | builder.execute(); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/org/youseed/spider/saver/mongo/UpdateHash.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider.saver.mongo; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.logging.log4j.LogManager; 6 | import org.apache.logging.log4j.Logger; 7 | import org.youseed.spider.MongoConn; 8 | import org.youseed.spider.RabbitMQConn; 9 | import org.youseed.spider.saver.MongoBasic; 10 | 11 | import com.alibaba.fastjson.JSON; 12 | import com.alibaba.fastjson.JSONArray; 13 | import com.rabbitmq.client.AMQP; 14 | import com.rabbitmq.client.Channel; 15 | import com.rabbitmq.client.DefaultConsumer; 16 | import com.rabbitmq.client.Envelope; 17 | 18 | /** 19 | * 更新将消息队列中的Hash 20 | */ 21 | public class UpdateHash extends MongoBasic { 22 | 23 | /** 24 | * 消费者标签 25 | */ 26 | static final String CONSUME_TAG = "mongo-update-consumer"; 27 | 28 | private static Logger logger = LogManager.getLogger(UpdateHash.class); 29 | 30 | RabbitMQConn mq = new RabbitMQConn(); 31 | 32 | MongoConn mongo = new MongoConn(); 33 | 34 | public UpdateHash() { 35 | super(); 36 | } 37 | 38 | /** 39 | * 更新Hash 40 | */ 41 | public void consume() throws IOException { 42 | 43 | //监听消息 44 | Channel channel = mq.getChannel(); 45 | channel.queueDeclare(mqMongoUpdateQueue, true, false, false, null); 46 | channel.queueBind(mqMongoUpdateQueue, mqMongoExchange, mqMongoUpdateRouting); 47 | channel.basicConsume(mqMongoUpdateQueue, false, CONSUME_TAG, new DefaultConsumer(channel) { 48 | 49 | @Override 50 | public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, 51 | byte[] body) { 52 | 53 | long deliveryTag = envelope.getDeliveryTag(); 54 | 55 | //1解析数据 56 | JSONArray data = null; 57 | try { 58 | data = JSON.parseArray(new String(body, "UTF-8")); 59 | } catch (Exception e) { 60 | logger.error("JSON解析失败，提交成功并跳过消息：" + e.getMessage(), e); 61 | confirmMsg(channel, deliveryTag); 62 | return; 63 | } 64 | 65 | //2更新数据 66 | //2.1解析数据 67 | int size = data.size(); 68 | if(size == 0) { 69 | logger.info("空数据，跳过"); 70 | confirmMsg(channel, deliveryTag); 71 | return; 72 | } 73 | 74 | logger.info("待更新：" + size); 75 | 76 | //2.2更新 77 | try { 78 | int cnt = bulkUpdate(mongo, data); 79 | logger.info("实际更新：" + cnt); 80 | }catch(Exception e) { 81 | logger.error("更新失败：" + e.getMessage(), e); 82 | testMongoConn(mongo, channel, deliveryTag); 83 | return; 84 | } 85 | 86 | //3确认消息 87 | confirmMsg(channel, deliveryTag); 88 | } 89 | }); 90 | } 91 | 92 | //test 93 | public static void main(String[] args) throws IOException { 94 | UpdateHash main = new UpdateHash(); 95 | main.consume(); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/org/youseed/spider/saver/es/UpdateHash.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider.saver.es; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.logging.log4j.LogManager; 6 | import org.apache.logging.log4j.Logger; 7 | import org.elasticsearch.client.transport.TransportClient; 8 | import org.youseed.spider.ESConn; 9 | import org.youseed.spider.RabbitMQConn; 10 | import org.youseed.spider.saver.ESBasic; 11 | 12 | import com.alibaba.fastjson.JSON; 13 | import com.alibaba.fastjson.JSONArray; 14 | import com.rabbitmq.client.AMQP; 15 | import com.rabbitmq.client.Channel; 16 | import com.rabbitmq.client.DefaultConsumer; 17 | import com.rabbitmq.client.Envelope; 18 | 19 | /** 20 | * 更新Hash热度（用于不间断监听更新） 21 | */ 22 | public class UpdateHash extends ESBasic { 23 | 24 | /** 25 | * 消费者标签 26 | */ 27 | static final String CONSUME_TAG = "es-update-consumer"; 28 | 29 | private static Logger logger = LogManager.getLogger(UpdateHash.class); 30 | 31 | RabbitMQConn mq = new RabbitMQConn(); 32 | ESConn es = new ESConn(); 33 | 34 | /** 35 | * 构造函数 36 | */ 37 | public UpdateHash() { 38 | super(); 39 | logger.info("starting update hash consumer..."); 40 | } 41 | 42 | /** 43 | * 更新Hash 44 | */ 45 | public void consume() throws IOException { 46 | 47 | TransportClient client = es.getClient(); 48 | Channel channel = mq.getChannel(); 49 | 50 | channel.queueDeclare(mqEsUpdateQueue, true, false, false, null); 51 | channel.queueBind(mqEsUpdateQueue, mqEsExchange, mqEsUpdateRouting); 52 | channel.basicConsume(mqEsUpdateQueue, false, CONSUME_TAG, new DefaultConsumer(channel) { 53 | 54 | @Override 55 | public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, byte[] body) { 56 | 57 | logger.info("---------------------"); 58 | long deliveryTag = envelope.getDeliveryTag(); 59 | 60 | //1解析数据 61 | JSONArray data = null; 62 | try { 63 | data = JSON.parseArray(new String(body, "UTF-8")); 64 | logger.info("获取到数据，条目: " + data.size()); 65 | } catch (Exception e) { 66 | logger.error("JSON解析失败，提交成功并跳过消息：" + e.getMessage(), e); 67 | confirmMsg(channel, deliveryTag); 68 | return; 69 | } 70 | 71 | //2提交处理 72 | try { 73 | int cnt = batchUpdateHash(client, data); 74 | logger.info("更新条目: " + cnt); 75 | }catch(Exception e) { 76 | logger.error("更新出错: " + e.getMessage(), e); 77 | testEsConn(client, channel, deliveryTag); 78 | return; 79 | } 80 | 81 | //3确认消息 82 | confirmMsg(channel, deliveryTag); 83 | } 84 | }); 85 | } 86 | 87 | 88 | /** 89 | * 测试 90 | */ 91 | public static void main(String[] args) throws IOException { 92 | UpdateHash main = new UpdateHash(); 93 | main.consume(); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/org/youseed/spider/saver/mongo/SaveStat.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider.saver.mongo; 2 | 3 | import java.io.IOException; 4 | import java.text.SimpleDateFormat; 5 | 6 | import org.apache.logging.log4j.LogManager; 7 | import org.apache.logging.log4j.Logger; 8 | import org.bson.Document; 9 | import org.youseed.spider.MongoConn; 10 | import org.youseed.spider.RabbitMQConn; 11 | import org.youseed.spider.SpiderConfig; 12 | import org.youseed.spider.saver.MongoBasic; 13 | 14 | import com.alibaba.fastjson.JSON; 15 | import com.alibaba.fastjson.JSONObject; 16 | import com.rabbitmq.client.AMQP; 17 | import com.rabbitmq.client.Channel; 18 | import com.rabbitmq.client.DefaultConsumer; 19 | import com.rabbitmq.client.Envelope; 20 | 21 | /** 22 | * 保存爬虫统计信息 23 | */ 24 | public class SaveStat extends MongoBasic { 25 | 26 | /** 27 | * 消费者标签 28 | */ 29 | static final String CONSUME_TAG = "mongo-stat-consumer"; 30 | 31 | private static Logger logger = LogManager.getLogger(SaveStat.class); 32 | 33 | RabbitMQConn mq = new RabbitMQConn(); 34 | 35 | MongoConn mongo = new MongoConn(); 36 | 37 | public SaveStat() { 38 | super(); 39 | } 40 | 41 | /** 42 | * 处理Hash 43 | */ 44 | public void consume() throws IOException { 45 | 46 | //监听消息 47 | Channel channel = mq.getChannel(); 48 | channel.queueDeclare(mqMongoStatQueue, true, false, false, null); 49 | channel.queueBind(mqMongoStatQueue, mqMongoExchange, mqMongoStatRouting); 50 | channel.basicConsume(mqMongoStatQueue, false, CONSUME_TAG, new DefaultConsumer(channel) { 51 | 52 | @Override 53 | public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, 54 | byte[] body) { 55 | 56 | long deliveryTag = envelope.getDeliveryTag(); 57 | 58 | //1解析数据 59 | JSONObject data = null; 60 | try { 61 | data = JSON.parseObject(new String(body, "UTF-8")); 62 | 63 | String dateStr = data.getString("date"); 64 | data.put("date", new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").parse(dateStr)); 65 | logger.info("已保存，爬虫" + data.getString("spider") + "|" + dateStr); 66 | } catch (Exception e) { 67 | logger.error("JSON解析失败，提交成功并跳过消息：" + e.getMessage(), e); 68 | confirmMsg(channel, deliveryTag); 69 | return; 70 | } 71 | 72 | //2保存数据 73 | try { 74 | mongo.save(SpiderConfig.COLL_STATE, new Document(data)); 75 | } catch (Exception e) { 76 | logger.error("保存失败：" + e.getMessage(), e); 77 | testMongoConn(mongo, channel, deliveryTag); 78 | return; 79 | } 80 | 81 | //3确认消息 82 | confirmMsg(channel, deliveryTag); 83 | } 84 | 85 | }); 86 | } 87 | 88 | //test 89 | public static void main(String[] args) throws IOException { 90 | SaveStat main = new SaveStat(); 91 | main.consume(); 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Youseed磁力爬虫入库程序 # 2 | 3 | 此程序使用Java编写，负责将rabbitMQ消息队列中的数据保存至数据库或者搜索引擎。 4 | 5 | ![](img/work.png) 6 | 7 | 注意：此程序是上图右侧方框“保存磁力数据”的部分。 8 | 9 | - DHT爬虫：[https://github.com/dht-open/youseed-spider-public](https://github.com/DHT-open/youseed-spider-public) 10 | - 爬虫数据入库：[https://github.com/DHT-open/youseed-spider-saver-public](https://github.com/DHT-open/youseed-spider-saver-public) 11 | 12 | *此程序仅用作技术学习和研究* 13 | 14 | # 功能 # 15 | 16 | 读取消息队列，将爬虫抓取到的数据保存至： 17 | 18 | - Youseed Mongodb数据库； 19 | - Youseed Elasticsearch搜索引擎； 20 | - “纸上烤鱼磁力搜索引擎”数据库 21 | 22 | 23 | **注意**：此爬虫程序主要负责保存数据，需要配合“dht_spider.py”，或者“dht_spider_zsky.py”爬虫程序使用。 24 | 25 | # 程序特点 # 26 | 27 | 1. 兼容性：支持Mongodb、Mysql和Elasticsearch搜索引擎； 28 | 2. 实时和定时：支持Elasticsearch中新资源的实时索引，支持旧资源的定时更新； 29 | 3. 支持不良资源鉴定：依据`spam.txt`中的关键字鉴别不良资源，并予以标记 30 | 31 | 32 | # 硬件要求 # 33 | 34 | - 内存：约200M 35 | 36 | # 软件要求 # 37 | 38 | 需要安装以下软件： 39 | 40 | - jdk运行环境 41 | 42 | # 安装（以centos7为例） # 43 | 44 | ## 安装JDK ## 45 | 46 | yum install java-1.8.0-openjdk.x86_64 47 | 48 | ## 下载程序 ## 49 | 50 | 将编译好的jar包`spider-saver-public-1.0.0.jar`和配置文件`youseed-spider-saver.yml`下载至本地。 51 | 52 | ## 修改配置 ## 53 | 54 | 编辑文件`youseed-spider-saver.yml`，修改连接配置： 55 | 56 | #MongoDB连接配置 57 | mongo: 58 | url: 127.0.0.1 59 | port: 27017 60 | db: seed 61 | admindb: 62 | user: 63 | psw: 64 | 65 | #ES搜索引擎连接配置 66 | es: 67 | url: 127.0.0.1 68 | port: 9300 69 | 70 | #mysql连接配置（for 纸上烤鱼） <------------------纸上烤鱼程序修改这个连接 71 | mysql: 72 | url: jdbc:mysql://localhost:3306/zsky?useUnicode=true&characterEncoding=utf-8&serverTimezone=GMT%2B8 73 | user: root 74 | psw: 75 | 76 | 77 | # 运行 # 78 | 79 | 编译好的程序在这里下载： 80 | [https://github.com/DHT-open/youseed-spider-saver-public/releases](https://github.com/DHT-open/youseed-spider-saver-public/releases "https://github.com/DHT-open/youseed-spider-saver-public/releases") 81 | 82 | ## 控制台运行 ## 83 | 使用如下命令在控制台运行入库程序（注意--config的值必须是绝对路径）： 84 | 85 | java -jar -Xms50m -Xmx128m /opt/spider/app/youseed-spider-saver-public-1.0.0.jar --config=/opt/spider/app/youseed-spider-saver.yml 86 | 87 | 程序会输出可选命令： 88 | 89 | m: 写入/更新Mongodb 90 | m1: |-------写入新资源到Mongo 91 | m2: |-------更新Mongo 92 | m3: |-------写入统计到Mongo 93 | es: 写入/更新ES（根据esUpdateTime设置，自动选择实时或定时更新） 94 | es1: |-------写入新资源到ES 95 | es2: |-------更新ES（常驻内存并实时更新） 96 | es3: |-------更新ES（更新完毕当前批次后关闭） 97 | zsky: 写入/更新纸上烤鱼（zsky） 98 | zsky1: |-------写入新资源到Mysql 99 | zsky2: |-------更新Mysql 100 | zsky3: |-------写入统计到Mysql 101 | 102 | 请选择一项操作（输入编号后回车）: 103 | 104 | 接下来输入`zsky`保存到“纸上烤鱼”数据库 105 | 106 | ## 后台运行 ## 107 | 输入如下命令，后台启动“纸上烤鱼”入库 108 | 109 | nohup java -jar -Xms50m -Xmx128m /opt/spider/app/youseed-spider-saver-public-1.0.0.jar --config=/opt/spider/app/youseed-spider-saver.yml zsky > /opt/spider/logs/spider-saver-mongo.log 2>&1 & 110 | 111 | -------------------------------------------------------------------------------- /src/org/youseed/spider/saver/zsky/SaveStat.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider.saver.zsky; 2 | 3 | import java.io.IOException; 4 | import java.text.SimpleDateFormat; 5 | 6 | import org.apache.logging.log4j.LogManager; 7 | import org.apache.logging.log4j.Logger; 8 | import org.rex.DB; 9 | import org.rex.db.Ps; 10 | import org.youseed.spider.MongoConn; 11 | import org.youseed.spider.RabbitMQConn; 12 | import org.youseed.spider.saver.MysqlBasic; 13 | 14 | import com.alibaba.fastjson.JSON; 15 | import com.alibaba.fastjson.JSONObject; 16 | import com.rabbitmq.client.AMQP; 17 | import com.rabbitmq.client.Channel; 18 | import com.rabbitmq.client.DefaultConsumer; 19 | import com.rabbitmq.client.Envelope; 20 | 21 | /** 22 | * 保存爬虫统计信息 23 | */ 24 | public class SaveStat extends MysqlBasic { 25 | 26 | /** 27 | * 消费者标签 28 | */ 29 | static final String CONSUME_TAG = "mysql-zsky-stat-consumer"; 30 | 31 | private static Logger logger = LogManager.getLogger(SaveStat.class); 32 | 33 | RabbitMQConn mq = new RabbitMQConn(); 34 | 35 | MongoConn mongo = new MongoConn(); 36 | 37 | static final String INSERT_REPORT = "INSERT INTO search_statusreport(date,new_hashes,total_requests, valid_requests) VALUES(?,?,?,?)"; 38 | 39 | public SaveStat() { 40 | super(); 41 | } 42 | 43 | /** 44 | * 处理Hash 45 | */ 46 | public void consume() throws IOException { 47 | 48 | //监听消息 49 | Channel channel = mq.getChannel(); 50 | channel.queueDeclare(mqMysqlStatQueue, true, false, false, null); 51 | channel.queueBind(mqMysqlStatQueue, mqMysqlExchange, mqMysqlStatRouting); 52 | channel.basicConsume(mqMysqlStatQueue, false, CONSUME_TAG, new DefaultConsumer(channel) { 53 | 54 | @Override 55 | public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, 56 | byte[] body) { 57 | 58 | long deliveryTag = envelope.getDeliveryTag(); 59 | 60 | //1解析数据 61 | JSONObject data = null; 62 | try { 63 | data = JSON.parseObject(new String(body, "UTF-8")); 64 | 65 | String dateStr = data.getString("date"); 66 | data.put("date", new SimpleDateFormat("yyyy-MM-dd hh:mm:ss").parse(dateStr)); 67 | logger.info("爬虫" + data.getString("spider") + "|" + dateStr); 68 | } catch (Exception e) { 69 | logger.error("JSON解析失败，提交成功并跳过消息：" + e.getMessage(), e); 70 | confirmMsg(channel, deliveryTag); 71 | return; 72 | } 73 | 74 | //2保存数据 75 | try { 76 | int newHash = data.getIntValue("num_new"); 77 | int total = data.containsKey("total") ? data.getIntValue("total") : 0; 78 | int valid = data.getIntValue("num_new") + data.getIntValue("num_stored"); 79 | 80 | DB.update(INSERT_REPORT, new Ps(data.get("date"), newHash, total, valid)); 81 | 82 | } catch (Exception e) { 83 | logger.error("保存失败：" + e.getMessage(), e); 84 | testMysqlConn(channel, deliveryTag); 85 | return; 86 | } 87 | 88 | //3确认消息 89 | confirmMsg(channel, deliveryTag); 90 | } 91 | 92 | }); 93 | } 94 | 95 | //test 96 | public static void main(String[] args) throws IOException { 97 | SaveStat main = new SaveStat(); 98 | main.consume(); 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /src/org/youseed/spider/saver/zsky/UpdateHash.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider.saver.zsky; 2 | 3 | import java.io.IOException; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import org.apache.logging.log4j.LogManager; 8 | import org.apache.logging.log4j.Logger; 9 | import org.rex.DB; 10 | import org.rex.db.Ps; 11 | import org.youseed.spider.MysqlConn; 12 | import org.youseed.spider.RabbitMQConn; 13 | import org.youseed.spider.saver.MysqlBasic; 14 | 15 | import com.alibaba.fastjson.JSON; 16 | import com.alibaba.fastjson.JSONArray; 17 | import com.rabbitmq.client.AMQP; 18 | import com.rabbitmq.client.Channel; 19 | import com.rabbitmq.client.DefaultConsumer; 20 | import com.rabbitmq.client.Envelope; 21 | 22 | /** 23 | * 更新将消息队列中的Hash 24 | */ 25 | public class UpdateHash extends MysqlBasic { 26 | 27 | /** 28 | * 消费者标签 29 | */ 30 | static final String CONSUME_TAG = "mysql-zsky-update-consumer"; 31 | 32 | private static Logger logger = LogManager.getLogger(UpdateHash.class); 33 | 34 | RabbitMQConn mq = new RabbitMQConn(); 35 | 36 | public UpdateHash() { 37 | super(); 38 | } 39 | 40 | final static String UPDATE_HASH = "UPDATE search_hash SET last_seen=now(),requests=requests+1 WHERE info_hash like ?"; 41 | 42 | /** 43 | * 更新Hash 44 | */ 45 | public void consume() throws IOException { 46 | 47 | //初始化数据库连接 48 | MysqlConn.initDB(); 49 | 50 | //监听消息 51 | Channel channel = mq.getChannel(); 52 | channel.queueDeclare(mqMysqlUpdateQueue, true, false, false, null); 53 | channel.queueBind(mqMysqlUpdateQueue, mqMysqlExchange, mqMysqlUpdateRouting); 54 | channel.basicConsume(mqMysqlUpdateQueue, false, CONSUME_TAG, new DefaultConsumer(channel) { 55 | 56 | @Override 57 | public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, 58 | byte[] body) { 59 | 60 | long deliveryTag = envelope.getDeliveryTag(); 61 | 62 | //1解析数据 63 | JSONArray data = null; 64 | try { 65 | data = JSON.parseArray(new String(body, "UTF-8")); 66 | } catch (Exception e) { 67 | logger.error("JSON解析失败，提交成功并跳过消息：" + e.getMessage(), e); 68 | confirmMsg(channel, deliveryTag); 69 | return; 70 | } 71 | 72 | //2更新数据 73 | //2.1解析数据 74 | int size = data.size(); 75 | if(size == 0) { 76 | logger.info("空数据，跳过"); 77 | confirmMsg(channel, deliveryTag); 78 | return; 79 | } 80 | 81 | logger.info("待更新：" + size); 82 | 83 | //2.2更新 84 | try { 85 | List pss = new ArrayList(); 86 | for (int i = 0; i < data.size(); i++) { 87 | pss.add(new Ps().add(data.get(i) + "%")); 88 | } 89 | 90 | int[] cnts = DB.batchUpdate(UPDATE_HASH, pss); 91 | 92 | int n = 0; 93 | for (int i = 0; i < cnts.length; i++) { 94 | n += cnts[i]; 95 | } 96 | logger.info("实际更新：" + n); 97 | }catch(Exception e) { 98 | logger.error("更新失败：" + e.getMessage(), e); 99 | testMysqlConn(channel, deliveryTag); 100 | return; 101 | } 102 | 103 | //3确认消息 104 | confirmMsg(channel, deliveryTag); 105 | } 106 | }); 107 | } 108 | 109 | //test 110 | public static void main(String[] args) throws IOException { 111 | UpdateHash main = new UpdateHash(); 112 | main.consume(); 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/org/youseed/spider/saver/es/UpdateHashOnTime.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider.saver.es; 2 | 3 | import java.io.IOException; 4 | import java.util.concurrent.TimeoutException; 5 | 6 | import org.apache.logging.log4j.LogManager; 7 | import org.apache.logging.log4j.Logger; 8 | import org.elasticsearch.client.transport.TransportClient; 9 | import org.youseed.spider.ESConn; 10 | import org.youseed.spider.RabbitMQConn; 11 | import org.youseed.spider.saver.ESBasic; 12 | 13 | import com.alibaba.fastjson.JSON; 14 | import com.alibaba.fastjson.JSONArray; 15 | import com.rabbitmq.client.AMQP; 16 | import com.rabbitmq.client.Channel; 17 | import com.rabbitmq.client.DefaultConsumer; 18 | import com.rabbitmq.client.Envelope; 19 | 20 | /** 21 | * 更新Hash热度(用于定时任务调用，例如每天执行一次更新，完毕后即退出) 22 | */ 23 | public class UpdateHashOnTime extends ESBasic { 24 | 25 | /** 26 | * 消费者标签 27 | */ 28 | static final String CONSUME_TAG = "es-update-consumer"; 29 | 30 | private static Logger logger = LogManager.getLogger(UpdateHashOnTime.class); 31 | 32 | RabbitMQConn mq = new RabbitMQConn(); 33 | ESConn es = new ESConn(); 34 | 35 | /** 36 | * 构造函数 37 | */ 38 | public UpdateHashOnTime() { 39 | super(); 40 | logger.info("starting update hash(on time) consumer..."); 41 | } 42 | 43 | /** 44 | * 更新Hash 45 | */ 46 | public void consume() throws IOException { 47 | TransportClient client = es.getClient(); 48 | Channel channel = mq.getChannel(); 49 | 50 | channel.queueDeclare(mqEsUpdateQueue, true, false, false, null); 51 | channel.queueBind(mqEsUpdateQueue, mqEsExchange, mqEsUpdateRouting); 52 | 53 | final long count = channel.messageCount(mqEsUpdateQueue); 54 | logger.info("待更新Hash批次数：" + count); 55 | 56 | channel.basicConsume(mqEsUpdateQueue, false, CONSUME_TAG, new DefaultConsumer(channel) { 57 | 58 | //需执行的批次 59 | long batch = count; 60 | 61 | @Override 62 | public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, byte[] body) { 63 | 64 | //判断此批次是否已经处理完成 65 | if(batch <= 0) { 66 | try { 67 | channel.close(); 68 | logger.info("批次任务已经更新完毕，程序退出。"); 69 | return; 70 | } catch (IOException | TimeoutException e) { 71 | } 72 | 73 | }else { 74 | logger.info("剩余更新批次：" + batch); 75 | batch--; 76 | } 77 | 78 | logger.info("---------------------"); 79 | long deliveryTag = envelope.getDeliveryTag(); 80 | 81 | //1解析数据 82 | JSONArray data = null; 83 | try { 84 | data = JSON.parseArray(new String(body, "UTF-8")); 85 | logger.info("获取到数据，条目: " + data.size()); 86 | } catch (Exception e) { 87 | logger.error("JSON解析失败，提交成功并跳过消息：" + e.getMessage(), e); 88 | confirmMsg(channel, deliveryTag); 89 | return; 90 | } 91 | 92 | //2提交处理 93 | try { 94 | int cnt = batchUpdateHash(client, data); 95 | logger.info("更新条目: " + cnt); 96 | }catch(Exception e) { 97 | logger.error("更新出错: " + e.getMessage(), e); 98 | testEsConn(client, channel, deliveryTag); 99 | return; 100 | } 101 | 102 | //3确认消息 103 | confirmMsg(channel, deliveryTag); 104 | } 105 | }); 106 | } 107 | 108 | 109 | /** 110 | * 测试 111 | */ 112 | public static void main(String[] args) throws IOException { 113 | UpdateHashOnTime main = new UpdateHashOnTime(); 114 | main.consume(); 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/org/youseed/spider/ConfigUtil.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider; 2 | 3 | import java.io.ByteArrayOutputStream; 4 | import java.io.FileNotFoundException; 5 | import java.io.FileReader; 6 | import java.io.IOException; 7 | import java.io.InputStream; 8 | import java.util.Map; 9 | 10 | import org.apache.logging.log4j.LogManager; 11 | import org.apache.logging.log4j.Logger; 12 | import org.yaml.snakeyaml.Yaml; 13 | 14 | import com.alibaba.fastjson.JSONObject; 15 | 16 | /** 17 | * 读取配置文件 18 | */ 19 | public class ConfigUtil { 20 | 21 | /** 22 | * 配置文件路径 23 | */ 24 | static final String DEFAULT_CONF = "config.yml"; 25 | 26 | static String conf = null; 27 | 28 | private static Logger logger = LogManager.getLogger(ConfigUtil.class); 29 | 30 | //-------------------------------获取properties格式配置 31 | /** 32 | * 设置配置路径 33 | */ 34 | public static void setConfPath(String path) { 35 | conf = path; 36 | logger.info("指定配置文件：" + path); 37 | } 38 | 39 | /** 40 | * 获取扁平配置，便于取值 41 | */ 42 | public static JSONObject getProperties() { 43 | JSONObject prop = new JSONObject(); 44 | JSONObject config = getConfig(); 45 | iterMaps(prop, null, config); 46 | return prop; 47 | } 48 | 49 | private static void iterMaps(JSONObject prop, String key, JSONObject val) { 50 | for (Map.Entry entry : val.entrySet()) { 51 | 52 | String k = entry.getKey(); 53 | Object v = entry.getValue(); 54 | 55 | String flatKey = key == null ? k : key + "." + k; 56 | if(v instanceof Map) { 57 | iterMaps(prop, flatKey, new JSONObject((Map)v)); 58 | }else { 59 | prop.put(flatKey, v); 60 | } 61 | } 62 | } 63 | 64 | 65 | //-------------------------------获取JSON格式配置 66 | /** 67 | * 获取FastJSON类型配置 68 | */ 69 | public static JSONObject getConfig() { 70 | Yaml yaml = new Yaml(); 71 | 72 | JSONObject config = null; 73 | if(conf == null) { 74 | logger.info("加载默认配置：" + DEFAULT_CONF); 75 | config = yaml.loadAs(Thread.currentThread().getContextClassLoader().getResourceAsStream(DEFAULT_CONF), 76 | JSONObject.class); 77 | }else { 78 | try { 79 | config = yaml.loadAs(new FileReader(conf), JSONObject.class); 80 | logger.info("加载配置：" + conf); 81 | } catch (FileNotFoundException e) { 82 | 83 | logger.error("加载配置文件出错：" + e.getMessage()); 84 | logger.info("加载默认配置：" + DEFAULT_CONF); 85 | config = yaml.loadAs(Thread.currentThread().getContextClassLoader().getResourceAsStream(DEFAULT_CONF), 86 | JSONObject.class); 87 | } 88 | } 89 | 90 | return config; 91 | } 92 | 93 | 94 | //-------------------------------输出banner 95 | /** 96 | * 输出文本内容，默认读取配置中的banner 97 | */ 98 | public static void printBanner() { 99 | printBanner(getConfig().getString("banner")); 100 | } 101 | 102 | /** 103 | * 输出文本中的内容 104 | */ 105 | public static void printBanner(String path) { 106 | InputStream is = ConfigUtil.class.getClassLoader().getResourceAsStream(path); 107 | byte[] txt; 108 | try { 109 | txt = readStream(is); 110 | System.out.println(new String(txt)); 111 | } catch (Exception e) { 112 | e.printStackTrace(); 113 | }finally { 114 | try { 115 | is.close(); 116 | } catch (IOException e) { 117 | } 118 | } 119 | } 120 | 121 | /** 122 | * 读取流 123 | */ 124 | private static byte[] readStream(InputStream inStream) throws Exception { 125 | ByteArrayOutputStream outSteam = new ByteArrayOutputStream(); 126 | byte[] buffer = new byte[1024]; 127 | int len = -1; 128 | while ((len = inStream.read(buffer)) != -1) { 129 | outSteam.write(buffer, 0, len); 130 | } 131 | outSteam.close(); 132 | inStream.close(); 133 | return outSteam.toByteArray(); 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /src/org/youseed/spider/saver/zsky/NewHash.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider.saver.zsky; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.logging.log4j.LogManager; 6 | import org.apache.logging.log4j.Logger; 7 | import org.rex.DB; 8 | import org.rex.db.Ps; 9 | import org.rex.db.exception.DBException; 10 | import org.youseed.spider.MysqlConn; 11 | import org.youseed.spider.RabbitMQConn; 12 | import org.youseed.spider.saver.MysqlBasic; 13 | 14 | import com.alibaba.fastjson.JSON; 15 | import com.alibaba.fastjson.JSONArray; 16 | import com.alibaba.fastjson.JSONObject; 17 | import com.rabbitmq.client.AMQP; 18 | import com.rabbitmq.client.Channel; 19 | import com.rabbitmq.client.DefaultConsumer; 20 | import com.rabbitmq.client.Envelope; 21 | 22 | import io.netty.util.internal.StringUtil; 23 | 24 | /** 25 | * 保存将消息队列中的新Hash 26 | * XXX:未使用批量写入，没有丢数据风险，但是性能比较差 27 | */ 28 | public class NewHash extends MysqlBasic { 29 | 30 | /** 31 | * 消费者标签 32 | */ 33 | static final String CONSUME_TAG = "mysql-zsky-new-consumer"; 34 | 35 | private static Logger logger = LogManager.getLogger(NewHash.class); 36 | 37 | RabbitMQConn mq = new RabbitMQConn(); 38 | 39 | public NewHash() { 40 | super(); 41 | } 42 | 43 | //新hash 44 | final static String INSERT_HASH = "INSERT IGNORE INTO search_hash(info_hash, category, data_hash, name, extension, source_ip, length, create_time, last_seen, requests) " 45 | + "VALUES (?,?,?,?,?,?,?,now(),now(),1)"; 46 | //新filelist 47 | final static String INSERT_FILELIST = "INSERT IGNORE INTO search_filelist(info_hash, file_list) VALUES (?,?)"; 48 | 49 | /** 50 | * 处理Hash 51 | */ 52 | public void consume() throws IOException { 53 | 54 | //初始化数据库连接 55 | MysqlConn.initDB(); 56 | 57 | //监听消息 58 | Channel channel = mq.getChannel(); 59 | channel.queueDeclare(mqMysqlNewQueue, true, false, false, null); 60 | channel.queueBind(mqMysqlNewQueue, mqMysqlExchange, mqMysqlNewRouting); 61 | channel.basicConsume(mqMysqlNewQueue, false, CONSUME_TAG, new DefaultConsumer(channel) { 62 | 63 | @Override 64 | public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, 65 | byte[] body) { 66 | 67 | long deliveryTag = envelope.getDeliveryTag(); 68 | 69 | //1.解析数据 70 | JSONObject data = null; 71 | try { 72 | data = JSON.parseObject(new String(body, "UTF-8")); 73 | } catch (Exception e) { 74 | logger.error("JSON解析失败，提交成功并跳过消息：" + e.getMessage(), e); 75 | confirmMsg(channel, deliveryTag); 76 | return; 77 | } 78 | 79 | //2.保存数据 80 | //2.1格式校验 81 | String infoHash = data.getString("info_hash"); 82 | String name = data.getString("name"); 83 | if(StringUtil.isNullOrEmpty(infoHash) || StringUtil.isNullOrEmpty(name)) { 84 | logger.info("数据格式不正确，跳过消息"); 85 | confirmMsg(channel, deliveryTag); 86 | return; 87 | } 88 | 89 | logger.info("新资源: " + infoHash); 90 | 91 | // 3保存数据 92 | try { 93 | //2.2hash 94 | DB.update(INSERT_HASH,new Ps( 95 | infoHash, 96 | data.getString("category"), 97 | data.getString("data_hash"), 98 | data.getString("name"), 99 | data.getString("extension") == null ? null : data.getString("extension").trim(), 100 | data.getString("source_ip"), 101 | data.getLong("length"))); 102 | 103 | //2.3files 104 | JSONArray filelist = data.getJSONArray("filelist"); 105 | if (filelist != null) { 106 | DB.update(INSERT_FILELIST, new Ps(infoHash, filelist.toJSONString())); 107 | } 108 | 109 | } catch (Exception e) { 110 | logger.error("保存失败：" + e.getMessage(), e); 111 | testMysqlConn(channel, deliveryTag); 112 | return; 113 | } 114 | 115 | //4确认消息 116 | confirmMsg(channel, deliveryTag); 117 | } 118 | 119 | }); 120 | } 121 | 122 | //test 123 | public static void main(String[] args) throws IOException, DBException { 124 | NewHash main = new NewHash(); 125 | main.consume(); 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /src/org/youseed/spider/saver/MysqlBasic.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider.saver; 2 | 3 | import java.sql.SQLException; 4 | 5 | import org.apache.logging.log4j.LogManager; 6 | import org.apache.logging.log4j.Logger; 7 | import org.rex.DB; 8 | import org.rex.db.exception.DBException; 9 | import org.youseed.spider.ConfigUtil; 10 | 11 | import com.alibaba.fastjson.JSONObject; 12 | import com.rabbitmq.client.Channel; 13 | 14 | /** 15 | * Mysql for zsky 16 | */ 17 | public class MysqlBasic extends MQBasic{ 18 | 19 | private static Logger logger = LogManager.getLogger(MysqlBasic.class); 20 | 21 | protected String mqMysqlExchange = "store"; 22 | 23 | protected String mqMysqlNewQueue = "store.new"; 24 | protected String mqMysqlNewRouting = "*.new"; 25 | 26 | protected String mqMysqlUpdateQueue = "store.update"; 27 | protected String mqMysqlUpdateRouting = "*.update"; 28 | 29 | protected String mqMysqlStatQueue = "store.stat"; 30 | protected String mqMysqlStatRouting = "*.stat"; 31 | 32 | protected String tableHash = "search_hash"; 33 | protected String tableFilelist = "search_filelist"; 34 | protected String tableStat = "search_statusreport"; 35 | 36 | /** 37 | * 构造函数 38 | */ 39 | public MysqlBasic() { 40 | ConfigUtil.printBanner(); 41 | 42 | JSONObject config = ConfigUtil.getProperties(); 43 | mqMysqlExchange = config.containsKey("binding.mysql.exchage") ? config.getString("binding.mysql.exchage") : mqMysqlExchange; 44 | mqMysqlNewQueue = config.containsKey("binding.mysql.new.queue") ? config.getString("binding.mysql.new.queue") : mqMysqlNewQueue; 45 | mqMysqlNewRouting = config.containsKey("binding.mysql.new.routing") ? config.getString("binding.mysql.new.routing") : mqMysqlNewRouting; 46 | mqMysqlUpdateQueue = config.containsKey("binding.mysql.update.queue") ? config.getString("binding.mysql.update.queue") : mqMysqlUpdateQueue; 47 | mqMysqlUpdateRouting = config.containsKey("binding.mysql.update.routing") ? config.getString("binding.mysql.update.routing") : mqMysqlUpdateRouting; 48 | mqMysqlStatQueue = config.containsKey("binding.mysql.stat.queue") ? config.getString("binding.mysql.stat.queue") : mqMysqlStatQueue; 49 | mqMysqlStatRouting = config.containsKey("binding.mysql.stat.routing") ? config.getString("binding.mysql.stat.routing") : mqMysqlStatRouting; 50 | 51 | tableHash = config.containsKey("store.mysql.hash") ? config.getString("store.mysql.hash") : tableHash; 52 | tableFilelist = config.containsKey("store.mysql.filelist") ? config.getString("store.mysql.filelist") : tableFilelist; 53 | tableStat = config.containsKey("store.mysql.stat") ? config.getString("store.mysql.stat") : tableStat; 54 | 55 | logger.info("---------RabbitMQ/Mysql-ZSKY绑定配置------------"); 56 | logger.info("交换器|binding.mysql.exchage: " + mqMysqlExchange); 57 | logger.info("新资源队列|binding.mysql.new.queue: " + mqMysqlNewQueue); 58 | logger.info("新资源路由|binding.mysql.new.routing: " + mqMysqlNewRouting); 59 | logger.info("更新资源队列|binding.mysql.update.queue: " + mqMysqlUpdateQueue); 60 | logger.info("更新资源路由|binding.mysql.update.routing: " + mqMysqlUpdateRouting); 61 | logger.info("爬虫统计队列|binding.mysql.stat.queue: " + mqMysqlStatQueue); 62 | logger.info("爬虫统计路由|binding.mysql.stat.routing: " + mqMysqlStatRouting); 63 | logger.info("资源明细|store.mysql.index: " + tableHash); 64 | logger.info("文件列表|store.mysql.type: " + tableFilelist); 65 | logger.info("爬虫统计信息|store.mysql.stat: " + tableStat); 66 | logger.info("---------------------------------------------"); 67 | } 68 | 69 | /** 70 | * 测试Mysql连接是否正常，并进行相应的消息处理 71 | */ 72 | public void testMysqlConn(Channel channel, long deliveryTag) { 73 | 74 | boolean alive = false; 75 | try { 76 | alive = DB.getConnection().isValid(1000);//连接是否健在 77 | } catch (SQLException | DBException e) { 78 | logger.error("测试数据库连接失败："+e.getMessage(), e); 79 | } 80 | 81 | if(alive) { 82 | logger.info("Mysql连接正常，提交确认至消息队列"); 83 | confirmMsg(channel, deliveryTag); 84 | } else { 85 | logger.info("Mysql连接失败，退回当前数据至队列"); 86 | rejectMsg(channel, deliveryTag); 87 | try { 88 | logger.info("暂停60秒..."); 89 | Thread.sleep(60000); 90 | return; 91 | } catch (InterruptedException e1) { 92 | } 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/org/youseed/spider/saver/mongo/NewHash.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider.saver.mongo; 2 | 3 | import java.io.IOException; 4 | import java.util.Calendar; 5 | import java.util.Date; 6 | 7 | import org.apache.logging.log4j.LogManager; 8 | import org.apache.logging.log4j.Logger; 9 | import org.bson.Document; 10 | import org.youseed.spider.MongoConn; 11 | import org.youseed.spider.RabbitMQConn; 12 | import org.youseed.spider.SpiderConfig; 13 | import org.youseed.spider.saver.MongoBasic; 14 | import org.youseed.spider.saver.SpamAnalyzer; 15 | 16 | import com.alibaba.fastjson.JSON; 17 | import com.alibaba.fastjson.JSONArray; 18 | import com.alibaba.fastjson.JSONObject; 19 | import com.rabbitmq.client.AMQP; 20 | import com.rabbitmq.client.Channel; 21 | import com.rabbitmq.client.DefaultConsumer; 22 | import com.rabbitmq.client.Envelope; 23 | 24 | import io.netty.util.internal.StringUtil; 25 | 26 | /** 27 | * 保存将消息队列中的新Hash 28 | */ 29 | public class NewHash extends MongoBasic { 30 | 31 | /** 32 | * 消费者标签 33 | */ 34 | static final String CONSUME_TAG = "mongo-new-consumer"; 35 | 36 | private static Logger logger = LogManager.getLogger(NewHash.class); 37 | 38 | RabbitMQConn mq = new RabbitMQConn(); 39 | 40 | MongoConn mongo = new MongoConn(); 41 | 42 | public NewHash() { 43 | super(); 44 | } 45 | 46 | /** 47 | * 处理Hash 48 | */ 49 | public void consume() throws IOException { 50 | //监听消息 51 | Channel channel = mq.getChannel(); 52 | channel.queueDeclare(mqMongoNewQueue, true, false, false, null); 53 | channel.queueBind(mqMongoNewQueue, mqMongoExchange, mqMongoNewRouting); 54 | channel.basicConsume(mqMongoNewQueue, false, CONSUME_TAG, new DefaultConsumer(channel) { 55 | 56 | @Override 57 | public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, 58 | byte[] body) { 59 | 60 | long deliveryTag = envelope.getDeliveryTag(); 61 | 62 | //1.解析数据 63 | JSONObject data = null; 64 | try { 65 | data = JSON.parseObject(new String(body, "UTF-8")); 66 | } catch (Exception e) { 67 | logger.error("JSON解析失败，提交成功并跳过消息：" + e.getMessage(), e); 68 | confirmMsg(channel, deliveryTag); 69 | return; 70 | } 71 | 72 | //2.保存数据 73 | //2.1格式校验 74 | String infoHash = data.getString("info_hash"); 75 | String name = data.getString("name"); 76 | if(StringUtil.isNullOrEmpty(infoHash) || StringUtil.isNullOrEmpty(name)) { 77 | logger.info("数据格式不正确，跳过消息"); 78 | confirmMsg(channel, deliveryTag); 79 | return; 80 | } 81 | 82 | //2.2hash 83 | String shortHash = infoHash.substring(0, 16); 84 | logger.info("新资源: " + shortHash); 85 | 86 | //2.3处理文件列表 87 | int fileCount = 0; 88 | JSONArray filelist_5 = null; 89 | 90 | JSONArray filelist = data.getJSONArray("filelist"); 91 | if (filelist != null) { 92 | fileCount = data.getIntValue("file_count"); 93 | filelist_5 = filelist.size() > 5 ? new JSONArray(filelist.subList(0, 5)) : filelist; 94 | } else { 95 | fileCount = 0; 96 | filelist_5 = new JSONArray(); 97 | } 98 | 99 | //2.4准备files数据 100 | Document file = null; 101 | 102 | if (filelist != null) { 103 | file = new Document(); 104 | file.put("short_hash", shortHash); 105 | file.put("info_hash", infoHash); 106 | file.put("file_count", fileCount); 107 | file.put("file_list", filelist); 108 | } 109 | 110 | //2.5准备hash表数据 111 | Document hash = new Document(); 112 | 113 | Date now = Calendar.getInstance().getTime(); 114 | hash.put("short_hash", shortHash); 115 | hash.put("info_hash", infoHash); 116 | hash.put("category", data.getString("category")); 117 | hash.put("data_hash", data.getString("data_hash")); 118 | hash.put("name", data.getString("name")); 119 | hash.put("file_count", fileCount); 120 | hash.put("filelist_5", filelist_5); 121 | hash.put("extension", data.getString("extension") == null ? null : data.getString("extension").trim()); 122 | hash.put("source_ip", data.getString("source_ip")); 123 | hash.put("length", data.getLong("length")); 124 | hash.put("create_time", now); 125 | hash.put("last_seen", now); 126 | hash.put("requests", 1); 127 | 128 | boolean spam = SpamAnalyzer.isSpam(data.getString("name"), filelist_5); 129 | if(spam) 130 | data.put("spam", spam); 131 | 132 | 133 | // 3保存数据 134 | try { 135 | mongo.save(SpiderConfig.COLL_HASH, hash); 136 | if (file != null) 137 | mongo.save(SpiderConfig.COLL_FILE, file); 138 | } catch (Exception e) { 139 | logger.error("保存失败：" + e.getMessage(), e); 140 | testMongoConn(mongo, channel, deliveryTag); 141 | return; 142 | } 143 | 144 | //4确认消息 145 | confirmMsg(channel, deliveryTag); 146 | } 147 | 148 | }); 149 | } 150 | 151 | //test 152 | public static void main(String[] args) throws IOException { 153 | NewHash main = new NewHash(); 154 | main.consume(); 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /src/org/youseed/spider/saver/es/NewHash.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider.saver.es; 2 | 3 | import java.io.IOException; 4 | import java.util.Arrays; 5 | import java.util.Calendar; 6 | import java.util.List; 7 | 8 | import org.apache.logging.log4j.LogManager; 9 | import org.apache.logging.log4j.Logger; 10 | import org.elasticsearch.action.index.IndexResponse; 11 | import org.elasticsearch.client.transport.TransportClient; 12 | import org.youseed.spider.ConfigUtil; 13 | import org.youseed.spider.ESConn; 14 | import org.youseed.spider.RabbitMQConn; 15 | import org.youseed.spider.SpiderConfig; 16 | import org.youseed.spider.saver.ESBasic; 17 | import org.youseed.spider.saver.SpamAnalyzer; 18 | 19 | import com.alibaba.fastjson.JSON; 20 | import com.alibaba.fastjson.JSONArray; 21 | import com.alibaba.fastjson.JSONObject; 22 | import com.rabbitmq.client.AMQP; 23 | import com.rabbitmq.client.Channel; 24 | import com.rabbitmq.client.DefaultConsumer; 25 | import com.rabbitmq.client.Envelope; 26 | 27 | import io.netty.util.internal.StringUtil; 28 | 29 | /** 30 | * 写入新资源到ES 31 | */ 32 | public class NewHash extends ESBasic { 33 | 34 | /** 35 | * 消费者标签 36 | */ 37 | static final String CONSUME_TAG_PREFIX = "es-new-consumer"; 38 | 39 | private static Logger logger = LogManager.getLogger(NewHash.class); 40 | 41 | RabbitMQConn mq = new RabbitMQConn(); 42 | 43 | ESConn es = new ESConn(); 44 | 45 | /** 46 | * 构造函数 47 | */ 48 | public NewHash() { 49 | super(); 50 | logger.info("starting new hash consumer..."); 51 | } 52 | 53 | /** 54 | * 处理Hash 55 | */ 56 | public void consume() throws IOException { 57 | 58 | //只入库指定的分类 59 | String catsConf = ConfigUtil.getConfig().getString("includeCategories"); 60 | final List cats = StringUtil.isNullOrEmpty(catsConf) ? null : Arrays.asList(catsConf.split(",")); 61 | 62 | //开始连接MQ 63 | TransportClient client = es.getClient(); 64 | Channel channel = mq.getChannel(); 65 | 66 | channel.queueDeclare(mqEsNewQueue, true, false, false, null); 67 | channel.queueBind(mqEsNewQueue, mqEsExchange, mqEsNewRouting); 68 | channel.basicConsume(mqEsNewQueue, false, CONSUME_TAG_PREFIX, new DefaultConsumer(channel) { 69 | 70 | @Override 71 | public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, 72 | byte[] body) { 73 | 74 | long deliveryTag = envelope.getDeliveryTag(); 75 | 76 | //1解析数据 77 | JSONObject data = null; 78 | try { 79 | data = JSON.parseObject(new String(body, "UTF-8")); 80 | } catch (Exception e) { 81 | logger.error("JSON解析失败，提交成功并跳过消息：" + e.getMessage(), e); 82 | confirmMsg(channel, deliveryTag); 83 | return; 84 | } 85 | 86 | //2分析数据 87 | //2.1格式校验 88 | String infoHash = data.getString("info_hash"); 89 | String cat = data.getString("category"); 90 | String name = data.getString("name"); 91 | if(StringUtil.isNullOrEmpty(cat) || StringUtil.isNullOrEmpty(infoHash) || StringUtil.isNullOrEmpty(name)) { 92 | logger.info("数据格式不正确，跳过消息"); 93 | confirmMsg(channel, deliveryTag); 94 | return; 95 | } 96 | 97 | //2.2检查分类 98 | if(cats != null) { 99 | //2.2.1类型不在设定范围，跳过消息 100 | if(!cats.contains(cat)) { 101 | confirmMsg(channel, deliveryTag); 102 | logger.info("skip: " + cat); 103 | return; 104 | } 105 | 106 | 107 | //2.2.2如果只设定了一个类型，将文件扩展名视为分类 108 | if(cats.size() == 1) { 109 | String ext = data.getString("extension").trim(); 110 | ext = ext.substring(1); 111 | data.put("category", ext); 112 | data.remove("extension"); 113 | } 114 | 115 | logger.info("store: " + data.get("category")); 116 | } 117 | 118 | //2.2hash 119 | String shortHash = infoHash.substring(0, 16); 120 | logger.info("新资源: " + shortHash); 121 | 122 | //2.3如果没有文件，写空数据 123 | if(!data.containsKey("filelist_5")) { 124 | data.put("file_count", 0); 125 | data.put("filelist_5", new JSONArray()); 126 | } 127 | 128 | //2.4敏感词（校验name和fileslist_5） 129 | boolean spam = SpamAnalyzer.isSpam(name, data.getJSONArray("filelist_5")); 130 | if(spam) { 131 | data.put("spam", true); 132 | } 133 | 134 | //2.5日期 135 | long now = Calendar.getInstance().getTimeInMillis()/1000; 136 | data.put("create_time", now); 137 | data.put("last_seen", now); 138 | 139 | //2.6移除多余字段 140 | data.remove("data_hash"); 141 | data.remove("extension"); 142 | data.remove("source_ip"); 143 | 144 | //3写入搜索引擎 145 | try { 146 | IndexResponse resp = client.prepareIndex(SpiderConfig.ES_INDEX_SEED, SpiderConfig.ES_TYPE_SEED) 147 | .setId(shortHash).setSource(data).execute().actionGet(); 148 | 149 | }catch(Exception e) { 150 | logger.error("写入Hash出错: " + e.getMessage(), e); 151 | testEsConn(client, channel, deliveryTag); 152 | return; 153 | } 154 | 155 | //4确认消息 156 | confirmMsg(channel, deliveryTag); 157 | } 158 | }); 159 | } 160 | 161 | //test 162 | public static void main(String[] args) throws IOException { 163 | NewHash main = new NewHash(); 164 | main.consume(); 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/org/youseed/spider/saver/ESBasic.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider.saver; 2 | 3 | import java.util.Calendar; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | import org.apache.logging.log4j.LogManager; 8 | import org.apache.logging.log4j.Logger; 9 | import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; 10 | import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; 11 | import org.elasticsearch.action.bulk.BulkItemResponse; 12 | import org.elasticsearch.action.bulk.BulkRequestBuilder; 13 | import org.elasticsearch.action.bulk.BulkResponse; 14 | import org.elasticsearch.action.update.UpdateRequest; 15 | import org.elasticsearch.client.transport.TransportClient; 16 | import org.elasticsearch.script.Script; 17 | import org.elasticsearch.script.ScriptType; 18 | import org.youseed.spider.ConfigUtil; 19 | import org.youseed.spider.SpiderConfig; 20 | 21 | import com.alibaba.fastjson.JSONArray; 22 | import com.alibaba.fastjson.JSONObject; 23 | import com.rabbitmq.client.Channel; 24 | 25 | /** 26 | * 写入ES通用方法 27 | */ 28 | public class ESBasic extends MQBasic{ 29 | 30 | private static Logger logger = LogManager.getLogger(ESBasic.class); 31 | 32 | protected String mqEsExchange = "es"; 33 | 34 | protected String mqEsNewQueue = "es.new"; 35 | protected String mqEsNewRouting = "*.new"; 36 | 37 | protected String mqEsUpdateQueue = "es.update"; 38 | protected String mqEsUpdateRouting = "*.update"; 39 | 40 | protected String esIndex = "seed"; 41 | protected String esType = "seed"; 42 | 43 | /** 44 | * 构造函数 45 | */ 46 | public ESBasic() { 47 | JSONObject config = ConfigUtil.getProperties(); 48 | 49 | mqEsExchange = config.containsKey("binding.es.exchage") ? config.getString("binding.es.exchage") : mqEsExchange; 50 | mqEsNewQueue = config.containsKey("binding.es.new.queue") ? config.getString("binding.es.new.queue") : mqEsNewQueue; 51 | mqEsNewRouting = config.containsKey("binding.es.new.routing") ? config.getString("binding.es.new.routing") : mqEsNewRouting; 52 | mqEsUpdateQueue = config.containsKey("binding.es.update.queue") ? config.getString("binding.es.update.queue") : mqEsUpdateQueue; 53 | mqEsUpdateRouting = config.containsKey("binding.es.update.routing") ? config.getString("binding.es.update.routing") : mqEsUpdateRouting; 54 | esIndex = config.containsKey("store.es.index") ? config.getString("store.es.index") : esIndex; 55 | esType = config.containsKey("store.es.type") ? config.getString("store.es.type") : esType; 56 | 57 | logger.info("---------RabbitMQ/Elasticsearch绑定配置------------"); 58 | logger.info("交换器|binding.es.exchage: " + mqEsExchange); 59 | logger.info("新资源队列|binding.es.new.queue: " + mqEsNewQueue); 60 | logger.info("新资源路由|binding.es.new.routing: " + mqEsNewRouting); 61 | logger.info("更新资源队列|binding.es.update.queue : " + mqEsUpdateQueue); 62 | logger.info("更新资源路由|binding.es.update.routing: " + mqEsUpdateRouting); 63 | logger.info("索引|store.es.index : " + esIndex); 64 | logger.info("索引类型|store.es.type: " + esType); 65 | logger.info("---------------------------------------------"); 66 | } 67 | 68 | /** 69 | * 批量更新hash，返回成功条目 70 | */ 71 | public int batchUpdateHash(TransportClient client, JSONArray data) { 72 | int now = (int)Calendar.getInstance().getTimeInMillis()/1000; 73 | Map params = new HashMap(); 74 | params.put("now", now); 75 | 76 | //1更新 77 | BulkRequestBuilder bulkRequest = client.prepareBulk(); 78 | for (int i = 0; i < data.size(); i++) { 79 | String shortHash = data.getString(i); 80 | UpdateRequest updateRequest = new UpdateRequest(SpiderConfig.ES_INDEX_SEED, SpiderConfig.ES_TYPE_SEED, shortHash); 81 | updateRequest.script(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, 82 | "ctx._source.requests+=1;ctx._source.last_seen=params.now", params)); 83 | 84 | bulkRequest.add(updateRequest); 85 | } 86 | 87 | //2读取结果 88 | HashMap errList = new HashMap(); 89 | BulkResponse resp = (BulkResponse)bulkRequest.execute().actionGet(); 90 | if(resp.hasFailures()) { 91 | BulkItemResponse[] item = resp.getItems(); 92 | for(int i = 0; i < item.length; i++) { 93 | if(item[i].isFailed()) { 94 | errList.put(item[i].getId(), item[i].getFailure().getMessage()); 95 | } 96 | } 97 | logger.error("失败条目: " + errList.size() + "|" + errList.values()); 98 | } 99 | return data.size() - errList.size(); 100 | } 101 | 102 | /** 103 | * 测试ES连接是否正常，并进行相应的消息处理 104 | */ 105 | public void testEsConn(TransportClient client, Channel channel, long deliveryTag) { 106 | logger.info("测试 es连接...."); 107 | try { 108 | NodesInfoResponse response = client.admin().cluster().nodesInfo(new NodesInfoRequest().timeout("30s")).actionGet(); 109 | response.getNodesMap(); 110 | 111 | logger.info("es连接正常，提交确认至消息队列"); 112 | confirmMsg(channel, deliveryTag); 113 | } catch (Exception e) { 114 | logger.info("es连接失败，退回当前数据至队列", e); 115 | rejectMsg(channel, deliveryTag); 116 | 117 | try { 118 | logger.info("暂停60秒..."); 119 | Thread.sleep(60000); 120 | } catch (InterruptedException e2) { 121 | } 122 | } 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /src/org/youseed/spider/saver/MongoBasic.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider.saver; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Calendar; 5 | import java.util.Date; 6 | import java.util.List; 7 | 8 | import org.apache.logging.log4j.LogManager; 9 | import org.apache.logging.log4j.Logger; 10 | import org.bson.Document; 11 | import org.youseed.spider.ConfigUtil; 12 | import org.youseed.spider.MongoConn; 13 | import org.youseed.spider.SpiderConfig; 14 | 15 | import com.alibaba.fastjson.JSONArray; 16 | import com.alibaba.fastjson.JSONObject; 17 | import com.mongodb.bulk.BulkWriteResult; 18 | import com.mongodb.client.MongoCollection; 19 | import com.mongodb.client.model.UpdateOneModel; 20 | import com.mongodb.client.model.UpdateOptions; 21 | import com.mongodb.client.model.WriteModel; 22 | import com.rabbitmq.client.Channel; 23 | 24 | /** 25 | * 测试Mongo连接 26 | */ 27 | public class MongoBasic extends MQBasic{ 28 | 29 | private static Logger logger = LogManager.getLogger(MongoBasic.class); 30 | 31 | protected String mqMongoExchange = "store"; 32 | 33 | protected String mqMongoNewQueue = "store.new"; 34 | protected String mqMongoNewRouting = "*.new"; 35 | 36 | protected String mqMongoUpdateQueue = "store.update"; 37 | protected String mqMongoUpdateRouting = "*.update"; 38 | 39 | protected String mqMongoStatQueue = "store.stat"; 40 | protected String mqMongoStatRouting = "*.stat"; 41 | 42 | protected String collHash = "seed_hash"; 43 | protected String collFilelist = "seed_filelist"; 44 | protected String collStat = "seed_stat"; 45 | 46 | /** 47 | * 构造函数 48 | */ 49 | public MongoBasic() { 50 | ConfigUtil.printBanner(); 51 | 52 | JSONObject config = ConfigUtil.getProperties(); 53 | mqMongoExchange = config.containsKey("binding.mongo.exchage") ? config.getString("binding.mongo.exchage") : mqMongoExchange; 54 | mqMongoNewQueue = config.containsKey("binding.mongo.new.queue") ? config.getString("binding.mongo.new.queue") : mqMongoNewQueue; 55 | mqMongoNewRouting = config.containsKey("binding.mongo.new.routing") ? config.getString("binding.mongo.new.routing") : mqMongoNewRouting; 56 | mqMongoUpdateQueue = config.containsKey("binding.mongo.update.queue") ? config.getString("binding.mongo.update.queue") : mqMongoUpdateQueue; 57 | mqMongoUpdateRouting = config.containsKey("binding.mongo.update.routing") ? config.getString("binding.mongo.update.routing") : mqMongoUpdateRouting; 58 | mqMongoStatQueue = config.containsKey("binding.mongo.stat.queue") ? config.getString("binding.mongo.stat.queue") : mqMongoStatQueue; 59 | mqMongoStatRouting = config.containsKey("binding.mongo.stat.routing") ? config.getString("binding.mongo.stat.routing") : mqMongoStatRouting; 60 | 61 | collHash = config.containsKey("store.mongo.hash") ? config.getString("store.mongo.hash") : collHash; 62 | collFilelist = config.containsKey("store.mongo.filelist") ? config.getString("store.mongo.filelist") : collFilelist; 63 | collStat = config.containsKey("store.mongo.stat") ? config.getString("store.mongo.stat") : collStat; 64 | 65 | 66 | logger.info("---------RabbitMQ/Mongodb绑定配置------------"); 67 | logger.info("交换器|binding.mongo.exchage: " + mqMongoExchange); 68 | logger.info("新资源队列|binding.mongo.new.queue: " + mqMongoNewQueue); 69 | logger.info("新资源路由|binding.mongo.new.routing: " + mqMongoNewRouting); 70 | logger.info("更新资源队列|binding.mongo.update.queue: " + mqMongoUpdateQueue); 71 | logger.info("更新资源路由|binding.mongo.update.routing: " + mqMongoUpdateRouting); 72 | logger.info("爬虫统计队列|binding.mongo.stat.queue: " + mqMongoStatQueue); 73 | logger.info("爬虫统计路由|binding.mongo.stat.routing: " + mqMongoStatRouting); 74 | logger.info("资源明细|store.mongo.index: " + collHash); 75 | logger.info("文件列表|store.mongo.type: " + collFilelist); 76 | logger.info("爬虫统计信息|store.mongo.stat: " + collStat); 77 | logger.info("---------------------------------------------"); 78 | } 79 | 80 | /** 81 | * 如果存在，则设置值 82 | */ 83 | private void setIfExists(JSONObject prop, String param, String key) { 84 | if(prop.containsKey(key)) { 85 | param = prop.getString(key); 86 | } 87 | } 88 | 89 | /** 90 | * 批量更新hash，返回成功条目 91 | */ 92 | public int bulkUpdate(MongoConn mongo, JSONArray shortHashs){ 93 | Date now = Calendar.getInstance().getTime(); 94 | MongoCollection coll = mongo.getCollection(SpiderConfig.COLL_HASH); 95 | 96 | List> requests = new ArrayList>(); 97 | for (int i = 0; i < shortHashs.size(); i++) { 98 | String shortHash = shortHashs.getString(i); 99 | UpdateOneModel uom = new UpdateOneModel( 100 | new Document("short_hash", shortHash), 101 | new Document().append("$set", new Document("last_seen", now)).append("$inc", new Document("requests", 1)), 102 | new UpdateOptions().upsert(false)); 103 | requests.add(uom); 104 | } 105 | BulkWriteResult bulkWriteResult = coll.bulkWrite(requests); 106 | return bulkWriteResult.getModifiedCount(); 107 | } 108 | 109 | /** 110 | * 测试Mongo连接是否正常，并进行相应的消息处理 111 | */ 112 | public void testMongoConn(MongoConn mongo, Channel channel, long deliveryTag) { 113 | 114 | boolean ok = testMongoConn(mongo, SpiderConfig.COLL_HASH); 115 | if(ok) { 116 | logger.info("mongodb连接正常，提交确认至消息队列"); 117 | confirmMsg(channel, deliveryTag); 118 | } else { 119 | logger.info("mongodb连接失败，退回当前数据至队列"); 120 | rejectMsg(channel, deliveryTag); 121 | 122 | try { 123 | logger.info("暂停60秒..."); 124 | Thread.sleep(60000); 125 | return; 126 | } catch (InterruptedException e1) { 127 | } 128 | } 129 | } 130 | 131 | /** 132 | * 测试Mongo连接 133 | */ 134 | public boolean testMongoConn(MongoConn mongo, String collName) { 135 | try { 136 | mongo.getCollection(collName).find().first(); 137 | return true; 138 | } catch (Exception e) { 139 | return false; 140 | } 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /src/org/youseed/Main.java: -------------------------------------------------------------------------------- 1 | package org.youseed; 2 | 3 | import java.io.IOException; 4 | import java.text.ParseException; 5 | import java.text.SimpleDateFormat; 6 | import java.util.Calendar; 7 | import java.util.Date; 8 | import java.util.LinkedHashMap; 9 | import java.util.Map; 10 | import java.util.Scanner; 11 | import java.util.Timer; 12 | import java.util.TimerTask; 13 | 14 | import org.apache.logging.log4j.LogManager; 15 | import org.apache.logging.log4j.Logger; 16 | import org.youseed.spider.ConfigUtil; 17 | import org.youseed.spider.saver.mongo.SaveStat; 18 | 19 | import com.alibaba.fastjson.JSONObject; 20 | 21 | /** 22 | * 执行入口 23 | */ 24 | public class Main { 25 | 26 | private static Logger logger = LogManager.getLogger(Main.class); 27 | 28 | public static void main(String[] args) throws IOException { 29 | 30 | String order = null; 31 | 32 | //1.分析参数，支持 “--config=/opt/config.yml”、“m1”这两种 33 | if ((args != null) && (args.length > 0)) { 34 | for (int i = 0; i < args.length; i++) { 35 | //设置配置路径 36 | if(args[i].startsWith("--config=")) { 37 | String path = args[i].substring(9); 38 | 39 | if(!"".equals(path.trim())) 40 | ConfigUtil.setConfPath(path); 41 | }else { 42 | order = args[i]; 43 | } 44 | } 45 | } 46 | 47 | if(order == null){ 48 | order = getOperId(); 49 | } 50 | 51 | //2.输出banner 52 | ConfigUtil.printBanner(); 53 | 54 | //3.执行选择的操作 55 | 56 | //--Mongodb写入+更新操作 57 | if ("m".equals(order)) { 58 | new org.youseed.spider.saver.mongo.NewHash().consume(); 59 | new org.youseed.spider.saver.mongo.UpdateHash().consume(); 60 | new SaveStat().consume(); 61 | } 62 | 63 | //--写入新资源到Mongodb 64 | else if ("m1".equals(order)) { 65 | new org.youseed.spider.saver.mongo.NewHash().consume(); 66 | } 67 | 68 | //--更新资源到Mongodb 69 | else if ("m2".equals(order)) { 70 | new org.youseed.spider.saver.mongo.UpdateHash().consume(); 71 | } 72 | 73 | //--写入爬虫日志到Mongodb 74 | else if ("m3".equals(order)) { 75 | new SaveStat().consume(); 76 | } 77 | 78 | //--ES的写入和更新操作（根据esUpdateTime设置自动选择实时或定时更新） 79 | else if ("es".equals(order)) { 80 | 81 | //1.启动写入新资源的消费者 82 | new org.youseed.spider.saver.es.NewHash().consume(); 83 | 84 | //2.根据定时配置，启动持续或者定时更新消费者 85 | JSONObject config = ConfigUtil.getProperties(); 86 | String t = config.getString("esUpdateTime"); 87 | 88 | //2.1没设置时间，常驻监听模式 89 | if(t == null || "".equals(t.trim())) { 90 | logger.info("常驻内存随时更新"); 91 | new org.youseed.spider.saver.es.UpdateHash().consume(); 92 | return; 93 | } 94 | 95 | Date time = null; 96 | SimpleDateFormat sdf = new SimpleDateFormat("HH:mm:ss"); 97 | try { 98 | Calendar calNow = Calendar.getInstance(); 99 | 100 | Calendar cal = Calendar.getInstance(); 101 | cal.setTime(sdf.parse(t)); 102 | cal.set(Calendar.YEAR, calNow.get(Calendar.YEAR)); 103 | cal.set(Calendar.MONTH, calNow.get(Calendar.MONTH)); 104 | cal.set(Calendar.DATE, calNow.get(Calendar.DATE)); 105 | 106 | if (cal.getTime().before(calNow.getTime())) { 107 | cal.add(Calendar.DAY_OF_MONTH, 1); 108 | } 109 | 110 | time = cal.getTime(); 111 | 112 | } catch (ParseException e) { 113 | logger.error("格式化定时更新ES出错" + e.getMessage(), e); 114 | } 115 | 116 | //2.2.格式化时间错误，常驻监听模式 117 | if(time == null) { 118 | logger.info("常驻内存随时更新"); 119 | new org.youseed.spider.saver.es.UpdateHash().consume(); 120 | return; 121 | } 122 | 123 | //2.3.时间设置无误，启动定时任务 124 | logger.info("已设置为定时更新资源，每日更新时间：" + sdf.format(time) + 125 | "，首次执行时间：" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(time)); 126 | 127 | Timer timer = new Timer(); 128 | timer.schedule(new TimerTask(){ 129 | @Override 130 | public void run() { 131 | try { 132 | logger.info("开始执行更新任务"); 133 | new org.youseed.spider.saver.es.UpdateHashOnTime().consume(); 134 | } catch (IOException e) { 135 | logger.error("更新任务执行出错" + e.getMessage(), e); 136 | } 137 | logger.info("更新任务执行完毕"); 138 | } 139 | }, time, 24 * 60 * 60 * 1000); 140 | 141 | } 142 | 143 | //--写入新资源到ES 144 | else if ("es1".equals(order)) { 145 | new org.youseed.spider.saver.es.NewHash().consume(); 146 | } 147 | 148 | //--更新资源到ES（常驻并实时更新） 149 | else if ("es2".equals(order)) { 150 | new org.youseed.spider.saver.es.UpdateHash().consume(); 151 | } 152 | 153 | //--更新资源到ES（更新完毕当前批次后关闭） 154 | else if ("es3".equals(order)) { 155 | new org.youseed.spider.saver.es.UpdateHashOnTime().consume(); 156 | } 157 | 158 | //--zsky写入+更新操作 159 | if ("zsky".equals(order)) { 160 | new org.youseed.spider.saver.zsky.NewHash().consume(); 161 | new org.youseed.spider.saver.zsky.UpdateHash().consume(); 162 | new org.youseed.spider.saver.zsky.SaveStat().consume(); 163 | } 164 | 165 | //--写入新资源到zsky 166 | else if ("zsky1".equals(order)) { 167 | new org.youseed.spider.saver.zsky.NewHash().consume(); 168 | } 169 | 170 | //--更新资源到zsky 171 | else if ("zsky2".equals(order)) { 172 | new org.youseed.spider.saver.zsky.UpdateHash().consume(); 173 | } 174 | 175 | //--写入爬虫日志到zsky 176 | else if ("zsky3".equals(order)) { 177 | new org.youseed.spider.saver.zsky.SaveStat().consume(); 178 | } 179 | } 180 | 181 | public static String getOperId() { 182 | Map oper = new LinkedHashMap(); 183 | oper.put("m", "写入/更新Mongodb"); 184 | oper.put("m1", "\t|-------写入新资源到Mongo"); 185 | oper.put("m2", "\t|-------更新Mongo"); 186 | oper.put("m3", "\t|-------写入统计到Mongo"); 187 | oper.put("es", "写入/更新ES（根据esUpdateTime设置，自动选择实时或定时更新）"); 188 | oper.put("es1", "\t|-------写入新资源到ES"); 189 | oper.put("es2", "\t|-------更新ES（常驻内存并实时更新）"); 190 | oper.put("es3", "\t|-------更新ES（更新完毕当前批次后关闭）"); 191 | oper.put("zsky", "写入/更新纸上烤鱼（zsky）"); 192 | oper.put("zsky1", "\t|-------写入新资源到Mysql"); 193 | oper.put("zsky2", "\t|-------更新Mysql"); 194 | oper.put("zsky3", "\t|-------写入统计到Mysql"); 195 | 196 | Scanner sc = new Scanner(System.in); 197 | 198 | String order = null; 199 | for (;;) { 200 | if (oper.containsKey(order)) { 201 | sc.close(); 202 | return order; 203 | } 204 | for (Map.Entry entry : oper.entrySet()) { 205 | String id = (String) entry.getKey(); 206 | String desc = (String) entry.getValue(); 207 | 208 | System.out.println(id + ": " + desc); 209 | } 210 | System.out.println(""); 211 | System.out.println("请选择一项操作（输入编号后回车）:"); 212 | 213 | order = sc.next(); 214 | } 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /src/org/youseed/spider/MongoConn.java: -------------------------------------------------------------------------------- 1 | package org.youseed.spider; 2 | 3 | import java.util.ArrayList; 4 | import java.util.LinkedHashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | import java.util.regex.Pattern; 8 | 9 | import org.apache.logging.log4j.LogManager; 10 | import org.apache.logging.log4j.Logger; 11 | import org.bson.Document; 12 | 13 | import com.alibaba.fastjson.JSONObject; 14 | import com.mongodb.BasicDBObject; 15 | import com.mongodb.MongoClient; 16 | import com.mongodb.MongoClientOptions; 17 | import com.mongodb.MongoCredential; 18 | import com.mongodb.ServerAddress; 19 | import com.mongodb.client.MongoCollection; 20 | import com.mongodb.client.MongoCursor; 21 | import com.mongodb.client.MongoDatabase; 22 | import com.mongodb.client.model.Filters; 23 | import com.mongodb.client.model.IndexOptions; 24 | import com.mongodb.client.model.InsertManyOptions; 25 | import com.mongodb.client.result.DeleteResult; 26 | import com.mongodb.client.result.UpdateResult; 27 | 28 | /** 29 | * 获取Mongo连接 30 | */ 31 | public class MongoConn { 32 | 33 | private static Logger logger = LogManager.getLogger(MongoConn.class); 34 | 35 | MongoClient mongoClient; 36 | MongoDatabase db; 37 | 38 | public MongoConn() { 39 | 40 | JSONObject mongo = ConfigUtil.getConfig().getJSONObject("mongo"); 41 | 42 | String url = mongo.getString("url"); 43 | int port = mongo.getIntValue("port"); 44 | String dbName = mongo.getString("db"); 45 | 46 | String user = mongo.getString("user"); 47 | String admindb = mongo.getString("admindb"); 48 | String psw = mongo.getString("psw"); 49 | 50 | logger.info("---------Mongodb配置------------"); 51 | logger.info("地址|mongo.url: " + url); 52 | logger.info("端口|mongo.port: " + port); 53 | logger.info("数据库|mongo.db: " + dbName); 54 | logger.info("用户名|mongo.user: " + user); 55 | logger.info("账户验证数据库|mongo.admindb: " + admindb); 56 | logger.info("---------------------------------------------"); 57 | 58 | 59 | //设置连接超时 60 | MongoClientOptions options = MongoClientOptions.builder() 61 | // .threadsAllowedToBlockForConnectionMultiplier(20) 62 | // .connectTimeout(5000) 63 | // .maxWaitTime(5000) 64 | // .socketTimeout(5000) 65 | .build(); 66 | 67 | 68 | if(mongo.get("user") == null) { 69 | ServerAddress add = new ServerAddress(url, port); 70 | mongoClient = new MongoClient(add, options); 71 | db = mongoClient.getDatabase(dbName); 72 | }else { 73 | ServerAddress add = new ServerAddress(url, port); 74 | List seeds = new ArrayList(); 75 | seeds.add(add); 76 | 77 | MongoCredential cre = MongoCredential.createCredential(user, admindb, psw.toCharArray()); 78 | mongoClient = new MongoClient(seeds, cre, options); 79 | db = mongoClient.getDatabase(dbName); 80 | } 81 | 82 | } 83 | 84 | /** 85 | * 创建集合 86 | */ 87 | public void createColl(String collectionName) { 88 | db.createCollection(collectionName); 89 | } 90 | 91 | /** 92 | * 创建唯一索引 93 | */ 94 | public void createIndexUnique(String collectionName, String index) { 95 | IndexOptions indexOptions = new IndexOptions(); 96 | indexOptions.unique(true); 97 | getCollection(collectionName).createIndex(new Document().append(index, -1), indexOptions); 98 | } 99 | 100 | /** 101 | * 创建唯一索引 102 | */ 103 | public void createIndexUnique(String collectionName, String[] index) { 104 | IndexOptions indexOptions = new IndexOptions(); 105 | indexOptions.unique(true); 106 | 107 | Document idx = new Document(); 108 | for (int i = 0; i < index.length; i++) { 109 | idx.append(index[i], 1); 110 | } 111 | 112 | getCollection(collectionName).createIndex(idx, indexOptions); 113 | } 114 | 115 | /** 116 | * 创建索引 117 | */ 118 | public void createIndex(String collectionName, String index) { 119 | getCollection(collectionName).createIndex(new Document().append(index, -1)); 120 | } 121 | 122 | /** 123 | * 保存一个文档 124 | */ 125 | public void save(String collectionName, String json) { 126 | Document doc = Document.parse(json); 127 | save(collectionName, doc); 128 | } 129 | 130 | /** 131 | * 保存一个文档 132 | */ 133 | public void save(String collectionName, Document doc) { 134 | MongoCollection collection = db.getCollection(collectionName, Document.class); 135 | collection.insertOne(doc); 136 | } 137 | 138 | /** 139 | * 保存多个文档 140 | */ 141 | public void save(String collectionName, List docs) { 142 | MongoCollection collection = db.getCollection(collectionName, Document.class); 143 | InsertManyOptions options = new InsertManyOptions();//不开启验证/ 效率更快 144 | options.ordered(false); 145 | 146 | collection.insertMany(docs, options); 147 | } 148 | 149 | /** 150 | * 获取一个文档 151 | */ 152 | public MongoCollection getCollection(String collectionName){ 153 | return db.getCollection(collectionName); 154 | } 155 | 156 | /** 157 | * 获取文档中所有编号 158 | */ 159 | public Map getIds(String collName) { 160 | Map ids = new LinkedHashMap(); 161 | 162 | MongoCollection movieColl = getCollection(collName); 163 | Document fields = new Document().append("_id", 1); 164 | 165 | MongoCursor iter = movieColl.find().projection(fields).iterator(); 166 | while(iter.hasNext()) { 167 | Document doc = iter.next(); 168 | Object _id = doc.get("_id"); 169 | ids.put(_id, null); 170 | } 171 | 172 | return ids; 173 | } 174 | 175 | /** 176 | * 模糊查询 177 | */ 178 | public Map getIdsFuzzy(String collName, String preg) { 179 | Map ids = new LinkedHashMap(); 180 | 181 | MongoCollection movieColl = getCollection(collName); 182 | 183 | Pattern pattern = Pattern.compile(preg, Pattern.CASE_INSENSITIVE); 184 | BasicDBObject query = new BasicDBObject(); 185 | query.put("_id", pattern); 186 | 187 | Document fields = new Document().append("_id", 1); 188 | 189 | MongoCursor iter = movieColl.find(query).projection(fields).iterator(); 190 | while(iter.hasNext()) { 191 | Document doc = iter.next(); 192 | Object _id = doc.get("_id"); 193 | ids.put(_id, null); 194 | } 195 | 196 | return ids; 197 | } 198 | 199 | /** 200 | * 删除一条记录 201 | */ 202 | public long delete(String collName, Object id) { 203 | MongoCollection coll = getCollection(collName); 204 | DeleteResult r = coll.deleteOne(new Document().append("_id", id)); 205 | return r.getDeletedCount(); 206 | } 207 | 208 | /** 209 | * 删除一条记录 210 | */ 211 | public long delete(String collName, String key, Object val) { 212 | MongoCollection coll = getCollection(collName); 213 | DeleteResult r = coll.deleteOne(new Document().append(key, val)); 214 | return r.getDeletedCount(); 215 | } 216 | 217 | /** 218 | * 删除符合条件的所有记录 219 | */ 220 | public long deleteMany(String collName, Document cond) { 221 | MongoCollection coll = getCollection(collName); 222 | DeleteResult r = coll.deleteMany(cond); 223 | return r.getDeletedCount(); 224 | } 225 | 226 | 227 | /** 228 | * 更新一条记录 229 | */ 230 | public long updateOne(String collName, Object id, Document cond) { 231 | MongoCollection coll = getCollection(collName); 232 | UpdateResult r = coll.updateOne(Filters.eq("_id", id), new Document("$set", cond)); 233 | return r.getModifiedCount(); 234 | } 235 | 236 | /** 237 | * 更新所有记录 238 | */ 239 | public long updateAll(String collName, Document cond) { 240 | MongoCollection coll = getCollection(collName); 241 | UpdateResult r = coll.updateMany(Document.parse("{}"), cond); 242 | return r.getModifiedCount(); 243 | } 244 | 245 | } 246 | -------------------------------------------------------------------------------- /src/org/rex/db/configuration/Configuration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2016 the Rex-Soft Group. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package org.rex.db.configuration; 17 | 18 | import java.io.InputStream; 19 | import java.util.Properties; 20 | 21 | import javax.sql.DataSource; 22 | 23 | import org.rex.db.datasource.DataSourceManager; 24 | import org.rex.db.dialect.Dialect; 25 | import org.rex.db.dialect.DialectManager; 26 | import org.rex.db.dynamic.javassist.BeanConvertorManager; 27 | import org.rex.db.exception.DBException; 28 | import org.rex.db.exception.DBRuntimeException; 29 | import org.rex.db.exception.ExceptionResourceFactory; 30 | import org.rex.db.listener.DBListener; 31 | import org.rex.db.listener.ListenerManager; 32 | import org.rex.db.logger.Logger; 33 | import org.rex.db.logger.LoggerFactory; 34 | import org.rex.db.transaction.Definition; 35 | import org.rex.db.util.ReflectUtil; 36 | import org.rex.db.util.ResourceUtil; 37 | 38 | /** 39 | * Framework Configuration. 40 | * 41 | * @version 1.0, 2016-03-28 42 | * @since Rexdb-1.0 43 | */ 44 | public class Configuration { 45 | 46 | //-----------------------------Singleton instance 47 | private static final Logger LOGGER = LoggerFactory.getLogger(Configuration.class); 48 | 49 | private static final String DEFAULT_CONFIG_PATH = "rexdb.xml"; 50 | 51 | private static volatile Configuration instance; 52 | 53 | //-----------------------------configuration 54 | /** 55 | * properties 56 | */ 57 | private volatile Properties variables; 58 | 59 | //--------settings 60 | /** 61 | * language: zh-cn, en 62 | */ 63 | private volatile String lang; 64 | 65 | /** 66 | * Disables all logs? 67 | */ 68 | private volatile boolean nolog = false; 69 | 70 | /** 71 | * Validates SQLs before executing. 72 | */ 73 | private volatile boolean validateSql = true; 74 | 75 | /** 76 | * Retrieves warnings reported by calls on the Connection, Statement and ResultSet object. 77 | */ 78 | private volatile boolean checkWarnings = false; 79 | 80 | /** 81 | * Timeout for each query. 82 | */ 83 | private volatile int queryTimeout = -1; 84 | 85 | /** 86 | * Timeout for each transaction. 87 | */ 88 | private volatile int transactionTimeout = -1; 89 | 90 | /** 91 | * Rolls back the transaction on committing failure. 92 | */ 93 | private volatile boolean autoRollback = false; 94 | 95 | /** 96 | * Transaction isolation. 97 | */ 98 | private volatile String transactionIsolation; 99 | 100 | /** 101 | * Caches beanInfo, parameter types etc in reflections. 102 | */ 103 | private volatile boolean reflectCache = true; 104 | 105 | /** 106 | * Uses dynamic classes instead of reflections. 107 | */ 108 | private volatile boolean dynamicClass = true; 109 | 110 | /** 111 | * Automatically converts Date and Time to java.sql.Timestamp. 112 | */ 113 | private volatile boolean dateAdjust = true; 114 | 115 | /** 116 | * Automatically begins a transaction before batch updating. 117 | */ 118 | private volatile boolean batchTransaction = true; 119 | 120 | //--------managers 121 | /** 122 | * DataSource Manager. 123 | */ 124 | private final DataSourceManager dataSourceManager; 125 | 126 | /** 127 | * Listener Manager. 128 | */ 129 | private final ListenerManager listenerManager; 130 | 131 | /** 132 | * Dialect Manager. 133 | */ 134 | private final DialectManager dialectManager; 135 | 136 | static{ 137 | try { 138 | LOGGER.info("loading default configuration {0}.", DEFAULT_CONFIG_PATH); 139 | loadDefaultConfiguration(); 140 | LOGGER.info("default configuration {0} loaded.", DEFAULT_CONFIG_PATH); 141 | } catch (DBException e) { 142 | LOGGER.warn("could not load default configuration {0} from classpath, rexdb is not initialized, cause {1}", DEFAULT_CONFIG_PATH, e.getMessage()); 143 | } 144 | } 145 | 146 | public Configuration(){ 147 | dataSourceManager = new DataSourceManager(); 148 | listenerManager = new ListenerManager(); 149 | dialectManager = new DialectManager(); 150 | } 151 | 152 | public static Configuration getInstance() { 153 | return instance; 154 | } 155 | 156 | public static Configuration setInstance(Configuration conf) { 157 | return instance = conf; 158 | } 159 | 160 | /** 161 | * Loads the default XML configuration. 162 | * 163 | * @throws DBException if the default XML file does not exist, could not parse the file, etc. 164 | */ 165 | public synchronized static void loadDefaultConfiguration() throws DBException{ 166 | if(instance != null) 167 | throw new DBException("DB-F0007", DEFAULT_CONFIG_PATH); 168 | 169 | InputStream inputStream = ResourceUtil.getResourceAsStream(DEFAULT_CONFIG_PATH); 170 | if(inputStream == null){ 171 | LOGGER.warn("could not find configuration {0} in classpath.", DEFAULT_CONFIG_PATH); 172 | }else 173 | instance = new XMLConfigurationLoader().load(inputStream); 174 | } 175 | 176 | /** 177 | * Loads the default XML configuration from ClassPath. 178 | * @param path the path of the XML file. 179 | * @throws DBException if the default XML file does not exist, could not parse the file, etc. 180 | */ 181 | public synchronized static void loadConfigurationFromClasspath(String path) throws DBException{ 182 | if(instance != null) 183 | throw new DBException("DB-F0007", path); 184 | 185 | LOGGER.info("loading configuration {0} from classpath.", path); 186 | instance = new XMLConfigurationLoader().loadFromClasspath(path); 187 | LOGGER.info("configuration {0} loaded.", path); 188 | } 189 | 190 | /** 191 | * Loads the default XML configuration from ClassPath. 192 | * @param path the absolute path of the XML file . 193 | * @throws DBException if the default XML file does not exist, could not parse the file, etc. 194 | */ 195 | public synchronized static void loadConfigurationFromFileSystem(String path) throws DBException{ 196 | if(instance != null) 197 | throw new DBException("DB-F0007", path); 198 | 199 | LOGGER.info("loading configuration {0} from file system.", path); 200 | instance = new XMLConfigurationLoader().loadFromFileSystem(path); 201 | LOGGER.info("configuration {0} loaded.", path); 202 | } 203 | 204 | /** 205 | * Returns the current configuration. 206 | */ 207 | public static Configuration getCurrentConfiguration() throws DBRuntimeException{ 208 | if(instance == null){ 209 | try { 210 | loadDefaultConfiguration(); 211 | } catch (DBException e) { 212 | throw new DBRuntimeException(e); 213 | } 214 | } 215 | 216 | if(instance == null) 217 | throw new DBRuntimeException("DB-F0008", DEFAULT_CONFIG_PATH); 218 | 219 | return instance; 220 | } 221 | 222 | 223 | //--------------------------- 224 | public void applySettings(){ 225 | //lang 226 | if(lang != null){ 227 | ExceptionResourceFactory.getInstance().setLang(lang); 228 | } 229 | 230 | //reflectCache 231 | if(!reflectCache){ 232 | ReflectUtil.setCacheEnabled(false); 233 | } 234 | 235 | //nolog 236 | if(nolog){ 237 | LoggerFactory.setNolog(true); 238 | } 239 | 240 | } 241 | 242 | //--------------------------- 243 | public void addVariables(Properties variables) { 244 | if(this.variables == null) 245 | this.variables = variables; 246 | else 247 | this.variables.putAll(variables); 248 | } 249 | 250 | public Properties getVariables() { 251 | return variables; 252 | } 253 | 254 | public String getLang() { 255 | return lang; 256 | } 257 | 258 | public void setLang(String lang) { 259 | this.lang = lang; 260 | } 261 | 262 | public boolean isNolog() { 263 | return nolog; 264 | } 265 | 266 | public void setNolog(boolean nolog) { 267 | this.nolog = nolog; 268 | } 269 | 270 | public boolean isValidateSql() { 271 | return validateSql; 272 | } 273 | 274 | public void setValidateSql(boolean validateSql) { 275 | LOGGER.info("sql validate has switched to {0}.", validateSql); 276 | this.validateSql = validateSql; 277 | } 278 | 279 | public boolean isCheckWarnings() { 280 | return checkWarnings; 281 | } 282 | 283 | public void setCheckWarnings(boolean checkWarnings) { 284 | LOGGER.info("check warnings has switched to {0}.", validateSql); 285 | this.checkWarnings = checkWarnings; 286 | } 287 | 288 | public int getQueryTimeout() { 289 | return queryTimeout; 290 | } 291 | 292 | public void setQueryTimeout(int queryTimeout) { 293 | this.queryTimeout = queryTimeout; 294 | } 295 | 296 | public int getTransactionTimeout() { 297 | return transactionTimeout; 298 | } 299 | 300 | public void setTransactionTimeout(int transactionTimeout) { 301 | this.transactionTimeout = transactionTimeout; 302 | } 303 | 304 | public boolean isAutoRollback() { 305 | return autoRollback; 306 | } 307 | 308 | public void setAutoRollback(boolean autoRollback) { 309 | this.autoRollback = autoRollback; 310 | } 311 | 312 | public String getTransactionIsolation() { 313 | return transactionIsolation; 314 | } 315 | 316 | public void setTransactionIsolation(String transactionIsolation) { 317 | this.transactionIsolation = Definition.ISOLATION_CONSTANT_PREFIX + '_' +transactionIsolation; 318 | } 319 | 320 | public boolean isReflectCache() { 321 | return reflectCache; 322 | } 323 | 324 | public void setReflectCache(boolean reflectCache) { 325 | this.reflectCache = reflectCache; 326 | } 327 | 328 | public boolean isDynamicClass() { 329 | return dynamicClass; 330 | } 331 | 332 | public void setDynamicClass(boolean dynamicClass) { 333 | if(dynamicClass){//test dynamic 334 | try{ 335 | BeanConvertorManager m = new BeanConvertorManager(); 336 | BeanConvertorManager.getConvertor(Configuration.class); 337 | this.dynamicClass = true; 338 | }catch(Throwable e){ 339 | this.dynamicClass = false; 340 | LOGGER.warn("dynamic class setting is true, but could not pass the validation test, now automatically switch to false, {0}", e, e.getMessage()); 341 | } 342 | }else 343 | this.dynamicClass = false; 344 | } 345 | 346 | public boolean isDateAdjust() { 347 | return dateAdjust; 348 | } 349 | 350 | public void setDateAdjust(boolean dateAdjust) { 351 | this.dateAdjust = dateAdjust; 352 | } 353 | 354 | public boolean isBatchTransaction() { 355 | return batchTransaction; 356 | } 357 | 358 | public void setBatchTransaction(boolean batchTransaction) { 359 | this.batchTransaction = batchTransaction; 360 | } 361 | 362 | //----------- 363 | public void setDefaultDataSource(DataSource dataSource){ 364 | dataSourceManager.setDefault(dataSource); 365 | } 366 | 367 | public void setDataSource(String id, DataSource dataSource){ 368 | dataSourceManager.add(id, dataSource); 369 | } 370 | 371 | public DataSourceManager getDataSourceManager() { 372 | return dataSourceManager; 373 | } 374 | 375 | public void addListener(DBListener listener){ 376 | listenerManager.registe(listener); 377 | } 378 | 379 | public ListenerManager getListenerManager() { 380 | return listenerManager; 381 | } 382 | 383 | public void addDialect(DataSource dataSource, Dialect dialect){ 384 | dialectManager.setDialect(dataSource, dialect);; 385 | } 386 | 387 | public DialectManager getDialectManager() { 388 | return dialectManager; 389 | } 390 | 391 | public String toString() { 392 | return "Configuration [variables=" + variables + ", lang=" + lang + ", dataSourceManager=" + dataSourceManager + ", listenerManager=" 393 | + listenerManager + ", dialectManager=" + dialectManager + "]"; 394 | } 395 | 396 | } 397 | -------------------------------------------------------------------------------- /resource/spam.txt: -------------------------------------------------------------------------------- 1 | 爱女人 2 | 爱液 3 | 按摩棒 4 | 拔出来 5 | 爆草 6 | 包二奶 7 | 暴干 8 | 暴奸 9 | 暴乳 10 | 爆乳 11 | 暴淫 12 | 屄 13 | 被操 14 | 被插 15 | 被干 16 | 逼奸 17 | 仓井空 18 | 插暴 19 | 操逼 20 | 操黑 21 | 操烂 22 | 肏你 23 | 肏死 24 | 操死 25 | 操我 26 | 厕奴 27 | 插比 28 | 插b 29 | 插逼 30 | 插进 31 | 插你 32 | 插我 33 | 插阴 34 | 潮吹 35 | 潮喷 36 | 成人dv 37 | 成人电影 38 | 成人论坛 39 | 成人小说 40 | 成人电 41 | 成人电影 42 | 成人卡通 43 | 成人聊 44 | 成人片 45 | 成人视 46 | 成人图 47 | 成人文 48 | 成人小 49 | 成人电影 50 | 成人论坛 51 | 成人色情 52 | 成人网站 53 | 成人文学 54 | 成人小说 55 | 艳情小说 56 | 成人游戏 57 | 吃精 58 | 赤裸 59 | 抽插 60 | 扌由插 61 | 抽一插 62 | 春药 63 | 大波 64 | 大力抽送 65 | 大乳 66 | 荡妇 67 | 荡女 68 | 盗撮 69 | 多人轮 70 | 发浪 71 | 放尿 72 | 肥逼 73 | 粉穴 74 | 封面女郎 75 | 风月大陆 76 | 干死你 77 | 干穴 78 | 肛交 79 | 肛门 80 | 龟头 81 | 裹本 82 | 国产av 83 | 好嫩 84 | 豪乳 85 | 黑逼 86 | 后庭 87 | 后穴 88 | 虎骑 89 | 花花公子 90 | 换妻俱乐部 91 | 黄片 92 | 几吧 93 | 鸡吧 94 | 鸡巴 95 | 鸡奸 96 | 寂寞男 97 | 寂寞女 98 | 妓女 99 | 激情 100 | 集体淫 101 | 奸情 102 | 叫床 103 | 脚交 104 | 金鳞岂是池中物 105 | 金麟岂是池中物 106 | 精液 107 | 就去日 108 | 巨屌 109 | 菊花洞 110 | 菊门 111 | 巨奶 112 | 巨乳 113 | 菊穴 114 | 开苞 115 | 口爆 116 | 口活 117 | 口交 118 | 口射 119 | 口淫 120 | 裤袜 121 | 狂操 122 | 狂插 123 | 浪逼 124 | 浪妇 125 | 浪叫 126 | 浪女 127 | 狼友 128 | 聊性 129 | 流淫 130 | 铃木麻 131 | 凌辱 132 | 漏乳 133 | 露b 134 | 乱交 135 | 乱伦 136 | 轮暴 137 | 轮操 138 | 轮奸 139 | 裸陪 140 | 买春 141 | 美逼 142 | 美少妇 143 | 美乳 144 | 美腿 145 | 美穴 146 | 美幼 147 | 秘唇 148 | 迷奸 149 | 密穴 150 | 蜜穴 151 | 蜜液 152 | 摸奶 153 | 摸胸 154 | 母奸 155 | 奈美 156 | 奶子 157 | 男奴 158 | 内射 159 | 嫩逼 160 | 嫩女 161 | 嫩穴 162 | 捏弄 163 | 女优 164 | 炮友 165 | 砲友 166 | 喷精 167 | 屁眼 168 | 品香堂 169 | 前凸后翘 170 | 强jian 171 | 强暴 172 | 强奸处女 173 | 情趣用品 174 | 情色 175 | 拳交 176 | 全裸 177 | 群交 178 | 惹火身材 179 | 人妻 180 | 人兽 181 | 日逼 182 | 日烂 183 | 肉棒 184 | 肉逼 185 | 肉唇 186 | 肉洞 187 | 肉缝 188 | 肉棍 189 | 肉茎 190 | 肉具 191 | 揉乳 192 | 肉穴 193 | 肉欲 194 | 乳爆 195 | 乳房 196 | 乳沟 197 | 乳交 198 | 乳头 199 | 三级片 200 | 骚逼 201 | 骚比 202 | 骚女 203 | 骚水 204 | 骚穴 205 | 色逼 206 | 色界 207 | 色猫 208 | 色盟 209 | 色情网站 210 | 色区 211 | 色色 212 | 色诱 213 | 色欲 214 | 色b 215 | 少年阿宾 216 | 少修正 217 | 射爽 218 | 射颜 219 | 食精 220 | 释欲 221 | 兽奸 222 | 兽交 223 | 手淫 224 | 兽欲 225 | 熟妇 226 | 熟母 227 | 熟女 228 | 爽片 229 | 爽死我了 230 | 双臀 231 | 死逼 232 | 丝袜 233 | 丝诱 234 | 松岛枫 235 | 酥痒 236 | 汤加丽 237 | 套弄 238 | 体奸 239 | 体位 240 | 舔脚 241 | 舔阴 242 | 调教 243 | 偷欢 244 | 偷拍 245 | 推油 246 | 脱内裤 247 | 文做 248 | 我就色 249 | 无码 250 | 舞女 251 | 无修正 252 | 吸精 253 | 夏川纯 254 | 相奸 255 | 小逼 256 | 校鸡 257 | 小穴 258 | 小xue 259 | 写真 260 | 性感妖娆 261 | 性感诱惑 262 | 性虎 263 | 性饥渴 264 | 性技巧 265 | 性交 266 | 性奴 267 | 性虐 268 | 性息 269 | 性欲 270 | 胸推 271 | 穴口 272 | 学生妹 273 | 穴图 274 | 亚情 275 | 颜射 276 | 阳具 277 | 杨思敏 278 | 要射了 279 | 夜勤病栋 280 | 一本道 281 | 一夜欢 282 | 一夜情 283 | 一ye情 284 | 阴部 285 | 淫虫 286 | 阴唇 287 | 淫荡 288 | 阴道 289 | 淫电影 290 | 阴阜 291 | 淫妇 292 | 淫河 293 | 阴核 294 | 阴户 295 | 淫贱 296 | 淫叫 297 | 淫教师 298 | 阴茎 299 | 阴精 300 | 淫浪 301 | 淫媚 302 | 淫糜 303 | 淫魔 304 | 淫母 305 | 淫女 306 | 淫虐 307 | 淫妻 308 | 淫情 309 | 淫色 310 | 淫声浪语 311 | 淫兽学园 312 | 淫书 313 | 淫术炼金士 314 | 淫水 315 | 淫娃 316 | 淫威 317 | 淫亵 318 | 淫样 319 | 淫液 320 | 淫照 321 | 阴b 322 | 应召 323 | 幼交 324 | 幼男 325 | 幼女 326 | 欲火 327 | 欲女 328 | 玉女心经 329 | 玉蒲团 330 | 玉乳 331 | 欲仙欲死 332 | 玉穴 333 | 援交 334 | 原味内衣 335 | 援助交际 336 | 张筱雨 337 | 招鸡 338 | 招妓 339 | 中年美妇 340 | 抓胸 341 | 自拍 342 | 自慰 343 | 作爱 344 | 18禁 345 | 99bb 346 | a4u 347 | a4y 348 | adult 349 | amateur 350 | anal 351 | a片 352 | fuck 353 | gay片 354 | g点 355 | g片 356 | hardcore 357 | h动画 358 | h动漫 359 | incest 360 | porn 361 | secom 362 | sexinsex 363 | sm女王 364 | xiao77 365 | xing伴侣 366 | tokyohot 367 | yin荡 368 | 贱人 369 | 装b 370 | 大sb 371 | 傻逼 372 | 傻b 373 | 煞逼 374 | 煞笔 375 | 刹笔 376 | 傻比 377 | 沙比 378 | 欠干 379 | 婊子养的 380 | 我日你 381 | 我操 382 | 我草 383 | 卧艹 384 | 卧槽 385 | 爆你菊 386 | 艹你 387 | cao你 388 | 你他妈 389 | 真他妈 390 | 别他吗 391 | 草你吗 392 | 草你丫 393 | 操你妈 394 | 擦你妈 395 | 操你娘 396 | 操他妈 397 | 日你妈 398 | 干你妈 399 | 干你娘 400 | 娘西皮 401 | 狗操 402 | 狗草 403 | 狗杂种 404 | 狗日的 405 | 操你祖宗 406 | 操你全家 407 | 操你大爷 408 | 妈逼 409 | 你麻痹 410 | 麻痹的 411 | 妈了个逼 412 | 马勒 413 | 狗娘养 414 | 贱比 415 | 贱b 416 | 下贱 417 | 死全家 418 | 全家死光 419 | 全家不得好死 420 | 全家死绝 421 | 白痴 422 | 无耻 423 | 杀b 424 | 你吗b 425 | 你妈的 426 | 婊子 427 | 贱货 428 | 人渣 429 | 混蛋 430 | 媚外 431 | 和弦 432 | 兼职 433 | 限量 434 | 铃声 435 | 性伴侣 436 | 男公关 437 | 火辣 438 | 精子 439 | 射精 440 | 诱奸 441 | 强奸 442 | 做爱 443 | 性爱 444 | 发生关系 445 | 按摩 446 | 快感 447 | 处男 448 | 猛男 449 | 少妇 450 | 屌 451 | 屁股 452 | 下体 453 | a片 454 | 内裤 455 | 浑圆 456 | 咪咪 457 | 发情 458 | 刺激 459 | 白嫩 460 | 粉嫩 461 | 兽性 462 | 风骚 463 | 呻吟 464 | 阉割 465 | 高潮 466 | 裸露 467 | 不穿 468 | 一丝不挂 469 | 脱光 470 | 干你 471 | 干死 472 | 我干 473 | 裙中性运动 474 | 乱奸 475 | 乱伦 476 | 乱伦类 477 | 乱伦小 478 | 伦理大 479 | 伦理电影 480 | 伦理毛 481 | 伦理片 482 | 裸聊 483 | 裸聊网 484 | 裸体写真 485 | 裸舞视 486 | 裸照 487 | 美女裸体 488 | 美女写真 489 | 美女上门 490 | 美艳少妇 491 | 妹按摩 492 | 妹上门 493 | 迷幻药 494 | 迷幻藥 495 | 迷昏口 496 | 迷昏药 497 | 迷昏藥 498 | 迷魂香 499 | 迷魂药 500 | 迷魂藥 501 | 迷奸粉 502 | 迷奸药 503 | 迷情粉 504 | 迷情水 505 | 迷情药 506 | 迷药 507 | 迷藥 508 | 谜奸药 509 | 骚妇 510 | 骚货 511 | 骚浪 512 | 骚女 513 | 骚嘴 514 | 色电影 515 | 色妹妹 516 | 色情表演 517 | 色情电影 518 | 色情服务 519 | 色情图片 520 | 色情小说 521 | 色情影片 522 | 色情表演 523 | 色情电影 524 | 色情服务 525 | 色情片 526 | 色视频 527 | 色小说 528 | 性伴侣 529 | 性服务 530 | 性福情 531 | 性感少 532 | 性伙伴 533 | 性交 534 | 性交视频 535 | 性交图片 536 | 性奴 537 | 性奴集中营 538 | 性虐 539 | 阴唇 540 | 阴道 541 | 阴蒂 542 | 阴户 543 | 阴间来电 544 | 阴茎 545 | 阴茎增大 546 | 阴茎助勃 547 | 阴毛 548 | 陰唇 549 | 陰道 550 | 陰戶 551 | 淫荡 552 | 淫荡美女 553 | 淫荡视频 554 | 淫荡照片 555 | 淫乱 556 | 淫靡 557 | 淫魔 558 | 淫魔舞 559 | 淫女 560 | 淫情女 561 | 淫肉 562 | 淫騷妹 563 | 淫兽 564 | 淫兽学 565 | 淫水 566 | 淫穴 567 | morphine 568 | 摇头丸 569 | 迷药 570 | 乖乖粉 571 | narcotic 572 | 麻醉药 573 | 精神药品 574 | 爱女人 575 | 爱液 576 | 按摩棒 577 | 拔出来 578 | 爆草 579 | 包二奶 580 | 暴干 581 | 暴奸 582 | 暴乳 583 | 爆乳 584 | 暴淫 585 | 屄 586 | 被操 587 | 被插 588 | 被干 589 | 逼奸 590 | 仓井空 591 | 插暴 592 | 操逼 593 | 操黑 594 | 操烂 595 | 肏你 596 | 肏死 597 | 操死 598 | 操我 599 | 厕奴 600 | 插比 601 | 插b 602 | 插逼 603 | 插进 604 | 插你 605 | 插我 606 | 插阴 607 | 潮吹 608 | 潮喷 609 | 成人电影 610 | 成人论坛 611 | 成人色情 612 | 成人网站 613 | 成人文学 614 | 成人小说 615 | 艳情小说 616 | 成人游戏 617 | 吃精 618 | 赤裸 619 | 抽插 620 | 扌由插 621 | 抽一插 622 | 春药 623 | 大波 624 | 大力抽送 625 | 大乳 626 | 荡妇 627 | 荡女 628 | 盗撮 629 | 多人轮 630 | 发浪 631 | 放尿 632 | 肥逼 633 | 粉穴 634 | 封面女郎 635 | 风月大陆 636 | 干死你 637 | 干穴 638 | 肛交 639 | 肛门 640 | 龟头 641 | 裹本 642 | 国产av 643 | 好嫩 644 | 豪乳 645 | 黑逼 646 | 后庭 647 | 后穴 648 | 虎骑 649 | 花花公子 650 | 换妻俱乐部 651 | 黄片 652 | 几吧 653 | 鸡吧 654 | 鸡巴 655 | 鸡奸 656 | 寂寞男 657 | 寂寞女 658 | 妓女 659 | 激情 660 | 集体淫 661 | 奸情 662 | 叫床 663 | 脚交 664 | 金鳞岂是池中物 665 | 金麟岂是池中物 666 | 精液 667 | 就去日 668 | 巨屌 669 | 菊花洞 670 | 菊门 671 | 巨奶 672 | 巨乳 673 | 菊穴 674 | 开苞 675 | 口爆 676 | 口活 677 | 口交 678 | 口射 679 | 口淫 680 | 裤袜 681 | 狂操 682 | 狂插 683 | 浪逼 684 | 浪妇 685 | 浪叫 686 | 浪女 687 | 狼友 688 | 聊性 689 | 流淫 690 | 铃木麻 691 | 凌辱 692 | 漏乳 693 | 露b 694 | 乱交 695 | 乱伦 696 | 轮暴 697 | 轮操 698 | 轮奸 699 | 裸陪 700 | 买春 701 | 美逼 702 | 美少妇 703 | 美乳 704 | 美腿 705 | 美穴 706 | 美幼 707 | 秘唇 708 | 迷奸 709 | 密穴 710 | 蜜穴 711 | 蜜液 712 | 摸奶 713 | 摸胸 714 | 母奸 715 | 奈美 716 | 奶子 717 | 男奴 718 | 内射 719 | 嫩逼 720 | 嫩女 721 | 嫩穴 722 | 捏弄 723 | 女优 724 | 炮友 725 | 砲友 726 | 喷精 727 | 屁眼 728 | 品香堂 729 | 前凸后翘 730 | 强jian 731 | 强暴 732 | 强奸处女 733 | 情趣用品 734 | 情色 735 | 拳交 736 | 全裸 737 | 群交 738 | 惹火身材 739 | 人妻 740 | 人兽 741 | 日逼 742 | 日烂 743 | 肉棒 744 | 肉逼 745 | 肉唇 746 | 肉洞 747 | 肉缝 748 | 肉棍 749 | 肉茎 750 | 肉具 751 | 揉乳 752 | 肉穴 753 | 肉欲 754 | 乳爆 755 | 乳房 756 | 乳沟 757 | 乳交 758 | 乳头 759 | 三级片 760 | 骚逼 761 | 骚比 762 | 骚女 763 | 骚水 764 | 骚穴 765 | 色逼 766 | 色界 767 | 色猫 768 | 色盟 769 | 色情网站 770 | 色区 771 | 色色 772 | 色诱 773 | 色欲 774 | 色b 775 | 少年阿宾 776 | 少修正 777 | 射爽 778 | 射颜 779 | 食精 780 | 释欲 781 | 兽奸 782 | 兽交 783 | 手淫 784 | 兽欲 785 | 熟妇 786 | 熟母 787 | 熟女 788 | 爽片 789 | 爽死我了 790 | 双臀 791 | 死逼 792 | 丝袜 793 | 丝诱 794 | 松岛枫 795 | 酥痒 796 | 汤加丽 797 | 套弄 798 | 体奸 799 | 体位 800 | 舔脚 801 | 舔阴 802 | 调教 803 | 偷欢 804 | 偷拍 805 | 推油 806 | 脱内裤 807 | 文做 808 | 我就色 809 | 无码 810 | 舞女 811 | 无修正 812 | 吸精 813 | 夏川纯 814 | 相奸 815 | 小逼 816 | 校鸡 817 | 小穴 818 | 小xue 819 | 写真 820 | 性感妖娆 821 | 性感诱惑 822 | 性虎 823 | 性饥渴 824 | 性技巧 825 | 性交 826 | 性奴 827 | 性虐 828 | 性息 829 | 性欲 830 | 胸推 831 | 穴口 832 | 学生妹 833 | 穴图 834 | 亚情 835 | 颜射 836 | 阳具 837 | 杨思敏 838 | 要射了 839 | 夜勤病栋 840 | 一本道 841 | 一夜欢 842 | 一夜情 843 | 一ye情 844 | 阴部 845 | 淫虫 846 | 阴唇 847 | 淫荡 848 | 阴道 849 | 淫电影 850 | 阴阜 851 | 淫妇 852 | 淫河 853 | 阴核 854 | 阴户 855 | 淫贱 856 | 淫叫 857 | 淫教师 858 | 阴茎 859 | 阴精 860 | 淫浪 861 | 淫媚 862 | 淫糜 863 | 淫魔 864 | 淫母 865 | 淫女 866 | 淫虐 867 | 淫妻 868 | 淫情 869 | 淫色 870 | 淫声浪语 871 | 淫兽学园 872 | 淫书 873 | 淫术炼金士 874 | 淫水 875 | 淫娃 876 | 淫威 877 | 淫亵 878 | 淫样 879 | 淫液 880 | 淫照 881 | 阴b 882 | 应召 883 | 幼交 884 | 幼男 885 | 幼女 886 | 欲火 887 | 欲女 888 | 玉女心经 889 | 玉蒲团 890 | 玉乳 891 | 欲仙欲死 892 | 玉穴 893 | 援交 894 | 原味内衣 895 | 援助交际 896 | 张筱雨 897 | 招鸡 898 | 招妓 899 | 中年美妇 900 | 抓胸 901 | 自拍 902 | 自慰 903 | 作爱 904 | 18禁 905 | 99bb 906 | a4u 907 | a4y 908 | adult 909 | amateur 910 | anal 911 | a片 912 | fuck 913 | gay片 914 | g点 915 | g片 916 | hardcore 917 | h动画 918 | h动漫 919 | incest 920 | porn 921 | secom 922 | sexinsex 923 | sm女王 924 | xiao77 925 | xing伴侣 926 | tokyohot 927 | yin荡 928 | 福音会 929 | 中国教徒 930 | 统一教 931 | 观音法门 932 | 清海无上师 933 | 李洪志 934 | 志洪李 935 | 李宏志 936 | 轮功 937 | 法轮 938 | 轮法功 939 | 三去车仑 940 | 氵去车仑 941 | 发论工 942 | 法x功 943 | 法o功 944 | 法0功 945 | 法一轮一功 946 | 轮子功 947 | 车仑工力 948 | 法lun 949 | fa轮 950 | 法lg 951 | flg 952 | fl功 953 | falungong 954 | 大法弟子 955 | 大纪元 956 | dajiyuan 957 | 明慧网 958 | 明慧周报 959 | 正见网 960 | 新唐人 961 | 伪火 962 | 退党 963 | tuidang 964 | 退dang 965 | 超越红墙 966 | 自fen 967 | 真善忍 968 | 九评 969 | 9评 970 | 9ping 971 | 九ping 972 | jiuping 973 | 藏字石 974 | 集体自杀 975 | 自sha 976 | zi杀 977 | suicide 978 | titor 979 | 逢8必灾 980 | 逢八必灾 981 | 逢9必乱 982 | 逢九必乱 983 | 朱瑟里诺 984 | 根达亚文明 985 | 诺查丹玛斯 986 | 人类灭亡进程表 987 | 按照马雅历法 988 | 推背图 989 | 推bei图 990 | 济世灵文 991 | 诸世纪 992 | 电狗 993 | 电话定位器 994 | 电话拦截器 995 | 电话窃听 996 | 电话监 997 | 电话交友 998 | 电话追杀系统 999 | 电击枪 1000 | 电鸡 1001 | 电警棒 1002 | 枪出售 1003 | 枪的制 1004 | 枪货到 1005 | 枪决女犯 1006 | 枪模 1007 | 枪手 1008 | 枪销售 1009 | 枪械制 1010 | 枪子弹 1011 | 售步枪 1012 | 售纯度 1013 | 售单管 1014 | 售弹簧刀 1015 | 售防身 1016 | 售狗子 1017 | 售虎头 1018 | 售火药 1019 | 售假币 1020 | 售健卫 1021 | 售军用 1022 | 售猎枪 1023 | 售氯胺 1024 | 售麻醉 1025 | 售枪支 1026 | 售热武 1027 | 售三棱 1028 | 售手枪 1029 | 售五四 1030 | 售一元硬 1031 | 售子弹 1032 | 售左轮 1033 | 亚砷（酸）酐 1034 | 亚砷酸钾 1035 | 亚砷酸钠 1036 | 亚硒酸 1037 | 亚硒酸二钠 1038 | 亚硒酸镁 1039 | 亚硒酸钠 1040 | 亚硒酸氢钠 1041 | 亚硝酸乙酯 1042 | 亚硝酰乙氧 1043 | 氧化二丁基锡 1044 | 氧化汞 1045 | 氧化铊 1046 | 氧化亚铊 1047 | 氧氯化磷 1048 | 原装弹 1049 | 原子弹方法 1050 | 原子弹清单 1051 | 安眠酮 1052 | 代血浆 1053 | 普萘洛尔 1054 | 呋塞米 1055 | 西布曲明 1056 | testosterone 1057 | 胰岛素样生长因子 1058 | 促红细胞生成素 1059 | 地西泮 1060 | 尼可刹米 1061 | 甲睾酮 1062 | adrenaline 1063 | erythropoietin 1064 | 地奈德 1065 | 莫达非尼 1066 | 氯噻嗪 1067 | 苯巴比妥 1068 | 促性腺激素 1069 | 泼尼松 1070 | 麻黄草 1071 | 雄烯二醇 1072 | 地塞米松 1073 | tamoxifen 1074 | strychnine 1075 | androst 1076 | 新型毒品 1077 | 杜冷丁 1078 | 兴奋剂 1079 | mdma 1080 | 海洛因 1081 | 海luo因 1082 | heroin 1083 | diamorphine 1084 | diacetylmorphine 1085 | 鸦片 1086 | 阿芙蓉 1087 | 咖啡因 1088 | cocain 1089 | 三唑仑 1090 | 美沙酮 1091 | 麻古 1092 | k粉 1093 | 凯他敏 1094 | ketamine 1095 | 冰毒 1096 | 苯丙胺 1097 | cannabis 1098 | 大麻 1099 | 爱他死 1100 | 氯胺酮 1101 | benzodiazepines 1102 | 甲基安非他明 1103 | 安非他命 1104 | 吗啡 1105 | 腐败中国 1106 | 三个呆婊 1107 | 你办事我放心 1108 | 社会主义灭亡 1109 | 打倒中国 1110 | 打倒共产党 1111 | 打倒共产主义 1112 | 打倒胡锦涛 1113 | 打倒江泽民 1114 | 打倒江主席 1115 | 打倒李鹏 1116 | 打倒罗干 1117 | 打倒温家宝 1118 | 打倒中共 1119 | 打倒朱镕 1120 | 抵制共产党 1121 | 抵制共产主义 1122 | 抵制胡锦涛 1123 | 抵制江泽民 1124 | 抵制江主席 1125 | 抵制李鹏 1126 | 抵制罗干 1127 | 抵制温家宝 1128 | 抵制中共 1129 | 抵制朱镕基 1130 | 灭亡中国 1131 | 亡党亡国 1132 | 粉碎四人帮 1133 | 激流中国 1134 | 特供 1135 | 特贡 1136 | 特共 1137 | zf大楼 1138 | 殃视 1139 | 贪污腐败 1140 | 强制拆除 1141 | 形式主义 1142 | 政治风波 1143 | 太子党 1144 | 上海帮 1145 | 北京帮 1146 | 清华帮 1147 | 红色贵族 1148 | 权贵集团 1149 | 河蟹社会 1150 | 喝血社会 1151 | 九风 1152 | 9风 1153 | 十七大 1154 | 十7大 1155 | 17da 1156 | 九学 1157 | 9学 1158 | 四风 1159 | 4风 1160 | 双规 1161 | 南街村 1162 | 最淫官员 1163 | 警匪 1164 | 官匪 1165 | 独夫民贼 1166 | 官商勾结 1167 | 城管暴力执法 1168 | 强制捐款 1169 | 毒豺 1170 | 一党执政 1171 | 一党专制 1172 | 一党专政 1173 | 专制政权 1174 | 宪法法院 1175 | 胡平 1176 | 苏晓康 1177 | 贺卫方 1178 | 谭作人 1179 | 焦国标 1180 | 万润南 1181 | 张志新 1182 | 辛灝年 1183 | 高勤荣 1184 | 王炳章 1185 | 高智晟 1186 | 司马璐 1187 | 刘晓竹 1188 | 刘宾雁 1189 | 魏京生 1190 | 寻找林昭的灵魂 1191 | 别梦成灰 1192 | 谁是新中国 1193 | 讨伐中宣部 1194 | 异议人士 1195 | 民运人士 1196 | 启蒙派 1197 | 选国家主席 1198 | 民一主 1199 | min主 1200 | 民竹 1201 | 民珠 1202 | 民猪 1203 | chinesedemocracy 1204 | 大赦国际 1205 | 国际特赦 1206 | da选 1207 | 投公 1208 | 公头 1209 | 宪政 1210 | 平反 1211 | 党章 1212 | 维权 1213 | 昝爱宗 1214 | 宪章 1215 | 08宪 1216 | 08xz 1217 | 抿主 1218 | 敏主 1219 | 人拳 1220 | 人木又 1221 | 人quan 1222 | renquan 1223 | 中国人权 1224 | 中国新民党 1225 | 群体事件 1226 | 群体性事件 1227 | 上中央 1228 | 去中央 1229 | 讨说法 1230 | 请愿 1231 | 请命 1232 | 公开信 1233 | 联名上书 1234 | 万人大签名 1235 | 万人骚动 1236 | 截访 1237 | 上访 1238 | shangfang 1239 | 信访 1240 | 访民 1241 | 集合 1242 | 集会 1243 | 组织集体 1244 | 静坐 1245 | 静zuo 1246 | jing坐 1247 | 示威 1248 | 示wei 1249 | 游行 1250 | you行 1251 | 油行 1252 | 游xing 1253 | youxing 1254 | 官逼民反 1255 | 反party 1256 | 反共 1257 | 抗议 1258 | 亢议 1259 | 抵制 1260 | 低制 1261 | 底制 1262 | di制 1263 | 抵zhi 1264 | dizhi 1265 | boycott 1266 | 血书 1267 | 焚烧中国国旗 1268 | baoluan 1269 | 流血冲突 1270 | 出现暴动 1271 | 发生暴动 1272 | 引起暴动 1273 | baodong 1274 | 灭共 1275 | 杀毙 1276 | 罢工 1277 | 霸工 1278 | 罢考 1279 | 罢餐 1280 | 霸餐 1281 | 罢参 1282 | 罢饭 1283 | 罢吃 1284 | 罢食 1285 | 罢课 1286 | 罢ke 1287 | 霸课 1288 | ba课 1289 | 罢教 1290 | 罢学 1291 | 罢运 1292 | 网特 1293 | 网评员 1294 | 网络评论员 1295 | 五毛党 1296 | 五毛们 1297 | 5毛党 1298 | 戒严 1299 | jieyan 1300 | jie严 1301 | 戒yan 1302 | 8的平方事件 1303 | 知道64 1304 | 八九年 1305 | 贰拾年 1306 | 2o年 1307 | 20和谐年 1308 | 贰拾周年 1309 | 六四 1310 | 六河蟹四 1311 | 六百度四 1312 | 六和谐四 1313 | 陆四 1314 | 陆肆 1315 | 198964 1316 | 5月35 1317 | 89年春夏之交 1318 | 64惨案 1319 | 64时期 1320 | 64运动 1321 | 4事件 1322 | 四事件 1323 | 北京风波 1324 | 学潮 1325 | 学chao 1326 | xuechao 1327 | 学百度潮 1328 | 门安天 1329 | 天按门 1330 | 坦克压大学生 1331 | 民主女神 1332 | 历史的伤口 1333 | 高自联 1334 | 北高联 1335 | 血洗京城 1336 | 四二六社论 1337 | 王丹 1338 | 柴玲 1339 | 沈彤 1340 | 封从德 1341 | 王超华 1342 | 王维林 1343 | 吾尔开希 1344 | 吾尔开西 1345 | 侯德健 1346 | 阎明复 1347 | 方励之 1348 | 蒋捷连 1349 | 丁子霖 1350 | 辛灏年 1351 | 蒋彦永 1352 | 严家其 1353 | 陈一咨 1354 | 中华局域网 1355 | 党的喉舌 1356 | 互联网审查 1357 | 当局严密封锁 1358 | 新闻封锁 1359 | 封锁消息 1360 | 爱国者同盟 1361 | 关闭所有论坛 1362 | 网络封锁 1363 | 金盾工程 1364 | gfw 1365 | 无界浏览 1366 | 无界网络 1367 | 自由门 1368 | 何清涟 1369 | 中国的陷阱 1370 | 汪兆钧 1371 | 记者无疆界 1372 | 境外媒体 1373 | 维基百科 1374 | 纽约时报 1375 | bbc中文网 1376 | 华盛顿邮报 1377 | 世界日报 1378 | 东森新闻网 1379 | 东森电视 1380 | 星岛日报 1381 | wikipedia 1382 | youtube 1383 | googleblogger 1384 | 美国广播公司 1385 | 英国金融时报 1386 | 自由亚洲 1387 | 自由时报 1388 | 中国时报 1389 | 反分裂 1390 | 威胁论 1391 | 左翼联盟 1392 | 钓鱼岛 1393 | 保钓组织 1394 | 主权 1395 | 弓单 1396 | 火乍 1397 | 木仓 1398 | 石肖 1399 | 核蛋 1400 | 步qiang 1401 | bao炸 1402 | 爆zha 1403 | baozha 1404 | zha药 1405 | zha弹 1406 | 炸dan 1407 | 炸yao 1408 | zhadan 1409 | zhayao 1410 | hmtd 1411 | 三硝基甲苯 1412 | 六氟化铀 1413 | 炸药配方 1414 | 弹药配方 1415 | 炸弹配方 1416 | 皮箱炸弹 1417 | 火药配方 1418 | 人体炸弹 1419 | 人肉炸弹 1420 | 解放军 1421 | 兵力部署 1422 | 军转 1423 | 军事社 1424 | 8341部队 1425 | 第21集团军 1426 | 七大军区 1427 | 7大军区 1428 | 北京军区 1429 | 沈阳军区 1430 | 济南军区 1431 | 成都军区 1432 | 广州军区 1433 | 南京军区 1434 | 兰州军区 1435 | 颜色革命 1436 | 规模冲突 1437 | 塔利班 1438 | 基地组织 1439 | 恐怖分子 1440 | 恐怖份子 1441 | 三股势力 1442 | 印尼屠华 1443 | 印尼事件 1444 | 蒋公纪念歌 1445 | 马英九 1446 | mayingjiu 1447 | 李天羽 1448 | 苏贞昌 1449 | 林文漪 1450 | 陈水扁 1451 | 陈s扁 1452 | 陈随便 1453 | 阿扁 1454 | a扁 1455 | 告全国同胞书 1456 | 台百度湾 1457 | 台完 1458 | 台wan 1459 | taiwan 1460 | 台弯 1461 | 湾台 1462 | 台湾国 1463 | 台湾共和国 1464 | 台军 1465 | 台独 1466 | 台毒 1467 | 台du 1468 | taidu 1469 | twdl 1470 | 一中一台 1471 | 打台湾 1472 | 两岸战争 1473 | 攻占台湾 1474 | 支持台湾 1475 | 进攻台湾 1476 | 占领台湾 1477 | 统一台湾 1478 | 收复台湾 1479 | 登陆台湾 1480 | 解放台湾 1481 | 解放tw 1482 | 解决台湾 1483 | 光复民国 1484 | 台湾独立 1485 | 台湾问题 1486 | 台海问题 1487 | 台海危机 1488 | 台海统一 1489 | 台海大战 1490 | 台海战争 1491 | 台海局势 1492 | 入联 1493 | 入耳关 1494 | 中华联邦 1495 | 国民党 1496 | x民党 1497 | 民进党 1498 | 青天白日 1499 | 闹独立 1500 | duli 1501 | fenlie 1502 | 日本万岁 1503 | 小泽一郎 1504 | 劣等民族 1505 | 汉人 1506 | 汉维 1507 | 维汉 1508 | 维吾 1509 | 吾尔 1510 | 热比娅 1511 | 伊力哈木 1512 | 疆独 1513 | 东突厥斯坦解放组织 1514 | 东突解放组织 1515 | 蒙古分裂分子 1516 | 列确 1517 | 阿旺晋美 1518 | 藏人 1519 | 臧人 1520 | zang人 1521 | 藏民 1522 | 藏m 1523 | 达赖 1524 | 赖达 1525 | dalai 1526 | 哒赖 1527 | dl喇嘛 1528 | 丹增嘉措 1529 | 打砸抢 1530 | 西独 1531 | 藏独 1532 | 葬独 1533 | 臧独 1534 | 藏毒 1535 | 藏du 1536 | zangdu 1537 | 支持zd 1538 | 藏暴乱 1539 | 藏青会 1540 | 雪山狮子旗 1541 | 拉萨 1542 | 啦萨 1543 | 啦沙 1544 | 啦撒 1545 | 拉sa 1546 | lasa 1547 | la萨 1548 | 西藏 1549 | 藏西 1550 | 藏春阁 1551 | 藏獨 1552 | 藏独 1553 | 藏独立 1554 | 藏妇会 1555 | 藏青会 1556 | 藏字石 1557 | xizang 1558 | xi藏 1559 | x藏 1560 | 西z 1561 | tibet 1562 | 希葬 1563 | 希藏 1564 | 硒藏 1565 | 稀藏 1566 | 西脏 1567 | 西奘 1568 | 西葬 1569 | 西臧 1570 | 援藏 1571 | bjork 1572 | 王千源 1573 | 安拉 1574 | 回教 1575 | 回族 1576 | 回回 1577 | 回民 1578 | 穆斯林 1579 | 穆罕穆德 1580 | 穆罕默德 1581 | 默罕默德 1582 | 伊斯兰 1583 | 圣战组织 1584 | 清真 1585 | 清zhen 1586 | qingzhen 1587 | 真主 1588 | 阿拉伯 1589 | 高丽棒子 1590 | 韩国狗 1591 | 满洲第三帝国 1592 | 满狗 1593 | 鞑子 1594 | 江丑闻 1595 | 江嫡系 1596 | 江毒 1597 | 江独裁 1598 | 江蛤蟆 1599 | 江核心 1600 | 江黑心 1601 | 江胡内斗 1602 | 江祸心 1603 | 江家帮 1604 | 江绵恒 1605 | 江派和胡派 1606 | 江派人马 1607 | 江泉集团 1608 | 江人马 1609 | 江三条腿 1610 | 江氏集团 1611 | 江氏家族 1612 | 江氏政治局 1613 | 江氏政治委员 1614 | 江梳头 1615 | 江太上 1616 | 江戏子 1617 | 江系人 1618 | 江系人马 1619 | 江宰民 1620 | 江贼 1621 | 江贼民 1622 | 江主席 1623 | 麻果丸 1624 | 麻将透 1625 | 麻醉弹 1626 | 麻醉狗 1627 | 麻醉枪 1628 | 麻醉槍 1629 | 麻醉药 1630 | 麻醉藥 1631 | 台独 1632 | 台湾版假币 1633 | 台湾独立 1634 | 台湾国 1635 | 台湾应该独立 1636 | 台湾有权独立 1637 | 天灭中共 1638 | 中共帮凶 1639 | 中共保命 1640 | 中共裁 1641 | 中共党文化 1642 | 中共腐败 1643 | 中共的血旗 1644 | 中共的罪恶 1645 | 中共帝国 1646 | 中共独裁 1647 | 中共封锁 1648 | 中共封网 1649 | 中共腐败 1650 | 中共黑 1651 | 中共黑帮 1652 | 中共解体 1653 | 中共近期权力斗争 1654 | 中共恐惧 1655 | 中共权力斗争 1656 | 中共任用 1657 | 中共退党 1658 | 中共洗脑 1659 | 中共邪教 1660 | 中共邪毒素 1661 | 中共政治游戏 --------------------------------------------------------------------------------