├── .DS_Store
├── .idea
├── Tspider.iml
├── copyright
│ └── profiles_settings.xml
├── dictionaries
│ └── rudytan.xml
├── misc.xml
├── modules.xml
├── vcs.xml
└── workspace.xml
├── example
├── .DS_Store
└── lol_hero_god_rank
│ ├── .DS_Store
│ ├── .idea
│ ├── copyright
│ │ └── profiles_settings.xml
│ ├── dictionaries
│ │ └── rudytan.xml
│ ├── lol_hero_rank.iml
│ ├── misc.xml
│ ├── modules.xml
│ ├── vcs.xml
│ └── workspace.xml
│ ├── application
│ ├── .DS_Store
│ ├── config
│ │ ├── .DS_Store
│ │ └── main.php
│ ├── extension
│ │ ├── Common.php
│ │ └── Http.php
│ ├── task
│ │ ├── ExternalStatisticsRequestTimer.php
│ │ ├── HeroGodRankRequestTimer.php
│ │ ├── MonitorDownloadTask.php
│ │ └── MonitorTimer.php
│ └── worker
│ │ ├── ExternalStatisticsWorker.php
│ │ └── HeroGodRankWorker.php
│ ├── framework
│ ├── .DS_Store
│ ├── TSpider.php
│ ├── base
│ │ ├── BaseWorker.php
│ │ ├── Logger.php
│ │ ├── Request.php
│ │ └── Response.php
│ ├── component
│ │ ├── Component.php
│ │ ├── MysqlComponent.php
│ │ ├── RedisComponent.php
│ │ ├── RedisLockComponent.php
│ │ └── SmsWoquComponent.php
│ ├── config
│ │ └── default.php
│ ├── coroutine
│ │ ├── ExampleTask.php
│ │ ├── Task.php
│ │ ├── TaskScheduler.php
│ │ └── TimerTask.php
│ ├── extension
│ │ ├── .DS_Store
│ │ ├── Common.php
│ │ ├── Http.php
│ │ ├── TLock.php
│ │ ├── TMysql.php
│ │ ├── TRedis.php
│ │ └── simple_html_dom.php
│ ├── queue
│ │ ├── LocalRequestQueue.php
│ │ └── RequestQueue.php
│ └── task
│ │ ├── AddRequestTask.php
│ │ ├── DelayTimer.php
│ │ └── DownloadTask.php
│ ├── index.php
│ ├── log
│ └── .DS_Store
│ ├── readme.md
│ └── snatch_lol.sql
├── framework
├── .DS_Store
├── TSpider.php
├── base
│ ├── BaseWorker.php
│ ├── Request.php
│ └── Response.php
├── component
│ ├── Component.php
│ ├── MysqlComponent.php
│ ├── RedisComponent.php
│ ├── RedisLockComponent.php
│ └── SmsWoquComponent.php
├── config
│ └── default.php
├── coroutine
│ ├── ExampleTask.php
│ ├── Task.php
│ ├── TaskScheduler.php
│ └── TimerTask.php
├── extension
│ ├── .DS_Store
│ ├── Common.php
│ ├── THttp.php
│ ├── TLock.php
│ ├── TLogger.php
│ ├── TMysql.php
│ ├── TRedis.php
│ └── simple_html_dom.php
├── queue
│ ├── LocalRequestQueue.php
│ └── RequestQueue.php
└── task
│ ├── AddRequestTask.php
│ ├── DelayTimer.php
│ └── DownloadTask.php
└── readme.md
/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hirudy/Tspider/93ba6b4f9ac1725519af3386c7ca28b33ceb8770/.DS_Store
--------------------------------------------------------------------------------
/.idea/Tspider.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/copyright/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/.idea/dictionaries/rudytan.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
28 |
29 |
30 |
31 |
32 |
33 |
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/example/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hirudy/Tspider/93ba6b4f9ac1725519af3386c7ca28b33ceb8770/example/.DS_Store
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hirudy/Tspider/93ba6b4f9ac1725519af3386c7ca28b33ceb8770/example/lol_hero_god_rank/.DS_Store
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/.idea/copyright/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/.idea/dictionaries/rudytan.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/.idea/lol_hero_rank.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/application/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hirudy/Tspider/93ba6b4f9ac1725519af3386c7ca28b33ceb8770/example/lol_hero_god_rank/application/.DS_Store
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/application/config/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hirudy/Tspider/93ba6b4f9ac1725519af3386c7ca28b33ceb8770/example/lol_hero_god_rank/application/config/.DS_Store
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/application/config/main.php:
--------------------------------------------------------------------------------
1 | 'lolHeroGodRankSpider',
12 | 'logs' => array(
13 | 'beforeDownload' => array(
14 | 'logName' => 'beforeDownload',
15 | 'frequency' => \framework\base\Logger::LOG_FREQUENCY_DAY,
16 | ),
17 | 'afterDownload' => array(
18 | 'logName' => 'afterDownload',
19 | 'frequency' => \framework\base\Logger::LOG_FREQUENCY_DAY,
20 | ),
21 | 'worker' => array(
22 | 'logName' => 'worker',
23 | ),
24 | 'errorDownload' => array(
25 | 'logName' => 'errorDownload',
26 | 'mode' => \framework\base\Logger::LOG_MODE_BOTH
27 | ),
28 | 'errorParse' => array(
29 | 'logName' => 'errorParse',
30 | 'mode' => \framework\base\Logger::LOG_MODE_BOTH
31 | ),
32 | 'common' => array(
33 | 'logName' => 'common',
34 | 'mode' => \framework\base\Logger::LOG_MODE_BOTH,
35 | ),
36 | 'monitor' => array(
37 | 'logName' => 'monitor',
38 | 'mode' => \framework\base\Logger::LOG_MODE_BOTH,
39 | 'frequency' => \framework\base\Logger::LOG_FREQUENCY_DAY,
40 | ),
41 | ),
42 | 'request' => array(
43 | 'maxRepeat' => 3,
44 | 'timeOut' => 300,
45 | 'requestQueue' => 'framework\queue\LocalRequestQueue'
46 | ),
47 | 'downloader' => array(
48 | 'className' => 'application\task\MonitorDownloadTask',
49 | 'windowSize' => 100,
50 | ),
51 | 'tasks' => array(
52 | 'common' => array(
53 | 'application\task\MonitorTimer' => array(),
54 | ),
55 | 'hero_god' => array(
56 | 'application\task\HeroGodRankRequestTimer' => array(),
57 | ),
58 | 'external' => array(
59 | 'application\task\ExternalStatisticsRequestTimer' => array(),
60 | )
61 | ),
62 |
63 | 'component' => array(
64 | 'dbSnatchLol' => array(
65 | 'className' => 'framework\component\MysqlComponent',
66 | 'host'=> '*.*.*.*',
67 | 'userName' => '****',
68 | 'password' => '*****',
69 | 'dbName' => 'snatch_lol',
70 | 'port' => '3306',
71 | 'checkConnection' => true
72 | ),
73 | 'sms' => array(
74 | 'className' => 'framework\component\SmsWoquComponent',
75 | 'serverUrl' => 'http://*.*.*.*:8080/send_sms',
76 | 'Module' => 'spider',
77 | 'To' => '',
78 | 'MsgType' => '0',
79 | )
80 | ),
81 | );
82 |
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/application/extension/Common.php:
--------------------------------------------------------------------------------
1 | $value){
23 | $options[$key] =$value;
24 | }
25 | if(!empty($postData) && is_array($postData)){
26 | $options[CURLOPT_POST] = true;
27 | $options[CURLOPT_POSTFIELDS] = http_build_query($postData);
28 | }
29 | if(stripos($url,'https') === 0){
30 | $options[CURLOPT_SSL_VERIFYPEER] = false;
31 | }
32 | $ch = curl_init();
33 | curl_setopt_array($ch,$options);
34 | $rel = curl_exec($ch);
35 | if($rel == false){
36 | print_r(curl_getinfo($ch));
37 | }
38 | curl_close($ch);
39 | return $rel;
40 | }
41 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/application/task/ExternalStatisticsRequestTimer.php:
--------------------------------------------------------------------------------
1 | requestGenerator = $this->getOneRequest();
27 | parent::__construct($intervalTime, $taskName);
28 | }
29 |
30 | /**
31 | * 获取英雄信息
32 | * @return array|bool
33 | * @throws \framework\base\Exception
34 | */
35 | public static function getChampionInfo(){
36 | if(empty(self::$championInfoList)){
37 | $url = 'http://lol.zhangyoubao.com/apis/rest/RolesService/championInfo?iamsuperman=2';
38 | $championInfoList = false;
39 | for ($i=0;$i < 5;$i++){
40 | $data = Http::request($url);
41 | $data = json_decode($data,true);
42 | if(is_array($data)){
43 | $championInfoList = $data;
44 | break;
45 | }
46 | }
47 | if($championInfoList == false){
48 | $str = "spider exit: championInfo get error;";
49 | Logger::factory('common')->error($str);
50 | TSpider::$component->sms->sendMessage($str,true);
51 | exit($str);
52 | }else{
53 | foreach ($championInfoList as $key=>$row){
54 | if($row['id'] == 1){
55 | // 奎恩
56 | $championInfoList[$key]['enname'] = 'Quinn';
57 | }
58 | self::$championInfoList[$championInfoList[$key]['enname']] =$championInfoList[$key];
59 | }
60 | }
61 | }
62 | return self::$championInfoList;
63 | }
64 |
65 | public function getOneRequest(){
66 | $championInfoList = self::getChampionInfo();
67 | foreach ($championInfoList as $enname => $row){
68 | yield self::createRequest($enname,$row);
69 | }
70 | }
71 |
72 | public static function createRequest($championAlias,$championInfo){
73 | $url = "http://champion.gg/champion/{$championAlias}";
74 | return new Request($url,'application\worker\ExternalStatisticsWorker',array(),array('championAlias'=>$championAlias,'championInfo'=>$championInfo));
75 | }
76 |
77 | /**
78 | * 执行入口,复写父类相同名称方法
79 | */
80 | function execute(){
81 | for($i=0;$i<20;$i++){
82 | $request = $this->requestGenerator->current();
83 | if($request instanceof Request){
84 | if(!$request->download()){
85 | break;
86 | }
87 | $this->requestGenerator->next();
88 | }else{
89 | break;
90 | }
91 | }
92 | }
93 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/application/task/HeroGodRankRequestTimer.php:
--------------------------------------------------------------------------------
1 | requestGenerator = $this->getOneRequest();
25 | parent::__construct($intervalTime, $taskName);
26 | }
27 |
28 | public static function createRequest($tgpId,$page=1){
29 | $time = time().''.rand(100,999);
30 | $url = "http://img.lol.qq.com/js/cevRank/{$tgpId}/{$page}.js?t={$time}";
31 | return new Request($url,'application\worker\HeroGodRankWorker',array(),array('tgpId'=>$tgpId,'page'=>$page));
32 | }
33 |
34 | public function getOneRequest(){
35 | $tgpIdList = Http::request('http://lol.anzogame.com/apis/rest/RolesService/tgpId2IdForPkg?iamsuperman=2');
36 | $tgpIdList = json_decode($tgpIdList,true);
37 | $tgpIdList = array_keys($tgpIdList);
38 | if(empty($tgpIdList)){
39 | $str = 'get tgpId error! exit spider';
40 | Logger::factory('common')->error($str);
41 | TSpider::$component->sms->sendMessage($str,true);
42 | exit($str);
43 | }
44 |
45 | foreach($tgpIdList as $tgpId){
46 | yield self::createRequest($tgpId);
47 | }
48 | }
49 |
50 | /**
51 | * 执行入口,复写父类相同名称方法
52 | */
53 | function execute(){
54 | for($i=0;$i<30;$i++){
55 | $request = $this->requestGenerator->current();
56 | if($request instanceof Request){
57 | if(!$request->download()){
58 | break;
59 | }
60 | $this->requestGenerator->next();
61 | }else{
62 | break;
63 | }
64 | }
65 | }
66 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/application/task/MonitorDownloadTask.php:
--------------------------------------------------------------------------------
1 | beforeDownloadLog = Logger::factory('beforeDownload');
35 | $this->afterDownloadLog = Logger::factory('afterDownload');
36 | $this->errorDownloadLog = Logger::factory('errorDownload');
37 | // $this->proxyLog = Logger::factory('proxy');
38 | parent::__construct($taskName);
39 | }
40 |
41 | public function countSpeed(){
42 | static $lastTime;
43 | static $downloadNum;
44 |
45 | $intLastTime = (int)$lastTime;
46 | $downloadNum = (int)$downloadNum;
47 | $downloadNum ++;
48 | if(time()-$intLastTime>= self::$speedSecond){
49 | self::$statisticsDownloadSpeed = "{$downloadNum}/m";
50 | $downloadNum = 0;
51 | $lastTime =time();
52 | }
53 | }
54 |
55 |
56 | public function beforeDownload(Request $request){
57 | self::$statisticsAllRequestNum ++;
58 | $this->beforeDownloadLog->log((string)$request);
59 | return $request;
60 | }
61 |
62 | public function afterDownload(Request $request,Response $response){
63 | self::$statisticsAllDownloadNum ++;
64 | $this->countSpeed();
65 | if($response->code == 200 || $response->code == 404){
66 | $this->afterDownloadLog->log($request->url.'|'.$response->code);
67 | }else{
68 | self::$statisticsErrorNum ++;
69 | $this->afterDownloadLog->error($request->url.'|'.$response->code.'|'.$request->getRepeatNum().'|'.$response->getError());
70 |
71 | // 重新下载
72 | if(!$request->redownload()){
73 | // 最终下载失败的
74 | $url = substr($request->url,-35);
75 | TSpider::$component->sms->sendMessage("download error:$url");
76 | $this->errorDownloadLog->error($request->url.'|'.$response->code.'|'.$request->getRepeatNum().'|'.$response->getError());
77 | }
78 | return false;
79 | }
80 | return null;
81 | }
82 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/application/task/MonitorTimer.php:
--------------------------------------------------------------------------------
1 | log = Logger::factory('monitor');
26 | parent::__construct($intervalTime, $taskName);
27 | }
28 |
29 | /**
30 | * 保存通用爬虫参数
31 | * @param $prefix
32 | * @param $dbName
33 | * @throws \framework\base\Exception
34 | */
35 | public function saveCrawlData($prefix,$dbName){
36 | $data = array(
37 | $prefix.'pre_version' => TSpider::$startCrawlTime,
38 | $prefix.'update_time' => date('Y-m-d H:i:s',TSpider::$startCrawlTime),
39 | $prefix.'use_time' => time()-TSpider::$startCrawlTime,
40 | $prefix.'version' => TSpider::$startCrawlTime
41 | );
42 |
43 | foreach($data as $key=>$value){
44 | $sql = '';
45 | try{
46 | $connection = TSpider::$component->$dbName;
47 | if($prefix.'pre_version' == $key){
48 | $sql = "select `value` from control where `key`='{$prefix}version' limit 1;";
49 | $rel = $connection->query($sql);
50 | if(!empty($rel)){
51 | $value = $rel[0]['value'];
52 | }
53 | }
54 | $sql = "select `value` from control where `key`='{$key}';";
55 | $rel = $connection->query($sql);
56 | if(empty($rel)){
57 | $sql = "insert control(`key`,`value`) values('{$key}','{$value}')";
58 | }else{
59 | $sql = "update control set `value`='{$value}' where `key`='{$key}';";
60 | }
61 |
62 | $connection->query($sql);
63 | }catch(\Exception $e){
64 | Logger::factory('common')->error('sql error: '.$sql.' | '.$e->getMessage());
65 | }
66 | }
67 | }
68 |
69 | function execute(){
70 | $name = TSpider::$spiderName;
71 | $requestQueueCount = TSpider::$requestQueue->count();
72 | $requestNum = MonitorDownloadTask::$statisticsAllRequestNum;
73 | $downloadNum = MonitorDownloadTask::$statisticsAllDownloadNum;
74 | $downloadErrorNum = MonitorDownloadTask::$statisticsErrorNum;
75 | $downloadSpeed = MonitorDownloadTask::$statisticsDownloadSpeed;
76 | $parserNum = BaseWorker::$statisticsParserNum;
77 | $parserErrorNum = BaseWorker::$statisticsParserErrorNum;
78 | $saveNum = BaseWorker::$statisticsSaveNum;
79 | $saveErrorNum = BaseWorker::$statisticsSaveErrorNum;
80 | $memory = Common::getMemoryUsedSizeShow();
81 | $str = "Memory:{$memory}|Queue:{$requestQueueCount}|Downloader:{$requestNum}-{$downloadNum}-{$downloadErrorNum}|Parser:{$parserNum}-{$parserErrorNum}-{$saveNum}-{$saveErrorNum}|Speed:{$downloadSpeed}";
82 | $this->log->log($str);
83 |
84 | // 当下载器空闲多长时间,关闭爬虫
85 | $downloaderSpareTime = TSpider::$downloader->getSpareTime();
86 | if($downloaderSpareTime >= 10){
87 | TaskScheduler::closeAllTask();
88 | // 保存操作变量
89 | switch (TSpider::$taskGroup){
90 | case 'hero_god':{
91 | $this->saveCrawlData('hero_god_rank_','dbSnatchLol');
92 | }break;
93 | case 'external':{
94 | $this->saveCrawlData('hero_external_','dbSnatchLol');
95 | }break;
96 | default:{
97 | $this->saveCrawlData('hero_god_rank_','dbSnatchLol');
98 | $this->saveCrawlData('hero_external_','dbSnatchLol');
99 | }
100 | }
101 | // 发送结束短信
102 | $str = "{$name} End:an-{$requestNum},dn-{$downloadNum},de-{$downloadErrorNum},pn-{$parserNum},pe-{$parserErrorNum},sn-{$saveNum},se-{$saveErrorNum}";
103 | TSpider::$component->sms->sendMessage($str,true);
104 | Logger::factory('common')->log($str);
105 | }
106 | }
107 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/application/worker/ExternalStatisticsWorker.php:
--------------------------------------------------------------------------------
1 | errorLog = Logger::factory('errorParse');
36 | $this->log = Logger::factory('worker');
37 | }
38 |
39 |
40 | /**
41 | * 重写解析方法
42 | * @param Request $request
43 | * @param Response $response
44 | */
45 | public function parse(Request $request,Response $response){
46 | if($response->code != 200){
47 | $this->errorLog->log($request->url."| ".$response->code." | ".$response->getError().' | '.json_encode($request->extData['championInfo']));
48 | return null;
49 | }
50 |
51 | $extractResult = $this->extract($response->getData(),$request);
52 | if($extractResult == false){
53 | // 重新下载
54 | if(!$request->redownload()){
55 | // 最终下载失败的
56 | $url = substr($request->url,-35);
57 | TSpider::$component->sms->sendMessage("parse error:$url");
58 | $this->errorLog->error('parse error:'.$request->url.'|'.$response->code.'|'.$response->getError().'|'.$this->lastError);
59 | }
60 | return;
61 | }
62 |
63 | $this->log->log($request->url.' | '.$request->getRepeatNum());
64 | $this->save($extractResult,$request);
65 | }
66 |
67 | /**
68 | * 提取数据
69 | * @param $data
70 | * @return array|bool
71 | */
72 | public function extract($data,$rawRequest){
73 | $response = array();
74 | $dom = str_get_html($data);
75 | if(!$this->extractItemUse($dom,$response,$rawRequest)){
76 | return false;
77 | }
78 |
79 | if(!$this->extractStatisticsChart($dom,$response,$rawRequest)){
80 | return false;
81 | }
82 |
83 | return $response;
84 | }
85 |
86 | /**
87 | * 获取物品的数据,结果写入response中
88 | * @param $dom
89 | * @param $response
90 | * @return bool
91 | */
92 | protected function extractItemUse($dom,&$response,$rawRequest){
93 | do{
94 | // 解析装备使用推荐
95 | $nodeList = $dom->find('div.col-md-7 div.build-wrapper');
96 | if(!is_array($nodeList) || count($nodeList) != 2){
97 | $this->lastError ='wrapper parse error';
98 | break;
99 | }
100 | $itemData = array();
101 | $itemUseData = array();
102 | foreach ($nodeList as $node){
103 | // 获取itemID
104 | $img_node_list = $node->find('a img');
105 | $tempImg = array();
106 | foreach ($img_node_list as $img_node){
107 | $src = $img_node->src;
108 | if (preg_match('#(\d+)\.png#',$src,$match)){
109 | $itemId = (int)$match[1];
110 | if($itemId > 0){
111 | $tempImg[] = $itemId;
112 | }
113 | }
114 | }
115 | if(count($tempImg) == 6){
116 | $itemData[] = $tempImg;
117 | }
118 |
119 | // 获取item使用情况
120 | $use_node_list = $node->find('div.build-text strong');
121 | $tempUse = array();
122 | foreach ($use_node_list as $use_node){
123 | $text = (float)$use_node->innertext;
124 | if($text > 0){
125 | $tempUse[] = $text;
126 | }
127 | }
128 |
129 | if(count($tempUse) == 2){
130 | $itemUseData[] = $tempUse;
131 | }
132 | }
133 | if(count($itemData) != 2 && count($itemUseData) != 2){
134 | $this->lastError = 'item win rate num error';
135 | $this->errorLog->error($this->lastError.'|'.$rawRequest->url);
136 | // break;
137 | }
138 |
139 | $totalData = array();
140 | foreach ($itemData as $key=>$value){
141 | if($key == 0){
142 | $name = self::ITEM_FREQUENT;
143 | }elseif($key == 1){
144 | $name = self::ITEM_WIN;
145 | }else{
146 | break;
147 | }
148 | $totalData[$name]['item'] = $value;
149 | $totalData[$name]['win_rate'] = $itemUseData[$key][0];
150 | $totalData[$name]['game_count'] = $itemUseData[$key][1];
151 | }
152 |
153 | // 结果写入返回
154 | $response['item_data'] = $totalData;
155 | return true;
156 | }while(false);
157 | return false;
158 | }
159 |
160 | /**
161 | * 提取4个统计图表数据,结果写入response中
162 | * @param $dom
163 | * @param $response
164 | * @return bool
165 | */
166 | protected function extractStatisticsChart($dom,&$response,$rawRequest){
167 | $script_dom_list = $dom->find('script');
168 | $data_text = '';
169 | foreach ($script_dom_list as $script_dom){
170 | $text = $script_dom->innertext;
171 | if(stripos($text,'matchupData.championData') !== false) {
172 | $data_text = $text;
173 | break;
174 | }
175 | }
176 |
177 | do{
178 | if(empty($data_text)){
179 | $this->lastError = 'no js data';
180 | break;
181 | }
182 | // 解析general_role
183 | if(!preg_match('#matchupData\.generalRole *= *(\{.+?\});#',$data_text,$match)){
184 | $this->lastError = 'preg_match error:generalRole';
185 | break;
186 | }
187 | $data_general_role = json_decode($match[1],true);
188 | if(!is_array($data_general_role)){
189 | $this->lastError = 'json_decode error:generalRole';
190 | break;
191 | }
192 |
193 | // 解析champion_data
194 | if(!preg_match('#matchupData\.championData *= *(\{.+?\});#',$data_text,$match)){
195 | $this->lastError = 'preg_match error:championData';
196 | break;
197 | }
198 | $data_champion_data = json_decode($match[1],true);
199 | if(!is_array($data_general_role)){
200 | $this->lastError = 'json_decode error:championData';
201 | break;
202 | }
203 |
204 | // 解析patch_history
205 | if(!preg_match('#matchupData\.patchHistory *= *(\[.+?\]);#',$data_text,$match)){
206 | $this->lastError = 'preg_match error:patchHistory';
207 | break;
208 | }
209 | $data_patch_history = json_decode($match[1],true);
210 | if(!is_array($data_general_role)){
211 | $this->lastError = 'json_decode error:patchHistory';
212 | break;
213 | }
214 |
215 | $result = array();
216 |
217 | $championName = $rawRequest->extData['championInfo']['name'];
218 | // 添加patch_win
219 | $temp_arr = array('target'=>array('y_prefix'=>'','y_suffix'=>'%'),'series'=>array());
220 | $x_show = $data_patch_history;
221 | $serie1 = array('name'=>'英雄平均胜率','data'=>array());
222 | $serie2 = array('name'=>$championName,'data'=>array());
223 | $y_2 = isset($data_champion_data['patchWin'])?$data_champion_data['patchWin']:array();
224 | if(count($x_show) !== count($y_2)){
225 | $this->lastError = 'patch_win:y2 num error';
226 | break;
227 | }
228 | foreach ($x_show as $key=>$value){
229 | $serie1['data'][] = array($value,50);
230 | $serie2['data'][] = array($value,(float)$y_2[$key]);
231 | }
232 | $temp_arr['series'][] = $serie1;
233 | $temp_arr['series'][] = $serie2;
234 | $result[self::CHART_PATCH_WIN] = $temp_arr;
235 |
236 | // 添加patch_play
237 | $temp_arr = array('target'=>array('y_prefix'=>'','y_suffix'=>'%'),'series'=>array());
238 | $x_show = $data_patch_history;
239 | $serie1 = array('name'=>'英雄平均出场率','data'=>array());
240 | $serie2 = array('name'=>$championName,'data'=>array());
241 | $y_1 = isset($data_general_role['patchPlay'])?$data_general_role['patchPlay']:array();
242 | $y_2 = isset($data_champion_data['patchPlay'])?$data_champion_data['patchPlay']:array();
243 | if(count($x_show) !== count($y_2) || count($x_show) !== count($y_1)){
244 | $this->lastError = 'patch_play:y1/y2 num error';
245 | break;
246 | }
247 | foreach ($x_show as $key=>$value){
248 | $serie1['data'][] = array($value,(float)$y_1[$key]);
249 | $serie2['data'][] = array($value,(float)$y_2[$key]);
250 | }
251 | $temp_arr['series'][] = $serie1;
252 | $temp_arr['series'][] = $serie2;
253 | $result[self::CHART_PATCH_PLAY] = $temp_arr;
254 |
255 | // 添加game_length_win
256 | $temp_arr = array('target'=>array('y_prefix'=>'','y_suffix'=>'%'),'series'=>array());
257 | $x_show = array('0-25','25-30','30-35','35-40','40+');
258 | $serie1 = array('name'=>'英雄平均胜率','data'=>array());
259 | $serie2 = array('name'=>$championName,'data'=>array());
260 | $y_2 = isset($data_champion_data['gameLength'])?$data_champion_data['gameLength']:array();
261 | if(count($x_show) !== count($y_2)){
262 | $this->lastError = 'game_length_win:y2 num error';
263 | break;
264 | }
265 | foreach ($x_show as $key=>$value){
266 | $serie1['data'][] = array($value,50);
267 | $serie2['data'][] = array($value,(float)$y_2[$key]);
268 | }
269 | $temp_arr['series'][] = $serie1;
270 | $temp_arr['series'][] = $serie2;
271 | $result[self::CHART_GAME_LENGTH_WIN] = $temp_arr;
272 |
273 | // 添加game_play_win
274 | $temp_arr = array('target'=>array('y_prefix'=>'','y_suffix'=>'%'),'series'=>array());
275 | $x_show = array('1-5','5-15','15-50','50-125','125+');
276 | $serie1 = array('name'=>'英雄平均胜率','data'=>array());
277 | $serie2 = array('name'=>$championName,'data'=>array());
278 | $y_1 = array_fill(0,count($x_show),end($data_champion_data['patchWin']));
279 | $y_2 = isset($data_champion_data['experienceRate'])?$data_champion_data['experienceRate']:array();
280 | if(count($x_show) !== count($y_2)){
281 | $this->lastError = 'game_length_win:y2 num error';
282 | break;
283 | }
284 | foreach ($x_show as $key=>$value){
285 | $serie1['data'][] = array($value,(float)$y_1[$key]);
286 | $serie2['data'][] = array($value,(float)$y_2[$key]);
287 | }
288 | $temp_arr['series'][] = $serie1;
289 | $temp_arr['series'][] = $serie2;
290 | $result[self::CHART_GAME_PLAY_WIN] = $temp_arr;
291 |
292 | $response['statistics_chart'] = $result;
293 | return true;
294 | }while(false);
295 |
296 | return false;
297 | }
298 |
299 | /**
300 | * 存储经过提取后的数据
301 | * @param $extractData
302 | * @throws \Exception
303 | */
304 | public function save($extractData,$request){
305 | try{
306 | $fields = array(
307 | 'version' => TSpider::$startCrawlTime,
308 | 'champion_id' => (int)$request->extData['championInfo']['tgp_id'],
309 | 'item_frequent' => json_encode(Common::getArrayValue($extractData['item_data'],self::ITEM_FREQUENT,array())),
310 | 'item_win' => json_encode(Common::getArrayValue($extractData['item_data'],self::ITEM_WIN,array())),
311 | 'chart_patch_win' => json_encode(Common::getArrayValue($extractData['statistics_chart'],self::CHART_PATCH_WIN,array())),
312 | 'chart_patch_play' => json_encode(Common::getArrayValue($extractData['statistics_chart'],self::CHART_PATCH_PLAY,array())),
313 | 'chart_game_length_win' => json_encode(Common::getArrayValue($extractData['statistics_chart'],self::CHART_GAME_LENGTH_WIN,array())),
314 | 'chart_game_play_win' => json_encode(Common::getArrayValue($extractData['statistics_chart'],self::CHART_GAME_PLAY_WIN,array())),
315 | );
316 | $rel = TSpider::$component->dbSnatchLol->insert('hero_external_statistics',$fields);
317 | if($rel <= 0){
318 | throw new \Exception('insert error:',-1);
319 | }
320 | self::$statisticsSaveNum ++;
321 | }catch(\Exception $e){
322 | $temp = substr($e->getMessage(),-35);
323 | TSpider::$component->sms->sendMessage("save error:{$temp}");
324 | $this->errorLog->error('save error:'.$e->getMessage().'|'.json_encode($extractData));
325 | }
326 | }
327 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/application/worker/HeroGodRankWorker.php:
--------------------------------------------------------------------------------
1 | errorLog = Logger::factory('errorParse');
25 | $this->log = Logger::factory('worker');
26 | }
27 |
28 |
29 | /**
30 | * 重写解析方法
31 | * @param Request $request
32 | * @param Response $response
33 | */
34 | public function parse(Request $request,Response $response){
35 | if($response->code == 200){
36 | $newRequest = HeroGodRankRequestTimer::createRequest($request->extData['tgpId'],$request->extData['page']+1);
37 | $newRequest->download();
38 | }else{
39 | return null;
40 | }
41 | $extractResult = $this->extract($response->getData());
42 | if($extractResult == false){
43 | // 重新下载
44 | if(!$request->redownload()){
45 | // 最终下载失败的
46 | $url = substr($request->url,-35);
47 | TSpider::$component->sms->sendMessage("parse error:$url");
48 | $this->errorLog->error('parse error:'.$request->url.'|'.$response->code.'|'.$response->getData());
49 | }
50 | return;
51 | }
52 |
53 | $this->log->log($request->url.' | '.$request->getRepeatNum());
54 | $this->save($extractResult,$request->extData['tgpId']);
55 | }
56 |
57 | /**
58 | * 提取数据
59 | * @param $data
60 | * @return array|bool
61 | */
62 | public function extract($data){
63 | $data = str_replace(array("try{heroSkillCallback(",")}catch(e){}"),array('',''),$data);
64 | $data = json_decode($data,true);
65 | if(!isset($data['retCode']) || !isset($data['data']['skillRank']) || $data['retCode'] !== 0){
66 | return false;
67 | }
68 | return $data['data']['skillRank'];
69 | }
70 |
71 |
72 | /**
73 | * 存储经过提取后的数据
74 | * @param $extractData
75 | * @throws \Exception
76 | */
77 | public function save($extractData,$heroId=0){
78 | try{
79 | $data_list = array();
80 | foreach($extractData as $row){
81 | $temp = array();
82 | $temp['version'] = TSpider::$startCrawlTime;
83 | $temp['hero_id'] = $heroId;
84 | $temp['rank'] = $row['index'];
85 | $temp['area_id'] = $row['area_id'];
86 | $temp['area_name'] = $row['areaName'];
87 | $temp['icon_id'] = $row['iconId'];
88 | $temp['uin'] = $row['uin'];
89 | $temp['username'] = $row['uName'];
90 | $temp['proficiency'] = $row['cevValue'];
91 |
92 | $data_list[] = $temp;
93 | }
94 | $rel = TSpider::$component->dbSnatchLol->insertMulti('hero_god_rank',$data_list);
95 | if($rel <= 0){
96 | throw new \Exception('insert error:',-1);
97 | }
98 | self::$statisticsSaveNum ++;
99 | }catch(\Exception $e){
100 | $temp = substr($e->getMessage(),-35);
101 | TSpider::$component->sms->sendMessage("save error:{$temp}");
102 | $this->errorLog->error('save error:'.$e->getMessage().'|'.json_encode($extractData));
103 | }
104 | }
105 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hirudy/Tspider/93ba6b4f9ac1725519af3386c7ca28b33ceb8770/example/lol_hero_god_rank/framework/.DS_Store
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/TSpider.php:
--------------------------------------------------------------------------------
1 | init($protectedName);
48 | }
49 |
50 | public static function init($protectedName){
51 | self::$startCrawlTime = time();
52 | self::$protectedName = $protectedName;
53 | self::$frameworkBasePath = dirname(__FILE__);
54 | self::$basePath = dirname(self::$frameworkBasePath);
55 | self::$applicationPath = self::$basePath.DIRECTORY_SEPARATOR.self::$protectedName;
56 |
57 | ini_set('date.timezone','Asia/Shanghai');
58 | spl_autoload_register('\framework\TSpider::autoLoadFile');
59 |
60 | self::checkEnvironment();
61 |
62 | // 解析命令
63 | self::parseCommand();
64 |
65 | // 导入引用的三方扩展
66 | self::searchAndInclude(self::$frameworkBasePath.'/extension');
67 | self::searchAndInclude(self::$applicationPath.'/extension');
68 |
69 | // 加载配置文件
70 | $default_config = include self::$frameworkBasePath.DIRECTORY_SEPARATOR.'config'.DIRECTORY_SEPARATOR.'default.php';
71 | $user_config_path = self::$applicationPath.DIRECTORY_SEPARATOR.'config'.DIRECTORY_SEPARATOR.'main.php';
72 | if(!is_file($user_config_path)){
73 | throw new \Exception("config file not found:{$user_config_path}",-1);
74 | }
75 | $user_config = include $user_config_path;
76 | if(!is_array($user_config)){
77 | throw new \Exception("config file not load error:{$user_config_path}",-1);
78 | }
79 |
80 | self::$config = array_merge($default_config,$user_config);
81 |
82 | self::$spiderName = Common::getArrayValue(self::$config,'name','defaultSpider');
83 | self::$isDebug = Common::getArrayValue(self::$config,'debug',false);
84 |
85 | if(self::$isDebug){
86 | ini_set("display_errors", "on");
87 | error_reporting(E_ALL^E_STRICT);
88 | }
89 |
90 | // 加载logger
91 | Logger::loadOneConfig(array('logName'=>'system'));
92 | Logger::$g_basePath = self::$config['logPath'];
93 | Logger::loadConfig(self::$config['logs']);
94 | self::$log = Logger::factory('system');
95 |
96 | // 设置request相关
97 | Request::$maxRepeat = self::$config['request']['maxRepeat'];
98 | Request::$timeOut = self::$config['request']['timeOut'];
99 | self::$requestQueue = new self::$config['request']['requestQueue']();
100 | if(!(self::$requestQueue instanceof RequestQueue)){
101 | $temp = self::$config['request']['requestQueue'];
102 | throw new \Exception("requestQueue error:{$temp}",-1);
103 | }
104 |
105 | // 加载必须任务,初始化任务调度器,添加下载器任务
106 | self::$taskScheduler = new TaskScheduler();
107 | if(!isset(self::$config['downloader'])){
108 | throw new \Exception('there is no download class config');
109 | }
110 | $downloaderClass = self::$config['downloader']['className'];
111 | self::$downloader = new $downloaderClass();
112 | self::$downloader->windowSize = self::$config['downloader']['windowSize'];
113 | self::$taskScheduler->addTask(self::$downloader);
114 |
115 | // 添加请求队列读取任务,以及自定义任务,all分组表示运行所有
116 | $addRequestTask = new AddRequestTask(self::$downloader);
117 | self::$taskScheduler->addTask($addRequestTask);
118 | if(self::$taskGroup == 'all'){
119 | foreach(self::$config['tasks'] as $key=>$value){
120 | if(is_array($value)){
121 | foreach ($value as $className=>$params){
122 | $tempTask = new $className();
123 | self::$taskScheduler->addTask($tempTask);
124 | }
125 | }
126 | }
127 | }else{
128 | if(isset(self::$config['tasks']['common'])){
129 | foreach(self::$config['tasks']['common'] as $key=>$value){
130 | $tempTask = new $key();
131 | self::$taskScheduler->addTask($tempTask);
132 | }
133 | }
134 |
135 | if(isset(self::$config['tasks'][self::$taskGroup]) && self::$taskGroup != 'common'){
136 | foreach(self::$config['tasks'][self::$taskGroup] as $className=>$params){
137 | $tempTask = new $className();
138 | self::$taskScheduler->addTask($tempTask);
139 | }
140 | }else{
141 | $taskGroup = self::$taskGroup;
142 | exit("taskGroup not found:{$taskGroup}\n");
143 | }
144 | }
145 |
146 | // 初始化组件配置文件
147 | if(isset(self::$config['component']) && is_array(self::$config['component'])){
148 | Component::$config = self::$config['component'];
149 | }
150 | self::$component = new Component();
151 |
152 | // 设置进程名称
153 | $title = self::$spiderName;
154 | $taskGroup = self::$taskGroup;
155 | $startTime = date('Y-m-d H:i:s',self::$startCrawlTime);
156 | self::setProcessTitle("TSpider:{$title}-{$taskGroup} startTime:{$startTime}");
157 | }
158 |
159 | public static function autoLoadFile($rawName){
160 | $name = str_replace('\\', DIRECTORY_SEPARATOR ,$rawName);
161 | $classFile = self::$basePath . DIRECTORY_SEPARATOR . $name . '.php';
162 |
163 | if(is_file($classFile)){
164 | if(!class_exists($rawName,true)){
165 | $rel = require $classFile;
166 | return $rel;
167 | }
168 | }else{
169 | throw new Exception("auto load File error:{$classFile}",-1);
170 | }
171 | return false;
172 | }
173 |
174 | /**
175 | * 解析接收的命令
176 | */
177 | public static function parseCommand(){
178 | if(isset($_SERVER['argv']) && count($_SERVER['argv']) >=2){
179 | $command = $_SERVER['argv'][1];
180 | if(strlen($command)){
181 | self::$taskGroup = $command;
182 | }else{
183 | self::$taskGroup = '';
184 | }
185 | }else{
186 | self::$taskGroup = 'all';
187 | }
188 | echo 'taskGroup:',self::$taskGroup,"\n";
189 | }
190 |
191 |
192 | /**
193 | * 检测运行环境
194 | */
195 | public static function checkEnvironment(){
196 | echo 'PHP-version:',PHP_VERSION,' TSpider-version:',self::VERSION,' start-time:',date('Y-m-d H:i:s',self::$startCrawlTime),"\n";
197 |
198 | if(version_compare(PHP_VERSION,'5.5.0','<=')){
199 | exit('php version must greater than 5.5.0');
200 | }
201 |
202 | if(substr(php_sapi_name(), 0, 3) != 'cli'){
203 | exit('this program must be running in cli mode ');
204 | }
205 |
206 | if(extension_loaded('posix')){
207 | $userInfo = posix_getpwuid(posix_getuid());
208 | echo 'pid:',posix_getpid()," ",'running-user:',$userInfo['name'],"\n";
209 | }
210 | }
211 |
212 | /**
213 | * 设置进程名称
214 | * @param $title
215 | */
216 | public static function setProcessTitle($title)
217 | {
218 | if (function_exists('cli_set_process_title')) {
219 | @cli_set_process_title($title);
220 | } elseif (extension_loaded('proctitle') && function_exists('setproctitle')) {
221 | @setproctitle($title);
222 | }
223 | }
224 |
225 | /**
226 | * 导入某个文件夹中的所有文件
227 | * @param $rootPath
228 | */
229 | public static function searchAndInclude($rootPath){
230 | if(is_dir($rootPath)){
231 | $rootPath = rtrim($rootPath,'/');
232 | $files = glob($rootPath.'/*');
233 | foreach ($files as $file){
234 | if(is_dir($file)){
235 | self::searchAndInclude($file);
236 | }else{
237 | include_once $file;
238 | }
239 | }
240 | }
241 | }
242 |
243 | public static function run(){
244 | self::$taskScheduler->run();
245 | }
246 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/base/BaseWorker.php:
--------------------------------------------------------------------------------
1 | code} : {$request->url} \n";
26 | }
27 |
28 | /**
29 | * 解析前hook
30 | * @return bool
31 | */
32 | public function beforeParse(){
33 | self::$statisticsParserNum ++;
34 | return true;
35 | }
36 |
37 | /**
38 | * 解析后hook
39 | */
40 | public function afterParse(){
41 | // TODO: Implement afterParse() method.
42 | }
43 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/base/Logger.php:
--------------------------------------------------------------------------------
1 | array(
42 | 'isLogging' => true,
43 | 'basePath' => '',
44 | 'suffix' => 'log',
45 | 'level' => array(self::LOG_LEVEL_ERROR,self::LOG_LEVEL_WARN,self::LOG_LEVEL_INFO),
46 | 'mode' => self::LOG_MODE_NORMAL,
47 | 'frequency' => self::LOG_FREQUENCY_NONE,
48 | ),
49 | );
50 |
51 | protected static $logPool = array(); // 日志对象池子
52 |
53 | protected $isLoging; //当前日志,是否记录
54 | protected $logName; //当前日志,日志名称
55 | protected $basePath; //当前日志,存储路径
56 | protected $suffix; //当前日志,日志文件后缀
57 | protected $level; //当前日志,运行记录的日志等级
58 | protected $mode; //当前日志,记录方式
59 | protected $frequency; //当前日志,日志记录每隔一(分钟/小时/天/月)换一个文件记录
60 |
61 | private $logFilePath; //完整的日志路径
62 | private $timeLength; //存储时间长度
63 |
64 | /**
65 | * 以二维数组的形式加载多个日志配置文件
66 | * @param $arr array 二维数组
67 | */
68 | public static function loadConfig($arr){
69 | if(!empty($arr) && is_array($arr)){
70 | foreach($arr as $name =>$config){
71 | self::loadOneConfig($config,$name);
72 | }
73 | }
74 | }
75 |
76 | /**
77 | * 加载一个日志配置文件
78 | * @param $arr
79 | * @param string $name
80 | * @return bool
81 | */
82 | public static function loadOneConfig($arr,$name=''){
83 | if(!is_array($arr)){
84 | $arr = array();
85 | }
86 | $name = isset($arr['logName'])?$arr['logName']:$name;
87 | $name = str_replace(array('\\','/'),'_',$name);
88 | if(!is_string($name) || empty($name) || $name == 'default'){
89 | return false;
90 | }
91 | unset($arr['logName']);
92 | self::$g_config_arr[$name] = $arr;
93 |
94 | return true;
95 | }
96 |
97 |
98 | /**
99 | * 根据日志名称,获取一个日志实例
100 | * @param string $logName 配置名称
101 | * @param bool $isNew 是否生成一个新的日志对象
102 | * @return Logger
103 | * @throws Exception
104 | */
105 | public static function factory($logName='default',$isNew=false){
106 | $logName = str_replace(array('\\','/'),'_',$logName);
107 | if(!isset(self::$logPool[$logName]) || $isNew == true){
108 | if(isset(self::$logPool[$logName])){
109 | unset(self::$logPool[$logName]);
110 | }
111 |
112 | if(empty($logName) || !is_string($logName) || !isset(self::$g_config_arr[$logName])){
113 | throw new Exception("Make sure that the log configuration which name is '{$logName}' is loaded successfully");
114 | }
115 | self::$logPool[$logName] = new self(self::$g_config_arr[$logName],$logName);
116 | }
117 | return self::$logPool[$logName];
118 | }
119 |
120 | /**
121 | * Logger 构造函数,不能直接new Logger()
122 | * @param string $logName
123 | * @param array $config
124 | * @throws Exception
125 | */
126 | protected function __construct($config = array(),$logName='default'){
127 | $this->isLoging = (isset($config['isLogging']))?$config['isLogging']:self::$g_config_arr['default']['isLogging'];
128 | $this->logName = (empty($logName) || !is_string($logName))?'default':$logName;
129 | $this->basePath = (isset($config['basePath']) && !empty($config['basePath']))?$config['basePath']:self::$g_basePath;
130 | $this->suffix = isset($config['suffix'])?$config['suffix']:self::$g_config_arr['default']['suffix'];
131 | $this->level = (isset($config['level']) && is_array($config['level']))?$config['level']:self::$g_config_arr['default']['level'];
132 | $this->mode = isset($config['mode'])?$config['mode']:self::$g_config_arr['default']['mode'];
133 | $this->frequency = isset($config['frequency'])?$config['frequency']:self::$g_config_arr['default']['frequency'];
134 |
135 | $this->basePath = rtrim($this->basePath,"\\/");
136 | if(!is_dir($this->basePath)){
137 | if( !mkdir($this->basePath,0755,true)){
138 | throw new Exception("create directory fail:".$this->basePath);
139 | }
140 | }
141 | $this->logFilePath = $this->basePath.DIRECTORY_SEPARATOR.$this->logName.'.'.$this->suffix;
142 | switch($this->frequency){
143 | case self::LOG_FREQUENCY_MINUTE: $this->timeLength = 12;break;
144 | case self::LOG_FREQUENCY_HOUR: $this->timeLength = 10;break;
145 | case self::LOG_FREQUENCY_DAY: $this->timeLength = 8;break;
146 | case self::LOG_FREQUENCY_MONTH: $this->timeLength = 6;break;
147 | default:
148 | $this->timeLength = -1;
149 | }
150 | }
151 |
152 |
153 | /**
154 | * 受保护的写日志方法
155 | * @param $filePath
156 | * @param $content
157 | * @return bool|int
158 | */
159 | protected function write($filePath,$content){
160 | $return_value = false;
161 | $content = $content."\n";
162 | switch($this->mode){
163 | case self::LOG_MODE_NORMAL:{
164 | $return_value = file_put_contents($filePath,$content,LOCK_EX | FILE_APPEND);
165 | $return_value = (int)$return_value > 0 ?true:false;
166 | }break;
167 | case self::LOG_MODE_PRINT:{
168 | echo $this->logName,':',$content;
169 | $return_value = true;
170 | }break;
171 | case self::LOG_MODE_BOTH:{
172 | echo $this->logName,':',$content;
173 | file_put_contents($filePath,$content,LOCK_EX | FILE_APPEND);
174 | $return_value = (int)$return_value > 0 ?true:false;
175 | }break;
176 | }
177 | return $return_value;
178 | }
179 |
180 |
181 | /**
182 | * 原始记录日志函数
183 | * @param $content
184 | * @param int $level
185 | * @return bool|int
186 | */
187 | public function log($content,$level = self::LOG_LEVEL_INFO){
188 | if(!self::$g_isLogging || !$this->isLoging || !in_array($level,$this->level)){
189 | return false;
190 | }
191 |
192 | if(!is_string($content)){
193 | if(is_array($content)){
194 | $content = json_encode($content,JSON_UNESCAPED_UNICODE);
195 | }else{
196 | $content = serialize($content);
197 | }
198 | }
199 |
200 | $logTime = time(); //记录日志时间
201 |
202 | // 检测是否需要对日志文件进行重命名
203 | if($this->timeLength > 0){
204 | $fileCreateTime = @filectime($this->logFilePath);
205 | if($fileCreateTime){
206 | $logTimeFormat = substr(@date('YmdHis',$logTime),0,$this->timeLength);
207 | $createTimeFormat = substr(@date('YmdHis',$fileCreateTime),0,$this->timeLength);
208 | if(strcmp($logTimeFormat,$createTimeFormat) !== 0){
209 | $newLogFilePath = $this->basePath.DIRECTORY_SEPARATOR.$this->logName.'_'.$createTimeFormat.'.'.$this->suffix;
210 | rename($this->logFilePath,$newLogFilePath);
211 | }
212 | }
213 | }
214 |
215 | //构造日志记录格式
216 | switch($level){
217 | case self::LOG_LEVEL_ERROR:{
218 | $content = sprintf('[%s %s] %s',@date('Y-m-d H:i:s',$logTime),'error',$content);
219 | }break;
220 | case self::LOG_LEVEL_WARN:{
221 | $content = sprintf('[%s %s] %s',@date('Y-m-d H:i:s',$logTime),'warn',$content);
222 | }break;
223 | default:
224 | $content = sprintf('[%s %s] %s',@date('Y-m-d H:i:s',$logTime),'info',$content);
225 | }
226 |
227 | //记录日志
228 | return $this->write($this->logFilePath,$content);
229 | }
230 |
231 | /**
232 | * 记录错误信息
233 | * @param $content
234 | * @return bool|int
235 | */
236 | public function error($content){
237 | return $this->log($content,self::LOG_LEVEL_ERROR);
238 | }
239 |
240 | /**
241 | * 记录警告信息
242 | * @param $content
243 | * @return bool|int
244 | */
245 | public function warn($content){
246 | return $this->log($content,self::LOG_LEVEL_WARN);
247 | }
248 | }
249 |
250 | //测试
251 | if(strtolower(PHP_SAPI) == 'cli' && isset($argv) && basename(__FILE__) == basename($argv[0])){
252 | $config = array( // 日志配置文件数组,default是默认配置项
253 | 'test1/a' => array(
254 | 'logName' => 'bb'
255 | ),
256 | );
257 |
258 | Logger::loadConfig($config);
259 | $log = Logger::factory('bb');
260 | $data = $log->error('hello 哈哈1中文');
261 | echo $data,"\n";
262 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/base/Request.php:
--------------------------------------------------------------------------------
1 | url = $url;
37 | $this->workerName = $workerName;
38 |
39 | $this->options[CURLOPT_URL] = $url;
40 | $this->options[CURLOPT_TIMEOUT] = self::$timeOut;
41 | $this->options[CURLOPT_USERAGENT] = 'Mozilla/5.0 (Windows NT 6.2; WOW64; Trident/7.0; rv:11.0) like Gecko';
42 | $this->options[CURLOPT_ENCODING] = 'gzip, deflate ';
43 | $this->extData = $extData;
44 | if(empty($postData) || !is_array($postData)){
45 | $this->type = 'GET';
46 | }else{
47 | $this->type = 'POST';
48 | $this->postData = $postData;
49 | $this->options[CURLOPT_POST] = true;
50 | $this->options[CURLOPT_POSTFIELDS] = $postData;
51 | }
52 | }
53 |
54 | /**
55 | * 对象字符串化
56 | * @return string
57 | */
58 | public function __toString(){
59 | $json_arr = array(
60 | 'url' => $this->url,
61 | 'workerName' => $this->workerName,
62 | 'options' => $this->options,
63 | 'postData' => $this->postData,
64 | 'repeat' => $this->repeat,
65 | 'header' => $this->header,
66 | );
67 | $returnString = json_encode($json_arr);
68 | if(!is_string($returnString)){
69 | $returnString = '';
70 | }
71 | return $returnString;
72 | }
73 |
74 | /**
75 | * 设置头部信息数组
76 | * @param $options
77 | */
78 | public function setOptions($options){
79 | foreach($options as $key=>$value){
80 | $this->options[$key] = $value;
81 | }
82 | }
83 |
84 | /**
85 | * 累加重复次数
86 | */
87 | public function addRepeat(){
88 | $this->repeat ++;
89 | }
90 |
91 | /**
92 | * 获取重复次数
93 | * @return int
94 | */
95 | public function getRepeatNum(){
96 | return $this->repeat;
97 | }
98 |
99 | /**
100 | * 是否能重复下载
101 | * @return bool
102 | */
103 | public function canRepeat(){
104 | if(self::$maxRepeat < $this->repeat){
105 | return false;
106 | }
107 | return true;
108 | }
109 |
110 | /**
111 | * 重新下载
112 | * @return bool true-添加成功,false-添加失败
113 | */
114 | public function redownload(){
115 | $this->addRepeat();
116 | if($this->canRepeat()) {
117 | TSpider::$requestQueue->add($this);
118 | return true;
119 | }
120 | return false;
121 | }
122 |
123 |
124 | /**
125 | * 将请求放入下载队列中,队列满了,返回失败
126 | * @return bool
127 | */
128 | public function download(){
129 | if(TSpider::$requestQueue->isFull()){
130 | return false;
131 | }
132 | TSpider::$requestQueue->add($this);
133 | return true;
134 | }
135 |
136 |
137 | public function createCurlObject(){
138 | $ch = curl_init();
139 | if($this->canRepeat()){
140 | if(!empty($this->options)){
141 | curl_setopt_array($ch,$this->options);
142 | }
143 | curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
144 | }else{
145 | $ch = false;
146 | }
147 | return $ch;
148 | }
149 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/base/Response.php:
--------------------------------------------------------------------------------
1 | info = $info;
22 | $this->data = $data;
23 | $this->error = $error;
24 | if(isset($this->info['http_code'])){
25 | $this->code = $this->info['http_code'];
26 | }
27 | }
28 |
29 | public function getData(){
30 | return $this->data;
31 | }
32 |
33 | public function getError(){
34 | return $this->error;
35 | }
36 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/component/Component.php:
--------------------------------------------------------------------------------
1 | getInstance();
31 | }
32 |
33 | $className = isset(self::$config[$name]['className'])?self::$config[$name]['className']:'';
34 | if(!class_exists($className)){
35 | throw new \Exception("have no component className:{$className}\n",-1);
36 | }
37 |
38 | $component = new $className($name,self::$config[$name]);
39 | if(!($component instanceof Component)){
40 | throw new \Exception("className:{$className} is not Component\n",-1);
41 | }
42 |
43 | self::$instanceMap[$name] = $component;
44 |
45 | return self::$instanceMap[$name]->getInstance();
46 | }
47 |
48 | public function __construct($componentName='',$config=array()){
49 | $this->componentName = $componentName;
50 | $this->oneConfig = $config;
51 | }
52 |
53 |
54 |
55 | public function getInstance(){
56 | return $this;
57 | }
58 |
59 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/component/MysqlComponent.php:
--------------------------------------------------------------------------------
1 | isLoadConfig){
19 | $this->oneConfig['connectionName'] = $this->componentName;
20 | TMysql::loadOneConfig($this->oneConfig);
21 | $this->isLoadConfig = true;
22 | }
23 | return TMysql::getConnection($this->componentName);
24 | }
25 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/component/RedisComponent.php:
--------------------------------------------------------------------------------
1 | isLoadConfig){
19 | $this->oneConfig['connectionName'] = $this->componentName;
20 | TRedis::loadOneConfig($this->oneConfig);
21 | $this->isLoadConfig = true;
22 | }
23 | return TRedis::getConnection($this->componentName);
24 | }
25 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/component/RedisLockComponent.php:
--------------------------------------------------------------------------------
1 | isLoadConfig){
21 | $this->oneConfig['connectionName'] = $this->componentName;
22 | TRedis::loadOneConfig($this->oneConfig);
23 | $this->isLoadConfig = true;
24 | }
25 |
26 | return new TLock($this->componentName,$this->getArrayValue($this->oneConfig,'prefix','lock_'),$this->getArrayValue($this->oneConfig,'expire',3600));
27 | }
28 |
29 |
30 | /**
31 | * 安全地获取数组的值
32 | * @param $arr array 数组
33 | * @param $key String 键名
34 | * @param string $default 默认值
35 | * @return string
36 | */
37 | public static function getArrayValue($arr,$key,$default=''){
38 | if(isset($arr[$key])){
39 | return $arr[$key];
40 | }
41 | return $default;
42 | }
43 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/component/SmsWoquComponent.php:
--------------------------------------------------------------------------------
1 | lastSendTime) > $interval_time){
37 | $this->lastSendTime = $now;
38 | $this->sendMessageNum = 1;
39 | return true;
40 | }
41 | if($this->sendMessageNum <= $maxSendNum){
42 | $this->sendMessageNum ++;
43 | return true;
44 | }
45 | return false;
46 | }
47 |
48 |
49 | /**
50 | * 发送消息函数
51 | * @param string $content 发送的消息内容
52 | * @param bool $needSend 当前短信是否必须发送出去
53 | * @return bool
54 | */
55 | public function sendMessage($content,$needSend=false){
56 | if(!($this->canSendMessage() || $needSend)){
57 | return false;
58 | }
59 | $phoneNumbers = explode(',',$this->oneConfig['To']);
60 | if(empty($this->oneConfig['To'])){
61 | Http::request($this->oneConfig['serverUrl'],array(
62 | 'Module' => $this->oneConfig['Module'],
63 | 'MsgType' => $this->oneConfig['MsgType'],
64 | 'To' => '',
65 | 'MsgText' => $content));
66 | }else{
67 | foreach($phoneNumbers as $key=>$phone){
68 | Http::request($this->oneConfig['serverUrl'],array(
69 | 'Module' => $this->oneConfig['Module'],
70 | 'MsgType' => $this->oneConfig['MsgType'],
71 | 'To' => $phone,
72 | 'MsgText' => $content));
73 | }
74 | }
75 | return true;
76 | }
77 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/config/default.php:
--------------------------------------------------------------------------------
1 | 'defaultSpider',
12 | 'debug' => false,
13 | 'logPath' => dirname(dirname(dirname(__FILE__))).DIRECTORY_SEPARATOR.'log'.DIRECTORY_SEPARATOR,
14 | 'logs' => array(
15 | 'system' => array(
16 | 'logName' => 'system',
17 | )
18 | ),
19 | 'downloader' => array(
20 | 'className' => 'framework\task\DownloadTask',
21 | 'windowSize' => 50,
22 | ),
23 | 'request' => array(
24 | 'maxRepeat' => 5,
25 | 'timeOut' => 300,
26 | 'requestQueue' => 'framework\queue\LocalRequestQueue'
27 | ),
28 | 'tasks' => array(
29 | 'common' => array(
30 | 'framework\task\DelayTimer'=>array()
31 | )
32 | ),
33 | 'component' => array(
34 | // 填写相应组件
35 | ),
36 | );
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/coroutine/ExampleTask.php:
--------------------------------------------------------------------------------
1 | taskId = ++self::$maxTaskId;
24 |
25 | $this->taskName = $taskName;
26 | if(empty($taskName)){
27 | $this->taskName = get_class();
28 | }
29 | $taskContent = $this->coroutine();
30 | if($taskContent instanceof \Generator){
31 | $this->taskContent = $taskContent;
32 | }else{
33 | throw new Exception('Task is not a coroutine',-1);
34 | }
35 | }
36 |
37 | public function getTaskId() {
38 | return $this->taskId;
39 | }
40 |
41 | public function getTaskName() {
42 | return $this->taskName;
43 | }
44 |
45 | public function setSendValue($sendValue) {
46 | $this->sendValue = $sendValue;
47 | }
48 |
49 | abstract public function coroutine();
50 |
51 | public function run() {
52 | if ($this->beforeFirstYield) {
53 | $this->beforeFirstYield = false;
54 | return $this->taskContent->current();
55 | } else {
56 | $retval = $this->taskContent->send($this->sendValue);
57 | $this->sendValue = null;
58 | return $retval;
59 | }
60 | }
61 |
62 | public function isFinished() {
63 | return !$this->taskContent->valid();
64 | }
65 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/coroutine/TaskScheduler.php:
--------------------------------------------------------------------------------
1 | task
16 | protected $taskQueue = null;
17 |
18 | protected static $is_closed = false;
19 |
20 | public function __construct() {
21 | $this->taskQueue = new \SplQueue();
22 | }
23 |
24 | public static function closeAllTask(){
25 | self::$is_closed = true;
26 | }
27 |
28 | public static function isAllTaskClosed(){
29 | return self::$is_closed;
30 | }
31 |
32 | public function addTask(Task $task){
33 | $taskId = $task->getTaskId();
34 | $this->taskMap[$taskId] = $task;
35 | $this->schedule($task);
36 | return $taskId;
37 | }
38 |
39 | public function schedule(Task $task) {
40 | $this->taskQueue->enqueue($task);
41 | }
42 |
43 | public function run() {
44 | while (!$this->taskQueue->isEmpty()) {
45 | $task = $this->taskQueue->dequeue();
46 | $task->run();
47 |
48 | if ($task->isFinished()) {
49 | unset($this->taskMap[$task->getTaskId()]);
50 | } else {
51 | $this->schedule($task);
52 | }
53 | if(TaskScheduler::isAllTaskClosed()){
54 | break;
55 | }
56 | }
57 | }
58 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/coroutine/TimerTask.php:
--------------------------------------------------------------------------------
1 | 0){
21 | $this->intervalTime = $intervalTime;
22 | }
23 | if(empty($taskName)){
24 | $taskName = get_class($this);
25 | }
26 | $this->expireTime = time()+$this->intervalTime;
27 | parent::__construct($taskName);
28 | }
29 |
30 | public function coroutine(){
31 | while(true){
32 | if($this->expireTime <= time()){
33 | $rel = $this->execute();
34 | $this->expireTime = time()+$this->intervalTime;
35 | if($rel === false){
36 | break;
37 | }
38 | }
39 | yield true;
40 | }
41 | }
42 |
43 | public abstract function execute();
44 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/extension/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hirudy/Tspider/93ba6b4f9ac1725519af3386c7ca28b33ceb8770/example/lol_hero_god_rank/framework/extension/.DS_Store
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/extension/Common.php:
--------------------------------------------------------------------------------
1 | '','errorList'=>array());
14 |
15 | public static function request($url,$postData=array(),$header=array()){
16 | $options = array();
17 | $url = trim($url);
18 | $options[CURLOPT_URL] = $url;
19 | $options[CURLOPT_TIMEOUT] = 10;
20 | $options[CURLOPT_USERAGENT] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.89 Safari/537.36';
21 | $options[CURLOPT_RETURNTRANSFER] = true;
22 | foreach($header as $key=>$value){
23 | $options[$key] =$value;
24 | }
25 | if(!empty($postData) && is_array($postData)){
26 | $options[CURLOPT_POST] = true;
27 | $options[CURLOPT_POSTFIELDS] = http_build_query($postData);
28 | }
29 | if(stripos($url,'https') === 0){
30 | $options[CURLOPT_SSL_VERIFYPEER] = false;
31 | }
32 | $ch = curl_init();
33 | curl_setopt_array($ch,$options);
34 | $rel = curl_exec($ch);
35 | if($rel == false){
36 | $errno = curl_errno( $ch );
37 | $error = curl_error($ch);
38 | self::$lastError['error'] = "({$errno})$error";
39 | self::$lastError['errorList'] = curl_getinfo($ch);
40 | }
41 | curl_close($ch);
42 | return $rel;
43 | }
44 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/extension/TLock.php:
--------------------------------------------------------------------------------
1 | connectionName = $connectionName;
21 | $this->uniqueId = uniqid();
22 | $this->prefix = $prefix;
23 | $this->expire = $expire;
24 | }
25 |
26 |
27 | /**
28 | * 获取一个非阻塞锁 true-获取到,false-没有获取到
29 | * @param $resource_name
30 | * @return bool
31 | * @throws RedisOperationException
32 | */
33 | public function lock($resource_name){
34 | if(!empty($this->resource_name)){
35 | return false;
36 | }
37 | $this->resource_name = $resource_name;
38 |
39 | $redis = TRedis::getConnection($this->connectionName);
40 | if($redis->setNx($this->resource_name,$this->uniqueId)){
41 | $redis->expire($this->resource_name,$this->expire);
42 | return true;
43 | }
44 | return false;
45 | }
46 |
47 | /**
48 | * 获得一个阻塞锁
49 | * @param $resource_name
50 | * @return bool
51 | * @throws RedisOperationException
52 | */
53 | public function lockWait($resource_name){
54 | if(!empty($this->resource_name)){
55 | return false;
56 | }
57 | $this->resource_name = $resource_name;
58 |
59 | do{
60 | $redis = TRedis::getConnection($this->connectionName);
61 | if($redis->setNx($this->resource_name,$this->uniqueId)){
62 | $redis->expire($this->resource_name,$this->expire);
63 | break;
64 | }
65 |
66 | sleep(1);
67 | }while(true);
68 | return true;
69 | }
70 |
71 | /**
72 | * 解锁,只能解锁自己对象的说
73 | * @throws RedisOperationException
74 | */
75 | public function unlock(){
76 | $redis = TRedis::getConnection($this->connectionName);
77 | $rel = $redis->get($this->resource_name);
78 | if($rel == $this->uniqueId){
79 | $redis->delete($this->resource_name);
80 | }
81 | }
82 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/extension/TMysql.php:
--------------------------------------------------------------------------------
1 | connectionName = $connectionName;
17 | $this->dbName = $dbname;
18 | $dbConnection = new \mysqli($host, $username, $passwd, $dbname, $port, $socket);
19 | $dbConnection->options(MYSQLI_OPT_CONNECT_TIMEOUT,5);
20 | if ($dbConnection->connect_error){
21 | throw new \Exception("mysql {$connectionName} connection failure:({$dbConnection->connect_errno}){$dbConnection->connect_error}",-1);
22 | }
23 | $dbConnection->set_charset($charset);
24 | $this->dbConnection = $dbConnection;
25 | }
26 |
27 | public function __destruct(){
28 | if ($this->dbConnection){
29 | // 操作对象销毁时候,关闭连接
30 | @$this->dbConnection->close();
31 | }
32 | }
33 |
34 | /**
35 | * 查看当前连接是否关闭
36 | * @return bool
37 | */
38 | public function ping(){
39 | // $autoReconnect = empty(ini_get('mysqli.reconnect'));
40 | try{
41 | if(@$this->dbConnection->ping()){
42 | return true;
43 | }else{
44 | @$this->dbConnection->close();
45 | return false;
46 | }
47 | }catch (\Exception $e){
48 | @$this->dbConnection->close();
49 | return false;
50 | }
51 | }
52 |
53 | /**
54 | * 获取当前默认的字符集
55 | * @return string 字符集
56 | */
57 | public function getCharset(){
58 | return $this->dbConnection->character_set_name();
59 | }
60 |
61 |
62 | /**
63 | * 获取select结果集中的所有结果
64 | * @param $option
65 | * @return array
66 | */
67 | protected function fetch_all($option){
68 | $result = array();
69 | if(method_exists($option,'fetch_all')){
70 | $result = $option->fetch_all(MYSQLI_ASSOC);
71 | }else{
72 | while($row = $option->fetch_assoc()){
73 | $result[] = $row;
74 | }
75 | }
76 |
77 | return $result;
78 | }
79 |
80 |
81 |
82 | /**
83 | * 切换默认数据库
84 | * @param $dbName String 要切换的数据库名称
85 | * @return $this 做链式访问
86 | * @throws Exception
87 | */
88 | public function switchDb($dbName){
89 | if (is_string($dbName) && !empty($dbName)){
90 | if($this->dbConnection->select_db($dbName)){
91 | return $this;
92 | };
93 | throw new \Exception("mysql {$this->connectionName} switch db from {$this->dbName} to {$dbName} failure:({$this->dbConnection->errno}){$this->dbConnection->error}",-1);
94 | }
95 | throw new \Exception("mysql {$this->connectionName} switch db from {$this->dbName} to {$dbName} failure:param error",-1);
96 | }
97 |
98 |
99 | /**
100 | * 执行数据库操作
101 | * @param $sql String 要执行的sql语句
102 | * @param array $params 如果不为空,进行参数预处理
103 | * @return bool|int|mixed 执行select等返回结果集数组,执行insert返回自增值,执行其他返回受影响的行数。失败返回false
104 | * @throws Exception 支持出错抛出异常。
105 | */
106 | public function query($sql,$params=array()){
107 | $returnValue = true;
108 | $sql = trim($sql,' ');
109 | if (empty($params)){
110 | $result = $this->dbConnection->query($sql);
111 | if ($result === false){
112 | if ($this->dbConnection->error){
113 | throw new \Exception("{$this->connectionName} query error:({$this->dbConnection->errno}){$this->dbConnection->error}");
114 | }else{
115 | return false;
116 | }
117 | }
118 |
119 | if ($result === true){
120 | if (stripos($sql,'insert') === false){
121 | return $this->dbConnection->affected_rows;
122 | }else{
123 | return $this->dbConnection->insert_id;
124 | }
125 | }
126 | $returnValue = $this->fetch_all($result);
127 | $result->close();
128 | }else{
129 | $statement = $this->dbConnection->prepare($sql);
130 | if ($statement == false){
131 | throw new \Exception("{$this->connectionName} prepare error:({$this->dbConnection->errno}){$this->dbConnection->error}");
132 | }
133 | $types = '';
134 | $data = array('');
135 | foreach($params as $key=>$value){
136 | $type = gettype($value);
137 | switch($type){
138 | case 'string':{
139 | $types .= 's';
140 | $data[] = &$params[$key];
141 | }break;
142 | case 'integer':{
143 | $types .= 'i';
144 | $data[] = &$params[$key];
145 | }break;
146 | case 'double':{
147 | $types .= 'd';
148 | $data[] = &$params[$key];
149 | }break;
150 | default:
151 | $types .= 'b';
152 | $data[] = &$params[$key];
153 | }
154 | }
155 | $data[0] = $types;
156 | $method = new \ReflectionMethod($statement,'bind_param');
157 | if ($method->invokeArgs($statement,$data) == false){
158 | throw new \Exception("{$this->connectionName} bind_param error:({$statement->errno}){$statement->error}");
159 | }
160 | $rel = $statement->execute();
161 | if ($rel == false){
162 | throw new \Exception("{$this->connectionName} execute error:({$statement->errno}){$statement->error}");
163 | }
164 | $arr = explode(' ',$sql);
165 | $type = isset($arr[0])?strtolower($arr[0]):'';
166 | switch($type){
167 | case 'explain':
168 | case 'select':{
169 | $rel = $statement->get_result();
170 | if ($rel === false){
171 | throw new \Exception("{$this->connectionName} select error:({$this->dbConnection->errno}){$this->dbConnection->error}");
172 | }
173 | $returnValue = $this->fetch_all($rel);
174 | }break;
175 | case 'insert':{
176 | $returnValue = $this->dbConnection->insert_id;
177 | }break;
178 | case 'update':
179 | case 'delete':{
180 | $returnValue = $this->dbConnection->affected_rows;
181 | };
182 | }
183 | $statement->close();
184 | }
185 |
186 | return $returnValue;
187 | }
188 |
189 | /**
190 | * 根据数组插入一条记录
191 | * @param string $table 要插入的表名
192 | * @param array $arr 插入的数组(关联数组)
193 | * @return bool|int|mixed
194 | * @throws \Exception
195 | */
196 | public function insert($table,$arr = array()){
197 | if(empty($arr) || empty($table) || !is_string($table) || !is_array($arr)){
198 | return false;
199 | }
200 | $fields = array_keys($arr);
201 | $tempArr = array_fill(0,count($fields),'?');
202 | $params = array_values($arr);
203 |
204 | foreach ($fields as $index => $row){
205 | $fields[$index] = "`{$row}`";
206 | }
207 | $fields = '('.implode(',',$fields).')';
208 | $tempArr = '('.implode(',',$tempArr).')';
209 | $sql = "insert `{$table}`{$fields} values{$tempArr};";
210 | return $this->query($sql,$params);
211 | }
212 |
213 | /**
214 | * 一次性插入多条记录
215 | * @param string $table 插入的表名称
216 | * @param array $arr 二维数组,具有相同的表结构
217 | * @return int 返回插入的最后一条记录的自增id
218 | */
219 | public function insertMulti($table,$arr=array(array(),array())){
220 | if(empty($arr) || empty($table) || !is_string($table) || !is_array($arr) || !is_array($arr[0])){
221 | return false;
222 | }
223 | $fields = array_keys($arr[0]);
224 | $tempArr = array_fill(0,count($fields),'?');
225 | foreach ($fields as $index => $row){
226 | $fields[$index] = "`{$row}`";
227 | }
228 | $fields = '('.implode(',',$fields).')';
229 |
230 | $tempArrString = '('.implode(',',$tempArr).')';
231 | $params = array();
232 | $tempArr = array();
233 | foreach ($arr as $row){
234 | $tempArr[] = $tempArrString;
235 | $temp = array_values($row);
236 | $params = array_merge($params,$temp);
237 | }
238 | $tempArrString = implode(',',$tempArr);
239 |
240 |
241 | $sql = "insert `{$table}`{$fields} values{$tempArrString};";
242 | return $this->query($sql,$params);
243 | }
244 |
245 |
246 | /**
247 | * 返回mysqli对象
248 | * @return mysqli|null
249 | */
250 | public function getConnection(){
251 | $connection = $this->dbConnection;
252 | if($connection instanceof \mysqli){
253 | return $connection;
254 | }else{
255 | return null;
256 | }
257 | }
258 | }
259 |
260 |
261 | class TMysql{
262 | private static $dbConnectionPool = array();
263 | private static $configs = array();
264 |
265 | public static function loadOneConfig($arr){
266 | mysqli_report(MYSQLI_REPORT_ALL^MYSQLI_REPORT_INDEX);
267 | if (!extension_loaded('mysqli')){
268 | throw new \Exception('need extension mysqli!', -1);
269 | }
270 | if (!is_array($arr) || empty($arr)){
271 | throw new \Exception('config not empty!', -1);
272 | }
273 | if (!isset($arr['connectionName']) || empty($arr['connectionName']) || !is_string($arr['connectionName'])){
274 | throw new \Exception('connectionName not exit or not a string!',-1);
275 | }
276 |
277 | $config = array();
278 | $config['connectionName'] = $arr['connectionName'];
279 | $config['host'] = isset($arr['host'])?$arr['host']:'';
280 | $config['userName'] = isset($arr['userName'])?$arr['userName']:'';
281 | $config['password'] = isset($arr['password'])?$arr['password']:'';
282 | $config['dbName'] = isset($arr['dbName'])?$arr['dbName']:'';
283 | $config['port'] = isset($arr['port'])?$arr['port']:3306;
284 | $config['checkConnection'] = isset($arr['checkConnection'])?isset($arr['checkConnection']):false;
285 | $config['charset'] = isset($arr['charset'])?$arr['charset']:'utf8';
286 |
287 | self::$configs[$config['connectionName']] = $config;
288 | if (isset(self::$dbConnectionPool[$config['connectionName']])){
289 | unset(self::$dbConnectionPool[$config['connectionName']]);
290 | }
291 | return true;
292 | }
293 |
294 | public static function getConnection($connectionName){
295 | if (!is_string($connectionName) || !isset(self::$configs[$connectionName])){
296 | throw new \Exception("mysql config : {$connectionName} is't loaded !",-1);
297 | }
298 |
299 | // 获取连接前,判断连接是否可用
300 | if (isset(self::$dbConnectionPool[$connectionName]) && self::$configs[$connectionName]['checkConnection']){
301 | $rel = self::$dbConnectionPool[$connectionName]->ping();
302 | if($rel == false){
303 | unset(self::$dbConnectionPool[$connectionName]);
304 | }
305 | }
306 |
307 | // 如果不存在连接,新建一个连接
308 | if(!isset(self::$dbConnectionPool[$connectionName])){
309 | $config = self::$configs[$connectionName];
310 | $dbConnection = new MysqlOperation($connectionName,$config['host'],$config['userName'],$config['password'],$config['dbName'],$config['port'],$config['charset']);
311 | self::$dbConnectionPool[$connectionName] = $dbConnection;
312 | }
313 |
314 | return self::$dbConnectionPool[$connectionName];
315 | }
316 |
317 | public static function clearAllConnection(){
318 | self::$dbConnectionPool = array();
319 | }
320 | }
321 |
322 |
323 | //测试
324 | if(strtolower(PHP_SAPI) == 'cli' && isset($argv) && basename(__FILE__) == basename($argv[0])){
325 | $config = array(
326 | 'connectionName' => 'dbLol',
327 | 'host'=> '*.*.*',
328 | 'userName' => 'xxxx',
329 | 'password' => '*****',
330 | 'dbName' => 'test',
331 | 'port' => '3306'
332 | );
333 |
334 | TMysql::loadOneConfig($config);
335 | $lolConnection = TMysql::getConnection('dbLol');
336 | $data = $lolConnection->query('select * from test where id<=?;',array(10));
337 | print_r($data);
338 | }
339 |
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/extension/TRedis.php:
--------------------------------------------------------------------------------
1 | close();
14 | }
15 | }
16 |
17 | class RedisOperationException extends \RedisException{
18 |
19 | }
20 |
21 | class TRedis {
22 | public static $maxConnectionTime = 5; // 最大连接时间
23 | protected static $connectionPool = array(); // 连接池对象
24 |
25 | protected static $configs = array(); // 配置项
26 |
27 | /**
28 | * 获取数组值
29 | * @param array $arr
30 | * @param string $key
31 | * @param null $default
32 | * @return mixed|null
33 | */
34 | public static function getArrayValue($arr=array(), $key='',$default=null){
35 | $response = null;
36 | if(is_array($arr) && isset($arr[$key])){
37 | $response = $arr[$key];
38 | }
39 | if($response == null){
40 | $response = $default;
41 | }
42 | return $response;
43 | }
44 |
45 | /**
46 | * 加载单个配置文件
47 | * @param array $configArr
48 | */
49 | public static function loadOneConfig($configArr= array()){
50 | $tempConfig = array();
51 | $tempConfig['connectionName'] = self::getArrayValue($configArr,'connectionName'); // 连接名称
52 | $tempConfig['host'] = self::getArrayValue($configArr,'host'); // 连接host
53 | $tempConfig['port'] = self::getArrayValue($configArr,'port',6379); // 端口号
54 | $tempConfig['database'] = self::getArrayValue($configArr,'database',0); // 使用数据库索引
55 | $tempConfig['password'] = self::getArrayValue($configArr,'password'); // 使用密码
56 | $tempConfig['prefix'] = self::getArrayValue($configArr,'prefix',''); // 数据库查询中的key前缀
57 | $tempConfig['checkConnection'] = self::getArrayValue($configArr,'checkConnection',false);
58 | if(empty($tempConfig['connectionName']) || !is_string($tempConfig['connectionName'])){
59 | $tempConfig['name'] = 'default';
60 | }
61 | self::$configs[$tempConfig['connectionName']] = $tempConfig;
62 | }
63 |
64 | /**
65 | * 获取一个连接对象
66 | * @param string $connectionName
67 | * @return mixed
68 | * @throws \Exception
69 | * @throws \RedisException
70 | */
71 | public static function getConnection($connectionName ='default'){
72 | if (!is_string($connectionName) || !isset(self::$configs[$connectionName])){
73 | throw new RedisOperationException("redis config : {$connectionName} is't loaded !",-1);
74 | }
75 |
76 | // 检测连接是否断开,断开重连
77 | if(isset(self::$connectionPool[$connectionName]) && self::$configs[$connectionName]['checkConnection']){
78 | try{
79 | $redisObject = self::$connectionPool[$connectionName];
80 | if($redisObject->ping() != '+PONG'){
81 | throw new RedisOperationException("redis ping error",-1);
82 | }
83 | }catch (RedisOperationException $e){
84 | @self::$connectionPool[$connectionName]->close();
85 | unset(self::$connectionPool[$connectionName]);
86 | }
87 | }
88 |
89 | // 创建连接
90 | if(!isset(self::$connectionPool[$connectionName])){
91 | $redisObject = new RedisOperation();
92 | $rel = $redisObject->connect(self::$configs[$connectionName]['host'],self::$configs[$connectionName]['port'],self::$maxConnectionTime);
93 | if(!$rel){
94 | throw new RedisOperationException("redis connection error:{$connectionName}",-1);
95 | }
96 | if(!empty(self::$configs[$connectionName]['password'])){
97 | $rel = $redisObject->auth(self::$configs[$connectionName]['password']);
98 | if(!$rel){
99 | $redisObject->close();
100 | throw new RedisOperationException("redis connection auth error:{$connectionName}",-1);
101 | }
102 | }
103 | $db = (int)self::$configs[$connectionName]['database'];
104 | $rel = $redisObject->select($db);
105 | if(!$rel){
106 | $redisObject->close();
107 | throw new RedisOperationException("redis select db error:name-{$connectionName},db-{$db}",-1);
108 | }
109 | if(is_string(self::$configs[$connectionName]['prefix']) && !empty(self::$configs[$connectionName]['prefix'])){
110 | $redisObject->setOption(\Redis::OPT_PREFIX,self::$configs[$connectionName]['prefix']);
111 | }
112 |
113 |
114 | self::$connectionPool[$connectionName] = $redisObject;
115 | }
116 | return self::$connectionPool[$connectionName];
117 | }
118 |
119 | /**
120 | * 清理连接
121 | */
122 | public static function clearAllConnection(){
123 | self::$connectionPool = array();
124 | }
125 | }
126 |
127 |
128 | //测试
129 | if(strtolower(PHP_SAPI) == 'cli' && isset($argv) && basename(__FILE__) == basename($argv[0])){
130 | $config = array(
131 | 'connectionName' => 'cacheRd',
132 | 'host'=> '*.*.*.*',
133 | // 'password' => '*****',
134 | 'database' => 1,
135 | 'port' => 6379,
136 | 'prefix' => 'redis_test_',
137 | 'checkConnection' => true
138 | );
139 |
140 | TRedis::loadOneConfig($config);
141 | $redis = TRedis::getConnection('cacheRd');
142 | // $redis->hSet('h', 'key1', 'hello');
143 | // $redis->expire('h',10);
144 | echo $redis->dbSize();
145 | print_r($redis->keys('*'));
146 | // print_r($testString);
147 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/queue/LocalRequestQueue.php:
--------------------------------------------------------------------------------
1 | queue = new \SplQueue();
19 | }
20 |
21 | public function add(Request $request){
22 | $this->queue->enqueue($request);
23 | }
24 |
25 | public function get(){
26 | if($this->isEmpty()){
27 | return null;
28 | }
29 | return $this->queue->dequeue();
30 | }
31 |
32 | public function isEmpty(){
33 | return $this->queue->isEmpty();
34 | }
35 |
36 | public function count(){
37 | return $this->queue->count();
38 | }
39 |
40 | public function isFull(){
41 | if($this->count() >= $this->maxCount){
42 | return true;
43 | }
44 | return false;
45 | }
46 |
47 | public function __destruct(){
48 | unset($this->queue);
49 | }
50 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/queue/RequestQueue.php:
--------------------------------------------------------------------------------
1 | downloader = $downloader;
22 | parent::__construct($taskName);
23 | }
24 |
25 | public function coroutine(){
26 | do{
27 | if (!TSpider::$requestQueue->isEmpty()){
28 | do{
29 | $request = TSpider::$requestQueue->get();
30 | $rel = $this->downloader->addRequest($request);
31 |
32 | // 如果下载队列添加失败,将请求重新放回请求队列中
33 | if($request instanceof Request && !$rel){
34 | TSpider::$requestQueue->add($request);
35 | }
36 | }while($rel);
37 | }
38 | yield true;
39 | }while(true);
40 | }
41 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/framework/task/DelayTimer.php:
--------------------------------------------------------------------------------
1 | afterDownload($request,$response);
30 | if($rel !== false){
31 | $worker = new $request->workerName();
32 |
33 | if($worker->beforeParse() !== false){
34 | $worker->parse($request,$response);
35 | }
36 | $worker->afterParse();
37 | }
38 | }
39 |
40 | /**
41 | * 初始化下载器
42 | * DownloadTask constructor.
43 | * @param string $taskName
44 | */
45 | public function __construct($taskName=''){
46 | $this->multiDownloader = curl_multi_init();
47 | parent::__construct($taskName);
48 | }
49 |
50 | /**
51 | *
52 | * @param $request
53 | * @return bool
54 | */
55 | public function addRequest($request){
56 | $response = false;
57 | do{
58 | if($this->multiDownloader == null){
59 | break;
60 | }
61 |
62 | if(!($request instanceof Request)){
63 | break;
64 | }
65 |
66 | $currentRequestNum = count($this->currentRequestMap);
67 | if($currentRequestNum >= $this->windowSize){
68 | break;
69 | }
70 |
71 | $rel = $this->beforeDownload($request);
72 | if($rel === false){
73 | break;
74 | }
75 |
76 | if($rel instanceof Request){
77 | $request = $rel;
78 | }
79 | $ch = $request->createCurlObject();
80 | $key = (string)$ch;
81 | $this->currentRequestMap[$key] = $request;
82 | curl_multi_add_handle($this->multiDownloader, $ch);
83 |
84 | $this->spareTime = 0;
85 | $response = true;
86 | }while(false);
87 |
88 | return $response;
89 | }
90 |
91 | public function beforeDownload(Request $request){
92 | return null;
93 | }
94 |
95 | public function afterDownload(Request $request,Response $response){
96 | return null;
97 | }
98 |
99 | // 获取下载器闲置时间
100 | public function getSpareTime(){
101 | if($this->spareTime == 0){
102 | return 0;
103 | }else{
104 | return time()- $this->spareTime;
105 | }
106 | }
107 |
108 | public function coroutine()
109 | {
110 | do {
111 | while (($execrun = curl_multi_exec($this->multiDownloader, $running)) == CURLM_CALL_MULTI_PERFORM) ;
112 | if ($execrun != CURLM_OK) {
113 | if($this->spareTime == 0){
114 | $this->spareTime = time();
115 | }
116 | }
117 |
118 | // 一旦有一个请求完成,找出来,因为curl底层是select,所以最大受限于1024
119 | while ($done = curl_multi_info_read($this->multiDownloader))
120 | {
121 | // 从请求中获取信息、内容、错误
122 | $info = curl_getinfo($done['handle']);
123 | $output = curl_multi_getcontent($done['handle']);
124 | $error = curl_error($done['handle']);
125 | $response = new Response($info,$output,$error);
126 |
127 | $key = (string)$done['handle'];
128 | $request = $this->currentRequestMap[$key];
129 | $this->dispatch($request,$response);
130 |
131 | // 把请求已经完成了得 curl handle 删除
132 | unset($this->currentRequestMap[$key]);
133 | curl_multi_remove_handle($this->multiDownloader, $done['handle']);
134 | }
135 |
136 | // 当没有数据的时候进行堵塞,把 CPU 使用权交出来,避免上面 do 死循环空跑数据导致 CPU 100%
137 | if ($running) {
138 | $rel = curl_multi_select($this->multiDownloader, 1);
139 | if($rel == -1){
140 | usleep(1000);
141 | }
142 | }
143 |
144 | if( $running == false){
145 | if($this->spareTime == 0){
146 | $this->spareTime = time();
147 | }
148 | }
149 | yield true;
150 | } while (true);
151 |
152 | // 关闭任务
153 | curl_multi_close($this->multiDownloader);
154 | unset($this->multiDownloader);
155 | }
156 | }
--------------------------------------------------------------------------------
/example/lol_hero_god_rank/index.php:
--------------------------------------------------------------------------------
1 | init($protectedName);
48 | }
49 |
50 | public static function init($protectedName){
51 | self::$startCrawlTime = time();
52 | self::$protectedName = $protectedName;
53 | self::$frameworkBasePath = dirname(__FILE__);
54 | self::$basePath = dirname(self::$frameworkBasePath);
55 | self::$applicationPath = self::$basePath.DIRECTORY_SEPARATOR.self::$protectedName;
56 |
57 | ini_set('date.timezone','Asia/Shanghai');
58 | spl_autoload_register('\framework\TSpider::autoLoadFile');
59 |
60 | self::checkEnvironment();
61 |
62 | // 解析命令
63 | self::parseCommand();
64 |
65 | // 导入引用的三方扩展
66 | self::searchAndInclude(self::$frameworkBasePath.'/extension');
67 | self::searchAndInclude(self::$applicationPath.'/extension');
68 |
69 | // 加载配置文件
70 | $default_config = include self::$frameworkBasePath.DIRECTORY_SEPARATOR.'config'.DIRECTORY_SEPARATOR.'default.php';
71 | $user_config_path = self::$applicationPath.DIRECTORY_SEPARATOR.'config'.DIRECTORY_SEPARATOR.'main.php';
72 | if(!is_file($user_config_path)){
73 | throw new \Exception("config file not found:{$user_config_path}",-1);
74 | }
75 | $user_config = include $user_config_path;
76 | if(!is_array($user_config)){
77 | throw new \Exception("config file not load error:{$user_config_path}",-1);
78 | }
79 |
80 | self::$config = array_merge($default_config,$user_config);
81 |
82 | self::$spiderName = Common::getArrayValue(self::$config,'name','defaultSpider');
83 | self::$isDebug = Common::getArrayValue(self::$config,'debug',false);
84 |
85 | if(self::$isDebug){
86 | ini_set("display_errors", "on");
87 | error_reporting(E_ALL^E_STRICT);
88 | }
89 |
90 | // 加载logger
91 | Logger::loadOneConfig(array('logName'=>'system'));
92 | Logger::$g_basePath = self::$config['logPath'];
93 | Logger::loadConfig(self::$config['logs']);
94 | self::$log = Logger::factory('system');
95 |
96 | // 设置request相关
97 | Request::$maxRepeat = self::$config['request']['maxRepeat'];
98 | Request::$timeOut = self::$config['request']['timeOut'];
99 | self::$requestQueue = new self::$config['request']['requestQueue']();
100 | if(!(self::$requestQueue instanceof RequestQueue)){
101 | $temp = self::$config['request']['requestQueue'];
102 | throw new \Exception("requestQueue error:{$temp}",-1);
103 | }
104 |
105 | // 加载必须任务,初始化任务调度器,添加下载器任务
106 | self::$taskScheduler = new TaskScheduler();
107 | if(!isset(self::$config['downloader'])){
108 | throw new \Exception('there is no download class config');
109 | }
110 | $downloaderClass = self::$config['downloader']['className'];
111 | self::$downloader = new $downloaderClass();
112 | self::$downloader->windowSize = self::$config['downloader']['windowSize'];
113 | self::$taskScheduler->addTask(self::$downloader);
114 |
115 | // 添加请求队列读取任务,以及自定义任务,all分组表示运行所有
116 | $addRequestTask = new AddRequestTask(self::$downloader);
117 | self::$taskScheduler->addTask($addRequestTask);
118 | if(self::$taskGroup == 'all'){
119 | foreach(self::$config['tasks'] as $key=>$value){
120 | if(is_array($value)){
121 | foreach ($value as $className=>$params){
122 | $tempTask = new $className();
123 | self::$taskScheduler->addTask($tempTask);
124 | }
125 | }
126 | }
127 | }else{
128 | if(isset(self::$config['tasks']['common'])){
129 | foreach(self::$config['tasks']['common'] as $key=>$value){
130 | $tempTask = new $key();
131 | self::$taskScheduler->addTask($tempTask);
132 | }
133 | }
134 |
135 | if(isset(self::$config['tasks'][self::$taskGroup]) && self::$taskGroup != 'common'){
136 | foreach(self::$config['tasks'][self::$taskGroup] as $className=>$params){
137 | $tempTask = new $className();
138 | self::$taskScheduler->addTask($tempTask);
139 | }
140 | }else{
141 | $taskGroup = self::$taskGroup;
142 | exit("taskGroup not found:{$taskGroup}\n");
143 | }
144 | }
145 |
146 | // 初始化组件配置文件
147 | if(isset(self::$config['component']) && is_array(self::$config['component'])){
148 | Component::$config = self::$config['component'];
149 | }
150 | self::$component = new Component();
151 |
152 | // 设置进程名称
153 | $title = self::$spiderName;
154 | $taskGroup = self::$taskGroup;
155 | $startTime = date('Y-m-d H:i:s',self::$startCrawlTime);
156 | self::setProcessTitle("TSpider:{$title}-{$taskGroup} startTime:{$startTime}");
157 | }
158 |
159 | public static function autoLoadFile($rawName){
160 | $name = str_replace('\\', DIRECTORY_SEPARATOR ,$rawName);
161 | $classFile = self::$basePath . DIRECTORY_SEPARATOR . $name . '.php';
162 |
163 | if(is_file($classFile)){
164 | if(!class_exists($rawName,true)){
165 | $rel = require $classFile;
166 | return $rel;
167 | }
168 | }else{
169 | throw new Exception("auto load File error:{$classFile}",-1);
170 | }
171 | return false;
172 | }
173 |
174 | /**
175 | * 解析接收的命令
176 | */
177 | public static function parseCommand(){
178 | if(isset($_SERVER['argv']) && count($_SERVER['argv']) >=2){
179 | $command = $_SERVER['argv'][1];
180 | if(strlen($command)){
181 | self::$taskGroup = $command;
182 | }else{
183 | self::$taskGroup = '';
184 | }
185 | }else{
186 | self::$taskGroup = 'all';
187 | }
188 | echo 'taskGroup:',self::$taskGroup,"\n";
189 | }
190 |
191 |
192 | /**
193 | * 检测运行环境
194 | */
195 | public static function checkEnvironment(){
196 | echo 'PHP-version:',PHP_VERSION,' TSpider-version:',self::VERSION,' start-time:',date('Y-m-d H:i:s',self::$startCrawlTime),"\n";
197 |
198 | if(version_compare(PHP_VERSION,'5.5.0','<=')){
199 | exit('php version must greater than 5.5.0');
200 | }
201 |
202 | if(substr(php_sapi_name(), 0, 3) != 'cli'){
203 | exit('this program must be running in cli mode ');
204 | }
205 |
206 | if(extension_loaded('posix')){
207 | $userInfo = posix_getpwuid(posix_getuid());
208 | echo 'pid:',posix_getpid()," ",'running-user:',$userInfo['name'],"\n";
209 | }
210 | }
211 |
212 | /**
213 | * 设置进程名称
214 | * @param $title
215 | */
216 | public static function setProcessTitle($title)
217 | {
218 | if (function_exists('cli_set_process_title')) {
219 | @cli_set_process_title($title);
220 | } elseif (extension_loaded('proctitle') && function_exists('setproctitle')) {
221 | @setproctitle($title);
222 | }
223 | }
224 |
225 | /**
226 | * 导入某个文件夹中的所有文件
227 | * @param $rootPath
228 | */
229 | public static function searchAndInclude($rootPath){
230 | if(is_dir($rootPath)){
231 | $rootPath = rtrim($rootPath,'/');
232 | $files = glob($rootPath.'/*');
233 | foreach ($files as $file){
234 | if(is_dir($file)){
235 | self::searchAndInclude($file);
236 | }else{
237 | include_once $file;
238 | }
239 | }
240 | }
241 | }
242 |
243 | public static function run(){
244 | self::$taskScheduler->run();
245 | }
246 | }
--------------------------------------------------------------------------------
/framework/base/BaseWorker.php:
--------------------------------------------------------------------------------
1 | code} : {$request->url} \n";
26 | }
27 |
28 | /**
29 | * 解析前hook
30 | * @return bool
31 | */
32 | public function beforeParse(){
33 | self::$statisticsParserNum ++;
34 | return true;
35 | }
36 |
37 | /**
38 | * 解析后hook
39 | */
40 | public function afterParse(){
41 | // TODO: Implement afterParse() method.
42 | }
43 | }
--------------------------------------------------------------------------------
/framework/base/Request.php:
--------------------------------------------------------------------------------
1 | url = $url;
37 | $this->workerName = $workerName;
38 |
39 | $this->options[CURLOPT_URL] = $url;
40 | $this->options[CURLOPT_TIMEOUT] = self::$timeOut;
41 | $this->options[CURLOPT_USERAGENT] = 'Mozilla/5.0 (Windows NT 6.2; WOW64; Trident/7.0; rv:11.0) like Gecko';
42 | $this->options[CURLOPT_ENCODING] = 'gzip, deflate ';
43 | $this->extData = $extData;
44 | if(empty($postData) || !is_array($postData)){
45 | $this->type = 'GET';
46 | }else{
47 | $this->type = 'POST';
48 | $this->postData = $postData;
49 | $this->options[CURLOPT_POST] = true;
50 | $this->options[CURLOPT_POSTFIELDS] = $postData;
51 | }
52 | }
53 |
54 | /**
55 | * 对象字符串化
56 | * @return string
57 | */
58 | public function __toString(){
59 | $json_arr = array(
60 | 'url' => $this->url,
61 | 'workerName' => $this->workerName,
62 | 'options' => $this->options,
63 | 'postData' => $this->postData,
64 | 'repeat' => $this->repeat,
65 | 'header' => $this->header,
66 | );
67 | $returnString = json_encode($json_arr);
68 | if(!is_string($returnString)){
69 | $returnString = '';
70 | }
71 | return $returnString;
72 | }
73 |
74 | /**
75 | * 设置头部信息数组
76 | * @param $options
77 | */
78 | public function setOptions($options){
79 | foreach($options as $key=>$value){
80 | $this->options[$key] = $value;
81 | }
82 | }
83 |
84 | /**
85 | * 累加重复次数
86 | */
87 | public function addRepeat(){
88 | $this->repeat ++;
89 | }
90 |
91 | /**
92 | * 获取重复次数
93 | * @return int
94 | */
95 | public function getRepeatNum(){
96 | return $this->repeat;
97 | }
98 |
99 | /**
100 | * 是否能重复下载
101 | * @return bool
102 | */
103 | public function canRepeat(){
104 | if(self::$maxRepeat < $this->repeat){
105 | return false;
106 | }
107 | return true;
108 | }
109 |
110 | /**
111 | * 重新下载
112 | * @return bool true-添加成功,false-添加失败
113 | */
114 | public function redownload(){
115 | $this->addRepeat();
116 | if($this->canRepeat()) {
117 | TSpider::$requestQueue->add($this);
118 | return true;
119 | }
120 | return false;
121 | }
122 |
123 |
124 | /**
125 | * 将请求放入下载队列中,队列满了,返回失败
126 | * @return bool
127 | */
128 | public function download(){
129 | if(TSpider::$requestQueue->isFull()){
130 | return false;
131 | }
132 | TSpider::$requestQueue->add($this);
133 | return true;
134 | }
135 |
136 |
137 | public function createCurlObject(){
138 | $ch = curl_init();
139 | if($this->canRepeat()){
140 | if(!empty($this->options)){
141 | curl_setopt_array($ch,$this->options);
142 | }
143 | curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
144 | }else{
145 | $ch = false;
146 | }
147 | return $ch;
148 | }
149 | }
--------------------------------------------------------------------------------
/framework/base/Response.php:
--------------------------------------------------------------------------------
1 | info = $info;
22 | $this->data = $data;
23 | $this->error = $error;
24 | if(isset($this->info['http_code'])){
25 | $this->code = $this->info['http_code'];
26 | }
27 | }
28 |
29 | public function getData(){
30 | return $this->data;
31 | }
32 |
33 | public function getError(){
34 | return $this->error;
35 | }
36 | }
--------------------------------------------------------------------------------
/framework/component/Component.php:
--------------------------------------------------------------------------------
1 | getInstance();
31 | }
32 |
33 | $className = isset(self::$config[$name]['className'])?self::$config[$name]['className']:'';
34 | if(!class_exists($className)){
35 | throw new \Exception("have no component className:{$className}\n",-1);
36 | }
37 |
38 | $component = new $className($name,self::$config[$name]);
39 | if(!($component instanceof Component)){
40 | throw new \Exception("className:{$className} is not Component\n",-1);
41 | }
42 |
43 | self::$instanceMap[$name] = $component;
44 |
45 | return self::$instanceMap[$name]->getInstance();
46 | }
47 |
48 | public function __construct($componentName='',$config=array()){
49 | $this->componentName = $componentName;
50 | $this->oneConfig = $config;
51 | }
52 |
53 |
54 |
55 | public function getInstance(){
56 | return $this;
57 | }
58 |
59 | }
--------------------------------------------------------------------------------
/framework/component/MysqlComponent.php:
--------------------------------------------------------------------------------
1 | isLoadConfig){
19 | $this->oneConfig['connectionName'] = $this->componentName;
20 | TMysql::loadOneConfig($this->oneConfig);
21 | $this->isLoadConfig = true;
22 | }
23 | return TMysql::getConnection($this->componentName);
24 | }
25 | }
--------------------------------------------------------------------------------
/framework/component/RedisComponent.php:
--------------------------------------------------------------------------------
1 | isLoadConfig){
19 | $this->oneConfig['connectionName'] = $this->componentName;
20 | TRedis::loadOneConfig($this->oneConfig);
21 | $this->isLoadConfig = true;
22 | }
23 | return TRedis::getConnection($this->componentName);
24 | }
25 | }
--------------------------------------------------------------------------------
/framework/component/RedisLockComponent.php:
--------------------------------------------------------------------------------
1 | isLoadConfig){
21 | $this->oneConfig['connectionName'] = $this->componentName;
22 | TRedis::loadOneConfig($this->oneConfig);
23 | $this->isLoadConfig = true;
24 | }
25 |
26 | return new TLock($this->componentName,$this->getArrayValue($this->oneConfig,'prefix','lock_'),$this->getArrayValue($this->oneConfig,'expire',3600));
27 | }
28 |
29 |
30 | /**
31 | * 安全地获取数组的值
32 | * @param $arr array 数组
33 | * @param $key String 键名
34 | * @param string $default 默认值
35 | * @return string
36 | */
37 | public static function getArrayValue($arr,$key,$default=''){
38 | if(isset($arr[$key])){
39 | return $arr[$key];
40 | }
41 | return $default;
42 | }
43 | }
--------------------------------------------------------------------------------
/framework/component/SmsWoquComponent.php:
--------------------------------------------------------------------------------
1 | lastSendTime) > $interval_time){
37 | $this->lastSendTime = $now;
38 | $this->sendMessageNum = 1;
39 | return true;
40 | }
41 | if($this->sendMessageNum <= $maxSendNum){
42 | $this->sendMessageNum ++;
43 | return true;
44 | }
45 | return false;
46 | }
47 |
48 |
49 | /**
50 | * 发送消息函数
51 | * @param string $content 发送的消息内容
52 | * @param bool $needSend 当前短信是否必须发送出去
53 | * @return bool
54 | */
55 | public function sendMessage($content,$needSend=false){
56 | if(!($this->canSendMessage() || $needSend)){
57 | return false;
58 | }
59 | $phoneNumbers = explode(',',$this->oneConfig['To']);
60 | if(empty($this->oneConfig['To'])){
61 | Http::request($this->oneConfig['serverUrl'],array(
62 | 'Module' => $this->oneConfig['Module'],
63 | 'MsgType' => $this->oneConfig['MsgType'],
64 | 'To' => '',
65 | 'MsgText' => $content));
66 | }else{
67 | foreach($phoneNumbers as $key=>$phone){
68 | Http::request($this->oneConfig['serverUrl'],array(
69 | 'Module' => $this->oneConfig['Module'],
70 | 'MsgType' => $this->oneConfig['MsgType'],
71 | 'To' => $phone,
72 | 'MsgText' => $content));
73 | }
74 | }
75 | return true;
76 | }
77 | }
--------------------------------------------------------------------------------
/framework/config/default.php:
--------------------------------------------------------------------------------
1 | 'defaultSpider',
12 | 'debug' => false,
13 | 'logPath' => dirname(dirname(dirname(__FILE__))).DIRECTORY_SEPARATOR.'log'.DIRECTORY_SEPARATOR,
14 | 'logs' => array(
15 | 'system' => array(
16 | 'logName' => 'system',
17 | )
18 | ),
19 | 'downloader' => array(
20 | 'className' => 'framework\task\DownloadTask',
21 | 'windowSize' => 50,
22 | ),
23 | 'request' => array(
24 | 'maxRepeat' => 5,
25 | 'timeOut' => 300,
26 | 'requestQueue' => 'framework\queue\LocalRequestQueue'
27 | ),
28 | 'tasks' => array(
29 | 'common' => array(
30 | 'framework\task\DelayTimer'=>array()
31 | )
32 | ),
33 | 'component' => array(
34 | // 填写相应组件
35 | ),
36 | );
--------------------------------------------------------------------------------
/framework/coroutine/ExampleTask.php:
--------------------------------------------------------------------------------
1 | taskId = ++self::$maxTaskId;
24 |
25 | $this->taskName = $taskName;
26 | if(empty($taskName)){
27 | $this->taskName = get_class();
28 | }
29 | $taskContent = $this->coroutine();
30 | if($taskContent instanceof \Generator){
31 | $this->taskContent = $taskContent;
32 | }else{
33 | throw new Exception('Task is not a coroutine',-1);
34 | }
35 | }
36 |
37 | public function getTaskId() {
38 | return $this->taskId;
39 | }
40 |
41 | public function getTaskName() {
42 | return $this->taskName;
43 | }
44 |
45 | public function setSendValue($sendValue) {
46 | $this->sendValue = $sendValue;
47 | }
48 |
49 | abstract public function coroutine();
50 |
51 | public function run() {
52 | if ($this->beforeFirstYield) {
53 | $this->beforeFirstYield = false;
54 | return $this->taskContent->current();
55 | } else {
56 | $retval = $this->taskContent->send($this->sendValue);
57 | $this->sendValue = null;
58 | return $retval;
59 | }
60 | }
61 |
62 | public function isFinished() {
63 | return !$this->taskContent->valid();
64 | }
65 | }
--------------------------------------------------------------------------------
/framework/coroutine/TaskScheduler.php:
--------------------------------------------------------------------------------
1 | task
16 | protected $taskQueue = null;
17 |
18 | protected static $is_closed = false;
19 |
20 | public function __construct() {
21 | $this->taskQueue = new \SplQueue();
22 | }
23 |
24 | public static function closeAllTask(){
25 | self::$is_closed = true;
26 | }
27 |
28 | public static function isAllTaskClosed(){
29 | return self::$is_closed;
30 | }
31 |
32 | public function addTask(Task $task){
33 | $taskId = $task->getTaskId();
34 | $this->taskMap[$taskId] = $task;
35 | $this->schedule($task);
36 | return $taskId;
37 | }
38 |
39 | public function schedule(Task $task) {
40 | $this->taskQueue->enqueue($task);
41 | }
42 |
43 | public function run() {
44 | while (!$this->taskQueue->isEmpty()) {
45 | $task = $this->taskQueue->dequeue();
46 | $task->run();
47 |
48 | if ($task->isFinished()) {
49 | unset($this->taskMap[$task->getTaskId()]);
50 | } else {
51 | $this->schedule($task);
52 | }
53 | if(TaskScheduler::isAllTaskClosed()){
54 | break;
55 | }
56 | }
57 | }
58 | }
--------------------------------------------------------------------------------
/framework/coroutine/TimerTask.php:
--------------------------------------------------------------------------------
1 | 0){
21 | $this->intervalTime = $intervalTime;
22 | }
23 | if(empty($taskName)){
24 | $taskName = get_class($this);
25 | }
26 | $this->expireTime = time()+$this->intervalTime;
27 | parent::__construct($taskName);
28 | }
29 |
30 | public function coroutine(){
31 | while(true){
32 | if($this->expireTime <= time()){
33 | $rel = $this->execute();
34 | $this->expireTime = time()+$this->intervalTime;
35 | if($rel === false){
36 | break;
37 | }
38 | }
39 | yield true;
40 | }
41 | }
42 |
43 | public abstract function execute();
44 | }
--------------------------------------------------------------------------------
/framework/extension/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hirudy/Tspider/93ba6b4f9ac1725519af3386c7ca28b33ceb8770/framework/extension/.DS_Store
--------------------------------------------------------------------------------
/framework/extension/Common.php:
--------------------------------------------------------------------------------
1 | $value){
42 | $options[$key] =$value;
43 | }
44 |
45 | // 是否是post请求
46 | if(!empty($postData) && is_array($postData)){
47 | $options[CURLOPT_POST] = true;
48 | $options[CURLOPT_POSTFIELDS] = http_build_query($postData);
49 | }
50 |
51 | // 是否是https
52 | if(stripos($url,'https') === 0){
53 | $options[CURLOPT_SSL_VERIFYPEER] = false;
54 | }
55 |
56 | // 返回curl对象
57 | $ch = curl_init();
58 | curl_setopt_array($ch,$options);
59 | return $ch;
60 | }
61 |
62 | /**
63 | * 将字符串解析为http响应头部数组
64 | * @param $strResponseHeader
65 | * @return array
66 | */
67 | private static function parseResponseHeader($strResponseHeader){
68 | $headerList = array();
69 | $tempHeaderList = explode("\r\n",$strResponseHeader);
70 | foreach ($tempHeaderList as $row){
71 | if (stripos($row,':') === false){
72 | $tmp = explode(" ",$row);
73 | $headerList['Protocol'] = isset($tmp[0])?$tmp[0]:'';
74 | $headerList['Status'] = (int)(isset($tmp[1])?$tmp[1]:0);
75 | $headerList['Message'] = isset($tmp[2])?$tmp[2]:'';
76 | }else{
77 | $tmp = explode(":",$row, 2);
78 | if (count($tmp) != 2){
79 | continue;
80 | }
81 | $key = trim($tmp[0]);
82 | $value = trim($tmp[1]);
83 | if ($key == 'Set-Cookie'){
84 | if (!isset($headerList[$key])){
85 | $headerList[$key] = array();
86 | }
87 | $tmpCookieList = explode(";",$value);
88 | foreach ($tmpCookieList as $oneCookie){
89 | $tmpCookie = explode("=",$oneCookie,2);
90 | if (count($tmpCookie) != 2){
91 | continue;
92 | }
93 | $key_cookie = trim($tmpCookie[0]);
94 | $value_cookie = trim($tmpCookie[1]);
95 | $headerList[$key][$key_cookie] = $value_cookie;
96 | }
97 | }else{
98 | $headerList[$key] = $value;
99 | }
100 | }
101 | }
102 |
103 | return $headerList;
104 | }
105 |
106 | /**
107 | * 从curl对象中提取请求的结果
108 | * @param $ch
109 | * @param $rel
110 | * @return array
111 | */
112 | private static function fetchResponse($ch, $rel){
113 | $response = array(
114 | 'status' => false,
115 | 'code' => 0,
116 | 'header' => array(),
117 | 'body' => '',
118 | 'extraInfo' => array(),
119 | 'errorInfo' => array()
120 | );
121 | $response['extraInfo'] = curl_getinfo($ch);
122 | if($rel == false){
123 | $error = array();
124 | $error['code'] = curl_errno( $ch );
125 | $error['info'] = curl_error($ch);
126 | $response['errorInfo'] = $error;
127 | }else{
128 | // 切割header 与 body
129 | $header_body = explode("\r\n\r\n",$rel);
130 | do{
131 | if (count($header_body) !== 2){
132 | $error =array();
133 | $error['code'] = 0;
134 | $error['info'] = 'split header and body error';
135 | $error['list'] = $rel;
136 | $response['errorInfo'] = $error;
137 | break;
138 | }
139 |
140 | // 格式化返回结果
141 | $response['body'] = $header_body[1];
142 | $response['header'] = self::parseResponseHeader($header_body[0]);
143 | $response['code'] = $response['header']['Status'];
144 | $response['status'] = true;
145 | }while(false);
146 | }
147 |
148 | return $response;
149 | }
150 |
151 | /**
152 | * 通用请求方法
153 | * @param string $url 请求url地址
154 | * @param array $postData 请求post参数数组
155 | * @param array $header 请求附带请求头部数组
156 | * @param int $timeOut 超时时间
157 | * @param string $proxy 代理设置
158 | * @return array
159 | */
160 | public static function request($url,$postData=array(),$header=array(),$timeOut=self::DEFAULT_TIMEOUT, $proxy=''){
161 | // 执行请求
162 | $ch = self::buildCurlObject($url, $postData, $header, $timeOut, $proxy);
163 | $rel = curl_exec($ch);
164 | $response = self::fetchResponse($ch, $rel);
165 | curl_close($ch);
166 | return $response;
167 | }
168 |
169 | /**
170 | * 简单返回请求方法
171 | * @param string $url 请求url地址
172 | * @param array $postData 请求post参数数组
173 | * @param array $header 请求附带请求头部数组
174 | * @param int $timeOut 超时时间
175 | * @param string $proxy 代理设置
176 | * @return bool|mixed 成功返回body字符串,失败返回false
177 | */
178 | public static function simpleResponseRequest($url,$postData=array(), $header=array(), $timeOut=self::DEFAULT_TIMEOUT, $proxy=''){
179 | $result = self::request($url, $postData, $header, $timeOut, $proxy);
180 | if ($result['status']){
181 | return $result['body'];
182 | }else{
183 | print_r($result['extraInfo']);
184 | print_r($result['errorInfo']);
185 | return false;
186 | }
187 | }
188 |
189 | /**
190 | * 多个http请求并行执行
191 | * @param array $requestList 参数同self::request,只不过改成了数组
192 | * @return arrayMAX_REDIRECT_NUM = 5
193 | */
194 | public static function multiRequest(Array $requestList){
195 | // 创建curl对象,存放到数组,添加到下载器中
196 | $requestCurlObjectList = array();
197 | $downloader = curl_multi_init();
198 | foreach ($requestList as $row){
199 |
200 | $url = isset($row['url'])?$row['url']:'';
201 | $postData = isset($row['postData'])?$row['postData']:array();
202 | $header = isset($row['header'])?$row['header']:array();
203 | $timeOut = isset($row['timeOut'])?$row['timeOut']:self::DEFAULT_TIMEOUT;
204 | $proxy = isset($row['proxy'])?$row['proxy']:'';
205 | $tmpCurlObject = self::buildCurlObject($url,$postData,$header,$timeOut,$proxy);
206 | $requestCurlObjectList[] = $tmpCurlObject;
207 | curl_multi_add_handle($downloader,$tmpCurlObject);
208 | }
209 |
210 | // 并行执行多个curl对象,等待所有请求完毕退出循环
211 | $active = true;
212 | $mrc = CURLM_OK;
213 | while ($active && $mrc == CURLM_OK) {
214 | do {
215 | $mrc = curl_multi_exec($downloader, $active);
216 | } while ($mrc == CURLM_CALL_MULTI_PERFORM);
217 |
218 | if (curl_multi_select($downloader) == -1) {
219 | usleep(100);
220 | }
221 | }
222 |
223 | // 解析每一个请求对象
224 | $responseList = array();
225 | foreach ($requestCurlObjectList as $key=>$ch){
226 | $rel = curl_multi_getcontent($ch);
227 | $response = self::fetchResponse($ch, $rel);
228 | $responseList[$key] = $response;
229 | curl_multi_remove_handle($downloader, $ch);
230 | curl_close($ch);
231 | }
232 | curl_multi_close($downloader);
233 | return $responseList;
234 | }
235 | }
236 |
237 | //测试
238 | if(strtolower(PHP_SAPI) == 'cli' && isset($argv) && basename(__FILE__) == basename($argv[0])){
239 | // 串行请求
240 | $start_time = microtime(true);
241 | $response1 = THttp::request('https://www.baidu.com/');
242 | $response2 = THttp::request('http://www.jd.com');
243 | $response3 = THttp::request('http://www.jianshu.com/');
244 | $response4 = THttp::request('http://www.zhihu.com/');
245 | $response5 = THttp::request('http://www.php.net/');
246 | $response6 = THttp::request('https://github.com/hirudy');
247 | $response7 = THttp::request('http://www.toutiao.com/');
248 | $response8 = THttp::request('http://www.mi.com/');
249 | // $response9 = THttp::request('https://www.google.com');
250 | echo "serial request take time : ", microtime(true)-$start_time,"\n";
251 |
252 | // 并行请求
253 | $start_time = microtime(true);
254 | $responseList = THttp::multiRequest(array(
255 | array('url'=>'https://www.baidu.com/'),
256 | array('url'=>'http://www.jd.com'),
257 | array('url'=>'http://www.jianshu.com/'),
258 | array('url'=>'http://www.zhihu.com/'),
259 | array('url'=>'http://www.php.net/'),
260 | array('url'=>'https://github.com/hirudy'),
261 | array('url'=>'http://www.toutiao.com/'),
262 | array('url'=>'http://www.mi.com/'),
263 | // array('url'=>'https://www.google.com')
264 | ));
265 | echo "parallel requests take time : ", microtime(true)-$start_time,"\n";
266 |
267 | }
--------------------------------------------------------------------------------
/framework/extension/TLock.php:
--------------------------------------------------------------------------------
1 | connectionName = $connectionName;
21 | $this->uniqueId = uniqid();
22 | $this->prefix = $prefix;
23 | $this->expire = $expire;
24 | }
25 |
26 |
27 | /**
28 | * 获取一个非阻塞锁 true-获取到,false-没有获取到
29 | * @param $resource_name
30 | * @return bool
31 | * @throws RedisOperationException
32 | */
33 | public function lock($resource_name){
34 | if(!empty($this->resource_name)){
35 | return false;
36 | }
37 | $this->resource_name = $resource_name;
38 |
39 | $redis = TRedis::getConnection($this->connectionName);
40 | if($redis->setNx($this->resource_name,$this->uniqueId)){
41 | $redis->expire($this->resource_name,$this->expire);
42 | return true;
43 | }
44 | return false;
45 | }
46 |
47 | /**
48 | * 获得一个阻塞锁
49 | * @param $resource_name
50 | * @return bool
51 | * @throws RedisOperationException
52 | */
53 | public function lockWait($resource_name){
54 | if(!empty($this->resource_name)){
55 | return false;
56 | }
57 | $this->resource_name = $resource_name;
58 |
59 | do{
60 | $redis = TRedis::getConnection($this->connectionName);
61 | if($redis->setNx($this->resource_name,$this->uniqueId)){
62 | $redis->expire($this->resource_name,$this->expire);
63 | break;
64 | }
65 |
66 | sleep(1);
67 | }while(true);
68 | return true;
69 | }
70 |
71 | /**
72 | * 解锁,只能解锁自己对象的说
73 | * @throws RedisOperationException
74 | */
75 | public function unlock(){
76 | $redis = TRedis::getConnection($this->connectionName);
77 | $rel = $redis->get($this->resource_name);
78 | if($rel == $this->uniqueId){
79 | $redis->delete($this->resource_name);
80 | }
81 | }
82 | }
--------------------------------------------------------------------------------
/framework/extension/TLogger.php:
--------------------------------------------------------------------------------
1 | 'fatal',
44 | TLogger::LOG_LEVEL_ERROR => 'error',
45 | TLogger::LOG_LEVEL_WARN => 'warn',
46 | TLogger::LOG_LEVEL_INFO => 'info',
47 | TLogger::LOG_LEVEL_DEBUG => 'debug'
48 | );
49 |
50 | // 具体的写类
51 | protected abstract function write($message);
52 |
53 | protected function log($message, $level){
54 | // 是否需要记录
55 | if(!TLogger::$g_isLogging || !$this->isLogging || $level > $this->level){
56 | return false;
57 | }
58 |
59 | // 序列化消息
60 | if(!is_string($message)){
61 | if(is_array($message)){
62 | $message = json_encode($message,JSON_UNESCAPED_UNICODE);
63 | }else if (is_object($message)){
64 | $message = serialize($message);
65 | }else{
66 | $message = (string)$message;
67 | }
68 | }
69 |
70 | // 格式化消息输出
71 | $levelInfo = TLogger::getArrayValue(self::$logLevelMap, $level, TLogger::LOG_LEVEL_INFO);
72 | $message = sprintf('[%s %s] %s',@date('Y-m-d H:i:s'), $levelInfo, $message);
73 |
74 | return $this->write($message);
75 | }
76 |
77 | public function fatal($message){
78 | return $this->log($message, TLogger::LOG_LEVEL_FATAL);
79 | }
80 |
81 | public function error($message){
82 | return $this->log($message, TLogger::LOG_LEVEL_ERROR);
83 | }
84 |
85 | public function warn($message){
86 | return $this->log($message, TLogger::LOG_LEVEL_WARN);
87 | }
88 |
89 | public function info($message){
90 | return $this->log($message, TLogger::LOG_LEVEL_INFO);
91 | }
92 |
93 | public function debug($message){
94 | return $this->log($message, TLogger::LOG_LEVEL_DEBUG);
95 | }
96 | }
97 |
98 | /**
99 | * 控制台操作类
100 | * Class ConsoleHandle
101 | */
102 | class ConsoleHandle extends LoggerHandle{
103 |
104 | public static function parseConfig($rawConfig){
105 | // 控制台模式日志配置格式
106 | $result = array(
107 | 'name' => TLogger::getArrayValue($rawConfig, 'name', 'default', 'string'), // 日志名称
108 | 'isLogging' => TLogger::getArrayValue($rawConfig, 'isLogging', TLogger::$g_isLogging, 'boolean'), // 当前日志是否记录
109 | 'mode' => TLogger::LOG_MODE_CONSOLE, // 记录模式
110 | 'level' => TLogger::getArrayValue($rawConfig, 'level', TLogger::LOG_LEVEL_DEBUG,'integer'), // 日志等级
111 | );
112 |
113 | // 判断记录等级是否合法
114 | if ($result['level'] > TLogger::LOG_LEVEL_DEBUG || $result['level'] < TLogger::LOG_LEVEL_FATAL){
115 | throw new TLoggerException("({$result['name']}) config level set error");
116 | }
117 | return $result;
118 | }
119 |
120 | public function __construct($config){
121 | $this->isLogging = $config['isLogging'];
122 | $this->name = $config['name'];
123 | $this->config = $config;
124 | $this->mode = $config['mode'];
125 | $this->level = $config['level'];
126 | }
127 |
128 | protected function write($message){
129 | echo $message, PHP_EOL;
130 | return true;
131 | }
132 | }
133 |
134 | /**
135 | * 文件操作类
136 | * Class FileHandle
137 | */
138 | class FileHandle extends LoggerHandle{
139 | protected $basePath; // 存储路径
140 | protected $frequency; // 切割日志方式
141 | protected $suffix = '.log'; // 日志文件后缀
142 |
143 | private $logFilePath; // 完整的日志路径
144 | private $timeLength; // 存储时间长度
145 |
146 | public static function parseConfig($rawConfig){
147 | // 文件模式日志配置格式化
148 | $result = array(
149 | 'name' => TLogger::getArrayValue($rawConfig, 'name', 'default', 'string'), // 日志名称
150 | 'isLogging' => TLogger::getArrayValue($rawConfig, 'isLogging', TLogger::$g_isLogging, 'boolean'), // 当前日志是否记录
151 | 'mode' => TLogger::LOG_MODE_FILE, // 记录模式
152 | 'level' => TLogger::getArrayValue($rawConfig, 'level', TLogger::LOG_LEVEL_DEBUG,'integer'), // 日志等级
153 |
154 | 'basePath' => TLogger::getArrayValue($rawConfig, 'basePath', TLogger::$g_basePath, 'string'), // 当前日志的记录文件根目录
155 | 'frequency' => TLogger::getArrayValue($rawConfig, 'frequency', TLogger::LOG_FREQUENCY_NONE, 'integer') // 切割日志方式
156 | );
157 |
158 | // 判断记录等级是否合法
159 | if ($result['level'] > TLogger::LOG_LEVEL_DEBUG || $result['level'] < TLogger::LOG_LEVEL_FATAL){
160 | throw new TLoggerException("({$result['name']}) config level set error");
161 | }
162 |
163 | // 判断记录日志切割是否合法
164 | if ($result['frequency'] > TLogger::LOG_FREQUENCY_MONTH || $result['frequency'] < TLogger::LOG_FREQUENCY_NONE){
165 | throw new TLoggerException("({$result['name']}) config frequency set error");
166 | }
167 |
168 | // 初始化日志记录根目录
169 | $result['basePath'] = rtrim($result['basePath'], DIRECTORY_SEPARATOR);
170 | if(!is_dir($result['basePath'])){
171 | if( !mkdir($result['basePath'], 0775, true)){
172 | throw new TLoggerException("({$result['name']}) config create directory fail:".$result['basePath']);
173 | }
174 | }
175 |
176 | return $result;
177 | }
178 |
179 | public function __construct($config){
180 | $this->isLogging = $config['isLogging'];
181 | $this->name = $config['name'];
182 | $this->config = $config;
183 | $this->mode = $config['mode'];
184 | $this->level = $config['level'];
185 | $this->frequency = $config['frequency'];
186 | $this->basePath = $config['basePath'];
187 |
188 | $this->logFilePath = $this->basePath.DIRECTORY_SEPARATOR.$this->name;
189 | switch($this->frequency){
190 | case TLogger::LOG_FREQUENCY_MINUTE: $this->timeLength = 12;break;
191 | case TLogger::LOG_FREQUENCY_HOUR: $this->timeLength = 10;break;
192 | case TLogger::LOG_FREQUENCY_DAY: $this->timeLength = 8;break;
193 | case TLogger::LOG_FREQUENCY_MONTH: $this->timeLength = 6;break;
194 | default:
195 | $this->timeLength = -1;
196 | }
197 | }
198 |
199 | protected function write($message){
200 | $logTime = time();
201 | // 检测是否需要对日志文件进行重命名
202 | if($this->timeLength > 0){
203 | $fileCreateTime = @filectime($this->logFilePath.$this->suffix);
204 | if($fileCreateTime){
205 | $logTimeFormat = substr(@date('YmdHis',$logTime),0,$this->timeLength);
206 | $createTimeFormat = substr(@date('YmdHis',$fileCreateTime),0,$this->timeLength);
207 | if(strcmp($logTimeFormat,$createTimeFormat) !== 0){
208 | $newLogFilePath = $this->logFilePath.'_'.$createTimeFormat.$this->suffix;
209 | rename($this->logFilePath.$this->suffix,$newLogFilePath);
210 | }
211 | }
212 | }
213 | $message .= PHP_EOL;
214 | // 写入日志文件中
215 | return file_put_contents($this->logFilePath.$this->suffix,$message,LOCK_EX | FILE_APPEND);
216 | }
217 | }
218 |
219 |
220 | /**
221 | * 日志管理类
222 | * Class TLogger
223 | */
224 | class TLogger{
225 | const VERSION = '1.0.1'; // 日志组件版本号
226 |
227 | const LOG_MODE_CONSOLE = 'Console'; // 记录模式,记录到控制台
228 | const LOG_MODE_FILE = 'File'; // 记录模式,记录到文件
229 | const LOG_MODE_TCP = 'Tcp'; // 记录模式,记录到TCP
230 | const LOG_MODE_UDP = 'Udp'; // 记录模式,记录到UDP
231 |
232 | const LOG_FREQUENCY_NONE = 0; // 切割日志方式,存放的日志文件始终只有一个文件,形如 default.log
233 | const LOG_FREQUENCY_MINUTE = 1; // 切割日志方式,存放日志每隔一分钟换一个,形如 default_201601192357.log
234 | const LOG_FREQUENCY_HOUR = 2; // 切割日志方式,存放日志每隔一小时换一个,形如 default_2016011923.log
235 | const LOG_FREQUENCY_DAY = 3; // 切割日志方式,存放日志每隔一天换一个,形如 default_20160119.log
236 | const LOG_FREQUENCY_MONTH = 4; // 切割日志方式,存放日志每隔一月换一个,形如 default_201601.log
237 |
238 | const LOG_LEVEL_FATAL = 0; // 日志等级,严重错误
239 | const LOG_LEVEL_ERROR = 1; // 日志等级,错误
240 | const LOG_LEVEL_WARN = 2; // 日志等级,警告
241 | const LOG_LEVEL_INFO = 3; // 日志等级,信息记录
242 | const LOG_LEVEL_DEBUG = 4; // 日志等级,调试
243 |
244 | public static $g_isLogging = true; // 总开关,是否记录日志
245 | public static $g_basePath = '/data/log'; // 默认存储路径
246 |
247 | protected static $g_config_arr = array(); // 日志配置文件数组
248 | protected static $logPool = array(); // 日志对象池子
249 |
250 |
251 | /**
252 | * 获取数组中某个值
253 | * @param array $arr 数组
254 | * @param string $key key值
255 | * @param string $default 默认值
256 | * @param string $valueType 不为空,检查变量数据类型
257 | * @return mixed
258 | */
259 | public static function getArrayValue(Array $arr, $key, $default='', $valueType=''){
260 | $result = $default;
261 | if (isset($arr[$key])){
262 | $result = $arr[$key];
263 | }
264 |
265 | if (!empty($valueType) && gettype($result)!= $valueType){
266 | $result = $default;
267 | }
268 |
269 | return $result;
270 | }
271 |
272 | /**
273 | * 以二维数组的形式加载多个日志配置文件
274 | * @param $arr array 二维数组
275 | */
276 | public static function loadConfig($arr){
277 | if(!empty($arr) && is_array($arr)){
278 | foreach($arr as $name =>$config){
279 | self::loadOneConfig($config,$name);
280 | }
281 | }
282 | }
283 |
284 | /**
285 | * 加载一个日志配置文件
286 | * @param array $arr
287 | * @param string $name
288 | * @return bool
289 | * @throws TLoggerException
290 | */
291 | public static function loadOneConfig(Array $arr,$name=''){
292 | $name = self::getArrayValue($arr,'name',$name,"string");
293 | if (empty($name)){
294 | throw new TLoggerException("no config name");
295 | }
296 | $arr['name'] = $name;
297 |
298 | $handle = self::getArrayValue($arr,'mode',self::LOG_MODE_FILE,'string');
299 | $handle = $handle.'Handle';
300 | if (!class_exists($handle,false) || get_parent_class($handle) != 'LoggerHandle'){
301 | throw new TLoggerException("({$name}) logger handle '{$handle}' not defined or is not sub-class of LoggerHandle");
302 | }
303 |
304 | $arr = call_user_func(array($handle,'parseConfig'),$arr);
305 | $arr['handleClass'] = $handle;
306 | if (empty($arr)){
307 | throw new TLoggerException("({$name}) logger handle {$$handle} config parse error");
308 | }
309 |
310 | self::$g_config_arr[$name] = $arr;
311 | if (isset(self::$logPool[$name])){
312 | unset(self::$logPool[$name]);
313 | }
314 | return true;
315 | }
316 |
317 |
318 | /**
319 | * 根据日志名称,获取一个日志实例
320 | * @param string $name 配置名称
321 | * @param bool $isNew 是否生成一个新的日志对象
322 | * @return object logger
323 | * @throws Exception
324 | */
325 | public static function getLogger($name='default',$isNew=false){
326 | if(!isset(self::$logPool[$name]) || $isNew == true){
327 | // 是否加载了默认配置文件
328 | if (!isset(self::$g_config_arr['default'])){
329 | self::loadOneConfig(array(
330 | 'name' => 'default', // 日志名称
331 | 'isLogging' => true, // 当前日志是否记录
332 | 'basePath' => self::$g_basePath, // 当前日志的记录根目录,没有,默认全局目录:g_basePath
333 | 'mode' => self::LOG_MODE_FILE, // 记录模式
334 | 'level' => self::LOG_LEVEL_DEBUG, // 日志等级
335 | 'frequency' => self::LOG_FREQUENCY_NONE, // 切割日志方式
336 | ));
337 | }
338 |
339 | // 删除已有的logger对象
340 | if(isset(self::$logPool[$name])){
341 | unset(self::$logPool[$name]);
342 | }
343 |
344 | // 检测配置文件是否加载
345 | if(!isset(self::$g_config_arr[$name])){
346 | throw new TLoggerException("Make sure that the log configuration which name is '{$name}' is loaded successfully");
347 | }
348 |
349 | // 根据配置文件实例化logger,并保存到logger单例数组中
350 | $config = self::$g_config_arr[$name];
351 | $reflector = new ReflectionClass($config['handleClass']);
352 | $logger = $reflector->newInstance($config);
353 | self::$logPool[$name] = $logger;
354 | }
355 |
356 | return self::$logPool[$name];
357 | }
358 | }
359 |
360 | //测试
361 | if(strtolower(PHP_SAPI) == 'cli' && isset($argv) && basename(__FILE__) == basename($argv[0])){
362 | $config = array( // 日志配置文件数组,default是默认配置项
363 | 'name' => 'test',
364 | 'level' => TLogger::LOG_LEVEL_INFO,
365 | 'frequency' => TLogger::LOG_FREQUENCY_MINUTE
366 | );
367 | TLogger::$g_basePath = __DIR__.DIRECTORY_SEPARATOR.'log';
368 | TLogger::loadOneConfig($config);
369 |
370 | $logger = TLogger::getLogger("test");
371 | $logger->debug("this is debug info ");
372 | $logger->info(array("is","info","recode"));
373 | $logger->warn(21);
374 | $logger->error("error info ");
375 | $logger->fatal($logger);
376 | }
--------------------------------------------------------------------------------
/framework/extension/TMysql.php:
--------------------------------------------------------------------------------
1 | connectionName = $connectionName;
17 | $this->dbName = $dbname;
18 | $dbConnection = new \mysqli($host, $username, $passwd, $dbname, $port, $socket);
19 | $dbConnection->options(MYSQLI_OPT_CONNECT_TIMEOUT,5);
20 | if ($dbConnection->connect_error){
21 | throw new \Exception("mysql {$connectionName} connection failure:({$dbConnection->connect_errno}){$dbConnection->connect_error}",-1);
22 | }
23 | $dbConnection->set_charset($charset);
24 | $this->dbConnection = $dbConnection;
25 | }
26 |
27 | public function __destruct(){
28 | if ($this->dbConnection){
29 | // 操作对象销毁时候,关闭连接
30 | @$this->dbConnection->close();
31 | }
32 | }
33 |
34 | /**
35 | * 查看当前连接是否关闭
36 | * @return bool
37 | */
38 | public function ping(){
39 | // $autoReconnect = empty(ini_get('mysqli.reconnect'));
40 | try{
41 | if(@$this->dbConnection->ping()){
42 | return true;
43 | }else{
44 | @$this->dbConnection->close();
45 | return false;
46 | }
47 | }catch (\Exception $e){
48 | @$this->dbConnection->close();
49 | return false;
50 | }
51 | }
52 |
53 | /**
54 | * 获取当前默认的字符集
55 | * @return string 字符集
56 | */
57 | public function getCharset(){
58 | return $this->dbConnection->character_set_name();
59 | }
60 |
61 |
62 | /**
63 | * 获取select结果集中的所有结果
64 | * @param $option
65 | * @return array
66 | */
67 | protected function fetch_all($option){
68 | $result = array();
69 | if(method_exists($option,'fetch_all')){
70 | $result = $option->fetch_all(MYSQLI_ASSOC);
71 | }else{
72 | while($row = $option->fetch_assoc()){
73 | $result[] = $row;
74 | }
75 | }
76 |
77 | return $result;
78 | }
79 |
80 |
81 |
82 | /**
83 | * 切换默认数据库
84 | * @param $dbName String 要切换的数据库名称
85 | * @return $this 做链式访问
86 | * @throws Exception
87 | */
88 | public function switchDb($dbName){
89 | if (is_string($dbName) && !empty($dbName)){
90 | if($this->dbConnection->select_db($dbName)){
91 | return $this;
92 | };
93 | throw new \Exception("mysql {$this->connectionName} switch db from {$this->dbName} to {$dbName} failure:({$this->dbConnection->errno}){$this->dbConnection->error}",-1);
94 | }
95 | throw new \Exception("mysql {$this->connectionName} switch db from {$this->dbName} to {$dbName} failure:param error",-1);
96 | }
97 |
98 |
99 | /**
100 | * 执行数据库操作
101 | * @param $sql String 要执行的sql语句
102 | * @param array $params 如果不为空,进行参数预处理
103 | * @return bool|int|mixed 执行select等返回结果集数组,执行insert返回自增值,执行其他返回受影响的行数。失败返回false
104 | * @throws Exception 支持出错抛出异常。
105 | */
106 | public function query($sql,$params=array()){
107 | $returnValue = true;
108 | $sql = trim($sql,' ');
109 | if (empty($params)){
110 | $result = $this->dbConnection->query($sql);
111 | if ($result === false){
112 | if ($this->dbConnection->error){
113 | throw new \Exception("{$this->connectionName} query error:({$this->dbConnection->errno}){$this->dbConnection->error}");
114 | }else{
115 | return false;
116 | }
117 | }
118 |
119 | if ($result === true){
120 | if (stripos($sql,'insert') === false){
121 | return $this->dbConnection->affected_rows;
122 | }else{
123 | return $this->dbConnection->insert_id;
124 | }
125 | }
126 | $returnValue = $this->fetch_all($result);
127 | $result->close();
128 | }else{
129 | $statement = $this->dbConnection->prepare($sql);
130 | if ($statement == false){
131 | throw new \Exception("{$this->connectionName} prepare error:({$this->dbConnection->errno}){$this->dbConnection->error}");
132 | }
133 | $types = '';
134 | $data = array('');
135 | foreach($params as $key=>$value){
136 | $type = gettype($value);
137 | switch($type){
138 | case 'string':{
139 | $types .= 's';
140 | $data[] = &$params[$key];
141 | }break;
142 | case 'integer':{
143 | $types .= 'i';
144 | $data[] = &$params[$key];
145 | }break;
146 | case 'double':{
147 | $types .= 'd';
148 | $data[] = &$params[$key];
149 | }break;
150 | default:
151 | $types .= 'b';
152 | $data[] = &$params[$key];
153 | }
154 | }
155 | $data[0] = $types;
156 | $method = new \ReflectionMethod($statement,'bind_param');
157 | if ($method->invokeArgs($statement,$data) == false){
158 | throw new \Exception("{$this->connectionName} bind_param error:({$statement->errno}){$statement->error}");
159 | }
160 | $rel = $statement->execute();
161 | if ($rel == false){
162 | throw new \Exception("{$this->connectionName} execute error:({$statement->errno}){$statement->error}");
163 | }
164 | $arr = explode(' ',$sql);
165 | $type = isset($arr[0])?strtolower($arr[0]):'';
166 | switch($type){
167 | case 'explain':
168 | case 'select':{
169 | $rel = $statement->get_result();
170 | if ($rel === false){
171 | throw new \Exception("{$this->connectionName} select error:({$this->dbConnection->errno}){$this->dbConnection->error}");
172 | }
173 | $returnValue = $this->fetch_all($rel);
174 | }break;
175 | case 'insert':{
176 | $returnValue = $this->dbConnection->insert_id;
177 | }break;
178 | case 'update':
179 | case 'delete':{
180 | $returnValue = $this->dbConnection->affected_rows;
181 | };
182 | }
183 | $statement->close();
184 | }
185 |
186 | return $returnValue;
187 | }
188 |
189 | /**
190 | * 根据数组插入一条记录
191 | * @param string $table 要插入的表名
192 | * @param array $arr 插入的数组(关联数组)
193 | * @return bool|int|mixed
194 | * @throws \Exception
195 | */
196 | public function insert($table,$arr = array()){
197 | if(empty($arr) || empty($table) || !is_string($table) || !is_array($arr)){
198 | return false;
199 | }
200 | $fields = array_keys($arr);
201 | $tempArr = array_fill(0,count($fields),'?');
202 | $params = array_values($arr);
203 |
204 | foreach ($fields as $index => $row){
205 | $fields[$index] = "`{$row}`";
206 | }
207 | $fields = '('.implode(',',$fields).')';
208 | $tempArr = '('.implode(',',$tempArr).')';
209 | $sql = "insert `{$table}`{$fields} values{$tempArr};";
210 | return $this->query($sql,$params);
211 | }
212 |
213 | /**
214 | * 一次性插入多条记录
215 | * @param string $table 插入的表名称
216 | * @param array $arr 二维数组,具有相同的表结构
217 | * @return int 返回插入的最后一条记录的自增id
218 | */
219 | public function insertMulti($table,$arr=array(array(),array())){
220 | if(empty($arr) || empty($table) || !is_string($table) || !is_array($arr) || !is_array($arr[0])){
221 | return false;
222 | }
223 | $fields = array_keys($arr[0]);
224 | $tempArr = array_fill(0,count($fields),'?');
225 | foreach ($fields as $index => $row){
226 | $fields[$index] = "`{$row}`";
227 | }
228 | $fields = '('.implode(',',$fields).')';
229 |
230 | $tempArrString = '('.implode(',',$tempArr).')';
231 | $params = array();
232 | $tempArr = array();
233 | foreach ($arr as $row){
234 | $tempArr[] = $tempArrString;
235 | $temp = array_values($row);
236 | $params = array_merge($params,$temp);
237 | }
238 | $tempArrString = implode(',',$tempArr);
239 |
240 |
241 | $sql = "insert `{$table}`{$fields} values{$tempArrString};";
242 | return $this->query($sql,$params);
243 | }
244 |
245 |
246 | /**
247 | * 返回mysqli对象
248 | * @return mysqli|null
249 | */
250 | public function getConnection(){
251 | $connection = $this->dbConnection;
252 | if($connection instanceof \mysqli){
253 | return $connection;
254 | }else{
255 | return null;
256 | }
257 | }
258 | }
259 |
260 |
261 | class TMysql{
262 | const VERSION = '1.0.1'; // 组件版本号
263 | private static $dbConnectionPool = array();
264 | private static $configs = array();
265 |
266 | public static function loadOneConfig($arr){
267 | mysqli_report(MYSQLI_REPORT_ALL^MYSQLI_REPORT_INDEX);
268 | if (!extension_loaded('mysqli')){
269 | throw new \Exception('need extension mysqli!', -1);
270 | }
271 | if (!is_array($arr) || empty($arr)){
272 | throw new \Exception('config not empty!', -1);
273 | }
274 | if (!isset($arr['connectionName']) || empty($arr['connectionName']) || !is_string($arr['connectionName'])){
275 | throw new \Exception('connectionName not exit or not a string!',-1);
276 | }
277 |
278 | $config = array();
279 | $config['connectionName'] = $arr['connectionName'];
280 | $config['host'] = isset($arr['host'])?$arr['host']:'';
281 | $config['userName'] = isset($arr['userName'])?$arr['userName']:'';
282 | $config['password'] = isset($arr['password'])?$arr['password']:'';
283 | $config['dbName'] = isset($arr['dbName'])?$arr['dbName']:'';
284 | $config['port'] = isset($arr['port'])?$arr['port']:3306;
285 | $config['checkConnection'] = isset($arr['checkConnection'])?isset($arr['checkConnection']):false;
286 | $config['charset'] = isset($arr['charset'])?$arr['charset']:'utf8';
287 |
288 | self::$configs[$config['connectionName']] = $config;
289 | if (isset(self::$dbConnectionPool[$config['connectionName']])){
290 | unset(self::$dbConnectionPool[$config['connectionName']]);
291 | }
292 | return true;
293 | }
294 |
295 | public static function getConnection($connectionName){
296 | if (!is_string($connectionName) || !isset(self::$configs[$connectionName])){
297 | throw new \Exception("mysql config : {$connectionName} is't loaded !",-1);
298 | }
299 |
300 | // 获取连接前,判断连接是否可用
301 | if (isset(self::$dbConnectionPool[$connectionName]) && self::$configs[$connectionName]['checkConnection']){
302 | $rel = self::$dbConnectionPool[$connectionName]->ping();
303 | if($rel == false){
304 | unset(self::$dbConnectionPool[$connectionName]);
305 | }
306 | }
307 |
308 | // 如果不存在连接,新建一个连接
309 | if(!isset(self::$dbConnectionPool[$connectionName])){
310 | $config = self::$configs[$connectionName];
311 | $dbConnection = new MysqlOperation($connectionName,$config['host'],$config['userName'],$config['password'],$config['dbName'],$config['port'],$config['charset']);
312 | self::$dbConnectionPool[$connectionName] = $dbConnection;
313 | }
314 |
315 | return self::$dbConnectionPool[$connectionName];
316 | }
317 |
318 | public static function clearAllConnection(){
319 | self::$dbConnectionPool = array();
320 | }
321 | }
322 |
323 |
324 | //测试
325 | if(strtolower(PHP_SAPI) == 'cli' && isset($argv) && basename(__FILE__) == basename($argv[0])){
326 | $config = array(
327 | 'connectionName' => 'dbLol',
328 | 'host'=> '*.*.*',
329 | 'userName' => 'xxxx',
330 | 'password' => '*****',
331 | 'dbName' => 'test',
332 | 'port' => '3306'
333 | );
334 |
335 | TMysql::loadOneConfig($config);
336 | $lolConnection = TMysql::getConnection('dbLol');
337 | $data = $lolConnection->query('select * from test where id<=?;',array(10));
338 | print_r($data);
339 | }
340 |
--------------------------------------------------------------------------------
/framework/extension/TRedis.php:
--------------------------------------------------------------------------------
1 | close();
14 | }
15 | }
16 |
17 | class RedisOperationException extends \RedisException{
18 |
19 | }
20 |
21 | class TRedis {
22 | const VERSION = '1.0.1'; // 组件版本号
23 | public static $maxConnectionTime = 5; // 最大连接时间
24 | protected static $connectionPool = array(); // 连接池对象
25 |
26 | protected static $configs = array(); // 配置项
27 |
28 | /**
29 | * 获取数组值
30 | * @param array $arr
31 | * @param string $key
32 | * @param null $default
33 | * @return mixed|null
34 | */
35 | public static function getArrayValue($arr=array(), $key='',$default=null){
36 | $response = null;
37 | if(is_array($arr) && isset($arr[$key])){
38 | $response = $arr[$key];
39 | }
40 | if($response == null){
41 | $response = $default;
42 | }
43 | return $response;
44 | }
45 |
46 | /**
47 | * 加载单个配置文件
48 | * @param array $configArr
49 | */
50 | public static function loadOneConfig($configArr= array()){
51 | $tempConfig = array();
52 | $tempConfig['connectionName'] = self::getArrayValue($configArr,'connectionName'); // 连接名称
53 | $tempConfig['host'] = self::getArrayValue($configArr,'host'); // 连接host
54 | $tempConfig['port'] = self::getArrayValue($configArr,'port',6379); // 端口号
55 | $tempConfig['database'] = self::getArrayValue($configArr,'database',0); // 使用数据库索引
56 | $tempConfig['password'] = self::getArrayValue($configArr,'password'); // 使用密码
57 | $tempConfig['prefix'] = self::getArrayValue($configArr,'prefix',''); // 数据库查询中的key前缀
58 | $tempConfig['checkConnection'] = self::getArrayValue($configArr,'checkConnection',false);
59 | if(empty($tempConfig['connectionName']) || !is_string($tempConfig['connectionName'])){
60 | $tempConfig['name'] = 'default';
61 | }
62 | self::$configs[$tempConfig['connectionName']] = $tempConfig;
63 | }
64 |
65 | /**
66 | * 获取一个连接对象
67 | * @param string $connectionName
68 | * @return mixed
69 | * @throws \Exception
70 | * @throws \RedisException
71 | */
72 | public static function getConnection($connectionName ='default'){
73 | if (!is_string($connectionName) || !isset(self::$configs[$connectionName])){
74 | throw new RedisOperationException("redis config : {$connectionName} is't loaded !",-1);
75 | }
76 |
77 | // 检测连接是否断开,断开重连
78 | if(isset(self::$connectionPool[$connectionName]) && self::$configs[$connectionName]['checkConnection']){
79 | try{
80 | $redisObject = self::$connectionPool[$connectionName];
81 | if($redisObject->ping() != '+PONG'){
82 | throw new RedisOperationException("redis ping error",-1);
83 | }
84 | }catch (RedisOperationException $e){
85 | @self::$connectionPool[$connectionName]->close();
86 | unset(self::$connectionPool[$connectionName]);
87 | }
88 | }
89 |
90 | // 创建连接
91 | if(!isset(self::$connectionPool[$connectionName])){
92 | $redisObject = new RedisOperation();
93 | $rel = $redisObject->connect(self::$configs[$connectionName]['host'],self::$configs[$connectionName]['port'],self::$maxConnectionTime);
94 | if(!$rel){
95 | throw new RedisOperationException("redis connection error:{$connectionName}",-1);
96 | }
97 | if(!empty(self::$configs[$connectionName]['password'])){
98 | $rel = $redisObject->auth(self::$configs[$connectionName]['password']);
99 | if(!$rel){
100 | $redisObject->close();
101 | throw new RedisOperationException("redis connection auth error:{$connectionName}",-1);
102 | }
103 | }
104 | $db = (int)self::$configs[$connectionName]['database'];
105 | $rel = $redisObject->select($db);
106 | if(!$rel){
107 | $redisObject->close();
108 | throw new RedisOperationException("redis select db error:name-{$connectionName},db-{$db}",-1);
109 | }
110 | if(is_string(self::$configs[$connectionName]['prefix']) && !empty(self::$configs[$connectionName]['prefix'])){
111 | $redisObject->setOption(\Redis::OPT_PREFIX,self::$configs[$connectionName]['prefix']);
112 | }
113 |
114 |
115 | self::$connectionPool[$connectionName] = $redisObject;
116 | }
117 | return self::$connectionPool[$connectionName];
118 | }
119 |
120 | /**
121 | * 清理连接
122 | */
123 | public static function clearAllConnection(){
124 | self::$connectionPool = array();
125 | }
126 | }
127 |
128 |
129 | //测试
130 | if(strtolower(PHP_SAPI) == 'cli' && isset($argv) && basename(__FILE__) == basename($argv[0])){
131 | $config = array(
132 | 'connectionName' => 'cacheRd',
133 | 'host'=> '115.28.149.242',
134 | 'password' => '',
135 | 'database' => 0,
136 | 'port' => 6379,
137 | 'prefix' => '',
138 | 'checkConnection' => false
139 | );
140 |
141 | TRedis::loadOneConfig($config);
142 | $redis = TRedis::getConnection('cacheRd');
143 | echo $redis->dbSize();
144 | print_r($redis->keys('*'));
145 | }
--------------------------------------------------------------------------------
/framework/queue/LocalRequestQueue.php:
--------------------------------------------------------------------------------
1 | queue = new \SplQueue();
19 | }
20 |
21 | public function add(Request $request){
22 | $this->queue->enqueue($request);
23 | }
24 |
25 | public function get(){
26 | if($this->isEmpty()){
27 | return null;
28 | }
29 | return $this->queue->dequeue();
30 | }
31 |
32 | public function isEmpty(){
33 | return $this->queue->isEmpty();
34 | }
35 |
36 | public function count(){
37 | return $this->queue->count();
38 | }
39 |
40 | public function isFull(){
41 | if($this->count() >= $this->maxCount){
42 | return true;
43 | }
44 | return false;
45 | }
46 |
47 | public function __destruct(){
48 | unset($this->queue);
49 | }
50 | }
--------------------------------------------------------------------------------
/framework/queue/RequestQueue.php:
--------------------------------------------------------------------------------
1 | downloader = $downloader;
22 | parent::__construct($taskName);
23 | }
24 |
25 | public function coroutine(){
26 | do{
27 | if (!TSpider::$requestQueue->isEmpty()){
28 | do{
29 | $request = TSpider::$requestQueue->get();
30 | $rel = $this->downloader->addRequest($request);
31 |
32 | // 如果下载队列添加失败,将请求重新放回请求队列中
33 | if($request instanceof Request && !$rel){
34 | TSpider::$requestQueue->add($request);
35 | }
36 | }while($rel);
37 | }
38 | yield true;
39 | }while(true);
40 | }
41 | }
--------------------------------------------------------------------------------
/framework/task/DelayTimer.php:
--------------------------------------------------------------------------------
1 | afterDownload($request,$response);
30 | if($rel !== false){
31 | $worker = new $request->workerName();
32 |
33 | if($worker->beforeParse() !== false){
34 | $worker->parse($request,$response);
35 | }
36 | $worker->afterParse();
37 | }
38 | }
39 |
40 | /**
41 | * 初始化下载器
42 | * DownloadTask constructor.
43 | * @param string $taskName
44 | */
45 | public function __construct($taskName=''){
46 | $this->multiDownloader = curl_multi_init();
47 | parent::__construct($taskName);
48 | }
49 |
50 | /**
51 | *
52 | * @param $request
53 | * @return bool
54 | */
55 | public function addRequest($request){
56 | $response = false;
57 | do{
58 | if($this->multiDownloader == null){
59 | break;
60 | }
61 |
62 | if(!($request instanceof Request)){
63 | break;
64 | }
65 |
66 | $currentRequestNum = count($this->currentRequestMap);
67 | if($currentRequestNum >= $this->windowSize){
68 | break;
69 | }
70 |
71 | $rel = $this->beforeDownload($request);
72 | if($rel === false){
73 | break;
74 | }
75 |
76 | if($rel instanceof Request){
77 | $request = $rel;
78 | }
79 | $ch = $request->createCurlObject();
80 | $key = (string)$ch;
81 | $this->currentRequestMap[$key] = $request;
82 | curl_multi_add_handle($this->multiDownloader, $ch);
83 |
84 | $this->spareTime = 0;
85 | $response = true;
86 | }while(false);
87 |
88 | return $response;
89 | }
90 |
91 | public function beforeDownload(Request $request){
92 | return null;
93 | }
94 |
95 | public function afterDownload(Request $request,Response $response){
96 | return null;
97 | }
98 |
99 | // 获取下载器闲置时间
100 | public function getSpareTime(){
101 | if($this->spareTime == 0){
102 | return 0;
103 | }else{
104 | return time()- $this->spareTime;
105 | }
106 | }
107 |
108 | public function coroutine()
109 | {
110 | do {
111 | while (($execrun = curl_multi_exec($this->multiDownloader, $running)) == CURLM_CALL_MULTI_PERFORM) ;
112 | if ($execrun != CURLM_OK) {
113 | if($this->spareTime == 0){
114 | $this->spareTime = time();
115 | }
116 | }
117 |
118 | // 一旦有一个请求完成,找出来,因为curl底层是select,所以最大受限于1024
119 | while ($done = curl_multi_info_read($this->multiDownloader))
120 | {
121 | // 从请求中获取信息、内容、错误
122 | $info = curl_getinfo($done['handle']);
123 | $output = curl_multi_getcontent($done['handle']);
124 | $error = curl_error($done['handle']);
125 | $response = new Response($info,$output,$error);
126 |
127 | $key = (string)$done['handle'];
128 | $request = $this->currentRequestMap[$key];
129 | $this->dispatch($request,$response);
130 |
131 | // 把请求已经完成了得 curl handle 删除
132 | unset($this->currentRequestMap[$key]);
133 | curl_multi_remove_handle($this->multiDownloader, $done['handle']);
134 | }
135 |
136 | // 当没有数据的时候进行堵塞,把 CPU 使用权交出来,避免上面 do 死循环空跑数据导致 CPU 100%
137 | if ($running) {
138 | $rel = curl_multi_select($this->multiDownloader, 1);
139 | if($rel == -1){
140 | usleep(1000);
141 | }
142 | }
143 |
144 | if( $running == false){
145 | if($this->spareTime == 0){
146 | $this->spareTime = time();
147 | }
148 | }
149 | yield true;
150 | } while (true);
151 |
152 | // 关闭任务
153 | curl_multi_close($this->multiDownloader);
154 | unset($this->multiDownloader);
155 | }
156 | }
--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
1 | ## TSpider
2 | 单进程,事件循环+协程方式实现的php爬虫框架。可以参考如何使用php的协程与curl_multi_*的使用。
3 |
4 |
5 | [php中curl_multi函数集的用法](https://github.com/hirudy/article/blob/master/php/php%E4%B8%ADcurl_multi%E5%87%BD%E6%95%B0%E9%9B%86%E7%9A%84%E7%94%A8%E6%B3%95.md)
6 |
7 | [转】在PHP中使用协程实现多任务调度](http://www.jianshu.com/p/9904b03a140c)
8 |
9 | ## 例子
10 | 参见./example中的例子。
11 |
--------------------------------------------------------------------------------