├── startApp.cmd
├── config.json
├── package.json
├── README.md
└── app.js


/startApp.cmd:
--------------------------------------------------------------------------------
1 | node app.js


--------------------------------------------------------------------------------
/config.json:
--------------------------------------------------------------------------------
1 | {
2 |   "address":"新街口",
3 |   "jobarea":"070200",
4 |   "startpage":0
5 | }


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "51job",
 3 |   "version": "0.0.1",
 4 |   "description": "51job招聘信息下载",
 5 |   "main": "app.js",
 6 |   "dependencies": {
 7 |     "bufferhelper": "^0.2.0",
 8 |     "async": "^0.2.9",
 9 |     "cheerio": "^0.12.2",
10 |     "iconv-lite": "^0.2.11"
11 |   },
12 |   "devDependencies": {},
13 |   "scripts": {
14 |     "test": "echo \"Error: no test specified\" && exit 1"
15 |   },
16 |   "repository": {
17 |     "type": "git",
18 |     "url": "https://github.com/lzxue/51job"
19 |   },
20 |   "keywords": [
21 |     "thinkgis",
22 |     "gis",
23 |     "招聘信息",
24 |     "51job"
25 |   ],
26 |   "author": "lzxue",
27 |   "license": "ISC",
28 |   "bugs": {
29 |     "url": "https://github.com/lzxue/51job/issues"
30 |   },
31 |   "homepage": "https://github.com/lzxue/51job"
32 | }
33 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | ###下载51job招聘信息的爬虫程序
 2 |  1.程序使用NodeJS编写，运行安装nodejs
 3 | 
 4 |  2.下载的数据存储在文本文档中以&字符为分隔符，一条招聘信息为一行数据。
 5 | 
 6 |  3.下载51job招聘信息，jobid，companyid 公司简介，岗位简介，经纬度等
 7 | 
 8 |  
 9 | 
10 |  [原数据来自51job地图搜索](http://search.51job.com/jobsearch/search_result.php?fromJs=1&jobarea=010000&industrytype=00&issuedate=9&keywordtype=2&lang=c&stype=3&address=%E4%B8%AD%E5%85%B3%E6%9D%91&radius=0.03&fromType=20) 可根据自身需要修改url参数实现分类下载招聘数据。
11 | 
12 | 地区查询一次下载整个地区的数据，程序中默认将查询半径设radius置1000(100000公里)旨在获取该地区的所有数据。但是数据最多返回10w条，向北京这样城市超过10w条数据因此需要设置多个中心点然后去重。
13 | 
14 | 下载参数设置在config,json中 
15 |   
16 |      {
17 |    
18 |        "address":"中关村",    //搜索中心 
19 |         "jobarea":"010000",   //搜索城市代码
20 |        " startpage":0        //下载起始页码，默认为0 在下载出错时修改该值，改完出错时中断的页码，实现继续上次中断下载
21 |      }
22 |  
23 | 
24 | windows下启动，修改下载参数，双击startapp.cmd 开始下载
25 | 
26 | 城市代码表：
27 | 
28 | ***
29 | 
30 | 				<a name="SelectMapArea" codestr="010000" city="北京" href="javascript:void(0);">北京</a></span>
31 | 				<a name="SelectMapArea" codestr="050000" city="天津" href="javascript:void(0);">天津</a>
32 | 				<a name="SelectMapArea" codestr="230300" city="大连" href="javascript:void(0);">大连</a>
33 | 				<a name="SelectMapArea" codestr="230200" city="沈阳" href="javascript:void(0);">沈阳</a>
34 | 				<a name="SelectMapArea" codestr="240200" city="长春" href="javascript:void(0);">长春</a>
35 | 				<a name="SelectMapArea" codestr="220200" city="哈尔滨" href="javascript:void(0);">哈尔滨</a>
36 | 				<a name="SelectMapArea" codestr="160200" city="石家庄" href="javascript:void(0);">石家庄</a>
37 | 			</td>
38 | 		</tr>
39 | 		<tr>
40 | 
41 | 			<td class="normalfont" nowrap="nowrap" valign="top"><strong>华东地区：</strong></td>
42 | 			<td>
43 | 				<a name="SelectMapArea" codestr="020000" city="上海" href="javascript:void(0);">上海</a>
44 | 				<a name="SelectMapArea" codestr="070200" city="南京" href="javascript:void(0);">南京</a>
45 | 				<a name="SelectMapArea" codestr="070300" city="苏州" href="javascript:void(0);">苏州</a>
46 | 				<a name="SelectMapArea" codestr="080200" city="杭州" href="javascript:void(0);">杭州</a>
47 | 				<a name="SelectMapArea" codestr="080300" city="宁波" href="javascript:void(0);">宁波</a>
48 | 				<a name="SelectMapArea" codestr="150200" city="合肥" href="javascript:void(0);">合肥</a>
49 | 				<a name="SelectMapArea" codestr="120200" city="济南" href="javascript:void(0);">济南</a>
50 | 				<a name="SelectMapArea" codestr="120300" city="青岛" href="javascript:void(0);">青岛</a>
51 | 				<a name="SelectMapArea" codestr="110200" city="福州" href="javascript:void(0);">福州</a>
52 | 				<a name="SelectMapArea" codestr="110300" city="厦门" href="javascript:void(0);">厦门</a>
53 | 				<br/>
54 | 				<a name="SelectMapArea" codestr="130200" city="南昌" href="javascript:void(0);">南昌</a>
55 | 				<a name="SelectMapArea" codestr="070400" city="无锡" href="javascript:void(0);">无锡</a>
56 | 				<a name="SelectMapArea" codestr="070500" city="常州" href="javascript:void(0);">常州</a>
57 | 			</td>                     
58 |                                 
59 | 		</tr>
60 | 		<tr>
61 | 			<td class="normalfont" nowrap="nowrap"><strong>华南-华中：</strong></td>
62 | 			<td>
63 | 				<a name="SelectMapArea" codestr="030200"  city="广州" href="javascript:void(0);">广州</a>
64 | 				<a name="SelectMapArea" codestr="040000"  city="深圳" href="javascript:void(0);">深圳</a>
65 | 				<a name="SelectMapArea" codestr="030800"  city="东莞" href="javascript:void(0);">东莞</a>
66 | 				<a name="SelectMapArea" codestr="180200"  city="武汉" href="javascript:void(0);">武汉</a>
67 | 				<a name="SelectMapArea" codestr="190200"  city="长沙" href="javascript:void(0);">长沙</a>
68 | 				<a name="SelectMapArea" codestr="170200"  city="郑州" href="javascript:void(0);">郑州</a>	
69 | 			</td>                            
70 |                
71 | 		</tr>                                                            
72 | 		<tr>                                            
73 | 			<td class="normalfont" nowrap="nowrap"><strong>西北-西南：</strong></td>
74 | 			<td>
75 | 				<a name="SelectMapArea" codestr="200200" city="西安" href="javascript:void(0);">西安</a>
76 | 				<a name="SelectMapArea" codestr="090200" city="成都" href="javascript:void(0);">成都</a>
77 | 				<a name="SelectMapArea" codestr="060000" city="重庆" href="javascript:void(0);">重庆</a>
78 | 				<a name="SelectMapArea" codestr="250200" city="昆明" href="javascript:void(0);">昆明</a>
79 | 			</td>                     
80 | 		</tr>                       
81 |  
82 | ***
83 | 
84 |  **codestr即为参数中的**jobarea
85 | 
86 |   address为 原网站 http://search.51job.com/list/010000%252C00,000000,0000,00,9,99,%2B,0,1.html?lang=c&stype=3&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=01&companysize=99&address=%D6%D0%B9%D8%B4%E5&lonlat=116.323066%2C39.989956&radius=0.05&ord_field=1&list_type=1&confirmdate=9&fromType=10选取该城市的任何一个 中心即可。
87 | 
88 | ***
89 | 数据导入excel示例
90 | ![gis](http://thinkgis.qiniudn.com/sinajobxx.png)
91 | 


--------------------------------------------------------------------------------
/app.js:
--------------------------------------------------------------------------------
  1 | var http=require("http");
  2 | var cheerio = require('cheerio');
  3 | var iconv = require('iconv-lite');
  4 | var BufferHelper = require('bufferhelper');
  5 | var async=require("async");
  6 | var fs=require("fs");
  7 | var config=require("./config.json");
  8 | var jobarea=config.jobarea;//城市代码
  9 | var address=config.address;//城市中心点 该城市任何一个均可
 10 | var startpage=config.startpage;//起始下载页码
 11 | var curr_page=0;
 12 | var url="http://search.51job.com/jobsearch/search_result.php?fromJs=1&jobarea="+jobarea+"&funtype=0000&industrytype=00&curr_page="+curr_page+"&keywordtype=2&lang=c&stype=3&postchannel=0000&address="+address+"&radius=10000&fromType=1";
 13 | var fileName=jobarea+address+".txt";
 14 | async.waterfall([
 15 |     function(callback){
 16 |         pages(url,startpage,function(pages){
 17 |             console.log("开始下载地区代号为"+jobarea+"的数据……");
 18 |             console.log("共"+pages.length+"页");
 19 |             callback(null, pages);
 20 |         })
 21 | 
 22 |     },
 23 |     function(pages, callback){
 24 |         // arg1 now equals 'three'
 25 |         async.eachLimit(pages,5,function(page,cb){
 26 |             console.log("正在下载第"+page+"页");
 27 |             url="http://search.51job.com/jobsearch/search_result.php?fromJs=1&jobarea="+jobarea+"&funtype=0000&industrytype=00&curr_page="+page+"&keywordtype=2&lang=c&stype=3&postchannel=0000&address="+address+"&radius=10000&fromType=1";
 28 | 
 29 |             getdata(url,function(data){
 30 | 
 31 |                  //处理数据
 32 |                 for(var i=0;i<data.length;i++){
 33 |                    var str=data[i].jobID+"&"+data[i].id+"&"+data[i].jobName+"&"+data[i].conName+"&"+data[i].locate+"&"+data[i].time+"&"+data[i].description+"&"+data[i].jobrequest+"&"+data[i].jobinstroduction+"&"+data[i].lon+"&"+data[i].lat+"&"+data[i].jobcount+"\n";
 34 |                  fs.appendFileSync(fileName,str,{encoding:null})
 35 |                 }
 36 |                  console.log("第"+page+"页下载完成");
 37 |                  cb();
 38 |              })
 39 | 
 40 |         },function(err){
 41 |             callback(null, 'done');
 42 |         })
 43 | 
 44 |     }
 45 | ], function (err, result) {
 46 |     console.log("代号为"+jobarea+"的数据下载完成");
 47 | });
 48 | 
 49 | function parseHtml(html){
 50 |     var $ = cheerio.load(html);
 51 |     var data=[];
 52 |     var reg = /var lonlatInfo=\{([\s\S]*)\}\}/m;//匹配经纬度信息
 53 |     var reg2=/M_\d*/g;
 54 |     var lonlatInfo=reg.exec(html)[0].replace(/\n/g,"").split("=")[1].replace(/key/g,"\"key\"").replace(/jobids/g,"\"jobids\"").replace(/lonlat/g,"\"lonlat\"").replace(/'/g,"\"");
 55 |     var ids=lonlatInfo.match(reg2);
 56 |     for(var i=0;i<ids.length;i++){//格式化经纬度信息
 57 |         var regm=new RegExp(ids[i]);
 58 |         lonlatInfo=lonlatInfo.replace(regm,"\""+ids[i]+"\"")
 59 |     }
 60 |     lonlatInfo=JSON.parse(lonlatInfo);
 61 |     var count=$("table .navBold").text().split("/")[1];
 62 |     var pages=parseInt(count/50)+1;
 63 | 
 64 |     $(".resultListDiv table .tr0").each(function (i,item) {
 65 |         var items={};
 66 |         items.jobID=cheerio.load($(item).html())(".td0 input").val();
 67 |         items.id=cheerio.load($(item).html())(".td1 img").attr("name");
 68 |         items.jobName=cheerio.load($(item).html())(".td1 .jobname").text();
 69 |         items.conName=cheerio.load($(item).html())(".td2").text();
 70 |         items.locate=cheerio.load($(item).html())(".td3").text();
 71 |         items.time=cheerio.load($(item).html())(".td4").text();
 72 |         items.description=cheerio.load($(item).html())(".td5").text();
 73 |         items.jobrequest=$(".resultListDiv table .tr1 .td1234").text().replace(/\s/g,"");
 74 |         items.jobinstroduction=$(".resultListDiv table .tr2 .td1234 span").text();
 75 |         items.lon=Number(lonlatInfo[items.id].lonlat.split(",")[0]);
 76 |         items.lat=Number(lonlatInfo[items.id].lonlat.split(",")[1]);
 77 |         items.jobcount=lonlatInfo[items.id].jobids.length;
 78 |         data.push(items);
 79 |     });
 80 |     return data;
 81 | }
 82 | function pages(url,startpage,cb){
 83 |     http.get(url,function(res) {
 84 |             var bufferHelper = new BufferHelper();
 85 |             res.on("data", function (chunk) {
 86 | 
 87 |                 bufferHelper.concat(chunk);
 88 |             });
 89 | 
 90 |             res.on("end", function () {
 91 |                 var page=[];
 92 |                 var html=iconv.decode(bufferHelper.toBuffer(), 'GB2312');
 93 |                 var $ = cheerio.load(html);
 94 |                 var count=$("table .navBold").text().split("/")[1];
 95 |                 console.log("共"+count+"条招聘记录");
 96 |                 var pages=parseInt(count/50)+1;
 97 |                 for(var i=startpage;i<pages;i++){
 98 |                     page.push(i+1);
 99 |                 }
100 |                 cb(page)
101 |             })
102 |         }
103 |     )
104 | 
105 | }
106 | 
107 | function getdata(url,cb){
108 |     http.get(url,function(res) {
109 |             var bufferHelper = new BufferHelper();
110 |             res.on("data", function (chunk) {
111 | 
112 |                 bufferHelper.concat(chunk);
113 |             });
114 | 
115 |             res.on("end", function () {
116 | 
117 |                 var html=iconv.decode(bufferHelper.toBuffer(), 'GB2312');
118 |                 var data=parseHtml(html);
119 |                 cb(data);
120 |             })
121 |         }
122 |     )
123 | }


--------------------------------------------------------------------------------