9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 | {{ props.row.json[key] }}
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
44 |
45 |
46 |
71 |
72 |
73 |
--------------------------------------------------------------------------------
/utils/domain/cdn.js:
--------------------------------------------------------------------------------
1 | const fs = require('fs');
2 | const path = require('path');
3 |
4 | const scan = function (options) {
5 | //Web指纹探测
6 | let scan_task_finger = require(path.join(__dirname, 'finger.js'));
7 | scan_task_finger.scan({
8 | uri: search_uri
9 | });
10 | //WHOIS查询
11 | let scan_task_whois = require(path.join(__dirname, 'whois.js'));
12 | scan_task_whois.scan({
13 | uri: search_uri
14 | });
15 | //子域名段查询
16 | let scan_task_subdomain = require(path.join(__dirname, 'subdomain.js'));
17 | scan_task_subdomain.scan({
18 | uri: search_uri
19 | });
20 | //敏感信息爬取
21 | let scan_task_sensitive = require(path.join(__dirname, 'static_resouce.js'));
22 | scan_task_sensitive.scan({
23 | uri: options.original_uri
24 | });
25 |
26 | // let electron_path = path.join(__dirname, '..\\node_modules\\electron\\dist\\electron.exe');
27 | const Nightmare = require('nightmare')
28 | let result = {
29 | sources: '',
30 | name: 'CDN判断',
31 | site: options.uri,
32 | html: '',
33 | json: {},
34 | }
35 | const nightmare = Nightmare({
36 | electronPath: electron_path,
37 | // show: true,
38 | // openDevTools: {
39 | // mode: 'detach'
40 | // },
41 | })
42 |
43 | nightmare
44 | .goto('https://ping.aizhan.com/')
45 | .insert('#domain', options.uri)
46 | .click('.search-button')
47 | .wait('#ping_container')
48 | .wait(7000)
49 | .evaluate(() => {
50 | let res = ''
51 | ips = []
52 | res = document.getElementById('main_container').innerHTML
53 | var tableId = document.getElementById("table");
54 | for (var i = 1; i < tableId.rows.length; i++) {
55 | ips.push(tableId.rows[i].cells[1].innerText);
56 | }
57 | return [res, ips]
58 | })
59 | .then(res => {
60 | html = res[0]
61 | ips = res[1]
62 | result.sources = result.sources + 'ping.aizhan.com' + ','
63 | result.html = result.html + html + '
'
64 | nightmare
65 | .goto('http://tool.chinaz.com/speedtest/' + options.uri)
66 | .wait('.svggroup')
67 | .wait(20000)
68 | .evaluate(() => {
69 | let html = ''
70 | html = document.getElementById('speedlist').innerHTML
71 | ips = [];
72 | for (var i = 2; i < document.getElementsByClassName('col-3').length; i = i + 2) {
73 | if (/^\d+/.test(document.getElementsByClassName('col-3')[i].lastElementChild.innerText)) {
74 | ips.push(document.getElementsByClassName('col-3')[i].lastElementChild.innerText)
75 | }
76 | }
77 | return [html,ips]
78 | })
79 | // 最后一次查询再end
80 | .end()
81 | .then(res => {
82 | html = res[0]
83 | ips_second = res[1]
84 | console.log(ips)
85 | result.sources = result.sources + 'tool.chinaz.com' + ','
86 | result.html = result.html + html + '
'
87 | ips = ips.concat(ips_second)
88 | console.log(ips)
89 | ips = Array.from(new Set(ips))
90 | console.log(ips)
91 | if(ips.length == 1){
92 | result.json.potential_cdn = 0
93 | result.json.ip = ips[0]
94 | }else{
95 | result.json.potential_cdn = 1
96 | }
97 | db.get('sites')
98 | .push(result)
99 | .write()
100 |
101 | if (result.json.potential_cdn == 1) {
102 | //查一下DNS 历史，可能发现真实IP
103 | let scan_task_dns = require(path.join(__dirname, 'dns_history.js'));
104 | scan_task_dns.scan({
105 | uri: search_uri
106 | });
107 | }else{
108 | var readDir = fs.readdirSync("./utils/ip");
109 | for (let index = 0; index < readDir.length; index++) {
110 | console.log(path.join(__dirname, '../ip/' + readDir[index]))
111 | let scan_task = require(path.join(__dirname, '../ip/' + readDir[index]));
112 | scan_task.scan({
113 | uri: result.json.ip
114 | });
115 | }
116 | }
117 | })
118 | .catch((error) => {
119 | console.error('CDN Search failed:', error);
120 | });
121 | })
122 | .catch((error) => {
123 | console.error('CDN Search failed:', error);
124 | });
125 | }
126 |
127 |
128 | module.exports.scan = scan;
--------------------------------------------------------------------------------
/utils/domain/dns_history.js:
--------------------------------------------------------------------------------
1 | const fs = require('fs');
2 | const path = require('path');
3 |
4 | const scan = function (options) {
5 | const Nightmare = require('nightmare')
6 | let result = {
7 | sources: '',
8 | name: 'DNS解析记录',
9 | site: options.uri,
10 | html: '',
11 | json: {},
12 | }
13 | const nightmare = Nightmare({
14 | electronPath: electron_path,
15 | show: true,
16 | // openDevTools: {
17 | // mode: 'detach'
18 | // },
19 | })
20 |
21 | nightmare
22 | .goto('https://viewdns.info/iphistory/?domain=' + options.uri)
23 | .wait('[border="1"]')
24 | .wait(2000)
25 | .evaluate(() => {
26 | let res = ''
27 | res = document.querySelector('[border="1"]').outerHTML
28 | return res
29 | })
30 | .then(html => {
31 | result.sources = result.sources + 'viewdns.info' + ','
32 | result.html = result.html + html + '
'
33 | nightmare
34 | .goto('http://site.ip138.com/' + options.uri)
35 | .wait('[class="panel"]')
36 | .wait(2000)
37 | .evaluate(() => {
38 | let res = ''
39 | res = document.querySelector('[class="panel"]').outerHTML
40 | return res
41 | })
42 | // .end()
43 | .then(html => {
44 | result.sources = result.sources + 'site.ip138.com' + ','
45 | result.html = result.html + html + '
'
46 | nightmare
47 | .goto('https://securitytrails.com/domain/' + options.uri + '/history/a')
48 | .wait('[class="table table-hover"]')
49 | .wait(2000)
50 | .evaluate(() => {
51 | let res = ''
52 | res = document.querySelector('[class="table table-hover"]').outerHTML
53 | return res
54 | })
55 | .then(html => {
56 | result.sources = result.sources + 'securitytrails.com' + ','
57 | result.html = result.html + html + '
'
58 | db.get('sites')
59 | .push(result)
60 | .write()
61 | })
62 | })
63 | .catch((error) => {
64 | console.error('DNS history Search failed:', error);
65 | });
66 | })
67 | .catch((error) => {
68 | console.error('DNS history Search failed:', error);
69 | });
70 | }
71 |
72 |
73 | module.exports.scan = scan;
--------------------------------------------------------------------------------
/utils/domain/finger.js:
--------------------------------------------------------------------------------
1 | const fs = require('fs');
2 | const path = require('path');
3 |
4 | const scan = function (options) {
5 | const Nightmare = require('nightmare')
6 | let result = {
7 | sources: '',
8 | name: 'Web指纹',
9 | site: options.uri,
10 | html: '',
11 | json: {},
12 | }
13 | const nightmare = Nightmare({
14 | electronPath: electron_path,
15 | show: true,
16 | // openDevTools: {
17 | // mode: 'detach'
18 | // },
19 | })
20 |
21 | nightmare
22 | .goto('https://www.wappalyzer.com/')
23 | .insert('[class="input is-medium"]', options.uri)
24 | .click('[class="button is-primary is-medium"]')
25 | .wait(7000)
26 | .evaluate(() => {
27 | let res = ''
28 | let finger = ''
29 | res = document.getElementsByClassName('lookup-results')[0].innerHTML
30 | finger = document.getElementsByClassName('lookup-results')[0].firstElementChild.innerText
31 | return [res, finger]
32 | })
33 | .then(res => {
34 | html = res[0]
35 | finger = res[1]
36 | result.sources = result.sources + 'www.wappalyzer.com' + ','
37 | result.html = result.html + html + '
'
38 | result.json.finger = finger
39 | db.get('sites')
40 | .push(result)
41 | .write()
42 | nightmare
43 | .goto('http://www.yunsee.cn/finger.html')
44 | .insert('#input1', options.uri)
45 | .click('#chaxun-btn')
46 | .catch((error) => {
47 | console.error('Search failed:', error);
48 | });
49 | })
50 | .catch((error) => {
51 | console.error('Search failed:', error);
52 | });
53 | }
54 |
55 |
56 | module.exports.scan = scan;
--------------------------------------------------------------------------------
/utils/domain/static_resouce.js:
--------------------------------------------------------------------------------
1 | const fs = require('fs');
2 | const path = require('path');
3 | const cheerio = require("cheerio")
4 | // let electron_path = path.join(__dirname, '..\\..\\node_modules\\electron\\dist\\electron.exe');
5 | const superagent = require("superagent")
6 |
7 | const scan = function (options) {
8 | const Nightmare = require('nightmare')
9 | let result = {
10 | sources: '',
11 | name: '敏感信息',
12 | site: options.uri,
13 | html: '',
14 | json: {},
15 | }
16 | const nightmare = Nightmare({
17 | electronPath: electron_path,
18 | show: true,
19 | // openDevTools: {
20 | // mode: 'detach'
21 | // },
22 | })
23 |
24 | if (!(options.uri.substr(0, 4) == "http")) {
25 | options.uri = "http://" + options.uri
26 | }
27 |
28 | nightmare
29 | .goto(options.uri)
30 | .wait(6000)
31 | .evaluate(() => {
32 | let res = ''
33 | res = document.documentElement.innerHTML
34 | return res
35 | })
36 | .end()
37 | .then(html => {
38 | // Array Remove - By John Resig (MIT Licensed)
39 | Array.prototype.remove = function (from, to) {
40 | var rest = this.slice((to || from) + 1 || this.length);
41 | this.length = from < 0 ? this.length + from : from;
42 | return this.push.apply(this, rest);
43 | };
44 |
45 | function getJs(scripts) {
46 | var res = [];
47 | scripts.each(function (i, script) {
48 | var src = script.attribs.src;
49 | if (!src)
50 | return;
51 | var index = src.indexOf('http');
52 | if (index != 0) {
53 | src = options.uri + src
54 | }
55 | res.push(src);
56 | });
57 | return Array.from(new Set(res));
58 | }
59 |
60 | function getCss(stylesheets) {
61 | var res = [];
62 | stylesheets.each(function (i, style) {
63 | var src = style.attribs.href;
64 | if (!src)
65 | return;
66 | var index = src.indexOf('http');
67 | if (index != 0) {
68 | src = options.uri + src
69 | }
70 | res.push(src);
71 | });
72 | return Array.from(new Set(res));
73 | }
74 |
75 | function get_resource(resource) {
76 | superagent
77 | .get(resource)
78 | .end(function (req, res) {
79 | // if (res.ok) {
80 | // console.log(res.text)
81 | return res.text
82 | // }
83 | })
84 | }
85 |
86 | function findMatch(reg, raw) {
87 | var data = {}
88 | var arr = []
89 | while (true) {
90 | data = reg.exec(raw);
91 | if (data != null) {
92 | arr.push(data[1]);
93 | } else {
94 | break;
95 | }
96 | }
97 | return arr;
98 | }
99 |
100 | var $ = cheerio.load(html);
101 | var js, css, all_links = [];
102 | var linkReg = /(?:"|')((\w+@\w+(\.\w+)+)|(\d{18}|\d{15})|((\(\d{3,4}\)|\d{3,4}-|\s)?\d{7,14})|((?:[a-zA-Z]{1,10}:\/\/|\/\/)*[-*a-zA-Z0-9]{1,}\.[a-zA-Z0-9]{2,}[^"']{0,})|((?:\/|\.\.\/|\.\/)[^"'><,;| *()(%%$^\/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-\/]{1,}\/[a-zA-Z0-9_\-\/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|\/][^"|']{0,}|))|([a-zA-Z0-9_\-]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:\?[^"|']{0,}|)))(?:"|')/g
103 | //获取js列表
104 | var scripts = $("script");
105 | js = getJs(scripts);
106 | //获取css和部分js、html等
107 | var stylesheets = $("link");
108 | css = getCss(stylesheets);
109 | //合并所有链接
110 | var static_resource = js.concat(css)
111 | for (var index = 0; index < static_resource.length; index++) {
112 | var resource = get_resource(static_resource[index]);
113 | if (resource != '') {
114 | links = findMatch(linkReg, resource);
115 | // links = remove_pictures(links)
116 | links = Array.from(new Set(links))
117 | all_links = all_links.concat(links)
118 | // console.log(all_links)
119 | key = `${static_resource[index]}`;
120 | result.json[key] = links;
121 | } else {
122 | console.log("crawl static resource failed.............")
123 | }
124 | }
125 | //从动态加载后的结果（html）中提取
126 | links = findMatch(linkReg, html);
127 | links = Array.from(new Set(links))
128 | all_links = all_links.concat(links)
129 | all_links = Array.from(new Set(all_links))
130 | // console.log(all_links.length)
131 | var all_links_without_pics = []
132 |
133 | function remove_pictures(all_links) {
134 | for (var index = 0; index < all_links.length; index++) {
135 | if (/\.(png|jpg|gif)$/g.test(all_links[index]) || /\.(png|jpg|gif)\?.*/g.test(all_links[index])) {
136 | all_links.remove(index)
137 | // all_links.splice(index, 1);
138 | console.log(all_links[index]);
139 | } else {
140 | all_links_without_pics.push(all_links[index])
141 | }
142 | }
143 | return all_links_without_pics
144 | }
145 | all_links = remove_pictures(all_links)
146 | all_links = Array.from(new Set(all_links))
147 | // console.log(all_links.length);
148 | // console.log(all_links);
149 | // console.log(all_links_without_pics.length);
150 | // console.log(all_links_without_pics);
151 | key = `${options.uri}`
152 | result.json[key] = links;
153 | result.json.links = all_links;
154 |
155 | // var meta = $("meta");
156 | // console.log(meta.html());
157 | db.get('sites')
158 | .push(result)
159 | .write()
160 | })
161 | .catch((error) => {
162 | console.error('sensitive info Search failed:', error);
163 | });
164 | };
165 | // scan({
166 | // uri: 'map.qq.com/'
167 | // })
168 |
169 | module.exports.scan = scan;
--------------------------------------------------------------------------------
/utils/domain/subdomain.js:
--------------------------------------------------------------------------------
1 | const fs = require('fs');
2 | const path = require('path');
3 |
4 | const scan = function (options) {
5 | const Nightmare = require('nightmare')
6 | let result = {
7 | sources: '',
8 | name: '子域信息',
9 | site: options.uri,
10 | html: '',
11 | json: {},
12 | }
13 | const nightmare = Nightmare({
14 | electronPath: electron_path,
15 | show: true,
16 | // openDevTools: {
17 | // mode: 'detach'
18 | // },
19 | })
20 |
21 | nightmare
22 | .goto('http://site.ip138.com/' + options.uri + '/domain.htm')
23 | .wait('[class="panel"]')
24 | .wait(2000)
25 | .evaluate(() => {
26 | let res = ''
27 | res = document.querySelector('[class="panel"]').innerHTML
28 | return res
29 | })
30 | .then(html => {
31 | result.sources = result.sources + 'site.ip138.com' + ','
32 | result.html = result.html + html + '
'
33 | nightmare
34 | .goto('https://securitytrails.com/list/apex_domain/' + options.uri)
35 | .catch((error) => {
36 | console.error('subdomain Search failed:', error);
37 | });
38 | })
39 | .catch((error) => {
40 | console.error('subdomain Search failed:', error);
41 | });
42 | }
43 |
44 |
45 | module.exports.scan = scan;
--------------------------------------------------------------------------------
/utils/domain/whois.js:
--------------------------------------------------------------------------------
1 | const fs = require('fs');
2 | const path = require('path');
3 |
4 | const scan = function (options) {
5 | const Nightmare = require('nightmare')
6 | let result = {
7 | sources: '',
8 | name: 'WHOIS信息',
9 | site: options.uri,
10 | html: '',
11 | json: {},
12 | }
13 | const nightmare = Nightmare({
14 | electronPath: electron_path,
15 | show: true,
16 | // openDevTools: {
17 | // mode: 'detach'
18 | // },
19 | })
20 |
21 | nightmare
22 | .goto('https://whois.aliyun.com/')
23 | .insert('[class="whois-search-input"]', options.uri)
24 | .click('[class="whois-search-btn"]')
25 | .wait('#info_whois_title')
26 | .wait(3000)
27 | .evaluate(() => {
28 | let res = ''
29 | res = document.querySelector('[class="whois-data"]').innerHTML
30 | return res
31 | })
32 | .then(html => {
33 | result.sources = result.sources + 'whois.aliyun.com' + ','
34 | result.html = result.html + html + '
'
35 | nightmare
36 | .goto('http://whois.chinaz.com/' + options.uri)
37 | .wait('#sh_info')
38 | .wait(3000)
39 | .evaluate(() => {
40 | let res = ''
41 | res = document.querySelector('#sh_info').outerHTML
42 | return res
43 | })
44 | .end()
45 | .then(html => {
46 | result.sources = result.sources + 'whois.chinaz.com' + ','
47 | result.html = result.html + html + '
'
48 | db.get('sites')
49 | .push(result)
50 | .write()
51 | })
52 | .catch((error) => {
53 | console.error('WHOIS Search failed:', error);
54 | });
55 | })
56 | .catch((error) => {
57 | console.error('WHOIS Search failed:', error);
58 | });
59 | }
60 |
61 |
62 | module.exports.scan = scan;
--------------------------------------------------------------------------------
/utils/ip/CIDR.js:
--------------------------------------------------------------------------------
1 | const fs = require('fs');
2 | const path = require('path');
3 |
4 | const scan = function (options) {
5 | const Nightmare = require('nightmare')
6 | let result = {
7 | sources: '',
8 | name: '扫描IP段',
9 | site: options.uri,
10 | html: '',
11 | json: {},
12 | }
13 | const nightmare = Nightmare({
14 | electronPath: electron_path,
15 | show: true,
16 | switches: {
17 | 'ignore-certificate-errors': true
18 | }
19 | // openDevTools: {
20 | // mode: 'detach'
21 | // },
22 | })
23 |
24 | nightmare
25 | .goto('https://phpinfo.me/bing.php')
26 | .insert('#domain', options.uri)
27 | .click('#getip')
28 | .wait(() =>{
29 | return document.getElementById('ip').innerText != ''
30 | })
31 | .click('#query')
32 | .catch((error) => {
33 | console.error('CIDR Search failed:', error);
34 | });
35 |
36 | const nightmare_1 = Nightmare({
37 | electronPath: electron_path,
38 | show: true,
39 | // openDevTools: {
40 | // mode: 'detach'
41 | // },
42 | })
43 | nightmare_1
44 | .goto('https://www.chinabaiker.com/cduan.php')
45 | .insert('#domain', options.uri)
46 | .click('#getip')
47 | .wait(() => {
48 | return document.getElementById('ip').innerText != ''
49 | })
50 | .click('#query')
51 | .catch((error) => {
52 | console.error('CIDR Search failed:', error);
53 | });
54 | }
55 |
56 |
57 | module.exports.scan = scan;
--------------------------------------------------------------------------------
/utils/ip/finger.js:
--------------------------------------------------------------------------------
1 | const fs = require('fs');
2 | const path = require('path');
3 |
4 | const scan = function (options) {
5 | const Nightmare = require('nightmare')
6 | let result = {
7 | sources: '',
8 | name: 'Web指纹',
9 | site: options.uri,
10 | html: '',
11 | json: {},
12 | }
13 | const nightmare = Nightmare({
14 | electronPath: electron_path,
15 | show: true,
16 | // openDevTools: {
17 | // mode: 'detach'
18 | // },
19 | })
20 |
21 | nightmare
22 | .goto('https://www.wappalyzer.com/')
23 | .insert('[class="input is-medium"]', options.uri)
24 | .click('[class="button is-primary is-medium"]')
25 | .wait(7000)
26 | .evaluate(() => {
27 | let res = ''
28 | let finger = ''
29 | res = document.getElementsByClassName('lookup-results')[0].innerHTML
30 | finger = document.getElementsByClassName('lookup-results')[0].firstElementChild.innerText
31 | return [res, finger]
32 | })
33 | .then(res => {
34 | html = res[0]
35 | finger = res[1]
36 | result.sources = result.sources + 'www.wappalyzer.com' + ','
37 | result.html = result.html + html + '
'
38 | result.json.finger = finger
39 | db.get('sites')
40 | .push(result)
41 | .write()
42 | nightmare
43 | .goto('http://www.yunsee.cn/finger.html')
44 | .insert('#input1', options.uri)
45 | .click('#chaxun-btn')
46 | .catch((error) => {
47 | console.error('Search failed:', error);
48 | });
49 | })
50 | .catch((error) => {
51 | console.error('Search failed:', error);
52 | });
53 | }
54 |
55 |
56 | module.exports.scan = scan;
--------------------------------------------------------------------------------
/utils/ip/ip_location.js:
--------------------------------------------------------------------------------
1 | const fs = require('fs');
2 | const path = require('path');
3 |
4 | const scan = function (options) {
5 | const Nightmare = require('nightmare');
6 | const superagent = require('superagent');
7 | let result = {
8 | sources: '',
9 | name: 'IP地理位置',
10 | site: options.uri,
11 | html: '',
12 | json: {},
13 | }
14 | const nightmare = Nightmare({
15 | electronPath: electron_path,
16 | show: true,
17 | // openDevTools: {
18 | // mode: 'detach'
19 | // },
20 | })
21 |
22 | superagent
23 | .get('http://api.db-ip.com/v2/free/' + options.uri)
24 | .end(function (req, res) {
25 | console.log(res.body)
26 | result.json = res.body
27 | result.sources = result.sources + 'db-ip.com' + ','
28 | db.get('sites')
29 | .push(result)
30 | .write()
31 | })
32 |
33 | nightmare
34 | .goto('https://www.ipplus360.com/search/ip/?ip=' + options.uri)
35 | .catch((error) => {
36 | console.error('IP location Search failed:', error);
37 | });
38 | }
39 |
40 |
41 | module.exports.scan = scan;
--------------------------------------------------------------------------------
/utils/ip/reverse_to_domain.js:
--------------------------------------------------------------------------------
1 | const fs = require('fs');
2 | const path = require('path');
3 |
4 | const scan = function (options) {
5 | // console.log(options)
6 | // let electron_path = path.join(__dirname, '..\\node_modules\\electron\\dist\\electron.exe');
7 | const Nightmare = require('nightmare')
8 | let result = {
9 | sources: '',
10 | name: 'IP反查',
11 | site: options.uri,
12 | html: '',
13 | json: {},
14 | }
15 | const nightmare = Nightmare({
16 | electronPath: electron_path,
17 | show: true,
18 | openDevTools: {
19 | mode: 'detach'
20 | },
21 | })
22 |
23 | nightmare
24 | .goto('https://tools.ipip.net/ipdomain.php')
25 | .insert('#ip', options.uri)
26 | .click('#btn')
27 | .wait('.col-sm-10')
28 | .evaluate(() => {
29 | let res = ''
30 | res = document.getElementsByClassName('col-sm-10 col-sm-offset-1')[0].innerHTML
31 | return res
32 | })
33 | // .end()
34 | .then(html => {
35 | result.sources = result.sources + 'tools.ipip.net' + ','
36 | result.html = result.html + html + '
'
37 | nightmare
38 | .goto('https://dns.aizhan.com')
39 | .insert('#domain')
40 | .insert('#domain', options.uri)
41 | .click('.search-button')
42 | // .wait(() => {
43 | // return document.documentElement.innerHTML.indexOf('暂无域名解析到该IP') < 0
44 | // })
45 | .wait(3000)
46 | .evaluate(() => {
47 | let res = ''
48 | res = document.getElementsByClassName('dns-content')[0].innerHTML
49 | return res
50 | })
51 | // 最后一次查询再end
52 | .end()
53 | .then(html => {
54 | console.log(html)
55 | result.sources = result.sources + 'dns.aizhan.com' + ','
56 | result.html = result.html + html + '
'
57 | db.get('sites')
58 | .push(result)
59 | .write()
60 | })
61 | .catch((error) => {
62 | console.error('Search failed:', error);
63 | });
64 | })
65 | .catch((error) => {
66 | console.error('Search failed:', error);
67 | });
68 | }
69 |
70 |
71 | module.exports.scan = scan;
--------------------------------------------------------------------------------
/utils/ip/static_resouce.js:
--------------------------------------------------------------------------------
1 | const fs = require('fs');
2 | const path = require('path');
3 | const cheerio = require("cheerio")
4 | // let electron_path = path.join(__dirname, '..\\..\\node_modules\\electron\\dist\\electron.exe');
5 | const superagent = require("superagent")
6 |
7 | const scan = function (options) {
8 | const Nightmare = require('nightmare')
9 | let result = {
10 | sources: '',
11 | name: '敏感信息',
12 | site: options.uri,
13 | html: '',
14 | json: {},
15 | }
16 | const nightmare = Nightmare({
17 | electronPath: electron_path,
18 | show: true,
19 | // openDevTools: {
20 | // mode: 'detach'
21 | // },
22 | })
23 |
24 | if (!(options.uri.substr(0, 4) == "http")) {
25 | options.uri = "http://" + options.uri
26 | }
27 |
28 | nightmare
29 | .goto(options.uri)
30 | .wait(6000)
31 | .evaluate(() => {
32 | let res = ''
33 | res = document.documentElement.innerHTML
34 | return res
35 | })
36 | .end()
37 | .then(html => {
38 | // Array Remove - By John Resig (MIT Licensed)
39 | Array.prototype.remove = function (from, to) {
40 | var rest = this.slice((to || from) + 1 || this.length);
41 | this.length = from < 0 ? this.length + from : from;
42 | return this.push.apply(this, rest);
43 | };
44 |
45 | function getJs(scripts) {
46 | var res = [];
47 | scripts.each(function (i, script) {
48 | var src = script.attribs.src;
49 | if (!src)
50 | return;
51 | var index = src.indexOf('http');
52 | if (index != 0) {
53 | src = options.uri + src
54 | }
55 | res.push(src);
56 | });
57 | return Array.from(new Set(res));
58 | }
59 |
60 | function getCss(stylesheets) {
61 | var res = [];
62 | stylesheets.each(function (i, style) {
63 | var src = style.attribs.href;
64 | if (!src)
65 | return;
66 | var index = src.indexOf('http');
67 | if (index != 0) {
68 | src = options.uri + src
69 | }
70 | res.push(src);
71 | });
72 | return Array.from(new Set(res));
73 | }
74 |
75 | function get_resource(resource) {
76 | superagent
77 | .get(resource)
78 | .end(function (req, res) {
79 | // if (res.ok) {
80 | // console.log(res.text)
81 | return res.text
82 | // }
83 | })
84 | }
85 |
86 | function findMatch(reg, raw) {
87 | var data = {}
88 | var arr = []
89 | while (true) {
90 | data = reg.exec(raw);
91 | if (data != null) {
92 | arr.push(data[1]);
93 | } else {
94 | break;
95 | }
96 | }
97 | return arr;
98 | }
99 |
100 | var $ = cheerio.load(html);
101 | var js, css, all_links = [];
102 | var linkReg = /(?:"|')((\w+@\w+(\.\w+)+)|(\d{18}|\d{15})|((\(\d{3,4}\)|\d{3,4}-|\s)?\d{7,14})|((?:[a-zA-Z]{1,10}:\/\/|\/\/)*[-*a-zA-Z0-9]{1,}\.[a-zA-Z0-9]{2,}[^"']{0,})|((?:\/|\.\.\/|\.\/)[^"'><,;| *()(%%$^\/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-\/]{1,}\/[a-zA-Z0-9_\-\/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|\/][^"|']{0,}|))|([a-zA-Z0-9_\-]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:\?[^"|']{0,}|)))(?:"|')/g
103 | //获取js列表
104 | var scripts = $("script");
105 | js = getJs(scripts);
106 | //获取css和部分js、html等
107 | var stylesheets = $("link");
108 | css = getCss(stylesheets);
109 | //合并所有链接
110 | var static_resource = js.concat(css)
111 | for (var index = 0; index < static_resource.length; index++) {
112 | var resource = get_resource(static_resource[index]);
113 | if (resource != '') {
114 | links = findMatch(linkReg, resource);
115 | // links = remove_pictures(links)
116 | links = Array.from(new Set(links))
117 | all_links = all_links.concat(links)
118 | // console.log(all_links)
119 | key = `${static_resource[index]}`;
120 | result.json[key] = links;
121 | } else {
122 | console.log("crawl static resource failed.............")
123 | }
124 | }
125 | //从动态加载后的结果（html）中提取
126 | links = findMatch(linkReg, html);
127 | links = Array.from(new Set(links))
128 | all_links = all_links.concat(links)
129 | all_links = Array.from(new Set(all_links))
130 | // console.log(all_links.length)
131 | var all_links_without_pics = []
132 |
133 | function remove_pictures(all_links) {
134 | for (var index = 0; index < all_links.length; index++) {
135 | if (/\.(png|jpg|gif)$/g.test(all_links[index]) || /\.(png|jpg|gif)\?.*/g.test(all_links[index])) {
136 | all_links.remove(index)
137 | // all_links.splice(index, 1);
138 | console.log(all_links[index]);
139 | } else {
140 | all_links_without_pics.push(all_links[index])
141 | }
142 | }
143 | return all_links_without_pics
144 | }
145 | all_links = remove_pictures(all_links)
146 | all_links = Array.from(new Set(all_links))
147 | // console.log(all_links.length);
148 | // console.log(all_links);
149 | // console.log(all_links_without_pics.length);
150 | // console.log(all_links_without_pics);
151 | key = `${options.uri}`
152 | result.json[key] = links;
153 | result.json.links = all_links;
154 |
155 | // var meta = $("meta");
156 | // console.log(meta.html());
157 | db.get('sites')
158 | .push(result)
159 | .write()
160 | })
161 | .catch((error) => {
162 | console.error('sensitive info Search failed:', error);
163 | });
164 | };
165 | // scan({
166 | // uri: 'map.qq.com/'
167 | // })
168 |
169 | module.exports.scan = scan;
--------------------------------------------------------------------------------
/webinfo.json:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/MichaelWayneLIU/InfoScraper/da631aec9f8d1a3a03665e3e871d75b1beb2d23f/webinfo.json
--------------------------------------------------------------------------------