*/
172 | if (stack[length - 1].tokenType === lexer_1.TOKEN_CONTENT_TEXT /*contentText*/) {
173 | if ((lexer.sourceCode[0] === "<" &&
174 | lexer_1.regexName.test(lexer.sourceCode[1])) /*contentText
*/ ||
175 | lexer.sourceCode.slice(0, 2) === " || */) {
177 | return false;
178 | }
179 | else {
180 | /*contentText
*/
181 | if ((lexer.sourceCode[0] === "<"
182 | && lexer_1.regexName.test(lexer.sourceCode[1]))) {
183 | let parseRes = Html_1.parseHtml(lexer);
184 | if (parseRes.selfClose) {
185 | return false;
186 | }
187 | }
188 | return true;
189 | }
190 | }
191 | // return true
192 | throw new Error(`not find contentEnd! at line ${lexer.GetLineNum()} ${lexer.sourceCode.slice(0, 100)}`);
193 | }
194 | function parseText(lexer) {
195 | lexer.hasCache = false;
196 | let node = new Node();
197 | if (!lexer.check) {
198 | node.nextSibling = null;
199 | }
200 | // lexer.isIgnored();
201 | node.LineNum = lexer.GetLineNum();
202 | let content = "";
203 | while (contentEnd(lexer) && !lexer.isEmpty()) {
204 | if (lexer.nextSourceCodeIs("\r\n") || lexer.nextSourceCodeIs("\n\r")) {
205 | lexer.lineNum += 1;
206 | content += lexer.sourceCode.slice(0, 2);
207 | lexer.skipSourceCode(2);
208 | }
209 | else {
210 | if (lexer.isNewLine(lexer.sourceCode[0])) {
211 | lexer.lineNum += 1;
212 | content += lexer.sourceCode[0];
213 | lexer.skipSourceCode(1);
214 | }
215 | else {
216 | content += lexer.sourceCode[0];
217 | lexer.skipSourceCode(1);
218 | }
219 | }
220 | }
221 | if (lexer.stack.length >= 3 &&
222 | tagClose_1.isSpecialTag({ tag: lexer.stack[lexer.stack.length - 3].token })) {
223 | let token = lexer.stack[lexer.stack.length - 3].token;
224 | let tokenLen = ``.length;
225 | if (lexer.sourceCode.slice(0, tokenLen) === ``) {
226 | lexer.skipSourceCode(2);
227 | let res = { lineNum: lexer.lineNum, tokenType: lexer_1.TOKEN_CLOSE, token: "
34 | // lexer.isIgnored()
35 | if (isSpecialTag(node)) {
36 | return null;
37 | }
38 | return node;
39 | }
40 | exports.parseClose = parseClose;
41 |
--------------------------------------------------------------------------------
/mycheck/check-dist.js:
--------------------------------------------------------------------------------
1 | const fs = require('fs')
2 | const path = require('path')
3 | const paser = require('../dist/index').paser
4 |
5 | let paths = [
6 | "demo.html",
7 | "demo1.html",
8 | "demo2.html",
9 | "demo3.html",
10 | "demo4.html",
11 | "demo5.html",
12 | "test4.html",
13 | "test5.html",
14 | "test6.html",
15 | "test7.html",
16 | "test8.html",
17 | "MDN_HTML.html",
18 | "MDN_JavaScript.html",
19 | "CSDN.html",
20 | "CSDN_SPM.html", // error
21 | "test9.html",
22 | "test10.html",
23 | "test11.html",
24 | "test12.html",
25 | "test13.html",
26 | "test14.html",
27 | "test15.html",
28 | "test16.html",
29 | "test17.html",
30 | "test18.html",
31 | "test19.html",
32 | "test20.html",
33 | "test21.html",
34 | "test22.html",
35 | "test23.html",
36 | "google.html",
37 | "005055fd7e2625aba5e8d2d370ea4914a152fe50d16620f896cdf4b1a68ba741.html",
38 | "005055fd7e2625aba5e8d2d370ea4914a152fe50d16620f896cdf4b1a68ba741-origin.html",
39 | "039c4b966d1f2a0c589ac0aad211fe65500ad1cb58c7f45b34251db7056803ec-origin.html",
40 | "0475e5eeadaaca857eea3f36d0eda01937fe672d48be7f98ba6bc7f25ecd63d0.html",
41 | "078cdb456d1beb698aeed86e0f2161e442e9431c4580295f1ba4ece22741068c.html",
42 | "0e55dcdbeb54c88ee87942b9fef7ea5398fa9a1e83493d55844b479506a80fd8.html",
43 | "qidian.html",
44 | "test24.html",
45 | "test25.html",
46 | "test26.html",
47 | ]
48 |
49 | for (let p of paths) {
50 | let code = fs.readFileSync(path.resolve(__dirname, `../demo/${p}`), {encoding: 'utf-8'})
51 | // console.log(code, 'code')
52 | if (code.length > 0) {
53 | console.time("test")
54 | let ast = paser(code)
55 | console.timeEnd("test")
56 | // console.log(__dirname, __filename)
57 | console.log(path.resolve(__dirname, "../out/", `./${p}.ast.json`))
58 | fs.writeFileSync(path.resolve(__dirname, "../out/", `./${p}.ast.json`), JSON.stringify(ast, null, 4))
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/mycheck/check.ts:
--------------------------------------------------------------------------------
1 | const fs = require('fs')
2 | const path = require('path')
3 | import { paser } from '../src/index'
4 |
5 | let paths = [
6 | // "demo.html",
7 | // "demo1.html",
8 | // "demo2.html",
9 | // "demo3.html",
10 | // "demo4.html",
11 | // "demo5.html",
12 | // "test4.html",
13 | // "test5.html",
14 | // "test6.html",
15 | // "test7.html",
16 | // "test8.html",
17 | // "MDN_HTML.html",
18 | // "MDN_JavaScript.html",
19 | // "CSDN.html",
20 | // "CSDN_SPM.html",
21 | // "test9.html",
22 | // "test10.html",
23 | // "test11.html",
24 | // "test12.html",
25 | // "test13.html",
26 | // "test14.html",
27 | // "test15.html",
28 | // "test16.html",
29 | // "test17.html",
30 | // "test18.html",
31 | // "test19.html",
32 | // "test20.html",
33 | // "test21.html",
34 | // "test22.html",
35 | // "test23.html",
36 | // "google.html",
37 | // "005055fd7e2625aba5e8d2d370ea4914a152fe50d16620f896cdf4b1a68ba741.html",
38 | // "005055fd7e2625aba5e8d2d370ea4914a152fe50d16620f896cdf4b1a68ba741-origin.html",
39 | // "039c4b966d1f2a0c589ac0aad211fe65500ad1cb58c7f45b34251db7056803ec-origin.html",
40 | // "0475e5eeadaaca857eea3f36d0eda01937fe672d48be7f98ba6bc7f25ecd63d0.html",
41 | // "078cdb456d1beb698aeed86e0f2161e442e9431c4580295f1ba4ece22741068c.html",
42 | // "0e55dcdbeb54c88ee87942b9fef7ea5398fa9a1e83493d55844b479506a80fd8.html",
43 | // "qidian.html",
44 | // "test24.html",
45 | // "test25.html",
46 | // "test26.html",
47 | // "qidian1.html",
48 | // "test27.html",
49 | // "test28.html",
50 | // "test30.html",
51 | // "test31.html",
52 | // "test32.html",
53 | // "test33.html",
54 | // "test34.html",
55 | // "test35.html",
56 | // "test36.html",
57 | // "test38.html",
58 | // "test40.html",
59 | ]
60 |
61 | for (let p of paths) {
62 | let code = fs.readFileSync(path.resolve(__dirname, `../demo/${p}`), { encoding: 'utf-8' })
63 | // console.log(code, 'code')
64 | if (code.length > 0) {
65 | // console.time("test")
66 | let ast = paser(code, true)
67 | // console.timeEnd("test")
68 | // console.log(__dirname, __filename)
69 | console.log(path.resolve(__dirname, "../out/", `./${p}.ast.json`))
70 | fs.writeFileSync(path.resolve(__dirname, "../out/", `./${p}.ast.json`), JSON.stringify(ast, null, 4))
71 | }
72 | }
73 |
--------------------------------------------------------------------------------
/mycheck/checkfile.js:
--------------------------------------------------------------------------------
1 | const fs = require('fs')
2 | const path = require('path')
3 | const paser = require('../dist/index').paser
4 |
5 | const files = fs.readdirSync(path.resolve(__dirname, "../files"))
6 |
7 | for (let i = 10; i < 20; i++) {
8 | console.log("-----------------------------------------------------")
9 | console.log(files[i], ` ${i}`)
10 | let code = fs.readFileSync(path.resolve(__dirname, `../files/${
11 | files[i]
12 | }`), {encoding: 'utf-8'})
13 |
14 | console.time("test")
15 | let ast = paser(code)
16 | console.timeEnd("test")
17 |
18 | }
19 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "@liulinboyi/htmlparser",
3 | "main": "./dist/index.js",
4 | "files": [
5 | "dist"
6 | ],
7 | "devDependencies": {
8 | "@playwright/test": "^1.13.1",
9 | "@types/node": "^14.14.20",
10 | "cross-env": "^7.0.3",
11 | "playwright": "^1.13.1",
12 | "prettier": "2.2.1",
13 | "pretty-quick": "3.1.0",
14 | "request": "^2.88.2",
15 | "request-promise": "^4.2.6",
16 | "ts-lint": "4.5.1",
17 | "ts-loader": "8.0.15",
18 | "ts-node": "^9.1.1",
19 | "tslint": "5.20.1",
20 | "tslint-config-prettier": "1.18.0",
21 | "typescript": "^4.1.3"
22 | },
23 | "scripts": {
24 | "test-ts": "ts-node ./mycheck/check.ts",
25 | "build": "rm -rf ./dist/ && tsc --project ./tsconfig.json",
26 | "build-es": "rm -rf ./dist-esmodule/ && tsc --project ./tsconfig-esmodule.json && node ./script/addSuffixJs.js",
27 | "build-all": "npm run build && npm run build-es",
28 | "test": "playwright test testall.spec.ts",
29 | "test-all": "node test-server.js"
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/script/addSuffixJs.js:
--------------------------------------------------------------------------------
1 | const fs = require("fs").promises
2 | const path = require("path")
3 |
4 | void async function () {
5 | try {
6 | let p = path.resolve(__dirname, "../dist-esmodule")
7 | let paths = await fs.readdir(p)
8 | // console.log(paths)
9 | let stack = [... paths]
10 | while (stack.length) {
11 | let top = stack.pop()
12 | let pat = path.resolve(p, top)
13 | let stat = await fs.stat(pat)
14 | if (stat.isDirectory()) {
15 | let temp = await fs.readdir(pat)
16 | if (temp) {
17 | for (let i of temp) {
18 | stack.push(path.join(top, i))
19 | }
20 | }
21 | } else {
22 | // console.log(pat)
23 |
24 | let personList = await fs.readFile(pat, {encoding: "utf8"})
25 |
26 | var regexpNames = /(?:export|import)(?:\s)*?(?:\{)??.*?(?:\})??(?:\s)*?from(?:\s)*?"(.+?)"/gm
27 |
28 | var match = personList.matchAll(regexpNames);
29 |
30 | let count = 0
31 | for (let item of match) {
32 | if (/.js$/.test(item[1])) {
33 | continue
34 | }
35 | let temp = item[0]
36 | let index = item.index + count
37 | let now = temp.replace(item[1], `${
38 | item[1]
39 | }.js`)
40 | let past = personList.slice(0, index)
41 | let feature = personList.slice(index + temp.length, personList.length)
42 | personList = `${past}${now}${feature}`
43 | count = count + 3
44 | }
45 |
46 | await fs.writeFile(pat, personList, {encoding: "utf8"})
47 | }
48 | }
49 | // for (let item of paths) {
50 | // let pat = path.resolve(p, item)
51 | // console.log(pat)
52 | // let stat = await fs.stat(pat)
53 | // console.log(stat.isDirectory())
54 | // }
55 | } catch (error) {
56 | console.log(error)
57 | }
58 |
59 | }()
60 |
--------------------------------------------------------------------------------
/server.js:
--------------------------------------------------------------------------------
1 | let http = require('http');
2 | let url = require('url');
3 | let util = require('util');
4 | let fs = require('fs');
5 | let path = require("path")
6 | const {spawn} = require('child_process');
7 | const process = require("process");
8 | const { URL, URLSearchParams } = require('url');
9 |
10 | const pour = (cmd, args, opts = {
11 | encoding: 'utf8'
12 | }, stdout = process.stdout, stderr = process.stderr) => {
13 | return new Promise((resolve, reject) => {
14 | const p = spawn(cmd, args, opts);
15 | p.stdout.setEncoding('utf-8');
16 | p.stdout.on('data', data => {
17 | stdout.write(data, "utf8");
18 | });
19 | p.stderr.on('data', data => {
20 | stderr.write(data);
21 | });
22 | p.on('close', code => {
23 | resolve(code);
24 | });
25 | });
26 | }
27 |
28 | async function exec(shell, args, opt) {
29 | console.log(`${shell} ${
30 | args.join(" ")
31 | }`)
32 | await pour(shell, args, opt);
33 | }
34 |
35 | let server = http.createServer((req, res) => {
36 | if (path.normalize(decodeURIComponent(req.url)) !== decodeURIComponent(req.url)) {
37 | res.statusCode = 403;
38 | res.end();
39 | return;
40 | }
41 | const u = decodeURIComponent(req.url)
42 | console.log("decodeURIComponent",u)
43 | // var pathname = url.parse(u).pathname; // 获取url的pathname (/index.html)
44 | console.log("file:" + u.substring(1)) // 将‘/’去掉
45 | // console.log(__dirname, __filename)
46 | let curPath = path.join(__dirname, u.substring(1))
47 | console.log(curPath)
48 | fs.readFile(curPath, function (err, data) { // fs模块加载文件
49 | if (err) {
50 | res.writeHead(404, {'Content-Type': 'text/html'});
51 | } else {
52 | res.writeHead(200, {'Content-Type': 'text/html'});
53 | res.write(data.toString());
54 | }
55 | res.end();
56 | });
57 |
58 | });
59 |
60 | server.listen(3000, '127.0.0.1', async () => {
61 | console.log("服务器已经运行，请打开浏览,输入:http://127.0.0.1:3000/ 来进行访问.")
62 | // await exec(process.platform === 'win32' ? 'npm.cmd' : "npm", ["run", "test"])
63 | // process.exit(0);
64 | });
65 |
--------------------------------------------------------------------------------
/src/definition.ts:
--------------------------------------------------------------------------------
1 |
2 | export interface Keywords {
3 | [index: string]: any
4 | }
5 |
6 | export interface TokenNameMap {
7 | [index: number]: any
8 | }
9 |
--------------------------------------------------------------------------------
/src/index.ts:
--------------------------------------------------------------------------------
1 | export { parse as paser } from "./parser"
2 |
--------------------------------------------------------------------------------
/src/parser.ts:
--------------------------------------------------------------------------------
1 | import { COMMENT, DIRECTIVE, Lexer, NewLexer, TOKEN_CLOSE, TOKEN_CONTENT_TEXT, TOKEN_DTD, TOKEN_EOF, TOKEN_IGNORED, TOKEN_LEFT_PAREN } from "./lexer"
2 | import { paseComment } from "./parser/Comment"
3 | import { paseDirective } from "./parser/Directive"
4 | import { parseDtd } from "./parser/DTD"
5 | import { parseHtml } from "./parser/Html"
6 | import { parseText } from "./parser/parseText"
7 | import { isSpecialTag, parseClose } from "./parser/tagClose"
8 |
9 | export interface Program {
10 | type?: string,
11 | LineNum?: number,
12 | children: Array
,
13 | }
14 |
15 | export class Program {
16 | constructor() {
17 | this.type = 'root'
18 | this.children = []
19 | }
20 | }
21 |
22 |
23 | // SourceCode ::= Statement+
24 | function parseSourceCode(lexer: Lexer, check: boolean) {
25 | let LineNum = lexer.GetLineNum()
26 | let root = parseStatements(lexer, check)
27 | root.LineNum = LineNum
28 | return root
29 | }
30 |
31 | /**
32 | * 将children中的多余的text节点去除
33 | * @param children
34 | * @returns
35 | */
36 | function filterText(children: any) {
37 | for (let start = 0; start < children.length; start++) {
38 | if (children[start].type === "text") {
39 | // 从实践中知道，如果有去除body后多余的text节点，则最多是两个取一个，所以有下面代码
40 | let i = start + 1
41 | if (i < children.length && children[i].type === "text") {
42 | // 其中重要的特征就是，里面是只有\r\n和空格
43 | // 只要当前标签和下一个标签这两个标签，则一定会删除一个"空标签(只包含\r\n和空格)"
44 | if (!children[i].content.replace(/[\r\n]+/g, "").trim()) {
45 | children[i].delete = true // 添加上delete属性，后面好处理
46 | } else {
47 | children[start].delete = true // 添加上delete属性，后面好处理
48 | }
49 | }
50 | }
51 | }
52 | // 删除delete为true的标签
53 | return children.filter((item: any) => !item.delete)
54 | }
55 |
56 | // Statement
57 | function parseStatements(lexer: Lexer, check: boolean) {
58 |
59 | if (check) {
60 | lexer.check = true
61 | }
62 |
63 | let root: any = {
64 | type: "root",
65 | children: [],
66 | LineNum: 1
67 | }
68 |
69 | let statements: Array = [root]
70 |
71 | let Block_level_elements = [ // 块级元素
72 | "address",
73 | "article",
74 | "aside",
75 | "audio",
76 | "blockquote",
77 | "canvas",
78 | "dd",
79 | // "div",
80 | "dl",
81 | "fieldset",
82 | "figcaption",
83 | "figure",
84 | "figcaption",
85 | "footer",
86 | "form",
87 | "header",
88 | "hgroup",
89 | "hr",
90 | "noscript",
91 | "ol",
92 | "output",
93 | "p",
94 | "pre",
95 | "section",
96 | "table",
97 | "tfoot",
98 | "ul",
99 | "video"
100 | ]
101 |
102 | let inlInline_elementsine = [ // 行内元素
103 | "b",
104 | "big",
105 | "i",
106 | "small",
107 | "tt",
108 | "abbr",
109 | "acronym",
110 | "cite",
111 | "code",
112 | "dfn",
113 | "em",
114 | "kbd",
115 | "strong",
116 | "samp",
117 | "var",
118 | "a",
119 | "bdo",
120 | "br",
121 | "img",
122 | "map",
123 | "object",
124 | "q",
125 | "script",
126 | "span",
127 | "sub",
128 | "sup",
129 | "button",
130 | "input",
131 | "label",
132 | "select",
133 | "textarea"
134 | ]
135 |
136 | let notInSelf = [ // 不能包含自己的元素
137 | "a",
138 | "br",
139 | "img",
140 | "script",
141 | "button",
142 | "input",
143 | ]
144 | // select ,
6 | attr: Array,
7 | type?: string,
8 | tag?: string,
9 | selfClose?: boolean,
10 | parent?: any,
11 | }
12 |
13 | let temp = Symbol("temp")
14 | let nextSibling = temp
15 |
16 | export class Node {
17 | constructor() {
18 | this.children = []
19 | this.attr = []
20 | }
21 | get nextSibling() {
22 | if (nextSibling !== temp) return nextSibling
23 | if (!this.parent) return null
24 | let lengtn = this.parent.children.length
25 | let index = -1
26 | for (let item of this.parent.children) {
27 | index++
28 | if (item === this) {
29 | break
30 | }
31 | }
32 | if (index + 1 > lengtn) {
33 | return null
34 | }
35 | return this.parent.children[index + 1]
36 | }
37 |
38 | set nextSibling(value: any) {
39 | nextSibling = value
40 | }
41 | }
42 |
43 | export function parseTag(lexer: Lexer) {
44 | return lexer.NextTokenIs(TOKEN_NAME).nowToken; // tag_name
45 | }
46 |
47 | export function parseName(lexer: Lexer, node: any) {
48 | let attrReg = /[^\s"'>/=[\u0000-\u001f]+/.exec(lexer.sourceCode)
49 | let name = ""
50 | if (attrReg) {
51 | name = attrReg[0]
52 | }
53 | if (name.includes("<")) {
54 | /*
55 |
57 |
58 |
59 |
60 | 暂时当做selfClose标签处理
61 | */
62 | node.selfClose = true
63 | }
64 | lexer.skipSourceCode(name.length)
65 | return name
66 | }
67 |
68 | function genereteAttr(name: any, value?: any) {
69 | return {
70 | name,
71 | value,
72 | }
73 | }
74 |
75 | export function parseValue(lexer: Lexer) {
76 | if (lexer.sourceCode[0] === "'") {
77 | return parseSingleQuotedAttr(lexer)
78 | } else if (lexer.sourceCode[0] === '"') {
79 | return parseDoubleQuotedAttr(lexer)
80 | } else {
81 | return parseString(lexer)
82 | }
83 | }
84 |
85 | export function parseString(lexer: Lexer) {
86 | let value = ""
87 | // lexer.NextTokenIs(TOKEN_SINGLE_QUOTE);
88 | // lexer.stack.pop()
89 | let res = /[^\s><]*/.exec(lexer.sourceCode)
90 | if (res) {
91 | value = res[0]
92 | }
93 | lexer.skipSourceCode(value.length)
94 | // lexer.NextTokenIs(TOKEN_SINGLE_QUOTE);
95 | // lexer.stack.pop()
96 | return value
97 | }
98 |
99 | export function parseSingleQuotedAttr(lexer: Lexer) {
100 | let value = ""
101 | lexer.NextTokenIs(TOKEN_SINGLE_QUOTE);
102 | lexer.stack.pop()
103 | let res = /[^']*/.exec(lexer.sourceCode)
104 | if (res) {
105 | value = res[0]
106 | }
107 | lexer.skipSourceCode(value.length)
108 | lexer.NextTokenIs(TOKEN_SINGLE_QUOTE);
109 | lexer.stack.pop()
110 | return value
111 | }
112 |
113 | export function parseDoubleQuotedAttr(lexer: Lexer) {
114 | let value = ""
115 | lexer.NextTokenIs(TOKEN_QUOTE);
116 | lexer.stack.pop()
117 | let res = /[^"]*/.exec(lexer.sourceCode)
118 | if (res) {
119 | value = res[0]
120 | }
121 | lexer.skipSourceCode(value.length)
122 | lexer.NextTokenIs(TOKEN_QUOTE);
123 | lexer.stack.pop()
124 | return value
125 | }
126 |
127 | export function parseAttr(lexer: Lexer, node: any) {
128 | let attrItem: {
129 | name?: string,
130 | value?: string,
131 | } = {}
132 | lexer.isIgnored(); // 空格
133 | let tag = parseName(lexer, node)
134 | lexer.isIgnored() // 空格
135 | if (tag) {
136 | let attr = tag
137 | attrItem = genereteAttr(attr); // name
138 | lexer.isIgnored(); // 空格
139 | if (lexer.sourceCode[0] === "=") {
140 | lexer.NextTokenIs(TOKEN_EQUAL) // =
141 | lexer.stack.pop()
142 | lexer.isIgnored(); // 空格
143 | attrItem.value = parseValue(lexer)
144 | lexer.isIgnored(); // 空格
145 | } else {
146 | attrItem.value = "true"
147 | lexer.isIgnored(); // 空格
148 | }
149 | }
150 | return attrItem
151 | }
152 |
153 | function checkAttrEnd(lexer: Lexer, node: Node) {
154 | if (lexer.sourceCode[0] === ">") {
155 | lexer.skipSourceCode(1)
156 | lexer.stack.push({ lineNum: lexer.lineNum, tokenType: TOKEN_RIGHT_PAREN /*>*/, token: ">" })
157 | return false
158 | } else if (lexer.sourceCode.slice(0, 2) === "/>") {
159 | node.selfClose = true
160 | lexer.skipSourceCode(2);
161 | lexer.stack.push({ lineNum: lexer.lineNum, tokenType: TOKEN_SELF_CLOSE /*/> ,
9 | content: string,
10 | type?: string,
11 | selfClose?: boolean,
12 | parent?: any,
13 | }
14 |
15 | let temp = Symbol("temp")
16 | let nextSibling = temp
17 |
18 | export class Node {
19 | constructor() {
20 | this.content = ""
21 | }
22 | get nextSibling() {
23 | if (nextSibling !== temp) return nextSibling
24 | if (!this.parent) return null
25 | let lengtn = this.parent.children.length
26 | let index = -1
27 | for (let item of this.parent.children) {
28 | index++
29 | if (item === this) {
30 | break
31 | }
32 | }
33 | if (index + 1 > lengtn) {
34 | return null
35 | }
36 | return this.parent.children[index + 1]
37 | }
38 |
39 | set nextSibling(value: any) {
40 | nextSibling = value
41 | }
42 | }
43 |
44 | export function isClose(lexer: Lexer) {
45 | const length = lexer.stack.length
46 | const topTwo = length >= 2 ? lexer.stack[length - 2].tokenType : ""
47 |
48 | const isTOKEN_DTD = topTwo === TOKEN_DTD
49 | const isCOMMENT = topTwo === COMMENT
50 | if (length >= 2 &&
51 | (isTOKEN_DTD /*dtd*/ ||
52 | isCOMMENT /*comment*/)
53 | ) {
54 | return true
55 | }
56 |
57 | if (length < 4) return false
58 |
59 | const topThree = lexer.stack[length - 3].tokenType
60 | const topFour = lexer.stack[length - 4].tokenType
61 | const isTOKEN_RIGHT_PAREN = topTwo === TOKEN_RIGHT_PAREN
62 | const isTOKEN_NAME = topThree === TOKEN_NAME
63 | //
64 | let one = isTOKEN_RIGHT_PAREN /*>*/ &&
65 | isTOKEN_NAME /*tag_name*/ &&
66 | topFour === TOKEN_CLOSE /*
68 | let close = isTOKEN_RIGHT_PAREN /*>*/ &&
69 | topThree === TOKEN_NAME /*tag_name*/ &&
70 | topFour === TOKEN_LEFT_PAREN /*<*/;
71 | // />
72 | let selfClose = topTwo === TOKEN_SELF_CLOSE; // /> contentText
101 | // contentText
102 | //
contentText
103 | //
contentText
||
contentText
contentText
143 | //
contentText
144 | //
contentText
145 | //
155 | //
157 | //
169 | // contentText
170 | // contentText
171 | // contentText
179 | // contentText
180 | // contentText
181 | // contentText*/
189 | /*contentText || */
191 | /*contentText
*/
192 | if (stack[length - 1].tokenType === TOKEN_CONTENT_TEXT /*contentText*/) {
193 | if ((lexer.sourceCode[0] === "<" &&
194 | regexName.test(lexer.sourceCode[1])) /*contentText
*/ ||
195 | lexer.sourceCode.slice(0, 2) === " || */
197 | ) {
198 | return false
199 | } else {
200 | /*contentText
*/
201 | if ((lexer.sourceCode[0] === "<"
202 | && regexName.test(lexer.sourceCode[1]))) {
203 | let parseRes = parseHtml(lexer)
204 | if (parseRes.selfClose) {
205 | return false
206 | }
207 | }
208 | return true
209 | }
210 | }
211 |
212 |
213 | // return true
214 | throw new Error(`not find contentEnd! at line ${lexer.GetLineNum()} ${lexer.sourceCode.slice(0, 100)}`)
215 |
216 | }
217 |
218 | export function parseText(lexer: Lexer) {
219 | lexer.hasCache = false
220 | let node = new Node()
221 | if (!lexer.check) {
222 | node.nextSibling = null
223 | }
224 |
225 | // lexer.isIgnored();
226 | node.LineNum = lexer.GetLineNum()
227 |
228 | let content = ""
229 | while (contentEnd(lexer) && !lexer.isEmpty()) {
230 | if (lexer.nextSourceCodeIs("\r\n") || lexer.nextSourceCodeIs("\n\r")) {
231 | lexer.lineNum += 1
232 | content += lexer.sourceCode.slice(0, 2)
233 | lexer.skipSourceCode(2)
234 | } else {
235 | if (lexer.isNewLine(lexer.sourceCode[0])) {
236 | lexer.lineNum += 1
237 | content += lexer.sourceCode[0]
238 | lexer.skipSourceCode(1)
239 | } else {
240 | content += lexer.sourceCode[0]
241 | lexer.skipSourceCode(1)
242 | }
243 | }
244 | }
245 |
246 | if (
247 | lexer.stack.length >= 3 &&
248 | isSpecialTag({ tag: lexer.stack[lexer.stack.length - 3].token })) {
249 | let token = lexer.stack[lexer.stack.length - 3].token
250 | let tokenLen = ``.length
251 | if (lexer.sourceCode.slice(0, tokenLen) === ``) {
252 | lexer.skipSourceCode(2)
253 | let res = { lineNum: lexer.lineNum, tokenType: TOKEN_CLOSE, token: ",
7 | attr: Array
,
8 | type?: string,
9 | tag?: string,
10 | selfClose?: boolean
11 | closeTag?: boolean
12 | }
13 |
14 | export class Node {
15 | constructor() {
16 | this.children = []
17 | this.attr = []
18 | }
19 | }
20 |
21 | export function isSpecialTag(node: any) {
22 | let tags = [
23 | "img",
24 | "source",
25 | "link",
26 | "meta",
27 | "area",
28 | "input",
29 | "br"
30 | ]
31 | return tags.includes(node.tag)
32 | }
33 |
34 | export function parseClose(lexer: Lexer) {
35 | lexer.hasCache = false
36 | let node = new Node()
37 | node.closeTag = true
38 | node.LineNum = lexer.GetLineNum()
39 | node.type = "tag"
40 | node.tag = parseTag(lexer)
41 | lexer.NextTokenIs(TOKEN_RIGHT_PAREN) // >
42 | // lexer.isIgnored()
43 | if (isSpecialTag(node)) {
44 | return null
45 | }
46 | return node
47 | }
--------------------------------------------------------------------------------
/test-server.js:
--------------------------------------------------------------------------------
1 | let http = require('http');
2 | let url = require('url');
3 | let util = require('util');
4 | let fs = require('fs');
5 | let path = require("path")
6 | const {spawn} = require('child_process');
7 | const process = require("process");
8 |
9 | const pour = (cmd, args, opts = {
10 | encoding: 'utf8'
11 | }, stdout = process.stdout, stderr = process.stderr) => {
12 | return new Promise((resolve, reject) => {
13 | const p = spawn(cmd, args, opts);
14 | p.stdout.setEncoding('utf-8');
15 | p.stdout.on('data', data => {
16 | stdout.write(data, "utf8");
17 | });
18 | p.stderr.on('data', data => {
19 | stderr.write(data);
20 | });
21 | p.on('close', code => {
22 | resolve(code);
23 | });
24 | });
25 | }
26 |
27 | async function exec(shell, args, opt) {
28 | console.log(`${shell} ${
29 | args.join(" ")
30 | }`)
31 | await pour(shell, args, opt);
32 | }
33 |
34 | let server = http.createServer((req, res) => {
35 | var pathname = url.parse(req.url).pathname; // 获取url的pathname (/index.html)
36 | // console.log("file:" + pathname.substring(1)) // 将‘/’去掉
37 | // console.log(__dirname, __filename)
38 | let curPath = path.join(__dirname, pathname)
39 | // console.log(curPath)
40 | fs.readFile(curPath, function (err, data) { // fs模块加载文件
41 | if (err) {
42 | res.writeHead(404, {'Content-Type': 'text/html'});
43 | } else {
44 | res.writeHead(200, {'Content-Type': 'text/html'});
45 | res.write(data.toString());
46 | }
47 | res.end();
48 | });
49 |
50 | });
51 |
52 | server.listen(3000, '127.0.0.1', async () => {
53 | console.log("服务器已经运行，请打开浏览,输入:http://127.0.0.1:3000/ 来进行访问.")
54 | await exec(process.platform === 'win32' ? 'npm.cmd' : "npm", ["run", "test"])
55 | process.exit(0);
56 | });
57 |
--------------------------------------------------------------------------------
/test/test24.spec.ts:
--------------------------------------------------------------------------------
1 | import { test, expect } from '@playwright/test';
2 | import { webkit } from 'playwright'
3 | const request = require("request-promise")
4 | import { parse } from '../src/parser'
5 |
6 | function getAll(source, type) {
7 | let current = source;
8 | let res = []
9 | let stack = Array.isArray(current) ? current : [current];
10 | while (stack.length) {
11 | let top = stack.pop();
12 | // console.log([top])
13 | if (type === "browser") {
14 | if (top.nodeName === "#comment") {
15 | continue
16 | }
17 | if (top.nodeName === "#text" && /[\r\n]+/.test(top.data)) {
18 | continue
19 | }
20 | res.push({ tag: top.tagName ? top.tagName : "text", content: top.data })
21 | } else if (type === "parser") {
22 | if (top.type === "comment") {
23 | continue
24 | }
25 | if (top.type === "text" && /[\r\n]+/.test(top.content)) {
26 | continue
27 | }
28 | res.push({ tag: top.tag ? top.tag : "text", content: top.content })
29 | }
30 | if (top.childNodes) {
31 | stack.push(...top.childNodes)
32 | } else if (top.children) {
33 | stack.push(...top.children)
34 | }
35 | }
36 | return res
37 | }
38 |
39 | test('test24.html', async (/*{ page }*/) => {
40 | // encodeURIComponent
41 | let url = "http://127.0.0.1:3000/demo/test24.html"
42 | const web = await webkit.launch();
43 | const context = await web.newContext({
44 | javaScriptEnabled: false
45 | });
46 | const newpage = await context.newPage();
47 | await newpage.goto(url, {
48 | referer: "",
49 | // timeout: 30,
50 | waitUntil: "domcontentloaded"
51 | });
52 |
53 | await newpage.evaluateHandle(`document.body.classList.add("body")`)
54 |
55 | // const name = await page.innerText('title');
56 | // expect(name).toBe('Document');
57 | // const elementHandle = await page.$('body');
58 | // console.log(elementHandle)
59 | // const bodyElement = elementHandle.asElement()
60 | // console.log(bodyElement)
61 |
62 | // const aHandle = await page.evaluateHandle('document');
63 | // console.log(aHandle)
64 |
65 | // const aHandle = await page.evaluateHandle(() => document.body);
66 | // let doc = await aHandle.jsonValue()
67 | // console.log(doc)
68 | // const resultHandle = await page.evaluateHandle(body => body.innerHTML, aHandle);
69 | // const jsonValue = await resultHandle.jsonValue()
70 | // console.log(jsonValue);
71 | // await resultHandle.dispose();
72 |
73 | const aHandle = await newpage.evaluateHandle(`(${getAll.toString()})(Array.from(document.childNodes),"browser")`);
74 | // console.log(aHandle)
75 | const browser: any = await aHandle.jsonValue()
76 |
77 | let html = await request({
78 | method: "GET",
79 | uri: url,
80 | headers: {
81 | "Accept": "*/*",
82 | "Accept-Encoding": "deflate, br",
83 | "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
84 | "Connection": "keep-alive",
85 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36 Edg/91.0.864.59",
86 | },
87 | })
88 |
89 | let ast = parse(html)
90 |
91 | // let body = search(ast, "body")
92 |
93 | let parser = getAll(ast.children, "parser")
94 |
95 | // console.log(browser, parser)
96 | let count = browser.length > parser.length ? browser.length : parser.length;
97 | for (let i = 0; i < count; i++) {
98 | // console.log(browser[i] ? browser[i].tag : "undefined", parser[i] ? parser[i].tag : "undefined")
99 | expect(browser[i].tag.toLowerCase()).toBe(parser[i].tag)
100 | // console.assert(browser[i].tag.toLowerCase() === parser[i].tag, `${browser[i] ? browser[i].tag : "undefined"}, ${parser[i] ? parser[i].tag : "undefined"}`)
101 | }
102 | });
--------------------------------------------------------------------------------
/test/test25.spec.ts:
--------------------------------------------------------------------------------
1 | import { test, expect } from '@playwright/test';
2 | const request = require("request-promise")
3 | import { parse } from '../src/parser'
4 |
5 | function getAll(source, type) {
6 | let current = source;
7 | let res = []
8 | let stack = Array.isArray(current) ? current : [current];
9 | while (stack.length) {
10 | let top = stack.pop();
11 | // console.log([top])
12 | if (type === "browser") {
13 | if (top.nodeName === "#comment") {
14 | continue
15 | }
16 | if (top.nodeName === "#text" && /[\r\n]+/.test(top.data)) {
17 | continue
18 | }
19 | res.push({ tag: top.tagName ? top.tagName : "text", content: top.data })
20 | } else if (type === "parser") {
21 | if (top.type === "comment") {
22 | continue
23 | }
24 | if (top.type === "text" && /[\r\n]+/.test(top.content)) {
25 | continue
26 | }
27 | res.push({ tag: top.tag ? top.tag : "text", content: top.content })
28 | }
29 | if (top.childNodes) {
30 | stack.push(...top.childNodes)
31 | } else if (top.children) {
32 | stack.push(...top.children)
33 | }
34 | }
35 | return res
36 | }
37 |
38 | test('test25.html', async ({ page }) => {
39 | // encodeURIComponent
40 | let url = "http://127.0.0.1:3000/demo/test25.html"
41 | await page.goto(url, {
42 | referer: "",
43 | // timeout: 30,
44 | waitUntil: "domcontentloaded"
45 | });
46 |
47 | await page.evaluateHandle(`document.body.classList.add("body")`)
48 |
49 | // const name = await page.innerText('title');
50 | // expect(name).toBe('Document');
51 | // const elementHandle = await page.$('body');
52 | // console.log(elementHandle)
53 | // const bodyElement = elementHandle.asElement()
54 | // console.log(bodyElement)
55 |
56 | // const aHandle = await page.evaluateHandle('document');
57 | // console.log(aHandle)
58 |
59 | // const aHandle = await page.evaluateHandle(() => document.body);
60 | // let doc = await aHandle.jsonValue()
61 | // console.log(doc)
62 | // const resultHandle = await page.evaluateHandle(body => body.innerHTML, aHandle);
63 | // const jsonValue = await resultHandle.jsonValue()
64 | // console.log(jsonValue);
65 | // await resultHandle.dispose();
66 |
67 | const aHandle = await page.evaluateHandle(`(${getAll.toString()})(Array.from(document.childNodes),"browser")`);
68 | // console.log(aHandle)
69 | const browser: any = await aHandle.jsonValue()
70 |
71 | let html = await request({
72 | method: "GET",
73 | uri: url,
74 | headers: {
75 | "Accept": "*/*",
76 | "Accept-Encoding": "deflate, br",
77 | "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
78 | "Connection": "keep-alive",
79 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36 Edg/91.0.864.59",
80 | },
81 | })
82 |
83 | let ast = parse(html)
84 |
85 | // let body = search(ast, "body")
86 |
87 | let parser = getAll(ast.children, "parser")
88 |
89 | // console.log(browser, parser)
90 | let count = browser.length > parser.length ? browser.length : parser.length;
91 | for (let i = 0; i < count; i++) {
92 | // console.log(browser[i] ? browser[i].tag : "undefined", parser[i] ? parser[i].tag : "undefined")
93 | expect(browser[i].tag.toLowerCase()).toBe(parser[i].tag)
94 | // console.assert(browser[i].tag.toLowerCase() === parser[i].tag, `${browser[i] ? browser[i].tag : "undefined"}, ${parser[i] ? parser[i].tag : "undefined"}`)
95 | }
96 | });
--------------------------------------------------------------------------------
/test/test26.spec.ts:
--------------------------------------------------------------------------------
1 | import { test, expect } from '@playwright/test';
2 | const request = require("request-promise")
3 | import { parse } from '../src/parser'
4 |
5 | function getAll(source, type) {
6 | let current = source;
7 | let res = []
8 | let stack = Array.isArray(current) ? current : [current];
9 | while (stack.length) {
10 | let top = stack.pop();
11 | // console.log([top])
12 | if (type === "browser") {
13 | if (top.nodeName === "#comment") {
14 | continue
15 | }
16 | if (top.nodeName === "#text" && /[\r\n]+/.test(top.data)) {
17 | continue
18 | }
19 | res.push({ tag: top.tagName ? top.tagName : "text", content: top.data })
20 | } else if (type === "parser") {
21 | if (top.type === "comment") {
22 | continue
23 | }
24 | if (top.type === "text" && /[\r\n]+/.test(top.content)) {
25 | continue
26 | }
27 | res.push({ tag: top.tag ? top.tag : "text", content: top.content })
28 | }
29 | if (top.childNodes) {
30 | stack.push(...top.childNodes)
31 | } else if (top.children) {
32 | stack.push(...top.children)
33 | }
34 | }
35 | return res
36 | }
37 |
38 | test('test26.html', async ({ page }) => {
39 | // encodeURIComponent
40 | let url = "http://127.0.0.1:3000/demo/test26.html"
41 | await page.goto(url, {
42 | referer: "",
43 | // timeout: 30,
44 | waitUntil: "domcontentloaded"
45 | });
46 |
47 | await page.evaluateHandle(`document.body.classList.add("body")`)
48 |
49 | // const name = await page.innerText('title');
50 | // expect(name).toBe('Document');
51 | // const elementHandle = await page.$('body');
52 | // console.log(elementHandle)
53 | // const bodyElement = elementHandle.asElement()
54 | // console.log(bodyElement)
55 |
56 | // const aHandle = await page.evaluateHandle('document');
57 | // console.log(aHandle)
58 |
59 | // const aHandle = await page.evaluateHandle(() => document.body);
60 | // let doc = await aHandle.jsonValue()
61 | // console.log(doc)
62 | // const resultHandle = await page.evaluateHandle(body => body.innerHTML, aHandle);
63 | // const jsonValue = await resultHandle.jsonValue()
64 | // console.log(jsonValue);
65 | // await resultHandle.dispose();
66 |
67 | const aHandle = await page.evaluateHandle(`(${getAll.toString()})(Array.from(document.childNodes),"browser")`);
68 | // console.log(aHandle)
69 | const browser: any = await aHandle.jsonValue()
70 |
71 | let html = await request({
72 | method: "GET",
73 | uri: url,
74 | headers: {
75 | "Accept": "*/*",
76 | "Accept-Encoding": "deflate, br",
77 | "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
78 | "Connection": "keep-alive",
79 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36 Edg/91.0.864.59",
80 | },
81 | })
82 |
83 | let ast = parse(html)
84 |
85 | // let body = search(ast, "body")
86 |
87 | let parser = getAll(ast.children, "parser")
88 |
89 | // console.log(browser, parser)
90 | let count = browser.length > parser.length ? browser.length : parser.length;
91 | for (let i = 0; i < count; i++) {
92 | // console.log(browser[i] ? browser[i].tag : "undefined", parser[i] ? parser[i].tag : "undefined")
93 | expect(browser[i].tag.toLowerCase()).toBe(parser[i].tag)
94 | // console.assert(browser[i].tag.toLowerCase() === parser[i].tag, `${browser[i] ? browser[i].tag : "undefined"}, ${parser[i] ? parser[i].tag : "undefined"}`)
95 | }
96 | });
--------------------------------------------------------------------------------
/test/testall.spec.ts:
--------------------------------------------------------------------------------
1 | import { test, expect } from '@playwright/test';
2 | import { chromium } from 'playwright'
3 | const request = require("request-promise")
4 | import { parse } from '../src/parser'
5 | const path = require("path")
6 | const fs = require("fs")
7 |
8 | function getAll(source, type) {
9 | let current = source;
10 | let res = []
11 | let stack = Array.isArray(current) ? current : [current];
12 | while (stack.length) {
13 | let top = stack.pop();
14 | // console.log([top])
15 | if (type === "browser") {
16 | if (top.nodeName === "#comment") {
17 | continue
18 | }
19 | if (top.nodeName === "#text" && /[\r\n]+/.test(top.data)) {
20 | continue
21 | }
22 | if (top.nodeName === "html") { // 浏览器中DocumentType的nodeName位html
23 | res.push({ tag: "DTD", content: null })
24 | continue
25 | }
26 | res.push({ tag: top.nodeName === "#text" ? "text" : top.nodeName, content: top.nodeName === "#text" ? top.data : null })
27 | } else if (type === "parser") {
28 | if (top.type === "comment") {
29 | continue
30 | }
31 | // top.content.replace(/[\r\n]+/g, "").trim()
32 | if (top.type === "text" && /[\r\n]+/.test(top.content)) {
33 | continue
34 | }
35 | if (top.type === "DTD") {
36 | res.push({ tag: "dtd", content: top.content, LineNum: top.LineNum })
37 | continue
38 | }
39 | res.push({ tag: top.tag ? top.tag : "text", content: top.content, LineNum: top.LineNum })
40 | }
41 | if (top.childNodes) {
42 | stack.push(...top.childNodes)
43 | } else if (top.children) {
44 | // debugger
45 | stack.push(...top.children)
46 | }
47 | }
48 | return res
49 | }
50 |
51 |
52 | let paths = [
53 | "demo.html",
54 | "demo1.html",
55 | "demo2.html",
56 | "demo3.html",
57 | "demo4.html",
58 | "demo5.html",
59 | "test4.html",
60 | "test5.html",
61 | "test6.html",
62 | "test7.html",
63 | "test8.html",
64 | "MDN_HTML.html",
65 | "MDN_JavaScript.html",
66 | "CSDN.html",
67 |
68 | // "CSDN_SPM.html",
69 |
70 | "test9.html",
71 | "test10.html",
72 | "test11.html",
73 | "test12.html",
74 | "test13.html",
75 | "test14.html",
76 | "test15.html",
77 | "test16.html",
78 | "test17.html",
79 | "test18.html",
80 | "test19.html",
81 | "test20.html",
82 | "test21.html",
83 | "test22.html",
84 | "test23.html",
85 | "google.html",
86 |
87 | // "005055fd7e2625aba5e8d2d370ea4914a152fe50d16620f896cdf4b1a68ba741.html",
88 | // "005055fd7e2625aba5e8d2d370ea4914a152fe50d16620f896cdf4b1a68ba741-origin.html",
89 | // "039c4b966d1f2a0c589ac0aad211fe65500ad1cb58c7f45b34251db7056803ec-origin.html",
90 | // ok
91 | "0475e5eeadaaca857eea3f36d0eda01937fe672d48be7f98ba6bc7f25ecd63d0.html",
92 | "078cdb456d1beb698aeed86e0f2161e442e9431c4580295f1ba4ece22741068c.html",
93 | "0e55dcdbeb54c88ee87942b9fef7ea5398fa9a1e83493d55844b479506a80fd8.html",
94 | "qidian.html",
95 | "test24.html",
96 | "test25.html",
97 | "test26.html",
98 | "qidian1.html",
99 | "test27.html",
100 | "test28.html",
101 | "test29.html",
102 | "test30.html",
103 | "test31.html",
104 | "test32.html",
105 | "test33.html",
106 | "test34.html",
107 | "test35.html",
108 | "test36.html",
109 | "test38.html",
110 | "test39.html",
111 | ]
112 |
113 | for (let item of paths) {
114 | test(`${item}`, async (/*{ page }*/) => {
115 | let url = `http://127.0.0.1:3000/demo/${item}`
116 | // console.log(url)
117 | const web = await chromium.launch();
118 | const context = await web.newContext({
119 | javaScriptEnabled: false
120 | });
121 | const newpage = await context.newPage();
122 | await newpage.goto(url, {
123 | referer: "",
124 | // timeout: 30,
125 | waitUntil: "domcontentloaded"
126 | });
127 |
128 | await newpage.evaluateHandle(`document.body.classList.add("body")`)
129 |
130 | // const name = await page.innerText('title');
131 | // expect(name).toBe('Document');
132 | // const elementHandle = await page.$('body');
133 | // console.log(elementHandle)
134 | // const bodyElement = elementHandle.asElement()
135 | // console.log(bodyElement)
136 |
137 | // const aHandle = await page.evaluateHandle('document');
138 | // console.log(aHandle)
139 |
140 | // const aHandle = await page.evaluateHandle(() => document.body);
141 | // let doc = await aHandle.jsonValue()
142 | // console.log(doc)
143 | // const resultHandle = await page.evaluateHandle(body => body.innerHTML, aHandle);
144 | // const jsonValue = await resultHandle.jsonValue()
145 | // console.log(jsonValue);
146 | // await resultHandle.dispose();
147 |
148 | const aHandle = await newpage.evaluateHandle(`(${getAll.toString()})(Array.from(document.childNodes),"browser")`);
149 | // console.log(aHandle)
150 | const browser: any = await aHandle.jsonValue()
151 |
152 | let html = await request({
153 | method: "GET",
154 | uri: url,
155 | headers: {
156 | "Accept": "*/*",
157 | "Accept-Encoding": "deflate, br",
158 | "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
159 | "Connection": "keep-alive",
160 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36 Edg/91.0.864.59",
161 | },
162 | })
163 |
164 | let ast = parse(html, false)
165 |
166 | // let body = search(ast, "body")
167 |
168 | let parser = getAll(ast.children, "parser")
169 | if (!fs.existsSync(path.resolve(__dirname, "../out/"))) {
170 | fs.mkdirSync(path.resolve(__dirname, "../out/"))
171 | }
172 | fs.writeFileSync(path.resolve(__dirname, "../out/", `./$parser.ast.json`), JSON.stringify(parser, null, 4))
173 | fs.writeFileSync(path.resolve(__dirname, "../out/", `./$browser.ast.json`), JSON.stringify(browser, null, 4))
174 |
175 | // console.log(browser, parser)
176 | let count = browser.length > parser.length ? browser.length : parser.length;
177 | for (let i = 0; i < count; i++) {
178 | // console.log(browser[i] ? browser[i].tag : "undefined", parser[i] ? parser[i].tag : "undefined")
179 |
180 | // if (browser[i] && parser[i]) {
181 | expect(browser[i].tag.toLowerCase()).toBe(parser[i].tag)
182 | // } else {
183 | // console.log(i)
184 | // console.log(browser[i], parser[i])
185 | // }
186 |
187 | // console.assert(browser[i].tag.toLowerCase() === parser[i].tag, `${browser[i] ? browser[i].tag : "undefined"}, ${parser[i] ? parser[i].tag : "undefined"}`)
188 | }
189 | });
190 | }
191 |
--------------------------------------------------------------------------------
/tsconfig-esmodule.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | /* Visit https://aka.ms/tsconfig.json to read more about this file */
4 |
5 | /* Basic Options */
6 | // "incremental": true, /* Enable incremental compilation */
7 | "target": "ESNEXT", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */
8 | "module": "ESNext", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
9 | "lib": ["ESNext", "DOM"], /* Specify library files to be included in the compilation. */
10 | "allowJs": true, /* Allow javascript files to be compiled. */
11 | // "checkJs": true, /* Report errors in .js files. */
12 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
13 | // "declaration": true, /* Generates corresponding '.d.ts' file. */
14 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
15 | // "sourceMap": true, /* Generates corresponding '.map' file. */
16 | // "outFile": "./", /* Concatenate and emit output to single file. */
17 | "outDir": "./dist-esmodule", /* Redirect output structure to the directory. */
18 | "rootDir": "./src", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
19 | // "composite": true, /* Enable project compilation */
20 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */
21 | // "removeComments": true, /* Do not emit comments to output. */
22 | // "noEmit": true, /* Do not emit outputs. */
23 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */
24 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
25 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
26 |
27 | /* Strict Type-Checking Options */
28 | "strict": true, /* Enable all strict type-checking options. */
29 | // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */
30 | // "strictNullChecks": true, /* Enable strict null checks. */
31 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */
32 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
33 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */
34 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */
35 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */
36 |
37 | /* Additional Checks */
38 | // "noUnusedLocals": true, /* Report errors on unused locals. */
39 | // "noUnusedParameters": true, /* Report errors on unused parameters. */
40 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */
41 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
42 | // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */
43 |
44 | /* Module Resolution Options */
45 | // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
46 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */
47 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
48 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */
49 | // "typeRoots": [], /* List of folders to include type definitions from. */
50 | // "types": [], /* Type declaration files to be included in compilation. */
51 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
52 | "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
53 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */
54 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
55 |
56 | /* Source Map Options */
57 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */
58 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
59 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */
60 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
61 |
62 | /* Experimental Options */
63 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */
64 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */
65 |
66 | /* Advanced Options */
67 | "skipLibCheck": true, /* Skip type checking of declaration files. */
68 | "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */
69 | },
70 | "exclude": [
71 | "vm",
72 | "examples",
73 | "test",
74 | "app",
75 | "dist",
76 | "script",
77 | "copy",
78 | "mycheck",
79 | "coverage",
80 | "matchtest",
81 | "compare.js",
82 | "jest-playwright.config.js",
83 | "jest.config.js",
84 | "server.js",
85 | "test-server.js",
86 | ],
87 | }
88 |
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | /* Visit https://aka.ms/tsconfig.json to read more about this file */
4 |
5 | /* Basic Options */
6 | // "incremental": true, /* Enable incremental compilation */
7 | "target": "ESNEXT", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */
8 | "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
9 | "lib": ["ESNext", "DOM"], /* Specify library files to be included in the compilation. */
10 | "allowJs": true, /* Allow javascript files to be compiled. */
11 | // "checkJs": true, /* Report errors in .js files. */
12 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
13 | // "declaration": true, /* Generates corresponding '.d.ts' file. */
14 | // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
15 | // "sourceMap": true, /* Generates corresponding '.map' file. */
16 | // "outFile": "./", /* Concatenate and emit output to single file. */
17 | "outDir": "./dist", /* Redirect output structure to the directory. */
18 | "rootDir": "./src", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
19 | // "composite": true, /* Enable project compilation */
20 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */
21 | // "removeComments": true, /* Do not emit comments to output. */
22 | // "noEmit": true, /* Do not emit outputs. */
23 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */
24 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
25 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
26 |
27 | /* Strict Type-Checking Options */
28 | "strict": true, /* Enable all strict type-checking options. */
29 | // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */
30 | // "strictNullChecks": true, /* Enable strict null checks. */
31 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */
32 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
33 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */
34 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */
35 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */
36 |
37 | /* Additional Checks */
38 | // "noUnusedLocals": true, /* Report errors on unused locals. */
39 | // "noUnusedParameters": true, /* Report errors on unused parameters. */
40 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */
41 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
42 | // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */
43 |
44 | /* Module Resolution Options */
45 | // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
46 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */
47 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
48 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */
49 | // "typeRoots": [], /* List of folders to include type definitions from. */
50 | // "types": [], /* Type declaration files to be included in compilation. */
51 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
52 | "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
53 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */
54 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
55 |
56 | /* Source Map Options */
57 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */
58 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
59 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */
60 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
61 |
62 | /* Experimental Options */
63 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */
64 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */
65 |
66 | /* Advanced Options */
67 | "skipLibCheck": true, /* Skip type checking of declaration files. */
68 | "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */
69 | },
70 | "exclude": [
71 | "vm",
72 | "examples",
73 | "test",
74 | "app",
75 | "dist-esmodule",
76 | "script",
77 | "copy",
78 | "mycheck",
79 | "coverage",
80 | "matchtest",
81 | "compare.js",
82 | "jest-playwright.config.js",
83 | "jest.config.js",
84 | "server.js",
85 | "test-server.js",
86 | ],
87 | }
88 |
--------------------------------------------------------------------------------
13 | 9(MDAyMTc0MTg1MDEyNTUxMTQxMDRkODNiYw004))
54 | 
24 | CSDN博客技术专家
25 |