├── .drone.yml
├── .gitignore
├── README.md
├── countClick
    └── countClick.js
├── getDir
    └── getDir.js
├── logs
    └── logUtils.js
├── package.json
├── picture
    └── BuriedPointPro.png
├── puppeteerScripts
    └── demo.js
├── resultExcels
    └── exportResult.js
├── test.js
├── test
    ├── testCountClick.js
    ├── testExportResult.js
    ├── testGetDir.js
    ├── testGetDirContent.js
    └── testLog.js
└── timeWait
    └── timeWait.js


/.drone.yml:
--------------------------------------------------------------------------------
1 | kind: pipeline
2 | name: default
3 | steps:
4 |   - name: sonar-scanner
5 |     image: mercuriete/sonar-scanner:4.2.0.1873
6 |     commands: 
7 |       - sonar-scanner -Dsonar.sources=. -Dsonar.host.url=http://111.230.120.184 -Dsonar.login=39800f87119cb03318150a013cb3ef979213c822 -Dsonar.projectKey=drone_demo


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules/
2 | /puppeteerScripts/*
3 | !/puppeteerScripts/demo.js
4 | /result.xlsx
5 | test/result.xlsx
6 | logs/logs/
7 | test/logs/


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # 最新
 2 | 
 3 | 
 4 | 该项目功能已被集成到 [ShaoNianyr/Scripts_Web_UI_Autotest](https://github.com/ShaoNianyr/Scripts_Web_UI_Autotest)，并进行优化，且集成了 breakpoint & mock 的功能。
 5 | 
 6 | Buried-Point-Pro 埋点测试工具的不足:
 7 | 
 8 | 1. 只是监听了埋点打印在 console 里面的内容，具体有没有真正上报成功不知道。
 9 | 
10 | 2. 打印在 console 的是神策埋点的特点，其他的埋点不一定这个方式，拦截埋点上报的链接做测试更精准，更普遍。
11 | 
12 | 3. 没有模拟触发测试异常场景的功能。（如请求失败，请求中断等记录异常错误的埋点）
13 | 
14 | 基于以上的一些优化，也就有了这个新的工具，更多详情见 [ShaoNianyr/Scripts_Web_UI_Autotest](https://github.com/ShaoNianyr/Scripts_Web_UI_Autotest) 。
15 | 
16 | 技术交流 QQ 群：552643038
17 | 
18 | 
19 | 
20 | 
21 | --------------------------- 以下为原文 -------------------------------
22 | 
23 | # 埋点自动化测试 Pro
24 | 
25 | Buried-Point-Pro 项目的前身：[maidianDemoTest](https://github.com/ShaoNianyr/maidianDemoTest)
26 | 
27 | Buried-Point-Pro，基于 nodejs 和 puppeteer 开发的埋点自动化测试框架，对外暴露 puppeteerScripts 的脚本文件夹，可以放置所有写好或录制好的 puppeteer 的业务流程脚本，并自动遍历执行所有脚本，监听并记录所有脚本流程的埋点信息。
28 | 
29 | 框架仅输出每个流程的所有埋点信息的 excel 表，每个脚本分不同的 sheet 记录，以及根据点击元素的 name 属性和该元素上报的事件名字进行对比校验。对埋点的校验方式支持二次扩展开发。
30 | 
31 | 更多详情见下方项目原理图。
32 | 
33 | ## 项目原理（我觉得还是看图吧！）
34 | 
35 | [项目原理图高清链接](https://www.processon.com/view/link/5dd38659e4b01da3459348c7)
36 | 
37 | <img src="https://github.com/ShaoNianyr/Buried-Point-Pro/blob/master/picture/BuriedPointPro.png">
38 | 
39 | 
40 | ## 快速体验
41 | 
42 | ```
43 |     // 首先要准备好 nodejs && npm 环境
44 |     git clone https://github.com/ShaoNianyr/Buried-Point-Pro.git
45 |     cd Buried-Point-Pro
46 |     npm config set puppeteer_download_host=https://npm.taobao.org/mirrors
47 |     npm install --registry=https://registry.npm.taobao.org
48 |     node test.js 
49 | ```
50 | 
51 | ## 实际使用
52 | 
53 | -   通过第三方工具 [puppeteer-recorder](https://github.com/checkly/puppeteer-recorder) 一键录制复杂的脚本 / 手写 puppeteer 脚本 放到 puppeteerScripts 文件夹下
54 | 
55 | -   注释掉 test.js 中的 " // demo 演示代码... "下的代码，启用 " // 实际业务代码... "下的代码
56 | 
57 | -   node test.js
58 | 
59 | ## 为什么要做埋点 Pro
60 | 
61 | maidianDemoTest 在投入使用的过程中发现，只能对静态页面的埋点进行检测，动态页面需要根据业务逻辑手动去写代码并提 excel 来设计，非常的麻烦，尤其是当业务流程很复杂又很多的时候，一样显得非常麻烦。在这样的一种前提下，重写了整个项目，由此而来的就是 Buried-Point-Pro. 
62 | 
63 | Buried-Point-Pro 换了另外一种方式来执行埋点的脚本，可以通过第三方工具 [puppeteer-recorder](https://github.com/checkly/puppeteer-recorder) 一键录制复杂的脚本，无需手动 copy selector 定位和写大量重复的 click 语句，脚本运行的时候，会遍历 puppeteerScripts 文件夹下所有的执行脚本，并记录过程中的上报埋点，并将这一系列的流程保存到单独的 sheet 里面，方便定位到具体某个业务流程的埋点。这样无惧复杂的业务流程，且但项目迭代时，你只需要重新录制一遍相关改动的业务流程即可，其他的脚本就可以作为埋点的回归测试。
64 | 


--------------------------------------------------------------------------------
/countClick/countClick.js:
--------------------------------------------------------------------------------
 1 | const readFileList = require('../getDir/getDir.js');
 2 | 
 3 | function countClick(list) {
 4 |     var countClickList = [];
 5 |     var clickList = [];
 6 |     for (var i = 0; i < list.length; i++) {
 7 |         clickList = list[i].content.match(/click/g);
 8 |         countClickList.push({
 9 |             script: list[i].file,
10 |             count: clickList.length,
11 |         })
12 |     }
13 |     return countClickList;
14 | }
15 | 
16 | module.exports = {
17 |     countClick
18 | }


--------------------------------------------------------------------------------
/getDir/getDir.js:
--------------------------------------------------------------------------------
 1 | var fs = require('fs');
 2 | var path = require('path');
 3 | 
 4 | function getDir(dir, filesList = []) {
 5 |     var dir = path.resolve(dir);
 6 |     const files = fs.readdirSync(dir);
 7 |     files.forEach((item, index) => {
 8 |         var fullPath = path.join(dir, item);
 9 |         const stat = fs.statSync(fullPath);
10 |         if (stat.isDirectory()) {
11 |             getDir(path.join(dir, item), filesList); //递归读取文件
12 |         } else {        
13 |             filesList.push(path.parse(fullPath));
14 |         }
15 |     });
16 |     return filesList;
17 | }
18 | 
19 | function getDirContent(dir) {
20 |     var dir = path.resolve(dir);
21 |     const files = fs.readdirSync(dir);
22 |     var contentList = [];
23 |     files.forEach((item, index) => {
24 |         var fullPath = path.join(dir, item);
25 |         const stat = fs.statSync(fullPath);
26 |         if (stat.isDirectory()) {
27 |             getDir(path.join(dir, item)); //递归读取文件
28 |         } else {
29 |             var content = fs.readFileSync(fullPath, 'utf-8');
30 |             contentList.push({
31 |                 file: path.parse(fullPath).name,
32 |                 content: content,
33 |             });
34 |         }
35 |     });
36 |     return contentList;
37 | }
38 | 
39 | module.exports = {
40 |     getDir,
41 |     getDirContent
42 | }


--------------------------------------------------------------------------------
/logs/logUtils.js:
--------------------------------------------------------------------------------
 1 | const log4js = require('log4js');
 2 | 
 3 | log4js.configure(
 4 | {
 5 |   appenders:
 6 |   {
 7 |     console:
 8 |     {
 9 |         type: 'console',
10 |     },
11 |     datelogSuc:
12 |     {
13 |         type: 'dateFile',
14 |         filename: './logs/logs/infolog',
15 |         pattern: ".yyyy-MM-dd.txt",
16 |         // alwaysIncludePattern: true,
17 |         // maxLogSize: 10, // 无效
18 |         // backups: 5, // 无效
19 |         compress: true,
20 |         daysToKeep: 2,
21 |     },
22 |     datelogFail:
23 |     {
24 |         type: 'dateFile',
25 |         filename: './logs/logs/debuglog',
26 |         pattern: ".yyyy-MM-dd.txt",
27 |         compress: true,
28 |         daysToKeep: 2,
29 |     },
30 |     // more...
31 |   },
32 |   categories:
33 |   {
34 |       default:
35 |       {
36 |           appenders: ['console'],
37 |           level: 'debug',
38 |       },
39 |       datelogSuc:
40 |       {
41 |           // 指定为上面定义的appender，如果不指定，无法写入
42 |           appenders: ['console', 'datelogSuc'],
43 |           level: 'debug', // 指定等级
44 |       },
45 |       datelogFail:
46 |       {
47 |           appenders: ['console', 'datelogFail'],
48 |           level: 'debug',
49 |       },
50 |       // more...
51 |   },
52 |   
53 |   // for pm2...
54 |   pm2: true,
55 |   disableClustering: true, // not sure...
56 | }
57 | );
58 | 
59 | 
60 | function getLogger(type)
61 | {
62 |     return log4js.getLogger(type);
63 | }
64 | 
65 | module.exports = {
66 |     getLogger,
67 | }


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "maidian",
 3 |   "version": "1.0.0",
 4 |   "description": "",
 5 |   "main": "test.js",
 6 |   "scripts": {
 7 |     "test": "echo \"Error: no test specified\" && exit 1"
 8 |   },
 9 |   "author": "shaonian",
10 |   "license": "ISC",
11 |   "dependencies": {
12 |     "fs": "^0.0.1-security",
13 |     "log4js": "^6.0.0",
14 |     "node-xlsx": "^0.15.0",
15 |     "puppeteer": "^2.0.0",
16 |     "string-random": "^0.1.3"
17 |   }
18 | }
19 | 


--------------------------------------------------------------------------------
/picture/BuriedPointPro.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/shaonianyr/Buried-Point-Pro/fdd8dc868c12531326392cbad4a15362e3c95f76/picture/BuriedPointPro.png


--------------------------------------------------------------------------------
/puppeteerScripts/demo.js:
--------------------------------------------------------------------------------
 1 | const time = require('../timeWait/timeWait.js');
 2 | 
 3 | module.exports = async (page) => {
 4 | 
 5 |     // 复制录制好的 puppeteer 脚本
 6 |     // ..
 7 | 
 8 |     // 此次没有复制实际业务作 demo，用 puppeteer 伪装一个事件的上报 
 9 |     await page.evaluate(() => console.log('{"properties": { "$url": "www.example.com", "$element_name": "name1", "$element_selector": "#__layout > div", "$title": "page1" }, "event": "$Webclick"}'));
10 | 
11 |     await time.sleep(1000)
12 | };


--------------------------------------------------------------------------------
/resultExcels/exportResult.js:
--------------------------------------------------------------------------------
 1 | const xlsx = require('node-xlsx');
 2 | const fs = require('fs');
 3 | const log4js = require('../logs/logUtils.js'); 
 4 | const loggerSuc = log4js.getLogger('datelogSuc'); 
 5 | const loggerFail = log4js.getLogger('datelogFail'); 
 6 | 
 7 | function exportResult(data)
 8 | {
 9 |     var buffer = xlsx.build(data);
10 |     fs.writeFile('./result.xlsx', buffer, function(err) {
11 |         if (err) {
12 |             loggerFail.error("\n写入 excel 错误: " + err);
13 |             return;
14 |         }
15 |         loggerSuc.info("\n数据已写入 excel 表.");
16 |     })
17 | }
18 | 
19 | function getExcel(path)
20 | {
21 |     var sheets = xlsx.parse(path);
22 |     var arrTotal = [];
23 |     sheets.forEach(function(sheet) {
24 |         var arr = [];
25 |         for(var i = 1; i < sheet["data"].length; i++) {
26 |             var row = sheet['data'][i];
27 |             if(row && row.length > 0){
28 |                 arr.push({
29 |                     script: sheet['name'],
30 |                     key: row[0],
31 |                     url: row[1],
32 |                     event: row[2],
33 |                     expectName: row[3],
34 |                     actualName: row[4],
35 |                     loginPositin: row[5],
36 |                     report: row[6],
37 |                 });
38 |             }
39 |         }
40 |         arrTotal.push(arr);
41 |     });
42 |     return arrTotal;
43 | }
44 | 
45 | module.exports = {
46 |     exportResult,
47 |     getExcel
48 | }


--------------------------------------------------------------------------------
/test.js:
--------------------------------------------------------------------------------
  1 | const puppeteer = require('puppeteer');
  2 | const devices = require('puppeteer/DeviceDescriptors');
  3 | const resultExcels = require('./resultExcels/exportResult.js');
  4 | const readFileList = require('./getDir/getDir.js');
  5 | const log4js = require('./logs/logUtils.js'); 
  6 | const loggerSuc = log4js.getLogger('datelogSuc'); 
  7 | const loggerFail = log4js.getLogger('datelogFail'); 
  8 | const countClick = require('./countClick/countClick.js');
  9 | const scriptPath = './puppeteerScripts/';
 10 | const resultPath = './result.xlsx';
 11 | 
 12 | 
 13 | (async () => {
 14 |     var arr = [];
 15 |     var key;
 16 |     var sheetName;
 17 |     var sheetNum = 0;
 18 |     var filesList = [];
 19 |     readFileList.getDir(scriptPath, filesList);
 20 | 
 21 |     for (var i = 0; i < filesList.length; i++) {
 22 | 
 23 |         // debug 使用
 24 |         const browser = await puppeteer.launch({ headless: false });
 25 | 
 26 |         // 实际业务使用
 27 |         // const browser = await puppeteer.launch();
 28 | 
 29 |         const page = await browser.newPage();
 30 | 
 31 |         // 开启移动端模拟
 32 |         await page.emulate(devices['iPhone X']);
 33 | 
 34 |         page.on('console', msg => {
 35 |             if (typeof msg === 'object') {
 36 | 
 37 |                 // debug 使用
 38 |                 console.log(msg);
 39 | 
 40 |                 try {
 41 |                     var obj = JSON.parse(msg._text);
 42 | 
 43 |                     if (obj.properties.$element_name !== undefined) {
 44 |                         // debug 使用
 45 |                         loggerSuc.info('\n上报事件：\n', obj);
 46 | 
 47 |                         // Excel记录逻辑
 48 |                         (async () => {
 49 |                             // 实际业务代码获取定位元素的 name 属性
 50 |                             // const name = await page.$eval(obj.properties.$element_selector, el => el.getAttribute('name'));
 51 |                             
 52 |                             // demo 演示代码直接自定义 name 属性
 53 |                             const name = 'name1';
 54 |                             
 55 |                             loggerSuc.info('\n埋点事件信息记录：\n', obj.event, obj.properties.$url, '\n预期上报名字：', name, '\n实际上报名字：', obj.properties.$element_name);
 56 |                             var list = [];
 57 |                             if (name === obj.properties.$element_name) {
 58 |                                 list.push(key.toString(), obj.properties.$url, obj.event, name, obj.properties.$element_name, '-', '1');
 59 |                             } else {
 60 |                                 list.push(key.toString(), obj.properties.$url, obj.event, name, obj.properties.$element_name, '-', '0');
 61 |                             }
 62 |                             arr[sheetNum].data.push(list);
 63 |                             key++;
 64 |                         })();
 65 |                     }
 66 | 
 67 |                     // 自定义事件
 68 |                     if (obj.properties.login_position !== undefined) {
 69 |                         // debug 使用
 70 |                         loggerSuc.info('\n上报事件：\n', obj);
 71 | 
 72 |                         // Excel记录逻辑
 73 |                         (async () => {
 74 |                             loggerSuc.info('\n埋点登陆位置记录：\n', '上报登陆位置：', obj.properties.login_position);
 75 |                             var list = [];
 76 |                             list.push(key.toString(), '-', '-', '-', '-', obj.properties.login_position, '1');
 77 |                             arr[sheetNum].data.push(list);
 78 |                             key++;
 79 |                         })();
 80 |                     }
 81 |                   } catch (error) {
 82 |                     var obj = {};
 83 |                   }
 84 |             }
 85 |         });
 86 | 
 87 |         // demo 演示代码，指定运行脚本目录下的 demo.js 脚本
 88 |         if (filesList[i].base === 'demo.js') {
 89 |             var customPathFunction = require(scriptPath + filesList[i].base);
 90 |             sheetName = filesList[i].name;
 91 |             key = 1;
 92 |             var keyName = {
 93 |                 name: sheetName,
 94 |                 data: [['key', 'url', 'event', 'expectName', 'actualName', 'loginPosition', 'report'],]
 95 |             }
 96 |             arr.push(keyName);
 97 |             await customPathFunction(page);
 98 |             sheetNum++;
 99 |             await browser.close();
100 |         } else {
101 |             await browser.close();
102 |         }
103 | 
104 |         // 实际业务代码
105 |         // if (filesList[i].base != 'demo.js') {
106 |         //     var customPathFunction = require(scriptPath + filesList[i].base);
107 |         //     sheetName = filesList[i].name;
108 |         //     key = 1;
109 |         //     var keyName = {
110 |         //         name: sheetName,
111 |         //         data: [['key', 'url', 'event', 'expectName', 'actualName', 'loginPosition', 'report'],]
112 |         //     }
113 |         //     arr.push(keyName);
114 |         //     await customPathFunction(page);
115 |         //     sheetNum++;
116 |         //     await browser.close();
117 |         // } else {
118 |         //     await browser.close();
119 |         // }
120 |     }
121 | 
122 |     resultExcels.exportResult(arr);
123 | 
124 |     var promise = new Promise(function(resolve, reject) {
125 |         setTimeout(function() {
126 |             resolve(resultExcels.getExcel(resultPath));
127 |         }, 2000);
128 |     });
129 | 
130 |     promise.then(function(value) {
131 |         // 打印写好的 excel 表的内容
132 |         loggerSuc.info('\nexcel 表中记录的埋点数据:\n', value);
133 |         // var excel = value;
134 | 
135 |         // const fileContentList = readFileList.getDirContent(scriptPath);
136 |         // const countClickList = countClick.countClick(fileContentList);
137 | 
138 |         // for (var i = 0; i < excel.length; i++) {
139 |         //     for (var j = 0; j < countClickList.length; j++) {
140 |         //         if (excel[i][0].script === countClickList[j].script) {
141 |         //             if (excel[i].length === countClickList[j].count) {
142 |         //                 loggerSuc.info(countClickList[j].script, "埋点上报数目符合");
143 |         //             } else {
144 |         //                 loggerFail.error(countClickList[j].script, "埋点上报数目不符合");
145 |         //                 loggerFail.error(excel[i].length, "!=", countClickList[j].count);
146 |         //             }
147 |         //         }
148 |         //     }
149 |         // }
150 |     });
151 | })()


--------------------------------------------------------------------------------
/test/testCountClick.js:
--------------------------------------------------------------------------------
1 | const readFileList = require('../getDir/getDir.js');
2 | const fileContentList = readFileList.getDirContent('../puppeteerScripts/');
3 | const countClick = require('../countClick/countClick.js');
4 | 
5 | const countClickList = countClick.countClick(fileContentList);
6 | console.log(countClickList);


--------------------------------------------------------------------------------
/test/testExportResult.js:
--------------------------------------------------------------------------------
 1 | const resultExcels = require('../resultExcels/exportResult.js');
 2 | const path = './result.xlsx';
 3 | 
 4 | var arr = [{  name: 'sheet1', 
 5 |               data: [
 6 |                         [
 7 |                             'key',
 8 |                             'url',
 9 |                             'event',
10 |                             'expectName',
11 |                             'actualName',
12 |                             'loginPosition',
13 |                             'report'
14 |                         ],
15 |                     ]
16 |             }];
17 | 
18 | var list1 = ['1', 'http://www.baidu.com', '$webclick', 'name1', 'name2', '/', '0'];
19 | var list2 = ['2', 'http://www.baidu.com', '$webclick', 'name1', 'name2', '/', '1'];
20 | var list3 = ['3', 'http://www.baidu.com', '$loginPosition', '/', '/', 'location1', '1'];
21 | var sheetName = 'sheet2'
22 | var sheet2 = {  name: sheetName,
23 |                 data: [
24 |                         [
25 |                             'key',
26 |                             'url',
27 |                             'event',
28 |                             'expectName',
29 |                             'actualName',
30 |                             'loginPosition',
31 |                             'report'
32 |                         ],
33 |                     ]
34 |              }
35 | 
36 | arr[0].data.push(list1);
37 | arr[0].data.push(list3);
38 | arr.push(sheet2);
39 | arr[1].data.push(list2);
40 | 
41 | resultExcels.exportResult(arr);
42 | 
43 | var promise = new Promise(function(resolve, reject) {
44 |     setTimeout(function() {
45 |         resolve(resultExcels.getExcel(path));
46 |     }, 1000);
47 | });
48 | 
49 | promise.then(function(value) {
50 |     console.log('Writing data:');
51 |     console.log(value);
52 |     // console.log(value[0]);
53 |     console.log(value[0][1]);
54 |     console.log(value[0].length);
55 |     console.log(value[0][1].script);
56 |     // console.log(value[1].[0].script);
57 | 
58 |     // console.log(value[0].script);
59 | });
60 | // arr.push(sheet2);
61 | // // arr[0].data.push(list1);
62 | // // arr[1].data.push(list2);
63 | // console.log(arr);


--------------------------------------------------------------------------------
/test/testGetDir.js:
--------------------------------------------------------------------------------
1 | const readFileList = require('../getDir/getDir.js');
2 | const path = require('path');
3 | var filesList = [];
4 | readFileList.getDir('../puppeteerScripts/',filesList);
5 | 
6 | for (var i = 0; i < filesList.length; i++) {
7 |     // console.log(filesList[i]);
8 |     console.log('./puppeteerScripts/' + filesList[i].base);
9 | }


--------------------------------------------------------------------------------
/test/testGetDirContent.js:
--------------------------------------------------------------------------------
1 | const readFileList = require('../getDir/getDir.js');
2 | const fileContentList = readFileList.getDirContent('../puppeteerScripts/');
3 | 
4 | for (var i = 0; i < fileContentList.length; i++) {
5 |     console.log(fileContentList[i]);
6 | }


--------------------------------------------------------------------------------
/test/testLog.js:
--------------------------------------------------------------------------------
1 | const log4js = require('../logs/logUtils.js'); // 引入库
2 | const loggerSuc = log4js.getLogger('datelogSuc'); // 获取指定的输出源
3 | const loggerFail = log4js.getLogger('datelogFail'); // 获取指定的输出源
4 | loggerSuc.info('info'); // 打印
5 | loggerFail.warn('warn'); // 打印
6 | loggerFail.debug('debug'); // 打印
7 | loggerFail.error('error'); // 打印


--------------------------------------------------------------------------------
/timeWait/timeWait.js:
--------------------------------------------------------------------------------
1 | function sleep (ms) {
2 |     return new Promise(resolve => setTimeout(resolve, ms));
3 | }
4 | 
5 | module.exports = {
6 |     sleep
7 | }


--------------------------------------------------------------------------------