├── .gitignore ├── README.md ├── config.json ├── index.js ├── package-lock.json ├── package.json └── result.json /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | config_temp.json -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gitlab 数据统计 2 | 3 | 可统计以下数据 4 | 5 | ``` 6 | push_count: 0, // 推送次数 7 | commit_count: 0, // commit 次数 8 | commit_line: 0, // 代码变更总行数 9 | mr_open_count:0, // 合并代码最多 10 | mr_merged_count:0, // 合并代码最多 11 | issues_open_count:0, // 提问题最多 12 | issues_close_count:0, // 提问题最多 13 | ``` 14 | -------------------------------------------------------------------------------- /config.json: -------------------------------------------------------------------------------- 1 | { 2 | "host": "https://xxx", 3 | "token": "", 4 | "startDate": "2020-03-01" 5 | } 6 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | const Gitlab = require("@gitbeaker/node").Gitlab; 3 | const results = require("./result.json") || {}; 4 | const config = require("./config.json"); 5 | const users = {}; 6 | const api = new Gitlab(config); 7 | const getUserInfo = async function (user) { 8 | const key = user.username; 9 | const name = user.name; 10 | console.log("begin fetch events:", key); 11 | 12 | const data = (results[key] && results[key].data) || { 13 | push_count: 0, 14 | commit_count: 0, 15 | commit_line: 0, 16 | mr_open_count: 0, // 合并代码最多 17 | mr_merged_count: 0, // 合并代码最多 18 | issues_open_count: 0, // 提问题最多 19 | issues_close_count: 0, // 提问题最多 20 | }; 21 | // 如果之前的数据没跑过这块 22 | if ( 23 | !results[key] || 24 | (results[key] && !results[key].data.hasOwnProperty("mr_open_count")) 25 | ) { 26 | // 跑 issues 数据 27 | const events = await api.Users.events(key, { 28 | after: new Date(config.startDate), 29 | target_type: "issue", 30 | }); 31 | events.forEach((event) => { 32 | if (event.action_name == "opened") { 33 | data.issues_open_count += 1; 34 | } else if (event.action_name == "closed") { 35 | data.issues_close_count += 1; 36 | } 37 | }); 38 | // 跑 pr 数据 39 | const mr_events = await api.Users.events(key, { 40 | after: new Date(config.startDate), 41 | target_type: "merge_request", 42 | }); 43 | mr_events.forEach((event) => { 44 | if (event.action_name == "opened") { 45 | data.mr_open_count += 1; 46 | } else if (event.action_name == "merged") { 47 | data.mr_merged_count += 1; 48 | } 49 | }); 50 | } 51 | 52 | if (results[key]) { 53 | console.log("exist"); 54 | // fs.writeFileSync('./result.json',JSON.stringify(results,null,2)); 55 | // 下面这些数据跑过就不跑了 56 | return; 57 | } 58 | const events = await api.Users.events(key, { 59 | after: new Date(config.startDate), 60 | // target_type: 'commented', 61 | }); 62 | 63 | function timeout(ms) { 64 | return new Promise((resolve) => setTimeout(resolve, ms)); 65 | } 66 | console.log("begin fetch events:", events.length); 67 | for (var i = 0; i < events.length; i++) { 68 | var event = events[i]; 69 | const commit_ids = []; 70 | if ((event.action_name = "pushed to")) { 71 | data.push_count += 1; 72 | var push_data = event.push_data; 73 | if (push_data) { 74 | data.commit_count += push_data.commit_count; 75 | if (push_data.commit_count > 1) { 76 | // await timeout(10); 77 | const compare_result = await api.Repositories.compare( 78 | event.project_id, 79 | push_data.commit_from, 80 | push_data.commit_to 81 | ); 82 | const commits = compare_result.commits; 83 | commits.forEach((c) => { 84 | commit_ids.push(c.id); 85 | }); 86 | } else { 87 | push_data.commit_to && commit_ids.push(push_data.commit_to); 88 | push_data.commit_from && commit_ids.push(push_data.commit_from); 89 | } 90 | } 91 | console.log("begin fetch commits:", commit_ids.length); 92 | for ( 93 | var commit_index = 0; 94 | commit_index < commit_ids.length; 95 | commit_index++ 96 | ) { 97 | var commit_id = commit_ids[commit_index]; 98 | // await timeout(10); 99 | try { 100 | const commit_info = await api.Commits.show( 101 | event.project_id, 102 | commit_id 103 | ); 104 | const stats = commit_info.stats; 105 | data.commit_line += stats.additions; 106 | } catch (e) { 107 | console.log(e); 108 | console.log(event, commit_id); 109 | } 110 | } 111 | } 112 | } 113 | results[key] = { 114 | data: data, 115 | user: user, 116 | }; 117 | // console.log(results); 118 | fs.writeFileSync("./result.json", JSON.stringify(results, null, 2)); 119 | }; 120 | 121 | const run = async function () { 122 | const all_users = await api.Users.all(); 123 | for (var i = 0; i < all_users.length; i++) { 124 | var user = all_users[i]; 125 | if (user.state == "active") { 126 | users[user.name] = { 127 | username: user.username, 128 | email: user.email, 129 | name: user.name, 130 | }; 131 | } 132 | } 133 | console.log("begin run users:", Object.keys(users).length); 134 | for (const name in users) { 135 | try { 136 | await getUserInfo(users[name]); 137 | } catch (e) { 138 | console.log(e); 139 | console.log("抓取错误：", users[name]); 140 | } 141 | } 142 | console.log(results); 143 | }; 144 | run(); 145 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gitlab-crawler", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "@gitbeaker/core": { 8 | "version": "17.0.1", 9 | "resolved": "http://registry.npm.souche-inc.com/@gitbeaker/core/download/@gitbeaker/core-17.0.1.tgz", 10 | "integrity": "sha1-c5hLhdJ7oEd67s+ddv3RAaZHLao=", 11 | "requires": { 12 | "@gitbeaker/requester-utils": "^17.0.0", 13 | "form-data": "^3.0.0", 14 | "li": "^1.3.0", 15 | "xcase": "^2.0.1" 16 | } 17 | }, 18 | "@gitbeaker/node": { 19 | "version": "17.0.1", 20 | "resolved": "http://registry.npm.souche-inc.com/@gitbeaker/node/download/@gitbeaker/node-17.0.1.tgz", 21 | "integrity": "sha1-DgnlhOyb61fRqJQ6LgitSIl8WcI=", 22 | "requires": { 23 | "@gitbeaker/core": "^17.0.0", 24 | "@gitbeaker/requester-utils": "^17.0.0", 25 | "form-data": "^3.0.0", 26 | "got": "^10.7.0", 27 | "xcase": "^2.0.1" 28 | } 29 | }, 30 | "@gitbeaker/requester-utils": { 31 | "version": "17.0.1", 32 | "resolved": "http://registry.npm.souche-inc.com/@gitbeaker/requester-utils/download/@gitbeaker/requester-utils-17.0.1.tgz", 33 | "integrity": "sha1-/xsTtL4bWSCWTV6MrpE75Iqo4A8=", 34 | "requires": { 35 | "form-data": "^3.0.0", 36 | "query-string": "^6.11.1", 37 | "xcase": "^2.0.1" 38 | } 39 | }, 40 | "@sindresorhus/is": { 41 | "version": "2.1.0", 42 | "resolved": "http://registry.npm.souche-inc.com/@sindresorhus/is/download/@sindresorhus/is-2.1.0.tgz", 43 | "integrity": "sha1-atTKYQ9pYJjpKVSrQx/4O+oM4T8=" 44 | }, 45 | "@szmarczak/http-timer": { 46 | "version": "4.0.5", 47 | "resolved": "http://registry.npm.souche-inc.com/@szmarczak/http-timer/download/@szmarczak/http-timer-4.0.5.tgz", 48 | "integrity": "sha1-v71QIR6d+lG6B9pYoUzf0zMgUVI=", 49 | "requires": { 50 | "defer-to-connect": "^2.0.0" 51 | } 52 | }, 53 | "@types/cacheable-request": { 54 | "version": "6.0.1", 55 | "resolved": "http://registry.npm.souche-inc.com/@types/cacheable-request/download/@types/cacheable-request-6.0.1.tgz", 56 | "integrity": "sha1-XSLz3e0f06hMC761A5p0GcLJGXY=", 57 | "requires": { 58 | "@types/http-cache-semantics": "*", 59 | "@types/keyv": "*", 60 | "@types/node": "*", 61 | "@types/responselike": "*" 62 | } 63 | }, 64 | "@types/http-cache-semantics": { 65 | "version": "4.0.0", 66 | "resolved": "http://registry.npm.souche-inc.com/@types/http-cache-semantics/download/@types/http-cache-semantics-4.0.0.tgz", 67 | "integrity": "sha1-kUB3lzaqJlVjXudW4kZ9eHz+iio=" 68 | }, 69 | "@types/keyv": { 70 | "version": "3.1.1", 71 | "resolved": "http://registry.npm.souche-inc.com/@types/keyv/download/@types/keyv-3.1.1.tgz", 72 | "integrity": "sha1-5FpFMk/KnatxarEjDuJJyftSz6c=", 73 | "requires": { 74 | "@types/node": "*" 75 | } 76 | }, 77 | "@types/node": { 78 | "version": "13.9.8", 79 | "resolved": "http://registry.npm.souche-inc.com/@types/node/download/@types/node-13.9.8.tgz", 80 | "integrity": "sha1-CZdkIPyAp6AL9AaAxjgV7Yx2FvQ=" 81 | }, 82 | "@types/responselike": { 83 | "version": "1.0.0", 84 | "resolved": "http://registry.npm.souche-inc.com/@types/responselike/download/@types/responselike-1.0.0.tgz", 85 | "integrity": "sha1-JR9P59FU0rrRJavhtCmyOv0mLik=", 86 | "requires": { 87 | "@types/node": "*" 88 | } 89 | }, 90 | "asynckit": { 91 | "version": "0.4.0", 92 | "resolved": "http://registry.npm.souche-inc.com/asynckit/download/asynckit-0.4.0.tgz", 93 | "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" 94 | }, 95 | "cacheable-lookup": { 96 | "version": "2.0.1", 97 | "resolved": "http://registry.npm.souche-inc.com/cacheable-lookup/download/cacheable-lookup-2.0.1.tgz", 98 | "integrity": "sha1-h75koYuSUjSHXhCpux68pK3Oazg=", 99 | "requires": { 100 | "@types/keyv": "^3.1.1", 101 | "keyv": "^4.0.0" 102 | } 103 | }, 104 | "cacheable-request": { 105 | "version": "7.0.1", 106 | "resolved": "http://registry.npm.souche-inc.com/cacheable-request/download/cacheable-request-7.0.1.tgz", 107 | "integrity": "sha1-BiAxwoViMngu1pSiV/o12pOUKlg=", 108 | "requires": { 109 | "clone-response": "^1.0.2", 110 | "get-stream": "^5.1.0", 111 | "http-cache-semantics": "^4.0.0", 112 | "keyv": "^4.0.0", 113 | "lowercase-keys": "^2.0.0", 114 | "normalize-url": "^4.1.0", 115 | "responselike": "^2.0.0" 116 | } 117 | }, 118 | "clone-response": { 119 | "version": "1.0.2", 120 | "resolved": "http://registry.npm.souche-inc.com/clone-response/download/clone-response-1.0.2.tgz", 121 | "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=", 122 | "requires": { 123 | "mimic-response": "^1.0.0" 124 | }, 125 | "dependencies": { 126 | "mimic-response": { 127 | "version": "1.0.1", 128 | "resolved": "http://registry.npm.souche-inc.com/mimic-response/download/mimic-response-1.0.1.tgz", 129 | "integrity": "sha1-SSNTiHju9CBjy4o+OweYeBSHqxs=" 130 | } 131 | } 132 | }, 133 | "combined-stream": { 134 | "version": "1.0.8", 135 | "resolved": "http://registry.npm.souche-inc.com/combined-stream/download/combined-stream-1.0.8.tgz", 136 | "integrity": "sha1-w9RaizT9cwYxoRCoolIGgrMdWn8=", 137 | "requires": { 138 | "delayed-stream": "~1.0.0" 139 | } 140 | }, 141 | "decode-uri-component": { 142 | "version": "0.2.0", 143 | "resolved": "http://registry.npm.souche-inc.com/decode-uri-component/download/decode-uri-component-0.2.0.tgz", 144 | "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=" 145 | }, 146 | "decompress-response": { 147 | "version": "5.0.0", 148 | "resolved": "http://registry.npm.souche-inc.com/decompress-response/download/decompress-response-5.0.0.tgz", 149 | "integrity": "sha1-eEk5boDj0euoyy9170kw92Rhyw8=", 150 | "requires": { 151 | "mimic-response": "^2.0.0" 152 | } 153 | }, 154 | "defer-to-connect": { 155 | "version": "2.0.0", 156 | "resolved": "http://registry.npm.souche-inc.com/defer-to-connect/download/defer-to-connect-2.0.0.tgz", 157 | "integrity": "sha1-g9axmdsEFZOshNeBtSIjCMz0wsE=" 158 | }, 159 | "delayed-stream": { 160 | "version": "1.0.0", 161 | "resolved": "http://registry.npm.souche-inc.com/delayed-stream/download/delayed-stream-1.0.0.tgz", 162 | "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" 163 | }, 164 | "duplexer3": { 165 | "version": "0.1.4", 166 | "resolved": "http://registry.npm.souche-inc.com/duplexer3/download/duplexer3-0.1.4.tgz", 167 | "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=" 168 | }, 169 | "end-of-stream": { 170 | "version": "1.4.4", 171 | "resolved": "http://registry.npm.souche-inc.com/end-of-stream/download/end-of-stream-1.4.4.tgz", 172 | "integrity": "sha1-WuZKX0UFe682JuwU2gyl5LJDHrA=", 173 | "requires": { 174 | "once": "^1.4.0" 175 | } 176 | }, 177 | "form-data": { 178 | "version": "3.0.0", 179 | "resolved": "http://registry.npm.souche-inc.com/form-data/download/form-data-3.0.0.tgz", 180 | "integrity": "sha1-MbfjnIXxNVtxOe4MZHzw3n+DxoI=", 181 | "requires": { 182 | "asynckit": "^0.4.0", 183 | "combined-stream": "^1.0.8", 184 | "mime-types": "^2.1.12" 185 | } 186 | }, 187 | "get-stream": { 188 | "version": "5.1.0", 189 | "resolved": "http://registry.npm.souche-inc.com/get-stream/download/get-stream-5.1.0.tgz", 190 | "integrity": "sha1-ASA83JJZf5uQkGfD5lbMH008Tck=", 191 | "requires": { 192 | "pump": "^3.0.0" 193 | } 194 | }, 195 | "got": { 196 | "version": "10.7.0", 197 | "resolved": "http://registry.npm.souche-inc.com/got/download/got-10.7.0.tgz", 198 | "integrity": "sha1-YoidvNbMoyzWoVTMLQxolRIdCR8=", 199 | "requires": { 200 | "@sindresorhus/is": "^2.0.0", 201 | "@szmarczak/http-timer": "^4.0.0", 202 | "@types/cacheable-request": "^6.0.1", 203 | "cacheable-lookup": "^2.0.0", 204 | "cacheable-request": "^7.0.1", 205 | "decompress-response": "^5.0.0", 206 | "duplexer3": "^0.1.4", 207 | "get-stream": "^5.0.0", 208 | "lowercase-keys": "^2.0.0", 209 | "mimic-response": "^2.1.0", 210 | "p-cancelable": "^2.0.0", 211 | "p-event": "^4.0.0", 212 | "responselike": "^2.0.0", 213 | "to-readable-stream": "^2.0.0", 214 | "type-fest": "^0.10.0" 215 | } 216 | }, 217 | "http-cache-semantics": { 218 | "version": "4.1.0", 219 | "resolved": "http://registry.npm.souche-inc.com/http-cache-semantics/download/http-cache-semantics-4.1.0.tgz", 220 | "integrity": "sha1-SekcXL82yblLz81xwj1SSex045A=" 221 | }, 222 | "json-buffer": { 223 | "version": "3.0.1", 224 | "resolved": "http://registry.npm.souche-inc.com/json-buffer/download/json-buffer-3.0.1.tgz", 225 | "integrity": "sha1-kziAKjDTtmBfvgYT4JQAjKjAWhM=" 226 | }, 227 | "keyv": { 228 | "version": "4.0.0", 229 | "resolved": "http://registry.npm.souche-inc.com/keyv/download/keyv-4.0.0.tgz", 230 | "integrity": "sha1-LR2raUkmstQn5MdIBKEIUL5EwS8=", 231 | "requires": { 232 | "json-buffer": "3.0.1" 233 | } 234 | }, 235 | "li": { 236 | "version": "1.3.0", 237 | "resolved": "http://registry.npm.souche-inc.com/li/download/li-1.3.0.tgz", 238 | "integrity": "sha1-IsWbyu+qmo7zWc91l4TkvxBq6hs=" 239 | }, 240 | "lowercase-keys": { 241 | "version": "2.0.0", 242 | "resolved": "http://registry.npm.souche-inc.com/lowercase-keys/download/lowercase-keys-2.0.0.tgz", 243 | "integrity": "sha1-JgPni3tLAAbLyi+8yKMgJVislHk=" 244 | }, 245 | "mime-db": { 246 | "version": "1.43.0", 247 | "resolved": "http://registry.npm.souche-inc.com/mime-db/download/mime-db-1.43.0.tgz", 248 | "integrity": "sha1-ChLgUCZQ5HPXNVNQUOfI9OtPrlg=" 249 | }, 250 | "mime-types": { 251 | "version": "2.1.26", 252 | "resolved": "http://registry.npm.souche-inc.com/mime-types/download/mime-types-2.1.26.tgz", 253 | "integrity": "sha1-nJIfwJt+FJpl39wNpNIJlyALCgY=", 254 | "requires": { 255 | "mime-db": "1.43.0" 256 | } 257 | }, 258 | "mimic-response": { 259 | "version": "2.1.0", 260 | "resolved": "http://registry.npm.souche-inc.com/mimic-response/download/mimic-response-2.1.0.tgz", 261 | "integrity": "sha1-0Tdj019hPQnsN+uzC6wEacDuj0M=" 262 | }, 263 | "normalize-url": { 264 | "version": "4.5.0", 265 | "resolved": "http://registry.npm.souche-inc.com/normalize-url/download/normalize-url-4.5.0.tgz", 266 | "integrity": "sha1-RTNUCH5sqWlXvY9br3U/WYIUISk=" 267 | }, 268 | "once": { 269 | "version": "1.4.0", 270 | "resolved": "http://registry.npm.souche-inc.com/once/download/once-1.4.0.tgz", 271 | "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", 272 | "requires": { 273 | "wrappy": "1" 274 | } 275 | }, 276 | "p-cancelable": { 277 | "version": "2.0.0", 278 | "resolved": "http://registry.npm.souche-inc.com/p-cancelable/download/p-cancelable-2.0.0.tgz", 279 | "integrity": "sha1-SjdA9b2vXtXXw+NIgsb7XWsmam4=" 280 | }, 281 | "p-event": { 282 | "version": "4.1.0", 283 | "resolved": "http://registry.npm.souche-inc.com/p-event/download/p-event-4.1.0.tgz", 284 | "integrity": "sha1-6Su4Ztfo5bcyKTscgmnTjpmCv44=", 285 | "requires": { 286 | "p-timeout": "^2.0.1" 287 | } 288 | }, 289 | "p-finally": { 290 | "version": "1.0.0", 291 | "resolved": "http://registry.npm.souche-inc.com/p-finally/download/p-finally-1.0.0.tgz", 292 | "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" 293 | }, 294 | "p-timeout": { 295 | "version": "2.0.1", 296 | "resolved": "http://registry.npm.souche-inc.com/p-timeout/download/p-timeout-2.0.1.tgz", 297 | "integrity": "sha1-2N0ZeVldLcATnh/ka4tkbLPN8Dg=", 298 | "requires": { 299 | "p-finally": "^1.0.0" 300 | } 301 | }, 302 | "pump": { 303 | "version": "3.0.0", 304 | "resolved": "http://registry.npm.souche-inc.com/pump/download/pump-3.0.0.tgz", 305 | "integrity": "sha1-tKIRaBW94vTh6mAjVOjHVWUQemQ=", 306 | "requires": { 307 | "end-of-stream": "^1.1.0", 308 | "once": "^1.3.1" 309 | } 310 | }, 311 | "query-string": { 312 | "version": "6.11.1", 313 | "resolved": "http://registry.npm.souche-inc.com/query-string/download/query-string-6.11.1.tgz", 314 | "integrity": "sha1-qwIfJ11GPOG2HojwzmmIs+j+fCw=", 315 | "requires": { 316 | "decode-uri-component": "^0.2.0", 317 | "split-on-first": "^1.0.0", 318 | "strict-uri-encode": "^2.0.0" 319 | } 320 | }, 321 | "responselike": { 322 | "version": "2.0.0", 323 | "resolved": "http://registry.npm.souche-inc.com/responselike/download/responselike-2.0.0.tgz", 324 | "integrity": "sha1-JjkbzDF091D5p56sxAoSpcQtdyM=", 325 | "requires": { 326 | "lowercase-keys": "^2.0.0" 327 | } 328 | }, 329 | "split-on-first": { 330 | "version": "1.1.0", 331 | "resolved": "http://registry.npm.souche-inc.com/split-on-first/download/split-on-first-1.1.0.tgz", 332 | "integrity": "sha1-9hCv7uOxK84dDDBCXnY5i3gkml8=" 333 | }, 334 | "strict-uri-encode": { 335 | "version": "2.0.0", 336 | "resolved": "http://registry.npm.souche-inc.com/strict-uri-encode/download/strict-uri-encode-2.0.0.tgz", 337 | "integrity": "sha1-ucczDHBChi9rFC3CdLvMWGbONUY=" 338 | }, 339 | "to-readable-stream": { 340 | "version": "2.1.0", 341 | "resolved": "http://registry.npm.souche-inc.com/to-readable-stream/download/to-readable-stream-2.1.0.tgz", 342 | "integrity": "sha1-gogDFhIb6mYs3CJq2zCt21DLBug=" 343 | }, 344 | "type-fest": { 345 | "version": "0.10.0", 346 | "resolved": "http://registry.npm.souche-inc.com/type-fest/download/type-fest-0.10.0.tgz", 347 | "integrity": "sha1-fwayufv8WBBo0TQf+r0DSc6vxkI=" 348 | }, 349 | "wrappy": { 350 | "version": "1.0.2", 351 | "resolved": "http://registry.npm.souche-inc.com/wrappy/download/wrappy-1.0.2.tgz", 352 | "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" 353 | }, 354 | "xcase": { 355 | "version": "2.0.1", 356 | "resolved": "http://registry.npm.souche-inc.com/xcase/download/xcase-2.0.1.tgz", 357 | "integrity": "sha1-x/pyyqD0QNt4/VZzQyA4rJhEULk=" 358 | } 359 | } 360 | } 361 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gitlab-crawler", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "@gitbeaker/node": "^17.0.1" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /result.json: -------------------------------------------------------------------------------- 1 | {} 2 | --------------------------------------------------------------------------------