3 |
10 |
11 | 
12 | GitHub 监控平台
13 |
14 |
15 | 概览
16 |
17 |
18 | 配置
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
42 |
43 |
49 |
--------------------------------------------------------------------------------
/client/src/config/api.js:
--------------------------------------------------------------------------------
1 | import Vue from "vue";
2 | const apiUri = "/api";
3 | const trend = `${apiUri}/trend`;
4 | const statistic = `${apiUri}/statistic`;
5 | const leakage = `${apiUri}/leakage`;
6 | const leakageInfo = `${apiUri}/leakage/info`;
7 | const leakageCode = `${apiUri}/leakage/code`;
8 | const settingBlacklist = `${apiUri}/setting/blacklist`;
9 | const settingQuery = `${apiUri}/setting/query`;
10 | const settingCron = `${apiUri}/setting/cron`;
11 | const settingNotice = `${apiUri}/setting/notice`;
12 | const settingMail = `${apiUri}/setting/mail`;
13 | const settingWebHook = `${apiUri}/setting/webhook`;
14 | const settingGithub = `${apiUri}/setting/github`;
15 | const api = {
16 | leakage,
17 | leakageCode,
18 | leakageInfo,
19 | settingBlacklist,
20 | settingQuery,
21 | settingNotice,
22 | settingCron,
23 | settingGithub,
24 | settingWebHook,
25 | settingMail,
26 | statistic,
27 | trend
28 | };
29 | export default api;
30 | Vue.prototype.api = api;
31 |
--------------------------------------------------------------------------------
/client/src/main.js:
--------------------------------------------------------------------------------
1 | import Vue from "vue";
2 | import App from "./App.vue";
3 | import router from "./router";
4 | import "@/config/api";
5 | import axios from "axios";
6 | import VueHighlightJS from "vue-highlight.js";
7 | import python from "highlight.js/lib/languages/python";
8 | import Element from "element-ui";
9 | import "element-ui/lib/theme-chalk/index.css";
10 | import {b64Decode, timeAgo, toThousands, dateFormat} from "./utils";
11 |
12 | Vue.use(VueHighlightJS, {
13 | languages: {
14 | python
15 | }
16 | });
17 | Vue.filter("b64Decode", b64Decode);
18 | Vue.filter("timeAgo", timeAgo);
19 | Vue.filter("dateFormat", dateFormat);
20 | Vue.filter("toThousands", toThousands);
21 | Vue.config.productionTip = false;
22 | Vue.use(VueHighlightJS);
23 | Vue.use(Element);
24 | Vue.prototype.axios = axios;
25 |
26 | new Vue({
27 | router,
28 | render: h => h(App)
29 | }).$mount("#app");
30 |
--------------------------------------------------------------------------------
/client/src/router.js:
--------------------------------------------------------------------------------
1 | import Vue from "vue";
2 | import Router from "vue-router";
3 |
4 | const LeakageData = () => import("@/views/result/Index");
5 | const Detail = () => import("@/views/result/Detail");
6 | const Setting = () => import("@/views/setting/Setting");
7 |
8 | Vue.use(Router);
9 | const routes = [
10 | {
11 | path: "/", name: "index", meta: {name: '首页'}, component: LeakageData
12 | },
13 | {path: "/view/leakage/:id", name: 'detail', meta: {name: '详情'}, component: Detail},
14 | {path: "/setting/:tab", name: "setting", meta: {name: '设置'}, component: Setting}
15 | ];
16 | export default new Router({
17 | mode: "history",
18 | routes
19 | });
20 |
--------------------------------------------------------------------------------
/client/src/style/app.css:
--------------------------------------------------------------------------------
1 |
2 | .search-result-item .el-card .el-button-group {
3 | float: right;
4 | margin-bottom: 10px;
5 | }
6 |
7 | .search-result-item .el-card .el-tag {
8 | float: left;
9 | margin-right: 10px;
10 |
11 | }
12 |
13 | .el-menu {
14 | border-radius: 0 !important;
15 | }
16 |
17 | body {
18 | font-family: "Haas Grot Text R Web", "Helvetica Neue", Helvetica, Arial, sans-serif;
19 | font-size: 14px;
20 | line-height: 1.5;
21 | color: #24292e;
22 | margin: 0;
23 | background-color: rgb(245, 246, 247);
24 | }
25 |
26 | a {
27 | text-decoration: none !important;
28 | background-color: transparent !important;
29 | }
30 |
31 | .el-message {
32 | border-radius: 1px !important;
33 | }
34 |
35 | .el-message--success {
36 | background-color: #fff !important;
37 | border-color: #fff !important;
38 | box-shadow: 0 2px 4px rgba(0, 0, 0, 0.12), 0 0 6px rgba(0, 0, 0, 0.04) !important;
39 | }
40 |
41 | .el-message--error {
42 | background-color: #fff !important;
43 | border-color: #fff !important;
44 | box-shadow: 0 2px 4px rgba(0, 0, 0, 0.12), 0 0 6px rgba(0, 0, 0, 0.04) !important;
45 | }
46 |
47 | .el-notification {
48 | width: 330px;
49 | padding: 20px;
50 | box-sizing: border-box;
51 | border-radius: 2px !important;
52 | position: fixed;
53 | right: 16px;
54 | background-color: #fff !important;
55 | box-shadow: 0 2px 4px rgba(0, 0, 0, 0.12), 0 0 6px rgba(0, 0, 0, 0.04) !important;
56 | transition: opacity 0.3s, transform 0.3s, right 0.3s, top 0.4s;
57 | overflow: hidden;
58 | }
59 |
60 | .float-right {
61 | padding-top: 0;
62 | float: right !important;
63 | }
64 |
65 | .text-gray {
66 | color: #586069 !important;
67 | }
68 |
69 | .f6 {
70 | font-size: 12px !important;
71 | }
72 |
73 | .page {
74 | margin-top: 20px;
75 | margin-bottom: 20px;
76 | float: right;
77 | }
78 |
79 | .el-tag {
80 | border: none !important;
81 | font-weight: normal !important;
82 | }
83 |
84 | .el-tag a {
85 | color: rgb(24, 144, 255)
86 | }
87 |
88 | .el-header {
89 | padding: 0 0 0 0 !important;
90 |
91 | }
92 |
93 | .el-footer {
94 | position: relative;
95 | text-align: center;
96 | bottom: 0 !important;
97 | }
98 |
99 | .el-menu-item.is-disabled {
100 | opacity: .95 !important;
101 | color: #F56C6C !important;
102 | cursor: pointer !important;
103 | margin-right: 20px;
104 | }
105 |
106 | .el-menu-item:hover {
107 | background-color: rgb(36, 41, 46) !important;
108 | }
109 |
110 | .el-menu-item {
111 | background-color: rgb(36, 41, 46) !important;
112 | }
113 |
114 | .el-collapse-item__header {
115 | border-bottom: none !important;
116 | }
117 |
118 | .el-collapse {
119 | border-bottom: none !important;
120 | border-top: none !important;
121 | }
122 |
123 | .el-collapse-item__wrap {
124 | border-bottom: none !important;
125 |
126 | }
127 | .clearfix {
128 | margin: 5px;
129 | }
130 |
131 | .card-panel {
132 | border: 1px #ebeef5;
133 | border-radius: 4px;
134 | padding: 5px;
135 | overflow: hidden;
136 | margin-bottom: 5px;
137 |
138 | }
139 |
140 | .hljs {
141 | background-color: #fff !important;
142 | }
143 |
144 | a {
145 | font-size: 14px;
146 | color: rgb(24, 144, 255) !important;
147 | text-decoration: none;
148 | }
149 |
150 | a:hover {
151 | text-decoration: underline;
152 | }
153 |
154 | .el-message-box__status.el-icon-warning, .el-alert__icon.el-icon-warning {
155 | color: orange !important;
156 | }
157 |
158 | .el-form-item__label {
159 | color: #666 !important;
160 | font-weight: 500;
161 | font-size: 16px;
162 | text-align: right;
163 | vertical-align: middle;
164 | line-height: 23px !important;
165 | display: inline-block;
166 | overflow: hidden;
167 | white-space: nowrap;
168 | }
169 |
170 | .el-form-item__content {
171 | font-weight: 400;
172 | margin: 0;
173 | font-size: 13px;
174 | line-height: 23px !important;
175 | }
176 |
177 | .avatar {
178 | display: inline-block;
179 | overflow: hidden;
180 | line-height: 1;
181 | vertical-align: middle;
182 | border-radius: 3px;
183 | }
184 |
185 | .el-radio-button__orig-radio:checked + .el-radio-button__inner {
186 | color: #409EFF !important;
187 | background-color: #ffffff !important;
188 | border-color: #409EFF;
189 | -webkit-box-shadow: -1px 0 0 0 #409EFF;
190 | box-shadow: -1px 0 0 0 #409EFF;
191 | }
--------------------------------------------------------------------------------
/client/src/utils/index.js:
--------------------------------------------------------------------------------
1 | import dayjs from "dayjs";
2 | import "dayjs/locale/zh-cn";
3 | import { Base64 } from "js-base64";
4 | import relativeTime from "dayjs/plugin/relativeTime";
5 |
6 | dayjs.extend(relativeTime);
7 | dayjs.locale("zh-cn");
8 |
9 | export const timeAgo = time => {
10 | return dayjs().from(dayjs(time));
11 | };
12 |
13 | export const dateFormat = time => {
14 | return dayjs(time).format("YYYY-MM-DD HH:mm");
15 | };
16 | export const dateTimeFormat = time => {
17 | return dayjs(time).format("YYYY-MM-DD");
18 | };
19 | export const b64Decode = val => {
20 | return Base64.decode(val);
21 | };
22 | export const toThousands = num => {
23 | return (num || 0).toString().replace(/(\d)(?=(?:\d{3})+$)/g, "$1,");
24 | };
25 |
--------------------------------------------------------------------------------
/client/src/views/result/Dashboard.vue:
--------------------------------------------------------------------------------
1 |
2 |
12 | GitHub 监控平台
13 |
6 |
7 |
13 |
8 |
12 | 泄露总数(个)
9 | {{trend.all.total}}
10 | 今日：{{trend.today.total}}
11 |
18 |
19 |
25 |
20 |
24 | 已确认(个)
21 | {{trend.all.risk}}
22 | 今日：{{trend.today.risk}}
23 |
30 |
31 |
37 |
32 |
36 | 已忽略(个)
33 | {{trend.all.ignore}}
34 | 今日：{{trend.today.ignore}}
35 |
42 |
43 |
49 |
44 |
48 | 查询引擎
45 | {{trend.engine.status?'正常':'离线'}}
46 | 最近：{{trend.engine.last*1000|timeAgo}}
47 |
6 |
7 |
8 | ![]()
11 |
12 |
13 | {{leakageInfo.project}}
15 | –
16 | {{leakageInfo.filename}}
17 |
18 |
19 |
28 |
29 | {{leakageInfo.language}}
30 |
31 |
32 | {{leakageInfo.datetime|dateFormat}}
33 |
34 |
35 |
36 | {{leakageInfo.tag}}
37 |
38 |
39 |
40 |
41 |
42 |
20 | ![]()
22 | ![]()
24 | ![]()
26 |
27 |
47 |
48 |
49 |
50 |
51 |
52 | 安全
53 | 涉密
54 |
55 |
56 |
57 |
58 |
59 | 忽略
60 | 监控
61 |
62 |
63 |
64 |
65 |
68 |
69 | 确认
70 | 快速排查
71 |
72 |
73 |
74 |
75 |
79 |
80 |
81 |
85 |
89 |
90 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
102 |
103 | 可疑文件
104 |
105 |
3 |
5 |
6 |
7 |
8 |
9 |
17 |
21 | {{item._id}} {{item.value}}
22 |
23 |
24 |
25 |
26 |
27 |
28 |
36 |
40 | {{item._id}} {{item.value}}
41 |
42 |
43 |
44 |
45 |
46 |
47 |
49 |
50 | 不限
51 |
52 |
53 | 待审
54 |
55 |
56 | 确认
57 |
58 |
59 | 误报
60 |
61 |
62 |
63 |
64 |
65 |
66 |
75 |
76 |
77 |
78 |
79 |
80 |
282 |
283 |
312 |
--------------------------------------------------------------------------------
/client/src/views/setting/Blacklist.vue:
--------------------------------------------------------------------------------
1 |
2 |
68 |
71 |
72 |
73 |
74 |
3 |
8 |
9 |
10 |
17 |
18 | 添加
19 |
23 |
27 |
28 |
32 |
33 | 删除
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
102 |
103 |
115 |
--------------------------------------------------------------------------------
/client/src/views/setting/Github.vue:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
7 |
8 |
11 |
12 | 添加
14 |
15 |
16 |
17 |
23 |
27 |
28 |
32 |
33 |
37 |
38 |
39 |
41 |
42 |
43 |
47 |
48 | 删除
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
120 |
--------------------------------------------------------------------------------
/client/src/views/setting/Notice.vue:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
16 |
17 |
18 |
19 |
21 |
37 |
38 |
39 |
40 |
48 |
49 |
50 |
52 |
53 |
54 |
56 |
57 |
65 |
66 |
68 |
69 |
73 |
74 |
75 |
76 |
78 |
79 |
81 |
82 |
85 |
86 |
90 |
91 |
92 |
93 |
94 |
98 |
99 |
101 |
107 |
108 |
109 |
112 |
113 |
116 |
117 |
121 |
122 |
123 |
124 | 发送一条测试消息
126 |
127 |
128 |
129 |
130 |
134 |
135 |
166 |
167 |
364 |
376 |
377 |
--------------------------------------------------------------------------------
/client/src/views/setting/Rule.vue:
--------------------------------------------------------------------------------
1 |
2 |
7 |
8 |
15 |
9 |
14 | 邮件
10 | 状态：{{smtp_server.enabled?'开启':'关闭'}}
12 |
13 |
22 |
24 |
26 |
36 |
27 |
35 | 钉钉
28 | 企业微信
29 | {{webhook.webhook.split('=')[1].slice(0,8)}}
30 | 状态：{{webhook.enabled?'开启':'关闭'}}
32 |
33 |
34 |
41 |
42 |
47 |
43 |
46 | 添加钉钉/微信告警
44 | oapi.dingtalk.com qyapi.weixin.qq.com
45 |
102 | webhook 配置
103 | 目前支持 钉钉/企业微信
104 |
105 |
106 |
136 |
143 |
144 | 添加
145 |
149 |
153 |
154 |
157 |
158 | 删除
159 |
160 |
161 |
162 |
163 |
164 |
165 |
3 |
5 |
6 |
7 | 若存在相同名称，会覆盖已有值
13 |
14 |
15 |
17 |
18 | 熟悉
24 | GitHub 搜索语法可以提高监控效率: OR/AND/NOT
27 |
28 |
29 |
31 |
32 |
33 |
37 |
38 |
39 |
40 |
44 |
45 |
46 | 添加
47 |
48 |
49 |
50 |
51 |
52 | {{scope.row.tag}}
53 |
54 |
55 |
56 |
57 |
61 |
62 | {{scope.row.keyword}}
66 |
67 |
68 |
75 |
76 |
77 | {{ scope.row.last * 1000|timeAgo }}
78 |
79 |
80 |
86 |
87 |
93 |
94 |
100 |
101 |
106 |
107 |
108 |
109 |
110 |
114 |
115 |
116 |
117 |
118 | 编辑
123 |
124 | 删除
129 |
130 |
144 |
145 |
146 |
147 |
148 |
149 |
150 |
151 |
249 |
--------------------------------------------------------------------------------
/client/src/views/setting/Setting.vue:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
11 |
12 |
19 |
20 | 确认
21 |
22 |
23 |
24 |
25 |
26 |
70 |
--------------------------------------------------------------------------------
/client/vue.config.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | runtimeCompiler: true,
3 |
4 | css: undefined,
5 |
6 | chainWebpack: config => {
7 | config.module
8 | .rule("vue")
9 | .use("vue-loader")
10 | .loader("vue-loader")
11 | .tap(options => {
12 | return options;
13 | });
14 | },
15 | devServer: {
16 | proxy: {
17 | "/api/": {
18 | target: "http://0.0.0.0:8999",
19 | changeOrigin: true
20 | }
21 | }
22 | },
23 | baseUrl: undefined,
24 | outputDir: undefined,
25 | assetsDir: undefined,
26 | filenameHashing: true,
27 | productionSourceMap: false,
28 | parallel: undefined
29 | };
30 |
--------------------------------------------------------------------------------
/deploy/apt/sources.list:
--------------------------------------------------------------------------------
1 | deb http://mirrors.aliyun.com/debian/ stretch main non-free contrib
2 | deb-src http://mirrors.aliyun.com/debian/ stretch main non-free contrib
3 | deb http://mirrors.aliyun.com/debian-security stretch/updates main
4 | deb-src http://mirrors.aliyun.com/debian-security stretch/updates main
5 | deb http://mirrors.aliyun.com/debian/ stretch-updates main non-free contrib
6 | deb-src http://mirrors.aliyun.com/debian/ stretch-updates main non-free contrib
7 | deb http://mirrors.aliyun.com/debian/ stretch-backports main non-free contrib
8 | deb-src http://mirrors.aliyun.com/debian/ stretch-backports main non-free contrib
--------------------------------------------------------------------------------
/deploy/nginx/Hawkeye.conf:
--------------------------------------------------------------------------------
1 | server {
2 | listen 80;
3 | charset utf-8;
4 | location / {
5 | gzip on;
6 | gzip_min_length 1k;
7 | gzip_buffers 16 64k;
8 | gzip_http_version 1.1;
9 | gzip_comp_level 9;
10 | gzip_types text/plain text/javascript application/javascript image/jpeg image/gif image/png application/font-woff application/x-javascript text/css application/xml;
11 | gzip_vary on;
12 | root /Hawkeye/client/dist;
13 | try_files $uri /index.html;
14 | index index.html index.htm index.php;
15 | }
16 | location /api {
17 | gzip_min_length 1k;
18 | gzip_buffers 16 64k;
19 | gzip_http_version 1.1;
20 | gzip_comp_level 9;
21 | gzip_types text/plain text/javascript application/javascript image/jpeg image/gif image/png application/font-woff application/x-javascript text/css application/xml;
22 | gzip_vary on;
23 | proxy_pass http://127.0.0.1:8888;
24 | proxy_set_header Host $host;
25 | proxy_set_header X-Real-IP $remote_addr;
26 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
27 | }
28 | location /static {
29 | gzip_min_length 1k;
30 | gzip_buffers 16 64k;
31 | gzip_http_version 1.1;
32 | gzip_comp_level 9;
33 | gzip_types text/plain text/javascript application/javascript image/jpeg image/gif image/png application/font-woff application/x-javascript text/css application/xml;
34 | gzip_vary on;
35 | root /Hawkeye/client/dist;
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/deploy/nginx/nginx.conf:
--------------------------------------------------------------------------------
1 | user root;
2 | worker_processes 10;
3 | error_log logs/error.log;
4 | events {
5 | worker_connections 1024;
6 | }
7 | http {
8 | include mime.types;
9 | include Hawkeye.conf;
10 | default_type application/octet-stream;
11 | log_format main '$remote_addr - $remote_user [$time_local] "$request" '
12 | '$status $body_bytes_sent "$http_referer" '
13 | '"$http_user_agent" "$http_x_forwarded_for"';
14 | access_log logs/access.log main;
15 | client_max_body_size 50m;
16 | sendfile on;
17 | keepalive_timeout 65;
18 | gzip on;
19 | }
20 |
--------------------------------------------------------------------------------
/deploy/nginx/pubkey.gpg:
--------------------------------------------------------------------------------
1 | -----BEGIN PGP PUBLIC KEY BLOCK-----
2 |
3 | mQENBFkg3CEBCADG5Vem2p+1p6yV2jZfNsbJBPY1KYzR9weF/K3hmLODcrTaWfiD
4 | EugHwKlAptGDGBtrMsERjUiOWUNUS8IHa+R6tzhnQePG6wO7/yGWBC4J82BkCT2x
5 | M7zCDgldtNYkNqoBc0UfE4ln+WR/RX1DuzPM+DTBZBXLqRJVJFyFtHVJn8I5HPO2
6 | hj51uYqHsewTyAkGzABV4gmSIETSmcU5KDisQ9Vt5OllE0ylh7+kakDFZklyBCHT
7 | 3IAuZhA18mw2qk1z5bnn/GpQ4fJi5w25lb9sqhhxta3ogwWWdJzXA+Nevb2dez8i
8 | bpzPeFnba9q0UVD2VJ25e99DpG15aPvNt+tbABEBAAG0JU9wZW5SZXN0eSBBZG1p
9 | biA8YWRtaW5Ab3BlbnJlc3R5LmNvbT6JATgEEwECACIFAlkg3CECGwMGCwkIBwMC
10 | BhUIAgkKCwQWAgMBAh4BAheAAAoJEJfbdEPV7et08k0H/iIOiZmDavSKE7NSPLxS
11 | ddRLh4+OGL3QW8JZh/+UaX1Z17G3q8kKSJwOemZBL/jIDkoMuHs1Hq0yp9vJ8BaS
12 | 5unX+FRmivwdS5yvkS9s3oA3iJbHagXw0KMnr+baDDNrUwo9MeO0m9muNF/eDoRz
13 | FZ9SpOrgxwhz0kOOt8j+gxWk3TaQ6/JonH4rm3XtP4GMKOKQuUo6l8+pMPEfM209
14 | nMv82kRPAxRV2TwRYToB+TithLTQytJTBytLA+ck5Ny8sGoO5PQWRyx6gj+Bhg0O
15 | rFXfg7/sP2/FEeiDZcF2qn/VMDPvnC7ux2EQdI05MMGFY/pkjVYtLJC2Nb17Bcqj
16 | DH25AQ0EWSDcIQEIAJOoTY0vf0mr+PGUbnv0KKtk65CTzKmICmWIAkCxZaTH+o/3
17 | Lt9ZDtANH1ot3xVTkKg+qBuexh53jnyXyIaIfNqavH1gm+9JusrApVOad2ruODT5
18 | XeVamz0blq37LTmJ7A4T1i8WvB0BQ3j1vh6XkVW6xq1URzVOYyhVqNNq2UIP9M9Y
19 | wtiIIans5i11qmDtZwqxcSYoqSjgz+03M6Dn0UPB1OQdHjOPx7GwHG8+6sVyr+8A
20 | 9G8SlKWre2/qdDyZNdgxalOi5ManCwWSURJRuY7s858qFUm0/5dLMAtWWEbYEmYc
21 | EUxbxQM2jPEaDmvvauZNup+a5DZXpRjpWcg19c0AEQEAAYkBHwQYAQIACQUCWSDc
22 | IQIbDAAKCRCX23RD1e3rdG0dB/9EWT8sTVPOlgFAF2WVZT3bFiqiIC9Dg6Wblt/K
23 | Id/p73gbDNTkeeTvGErAPPQwsKkbD1w2rIYoRzEJ1zVrLgaAbeH/frbQaYNu7c+3
24 | Wm93gxBxjL9Jyrs3jq5jwR4kJ5j+a/GEPtTDqtXzZHvyCP2PWDoQWANNAQDuTpYE
25 | LGHfDF9pmTVwuhkh2IFcH/ZBZUvcxP/w3jXqEiPti/rFN8wKSQtBgWI0pBpXGdrJ
26 | Tl3mIE4jLbPmkxidP1yUFx9wzEVu3soXViehMua9nOeotGOKF4DgekzCnFuXNnd3
27 | h2EiDJbMKk+QJcMPliIePZCP9JWj7n0ok9ccLg5XcNwiFEtn
28 | =U4Wk
29 | -----END PGP PUBLIC KEY BLOCK-----
--------------------------------------------------------------------------------
/deploy/pyenv/pip.conf:
--------------------------------------------------------------------------------
1 | [global]
2 | index-url = https://pypi.tuna.tsinghua.edu.cn/simple
--------------------------------------------------------------------------------
/deploy/pyenv/requirements.txt:
--------------------------------------------------------------------------------
1 | aniso8601==3.0.2
2 | certifi==2018.8.24
3 | chardet==3.0.4
4 | Click==7.0
5 | Deprecated==1.2.3
6 | Flask==1.0.2
7 | Flask-RESTful==0.3.6
8 | gunicorn==19.9.0
9 | huey==1.10.2
10 | idna==2.7
11 | itsdangerous==0.24
12 | Jinja2==2.10
13 | MarkupSafe==1.0
14 | psutil==5.4.7
15 | PyGithub==1.43.2
16 | PyJWT==1.6.4
17 | pymongo==3.7.1
18 | pytz==2018.5
19 | redis==2.10.6
20 | requests==2.19.1
21 | requests-file==1.4.3
22 | six==1.11.0
23 | tldextract==2.2.0
24 | urllib3==1.23
25 | Werkzeug==0.14.1
26 | wrapt==1.10.11
27 |
--------------------------------------------------------------------------------
/deploy/supervisor/hawkeye.conf:
--------------------------------------------------------------------------------
1 | [program:hawkeye]
2 | command=/usr/local/bin/gunicorn -w10 -b127.0.0.1:8888 api:app
3 | directory=/Hawkeye/server
4 | startsecs=5
5 | stopwaitsecs=0
6 | autostart=true
7 | autorestart=true
--------------------------------------------------------------------------------
/deploy/supervisor/huey.conf:
--------------------------------------------------------------------------------
1 | [program:huey]
2 | command=/usr/local/bin/huey_consumer.py task.huey -k process -w 4
3 | directory=/Hawkeye/server
4 | startsecs=5
5 | stopwaitsecs=0
6 | autostart=true
7 | autorestart=true
8 |
--------------------------------------------------------------------------------
/deploy/supervisor/openresty.conf:
--------------------------------------------------------------------------------
1 | [program:openresty]
2 | command=/usr/local/openresty/bin/openresty -g "daemon off;"
3 | directory=/tmp
4 | startsecs=5
5 | stopwaitsecs=0
6 | autostart=true
7 | autorestart=true
--------------------------------------------------------------------------------
/deploy/supervisor/redis.conf:
--------------------------------------------------------------------------------
1 | [program:redis]
2 | command=/usr/bin/redis-server
3 | directory=/tmp
4 | startsecs=10
5 | stopwaitsecs=0
6 | autostart=true
7 | autorestart=true
--------------------------------------------------------------------------------
/docker-entrypoint.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | /usr/bin/supervisord -n -c /etc/supervisor/supervisord.conf&&tail -f /var/log/supervisor/*
--------------------------------------------------------------------------------
/server/api.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # -*- coding: utf-8 -*-
3 | from flask import Flask
4 | from flask_restful import Api
5 | from controllers.result import Leakage, LeakageCode, LeakageInfo
6 | from controllers.setting import Blacklist, Cron, Notice, Query, GithubAccount, SMTPServer, WebHookNotice
7 | from controllers.statistic import Dashboard, Statistic
8 | from controllers.health import Status
9 | import re
10 | import sys
11 |
12 | if sys.hexversion > 0x03070000:
13 | re._pattern_type = re.Pattern
14 |
15 | app = Flask(__name__)
16 | api = Api(app)
17 |
18 | api.add_resource(Status, '/api/health')
19 | api.add_resource(Leakage, '/api/leakage')
20 | api.add_resource(Cron, '/api/setting/cron')
21 | api.add_resource(Query, '/api/setting/query')
22 | api.add_resource(Notice, '/api/setting/notice')
23 | api.add_resource(GithubAccount, '/api/setting/github')
24 | api.add_resource(WebHookNotice, '/api/setting/webhook')
25 | api.add_resource(SMTPServer, '/api/setting/mail')
26 | api.add_resource(Dashboard, '/api/trend')
27 | api.add_resource(Statistic, '/api/statistic')
28 | api.add_resource(LeakageCode, '/api/leakage/code')
29 | api.add_resource(LeakageInfo, '/api/leakage/info')
30 | api.add_resource(Blacklist, '/api/setting/blacklist')
31 |
32 | if __name__ == '__main__':
33 | app.run(host='0.0.0.0', threaded=True, use_reloader=True)
34 |
--------------------------------------------------------------------------------
/server/config/database.py:
--------------------------------------------------------------------------------
1 | from pymongo import MongoClient, DESCENDING
2 | from redis import Redis
3 | import os
4 |
5 | if os.environ.get('MONGODB_URI'):
6 | MONGODB_URI = os.environ.get('MONGODB_URI')
7 | else:
8 | MONGODB_URI = 'mongodb://localhost:27017'
9 |
10 | client = MongoClient(MONGODB_URI, connect=False)
11 | db = client.get_database('hawkeye')
12 |
13 | if os.environ.get('MONGODB_USER'):
14 | MONGODB_USER = os.environ.get('MONGODB_USER')
15 | MONGODB_PASSWORD = os.environ.get('MONGODB_PASSWORD')
16 | db.authenticate(MONGODB_USER, MONGODB_PASSWORD)
17 |
18 | result_col = db.get_collection('result')
19 | query_col = db.get_collection('query')
20 | blacklist_col = db.get_collection('blacklist')
21 | task_col = db.get_collection('task')
22 | notice_col = db.get_collection('notice')
23 | github_col = db.get_collection('github')
24 | setting_col = db.get_collection('setting')
25 |
26 | if os.environ.get('REDIS_HOST'):
27 | REDIS_HOST = os.environ.get('REDIS_HOST')
28 | else:
29 | REDIS_HOST = 'localhost'
30 | if os.environ.get('REDIS_PORT'):
31 | REDIS_PORT = int(os.environ.get('REDIS_PORT'))
32 | else:
33 | REDIS_PORT = 6379
34 | result_cache = Redis(host=REDIS_HOST, port=REDIS_PORT,
35 | db=1, decode_responses=True)
36 |
37 |
38 | def create_indexes():
39 | for filed in ['language', 'tag', 'datetime', 'security', 'desc', 'ignore', 'timestamp']:
40 | try:
41 | result_col.create_index(filed, background=True)
42 | except:
43 | pass
44 |
--------------------------------------------------------------------------------
/server/controllers/account.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/0xbug/Hawkeye/3ef48c27c47c86c8bfe013b4a266e08534c821d4/server/controllers/account.py
--------------------------------------------------------------------------------
/server/controllers/health.py:
--------------------------------------------------------------------------------
1 | from flask import jsonify
2 | from flask_restful import Resource
3 | from pymongo import MongoClient
4 | import os
5 | import requests
6 |
7 |
8 | class Status(Resource):
9 | def get(self):
10 | try:
11 | github = 'api.github.com' in requests.get('https://api.github.com/', timeout=30).text
12 | except Exception as error:
13 | github = str(error)
14 | try:
15 | if os.environ.get('MONGODB_URI'):
16 | MONGODB_URI = os.environ.get('MONGODB_URI')
17 | else:
18 | MONGODB_URI = 'mongodb://localhost:27017'
19 |
20 | client = MongoClient(MONGODB_URI, connect=False, socketTimeoutMS=50)
21 | db = client.get_database('hawkeye')
22 | if os.environ.get('MONGODB_USER'):
23 | MONGODB_USER = os.environ.get('MONGODB_USER')
24 | MONGODB_PASSWORD = os.environ.get('MONGODB_PASSWORD')
25 | db.authenticate(MONGODB_USER, MONGODB_PASSWORD)
26 | mongodb = db.last_status()
27 | except Exception as error:
28 | mongodb = str(error)
29 | data = {
30 | 'github': github,
31 | 'mongodb': mongodb,
32 | }
33 | return jsonify(data)
34 |
--------------------------------------------------------------------------------
/server/controllers/result.py:
--------------------------------------------------------------------------------
1 | from flask import jsonify
2 | from flask_restful import Resource, reqparse
3 | from config.database import result_col, DESCENDING,create_indexes
4 | import json
5 |
6 |
7 | class Leakage(Resource):
8 | def get(self):
9 | parser = reqparse.RequestParser()
10 | parser.add_argument('status', type=str, help='')
11 | parser.add_argument('tag', type=str, help='')
12 | parser.add_argument('language', type=str, help='')
13 | parser.add_argument('limit', type=int, default=10, help='')
14 | parser.add_argument('from', type=int, default=1, help='')
15 | args = parser.parse_args()
16 | status = json.loads(args.get('status'))
17 | filters = status
18 | create_indexes()
19 | if args.get('tag'):
20 | filters = dict({'tag': args.get('tag')}, **filters)
21 | if args.get('language'):
22 | filters = dict({'language': args.get('language')}, **filters)
23 | results = list(
24 | result_col.find(filters, {'code': 0, 'affect': 0}).sort('datetime', DESCENDING).limit(
25 | args.get('limit')).skip(
26 | args.get('limit') * (args.get('from') - 1)))
27 | total = result_col.count(filters)
28 | if total:
29 | msg = '共 {} 条记录'.format(total)
30 | else:
31 | msg = '暂无数据'
32 | data = {
33 | 'msg': msg,
34 | 'status': 200,
35 | 'result': results,
36 | 'total': total
37 | }
38 | return jsonify(data)
39 |
40 | def patch(self):
41 | parser = reqparse.RequestParser()
42 | parser.add_argument('id', type=str, help='')
43 | parser.add_argument('project', type=str, help='')
44 | parser.add_argument('ignore', type=int, choices=(0, 1), help='')
45 | parser.add_argument('security', type=int, choices=(0, 1), help='')
46 | parser.add_argument('desc', type=str, default='', help='')
47 | args = parser.parse_args()
48 | desc = args.get('desc')
49 | result_col.update({'_id': args.get('id')}, {'$set': {'security': int(
50 | args.get('security')), 'ignore': int(args.get('ignore')), 'desc': desc}})
51 | if not args.get('security'):
52 | if not args.get('ignore'):
53 | result_col.update_many({'project': args.get('project')}, {
54 | '$set': {'security': 0, 'ignore': 0, 'desc': desc}})
55 | if args.get('security') and args.get('ignore'):
56 | result_col.update_many({'project': args.get('project')}, {
57 | '$set': {'security': 1, 'ignore': 1, 'desc': desc}})
58 | data = {'status': 201, 'msg': '处理成功', 'result': []}
59 | return jsonify(data)
60 |
61 |
62 | class LeakageCode(Resource):
63 | def get(self):
64 | parser = reqparse.RequestParser()
65 | parser.add_argument('id', type=str, help='leakage_id')
66 | args = parser.parse_args()
67 | leakage_id = args.get('id')
68 | result = result_col.find_one(
69 | {'_id': leakage_id}, {'_id': 0, 'code': 1, 'affect': 1})
70 | data = {'status': 200, 'msg': '获取信息成功', 'result': result}
71 | return jsonify(data)
72 |
73 |
74 | class LeakageInfo(Resource):
75 | def get(self):
76 | parser = reqparse.RequestParser()
77 | parser.add_argument('id', type=str, help='leakage_id')
78 | args = parser.parse_args()
79 | leakage_id = args.get('id')
80 | result = result_col.find_one(
81 | {'_id': leakage_id}, {'_id': 0, 'code': 0})
82 | data = {'status': 200, 'msg': '获取信息成功', 'result': result}
83 | return jsonify(data)
84 |
--------------------------------------------------------------------------------
/server/controllers/setting.py:
--------------------------------------------------------------------------------
1 | from flask import jsonify, request
2 | from flask_restful import Resource, reqparse, inputs
3 | from urllib.parse import urlparse
4 | from config.database import blacklist_col, result_col, query_col, notice_col, github_col, setting_col
5 | from utils.hash import md5
6 | from utils.date import timestamp
7 | from github import Github, GithubException, BadCredentialsException
8 | import signal
9 | import os
10 | import requests
11 |
12 |
13 | class System(Resource):
14 | def get(self):
15 | result = setting_col.find({'key': 'system'}, {'_id': 0})
16 | data = {'status': 200, 'msg': '获取信息成功', 'result': result}
17 | return jsonify(data)
18 |
19 | def post(self):
20 | parser = reqparse.RequestParser()
21 | parser.add_argument('argument', type=str, default=request.host, help='setting argument')
22 | parser.add_argument('value', type=str, default=request.host, help='setting value')
23 | parser.add_argument('unique', type=inputs.boolean, default=False, help='setting unique')
24 | args = parser.parse_args()
25 | value = args.get('value')
26 | argument = args.get('argument')
27 |
28 | setting_col.update_many({'key': 'system', 'argument': argument},
29 | {'$set': {'key': 'system', 'argument': argument, 'value': value}}, upsert=True)
30 |
31 | result = list(setting_col.find({}, {'_id': 0}))
32 | data = {'status': 201, 'msg': '设置成功', 'result': result}
33 | return jsonify(data)
34 |
35 |
36 | class Cron(Resource):
37 | def get(self):
38 | result = setting_col.find_one({'key': 'task'}, {'_id': 0})
39 | if result:
40 | data = {'status': 200, 'msg': '获取信息成功', 'result': result}
41 | else:
42 | data = {'status': 400, 'msg': '请配置查询页数和周期', 'result': result}
43 | return jsonify(data)
44 |
45 | def post(self):
46 | parser = reqparse.RequestParser()
47 | parser.add_argument('page', type=int, default=1, help='')
48 | parser.add_argument('minute', type=int, default=10, help='')
49 | args = parser.parse_args()
50 | page = args.get('page')
51 | minute = args.get('minute')
52 | setting_col.update_many({'key': 'task'}, {'$set': {'key': 'task', 'page': page, 'minute': minute}}, upsert=True)
53 | try:
54 | os.kill(setting_col.find_one({'key': 'task'}).get('pid'), signal.SIGHUP)
55 | except ProcessLookupError:
56 | pass
57 | result = list(setting_col.find({}, {'_id': 0}))
58 | data = {'status': 201, 'msg': '设置成功', 'result': result}
59 | return jsonify(data)
60 |
61 |
62 | class GithubAccount(Resource):
63 | def get(self):
64 | result = list(github_col.find({}, {'_id': 0, 'password': 0}))
65 | data = {'status': 200, 'msg': '获取信息成功', 'result': result}
66 | return jsonify(data)
67 |
68 | def post(self):
69 | parser = reqparse.RequestParser()
70 | parser.add_argument('username', type=str, help='')
71 | parser.add_argument('password', type=str, help='')
72 | args = parser.parse_args()
73 | username = args.get('username')
74 | password = args.get('password')
75 | try:
76 | g = Github(username, password)
77 | github_col.save({'_id': md5(username), 'username': username, 'password': password,
78 | 'mask_password': password.replace(''.join(password[2:-2]), '****'), 'addat': timestamp(),
79 | 'rate_limit': int(g.get_rate_limit().search.limit),
80 | 'rate_remaining': int(g.get_rate_limit().search.remaining)})
81 | result = list(github_col.find({}, {'_id': 0}))
82 | data = {'status': 201, 'msg': '添加成功', 'result': result}
83 | except BadCredentialsException:
84 | data = {'status': 401, 'msg': '认证失败，请检查账号是否可用', 'result': []}
85 | return jsonify(data)
86 |
87 | def delete(self):
88 | parser = reqparse.RequestParser()
89 | parser.add_argument('username', type=str, help='')
90 | args = parser.parse_args()
91 | github_col.delete_many({'username': args.get('username')})
92 | result = list(github_col.find({}, {'_id': 0}))
93 | data = {'status': 404, 'msg': '删除成功', 'result': result}
94 | return jsonify(data)
95 |
96 |
97 | class Blacklist(Resource):
98 | def get(self):
99 | result = list(blacklist_col.find({}, {'_id': 0}))
100 | data = {'status': 200, 'msg': '获取信息成功', 'result': result}
101 | return jsonify(data)
102 |
103 | def post(self):
104 | parser = reqparse.RequestParser()
105 | parser.add_argument('text', type=str, help='')
106 | args = parser.parse_args()
107 | text = args.get('text')
108 | text = text.strip().replace(' ', '')
109 | blacklist_col.save({'_id': md5(text), 'text': text})
110 | result = list(blacklist_col.find({}, {'_id': 0}))
111 | data = {'status': 201, 'msg': '添加成功', 'result': result}
112 | return jsonify(data)
113 |
114 | def delete(self):
115 | parser = reqparse.RequestParser()
116 | parser.add_argument('text', type=str, help='')
117 | args = parser.parse_args()
118 | blacklist_col.delete_many({'text': args.get('text')})
119 | result = list(blacklist_col.find({}, {'_id': 0}))
120 | data = {'status': 404, 'msg': '删除成功', 'result': result}
121 | return jsonify(data)
122 |
123 |
124 | class SMTPServer(Resource):
125 | def get(self):
126 | result = setting_col.find_one({'key': 'mail'}, {'_id': 0})
127 | data = {'status': 200, 'msg': '获取信息成功', 'result': result}
128 | return jsonify(data)
129 |
130 | def post(self):
131 | parser = reqparse.RequestParser()
132 | parser.add_argument('from', type=str, help='From (sender email)')
133 | parser.add_argument('host', type=str, help='SMTPServer Host')
134 | parser.add_argument('port', type=int, help='SMTPServer Port')
135 | parser.add_argument('tls', type=inputs.boolean, default=False, help='Force TLS')
136 | parser.add_argument('username', type=str, help='Username')
137 | parser.add_argument('password', type=str, help='Password')
138 | parser.add_argument('domain', type=str, help='System URL Host')
139 | parser.add_argument('enabled', type=inputs.boolean, default=False, help='Enabled Mail Notice')
140 | parser.add_argument('test', type=inputs.boolean, default=False, help='Test Mail Notice')
141 | args = parser.parse_args()
142 | __setting = args
143 | setting_col.update_many({'key': 'mail'}, {'$set': dict({'key': 'mail'}, **__setting)}, upsert=True)
144 | result = setting_col.find_one({'key': 'mail'}, {'_id': 0})
145 | data = {'status': 201, 'msg': '设置成功', 'result': result}
146 | return jsonify(data)
147 |
148 |
149 | class WebHookNotice(Resource):
150 | """
151 | WebHook 通知
152 | """
153 |
154 | def get(self):
155 | result = list(setting_col.find({'webhook': {'$exists': True}}, {'_id': 0}))
156 | data = {'status': 200, 'msg': '获取信息成功', 'result': result}
157 | return jsonify(data)
158 |
159 | def delete(self):
160 | parser = reqparse.RequestParser()
161 | parser.add_argument('webhook', type=str, required=True, help='WebHook URL')
162 | args = parser.parse_args()
163 | delete_result = setting_col.delete_one({'webhook': args.get('webhook')})
164 | if delete_result.deleted_count == 1:
165 | data = {'status': 200, 'msg': '删除成功', 'result': []}
166 | else:
167 | data = {'status': 404, 'msg': '删除失败', 'result': []}
168 | return jsonify(data)
169 |
170 | def post(self):
171 | parser = reqparse.RequestParser()
172 | parser.add_argument('webhook', type=str, required=True, help='WebHook URL')
173 | parser.add_argument('domain', type=str, help='System URL Host')
174 | parser.add_argument('enabled', type=inputs.boolean, default=False, help='Enabled Notice')
175 | parser.add_argument('test', type=inputs.boolean, default=False, help='Test Notice')
176 | args = parser.parse_args()
177 | if urlparse(args.get('webhook')).netloc not in ['oapi.dingtalk.com', 'qyapi.weixin.qq.com'] or urlparse(
178 | args.get('webhook')).scheme != 'https':
179 | data = {'status': 400, 'msg': '错误的 webhook 地址', 'result': []}
180 | return jsonify(data)
181 | if args.get('test'):
182 | if urlparse(args.get('webhook')).netloc == 'oapi.dingtalk.com':
183 | test_content = {
184 | "msgtype": "markdown",
185 | "markdown": {"title": "GitHub泄露",
186 | "text": '### 规则名称: [WebHook告警测试]({})'.format(args.get('domain'))
187 | },
188 | "at": {
189 | "atMobiles": [
190 |
191 | ],
192 | "isAtAll": False
193 | }
194 | }
195 | else:
196 | test_content = {
197 | "msgtype": "markdown",
198 | "markdown": {
199 | "content": '### 规则名称: [WebHook告警测试]({})'.format(args.get('domain'))
200 | }
201 | }
202 |
203 | response = requests.post(
204 | args.get('webhook'),
205 | json=test_content)
206 | if response.ok:
207 | if response.json().get('errmsg') == 'ok':
208 | data = {'status': 201, 'msg': '已发送，请前往钉钉/企业微信群查看', 'result': []}
209 | else:
210 | data = {'status': 400, 'msg': '发送失败，WebHook 响应: {}'.format(response.json().get('errmsg')),
211 | 'result': []}
212 | return jsonify(data)
213 | else:
214 | data = {'status': 400, 'msg': '发送失败，请检查服务器网络', 'result': []}
215 | return jsonify(data)
216 | del args['test']
217 | setting_col.update_one({'webhook': args.get('webhook')}, {'$set': args}, upsert=True)
218 | result = setting_col.count({'webhook': args.get('webhook')})
219 | if result > 0:
220 | data = {'status': 201, 'msg': '设置成功', 'result': result}
221 | else:
222 | data = {'status': 400, 'msg': '设置失败', 'result': result}
223 | return jsonify(data)
224 |
225 |
226 | class Notice(Resource):
227 | def get(self):
228 | result = list(notice_col.find({}, {'_id': 0}))
229 | data = {'status': 200, 'msg': '获取信息成功', 'result': result}
230 | return jsonify(data)
231 |
232 | def post(self):
233 | parser = reqparse.RequestParser()
234 | parser.add_argument('mail', type=str, help='')
235 | args = parser.parse_args()
236 | mail = args.get('mail')
237 | mail = mail.strip().replace(' ', '')
238 | notice_col.insert_one({'_id': md5(mail), 'mail': mail})
239 | result = list(notice_col.find({}, {'_id': 0}))
240 | data = {'status': 201, 'msg': '添加成功', 'result': result}
241 | return jsonify(data)
242 |
243 | def delete(self):
244 | parser = reqparse.RequestParser()
245 | parser.add_argument('mail', type=str, help='')
246 | args = parser.parse_args()
247 | notice_col.delete_many({'mail': args.get('mail')})
248 | result = list(notice_col.find({}, {'_id': 0}))
249 | data = {'status': 404, 'msg': '删除成功', 'result': result}
250 | return jsonify(data)
251 |
252 |
253 | class Rule(Resource):
254 | def get(self):
255 | result = list(query_col.find({}).sort('enabled', -1))
256 | data = {'status': 200, 'msg': '获取信息成功', 'result': result}
257 | return jsonify(data)
258 |
259 | def post(self):
260 | parser = reqparse.RequestParser()
261 | parser.add_argument('keyword', type=str, help='')
262 | parser.add_argument('tag', type=str, help='')
263 | parser.add_argument('enabled', type=inputs.boolean, default=True, help='')
264 | args = parser.parse_args()
265 | if query_col.count({'tag': args.get('tag')}):
266 | query_col.update_one({'tag': args.get('tag')}, {'$set': args})
267 | msg = '更新成功'
268 | else:
269 | new_query = args
270 | new_query['_id'] = md5(''.join([str(v) for v in new_query.values()]))
271 | query_col.insert_one(new_query)
272 | msg = '添加成功'
273 | result = list(query_col.find({}).sort('enabled', -1))
274 | data = {'status': 200, 'msg': msg, 'result': result}
275 | return jsonify(data)
276 |
277 | def delete(self):
278 | parser = reqparse.RequestParser()
279 | parser.add_argument('_id', type=str, help='')
280 | parser.add_argument('tag', type=str, help='')
281 | args = parser.parse_args()
282 | query_col.delete_many({'_id': args.get('_id')})
283 | result_col.delete_many({'tag': args.get('tag')})
284 | result = list(query_col.find({}).sort('enabled', -1))
285 | data = {'status': 404, 'msg': '删除成功', 'result': result}
286 | return jsonify(data)
287 |
288 |
289 | class Query(Resource):
290 | def get(self):
291 | result = list(query_col.find({}).sort('enabled', -1))
292 | data = {'status': 200, 'msg': '获取信息成功', 'result': result}
293 | return jsonify(data)
294 |
295 | def post(self):
296 | parser = reqparse.RequestParser()
297 | parser.add_argument('keyword', type=str, help='')
298 | parser.add_argument('tag', type=str, help='')
299 | parser.add_argument('enabled', type=inputs.boolean, default=True, help='')
300 | args = parser.parse_args()
301 | if query_col.count({'tag': args.get('tag')}):
302 | query_col.update_one({'tag': args.get('tag')}, {'$set': args})
303 | msg = '更新成功'
304 | else:
305 | new_query = args
306 | new_query['_id'] = md5(''.join([str(v) for v in new_query.values()]))
307 | query_col.insert_one(new_query)
308 | msg = '添加成功'
309 | result = list(query_col.find({}).sort('enabled', -1))
310 | data = {'status': 200, 'msg': msg, 'result': result}
311 | return jsonify(data)
312 |
313 | def delete(self):
314 | parser = reqparse.RequestParser()
315 | parser.add_argument('_id', type=str, help='')
316 | parser.add_argument('tag', type=str, help='')
317 | args = parser.parse_args()
318 | query_col.delete_many({'_id': args.get('_id')})
319 | result_col.delete_many({'tag': args.get('tag')})
320 | result = list(query_col.find({}).sort('enabled', -1))
321 | data = {'status': 404, 'msg': '删除成功', 'result': result}
322 | return jsonify(data)
323 |
--------------------------------------------------------------------------------
/server/controllers/statistic.py:
--------------------------------------------------------------------------------
1 | from flask import jsonify
2 | from flask_restful import Resource, reqparse
3 | from config.database import blacklist_col, result_col, query_col, notice_col, github_col, setting_col
4 |
5 | from utils.date import today_start
6 | import psutil
7 |
8 |
9 | class Dashboard(Resource):
10 | def get(self):
11 | parser = reqparse.RequestParser()
12 | parser.add_argument('tag', type=str, help='')
13 | args = parser.parse_args()
14 | tag = args.get('tag')
15 | if tag:
16 | total = {'total': result_col.count({'tag': tag}),
17 | 'ignore': result_col.count({'tag': tag, 'security': 1}),
18 | 'risk': result_col.count(
19 | {'tag': tag, 'security': 0, "desc": {"$exists": True}})}
20 | today = {
21 | 'total': result_col.count({'tag': tag, 'timestamp': {'$gte': today_start()}}),
22 | 'ignore': result_col.count({'tag': tag, 'timestamp': {'$gte': today_start()}, 'security': 1}),
23 | 'risk': result_col.count(
24 | {'tag': tag, 'timestamp': {'$gte': today_start()}, 'security': 0, "desc": {"$exists": True}}),
25 | }
26 | else:
27 | total = {'total': result_col.count(),
28 | 'ignore': result_col.count({'security': 1}),
29 | 'risk': result_col.count(
30 | {'security': 0, "desc": {"$exists": True}})}
31 | today = {
32 | 'total': result_col.count({'timestamp': {'$gte': today_start()}}),
33 | 'ignore': result_col.count({'timestamp': {'$gte': today_start()}, 'security': 1}),
34 | 'risk': result_col.count(
35 | {'timestamp': {'$gte': today_start()}, 'security': 0, "desc": {"$exists": True}}),
36 | }
37 | if setting_col.count({'key': 'task'}):
38 | status = psutil.pid_exists(int(setting_col.find_one({'key': 'task'}).get('pid')))
39 | last = setting_col.find_one({'key': 'task'}).get('last')
40 | else:
41 | status = False
42 | last = 0
43 | engine = {
44 | 'status': status,
45 | 'last': last,
46 | }
47 | result = {
48 | 'all': total,
49 | 'today': today,
50 | 'engine': engine
51 | }
52 | data = {'status': 200, 'msg': '获取信息成功', 'result': result}
53 | return jsonify(data)
54 |
55 |
56 | class Statistic(Resource):
57 | def get(self):
58 | parser = reqparse.RequestParser()
59 | parser.add_argument('by', type=str, default='tag', help='')
60 | parser.add_argument('tag', type=str, help='')
61 | args = parser.parse_args()
62 | by = args.get('by')
63 | tag = args.get('tag')
64 | if len(tag):
65 | filters = {'tag': tag, 'security': 0}
66 | else:
67 | filters = {'security': 0}
68 | pipeline = [
69 | {'$match': filters},
70 | {'$group': {'_id': '${}'.format(by), 'value': {'$sum': 1}}},
71 | ]
72 |
73 | result = list(result_col.aggregate(pipeline))
74 | if not len(result):
75 | pipeline = [
76 | {'$group': {'_id': '${}'.format(by), 'value': {'$sum': 0}}},
77 | ]
78 | result = list(result_col.aggregate(pipeline))
79 | data = {'status': 200, 'msg': '获取信息成功', 'result': result}
80 | return jsonify(data)
81 |
--------------------------------------------------------------------------------
/server/task.py:
--------------------------------------------------------------------------------
1 | import datetime
2 | import os
3 | import re
4 | import random
5 | import requests
6 | import smtplib
7 | import time
8 | from ipaddress import ip_address
9 | import tldextract
10 | from github import Github
11 | from huey import RedisHuey, crontab
12 | from pymongo import errors, DESCENDING, ASCENDING
13 | from config.database import result_col, query_col, blacklist_col, notice_col, github_col, setting_col, REDIS_HOST, \
14 | REDIS_PORT
15 | from utils.date import timestamp
16 | from utils.log import logger
17 | from utils.notice import mail_notice
18 |
19 | huey = RedisHuey('hawkeye', host=REDIS_HOST, port=int(REDIS_PORT))
20 | base_path = os.path.split(os.path.realpath(__file__))[0]
21 | extract = tldextract.TLDExtract(cache_file='{}/.tld_set'.format(base_path))
22 |
23 | if setting_col.count({'key': 'task', 'minute': {'$exists': True}, 'page': {'$exists': True}}):
24 | minute = int(setting_col.find_one({'key': 'task'}).get('minute'))
25 | setting_col.update_one({'key': 'task'}, {'$set': {'key': 'task', 'pid': os.getpid(), 'last': timestamp()}},
26 | upsert=True)
27 |
28 | else:
29 | minute = 10
30 | setting_col.update_one({'key': 'task'},
31 | {'$set': {'key': 'task', 'pid': os.getpid(), 'minute': 10, 'page': 3, 'last': timestamp()}},
32 | upsert=True)
33 |
34 |
35 | @huey.task()
36 | def search(query, page, g, github_username):
37 | mail_notice_list = []
38 | webhook_notice_list = []
39 | logger.info('开始抓取: tag is {} keyword is {}, page is {}'.format(
40 | query.get('tag'), query.get('keyword'), page + 1))
41 | try:
42 | repos = g.search_code(query=query.get('keyword'),
43 | sort="indexed", order="desc")
44 | github_col.update_one({'username': github_username},
45 | {'$set': {'rate_remaining': int(g.get_rate_limit().search.remaining)}})
46 |
47 | except Exception as error:
48 | logger.critical(error)
49 | logger.critical("触发限制啦")
50 | return
51 | try:
52 | for repo in repos.get_page(page):
53 | setting_col.update_one({'key': 'task'}, {'$set': {'key': 'task', 'pid': os.getpid(), 'last': timestamp()}},
54 | upsert=True)
55 | if not result_col.count({'_id': repo.sha}):
56 | try:
57 | code = str(repo.content).replace('\n', '')
58 | except:
59 | code = ''
60 | leakage = {
61 | 'link': repo.html_url,
62 | 'project': repo.repository.full_name,
63 | 'project_url': repo.repository.html_url,
64 | '_id': repo.sha,
65 | 'language': repo.repository.language,
66 | 'username': repo.repository.owner.login,
67 | 'avatar_url': repo.repository.owner.avatar_url,
68 | 'filepath': repo.path,
69 | 'filename': repo.name,
70 | 'security': 0,
71 | 'ignore': 0,
72 | 'tag': query.get('tag'),
73 | 'code': code,
74 | }
75 | try:
76 | leakage['affect'] = get_affect_assets(repo.decoded_content)
77 | except Exception as error:
78 | logger.critical('{} {}'.format(error, leakage.get('link')))
79 | leakage['affect'] = []
80 | if int(repo.raw_headers.get('x-ratelimit-remaining')) == 0:
81 | logger.critical('剩余使用次数: {}'.format(
82 | repo.raw_headers.get('x-ratelimit-remaining')))
83 | return
84 | last_modified = datetime.datetime.strptime(
85 | repo.last_modified, '%a, %d %b %Y %H:%M:%S %Z')
86 | leakage['datetime'] = last_modified
87 | leakage['timestamp'] = last_modified.timestamp()
88 | in_blacklist = False
89 | for blacklist in blacklist_col.find({}):
90 | if blacklist.get('text').lower() in leakage.get('link').lower():
91 | logger.warning('{} 包含白名单中的 {}'.format(
92 | leakage.get('link'), blacklist.get('text')))
93 | in_blacklist = True
94 | if in_blacklist:
95 | continue
96 | if result_col.count({"project": leakage.get('project'), "ignore": 1}):
97 | continue
98 | if not result_col.count(
99 | {"project": leakage.get('project'), "filepath": leakage.get("filepath"), "security": 0}):
100 | mail_notice_list.append(
101 | '上传时间:{} 地址: {}/{}'.format(leakage.get('datetime'), leakage.get('link'),
102 | leakage.get('project'), leakage.get('filename')))
103 | webhook_notice_list.append(
104 | '[{}/{}]({}) 上传于 {}'.format(leakage.get('project').split('.')[-1],
105 | leakage.get('filename'), leakage.get('link'),
106 | leakage.get('datetime')))
107 | try:
108 | result_col.insert_one(leakage)
109 | logger.info(leakage.get('project'))
110 | except errors.DuplicateKeyError:
111 | logger.info('已存在')
112 |
113 | logger.info('抓取关键字：{} {}'.format(query.get('tag'), leakage.get('link')))
114 | except Exception as error:
115 | if 'Not Found' not in error.data:
116 | g, github_username = new_github()
117 | search.schedule(
118 | args=(query, page, g, github_username),
119 | delay=huey.pending_count() + huey.scheduled_count())
120 | logger.critical(error)
121 | logger.error('抓取: tag is {} keyword is {}, page is {} 失败'.format(
122 | query.get('tag'), query.get('keyword'), page + 1))
123 |
124 | return
125 | logger.info('抓取: tag is {} keyword is {}, page is {} 成功'.format(
126 | query.get('tag'), query.get('keyword'), page + 1))
127 | query_col.update_one({'tag': query.get('tag')},
128 | {'$set': {'last': int(time.time()), 'status': 1, 'reason': '抓取第{}页成功'.format(page),
129 | 'api_total': repos.totalCount,
130 | 'found_total': result_col.count({'tag': query.get('tag')})}})
131 | if setting_col.count({'key': 'mail', 'enabled': True}) and len(mail_notice_list):
132 | main_content = '规则名称: {}
{}'.format(query.get('tag'), '
'.join(mail_notice_list)) 133 | send_mail(main_content) 134 | logger.info(len(webhook_notice_list)) 135 | webhook_notice(query.get('tag'), webhook_notice_list) 136 | 137 | 138 | @huey.task() 139 | def webhook_notice(tag, results): 140 | if len(results): 141 | for webhook_setting in setting_col.find({'webhook': {'$exists': True}, 'enabled': True}, 142 | {'domain': 1, 'webhook': 1, '_id': 0}): 143 | hostname = webhook_setting.get('domain') 144 | webhook = webhook_setting.get('webhook') 145 | if 'oapi.dingtalk.com' in webhook: 146 | content = { 147 | "msgtype": "markdown", 148 | "markdown": {"title": "GitHub泄露", 149 | "text": '#### [规则名称: {}]({}/?tag={})\n\n- {}'.format(tag, hostname, tag, 150 | '\n- '.join(results)) 151 | }, 152 | "at": { 153 | "atMobiles": [ 154 | 155 | ], 156 | "isAtAll": False 157 | } 158 | } 159 | else: 160 | 161 | content = { 162 | "msgtype": "markdown", 163 | "markdown": { 164 | "content": '#### [规则名称: {}]({}/?tag={})\n\n- {}'.format(tag, hostname, tag, 165 | '\n- '.join(results)) 166 | } 167 | } 168 | try: 169 | requests.post( 170 | webhook, 171 | json=content) 172 | except Exception as error: 173 | logger.error(error) 174 | 175 | 176 | def get_domain(target): 177 | result = extract(target) 178 | if bool(len(result.suffix)) and bool(len(result.domain)): 179 | return '{}.{}'.format(result.domain, result.suffix) 180 | else: 181 | return False 182 | 183 | 184 | def get_affect_assets(code): 185 | """ 186 | 187 | :param code: 188 | :return: 189 | """ 190 | code = str(code) 191 | affect = [] 192 | domain_pattern = '(?!\-)(?:[a-zA-Z\d\-]{0,62}[a-zA-Z\d]\.){1,126}(?!\d+)[a-zA-Z\d]{1,63}' 193 | ip_pattern = "(\d+\.\d+\.\d+\.\d+)" 194 | email_pattern = "[\w!#$%&'*+/=?^_`{|}~-]+(?:\.[\w!#$%&'*+/=?^_`{|}~-]+)*@(?:[\w](?:[\w-]*[\w])?\.)+[\w](?:[\w-]*[\w])?" 195 | affect_assets = { 196 | 'domain': list(set(re.findall(domain_pattern, code))), 197 | 'email': list(set(re.findall(email_pattern, code))), 198 | 'ip': list(set(re.findall(ip_pattern, code))), 199 | } 200 | for assets in affect_assets.keys(): 201 | if len(affect_assets.get(assets)) > 100: 202 | affect_assets[assets] = [] 203 | continue 204 | for asset in affect_assets.get(assets): 205 | if assets == 'ip': 206 | if not is_ip(asset): 207 | continue 208 | if assets == 'domain': 209 | if not get_domain(asset): 210 | continue 211 | if assets == 'email': 212 | if not get_domain(asset.split('@')[-1]): 213 | continue 214 | affect.append({'type': assets, 'value': asset.replace( 215 | "'", "").replace('"', '').replace("`", "").lower()}) 216 | return affect 217 | 218 | 219 | def is_ip(ip): 220 | """ 221 | 222 | :param ip: 223 | :return: 224 | """ 225 | try: 226 | ip_address(ip) 227 | return True 228 | except ValueError: 229 | return False 230 | 231 | 232 | @huey.task() 233 | def send_mail(content): 234 | smtp_config = setting_col.find_one({'key': 'mail'}) 235 | receivers = [data.get('mail') for data in notice_col.find({})] 236 | try: 237 | if mail_notice(smtp_config, receivers, content): 238 | logger.info('邮件发送成功') 239 | else: 240 | logger.critical('Error: 无法发送邮件') 241 | 242 | except smtplib.SMTPException as error: 243 | logger.critical('Error: 无法发送邮件 {}'.format(error)) 244 | 245 | 246 | @huey.periodic_task(crontab(minute='*/2')) 247 | def update_rate_remain(): 248 | for account in github_col.find(): 249 | github_username = account.get('username') 250 | github_password = account.get('password') 251 | try: 252 | g = Github(github_username, github_password) 253 | github_col.update_one({'username': github_username}, 254 | {'$set': {'rate_remaining': int(g.get_rate_limit().search.remaining), 255 | 'rate_limit': int(g.get_rate_limit().search.limit)}}) 256 | except Exception as error: 257 | logger.error(error) 258 | 259 | 260 | def new_github(): 261 | if github_col.count({'rate_remaining': {'$gt': 5}}): 262 | pass 263 | else: 264 | logger.error('请配置github账号') 265 | return 266 | github_account = random.choice(list(github_col.find({"rate_limit": {"$gt": 5}}).sort('rate_remaining', DESCENDING))) 267 | github_username = github_account.get('username') 268 | github_password = github_account.get('password') 269 | g = Github(github_username, github_password) 270 | return g, github_username 271 | 272 | 273 | @huey.periodic_task(crontab(minute='*/{}'.format(minute))) 274 | def check(): 275 | setting_col.update_one({'key': 'task'}, {'$set': {'key': 'task', 'pid': os.getpid()}}, upsert=True) 276 | query_count = query_col.count({'enabled': True}) 277 | logger.info('需要处理的关键词总数: {}'.format(query_count)) 278 | if query_count: 279 | logger.info('需要处理的关键词总数: {}'.format(query_count)) 280 | else: 281 | logger.warning('请添加关键词') 282 | return 283 | if github_col.count({'rate_remaining': {'$gt': 5}}): 284 | pass 285 | else: 286 | logger.error('请配置github账号') 287 | return 288 | 289 | if setting_col.count({'key': 'task', 'page': {'$exists': True}}): 290 | setting_col.update_one({'key': 'task'}, {'$set': {'pid': os.getpid()}}) 291 | page = int(setting_col.find_one({'key': 'task'}).get('page')) 292 | for p in range(0, page): 293 | for query in query_col.find({'enabled': True}).sort('last', ASCENDING): 294 | github_account = random.choice( 295 | list(github_col.find({"rate_limit": {"$gt": 5}}).sort('rate_remaining', DESCENDING))) 296 | github_username = github_account.get('username') 297 | github_password = github_account.get('password') 298 | rate_remaining = github_account.get('rate_remaining') 299 | logger.info(github_username) 300 | logger.info(rate_remaining) 301 | g = Github(github_username, github_password, 302 | user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36') 303 | search.schedule(args=(query, p, g, github_username), 304 | delay=huey.pending_count() + huey.scheduled_count()) 305 | else: 306 | logger.error('请在页面上配置任务参数') 307 | -------------------------------------------------------------------------------- /server/utils/asset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/0xbug/Hawkeye/3ef48c27c47c86c8bfe013b4a266e08534c821d4/server/utils/asset.py -------------------------------------------------------------------------------- /server/utils/date.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | 4 | def timestamp(): 5 | return int(time.time()) 6 | 7 | 8 | def today_start(): 9 | zero_timestamp = time.mktime( 10 | time.strptime(time.strftime('%Y-%m-%d 00:00:00', time.localtime(time.time())), '%Y-%m-%d %H:%M:%S')) 11 | 12 | return int(zero_timestamp) 13 | -------------------------------------------------------------------------------- /server/utils/hash.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | 3 | 4 | def sha256(data): 5 | return hashlib.sha256(data.encode('utf-8')).hexdigest() 6 | 7 | 8 | def sha1(data): 9 | return hashlib.sha1(data.encode('utf-8')).hexdigest() 10 | 11 | 12 | def md5(data): 13 | m = hashlib.md5() 14 | m.update(data.encode('utf-8')) 15 | results = m.hexdigest() 16 | return results 17 | -------------------------------------------------------------------------------- /server/utils/log.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import syslog 4 | 5 | FORMAT = '%(asctime)s [%(name)s] [%(levelname)s] [%(pathname)s:%(lineno)s %(funcName)s] %(message)s' 6 | logging.basicConfig(filename='Hawkeye.log', level=logging.INFO, format=FORMAT) 7 | logger = logging.getLogger() 8 | -------------------------------------------------------------------------------- /server/utils/notice.py: -------------------------------------------------------------------------------- 1 | import smtplib 2 | from email.header import Header 3 | from email.mime.text import MIMEText 4 | 5 | 6 | class SMTPServer(object): 7 | def __init__(self, smtp_config): 8 | self.host = smtp_config.get('host') 9 | self.port = int(smtp_config.get('port')) 10 | self.tls = smtp_config.get('tls') 11 | self.username = smtp_config.get('username') 12 | self.password = smtp_config.get('password') 13 | try: 14 | if self.tls: 15 | self.smtp = smtplib.SMTP(self.host, self.port, timeout=300) 16 | else: 17 | self.smtp = smtplib.SMTP_SSL(self.host, self.port, timeout=300) 18 | except Exception as error: 19 | print(error) 20 | 21 | def login(self): 22 | try: 23 | if self.tls: 24 | self.smtp.starttls() 25 | self.smtp.login(self.username, self.password) 26 | except Exception as error: 27 | print(error) 28 | 29 | def sendmail(self, receivers, message): 30 | """ 31 | 32 | :param receivers: 33 | :param message: 34 | :return: 35 | """ 36 | message = message.as_string() 37 | self.smtp.sendmail(self.username, receivers, message) 38 | 39 | 40 | def mail_notice(smtp_config, receivers, content): 41 | """ 42 | :param receivers: 43 | :param smtp_config: 44 | :param content: 45 | :return: 46 | """ 47 | 48 | message = MIMEText(content, _subtype='html', _charset='utf-8') 49 | message['From'] = Header('{}<{}>'.format(smtp_config.get('from'), smtp_config.get('username')), 'utf-8') 50 | message['To'] = Header(';'.join(receivers), 'utf-8') 51 | message['Subject'] = Header('[GitHub] 监控告警', 'utf-8') 52 | try: 53 | smtp = SMTPServer(smtp_config) 54 | print('login') 55 | smtp.login() 56 | smtp.sendmail(receivers, message) 57 | return True 58 | 59 | except smtplib.SMTPException: 60 | return False 61 | --------------------------------------------------------------------------------