(.*?)
', _desc, re.DOTALL)[0].strip() 79 | cve.title = cve.info 80 | return cve 81 | -------------------------------------------------------------------------------- /src/crawler/vas.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Author : EXP 4 | # @Time : 2020/4/28 14:38 5 | # @File : vas.py 6 | # ----------------------------------------------- 7 | # 斗象:https://vas.riskivy.com/vuln 8 | # ----------------------------------------------- 9 | 10 | from src.bean.cve_info import CVEInfo 11 | from src.crawler._base_crawler import BaseCrawler 12 | from color_log.clog import log 13 | import requests 14 | import json 15 | import re 16 | import time 17 | 18 | 19 | class Vas(BaseCrawler): 20 | 21 | def __init__(self): 22 | BaseCrawler.__init__(self) 23 | self.name_ch = '斗象' 24 | self.name_en = 'vas' 25 | self.home_page = 'https://vas.riskivy.com/vuln' 26 | self.url_list = 'https://console.riskivy.com/vas' 27 | self.url_details = 'https://console.riskivy.com/vas/' 28 | self.url_cve = 'https://vas.riskivy.com/vuln-detail?id=' 29 | 30 | 31 | def NAME_CH(self): 32 | return self.name_ch 33 | 34 | 35 | def NAME_EN(self): 36 | return self.name_en 37 | 38 | 39 | def HOME_PAGE(self): 40 | return self.home_page 41 | 42 | 43 | def get_cves(self, limit = 5): 44 | params = { 45 | 'title': '', 46 | 'cve' : '', 47 | 'cnvd': '', 48 | 'cnnvd': '', 49 | 'order': 'update', 50 | 'has_poc': '', 51 | 'has_repair': '', 52 | 'bug_level': '', 53 | 'page': 1, 54 | 'per-page': limit, 55 | } 56 | 57 | response = requests.get( 58 | self.url_list, 59 | headers = self.headers(), 60 | params = params, 61 | timeout = self.timeout 62 | ) 63 | 64 | cves = [] 65 | if response.status_code == 200: 66 | json_obj = json.loads(response.text) 67 | for obj in json_obj.get('data').get('items'): 68 | cve = self.to_cve(obj) 69 | if cve.is_vaild(): 70 | cves.append(cve) 71 | # log.debug(cve) 72 | else: 73 | log.warn('获取 [%s] 威胁情报失败: [HTTP Error %i]' % (self.NAME_CH(), response.status_code)) 74 | return cves 75 | 76 | 77 | def to_cve(self, json_obj): 78 | cve = CVEInfo() 79 | cve.src = self.NAME_CH() 80 | 81 | id = str(json_obj.get('id')) or '' 82 | cve.url = self.url_cve + id 83 | cve.title = json_obj.get('bug_title') or '' 84 | 85 | seconds = json_obj.get('updated_at') or 0 86 | localtime = time.localtime(seconds) 87 | cve.time = time.strftime('%Y-%m-%d %H:%M:%S', localtime) 88 | 89 | self.get_cve_info(cve, id) 90 | return cve 91 | 92 | 93 | def get_cve_info(self, cve, id): 94 | url = self.url_details + id 95 | response = requests.get( 96 | url, 97 | headers = self.headers(), 98 | timeout = self.timeout 99 | ) 100 | 101 | if response.status_code == 200: 102 | json_obj = json.loads(response.text) 103 | cve.id = json_obj.get('data').get('bug_cve').replace(',', ', ') 104 | cve.info = json_obj.get('data').get('detail').get('bug_description') 105 | cve.info = re.sub(r'<.*?>', '', cve.info) 106 | 107 | time.sleep(0.1) 108 | 109 | -------------------------------------------------------------------------------- /src/dao/_base.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Author : EXP 4 | # @Time : 2020/4/29 23:31 5 | # @File : _base.py 6 | # ----------------------------------------------- 7 | # 数据访问对象:基类 8 | # ----------------------------------------------- 9 | 10 | from color_log.clog import log 11 | 12 | 13 | class BaseDao: 14 | """ 15 | Dao 基类 16 | """ 17 | 18 | # TODO 需子类实现 19 | TABLE_NAME = "" 20 | SQL_COUNT = "" 21 | SQL_TRUNCATE = "" 22 | SQL_INSERT = "" 23 | SQL_DELETE = "" 24 | SQL_UPDATE = "" 25 | SQL_SELECT = "" 26 | CHARSET = "utf-8" 27 | 28 | def __init__(self): 29 | pass 30 | 31 | 32 | def count(self, conn): 33 | """ 34 | 统计行数 35 | :param conn: 数据库连接 36 | :return: 表行数 37 | """ 38 | cnt = 0 39 | try: 40 | cursor = conn.cursor() 41 | cursor.execute(self.SQL_COUNT) 42 | cnt = cursor.fetchone()[0] 43 | cursor.close() 44 | except: 45 | log.error("统计表 [%s] 行数失败" % self.TABLE_NAME) 46 | return cnt 47 | 48 | 49 | def truncate(self, conn): 50 | """ 51 | 清空表 52 | :param conn: 数据库连接 53 | :return: 是否清空成功 54 | """ 55 | is_ok = False 56 | try: 57 | cursor = conn.cursor() 58 | cursor.execute(self.SQL_TRUNCATE) 59 | conn.commit() 60 | cursor.close() 61 | is_ok = True 62 | except: 63 | log.error("清空表 [%s] 失败" % self.TABLE_NAME) 64 | return is_ok 65 | 66 | 67 | def insert(self, conn, bean): 68 | """ 69 | 插入单条数据 70 | :param conn: 数据库连接 71 | :param bean: 数据模型实例 72 | :return: 是否插入成功 73 | """ 74 | is_ok = False 75 | try: 76 | cursor = conn.cursor() 77 | params = bean.params() 78 | cursor.execute(self.SQL_INSERT, params) 79 | conn.commit() 80 | cursor.close() 81 | is_ok = True 82 | except: 83 | log.error("插入数据到表 [%s] 失败" % self.TABLE_NAME) 84 | return is_ok 85 | 86 | 87 | def insert_all(self, conn, beans): 88 | """ 89 | 插入多条数据 90 | :param conn: 数据库连接 91 | :param beans: 数据模型实例队列 92 | :return: 成功插入个数 93 | """ 94 | cnt = 0 95 | try: 96 | cursor = conn.cursor() 97 | for bean in beans: 98 | try: 99 | params = bean.params() 100 | cursor.execute(self.SQL_INSERT, params) 101 | cnt += 1 102 | except: 103 | log.error("插入数据到表 [%s] 失败" % self.TABLE_NAME) 104 | conn.commit() 105 | cursor.close() 106 | except: 107 | log.error("插入数据集到表 [%s] 失败" % self.TABLE_NAME) 108 | return cnt 109 | 110 | 111 | def delete(self, conn, wheres={}): 112 | """ 113 | 删除数据 114 | :param conn: 数据库连接 115 | :param wheres: 条件键值对, 要求键值包含操作符,如: { 'column1 like': 'xyz', 'column2 =': 'abc' } 116 | :return: 是否删除成功 117 | """ 118 | is_ok = False 119 | try: 120 | cursor = conn.cursor() 121 | sql = self._append(self.SQL_DELETE, wheres.keys()) 122 | cursor.execute(sql, wheres.values()) 123 | conn.commit() 124 | cursor.close() 125 | is_ok = True 126 | except: 127 | log.error("从表 [%s] 删除数据失败" % self.TABLE_NAME) 128 | return is_ok 129 | 130 | 131 | def update(self, conn, bean): 132 | """ 133 | 更新数据 134 | :param conn: 数据库连接 135 | :param bean: 数据模型实例 136 | :return: 是否更新成功 137 | """ 138 | is_ok = False 139 | try: 140 | cursor = conn.cursor() 141 | sql = self._append(self.SQL_UPDATE, ["%s = " % bean.i_id]) 142 | params = bean.params() + (bean.id,) 143 | cursor.execute(sql, params) 144 | conn.commit() 145 | cursor.close() 146 | is_ok = True 147 | except: 148 | log.error("更新数据到表 [%s] 失败" % self.TABLE_NAME) 149 | return is_ok 150 | 151 | 152 | def query_all(self, conn): 153 | """ 154 | 查询表中所有数据 155 | :param conn: 数据库连接 156 | :return: 数据模型实例队列(失败返回 [] ,不会为 None) 157 | """ 158 | return self.query_some(conn) 159 | 160 | 161 | def query_some(self, conn, wheres={}): 162 | """ 163 | 查询表中部分数据 164 | :param conn: 数据库连接 165 | :param wheres: 条件键值对, 要求键值包含操作符,如: { 'column1 like': 'xyz', 'column2 =': 'abc' } 166 | :return: 数据模型实例队列(失败返回 [] ,不会为 None) 167 | """ 168 | beans = [] 169 | try: 170 | cursor = conn.cursor() 171 | sql = self._append(self.SQL_SELECT, wheres.keys()) 172 | cursor.execute(sql, wheres.values()) 173 | rows = cursor.fetchall() 174 | for row in rows: 175 | bean = self._to_bean(row) 176 | beans.append(bean) 177 | cursor.close() 178 | except: 179 | log.error("从表 [%s] 查询数据失败" % self.TABLE_NAME) 180 | return beans 181 | 182 | 183 | def query_one(self, conn, wheres={}): 184 | """ 185 | 查询表中一条数据 186 | :param conn: 数据库连接 187 | :param wheres: 条件键值对, 要求键值包含操作符,如: { 'column1 like': 'xyz', 'column2 =': 'abc' } 188 | :return: 数据模型实例队列(若多个满足则返回第 1 个,没有满足则返回 None) 189 | """ 190 | bean = None 191 | try: 192 | cursor = conn.cursor() 193 | sql = self._append(self.SQL_SELECT, wheres.keys()) 194 | cursor.execute(sql, wheres.values()) 195 | row = cursor.fetchone() 196 | bean = self._to_bean(row) 197 | conn.commit() 198 | cursor.close() 199 | except: 200 | log.error("从表 [%s] 查询数据失败" % self.TABLE_NAME) 201 | return bean 202 | 203 | 204 | def _append(self, sql, keys): 205 | """ 206 | 追加 where 条件到 sql, 条件之间只为 and 关系(目的只是支持简单的数据库操作) 207 | :param sql: 语句 208 | :param keys: 条件键值集合, 要求键值包含操作符,如: [ 'column1 like', 'column2 =' ] 209 | :return: 追加 where 条件后的 sql 210 | """ 211 | _sql = sql 212 | if keys: 213 | for key in keys: 214 | _sql = " ".join((_sql, "and", key, " ?")) # ? 是 sql 占位符,目的是防注入 215 | return _sql 216 | 217 | 218 | def _to_bean(self, row): 219 | """ 220 | 把数据库查询的单行结果转换成模型实例对象 221 | :param row: 单行查询结果 222 | :return: 模型实例对象 223 | """ 224 | # 需子类实现 225 | return row 226 | 227 | 228 | def _to_val(self, row, idx): 229 | """ 230 | 把 unicode 编码的字符串转换成 utf8 231 | :param row: 行对象 232 | :param idx: 列索引 233 | :return: 列值(utf8 编码) 234 | """ 235 | val = None 236 | try: 237 | val = row[idx] 238 | if val is not None and isinstance(val, unicode): 239 | val = val.encode(self.CHARSET) 240 | except: 241 | pass 242 | return val 243 | 244 | -------------------------------------------------------------------------------- /src/dao/t_cves.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Author : EXP 4 | # @Time : 2020/4/29 23:32 5 | # @File : t_cves.py 6 | # ----------------------------------------------- 7 | # DAO: t_cves 8 | # ----------------------------------------------- 9 | 10 | from src.bean.t_cves import TCves 11 | from src.dao._base import BaseDao 12 | 13 | 14 | class TCvesDao(BaseDao): 15 | 16 | TABLE_NAME = "t_cves" 17 | SQL_COUNT = "select count(1) from t_cves" 18 | SQL_TRUNCATE = "truncate table t_cves" 19 | SQL_INSERT = "insert into t_cves(s_md5, s_src, s_cves, s_title, s_time, s_info, s_url) values (?, ?, ?, ?, ?, ?, ?)" 20 | SQL_DELETE = "delete from t_cves where 1 = 1 " 21 | SQL_UPDATE = "update t_cves set s_md5 = ?, s_src = ?, s_cves = ?, s_title = ?, s_time = ?, s_info = ?, s_url = ? where 1 = 1 " 22 | SQL_SELECT = "select s_md5, s_src, s_cves, s_title, s_time, s_info, s_url from t_cves where 1 = 1 " 23 | 24 | 25 | def __init__(self): 26 | BaseDao.__init__(self) 27 | 28 | 29 | def _to_bean(self, row): 30 | bean = None 31 | if row: 32 | bean = TCves() 33 | bean.md5 = self._to_val(row, 0) 34 | bean.src = self._to_val(row, 1) 35 | bean.cves = self._to_val(row, 2) 36 | bean.title = self._to_val(row, 3) 37 | bean.time = self._to_val(row, 4) 38 | bean.info = self._to_val(row, 5) 39 | bean.url = self._to_val(row, 6) 40 | return bean 41 | 42 | -------------------------------------------------------------------------------- /src/notice/mail.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Author : EXP 4 | # @Time : 2020/4/30 23:29 5 | # ----------------------------------------------- 6 | # 通过邮件发送威胁情报 7 | # ----------------------------------------------- 8 | 9 | import os 10 | import re 11 | import smtplib 12 | from email.mime.text import MIMEText 13 | from email.header import Header 14 | import src.config as config 15 | from color_log.clog import log 16 | from src.utils import _git 17 | 18 | MAIL_TPL_PATH = '%s/tpl/mail.tpl' % config.PRJ_DIR 19 | MAIL_RECV_DIR = '%s/recv' % config.PRJ_DIR 20 | 21 | MAIL_CONTENT_CACHE = '%s/cache/mail_content.dat' % config.PRJ_DIR 22 | MAIL_RECV_CACHE = '%s/cache/mail_recvs.dat' % config.PRJ_DIR 23 | 24 | 25 | def to_mail(gtk, cves, smtp, sender, password): 26 | content = format_content(cves) 27 | receivers = load_local_receivers() 28 | if gtk: 29 | log.info('[邮件] 正在通过 Github Actions 推送威胁情报...') 30 | recvs = load_issue_receivers(gtk) 31 | recvs.update(receivers) 32 | to_cache(','.join(recvs), MAIL_RECV_CACHE) 33 | to_cache(content, MAIL_CONTENT_CACHE) 34 | 35 | else: 36 | log.info('[邮件] 正在推送威胁情报...') 37 | email = MIMEText(content, 'html', config.CHARSET) # 以 html 格式发送邮件内容 38 | email['From'] = sender 39 | email['To'] = ', '.join(receivers) # 此处收件人列表必须为逗号分隔的 str 40 | log.info('[邮件] 收件人清单: %s' % receivers) 41 | subject = '威胁情报播报' 42 | email['Subject'] = Header(subject, 'utf-8') 43 | 44 | try: 45 | smtpObj = smtplib.SMTP(smtp) 46 | smtpObj.login(sender, password) 47 | smtpObj.sendmail(sender, receivers, email.as_string()) # 此处收件人列表必须为 list 48 | log.info('[邮件] 推送威胁情报成功') 49 | except: 50 | log.error('[邮件] 推送威胁情报失败') 51 | 52 | 53 | def format_content(cves): 54 | src_tpl = '![]() |
35 | ![]() |
36 |
---|---|
团队GitHub | 39 |团队公众号 | 40 |
%(src)s [TOP %(top)d] | 6 |CVES | 7 |TIME | 8 |TITLE | 9 |URL | 10 |
---|