", "").replace("
", "")) 86 | count = int(content['pulse_info']['count']) 87 | subs_list = [] 88 | for i in range(count): 89 | subs_list.append(content['pulse_info']['pulses'][i]['subscriber_count']) 90 | tactic_list = [] 91 | for i, item in enumerate(subs_list): 92 | max_test = max(subs_list) 93 | if content['pulse_info']['pulses'][i]['attack_ids']: 94 | for j in range(len(content['pulse_info']['pulses'][i]['attack_ids'])): 95 | tactic_list.append(content['pulse_info']['pulses'][i]['attack_ids'][j]['id']) 96 | break 97 | else: 98 | subs_list.remove(max_test) 99 | else: 100 | return f'NO TTP' 101 | tactic_list = list(set(tactic_list)) # replace replications 102 | technique_list = [] 103 | for i, item in enumerate(tactic_list): 104 | if len(str(item)) > 6: 105 | technique_list.append(str(item).replace(".", "/")) 106 | 107 | result_list = [] 108 | if technique_list: 109 | for tactic_id in technique_list: 110 | if get_mitigations(tactic_id): 111 | for item in get_mitigations(tactic_id): 112 | result_list.append(item) 113 | return list(set(result_list)) 114 | else: 115 | for tactic_id in tactic_list: 116 | if get_mitigations(tactic_id): 117 | for item in get_mitigations(tactic_id): 118 | result_list.append(item) 119 | return list(set(result_list)) 120 | 121 | 122 | # check info for cve on microsoft--------------------------------------------------------------------------------------- 123 | def check_microsoft(cve): 124 | # print('check_microsoft') # DEBUG 125 | msrc_url = f"https://api.msrc.microsoft.com/cvrf/v2.0/Updates('{cve}')" 126 | get_cvrf_link = requests.get(msrc_url, verify=False) 127 | return get_cvrf_link.status_code 128 | 129 | 130 | # get KB links for cve-------------------------------------------------------------------------------------------------- 131 | def get_kb(cve): 132 | # print('get_kb') # DEBUG 133 | msrc_url = f"https://api.msrc.microsoft.com/cvrf/v2.0/Updates('{cve}')" 134 | get_cvrf_link = requests.get(msrc_url, verify=False) 135 | id_for_cvrf = re.search(r'\d{4}-\w{3}', get_cvrf_link.text) 136 | cvrf_url = f'https://api.msrc.microsoft.com/cvrf/v2.0/document/{id_for_cvrf[0]}' 137 | get_info = requests.get(cvrf_url, verify=False) 138 | soup = BeautifulSoup(get_info.text, "html.parser") 139 | parse_list = [] 140 | buff = '' 141 | for item in soup.text: 142 | if item == '\n': 143 | parse_list.append(buff) 144 | buff = '' 145 | else: 146 | buff += item 147 | parse_string = '' 148 | for j, item in enumerate(parse_list): 149 | regex = re.findall(cve, parse_list[j]) 150 | if regex: 151 | parse_string = parse_list[j] 152 | kb_list = re.findall(r'KB\d{7}', parse_string) 153 | not_remove_list_of_kb = [] 154 | for kb in kb_list: 155 | if kb not in not_remove_list_of_kb: 156 | not_remove_list_of_kb.append(kb) 157 | link_list = [] 158 | for kb in not_remove_list_of_kb: 159 | kb_url = f'https://catalog.update.microsoft.com/v7/site/Search.aspx?q={kb}' 160 | test = requests.get(kb_url, verify=False) 161 | if test.status_code == 200: 162 | url_get_product = f'https://www.catalog.update.microsoft.com/Search.aspx?q={kb}' 163 | get_product = requests.get(url_get_product, verify=False) 164 | soup_get_product = BeautifulSoup(get_product.text, "html.parser") 165 | product_buff = '' 166 | for item in soup_get_product.find_all('a', class_='contentTextItemSpacerNoBreakLink'): 167 | product_buff = item.text 168 | product = product_buff.strip() 169 | # Output: Windows 10 Version 1809 for x86-based Systems 170 | # link_list.append(f'[{kb}]({kb_url}) - {(product.partition("for")[2])[:-12]}') 171 | if product: 172 | # Output: 2022-01 Cumulative Update for Windows 10 Version 1809 for x86-based Systems (KB5009557) 173 | link_list.append(f'[{kb}]({kb_url}) - {product}') 174 | return link_list 175 | 176 | 177 | # check https://github.com/nu11secur1ty/ ------------------------------------------------------------------------------- 178 | def get_exploit_info(cve): 179 | # print('get_exploit_info') # DEBUG 180 | link = 'https://github.com/nu11secur1ty/CVE-mitre' 181 | link_2 = 'https://github.com/nu11secur1ty/CVE-mitre/tree/main/2022' 182 | default_link = '' 183 | poc_cve_list = [] 184 | r = requests.get(link) 185 | soup = BeautifulSoup(r.text, "html.parser") 186 | for cve_id in soup.find_all("span", class_="css-truncate css-truncate-target d-block width-fit"): 187 | regex = re.findall(r'CVE-\d{4}-\d{4,8}', cve_id.text) 188 | if regex: 189 | poc_cve_list.append(str(regex[0])) 190 | 191 | r = requests.get(link_2) 192 | soup = BeautifulSoup(r.text, "html.parser") 193 | for cve_id in soup.find_all("span", class_="css-truncate css-truncate-target d-block width-fit"): 194 | regex = re.findall(r'CVE-\d{4}-\d{4,8}', cve_id.text) 195 | if regex: 196 | poc_cve_list.append(str(regex[0])) 197 | 198 | for item in poc_cve_list: 199 | if cve == item: 200 | default_link = f'**nu11secur1ty** - https://github.com/nu11secur1ty/CVE-mitre/tree/main/{cve}' 201 | return default_link 202 | 203 | 204 | # check https://github.com/trickest/cve/ ------------------------------------------------------------------------------- 205 | def get_exploit_info_2(cve): 206 | # print('get_exploit_info_2') # DEBUG 207 | year = cve.split('-')[1] 208 | link = f'https://github.com/trickest/cve/tree/main/{year}' 209 | r = requests.get(link) 210 | soup = BeautifulSoup(r.text, "html.parser") 211 | default_link = '' 212 | for cve_id in soup.find_all("span", class_="css-truncate css-truncate-target d-block width-fit"): 213 | if f'{cve}.md' == cve_id.text: 214 | default_link = f'**trickest/cve** - https://github.com/trickest/cve/tree/main/{year}/{cve}.md' 215 | break 216 | return default_link 217 | 218 | 219 | # main function for upload information on YT---------------------------------------------------------------------------- 220 | def get_cve_data(cve, id): 221 | # print('get_cve_data') # DEBUG 222 | try: 223 | r = nvdlib.getCVE(cve, cpe_dict=False) 224 | cve_cpe_nodes = r.configurations.nodes 225 | cpe_nodes = ast.literal_eval(str(r.configurations)) 226 | 227 | # parse CVSSv3 Score and CVSSv3 Vector-------------------------------------------------------------------------- 228 | try: 229 | score = r.v3score 230 | vector = r.v3vector 231 | except AttributeError: 232 | score = 0.1 233 | vector = "Нет: cvss vector" 234 | if vector != "Нет: cvss vector": 235 | vector = r.v3vector[9:len(r.v3vector)] 236 | 237 | # find Exploit links-------------------------------------------------------------------------------------------- 238 | links = [] 239 | exploit_links = [] 240 | links.append(r.url) 241 | for t in r.cve.references.reference_data: 242 | links.append(t.url) 243 | if 'Exploit' in t.tags: 244 | exploit_links.append(t.url) 245 | if get_exploit_info(cve): # check https://github.com/nu11secur1ty/ 246 | exploit_links.append(get_exploit_info(cve)) 247 | if get_exploit_info_2(cve): # check https://github.com/trickest/cve/ 248 | exploit_links.append(get_exploit_info_2(cve)) 249 | value_exploit = "Да" 250 | if not exploit_links: 251 | value_exploit = "Нет" 252 | 253 | # parse CPE for further processing------------------------------------------------------------------------------ 254 | cpe_for_product_vendors = [] 255 | if cpe_nodes: 256 | for conf in cve_cpe_nodes: 257 | if conf.operator == 'AND': 258 | children = [conf.children[0]] 259 | else: 260 | children = [conf] 261 | for child in children: 262 | for cpe in child.cpe_match: 263 | cpe_for_product_vendors.append(cpe.cpe23Uri) 264 | 265 | # parse product and vector-------------------------------------------------------------------------------------- 266 | product_vendor_list = [] 267 | product_image_list = [] 268 | version_list = [] 269 | update_list = [] 270 | for cpe in cpe_for_product_vendors: 271 | cpe_parsed = CPE(cpe) 272 | product = cpe_parsed.get_product() 273 | vendor = cpe_parsed.get_vendor() 274 | product_vendor = vendor[0] + " " + product[0] if product != vendor else product[0] 275 | product_vendor_list.append(product_vendor) 276 | product_image_list.append(product[0]) 277 | version = cpe_parsed.get_version() 278 | update = cpe_parsed.get_update() 279 | update_list.append(update) 280 | if version[0] != '-' and version[0] != '*': 281 | version_list.append(f'{product[0]} - {version[0]}') 282 | 283 | temp1 = [] 284 | for item_1 in version_list: 285 | if item_1 not in temp1: 286 | temp1.append(item_1) 287 | versions = [] 288 | for item_2 in temp1: 289 | ver = {"name": item_2} 290 | versions.append(ver) 291 | 292 | prod = [] 293 | for item_3 in product_image_list: 294 | if item_3 not in prod: 295 | prod.append(item_3) 296 | 297 | content = [] 298 | for item_4 in product_vendor_list: 299 | con = {"name": item_4} 300 | content.append(con) 301 | 302 | # check regex name in cve (like: LPE, RCE...)------------------------------------------------------------------- 303 | cve_name = '' 304 | cve_info = r.cve.description.description_data[0].value 305 | for item_5 in pattern: 306 | if item_5.upper() in cve_info.upper(): 307 | cve_name = cve + " - " + item_5 308 | break 309 | else: 310 | cve_name = cve 311 | 312 | # check kb in cve---------------------------------------------------------------------------------------------- 313 | kb_links = '' 314 | try: 315 | if check_microsoft(cve) == 200: 316 | kb_links = get_kb(cve) 317 | except AttributeError: 318 | pass 319 | 320 | # check mitigations for cve------------------------------------------------------------------------------------- 321 | mitigations_links = '' 322 | try: 323 | links_mitigations_mitre = get_ttp(cve) 324 | value_mitigations = 'Нет' 325 | if links_mitigations_mitre != 'NO TTP' and links_mitigations_mitre: 326 | mitigations_links = links_mitigations_mitre 327 | value_mitigations = 'Да' 328 | except AttributeError: 329 | value_mitigations = 'Нет' 330 | 331 | # forming message for payload----------------------------------------------------------------------------------- 332 | data = { 333 | 'cve': cve_info, 334 | 'lastModifiedDate': r.lastModifiedDate[:-7], 335 | 'publishedDate': r.publishedDate[:-7], 336 | 'configurations': cpe_nodes, 337 | 'score': score, 338 | 'vector': vector, 339 | 'links': links, 340 | 'product_vendor_list': prod, 341 | 'exploit_links': exploit_links, 342 | 'kb_links': kb_links, 343 | 'mitigations_links': mitigations_links 344 | } 345 | message = jinja2.Template(template).render(d=data) 346 | 347 | # check for product_vendor-------------------------------------------------------------------------------------- 348 | headers_for_data_prod = { 349 | "Accept": "application/json", 350 | "Authorization": "Bearer {}".format(YOU_TRACK_TOKEN), 351 | "Content-Type": "application/json" 352 | } 353 | data_prod = requests.get(URL_GET_PRODUCTS, headers=headers_for_data_prod).json() 354 | 355 | upload_prod = [] 356 | for buff in product_vendor_list: 357 | upload_prod.append(buff) 358 | 359 | prod_vend = [] 360 | for i in data_prod: 361 | prod_vend.append(i['name']) 362 | 363 | temp = [] 364 | for upload_1 in upload_prod: 365 | if upload_1 not in prod_vend: 366 | temp.append(upload_1) 367 | 368 | for upload in temp: 369 | payload = { 370 | "id": "0", 371 | "&type": "FieldStyle", 372 | "name": upload 373 | } 374 | requests.post(URL_GET_PRODUCTS, headers=headers_for_data_prod, json=payload) 375 | 376 | # check for versions-------------------------------------------------------------------------------------------- 377 | data_ver = requests.get(URL_GET_VERSIONS, headers=headers_for_data_prod).json() 378 | ver_list = [] 379 | for item_6 in data_ver: 380 | ver_list.append(item_6['name']) 381 | 382 | temp2 = [] 383 | for item_7 in temp1: 384 | if item_7 not in ver_list: 385 | temp2.append(item_7) 386 | 387 | for upload in temp2: 388 | payload = { 389 | "id": "0", 390 | "&type": "FieldStyle", 391 | "name": upload 392 | } 393 | requests.post(URL_GET_VERSIONS, headers=headers_for_data_prod, json=payload) 394 | 395 | # replace vendor name on normal name---------------------------------------------------------------------------- 396 | buff_content = [] 397 | buff_versions = [] 398 | if product_vendor_list: 399 | if re.search(r'windows', str(product_vendor_list[0])): 400 | con = {"name": "Microsoft Windows"} 401 | buff_content.append(con) 402 | buff_versions = versions 403 | elif re.search(r'juniper', str(product_vendor_list[0])): 404 | con = {"name": "Juniper"} 405 | buff_content.append(con) 406 | buff_versions = versions 407 | elif re.search(r'adaptive_security_appliance', str(product_vendor_list[0])): 408 | con = {"name": "Cisco ASA"} 409 | buff_content.append(con) 410 | buff_versions = versions 411 | else: 412 | if content: 413 | buff_content.append(content[0]) 414 | if versions: 415 | buff_versions.append(versions[0]) 416 | 417 | # gradation of vulnerability criticality------------------------------------------------------------------------ 418 | priority = '' 419 | if isinstance(score, float): 420 | if 0.1 <= score <= 3.9: 421 | priority = 'Низкая' 422 | elif 4.0 <= score <= 6.9: 423 | priority = 'Средняя' 424 | elif 7.0 <= score <= 8.9: 425 | priority = 'Высокая' 426 | elif 9.0 <= score <= 10.0: 427 | priority = 'Критическая' 428 | 429 | # forming payload for YT issue---------------------------------------------------------------------------------- 430 | request_payload = { 431 | "project": { 432 | "id": YOU_TRACK_PROJECT_ID 433 | }, 434 | "summary": cve_name, 435 | "description": message, 436 | "customFields": [ 437 | { 438 | "name": "Продукт (пакет)", 439 | "$type": "MultiEnumIssueCustomField", 440 | "value": buff_content 441 | }, 442 | { 443 | "name": "Есть эксплоит", 444 | "$type": "SingleEnumIssueCustomField", 445 | "value": {"name": value_exploit} 446 | }, 447 | { 448 | "name": "Affected versions", 449 | "$type": "MultiEnumIssueCustomField", 450 | "value": buff_versions 451 | }, 452 | { 453 | "name": "CVSS Score", 454 | "$type": "SimpleIssueCustomField", 455 | "value": score 456 | 457 | }, 458 | { 459 | "name": "CVSS Vector", 460 | "$type": "SimpleIssueCustomField", 461 | "value": str(vector) 462 | 463 | }, 464 | { 465 | "name": "Priority", 466 | "$type": "SingleEnumIssueCustomField", 467 | "value": {"name": priority} 468 | }, 469 | { 470 | "name": "Есть mitigation", 471 | "$type": "SingleEnumIssueCustomField", 472 | "value": {"name": value_mitigations} 473 | }, 474 | ] 475 | } 476 | 477 | # upload information on cve------------------------------------------------------------------------------------- 478 | url_differences = f'{YOU_TRACK_BASE_URL}/issues/{id}' 479 | diff = requests.post(url_differences, headers=headers_for_data_prod, json=request_payload) 480 | return diff.status_code 481 | # return request_payload # DEBUG 482 | 483 | except LookupError: 484 | now_time = datetime.datetime.now() 485 | message = f'По состоянию на {now_time.strftime("%d-%m-%Y %H:%M")} информация об уязвимости отсутствует' 486 | headers = { 487 | "Accept": "application/json", 488 | "Authorization": "Bearer {}".format(YOU_TRACK_TOKEN), 489 | "Content-Type": "application/json" 490 | } 491 | request_payload = { 492 | "project": { 493 | "id": YOU_TRACK_PROJECT_ID 494 | }, 495 | "summary": cve, 496 | "description": message, 497 | } 498 | url_differences = f'{YOU_TRACK_BASE_URL}/issues/{id}' 499 | diff = requests.post(url_differences, headers=headers, json=request_payload) 500 | return diff.status_code 501 | 502 | 503 | # alert on mail--------------------------------------------------------------------------------------------------------- 504 | def email_alert(time_start, time_stop): 505 | recipients = [] 506 | recipients.append(USER1) 507 | msg = EmailMessage() 508 | msg['Subject'] = 'сhanging_issues' 509 | msg['From'] = EMAIL_HOST_USER 510 | msg['To'] = ", ".join(recipients) 511 | body = f'Программа начала работу {time_start}, отработала - {time_stop}' 512 | msg.set_content(body) 513 | msg.set_content(body) 514 | smtp_server = smtplib.SMTP_SSL(host=EMAIL_HOST, port=EMAIL_PORT) 515 | smtp_server.login(user=EMAIL_HOST_USER, password=EMAIL_HOST_PASSWORD) 516 | smtp_server.send_message(msg) 517 | print('Email sent {}'.format(msg['Subject'])) 518 | 519 | 520 | #alert on telegram bot-------------------------Использовать на свой страх и риск---------------------------------------- 521 | def telegram_alert(message): 522 | sticker = random.choice(stickers) 523 | # R Alert 524 | requests.get( 525 | f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage?chat_id={CHAT_ID_R}&text={message}&parse_mode=markdown") 526 | requests.get(f"https://api.telegram.org/bot{BOT_TOKEN}/"f"sendSticker?chat_id={CHAT_ID_R}&sticker={sticker}") 527 | # Dj Alert 528 | requests.get( 529 | f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage?chat_id={CHAT_ID_J}&text={message}&parse_mode=markdown") 530 | requests.get(f"https://api.telegram.org/bot{BOT_TOKEN}/sendSticker?chat_id={CHAT_ID_J}&sticker={sticker}") 531 | # A Alert 532 | requests.get( 533 | f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage?chat_id={CHAT_ID_A}&text={message}&parse_mode=markdown") 534 | requests.get(f"https://api.telegram.org/bot{BOT_TOKEN}/sendSticker?chat_id={CHAT_ID_A}&sticker={sticker}") 535 | 536 | 537 | # convert sec to normak time like 01:35:52------------------------------------------------------------------------------ 538 | def convert_to_preferred_format(sec): 539 | sec = sec % (24 * 3600) 540 | hour = sec // 3600 541 | sec %= 3600 542 | min = sec // 60 543 | sec %= 60 544 | return "%02d:%02d:%02d" % (hour, min, sec) 545 | 546 | # ----------------------------------------------MAIN-------------------------------------------------------------------- 547 | now = datetime.datetime.now() 548 | time_start = now.strftime("%d-%m-%Y %H:%M") 549 | start_time = time.time() 550 | headers_main = { 551 | "Accept": "application/json", 552 | "Authorization": "Bearer {}".format(YOU_TRACK_TOKEN), 553 | "Content-Type": "application/json" 554 | } 555 | list_summary = requests.get(MAIN_URL_CHANGING, headers=headers_main).json() # Получение задач с YouTrack 556 | print(len(list_summary)) # Количество заведенных задач 557 | 558 | cve_list = [] 559 | id_list = [] 560 | 561 | for i, item in enumerate(list_summary): 562 | regex = re.search(r'CVE-\d{4}-\d{4,8}', str(list_summary[i]['summary'])) 563 | # regex = re.search(r'CVE-2021-43899', str(list_summary[i]['summary'])) # DEBUG 564 | if regex: 565 | issue_state = list_summary[i]['customFields'][2]['value']['name'] 566 | state_1 = "Won't fix" 567 | state_2 = "Выработаны рекомендации" 568 | state_3 = "Направлены рекомендации исполнителям" 569 | # if issue_state == 'Open' or issue_state == 'In Progress': 570 | # Обновляется информация только для актуальных задач: 571 | if issue_state != state_1 and issue_state != state_2 and issue_state != state_3: 572 | cve_list.append(str(regex.group())) 573 | id_list.append(list_summary[i]['id']) 574 | 575 | # print(len(cve_list)) # DEBUG 576 | 577 | for i, item in enumerate(cve_list): 578 | # Перебор с конца списка 579 | # print(f'{i + 1} / {len(cve_list)} - {get_cve_data(cve_list[len(cve_list) - i - 1], id_list[len(cve_list) - 580 | # i - 1])} - {cve_list[len(cve_list) - i - 1]}') 581 | # Прямой перебор по списку 582 | result_status_code = get_cve_data(cve_list[i], id_list[i]) 583 | result_string = f'{i + 1} / {len(cve_list)} ({cve_list[i]})' 584 | subtraction = len(result_string) - 26 585 | escape = ' ' 586 | print(f'{result_string}{escape*abs(subtraction)} - {result_status_code}') # Красивый вывод информации 587 | # print(f'{i + 1} / {len(cve_list)} ({cve_list[i]}) - {result_status_code}') 588 | 589 | # Time manipulation 590 | time_stop = "за %s секунд" % (time.time() - start_time) 591 | current_time = str(time.time() - start_time) 592 | super_time = convert_to_preferred_format(int(current_time.split(".")[0])) 593 | start_date = time_start.split(" ")[0] 594 | start_time = time_start.split(" ")[1] 595 | message = f'*CHANGING*\nПрограмма начала работу {start_date} в {start_time} и отработала за ```{super_time}```' 596 | # telegram alert 597 | telegram_alert(message) 598 | # email_alert(time_start, time_stop) 599 | 600 | ''' 601 | # DEBUG ONLY 602 | 603 | for i in range(len(cve_list)): 604 | if cve_list[i] == 'CVE-2019-12353': 605 | print(cve_list[i]) 606 | print(id_list[i]) 607 | 608 | print(get_cve_data('CVE-2019-12353', '2-27790')) 609 | # print(get_kb('CVE-2017-0213')) 610 | ''' -------------------------------------------------------------------------------- /export_YT_to_RV.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3 2 | import requests 3 | import urllib3 4 | import re 5 | import datetime 6 | import time 7 | from dotenv import dotenv_values 8 | import os 9 | 10 | urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) 11 | 12 | dotenv_path = os.path.join(os.path.dirname(__file__), '.env') 13 | config = dotenv_values(dotenv_path) 14 | 15 | # imported variables from .env 16 | YOU_TRACK_PROJECT_ID = config.get("YOU_TRACK_PROJECT_ID") 17 | YOU_TRACK_BASE_URL = config.get("YOU_TRACK_BASE_URL") 18 | MAIN_URL_CHANGING = config.get("MAIN_URL_CHANGING") 19 | YOU_TRACK_TOKEN = config.get("YOU_TRACK_TOKEN") 20 | MAIN_URL_FOR_RV = config.get("MAIN_URL_FOR_RV") 21 | RV_FILTERS_LINK = config.get("RV_FILTERS_LINK") 22 | RV_USERNAME = config.get("RV_USERNAME") 23 | RV_PASSWORD = config.get("RV_PASSWORD") 24 | RV_X_TOKEN = config.get("RV_X_TOKEN") 25 | RV_NAME = config.get("RV_NAME") 26 | RV_URL = config.get("RV_URL") 27 | 28 | URL = str(YOU_TRACK_BASE_URL) + "/issues" # main url for YT 29 | 30 | unix_time = str(time.mktime(datetime.datetime.now().timetuple()))[:-2] 31 | session = requests.Session() 32 | 33 | 34 | def get_csrf_token(): 35 | global session 36 | 37 | url_get_token = RV_URL + f'csrfToken?{unix_time}' 38 | a = session.get(url_get_token, verify=False) 39 | return a.json()["_csrf"] 40 | 41 | 42 | def auth(task_name, task_description, product_list): 43 | global session 44 | 45 | csrf = get_csrf_token() 46 | url_auth = RV_URL + 'login' 47 | payload_auth = { 48 | "username": RV_USERNAME, 49 | "password": RV_PASSWORD, 50 | "tz": "Europe/Moscow", 51 | "_csrf": csrf 52 | } 53 | 54 | session.post(url_auth, verify=False, data=payload_auth) # get new csrf for POST request 55 | get_for_csrf_2 = session.get(RV_URL + f'csrfToken?{unix_time}', verify=False) 56 | csrf_2 = get_for_csrf_2.json()["_csrf"] 57 | headers = { 58 | 'X-Token': RV_X_TOKEN, 59 | 'X-Csrf-Token': csrf_2 60 | } 61 | 62 | start_date = datetime.datetime.today().strftime('%Y-%m-%d') 63 | date_format = datetime.datetime.strptime(start_date, '%Y-%m-%d') 64 | end_date = date_format + datetime.timedelta(days=30) 65 | duedate = str(end_date).replace(' ', 'T') + 'Z' 66 | 67 | # Информация по задаче 68 | payload_for_create_task = { 69 | "company_id": 4, # ID организации 70 | "name": task_name, # Example: Закрытие уязвимости CVE-2022-40444 71 | "type_id": 6, # Тип задачи (6 - Работа с уязвимостями) 72 | "description": task_description, # Описание задачи 73 | "duedate": duedate, # Срок исполнения 74 | "level_id": 2, 75 | "assignee_ids": [151, 125] # ID ответственных за исполнение задачи 76 | } 77 | url_for_create_task = RV_URL + 'api/v1/tm/tasks/' 78 | request_for_create_task = session.post(url_for_create_task, verify=False, headers=headers, 79 | data=payload_for_create_task) 80 | response_for_create_task = request_for_create_task.json() 81 | task_id = response_for_create_task['data'][0]['id'] 82 | if request_for_create_task.status_code == 200: 83 | print(f'Создана задача TSK-{task_id}') 84 | else: 85 | print(request_for_create_task.text) 86 | 87 | # Получение информации об ID оборудования 88 | ids_list = [] 89 | for product_name in product_list: 90 | url_get_id_active = RV_URL + f'api/v1/am/devices?_dc={unix_time}' + RV_FILTERS_LINK + f'{product_name}' + '"}]' 91 | request_get_id_active = session.get(url_get_id_active, verify=False, headers=headers) 92 | response_get_id_active = request_get_id_active.json() 93 | for i in range(len(response_get_id_active['data'])): 94 | if RV_NAME in response_get_id_active['data'][i]['assets_name']: 95 | ids_list.append(int(response_get_id_active['data'][i]['id'])) 96 | 97 | # Добавление активов к задаче 98 | payload_add_active = { 99 | "ids": ids_list 100 | } 101 | # Выгрузка информации об оборудовании 102 | url_add_active = RV_URL + f'api/v1/tm/tasks/{task_id}/devices' 103 | request_add_active = session.post(url_add_active, verify=False, headers=headers, data=payload_add_active) 104 | if request_add_active.status_code == 200: 105 | print(f'Добавлено оборудование к задаче TSK-{task_id}') 106 | else: 107 | print(request_add_active.text) 108 | 109 | return task_id 110 | 111 | 112 | # ----------------------------------------------MAIN-------------------------------------------------------------------- 113 | headers_main = { 114 | "Accept": "application/json", 115 | "Authorization": "Bearer {}".format(YOU_TRACK_TOKEN), 116 | "Content-Type": "application/json" 117 | } 118 | 119 | 120 | list_summary = requests.get(MAIN_URL_FOR_RV, headers=headers_main).json() # Получение задач с YouTrack 121 | # print(len(list_summary)) # Количество заведенных задач 122 | 123 | # Получение задач для экспорта (наименование, ID задачи) 124 | cve_list = [] 125 | id_list = [] 126 | for i, item in enumerate(list_summary): 127 | regex = re.search(r'CVE-\d{4}-\d{4,8}', str(list_summary[i]['summary'])) 128 | # regex = re.search(r'CVE-2021-43899', str(list_summary[i]['summary'])) # DEBUG 129 | if regex: 130 | issue_state = list_summary[i]['customFields'][2]['value']['name'] 131 | state = "Выработаны рекомендации" 132 | issue_status = list_summary[i]['customFields'][13]['value']['name'] 133 | status = 'to export' 134 | if issue_status == status and issue_state == state: 135 | cve_list.append(item) 136 | id_list.append(list_summary[i]['id']) 137 | 138 | # Получение продуктов 139 | product_list = [] 140 | cve_name = '' 141 | mitigations = '' 142 | if cve_list: 143 | for i in range(len(cve_list)): 144 | for j in range(len(cve_list[i]['customFields'][5]['value'])): 145 | name_of_product = cve_list[i]['customFields'][5]['value'][j]['name'].split(' - ')[0].replace('_', ' ') 146 | if 'windows' in name_of_product and name_of_product.split(' ')[-1:][0].isnumeric(): # Очень хитрый костыль 147 | product_list.append(name_of_product) 148 | product_list_no_repetitions = list(set(product_list)) # Получение списка продуктов 149 | 150 | # Получение ID задачи в YT 151 | issueID = id_list[i] 152 | 153 | # Получение наименования уязвимости или уязвимостей (из связанных задач) 154 | url_get_relations = URL + f'/{issueID}/links?fields=issues(summary)' 155 | response_get_VM = requests.get(url_get_relations, headers=headers_main, verify=False) 156 | buff = response_get_VM.json() 157 | name_list = [] 158 | for a, summary in enumerate(buff[0]['issues']): 159 | if len(summary['summary']) > 20: 160 | name_list.append(summary['summary'].split(' - ')[0]) 161 | else: 162 | name_list.append(summary['summary']) 163 | if len(name_list) > 1: 164 | cve_name = ", ".join(name_list) 165 | task_name = f'Задача по устранению уязвимостей {cve_name}' 166 | else: 167 | task_name = f'Задача по устранению уязвимости {name_list[0]}' 168 | 169 | # Получение рекомендаций 170 | url_comments = URL + f'/{issueID}/comments?fields=text' 171 | comment_list = requests.get(url_comments, headers=headers_main).json() 172 | for item in comment_list: 173 | if 'Рекомендации' in item['text']: 174 | to_text = item['text'] 175 | cosmetic = to_text.replace('#', '').replace('[', '').replace('](', ' -> ') 176 | mitigations = cosmetic.replace(')', '').replace('(', '').replace('**', '') 177 | 178 | # Вывод количества итераций 179 | print(f'{i+1} / {len(cve_list)}') 180 | 181 | # Создание задачи в RVision 182 | task_id = auth(task_name, mitigations, product_list_no_repetitions) 183 | 184 | # Добавление комментария в YouTrack с номером задачи из RVision 185 | url_add_comment = URL + f'/{issueID}/comments' 186 | payload_add_comment = { 187 | "text": f'Решение для закрытия уязвимости доведены ответственным через RVision. TSK-{task_id}', 188 | "visibility": { 189 | "permittedGroups": [ 190 | { 191 | "id": "1-8" 192 | } 193 | ], 194 | "$type": "UnlimitedVisibility" 195 | } 196 | } 197 | add_comment = requests.post(url_add_comment, verify=False, headers=headers_main, json=payload_add_comment) 198 | if add_comment.status_code == 200: 199 | print(f'Добавлен комментарий к задаче {issueID}') 200 | else: 201 | print(add_comment.text) 202 | 203 | # Смена состояний задачи в YT 204 | payload_change_fields = { 205 | "customFields": [ 206 | { 207 | "name": "State", 208 | "$type": "StateIssueCustomField", 209 | "value": {"name": "Направлены рекомендации исполнителям"} 210 | }, 211 | { 212 | "name": "Экспорт задач", 213 | "$type": "StateIssueCustomField", 214 | "value": {"name": "exported"} 215 | } 216 | ] 217 | } 218 | url_change_fields = URL + f'/{issueID}' 219 | change_field = requests.post(url_change_fields, headers=headers_main, verify=False, json=payload_change_fields) 220 | if change_field.status_code == 200: 221 | print(f'Состояние задачи {issueID} изменилось') 222 | else: 223 | print(change_field.text) 224 | else: 225 | print('Новых отресёрченных уязвимостей нет') 226 | -------------------------------------------------------------------------------- /get_cve_from_month_update.py: -------------------------------------------------------------------------------- 1 | import urllib3 2 | from bs4 import BeautifulSoup 3 | from selenium import webdriver 4 | import time 5 | import re 6 | import datetime 7 | import os 8 | from get_kb import get_kb 9 | 10 | 11 | urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) 12 | webdriver_path = os.path.join(os.path.dirname(__file__), 'yandexdriver.exe') 13 | 14 | 15 | def get_cve_from_microsoft(url): 16 | options = webdriver.ChromeOptions() 17 | driver = webdriver.Chrome(executable_path=webdriver_path, options=options) 18 | driver.get(url) 19 | time.sleep(5) # Если не успевает прогрузить страницу, то увеличить количество секунд на задержку 20 | page_source = driver.execute_script("return document.body.innerHTML;") 21 | s_response = BeautifulSoup(page_source, "html.parser") 22 | cve_list = [] 23 | for item in s_response.find_all('a'): 24 | regex = re.search(r'CVE-\d{4}-\d{4,8}', item.text) 25 | if regex: 26 | cve_list.append(item.text) 27 | return cve_list 28 | 29 | # Вывод списков cve по месяцам и годам 30 | ''' 31 | month_list = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] 32 | year_list = ['2018', '2019', '2020', '2021', '2022'] # Тут можно дополнить года 33 | for year in year_list: 34 | for month in month_list: 35 | url_get_cve = f'https://msrc.microsoft.com/update-guide/releaseNote/{year}-{month}' 36 | compare_cve_list = get_get_microsoft(url_get_cve) 37 | print(f'{year}-{month}:') 38 | print(compare_cve_list) 39 | ''' 40 | 41 | '''Список CVE на текущий месяц 42 | Обновления выходят каждый второй вторник каждого месяца, поэтому рекомендуемый порядок запуска в кроне: 43 | 0 0 15 * * python3 get_cve_from_month_update.py''' 44 | year = datetime.datetime.today().strftime('%Y') 45 | month = datetime.datetime.today().strftime('%B')[0:3] 46 | url_get_cve = f'https://msrc.microsoft.com/update-guide/releaseNote/{year}-{month}' 47 | compare_cve_list = get_cve_from_microsoft(url_get_cve) 48 | print(f'{year}-{month}:') 49 | for cve in compare_cve_list: 50 | print(cve) 51 | 52 | '''Можно подтянуть функцию из скрипта get_kb.py, чтобы "на лету" получать список необходимых 53 | для закрытия уязвимостей обновлений безопасности (KB)''' 54 | ''' 55 | for cve in compare_cve_list: 56 | update_list = get_kb(cve) 57 | print(cve) 58 | for item in update_list: 59 | print(item) 60 | ''' -------------------------------------------------------------------------------- /get_kb.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | import re 4 | import urllib3 5 | 6 | urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) 7 | 8 | def get_kb(cve): 9 | # print('get_kb') # DEBUG 10 | msrc_url = f"https://api.msrc.microsoft.com/cvrf/v2.0/Updates('{cve}')" 11 | get_cvrf_link = requests.get(msrc_url, verify=False) 12 | id_for_cvrf = re.search(r'\d{4}-\w{3}', get_cvrf_link.text) 13 | cvrf_url = f'https://api.msrc.microsoft.com/cvrf/v2.0/document/{id_for_cvrf[0]}' 14 | get_info = requests.get(cvrf_url, verify=False) 15 | soup = BeautifulSoup(get_info.text, "lxml") 16 | 17 | parse_list = [] 18 | buff = '' 19 | for item in soup.text: 20 | if item == '\n': 21 | parse_list.append(buff) 22 | buff = '' 23 | else: 24 | buff += item 25 | 26 | parse_string = '' 27 | for j, item in enumerate(parse_list): 28 | regex = re.findall(cve, parse_list[j]) 29 | if regex: 30 | parse_string = parse_list[j] 31 | 32 | kb_list = re.findall(r'KB\d{7}', parse_string) 33 | not_remove_list_of_kb = [] 34 | for kb in kb_list: 35 | if kb not in not_remove_list_of_kb: 36 | not_remove_list_of_kb.append(kb) 37 | 38 | link_list = [] 39 | for kb in not_remove_list_of_kb: 40 | kb_url = f'https://catalog.update.microsoft.com/v7/site/Search.aspx?q={kb}' 41 | test = requests.get(kb_url, verify=False) 42 | if test.status_code == 200: 43 | url_get_product = f'https://www.catalog.update.microsoft.com/Search.aspx?q={kb}' 44 | get_product = requests.get(url_get_product, verify=False) 45 | soup_get_product = BeautifulSoup(get_product.text, "lxml") 46 | product_buff = '' 47 | for item in soup_get_product.find_all('a', class_='contentTextItemSpacerNoBreakLink'): 48 | product_buff = item.text 49 | product = product_buff.strip() 50 | # Output: Windows 10 Version 1809 for x86-based Systems 51 | #link_list.append(f'[{kb}]({kb_url}) - {(product.partition("for")[2])[:-12]}') 52 | if product: 53 | # Output: 2022-01 Cumulative Update for Windows 10 Version 1809 for x86-based Systems (KB5009557) 54 | link_list.append(f'[{kb}]({kb_url}) - {product[:-12]}') 55 | 56 | return link_list 57 | 58 | ''' 59 | cve = 'CVE-2022-26809' 60 | link_list = get_kb(cve) 61 | for item in link_list: 62 | print(item) 63 | 64 | format_sting_1 = item.split(']')[1].replace('(', '').replace(')', '') # cosmetic output 65 | result_string = format_sting_1.split(' - ') # cosmetic output 66 | print(f'{result_string[1]} - {result_string[0]}') 67 | 68 | ''' 69 | -------------------------------------------------------------------------------- /parse_alienvault.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | import json 4 | import time 5 | 6 | API_KEY_ALIENVAULT = 'c63733bf4b4e763170cdda3b9b451c95dd2994aa07e5d4d4a4b216fd47275aed' # old api key 7 | 8 | def get_mitigations(tactic_id): 9 | tactic_url = f'https://attack.mitre.org/techniques/{tactic_id}' 10 | r_get_tactic_info = requests.get(tactic_url) 11 | s_get_tactic_info = BeautifulSoup(r_get_tactic_info.text, 'html.parser') 12 | 13 | # 0 - Procedure Examples 14 | # 1 - Mitigations 15 | # 2 - Detection 16 | try: 17 | mitigations = s_get_tactic_info.find_all("table", class_="table table-bordered table-alternate mt-2")[1].text 18 | buff_string_1 = str(mitigations).replace("ID", "").replace("Mitigation", "").replace("Description", "").replace("\n\n", "") 19 | buff_list = buff_string_1.split('\n') 20 | buff_list.pop(0) # Remove empty element, index - 0 21 | return_list = [] 22 | for i in range(int(len(buff_list) / 3)): 23 | id_mit = buff_list[i*3].rstrip().lstrip() 24 | name_mit = buff_list[i*3+1].rstrip().lstrip() 25 | desc_mit = buff_list[i*3+2].rstrip().lstrip() 26 | return_list.append(f'[{id_mit} - {name_mit}](https://attack.mitre.org/mitigations/{id_mit}) - {desc_mit}') 27 | return return_list 28 | except: 29 | pass 30 | 31 | def get_ttp(cve): 32 | headers = {'X-OTX-API-KEY': API_KEY_ALIENVAULT} 33 | url = f'https://otx.alienvault.com/api/v1/indicators/cve/{cve}' 34 | pattern = '{"detail": "endpoint not found"}' 35 | r_get_cve_info = requests.get(url, headers=headers) 36 | s_get_cve_info = BeautifulSoup(r_get_cve_info.text, 'lxml') 37 | tactic_list = [] 38 | if str(s_get_cve_info.p.contents).replace("['", "").replace("']", "") == pattern: 39 | print(f'no information for {cve}') # DEBUG 40 | else: 41 | content = json.loads(str(s_get_cve_info.p).replace("", "").replace("
","")) 42 | count = int(content['pulse_info']['count']) 43 | if count != 0: 44 | for i in range(count): 45 | for j in range(len(content['pulse_info']['pulses'][i]['attack_ids'])): 46 | tactic_list.append(content['pulse_info']['pulses'][i]['attack_ids'][j]['id']) 47 | else: 48 | return f'NO TTP' 49 | tactic_list = list(set(tactic_list)) # replace replications 50 | technique_list = [] 51 | for i, item in enumerate(tactic_list): 52 | if len(str(item)) > 6: 53 | technique_list.append(str(item).replace(".", "/")) 54 | 55 | result_list = [] 56 | if technique_list: 57 | for tactic_id in technique_list: 58 | if get_mitigations(tactic_id): 59 | for item in get_mitigations(tactic_id): 60 | result_list.append(item) 61 | return list(set(result_list)) 62 | else: 63 | for tactic_id in tactic_list: 64 | if get_mitigations(tactic_id): 65 | for item in get_mitigations(tactic_id): 66 | result_list.append(item) 67 | return list(set(result_list)) 68 | 69 | def get_ttp_1(cve): 70 | headers = {'X-OTX-API-KEY': API_KEY_ALIENVAULT} 71 | url = f'https://otx.alienvault.com/api/v1/indicators/cve/{cve}' 72 | pattern = '{"detail": "endpoint not found"}' 73 | r_get_cve_info = requests.get(url, headers=headers) 74 | s_get_cve_info = BeautifulSoup(r_get_cve_info.text, 'lxml') 75 | tactic_list = [] 76 | if str(s_get_cve_info.p.contents).replace("['", "").replace("']", "") == pattern: 77 | print(f'no information for {cve}') # DEBUG 78 | else: 79 | content = json.loads(str(s_get_cve_info.p).replace("", "").replace("
", "")) 80 | count = int(content['pulse_info']['count']) 81 | subs_list = [] 82 | for i in range(count): 83 | subs_list.append(content['pulse_info']['pulses'][i]['subscriber_count']) 84 | print(subs_list) 85 | tactic_list = [] 86 | for i, item in enumerate(subs_list): 87 | max_test = max(subs_list) 88 | if content['pulse_info']['pulses'][i]['attack_ids']: 89 | for j in range(len(content['pulse_info']['pulses'][i]['attack_ids'])): 90 | tactic_list.append(content['pulse_info']['pulses'][i]['attack_ids'][j]['id']) 91 | break 92 | else: 93 | subs_list.remove(max_test) 94 | else: 95 | return f'NO TTP' 96 | tactic_list = list(set(tactic_list)) # replace replications 97 | technique_list = [] 98 | for i, item in enumerate(tactic_list): 99 | if len(str(item)) > 6: 100 | technique_list.append(str(item).replace(".", "/")) 101 | 102 | result_list = [] 103 | if technique_list: 104 | for tactic_id in technique_list: 105 | if get_mitigations(tactic_id): 106 | for item in get_mitigations(tactic_id): 107 | result_list.append(item) 108 | return list(set(result_list)) 109 | else: 110 | for tactic_id in tactic_list: 111 | if get_mitigations(tactic_id): 112 | for item in get_mitigations(tactic_id): 113 | result_list.append(item) 114 | return list(set(result_list)) 115 | 116 | 117 | # DEBUG ONLY 118 | #cve = 'CVE-2021-4034' 119 | cve = 'CVE-2021-42001' 120 | mit = 'T1059/001' 121 | # print(get_ttp(cve)) 122 | a = get_ttp_1(cve) 123 | b = get_ttp(cve) 124 | print(len(a)) 125 | print(len(b)) 126 | 127 | 128 | ''' 129 | start_time = time.time() 130 | print(start_time) 131 | cve_list = ['CVE-2021-27365', 'CVE-2021-28313', 'CVE-2021-28315', 'CVE-2021-32761', 'CVE-2021-40444', 'CVE-2021-44228', 'CVE-2019-17571', 'CVE-2021-43803', 'CVE-2021-43808', 'CVE-2021-41270', 'CVE-2021-34787', 'CVE-2021-40125'] 132 | for cve in cve_list: 133 | links_mitigations_mitre = get_ttp(cve) 134 | if links_mitigations_mitre != 'NO TTP' and links_mitigations_mitre: 135 | print(links_mitigations_mitre) 136 | else: 137 | print(f'No mitigations for {cve}') 138 | time_stop = "Программа отработала за %s секунд" % (time.time() - start_time) 139 | print(time_stop) 140 | ''' 141 | -------------------------------------------------------------------------------- /parsing_cvetrends.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import jinja2 4 | import requests 5 | from bs4 import BeautifulSoup 6 | import urllib3 7 | from cpe import CPE 8 | import nvdlib 9 | import ast 10 | import re 11 | import smtplib 12 | from email.message import EmailMessage 13 | import datetime 14 | import os 15 | from dotenv import dotenv_values 16 | import random 17 | from stickers import stickers 18 | from usage import pattern, template 19 | 20 | urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) 21 | 22 | dotenv_path = os.path.join(os.path.dirname(__file__), '.env') 23 | config = dotenv_values(dotenv_path) 24 | # imported variables from .env 25 | YOU_TRACK_TOKEN = config.get("YOU_TRACK_TOKEN") 26 | YOU_TRACK_PROJECT_ID = config.get("YOU_TRACK_PROJECT_ID") 27 | YOU_TRACK_BASE_URL = config.get("YOU_TRACK_BASE_URL") 28 | URL_GET_PRODUCTS = config.get("URL_GET_PRODUCTS") 29 | URL_GET_VERSIONS = config.get("URL_GET_VERSIONS") 30 | EMAIL_HOST = config.get("EMAIL_HOST") 31 | EMAIL_PORT = config.get("EMAIL_PORT") 32 | EMAIL_HOST_PASSWORD = config.get("EMAIL_HOST_PASSWORD") 33 | EMAIL_HOST_USER = config.get("EMAIL_HOST_USER") 34 | USER1 = config.get("USER1") 35 | USER2 = config.get("USER2") 36 | MAIN_URL_CVETRENDS = config.get("MAIN_URL_CVETRENDS") 37 | BOT_TOKEN = config.get("BOT_TOKEN") 38 | CHAT_ID_J = config.get("CHAT_ID_J") 39 | CHAT_ID_R = config.get("CHAT_ID_R") 40 | CHAT_ID_A = config.get("CHAT_ID_A") 41 | 42 | URL = str(YOU_TRACK_BASE_URL) + "/issues" 43 | 44 | 45 | def get_top_cve_list(): 46 | link = 'https://cvetrends.com/api/cves/24hrs' 47 | r = requests.get(link) 48 | soup = BeautifulSoup(r.text, "lxml") 49 | regex = re.findall(r'"cve": "CVE-\d{4}-\d{4,8}"', soup.get_text()) 50 | top_cve_list = [] 51 | for item in regex: 52 | top_cve_list.append(re.search(r'CVE-\d{4}-\d{4,8}', item).group()) 53 | 54 | return top_cve_list 55 | 56 | 57 | def add_tag(id): 58 | request_payload = { 59 | "project": { 60 | "id": YOU_TRACK_PROJECT_ID 61 | }, 62 | "tags": [ 63 | { 64 | "name": "В тренде", 65 | "id": "6-27", 66 | "$type": "IssueTag" 67 | } 68 | ], 69 | } 70 | url_differences = f'{YOU_TRACK_BASE_URL}/issues/{id}' 71 | diff = requests.post(url_differences, headers=headers, json=request_payload) 72 | return diff.status_code 73 | 74 | 75 | def delete_tag(id): 76 | URL1 = f'{YOU_TRACK_BASE_URL}/issues/{id}/tags/6-27' 77 | delete = requests.delete(URL1, headers=headers) 78 | return delete.status_code 79 | 80 | 81 | def change_tag(id): 82 | request_payload = { 83 | "project": { 84 | "id": YOU_TRACK_PROJECT_ID 85 | }, 86 | "tags": [ 87 | { 88 | "name": "Была в тренде", 89 | "id": "6-28", 90 | "$type": "IssueTag" 91 | } 92 | ], 93 | } 94 | url_differences = f'{YOU_TRACK_BASE_URL}/issues/{id}' 95 | diff = requests.post(url_differences, headers=headers, json=request_payload) 96 | return diff.status_code 97 | 98 | 99 | # check https://github.com/nu11secur1ty/ ------------------------------------------------------------------------------- 100 | def get_exploit_info(cve): 101 | link = 'https://github.com/nu11secur1ty/CVE-mitre' 102 | link_2 = 'https://github.com/nu11secur1ty/CVE-mitre/tree/main/2022' 103 | default_link = '' 104 | poc_cve_list = [] 105 | r = requests.get(link) 106 | soup = BeautifulSoup(r.text, "html.parser") 107 | for cve_id in soup.find_all("span", class_="css-truncate css-truncate-target d-block width-fit"): 108 | regex = re.findall(r'CVE-\d{4}-\d{4,8}', cve_id.text) 109 | if regex: 110 | poc_cve_list.append(str(regex[0])) 111 | 112 | r = requests.get(link_2) 113 | soup = BeautifulSoup(r.text, "html.parser") 114 | for cve_id in soup.find_all("span", class_="css-truncate css-truncate-target d-block width-fit"): 115 | regex = re.findall(r'CVE-\d{4}-\d{4,8}', cve_id.text) 116 | if regex: 117 | poc_cve_list.append(str(regex[0])) 118 | 119 | for item in poc_cve_list: 120 | if cve == item: 121 | default_link = f'https://github.com/nu11secur1ty/CVE-mitre/tree/main/{cve}' 122 | return default_link 123 | 124 | 125 | # check https://github.com/trickest/cve/ ------------------------------------------------------------------------------- 126 | def get_exploit_info_2(cve): 127 | # print('get_exploit_info_2') # DEBUG 128 | year = cve.split('-')[1] 129 | link = f'https://github.com/trickest/cve/tree/main/{year}' 130 | r = requests.get(link) 131 | soup = BeautifulSoup(r.text, "html.parser") 132 | default_link = '' 133 | for cve_id in soup.find_all("span", class_="css-truncate css-truncate-target d-block width-fit"): 134 | if f'{cve}.md' == cve_id.text: 135 | default_link = f'**trickest/cve** - https://github.com/trickest/cve/tree/main/{year}/{cve}.md' 136 | break 137 | return default_link 138 | 139 | 140 | # main function for upload information on YT---------------------------------------------------------------------------- 141 | def get_cve_data(cve): 142 | try: 143 | r = nvdlib.getCVE(cve, cpe_dict=False) 144 | cve_cpe_nodes = r.configurations.nodes 145 | cpe_nodes = ast.literal_eval(str(r.configurations)) 146 | try: 147 | score = r.v3score 148 | vector = r.v3vector 149 | except: 150 | score = 0.1 151 | vector = "Нет: cvss vector" 152 | if vector != "Нет: cvss vector": 153 | vector = r.v3vector[9:len(r.v3vector)] 154 | 155 | links = [] 156 | exploit_links = [] 157 | links.append(r.url) 158 | for t in r.cve.references.reference_data: 159 | links.append(t.url) 160 | if 'Exploit' in t.tags: 161 | exploit_links.append(t.url) 162 | 163 | if get_exploit_info(cve): # check https://github.com/nu11secur1ty/ 164 | exploit_links.append(get_exploit_info(cve)) 165 | if get_exploit_info_2(cve): # check https://github.com/trickest/cve/ 166 | exploit_links.append(get_exploit_info_2(cve)) 167 | 168 | cpe_for_product_vendors = [] 169 | if cpe_nodes: 170 | for conf in cve_cpe_nodes: 171 | if conf.operator == 'AND': 172 | children = [conf.children[0]] 173 | else: 174 | children = [conf] 175 | for child in children: 176 | for cpe in child.cpe_match: 177 | cpe_for_product_vendors.append(cpe.cpe23Uri) 178 | 179 | # parse CPE--------------------------------------------------------------------------------------------------------- 180 | product_vendor_list = [] 181 | product_image_list = [] 182 | version_list = [] 183 | for cpe in cpe_for_product_vendors: 184 | cpe_parsed = CPE(cpe) 185 | product = cpe_parsed.get_product() 186 | vendor = cpe_parsed.get_vendor() 187 | product_vendor = vendor[0] + " " + product[0] if product != vendor else product[0] 188 | product_vendor_list.append(product_vendor) 189 | product_image_list.append(product[0]) 190 | version = cpe_parsed.get_version() 191 | if version[0] != '-' and version[0] != '*': 192 | version_list.append(f'{product[0]} - {version[0]}') 193 | 194 | temp1 = [] 195 | for item in version_list: 196 | if item not in temp1: 197 | temp1.append(item) 198 | versions = [] 199 | for item in temp1: 200 | ver = {"name": item} 201 | versions.append(ver) 202 | 203 | prod = [] 204 | for item in product_image_list: 205 | if item not in prod: 206 | prod.append(item) 207 | 208 | content = [] 209 | for item in product_vendor_list: 210 | con = {"name": item} 211 | content.append(con) 212 | 213 | value = "Да" 214 | if not exploit_links: 215 | value = "Нет" 216 | 217 | # check regex in cve------------------------------------------------------------------------------------------------ 218 | cve_name = '' 219 | cve_info = r.cve.description.description_data[0].value 220 | for item in pattern: 221 | if item.upper() in cve_info.upper(): 222 | cve_name = cve + " - " + item 223 | break 224 | else: 225 | cve_name = cve 226 | # message----------------------------------------------------------------------------------------------------------- 227 | data = { 228 | 'cve': cve_info, 229 | 'lastModifiedDate': r.lastModifiedDate[:-7], 230 | 'publishedDate': r.publishedDate[:-7], 231 | 'configurations': cpe_nodes, 232 | 'score': score, 233 | 'vector': r.v3vector, 234 | 'links': links, 235 | 'product_vendor_list': prod, 236 | 'exploit_links': exploit_links 237 | } 238 | message = jinja2.Template(template).render(d=data) 239 | 240 | # check for product_vendor------------------------------------------------------------------------------------------ 241 | headers = { 242 | "Accept": "application/json", 243 | "Authorization": "Bearer {}".format(YOU_TRACK_TOKEN), 244 | "Content-Type": "application/json" 245 | } 246 | data_prod = requests.get(URL_GET_PRODUCTS, headers=headers).json() 247 | 248 | upload_prod = [] 249 | for buff in product_vendor_list: 250 | upload_prod.append(buff) 251 | 252 | prod_vend = [] 253 | for i in data_prod: 254 | prod_vend.append(i['name']) 255 | 256 | temp = [] 257 | for iter in upload_prod: 258 | if iter not in prod_vend: 259 | temp.append(iter) 260 | 261 | for upload in temp: 262 | payload = { 263 | "id": "0", 264 | "&type": "FieldStyle", 265 | "name": upload 266 | } 267 | requests.post(URL_GET_PRODUCTS, headers=headers, json=payload) 268 | 269 | # check for versions------------------------------------------------------------------------------------------------ 270 | headers = { 271 | "Accept": "application/json", 272 | "Authorization": "Bearer {}".format(YOU_TRACK_TOKEN), 273 | "Content-Type": "application/json" 274 | } 275 | data_ver = requests.get(URL_GET_VERSIONS, headers=headers).json() 276 | 277 | ver_list = [] 278 | for i in data_ver: 279 | ver_list.append(i['name']) 280 | 281 | temp2 = [] 282 | for iter in temp1: 283 | if iter not in ver_list: 284 | temp2.append(iter) 285 | 286 | for upload in temp2: 287 | payload = { 288 | "id": "0", 289 | "&type": "FieldStyle", 290 | "name": upload 291 | } 292 | requests.post(URL_GET_VERSIONS, headers=headers, json=payload) 293 | 294 | # upload information on cve ---------------------------------------------------------------------------------------- 295 | buff_content = [] 296 | buff_versions = [] 297 | if re.search(r'windows', product_vendor_list[0]): 298 | con = {"name": "Microsoft Windows"} 299 | buff_content.append(con) 300 | buff_versions = versions 301 | elif re.search(r'juniper', product_vendor_list[0]): 302 | con = {"name": "Juniper"} 303 | buff_content.append(con) 304 | buff_versions = versions 305 | elif re.search(r'adaptive_security_appliance', product_vendor_list[0]): 306 | con = {"name": "Cisco ASA"} 307 | buff_content.append(con) 308 | buff_versions = versions 309 | else: 310 | if content: 311 | buff_content.append(content[0]) 312 | if versions: 313 | buff_versions.append(versions[0]) 314 | 315 | priority = '' 316 | if isinstance(score, float): 317 | if 0.1 <= score <= 3.9: 318 | priority = 'Низкая' 319 | elif 4.0 <= score <= 6.9: 320 | priority = 'Средняя' 321 | elif 7.0 <= score <= 8.9: 322 | priority = 'Высокая' 323 | elif 9.0 <= score <= 10.0: 324 | priority = 'Критическая' 325 | 326 | request_payload = { 327 | "project": { 328 | "id": YOU_TRACK_PROJECT_ID 329 | }, 330 | "summary": cve_name, 331 | "description": message, 332 | "tags": [ 333 | { 334 | "name": "В тренде", 335 | "id": "6-27", 336 | "$type": "IssueTag" 337 | } 338 | ], 339 | "customFields": [ 340 | { 341 | "name": "Продукт (пакет)", 342 | "$type": "MultiEnumIssueCustomField", 343 | "value": buff_content 344 | }, 345 | { 346 | "name": "Есть эксплоит", 347 | "$type": "SingleEnumIssueCustomField", 348 | "value": {"name": value} 349 | }, 350 | { 351 | "name": "Affected versions", 352 | "$type": "MultiEnumIssueCustomField", 353 | "value": buff_versions 354 | }, 355 | { 356 | "name": "CVSS Score", 357 | "$type": "SimpleIssueCustomField", 358 | "value": score 359 | 360 | }, 361 | { 362 | "name": "CVSS Vector", 363 | "$type": "SimpleIssueCustomField", 364 | "value": str(vector) 365 | 366 | }, 367 | { 368 | "name": "Priority", 369 | "$type": "SingleEnumIssueCustomField", 370 | "value": {"name": priority} 371 | }, 372 | ] 373 | } 374 | post = requests.post(URL, headers=headers, json=request_payload) # Выгрузка инфы о cve в YouTrack 375 | return post.status_code 376 | except: 377 | now = datetime.datetime.now() 378 | message = f'По состоянию на {now.strftime("%d-%m-%Y %H:%M")} информация об уязвимости отсутствует' 379 | headers = { 380 | "Accept": "application/json", 381 | "Authorization": "Bearer {}".format(YOU_TRACK_TOKEN), 382 | "Content-Type": "application/json" 383 | } 384 | request_payload = { 385 | "project": { 386 | "id": YOU_TRACK_PROJECT_ID 387 | }, 388 | "summary": cve, 389 | "description": message, 390 | "tags": [ 391 | { 392 | "name": "В тренде", 393 | "id": "6-27", 394 | "$type": "IssueTag" 395 | } 396 | ] 397 | } 398 | post = requests.post(URL, headers=headers, json=request_payload) # Выгрузка инфы о cve в YouTrack 399 | return post.status_code 400 | 401 | 402 | def email_alert(cve_list): 403 | recipients = [] 404 | recipients.append(USER1) 405 | recipients.append(USER2) 406 | msg = EmailMessage() 407 | msg['Subject'] = 'cvetrend.com' 408 | msg['From'] = EMAIL_HOST_USER 409 | msg['To'] = ", ".join(recipients) 410 | if len(cve_list) == 1: 411 | body = f'Добавлена информация о новой уязвимости {cve_list[0]}' 412 | else: 413 | body = f'Добавлена информация о новых уязвимостях\n {", ".join(cve_list)}' 414 | msg.set_content(body) 415 | msg.set_content(body) 416 | smtp_server = smtplib.SMTP_SSL(host=EMAIL_HOST, port=EMAIL_PORT) 417 | smtp_server.login(user=EMAIL_HOST_USER, password=EMAIL_HOST_PASSWORD) 418 | smtp_server.send_message(msg) 419 | print('Email sent {}'.format(msg['Subject'])) 420 | 421 | 422 | # alert on telegram bot--------------------Использовать на свой страх и риск-------------------------------------------- 423 | def telegram_alert(message): 424 | sticker = random.choice(stickers) 425 | # R Alert 426 | requests.get( 427 | f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage?chat_id={CHAT_ID_R}&text={message}&parse_mode=markdown") 428 | requests.get(f"https://api.telegram.org/bot{BOT_TOKEN}/"f"sendSticker?chat_id={CHAT_ID_R}&sticker={sticker}") 429 | # Dj Alert 430 | requests.get( 431 | f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage?chat_id={CHAT_ID_J}&text={message}&parse_mode=markdown") 432 | requests.get(f"https://api.telegram.org/bot{BOT_TOKEN}/sendSticker?chat_id={CHAT_ID_J}&sticker={sticker}") 433 | # A Alert 434 | requests.get( 435 | f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage?chat_id={CHAT_ID_A}&text={message}&parse_mode=markdown") 436 | requests.get(f"https://api.telegram.org/bot{BOT_TOKEN}/sendSticker?chat_id={CHAT_ID_A}&sticker={sticker}") 437 | 438 | 439 | # ------------------------------------------------------MAIN------------------------------------------------------------ 440 | headers = { 441 | "Accept": "application/json", 442 | "Authorization": "Bearer {}".format(YOU_TRACK_TOKEN), 443 | "Content-Type": "application/json" 444 | } 445 | 446 | list_summary = requests.get(MAIN_URL_CVETRENDS, headers=headers).json() # Получение задач с YouTrack 447 | 448 | # Получение информации по cve с YouTrack 449 | cve_list = [] # CVE id 450 | id_list = [] # ID Задачи 451 | tag_id = [] # ID Тэга 452 | for i, item in enumerate(list_summary): 453 | regex = re.search(r'CVE-\d{4}-\d{4,8}', str(list_summary[i]['summary'])) 454 | if regex != None: 455 | cve_list.append(str(regex.group())) 456 | id_list.append(list_summary[i]['id']) 457 | a = dict(list_summary[i]) 458 | try: 459 | tag_id.append(a.get('tags')[0].get('id')) 460 | except: 461 | tag_id.append('NO') 462 | 463 | top_cve_list = get_top_cve_list() # Получение топа cve с cvetrends.com 464 | 465 | # Добавление тега к задачам 466 | print(f'Добавление тега:') 467 | repetiotion_cve = [] 468 | for i in range(len(cve_list)): 469 | for j in range(len(top_cve_list)): 470 | if cve_list[i] == top_cve_list[j]: 471 | add_tag(id_list[i]) 472 | repetiotion_cve.append(top_cve_list[j]) 473 | 474 | # Удаление тега с задач, для которых он не актуален и добавление другого тега 475 | remove_list = [] 476 | delete_list = [] 477 | del_list = [] 478 | print(f'Удаление и изменение тега:') 479 | for i, item in enumerate(cve_list): 480 | if tag_id[i] == '6-27': 481 | remove_list.append(cve_list[i]) 482 | 483 | for item in remove_list: 484 | if item not in top_cve_list: 485 | delete_list.append(item) 486 | 487 | for i, item in enumerate(cve_list): 488 | for j in range(len(delete_list)): 489 | if cve_list[i] == delete_list[j]: 490 | del_list.append(id_list[i]) 491 | 492 | if del_list: 493 | for i, item in enumerate(del_list): 494 | print(f'{i+1} / {len(del_list)} - Удаление: {delete_tag(del_list[i])} - Изменение: {change_tag(del_list[i])}') 495 | else: 496 | print('None') 497 | 498 | # Формирование списка уязвимостей (CVE), которых нет в YouTrack 499 | add_new_cve = [] 500 | for item in top_cve_list: 501 | if item not in repetiotion_cve: 502 | add_new_cve.append(item) 503 | 504 | # Добавление информации о новых уязвимостях в YouTrack 505 | email_list = [] 506 | print(f'Добавление информации о новых уязвимостях:') 507 | if add_new_cve: 508 | time.sleep(10) 509 | for i in range(len(add_new_cve)): 510 | print(f'{i+1} / {len(add_new_cve)} - {get_cve_data(add_new_cve[i])}') 511 | if (get_cve_data(add_new_cve[i])) == 200: 512 | email_list.append(add_new_cve[i]) 513 | else: 514 | print('None') 515 | 516 | 517 | cve_list_new = [] 518 | if email_list: 519 | for item in email_list: 520 | cve_list_new.append(f'[{item}](https://nvd.nist.gov/vuln/detail/{item})') 521 | 522 | if cve_list_new: 523 | if len(cve_list_new) == 1: 524 | message = f'*CVETRENDS*\nДобавлена информация о новой уязвимости {cve_list_new[0]}' 525 | telegram_alert(message) 526 | else: 527 | message = f'*CVETRENDS*\nДобавлена информация о новых уязвимостях {", ".join(cve_list_new)}' 528 | telegram_alert(message) 529 | 530 | -------------------------------------------------------------------------------- /parsing_nkcki.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import urllib3 3 | from bs4 import BeautifulSoup 4 | import os 5 | import zipfile 6 | import json 7 | 8 | urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) 9 | PATH = os.path.join(os.path.dirname(__file__), 'bulletins') 10 | 11 | def get_links_for_bulletine(): 12 | links = [] 13 | for page_number in range(1): 14 | r = requests.get(f"https://safe-surf.ru/specialists/bulletins-nkcki/?PAGEN_1={page_number+1}") 15 | soup = BeautifulSoup(r.text, "html.parser") 16 | for vuln in soup.find_all('div', class_='cell-value'): 17 | a_tags = vuln.find_all('a') 18 | for tag in a_tags: 19 | href = tag.get('href') 20 | if href and "json.zip" in href and f"https://safe-surf.ru{href}" not in links: 21 | links.append(f"https://safe-surf.ru{href}") 22 | return links 23 | 24 | def download_json(url, directory="bulletins"): 25 | response = requests.get(url) 26 | response.raise_for_status() 27 | 28 | filename = os.path.basename(url.split("/")[-1]) 29 | file_path = os.path.join(directory, filename) 30 | with open(file_path, "wb") as file: 31 | file.write(response.content) 32 | 33 | with zipfile.ZipFile(file_path, "r") as zip_ref: 34 | zip_ref.extractall(directory) 35 | 36 | 37 | def get_info_json(): 38 | files = [] 39 | for i in os.listdir(PATH): 40 | if "zip" not in i: 41 | files.append(i) 42 | for file in files: 43 | with open(f"bulletins\\{file}", encoding='utf-8') as f: 44 | json_data = f.read() 45 | 46 | data = json.loads(json_data) 47 | total = data['total'] 48 | for j in range(total): 49 | mitre = data["data"][j]["vuln_id"]["MITRE"] 50 | date_published = data["data"][j]["date_published"] 51 | date_updated = data["data"][j]["date_updated"] 52 | print("-------------------------------------------------------------------") 53 | print("MITRE:", mitre) 54 | print("date_published:", date_published) 55 | print("date_updated:", date_updated) 56 | 57 | 58 | def remove_data(): 59 | for file in os.listdir(PATH): 60 | os.remove(f"bulletins\\{file}") 61 | 62 | 63 | links = get_links_for_bulletine() 64 | for link in links: 65 | download_json(link) 66 | get_info_json() 67 | remove_data() 68 | 69 | 70 | -------------------------------------------------------------------------------- /parsing_opencve.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3 2 | import time 3 | 4 | import jinja2 5 | import requests 6 | from bs4 import BeautifulSoup 7 | import urllib3 8 | from cpe import CPE 9 | import nvdlib 10 | import ast 11 | import re 12 | import smtplib 13 | from email.message import EmailMessage 14 | from dotenv import dotenv_values 15 | import os 16 | import random 17 | from stickers import stickers 18 | from usage import pattern, template 19 | 20 | urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) 21 | 22 | dotenv_path = os.path.join(os.path.dirname(__file__), '.env') 23 | config = dotenv_values(dotenv_path) 24 | # imported variables from .env 25 | YOU_TRACK_TOKEN = config.get("YOU_TRACK_TOKEN") 26 | YOU_TRACK_PROJECT_ID = config.get("YOU_TRACK_PROJECT_ID") 27 | YOU_TRACK_BASE_URL = config.get("YOU_TRACK_BASE_URL") 28 | URL_GET_PRODUCTS = config.get("URL_GET_PRODUCTS") 29 | URL_GET_VERSIONS = config.get("URL_GET_VERSIONS") 30 | EMAIL_HOST = config.get("EMAIL_HOST") 31 | EMAIL_PORT = config.get("EMAIL_PORT") 32 | EMAIL_HOST_PASSWORD = config.get("EMAIL_HOST_PASSWORD") 33 | EMAIL_HOST_USER = config.get("EMAIL_HOST_USER") 34 | USER1 = config.get("USER1") 35 | USER2 = config.get("USER2") 36 | MAIN_URL_OPENCVE = config.get("MAIN_URL_OPENCVE") 37 | USERNAME = config.get("USERNAME_OPENCVE") 38 | PASSWORD = config.get("PASSWORD_OPENCVE") 39 | BOT_TOKEN = config.get("BOT_TOKEN") 40 | CHAT_ID_J = config.get("CHAT_ID_J") 41 | CHAT_ID_R = config.get("CHAT_ID_R") 42 | CHAT_ID_A = config.get("CHAT_ID_A") 43 | 44 | 45 | URL = str(YOU_TRACK_BASE_URL) + "/issues" 46 | 47 | 48 | # check https://github.com/nu11secur1ty/ ------------------------------------------------------------------------------- 49 | def get_exploit_info(cve): 50 | link = 'https://github.com/nu11secur1ty/CVE-mitre' 51 | link_2 = 'https://github.com/nu11secur1ty/CVE-mitre/tree/main/2022' 52 | default_link = '' 53 | poc_cve_list = [] 54 | r = requests.get(link) 55 | soup = BeautifulSoup(r.text, "html.parser") 56 | for cve_id in soup.find_all("span", class_="css-truncate css-truncate-target d-block width-fit"): 57 | regex = re.findall(r'CVE-\d{4}-\d{4,8}', cve_id.text) 58 | if regex: 59 | poc_cve_list.append(str(regex[0])) 60 | 61 | r = requests.get(link_2) 62 | soup = BeautifulSoup(r.text, "html.parser") 63 | for cve_id in soup.find_all("span", class_="css-truncate css-truncate-target d-block width-fit"): 64 | regex = re.findall(r'CVE-\d{4}-\d{4,8}', cve_id.text) 65 | if regex: 66 | poc_cve_list.append(str(regex[0])) 67 | 68 | for item in poc_cve_list: 69 | if cve == item: 70 | default_link = f'https://github.com/nu11secur1ty/CVE-mitre/tree/main/{cve}' 71 | return default_link 72 | 73 | 74 | # check https://github.com/trickest/cve/ ------------------------------------------------------------------------------- 75 | def get_exploit_info_2(cve): 76 | # print('get_exploit_info_2') # DEBUG 77 | year = cve.split('-')[1] 78 | link = f'https://github.com/trickest/cve/tree/main/{year}' 79 | r = requests.get(link) 80 | soup = BeautifulSoup(r.text, "html.parser") 81 | default_link = '' 82 | for cve_id in soup.find_all("span", class_="css-truncate css-truncate-target d-block width-fit"): 83 | if f'{cve}.md' == cve_id.text: 84 | default_link = f'**trickest/cve** - https://github.com/trickest/cve/tree/main/{year}/{cve}.md' 85 | break 86 | return default_link 87 | 88 | 89 | # parse opencve.io------------------------------------------------------------------------------------------------------ 90 | def parsing_opencve(): 91 | url1 = 'https://www.opencve.io/login/' 92 | url2 = 'https://www.opencve.io/login' 93 | csrf_token = '' 94 | s = requests.Session() 95 | response = s.get(url1) 96 | soup = BeautifulSoup(response.text, 'lxml') 97 | 98 | # Get CSRF 99 | for a in soup.find_all('meta'): 100 | if 'name' in a.attrs: 101 | if a.attrs['name'] == 'csrf-token': 102 | csrf_token = a.attrs['content'] 103 | 104 | # Authentication 105 | s.post( 106 | url2, 107 | data={ 108 | 'username': USERNAME, 109 | 'password': PASSWORD, 110 | 'csrf_token': csrf_token, 111 | }, 112 | headers={'referer': 'https://www.opencve.io/login'}, 113 | verify=False 114 | ) 115 | # Get new CVE 116 | cve_line = [] 117 | for page_num in range(1, 20): 118 | pagination = f'https://www.opencve.io/?page={page_num}' 119 | resp = s.get(pagination) 120 | parse = BeautifulSoup(resp.text, 'lxml') 121 | for cve in parse.find_all('h3', class_='timeline-header'): 122 | index = cve.text.find('has changed') 123 | if index == -1: 124 | cve_line.append(cve.text.replace(' is a new CVE', '')) 125 | 126 | cve_line_no_replic = [] 127 | for item in cve_line: 128 | if item not in cve_line_no_replic: 129 | cve_line_no_replic.append(item[:-1]) 130 | return cve_line_no_replic 131 | 132 | 133 | # main function for upload information on YT---------------------------------------------------------------------------- 134 | def get_cve_data(cve): 135 | r = nvdlib.getCVE(cve, cpe_dict=False) 136 | cve_cpe_nodes = r.configurations.nodes 137 | cpe_nodes = ast.literal_eval(str(r.configurations)) 138 | try: 139 | score = r.v3score 140 | vector = r.v3vector 141 | except: 142 | score = 0.1 143 | vector = "Нет: cvss vector" 144 | 145 | if vector != "Нет: cvss vector": 146 | vector = r.v3vector[9:len(r.v3vector)] 147 | 148 | links = [] 149 | exploit_links = [] 150 | links.append(r.url) 151 | for t in r.cve.references.reference_data: 152 | links.append(t.url) 153 | if 'Exploit' in t.tags: 154 | exploit_links.append(t.url) 155 | 156 | if get_exploit_info(cve): # check https://github.com/nu11secur1ty/ 157 | exploit_links.append(get_exploit_info(cve)) 158 | if get_exploit_info_2(cve): # check https://github.com/trickest/cve/ 159 | exploit_links.append(get_exploit_info_2(cve)) 160 | 161 | cpe_for_product_vendors = [] 162 | if cpe_nodes: 163 | for conf in cve_cpe_nodes: 164 | if conf.operator == 'AND': 165 | children = [conf.children[0]] 166 | else: 167 | children = [conf] 168 | for child in children: 169 | for cpe in child.cpe_match: 170 | cpe_for_product_vendors.append(cpe.cpe23Uri) 171 | 172 | # parse CPE------------------------------------------------------------------------------------------------------------- 173 | product_vendor_list = [] 174 | product_image_list = [] 175 | version_list = [] 176 | for cpe in cpe_for_product_vendors: 177 | cpe_parsed = CPE(cpe) 178 | product = cpe_parsed.get_product() 179 | vendor = cpe_parsed.get_vendor() 180 | product_vendor = vendor[0] + " " + product[0] if product != vendor else product[0] 181 | product_vendor_list.append(product_vendor) 182 | product_image_list.append(product[0]) 183 | version = cpe_parsed.get_version() 184 | if version[0] != '-' and version[0] != '*': 185 | version_list.append(f'{product[0]} - {version[0]}') 186 | 187 | temp1 = [] 188 | for item in version_list: 189 | if item not in temp1: 190 | temp1.append(item) 191 | versions = [] 192 | for item in temp1: 193 | ver = {"name": item} 194 | versions.append(ver) 195 | 196 | prod = [] 197 | for item in product_image_list: 198 | if item not in prod: 199 | prod.append(item) 200 | 201 | content = [] 202 | for item in product_vendor_list: 203 | con = {"name": item} 204 | content.append(con) 205 | 206 | value = "Да" 207 | if not exploit_links: 208 | value = "Нет" 209 | 210 | # check regex in cve---------------------------------------------------------------------------------------------------- 211 | cve_name = '' 212 | cve_info = r.cve.description.description_data[0].value 213 | for item in pattern: 214 | if item.upper() in cve_info.upper(): 215 | cve_name = cve + " - " + item 216 | break 217 | else: 218 | cve_name = cve 219 | 220 | # message--------------------------------------------------------------------------------------------------------------- 221 | data = { 222 | 'cve': cve_info, 223 | 'lastModifiedDate': r.lastModifiedDate[:-7], 224 | 'publishedDate': r.publishedDate[:-7], 225 | 'configurations': cpe_nodes, 226 | 'score': score, 227 | 'vector': r.v3vector, 228 | 'links': links, 229 | 'product_vendor_list': prod, 230 | 'exploit_links': exploit_links 231 | } 232 | message = jinja2.Template(template).render(d=data) 233 | 234 | # check for product_vendor---------------------------------------------------------------------------------------------- 235 | headers = { 236 | "Accept": "application/json", 237 | "Authorization": "Bearer {}".format(YOU_TRACK_TOKEN), 238 | "Content-Type": "application/json" 239 | } 240 | data_prod = requests.get(URL_GET_PRODUCTS, headers=headers).json() 241 | 242 | upload_prod = [] 243 | for buff in product_vendor_list: 244 | upload_prod.append(buff) 245 | 246 | prod_vend = [] 247 | for i in data_prod: 248 | prod_vend.append(i['name']) 249 | 250 | temp = [] 251 | for iter in upload_prod: 252 | if iter not in prod_vend: 253 | temp.append(iter) 254 | 255 | for upload in temp: 256 | payload = { 257 | "id": "0", 258 | "&type": "FieldStyle", 259 | "name": upload 260 | } 261 | requests.post(URL_GET_PRODUCTS, headers=headers, json=payload) 262 | 263 | # check for versions---------------------------------------------------------------------------------------------------- 264 | headers = { 265 | "Accept": "application/json", 266 | "Authorization": "Bearer {}".format(YOU_TRACK_TOKEN), 267 | "Content-Type": "application/json" 268 | } 269 | data_ver = requests.get(URL_GET_VERSIONS, headers=headers).json() 270 | 271 | ver_list = [] 272 | for i in data_ver: 273 | ver_list.append(i['name']) 274 | 275 | temp2 = [] 276 | for iter in temp1: 277 | if iter not in ver_list: 278 | temp2.append(iter) 279 | 280 | for upload in temp2: 281 | payload = { 282 | "id": "0", 283 | "&type": "FieldStyle", 284 | "name": upload 285 | } 286 | requests.post(URL_GET_VERSIONS, headers=headers, json=payload) 287 | 288 | # upload information on cve--------------------------------------------------------------------------------------------- 289 | buff_content = [] 290 | buff_versions = [] 291 | check_mark = '✓' # easter_egg or bug (ya xz) 292 | if re.search(r'windows', str(product_vendor_list[0])): 293 | con = {"name": "Microsoft Windows"} 294 | buff_content.append(con) 295 | buff_versions = versions 296 | elif re.search(r'juniper', str(product_vendor_list[0])): 297 | con = {"name": "Juniper"} 298 | buff_content.append(con) 299 | buff_versions = versions 300 | elif re.search(r'adaptive_security_appliance', str(product_vendor_list[0])): 301 | con = {"name": "Cisco ASA"} 302 | buff_content.append(con) 303 | buff_versions = versions 304 | else: 305 | if content: 306 | buff_content.append(content[0]) 307 | if versions: 308 | buff_versions.append(versions[0]) 309 | 310 | priority = '' 311 | if isinstance(score, float): 312 | if 0.1 <= score <= 3.9: 313 | priority = 'Низкая' 314 | elif 4.0 <= score <= 6.9: 315 | priority = 'Средняя' 316 | elif 7.0 <= score <= 8.9: 317 | priority = 'Высокая' 318 | elif 9.0 <= score <= 10.0: 319 | priority = 'Критическая' 320 | 321 | request_payload = { 322 | "project": { 323 | "id": YOU_TRACK_PROJECT_ID 324 | }, 325 | "summary": cve_name, 326 | "description": message, 327 | "tags": [ 328 | { 329 | "name": "OpenCVE", 330 | "id": "6-20", 331 | "$type": "IssueTag" 332 | } 333 | ], 334 | "customFields": [ 335 | { 336 | "name": "Продукт (пакет)", 337 | "$type": "MultiEnumIssueCustomField", 338 | "value": buff_content 339 | }, 340 | { 341 | "name": "Есть эксплоит", 342 | "$type": "SingleEnumIssueCustomField", 343 | "value": {"name": value} 344 | }, 345 | { 346 | "name": "Affected versions", 347 | "$type": "MultiEnumIssueCustomField", 348 | "value": buff_versions 349 | }, 350 | { 351 | "name": "CVSS Score", 352 | "$type": "SimpleIssueCustomField", 353 | "value": score 354 | 355 | }, 356 | { 357 | "name": "CVSS Vector", 358 | "$type": "SimpleIssueCustomField", 359 | "value": str(vector) 360 | 361 | }, 362 | { 363 | "name": "Priority", 364 | "$type": "SingleEnumIssueCustomField", 365 | "value": {"name": priority} 366 | }, 367 | ] 368 | } 369 | # print(request_payload) #Debug 370 | post = requests.post(URL, headers=headers, json=request_payload) # Выгрузка информации о cve в YouTrack 371 | return post.status_code 372 | 373 | 374 | def email_alert(cve_list): 375 | recipients = [] 376 | recipients.append(USER1) 377 | recipients.append(USER2) 378 | msg = EmailMessage() 379 | msg['Subject'] = 'OpenCVE' 380 | msg['From'] = EMAIL_HOST_USER 381 | msg['To'] = ", ".join(recipients) 382 | if len(cve_list) == 1: 383 | body = f'Добавлена информация о новой уязвимости {cve_list[0]}' 384 | else: 385 | body = f'Добавлена информация о новых уязвимостях\n {", ".join(cve_list)}' 386 | msg.set_content(body) 387 | msg.set_content(body) 388 | smtp_server = smtplib.SMTP_SSL(host=EMAIL_HOST, port=EMAIL_PORT) 389 | smtp_server.login(user=EMAIL_HOST_USER, password=EMAIL_HOST_PASSWORD) 390 | smtp_server.send_message(msg) 391 | print('Email sent {}'.format(msg['Subject'])) 392 | 393 | 394 | # alert on telegram bot---------------------Использовать на свой страх и риск------------------------------------------- 395 | def telegram_alert(message): 396 | sticker = random.choice(stickers) 397 | sticker = random.choice(stickers) 398 | # R Alert 399 | requests.get( 400 | f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage?chat_id={CHAT_ID_R}&text={message}&parse_mode=markdown") 401 | requests.get(f"https://api.telegram.org/bot{BOT_TOKEN}/"f"sendSticker?chat_id={CHAT_ID_R}&sticker={sticker}") 402 | # Dj Alert 403 | requests.get( 404 | f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage?chat_id={CHAT_ID_J}&text={message}&parse_mode=markdown") 405 | requests.get(f"https://api.telegram.org/bot{BOT_TOKEN}/sendSticker?chat_id={CHAT_ID_J}&sticker={sticker}") 406 | # A Alert 407 | requests.get( 408 | f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage?chat_id={CHAT_ID_A}&text={message}&parse_mode=markdown") 409 | requests.get(f"https://api.telegram.org/bot{BOT_TOKEN}/sendSticker?chat_id={CHAT_ID_A}&sticker={sticker}") 410 | 411 | 412 | # -------------------------------------------------MAIN----------------------------------------------------------------- 413 | headers = { 414 | "Accept": "application/json", 415 | "Authorization": "Bearer {}".format(YOU_TRACK_TOKEN), 416 | "Content-Type": "application/json" 417 | } 418 | list_summary = requests.get(MAIN_URL_OPENCVE, headers=headers).json() # Получение задач с YouTrack 419 | cve_line = parsing_opencve() # Получение списка новых cve с сайта opencve.io 420 | 421 | sum_list = [] 422 | for n, item in enumerate(list_summary): # Получение описания для каждой уязвимости 423 | regex = re.search(r'CVE-\d{4}-\d{4,8}', list_summary[n]['summary']) 424 | if regex: 425 | sum_list.append(str(regex.group())) 426 | 427 | vuln_list = [] 428 | for item in cve_line: # Удаление идентификаторов CVE по которым уже заведены задачи 429 | if item not in sum_list: 430 | vuln_list.append(item) 431 | 432 | cve_list = [] 433 | for i, item in enumerate(vuln_list): # Заведение задач в YouTrack 434 | time.sleep(10) 435 | cve = vuln_list[i] 436 | post = get_cve_data(cve) 437 | if post == 200: 438 | cve_list.append(cve) 439 | print(f'{i + 1} / {len(vuln_list)} - {post}') 440 | 441 | 442 | cve_list_new = [] 443 | if cve_list: 444 | for item in cve_list: 445 | cve_list_new.append(f'[{item}](https://nvd.nist.gov/vuln/detail/{item})') 446 | 447 | # telegram alert 448 | if cve_list_new: 449 | if len(cve_list_new) == 1: 450 | message = f'*OPENCVE*\nДобавлена информация о новой уязвимости {cve_list_new[0]}' 451 | telegram_alert(message) 452 | else: 453 | message = f'*OPENCVE*\nДобавлена информация о новых уязвимостях {", ".join(cve_list_new)}' 454 | telegram_alert(message) 455 | 456 | -------------------------------------------------------------------------------- /parsing_pdf.py: -------------------------------------------------------------------------------- 1 | import json 2 | import PyPDF2 3 | import os.path 4 | import os 5 | import time 6 | import jinja2 7 | import requests 8 | from bs4 import BeautifulSoup 9 | import urllib3 10 | from cpe import CPE 11 | import nvdlib 12 | import ast 13 | import re 14 | import smtplib 15 | from email.message import EmailMessage 16 | from dotenv import dotenv_values 17 | import random 18 | from stickers import stickers 19 | from usage import pattern, template 20 | 21 | urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) 22 | 23 | dotenv_path = os.path.join(os.path.dirname(__file__), '.env') 24 | config = dotenv_values(dotenv_path) 25 | # imported variables from .env 26 | YOU_TRACK_TOKEN = config.get("YOU_TRACK_TOKEN") 27 | YOU_TRACK_PROJECT_ID = config.get("YOU_TRACK_PROJECT_ID") 28 | YOU_TRACK_BASE_URL = config.get("YOU_TRACK_BASE_URL") 29 | URL_GET_PRODUCTS = config.get("URL_GET_PRODUCTS") 30 | URL_GET_VERSIONS = config.get("URL_GET_VERSIONS") 31 | EMAIL_HOST = config.get("EMAIL_HOST") 32 | EMAIL_PORT = config.get("EMAIL_PORT") 33 | EMAIL_HOST_PASSWORD = config.get("EMAIL_HOST_PASSWORD") 34 | EMAIL_HOST_USER = config.get("EMAIL_HOST_USER") 35 | USER1 = config.get("USER1") 36 | USER2 = config.get("USER2") 37 | MAIN_URL_PDF = config.get("MAIN_URL_PDF") 38 | BOT_TOKEN = config.get("BOT_TOKEN") 39 | CHAT_ID_J = config.get("CHAT_ID_J") 40 | CHAT_ID_R = config.get("CHAT_ID_R") 41 | CHAT_ID_A = config.get("CHAT_ID_A") 42 | 43 | URL = str(YOU_TRACK_BASE_URL) + "/issues" 44 | PATH = os.path.join(os.path.dirname(__file__), 'bulletins') 45 | 46 | 47 | # check https://github.com/nu11secur1ty/ ------------------------------------------------------------------------------- 48 | def get_exploit_info(cve): 49 | link = 'https://github.com/nu11secur1ty/CVE-mitre' 50 | link_2 = 'https://github.com/nu11secur1ty/CVE-mitre/tree/main/2022' 51 | default_link = '' 52 | poc_cve_list = [] 53 | r = requests.get(link) 54 | soup = BeautifulSoup(r.text, "html.parser") 55 | for cve_id in soup.find_all("span", class_="css-truncate css-truncate-target d-block width-fit"): 56 | regex = re.findall(r'CVE-\d{4}-\d{4,8}', cve_id.text) 57 | if regex: 58 | poc_cve_list.append(str(regex[0])) 59 | 60 | r = requests.get(link_2) 61 | soup = BeautifulSoup(r.text, "html.parser") 62 | for cve_id in soup.find_all("span", class_="css-truncate css-truncate-target d-block width-fit"): 63 | regex = re.findall(r'CVE-\d{4}-\d{4,8}', cve_id.text) 64 | if regex: 65 | poc_cve_list.append(str(regex[0])) 66 | 67 | for item in poc_cve_list: 68 | if cve == item: 69 | default_link = f'https://github.com/nu11secur1ty/CVE-mitre/tree/main/{cve}' 70 | return default_link 71 | 72 | 73 | # check https://github.com/trickest/cve/ ------------------------------------------------------------------------------- 74 | def get_exploit_info_2(cve): 75 | # print('get_exploit_info_2') # DEBUG 76 | year = cve.split('-')[1] 77 | link = f'https://github.com/trickest/cve/tree/main/{year}' 78 | r = requests.get(link) 79 | soup = BeautifulSoup(r.text, "html.parser") 80 | default_link = '' 81 | for cve_id in soup.find_all("span", class_="css-truncate css-truncate-target d-block width-fit"): 82 | if f'{cve}.md' == cve_id.text: 83 | default_link = f'**trickest/cve** - https://github.com/trickest/cve/tree/main/{year}/{cve}.md' 84 | break 85 | return default_link 86 | 87 | 88 | # parse bulletine NKCKI------------------------------------------------------------------------------------------------ 89 | def get_cve_list(name_list): 90 | cve_list = [] 91 | cve_list_repeat = [] 92 | for name in name_list: 93 | pdf_file = open(f'{PATH}/{name}', 'rb') 94 | read_pdf = PyPDF2.PdfFileReader(pdf_file) 95 | number_of_pages = read_pdf.getNumPages() 96 | data = '' 97 | for i in range(number_of_pages): 98 | page = read_pdf.getPage(i) 99 | page_content = page.extractText() 100 | data1 = json.dumps(page_content) 101 | data += data1 102 | 103 | rez = '' 104 | for n, item in enumerate(data, start=0): 105 | if item != '\\': 106 | rez += data[n] 107 | 108 | result = '' 109 | for n, item in enumerate(rez, start=0): 110 | if item != 'n': 111 | result += rez[n] 112 | rez1 = result.replace(" ", "") 113 | 114 | regex = re.findall(r'CVE-\d{4}-\d{4,8}', rez1) 115 | for cve in regex: 116 | cve_list_repeat.append(cve) 117 | for item in cve_list_repeat: 118 | if item not in cve_list: 119 | cve_list.append(item) 120 | return cve_list 121 | 122 | 123 | def get_links_for_bulletine(): 124 | links = [] 125 | for page_number in range(2): 126 | r = requests.get(f"https://safe-surf.ru/specialists/bulletins-nkcki/?PAGEN_1={page_number+1}") 127 | soup = BeautifulSoup(r.text, "html.parser") 128 | for vuln in soup.find_all("h3"): 129 | vuln_name = re.findall(r"VULN-\d*.\d*", vuln.text) 130 | if vuln_name: 131 | full_name_vuln = vuln_name[0] + ".pdf" 132 | bulletin_pdf_url = "https://safe-surf.ru/upload/VULN/{}".format(full_name_vuln) 133 | links.append(bulletin_pdf_url) 134 | return links 135 | 136 | 137 | def create_pdf_file(links): 138 | for str in links: 139 | get = requests.get(str) 140 | name = str.replace("https://safe-surf.ru/upload/VULN/", "") 141 | check_file = os.path.exists(f'{PATH}/{name}') 142 | if check_file == 0: 143 | with open(f'{PATH}/{name}', 'wb') as f: 144 | f.write(get.content) 145 | f.close() 146 | else: 147 | print(f'file {name} already exists') 148 | f.close() 149 | 150 | 151 | def get_name_list(path): 152 | name_list = os.listdir(path) 153 | return name_list 154 | 155 | 156 | def remove_pdf_file(path): 157 | for item in path: 158 | os.remove(item) 159 | 160 | 161 | # main function for upload information on YT---------------------------------------------------------------------------- 162 | def get_cve_data(cve): 163 | try: 164 | r = nvdlib.getCVE(cve, cpe_dict=False) 165 | cve_cpe_nodes = r.configurations.nodes 166 | cpe_nodes = ast.literal_eval(str(r.configurations)) 167 | try: 168 | score = r.v3score 169 | vector = r.v3vector 170 | except: 171 | score = 0.1 172 | vector = "Нет: cvss vector" 173 | if vector != "Нет: cvss vector": 174 | vector = r.v3vector[9:len(r.v3vector)] 175 | 176 | links = [] 177 | exploit_links = [] 178 | links.append(r.url) 179 | for t in r.cve.references.reference_data: 180 | links.append(t.url) 181 | if 'Exploit' in t.tags: 182 | exploit_links.append(t.url) 183 | 184 | if get_exploit_info(cve): # check https://github.com/nu11secur1ty/ 185 | exploit_links.append(get_exploit_info(cve)) 186 | if get_exploit_info_2(cve): # check https://github.com/trickest/cve/ 187 | exploit_links.append(get_exploit_info_2(cve)) 188 | 189 | cpe_for_product_vendors = [] 190 | if cpe_nodes: 191 | for conf in cve_cpe_nodes: 192 | if conf.operator == 'AND': 193 | children = [conf.children[0]] 194 | else: 195 | children = [conf] 196 | for child in children: 197 | for cpe in child.cpe_match: 198 | cpe_for_product_vendors.append(cpe.cpe23Uri) 199 | 200 | # parse CPE--------------------------------------------------------------------------------------------------------- 201 | product_vendor_list = [] 202 | product_image_list = [] 203 | version_list = [] 204 | for cpe in cpe_for_product_vendors: 205 | cpe_parsed = CPE(cpe) 206 | product = cpe_parsed.get_product() 207 | vendor = cpe_parsed.get_vendor() 208 | product_vendor = vendor[0] + " " + product[0] if product != vendor else product[0] 209 | product_vendor_list.append(product_vendor) 210 | product_image_list.append(product[0]) 211 | version = cpe_parsed.get_version() 212 | if (version[0] != '-' and version[0] != '*'): 213 | version_list.append(f'{product[0]} - {version[0]}') 214 | 215 | temp1 = [] 216 | for item in version_list: 217 | if item not in temp1: 218 | temp1.append(item) 219 | versions = [] 220 | for item in temp1: 221 | ver = {"name": item} 222 | versions.append(ver) 223 | 224 | prod = [] 225 | for item in product_image_list: 226 | if item not in prod: 227 | prod.append(item) 228 | 229 | content = [] 230 | for item in product_vendor_list: 231 | con = {"name": item} 232 | content.append(con) 233 | 234 | value = "Да" 235 | if not exploit_links: 236 | value = "Нет" 237 | 238 | # check regex in cve------------------------------------------------------------------------------------------------ 239 | cve_name = '' 240 | cve_info = r.cve.description.description_data[0].value 241 | for item in pattern: 242 | if item.upper() in cve_info.upper(): 243 | cve_name = cve + " - " + item 244 | break 245 | else: 246 | cve_name = cve 247 | # message----------------------------------------------------------------------------------------------------------- 248 | data = { 249 | 'cve': cve_info, 250 | 'lastModifiedDate': r.lastModifiedDate[:-7], 251 | 'publishedDate': r.publishedDate[:-7], 252 | 'configurations': cpe_nodes, 253 | 'score': score, 254 | 'vector': r.v3vector, 255 | 'links': links, 256 | 'product_vendor_list': prod, 257 | 'exploit_links': exploit_links 258 | } 259 | message = jinja2.Template(template).render(d=data) 260 | 261 | # check for product_vendor------------------------------------------------------------------------------------------ 262 | headers = { 263 | "Accept": "application/json", 264 | "Authorization": "Bearer {}".format(YOU_TRACK_TOKEN), 265 | "Content-Type": "application/json" 266 | } 267 | data_prod = requests.get(URL_GET_PRODUCTS, headers=headers).json() 268 | 269 | upload_prod = [] 270 | for buff in product_vendor_list: 271 | upload_prod.append(buff) 272 | 273 | prod_vend = [] 274 | for i in data_prod: 275 | prod_vend.append(i['name']) 276 | 277 | temp = [] 278 | for iter in upload_prod: 279 | if iter not in prod_vend: 280 | temp.append(iter) 281 | 282 | for upload in temp: 283 | payload = { 284 | "id": "0", 285 | "&type": "FieldStyle", 286 | "name": upload 287 | } 288 | requests.post(URL_GET_PRODUCTS, headers=headers, json=payload) 289 | 290 | # check for versions------------------------------------------------------------------------------------------------ 291 | headers = { 292 | "Accept": "application/json", 293 | "Authorization": "Bearer {}".format(YOU_TRACK_TOKEN), 294 | "Content-Type": "application/json" 295 | } 296 | data_ver = requests.get(URL_GET_VERSIONS, headers=headers).json() 297 | 298 | ver_list = [] 299 | for i in data_ver: 300 | ver_list.append(i['name']) 301 | 302 | temp2 = [] 303 | for iter in temp1: 304 | if iter not in ver_list: 305 | temp2.append(iter) 306 | 307 | for upload in temp2: 308 | payload = { 309 | "id": "0", 310 | "&type": "FieldStyle", 311 | "name": upload 312 | } 313 | requests.post(URL_GET_VERSIONS, headers=headers, json=payload) 314 | 315 | # upload information on cve---------------------------------------------------------------------------------------- 316 | buff_content = [] 317 | buff_versions = [] 318 | if re.search(r'windows', product_vendor_list[0]): 319 | con = {"name": "Microsoft Windows"} 320 | buff_content.append(con) 321 | buff_versions = versions 322 | elif re.search(r'juniper', product_vendor_list[0]): 323 | con = {"name": "Juniper"} 324 | buff_content.append(con) 325 | buff_versions = versions 326 | elif re.search(r'adaptive_security_appliance', product_vendor_list[0]): 327 | con = {"name": "Cisco ASA"} 328 | buff_content.append(con) 329 | buff_versions = versions 330 | else: 331 | if content: 332 | buff_content.append(content[0]) 333 | if versions: 334 | buff_versions.append(versions[0]) 335 | 336 | priority = '' 337 | if isinstance(score, float): 338 | if 0.1 <= score <= 3.9: 339 | priority = 'Низкая' 340 | elif 4.0 <= score <= 6.9: 341 | priority = 'Средняя' 342 | elif 7.0 <= score <= 8.9: 343 | priority = 'Высокая' 344 | elif 9.0 <= score <= 10.0: 345 | priority = 'Критическая' 346 | 347 | request_payload = { 348 | "project": { 349 | "id": YOU_TRACK_PROJECT_ID 350 | }, 351 | "summary": cve_name, 352 | "description": message, 353 | "tags": [ 354 | { 355 | "name": "Бюллетени НКЦКИ", 356 | "id": "6-4", 357 | "$type": "IssueTag" 358 | } 359 | ], 360 | "customFields": [ 361 | { 362 | "name": "Продукт (пакет)", 363 | "$type": "MultiEnumIssueCustomField", 364 | "value": buff_content 365 | }, 366 | { 367 | "name": "Есть эксплоит", 368 | "$type": "SingleEnumIssueCustomField", 369 | "value": {"name": value} 370 | }, 371 | { 372 | "name": "Affected versions", 373 | "$type": "MultiEnumIssueCustomField", 374 | "value": buff_versions 375 | }, 376 | { 377 | "name": "CVSS Score", 378 | "$type": "SimpleIssueCustomField", 379 | "value": score 380 | 381 | }, 382 | { 383 | "name": "CVSS Vector", 384 | "$type": "SimpleIssueCustomField", 385 | "value": str(vector) 386 | 387 | }, 388 | { 389 | "name": "Priority", 390 | "$type": "SingleEnumIssueCustomField", 391 | "value": {"name": priority} 392 | }, 393 | ] 394 | } 395 | post = requests.post(URL, headers=headers, json=request_payload) # Выгрузка инфы о cve в YouTrack 396 | return post.status_code 397 | except: 398 | pass 399 | 400 | 401 | def email_alert(cve_list): 402 | recipients = [] 403 | recipients.append(USER1) 404 | recipients.append(USER2) 405 | msg = EmailMessage() 406 | msg['Subject'] = 'НКЦКИ' 407 | msg['From'] = EMAIL_HOST_USER 408 | msg['To'] = ", ".join(recipients) 409 | if len(cve_list) == 1: 410 | body = f'Добавлена информация о новой уязвимости {cve_list[0]}' 411 | else: 412 | body = f'Добавлена информация о новых уязвимостях\n {", ".join(cve_list)}' 413 | msg.set_content(body) 414 | msg.set_content(body) 415 | smtp_server = smtplib.SMTP_SSL(host=EMAIL_HOST, port=EMAIL_PORT) 416 | smtp_server.login(user=EMAIL_HOST_USER, password=EMAIL_HOST_PASSWORD) 417 | smtp_server.send_message(msg) 418 | print('Email sent {}'.format(msg['Subject'])) 419 | 420 | 421 | # alert on telegram bot------------------------Использовать на свой страх и риск---------------------------------------- 422 | def telegram_alert(message): 423 | sticker = random.choice(stickers) 424 | sticker = random.choice(stickers) 425 | # R Alert 426 | requests.get( 427 | f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage?chat_id={CHAT_ID_R}&text={message}&parse_mode=markdown") 428 | requests.get(f"https://api.telegram.org/bot{BOT_TOKEN}/"f"sendSticker?chat_id={CHAT_ID_R}&sticker={sticker}") 429 | # Dj Alert 430 | requests.get( 431 | f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage?chat_id={CHAT_ID_J}&text={message}&parse_mode=markdown") 432 | requests.get(f"https://api.telegram.org/bot{BOT_TOKEN}/sendSticker?chat_id={CHAT_ID_J}&sticker={sticker}") 433 | # A Alert 434 | requests.get( 435 | f"https://api.telegram.org/bot{BOT_TOKEN}/sendMessage?chat_id={CHAT_ID_A}&text={message}&parse_mode=markdown") 436 | requests.get(f"https://api.telegram.org/bot{BOT_TOKEN}/sendSticker?chat_id={CHAT_ID_A}&sticker={sticker}") 437 | 438 | 439 | # ---------------------------------------------------MAIN--------------------------------------------------------------- 440 | headers = { 441 | "Accept": "application/json", 442 | "Authorization": "Bearer {}".format(YOU_TRACK_TOKEN), 443 | "Content-Type": "application/json" 444 | } 445 | list_summary = requests.get(MAIN_URL_PDF, headers=headers).json() # Получение задач с YouTrack 446 | links = get_links_for_bulletine() 447 | create_pdf_file(links) 448 | name_list = get_name_list(PATH) # Получение имен скаченных файлов для парсинга 449 | cve_line = get_cve_list(name_list) # Получение списка cve из биллютеня НКЦКИ 450 | 451 | broke_sum_list = [] 452 | for n in range(len(list_summary)): # Получение описания для каждой уязвимости 453 | broke_sum_list.append(list_summary[n]['summary']) 454 | 455 | sum_list = [] 456 | for item in broke_sum_list: # Выдергивание идентификатора CVE из каждого описания уязвимости 457 | regex = re.search(r'CVE-\d{4}-\d{4,8}', item) 458 | if regex: 459 | sum_list.append(str(regex.group())) 460 | 461 | vuln_list = [] 462 | for item in cve_line: # Удаление идентификаторов CVE по которым уже заведены задачи 463 | if item not in sum_list: 464 | vuln_list.append(item) 465 | 466 | 467 | cve_list = [] 468 | for i, item in enumerate(vuln_list): # Заведение задач в YouTrack 469 | time.sleep(10) 470 | cve = vuln_list[i] 471 | post = get_cve_data(cve) 472 | if post == 200: 473 | cve_list.append(cve) 474 | print(f'{i + 1} / {len(vuln_list)} - {post} - {cve}') 475 | else: 476 | print(f'{i + 1} / {len(vuln_list)} - No information for {cve}') 477 | 478 | 479 | cve_list_new = [] 480 | if cve_list: 481 | for item in cve_list: 482 | cve_list_new.append(f'[{item}](https://nvd.nist.gov/vuln/detail/{item})') 483 | 484 | # telegram alert 485 | if cve_list_new: 486 | if len(cve_list_new) == 1: 487 | message = f'*NKCKI*\nДобавлена информация о новой уязвимости {cve_list_new[0]}' 488 | telegram_alert(message) 489 | else: 490 | message = f'*NKCKI*\nДобавлена информация о новых уязвимостях {", ".join(cve_list_new)}' 491 | telegram_alert(message) 492 | 493 | 494 | # remove pdf buffer----------------------------------------------------------------------------------------------------- 495 | time.sleep(5) 496 | path_to_remove_pdf = [] 497 | for item in name_list: 498 | path_to_remove_pdf.append(f'{PATH}/{item}') 499 | remove_pdf_file(path_to_remove_pdf) 500 | 501 | 502 | 503 | 504 | 505 | -------------------------------------------------------------------------------- /remove_repetitions.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import re 3 | import urllib3 4 | from dotenv import dotenv_values 5 | import os 6 | 7 | urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) 8 | 9 | dotenv_path = os.path.join(os.path.dirname(__file__), '.env') 10 | config = dotenv_values(dotenv_path) 11 | YOU_TRACK_TOKEN = config.get("YOU_TRACK_TOKEN") 12 | MAIN_URL_REMOVE = config.get("MAIN_URL_REMOVE") 13 | URL_REMOVE = config.get("URL_REMOVE") 14 | headers = { 15 | "Accept": "application/json", 16 | "Authorization": "Bearer {}".format(YOU_TRACK_TOKEN), 17 | "Content-Type": "application/json" 18 | } 19 | list_summary = requests.get(MAIN_URL_REMOVE, headers=headers).json() 20 | 21 | buff_cve_list = [] 22 | buff_id_list = [] 23 | for i in range(len(list_summary)): 24 | regex = re.search(r'CVE-\d{4}-\d{4,6}', str(list_summary[i]['summary'])) 25 | if regex is not None: 26 | buff_cve_list.append(str(regex.group())) 27 | buff_id_list.append(list_summary[i]['id']) 28 | 29 | no_repeat_list = [] 30 | repeat_list = [] 31 | for i in range(len(buff_cve_list)): 32 | if buff_cve_list[i] not in no_repeat_list: 33 | no_repeat_list.append(buff_cve_list[i]) 34 | else: 35 | repeat_list.append(buff_id_list[i]) 36 | 37 | for n, name in enumerate(repeat_list, start=1): 38 | URL1 = f'{URL_REMOVE}{name}' 39 | headers = { 40 | "Accept": "application/json", 41 | "Authorization": "Bearer {}".format(YOU_TRACK_TOKEN), 42 | "Content-Type": "application/json" 43 | } 44 | delete = requests.delete(URL1, headers=headers) 45 | if delete.status_code == 200: 46 | print(f'{n} / {len(repeat_list)} - OK') 47 | else: 48 | print(f'{n} / {len(repeat_list)} - {delete.json()}') -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Jinja2==3.1.1 2 | MarkupSafe==2.1.1 3 | PyPDF2==1.27.6 4 | beautifulsoup4==4.11.1 5 | certifi==2021.10.8 6 | charset-normalizer==2.0.12 7 | cpe==1.2.1 8 | idna==3.3 9 | lxml==4.8.0 10 | nvdlib==0.5.6 11 | python-dotenv==0.20.0 12 | requests==2.27.1 13 | setuptools==57.0.0 14 | soupsieve==2.3.2.post1 15 | urllib3==1.26.9 16 | wheel==0.36.2 17 | webdriver-manager==3.7.0 18 | selenium==4.2.0 -------------------------------------------------------------------------------- /stickers.py: -------------------------------------------------------------------------------- 1 | stickers = ['CAACAgIAAxkBAAEEwIRig7k4Kx0wERiO0KG7ZiUO2-83lQACJxgAApd54UvWARYS3sXfTyQE', 2 | 'CAACAgIAAxkBAAEEwItig7nT06HJDyRzu8NkPS2IEoAUOQAC1xgAAm4m4UsFYy3CmOv8qyQE', 3 | 'CAACAgIAAxkBAAEEwI9ig7nwn29hxgmUnm-vETat4hn2MwACKBYAAi8x4UvsAnm5hTjEHSQE', 4 | 'CAACAgIAAxkBAAEEwJNig7oGxDvg4d0amFCSg4suTeKqvQACthMAAj836UveLP8TsUC1LyQE', 5 | 'CAACAgIAAxkBAAEEwJVig7oddSwk4JUDF3RlTQ_PK5zg9wACQRUAAjL6OUqi9LEwcYo_nCQE', 6 | 'CAACAgIAAxkBAAEEwJ5ig7pGLz5sc9_ibS6I8atGYwS5MgACSRIAAtU86UsZ3n-O8Yj8qCQE', 7 | 'CAACAgIAAxkBAAEEwKBig7pauX5hbDCrjt2br-22esWDjAACSQkAAnwFBxuOiNjG04BODCQE', 8 | 'CAACAgIAAxkBAAEEwKJig7pxnTLFx1_t7KugXnp124voHAACLxAAAoF3MElJ-ZgtFAEtNCQE', 9 | 'CAACAgQAAxkBAAEEwKRig7p9bXY91DWx0KMU0PhY6zkk1wACOAEAAqghIQZTVv_n9B-txSQE', 10 | 'CAACAgIAAxkBAAEEwKZig7s-TX4jTW7ZlskuVon08-BcVQACQwkAAnwFBxvA4qMYIRAuFyQE', 11 | 'CAACAgIAAxkBAAEEwKhig7taD0bz4ENmolhlRl4N4aFcCAACHAoAAnwFBxuc89MsqMShzSQE', 12 | 'CAACAgIAAxkBAAEEwKpig7uO7dQUYH53tKahdZgZ8trHzgACHwkAAnwFBxu1CPHRDLongCQE', 13 | 'CAACAgIAAxkBAAEEwK5ig7u13IdNqt9lV4GN0gABgzjaeAIAAq4AA0BndBFiHoVVdUAAAYskBA', 14 | 'CAACAgIAAxkBAAEEwLJig7vO_EDLkijL0iWYJTX9gXRV1wACDgADSmsdAAEswKA7Mels6SQE', 15 | 'CAACAgIAAxkBAAEEwLZig7vcrc8WbWlptV-JGvt-bo8yywACDgADWgw3FTBFae29B1_gJAQ', 16 | 'CAACAgEAAxkBAAEEwYpihJS4AljAGtv4kuL-xNsgvaAuOwACvwEAAtlluEVflbhA19jCLiQE', 17 | 'CAACAgEAAxkBAAEEwYxihJTTxxMn-33XfpPUmrLtTuhnhQACywEAArN_qEcH7VIuAj29xSQE', 18 | 'CAACAgIAAxkBAAEEwY5ihJUEtQnw7IUkTKNLcxejd5YfpQACOBMAAhYsGUloBvxOqz_qgiQE', 19 | 'CAACAgIAAxkBAAEEwZBihJUcZunYKqpY3QM5LrD1eUXXCQAClgADQGd0ES7zulUzSkXgJAQ', 20 | 'CAACAgIAAxkBAAEEwZJihJUpT6q3i5MSdDB9u5JXhmp1DQACqwADQGd0ETIWiA8B9vQMJAQ', 21 | 'CAACAgIAAxkBAAEEwZRihJU9HqRdYjvVfCE0-5qh79ExTwAC1gADQGd0EV_pKKknvGNhJAQ', 22 | 'CAACAgEAAxkBAAEEwZZihJWbDFegnCe7Jfg8U2w7DVo7rQACOgEAAstisUUWqpO2euH8kyQE'] -------------------------------------------------------------------------------- /usage.py: -------------------------------------------------------------------------------- 1 | pattern = ['Stack-based buffer overflow', 'Arbitrary command execution', 'Obtain sensitive information', 2 | 'Local privilege escalation', 'Security Feature Bypass', 'Out-of-bounds read', 'Out of bounds read', 3 | 'Denial of service', 'Denial-of-service', 'Execute arbitrary code', 'Expose the credentials', 4 | 'Cross-site scripting (XSS)', 'Privilege escalation', 'Reflective XSS Vulnerability', 5 | 'Execution of arbitrary programs', 'Server-side request forgery (SSRF)', 'Stack overflow', 6 | 'Execute arbitrary commands', 'Obtain highly sensitive information', 'Bypass security', 7 | 'Remote Code Execution', 'Memory Corruption', 'Arbitrary code execution', 'CSV Injection', 8 | 'Heap corruption', 'Out of bounds memory access', 'Sandbox escape', 'NULL pointer dereference', 9 | 'Remote Code Execution', 'RCE', 'Authentication Error', 'Use-After-Free', 'Use After Free', 10 | 'Corrupt Memory', 'Execute Untrusted Code', 'Run Arbitrary Code', 'heap out-of-bounds write', 11 | 'OS Command injection', 'Elevation of Privilege', 'Race condition', 'Access violation', 'Infinite loop'] 12 | 13 | template = """ 14 | ### Описание 15 | 16 | {{d.cve}} 17 | 18 | ### Дата публикации 19 | 20 | {{d.lastModifiedDate}} 21 | 22 | ### Дата выявления 23 | 24 | {{d.publishedDate}} 25 | 26 | 27 | ### Продукт, вендор 28 | 29 |