├── .github └── workflows │ ├── run.yml │ └── site.yml ├── .gitignore ├── README.md ├── cve.sqlite ├── main.py ├── mkdocs.yml └── web └── README.md /.github/workflows/run.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: update md 4 | 5 | # Controls when the action will run. 6 | on: 7 | # Triggers the workflow on push or pull request events but only for the master branch 8 | schedule: 9 | - cron: "*/60 * * * *" 10 | 11 | # Allows you to run this workflow manually from the Actions tab 12 | workflow_dispatch: 13 | 14 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 15 | jobs: 16 | # This workflow contains a single job called "build" 17 | build: 18 | # The type of runner that the job will run on 19 | runs-on: ubuntu-latest 20 | 21 | # Steps represent a sequence of tasks that will be executed as part of the job 22 | steps: 23 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 24 | - uses: actions/checkout@v2 25 | 26 | # Runs a single command using the runners shell 27 | - name: Run a one-line script 28 | run: echo starting workflow! 29 | 30 | - name: Set up Python 3.7 31 | uses: actions/setup-python@v1 32 | with: 33 | python-version: 3.7 34 | - name: install python packages 35 | run: pip install peewee requests mkdocs mkdocs-bootstrap386 36 | - name: run script 37 | run: python main.py 38 | 39 | - name: commit 40 | run: | 41 | git config --global user.email action@github.com 42 | git config --global user.name action 43 | git add . 44 | git commit -m "update db" -a 45 | 46 | - name: Push changes 47 | uses: ad-m/github-push-action@master 48 | with: 49 | github_token: ${{ secrets.GITHUB_TOKEN }} 50 | -------------------------------------------------------------------------------- /.github/workflows/site.yml: -------------------------------------------------------------------------------- 1 | name: update website 2 | on: 3 | schedule: 4 | - cron: "15 */1 * * *" 5 | 6 | jobs: 7 | build: 8 | name: Deploy docs 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout master 12 | uses: actions/checkout@v1 13 | - name: install python packages 14 | run: pip install mkdocs-bootstrap386 15 | - name: run shell 16 | run: cp README.md web/ 17 | - name: Deploy docs 18 | uses: mhausenblas/mkdocs-deploy-gh-pages@master 19 | env: 20 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # File created using '.gitignore Generator' for Visual Studio Code: https://bit.ly/vscode-gig 2 | 3 | # Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,macos,python,vscode 4 | # Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,macos,python,vscode 5 | 6 | ### macOS ### 7 | # General 8 | .DS_Store 9 | .AppleDouble 10 | .LSOverride 11 | 12 | # Icon must end with two \r 13 | Icon 14 | 15 | 16 | # Thumbnails 17 | ._* 18 | 19 | # Files that might appear in the root of a volume 20 | .DocumentRevisions-V100 21 | .fseventsd 22 | .Spotlight-V100 23 | .TemporaryItems 24 | .Trashes 25 | .VolumeIcon.icns 26 | .com.apple.timemachine.donotpresent 27 | 28 | # Directories potentially created on remote AFP share 29 | .AppleDB 30 | .AppleDesktop 31 | Network Trash Folder 32 | Temporary Items 33 | .apdisk 34 | 35 | ### Python ### 36 | # Byte-compiled / optimized / DLL files 37 | __pycache__/ 38 | *.py[cod] 39 | *$py.class 40 | 41 | site/site/ 42 | # C extensions 43 | *.so 44 | 45 | # Distribution / packaging 46 | .Python 47 | build/ 48 | develop-eggs/ 49 | dist/ 50 | downloads/ 51 | eggs/ 52 | .eggs/ 53 | parts/ 54 | sdist/ 55 | var/ 56 | wheels/ 57 | pip-wheel-metadata/ 58 | share/python-wheels/ 59 | *.egg-info/ 60 | .installed.cfg 61 | *.egg 62 | MANIFEST 63 | 64 | # PyInstaller 65 | # Usually these files are written by a python script from a template 66 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 67 | *.manifest 68 | *.spec 69 | 70 | # Installer logs 71 | pip-log.txt 72 | pip-delete-this-directory.txt 73 | 74 | # Unit test / coverage reports 75 | htmlcov/ 76 | .tox/ 77 | .nox/ 78 | .coverage 79 | .coverage.* 80 | .cache 81 | nosetests.xml 82 | coverage.xml 83 | *.cover 84 | *.py,cover 85 | .hypothesis/ 86 | .pytest_cache/ 87 | pytestdebug.log 88 | 89 | # Translations 90 | *.mo 91 | *.pot 92 | 93 | # Django stuff: 94 | *.log 95 | local_settings.py 96 | db.sqlite3 97 | db.sqlite3-journal 98 | 99 | # Flask stuff: 100 | instance/ 101 | .webassets-cache 102 | 103 | # Scrapy stuff: 104 | .scrapy 105 | 106 | # Sphinx documentation 107 | docs/_build/ 108 | doc/_build/ 109 | 110 | # PyBuilder 111 | target/ 112 | 113 | # Jupyter Notebook 114 | .ipynb_checkpoints 115 | 116 | # IPython 117 | profile_default/ 118 | ipython_config.py 119 | 120 | # pyenv 121 | .python-version 122 | 123 | # pipenv 124 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 125 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 126 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 127 | # install all needed dependencies. 128 | #Pipfile.lock 129 | 130 | # poetry 131 | #poetry.lock 132 | 133 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 134 | __pypackages__/ 135 | 136 | # Celery stuff 137 | celerybeat-schedule 138 | celerybeat.pid 139 | 140 | # SageMath parsed files 141 | *.sage.py 142 | 143 | # Environments 144 | # .env 145 | .env/ 146 | .venv/ 147 | env/ 148 | venv/ 149 | ENV/ 150 | env.bak/ 151 | venv.bak/ 152 | pythonenv* 153 | 154 | # Spyder project settings 155 | .spyderproject 156 | .spyproject 157 | 158 | # Rope project settings 159 | .ropeproject 160 | 161 | # mkdocs documentation 162 | /site 163 | 164 | # mypy 165 | .mypy_cache/ 166 | .dmypy.json 167 | dmypy.json 168 | 169 | # Pyre type checker 170 | .pyre/ 171 | 172 | # pytype static type analyzer 173 | .pytype/ 174 | 175 | # operating system-related files 176 | *.DS_Store #file properties cache/storage on macOS 177 | Thumbs.db #thumbnail cache on Windows 178 | 179 | # profiling data 180 | .prof 181 | 182 | 183 | ### VisualStudioCode ### 184 | .vscode/ 185 | .vscode/* 186 | !.vscode/tasks.json 187 | !.vscode/launch.json 188 | *.code-workspace 189 | 190 | ### VisualStudioCode Patch ### 191 | # Ignore all local history of files 192 | .history 193 | .ionide 194 | 195 | ### vscode ### 196 | !.vscode/settings.json 197 | !.vscode/extensions.json 198 | 199 | # End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,macos,python,vscode 200 | 201 | # Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option) 202 | 203 | -------------------------------------------------------------------------------- /cve.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Mr-xn/cve_monitor/6835ff4970ce5d3478c84ac84078c2f41087a8ac/cve.sqlite -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Description: Editor's info in the top of the file 3 | Author: p1ay8y3ar 4 | Date: 2021-04-01 23:53:55 5 | LastEditor: p1ay8y3ar 6 | LastEditTime: 2021-04-15 00:13:48 7 | Email: p1ay8y3ar@gmail.com 8 | ''' 9 | 10 | import requests 11 | from peewee import * 12 | from datetime import datetime 13 | import time 14 | import random 15 | import math 16 | db = SqliteDatabase("cve.sqlite") 17 | 18 | 19 | class CVE_DB(Model): 20 | id = IntegerField() 21 | full_name = CharField(max_length=1024) 22 | description = CharField(max_length=4098) 23 | url = CharField(max_length=1024) 24 | created_at = CharField(max_length=128) 25 | 26 | class Meta: 27 | database = db 28 | 29 | 30 | db.connect() 31 | db.create_tables([CVE_DB]) 32 | 33 | 34 | def write_file(new_contents): 35 | with open("README.md") as f: 36 | #去除标题 37 | for _ in range(7): 38 | f.readline() 39 | 40 | old = f.read() 41 | new = new_contents + old 42 | with open("README.md", "w") as f: 43 | f.write(new) 44 | 45 | 46 | def craw_all(): 47 | # 这是爬取所有的,github api限制每分钟请求最多30次 48 | api = "https://api.github.com/search/repositories?q=CVE-{}&sort=updated&per_page=100&page={}" 49 | item_list = [] 50 | for i in range(1999, 2022, 1): 51 | try: 52 | reqtem = requests.get(api.format(i, 1)).json() 53 | total_count = reqtem["total_count"] 54 | print("{}年,共{}条".format(i, total_count)) 55 | for_count = math.ceil(total_count / 100) + 1 56 | time.sleep(random.randint(3, 15)) 57 | except Exception as e: 58 | print("请求数量的时候发生错误", e) 59 | continue 60 | 61 | for j in range(1, for_count, 1): 62 | try: 63 | req = requests.get(api.format(i, j)).json() 64 | items = req["items"] 65 | item_list.extend(items) 66 | print("{}年，第{}轮，爬取{}条".format(i, j, len(items))) 67 | time.sleep(random.randint(3, 15)) 68 | except Exception as e: 69 | print("网络发生错误", e) 70 | continue 71 | 72 | return item_list 73 | 74 | 75 | def get_info(year): 76 | # 监控用的 77 | try: 78 | 79 | api = "https://api.github.com/search/repositories?q=CVE-{}&sort=updated".format( 80 | year) 81 | # 请求API 82 | req = requests.get(api).json() 83 | items = req["items"] 84 | 85 | return items 86 | except Exception as e: 87 | print("网络请求发生错误", e) 88 | return None 89 | 90 | 91 | def db_match(items): 92 | r_list = [] 93 | for item in items: 94 | id = item["id"] 95 | if CVE_DB.select().where(CVE_DB.id == id).count() != 0: 96 | continue 97 | full_name = item["full_name"] 98 | description = item["description"] 99 | if description == "" or description == None: 100 | description = 'no description' 101 | else: 102 | description = description.strip() 103 | url = item["html_url"] 104 | created_at = item["created_at"] 105 | r_list.append({ 106 | "id": id, 107 | "full_name": full_name, 108 | "description": description, 109 | "url": url, 110 | "created_at": created_at 111 | }) 112 | CVE_DB.create(id=id, 113 | full_name=full_name, 114 | description=description, 115 | url=url, 116 | created_at=created_at) 117 | 118 | return sorted(r_list, key=lambda e: e.__getitem__('created_at')) 119 | 120 | 121 | def update_all(): 122 | sorted_list = craw_all() 123 | sorted = db_match(sorted_list) 124 | if len(sorted) != 0: 125 | print("更新{}条".format(len(sorted))) 126 | sorted_list.extend(sorted) 127 | newline = "" 128 | for s in sorted_list: 129 | line = "**{}** : [{}]({}) create time: {}\n\n".format( 130 | s["description"], s["full_name"], s["url"], s["created_at"]) 131 | newline = line + newline 132 | print(newline) 133 | if newline != "": 134 | newline = "# Automatic monitor github cve using Github Actions \n\n > update time: {} total: {} \n\n".format( 135 | datetime.now(), 136 | CVE_DB.select().where(CVE_DB.id != None).count()) + newline 137 | 138 | write_file(newline) 139 | 140 | 141 | def main(): 142 | # 下面是监控用的 143 | year = datetime.now().year 144 | sorted_list = [] 145 | for i in range(year, 1999, -1): 146 | item = get_info(i) 147 | if item is None or len(item) == 0: 148 | continue 149 | print("{}年,获取原始数据:{}条".format(i, len(item))) 150 | sorted = db_match(item) 151 | if len(sorted) != 0: 152 | print("{}年,更新{}条".format(i, len(sorted))) 153 | sorted_list.extend(sorted) 154 | count = random.randint(3, 15) 155 | time.sleep(count) 156 | # print(sorted_list) 157 | 158 | newline = "" 159 | for s in sorted_list: 160 | line = "**{}** : [{}]({}) create time: {}\n\n".format( 161 | s["description"], s["full_name"], s["url"], s["created_at"]) 162 | newline = line + newline 163 | print(newline) 164 | if newline != "": 165 | newline = "# Automatic monitor github cve using Github Actions \n\n > update time: {} total: {} \n\n \n ![star me](https://img.shields.io/badge/star%20me-click%20--%3E-orange) [cve monitor](https://github.com/p1ay8y3ar/cve_monitor) [Browsing through the web](https://p1ay8y3ar.github.io/cve_monitor/) ![visitors](https://visitor-badge.glitch.me/badge?page_id=cve_monitor) \n\n".format( 166 | datetime.now(), 167 | CVE_DB.select().where(CVE_DB.id != None).count()) + newline 168 | 169 | write_file(newline) 170 | 171 | 172 | if __name__ == "__main__": 173 | # update_all() 174 | main() -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_author: "p1ay8y3ar" 2 | repo_name: "p1ay8y3ar/cve_monitor" 3 | repo_url: "https://github.com/p1ay8y3ar/cve_monitor" 4 | docs_dir: web/ 5 | site_name: CVE Monitor 6 | theme: 7 | name: material 8 | 9 | 10 | markdown_extensions: 11 | - admonition 12 | - codehilite: 13 | guess_lang: false 14 | linenums: false 15 | - toc: 16 | permalink: true 17 | - footnotes 18 | - meta 19 | - def_list 20 | - pymdownx.arithmatex 21 | - pymdownx.betterem: 22 | smart_enable: all 23 | - pymdownx.caret 24 | - pymdownx.critic 25 | - pymdownx.details 26 | - pymdownx.emoji: 27 | emoji_generator: !!python/name:pymdownx.emoji.to_png 28 | - pymdownx.inlinehilite 29 | - pymdownx.magiclink 30 | - pymdownx.mark 31 | - pymdownx.smartsymbols 32 | - pymdownx.superfences 33 | - pymdownx.tasklist 34 | - pymdownx.tilde 35 | 36 | copyright: "https://github.com/p1ay8y3ar" 37 | google_analytics: ['G-T1CS54L5LC', 'p1ay8y3ar.github.io/cve_monitor/'] 38 | plugins: [] -------------------------------------------------------------------------------- /web/README.md: -------------------------------------------------------------------------------- 1 | 9 | --------------------------------------------------------------------------------