├── .gitignore ├── .vscode ├── launch.json └── settings.json ├── Dockerfile ├── LICENSE ├── README.md ├── apps ├── __init__.py ├── cmdb │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── migrations │ │ └── __init__.py │ ├── model │ │ └── __init__.py │ ├── serializer │ │ ├── __init__.py │ │ ├── serializer_assets.py │ │ └── serializer_cmdb.py │ ├── serializers.py │ ├── tests.py │ ├── urls.py │ ├── view │ │ ├── __init__.py │ │ ├── view_assets.py │ │ └── view_cmdb.py │ └── views.py ├── dashboard │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── migrations │ │ └── __init__.py │ ├── serializers.py │ ├── tests.py │ ├── urls.py │ └── views.py ├── deploy │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── consumers.py │ ├── documents.py │ ├── documents_order.py │ ├── ext_func.py │ ├── migrations │ │ └── __init__.py │ ├── rds_transfer.py │ ├── routing.py │ ├── serializers.py │ ├── serializers_order.py │ ├── signals.py │ ├── tests.py │ └── views.py ├── ucenter │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── migrations │ │ └── __init__.py │ ├── serializers.py │ ├── tests.py │ └── views.py ├── workflow │ ├── __init__.py │ ├── apps.py │ ├── callback_common.py │ ├── ext_func.py │ ├── lifecycle.py │ ├── migrations │ │ └── __init__.py │ ├── notice.py │ ├── serializers.py │ └── views │ │ ├── callback.py │ │ ├── category.py │ │ ├── my_related.py │ │ ├── my_request.py │ │ ├── my_upcoming.py │ │ ├── template.py │ │ └── workflow.py └── workflow_callback │ ├── __init__.py │ ├── apps.py │ ├── middleware.py │ ├── urls.py │ └── views │ ├── app.py │ └── base.py ├── celery.sh ├── celery_tasks ├── __init__.py ├── celery.py ├── celeryconfig.py └── tasks.py ├── common ├── CeleryRedisClusterBackend.py ├── MailSend.py ├── __init__.py ├── ansible_callback │ ├── profile_tasks.py │ └── pyyaml.py ├── custom_format.py ├── exception.py ├── ext_fun.py ├── extends │ ├── JwtAuth.py │ ├── __init__.py │ ├── authenticate.py │ ├── decorators.py │ ├── django_qcluster.py │ ├── fernet.py │ ├── filters.py │ ├── handler.py │ ├── models.py │ ├── pagination.py │ ├── permissions.py │ ├── q_redis_broker.py │ ├── renderers.py │ ├── serializers.py │ ├── storage.py │ └── viewsets.py ├── get_ip.py ├── kubernetes_utils.py ├── md5.py ├── recursive.py ├── timer.py ├── utils │ ├── AesCipher.py │ ├── AnsibleAPI.py │ ├── AnsibleCallback.py │ ├── AtlassianJiraAPI.py │ ├── DocumentRegistry.py │ ├── DyInventory.py │ ├── ElasticSearchAPI.py │ ├── GitLabAPI.py │ ├── HarborAPI.py │ ├── JenkinsAPI.py │ ├── JiraAPI.py │ ├── K8sAPI.py │ ├── RedisAPI.py │ └── __init__.py └── variables.py ├── config.py.sample ├── daemon.conf ├── dbapp ├── __init__.py ├── admin.py ├── apps.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ ├── initdata.py │ │ └── qtasks.py ├── migrations │ └── __init__.py ├── model │ ├── __init__.py │ ├── model_assets.py │ ├── model_cmdb.py │ ├── model_dashboard.py │ ├── model_deploy.py │ ├── model_ucenter.py │ └── model_workflow.py └── models.py ├── dev.sh ├── devops.conf ├── devops_backend ├── __init__.py ├── asgi.py ├── documents.py ├── routing.py ├── settings.py ├── urls.py ├── websocket.py └── wsgi.py ├── dist.tar.gz ├── gunicorn.sh ├── manage.py ├── media └── playbook │ ├── docker_deploy.yaml │ ├── jar_deploy.yaml │ ├── jar_rollback.yaml │ └── web_deploy.yaml ├── preview ├── a-app-extra.gif ├── a-cicd.gif ├── a-cilog.gif ├── a-cmdb.gif ├── a-login-dashboard.gif └── a-wflow.gif ├── qtasks ├── __init__.py ├── hooks │ ├── __init__.py │ └── tasks_hook.py ├── tasks.py ├── tasks_build.py └── tasks_deploy.py ├── requirements.txt ├── start.sh └── task.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | venv_extra/ 87 | .venv 88 | env/ 89 | venv/ 90 | ENV/ 91 | env.bak/ 92 | venv.bak/ 93 | 94 | # Spyder project settings 95 | .spyderproject 96 | .spyproject 97 | 98 | # Rope project settings 99 | .ropeproject 100 | 101 | # mkdocs documentation 102 | /site 103 | 104 | # mypy 105 | .mypy_cache/ 106 | 107 | # backup 108 | media/salt/deploy/*.bak 109 | 110 | *.db 111 | settings_local.py 112 | media/upload/ 113 | media/download/ 114 | media/salt/config 115 | !media 116 | # migrations/ 117 | # !migrations/__init__.py 118 | .idea 119 | saltconfig 120 | cookie.txt 121 | config.py 122 | soms.db 123 | media/uploads/ 124 | media/playbook/files 125 | media/roster/ 126 | media/inventory/ 127 | temp 128 | logs/* 129 | *.out 130 | *.pid 131 | *.http 132 | config.py.env 133 | deployment/redis/docker/.env 134 | Pipfile 135 | venv 136 | apps/*/migrations/* 137 | dbapp/migrations/* 138 | !dbapp/migrations/__init__.py 139 | !apps/*/migrations/__init__.py 140 | .save 141 | static/ 142 | soar 143 | a.py 144 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // 使用 IntelliSense 了解相关属性。 3 | // 悬停以查看现有属性的描述。 4 | // 欲了解更多信息，请访问: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "devops_backend", 9 | "type": "python", 10 | "request": "launch", 11 | "program": "${workspaceFolder}/venv/bin/gunicorn", 12 | "args": [ 13 | "devops_backend.wsgi", 14 | "-b", 15 | "0.0.0.0:9000", 16 | "--thread", 17 | "4", 18 | "--reload" 19 | ], 20 | "django": true, 21 | "justMyCode": true 22 | } 23 | ] 24 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.analysis.extraPaths": [ 3 | "./apps" 4 | ], 5 | "editor.tabSize": 4, 6 | "python.envFile": "${workspaceFolder}/venv", 7 | "editor.formatOnPaste": true, 8 | "editor.formatOnSave": true 9 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9.14-slim-buster 2 | LABEL maintainer="Charles Lai" 3 | 4 | ARG APP 5 | RUN mkdir -p /app/${APP} /data/nfs/web; apt update && apt -y install nginx libmariadb-dev-compat libmariadb-dev python3-dev python-dev libldap2-dev libsasl2-dev libssl-dev build-essential libffi-dev && apt -y autoclean 6 | COPY requirements.txt /data/requirements.txt 7 | COPY devops.conf /etc/nginx/sites-enabled/default 8 | 9 | RUN pip install -r /data/requirements.txt -i https://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com 10 | COPY . /app/${APP} 11 | ADD dist.tar.gz /app/${APP}/ 12 | WORKDIR /app/${APP} 13 | RUN chmod +x *.sh 14 | 15 | ENTRYPOINT ["supervisord", "-n", "-c", "daemon.conf"] 16 | 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## 演示环境 2 | 运维平台演示 https://demo.imaojia.com/ 3 | 4 | ## 功能预览 5 | 登录页及Dashboard 6 | ![登录及Dashborad](https://github.com/qitan/devops-backend-lite/blob/main/preview/a-login-dashboard.gif) 7 | CMDB产品应用管理、流水线及kubernetes模板编排 8 | ![cmdb模块](https://github.com/qitan/devops-backend-lite/blob/main/preview/a-cmdb.gif) 9 | CICD持续构建部署 10 | ![cicd模块](https://github.com/qitan/devops-backend-lite/blob/main/preview/a-cicd.gif) 11 | ![cicd实时日志](https://github.com/qitan/devops-backend-lite/blob/main/preview/a-cilog.gif) 12 | 应用重启、构建发布日志、容器配置及上下线 13 | ![应用操作](https://github.com/qitan/devops-backend-lite/blob/main/preview/a-app-extra.gif) 14 | 工单流程管理 15 | ![工单管理](https://github.com/qitan/devops-backend-lite/blob/main/preview/a-wflow.gif) 16 | 17 | ## Docker部署(功能体验) 18 | 19 | 镜像打包 20 | ``` 21 | docker build -t docker.imaojia.com/allinone/devops-backend:ce1.0 --build-arg 'APP=devops-backend' -f Dockerfile . 22 | docker run -it --name t1 -p 10000:80 -v `pwd`/config.py:/app/devops-backend/config.py -d docker.imaojia.com/allinone/devops-backend:ce1.0 23 | ``` 24 | 25 | 初始数据 26 | ``` 27 | python manage.py makemigrations 28 | python manage.py migrate 29 | python manage.py initdata --type all 30 | ``` 31 | 32 | 创建管理员 33 | ``` 34 | python manage.py createsuperuser 35 | ``` 36 | 37 | ## 环境依赖 38 | 39 | * Python 3.9 40 | * MySQL 8.0.25 41 | * ElasticSearch 7.14.0 42 | * Harbor v1.7 43 | 44 | ## Jenkins 45 | 46 | ### plugins: 47 | 48 | * http request 49 | * docker 50 | * Docker Pipeline 51 | 52 | ### Jenkins所在机器需要安装如下软件 53 | 54 | python3 55 | 56 | > 依赖 57 | > pycryptodome==3.9.8 58 | > xmltodict==0.12.0 59 | > requests==2.25.0 60 | > ansible==2.10.4 61 | 62 | ## 开发环境 63 | 64 | 部署MySQL 65 | 66 | ```shell script 67 | docker run -it --name mysqldb -p 43306:3306 -e MYSQL_ROOT_PASSWORD=password -e MYSQL_DATABASE=ydevopsdb -e MYSQL_USER=devops -e MYSQL_PASSWORD=ops123456 -d mysql:8.0.18 --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci 68 | ``` 69 | 70 | 部署redis 71 | 72 | ```shell script 73 | docker run -d --name redis -p 6379:6379 daocloud.io/redis --requirepass 'ops123456' 74 | ``` 75 | 76 | 部署ElasticSearch 77 | 78 | ```python 79 | docker run --name es -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e ES_JAVA_OPTS="-Xms512m -Xmx512m" -d elasticsearch:7.14.0 80 | ``` 81 | 82 | 部署GitLab 83 | 84 | ```shell script 85 | docker run -d --name gitlab -p 8090:8090 -p 2222:2222 gitlab/gitlab-ce 86 | ``` 87 | 88 | ## 依赖安装 89 | 90 | mysqlclient: 91 | 92 | * debian系 93 | 94 | ```shell script 95 | sudo apt install mysql-client-8.0 libmysqlclient-dev python3-dev python-dev libldap2-dev libsasl2-dev libssl-dev 96 | ``` 97 | 98 | * redhat系 99 | 100 | ```shell script 101 | -- 102 | ``` 103 | 104 | openldap: 105 | 106 | * debian系 107 | 108 | ```shell script 109 | sudo apt-get install libsasl2-dev python-dev libldap2-dev libssl-dev 110 | ``` 111 | 112 | * redhat系: 113 | 114 | ```shell script 115 | yum install python-devel openldap-devel 116 | ``` 117 | 118 | ## ansible依赖 119 | 120 | ```shell script 121 | yum -y install sshpass 122 | ``` 123 | 124 | ## RBAC 125 | 126 | ### 获取权限 127 | 128 | ```python 129 | from rest_framework.schemas.openapi import SchemaGenerator 130 | generator = SchemaGenerator(title='DevOps API') 131 | data = [] 132 | try: 133 | generator.get_schema() 134 | except BaseException as e: 135 | print(str(e)) 136 | finally: 137 | data = generator.endpoints 138 | ``` 139 | 140 | ### 初始化配置 141 | 142 | ```python 143 | python manage.py migrate 144 | # 注释好之后再执行初始化数据 145 | python manage.py initdata --help 146 | ``` 147 | 148 | ## Nginx配置 149 | 150 | ``` 151 | server { 152 | listen 9000; 153 | server_name localhost; 154 | error_log /usr/local/nginx/logs/devops_error.log; 155 | #access_log off; 156 | access_log /usr/local/nginx/logs/devops_access.log; 157 | error_page 404 /404.html; 158 | location = /404.html { 159 | root /etc/nginx; 160 | } 161 | error_page 500 502 503 504 /500.html; 162 | location = /500.html { 163 | root /etc/nginx; 164 | } 165 | underscores_in_headers on; 166 | client_max_body_size 2048m; 167 | 168 | location / { 169 | root /data/frontend/dist; 170 | index index.html index.htm; 171 | try_files $uri $uri/ /index.html; 172 | } 173 | location ~ ^/(admin|api) { 174 | proxy_pass http://localhost:8000; 175 | proxy_connect_timeout 1200s; 176 | proxy_read_timeout 1200s; 177 | proxy_send_timeout 1200s; 178 | proxy_set_header Host $host:$server_port; 179 | proxy_set_header X-Real-IP $remote_addr; 180 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 181 | } 182 | } 183 | ``` 184 | 185 | ## 表关联及报表展示模型配置 186 | 187 | 模型定义了扩展元数据 188 | 189 | * 表是否可关联由related控制 190 | * 报表是否可展示由dashboard控制 191 | 192 | ```python 193 | class ExtMeta: 194 | related = False 195 | dashboard = False 196 | ``` 197 | -------------------------------------------------------------------------------- /apps/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | """ 4 | @author: Charles Lai 5 | @file: __init__.py.py 6 | @time: 2022/9/8 11:47 7 | @contact: qqing_lai@hotmail.com 8 | @company: IMAOJIA Co,Ltd 9 | """ 10 | -------------------------------------------------------------------------------- /apps/cmdb/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/apps/cmdb/__init__.py -------------------------------------------------------------------------------- /apps/cmdb/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from dbapp.models import Idc, Environment 3 | 4 | # Register your models here. 5 | admin.site.register([Idc, Environment]) 6 | -------------------------------------------------------------------------------- /apps/cmdb/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class CmdbConfig(AppConfig): 5 | name = 'cmdb' 6 | -------------------------------------------------------------------------------- /apps/cmdb/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/apps/cmdb/migrations/__init__.py -------------------------------------------------------------------------------- /apps/cmdb/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/apps/cmdb/model/__init__.py -------------------------------------------------------------------------------- /apps/cmdb/serializer/__init__.py: -------------------------------------------------------------------------------- 1 | from .serializer_assets import * 2 | from .serializer_cmdb import * 3 | -------------------------------------------------------------------------------- /apps/cmdb/serializer/serializer_assets.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2021/05/13 下午3:44 7 | @FileName: serializer_assets 8 | @Blog : https://imaojia.com 9 | """ 10 | from dbapp.models import * 11 | 12 | from common.extends.serializers import ModelSerializer 13 | 14 | 15 | class IdcSerializers(ModelSerializer): 16 | class Meta: 17 | model = Idc 18 | fields = '__all__' 19 | 20 | 21 | class IdcListSerializers(ModelSerializer): 22 | class Meta: 23 | model = Idc 24 | fields = ('name', 'alias', 'type', 'supplier', 'desc') 25 | -------------------------------------------------------------------------------- /apps/cmdb/serializers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/9/15 下午7:46 7 | @FileName: serializers.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | 11 | import re 12 | from django.apps import apps 13 | from django.db.models import Q 14 | from rest_framework.relations import Hyperlink, PKOnlyObject 15 | from rest_framework.fields import ( # NOQA # isort:skip 16 | CreateOnlyDefault, CurrentUserDefault, SkipField, empty 17 | ) 18 | 19 | from elasticsearch_dsl import Document 20 | from elasticsearch_dsl.response import Hit 21 | 22 | from cmdb.serializer import * 23 | 24 | import pytz 25 | import datetime 26 | import logging 27 | 28 | logger = logging.getLogger('cmdb_es') 29 | 30 | 31 | class FileUploadSerializers(serializers.ModelSerializer): 32 | class Meta: 33 | model = FileUpload 34 | fields = '__all__' 35 | read_only_fields = ('uploader',) 36 | -------------------------------------------------------------------------------- /apps/cmdb/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /apps/cmdb/urls.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2021/05/13 下午1:53 7 | @FileName: urls 8 | @Blog : https://imaojia.com 9 | """ 10 | 11 | from django.conf.urls import url, include 12 | from django.urls import path 13 | from rest_framework import viewsets 14 | from rest_framework.routers import DefaultRouter 15 | from rest_framework_nested import routers 16 | from cmdb.view.view_cmdb import ProjectConfigViewSet 17 | 18 | from cmdb.views import IdcViewSet, \ 19 | ProductViewSet, \ 20 | EnvironmentViewSet, KubernetesClusterViewSet, \ 21 | ProjectViewSet, MicroAppViewSet, AppInfoViewSet, DevLanguageViewSet, \ 22 | RegionViewSet 23 | 24 | router = DefaultRouter() 25 | 26 | router.register('product', ProductViewSet) 27 | router.register('region', RegionViewSet) 28 | router.register('environment', EnvironmentViewSet) 29 | router.register('asset/idc', IdcViewSet) 30 | router.register('app/language', DevLanguageViewSet) 31 | router.register('app/service', AppInfoViewSet) 32 | router.register('app', MicroAppViewSet) 33 | router.register('project/config', ProjectConfigViewSet) 34 | router.register('project', ProjectViewSet) 35 | router.register('kubernetes', KubernetesClusterViewSet) 36 | 37 | router.register('cmdb', viewsets.ViewSet, basename='cmdb') 38 | cmdb_router = routers.NestedDefaultRouter(router, r'cmdb', lookup='table') 39 | 40 | urlpatterns = [ 41 | path(r'', include(router.urls)), 42 | path(r'', include(cmdb_router.urls)), 43 | ] 44 | -------------------------------------------------------------------------------- /apps/cmdb/view/__init__.py: -------------------------------------------------------------------------------- 1 | from .view_assets import * 2 | from .view_cmdb import * 3 | -------------------------------------------------------------------------------- /apps/cmdb/view/view_assets.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2021/05/13 下午4:03 7 | @FileName: view_assets 8 | @Blog : https://imaojia.com 9 | """ 10 | 11 | from rest_framework.decorators import action 12 | from rest_framework.response import Response 13 | from rest_framework import status 14 | from rest_framework import pagination 15 | from rest_framework.filters import SearchFilter, OrderingFilter 16 | 17 | import django_filters 18 | 19 | from dbapp.models import * 20 | from dbapp.model.model_ucenter import Menu, SystemConfig 21 | from cmdb.serializers import IdcSerializers 22 | 23 | from common.extends.viewsets import CustomModelViewSet 24 | from common.extends.filters import CustomSearchFilter 25 | 26 | 27 | class IdcViewSet(CustomModelViewSet): 28 | """ 29 | IT资产 - IDC视图 30 | 31 | ### IDC权限 32 | {'*': ('itasset_all', 'IT资产管理')}, 33 | {'get': ('itasset_list', '查看IT资产')}, 34 | {'post': ('itasset_create', '创建IT资产')}, 35 | {'put': ('itasset_edit', '编辑IT资产')}, 36 | {'delete': ('itasset_delete', '删除IT资产')} 37 | """ 38 | perms_map = ( 39 | {'*': ('admin', '管理员')}, 40 | {'*': ('itasset_all', 'IT资产管理')}, 41 | {'get': ('itasset_list', '查看IT资产')}, 42 | {'post': ('itasset_create', '创建IT资产')}, 43 | {'put': ('itasset_edit', '编辑IT资产')}, 44 | {'delete': ('itasset_delete', '删除IT资产')} 45 | ) 46 | queryset = Idc.objects.all() 47 | serializer_class = IdcSerializers 48 | filter_backends = (django_filters.rest_framework.DjangoFilterBackend, 49 | CustomSearchFilter, OrderingFilter) 50 | filter_fields = ('name', 'alias', 'forward', 'supplier') 51 | search_fields = ('name', 'alias', 'ops', 'desc') 52 | 53 | @action(methods=['GET'], url_path='repo', detail=False) 54 | def get_harbor_repo(self, request): 55 | harbors = SystemConfig.objects.filter(type='cicd-harbor', status=True) 56 | return Response({'code': 20000, 'data': [{'id': i.id, 'name': i.name} for i in harbors]}) 57 | -------------------------------------------------------------------------------- /apps/cmdb/views.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/9/15 下午7:48 7 | @FileName: views.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | 11 | from django.apps import apps 12 | from cmdb.view import * 13 | from cmdb.serializers import * 14 | from common.extends.filters import CustomFilter 15 | from common.extends.viewsets import CustomModelViewSet 16 | from common.md5 import md5 17 | from config import MEDIA_ROOT 18 | import logging 19 | from drf_yasg.utils import swagger_auto_schema 20 | 21 | logger = logging.getLogger('drf') 22 | 23 | 24 | class FileUploadViewSet(CustomModelViewSet): 25 | perms_map = () 26 | queryset = FileUpload.objects.all() 27 | serializer_class = FileUploadSerializers 28 | 29 | def create(self, request, *args, **kwargs): 30 | file_obj = request.data.get('name') 31 | _file_md5 = md5(file_obj) 32 | asset_type = request.data.get('type') 33 | platform = request.data.get('platform') 34 | try: 35 | FileUpload.objects.get(md5=_file_md5) 36 | return Response({'code': 20000, 'status': 'failed', 'message': '文件已存在'}) 37 | except BaseException as e: 38 | pass 39 | request.data['md5'] = _file_md5 40 | serializer = self.get_serializer(data=request.data) 41 | if not serializer.is_valid(): 42 | return Response({'code': 40000, 'status': 'failed', 'message': serializer.errors}) 43 | try: 44 | self.perform_create(serializer) 45 | except BaseException as e: 46 | return Response({'code': 50000, 'status': 'failed', 'message': str(e)}) 47 | data = serializer.data 48 | data['status'] = 'success' 49 | data['code'] = 20000 50 | return Response(data) 51 | -------------------------------------------------------------------------------- /apps/dashboard/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/apps/dashboard/__init__.py -------------------------------------------------------------------------------- /apps/dashboard/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /apps/dashboard/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class DashboardConfig(AppConfig): 5 | default_auto_field = 'django.db.models.BigAutoField' 6 | name = 'dashboard' 7 | -------------------------------------------------------------------------------- /apps/dashboard/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/apps/dashboard/migrations/__init__.py -------------------------------------------------------------------------------- /apps/dashboard/serializers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : serializers.py 6 | @time : 2023/04/18 19:25 7 | @contact : qqing_lai@hotmail.com 8 | ''' 9 | 10 | # here put the import lib 11 | from dbapp.model.model_dashboard import DashBoard 12 | 13 | from common.extends.serializers import ModelSerializer 14 | 15 | 16 | class DashBoardSerializers(ModelSerializer): 17 | 18 | class Meta: 19 | model = DashBoard 20 | fields = '__all__' 21 | read_only_fields = ('creator', ) 22 | -------------------------------------------------------------------------------- /apps/dashboard/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /apps/dashboard/urls.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : urls.py 6 | @time : 2023/04/18 19:26 7 | @contact : qqing_lai@hotmail.com 8 | ''' 9 | 10 | # here put the import lib 11 | from django.conf.urls import url, include 12 | from django.urls import path 13 | from rest_framework.routers import DefaultRouter 14 | 15 | from dashboard.views import DashBoardViewSet 16 | 17 | router = DefaultRouter() 18 | 19 | router.register('dashboard', DashBoardViewSet, basename='dashboard') 20 | 21 | urlpatterns = [ 22 | path(r'', include(router.urls)), 23 | ] 24 | -------------------------------------------------------------------------------- /apps/deploy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/apps/deploy/__init__.py -------------------------------------------------------------------------------- /apps/deploy/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /apps/deploy/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class DeployConfig(AppConfig): 5 | name = 'deploy' 6 | 7 | def ready(self): 8 | import deploy.signals 9 | -------------------------------------------------------------------------------- /apps/deploy/documents.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : documents.py 6 | @time : 2023/04/20 17:39 7 | @contact : qqing_lai@hotmail.com 8 | ''' 9 | 10 | # here put the import lib 11 | from common.variables import my_normalizer, CI_RESULT_INDEX, CD_RESULT_INDEX 12 | from dbapp.model.model_deploy import BuildJob, DeployJob 13 | 14 | from common.utils.ElasticSearchAPI import CustomDocument, Text, Date, Keyword, Integer, EsNested, Object 15 | from common.utils.DocumentRegistry import registry 16 | 17 | from config import CMDB_SOURCE_INDEX, ELASTICSEARCH_PREFIX 18 | 19 | import datetime 20 | 21 | from deploy.serializers import BuildJobEsListSerializer, \ 22 | DeployJobEsListSerializer 23 | 24 | 25 | class BuildJobResultDocument(CustomDocument): 26 | """ 27 | 构建结果 - 按月创建索引 28 | """ 29 | id = Integer() 30 | result = Text() 31 | console_output = Text() 32 | created_at = Date() 33 | status = Integer() 34 | 35 | class Index: 36 | name = ELASTICSEARCH_PREFIX + f'{CI_RESULT_INDEX}-*' 37 | aliases = {ELASTICSEARCH_PREFIX + CI_RESULT_INDEX: {}} 38 | 39 | def save(self, **kwargs): 40 | self.created_at = datetime.datetime.now() 41 | kwargs[ 42 | 'index'] = f"{ELASTICSEARCH_PREFIX}{CI_RESULT_INDEX}-{self.created_at.strftime('%Y%m')}" 43 | return super().save(**kwargs) 44 | 45 | 46 | class DeployJobResultDocument(CustomDocument): 47 | """ 48 | 发布结果 - 按月创建索引 49 | """ 50 | id = Integer() 51 | result = Text() 52 | created_at = Date() 53 | status = Integer() 54 | 55 | class Index: 56 | name = ELASTICSEARCH_PREFIX + f'{CD_RESULT_INDEX}-*' 57 | aliases = {ELASTICSEARCH_PREFIX + CD_RESULT_INDEX: {}} 58 | 59 | def save(self, **kwargs): 60 | self.created_at = datetime.datetime.now() 61 | kwargs[ 62 | 'index'] = f"{ELASTICSEARCH_PREFIX}{CD_RESULT_INDEX}-{self.created_at.strftime('%Y%m')}" 63 | return super().save(**kwargs) 64 | 65 | 66 | @registry.register_document 67 | class BuildJobDocument(CustomDocument): 68 | """ 69 | 持续构建索引文档 - 按年创建索引 70 | """ 71 | deployer_info = Object() # 构建人信息 72 | appinfo_obj_info = Object() # 应用信息 73 | project_info = Object() 74 | region_info = Object() 75 | environment_info = Object() 76 | 77 | class Index: 78 | name = ELASTICSEARCH_PREFIX + 'buildjob-*' 79 | aliases = {ELASTICSEARCH_PREFIX + 'buildjob': {}} 80 | 81 | class Django: 82 | model = BuildJob 83 | serializer = BuildJobEsListSerializer 84 | fields = '__all__' 85 | 86 | def save(self, **kwargs): 87 | kwargs['index'] = f"{ELASTICSEARCH_PREFIX}buildjob-{datetime.datetime.now().strftime('%Y')}" 88 | return super().save(**kwargs) 89 | 90 | 91 | @registry.register_document 92 | class DeployJobDocument(CustomDocument): 93 | """ 94 | 持续部署索引文档 - 按年创建索引 95 | """ 96 | deployer_info = Object( 97 | ) 98 | appinfo_obj_info = Object() # 应用信息 99 | project_info = Object() 100 | region_info = Object() 101 | environment_info = Object() 102 | 103 | class Index: 104 | name = ELASTICSEARCH_PREFIX + "deployjob-*" 105 | aliases = {ELASTICSEARCH_PREFIX + 'deployjob': {}} 106 | 107 | class Django: 108 | model = DeployJob 109 | serializer = DeployJobEsListSerializer 110 | fields_remap = {'kubernetes': Text(), 'result': Text()} # 定义需要重新映射的字段 111 | fields = '__all__' 112 | 113 | def save(self, **kwargs): 114 | kwargs['index'] = f"{ELASTICSEARCH_PREFIX}deployjob-{datetime.datetime.now().strftime('%Y')}" 115 | return super().save(**kwargs) 116 | -------------------------------------------------------------------------------- /apps/deploy/documents_order.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : documents_order.py 6 | @time : 2023/04/21 15:06 7 | @contact : qqing_lai@hotmail.com 8 | ''' 9 | 10 | # here put the import lib 11 | from dbapp.model.model_deploy import PublishApp 12 | 13 | from common.utils.ElasticSearchAPI import CustomDocument, Text, Date, Keyword, Integer, EsNested, Object 14 | from common.utils.DocumentRegistry import registry 15 | 16 | from config import CMDB_SOURCE_INDEX, ELASTICSEARCH_PREFIX 17 | 18 | from deploy.serializers_order import PublishAppEsListSerializer 19 | 20 | import datetime 21 | 22 | 23 | @registry.register_document 24 | class PublishAppDocument(CustomDocument): 25 | """ 26 | 工单应用索引文档 - 按年创建索引 27 | """ 28 | project_info = Object() 29 | product_info = Object() 30 | region_info = Object() 31 | 32 | class Index: 33 | name = ELASTICSEARCH_PREFIX + 'publishapp-*' 34 | aliases = {ELASTICSEARCH_PREFIX + 'publishapp': {}} 35 | 36 | class Django: 37 | model = PublishApp 38 | serializer = PublishAppEsListSerializer 39 | fields = '__all__' 40 | 41 | def save(self, **kwargs): 42 | kwargs['index'] = f"{ELASTICSEARCH_PREFIX}publishapp-{datetime.datetime.now().strftime('%Y')}" 43 | return super().save(**kwargs) 44 | -------------------------------------------------------------------------------- /apps/deploy/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/apps/deploy/migrations/__init__.py -------------------------------------------------------------------------------- /apps/deploy/rds_transfer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : rds_transfer.py 6 | @time : 2023/05/10 16:29 7 | @contact : qqing_lai@hotmail.com 8 | ''' 9 | 10 | # here put the import lib 11 | def rds_transfer_es(document, instance): 12 | document_serializer = getattr(document.Django, "serializer", None) 13 | if document_serializer: 14 | serializer = document_serializer(instance) 15 | data = serializer.data 16 | else: 17 | data = instance.__dict__ 18 | model_field_exclude = getattr(document.Django, "exclude", []) 19 | model_field_names = getattr(document.Django, "fields", []) 20 | if model_field_names == '__all__': 21 | data.pop('_state', None) 22 | else: 23 | _data = {} 24 | for field in model_field_names: 25 | _data[field] = getattr(instance, field, None) 26 | data = _data 27 | for i in model_field_exclude: 28 | data.pop(i, None) 29 | data['_id'] = data['id'] 30 | docu = document(**data) 31 | docu.save(skip_empty=False) 32 | -------------------------------------------------------------------------------- /apps/deploy/routing.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/5/13 下午5:08 7 | @FileName: routing.py 8 | @Blog : https://blog.imaojia.com 9 | """ 10 | 11 | from django.urls import re_path 12 | 13 | from deploy.consumers import BuildJobStageOutput, BuildJobConsoleOutput, WatchK8s, WatchK8sLog, \ 14 | WatchK8sDeployment 15 | 16 | websocket_urlpatterns = [ 17 | re_path('ws/build/(?P[0-9]+)/(?P[0-9]+)/(?P[^/]+)/stage/$', 18 | BuildJobStageOutput.as_asgi()), 19 | re_path('ws/build/(?P[0-9]+)/(?P[0-9]+)/(?P[^/]+)/console/$', 20 | BuildJobConsoleOutput.as_asgi()), 21 | re_path( 22 | 'ws/kubernetes/(?P[0-9]+)/(?P[^/]+)/(?P[^/]+)/watch/$', WatchK8s.as_asgi()), 23 | re_path( 24 | 'ws/kubernetes/(?P[0-9]+)/(?P[^/]+)/(?P[^/]+)/log/$', WatchK8sLog.as_asgi()), 25 | re_path( 26 | 'ws/kubernetes/(?P[0-9]+)/(?P[^/]+)/deployment/$', WatchK8sDeployment.as_asgi()), 27 | ] 28 | -------------------------------------------------------------------------------- /apps/deploy/serializers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/9/29 下午3:03 7 | @FileName: serializers.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | import json 11 | 12 | from dbapp.model.model_cmdb import Environment, MicroApp 13 | from common.ext_fun import get_datadict 14 | from django.db import transaction 15 | from rest_framework import serializers 16 | from dbapp.models import AppInfo 17 | from dbapp.model.model_deploy import BuildJob, PublishOrder, PublishApp, DockerImage, DeployJob, \ 18 | DeployJobResult, BuildJobResult 19 | 20 | from common.extends.serializers import ModelSerializer, EsSerializer 21 | 22 | import datetime 23 | import shortuuid 24 | 25 | 26 | class DockerImageSerializer(ModelSerializer): 27 | deployer_info = serializers.SerializerMethodField() 28 | 29 | def get_deployer_info(self, instance): 30 | return {'id': instance.deployer.id, 'username': instance.deployer.username, 31 | 'first_name': instance.deployer.first_name} 32 | 33 | class Meta: 34 | model = DockerImage 35 | fields = '__all__' 36 | 37 | 38 | class DeployJobListSerializer(ModelSerializer): 39 | deployer_info = serializers.SerializerMethodField() 40 | 41 | def get_deployer_info(self, instance): 42 | if instance.deployer: 43 | return {'id': instance.deployer.id, 'name': instance.deployer.name, 'position': instance.deployer.position} 44 | return {} 45 | 46 | class Meta: 47 | model = DeployJob 48 | fields = '__all__' 49 | 50 | 51 | class ListForCicdSerializer(ModelSerializer): 52 | appinfo_obj_info = serializers.SerializerMethodField() 53 | environment = serializers.SerializerMethodField() 54 | environment_info = serializers.SerializerMethodField() 55 | project_info = serializers.SerializerMethodField() 56 | product_info = serializers.SerializerMethodField() 57 | deployer_info = serializers.SerializerMethodField() 58 | 59 | @staticmethod 60 | def instance_app(appinfo_id): 61 | return AppInfo.objects.get(id=appinfo_id) 62 | 63 | def get_deployer_info(self, instance): 64 | if instance.deployer: 65 | return {'id': instance.deployer.id, 'name': instance.deployer.name, 'position': instance.deployer.position} 66 | else: 67 | return {} 68 | 69 | def get_appinfo_obj_info(self, instance): 70 | try: 71 | appinfo_obj = self.instance_app(instance.appinfo_id) 72 | return {'appid': appinfo_obj.app.appid, 'name': appinfo_obj.app.name, 'alias': appinfo_obj.app.alias, 73 | 'category': get_datadict(appinfo_obj.app.category), 74 | 'environment': {'id': appinfo_obj.environment.id, 'name': appinfo_obj.environment.name, 75 | 'alias': appinfo_obj.environment.alias}} 76 | except: 77 | return {} 78 | 79 | def get_environment(self, instance): 80 | try: 81 | appinfo_obj = self.instance_app(instance.appinfo_id) 82 | return appinfo_obj.environment.id 83 | except: 84 | return None 85 | 86 | def get_environment_info(self, instance): 87 | try: 88 | appinfo_obj = self.instance_app(instance.appinfo_id) 89 | return {'id': appinfo_obj.environment.id, 'name': appinfo_obj.environment.name, 90 | 'alias': appinfo_obj.environment.alias} 91 | except: 92 | return {} 93 | 94 | def get_project_info(self, instance): 95 | try: 96 | appinfo_obj = self.instance_app(instance.appinfo_id) 97 | return {'id': appinfo_obj.app.project.id, 'name': appinfo_obj.app.project.name, 98 | 'alias': appinfo_obj.app.project.alias} 99 | except: 100 | return {} 101 | 102 | def get_product_info(self, instance): 103 | try: 104 | appinfo_obj = self.instance_app(instance.appinfo_id) 105 | return {'id': appinfo_obj.app.project.product.id, 'name': appinfo_obj.app.project.product.name, 106 | 'alias': appinfo_obj.app.project.product.alias} 107 | except: 108 | return {} 109 | 110 | 111 | class DeployJobInfoSerializer(ListForCicdSerializer): 112 | """ 113 | ElasticSearch索引文档序列化 114 | """ 115 | 116 | class Meta: 117 | model = DeployJob 118 | fields = '__all__' 119 | 120 | 121 | class DeployJobSerializer(ModelSerializer): 122 | class Meta: 123 | model = DeployJob 124 | fields = '__all__' 125 | read_only_fields = ('uniq_id', 'status') 126 | 127 | 128 | class BuildJobListSerializer(ModelSerializer): 129 | deployer_info = serializers.SerializerMethodField() 130 | 131 | def get_deployer_info(self, instance): 132 | if instance.deployer: 133 | return {'id': instance.deployer.id, 'first_name': instance.deployer.first_name, 'username': instance.deployer.username, 'name': instance.deployer.name, 'position': instance.deployer.position} 134 | return {} 135 | 136 | class Meta: 137 | model = BuildJob 138 | fields = '__all__' 139 | 140 | 141 | class BuildJobEsListSerializer(EsSerializer, ListForCicdSerializer): 142 | """ 143 | ElasticSearch索引文档序列化 144 | """ 145 | 146 | class Meta: 147 | model = BuildJob 148 | fields = '__all__' 149 | 150 | 151 | class DeployJobListForRollbackSerializer(BuildJobListSerializer): 152 | commits = serializers.SerializerMethodField('get_commits') 153 | commit_tag = serializers.SerializerMethodField() 154 | build_number = serializers.IntegerField() 155 | 156 | def get_commits(self, instance): 157 | try: 158 | return json.loads(instance.commits) 159 | except BaseException as e: 160 | return {} 161 | 162 | def get_commit_tag(self, instance): 163 | try: 164 | return json.loads(instance.commit_tag) 165 | except BaseException as e: 166 | return {} 167 | 168 | class Meta: 169 | model = DeployJob 170 | fields = '__all__' 171 | 172 | 173 | class DeployJobEsListSerializer(EsSerializer, ListForCicdSerializer): 174 | """ 175 | ElasticSearch索引文档序列化 176 | """ 177 | 178 | class Meta: 179 | model = DeployJob 180 | fields = '__all__' 181 | 182 | 183 | class BuildJobListForCiSerializer(BuildJobListSerializer): 184 | pass 185 | 186 | 187 | class BuildJobSerializer(ModelSerializer): 188 | 189 | class Meta: 190 | model = BuildJob 191 | fields = '__all__' 192 | 193 | 194 | class ResultSerializer(ModelSerializer): 195 | result = serializers.SerializerMethodField() 196 | 197 | def get_result(self, instance): 198 | try: 199 | result = json.loads(instance.result) 200 | except BaseException as e: 201 | result = {} 202 | return result 203 | 204 | 205 | class DeployJobResultSerializer(ResultSerializer): 206 | class Meta: 207 | model = DeployJobResult 208 | fields = '__all__' 209 | 210 | 211 | class BuildJobResultSerializer(ResultSerializer): 212 | class Meta: 213 | model = BuildJobResult 214 | fields = '__all__' 215 | -------------------------------------------------------------------------------- /apps/deploy/signals.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : signals.py 6 | @time : 2023/04/20 17:49 7 | @contact : qqing_lai@hotmail.com 8 | ''' 9 | 10 | # here put the import lib 11 | from django.db.models.signals import pre_save, post_save, post_delete 12 | from django.dispatch import receiver 13 | 14 | from dbapp.model.model_deploy import BuildJob, DeployJob, PublishApp 15 | from deploy.rds_transfer import rds_transfer_es 16 | from devops_backend import documents 17 | 18 | 19 | import logging 20 | 21 | logger = logging.getLogger('elasticsearch') 22 | 23 | 24 | @receiver(post_save, sender=PublishApp, dispatch_uid='publishapp_record') 25 | @receiver(post_save, sender=DeployJob, dispatch_uid='deployjob_record') 26 | @receiver(post_save, sender=BuildJob, dispatch_uid='buildjob_record') 27 | def save_es_record(sender, instance, created, **kwargs): 28 | if created is False or sender._meta.object_name == 'PublishApp': 29 | document = getattr(documents, f"{sender._meta.object_name}Document") 30 | try: 31 | rds_transfer_es(document, instance) 32 | except BaseException as e: 33 | logger.error(f'模型转存ES失败，原因：{e}') 34 | -------------------------------------------------------------------------------- /apps/deploy/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /apps/ucenter/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/apps/ucenter/__init__.py -------------------------------------------------------------------------------- /apps/ucenter/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | from dbapp.models import UserProfile 5 | 6 | 7 | class UserProfileAdmin(admin.ModelAdmin): 8 | def save_model(self, request, obj, form, change): 9 | obj.set_password(obj.password) 10 | super().save_model(request, obj, form, change) 11 | 12 | 13 | admin.site.register(UserProfile, UserProfileAdmin) 14 | 15 | admin.site.site_title = 'DevOps平台' 16 | admin.site.site_header = 'DevOps平台管理' 17 | -------------------------------------------------------------------------------- /apps/ucenter/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class UcenterConfig(AppConfig): 5 | name = 'ucenter' 6 | -------------------------------------------------------------------------------- /apps/ucenter/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/apps/ucenter/migrations/__init__.py -------------------------------------------------------------------------------- /apps/ucenter/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /apps/workflow/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/apps/workflow/__init__.py -------------------------------------------------------------------------------- /apps/workflow/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class WorkflowConfig(AppConfig): 5 | name = 'workflow' 6 | -------------------------------------------------------------------------------- /apps/workflow/callback_common.py: -------------------------------------------------------------------------------- 1 | from dbapp.models import UserProfile 2 | from dbapp.models import WorkflowTemplate, Workflow 3 | import requests 4 | 5 | from common.ext_fun import get_redis_data 6 | import logging 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | def callback_work( 12 | callback_type, method, url, template_model_cls=WorkflowTemplate, 13 | creator_id=None, wid=None, node_name=None, topic=None, 14 | template_id=None, cur_node_form=None, first_node_form=None, 15 | workflow_node_history_id=None, 16 | headers: dict = None, cookies: dict = None, action='normal', timeout=10 17 | ): 18 | url = url.strip() 19 | method = method.lower() 20 | if not headers: 21 | headers = {} 22 | if not cookies: 23 | cookies = {} 24 | params = { 25 | '__workflow_node_callback_type__': callback_type, 26 | '__workflow_node_history_id__': workflow_node_history_id 27 | } 28 | data = {'action': action} 29 | # api地址是来自本项目的情况和来自其他http服务的情况 30 | platform_conf = get_redis_data('platform') 31 | if '://' not in url: 32 | platform_base_url = platform_conf.get('url') 33 | if not platform_base_url: 34 | raise ValueError('获取不到平台访问地址， 请设置 【系统设置】-【基本设置】-【平台访问地址】') 35 | url = f'{platform_base_url.rstrip("/")}/{url.lstrip("/")}' # 处理url地址 36 | template_obj = template_model_cls.objects.get(id=template_id) 37 | if not creator_id: 38 | creator_id = Workflow.objects.get(wid=wid).creator.id 39 | data['workflow'] = { 40 | 'wid': wid, 41 | 'name': topic, 42 | 'template': template_obj.name, 43 | 'node': node_name, 44 | 'creator_id': creator_id, 45 | } 46 | data['cur_node_form'] = cur_node_form 47 | first_node_name = template_obj.nodes[0]['name'] 48 | first_node_conf = template_obj.get_node_conf(first_node_name) 49 | form = {} 50 | for field_conf in first_node_conf['form_models']: 51 | name = field_conf['field'] 52 | cname = field_conf['title'] 53 | value = first_node_form.get(name, '') 54 | if name in first_node_form: 55 | form[cname] = value 56 | data['first_node_form'] = form 57 | data['node_name'] = first_node_name 58 | headers['Content-Type'] = 'application/json' 59 | 60 | is_exp, result, original_result = callback_request(method, url, headers=headers, params=params, json=data, 61 | cookies=cookies, timeout=timeout) 62 | return { 63 | 'type': callback_type, 64 | 'url': url, 65 | 'response': { 66 | 'code': is_exp and 500 or original_result.status_code, 67 | 'data': result 68 | } 69 | } 70 | 71 | 72 | def callback_request(method, url, **kwargs): 73 | func = getattr(requests, method) 74 | if not func: 75 | msg = f'非法的 HTTP 方法名：{method}' 76 | return True, msg, None 77 | try: 78 | res = func(url, **kwargs) 79 | res_str = res.text.strip('"') 80 | logger.info(f'请求回调 {url} {kwargs.get("params")} 结果 {res_str}') 81 | return False, res_str, res 82 | except Exception as e: 83 | msg = f'请求回调 {url} 发生错误 {e.__class__} {e}' 84 | logger.exception(msg) 85 | return True, msg, e 86 | -------------------------------------------------------------------------------- /apps/workflow/ext_func.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : ext_func.py 6 | @time : 2023/05/22 15:14 7 | @contact : qqing_lai@hotmail.com 8 | ''' 9 | 10 | # here put the import lib 11 | import logging 12 | from dbapp.models import MicroApp 13 | from dbapp.models import UserProfile 14 | 15 | from dbapp.models import Workflow, WorkflowTemplate, WorkflowTemplateRevisionHistory 16 | from workflow.notice import get_member_user_ids 17 | from workflow.serializers import WorkflowNodeHistorySerializer, WorkflowSerializer 18 | 19 | logger = logging.getLogger('drf') 20 | 21 | 22 | def members_handle(data, field, leader: UserProfile, owner: UserProfile = None): 23 | data[field] = list(set(data[field])) 24 | return data 25 | 26 | 27 | def create_workflow(form, workflow_data, workflow_template, user): 28 | template_id = workflow_template['id'] 29 | template_obj = WorkflowTemplate.objects.get(pk=template_id) 30 | template_obj.pk = None 31 | template_obj.__class__ = WorkflowTemplateRevisionHistory 32 | 33 | # 处理发版工单 34 | leader = None 35 | for _, v in form.items(): 36 | if isinstance(v, (dict,)) and v.get('applist', None): 37 | try: 38 | # 申请应用 39 | filter = {'appid': v['applist'][0]} 40 | if isinstance(v['applist'][0], (dict, )): 41 | # 发布应用 42 | filter = {'id': v['applist'][0]['app']['id']} 43 | microapp_obj = MicroApp.objects.get(**filter) 44 | leader = UserProfile.objects.get( 45 | id=microapp_obj.project.manager) 46 | except BaseException as e: 47 | logger.info(f'处理项目信息异常==={e}') 48 | 49 | for (index, i) in enumerate(template_obj.nodes): 50 | if index > 0: 51 | # 合并前端传递的处理人员 52 | # 排除发起节点 53 | extra_member = workflow_template['nodes'][index]['members'] 54 | i['members'].extend(extra_member) 55 | i['members'] = list(set(i['members'])) 56 | if i['pass_type'] == 'passed': 57 | # 节点无需审批，添加发起人为处理人 58 | i['members'].append( 59 | f'user@{user.id}@{user.first_name}') 60 | template_obj.save() 61 | template_nodes = template_obj.nodes 62 | if len(template_nodes) == 0: 63 | return False, f'工单模板 {template_obj.name} 没有配置节点' 64 | 65 | first_template_node = template_nodes[0] 66 | members = first_template_node.get('members', []) 67 | if members: 68 | user_ids = get_member_user_ids(members) 69 | if not user.is_superuser and str(user.id) not in user_ids: 70 | return False, f'发起工单失败，当前工单只允许指定人员发起' 71 | _flag = workflow_data.pop('flag', 'normal') 72 | deploy_method = workflow_data.pop('deploy_method', None) 73 | workflow_data['extra'] = {'deploy_method': deploy_method} 74 | workflow_data['workflow_flag'] = _flag 75 | workflow_data['template'] = template_obj.pk 76 | workflow_data['node'] = first_template_node['name'] 77 | workflow_data['creator'] = user.id 78 | workflow_data['status'] = Workflow.STATUS.wait 79 | serializer = WorkflowSerializer(data=workflow_data) 80 | if not serializer.is_valid(): 81 | return False, serializer.errors 82 | workflow_obj = serializer.save() 83 | # 生成工单号 84 | workflow_obj.generate_wid(save=True) 85 | 86 | # 判断发起节点中， 有没有表单类型是 节点处理人的 87 | first_node_form_models = first_template_node.get('form_models', []) 88 | for field in first_node_form_models: 89 | if field['type'] != 'nodeHandler': 90 | continue 91 | # 如果是节点处理人类型的， 根据 配置的节点， 将选中的人员， 改到对应的节点绑定人员中 92 | type_ext_conf = field['type_ext_conf'] 93 | node_handler_id = type_ext_conf['node_handler_id'] 94 | for node in template_nodes: 95 | if node['id'] != node_handler_id: 96 | continue 97 | selected_user_list = form[field['name']] 98 | if isinstance(selected_user_list, str): 99 | selected_user_list = [selected_user_list] 100 | mapping_to_node_members_list = [] 101 | for u in selected_user_list: 102 | username, _ = u.split('@') 103 | selected_user_obj = UserProfile.objects.get( 104 | username=username) 105 | mapping_to_node_members_list.append( 106 | f'user@{selected_user_obj.id}@{selected_user_obj.first_name}') 107 | # 此处将完全覆盖原来的绑定人员配置 108 | node['members'] = mapping_to_node_members_list 109 | 110 | template_obj.save() 111 | 112 | node_form = { 113 | 'workflow': workflow_obj.pk, 114 | 'node': workflow_obj.node, 115 | 'form': form, 116 | 'operator': user.id 117 | } 118 | node_serializer = WorkflowNodeHistorySerializer(data=node_form) 119 | if not node_serializer.is_valid(): 120 | return False, node_serializer.errors 121 | node_obj = node_serializer.save() 122 | return True, {'data': serializer.data, 'workflow_obj': workflow_obj, 'node_obj': node_obj} 123 | -------------------------------------------------------------------------------- /apps/workflow/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/apps/workflow/migrations/__init__.py -------------------------------------------------------------------------------- /apps/workflow/serializers.py: -------------------------------------------------------------------------------- 1 | """ 2 | @Author : Ken Chen 3 | @Contact : 316084217@qq.com 4 | @Time : 2021/11/2 上午9:50 5 | """ 6 | 7 | from rest_framework import serializers 8 | from dbapp.models import Product, Project 9 | 10 | from common.recursive import RecursiveField 11 | from dbapp.models import UserProfile 12 | from dbapp.models import WorkflowCategory, Workflow, WorkflowNodeHistory, WorkflowTemplate, \ 13 | WorkflowTemplateRevisionHistory, WorkflowNodeHistoryCallback 14 | from common.extends.serializers import ModelSerializer 15 | from django.conf import settings 16 | import logging 17 | 18 | logger = logging.getLogger(__name__) 19 | 20 | 21 | class WorkflowTemplateSerializer(ModelSerializer): 22 | projects_info = serializers.SerializerMethodField() 23 | env_info = serializers.SerializerMethodField() 24 | 25 | def get_env_info(self, instance): 26 | if instance.environment: 27 | return {'name': instance.environment.name, 'alias': instance.environment.alias} 28 | return {} 29 | 30 | def get_projects_info(self, instance): 31 | data = [] 32 | product_ids = {} 33 | for i in instance.projects: 34 | if i[0] not in product_ids: 35 | product_ids[i[0]] = [] 36 | product_ids[i[0]].append(i[1]) 37 | for k, v in product_ids.items(): 38 | product = Product.objects.get(id=k) 39 | _projects = Project.objects.filter(id__in=v) 40 | data.append({'value': product.id, 'name': product.name, 'label': product.alias, 41 | 'children': [{'value': i.id, 'name': i.name, 'label': i.alias} for i in _projects]}) 42 | return data 43 | 44 | class Meta: 45 | model = WorkflowTemplate 46 | fields = '__all__' 47 | 48 | 49 | class WorkflowTemplateForRetrieveSerializer(ModelSerializer): 50 | 51 | class Meta: 52 | model = WorkflowTemplate 53 | fields = '__all__' 54 | 55 | 56 | class WorkflowRevisionTemplateSerializer(ModelSerializer): 57 | class Meta: 58 | model = WorkflowTemplateRevisionHistory 59 | fields = '__all__' 60 | 61 | 62 | class WorkflowCategorySerializer(ModelSerializer): 63 | workflows = serializers.SerializerMethodField() 64 | 65 | def get_workflows(self, instance): 66 | qs = instance.workflowtemplate_set.filter(enabled=True) 67 | return [{'id': i.id, 'name': i.name, 'comment': i.comment} for i in qs] 68 | 69 | class Meta: 70 | model = WorkflowCategory 71 | fields = ['id', 'name', 'desc', 'sort', 'workflows'] 72 | 73 | 74 | class WorkflowNodeHistorySerializer(ModelSerializer): 75 | class Meta: 76 | model = WorkflowNodeHistory 77 | fields = '__all__' 78 | read_only_fields = ('created_time',) 79 | 80 | 81 | class WorkflowNodeHistoryOperatorSerializers(ModelSerializer): 82 | class Meta: 83 | model = UserProfile 84 | fields = ['id', 'username', 'first_name'] 85 | 86 | 87 | class WorkflowNodeHistoryListSerializer(WorkflowNodeHistorySerializer): 88 | operator = WorkflowNodeHistoryOperatorSerializers() 89 | created_time = serializers.DateTimeField(format=settings.DATETIME_FORMAT) 90 | callback_status = serializers.SerializerMethodField() 91 | 92 | def get_callback_status(self, instance): 93 | callback_objs = WorkflowNodeHistoryCallback.objects.filter( 94 | node_history=instance) 95 | if not callback_objs: 96 | return None 97 | return callback_objs.first().get_status_display() 98 | 99 | 100 | class WorkflowListSerializer(ModelSerializer): 101 | template = serializers.SerializerMethodField() 102 | template_id = serializers.SerializerMethodField() 103 | creator = serializers.SerializerMethodField() 104 | created_time = serializers.DateTimeField(format=settings.DATETIME_FORMAT) 105 | update_time = serializers.DateTimeField(format=settings.DATETIME_FORMAT) 106 | current_node_handler = serializers.SerializerMethodField() 107 | category = serializers.SerializerMethodField() 108 | 109 | def get_template(self, instance): 110 | return instance.template.__str__() 111 | 112 | def get_template_id(self, instance): 113 | return instance.template.id 114 | 115 | def get_current_node_handler(self, instance): 116 | if instance.status == '已完成': 117 | wnode = WorkflowNodeHistory.objects.filter(workflow=instance.id, 118 | node=instance.node).first() 119 | if wnode: 120 | return wnode.operator.__str__() 121 | return '' 122 | 123 | node_conf = instance.template.get_node_conf(instance.node) 124 | members = node_conf.get('members') 125 | members_str = '' 126 | for member in members: 127 | name = member.split('@')[-1] 128 | members_str += f'{name} ' 129 | return members_str 130 | 131 | def get_creator(self, instance): 132 | return instance.creator.__str__() 133 | 134 | def get_category(self, instance): 135 | return instance.template.category.name 136 | 137 | class Meta: 138 | model = Workflow 139 | fields = '__all__' 140 | read_only_fields = ('created_time', 'update_time') 141 | 142 | 143 | class WorkflowRetrieveSerializer(WorkflowListSerializer): 144 | template = WorkflowTemplateSerializer() 145 | creator_department = serializers.SerializerMethodField() 146 | 147 | def get_creator_department(self, instance): 148 | dep = instance.creator.department.all().first() 149 | return dep and dep.__str__() 150 | 151 | 152 | class WorkflowSerializer(ModelSerializer): 153 | template = serializers.PrimaryKeyRelatedField( 154 | queryset=WorkflowTemplateRevisionHistory.objects) 155 | 156 | class Meta: 157 | model = Workflow 158 | fields = '__all__' 159 | read_only_fields = ('created_time', 'update_time') 160 | 161 | 162 | class WorkflowNodeHistoryCallbackSerializer(ModelSerializer): 163 | trigger = WorkflowNodeHistoryOperatorSerializers() 164 | 165 | class Meta: 166 | model = WorkflowNodeHistoryCallback 167 | fields = '__all__' 168 | -------------------------------------------------------------------------------- /apps/workflow/views/callback.py: -------------------------------------------------------------------------------- 1 | from django.db import transaction 2 | from rest_framework.decorators import action 3 | from rest_framework.response import Response 4 | from django.utils import timezone 5 | from django_q.tasks import async_task 6 | from common.extends.viewsets import CustomModelViewSet 7 | from workflow.callback_common import callback_work 8 | from dbapp.models import WorkflowNodeHistoryCallback 9 | from workflow.serializers import WorkflowNodeHistoryCallbackSerializer 10 | import logging 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | class WorkflowNodeHistoryCallbackViewSet(CustomModelViewSet): 16 | """ 17 | 工单历史回调信息 18 | 19 | ### 工单历史回调信息 20 | perms_map = ( 21 | {'*': ('admin', '管理员')}, 22 | {'*': ('workflow_callback_all', '工单回调管理')}, 23 | {'get': ('workflow_callback_get', '获取工单回调')}, 24 | {'post': ('workflow_callback_exec', '执行工单回调')}, 25 | ) 26 | """ 27 | perms_map = ( 28 | {'*': ('admin', '管理员')}, 29 | {'*': ('workflow_callback', '工单回调管理')}, 30 | {'get': ('workflow_callback_get', '获取工单回调')}, 31 | {'post': ('workflow_callback_exec', '执行工单回调')}, 32 | {'post_retry': ('workflow_callback_retry', '重新执行工单回调')}, 33 | ) 34 | queryset = WorkflowNodeHistoryCallback.objects.all() 35 | serializer_class = WorkflowNodeHistoryCallbackSerializer 36 | 37 | @transaction.atomic 38 | def create(self, request, *args, **kwargs): 39 | """ 40 | """ 41 | data = request.data 42 | action = data.get('action', 'normal') 43 | callback_type = data['callbackType'] 44 | callback_conf = data['callbackConf'] 45 | callback_url = callback_conf['url'] 46 | init_point = transaction.savepoint() 47 | cb_obj = WorkflowNodeHistoryCallback.objects.create( 48 | trigger=request.user, 49 | trigger_type=WorkflowNodeHistoryCallback.TriggerType.MANUAL, 50 | callback_type=callback_type, 51 | callback_url=callback_url, 52 | ) 53 | try: 54 | res = callback_work( 55 | callback_type, 'POST', callback_url, 56 | creator_id=request.user.id, 57 | wid='00000000000', topic='测试回调', node_name=data['node'], 58 | template_id=data['template'], 59 | cur_node_form={'type': callback_conf['type'], 'comment': '测试'}, 60 | first_node_form=data['form'], 61 | workflow_node_history_id=0, 62 | headers=dict(request.headers), 63 | cookies=dict(request.COOKIES), 64 | action=action 65 | ) 66 | cb_obj.response_code = res['response']['code'] 67 | cb_obj.response_result = res['response']['data'] 68 | if cb_obj.response_code != 200: 69 | if action == 'simulate': 70 | # 回调模拟 71 | transaction.savepoint_rollback(init_point) 72 | raise ValueError(cb_obj.response_result) 73 | 74 | response = Response({ 75 | 'code': 20000, 76 | 'status': 'success', 77 | 'data': res 78 | }) 79 | except Exception as e: 80 | logger.exception(e) 81 | error = f'回调函数发生异常： {e.__class__} {e}' 82 | cb_obj.response_result = error 83 | response = Response({ 84 | 'code': 40000, 85 | 'status': 'failed', 86 | 'message': error, 87 | }) 88 | # 增加回调结果判断 89 | if cb_obj.response_code == 200: 90 | cb_obj.status = WorkflowNodeHistoryCallback.Status.SUCCESS 91 | else: 92 | cb_obj.status = WorkflowNodeHistoryCallback.Status.ERROR 93 | cb_obj.response_time = timezone.now() 94 | cb_obj.save() 95 | if action == 'simulate': 96 | # 回调模拟 97 | transaction.savepoint_rollback(init_point) 98 | return response 99 | 100 | @action(methods=['POST'], url_path='retry', detail=True) 101 | def retry(self, request, *args, **kwargs): 102 | """ 103 | 重新触发回调 104 | """ 105 | workflow_node_history_callback_obj = WorkflowNodeHistoryCallback.objects.get( 106 | id=kwargs['pk']) 107 | headers = { 108 | 'Authorization': dict(self.request.headers).get('Authorization') 109 | } 110 | cookies = {} 111 | # 预先初始化好回调数据 112 | handle_type = workflow_node_history_callback_obj.callback_type 113 | node_history = workflow_node_history_callback_obj.node_history 114 | callback_url = workflow_node_history_callback_obj.callback_url 115 | data = { 116 | 'node_history': node_history, 117 | 'trigger': request.user, 118 | 'trigger_type': WorkflowNodeHistoryCallback.TriggerType.MANUAL, 119 | 'callback_type': handle_type, 120 | 'callback_url': callback_url, 121 | } 122 | new_cb_obj = WorkflowNodeHistoryCallback.objects.create(**data) 123 | taskid = async_task('qtasks.tasks.workflow_callback', handle_type, node_history.id, new_cb_obj.id, 'post', callback_url, headers=headers, 124 | cookies=cookies) 125 | # 重试之后， 把原来的回调状态设置为 已重试 126 | workflow_node_history_callback_obj.status = WorkflowNodeHistoryCallback.Status.RETRY 127 | workflow_node_history_callback_obj.save() 128 | return Response({ 129 | 'code': 20000, 130 | 'status': 'success', 131 | 'message': '重新执行完毕', 132 | }) 133 | 134 | def get_queryset(self): 135 | node_history_id = self.request.query_params.get( 136 | 'node_history_id', None) 137 | if node_history_id: 138 | return self.queryset.filter(node_history__id=node_history_id) 139 | return self.queryset 140 | -------------------------------------------------------------------------------- /apps/workflow/views/category.py: -------------------------------------------------------------------------------- 1 | import django_filters 2 | from rest_framework import pagination 3 | from rest_framework.decorators import action 4 | from rest_framework.response import Response 5 | 6 | from common.extends.filters import CustomSearchFilter 7 | from common.extends.permissions import RbacPermission 8 | from common.extends.viewsets import CustomModelViewSet 9 | from dbapp.models import WorkflowCategory 10 | from workflow.serializers import WorkflowCategorySerializer, WorkflowTemplateSerializer 11 | 12 | from rest_framework.filters import OrderingFilter 13 | import logging 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | class WorkflowCategoryViewSet(CustomModelViewSet): 19 | """ 20 | 工单分类视图 21 | ### 工单分类视图权限 22 | {'*': ('admin', '管理员')}, 23 | {'*': ('workflow_category_all', '工单分类管理')}, 24 | {'get': ('workflow_category_list', '查看工单分类')}, 25 | {'post': ('workflow_category_create', '创建工单分类')}, 26 | {'put': ('workflow_category_edit', '编辑工单分类')}, 27 | {'delete': ('workflow_category_delete', '删除工单分类')} 28 | """ 29 | perms_map = ( 30 | {'*': ('admin', '管理员')}, 31 | {'*': ('workflow_category_all', '工单分类管理')}, 32 | {'get': ('workflow_category_list', '查看工单分类')}, 33 | {'post': ('workflow_category_create', '创建工单分类')}, 34 | {'put': ('workflow_category_edit', '编辑工单分类')}, 35 | {'delete': ('workflow_category_delete', '删除工单分类')} 36 | ) 37 | queryset = WorkflowCategory.objects.all() 38 | serializer_class = WorkflowCategorySerializer 39 | filter_backends = (django_filters.rest_framework.DjangoFilterBackend, 40 | CustomSearchFilter, OrderingFilter) 41 | filter_fields = ('name',) 42 | search_fields = ('name',) 43 | 44 | @action(methods=['GET'], url_path='template', detail=True) 45 | def category_ticket_template(self, request, pk=None): 46 | page_size = request.query_params.get('page_size') 47 | pagination.PageNumberPagination.page_size = page_size 48 | qs = self.queryset.get(pk=pk) 49 | queryset = qs.workflowtemplate_set.all() 50 | logger.debug('queryset === %s', queryset) 51 | page = self.paginate_queryset(queryset) 52 | if page is not None: 53 | serializer = self.get_serializer(page, many=True) 54 | return self.get_paginated_response(serializer.data) 55 | serializer = WorkflowTemplateSerializer(queryset, many=True) 56 | logger.debug('page === %s', page) 57 | if page is not None: 58 | return self.get_paginated_response(serializer.data) 59 | return Response({ 60 | 'code': 20000, 61 | 'status': 'success', 62 | 'data': { 63 | 'total': queryset.count(), 64 | 'items': serializer.data 65 | } 66 | }) 67 | -------------------------------------------------------------------------------- /apps/workflow/views/my_related.py: -------------------------------------------------------------------------------- 1 | from workflow.views.workflow import WorkflowViewSetAbstract, check_user_is_workflow_member 2 | 3 | from dbapp.models import WorkflowNodeHistory 4 | import logging 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class WorkflowMyRelatedViewSet(WorkflowViewSetAbstract): 10 | """ 11 | 我的关联 12 | 13 | ### 我的关联 权限 14 | {'*': ('admin', '管理员')}, 15 | {'*': ('workflow_all', '工单管理')}, 16 | {'get': ('workflow_my_relate_list', '查看我关联的工单')}, 17 | """ 18 | perms_map = ( 19 | {'*': ('admin', '管理员')}, 20 | {'*': ('workflow_all', '工单管理')}, 21 | {'get': ('workflow_my_relate_list', '查看我关联的工单')}, 22 | ) 23 | 24 | def extend_filter(self, queryset): 25 | return self._get_node_include_me_workflow(queryset) 26 | 27 | def _get_node_include_me_workflow(self, queryset): 28 | """ 29 | """ 30 | match_id_list = [] 31 | user_obj = self.request.user 32 | for i in queryset: 33 | if check_user_is_workflow_member(i, user_obj) or WorkflowNodeHistory.objects.filter( 34 | workflow=i, 35 | operator=self.request.user 36 | ).count() > 0: 37 | match_id_list.append(i.id) 38 | return queryset.filter(id__in=match_id_list) 39 | -------------------------------------------------------------------------------- /apps/workflow/views/my_request.py: -------------------------------------------------------------------------------- 1 | from workflow.views.workflow import WorkflowViewSetAbstract 2 | 3 | from workflow.serializers import WorkflowListSerializer 4 | 5 | 6 | class WorkflowMyRequestViewSet(WorkflowViewSetAbstract): 7 | """ 8 | 我的请求 9 | 10 | ### 工单 我的请求 权限 11 | {'*': ('admin', '管理员')}, 12 | {'*': ('workflow_all', '工单管理')}, 13 | {'get': ('workflow_my_request_list', '查看我创建的工单')}, 14 | """ 15 | perms_map = ( 16 | {'*': ('admin', '管理员')}, 17 | {'*': ('workflow_all', '工单管理')}, 18 | {'get': ('workflow_my_request_list', '查看我创建的工单')}, 19 | ) 20 | serializer_list_class = WorkflowListSerializer 21 | 22 | def extend_filter(self, queryset): 23 | return queryset.filter(creator=self.request.user.id) 24 | -------------------------------------------------------------------------------- /apps/workflow/views/my_upcoming.py: -------------------------------------------------------------------------------- 1 | from django.db import transaction 2 | from rest_framework.decorators import action 3 | from rest_framework.response import Response 4 | from workflow.views.workflow import WorkflowViewSetAbstract, check_workflow_permission, \ 5 | check_user_include_workflow_member, IS_NOTICE_ASYNC 6 | 7 | from common.exception import OkAPIException 8 | from common.extends.handler import log_audit 9 | from dbapp.models import UserProfile 10 | from ucenter.serializers import UserProfileListSerializers 11 | from workflow.lifecycle import LifeCycle 12 | from dbapp.models import WorkflowNodeHistory, Workflow 13 | from workflow.notice import get_member_user_ids 14 | from workflow.serializers import WorkflowNodeHistorySerializer 15 | 16 | import logging 17 | 18 | logger = logging.getLogger(__name__) 19 | 20 | 21 | class WorkflowMyUpComingViewSet(WorkflowViewSetAbstract): 22 | """ 23 | 我的待办 24 | 25 | ### 我的待办 权限 26 | {'*': ('admin', '管理员')}, 27 | {'*': ('workflow_all', '工单管理')}, 28 | {'get': ('workflow_my_upcoming_list', '查看我的待办工单')}, 29 | {'put': ('workflow_handle', '处理工单')}, 30 | """ 31 | perms_map = ( 32 | {'*': ('admin', '管理员')}, 33 | {'*': ('workflow_all', '工单管理')}, 34 | {'get': ('workflow_my_upcoming_list', '查看我的待办工单')}, 35 | {'put': ('workflow_handle', '处理工单')}, 36 | ) 37 | 38 | @transaction.atomic 39 | @check_workflow_permission() 40 | def update(self, request, *args, **kwargs): 41 | """ 42 | 处理流程 43 | :param request: 44 | :param args: 45 | :param kwargs: 46 | :return: 47 | """ 48 | filters = {'wid': kwargs['pk']} 49 | workflow_obj = self.queryset.get(**filters) 50 | current_node = request.data['node'] 51 | form = request.data['form'] 52 | passed = WorkflowNodeHistory.HandleType.PASSED 53 | handle_type = form.get('handle_type', passed) 54 | if not handle_type: 55 | if workflow_obj.template.node_list.index(workflow_obj.node) == 0: 56 | handle_type = passed 57 | form['handle_type'] = passed 58 | else: 59 | return Response({ 60 | 'message': '缺少参数 审批类型', 'code': 40000, 'status': 'failed' 61 | }) 62 | node_form = { 63 | 'workflow': workflow_obj.pk, 64 | 'node': workflow_obj.node, 65 | 'form': form, 66 | 'handle_type': handle_type == passed and passed or WorkflowNodeHistory.HandleType.REJECT, 67 | 'operator': request.user.id 68 | } 69 | # 将新的节点数据插入到数据库 70 | node_serializer = WorkflowNodeHistorySerializer(data=node_form) 71 | if not node_serializer.is_valid(): 72 | raise OkAPIException({ 73 | 'code': 40000, 74 | 'status': 'failed', 75 | 'message': node_serializer.errors 76 | }) 77 | node_obj = node_serializer.save() 78 | logger.debug(f'node_obj.handle_type === {node_obj.handle_type}') 79 | workflow_obj.status = workflow_obj.STATUS.wait 80 | # 判断是否指定了下一个节点处理人 81 | # 如果指定了， 直接修改当前工单绑定的模板节点数据 82 | next_handle_user_list = form.get('next_handle_user') 83 | if handle_type == passed and next_handle_user_list: 84 | template_obj = workflow_obj.template 85 | next_node_conf = template_obj.nodes[template_obj.node_list.index( 86 | workflow_obj.node) + 1] 87 | member_list = [] 88 | for user_id in next_handle_user_list: 89 | user_obj = UserProfile.objects.filter(id=user_id) 90 | if not user_obj: 91 | continue 92 | user_obj = user_obj.first() 93 | member_list.append(f'user@{user_obj.id}@{user_obj}') 94 | next_node_conf['members'] = member_list 95 | template_obj.save() 96 | 97 | workflow_obj.save() 98 | life_cycle = LifeCycle(request, workflow_obj, 99 | node_obj, form, is_async=IS_NOTICE_ASYNC) 100 | life_cycle.handle() 101 | data = node_serializer.data 102 | data['status'] = 'success' 103 | data['code'] = 20000 104 | log_audit( 105 | request, 106 | action_type=self.serializer_class.Meta.model.__name__, 107 | action='创建', content='', 108 | data=data 109 | ) 110 | return Response(data) 111 | 112 | @transaction.atomic 113 | @action(methods=['GET'], url_path='next_handle_users', detail=True) 114 | def next_handle_users(self, request, *args, **kwargs): 115 | filters = {'wid': kwargs['pk']} 116 | workflow_obj = self.queryset.get(**filters) 117 | template_obj = workflow_obj.template 118 | cur_node = request.GET['node'] 119 | next_node_index = template_obj.node_list.index(cur_node) + 1 120 | if next_node_index >= len(template_obj.nodes): 121 | return Response({ 122 | 'code': 40000, 123 | 'status': 'failed', 124 | 'message': '当前节点已经是最后一个节点' 125 | }) 126 | 127 | next_node_conf = template_obj.nodes[next_node_index] 128 | node_members = next_node_conf.get('members', []) 129 | member_ids = get_member_user_ids(node_members) 130 | member_objs = UserProfile.objects.filter( 131 | id__in=member_ids 132 | ) 133 | serializer = UserProfileListSerializers(member_objs, many=True) 134 | return Response({ 135 | 'total': member_objs.count(), 136 | 'items': serializer.data, 137 | 'code': 20000, 138 | 'status': 'success' 139 | }) 140 | 141 | def extend_filter(self, queryset): 142 | queryset = queryset.exclude( 143 | status__in=[Workflow.STATUS.complete, Workflow.STATUS.close]) 144 | queryset_list = self._get_current_node_members_include_me_workflow( 145 | queryset) 146 | ignore_id_list = [] 147 | for wf in queryset_list: 148 | if wf.cur_node_conf.get('pass_type', '') == 'countersigned': 149 | user_node_his_objs = WorkflowNodeHistory.objects.filter( 150 | operator=self.request.user, 151 | node=wf.node, 152 | handle_type=WorkflowNodeHistory.HandleType.PASSED 153 | ) 154 | if user_node_his_objs.count() > 0: 155 | ignore_id_list.append(wf.id) 156 | return queryset_list.exclude(id__in=ignore_id_list) 157 | 158 | def _get_current_node_members_include_me_workflow(self, queryset): 159 | match_id_list = [] 160 | user_obj = self.request.user 161 | user_departments_obj = UserProfile.objects.get( 162 | id=user_obj.id).department.all() 163 | for i in queryset: 164 | data_id = i.id 165 | node_name = i.node 166 | node_conf = i.cur_node_conf 167 | if not node_conf: 168 | continue 169 | members = node_conf.get('members', []) 170 | # 如果是驳回状态的流程 171 | # 判断发起人跟当前用户是否匹配 172 | if i.status == Workflow.STATUS.reject: 173 | if i.creator == user_obj: 174 | match_id_list.append(data_id) 175 | continue 176 | if len(members) > 0 and check_user_include_workflow_member(members, user_obj, user_departments_obj): 177 | match_id_list.append(data_id) 178 | return queryset.filter(id__in=match_id_list) 179 | 180 | @action(methods=['GET'], url_path='count', detail=False) 181 | def count(self, request, *args, **kwargs): 182 | queryset = self.filter_queryset(self.get_queryset()) 183 | return Response({'code': 20000, 'data': queryset.count()}) 184 | -------------------------------------------------------------------------------- /apps/workflow/views/template.py: -------------------------------------------------------------------------------- 1 | import django_filters 2 | from rest_framework.filters import OrderingFilter 3 | 4 | from common.extends.filters import CustomSearchFilter 5 | from common.extends.permissions import RbacPermission 6 | from common.extends.viewsets import CustomModelViewSet 7 | from dbapp.models import WorkflowTemplate 8 | from workflow.serializers import WorkflowTemplateSerializer 9 | 10 | import logging 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | class WorkflowTemplateViewSet(CustomModelViewSet): 16 | """ 17 | 工单模板 18 | ### 工单模板权限 19 | {'*': ('admin', '管理员')}, 20 | {'*': ('workflow_template_all', '工单模板管理')}, 21 | {'get': ('workflow_template_list', '查看工单模板')}, 22 | {'post': ('workflow_template_create', '创建工单模板')}, 23 | {'put': ('workflow_template_edit', '编辑工单模板')}, 24 | {'delete': ('workflow_template_delete', '删除工单模板')} 25 | """ 26 | perms_map = ( 27 | {'*': ('admin', '管理员')}, 28 | {'*': ('workflow_template_all', '工单模板管理')}, 29 | {'get': ('workflow_template_list', '查看工单模板')}, 30 | {'post': ('workflow_template_create', '创建工单模板')}, 31 | {'put': ('workflow_template_edit', '编辑工单模板')}, 32 | {'delete': ('workflow_template_delete', '删除工单模板')} 33 | ) 34 | queryset = WorkflowTemplate.objects.all() 35 | serializer_class = WorkflowTemplateSerializer 36 | filter_backends = (django_filters.rest_framework.DjangoFilterBackend, 37 | CustomSearchFilter, OrderingFilter) 38 | filter_fields = ('category', 'enabled',) 39 | search_fields = ('name',) 40 | 41 | def get_serializer_context(self): 42 | """ 43 | Extra context provided to the serializer class. 44 | """ 45 | return { 46 | 'request': self.request, 47 | 'format': self.format_kwarg, 48 | 'view': self 49 | } 50 | 51 | def update(self, request, *args, **kwargs): 52 | response = super().update(request, *args, **kwargs) 53 | # 更新一下版本号 54 | if response.data['status'] == 'success': 55 | filters = {'pk': kwargs['pk']} 56 | instance = self.queryset.get(**filters) 57 | instance.revision = instance.revision + 1 58 | instance.save() 59 | return response 60 | -------------------------------------------------------------------------------- /apps/workflow_callback/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/apps/workflow_callback/__init__.py -------------------------------------------------------------------------------- /apps/workflow_callback/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class WorkflowCallbackConfig(AppConfig): 5 | name = 'workflow_callback' 6 | -------------------------------------------------------------------------------- /apps/workflow_callback/middleware.py: -------------------------------------------------------------------------------- 1 | from django.http import HttpResponse 2 | from django.utils.deprecation import MiddlewareMixin 3 | import logging 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | 8 | class WorkflowCallbackMiddleware(MiddlewareMixin): 9 | def process_request(self, request): 10 | workflow_node_history_id = request.GET.get('__workflow_node_history_id__') 11 | if request.GET.get('__workflow_node_history_id__'): 12 | setattr(request, 'workflow_node_history_id', workflow_node_history_id) 13 | 14 | def process_exception(self, request, exception): 15 | if hasattr(request, 'workflow_node_history_id'): 16 | msg = f'工单回调发生异常： {exception.__class__} {exception}' 17 | logger.exception(f'工单回调发生异常 {exception}') 18 | return HttpResponse(msg, status=500) 19 | raise exception 20 | -------------------------------------------------------------------------------- /apps/workflow_callback/urls.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Ken Chen 5 | @Contact : chenxiaoshun@yl-scm.com 6 | @Time : 2021/12/10 下午1:53 7 | @FileName: urls 8 | """ 9 | 10 | from django.conf.urls import url, include 11 | from rest_framework.routers import DefaultRouter 12 | 13 | from workflow_callback.views.app import AppMemberAPIView 14 | 15 | router = DefaultRouter() 16 | 17 | urlpatterns = [ 18 | url(r'', include(router.urls)), 19 | url(r'app/member', AppMemberAPIView.as_view(), name='app-member'), 20 | ] 21 | -------------------------------------------------------------------------------- /apps/workflow_callback/views/app.py: -------------------------------------------------------------------------------- 1 | from django.db import transaction 2 | from dbapp.models import MicroApp 3 | from common.ext_fun import get_datadict, get_members 4 | from dbapp.models import UserProfile, DataDict 5 | from workflow_callback.views.base import CallbackAPIView 6 | from rest_framework.response import Response 7 | from dbapp.models import Workflow 8 | 9 | from asgiref.sync import sync_to_async 10 | from channels.db import database_sync_to_async 11 | import logging 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | class AppMemberAPIView(CallbackAPIView): 17 | 18 | @transaction.atomic 19 | def post(self, request): 20 | data = request.data 21 | first_node_form = data['first_node_form'] 22 | app_node = first_node_form.get('申请应用') 23 | app_value = app_node.get('applist', []) 24 | position = app_node.get('position', None) 25 | if not app_value: 26 | return Response('缺少 申请应用 参数', status=400) 27 | user_value = first_node_form.get('申请用户') 28 | if not user_value: 29 | return Response('缺少 申请用户 参数', status=400) 30 | 31 | # 获取工单 32 | wf = self.get_workflow(request) 33 | user_value = list(map(lambda x: x.split('@')[0], user_value)) 34 | app_objs = MicroApp.objects.filter(appid__in=app_value) 35 | user_objs = UserProfile.objects.filter(username__in=user_value) 36 | response_texts = [] 37 | init_point = transaction.savepoint() 38 | for app_obj in app_objs: 39 | for user_obj in user_objs: 40 | if user_obj.is_superuser: 41 | user_position = 'op' 42 | else: 43 | positions = get_datadict('POSITION', config=1) 44 | user_obj.position = position 45 | user_obj.save() 46 | user_position = user_obj.position 47 | if user_position not in [i['name'] for i in positions]: 48 | response_texts.append( 49 | f'{app_obj} 用户 {user_obj} position 不在映射中： {user_position}') 50 | continue 51 | member_list = app_obj.team_members.get(user_position, []) 52 | if user_obj.id in member_list: 53 | response_texts.append( 54 | f'{app_obj} {user_position} 用户已经存在：{user_obj}') 55 | continue 56 | member_list.append(user_obj.id) 57 | response_texts.append( 58 | f'{app_obj} {user_position} 添加用户：{user_obj}') 59 | app_obj.save() 60 | response_text = "
".join(response_texts) 61 | self.set_status(wf, 'complete') 62 | return Response(f'执行完毕
{response_text}') 63 | -------------------------------------------------------------------------------- /apps/workflow_callback/views/base.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : base.py 6 | @time : 2023/03/16 22:00 7 | @contact : qqing_lai@hotmail.com 8 | @company : IMAOJIA Co,Ltd 9 | ''' 10 | 11 | # here put the import lib 12 | import asyncio 13 | import logging 14 | from typing import Any 15 | 16 | from django.db import close_old_connections, connection 17 | 18 | from rest_framework.permissions import IsAuthenticated 19 | from rest_framework.views import APIView 20 | 21 | from dbapp.models import Workflow 22 | 23 | logger = logging.getLogger(__name__) 24 | 25 | 26 | def get_form_item_name(node_conf, cname): 27 | for field_conf in node_conf['form_models']: 28 | field_name = field_conf['field'] 29 | field_cname = field_conf['title'] 30 | if field_cname == cname: 31 | return field_name 32 | 33 | 34 | class CallbackAPIView(APIView): 35 | permission_classes = [IsAuthenticated] 36 | 37 | def get_workflow(self, request): 38 | try: 39 | wid = request.data['workflow']['wid'] 40 | wf = Workflow.objects.get(wid=wid) 41 | return wf 42 | except BaseException as e: 43 | logger.info(f'获取不到工单ID，原因：{e}') 44 | raise Exception('获取不到工单ID') 45 | 46 | def set_status(self, wf, status): 47 | wf.status = getattr(Workflow.STATUS, status) 48 | wf.save() 49 | 50 | def task_queue(self, *args): 51 | pass 52 | 53 | def handler(self, *args): 54 | tasks = self.task_queue(*args) 55 | loop = asyncio.new_event_loop() 56 | asyncio.set_event_loop(loop) 57 | results = [] 58 | try: 59 | results = loop.run_until_complete(asyncio.gather(*tasks)) 60 | except BaseException as e: 61 | logger.debug(f'err {e}') 62 | results = [(False, '执行异常')] 63 | loop.close() 64 | return results 65 | -------------------------------------------------------------------------------- /celery.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export PYTHONOPTIMIZE=1 3 | 4 | if [ $# -ne 0 ];then 5 | queue=$1 6 | elif [ ! -z $QUEUE ];then 7 | queue=$QUEUE 8 | else 9 | queue='celery' 10 | fi 11 | 12 | echo "Current Queue: $queue" 13 | 14 | celery -A celery_tasks worker --loglevel=info --without-gossip --without-mingle --without-heartbeat -E -Q $queue 15 | -------------------------------------------------------------------------------- /celery_tasks/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/6/19 下午3:35 7 | @FileName: __init__.py.py 8 | @Company : Vision Fund 9 | """ 10 | 11 | from __future__ import unicode_literals 12 | -------------------------------------------------------------------------------- /celery_tasks/celery.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/5/19 下午5:44 7 | @FileName: celery.py 8 | @Company : Vision Fund 9 | """ 10 | 11 | from __future__ import unicode_literals 12 | 13 | from celery import Celery 14 | from django.conf import settings 15 | from celery_tasks import celeryconfig 16 | import os 17 | from ansible import constants as C 18 | 19 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'devops_backend.settings') 20 | 21 | app = Celery('celery_tasks') 22 | app.config_from_object(celeryconfig) 23 | 24 | app.autodiscover_tasks(settings.INSTALLED_APPS) 25 | -------------------------------------------------------------------------------- /celery_tasks/celeryconfig.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/6/19 下午3:37 7 | @FileName: celeryconfig.py 8 | @Company : Vision Fund 9 | """ 10 | 11 | from __future__ import unicode_literals 12 | 13 | from celery.schedules import crontab 14 | 15 | import datetime 16 | 17 | from devops_backend import settings 18 | from config import CELERY_CONFIG 19 | 20 | import os 21 | import pytz 22 | 23 | 24 | def now_func(): return datetime.datetime.now(pytz.timezone(settings.TIME_ZONE)) 25 | 26 | 27 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'devops_backend.settings') 28 | 29 | # 使用 django logging 配置 30 | EREBUS_WORKER_HIJACK_ROOT_LOGGER = False 31 | CELERY_DEFAULT_QUEUE = CELERY_CONFIG.get('queue', 'celery') 32 | # 设置结果存储 33 | if CELERY_CONFIG['result_backend'].get('startup_nodes', None): 34 | # 存在startup_nodes配置项，使用集群redis 35 | CELERY_RESULT_BACKEND = 'common.CeleryRedisClusterBackend.RedisClusterBackend' 36 | CELERY_REDIS_CLUSTER_SETTINGS = {'startup_nodes': CELERY_CONFIG['result_backend']['startup_nodes'], 37 | 'password': CELERY_CONFIG['result_backend']['password_cluster']} 38 | else: 39 | CELERY_RESULT_BACKEND = f"redis://:{CELERY_CONFIG['result_backend'].get('password', '')}@{CELERY_CONFIG['result_backend']['host']}:{CELERY_CONFIG['result_backend']['port']}/{CELERY_CONFIG['result_backend']['db']}" 40 | CELERY_RESULT_SERIALIZER = 'json' 41 | # 设置代理人broker 42 | BROKER_URL = f"redis://:{CELERY_CONFIG['result_backend'].get('password', '')}@{CELERY_CONFIG['broker_url']['host']}:{CELERY_CONFIG['broker_url']['port']}/{CELERY_CONFIG['broker_url']['db']}" 43 | CELERYD_FORCE_EXECV = True 44 | CELERY_ENABLE_UTC = True 45 | CELERY_TIMEZONE = settings.TIME_ZONE 46 | DJANGO_CELERY_BEAT_TZ_AWARE = False 47 | CELERYBEAT_SCHEDULE = { 48 | } 49 | -------------------------------------------------------------------------------- /common/MailSend.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | # @Author: Charles Lai 4 | # @Email: qqing_lai@hotmail.com 5 | # @Site: https://imaojia.com 6 | # @File: MailSend.py 7 | # @Time: 18-3-7 下午1:48 8 | 9 | from fernet_fields import EncryptedField 10 | from django.core.mail import send_mail 11 | from django.core.mail import send_mass_mail 12 | from django.core.mail.backends.smtp import EmailBackend as BaseEmailBackend 13 | from django.conf import settings 14 | 15 | from dbapp.models import SystemConfig 16 | 17 | from common.utils.RedisAPI import RedisManage 18 | from common.ext_fun import get_redis_data 19 | 20 | import threading 21 | import json 22 | import logging 23 | 24 | logger = logging.getLogger(__name__) 25 | 26 | TICKET_STATUS = {0: '', 1: '', 2: '需要审批', 3: '审批通过，待执行', 4: '执行中', 5: '已处理完成，请前往平台确认是否结单', 27 | 6: '审批不通过', 7: '申请被驳回', 28 | 8: '因用户对处理结果有异议，请重新处理', 9: '用户已确认结单'} 29 | 30 | 31 | def check_key(key, data): 32 | if key in data: 33 | return data[key] 34 | else: 35 | return None 36 | 37 | 38 | class EmailBackend(BaseEmailBackend): 39 | """ 40 | A wrapper that manages the SMTP network connection. 41 | """ 42 | 43 | def __init__(self, host=None, port=None, username=None, password=None, 44 | use_tls=None, fail_silently=False, use_ssl=None, timeout=None, 45 | ssl_keyfile=None, ssl_certfile=None, 46 | **kwargs): 47 | super().__init__(fail_silently=fail_silently) 48 | self.mail_config = get_redis_data('mail') 49 | self.host = host or check_key( 50 | 'host', self.mail_config) or settings.EMAIL_HOST 51 | self.port = port or check_key( 52 | 'port', self.mail_config) or settings.EMAIL_PORT 53 | self.username = check_key('user', 54 | self.mail_config) or settings.EMAIL_HOST_USER if username is None else username 55 | self.password = check_key('password', 56 | self.mail_config) or settings.EMAIL_HOST_PASSWORD if password is None else password 57 | self.use_tls = check_key( 58 | 'tls', self.mail_config) or settings.EMAIL_USE_TLS if use_tls is None else use_tls 59 | self.use_ssl = check_key( 60 | 'ssl', self.mail_config) or settings.EMAIL_USE_SSL if use_ssl is None else use_ssl 61 | self.timeout = check_key( 62 | 'timeout', self.mail_config) or settings.EMAIL_TIMEOUT if timeout is None else timeout 63 | self.ssl_keyfile = check_key('key', 64 | self.mail_config) or settings.EMAIL_SSL_KEYFILE if ssl_keyfile is None else ssl_keyfile 65 | self.ssl_certfile = check_key('cert', 66 | self.mail_config) or settings.EMAIL_SSL_CERTFILE if ssl_certfile is None else ssl_certfile 67 | if self.use_ssl and self.use_tls: 68 | raise ValueError( 69 | "EMAIL_USE_TLS/EMAIL_USE_SSL are mutually exclusive, so only set " 70 | "one of those settings to True.") 71 | self.connection = None 72 | self._lock = threading.RLock() 73 | 74 | 75 | class OmsMail(object): 76 | def __init__(self): 77 | self.__email_config = get_redis_data('mail') 78 | self.__master = None 79 | if self.__email_config: 80 | self.__master = self.__email_config.get('user', None) 81 | self.__url = get_redis_data('platform')['url'] 82 | 83 | def send_mail(self, title, msg, receiver, is_html=False): 84 | self.__send_mail(title, msg, receiver, is_html=is_html) 85 | 86 | def __send_mail(self, title, msg, receiver, is_html=False): 87 | """ 88 | 89 | :param title: 90 | :param msg: 91 | :param receiver: 'a@yd.com,b@yd.com' 92 | :return: 93 | """ 94 | try: 95 | html_message = '' 96 | if is_html: 97 | html_message = msg 98 | send_mail( 99 | f"{self.__email_config['prefix']}{title}", 100 | msg, 101 | self.__master, receiver.split(','), 102 | html_message=html_message 103 | ) 104 | return {'status': 0} 105 | except Exception as e: 106 | print('err', e) 107 | return {'status': 1, 'msg': '发送邮件通知失败 %s' % str(e)} 108 | 109 | def ticket_process(self, ticket, title, status, user, receiver): 110 | msg = f"Hi {user}，\n你有新的工单{ticket}（标题：{title}）{TICKET_STATUS[status]}。\n请访问{self.__url} 进行处理。" 111 | self.__send_mail('工单跟踪', msg, receiver) 112 | 113 | def ticket_handle(self, ticket, title, status, user, receiver): 114 | msg = f"Hi {user}，\n工单{ticket}（标题：{title}）{TICKET_STATUS[status]}。\n请访问{self.__url} 进行处理。" 115 | self.__send_mail('工单跟踪', msg, receiver) 116 | 117 | def ticket_create(self, ticket, title, status, user, receiver): 118 | mail_title = '工单处理结果' 119 | if status == 4: 120 | mail_title = '工单处理中' 121 | msg = f"Hi {user}，\n你的工单{ticket}（标题：{title}）{TICKET_STATUS[status]}。\n请访问{self.__url} 查看更多信息。" 122 | self.__send_mail(mail_title, msg, receiver) 123 | 124 | def account_register(self, op_user, username, password, user, receiver): 125 | msg = f"Hi {user}，\n{op_user}已为你开通平台账号，相关信息如下：\n用户名：{username}\n密码：{password}\n登录地址：{self.__url}。" 126 | self.__send_mail('账号开通', msg, receiver) 127 | 128 | def deploy_notify(self, title, msg, receiver): 129 | self.__send_mail(title, msg, receiver) 130 | 131 | def test_notify(self, receiver): 132 | ret = self.__send_mail('邮件测试', "Hi，如果能看到此邮件，说明平台邮件服务配置成功", receiver) 133 | return ret 134 | -------------------------------------------------------------------------------- /common/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | """ 4 | @author: Charles Lai 5 | @file: __init__.py.py 6 | @time: 2022/9/23 10:14 7 | @contact: qqing_lai@hotmail.com 8 | @company: IMAOJIA Co,Ltd 9 | """ 10 | -------------------------------------------------------------------------------- /common/ansible_callback/profile_tasks.py: -------------------------------------------------------------------------------- 1 | # (C) 2016, Joel, https://github.com/jjshoe 2 | # (C) 2015, Tom Paine, 3 | # (C) 2014, Jharrod LaFon, @JharrodLaFon 4 | # (C) 2012-2013, Michael DeHaan, 5 | # (C) 2017 Ansible Project 6 | # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) 7 | 8 | # Make coding more python3-ish 9 | from __future__ import (absolute_import, division, print_function) 10 | 11 | __metaclass__ = type 12 | 13 | DOCUMENTATION = ''' 14 | callback: profile_tasks 15 | type: aggregate 16 | short_description: adds time information to tasks 17 | version_added: "2.0" 18 | description: 19 | - Ansible callback plugin for timing individual tasks and overall execution time. 20 | - "Mashup of 2 excellent original works: https://github.com/jlafon/ansible-profile, 21 | https://github.com/junaid18183/ansible_home/blob/master/ansible_plugins/callback_plugins/timestamp.py.old" 22 | - "Format: C( () )" 23 | - It also lists the top/bottom time consuming tasks in the summary (configurable) 24 | - Before 2.4 only the environment variables were available for configuration. 25 | requirements: 26 | - whitelisting in configuration - see examples section below for details. 27 | options: 28 | output_limit: 29 | description: Number of tasks to display in the summary 30 | default: 20 31 | env: 32 | - name: PROFILE_TASKS_TASK_OUTPUT_LIMIT 33 | ini: 34 | - section: callback_profile_tasks 35 | key: task_output_limit 36 | sort_order: 37 | description: Adjust the sorting output of summary tasks 38 | choices: ['descending', 'ascending', 'none'] 39 | default: 'descending' 40 | env: 41 | - name: PROFILE_TASKS_SORT_ORDER 42 | ini: 43 | - section: callback_profile_tasks 44 | key: sort_order 45 | ''' 46 | 47 | EXAMPLES = ''' 48 | example: > 49 | To enable, add this to your ansible.cfg file in the defaults block 50 | [defaults] 51 | callback_whitelist = profile_tasks 52 | sample output: > 53 | # 54 | # TASK: [ensure messaging security group exists] ******************************** 55 | # Thursday 11 June 2017 22:50:53 +0100 (0:00:00.721) 0:00:05.322 ********* 56 | # ok: [localhost] 57 | # 58 | # TASK: [ensure db security group exists] *************************************** 59 | # Thursday 11 June 2017 22:50:54 +0100 (0:00:00.558) 0:00:05.880 ********* 60 | # changed: [localhost] 61 | # 62 | ''' 63 | 64 | import collections 65 | import time 66 | 67 | from ansible.module_utils.six.moves import reduce 68 | from ansible.plugins.callback import CallbackBase 69 | 70 | # define start time 71 | t0 = tn = time.time() 72 | 73 | 74 | def secondsToStr(t): 75 | # http://bytes.com/topic/python/answers/635958-handy-short-cut-formatting-elapsed-time-floating-point-seconds 76 | def rediv(ll, b): 77 | return list(divmod(ll[0], b)) + ll[1:] 78 | 79 | return "%d:%02d:%02d.%03d" % tuple(reduce(rediv, [[t * 1000, ], 1000, 60, 60])) 80 | 81 | 82 | def filled(msg, fchar="*"): 83 | if len(msg) == 0: 84 | width = 79 85 | else: 86 | msg = "%s " % msg 87 | width = 79 - len(msg) 88 | if width < 3: 89 | width = 3 90 | filler = fchar * width 91 | return "%s%s " % (msg, filler) 92 | 93 | 94 | def timestamp(self): 95 | if self.current is not None: 96 | self.stats[self.current]['time'] = time.time() - self.stats[self.current]['time'] 97 | 98 | 99 | def tasktime(): 100 | global tn 101 | time_current = time.strftime('%A %d %B %Y %H:%M:%S %z') 102 | time_elapsed = secondsToStr(time.time() - tn) 103 | time_total_elapsed = secondsToStr(time.time() - t0) 104 | tn = time.time() 105 | return filled('%s (%s)%s%s' % (time_current, time_elapsed, ' ' * 7, time_total_elapsed)) 106 | 107 | 108 | class CallbackModule(CallbackBase): 109 | """ 110 | This callback module provides per-task timing, ongoing playbook elapsed time 111 | and ordered list of top 20 longest running tasks at end. 112 | """ 113 | CALLBACK_VERSION = 2.0 114 | CALLBACK_TYPE = 'aggregate' 115 | CALLBACK_NAME = 'profile_tasks' 116 | CALLBACK_NEEDS_WHITELIST = True 117 | 118 | def __init__(self): 119 | self.stats = collections.OrderedDict() 120 | self.current = None 121 | 122 | self.sort_order = None 123 | self.task_output_limit = None 124 | 125 | super(CallbackModule, self).__init__() 126 | 127 | def set_options(self, task_keys=None, var_options=None, direct=None): 128 | 129 | super(CallbackModule, self).set_options(task_keys=task_keys, var_options=var_options, direct=direct) 130 | 131 | self.sort_order = self.get_option('sort_order') 132 | if self.sort_order is not None: 133 | if self.sort_order == 'ascending': 134 | self.sort_order = False 135 | elif self.sort_order == 'descending': 136 | self.sort_order = True 137 | elif self.sort_order == 'none': 138 | self.sort_order = None 139 | 140 | self.task_output_limit = self.get_option('output_limit') 141 | if self.task_output_limit is not None: 142 | if self.task_output_limit == 'all': 143 | self.task_output_limit = None 144 | else: 145 | self.task_output_limit = int(self.task_output_limit) 146 | 147 | def _record_task(self, task): 148 | """ 149 | Logs the start of each task 150 | """ 151 | self._display.display(tasktime()) 152 | timestamp(self) 153 | 154 | # Record the start time of the current task 155 | self.current = task._uuid 156 | self.stats[self.current] = {'time': time.time(), 'name': task.get_name()} 157 | if self._display.verbosity >= 2: 158 | self.stats[self.current]['path'] = task.get_path() 159 | 160 | def v2_playbook_on_task_start(self, task, is_conditional): 161 | self._record_task(task) 162 | 163 | def v2_playbook_on_handler_task_start(self, task): 164 | self._record_task(task) 165 | 166 | def playbook_on_setup(self): 167 | self._display.display(tasktime()) 168 | 169 | def playbook_on_stats(self, stats): 170 | self._display.display(tasktime()) 171 | self._display.display(filled("", fchar="=")) 172 | 173 | timestamp(self) 174 | self.current = None 175 | 176 | results = list(self.stats.items()) 177 | 178 | # Sort the tasks by the specified sort 179 | if self.sort_order is not None: 180 | results = sorted( 181 | self.stats.items(), 182 | key=lambda x: x[1]['time'], 183 | reverse=self.sort_order, 184 | ) 185 | 186 | # Display the number of tasks specified or the default of 20 187 | results = list(results)[:self.task_output_limit] 188 | 189 | # Print the timings 190 | for uuid, result in results: 191 | msg = u"{0:-<{2}}{1:->9}".format(result['name'] + u' ', u' {0:.02f}s'.format(result['time']), 192 | self._display.columns - 9) 193 | if 'path' in result: 194 | msg += u"\n{0:-<{1}}".format(result['path'] + u' ', self._display.columns) 195 | self._display.display(msg) 196 | -------------------------------------------------------------------------------- /common/custom_format.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/5/14 下午4:31 7 | @FileName: custom_format.py 8 | @Blog : https://blog.imaojia.com 9 | """ 10 | 11 | import xml.etree.ElementTree as ET 12 | import xmltodict 13 | 14 | 15 | def convert_xml_to_str_with_pipeline(xml, url, secret, desc, jenkinsfile, scm=True): 16 | """ 17 | scm 18 | True: jenkinsfile为指定的git地址 19 | False: jenkinsfile为具体的pipeline 20 | """ 21 | xml_dict = xmltodict.parse(xml) 22 | if scm: 23 | xml_dict['flow-definition']['definition']['@class'] = 'org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition' 24 | xml_dict['flow-definition']['definition']['scm']['userRemoteConfigs']['hudson.plugins.git.UserRemoteConfig'][ 25 | 'url'] = url 26 | xml_dict['flow-definition']['definition']['scm']['userRemoteConfigs']['hudson.plugins.git.UserRemoteConfig'][ 27 | 'credentialsId'] = secret 28 | xml_dict['flow-definition']['definition']['scriptPath'] = jenkinsfile 29 | else: 30 | xml_dict['flow-definition']['definition']['@class'] = 'org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition' 31 | xml_dict['flow-definition']['definition']['script'] = jenkinsfile 32 | xml_dict['flow-definition']['definition']['sandbox'] = 'true' 33 | xml_dict['flow-definition']['description'] = desc 34 | result = xmltodict.unparse( 35 | xml_dict, short_empty_elements=True, pretty=True) 36 | return result 37 | -------------------------------------------------------------------------------- /common/exception.py: -------------------------------------------------------------------------------- 1 | from rest_framework.exceptions import APIException 2 | 3 | 4 | class OkAPIException(APIException): 5 | """ 6 | 新增一个 异常类型 7 | 因为接口返回状态重新封装过， stasus_code 都是200， 当需要引发一个 api 异常且需要返回数据的时候，比较麻烦 8 | 为什么需要引发异常： 9 | 好处1： 10 | 能够搭配 transaction.atomic， 当 异常产生的时候， transaction.atomic会回滚数据，防止产生脏数据 11 | """ 12 | status_code = 200 13 | -------------------------------------------------------------------------------- /common/extends/JwtAuth.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/7/2 上午11:00 7 | @FileName: JwtAuth.py 8 | @Company : Vision Fund 9 | """ 10 | 11 | from __future__ import unicode_literals 12 | 13 | from rest_framework import exceptions, status 14 | from rest_framework_simplejwt.authentication import JWTAuthentication as BaseJWTAuthentication 15 | from rest_framework_simplejwt.exceptions import TokenError, InvalidToken 16 | from rest_framework_simplejwt.serializers import TokenObtainPairSerializer as BaseTokenObtainPairSerializer, \ 17 | TokenRefreshSerializer as BaseTokenRefreshSerializer 18 | from rest_framework_simplejwt.tokens import Token as BaseToken, RefreshToken as BaseRefreshToken 19 | from rest_framework_simplejwt.settings import APISettings, DEFAULTS, IMPORT_STRINGS 20 | from django.utils.translation import ugettext_lazy as _ 21 | 22 | from devops_backend import settings 23 | 24 | from common.ext_fun import get_redis_data 25 | import datetime 26 | 27 | api_settings = APISettings( 28 | getattr(settings, 'SIMPLE_JWT', None), DEFAULTS, IMPORT_STRINGS) 29 | 30 | 31 | class JWTAuthentication(BaseJWTAuthentication): 32 | """ 33 | code: 40108 登录失败，40101 刷新Token失效，40100 Token已经过期 34 | """ 35 | 36 | def get_validated_token(self, raw_token): 37 | """ 38 | Validates an encoded JSON web token and returns a validated token 39 | wrapper object. 40 | """ 41 | messages = [] 42 | for AuthToken in api_settings.AUTH_TOKEN_CLASSES: 43 | try: 44 | return AuthToken(raw_token) 45 | except TokenError as e: 46 | messages.append( 47 | { 48 | "token_class": AuthToken.__name__, 49 | "token_type": AuthToken.token_type, 50 | "message": e.args[0], 51 | } 52 | ) 53 | 54 | raise CustomInvalidToken( 55 | { 56 | "detail": 'Token已经过期.', 57 | "code": 40100 58 | } 59 | ) 60 | 61 | 62 | class CustomInvalidToken(InvalidToken): 63 | status_code = status.HTTP_401_UNAUTHORIZED 64 | default_detail = 'Token不合法或者已经过期.' 65 | default_code = 40100 66 | 67 | 68 | class AccessToken(BaseToken): 69 | token_type = 'access' 70 | 71 | def __init__(self, token=None, verify=True): 72 | expired_time = get_redis_data('platform') 73 | lifetime = datetime.timedelta(minutes=expired_time[ 74 | 'access']) if expired_time and 'access' in expired_time else api_settings.ACCESS_TOKEN_LIFETIME 75 | self.lifetime = lifetime 76 | super().__init__(token, verify) 77 | 78 | 79 | class RefreshToken(BaseRefreshToken): 80 | token_type = 'refresh' 81 | 82 | def __init__(self, token=None, verify=True): 83 | expired_time = get_redis_data('platform') 84 | lifetime = datetime.timedelta(minutes=expired_time[ 85 | 'refresh']) if expired_time and 'refresh' in expired_time else api_settings.REFRESH_TOKEN_LIFETIME 86 | self.lifetime = lifetime 87 | super().__init__(token, verify) 88 | 89 | @property 90 | def access_token(self): 91 | """ 92 | Returns an access token created from this refresh token. Copies all 93 | claims present in this refresh token to the new access token except 94 | those claims listed in the `no_copy_claims` attribute. 95 | """ 96 | access = AccessToken() 97 | access.set_exp(from_time=self.current_time) 98 | 99 | no_copy = self.no_copy_claims 100 | for claim, value in self.payload.items(): 101 | if claim in no_copy: 102 | continue 103 | access[claim] = value 104 | 105 | return access 106 | 107 | 108 | class TokenRefreshSerializer(BaseTokenRefreshSerializer): 109 | 110 | def validate(self, attrs): 111 | refresh = RefreshToken(attrs['refresh']) 112 | data = {'access': str(refresh.access_token)} 113 | 114 | if api_settings.ROTATE_REFRESH_TOKENS: 115 | if api_settings.BLACKLIST_AFTER_ROTATION: 116 | try: 117 | # Attempt to blacklist the given refresh token 118 | refresh.blacklist() 119 | except AttributeError: 120 | # If blacklist app not installed, `blacklist` method will 121 | # not be present 122 | pass 123 | 124 | refresh.set_jti() 125 | refresh.set_exp() 126 | 127 | data['refresh'] = str(refresh) 128 | 129 | return data 130 | 131 | 132 | class TokenObtainPairSerializer(BaseTokenObtainPairSerializer): 133 | 134 | default_error_messages = { 135 | "no_active_account": "用户名或者密码错误！" 136 | } 137 | 138 | @classmethod 139 | def get_token(cls, user): 140 | token = RefreshToken.for_user(user) 141 | return token 142 | -------------------------------------------------------------------------------- /common/extends/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/9/15 下午4:56 7 | @FileName: __init__.py.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | -------------------------------------------------------------------------------- /common/extends/authenticate.py: -------------------------------------------------------------------------------- 1 | from rest_framework import exceptions 2 | from rest_framework.authentication import CSRFCheck 3 | from rest_framework_simplejwt.authentication import JWTAuthentication 4 | from django.conf import settings 5 | from rest_framework_simplejwt.settings import api_settings 6 | 7 | import logging 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | class CookiesAuthentication(JWTAuthentication): 13 | cookieName = 'visionToken' 14 | 15 | def authenticate(self, request): 16 | raw_token = request.COOKIES.get(self.cookieName) or None 17 | if raw_token is None: 18 | return None 19 | 20 | validated_token = self.get_validated_token(raw_token) 21 | return self.get_user(validated_token), validated_token 22 | -------------------------------------------------------------------------------- /common/extends/django_qcluster.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : django_qcluster.py 6 | @time : 2023/03/08 21:21 7 | @contact : qqing_lai@hotmail.com 8 | @company : IMAOJIA Co,Ltd 9 | ''' 10 | 11 | # here put the import lib 12 | from time import sleep 13 | from multiprocessing import current_process, Event, Process 14 | from django.utils.translation import gettext_lazy as _ 15 | from django_q.brokers import Broker 16 | from django_q.humanhash import humanize 17 | from django_q.status import Stat 18 | from django_q.cluster import scheduler, Cluster as BaseCluster, Sentinel as BaseSentinel 19 | from django_q.conf import ( 20 | Conf, 21 | logger 22 | ) 23 | 24 | 25 | class Sentinel(BaseSentinel): 26 | 27 | def __init__(self, stop_event, start_event, cluster_id, broker=None, timeout=Conf.TIMEOUT, start=True): 28 | super().__init__(stop_event, start_event, cluster_id, broker, timeout, start) 29 | 30 | def guard(self): 31 | logger.info( 32 | _( 33 | f"{current_process().name} guarding cluster {humanize(self.cluster_id.hex)}" 34 | ) 35 | ) 36 | self.start_event.set() 37 | Stat(self).save() 38 | logger.info(_(f"Q Cluster {humanize(self.cluster_id.hex)} running.")) 39 | counter = 0 40 | cycle = Conf.GUARD_CYCLE # guard loop sleep in seconds, 默认0.5 41 | # Guard loop. Runs at least once 42 | while not self.stop_event.is_set() or not counter: 43 | # Check Workers 44 | for p in self.pool: 45 | with p.timer.get_lock(): 46 | # Are you alive? 47 | if not p.is_alive() or p.timer.value == 0: 48 | self.reincarnate(p) 49 | continue 50 | # Decrement timer if work is being done 51 | if p.timer.value > 0: 52 | p.timer.value -= cycle 53 | # Check Monitor 54 | if not self.monitor.is_alive(): 55 | self.reincarnate(self.monitor) 56 | # Check Pusher 57 | if not self.pusher.is_alive(): 58 | self.reincarnate(self.pusher) 59 | # Call scheduler once a minute (or so) 60 | counter += cycle 61 | # 默认30 62 | if counter >= 10 and Conf.SCHEDULER: 63 | counter = 0 64 | scheduler(broker=self.broker) 65 | # Save current status 66 | Stat(self).save() 67 | sleep(cycle) 68 | self.stop() 69 | 70 | 71 | class Cluster(BaseCluster): 72 | 73 | def __init__(self, broker: Broker = None): 74 | super().__init__(broker) 75 | 76 | def start(self) -> int: 77 | # Start Sentinel 78 | self.stop_event = Event() 79 | self.start_event = Event() 80 | self.sentinel = Process( 81 | target=Sentinel, 82 | args=( 83 | self.stop_event, 84 | self.start_event, 85 | self.cluster_id, 86 | self.broker, 87 | self.timeout, 88 | ), 89 | ) 90 | self.sentinel.start() 91 | logger.info(_(f"Q Cluster {self.name} starting.")) 92 | while not self.start_event.is_set(): 93 | sleep(0.1) 94 | return self.pid 95 | -------------------------------------------------------------------------------- /common/extends/fernet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2021/1/14 下午7:34 7 | @FileName: fernet.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | 11 | from fernet_fields import EncryptedField 12 | from django.db.models import JSONField 13 | 14 | 15 | class EncryptedJsonField(EncryptedField, JSONField): 16 | pass 17 | -------------------------------------------------------------------------------- /common/extends/handler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2021/1/7 上午11:09 7 | @FileName: handler.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | 11 | from dbapp.models import AuditLog 12 | 13 | from common.get_ip import user_ip 14 | from common.ext_fun import mask_sensitive_data 15 | 16 | 17 | def log_audit(request, action_type, action, content=None, data=None, old_data=None, user=None): 18 | if user is None: 19 | user = request.user.first_name or request.user.username 20 | 21 | AuditLog.objects.create(user=user, type=action_type, action=action, 22 | action_ip=user_ip(request), 23 | content=f"{mask_sensitive_data(content)}\n请求方法：{request.method}，请求路径：{request.path}，UserAgent：{request.META['HTTP_USER_AGENT']}", 24 | data=mask_sensitive_data(data), 25 | old_data=mask_sensitive_data(old_data)) 26 | -------------------------------------------------------------------------------- /common/extends/models.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/12/31 下午3:29 7 | @FileName: models.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | 11 | from django.db import models 12 | 13 | 14 | class CommonParent(models.Model): 15 | parent = models.ForeignKey("self", null=True, blank=True, on_delete=models.SET_NULL, related_name='children') 16 | 17 | class Meta: 18 | abstract = True 19 | 20 | 21 | class TimeAbstract(models.Model): 22 | update_time = models.DateTimeField(auto_now=True, null=True, blank=True, verbose_name='更新时间') 23 | created_time = models.DateTimeField(auto_now_add=True, null=True, blank=True, verbose_name='创建时间') 24 | 25 | class ExtMeta: 26 | related = False 27 | dashboard = False 28 | 29 | class Meta: 30 | abstract = True 31 | ordering = ['-id'] 32 | 33 | 34 | class CreateTimeAbstract(models.Model): 35 | created_time = models.DateTimeField(auto_now_add=True, null=True, blank=True, verbose_name='创建时间') 36 | 37 | class ExtMeta: 38 | related = False 39 | dashboard = False 40 | 41 | class Meta: 42 | abstract = True 43 | 44 | 45 | class JobManager(models.Manager): 46 | def __init__(self, defer_fields=None): 47 | self.defer_fields = defer_fields 48 | super().__init__() 49 | 50 | def get_queryset(self): 51 | if self.defer_fields: 52 | return super().get_queryset().defer(*self.defer_fields) 53 | return super().get_queryset() 54 | -------------------------------------------------------------------------------- /common/extends/pagination.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/9/15 下午5:19 7 | @FileName: pagination.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | 11 | from rest_framework.pagination import PageNumberPagination, LimitOffsetPagination 12 | from rest_framework.response import Response 13 | 14 | 15 | class CustomPagination(PageNumberPagination): 16 | def get_paginated_response(self, data): 17 | return Response({ 18 | 'data': {'items': data, 'total': self.page.paginator.count}, 19 | 'code': 20000, 20 | 'next': self.get_next_link(), 21 | 'previous': self.get_previous_link() 22 | }) 23 | -------------------------------------------------------------------------------- /common/extends/permissions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/9/15 下午5:01 7 | @FileName: permissions.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | 11 | from rest_framework.permissions import BasePermission 12 | from dbapp.models import AuditLog 13 | 14 | from common.get_ip import user_ip 15 | from common.ext_fun import get_redis_data, get_members 16 | import logging 17 | 18 | logger = logging.getLogger('api') 19 | 20 | 21 | class RbacPermission(BasePermission): 22 | """ 23 | 自定义权限 24 | """ 25 | 26 | @classmethod 27 | def check_is_admin(cls, request): 28 | return request.user.is_authenticated and request.user.roles.filter(name='管理员').count() > 0 29 | 30 | @classmethod 31 | def get_permission_from_role(cls, request): 32 | try: 33 | perms = request.user.roles.values( 34 | 'permissions__method', 35 | ).distinct() 36 | return [p['permissions__method'] for p in perms] 37 | except AttributeError: 38 | return [] 39 | 40 | def _has_permission(self, request, view): 41 | """ 42 | :return: 43 | """ 44 | _method = request._request.method.lower() 45 | platform = get_redis_data('platform') 46 | url_whitelist = platform['whitelist'] if platform else [] 47 | url_whitelist.extend( 48 | [{'url': '/api/login/feishu/'}, {'url': '/api/login/gitlab/'}]) 49 | path_info = request.path_info 50 | for item in url_whitelist: 51 | url = item['url'] 52 | if url in path_info: 53 | logger.debug(f'请求地址 {path_info} 命中白名单 {url}， 放行') 54 | return True 55 | 56 | from_workflow = 'from_workflow' in request.GET 57 | if _method == 'get' and from_workflow: 58 | return True 59 | 60 | is_superuser = request.user.is_superuser 61 | if is_superuser: 62 | return True 63 | 64 | is_admin = RbacPermission.check_is_admin(request) 65 | perms = self.get_permission_from_role(request) 66 | if not is_admin and not perms: 67 | logger.debug(f'用户 {request.user} 不是管理员 且 权限列表为空， 直接拒绝') 68 | return False 69 | 70 | perms_map = view.perms_map 71 | 72 | action = view.action 73 | _custom_method = f'{_method}_{action}' 74 | for i in perms_map: 75 | for method, alias in i.items(): 76 | if is_admin and (method == '*' and alias[0] == 'admin'): 77 | return True 78 | if method == '*' and alias[0] in perms: 79 | return True 80 | if _custom_method and alias[0] in perms and (_custom_method == method or method == f'*_{action}'): 81 | return True 82 | if _method == method and alias[0] in perms: 83 | return True 84 | return False 85 | 86 | def has_permission(self, request, view): 87 | res = self._has_permission(request, view) 88 | # 记录权限异常的操作 89 | if not res: 90 | AuditLog.objects.create( 91 | user=request.user, type='', action='拒绝操作', 92 | action_ip=user_ip(request), 93 | content=f"请求方法：{request.method}，请求路径：{request.path}，UserAgent：{request.META['HTTP_USER_AGENT']}", 94 | data='', 95 | old_data='' 96 | ) 97 | return res 98 | 99 | 100 | class AdminPermission(BasePermission): 101 | 102 | def has_permission(self, request, view): 103 | if RbacPermission.check_is_admin(request): 104 | return True 105 | return False 106 | 107 | 108 | class ObjPermission(BasePermission): 109 | """ 110 | 密码管理对象级权限控制 111 | """ 112 | 113 | def has_object_permission(self, request, view, obj): 114 | perms = RbacPermission.get_permission_from_role(request) 115 | if 'admin' in perms: 116 | return True 117 | elif request.user.id == obj.uid_id: 118 | return True 119 | 120 | 121 | class AppPermission(BasePermission): 122 | def has_object_permission(self, request, view, obj): 123 | return True 124 | 125 | 126 | class AppInfoPermission(BasePermission): 127 | def has_object_permission(self, request, view, obj): 128 | return True 129 | 130 | 131 | class AppDeployPermission(BasePermission): 132 | def has_object_permission(self, request, view, obj): 133 | return True 134 | -------------------------------------------------------------------------------- /common/extends/q_redis_broker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : q_redis_broker.py 6 | @time : 2023/09/11 11:15 7 | @contact : qqing_lai@hotmail.com 8 | ''' 9 | 10 | # here put the import lib 11 | from django_q.brokers.redis_broker import Redis as BaseRedis 12 | from django_q.conf import Conf 13 | 14 | 15 | # bug: site-packages/django_q/brokers/redis_broker.py", line 44, in info 16 | # self._info = f"Redis {info['redis_version']}" 17 | # KeyError: 'redis_version' 18 | 19 | 20 | class Redis(BaseRedis): 21 | def __init__(self, list_key: str = Conf.PREFIX): 22 | super().__init__(list_key) 23 | 24 | def info(self) -> str: 25 | if not self._info: 26 | info = self.connection.info("server") 27 | try: 28 | # 尝试获取版本号 29 | self._info = f"Redis {info['redis_version']}" 30 | except BaseException as e: 31 | self._info = list(info.values())[0].get('redis_version', 'unknown') 32 | return self._info 33 | -------------------------------------------------------------------------------- /common/extends/renderers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/9/15 下午4:57 7 | @FileName: renderers.py 8 | @Blog ：https://imaojia.com 9 | @Desc: from https://stackoverflow.com/questions/53910545/how-overwrite-response-class-in-django-rest-framework-drf 10 | """ 11 | 12 | from rest_framework.renderers import BaseRenderer 13 | from rest_framework.utils import json 14 | 15 | 16 | class ApiRenderer(BaseRenderer): 17 | 18 | def render(self, data, accepted_media_type=None, renderer_context=None): 19 | response_dict = { 20 | 'status': 'failure', 21 | 'data': {}, 22 | 'message': '', 23 | } 24 | if data.get('data'): 25 | response_dict['data'] = data.get('data') 26 | if data.get('status'): 27 | response_dict['status'] = data.get('status') 28 | if data.get('message'): 29 | response_dict['message'] = data.get('message') 30 | data = response_dict 31 | return json.dumps(data) 32 | -------------------------------------------------------------------------------- /common/extends/serializers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2021/08/03 10:42 7 | @FileName: serializers.py 8 | @Blog : https://imaojia.com 9 | ''' 10 | from elasticsearch_dsl import Document 11 | from elasticsearch_dsl.response import Hit 12 | from rest_framework.serializers import Field, ModelSerializer as BaseModelSerializer, Serializer 13 | from rest_framework.fields import SkipField 14 | from rest_framework.relations import PKOnlyObject 15 | from django.utils.translation import ugettext_lazy as _ 16 | from collections import OrderedDict 17 | 18 | 19 | class ModelSerializer(BaseModelSerializer): 20 | 21 | def to_representation(self, instance): 22 | """ 23 | Object instance -> Dict of primitive datatypes. 24 | """ 25 | ret = OrderedDict() 26 | fields = self._readable_fields 27 | 28 | for field in fields: 29 | try: 30 | attribute = field.get_attribute(instance) 31 | except SkipField: 32 | continue 33 | 34 | # We skip `to_representation` for `None` values so that fields do 35 | # not have to explicitly deal with that case. 36 | # 37 | # For related fields with `use_pk_only_optimization` we need to 38 | # resolve the pk value. 39 | check_for_none = attribute.pk if isinstance( 40 | attribute, PKOnlyObject) else attribute 41 | if check_for_none is None: 42 | ret[field.field_name] = None 43 | else: 44 | if field.field_name == 'name': 45 | try: 46 | ret[field.field_name] = field.to_representation( 47 | attribute).lower() 48 | except: 49 | ret[field.field_name] = field.to_representation( 50 | attribute) 51 | else: 52 | ret[field.field_name] = field.to_representation(attribute) 53 | return ret 54 | 55 | 56 | class EsSerializer(BaseModelSerializer): 57 | """ 58 | ElasticSearch索引文档序列化 59 | """ 60 | 61 | def to_representation(self, instance): 62 | if isinstance(instance, (Document, Hit,)): 63 | return instance.to_dict() 64 | return super().to_representation(instance) 65 | 66 | 67 | class BooleanField(Field): 68 | default_error_messages = { 69 | 'invalid': _('"{input}" is not a valid boolean.') 70 | } 71 | initial = None 72 | TRUE_VALUES = { 73 | 't', 'T', 74 | 'y', 'Y', 'yes', 'YES', 75 | 'true', 'True', 'TRUE', 76 | 'on', 'On', 'ON', 77 | '1', 1, 78 | True 79 | } 80 | FALSE_VALUES = { 81 | 'f', 'F', 82 | 'n', 'N', 'no', 'NO', 83 | 'false', 'False', 'FALSE', 84 | 'off', 'Off', 'OFF', 85 | '0', 0, 0.0, 86 | False 87 | } 88 | NULL_VALUES = {'n', 'N', 'null', 'Null', 'NULL', '', None} 89 | 90 | def __init__(self, **kwargs): 91 | super(BooleanField, self).__init__(**kwargs) 92 | 93 | def to_internal_value(self, data): 94 | try: 95 | if data in self.TRUE_VALUES: 96 | return True 97 | elif data in self.FALSE_VALUES: 98 | return False 99 | elif data in self.NULL_VALUES: 100 | return None 101 | except TypeError: # Input is an unhashable type 102 | pass 103 | self.fail('invalid', input=data) 104 | 105 | def to_representation(self, value): 106 | if value in self.NULL_VALUES: 107 | return None 108 | if value in self.TRUE_VALUES: 109 | return True 110 | elif value in self.FALSE_VALUES: 111 | return False 112 | return bool(value) 113 | 114 | 115 | class UnusefulSerializer(Serializer): 116 | """无用的序列化类""" 117 | pass 118 | -------------------------------------------------------------------------------- /common/extends/storage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2021/12/27 12:44 7 | @FileName: storage.py 8 | @Blog : https://imaojia.com 9 | ''' 10 | 11 | from typing import Optional 12 | from django.core.files.storage import FileSystemStorage, Storage, DefaultStorage 13 | from django.utils._os import safe_join 14 | 15 | 16 | class FileUploadStorage(FileSystemStorage): 17 | """ 18 | 上传存储类 19 | """ 20 | 21 | def __init__(self, location=None, base_url=None, file_permissions_mode=None, 22 | directory_permissions_mode=None, upload_root=None): 23 | self.upload_root = upload_root 24 | super().__init__(location=location, base_url=base_url, file_permissions_mode=file_permissions_mode, 25 | directory_permissions_mode=directory_permissions_mode) 26 | 27 | def path(self, name: str) -> str: 28 | if self.upload_root: 29 | return safe_join(self.upload_root, name) 30 | return safe_join(self.location, name) 31 | -------------------------------------------------------------------------------- /common/get_ip.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/12/31 下午3:38 7 | @FileName: get_ip.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | 11 | 12 | def user_ip(request): 13 | """ 14 | 获取用户真实IP 15 | :param request: 16 | :return: 17 | """ 18 | if 'X-Real-IP' in request.META: 19 | return request.META['X-Real-IP'] 20 | if 'HTTP_X_FORWARDED_FOR' in request.META: 21 | return request.META['HTTP_X_FORWARDED_FOR'].split(',')[0] 22 | if 'REMOTE_ADDR' in request.META: 23 | return request.META['REMOTE_ADDR'].split(',')[0] 24 | -------------------------------------------------------------------------------- /common/kubernetes_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : kubernetes_utils.py 6 | @time : 2022/10/14 08:55 7 | @contact : qqing_lai@hotmail.com 8 | @company : IMAOJIA Co,Ltd 9 | ''' 10 | 11 | # here put the import lib 12 | import json 13 | import time 14 | import logging 15 | 16 | from dbapp.models import AppInfo, KubernetesCluster 17 | 18 | from common.ext_fun import get_datadict 19 | 20 | logger = logging.getLogger('drf') 21 | 22 | 23 | class DeploymentCheck(object): 24 | def __init__(self, cli, appinfo_obj: AppInfo, k8s: KubernetesCluster, tag=None, app_deploy_name=None): 25 | self.cli = cli 26 | self.appinfo_obj = appinfo_obj 27 | self.k8s = k8s 28 | self.tag = tag 29 | self.app_deploy_name = app_deploy_name 30 | self.check_police = get_datadict('DEPLOY_CHECK', 1) or { 31 | 'count': 30, 'interval': 6} 32 | self.count = self.check_police.get('count', 30) 33 | self.wait = self.check_police.get('interval', 6) 34 | self.namespace = self.appinfo_obj.namespace 35 | self.api_version = k8s.version.get('apiversion', 'apps/v1') 36 | 37 | def check_deployment(self, check_count): 38 | """ 39 | 检查 k8s deploy 状态 40 | :param check_count: 检查次数 41 | :return: 42 | """ 43 | check_count -= 1 44 | deployment = self.cli.fetch_deployment( 45 | self.app_deploy_name, self.namespace, self.api_version) 46 | if deployment.get('ecode', 200) > 399 or check_count < 0: 47 | check_desc = f"Kubernetes集群[{self.k8s.name}]: 应用{self.appinfo_obj.app.alias}[{self.app_deploy_name}]Deployment检测异常\n" 48 | return False, check_desc 49 | if all([ 50 | deployment['message']['metadata'].get('annotations', None), 51 | deployment['message']['spec'].get('selector', None) 52 | ]): 53 | if deployment['message']['metadata']['annotations'].get('deployment.kubernetes.io/revision', None): 54 | return True, deployment['message'] 55 | time.sleep(1) 56 | return self.check_deployment(check_count) 57 | 58 | def check_replica(self, deployment, check_count, pod_status=None): 59 | """ 60 | 检测 rs 和pods， 判断是否就绪 61 | :return: 62 | """ 63 | check_count -= 1 64 | if check_count < 0: 65 | # 如果超时了， 返回最后一次循环最后一个pod的状态信息 66 | check_desc = f"Kubernetes集群[{self.k8s.name}]: 应用{self.appinfo_obj.app.alias}[{self.app_deploy_name}] 未能在规定的时间内就绪，状态检测超时\n请查看Kubernetes Pod日志\n" 67 | return False, check_desc, pod_status 68 | labels = f"status-app-name-for-ops-platform={deployment['spec']['template']['metadata']['labels']['status-app-name-for-ops-platform']}" 69 | ret = self.cli.get_replica( 70 | self.namespace, self.api_version, **{"label_selector": labels}) 71 | if ret.get('ecode', 200) > 399: 72 | check_desc = f"Kubernetes集群[{self.k8s.name}]: 应用{self.appinfo_obj.app.alias}[{self.app_deploy_name}]访问rs信息异常\n" 73 | return False, check_desc, ret 74 | 75 | rs_message = ret['message'] 76 | if len(rs_message['items']) == 0 and check_count > 0: 77 | time.sleep(1) 78 | logger.debug('休眠1秒后继续查询replica') 79 | return self.check_replica(deployment, check_count) 80 | 81 | _key = 'deployment.kubernetes.io/revision' 82 | rs_list = [i for i in rs_message['items'] if 83 | i['metadata']['annotations'][_key] == deployment['metadata']['annotations'][_key]] 84 | if not rs_list: 85 | return self.check_replica(deployment, check_count) 86 | rs = rs_list[0] 87 | if self.tag: 88 | try: 89 | image = rs['spec']['template']['spec']['containers'][0]['image'] 90 | if image.split(':')[-1] != self.tag: 91 | logger.debug('当前镜像版本和部署版本不一致') 92 | return False, '部署状态检测结果: 当前运行版本和部署版本不一致，请查看Kubernetes Pod日志', { 93 | '当前运行版本': image, 94 | '部署版本': self.tag 95 | } 96 | except BaseException as e: 97 | logger.exception(f"运行版本和部署版本检测发生异常 {e.__class__} {e} ") 98 | 99 | rs_status = rs['status'] 100 | available_replicas = rs_status.get('availableReplicas', 0) 101 | fully_labeled_replicas = rs_status.get('fullyLabeledReplicas', 0) 102 | ready_replicas = rs_status.get('readyReplicas', 0) 103 | rs_ready_conditions = [ 104 | available_replicas, 105 | fully_labeled_replicas, 106 | ready_replicas, 107 | ] 108 | rs_labels = ','.join( 109 | [f'{k}={v}' for k, v in rs['spec']['selector']['matchLabels'].items()]) 110 | # 结合 rs 副本状态 和 pods 的状态， 只要 rs和其中一个pods完全就绪， 就算通过 111 | pods_ret = self.cli.get_pods( 112 | self.namespace, **{"label_selector": rs_labels}) 113 | pod_list = pods_ret['message']['items'] 114 | for pod in pod_list: 115 | pod_status = pod['status'] 116 | if all(rs_ready_conditions) and pod_status['phase'].lower() == 'running' and ( 117 | 'containerStatuses' in pod_status and 118 | pod_status['containerStatuses'][0]['ready'] is True and 119 | 'running' in pod_status['containerStatuses'][0]['state'] 120 | ): 121 | check_desc = f"{pod['metadata']['name']}\n 部署状态检测结果：当前Replica运行副本\n - availableReplicas: {rs_status.get('availableReplicas', 0)}\n - fullyLabeledReplicas: {rs_status.get('fullyLabeledReplicas', 0)}\n - readyReplicas: {rs_status.get('readyReplicas', 0)}\n" 122 | return True, check_desc, pod_status 123 | 124 | return self.check_replica(deployment, check_count, pod_status) 125 | 126 | def run(self): 127 | is_ok, deployment = self.check_deployment(5) 128 | if not is_ok: 129 | return {'status': 2, 'message': deployment, 'data': deployment} 130 | 131 | desc = '' 132 | log = {} 133 | while True: 134 | self.count -= 1 135 | is_ok, desc, log = self.check_replica(deployment, 10) 136 | if is_ok: 137 | logger.info( 138 | f"Kubernetes集群[{self.k8s.name}]: 应用{self.appinfo_obj.app.alias}[{self.appinfo_obj.app.name}]检测成功\n") 139 | return {'status': 1, 'message': desc, 'data': json.dumps(log)} 140 | if self.count < 0: 141 | break 142 | time.sleep(self.wait) 143 | return {'status': 2, 'message': desc, 'data': json.dumps(log)} 144 | 145 | 146 | def deployment_check(cli, appinfo_obj: AppInfo, k8s: KubernetesCluster, tag=None, app_deploy_name=None): 147 | if not app_deploy_name: 148 | app_deploy_name = appinfo_obj.app.name 149 | dc = DeploymentCheck(cli, appinfo_obj, k8s, tag=tag, 150 | app_deploy_name=app_deploy_name) 151 | return dc.run() 152 | -------------------------------------------------------------------------------- /common/md5.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/11/27 下午2:39 7 | @FileName: md5.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | 11 | from functools import partial 12 | import hashlib 13 | 14 | 15 | def md5(data, block_size=65536): 16 | # 创建md5对象 17 | m = hashlib.md5() 18 | # 对django中的文件对象进行迭代 19 | for item in iter(partial(data.read, block_size), b''): 20 | # 把迭代后的bytes加入到md5对象中 21 | m.update(item) 22 | 23 | return m.hexdigest() 24 | -------------------------------------------------------------------------------- /common/recursive.py: -------------------------------------------------------------------------------- 1 | # clone from https://github.com/heywbj/django-rest-framework-recursive 2 | 3 | import inspect 4 | import importlib 5 | from rest_framework.fields import Field 6 | from rest_framework.serializers import BaseSerializer 7 | 8 | 9 | def _signature_parameters(func): 10 | try: 11 | inspect.signature 12 | except AttributeError: 13 | # Python 2.x 14 | return inspect.getargspec(func).args 15 | else: 16 | # Python 3.x 17 | return inspect.signature(func).parameters.keys() 18 | 19 | 20 | class RecursiveField(Field): 21 | """ 22 | A field that gets its representation from its parent. 23 | 24 | This method could be used to serialize a tree structure, a linked list, or 25 | even a directed acyclic graph. As with all recursive things, it is 26 | important to keep the base case in mind. In the case of the tree serializer 27 | example below, the base case is a node with an empty list of children. In 28 | the case of the list serializer below, the base case is when `next==None`. 29 | Above all, beware of cyclical references. 30 | 31 | Examples: 32 | 33 | class TreeSerializer(self): 34 | children = ListField(child=RecursiveField()) 35 | 36 | class ListSerializer(self): 37 | next = RecursiveField(allow_null=True) 38 | """ 39 | 40 | # This list of attributes determined by the attributes that 41 | # `rest_framework.serializers` calls to on a field object 42 | PROXIED_ATTRS = ( 43 | # methods 44 | 'get_value', 45 | 'get_initial', 46 | 'run_validation', 47 | 'get_attribute', 48 | 'to_representation', 49 | 50 | # attributes 51 | 'field_name', 52 | 'source', 53 | 'read_only', 54 | 'default', 55 | 'source_attrs', 56 | 'write_only', 57 | ) 58 | 59 | def __init__(self, to=None, **kwargs): 60 | """ 61 | arguments: 62 | to - `None`, the name of another serializer defined in the same module 63 | as this serializer, or the fully qualified import path to another 64 | serializer. e.g. `ExampleSerializer` or 65 | `path.to.module.ExampleSerializer` 66 | """ 67 | self.to = to 68 | self.init_kwargs = kwargs 69 | self._proxied = None 70 | 71 | # need to call super-constructor to support ModelSerializer 72 | super_kwargs = dict( 73 | (key, kwargs[key]) 74 | for key in kwargs 75 | if key in _signature_parameters(Field.__init__) 76 | ) 77 | super(RecursiveField, self).__init__(**super_kwargs) 78 | 79 | def bind(self, field_name, parent): 80 | # Extra-lazy binding, because when we are nested in a ListField, the 81 | # RecursiveField will be bound before the ListField is bound 82 | self.bind_args = (field_name, parent) 83 | 84 | @property 85 | def proxied(self): 86 | if not self._proxied: 87 | if self.bind_args: 88 | field_name, parent = self.bind_args 89 | 90 | if hasattr(parent, 'child') and parent.child is self: 91 | # RecursiveField nested inside of a ListField 92 | parent_class = parent.parent.__class__ 93 | else: 94 | # RecursiveField directly inside a Serializer 95 | parent_class = parent.__class__ 96 | 97 | assert issubclass(parent_class, BaseSerializer) 98 | 99 | if self.to is None: 100 | proxied_class = parent_class 101 | else: 102 | try: 103 | module_name, class_name = self.to.rsplit('.', 1) 104 | except ValueError: 105 | module_name, class_name = parent_class.__module__, self.to 106 | 107 | try: 108 | proxied_class = getattr( 109 | importlib.import_module(module_name), class_name) 110 | except Exception as e: 111 | raise ImportError( 112 | 'could not locate serializer %s' % self.to, e) 113 | 114 | # Create a new serializer instance and proxy it 115 | proxied = proxied_class(**self.init_kwargs) 116 | proxied.bind(field_name, parent) 117 | self._proxied = proxied 118 | 119 | return self._proxied 120 | 121 | def __getattribute__(self, name): 122 | if name in RecursiveField.PROXIED_ATTRS: 123 | try: 124 | proxied = object.__getattribute__(self, 'proxied') 125 | return getattr(proxied, name) 126 | except AttributeError: 127 | pass 128 | 129 | return object.__getattribute__(self, name) 130 | -------------------------------------------------------------------------------- /common/timer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: UTF-8 -*- 2 | """ 3 | @author: hhyo 4 | @license: Apache Licence 5 | @file: timer.py 6 | @time: 2019/05/15 7 | """ 8 | import datetime 9 | 10 | __author__ = "hhyo" 11 | 12 | 13 | class FuncTimer(object): 14 | """ 15 | 获取执行时间的上下文管理器 16 | """ 17 | 18 | def __init__(self): 19 | self.start = None 20 | self.end = None 21 | self.cost = 0 22 | 23 | def __enter__(self): 24 | self.start = datetime.datetime.now() 25 | return self 26 | 27 | def __exit__(self, exc_type, exc_val, exc_tb): 28 | self.end = datetime.datetime.now() 29 | self.cost = (self.end - self.start).total_seconds() 30 | -------------------------------------------------------------------------------- /common/utils/AesCipher.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2021/1/20 下午2:24 7 | @FileName: AesCipher.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | 11 | import base64 12 | from Crypto.Cipher import AES 13 | 14 | 15 | class AesCipher(object): 16 | def __init__(self, secret_key='Devops SecretKey'): 17 | self.__secret_key = secret_key 18 | self.__aes = AES.new(str.encode(self.__secret_key), AES.MODE_ECB) 19 | 20 | def encrypt(self, data): 21 | while len(data) % 16 != 0: # 补足字符串长度为16的倍数 22 | data += (16 - len(data) % 16) * chr(16 - len(data) % 16) 23 | cipher_data = str(base64.encodebytes(self.__aes.encrypt(str.encode(data))), encoding='utf8').replace('\n', '') 24 | return cipher_data 25 | 26 | def decrypt(self, cipher_data): 27 | try: 28 | decrypted_text = self.__aes.decrypt(base64.decodebytes(bytes(cipher_data, encoding='utf8'))).decode("utf8") 29 | decrypted_text = decrypted_text[:-ord(decrypted_text[-1])] # 去除多余补位 30 | return decrypted_text 31 | except BaseException as e: 32 | print('data', e) 33 | raise Exception(e) 34 | -------------------------------------------------------------------------------- /common/utils/AnsibleCallback.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/12/1 下午4:05 7 | @FileName: AnsibleCallback.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | 11 | from __future__ import unicode_literals 12 | 13 | from common.utils.AnsibleAPI import AnsibleApi as BaseAnsibleApi 14 | from common.utils.AnsibleAPI import PlayBookResultsCollector as BasePlayBookResultsCollector 15 | import json 16 | import logging 17 | 18 | logger = logging.getLogger(__name__) 19 | 20 | 21 | class PlayBookResultsCollector(BasePlayBookResultsCollector): 22 | 23 | def __init__(self, redis_conn, chan, jid, channel, *args, debug=False, on_any_callback=None, **kwargs): 24 | super(PlayBookResultsCollector, self).__init__(*args, **kwargs) 25 | self.channel = channel 26 | self.jid = jid 27 | self.chan = chan 28 | self.redis_conn = redis_conn 29 | self.debug = debug 30 | self.on_any_callback = on_any_callback 31 | 32 | @staticmethod 33 | def result_pop(result): 34 | try: 35 | result._result['container'].pop('ResolvConfPath', None) 36 | result._result['container'].pop('HostnamePath', None) 37 | result._result['container'].pop('HostsPath', None) 38 | result._result['container'].pop('Platform', None) 39 | result._result['container'].pop('HostConfig', None) 40 | result._result['container'].pop('GraphDriver', None) 41 | result._result['container'].pop('NetworkSettings', None) 42 | except: 43 | pass 44 | if 'stdout' in result._result and 'stdout_lines' in result._result: 45 | result._result['stdout_lines'] = '' 46 | if 'results' in result._result: 47 | for i in result._result['results']: 48 | if 'stdout' in i and 'stdout_lines' in i: 49 | i['stdout_lines'] = '' 50 | return result 51 | 52 | def v2_runner_on_ok(self, result, *args, **kwargs): 53 | 'ansible_facts' in result._result and result._result.pop( 54 | 'ansible_facts') 55 | 'invocation' in result._result and result._result.pop('invocation') 56 | result = self.result_pop(result) 57 | res = { 58 | 'status': 'success', 59 | 'step_status': 'success', 60 | 'host': result._host.get_name(), 61 | 'task': result._task.get_name(), 62 | 'msg': result._result 63 | } 64 | 65 | self.redis_conn.rpush(self.jid, json.dumps( 66 | {res['task']: {res['host']: res}})) 67 | self.redis_conn.rpush('%s:%s:status' % (self.jid, res['task']), 0) 68 | self.task_ok[result._host.get_name()] = result 69 | logger.debug(f'v2_runner_on_ok ======== {self.task_ok}') 70 | 71 | def v2_runner_on_failed(self, result, *args, **kwargs): 72 | 'ansible_facts' in result._result and result._result.pop( 73 | 'ansible_facts') 74 | 'invocation' in result._result and result._result.pop('invocation') 75 | result = self.result_pop(result) 76 | res = { 77 | 'status': 'failed', 78 | 'step_status': 'error', 79 | 'host': result._host.get_name(), 80 | 'task': result._task.get_name(), 81 | 'msg': result._result 82 | } 83 | self.redis_conn.rpush(self.jid, json.dumps( 84 | {res['task']: {res['host']: res}})) 85 | self.redis_conn.rpush('%s:%s:status' % (self.jid, res['task']), 1) 86 | self.task_failed[result._host.get_name()] = result 87 | logger.debug(f'v2_runner_on_failed ======== {self.task_failed}') 88 | 89 | def v2_runner_on_unreachable(self, result): 90 | 'ansible_facts' in result._result and result._result.pop( 91 | 'ansible_facts') 92 | 'invocation' in result._result and result._result.pop('invocation') 93 | result = self.result_pop(result) 94 | res = { 95 | 'status': 'unreachable', 96 | 'step_status': 'error', 97 | 'host': result._host.get_name(), 98 | 'task': result._task.get_name(), 99 | 'msg': result._result 100 | } 101 | self.redis_conn.rpush(self.jid, json.dumps( 102 | {res['task']: {res['host']: res}})) 103 | self.redis_conn.rpush('%s:%s:status' % (self.jid, res['task']), 1) 104 | self.task_unreachable[result._host.get_name()] = result 105 | logger.debug( 106 | f'v2_runner_on_unreachable ======== {self.task_unreachable}') 107 | 108 | def v2_runner_on_skipped(self, result): 109 | 'ansible_facts' in result._result and result._result.pop( 110 | 'ansible_facts') 111 | 'invocation' in result._result and result._result.pop('invocation') 112 | result = self.result_pop(result) 113 | res = { 114 | 'status': 'skipped', 115 | 'step_status': 'finish', 116 | 'host': result._host.get_name(), 117 | 'task': result._task.get_name(), 118 | 'msg': result._result 119 | } 120 | self.redis_conn.rpush(self.jid, json.dumps( 121 | {res['task']: {res['host']: res}})) 122 | self.redis_conn.rpush('%s:%s:status' % (self.jid, res['task']), 0) 123 | self.task_skipped[result._host.get_name()] = result 124 | logger.debug(f'v2_runner_on_skipped ======== {self.task_skipped}') 125 | 126 | def v2_runner_on_changed(self, result): 127 | 'ansible_facts' in result._result and result._result.pop( 128 | 'ansible_facts') 129 | 'invocation' in result._result and result._result.pop('invocation') 130 | result = self.result_pop(result) 131 | res = { 132 | 'status': 'onchanged', 133 | 'step_status': 'finish', 134 | 'host': result._host.get_name(), 135 | 'task': result._task.get_name(), 136 | 'msg': result._result 137 | } 138 | self.redis_conn.rpush(self.jid, json.dumps( 139 | {res['task']: {res['host']: res}})) 140 | self.redis_conn.rpush('%s:%s:status' % (self.jid, res['task']), 0) 141 | self.task_changed[result._host.get_name()] = result 142 | logger.debug(f'v2_runner_on_changed ======== {self.task_changed}') 143 | 144 | def v2_playbook_on_no_hosts_matched(self): 145 | res = { 146 | 'task': '查找主机', 147 | 'status': 'unreachable', 148 | 'step_status': 'error', 149 | 'host': 'unmatched', 150 | 'msg': {'result': 'Could not match supplied host'} 151 | } 152 | self.redis_conn.rpush(self.jid, json.dumps( 153 | {res['task']: {res['host']: res}})) 154 | self.redis_conn.rpush('%s:%s:status' % (self.jid, res['task']), 0) 155 | 156 | def v2_on_any(self, result, *args, **kwargs): 157 | if self.on_any_callback: 158 | self.on_any_callback(self, result, *args, **kwargs) 159 | 160 | 161 | class AnsibleApi(BaseAnsibleApi): 162 | def __init__(self, redis_conn, chan, jid, channel, *args, on_any_callback=None, **kwargs): 163 | super(AnsibleApi, self).__init__(*args, **kwargs) 164 | 165 | self.playbook_callback = PlayBookResultsCollector(redis_conn, chan, jid, channel, 166 | on_any_callback=on_any_callback) 167 | self.channel = channel 168 | self.redis_conn = redis_conn 169 | self.jid = jid 170 | -------------------------------------------------------------------------------- /common/utils/AtlassianJiraAPI.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : AtlassianJiraAPI.py 6 | @time : 2023/04/14 10:41 7 | @contact : qqing_lai@hotmail.com 8 | ''' 9 | 10 | # here put the import lib 11 | from atlassian import Jira 12 | 13 | import logging 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | class JiraAPI(object): 19 | 20 | def __init__(self, url, user=None, password=None, token=None): 21 | self.__url = url 22 | self.__user = user 23 | self.__password = password 24 | self.__token = token 25 | 26 | if token: 27 | self.client = Jira(url=self.__url, token=self.__token) 28 | elif user and password: 29 | self.client = Jira( 30 | url=self.__url, username=self.__user, password=self.__password) 31 | else: 32 | raise Exception('未提供认证信息.') 33 | 34 | def list_issues(self, issue_text='', issue_key=None, project=None, include_status=None, exclude_status=None, max_results=20): 35 | """ 36 | 获取issues 37 | :params return: ['expand', 'startAt', 'maxResults', 'total', 'issues'] 38 | """ 39 | params = '' 40 | if not any([issue_text, issue_key, project]): 41 | return False, '缺少参数！' 42 | if issue_key: 43 | params = f'issueKey={issue_key}' 44 | if issue_text: 45 | params = f'text~{issue_text}' 46 | max_results = 50 47 | if project: 48 | params = f'project={project}' 49 | if include_status: 50 | # 存在包含的状态 51 | status = (',').join(include_status.split(',')) 52 | params = f'{params} and status in ({status})' 53 | elif exclude_status: 54 | # 没有配置包含的状态且存在要排除的状态 55 | status = (',').join(exclude_status.split(',')) 56 | params = f'{params} and status not in ({status})' 57 | try: 58 | issues = self.client.jql(params) 59 | return True, issues 60 | except BaseException as e: 61 | logger.debug(f'获取issue异常, {e.__dict__}, {e}') 62 | return False, str(e) 63 | 64 | def update_issue_status(self, issue_key, status): 65 | """ 66 | 更新issue状态 67 | """ 68 | try: 69 | result = self.client.set_issue_status( 70 | issue_key, status, fields=None) 71 | logger.debug(f'更新issue状态, result') 72 | return True, result 73 | except BaseException as e: 74 | logger.debug(f'更新issue状态异常，{e}') 75 | return False, str(e) 76 | -------------------------------------------------------------------------------- /common/utils/DocumentRegistry.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : DocumentRegistry.py 6 | @time : 2023/04/20 17:39 7 | @contact : qqing_lai@hotmail.com 8 | ''' 9 | 10 | # here put the import lib 11 | from django.core.exceptions import ImproperlyConfigured 12 | from elasticsearch_dsl import AttrDict 13 | 14 | from collections import defaultdict 15 | 16 | from common.variables import ES_MODEL_FIELD_MAP 17 | 18 | 19 | class DocumentRegistry(object): 20 | """ 21 | Registry of models classes to a set of Document classes. 22 | """ 23 | 24 | def __init__(self): 25 | self._models = defaultdict(set) 26 | 27 | def register_document(self, document): 28 | django_meta = getattr(document, 'Django') 29 | # Raise error if Django class can not be found 30 | if not django_meta: 31 | message = "You must declare the Django class inside {}".format( 32 | document.__name__) 33 | raise ImproperlyConfigured(message) 34 | 35 | # Keep all django related attribute in a django_attr AttrDict 36 | data = {'model': getattr(document.Django, 'model')} 37 | django_attr = AttrDict(data) 38 | 39 | if not django_attr.model: 40 | raise ImproperlyConfigured("You must specify the django model") 41 | 42 | # Add The model fields into elasticsearch mapping field 43 | model_field_names = getattr(document.Django, "fields", []) 44 | model_field_exclude = getattr(document.Django, "exclude", []) 45 | if model_field_names == '__all__': 46 | model_field_names = [ 47 | i.name for i in django_attr['model']._meta.fields if i.name not in model_field_exclude] 48 | model_field_remap = getattr(document.Django, "fields_remap", {}) 49 | for field_name in model_field_names: 50 | django_field = django_attr.model._meta.get_field(field_name) 51 | field_instance = ES_MODEL_FIELD_MAP[django_field.__class__] 52 | if field_name in model_field_remap: 53 | field_instance = model_field_remap[field_name] 54 | document._doc_type.mapping.field(field_name, field_instance) 55 | try: 56 | if not document._index.exists(): 57 | document.init() 58 | except: 59 | pass 60 | return document 61 | 62 | 63 | registry = DocumentRegistry() 64 | -------------------------------------------------------------------------------- /common/utils/DyInventory.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/12/8 上午10:26 7 | @FileName: DyInventory.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | 11 | import os 12 | import requests 13 | import argparse 14 | 15 | try: 16 | import json 17 | except ImportError: 18 | import simplejson as json 19 | 20 | 21 | class DyInventory(object): 22 | def __init__(self, url, username, password): 23 | self.__url = url 24 | self.__username = username 25 | self.__password = password 26 | self.__headers = {'Content-Type': 'application/json;charset=UTF-8'} 27 | 28 | self.inventory = {} 29 | self.read_cli_args() 30 | # self Called with `--list`. 31 | if self.args.list: 32 | self.inventory = self.get_hosts() 33 | print(json.dumps(self.inventory, indent=4)) 34 | elif self.args.host: 35 | # Not implemented, since we return _meta info `--list`. 36 | # self.inventory = self.empty_inventor() 37 | host = self.get_host_detail(self.args.host) 38 | print(json.dumps(host, indent=4)) 39 | # If no groups or vars are present, return empty inventory. 40 | else: 41 | self.inventory = self.empty_inventor() 42 | print(json.dumps(self.inventory, indent=4)) 43 | 44 | def get_token(self): 45 | url = f"{self.__url}/api/soms/user/login/" 46 | data = {'username': self.__username, 'password': self.__password} 47 | r = requests.post(url=url, json=data, headers=self.__headers) 48 | if r.status_code == 200: 49 | return r.json()['data']['access'] 50 | else: 51 | raise Exception('get token err') 52 | 53 | def get_hosts(self): 54 | app_id = os.environ['APP_ID'] 55 | env = os.environ['APP_ENV'] 56 | token = self.get_token() 57 | url = f"{self.__url}/api/soms/app/service/asset/?app_id={app_id}&environment={env}" 58 | self.__headers['Authorization'] = f"Bearer {token}" 59 | r = requests.get(url, headers=self.__headers) 60 | if r.status_code == 200: 61 | return r.json()['data'] 62 | else: 63 | raise Exception('get hosts err') 64 | 65 | # Empty inventory for testing. 66 | def empty_inventor(self): 67 | if not self.args.host: 68 | return {'_meta': {'hostvars': {}}} 69 | data = self.get_hosts() 70 | var = data.get('_meta', {}).get('hostvars', {}).get(self.args.host, '') 71 | return var or {} 72 | 73 | def get_host_detail(self, host): 74 | data = self.get_hosts()['_meta']['hostvars'][host] 75 | return {'ansible_ssh_host': data['hostname'], 'ansible_ssh_port': data['port'], 76 | 'ansible_ssh_user': data['username'], 'ansible_ssh_pass': data['password']} 77 | 78 | # Read the command line args passed to the script. 79 | def read_cli_args(self): 80 | parser = argparse.ArgumentParser() 81 | parser.add_argument('--list', action='store_true') 82 | parser.add_argument('--host', action='store') 83 | parser.add_argument('--app', action='store') 84 | parser.add_argument('--env', action='store') 85 | self.args = parser.parse_args() 86 | -------------------------------------------------------------------------------- /common/utils/ElasticSearchAPI.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2021/10/26 14:26 7 | @FileName: ElasticSearchAPI.py 8 | @Blog : https://imaojia.com 9 | ''' 10 | 11 | from elasticsearch_dsl.serializer import serializer 12 | from datetime import datetime 13 | from elasticsearch_dsl import Document, Date, Integer, Keyword, Text, connections 14 | 15 | from common.variables import ES_FIELD_MAP, ES_TYPE_MAP, my_normalizer, ES_MODEL_FIELD_MAP 16 | from elasticsearch_dsl import Object, Nested as EsNested 17 | 18 | from elasticsearch.exceptions import NotFoundError 19 | 20 | from elasticsearch.helpers import bulk 21 | from elasticsearch import Elasticsearch 22 | from elasticsearch.client import IndicesClient 23 | from datetime import datetime 24 | from elasticsearch_dsl import Search as BaseSearch, Index as BaseIndex, Document, Date, Integer, Keyword, Text, \ 25 | connections 26 | from elasticsearch_dsl import analyzer, tokenizer 27 | from six import iteritems, string_types 28 | 29 | 30 | from config import ELASTICSEARCH, ELASTICSEARCH_PREFIX 31 | 32 | 33 | class Mapping: 34 | 35 | @staticmethod 36 | def _generate_mapping(table): 37 | mapping = {} 38 | for field in table.fields.all(): 39 | if field.is_multi: 40 | mapping[field.name] = Keyword(multi=True) 41 | else: 42 | mapping[field.name] = ES_FIELD_MAP[field.type] 43 | if field.is_related: 44 | # 外键关联, 额外添加field_data字段 45 | mapping[f"{field.name}_data"] = EsNested() 46 | return mapping 47 | 48 | def generate_data_mapping(self, table): 49 | system_mapping = { 50 | "S-creator": ES_FIELD_MAP[0], 51 | "S-creation-time": ES_FIELD_MAP[6], 52 | "S-modified-time": ES_FIELD_MAP[6], 53 | "S-last-modified": ES_FIELD_MAP[0] 54 | } 55 | field_mapping = self._generate_mapping(table) 56 | return dict(**system_mapping, **field_mapping) 57 | 58 | def generate_history_data_mapping(self, table): 59 | system_mapping = { 60 | "S-data-id": ES_FIELD_MAP[0], 61 | "S-changer": ES_FIELD_MAP[0], 62 | "S-update-time": ES_FIELD_MAP[6] 63 | } 64 | field_mapping = self._generate_mapping(table) 65 | return dict(**system_mapping, **field_mapping) 66 | 67 | def generate_deleted_data_mapping(self, table): 68 | system_mapping = { 69 | "S-delete-people": ES_FIELD_MAP[0], 70 | "S-delete-time": ES_FIELD_MAP[6] 71 | } 72 | field_mapping = self._generate_mapping(table) 73 | return dict(**system_mapping, **field_mapping) 74 | 75 | 76 | class Search(BaseSearch): 77 | def __init__(self, prefix=False, **kwargs): 78 | if kwargs.get('index', None) and prefix: 79 | if isinstance(kwargs['index'], string_types): 80 | kwargs['index'] = f"{ELASTICSEARCH_PREFIX}{kwargs['index']}" 81 | elif isinstance(kwargs['index'], list): 82 | kwargs['index'] = [ 83 | f"{ELASTICSEARCH_PREFIX}{i}" for i in kwargs['index']] 84 | elif isinstance(kwargs['index'], tuple): 85 | kwargs['index'] = tuple( 86 | f"{ELASTICSEARCH_PREFIX}{i}" for i in kwargs['index']) 87 | else: 88 | raise Exception('索引名称格式错误!') 89 | super(Search, self).__init__(**kwargs) 90 | 91 | 92 | class Index(BaseIndex): 93 | 94 | def __init__(self, name, using="default"): 95 | name = f"{ELASTICSEARCH_PREFIX}{name}" 96 | super(Index, self).__init__(name, using=using) 97 | 98 | def rebuild_index(self, using=None, **kwargs): 99 | """ 100 | Creates the index in elasticsearch. 101 | 102 | Any additional keyword arguments will be passed to 103 | ``Elasticsearch.indices.create`` unchanged. 104 | """ 105 | return self._get_connection(using).reindex(body=kwargs) 106 | 107 | 108 | class CustomDocument(Document): 109 | 110 | @staticmethod 111 | def gen_data(data, index, pk=None): 112 | for i in data: 113 | i['_index'] = index 114 | if pk: 115 | i['instanceid'] = i[pk] 116 | i['_id'] = i[pk] 117 | yield i 118 | 119 | @classmethod 120 | def bulk_save(cls, data, using=None, index=None, pk=None, validate=True, skip_empty=True, return_doc_meta=False, 121 | **kwargs): 122 | """ 123 | 批量创建 124 | 125 | : param data: [{'instanceid': 's3', 'inner_ip': '1.1.1.3'},{'instanceid': 's4', 'inner_ip': '1.1.1.4'}] 126 | """ 127 | es = cls._get_connection(cls._get_using(using)) 128 | data = cls.gen_data(data, cls._default_index(index), pk) 129 | return bulk(es, data) 130 | 131 | 132 | def generate_docu(table, index_version=None): 133 | index_name = f"{table.name}-{index_version}" if index_version else table.name 134 | _tbindex = Index(index_name) 135 | _tbindex.analyzer(my_normalizer) 136 | _tbindex.settings(number_of_shards=3, number_of_replicas=1) 137 | _fields = Mapping().generate_data_mapping(table) 138 | docu = type(index_name, (CustomDocument,), _fields) 139 | return _tbindex.document(docu) 140 | 141 | 142 | def generate_history_docu(table): 143 | _tbindex = Index(table.name + '_history') 144 | _tbindex.settings(number_of_shards=3, number_of_replicas=1) 145 | _fields = Mapping().generate_history_data_mapping(table) 146 | docu = type(table.name + '_history', (Document,), _fields) 147 | return _tbindex.document(docu) 148 | -------------------------------------------------------------------------------- /common/utils/HarborAPI.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2021/07/14 18:14 7 | @FileName: HarborAPI.py 8 | @Blog : https://imaojia.com 9 | ''' 10 | 11 | from __future__ import unicode_literals 12 | 13 | import base64 14 | import ssl 15 | 16 | import requests 17 | 18 | import logging 19 | 20 | logger = logging.getLogger('drf') 21 | 22 | ssl._create_default_https_context = ssl._create_unverified_context 23 | 24 | 25 | class HarborAPI(object): 26 | def __init__(self, url, username, password): 27 | self.__url = url.rstrip('/') 28 | self.__user = username 29 | self.__password = password 30 | self.__token = base64.b64encode( 31 | bytes('%s:%s' % (self.__user, self.__password), encoding='utf-8')) 32 | self.__headers = dict() 33 | self.__headers["Accept"] = "application/json" 34 | self.__headers['authorization'] = 'Basic %s' % str( 35 | self.__token, encoding='utf-8') 36 | 37 | def request(self, method, obj=None, prefix='/'): 38 | try: 39 | if method == 'get': 40 | req = requests.request(method, '%s%s' % (self.__url, prefix), params=obj, headers=self.__headers, 41 | verify=False) 42 | if req.status_code > 399: 43 | return {'ecode': req.status_code, 'message': f'{req.content}\n{req.reason}'} 44 | res = {'ecode': req.status_code, 'data': req.json(), 'count': req.headers.get('X-Total-Count', None), 45 | 'next': req.headers.get('Link', None)} 46 | if method == 'delete': 47 | req = requests.request(method, '%s%s' % ( 48 | self.__url, prefix), headers=self.__headers, verify=False) 49 | if req.status_code > 399: 50 | return {'ecode': req.status_code, 'message': f'{req.content}\n{req.reason}'} 51 | res = {'ecode': req.status_code, 'data': req.content} 52 | if method in ['put', 'post']: 53 | req = requests.request(method, '%s%s' % (self.__url, prefix), json=obj, headers=self.__headers, 54 | verify=False) 55 | if req.status_code > 399: 56 | return {'ecode': req.status_code, 'message': f'{req.content}\n{req.reason}'} 57 | res = {'ecode': req.status_code, 'data': req.content} 58 | if method == 'head': 59 | req = requests.request(method, '%s%s' % ( 60 | self.__url, prefix), headers=self.__headers, verify=False) 61 | if req.status_code > 399: 62 | return {'ecode': req.status_code, 'message': f'{req.content}\n{req.reason}'} 63 | res = {'ecode': req.status_code, 'data': req.content} 64 | except BaseException as e: 65 | raise e 66 | return res 67 | 68 | def systeminfo(self): 69 | res = self.request('get', prefix='/systeminfo') 70 | return res 71 | 72 | def get_users(self): 73 | res = self.request('get', prefix='/users') 74 | return res 75 | 76 | def get_projects(self, project_name=None, page=1, page_size=20): 77 | """ 78 | :project_name: The name of project 79 | :page: default is 1. 80 | :page_size: default is 10, maximum is 100. 81 | """ 82 | params = {'page': page, 'page_size': page_size} 83 | if project_name: 84 | params['name'] = project_name 85 | try: 86 | res = self.request('get', params, prefix='/projects') 87 | return res 88 | except BaseException as e: 89 | return {'ecode': 500, 'message': e} 90 | 91 | def get_repositories(self, project_id, page=1, page_size=20, repo=None): 92 | params = {'project_id': project_id, 93 | 'page': page, 'page_size': page_size} 94 | if repo: 95 | params['q'] = repo 96 | try: 97 | res = self.request('get', params, '/repositories') 98 | return res 99 | except BaseException as e: 100 | return {'ecode': 500, 'message': e} 101 | 102 | def get_tags(self, repo): 103 | try: 104 | res = self.request('get', prefix='/repositories/%s/tags' % repo) 105 | tags = [ 106 | {'name': i['name'], 'created': i['created'], 'push_time': i.get( 107 | 'push_time', None), 'size': i['size']} 108 | for i in 109 | res['data']] 110 | tags.sort(key=lambda k: (k.get('created')), reverse=True) 111 | return {'ecode': 200, 'data': tags, 'count': len(tags)} 112 | except BaseException as e: 113 | return {'ecode': 500, 'message': e} 114 | 115 | def fetch_project(self, project_id): 116 | """ 117 | 获取项目信息 118 | """ 119 | try: 120 | res = self.request( 121 | 'get', {'project_id': project_id}, prefix=f'/projects/{project_id}') 122 | return res 123 | except BaseException as e: 124 | return {'ecode': 500, 'message': e} 125 | 126 | def fetch_tag(self, repo, tag): 127 | """ 128 | 获取指定镜像标签 129 | """ 130 | try: 131 | res = self.request( 132 | 'get', prefix=f'/repositories/{repo}/tags/{tag}') 133 | return res 134 | except BaseException as e: 135 | return {'ecode': 500, 'message': e} 136 | 137 | def create_project(self, project_name, public=True): 138 | """ 139 | 创建仓库项目 140 | """ 141 | try: 142 | data = {'project_name': project_name, 'metadata': { 143 | 'public': 'true' if public else 'false'}} 144 | res = self.request('post', obj=data, prefix='/projects') 145 | return res 146 | except BaseException as e: 147 | return {'ecode': 500, 'message': e} 148 | 149 | def update_project(self, project_id, *args, **kwargs): 150 | """ 151 | 更新仓库项目 152 | """ 153 | try: 154 | res = self.request('put', obj=kwargs, 155 | prefix=f'/projects/{project_id}') 156 | return res 157 | except BaseException as e: 158 | return {'ecode': 500, 'message': e} 159 | 160 | def project_exists(self, project_name): 161 | """ 162 | 查询项目是否存在 163 | """ 164 | try: 165 | res = self.request( 166 | 'head', prefix=f'/projects?project_name={project_name}') 167 | return res 168 | except BaseException as e: 169 | return {'ecode': 500, 'message': e} 170 | 171 | def patch_tag(self, repo, src_image, tag_name): 172 | """ 173 | 镜像打标签 174 | """ 175 | try: 176 | try: 177 | # 创建仓库项目 178 | res = self.create_project(repo.split('/')[0]) 179 | except BaseException as e: 180 | pass 181 | data = {'tag': tag_name, 'src_image': src_image, 'override': True} 182 | res = self.request( 183 | 'post', obj=data, prefix='/repositories/%s/tags' % repo) 184 | return res 185 | except BaseException as e: 186 | return {'ecode': 500, 'message': e} 187 | 188 | def delete_tag(self, repo, tag): 189 | """ 190 | 删除标签 191 | """ 192 | try: 193 | res = self.request( 194 | 'delete', prefix=f'/repositories/{repo}/tags/{tag}') 195 | return res 196 | except BaseException as e: 197 | logger.ex 198 | return {'ecode': 500, 'message': e} 199 | 200 | def search(self, query): 201 | """ 202 | 搜索 203 | """ 204 | try: 205 | res = self.request('get', {'q': query}, prefix='/search') 206 | return res 207 | except BaseException as e: 208 | logger.exception(e) 209 | return {'ecode': 500, 'message': e} 210 | -------------------------------------------------------------------------------- /common/utils/JiraAPI.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : JiraAPI.py 6 | @time : 2022/12/05 14:10 7 | @contact : qqing_lai@hotmail.com 8 | @company : IMAOJIA Co,Ltd 9 | ''' 10 | 11 | # here put the import lib 12 | from jira import JIRA 13 | import logging 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | class JiraAPI(object): 19 | 20 | def __init__(self, url, user=None, password=None, token=None): 21 | self.__url = url 22 | self.__user = user 23 | self.__password = password 24 | self.__token = token 25 | if user and token: 26 | self.client = JIRA(server=self.__url, basic_auth=( 27 | self.__user, self.__token)) 28 | elif token: 29 | self.client = JIRA(server=self.__url, token_auth=self.__token) 30 | elif user and password: 31 | self.client = JIRA(server=self.__url, basic_auth=( 32 | self.__user, self.__password)) 33 | 34 | def list_issues(self, issue_text='', issue_key=None, project=None, exclude_status=None, max_results=20): 35 | """ 36 | 获取issues 37 | :params return: ['expand', 'startAt', 'maxResults', 'total', 'issues'] 38 | """ 39 | params = '' 40 | if not any([issue_text, issue_key, project]): 41 | return False, '缺少参数！' 42 | if issue_key: 43 | params = f'issueKey={issue_key}' 44 | if issue_text: 45 | params = f'text~{issue_text}' 46 | max_results = 50 47 | if project: 48 | params = f'project={project}' 49 | if exclude_status: 50 | status = (',').join(exclude_status.split(',')) 51 | params = f'{params} and status not in ({status})' 52 | try: 53 | issues = self.client.search_issues( 54 | params, json_result=True, maxResults=max_results) 55 | return True, issues 56 | except BaseException as e: 57 | return False, e.text 58 | 59 | def get_issue(self, issue_key): 60 | """ 61 | 获取issue详情 62 | """ 63 | try: 64 | issue = self.client.issue(issue_key) 65 | return True, issue 66 | except BaseException as e: 67 | return False, e.text 68 | 69 | def update_issue(self, issue_key, **kwargs): 70 | """ 71 | 更新issue 72 | """ 73 | ok, issue = self.get_issue(issue_key) 74 | if ok: 75 | try: 76 | result = issue.update(**kwargs) 77 | return True, result 78 | except BaseException as e: 79 | return False, e.text 80 | return False, '获取issue失败.' 81 | -------------------------------------------------------------------------------- /common/utils/RedisAPI.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/5/13 下午5:40 7 | @FileName: RedisAPI.py 8 | @Blog : https://blog.imaojia.com 9 | """ 10 | 11 | import redis 12 | from django_redis.client import DefaultClient 13 | from rediscluster import ClusterConnectionPool, RedisCluster 14 | 15 | from config import REDIS_CONFIG, CACHE_CONFIG, REDIS_CLUSTER_CONFIG 16 | 17 | 18 | class CustomRedisCluster(DefaultClient): 19 | 20 | def connect(self, index): 21 | pool = ClusterConnectionPool(startup_nodes=CACHE_CONFIG['startup_nodes'], 22 | password=CACHE_CONFIG.get('password', ''), 23 | nodemanager_follow_cluster=True, 24 | skip_full_coverage_check=True, 25 | # decode_responses=True, 26 | ) 27 | return RedisCluster(connection_pool=pool, 28 | nodemanager_follow_cluster=True) 29 | 30 | 31 | class RedisManage(object): 32 | 33 | @classmethod 34 | def conn(cls): 35 | if REDIS_CLUSTER_CONFIG.get('startup_nodes', None): 36 | pool = ClusterConnectionPool(startup_nodes=REDIS_CLUSTER_CONFIG['startup_nodes'], 37 | password=REDIS_CLUSTER_CONFIG.get( 38 | 'password', ''), 39 | nodemanager_follow_cluster=True, 40 | decode_responses=True, ) 41 | return RedisCluster(connection_pool=pool, nodemanager_follow_cluster=True) 42 | pool = redis.ConnectionPool(host=REDIS_CONFIG['host'], port=REDIS_CONFIG['port'], db=REDIS_CONFIG['db'], 43 | password=REDIS_CONFIG.get('password', ''), decode_responses=True) 44 | return redis.Redis(connection_pool=pool) 45 | 46 | @staticmethod 47 | def get_pubsub(): 48 | r = redis.StrictRedis(host=REDIS_CONFIG['host'], port=REDIS_CONFIG['port'], db=REDIS_CONFIG['db'], 49 | password=REDIS_CONFIG.get('password', '')) 50 | return r.pubsub(ignore_subscribe_messages=True) 51 | -------------------------------------------------------------------------------- /common/utils/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/9/16 上午10:27 7 | @FileName: __init__.py.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | -------------------------------------------------------------------------------- /config.py.sample: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/9/15 下午1:58 7 | @FileName: config.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | 11 | from pathlib import Path 12 | import os 13 | 14 | BASE_DIR = Path(__file__).resolve().parent 15 | DEBUG = True 16 | SECRET_KEY = '7f=5@e+a=b(ghm-l*mtc_ile60xuvxqi(l5y$3&gfpk1!)3_4v' 17 | ALLOWED_HOSTS = ['*'] 18 | DATE_FORMAT = '%Y-%m-%d' 19 | DATETIME_FORMAT = f'{DATE_FORMAT} %H:%M:%S' 20 | 21 | # 数据库连接信息 22 | DATABASES_CONFIG = { 23 | 'NAME': 'ydevopsdb', 24 | 'HOST': '127.0.0.1', 25 | 'PORT': 43306, 26 | 'USER': 'devops', 27 | 'PASSWORD': 'ops123456', 28 | } 29 | 30 | # 日志配置 31 | LOG_CONFIG = { 32 | 'level': 'INFO', 33 | 'size': 300 * 1024 * 1024, # 日志大小 34 | 'backup': 7 # 日志保留份数 35 | } 36 | 37 | # token时间 38 | TOKEN_TIME = { 39 | 'ACCESS': { 40 | 'minutes': 5 41 | }, 42 | 'REFRESH': { 43 | 'minutes': 30 44 | } 45 | } 46 | 47 | # url白名单 48 | URL_WHITELIST = [ 49 | '/api/user/profile/info/', 50 | '/api/user/profile/menus/', 51 | '/api/user/login/', 52 | '/api/user/logout/', 53 | '/api/user/refresh/' 54 | ] 55 | 56 | REDIS_CONFIG = { 57 | 'host': '127.0.0.1', 58 | 'port': 6379, 59 | 'db': 10, 60 | 'password': 'ops123456' 61 | } 62 | 63 | STARTUP_NODES = [ 64 | {'host': '127.0.0.1', 'port': 7379}, 65 | {'host': '127.0.0.1', 'port': 7479}, 66 | {'host': '127.0.0.1', 'port': 7579}, 67 | {'host': '127.0.0.1', 'port': 7679}, 68 | {'host': '127.0.0.1', 'port': 7779}, 69 | {'host': '127.0.0.1', 'port': 7879}, 70 | ] 71 | 72 | REDIS_CLUSTER_CONFIG = { 73 | 'startup_nodes': STARTUP_NODES, 74 | 'password': 'ops123456' 75 | } 76 | 77 | CELERY_CONFIG = { 78 | 'queue': 'celery', 79 | 'broker_url': { 80 | 'host': '127.0.0.1', 81 | 'port': 6379, 82 | 'db': 4, 83 | 'password': 'ops123456' 84 | }, 85 | 'result_backend': { 86 | 'host': '127.0.0.1', # redis单机 87 | 'port': 6379, 88 | 'db': 7, 89 | 'startup_nodes': STARTUP_NODES, # redis集群 90 | 'password_cluster': 'ops123456', 91 | 'password': 'ops123456' 92 | }, 93 | } 94 | 95 | CACHE_CONFIG = { 96 | 'host': '127.0.0.1', # 单机redis 97 | 'port': '6379', 98 | 'db': '5', 99 | 'KEY_PREFIX': 'localdev', 100 | 'startup_nodes': STARTUP_NODES, # 集群redis 101 | 'password': 'ops123456' 102 | } 103 | 104 | CHANNEL_CONFIG = { 105 | 'host': '127.0.0.1', 106 | 'port': '6379', 107 | 'db': '6', 108 | 'password': 'ops123456' 109 | } 110 | 111 | Q_CLUSTER_CONFIG = { 112 | 'workers': 4, 113 | 'recycle': 500, 114 | 'timeout': 600, 115 | 'save_limit': 0, 116 | 'queue_limit': 1000, 117 | 'sync': False, # 本地调试可以修改为True，使用同步模式 118 | } 119 | 120 | MEDIA_ROOT = './media' 121 | 122 | UPLOAD_ROOT = './media/upload' 123 | UPLOAD_PATH = 'upload' 124 | PLAYBOOK_PATH = './media/playbook' 125 | 126 | COMPANY = '爱猫家' 127 | # 生产环境标识 128 | PROD_TAG = 'prod' 129 | 130 | # ES配置 131 | ELASTICSEARCH_CONFIG = { 132 | 'local': { 133 | 'host': ['localhost:9200'], 134 | 'username': '', 135 | 'password': '', 136 | 'ssl': False, 137 | 'timeout': 30 138 | }, 139 | 'prod': { 140 | 'host': [], 141 | 'username': '', 142 | 'password': '', 143 | 'ssl': False, 144 | 'timeout': 30 145 | } 146 | } 147 | ELASTICSEARCH = ELASTICSEARCH_CONFIG['local'] 148 | # 平台索引前缀 149 | ELASTICSEARCH_PREFIX = 'ydevops-' 150 | # CMDB资产上传/上报源数据及导入结果索引 151 | CMDB_SOURCE_INDEX = ELASTICSEARCH_PREFIX + 'cmdbupload' 152 | 153 | # 云平台AK/SK 154 | # 阿里云 155 | ALI_CONFIG = { 156 | 'key': '', 157 | 'secret': '', 158 | 'region': 'cn-shanghai' 159 | } 160 | # Tencent 161 | TENCENT_CONFIG = { 162 | 'key': '', 163 | 'secret': '', 164 | 'region': 'ap-shanghai' 165 | } 166 | # AWS 167 | AWS_CONFIG = { 168 | 'access_id': '', 169 | 'access_key': '', 170 | 'region': 'sa-east-1', 171 | 'owners': '' 172 | } 173 | 174 | GITLAB_ADMIN_TOKEN = '' 175 | 176 | CONFIG_CENTER_DEFAULT_USER = 'apollo' 177 | CONFIG_CENTER_DEFAULT_PASSWD = 'admin' 178 | 179 | PROXY_CONFIG = { 180 | 'http': 'http://', # 留空不启用代理 181 | 'https': 'http://' # 留空不启用代理 182 | } 183 | 184 | OSS_CONFIG = { 185 | 'key': '', 186 | 'secret': '', 187 | 'bucket': '', 188 | 'endpoint': '', 189 | } 190 | 191 | PHONE_CONFIG = { 192 | 'prod_id': 0 193 | } 194 | 195 | TEST_CONSUL_CONFIG = { 196 | 'url': '', 197 | 'token': '', 198 | } 199 | 200 | # 日志配置 201 | BASE_LOG_DIR = os.path.join(BASE_DIR, 'logs') 202 | 203 | # elasticsearch查询日志下载路径 204 | ES_LOG_DOWNLOAD_URL = 'log/download' 205 | 206 | JIRA_CONFIG = '' 207 | TB_CONFIG = '' 208 | CONFIG_CENTER_DEFAULT_USER = '' 209 | 210 | # 飞书登录 211 | FEISHU_URL = '' 212 | SOCIAL_AUTH_FEISHU_KEY = '' 213 | SOCIAL_AUTH_FEISHU_SECRET = '' 214 | 215 | # Gitlab配置 216 | GITLAB_ADMIN_TOKEN = '' 217 | SOCIAL_AUTH_GITLAB_KEY = '' 218 | SOCIAL_AUTH_GITLAB_SECRET = '' 219 | SOCIAL_AUTH_GITLAB_API_URL = '' 220 | 221 | -------------------------------------------------------------------------------- /daemon.conf: -------------------------------------------------------------------------------- 1 | [unix_http_server] 2 | file=/tmp/supervisor.sock ; path to your socket file 3 | 4 | [supervisord] 5 | logfile=/var/log/supervisord.log ; supervisord log file 6 | logfile_maxbytes=50MB ; maximum size of logfile before rotation 7 | logfile_backups=10 ; number of backed up logfiles 8 | loglevel=error ; info, debug, warn, trace 9 | pidfile=/var/run/supervisord.pid ; pidfile location 10 | nodaemon=true ; run supervisord as a daemon 11 | minfds=1024 ; number of startup file descriptors 12 | minprocs=200 ; number of process descriptors 13 | user=root ; default user 14 | childlogdir=/var/log/ ; where child log files will live 15 | 16 | [rpcinterface:supervisor] 17 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 18 | 19 | [supervisorctl] 20 | serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket 21 | 22 | [program:nginx] 23 | command=nginx -g 'daemon off;' 24 | autostart=true 25 | autorestart=true 26 | redirect_stderr=true 27 | priority=1 28 | [program:backend] 29 | command=gunicorn devops_backend.wsgi -b 0.0.0.0:8080 -t 60 --thread 20 30 | directory=/app/devops-backend 31 | autostart=true 32 | autorestart=true 33 | startsecs=3 34 | priority=1 35 | stopasgroup=true 36 | killasgroup=true 37 | stderr_logfile=/dev/sederr 38 | stderr_logfile_maxbytes=0 39 | stdout_logfile=/dev/stdout 40 | stdout_logfile_maxbytes=0 41 | [program:websocket] 42 | command=daphne devops_backend.asgi:application -b 0.0.0.0 -p 8081 43 | directory=/app/devops-backend 44 | autostart=true 45 | autorestart=true 46 | startsecs=3 47 | priority=1 48 | stopasgroup=true 49 | killasgroup=true 50 | stderr_logfile=/dev/sederr 51 | stderr_logfile_maxbytes=0 52 | stdout_logfile=/dev/stdout 53 | stdout_logfile_maxbytes=0 54 | [program:qtask] 55 | command=python manage.py qcluster 56 | directory=/app/devops-backend 57 | autostart=true 58 | autorestart=true 59 | startsecs=3 60 | priority=1 61 | stopasgroup=true 62 | killasgroup=true 63 | stderr_logfile=/dev/sederr 64 | stderr_logfile_maxbytes=0 65 | stdout_logfile=/dev/stdout 66 | stdout_logfile_maxbytes=0 67 | [program:celery] 68 | command=./celery.sh 69 | directory=/app/devops-backend 70 | autostart=true 71 | autorestart=true 72 | startsecs=3 73 | priority=1 74 | stopasgroup=true 75 | killasgroup=true 76 | stderr_logfile=/dev/sederr 77 | stderr_logfile_maxbytes=0 78 | stdout_logfile=/dev/stdout 79 | stdout_logfile_maxbytes=0 80 | 81 | -------------------------------------------------------------------------------- /dbapp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/dbapp/__init__.py -------------------------------------------------------------------------------- /dbapp/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /dbapp/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class DbappConfig(AppConfig): 5 | name = 'dbapp' 6 | -------------------------------------------------------------------------------- /dbapp/management/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2021/2/4 下午10:08 7 | @FileName: __init__.py.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | 11 | from __future__ import unicode_literals 12 | -------------------------------------------------------------------------------- /dbapp/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2021/2/4 下午10:08 7 | @FileName: __init__.py.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | 11 | from __future__ import unicode_literals 12 | -------------------------------------------------------------------------------- /dbapp/management/commands/qtasks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : qtasks.py 6 | @time : 2023/03/08 21:20 7 | @contact : qqing_lai@hotmail.com 8 | @company : IMAOJIA Co,Ltd 9 | ''' 10 | 11 | # here put the import lib 12 | from django.core.management.base import BaseCommand 13 | from django.utils.translation import gettext as _ 14 | 15 | from common.extends.django_qcluster import Cluster 16 | 17 | 18 | class Command(BaseCommand): 19 | # Translators: help text for qcluster management command 20 | help = _("Starts a Django Q Cluster.") 21 | 22 | def add_arguments(self, parser): 23 | parser.add_argument( 24 | "--run-once", 25 | action="store_true", 26 | dest="run_once", 27 | default=False, 28 | help="Run once and then stop.", 29 | ) 30 | 31 | def handle(self, *args, **options): 32 | q = Cluster() 33 | q.start() 34 | if options.get("run_once", False): 35 | q.stop() 36 | -------------------------------------------------------------------------------- /dbapp/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/dbapp/migrations/__init__.py -------------------------------------------------------------------------------- /dbapp/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/dbapp/model/__init__.py -------------------------------------------------------------------------------- /dbapp/model/model_assets.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2021/05/13 上午11:22 7 | @FileName: model_assets.py 8 | @Blog ： https://imaojia.com 9 | """ 10 | 11 | from django.contrib.auth.models import User 12 | from django.db import models 13 | 14 | from dbapp.model.model_ucenter import UserProfile 15 | 16 | from common.extends.fernet import EncryptedJsonField 17 | from common.extends.models import TimeAbstract, CreateTimeAbstract 18 | from common.extends.storage import FileUploadStorage 19 | from common.variables import * 20 | 21 | from config import UPLOAD_PATH, UPLOAD_ROOT 22 | 23 | 24 | class FileUpload(TimeAbstract): 25 | name = models.CharField(max_length=250, default='', verbose_name='文件名') 26 | file = models.FileField(upload_to=UPLOAD_PATH, storage=FileUploadStorage( 27 | upload_root=UPLOAD_ROOT), default=None) 28 | md5 = models.CharField(max_length=250) 29 | table = models.CharField(max_length=64, default='', verbose_name='关联表') 30 | uploader = models.ForeignKey( 31 | UserProfile, null=True, blank=True, on_delete=models.SET_NULL, verbose_name='上传者') 32 | # 0: 未处理, 1: 已处理 33 | status = models.SmallIntegerField(default=0, verbose_name='状态') 34 | 35 | def __str__(self): 36 | return self.name 37 | 38 | class Meta: 39 | db_table = 'cmdb_fileupload' 40 | default_permissions = () 41 | verbose_name = '文件上传' 42 | verbose_name_plural = verbose_name + '管理' 43 | 44 | 45 | class Region(TimeAbstract): 46 | name = models.CharField(max_length=100, unique=True, verbose_name='地域') 47 | alias = models.CharField(max_length=128, default='', verbose_name='地域别名') 48 | desc = models.TextField(verbose_name='详情描述', null=True, blank=True) 49 | extra = models.JSONField(default=dict, verbose_name='扩展字段') 50 | # {0: 禁用， 1： 启用} 51 | is_enable = models.SmallIntegerField( 52 | default=1, verbose_name='启用', help_text='状态 {0: 禁用， 1： 启用}，默认值为1') 53 | 54 | def __str__(self) -> str: 55 | return self.alias 56 | 57 | class ExtMeta: 58 | related = True 59 | dashboard = True 60 | 61 | class Meta: 62 | db_table = 'cmdb_region' 63 | verbose_name = '地域' 64 | verbose_name_plural = verbose_name + '管理' 65 | 66 | 67 | class Idc(TimeAbstract): 68 | """ 69 | Idc模型 70 | """ 71 | name = models.CharField(max_length=100, unique=True, verbose_name='名称') 72 | alias = models.CharField(max_length=128, unique=True, verbose_name='别名') 73 | region = models.ForeignKey( 74 | Region, blank=True, null=True, on_delete=models.PROTECT, verbose_name='区域') 75 | type = models.SmallIntegerField(default=0, choices=IDC_TYPE, verbose_name='机房类型', 76 | help_text=f"可选： {IDC_TYPE}") 77 | supplier = models.CharField( 78 | max_length=128, default=None, null=True, blank=True, verbose_name='服务商') 79 | config = EncryptedJsonField(default=dict, verbose_name='配置信息', 80 | help_text='阿里云：{"key":"key","secret":"secret","region":["cn-south-1"],"project":[]}\n华为云：{"domain":"domain","user":"user","password":password","project":[{"region":"region","project_id":"project_id"}]}') 81 | forward = models.BooleanField(default=False, verbose_name='是否中转') 82 | ops = models.CharField(max_length=100, blank=True, 83 | null=True, verbose_name='运维机器') 84 | repo = models.SmallIntegerField(default=0, verbose_name='镜像仓库') 85 | contact = models.JSONField(default=list, verbose_name='联系人') 86 | desc = models.TextField(default='', null=True, 87 | blank=True, verbose_name='备注') 88 | 89 | def __str__(self): 90 | return self.name 91 | 92 | class ExtMeta: 93 | related = True 94 | dashboard = True 95 | icon = 'international' 96 | 97 | class Meta: 98 | db_table = 'cmdb_idc' 99 | verbose_name = 'IDC机房' 100 | verbose_name_plural = verbose_name + '管理' 101 | -------------------------------------------------------------------------------- /dbapp/model/model_dashboard.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : models.py 6 | @time : 2023/04/18 19:24 7 | @contact : qqing_lai@hotmail.com 8 | ''' 9 | 10 | # here put the import lib 11 | 12 | from django.db import models 13 | 14 | from dbapp.model.model_ucenter import UserProfile 15 | 16 | from common.extends.models import TimeAbstract 17 | from common.variables import DASHBOARD_TYPE 18 | 19 | 20 | class DashBoard(TimeAbstract): 21 | name = models.CharField(max_length=128, unique=True, verbose_name='名称') 22 | config = models.JSONField(default=list, verbose_name='配置') 23 | type = models.CharField(max_length=16, choices=DASHBOARD_TYPE, default='index', 24 | verbose_name='报表类型', help_text=f"报表类型: {dict(DASHBOARD_TYPE)}") 25 | creator = models.ForeignKey( 26 | UserProfile, on_delete=models.CASCADE, verbose_name='创建人') 27 | 28 | def __str__(self): 29 | return self.name 30 | 31 | class Meta: 32 | db_table = 'dashboard_dashboard' 33 | default_permissions = () 34 | verbose_name = '报表配置' 35 | verbose_name_plural = verbose_name + '管理' 36 | -------------------------------------------------------------------------------- /dbapp/models.py: -------------------------------------------------------------------------------- 1 | from .model.model_ucenter import * 2 | from .model.model_assets import * 3 | from .model.model_cmdb import * 4 | from .model.model_dashboard import * 5 | from .model.model_deploy import * 6 | from .model.model_workflow import * 7 | -------------------------------------------------------------------------------- /dev.sh: -------------------------------------------------------------------------------- 1 | python manage.py runserver 0.0.0.0:8090 --insecure 2 | -------------------------------------------------------------------------------- /devops.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80; 3 | server_name localhost; 4 | access_log off; 5 | error_page 404 /404.html; 6 | location = /404.html { 7 | root /etc/nginx; 8 | } 9 | error_page 500 502 503 504 /500.html; 10 | location = /500.html { 11 | root /etc/nginx; 12 | } 13 | underscores_in_headers on; 14 | client_max_body_size 2048m; 15 | 16 | location / { 17 | root /app/devops-backend/dist; 18 | index index.html index.htm; 19 | try_files $uri $uri/ /index.html; 20 | } 21 | location /ws/ { 22 | proxy_pass http://localhost:8081; 23 | proxy_http_version 1.1; 24 | proxy_set_header Upgrade $http_upgrade; 25 | proxy_set_header Connection "Upgrade"; 26 | } 27 | location ~ ^/(admin|api) { 28 | proxy_pass http://localhost:8080; 29 | proxy_connect_timeout 1200s; 30 | proxy_read_timeout 1200s; 31 | proxy_send_timeout 1200s; 32 | proxy_set_header Host $host; 33 | proxy_set_header X-Real-IP $remote_addr; 34 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 35 | } 36 | } 37 | 38 | -------------------------------------------------------------------------------- /devops_backend/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/devops_backend/__init__.py -------------------------------------------------------------------------------- /devops_backend/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for devops_backend project. 3 | 4 | It exposes the ASGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/ 8 | """ 9 | 10 | import os 11 | import django 12 | 13 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'devops_backend.settings') 14 | django.setup() 15 | from django.core.asgi import get_asgi_application 16 | from channels.routing import ProtocolTypeRouter, URLRouter 17 | 18 | from channels.auth import AuthMiddlewareStack 19 | import deploy.routing 20 | 21 | django_application = get_asgi_application() 22 | 23 | application = ProtocolTypeRouter({ 24 | # Explicitly set 'http' key using Django's ASGI application. 25 | "http": django_application, 26 | 'websocket': AuthMiddlewareStack( 27 | URLRouter( 28 | deploy.routing.websocket_urlpatterns 29 | ) 30 | ), 31 | }) 32 | -------------------------------------------------------------------------------- /devops_backend/documents.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : documents.py 6 | @time : 2023/04/20 17:50 7 | @contact : qqing_lai@hotmail.com 8 | ''' 9 | 10 | # here put the import lib 11 | from deploy.documents import * 12 | from deploy.documents_order import * 13 | -------------------------------------------------------------------------------- /devops_backend/routing.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/5/13 下午5:01 7 | @FileName: routing.py 8 | @Blog : https://imaojia.com 9 | """ 10 | 11 | from channels.auth import AuthMiddlewareStack 12 | from channels.routing import ProtocolTypeRouter, URLRouter 13 | 14 | import deploy.routing 15 | 16 | application = ProtocolTypeRouter({ 17 | 'websocket': AuthMiddlewareStack( 18 | URLRouter( 19 | deploy.routing.websocket_urlpatterns 20 | ) 21 | ) 22 | }) 23 | -------------------------------------------------------------------------------- /devops_backend/urls.py: -------------------------------------------------------------------------------- 1 | """devops_backend URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/3.2/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from config import DEBUG 17 | from django.contrib import admin 18 | from django.urls import path, include 19 | 20 | from rest_framework.documentation import include_docs_urls 21 | from rest_framework.routers import DefaultRouter 22 | from rest_framework import permissions 23 | 24 | from drf_yasg.views import get_schema_view 25 | from drf_yasg import openapi 26 | 27 | from ucenter.views import SystemConfigViewSet, MenuViewSet, PermissionViewSet, RoleViewSet, OrganizationViewSet, \ 28 | UserViewSet, UserProfileViewSet, UserAuthTokenView, UserLogout, UserAuthTokenRefreshView, AuditLogViewSet, \ 29 | DataDictViewSet 30 | from dashboard.views import LiveCheck 31 | from deploy.views import BuildJobViewSet, PublishOrderViewSet, PublishAppViewSet, \ 32 | DeployJobViewSet 33 | from workflow.views.callback import WorkflowNodeHistoryCallbackViewSet 34 | from workflow.views.workflow import WorkflowViewSet 35 | from workflow.views.my_related import WorkflowMyRelatedViewSet 36 | from workflow.views.my_request import WorkflowMyRequestViewSet 37 | from workflow.views.my_upcoming import WorkflowMyUpComingViewSet 38 | from workflow.views.template import WorkflowTemplateViewSet 39 | from workflow.views.category import WorkflowCategoryViewSet 40 | 41 | from cmdb import urls as cmdb_urls 42 | from dashboard import urls as dashboard_urls 43 | from workflow_callback import urls as workflow_callback_urls 44 | 45 | schema_view = get_schema_view( 46 | openapi.Info( 47 | title="DevOps运维平台", 48 | default_version='v1', 49 | description="DevOps运维平台 接口文档", 50 | terms_of_service="", 51 | contact=openapi.Contact(email="qqing_lai@hotmail.com"), 52 | license=openapi.License(name="Apache License 2.0"), 53 | ), 54 | public=True, 55 | permission_classes=(permissions.IsAuthenticated, ) 56 | ) 57 | 58 | router = DefaultRouter() 59 | router.register('cicd/order/app', PublishAppViewSet) 60 | router.register('cicd/order', PublishOrderViewSet) 61 | router.register('cicd/deploy', DeployJobViewSet) 62 | router.register('cicd', BuildJobViewSet) 63 | # ucenter 64 | router.register('audit', AuditLogViewSet) 65 | router.register('system/data', DataDictViewSet) 66 | router.register('system', SystemConfigViewSet) 67 | router.register('menu', MenuViewSet) 68 | router.register('permission', PermissionViewSet) 69 | router.register('role', RoleViewSet) 70 | router.register('organization', OrganizationViewSet) 71 | router.register('users', UserViewSet) 72 | router.register('user/profile', UserProfileViewSet, basename='user-profile') 73 | # 新的工单系统 74 | router.register('workflow/node_history/callback', 75 | WorkflowNodeHistoryCallbackViewSet) 76 | router.register('workflow/category', WorkflowCategoryViewSet) 77 | router.register('workflow/template', WorkflowTemplateViewSet) 78 | router.register('workflow/my-request', WorkflowMyRequestViewSet) 79 | router.register('workflow/my-upcoming', WorkflowMyUpComingViewSet) 80 | router.register('workflow/my-related', WorkflowMyRelatedViewSet) 81 | router.register('workflow', WorkflowViewSet) 82 | 83 | extra = '/' 84 | 85 | urlpatterns = [ 86 | path('admin/', admin.site.urls), 87 | path('apidoc/', schema_view.with_ui('swagger', 88 | cache_timeout=0), name='schema-swagger-ui'), 89 | path('api/', include(router.urls)), 90 | path('api/workflow_callback/', include(workflow_callback_urls), 91 | name='workflow_callback'), 92 | path('api/check/', LiveCheck.as_view(), name='live-check'), 93 | path('api/user/login/', UserAuthTokenView.as_view(), name='user-login'), 94 | path('api/user/logout/', UserLogout.as_view(), name='user-logout'), 95 | path('api/user/refresh/', UserAuthTokenRefreshView.as_view(), 96 | name='token-refresh'), 97 | path('api/', include(cmdb_urls)), 98 | path('api/', include(dashboard_urls)), 99 | ] 100 | 101 | if DEBUG: 102 | # 兼容gunicorn启动 103 | from django.contrib.staticfiles.urls import staticfiles_urlpatterns 104 | urlpatterns += staticfiles_urlpatterns() 105 | -------------------------------------------------------------------------------- /devops_backend/websocket.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | @Author : Charles Lai 5 | @Contact : qqing_lai@hotmail.com 6 | @Time : 2020/10/9 下午2:29 7 | @FileName: websocket.py 8 | @Blog ：https://imaojia.com 9 | """ 10 | 11 | 12 | async def websocket_application(scope, receive, send): 13 | while True: 14 | # print(scope) 15 | event = await receive() 16 | 17 | if event['type'] == 'websocket.connect': 18 | await send({ 19 | 'type': 'websocket.accept' 20 | }) 21 | 22 | if event['type'] == 'websocket.disconnect': 23 | break 24 | 25 | if event['type'] == 'websocket.receive': 26 | print(scope['path']) 27 | if scope['path'] == '/build': 28 | await send({ 29 | 'type': 'websocket.send', 30 | 'text': 'ws send' 31 | }) 32 | if event['text'] == 'ping': 33 | await send({ 34 | 'type': 'websocket.send', 35 | 'text': 'pong' 36 | }) 37 | -------------------------------------------------------------------------------- /devops_backend/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for devops_backend project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'devops_backend.settings') 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /dist.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/dist.tar.gz -------------------------------------------------------------------------------- /gunicorn.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | touch logs/gunicorn.log 4 | gunicorn devops_backend.wsgi -b 0.0.0.0:8080 -t 60 --thread 20 --access-logfile logs/gunicorn.log -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | 7 | def main(): 8 | """Run administrative tasks.""" 9 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'devops_backend.settings') 10 | try: 11 | from django.core.management import execute_from_command_line 12 | except ImportError as exc: 13 | raise ImportError( 14 | "Couldn't import Django. Are you sure it's installed and " 15 | "available on your PYTHONPATH environment variable? Did you " 16 | "forget to activate a virtual environment?" 17 | ) from exc 18 | execute_from_command_line(sys.argv) 19 | 20 | 21 | if __name__ == '__main__': 22 | main() 23 | -------------------------------------------------------------------------------- /media/playbook/docker_deploy.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: "{{ target_hosts }}" #"{{ hosts }}" 3 | gather_facts: no 4 | vars: 5 | ansible_python_interpreter: /usr/bin/python3 6 | # 静态文件源路径 7 | src_path: "/data/nfs/web/{{ src_env }}/{{ app_name }}/{{ src_tag }}" 8 | # 生产仓库 9 | reg_path: "/data/nfs/web/{{ dest_env }}/{{ app_name }}" 10 | 11 | tasks: 12 | - name: Python3 Dependency 13 | pip: 14 | name: docker 15 | # 平台直接访问目标机器 16 | - name: Docker Login 17 | docker_login: 18 | registry_url: "{{ harbor_url }}" 19 | username: "{{ harbor_user }}" 20 | password: "{{ harbor_passwd }}" 21 | reauthorize: yes 22 | - name: Pull Image 23 | docker_image: 24 | name: "{{ image }}" 25 | source: pull 26 | 27 | - name: Update Container 28 | docker_container: 29 | name: "{{ app_name }}" 30 | image: "{{ image }}" 31 | command: 32 | - "{{ service_command }}" 33 | state: started 34 | recreate: true 35 | env: 36 | appiddd: "backend" 37 | appnamee: "migrate" 38 | -------------------------------------------------------------------------------- /media/playbook/jar_deploy.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: "{{ hosts }}" 3 | gather_facts: no 4 | vars: 5 | # 静态文件源路径 6 | src_path: "/data/nfs/web/{{ src_env }}/{{ app_name }}/{{ src_tag }}" 7 | # 生产仓库 8 | reg_path: "/data/nfs/web/{{ dest_env }}/{{ app_name }}" 9 | 10 | tasks: 11 | - name: 创建生产仓库目录 12 | file: 13 | path: "{{ reg_path }}" 14 | state: directory 15 | 16 | - name: 同步JAR包到生产仓库 17 | synchronize: 18 | src: "{{ src_path }}" 19 | dest: "{{ reg_path }}/" 20 | rsync_opts: 21 | - "-azP" 22 | - "--exclude=.git" 23 | delegate_to: 127.0.0.1 24 | 25 | - name: 同步JAR包到目标主机 26 | shell: "ssh {{ item }} mkdir -p {{ dest }};/usr/bin/rsync -azP {{ reg_path }}/{{ src_tag }}/ {{ item }}:{{ dest }}" 27 | with_items: "{{ target_hosts }}" 28 | -------------------------------------------------------------------------------- /media/playbook/jar_rollback.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: "{{ hosts }}" 3 | gather_facts: no 4 | vars: 5 | forward_dest: "/tmp/webtrans/{{ dest_env }}/{{ app_name }}/" 6 | # 静态文件源路径 7 | src_path: "/data/nfs/web/{{ src_env }}/{{ app_name }}/{{ src_tag }}" 8 | # 生产仓库 9 | reg_path: "/tmp/{{ dest_env }}/{{ app_name }}" 10 | 11 | tasks: 12 | - name: 目标主机JAR包回退 13 | shell: "ssh {{ item }} echo rollback && sleep 15" 14 | with_items: "{{ target_hosts }}" 15 | -------------------------------------------------------------------------------- /media/playbook/web_deploy.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: "{{ hosts }}" 3 | gather_facts: no 4 | vars: 5 | # 中转机器仓库 6 | forward_dest: "/data/webtrans/{{ dest_env }}/{{ app_name }}/" 7 | # 静态文件源路径 8 | src_path: "/data/nfs/web/{{ src_env }}/{{ app_name }}/{{ src_tag }}" 9 | # 生产仓库 10 | reg_path: "/data/nfs/web/{{ dest_env }}/{{ app_name }}" 11 | 12 | tasks: 13 | - name: 创建同步目录 14 | file: 15 | path: "{{ forward_dest }}" 16 | state: directory 17 | 18 | - name: 创建生产仓库目录 19 | file: 20 | path: "{{ reg_path }}" 21 | state: directory 22 | 23 | - name: 同步静态文件到生产仓库 24 | synchronize: 25 | src: "{{ src_path }}" 26 | dest: "{{ reg_path }}/" 27 | rsync_opts: 28 | - "-azP" 29 | - "--exclude=.git" 30 | delegate_to: 127.0.0.1 31 | 32 | - name: 同步静态文件到中转机器 33 | synchronize: 34 | src: "{{ src_path }}" 35 | dest: "{{ forward_dest }}" 36 | rsync_opts: 37 | - "-azP" 38 | - "--exclude=.git" 39 | become: no 40 | 41 | - name: 同步静态文件到Nginx机器 42 | shell: "ssh {{ item }} mkdir -p {{ dest }};/usr/bin/rsync -azP {{ forward_dest }}/{{ src_tag }}/ {{ item }}:{{ dest }}" 43 | with_items: "{{ target_hosts }}" 44 | -------------------------------------------------------------------------------- /preview/a-app-extra.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/preview/a-app-extra.gif -------------------------------------------------------------------------------- /preview/a-cicd.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/preview/a-cicd.gif -------------------------------------------------------------------------------- /preview/a-cilog.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/preview/a-cilog.gif -------------------------------------------------------------------------------- /preview/a-cmdb.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/preview/a-cmdb.gif -------------------------------------------------------------------------------- /preview/a-login-dashboard.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/preview/a-login-dashboard.gif -------------------------------------------------------------------------------- /preview/a-wflow.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/preview/a-wflow.gif -------------------------------------------------------------------------------- /qtasks/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : __init__.py 6 | @time : 2023/03/07 11:10 7 | @contact : qqing_lai@hotmail.com 8 | @company : IMAOJIA Co,Ltd 9 | ''' 10 | 11 | # here put the import lib 12 | -------------------------------------------------------------------------------- /qtasks/hooks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qitan/devops-backend/360e0428b26139bb9eda9fb16e6f8d1756df7933/qtasks/hooks/__init__.py -------------------------------------------------------------------------------- /qtasks/hooks/tasks_hook.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | ''' 4 | @author : Charles Lai 5 | @file : tasks_hook.py 6 | @time : 2023/03/09 11:38 7 | @contact : qqing_lai@hotmail.com 8 | @company : IMAOJIA Co,Ltd 9 | ''' 10 | 11 | # here put the import lib 12 | 13 | def print_result(task): 14 | print('回调结果', task.result) 15 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.8.1 2 | aioredis==1.3.1 3 | aioredis-cluster==1.5.2 4 | aiosignal==1.2.0 5 | alibabacloud-credentials==0.3.2 6 | alibabacloud-dyvmsapi20170525==2.1.4 7 | alibabacloud-ecs20140526==3.0.7 8 | alibabacloud-endpoint-util==0.0.3 9 | alibabacloud-gateway-spi==0.0.1 10 | alibabacloud-openapi-util==0.2.1 11 | alibabacloud-rocketmq20220801==1.0.5 12 | alibabacloud-tea==0.2.9 13 | alibabacloud-tea-openapi==0.3.7 14 | alibabacloud-tea-util==0.3.8 15 | alibabacloud-tea-xml==0.0.2 16 | aliyun-python-sdk-alidns==2.6.32 17 | aliyun-python-sdk-cdn==3.7.1 18 | aliyun-python-sdk-core==2.13.26 19 | aliyun-python-sdk-cs==4.8.1 20 | aliyun-python-sdk-domain==3.14.7 21 | aliyun-python-sdk-dyvmsapi==3.2.0 22 | aliyun-python-sdk-ecs==4.19.11 23 | aliyun-python-sdk-elasticsearch==3.0.29 24 | aliyun-python-sdk-kms==2.15.0 25 | aliyun-python-sdk-nas==3.14.0 26 | aliyun-python-sdk-ons==3.1.8 27 | aliyun-python-sdk-polardb==1.8.13 28 | aliyun-python-sdk-r-kvstore==2.20.2 29 | aliyun-python-sdk-rds==2.5.1 30 | aliyun-python-sdk-slb==3.3.9 31 | aliyun-python-sdk-waf-openapi==1.1.8 32 | amqp==2.6.1 33 | ansible==2.9.13 34 | arrow==1.2.3 35 | asgiref==3.4.1 36 | asttokens==2.0.8 37 | async-timeout==4.0.2 38 | atlassian-python-api==3.36.0 39 | attrs==20.2.0 40 | autobahn==20.7.1 41 | Automat==20.2.0 42 | autopep8==2.0.2 43 | backcall==0.2.0 44 | bcrypt==3.2.0 45 | billiard==3.6.4.0 46 | blessed==1.20.0 47 | boto3==1.18.32 48 | botocore==1.21.65 49 | cachetools==4.1.1 50 | celery==4.4.7 51 | certifi==2020.6.20 52 | cffi==1.14.2 53 | channels==3.0.4 54 | channels-redis==3.2.0 55 | chardet==3.0.4 56 | charset-normalizer==2.1.1 57 | colorama==0.4.3 58 | constantly==15.1.0 59 | coreapi==2.3.3 60 | coreschema==0.0.4 61 | crcmod==1.7 62 | croniter==1.3.8 63 | cryptography==3.4.8 64 | daphne==3.0.2 65 | decorator==5.1.1 66 | defusedxml==0.7.1 67 | Deprecated==1.2.13 68 | Django==3.2 69 | django-auth-ldap==2.4.0 70 | django-celery-beat==2.0.0 71 | django-cors-headers==3.5.0 72 | django-excel==0.0.10 73 | django-fernet-fields==0.6 74 | django-filter==2.3.0 75 | django-mirage-field==1.4.0 76 | django-oauth-toolkit==2.1.0 77 | django-picklefield==3.1 78 | django-q==1.3.9 79 | django-redis==4.12.1 80 | django-rest-framework-condition==0.1.1 81 | django-timezone-field==4.0 82 | djangorestframework==3.11.1 83 | djangorestframework-simplejwt==4.6.0 84 | dnspython==2.4.2 85 | docutils==0.15.2 86 | drf-nested-routers==0.93.4 87 | drf-social-oauth2==1.2.1 88 | drf-yasg==1.20.0 89 | ecdsa==0.18.0 90 | elasticsearch==7.15.1 91 | elasticsearch-dsl==7.4.0 92 | et-xmlfile==1.0.1 93 | executing==1.0.0 94 | frozenlist==1.3.1 95 | google-auth==1.22.1 96 | gunicorn==20.1.0 97 | hiredis==2.0.0 98 | hyperlink==20.0.1 99 | idna==2.10 100 | importlib-metadata==4.3.0 101 | incremental==17.5.0 102 | inflection==0.5.1 103 | ipython==8.5.0 104 | itypes==1.2.0 105 | jdcal==1.4.1 106 | jedi==0.18.1 107 | Jinja2==2.11.2 108 | jira==3.4.1 109 | jmespath==0.10.0 110 | jwcrypto==1.3.1 111 | kombu==4.6.11 112 | kubernetes==17.17.0 113 | ldap3==2.9.1 114 | lml==0.1.0 115 | Markdown==3.3.5 116 | MarkupSafe==1.1.1 117 | matplotlib-inline==0.1.6 118 | msgpack==1.0.2 119 | multi-key-dict==2.0.3 120 | multidict==6.0.2 121 | mybatis-mapper2sql==0.1.9 122 | mysqlclient==2.0.1 123 | numpy==1.26.1 124 | oauthlib==3.1.0 125 | openpyxl==3.1.2 126 | oss2==2.15.0 127 | packaging==20.9 128 | pandas==2.1.1 129 | paramiko==2.7.2 130 | parso==0.8.3 131 | pbr==5.5.0 132 | pexpect==4.8.0 133 | pickleshare==0.7.5 134 | Pillow==10.1.0 135 | prompt-toolkit==3.0.31 136 | psutil==5.8.0 137 | psycopg2-binary==2.9.9 138 | ptyprocess==0.7.0 139 | pure-eval==0.2.2 140 | pyasn1==0.4.8 141 | pyasn1-modules==0.2.8 142 | pycodestyle==2.10.0 143 | pycparser==2.20 144 | pycryptodome==3.9.8 145 | pyexcel==0.6.7 146 | pyexcel-io==0.6.4 147 | pyexcel-webio==0.1.4 148 | pyexcel-xlsx==0.6.0 149 | Pygments==2.13.0 150 | PyHamcrest==2.0.2 151 | PyJWT==2.5.0 152 | pymongo==3.11.0 153 | PyMySQL==1.0.2 154 | PyNaCl==1.4.0 155 | pyOpenSSL==19.1.0 156 | pyparsing==2.4.7 157 | python-crontab==2.5.1 158 | python-dateutil==2.8.2 159 | python-gitlab==3.15.0 160 | python-jenkins==1.7.0 161 | python-jose==3.3.0 162 | python-ldap==3.3.1 163 | python-redis-lock==3.7.0 164 | python3-openid==3.2.0 165 | pytz==2020.1 166 | PyYAML==5.3.1 167 | redis==3.5.3 168 | redis-py-cluster==2.1.3 169 | requests==2.31.0 170 | requests-oauthlib==1.3.0 171 | requests-toolbelt==1.0.0 172 | rest-social-auth==7.0.0 173 | rsa==4.5 174 | ruamel.yaml==0.17.35 175 | ruamel.yaml.clib==0.2.8 176 | s3transfer==0.5.2 177 | SchemaObject==0.5.9 178 | semantic-version==2.10.0 179 | service-identity==18.1.0 180 | setuptools-rust==1.5.2 181 | shortuuid==1.0.1 182 | simplejson==3.18.3 183 | six==1.15.0 184 | social-auth-app-django==5.0.0 185 | social-auth-core==4.3.0 186 | sqlparse==0.3.1 187 | stack-data==0.5.0 188 | tencentcloud-sdk-python==3.0.529 189 | texttable==1.6.4 190 | tomli==2.0.1 191 | tqdm==4.65.0 192 | traitlets==5.3.0 193 | Twisted==21.2.0 194 | txaio==20.4.1 195 | typing==3.7.4.3 196 | typing-extensions==3.10.0.0 197 | tzdata==2023.3 198 | uritemplate==3.0.1 199 | urllib3==1.25.10 200 | vine==1.3.0 201 | wcwidth==0.2.5 202 | websocket-client==0.57.0 203 | wrapt==1.14.1 204 | XlsxWriter==3.1.2 205 | xlwt==1.3.0 206 | xmltodict==0.12.0 207 | yarl==1.8.1 208 | zipp==3.4.1 209 | zope.interface==5.1.2 210 | orderedset==2.0.3 211 | supervisor==4.2.5 212 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | touch logs/daphne.log 4 | daphne devops_backend.asgi:application -b 0.0.0.0 -p 8080 --access-log logs/daphne.log 5 | -------------------------------------------------------------------------------- /task.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python manage.py qtasks 4 | --------------------------------------------------------------------------------