├── .gitignore ├── requirements.txt ├── images ├── publish_edit.png ├── commit_publish.png ├── cos_publish_list.png ├── oss_publish_list.png ├── s3_publish_list.png └── server_publish_list.png ├── k8s ├── __init__.py ├── bulid_code.py ├── cron.yaml ├── bulid_image.py ├── pull_code.py └── deploy_app.py ├── server ├── __init__.py ├── custom_operation.py ├── bulid_code.py ├── backup_code.py ├── deploy_code.py ├── pull_code.py └── upload_code.py ├── bucker ├── __init__.py ├── upload_cos.py ├── upload_oss.py └── upload_s3.py ├── LICENSE ├── settings.py ├── cmdb_api.py ├── publish_api.py ├── public.py ├── api_handler.py ├── get_publish_info.py └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .git 2 | __pycache__ -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | fire 3 | paramiko 4 | pyotp 5 | coscmd 6 | oss2 7 | awscli 8 | -------------------------------------------------------------------------------- /images/publish_edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendevops-cn/codo-publish/HEAD/images/publish_edit.png -------------------------------------------------------------------------------- /images/commit_publish.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendevops-cn/codo-publish/HEAD/images/commit_publish.png -------------------------------------------------------------------------------- /images/cos_publish_list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendevops-cn/codo-publish/HEAD/images/cos_publish_list.png -------------------------------------------------------------------------------- /images/oss_publish_list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendevops-cn/codo-publish/HEAD/images/oss_publish_list.png -------------------------------------------------------------------------------- /images/s3_publish_list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendevops-cn/codo-publish/HEAD/images/s3_publish_list.png -------------------------------------------------------------------------------- /images/server_publish_list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendevops-cn/codo-publish/HEAD/images/server_publish_list.png -------------------------------------------------------------------------------- /k8s/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Contact : 191715030@qq.com 5 | Author : shenshuo 6 | Date : 2019/1/14 7 | Desc : 8 | """ -------------------------------------------------------------------------------- /server/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/6 9:25 4 | # @Author : Fred Yang 5 | # @File : __init__.py.py 6 | # @Role : 说明脚本功能 -------------------------------------------------------------------------------- /bucker/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/11 14:32 4 | # @Author : Fred Yang 5 | # @File : __init__.py.py 6 | # @Role : Client客户端静态资源发布 -------------------------------------------------------------------------------- /server/custom_operation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/7 14:22 4 | # @Author : Fred Yang 5 | # @File : custom_operation.py 6 | # @Role : 下发代码后的自定义操作 7 | 8 | print('[INFO]: 这部分是你的自定义操作，比如下发完代码后需要重启xxx进程， systemctl restart nginx') 9 | -------------------------------------------------------------------------------- /server/bulid_code.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/4 16:39 4 | # @Author : Fred Yang 5 | # @File : bulid_code.py 6 | # @Role : 这里是自定义脚本编译代码 7 | 8 | 9 | print('[INFO]: 这部分是你的自定义操作，你可以来编译你的代码，比如：npm install') 10 | print('[INFO]: Start building code...') 11 | 12 | 13 | -------------------------------------------------------------------------------- /k8s/bulid_code.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/4 16:39 4 | # @Author : Fred Yang 5 | # @File : bulid_code.py 6 | # @Role : 这里是自定义脚本编译代码 7 | 8 | 9 | print('[INFO]: 这部分是你的自定义操作，你可以来编译你的代码，比如：npm install') 10 | print('[INFO]: Start building code...') 11 | print('[INFO]: Build code successfully...') 12 | 13 | 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 CloudOpenDevOps 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /settings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/24 16:57 4 | # @Author : Fred Yang 5 | # @File : settings.py 6 | # @Role : 发布配置信息 7 | 8 | 9 | # 系统配置中的API地址,网关地址 10 | api_gw = 'http://gw.shinezone.net.cn/api' 11 | mail_api = '{}/mg/v2/notifications/mail/'.format(api_gw) 12 | login_api = '{}/accounts/login/'.format(api_gw) 13 | publish_info_api = '{}/task/v2/task_other/publish_cd/'.format(api_gw) 14 | docker_registry_info_api = '{}/task/v2/task_other/docker_registry/'.format(api_gw) 15 | get_key_api = '{}/task/v2/task/accept/'.format(api_gw) 16 | 17 | ### cmdb 读取get，获取CSRF 邮件发送post 镜像仓库信息get 18 | api_user = 'publish' 19 | api_pwd = 'shenshuo' 20 | api_key = 'JJFTQNLXHBYWSVSCINNEWNDFJRKWGY2UNJTTSVTLMN3UGUKXPFDE4NDH' 21 | 22 | api_settings = dict( 23 | api_user=api_user, 24 | api_pwd=api_pwd, 25 | api_key=api_key, 26 | api_gw=api_gw, 27 | mail_api=mail_api, 28 | login_api=login_api, 29 | publish_info_api=publish_info_api, 30 | get_key_api=get_key_api, 31 | docker_registry_info_api=docker_registry_info_api 32 | ) 33 | 34 | # 发布接口账户密码 35 | publish_info = { 36 | 'user': 'publish', 37 | 'password': 'shenshuo', 38 | 'key': 'JJFTQNLXHBYWSVSCINNEWNDFJRKWGY2UNJTTSVTLMN3UGUKXPFDE4NDH' 39 | } 40 | 41 | # CMDB接口账户密码 42 | cmdb_info = { 43 | 'user': 'cmdb', 44 | 'password': 'password', 45 | 'key': 'O42EEODGMFHGMYT2OAM3KN3HJVBE4Z3CKI3TKRDHG5MWSUCGMNHEE6LI' 46 | } 47 | -------------------------------------------------------------------------------- /k8s/cron.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: ss1917--do-cron 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: ss1917--do-cron 10 | template: 11 | metadata: 12 | labels: 13 | app: ss1917--do-cron 14 | spec: 15 | containers: 16 | - name: ss1917--do-cron 17 | image: harbor.xx.com/shinezonetest/ss1917--do_cron:v0.20181226R1 18 | imagePullPolicy: IfNotPresent 19 | # lifecycle: 20 | # postStart: 21 | # exec: 22 | # command: ['/bin/sh', '-c', 'sed -i "s/s_hostname/$(hostname)/" /data1/www/templates/app.html'] 23 | ports: 24 | - name: http 25 | containerPort: 80 26 | volumeMounts: 27 | - name: cronconf 28 | mountPath: /var/www/do_cron/settings.py 29 | subPath: settings.py #以文件的形式挂载 30 | readOnly: true 31 | # - name: cronnginx 32 | # mountPath: /etc/nginx/conf.d/ 33 | 34 | volumes: 35 | - name: cronconf 36 | configMap: 37 | name: cron-conf #configmap要提前创建好 kubectl create configmap cmdb-conf --from-file=./cmdb.conf -n qa 38 | #- name: cronnginx 39 | # configMap: 40 | # name: cron-nginx-conf # kubectl create configmap cmdb-nginx-conf --from-file=./nginx_cmdb.conf -n qa 41 | 42 | --- 43 | apiVersion: v1 44 | kind: Service 45 | metadata: 46 | name: ss1917--do-cron #k8s内网对应的域名为 codo-cmdb.qa.svc.cluster.local 47 | labels: 48 | app: ss1917--do-cron 49 | spec: 50 | selector: 51 | app: ss1917--do-cron 52 | #type: NodePort 53 | type: ClusterIP 54 | ports: 55 | - name: http 56 | port: 80 57 | targetPort: 80 58 | 59 | --- 60 | # api不需要通过ingress反代到外网 61 | apiVersion: extensions/v1beta1 62 | kind: Ingress 63 | metadata: 64 | name: ss1917--do-cron-ingress 65 | annotations: 66 | kubernetes.io/ingress.class: traefik 67 | spec: 68 | rules: 69 | - host: ops-cron.xx.net.cn 70 | http: 71 | paths: 72 | - path: 73 | backend: 74 | serviceName: ss1917--do-cron 75 | servicePort: 80 -------------------------------------------------------------------------------- /cmdb_api.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/7 10:24 4 | # @Author : Fred Yang 5 | # @File : cmdb_api.py 6 | # @Role : CMDB API 7 | 8 | 9 | import pyotp 10 | import requests 11 | import json 12 | from settings import api_gw, cmdb_info 13 | 14 | 15 | class CMDB_API(): 16 | def __init__(self): 17 | self.url = api_gw 18 | self.user = cmdb_info.get('user') 19 | self.pwd = cmdb_info.get('password') 20 | self.key = cmdb_info.get('key') 21 | 22 | @property 23 | def get_mfa(self): 24 | t = pyotp.TOTP(self.key) 25 | return t.now() 26 | 27 | def login(self): 28 | try: 29 | headers = {"Content-Type": "application/json"} 30 | params = {"username": self.user, "password": self.pwd, "dynamic": self.get_mfa} 31 | result = requests.post('%s/accounts/login/' % self.url, data=json.dumps(params), headers=headers) 32 | ret = json.loads(result.text) 33 | if ret['code'] == 0: 34 | print(ret['auth_key']) 35 | return ret['auth_key'] 36 | else: 37 | print(ret) 38 | print(ret['msg']) 39 | exit(1) 40 | except Exception as e: 41 | print('[Error:] CMDB用户登陆失败，错误信息：{}'.format(e)) 42 | 43 | def get_ec2_info(self, publish_host_api): 44 | """ 45 | 获取主机信息 46 | :param publish_host_api: 前端传来 47 | :return: 48 | """ 49 | 50 | token = self.login() 51 | try: 52 | # res = requests.get('%s/cmdb/api/cmdb/server_list/' % self.url, params=params, cookies=dict(auth_key=token)) 53 | res = requests.get('{}'.format(publish_host_api), cookies=dict(auth_key=token)) 54 | print('CMDB_API request Status:{}'.format(res.status_code)) 55 | # ret = str(res.content,'utf-8') py3 56 | ret = str(res.text) 57 | data = json.loads(ret) 58 | return data 59 | except Exception as e: 60 | print('[Error:] CMDB获取机器信息失败，错误信息：{}'.format(e)) 61 | exit(-2) 62 | 63 | # if __name__ == '__main__': 64 | # obj = CMDB_API() 65 | # obj.login() 66 | -------------------------------------------------------------------------------- /publish_api.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/5 10:24 4 | # @Author : Fred Yang 5 | # @File : publish_api.py 6 | # @Role : 发布接口API 7 | 8 | 9 | import pyotp 10 | import requests 11 | import json 12 | from settings import api_gw, publish_info 13 | 14 | 15 | class Publish_API(): 16 | def __init__(self): 17 | self.url = api_gw 18 | self.user = publish_info.get('user') 19 | self.pwd = publish_info.get('password') 20 | self.key = publish_info.get('key') 21 | 22 | @property 23 | def get_mfa(self): 24 | t = pyotp.TOTP(self.key) 25 | return t.now() 26 | 27 | def login(self): 28 | try: 29 | headers = {"Content-Type": "application/json"} 30 | params = {"username": self.user, "password": self.pwd, "dynamic": self.get_mfa} 31 | result = requests.post('%s/accounts/login/' % self.url, data=json.dumps(params), headers=headers) 32 | 33 | ret = json.loads(result.text) 34 | if ret['code'] == 0: 35 | return ret['auth_key'] 36 | else: 37 | print(ret) 38 | print(ret['msg']) 39 | exit(1) 40 | except Exception as e: 41 | print('[Error:] Publish用户接口登陆失败，错误信息：{}'.format(e)) 42 | 43 | def get_publish_name_info(self, publish_name): 44 | token = self.login() 45 | try: 46 | params = {'key': 'publish_name', 'value': publish_name} 47 | res = requests.get('%s/task/v2/task_other/publish_cd/' % self.url, params=params, 48 | cookies=dict(auth_key=token)) 49 | ret = json.loads(res.content) 50 | if ret['code'] == 0: return ret['data'] 51 | except Exception as e: 52 | print('[Error:] Publish发布接口连接失败，错误信息：{}'.format(e)) 53 | exit(-2) 54 | 55 | def get_publish_all_info(self): 56 | '''获取发布配置所有信息''' 57 | try: 58 | token = self.login() 59 | except Exception as e: 60 | print(e) 61 | token = self.login() 62 | 63 | res = requests.get('%s/task/v2/task_other/publish_cd/' % self.url, cookies=dict(auth_key=token)) 64 | ret = json.loads(res.content) 65 | if ret['code'] == 0: return ret['data'] 66 | -------------------------------------------------------------------------------- /public.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/4 13:18 4 | # @Author : Fred Yang 5 | # @File : public.py 6 | # @Role : 公共工具 7 | 8 | import os 9 | import sys 10 | import re 11 | import time 12 | import subprocess 13 | import paramiko 14 | import concurrent.futures 15 | 16 | 17 | def lock_json(script_name): 18 | """文件锁""" 19 | pid_file = '/tmp/publish_data_json.pid' 20 | lock_count = 0 21 | while True: 22 | if os.path.isfile(pid_file): 23 | ###打开脚本运行进程id文件并读取进程id 24 | with open(pid_file, 'r') as fp_pid: 25 | process_id = fp_pid.read() 26 | 27 | ###判断pid文件取出的是否是数字 28 | if not process_id: 29 | break 30 | 31 | if not re.search(r'^\d', process_id): 32 | break 33 | 34 | ###确认此进程id是否还有进程 35 | lock_count += 1 36 | if lock_count > 30: 37 | print('1 min after this script is still exists') 38 | sys.exit(111) 39 | else: 40 | check_list = os.popen('/bin/ps %s|grep "%s"' % (process_id, script_name)).readlines() 41 | if check_list: 42 | print('check_list--->', check_list) 43 | print('cmd ----> /bin/ps %s|grep "%s"' % (process_id, script_name)) 44 | print("The script is running...... ,Please wait for a moment!") 45 | time.sleep(5) 46 | else: 47 | os.remove(pid_file) 48 | else: 49 | break 50 | 51 | ###把进程号写入文件 52 | with open(pid_file, 'w') as wp_pid: 53 | p_id = os.getpid() 54 | wp_pid.write(str(p_id)) 55 | 56 | 57 | def exec_shell(cmd): 58 | '''执行shell命令函数''' 59 | sub2 = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 60 | stdout, stderr = sub2.communicate() 61 | ret = sub2.returncode 62 | if ret == 0: 63 | return ret, stdout.decode('utf-8').split('\n') 64 | else: 65 | return ret, stdout.decode('utf-8').replace('\n', '') 66 | 67 | 68 | def ssh_connect(ip, port, user, password): 69 | try: 70 | _ssh_fd = paramiko.SSHClient() 71 | _ssh_fd.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 72 | _ssh_fd.connect(hostname=ip, port=port, username=user, password=password) 73 | return _ssh_fd 74 | except Exception as e: 75 | print('[Error]: ssh {}@{} {}'.format(user, ip, e)) 76 | 77 | 78 | def exec_thread(func, iterable1): 79 | ### 取所有主机 最多启动100个进程 80 | pool_num = 10 81 | with concurrent.futures.ProcessPoolExecutor(pool_num) as executor: 82 | executor.map(func, iterable1) 83 | -------------------------------------------------------------------------------- /server/backup_code.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/5 18:35 4 | # @Author : Fred Yang 5 | # @File : backup_code.py 6 | # @Role : 备份代码 7 | 8 | 9 | import os 10 | import sys 11 | 12 | Base_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 13 | sys.path.append(Base_DIR) 14 | import uuid 15 | from public import exec_shell 16 | from public import exec_thread 17 | 18 | from get_publish_info import get_publish_data, get_all_hosts 19 | import fire 20 | 21 | 22 | class BackupCode(): 23 | def __init__(self, data): 24 | self.publish_path = data.get('publish_path') # 发布目录 25 | self.repository = data.get('repository') # 代码仓库 26 | self.repo_name = self.repository.split('/')[-1].replace('.git', '') # 仓库名字 27 | self.local_dir = '/tmp/' 28 | self.backup_dir = '/tmp/code_backup/' 29 | self.uuid_file = '/tmp/publish_{}'.format(uuid.uuid1()) # 错误判断该使用。 30 | 31 | def code_backup(self, host): 32 | """ 33 | :param host: 主机信息，IP端口用户密码 34 | :return: 35 | """ 36 | if not isinstance(host, dict): 37 | raise ValueError() 38 | 39 | ip = host.get('ip') 40 | port = host.get('port', 22) 41 | user = host.get('user', 'root') 42 | password = host.get('password') 43 | code_path = self.publish_path + self.repo_name 44 | try: 45 | backup_cmd = '[ ! -d "{}" ] && mkdir -p {} ; [ ! -d "{}" ] && mkdir -p {} ; cp -aR {} {} && echo success'.format( 46 | self.backup_dir, self.backup_dir, code_path, code_path, code_path, self.backup_dir) 47 | ssh_cmd = "sshpass -p {} ssh -p {} -o StrictHostKeyChecking=no {}@{} '{}'".format(password, port, user, 48 | ip, 49 | backup_cmd) 50 | 51 | ssh_status, ssh_output = exec_shell(ssh_cmd) 52 | if ssh_status == 0: 53 | print('[Success]: Host: {} 备份路径: {}'.format(ip, self.backup_dir)) 54 | else: 55 | os.mknod(self.uuid_file) 56 | print('[Error]: Host: {} 备份失败，信息: {} faild'.format(ip, ssh_output)) 57 | except Exception as e: 58 | print(e) 59 | exit(-500) 60 | 61 | def check_err(self): 62 | if os.path.exists(self.uuid_file): 63 | print('[Error]') 64 | exit(-1) 65 | 66 | 67 | def main(flow_id): 68 | print('[INFO]: 这部分是备份你目标主机的代码,只保留一天备份，若不需要可以跳过此步骤') 69 | data = get_publish_data(flow_id) # 获取发布信息 70 | obj = BackupCode(data) 71 | all_hosts = get_all_hosts(flow_id) 72 | exec_thread(func=obj.code_backup, iterable1=all_hosts) 73 | obj.check_err() 74 | 75 | 76 | if __name__ == '__main__': 77 | fire.Fire(main) 78 | -------------------------------------------------------------------------------- /server/deploy_code.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/7 13:42 4 | # @Author : Fred Yang 5 | # @File : deploy_code.py 6 | # @Role : 部署代码，下发代码，此步将代码最终发布到代码目录 7 | 8 | 9 | import os 10 | import sys 11 | 12 | Base_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 13 | sys.path.append(Base_DIR) 14 | import uuid 15 | from public import exec_shell 16 | from public import exec_thread 17 | from get_publish_info import get_publish_data, get_all_hosts 18 | import fire 19 | 20 | 21 | class DeployCode(): 22 | 23 | def __init__(self, data): 24 | self.publish_path = data.get('publish_path') # 发布目录 25 | self.repository = data.get('repository') # 代码仓库 26 | self.repo_name = self.repository.split('/')[-1].replace('.git', '') # 仓库名字 27 | self.uuid_file = '/tmp/publish_{}'.format(uuid.uuid1()) #错误文件判断用。 28 | 29 | def code_deploy(self, host): 30 | if not isinstance(host, dict): 31 | raise ValueError() 32 | 33 | '''/tmp下的代码是upload_code脚本上传过来的''' 34 | tmp_code_path = '/tmp/{}'.format(self.repo_name) 35 | # if not os.path.exists(tmp_code_path): 36 | # print('[Error]: No code found') 37 | # sys.exit(-100) 38 | 39 | ip = host.get('ip') 40 | port = host.get('port', 22) 41 | user = host.get('user', 'root') 42 | password = host.get('password') 43 | # code_path = self.publish_path + self.repo_name 44 | # depoly_cmd = "sshpass -p {} rsync -ahqzt --delete -e 'ssh -p {} -o StrictHostKeyChecking=no ' {} {}@{}:{}".format( 45 | # password, port, tmp_code_path, user, ip, self.publish_path) 46 | rsync_cmd = 'rsync -ahqzt --delete {} {}'.format(tmp_code_path, self.publish_path) 47 | depoly_cmd = "sshpass -p {} ssh -p {} -o StrictHostKeyChecking=no {}@{} '{}'".format( 48 | password, 49 | port, 50 | user, ip, 51 | rsync_cmd) 52 | # print('[CMD:]', depoly_cmd) 53 | 54 | try: 55 | depoly_status, depoly_output = exec_shell(depoly_cmd) 56 | if depoly_status == 0: 57 | print('[Success]: Host:{} 发布成功'.format(ip)) 58 | else: 59 | os.mknod(self.uuid_file) 60 | print('[Error]: Host:{} 失败，错误信息: {}'.format(ip, depoly_output)) 61 | exit(-3) 62 | 63 | except Exception as e: 64 | print(e) 65 | exit(-500) 66 | 67 | def check_err(self): 68 | if os.path.exists(self.uuid_file): 69 | print('[Error]') 70 | exit(-1) 71 | 72 | def main(flow_id): 73 | print('[INFO]: 这部分是将代码正式下发/同步到你的代码目录') 74 | data = get_publish_data(flow_id) # 获取发布信息 75 | obj = DeployCode(data) 76 | all_hosts = get_all_hosts(flow_id) 77 | exec_thread(func=obj.code_deploy, iterable1=all_hosts) 78 | obj.check_err() 79 | 80 | 81 | if __name__ == '__main__': 82 | fire.Fire(main) 83 | -------------------------------------------------------------------------------- /k8s/bulid_image.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Contact : 191715030@qq.com 5 | Author : shenshuo 6 | Date : 2019/1/14 7 | Desc : 编译docker镜像 并上传 8 | """ 9 | import os 10 | import sys 11 | import fire 12 | 13 | Base_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 14 | sys.path.append(Base_DIR) 15 | from public import exec_shell 16 | from get_publish_info import get_publish_data 17 | from settings import api_settings 18 | from api_handler import API 19 | 20 | 21 | class BuildImage(object): 22 | def __init__(self, data, tag): 23 | self.git_tag = tag 24 | self.docker_registry = data.get('docker_registry') 25 | self.repository = data.get('repository') # 代码仓库 26 | self.app_name = self.repository.split('/')[-1].replace('.git', '') 27 | self.version_dir = os.path.join('/var/www/version/', self.repository.split('/')[-2]) 28 | self.code_dir = os.path.join(self.version_dir, self.app_name) 29 | self.image_name = "{}{}--{}:{}".format(self.docker_registry, self.repository.split('/')[-2], self.app_name, 30 | self.git_tag) 31 | 32 | def build_image(self): 33 | cmd = "cd {} && docker build . -t {}".format(self.code_dir, self.image_name) 34 | exec_status, exec_output = exec_shell(cmd) 35 | if exec_status == 0: 36 | print('[Success]: docker build {} OK...'.format(self.image_name)) 37 | else: 38 | print('[Error]: docker build {} failed ...\n[Error]: {}'.format(self.image_name, exec_output)) 39 | exit(-1) 40 | 41 | def login(self): 42 | obj = API() 43 | result = obj.get_api_info(api_settings['docker_registry_info_api']) 44 | for i in result: 45 | if i['registry_url'] == self.docker_registry: 46 | login_cmd = "docker login -u {} -p {} {}".format(i['user_name'], i['password'], i['registry_url']) 47 | print(login_cmd) 48 | exec_status, exec_output = exec_shell(login_cmd) 49 | if exec_status == 0: 50 | print('[Success]: docker login {} OK...'.format(self.docker_registry)) 51 | else: 52 | print('[Error]: docker login {} failed ...\n[Error]: {}'.format(self.docker_registry, exec_output)) 53 | exit(-1) 54 | 55 | def push_image(self): 56 | self.login() 57 | cmd = "docker push {}".format(self.image_name) 58 | exec_status, exec_output = exec_shell(cmd) 59 | if exec_status == 0: 60 | print('[Success]: docker push {} successfully...'.format(self.image_name)) 61 | else: 62 | print('[Error]: docker push {} failed ...\n[Error]: {}'.format(self.image_name, exec_output)) 63 | exit(-1) 64 | 65 | 66 | def main(flow_id, git_tag): 67 | """ 68 | :param flow_id: 订单ID 69 | :param git_tag: Git Tag名字 70 | :return: 71 | """ 72 | print('[INFO]: 这部分是用来在编译镜像，并且上传docker仓库') 73 | data = get_publish_data(flow_id) # 配置信息 74 | obj = BuildImage(data, git_tag) # 初始化类 75 | obj.build_image() # 编译镜像 76 | obj.push_image() # 上传镜像 77 | 78 | 79 | if __name__ == '__main__': 80 | fire.Fire(main) 81 | -------------------------------------------------------------------------------- /k8s/pull_code.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/4 11:00 4 | # @Author : Fred Yang 5 | # @File : pull_code.py 6 | # @Role : 02. 获取代码 7 | 8 | 9 | import os 10 | import sys 11 | import fire 12 | Base_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 13 | sys.path.append(Base_DIR) 14 | from public import exec_shell 15 | from get_publish_info import get_publish_data 16 | 17 | 18 | class PullCode(object): 19 | """ 20 | 通过git地址拉取代码 21 | 首次： git clone 22 | 存在： git fetch --all && git pull 23 | """ 24 | 25 | def __init__(self, data): 26 | self.publish_path = data.get('publish_path') # 发布目录 27 | self.repository = data.get('repository') # 代码仓库 28 | self.repo_name = self.repository.split('/')[-1].replace('.git', '') # 仓库名字 29 | self.version_dir = os.path.join('/var/www/version/',self.repository.split('/')[-2]) 30 | self.code_dir = os.path.join(self.version_dir, self.repo_name) 31 | 32 | def git_clone(self): 33 | """检测发布目录没有代码则进行git clone""" 34 | try: 35 | if not os.path.exists(self.code_dir): 36 | print('[INFO]: Start pulling a new codebase to {}'.format(self.code_dir)) 37 | ### 克隆代码 38 | git_clone_cmd = 'mkdir -p {} ;cd {} && git clone {}'.format(self.version_dir, self.version_dir, 39 | self.repository) 40 | print('[CMD:] ', git_clone_cmd) 41 | git_clone_status, git_clone_output = exec_shell(git_clone_cmd) 42 | if git_clone_status == 0: 43 | print('[Success]: git clone {} successfully...'.format(self.repository)) 44 | else: 45 | print('[Error]: git clone {} failed ...'.format(self.repository)) 46 | exit(404) 47 | else: 48 | print('[PASS]： The repository already exists, skip the clone and update directly') 49 | except Exception as e: 50 | print(e) 51 | 52 | def checkout_tag(self, git_tag): 53 | """切换分支""" 54 | git_fetch_cmd = 'cd {} && git fetch -t -p -f && git fetch --all'.format(self.code_dir) # 更新代码 55 | git_checkout_cmd = 'cd {} && git clean -df && git checkout {}'.format(self.code_dir, git_tag) # 切换分支 56 | 57 | try: 58 | git_fetch_status, git_fetch_output = exec_shell(git_fetch_cmd) 59 | if git_fetch_status == 0: 60 | git_check_status, git_check_output = exec_shell(git_checkout_cmd) 61 | if git_check_status == 0: 62 | print('[Success]: git checkout tag: {} successfully...'.format(git_tag)) 63 | else: 64 | print('[Error]: git checkout tag: {} failed ...'.format(git_tag)) 65 | exit(402) 66 | else: 67 | print('[Error]: git fetch failed ...') 68 | exit(403) 69 | except Exception as e: 70 | print(e) 71 | exit(-500) 72 | 73 | 74 | def main(flow_id, git_tag): 75 | """ 76 | :param flow_id: 订单ID 77 | :param git_tag: Git Tag名字 78 | :return: 79 | """ 80 | print('[INFO]: 这部分是用来在构建机器上拉取代码，切换Tag操作') 81 | data = get_publish_data(flow_id) # 配置信息 82 | obj = PullCode(data) # 初始化类 83 | obj.git_clone() # 克隆代码 84 | obj.checkout_tag(git_tag) # 切换Tag 85 | 86 | 87 | if __name__ == '__main__': 88 | fire.Fire(main) 89 | 90 | ### python3 /home/dev/python_dev/codo-publish/k8s/pull_code.py 231 v0.20181226R1 -------------------------------------------------------------------------------- /k8s/deploy_app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Contact : 191715030@qq.com 5 | Author : shenshuo 6 | Date : 2019/1/15 7 | Desc : 滚动升级 8 | """ 9 | 10 | import os 11 | import sys 12 | import fire 13 | 14 | Base_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 15 | sys.path.append(Base_DIR) 16 | from public import exec_shell 17 | from get_publish_info import get_publish_data 18 | 19 | 20 | class Deploy(object): 21 | def __init__(self, data, tag): 22 | self.git_tag = tag 23 | self.docker_registry = data.get('docker_registry') 24 | self.repository = data.get('repository') # 代码仓库 25 | self.app_name = self.repository.split('/')[-1].replace('.git', '') 26 | self.deploy_name = "{}--{}".format(self.repository.split('/')[-2], self.app_name).replace("_", "-") 27 | self.image_name = "{}{}--{}:{}".format(self.docker_registry, self.repository.split('/')[-2], self.app_name, 28 | self.git_tag) 29 | self.k8s_host = data.get('k8s_host') 30 | self.namespace = data.get('namespace') 31 | 32 | def run(self): 33 | cmd = "kubectl set image deployment/{} {}={} -n {}".format(self.deploy_name, self.deploy_name, self.image_name, 34 | self.namespace) 35 | exec_status, exec_output = exec_shell("ssh {} '{}'".format(self.k8s_host, cmd)) 36 | if exec_status == 0: 37 | print('[Success]: kubernetes rolling-update {} successfully...'.format(self.image_name)) 38 | else: 39 | print('[Error]: kubernetes rolling-update {} failed ...\n[Error]: {}'.format(self.image_name, exec_output)) 40 | exit(-1) 41 | 42 | def check(self): 43 | cmd = "ssh %s kubectl get pod -n %s -o wide -l app=%s |grep -v 'NAME' |awk '{print($3)}'" % (self.k8s_host, 44 | self.namespace, 45 | self.deploy_name) 46 | 47 | exec_status, exec_output = exec_shell(cmd) 48 | if exec_status == 0 and exec_output[0] == 'Running': 49 | print('[Success]: The pod status is running...') 50 | else: 51 | print('[Error]: The pod status is not running...\n[Error]: {}'.format(exec_output)) 52 | print('[Warning] rollout undo deployment start !!!!') 53 | rollback_cmd = "ssh {} kubectl rollout undo -n {} deployment {}".format(self.k8s_host, self.namespace, 54 | self.deploy_name) 55 | exec_status1, exec_output1 = exec_shell(rollback_cmd) 56 | print("{} {}".format(exec_status1, exec_output1)) 57 | print("Roll back to the last version end") 58 | exit(-2) 59 | 60 | 61 | def main(flow_id, git_tag): 62 | """ 63 | :param flow_id: 订单ID 64 | :param git_tag: Git Tag名字 65 | :return: 66 | """ 67 | print('[INFO]: 这部分是用来在编译镜像，并且上传docker仓库') 68 | data = get_publish_data(flow_id) # 配置信息 69 | obj = Deploy(data, git_tag) # 初始化类 70 | obj.run() # 滚动升级 71 | obj.check() # 检查POD状态 72 | 73 | 74 | if __name__ == '__main__': 75 | fire.Fire(main) 76 | 77 | ### 注意事项 ssh kubernetes master可到以直连 要设置镜像仓库的key。命名空间隔离，所以每个命名空间都要单独设置 78 | ### kubectl create secret docker-registry registrysecret --docker-server=harbor.shinezone.com --docker-username=admin --docker-password=xxxxxxxxxxxxx --docker-email=191715030@qq.com -n shenshuo 79 | ### kubectl patch serviceaccount default -p '{"imagePullSecrets": [{"name": "registrysecret"}]}' -n shenshuo 80 | ### python3 /home/dev/python_dev/codo-publish/k8s/deploy_app.py 234 v0.20181226R1 81 | -------------------------------------------------------------------------------- /server/pull_code.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/4 11:00 4 | # @Author : Fred Yang 5 | # @File : pull_code.py 6 | # @Role : 02. 获取代码 7 | 8 | 9 | import os 10 | import sys 11 | 12 | Base_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 13 | sys.path.append(Base_DIR) 14 | from public import exec_shell 15 | from get_publish_info import get_publish_data 16 | import fire 17 | 18 | 19 | class PullCode(): 20 | """ 21 | 通过git地址拉取代码 22 | 首次： git clone 23 | 存在： git tetch --all && git pull 24 | """ 25 | 26 | def __init__(self, data): 27 | self.publish_path = data.get('publish_path') # 发布目录 28 | self.repository = data.get('repository') # 代码仓库 29 | self.repo_name = self.repository.split('/')[-1].replace('.git', '') # 仓库名字 30 | 31 | def git_clone(self): 32 | '''检测发布目录没有代码则进行git clone''' 33 | code_dir = self.publish_path + self.repo_name 34 | try: 35 | if not os.path.exists(code_dir): 36 | print('[INFO]: Start pulling a new codebase to {}'.format(code_dir)) 37 | git_clone_cmd = '[ ! -d "{}" ] && mkdir {} ;cd {} && git clone {}'.format(self.publish_path, 38 | self.publish_path, 39 | self.publish_path, 40 | self.repository) # 克隆代码 41 | print('[CMD:] ', git_clone_cmd) 42 | git_clone_status, git_clone_output = exec_shell(git_clone_cmd) 43 | if git_clone_status == 0: 44 | print('[Success]: git clone {} sucess...'.format(self.repository)) 45 | else: 46 | print('[Error]: git clone {} faild...'.format(self.repository)) 47 | exit(404) 48 | else: 49 | print('[PASS]： The repository already exists, skip the clone and update directly') 50 | except Exception as e: 51 | print(e) 52 | 53 | def checkout_tag(self, git_tag): 54 | '''切换分支''' 55 | git_fetch_cmd = 'cd {}/{} && git fetch -t -p -f && git fetch --all'.format(self.publish_path, 56 | 57 | self.repo_name) # 更新代码 58 | git_checkout_cmd = 'cd {}/{} && git clean -df && git checkout {}'.format( 59 | self.publish_path, self.repo_name, git_tag) # 切换分支 60 | 61 | # print('[CMD:]', git_fetch_cmd) 62 | # print('[CMD:]', git_checkout_cmd) 63 | try: 64 | git_fetch_status, git_fetch_output = exec_shell(git_fetch_cmd) 65 | if git_fetch_status == 0: 66 | git_check_status, git_check_output = exec_shell(git_checkout_cmd) 67 | if git_check_status == 0: 68 | print('[Success]: git checkout tag: {} successfully...'.format(git_tag)) 69 | else: 70 | print('[Error]: git checkout tag: {} faild...'.format(git_tag)) 71 | exit(402) 72 | else: 73 | print('[Error]: git fetch faild...') 74 | exit(403) 75 | except Exception as e: 76 | print(e) 77 | exit(-500) 78 | 79 | 80 | def main(flow_id, git_tag): 81 | """ 82 | :param flow_id: 订单ID 83 | :param git_tag: Git Tag名字 84 | :return: 85 | """ 86 | print('[INFO]: 这部分是用来在构建机器上拉取代码，切换Tag操作') 87 | data = get_publish_data(flow_id) # 配置信息 88 | obj = PullCode(data) # 初始化类 89 | obj.git_clone() # 克隆代码 90 | obj.checkout_tag(git_tag) # 切换Tag 91 | 92 | 93 | if __name__ == '__main__': 94 | fire.Fire(main) 95 | -------------------------------------------------------------------------------- /api_handler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | Contact : 191715030@qq.com 5 | Author : shenshuo 6 | Date : 2018/12/25 7 | Desc : 8 | """ 9 | 10 | import pyotp 11 | import requests 12 | import json 13 | from settings import api_settings 14 | 15 | 16 | 17 | class API(object): 18 | def __init__(self): 19 | self.url = api_settings['api_gw'] 20 | self.user = api_settings['api_user'] 21 | self.pwd = api_settings['api_pwd'] 22 | self.key = api_settings.get('api_key') 23 | self.login_api = api_settings.get('login_api') 24 | self.publish_info_api = api_settings.get('publish_info_api') 25 | self.get_key_api = api_settings.get('get_key_api') 26 | self.mail_api = api_settings.get('mail_api') 27 | 28 | @property 29 | def get_mfa(self): 30 | t = pyotp.TOTP(self.key) 31 | return t.now() 32 | 33 | def login(self): 34 | try: 35 | headers = {"Content-Type": "application/json"} 36 | params = {"username": self.user, "password": self.pwd, "dynamic": self.get_mfa} 37 | result = requests.post(self.login_api, data=json.dumps(params), headers=headers) 38 | 39 | ret = json.loads(result.text) 40 | if ret['code'] == 0: 41 | return ret['auth_key'] 42 | else: 43 | print(ret) 44 | print(ret['msg']) 45 | exit(1) 46 | except Exception as e: 47 | print('[Error:] 用户:{} 接口登陆失败，错误信息：{}'.format(self.user, e)) 48 | 49 | def get_publish_name_info(self, publish_name): 50 | token = self.login() 51 | try: 52 | params = {'key': 'publish_name', 'value': publish_name} 53 | res = requests.get(self.publish_info_api, params=params, cookies=dict(auth_key=token)) 54 | ret = json.loads(res.content) 55 | if ret['code'] == 0: return ret['data'] 56 | except Exception as e: 57 | print('[Error:] Publish发布接口连接失败，错误信息：{}'.format(e)) 58 | exit(-2) 59 | 60 | def get_publish_all_info(self): 61 | '''获取发布配置所有信息''' 62 | try: 63 | token = self.login() 64 | except Exception as e: 65 | print(e) 66 | token = self.login() 67 | 68 | res = requests.get(self.publish_info_api, cookies=dict(auth_key=token)) 69 | ret = json.loads(res.content) 70 | if ret['code'] == 0: return ret['data'] 71 | 72 | def get_api_info(self, api_url): 73 | '''获取接口所有信息''' 74 | try: 75 | token = self.login() 76 | except Exception as e: 77 | print(e) 78 | token = self.login() 79 | 80 | res = requests.get(api_url, cookies=dict(auth_key=token)) 81 | ret = json.loads(res.content) 82 | return ret['data'] 83 | 84 | def send_mail_for_api(self, mail_list, mail_subject, mail_content): 85 | """发送邮件""" 86 | try: 87 | token = self.login() 88 | except Exception as e: 89 | print(e) 90 | token = self.login() 91 | 92 | req1 = requests.get(self.get_key_api, cookies=dict(auth_key=token)) 93 | if req1.status_code != 200: 94 | raise SystemExit(req1.status_code) 95 | 96 | ret1 = json.loads(req1.text) 97 | if ret1['code'] == 0: 98 | csrf_key = ret1['csrf_key'] 99 | else: 100 | raise SystemExit(ret1['code']) 101 | 102 | body = json.dumps({"to_list": mail_list, "subject": mail_subject, "content": mail_content, "subtype": "plain"}) 103 | res = requests.post(self.mail_api, data=body, cookies=dict(auth_key=token, csrf_key=csrf_key)) 104 | if res.status_code != 200: 105 | raise SystemExit(res.status_code) 106 | else: 107 | return json.loads(res.text)['msg'] 108 | -------------------------------------------------------------------------------- /get_publish_info.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/5 10:29 4 | # @Author : Fred Yang 5 | # @File : get_publish_info.py 6 | # @Role : 获取发布信息写文件 7 | 8 | 9 | import os 10 | import sys 11 | import json 12 | from publish_api import Publish_API 13 | from cmdb_api import CMDB_API 14 | from public import lock_json 15 | import fire 16 | 17 | 18 | def data_save(file_name, data): 19 | """ 20 | :param data: 通过API取到发布配置信息 21 | :param file_name: 配置信息存储文件路径 22 | :return: 23 | """ 24 | lock_json(sys.argv[0]) 25 | if not os.path.exists(file_name): 26 | os.system("echo {} > %s" % file_name) 27 | 28 | try: 29 | f = open(file_name, 'w', encoding='utf-8') 30 | jsObj = json.dumps(data) 31 | f.write(jsObj) 32 | f.close() 33 | print('[INFO]: Publish info has been written :{}'.format(file_name)) 34 | except Exception as e: 35 | print(e) 36 | print('[Error]: Publish info write falid') 37 | 38 | 39 | def get_publish_data(flow_id): 40 | """ 41 | 获取发布配置文件信息 42 | :param flow_id: 43 | :return: 44 | """ 45 | file_name = '/tmp/publish_{}.json'.format(flow_id) 46 | if not os.path.exists(file_name): 47 | print('[Error]: Not Fount config file... ') 48 | exit(400) 49 | else: 50 | f = open(file_name, 'r', encoding='utf-8') 51 | for line in f: 52 | ret = json.loads(line) 53 | for data in ret: 54 | return data 55 | 56 | 57 | def get_all_hosts(flow_id): 58 | """ 59 | 获取所有主机，需要处理以下： 60 | 1. 用户手动输入的主机 publish_host 61 | 2. 用户从CMDB里面调用的主机 cmdb_host 62 | :param flow_id: 63 | :return: 64 | """ 65 | # 将所有主机放到一个列表里面 66 | all_hosts = [] 67 | # 首先处理publish_host主机 68 | data = get_publish_data(flow_id) 69 | publish_hosts_str = data.get('publish_hosts') 70 | publish_hosts_list = publish_hosts_str.split('\n') 71 | keys_list = ['ip', 'port', 'user', 'password'] 72 | for publish_host in publish_hosts_list: 73 | values_list = publish_host.split(' ') 74 | host_dict = dict(zip(keys_list, values_list)) 75 | all_hosts.append(host_dict) 76 | 77 | # 处理CMDB主机类型 78 | publish_host_api = data.get('publish_hosts_api') 79 | if not publish_host_api: 80 | print('[INFO]: 没有获取到CMDB主机信息，自动跳过') 81 | else: 82 | cmdb_obj = CMDB_API() 83 | cmdb_hosts_list = cmdb_obj.get_ec2_info(publish_host_api) 84 | for cmdb_host in cmdb_hosts_list: 85 | cmdb_host_dict = { 86 | "ip": cmdb_host.get('ip'), 87 | "port": cmdb_host.get('port'), 88 | "user": cmdb_host.get('username'), 89 | "password": cmdb_host.get('password'), 90 | } 91 | all_hosts.append(cmdb_host_dict) 92 | 93 | return all_hosts 94 | 95 | # publish_hosts = {'172.16.0.20': ['22', 'root', 'None'], '172.16.0.228': ['22', 'root', '123456'], 96 | # # '172.16.0.101': ['22', 'root', '1']} 97 | # all_hosts = [] 98 | # for hosts_key in publish_hosts.keys(): 99 | # hosts_value = publish_hosts[hosts_key] 100 | # hosts_data = { 101 | # 'ip': hosts_key, 102 | # 'port': hosts_value[0], 103 | # 'user': hosts_value[1], 104 | # 'password': hosts_value[2] 105 | # } 106 | # all_hosts.append(hosts_data) 107 | # print('all_hosts----->',all_hosts) 108 | # return all_hosts 109 | 110 | 111 | def main(publish_name, flow_id): 112 | """ 113 | :param publish_name: 发布应用的名称 114 | :param flow_id: 订单ID 115 | :return: 116 | """ 117 | file_name = '/tmp/publish_{}.json'.format(flow_id) 118 | obj = Publish_API() 119 | data = obj.get_publish_name_info(publish_name) 120 | data_save(file_name, data) 121 | 122 | 123 | if __name__ == '__main__': 124 | fire.Fire(main) 125 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # codo-publish 3 | 4 | > codo平台发布示例，支持服务器发布和Bucket发布，只有后端代码逻辑，所有的配置信息都是从前端传过来的。 5 | 6 | ## Server发布 7 | - `get_publish_info.py`: 获取发布信息写文件 8 | - `pull_code.py`: 构建主机拉取代码 9 | - `bulid_code.py`: 编译代码，如：`npm install` 10 | - `upload_code.py`: 处理exclude过滤后将代码并发到目标主机的`/tmp`目录 11 | - `backup_code.py`: 备份目标主机的代码到：`/tmp/code_backup`，只保留最近一个版本 12 | - `depoly_code.py`: 部署/下发代码到目标主机的代码目录，并发操作 13 | - `custom_operation.py`: 自定义操作，比如下发完代码后需要重启xxx进程， `systemctl restart nginx` 14 | 15 | ## Bucket发布 16 | 17 | - `upload_s3`：上传资源到AWS S3 18 | - `upload_cos`: 上传资源到腾讯云COS 19 | - `upload_oss`: 上传资源到阿里云OSS 20 | 21 | 22 | 23 | ## 开始使用 24 | 25 | ### 发布配置 26 | 27 | > 预先配置你发布的应用信息，如：环境、主机、Git库等， 28 | 29 | 30 | 例举例一个简单发布示例： 31 | 32 | - ①创建命令 33 | - ②配置模板 34 | - ③配置仓库 35 | - ④新建应用 36 | - ⑤提交发布 37 | 38 | **创建命令** 39 | 40 | > 我们提供了基于`服务器`和`Bucket`基础发布脚本示例供你使用. 41 | 42 | 克隆代码 43 | ```shell 44 | ##将代码放到/opt/ops_scripts目录下，后续平台要使用到 45 | mkdir -p /opt/ops_scripts 46 | cd /opt/ops_scripts && git clone https://github.com/opendevops-cn/codo-publish.git 47 | cd codo-publish && pip3 install -r requirements.txt 48 | 49 | ##修改对应settings里面API网关地址，用户密码信息 50 | vim settings.py 51 | api_gw = 'gw.opendevops.cn' 52 | ``` 53 | 54 | 55 | 接着到平台添加命令，点开`任务模块`-`命令管理` 56 | 57 | ![bash_list](https://raw.githubusercontent.com/opendevops-cn/opendevops/master/docs/source/_static/images/publish_cmd_list.png) 58 | 59 | **配置模板** 60 | 点开`任务模块`-`模板管理` 61 | 62 | 63 | 64 | > 这里需要你创建一个模板，如：Server发布示例，添加你的命令，自由排版，模板管理文档参考：[任务模板](http://docs.opendevops.cn/zh/latest/task_template.html#) 65 | 66 | - 组：1的为第一组 67 | - 优先级： 按数字顺序执行 68 | - 参数：PUBLISH_NAME和FLOW_ID都是系统变量，不要修改 69 | - 执行用户： 默认127.0.0.1可不选，保持默认即可 70 | - 触发器：这里支持定时支持、人工干预、直接执行，默认是直接执行 71 | 72 | ![bash_list](https://raw.githubusercontent.com/opendevops-cn/opendevops/master/docs/source/_static/images/publish_server_temp.png) 73 | 74 | 75 | 76 | **配置代码仓库** 77 | 78 | 点开`作业配置`-`代码仓库` 79 | 80 | > 这里需要你配置一个git仓库地址，要发布的代码仓库。 81 | 82 | ![bash_list](https://raw.githubusercontent.com/opendevops-cn/opendevops/master/docs/source/_static/images/publish_code_repo.png) 83 | 84 | 85 | **新建应用** 86 | 点开`作业配置`-`发布配置` 87 | - 发布应用：建议一个有意义的名称 88 | - 发布类型：选择发布类型，支持(服务器、存储桶) 89 | - 构建主机：哪台主机执行操作，默认（127.0.0.1） 90 | - 代码仓库：下拉选择你的代码库 91 | - 关联模板：下拉选择你要关联的模板 92 | - 排除文件：exclude排除，如图，多个按行隔开 93 | - 发布类型：目前只支持简单发布 94 | - 目标主机：ip port user password格式，多个按行隔开 95 | - 目标主机组：CMDB里面API主机组的信息，可通过CMDB组里面点击获取API地址 96 | - 发布目录: 服务器的代码目录如：/var/wwww/yanghongfei/ 97 | 98 | ![](https://raw.githubusercontent.com/opendevops-cn/opendevops/master/docs/source/_static/images/publish_create_app.png) 99 | 100 | 101 | **提交发布** 102 | > 最后提交发布，发布是基于git tag的，需要你在要发布的版本库上打上tag 103 | 104 | ![](https://raw.githubusercontent.com/opendevops-cn/opendevops/master/docs/source/_static/images/publish_server_commit.png) 105 | 106 | 提交发布后会在订单列表里面显示出来你将要执行的任务， 107 | 当你点击到这个任务的时候，我们会把任务信息给展示出来方便你来确认，最后需要你进行审批，可选择时间执行。 108 | 109 | ![](https://raw.githubusercontent.com/opendevops-cn/opendevops/master/docs/source/_static/images/publish_audit.png) 110 | 111 | 如下图示例有一个手工干预的，需要你点击终止全部旁边的干预名称进行执行。 112 | 113 | ![](https://raw.githubusercontent.com/opendevops-cn/opendevops/master/docs/source/_static/images/publish_commit.png) 114 | 115 | 点击完审批后会进入等待执行，随后会按照你模板里面的顺序进行执行 116 | 117 | ![](https://raw.githubusercontent.com/opendevops-cn/opendevops/master/docs/source/_static/images/publish_wait.png) 118 | 最后执行完成后会显示状态OK， 119 | - 若执行途中有报错，会暂停任务，可根据`日志`进行排错，进行`重做` 120 | - 若执行途中有报错，且你不希望本组服务器进行执行了，可以点击`终止` 121 | - 若整个任务都不想要了，可以点击`终止全部` 122 | 123 | ![](https://raw.githubusercontent.com/opendevops-cn/opendevops/master/docs/source/_static/images/publish_ok.png) 124 | 125 | 126 | ### 数据类型 127 | 128 | - 接口数据类型如下 129 | 130 | ```json 131 | { 132 | "id": 1, 133 | "publish_name": "yanghongfeitest", 134 | "publish_type": "service", 135 | "repository": "git@gitlab.domai.comn:ops/yanghongfei.git", 136 | "build_host": "172.16.0.101", 137 | "exclude_file": "flash\n.git/", 138 | "temp_name": "yanghongfeitest", 139 | "publish_type1": null, 140 | "publish_path": "/var/www/", 141 | "publish_hosts": "172.16.0.20 22 root password", 142 | "publish_hosts_api": "http://gw.domain.com/cmdb/api/cmdb/server_list/?group=xxxx", 143 | "bucket_type": "oss", 144 | "region": "", 145 | "bucket_name": "", 146 | "bucket_path": "", 147 | "SecretID": "", 148 | "SecretKey": "", 149 | "docker_registry": "", 150 | "k8s_api": "", 151 | "namespace": "", 152 | "create_time": "2018-12-04 16:29:21" 153 | } 154 | ``` 155 | 156 | > PS: 代码写的有点搓，欢迎吐槽！ -------------------------------------------------------------------------------- /server/upload_code.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/4 16:47 4 | # @Author : Fred Yang 5 | # @File : upload_code.py 6 | # @Role : 发布代码到服务器 7 | 8 | 9 | import os 10 | import sys 11 | 12 | Base_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 13 | sys.path.append(Base_DIR) 14 | import shutil 15 | import uuid 16 | from public import exec_shell 17 | from public import exec_thread 18 | from get_publish_info import get_publish_data, get_all_hosts 19 | import fire 20 | 21 | 22 | class UploadCode(): 23 | """ 24 | 发布代码到目标主机，并发操作 25 | 01. 代码先同步到编译主机的/tmp下 26 | 02. 将本地/tmp下并发上传到目标主机的/tmp 27 | """ 28 | 29 | def __init__(self, data): 30 | self.publish_path = data.get('publish_path') # 发布目录 31 | self.repository = data.get('repository') # 代码仓库 32 | self.repo_name = self.repository.split('/')[-1].replace('.git', '') # 仓库名字 33 | self.local_dir = '/tmp/' 34 | self.uuid_file = '/tmp/publish_{}'.format(uuid.uuid1()) #错误判断该使用。 35 | 36 | def code_process(self, exclude_file): 37 | '''代码处理，如过滤，处理完放到编译主机/tmp''' 38 | code_dir = self.publish_path + self.repo_name 39 | try: 40 | if os.path.exists(code_dir): 41 | rsync_local_cmd = 'cd {} && rsync -ahqzt --delete --exclude-from="{}" {} {}'.format(code_dir, 42 | exclude_file, 43 | code_dir, 44 | self.local_dir) 45 | # print('[CMD:]', rsync_local_cmd) 46 | recode, stdout = exec_shell(rsync_local_cmd) 47 | if recode == 0: 48 | print('[Success]: 构建机器代码处理(如：exclude)到临时路径：/tmp/{} 完成 '.format(self.repo_name)) 49 | 50 | else: 51 | print('[Error]: Rsync bulid host:/tmp failed') 52 | exit(405) 53 | 54 | else: 55 | print('[Error]: Not fount git repo dir:{} '.format(code_dir)) 56 | exit(404) 57 | 58 | except Exception as e: 59 | print(e) 60 | 61 | def rsync_tmp(self, host): 62 | """ 63 | 发布代码到目标主机的/tmp目录 64 | :return: 65 | """ 66 | if not isinstance(host, dict): 67 | raise ValueError() 68 | 69 | ip = host.get('ip') 70 | port = host.get('port', 22) 71 | user = host.get('user', 'root') 72 | password = host.get('password') 73 | 74 | local_code_path = self.local_dir + self.repo_name # 处理后的本地代码路径 75 | rsync_tmp_cmd = "sshpass -p {} rsync -ahqzt --delete -e 'ssh -p {} -o StrictHostKeyChecking=no ' {} {}@{}:{}".format( 76 | password, port, local_code_path, user, ip, '/tmp/') 77 | # print('[CMD:] ', rsync_tmp_cmd) 78 | rsync_status, rsync_output = exec_shell(rsync_tmp_cmd) 79 | if rsync_status == 0: 80 | # 同步完成删除/tmp/repo_name目录 81 | print('[Success]: rsync host:{} to /tmp/{} sucess...'.format(ip, self.repo_name)) 82 | else: 83 | os.mknod(self.uuid_file) 84 | print('[Error]: rsync host:{} falied '.format(ip)) 85 | 86 | def delete_tmp(self): 87 | """删除临时代码目录""" 88 | tmp_path = '/tmp/{}'.format(self.repo_name) 89 | if os.path.exists(tmp_path): 90 | shutil.rmtree(tmp_path) 91 | 92 | def check_err(self): 93 | if os.path.exists(self.uuid_file): 94 | print('[Error]') 95 | exit(-1) 96 | 97 | def get_exclude_file(data): 98 | '''获取exclude文件内容写入临时文件,前端传来的只是字符串''' 99 | try: 100 | exclude_content = data.get('exclude_file') 101 | publish_name = data.get('publish_name') 102 | file_name = '/tmp/publish_exclude_{}_file.txt'.format(publish_name) 103 | f = open(file_name, 'w', encoding='utf-8') 104 | f.write(exclude_content) 105 | f.close() 106 | print('[Success]: Publish exclude_file has been written {}'.format(file_name)) 107 | return file_name 108 | except Exception as e: 109 | print(e) 110 | print('[Error]: Publish exclude_file write falid') 111 | exit(-500) 112 | 113 | 114 | def main(flow_id): 115 | """ 116 | 01. 处理exclude文件 117 | 02. 获取所有主机信息 118 | 03. 并发代码到目的主机/tmp 119 | :return: 120 | """ 121 | print('[INFO]: 这部分是处理exclude 将过滤后的代码并发到你的目标主机/tmp下，等待你的部署,如果rsync同步失败请确认服务器和目标主机都有rsync命令') 122 | data = get_publish_data(flow_id) # 获取发布信息 123 | exclude_file = get_exclude_file(data) # 过滤文件名称 124 | obj = UploadCode(data) 125 | obj.code_process(exclude_file) # 处理代码，如：exclude操作 126 | all_hosts = get_all_hosts(flow_id) 127 | exec_thread(func=obj.rsync_tmp, iterable1=all_hosts) 128 | obj.delete_tmp() 129 | obj.check_err() 130 | 131 | if __name__ == '__main__': 132 | fire.Fire(main) -------------------------------------------------------------------------------- /bucker/upload_cos.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/11 14:32 4 | # @Author : Fred Yang 5 | # @File : upload_cos.py 6 | # @Role : 上传资源到腾讯云COS 7 | 8 | 9 | import sys, os 10 | 11 | Base_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 12 | sys.path.append(Base_DIR) 13 | 14 | from publish_api import Publish_API 15 | from public import exec_shell 16 | import fire 17 | 18 | 19 | class Publish_COS(): 20 | def __init__(self, data): 21 | """ 22 | COS发布 23 | :param data: 前端传来的配置信息 24 | """ 25 | self.repository = data.get('repository') # 代码仓库 26 | self.repo_name = self.repository.split('/')[-1].replace('.git', '') # 仓库名字 27 | self.access_id = data.get('SecretID') 28 | self.access_key = data.get('SecretKey') 29 | self.region = data.get('region') # Bucket区域 30 | self.bucket_name = data.get('bucket_name') # Bucket名字 31 | self.bucket_path = data.get('bucket_path') # 目录路径 32 | self.clone_dir = '/opt' # 代码拉取地址 33 | 34 | def git_clone(self): 35 | '''检测发布目录没有代码则进行git clone''' 36 | try: 37 | if not os.path.exists(self.clone_dir): 38 | print('[INFO]: Start pulling a new codebase to {}'.format(self.clone_dir)) 39 | git_clone_cmd = '[ ! -d "{}" ] && mkdir {} ;cd {} && git clone {}'.format(self.clone_dir, 40 | self.clone_dir, 41 | self.clone_dir, 42 | self.repository) # 克隆代码 43 | # print('[CMD:] ', git_clone_cmd) 44 | git_clone_status, git_clone_output = exec_shell(git_clone_cmd) 45 | if git_clone_status == 0: 46 | print('[Success]: git clone {} sucess...'.format(self.repository)) 47 | else: 48 | print('[Error]: git clone {} faild...'.format(self.repository)) 49 | exit(404) 50 | else: 51 | print('[PASS]： The repository already exists, skip the clone and update directly') 52 | except Exception as e: 53 | print(e) 54 | 55 | def checkout_tag(self, git_tag): 56 | '''切换分支''' 57 | git_fetch_cmd = 'cd {}/{} && git fetch -t -p -f && git fetch --all'.format(self.clone_dir, 58 | 59 | self.repo_name) # 更新代码 60 | git_checkout_cmd = 'cd {}/{} && git clean -df && git checkout {}'.format( 61 | self.clone_dir, self.repo_name, git_tag) # 切换分支 62 | 63 | # print('[CMD:]', git_fetch_cmd) 64 | # print('[CMD:]', git_checkout_cmd) 65 | try: 66 | git_fetch_status, git_fetch_output = exec_shell(git_fetch_cmd) 67 | if git_fetch_status == 0: 68 | git_check_status, git_check_output = exec_shell(git_checkout_cmd) 69 | if git_check_status == 0: 70 | print('[Success]: git checkout tag: {} successfully...'.format(git_tag)) 71 | else: 72 | print('[Error]: git checkout tag: {} faild...'.format(git_tag)) 73 | exit(402) 74 | else: 75 | print('[Error]: git fetch faild...') 76 | exit(403) 77 | except Exception as e: 78 | print(e) 79 | exit(-500) 80 | 81 | def get_exclude_file(self, data): 82 | '''获取exclude文件内容写入临时文件,前端传来的只是字符串''' 83 | try: 84 | exclude_content = data.get('exclude_file') 85 | publish_name = data.get('publish_name') 86 | file_name = '/tmp/publish_exclude_{}_file.txt'.format(publish_name) 87 | f = open(file_name, 'w', encoding='utf-8') 88 | f.write(exclude_content) 89 | f.close() 90 | print('[Success]: Publish exclude_file has been written {}'.format(file_name)) 91 | return file_name 92 | except Exception as e: 93 | print(e) 94 | print('[Error]: Publish exclude_file write falid') 95 | exit(-500) 96 | 97 | def code_process(self, exclude_file): 98 | '''代码处理，如过滤，处理完放到编译主机/tmp''' 99 | try: 100 | if os.path.exists(self.clone_dir): 101 | rsync_local_cmd = 'cd {} && rsync -ahqzt --delete --exclude-from="{}" {}/{} {}'.format(self.clone_dir, 102 | exclude_file, 103 | self.clone_dir, 104 | self.repo_name, 105 | '/tmp') 106 | # print('[CMD:]', rsync_local_cmd) 107 | recode, stdout = exec_shell(rsync_local_cmd) 108 | if recode == 0: 109 | print('[Success]: 构建机器代码处理(如：exclude)到临时路径：/tmp/{} 完成 '.format(self.repo_name)) 110 | 111 | else: 112 | print('[Error]: Rsync bulid host:/tmp failed') 113 | exit(405) 114 | 115 | else: 116 | print('[Error]: Not fount git repo dir:{} '.format(self.clone_dir)) 117 | exit(404) 118 | 119 | except Exception as e: 120 | print(e) 121 | 122 | def login_cos(self): 123 | """ 124 | 登陆COS 125 | :return: 126 | """ 127 | print('[INFO:] 开始登陆COS') 128 | login_cos = 'coscmd config -a %s -s %s -b %s -r %s -m 20 >/dev/null' % ( 129 | self.access_id, self.access_key, self.bucket_name, self.region) 130 | login_cos_status, login_cos_output = exec_shell(login_cos) 131 | if login_cos_status == 0: 132 | print('[Success:] COS登陆成功') 133 | else: 134 | print('[Error:] COS登陆失败，错误信息：{}'.format(login_cos_output)) 135 | 136 | def cos_upload(self): 137 | """ 138 | 上传资源到COS 139 | :return: 140 | """ 141 | # 判断有没有权限 142 | print('[INFO:] 正在检查权限') 143 | check_cos = "coscmd getbucketacl" 144 | check_cos_status, check_cos_output = exec_shell(check_cos) 145 | if check_cos_status != 0: 146 | print('[Error:]Access Denied，请确认密钥是否有权限访问Bucket: {}'.format(self.bucket_name)) 147 | exit(-403) 148 | 149 | source_dir = '/tmp/%s' % (self.repo_name) 150 | 151 | # cmd = 'aws s3 sync %s s3://%s --exclude *.git* --profile %s --quiet'%(source_dir,BucketDict[self.app_name][self.tag_type],BucketDict[self.app_name]['s3_key']) 152 | upload_cmd = 'coscmd upload -rs %s/ %s >/dev/null 2>&1' % (source_dir, self.bucket_path) 153 | upload_cmd_status, upload_cmd_output = exec_shell(upload_cmd) 154 | if upload_cmd_status == 0: 155 | print('[Success:] 文件上传成功') 156 | else: 157 | print('[Error:] 上传失败，错误信息：{}'.format(upload_cmd_output)) 158 | exit(-404) 159 | 160 | 161 | def get_publish_info(publish_name): 162 | obj = Publish_API() 163 | publish_info = obj.get_publish_name_info(publish_name) 164 | for data in publish_info: 165 | return data 166 | 167 | 168 | def main(publish_name, git_tag): 169 | """ 170 | :param publish_name: 发布应用名称 171 | :param git_tag: 发布TAG名称 172 | :return: 173 | """ 174 | # 获取配置信息 175 | data = get_publish_info(publish_name) 176 | # 实例化 177 | obj = Publish_COS(data) 178 | # 克隆代码 179 | obj.git_clone() 180 | # 切换分支 181 | obj.checkout_tag(git_tag) 182 | # 获取过滤文件名字 183 | exclude_file_name = obj.get_exclude_file(data) 184 | # 文件处理，比如：过滤文件到/tmp临时文件 185 | obj.code_process(exclude_file_name) 186 | # 登陆COS 187 | obj.login_cos() 188 | # 上传文件 189 | obj.cos_upload() 190 | 191 | 192 | if __name__ == '__main__': 193 | fire.Fire(main) 194 | -------------------------------------------------------------------------------- /bucker/upload_oss.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/11 14:32 4 | # @Author : Fred Yang 5 | # @File : upload_cos.py 6 | # @Role : 上传资源到阿里云OSS 7 | 8 | 9 | import sys, os 10 | 11 | Base_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 12 | sys.path.append(Base_DIR) 13 | 14 | from publish_api import Publish_API 15 | from public import exec_shell 16 | import traceback 17 | import oss2 18 | import fire 19 | 20 | 21 | 22 | class Publish_OSS(): 23 | def __init__(self, data): 24 | """ 25 | COS发布 26 | :param data: 前端传来的配置信息 27 | """ 28 | self.repository = data.get('repository') # 代码仓库 29 | self.repo_name = self.repository.split('/')[-1].replace('.git', '') # 仓库名字 30 | self.access_id = data.get('SecretID') 31 | self.access_key = data.get('SecretKey') 32 | self.region = data.get('region') # Bucket区域 33 | self.bucket_name = data.get('bucket_name') # Bucket名字 34 | self.bucket_path = data.get('bucket_path') # 目录路径 35 | self.clone_dir = '/opt' # 代码拉取地址 36 | self.local_dir = '/tmp/{}/'.format(self.repo_name) # 处理后的临时存放代码路径 37 | self.ENDPOINT = 'http://oss-{}.aliyuncs.com'.format(self.region) # 节点信息 38 | self.bucket = oss2.Bucket(oss2.Auth(self.access_id, self.access_key), self.ENDPOINT, self.bucket_name) 39 | 40 | def git_clone(self): 41 | '''检测发布目录没有代码则进行git clone''' 42 | try: 43 | if not os.path.exists(self.clone_dir): 44 | print('[INFO]: Start pulling a new codebase to {}'.format(self.clone_dir)) 45 | git_clone_cmd = '[ ! -d "{}" ] && mkdir {} ;cd {} && git clone {}'.format(self.clone_dir, 46 | self.clone_dir, 47 | self.clone_dir, 48 | self.repository) # 克隆代码 49 | # print('[CMD:] ', git_clone_cmd) 50 | git_clone_status, git_clone_output = exec_shell(git_clone_cmd) 51 | if git_clone_status == 0: 52 | print('[Success]: git clone {} sucess...'.format(self.repository)) 53 | else: 54 | print('[Error]: git clone {} faild...'.format(self.repository)) 55 | exit(404) 56 | else: 57 | print('[PASS]： The repository already exists, skip the clone and update directly') 58 | except Exception as e: 59 | print(e) 60 | 61 | def checkout_tag(self, git_tag): 62 | """ 63 | 切换分支 64 | :param git_tag: 分支名称 65 | :return: 66 | """ 67 | 68 | git_fetch_cmd = 'cd {}/{} && git fetch -t -p -f && git fetch --all'.format(self.clone_dir, 69 | 70 | self.repo_name) # 更新代码 71 | git_checkout_cmd = 'cd {}/{} && git clean -df && git checkout {}'.format( 72 | self.clone_dir, self.repo_name, git_tag) # 切换分支 73 | 74 | # print('[CMD:]', git_fetch_cmd) 75 | # print('[CMD:]', git_checkout_cmd) 76 | try: 77 | git_fetch_status, git_fetch_output = exec_shell(git_fetch_cmd) 78 | if git_fetch_status == 0: 79 | git_check_status, git_check_output = exec_shell(git_checkout_cmd) 80 | if git_check_status == 0: 81 | print('[Success]: git checkout tag: {} successfully...'.format(git_tag)) 82 | else: 83 | print('[Error]: git checkout tag: {} faild...'.format(git_tag)) 84 | exit(402) 85 | else: 86 | print('[Error]: git fetch faild...') 87 | exit(403) 88 | except Exception as e: 89 | print(e) 90 | exit(-500) 91 | 92 | def get_exclude_file(self, data): 93 | """ 94 | 获取exclude文件内容写入临时文件,前端传来的只是字符串 95 | :param data: 发布配置信息，JSON 96 | :return: 97 | """ 98 | try: 99 | exclude_content = data.get('exclude_file') 100 | publish_name = data.get('publish_name') 101 | file_name = '/tmp/publish_exclude_{}_file.txt'.format(publish_name) 102 | f = open(file_name, 'w', encoding='utf-8') 103 | f.write(exclude_content) 104 | f.close() 105 | print('[Success]: Publish exclude_file has been written {}'.format(file_name)) 106 | return file_name 107 | except Exception as e: 108 | print(e) 109 | print('[Error]: Publish exclude_file write falid') 110 | exit(-500) 111 | 112 | def code_process(self, exclude_file): 113 | """ 114 | 代码处理，如过滤，处理完放到编译主机/tmp 115 | :param exclude_file: 过滤文件名称 116 | :return: 117 | """ 118 | 119 | try: 120 | if os.path.exists(self.clone_dir): 121 | rsync_local_cmd = 'cd {} && rsync -ahqzt --delete --exclude-from="{}" {}/{} {}'.format(self.clone_dir, 122 | exclude_file, 123 | self.clone_dir, 124 | self.repo_name, 125 | '/tmp') 126 | # print('[CMD:]', rsync_local_cmd) 127 | recode, stdout = exec_shell(rsync_local_cmd) 128 | if recode == 0: 129 | print('[Success]: 构建机器代码处理(如：exclude)到临时路径：/tmp/{} 完成 '.format(self.repo_name)) 130 | 131 | else: 132 | print('[Error]: Rsync bulid host:/tmp failed') 133 | exit(405) 134 | 135 | else: 136 | print('[Error]: Not fount git repo dir:{} '.format(self.clone_dir)) 137 | exit(404) 138 | 139 | except Exception as e: 140 | print(e) 141 | exit(406) 142 | 143 | def oss_upload(self): 144 | for root, dirs, files in os.walk(self.local_dir): 145 | for file in files: 146 | # print(os.path.join(root, file)) 147 | file_name = os.path.join(root, file) 148 | split_file_name = file_name.split(self.local_dir)[1] 149 | 150 | object_name = split_file_name 151 | try: 152 | exist = self.bucket.object_exists(object_name) 153 | if not exist: 154 | # print('[INFO:] FilaName:{}\nObjectName:{}'.format(split_file_name,object_name)) 155 | res = self.bucket.put_object_from_file(object_name, file_name) 156 | if res.status != 200: 157 | print('[Error:] 上传失败！') 158 | exit(-2) 159 | except Exception as e: 160 | print('[Error:] 发生错误，请检查你的信息是否正确，错误信息：{} '.format(e)) 161 | # except: 162 | # print(traceback.format_exc()) 163 | exit(-3) 164 | print('[Success:] 上传成功！') 165 | 166 | 167 | def get_publish_info(publish_name): 168 | """ 169 | 获取发布配置信息 170 | :param publish_name:发布应用名称 171 | :return: 172 | """ 173 | obj = Publish_API() 174 | publish_info = obj.get_publish_name_info(publish_name) 175 | for data in publish_info: 176 | return data 177 | 178 | 179 | def main(publish_name, git_tag): 180 | """ 181 | :param publish_name: 发布应用名称 182 | :param git_tag: 发布TAG名称 183 | :return: 184 | """ 185 | # 获取配置信息 186 | data = get_publish_info(publish_name) 187 | # 实例化 188 | obj = Publish_OSS(data) 189 | # 克隆代码 190 | obj.git_clone() 191 | # 切换分支 192 | obj.checkout_tag(git_tag) 193 | # 获取过滤文件名字 194 | exclude_file_name = obj.get_exclude_file(data) 195 | # 文件处理，比如：过滤文件到/tmp临时文件 196 | obj.code_process(exclude_file_name) 197 | # 资源上传 198 | obj.oss_upload() 199 | 200 | 201 | if __name__ == '__main__': 202 | fire.Fire(main) 203 | -------------------------------------------------------------------------------- /bucker/upload_s3.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # @Time : 2018/12/12 17:31 4 | # @Author : Fred Yang 5 | # @File : upload_s3.py 6 | # @Role : 上传资源到AWS S3 7 | 8 | 9 | import sys, os 10 | 11 | Base_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 12 | sys.path.append(Base_DIR) 13 | 14 | from publish_api import Publish_API 15 | from public import exec_shell 16 | import fire 17 | 18 | 19 | class Publish_S3(): 20 | def __init__(self, data): 21 | """ 22 | S3发布 23 | :param data: 前端传来的配置信息 24 | """ 25 | self.repository = data.get('repository') # 代码仓库 26 | self.repo_name = self.repository.split('/')[-1].replace('.git', '') # 仓库名字 27 | self.access_id = data.get('SecretID') 28 | self.access_key = data.get('SecretKey') 29 | self.region = data.get('region') # Bucket区域 30 | self.bucket_name = data.get('bucket_name') # Bucket名字 31 | self.clone_dir = '/opt' # 代码拉取地址 32 | self.local_dir = '/tmp/{}/'.format(self.repo_name) # 处理后的临时存放代码路径 33 | 34 | def git_clone(self): 35 | '''检测发布目录没有代码则进行git clone''' 36 | try: 37 | if not os.path.exists(self.clone_dir): 38 | print('[INFO]: Start pulling a new codebase to {}'.format(self.clone_dir)) 39 | git_clone_cmd = '[ ! -d "{}" ] && mkdir {} ;cd {} && git clone {}'.format(self.clone_dir, 40 | self.clone_dir, 41 | self.clone_dir, 42 | self.repository) # 克隆代码 43 | # print('[CMD:] ', git_clone_cmd) 44 | git_clone_status, git_clone_output = exec_shell(git_clone_cmd) 45 | if git_clone_status == 0: 46 | print('[Success]: git clone {} sucess...'.format(self.repository)) 47 | else: 48 | print('[Error]: git clone {} faild...'.format(self.repository)) 49 | exit(404) 50 | else: 51 | print('[PASS]： The repository already exists, skip the clone and update directly') 52 | except Exception as e: 53 | print(e) 54 | 55 | def checkout_tag(self, git_tag): 56 | """ 57 | 切换分支 58 | :param git_tag: 分支名称 59 | :return: 60 | """ 61 | 62 | git_fetch_cmd = 'cd {}/{} && git fetch -t -p -f && git fetch --all'.format(self.clone_dir, 63 | 64 | self.repo_name) # 更新代码 65 | git_checkout_cmd = 'cd {}/{} && git clean -df && git checkout {}'.format( 66 | self.clone_dir, self.repo_name, git_tag) # 切换分支 67 | 68 | # print('[CMD:]', git_fetch_cmd) 69 | # print('[CMD:]', git_checkout_cmd) 70 | try: 71 | git_fetch_status, git_fetch_output = exec_shell(git_fetch_cmd) 72 | if git_fetch_status == 0: 73 | git_check_status, git_check_output = exec_shell(git_checkout_cmd) 74 | if git_check_status == 0: 75 | print('[Success]: git checkout tag: {} successfully...'.format(git_tag)) 76 | else: 77 | print('[Error]: git checkout tag: {} faild...'.format(git_tag)) 78 | exit(402) 79 | else: 80 | print('[Error]: git fetch faild...') 81 | exit(403) 82 | except Exception as e: 83 | print(e) 84 | exit(-500) 85 | 86 | def get_exclude_file(self, data): 87 | """ 88 | 获取exclude文件内容写入临时文件,前端传来的只是字符串 89 | :param data: 发布配置信息，JSON 90 | :return: 91 | """ 92 | try: 93 | exclude_content = data.get('exclude_file') 94 | publish_name = data.get('publish_name') 95 | file_name = '/tmp/publish_exclude_{}_file.txt'.format(publish_name) 96 | f = open(file_name, 'w', encoding='utf-8') 97 | f.write(exclude_content) 98 | f.close() 99 | print('[Success]: Publish exclude_file has been written {}'.format(file_name)) 100 | return file_name 101 | except Exception as e: 102 | print(e) 103 | print('[Error]: Publish exclude_file write falid') 104 | exit(-500) 105 | 106 | def code_process(self, exclude_file): 107 | """ 108 | 代码处理，如过滤，处理完放到编译主机/tmp 109 | :param exclude_file: 过滤文件名称 110 | :return: 111 | """ 112 | 113 | try: 114 | if os.path.exists(self.clone_dir): 115 | rsync_local_cmd = 'cd {} && rsync -ahqzt --delete --exclude-from="{}" {}/{} {}'.format(self.clone_dir, 116 | exclude_file, 117 | self.clone_dir, 118 | self.repo_name, 119 | '/tmp') 120 | # print('[CMD:]', rsync_local_cmd) 121 | recode, stdout = exec_shell(rsync_local_cmd) 122 | if recode == 0: 123 | print('[Success]: 构建机器代码处理(如：exclude)到临时路径：/tmp/{} 完成 '.format(self.repo_name)) 124 | 125 | else: 126 | print('[Error]: Rsync bulid host:/tmp failed') 127 | exit(405) 128 | 129 | else: 130 | print('[Error]: Not fount git repo dir:{} '.format(self.clone_dir)) 131 | exit(404) 132 | 133 | except Exception as e: 134 | print(e) 135 | exit(406) 136 | 137 | def aws_configure(self): 138 | """ 139 | 配置AWS configure信息，使能访问AWS S3 140 | :return: 141 | """ 142 | ca_file_name = '/root/.aws/credentials' 143 | ca_file_content = "[default]\naws_access_key_id = {}\naws_secret_access_key = {}".format(self.access_id, 144 | self.access_key) 145 | aws_config_name = '/root/.aws/config' 146 | aws_config_content = '[default]\nregion = {}'.format(self.region) 147 | try: 148 | f = open(ca_file_name, 'w', encoding='utf-8') 149 | f.write(ca_file_content) 150 | f.close() 151 | print('[INFO]: AWS 证书文件 – 位于 ~/.aws/credentials') 152 | f = open(aws_config_name, 'w', encoding='utf-8') 153 | f.write(aws_config_content) 154 | f.close() 155 | print('[INFO]: CLI 配置文件 – 位于 ~/.aws/config') 156 | except Exception as e: 157 | print(e) 158 | print('[Error]: AWS 配置文件写入失败，也可以使用aws configure进行交互式配置') 159 | exit(-1) 160 | 161 | def upload_file(self): 162 | """ 163 | 将资源文件上传到AWS S3,使用AWSCLI的 AWS S3 Sync方式 164 | :return: 165 | """ 166 | source_dir = '/tmp/%s/' % (self.repo_name) 167 | upload_s3_cmd = 'aws s3 sync %s s3://%s --exclude *.git* --quiet' % (source_dir, self.bucket_name) 168 | print('[INFO]：Start upload for bucket: %s' % self.repo_name) 169 | upload_s3_status, upload_s3_ouput = exec_shell(upload_s3_cmd) 170 | if upload_s3_status == 0: 171 | print('[Success:] AWS S3 上传成功！') 172 | else: 173 | print('[Error:] AWS S3 上传失败！') 174 | 175 | 176 | def get_publish_info(publish_name): 177 | """ 178 | 获取发布配置信息 179 | :param publish_name:发布应用名称 180 | :return: 181 | """ 182 | obj = Publish_API() 183 | publish_info = obj.get_publish_name_info(publish_name) 184 | for data in publish_info: 185 | return data 186 | 187 | 188 | def main(publish_name, git_tag): 189 | """ 190 | :param publish_name: 发布应用名称 191 | :param git_tag: 发布TAG名称 192 | :return: 193 | """ 194 | # 获取配置信息 195 | data = get_publish_info(publish_name) 196 | # 实例化 197 | obj = Publish_S3(data) 198 | # 克隆代码 199 | obj.git_clone() 200 | # 切换分支 201 | obj.checkout_tag(git_tag) 202 | # 获取过滤文件名字 203 | exclude_file_name = obj.get_exclude_file(data) 204 | # 文件处理，比如：过滤文件到/tmp临时文件 205 | obj.code_process(exclude_file_name) 206 | # AWS 配置 207 | obj.aws_configure() 208 | # 资源上传 209 | obj.upload_file() 210 | 211 | 212 | if __name__ == '__main__': 213 | fire.Fire(main) 214 | --------------------------------------------------------------------------------