├── README.md ├── libs ├── __init__.py ├── log.py ├── loki.py ├── redisoj.py ├── storage.py └── utils.py ├── nginx ├── __init__.py ├── libs │ ├── __init__.py │ ├── conf.py │ ├── domains.py │ ├── global_id.py │ ├── link.sh │ ├── mysqloj.py │ ├── template.py │ └── upstream.py ├── template │ ├── server.conf │ └── upstream.conf └── web │ ├── __init__.py │ └── service.py ├── restart.sh ├── start.sh ├── stop.sh └── web ├── __init__.py ├── const.py └── main_service.py /README.md: -------------------------------------------------------------------------------- 1 | # 说明 2 | 3 | Nginx 配置管理, 基于 git, 并分产品线, 内网外网和机房三个维度. 4 | 5 | 6 | 1. 由于 Nginx 配置分为 产品线、内网或外网和机房, 不同纬度的 Nginx 的配置不一样; 7 | 8 | 2. 支持分支发布, 根据 Nginx 机器名和分支名获取配置文件链接; 9 | 10 | 3. 提供获取和增删改 Nginx upstream API; 11 | 12 | 4. Nginx upstream 信息放在 mysql, 根据模板生成upstream.conf, 机器列表实时从服务管理系统获取; 13 | 14 | 5. 支持 ip_hash 字段, online 字段表示 是否在线, 不为1 不生成配置; 15 | 16 | 6. 支持增加域名, 自动生成配置文件并 push 到 origin. 17 | 18 | 19 | 20 | 21 | # Nginx upstream 数据库 22 | ``` 23 | create database online_nginx_conf; 24 | create table nginx_upstream_template ( 25 | id int unsigned not null auto_increment, 26 | name varchar(100) not null, 27 | loki_id int unsigned not null, 28 | port int unsigned not null default 80, 29 | ip_hash int unsigned not null default 0, 30 | online int unsigned not null default 1, 31 | primary key (id), 32 | unique KEY `unq_name` (`name`); 33 | ); 34 | ``` 35 | 36 | 37 | 38 | # 依赖 39 | 40 | ``` 41 | pip install futures 42 | 43 | pip install ujson 44 | 45 | pip install tornado 46 | 47 | pip install DBUtils 48 | 49 | pip install jinja2 50 | 51 | yum -y install python-redis python-ldap MySQL-python``` 52 | ``` 53 | -------------------------------------------------------------------------------- /libs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex1528/nginx_conf/db661fb231deedf4c337aecb93e02e519daec5d3/libs/__init__.py -------------------------------------------------------------------------------- /libs/log.py: -------------------------------------------------------------------------------- 1 | #-*- coding: utf-8 -*- 2 | 3 | import os 4 | import logging 5 | 6 | from web.const import LOG_DIR 7 | from web.const import LOG_FILE 8 | 9 | 10 | def get_logger(name): 11 | logger_ = logging.getLogger(name) 12 | formatter = logging.Formatter( 13 | '[%(asctime)s] [%(levelname)-8s] [%(name)-8s] %(message)s', '%Y-%m-%d %H:%M:%S',) 14 | handler = logging.FileHandler(LOG_DIR + "/" + LOG_FILE) 15 | handler.setFormatter(formatter) 16 | logger_.addHandler(handler) 17 | logger_.setLevel(logging.DEBUG) 18 | return logger_ 19 | 20 | if not os.path.isdir(LOG_DIR): 21 | os.makedirs(LOG_DIR) 22 | -------------------------------------------------------------------------------- /libs/loki.py: -------------------------------------------------------------------------------- 1 | #-*- coding: utf-8 -*- 2 | 3 | import requests 4 | 5 | 6 | def get_path_from_hostname(hostname): 7 | url = "http://loki.internal.nosa.me"\ 8 | "/ptree/api/server_search?hostname=%s" % hostname 9 | 10 | ret = requests.get(url) 11 | dirs = ret.json()["data"]["dirs"] 12 | return [i["dir"] for i in dirs] 13 | 14 | 15 | def get_hostnames_from_id(node_id): 16 | url = "http://loki.internal.nosa.me"\ 17 | "/server/api/servers?type=recursive&node_id=%s" % node_id 18 | 19 | ret = requests.get(url) 20 | data = ret.json()["data"] 21 | return [i["hostname"] for i in data] 22 | -------------------------------------------------------------------------------- /libs/redisoj.py: -------------------------------------------------------------------------------- 1 | #-*- coding: utf-8 -*- 2 | 3 | import redis 4 | 5 | #from web.const import REDIS_HOST, REDIS_PORT, REDIS_PASSWD 6 | from web.const import REDIS_HOST, REDIS_PORT 7 | 8 | 9 | class PooledConnection(object): 10 | def __init__(self, redis_db): 11 | self.pool = redis.connection.BlockingConnectionPool( 12 | host=REDIS_HOST, 13 | port=REDIS_PORT, 14 | db=redis_db, 15 | #password=REDIS_PASSWD, 16 | max_connections=10, 17 | timeout=60 18 | ) 19 | 20 | def get(self): 21 | """ 返回一个连接. 22 | 23 | """ 24 | return redis.client.Redis(connection_pool=self.pool) 25 | -------------------------------------------------------------------------------- /libs/storage.py: -------------------------------------------------------------------------------- 1 | #-*- coding: utf-8 -*- 2 | 3 | """ 把一个文件存储到下载服务, 并返回 下载链接. 4 | 5 | """ 6 | 7 | import os 8 | 9 | from libs import utils 10 | 11 | 12 | def post(file_path): 13 | """ 上传文件到远程并拿到下载链接. 14 | 15 | 使用封装过的 wcdn 命令. 16 | 17 | """ 18 | cmd = " wcdn cp -f %s /cdn.internal.nosa.me/nginx_conf_deploy "\ 19 | "--no-verbose --md5" % file_path 20 | rc, so, se = utils.shell(cmd) 21 | if rc != 0: 22 | raise Exception(se) 23 | 24 | return "http://cdn.internal.nosa.me/nginx_conf_deploy/%s"\ 25 | % os.path.basename(file_path) 26 | -------------------------------------------------------------------------------- /libs/utils.py: -------------------------------------------------------------------------------- 1 | #-*- coding: utf-8 -*- 2 | 3 | import os 4 | import subprocess 5 | import re 6 | import random 7 | import time 8 | import getpass 9 | 10 | 11 | def shell(cmd): 12 | process = subprocess.Popen( 13 | args=cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 14 | std_out, std_err = process.communicate() 15 | return_code = process.poll() 16 | return return_code, std_out, std_err 17 | 18 | 19 | def get_hostname(): 20 | cmd = "hostname " 21 | rc, so, se = shell(cmd) 22 | 23 | if rc == 0: 24 | return so 25 | 26 | 27 | def get_inner_ip(): 28 | cmd = "hostname -i" 29 | rc, so, se = shell(cmd) 30 | 31 | if rc == 0: 32 | return so 33 | 34 | 35 | def get_extra_ip(): 36 | """ 获取公网 IP. 37 | 38 | """ 39 | cmd = ''' /sbin/ifconfig |grep "inet addr:" |egrep -v "127.0.0.1|10\.|192\.168\." |awk '{print $2}' |awk -F ":" '{print $2}' ''' 40 | rc, so, se = shell(cmd) 41 | 42 | if rc == 0: 43 | return so 44 | 45 | 46 | def is_valid_ip(ip): 47 | """ 检查 ip 是否合法. 48 | 49 | """ 50 | p = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") 51 | 52 | s = p.findall(ip.strip()) 53 | if s == []: 54 | return False 55 | 56 | return len([i for i in s[0].split('.') if (0 <= int(i) <= 255)]) == 4 57 | 58 | 59 | def mac_random(): 60 | mac = [0x00, 0x16, 0x3e, random.randint(0x00, 0x7f), 61 | random.randint(0x00, 0xff), random.randint(0x00, 0xff)] 62 | s = [] 63 | for item in mac: 64 | s.append(str("%02x" % item)) 65 | 66 | return ':'.join(s) 67 | 68 | 69 | def ping(ip): 70 | cmd = "ping -c 3 %s &>/dev/null " % ip 71 | 72 | rc, so, se = shell(cmd) 73 | if rc == 0: 74 | return True 75 | else: 76 | return False 77 | 78 | 79 | def dns_check(hostname): 80 | cmd = "nslookup %s &>/dev/null " % hostname 81 | 82 | rc, so, se = shell(cmd) 83 | if rc == 0: 84 | return True 85 | else: 86 | return False 87 | 88 | 89 | def check_wait(check_cmd, post_cmd, timeinit=0, interval=10, timeout=600): 90 | """ 循环等待某一条件成功,就执行 post_cmd,时间超过 timeout 就超时. 91 | 92 | """ 93 | timetotal = timeinit 94 | 95 | while timetotal < timeout: 96 | rc, ro, re = shell(check_cmd) 97 | if rc == 0: 98 | rc, ro, re = shell(post_cmd) 99 | if rc == 0: 100 | return True 101 | else: 102 | return False 103 | 104 | time.sleep(interval) 105 | timetotal += interval 106 | 107 | return False 108 | 109 | 110 | def check_wait_null(check_cmd, timeinit=0, interval=10, timeout=600): 111 | """ 循环等待某一条件成功,返回 True, 时间超过 timeout 就超时. 112 | 113 | """ 114 | timetotal = timeinit 115 | 116 | while timetotal < timeout: 117 | rc, ro, re = shell(check_cmd) 118 | if rc == 0: 119 | return True 120 | 121 | time.sleep(interval) 122 | timetotal += interval 123 | 124 | return False 125 | 126 | 127 | def dns_resolv(hostnames=None): 128 | if hostnames is None: 129 | return [] 130 | ips = list() 131 | for hostname in hostnames: 132 | cmd = ''' nslookup %s |grep -v "#53" |grep "Address:" ''' % hostname 133 | rc, so, se = shell(cmd) 134 | if rc != 0: 135 | return False 136 | 137 | ip = so.strip().split(":")[-1].strip() 138 | ips.append(ip) 139 | return ips 140 | 141 | 142 | def remote_cmd(host, cmd, user="", sshkey="", timeout=10): 143 | """ sshkey 是私钥文件路径. 144 | 145 | """ 146 | if user == "": 147 | user_host = host 148 | else: 149 | user_host = "%s@%s" % (user, host) 150 | 151 | if sshkey != "": 152 | cmd = ''' ssh -oConnectTimeout=%s -oStrictHostKeyChecking=no -i %s %s "%s" ''' % ( 153 | timeout, sshkey, user_host, cmd) 154 | else: 155 | cmd = ''' ssh -oConnectTimeout=%s -oStrictHostKeyChecking=no %s "%s" ''' % ( 156 | timeout, user_host, cmd) 157 | 158 | rc, so, se = shell(cmd) 159 | return rc, so, se 160 | 161 | 162 | def transfer_dir(hosts, local_dir, remote_dir, user="", sshkey="", timeout=10): 163 | # 如果不指定 user, 使用运行程序的用户. 164 | if user == "": 165 | user = getpass.getuser() 166 | 167 | # sshkey 是私钥文件路径 168 | if sshkey != "": 169 | sshkey_option = "-i %s" % sshkey 170 | else: 171 | sshkey_option = "" 172 | 173 | # 先修改目标文件夹权限, 否则可能因为权限问题无法传输. 174 | for host in hosts: 175 | cmd = "sudo chown %s:%s -R %s" % ( 176 | user, user, remote_dir) 177 | rc, so, se = remote_cmd(host, cmd, user) 178 | if rc != 0: 179 | return False 180 | 181 | for host in hosts: 182 | # 如果本地文件夹不存在或者为空, 不做处理. 183 | if not os.path.isdir(local_dir): 184 | continue 185 | if os.listdir(local_dir) == []: 186 | continue 187 | 188 | # 执行传输 189 | cmd = "scp -r -oConnectTimeout=%s -oStrictHostKeyChecking=no %s %s/* %s@%s:%s" % ( 190 | timeout, sshkey_option, local_dir, user, host, remote_dir) 191 | rc, so, se = shell(cmd) 192 | if rc != 0: 193 | return False 194 | 195 | return True 196 | -------------------------------------------------------------------------------- /nginx/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex1528/nginx_conf/db661fb231deedf4c337aecb93e02e519daec5d3/nginx/__init__.py -------------------------------------------------------------------------------- /nginx/libs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex1528/nginx_conf/db661fb231deedf4c337aecb93e02e519daec5d3/nginx/libs/__init__.py -------------------------------------------------------------------------------- /nginx/libs/conf.py: -------------------------------------------------------------------------------- 1 | #-*- coding: utf-8 -*- 2 | 3 | """ 返回 Nginx 配置文件的下载地址. 4 | 5 | """ 6 | 7 | import os 8 | import re 9 | import functools 10 | 11 | from libs import log, utils, loki, redisoj, storage 12 | from web.const import (NGINX_CONF_DIR, 13 | NGINX_TMP_STORAGE_DIR, 14 | NGINX_TEMPLATE_DIR, 15 | NGINX_UPSTREAM_TEMPLATE_FILE, 16 | NGINX_SSL_ORIGIN_DIR, 17 | NGINX_SSL_DEST_DIR, 18 | REDIS_DB_NGINX) 19 | from nginx.libs import global_id, template, upstream 20 | 21 | 22 | _redis_oj = redisoj.PooledConnection(REDIS_DB_NGINX) 23 | client = _redis_oj.get() 24 | 25 | logger = log.get_logger("Nginx CONF ") 26 | 27 | 28 | def _shell(cmd, _logger=logger): 29 | """ 执行命令, 记录日志. 30 | 31 | """ 32 | rc , so, se = utils.shell(cmd) 33 | if rc == 0: 34 | message = "cmd:%s" % cmd 35 | _logger.info(message) 36 | else: 37 | message = "cmd:%s, error:%s" % ( 38 | cmd, se) 39 | raise Exception(message) 40 | 41 | 42 | def _redis_lock(func): 43 | """ redis 锁. 44 | 45 | """ 46 | @functools.wraps(func) 47 | def wrapper(*args, **kw): 48 | while 1: 49 | if client.exists("lock"): 50 | import time 51 | time.sleep(0.1) 52 | else: 53 | client.set("lock", None, 2) # 2s 超时. 54 | return func(*args, **kw) 55 | client.delete("lock") 56 | break 57 | return wrapper 58 | 59 | 60 | @_redis_lock 61 | def _conf(branch, conf_base_dir, conf_tmp_dir): 62 | """ 拉取配置并拷贝到响应目录. 63 | 64 | 由于 git 并发操作会报错, 所以用了 redis 锁. 65 | 66 | """ 67 | # 拷贝最新 Nginx 配置. 68 | cmd = """cd %s &&\ 69 | git fetch origin -p &&\ 70 | git checkout %s &&\ 71 | git pull &&\ 72 | cp -a %s %s """ % ( conf_base_dir, 73 | branch, 74 | conf_base_dir, 75 | conf_tmp_dir) 76 | _shell(cmd) 77 | 78 | 79 | def get(nginx, branch): 80 | """ 根据 Nginx 机器名打包配置文件, 并上传到远端和获取下载地址. 81 | 82 | """ 83 | # 拿到 Nginx 机器的 产品线,类型,机房. 84 | loki_path = loki.get_path_from_hostname(nginx) 85 | if loki_path == []: 86 | raise Exception("%s has no loki path." % nginx) 87 | elif len(loki_path) > 1: 88 | raise Exception("%s has %s loki path - %s." % ( 89 | nginx, 90 | len(loki_path), 91 | loki_path)) 92 | 93 | tmp = loki_path[0].split("/") 94 | product = tmp[2] # 产品线 95 | _type = tmp[-2] # 类型, 表示内网还是内网. 96 | idc = tmp[-1] # 机房. 97 | del tmp 98 | logger.info("%s %s %s" % (product, _type, idc)) 99 | 100 | # 创建临时 Nginx 配置文件目录. 101 | if not os.path.exists(NGINX_TMP_STORAGE_DIR): 102 | os.mkdir(NGINX_TMP_STORAGE_DIR) 103 | _global_id = global_id.get() 104 | conf_tmp_dir = "%s/%s/%s" % (os.getcwd(), 105 | NGINX_TMP_STORAGE_DIR, 106 | _global_id) 107 | 108 | # 拷贝最新 Nginx 配置. 109 | _conf(branch, NGINX_CONF_DIR, conf_tmp_dir) 110 | 111 | # 拿到 upstream data. 112 | upstream_data = list() 113 | for i in upstream.get(): 114 | if i["online"] == 1: 115 | servers = loki.get_hostnames_from_id(i["loki_id"]) 116 | if servers == []: 117 | raise("%s has no servers." % i["loki_id"]) 118 | _dict = { 119 | "name": i["name"], 120 | "servers": servers, 121 | "port": i["port"], 122 | "ip_hash": i["ip_hash"] 123 | } 124 | upstream_data.append(_dict) 125 | 126 | # 生成 upstream.conf 文件. 127 | upstream_path = "%s/upstream.conf" % conf_tmp_dir 128 | template.gen_upstream(NGINX_TEMPLATE_DIR, 129 | NGINX_UPSTREAM_TEMPLATE_FILE, 130 | upstream_path, 131 | upstream_data) 132 | 133 | # 做软链. 134 | cmd = "sh nginx/libs/link.sh %s %s %s %s %s %s" % (conf_tmp_dir, product, 135 | _type, idc, 136 | NGINX_SSL_ORIGIN_DIR, 137 | NGINX_SSL_DEST_DIR) 138 | _shell(cmd) 139 | 140 | # 打包. 141 | package_name = "%s.tgz" % _global_id 142 | cmd = "cd %s &&tar czf ../%s *" % (conf_tmp_dir, 143 | package_name) 144 | _shell(cmd) 145 | 146 | # 把 Nginx 包上传到存储服务, 并返回下载连接. 147 | package_path = "%s/%s" % (NGINX_TMP_STORAGE_DIR, package_name) 148 | logger.info("package_path:%s" % package_path) 149 | return storage.post(package_path) 150 | -------------------------------------------------------------------------------- /nginx/libs/domains.py: -------------------------------------------------------------------------------- 1 | #-*- coding: utf-8 -*- 2 | 3 | """ 返回 Nginx 配置文件的下载地址. 4 | 5 | """ 6 | 7 | 8 | import os 9 | 10 | from libs import log, utils 11 | from web.const import (NGINX_CONF_DIR, 12 | NGINX_TEMPLATE_DIR, 13 | NGINX_SERVER_TEMPLATE_FILE) 14 | from nginx.libs import template, upstream 15 | 16 | 17 | logger = log.get_logger("Nginx DOMAINS ") 18 | 19 | 20 | def _shell(cmd, _logger=logger): 21 | """ 执行命令, 记录日志. 22 | 23 | """ 24 | rc , so, se = utils.shell(cmd) 25 | if rc == 0: 26 | message = "cmd:%s" % cmd 27 | _logger.info(message) 28 | else: 29 | message = "cmd:%s, error:%s" % (cmd, se) 30 | raise Exception(message) 31 | 32 | 33 | def add(product, _type, idc, name, server_names, log_name, log_format, upstream_node): 34 | """ 增加域名的配置. 35 | 36 | """ 37 | upstreams = upstream.get() 38 | upstream_nodes = [ i["name"] for i in upstreams ] 39 | if upstream_node not in upstream_nodes: 40 | raise Exception("%s not exist" % upstream_node) 41 | else: 42 | for i in upstreams: 43 | if i["name"] == upstream_node: 44 | if i["online"] != 1: 45 | raise Exception("%s is offline" % i["name"]) 46 | else: 47 | break 48 | 49 | domain_conf_dir = "%s/sites-available/%s/%s/%s/" % (NGINX_CONF_DIR, 50 | product, 51 | _type, 52 | idc) 53 | domain_conf_path = "%s/%s" % (domain_conf_dir, name) 54 | 55 | if not os.path.exists(domain_conf_dir): 56 | os.makedirs(domain_conf_dir) 57 | 58 | if os.path.exists(domain_conf_path): 59 | raise("%s exists" % domain_conf_path) 60 | 61 | template.gen_server(NGINX_TEMPLATE_DIR, NGINX_SERVER_TEMPLATE_FILE, 62 | domain_conf_path, server_names, log_name, 63 | log_format, upstream_node) 64 | 65 | cmds = """cd %s && 66 | git checkout master && 67 | git pull && 68 | git add sites-available/%s/%s/%s/%s && 69 | git commit -m 'add %s' && 70 | git push origin master """ % (NGINX_CONF_DIR, product, 71 | _type, idc, name, name) 72 | 73 | _shell(cmds) 74 | return True 75 | -------------------------------------------------------------------------------- /nginx/libs/global_id.py: -------------------------------------------------------------------------------- 1 | #-*- coding: utf-8 -*- 2 | 3 | """ 生成一个全局递增 id. 4 | 5 | """ 6 | 7 | import redis 8 | 9 | from libs import redisoj 10 | from web.const import REDIS_DB_NGINX 11 | 12 | 13 | _redis_oj = redisoj.PooledConnection(REDIS_DB_NGINX) 14 | client = _redis_oj.get() 15 | 16 | 17 | def get(): 18 | pipe = client.pipeline() 19 | 20 | while 1: 21 | try: 22 | pipe.watch('global_id') 23 | current_id = pipe.get('global_id') 24 | if current_id is None: 25 | next_id = 1 26 | else: 27 | next_id = int(current_id) + 1 28 | pipe.multi() 29 | pipe.set('global_id', next_id) 30 | pipe.execute() 31 | break 32 | except redis.exceptions.WatchError: 33 | continue 34 | finally: 35 | pipe.reset() 36 | 37 | return next_id 38 | 39 | 40 | if __name__ == '__main__': 41 | get() 42 | -------------------------------------------------------------------------------- /nginx/libs/link.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 做软链. 3 | 4 | 5 | # 参数判断. 6 | if [ "$#" != "6" ] 7 | then 8 | echo "Parameter num must be 4" 1>&2 9 | fi 10 | 11 | 12 | # 变量列表. 13 | nginx_dir=$1 14 | product=$2 15 | _type=$3 16 | idc=$4 17 | nginx_ssl_origin_dir=$5 18 | nginx_ssl_dest_dir=$6 19 | 20 | 21 | # 进入目录. 22 | cd $nginx_dir ||exit 1 23 | 24 | 25 | # 拷贝 ssl 证书. 26 | if ! test -d $nginx_ssl_dest_dir 27 | then 28 | mkdir $nginx_ssl_dest_dir 29 | fi 30 | /bin/cp -f $nginx_ssl_origin_dir/* $nginx_ssl_dest_dir/ 1>&2 ||exit 1 31 | 32 | 33 | # 进入 sites-enabled. 34 | if ! test -d sites-enabled 35 | then 36 | mkdir sites-enabled ||exit 1 37 | fi 38 | cd sites-enabled ||exit 1 39 | 40 | 41 | # 如果不是目录, 退出. 42 | if ! test -d ../sites-available/$product/${_type}/$idc/ 43 | then 44 | echo "../sites-available/$product/${_type}/$idc/ is not a directory." 1>&2 45 | exit 1 46 | fi 47 | 48 | 49 | # 如果目录为空, 退出. 50 | files=`ls ../sites-available/$product/${_type}/$idc/` 51 | if [ "$files" == "" ] 52 | then 53 | echo "../sites-available/$product/${_type}/$idc/ not include any files." 1>&2 54 | exit 1 55 | fi 56 | 57 | 58 | # 做软链. 59 | for file in $files 60 | do 61 | if ! ln -sf ../sites-available/$product/${_type}/$idc/$file $file 62 | then 63 | echo "exec ln -sf ../sites-available/$product/${_type}/$idc/$file $file failed." 1>&2 64 | exit 1 65 | fi 66 | done 67 | 68 | 69 | # 除了做域名的软链之外, 还要做日志格式的软链. 70 | if ! test -d ../log_format-enabled/ 71 | then 72 | mkdir ../log_format-enabled/ ||exit 1 73 | fi 74 | cd ../log_format-enabled/ ||exit 1 75 | if test -d ../log_format-available/$product/${_type}/$idc/ 76 | then 77 | files=`ls ../log_format-available/$product/${_type}/$idc/` 78 | for file in $files 79 | do 80 | if ! ln -sf ../log_format-available/$product/${_type}/$idc/$file $file 81 | then 82 | echo "exec ln -sf ../log_format-available/$product/${_type}/$idc/$file $file failed." 1>&2 83 | exit 1 84 | fi 85 | done 86 | fi 87 | 88 | 89 | exit 0 90 | -------------------------------------------------------------------------------- /nginx/libs/mysqloj.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import MySQLdb 4 | from DBUtils import PooledDB 5 | 6 | from web import const 7 | 8 | 9 | class PooledConnection(object): 10 | def __init__(self): 11 | self.pool = PooledDB.PooledDB(MySQLdb, 12 | user=const.MYSQL_USER, 13 | passwd=const.MYSQL_PASSWD, 14 | host=const.MYSQL_HOST, 15 | port=const.MYSQL_PORT, 16 | db=const.MYSQL_DATABASE, 17 | mincached=3, 18 | maxcached=10, 19 | maxshared=3, 20 | maxconnections=100 21 | ) 22 | 23 | def get(self): 24 | """ 返回一个数据库连接. 25 | 26 | """ 27 | return self.pool.connection() 28 | 29 | def select(self, sql): 30 | """ 查询数据库. 31 | 32 | """ 33 | conn = self.get() 34 | cur = conn.cursor() 35 | cur.execute(sql) 36 | # res = cur.fetchone() 37 | res = cur.fetchall() 38 | cur.close() 39 | conn.close() 40 | return res 41 | 42 | def change(self, sql): 43 | """ 修改数据库. 44 | 45 | """ 46 | conn = self.get() 47 | cur = conn.cursor() 48 | try: 49 | cur.execute(sql) 50 | conn.commit() 51 | return (True, None) 52 | except Exception, e: 53 | conn.rollback() 54 | finally: 55 | cur.close() 56 | conn.close() 57 | 58 | return (False, "%s" % e) 59 | 60 | def sqls(self, _sqls): 61 | """ 多条 sql 事务执行. 62 | 63 | """ 64 | conn = self.get() 65 | cur = conn.cursor() 66 | try: 67 | # cur.execute("set autocommit=0 ") 68 | for sql in _sqls: 69 | cur.execute(sql) 70 | conn.commit() 71 | res = cur.fetchall() 72 | return (True, res) 73 | except Exception, e: 74 | conn.rollback() 75 | finally: 76 | cur.close() 77 | conn.close() 78 | 79 | return (False, "%s" % e) 80 | -------------------------------------------------------------------------------- /nginx/libs/template.py: -------------------------------------------------------------------------------- 1 | #-*- coding: utf-8 -*- 2 | 3 | """ 生成 upstream nodes 的配置文件. 4 | 5 | """ 6 | 7 | 8 | from jinja2 import Environment, FileSystemLoader 9 | 10 | 11 | def gen_upstream(template_dir, template_file, template_dest, data): 12 | """ 生成 upstream nodes 配置文件. 13 | 14 | data 是一个 list, list 里面是一个 dict, dict 15 | 里面有 node_name, servers, port, servers 又是 16 | 一个 list. 17 | 18 | """ 19 | j2_env = Environment(loader=FileSystemLoader(template_dir), 20 | trim_blocks=True) 21 | ret = j2_env.get_template(template_file).render(data=data) 22 | with file(template_dest, 'w') as f: 23 | f.writelines(ret) 24 | 25 | 26 | def gen_server(template_dir, template_file, template_dest, 27 | server_names, log_name, log_format, upstream_node): 28 | """ 生成 server 配置文件. 29 | 30 | 传入变量: 31 | server_names 32 | log_name 33 | log_format 34 | upstream_node 35 | 36 | """ 37 | j2_env = Environment(loader=FileSystemLoader(template_dir), 38 | trim_blocks=True) 39 | ret = j2_env.get_template(template_file).render(server_names = server_names, 40 | log_name = log_name, 41 | log_format = log_format, 42 | upstream_node = upstream_node) 43 | with file(template_dest, 'w') as f: 44 | f.writelines(ret) 45 | -------------------------------------------------------------------------------- /nginx/libs/upstream.py: -------------------------------------------------------------------------------- 1 | #-*- coding: utf-8 -*- 2 | 3 | """ 获取, 修改和删除 Nginx upstream 信息. 4 | 5 | """ 6 | 7 | import numbers 8 | 9 | from libs import log 10 | 11 | from web import const 12 | from nginx.libs import mysqloj 13 | 14 | 15 | logger = log.get_logger("Nginx UPSTREAM ") 16 | 17 | 18 | def get(): 19 | """ 获取 upstream 信息. 20 | 21 | """ 22 | _mysql_oj = mysqloj.PooledConnection() 23 | sql = "select name, loki_id, port, ip_hash, online "\ 24 | "from %s;" % const.MYSQL_TABLE 25 | ret = _mysql_oj.select(sql) 26 | 27 | data = list() 28 | for i in ret: 29 | _dict = { 30 | "name": i[0], 31 | "loki_id": i[1], 32 | "port": i[2], 33 | "ip_hash": i[3], 34 | "online": i[4] 35 | } 36 | data.append(_dict) 37 | return data 38 | 39 | 40 | def add(name, loki_id, port, ip_hash, online): 41 | """ 增加 upstream. 42 | 43 | """ 44 | if not (isinstance(loki_id, numbers.Integral) and isinstance(port, numbers.Integral)): 45 | raise("loki_id and port must be int.") 46 | 47 | if ip_hash not in [0, 1] or online not in [0, 1]: 48 | raise("ip_hash and online must be 0 or 1.") 49 | 50 | sql = """insert into %s (name, loki_id, port, ip_hash, """\ 51 | """online) values ("%s", %s, %s, %s, %s); """ % (const.MYSQL_TABLE, 52 | name, loki_id, 53 | port, ip_hash, online) 54 | return _exec_change_sql(sql) 55 | 56 | 57 | def modify(name, _dict): 58 | """ 根据 name 修改 upstream. 59 | 60 | _dict 格式是: 61 | { 62 | "loki_id": loki_id, 63 | "port": port, 64 | "ip_hash": ip_hash, 65 | "online": online 66 | } 67 | key 可以一个或多个. 68 | 69 | """ 70 | _set_str = list() 71 | for key, value in _dict.items(): 72 | _set_str.append("%s=%s" % (key, value)) 73 | sql = "update %s set %s where %s.name='%s'" % (const.MYSQL_TABLE, 74 | ",".join(_set_str), 75 | const.MYSQL_TABLE, 76 | name) 77 | return _exec_change_sql(sql) 78 | 79 | 80 | def delete(name): 81 | """ 根据 name 删除 upstream. 82 | 83 | """ 84 | sql = "delete from %s where name='%s' " % (const.MYSQL_TABLE, name) 85 | return _exec_change_sql(sql) 86 | 87 | 88 | def _exec_change_sql(sql): 89 | """ 执行会改变数据库的 sql. 90 | 91 | """ 92 | _mysql_oj = mysqloj.PooledConnection() 93 | status, result = _mysql_oj.change(sql) 94 | message = "sql:%s, status:%s, result:%s" % (sql, status, result) 95 | logger.info(message) 96 | return (status, result) 97 | -------------------------------------------------------------------------------- /nginx/template/server.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80; 3 | server_name {% for server_name in server_names %} {{ server_name }}{% endfor %}; 4 | access_log /home/work/nginx/logs/{{log_name}}.access.log {{log_format}}; 5 | error_log /home/work/nginx/logs/{{log_name}}.error.log; 6 | 7 | location / { 8 | proxy_set_header Host $host; 9 | proxy_pass http://{{upstream_node}}; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /nginx/template/upstream.conf: -------------------------------------------------------------------------------- 1 | {% for _data in data %} 2 | upstream {{_data["name"]}} { 3 | {% if _data["ip_hash"] == 1 %} 4 | ip_hash; 5 | {% endif %} 6 | 7 | {% for _server in _data["servers"] %} 8 | server {{_server}}:{{_data["port"]}}; 9 | {% endfor %} 10 | 11 | check interval=3000 rise=2 fall=3 timeout=1000 type=http port=1023; 12 | check_http_send "HEAD /index.html HTTP/1.1\r\nConnection: keep-alive\r\nHost:nginx.check\r\n\r\n"; 13 | check_http_expect_alive http_2xx http_3xx; 14 | } 15 | 16 | {% endfor %} 17 | -------------------------------------------------------------------------------- /nginx/web/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex1528/nginx_conf/db661fb231deedf4c337aecb93e02e519daec5d3/nginx/web/__init__.py -------------------------------------------------------------------------------- /nginx/web/service.py: -------------------------------------------------------------------------------- 1 | #-*- coding: utf-8 -*- 2 | 3 | 4 | import socket 5 | import sys 6 | import os 7 | 8 | import ujson as json 9 | from concurrent.futures import ThreadPoolExecutor 10 | 11 | import tornado.web 12 | from tornado import gen 13 | 14 | from libs import utils 15 | from lvs.libs import info 16 | from nginx.libs import wstype, conf, upstream, domains 17 | from web.const import NGINX_CONF_DIR 18 | 19 | 20 | def _self_write(status, name, func, fail_msg=None, succ_msg=None): 21 | """ 代码重用. 22 | 23 | """ 24 | if fail_msg is None: 25 | fail = { 26 | "status": "failed", 27 | "message": "%s %sed failed" % (name, func) 28 | } 29 | else: 30 | fail = { 31 | "status": "failed", 32 | "message": fail_msg 33 | } 34 | 35 | if succ_msg is None: 36 | succ = { 37 | "status": "success", 38 | "message": "%s has been %sed successfully" % (name, func) 39 | } 40 | else: 41 | succ = { 42 | "status": "success", 43 | "message": succ_msg 44 | } 45 | 46 | if status: 47 | return json.dumps(succ) 48 | else: 49 | return json.dumps(fail) 50 | 51 | 52 | #class WstypeHander(tornado.web.RequestHandler): 53 | # 54 | # def get(self): 55 | # """ 拿到每一个 wstype 的 Nginx 列表. 56 | # 57 | # """ 58 | # cluster_nginxs = list() 59 | # _list = list() 60 | # 61 | # for cluster_lvs in info.cluster(): 62 | # for vip2ws in cluster_lvs["vip2ws"]: 63 | # _dict = {} 64 | # _wstype = "%s:%s" % (cluster_lvs["name"], vip2ws["wstype"]) 65 | # if _wstype not in _list: 66 | # _dict["wstype"] = _wstype 67 | # _dict["wss"] = vip2ws["wss"] 68 | # cluster_nginxs.append(_dict) 69 | # _list.append(_wstype) 70 | # else: 71 | # for cluster_nginx in cluster_nginxs: 72 | # if cluster_nginx["wstype"] == _wstype: 73 | # cluster_nginx["wss"].extend(vip2ws["wss"]) 74 | # 75 | # # wss 列能有重复, 需清理. 76 | # for cluster_nginx in cluster_nginxs: 77 | # cluster_nginx["wss"] = {}.fromkeys(cluster_nginx["wss"]).keys() 78 | # 79 | # self.write(json.dumps(cluster_nginxs)) 80 | 81 | 82 | class ConfHandler(tornado.web.RequestHandler): 83 | global executor 84 | executor = ThreadPoolExecutor(max_workers=3) 85 | 86 | @tornado.gen.coroutine 87 | def get(self): 88 | """ 根据 Nginx 机器生成 Nginx 配置文件包和并返回下载链接. 89 | 90 | """ 91 | nginx = self.get_argument('nginx') 92 | branch = self.get_argument('branch') 93 | 94 | _ret = yield executor.submit(conf.get, nginx, branch) 95 | ret = { 96 | "status": "success", 97 | "message": _ret 98 | } 99 | self.write(json.dumps(ret)) 100 | 101 | 102 | class BranchesHandler(tornado.web.RequestHandler): 103 | 104 | def get(self): 105 | """ 返回 Nginx 的分支. 106 | 107 | """ 108 | cmd = "cd %s &&git fetch origin -p &&git branch -r" % NGINX_CONF_DIR 109 | rc, so, se = utils.shell(cmd) 110 | branches = [ 111 | i.strip().replace("origin/", "") 112 | for i in so.strip().splitlines() 113 | if "origin/HEAD" not in i 114 | ] 115 | self.write(json.dumps(branches)) 116 | 117 | 118 | class UpstreamsHandler(tornado.web.RequestHandler): 119 | 120 | def get(self): 121 | """ 返回 upstream 列表. 122 | 123 | """ 124 | self.write(json.dumps(upstream.get())) 125 | 126 | def post(self): 127 | """ 增加 upstream. 128 | 129 | """ 130 | name = json.loads(self.get_argument('name')) 131 | loki_id = json.loads(self.get_argument('loki_id')) 132 | port = json.loads(self.get_argument('port')) 133 | ip_hash = json.loads(self.get_argument('ip_hash')) 134 | online = json.loads(self.get_argument('online')) 135 | status, result = upstream.add(name, loki_id, port, ip_hash, online) 136 | self.write(_self_write(status, name, "add", result)) 137 | 138 | 139 | class UpstreamHandler(tornado.web.RequestHandler): 140 | 141 | def patch(self, name): 142 | """ 增加 upstream. 143 | 144 | """ 145 | data = json.loads(self.get_argument('data')) 146 | status, result = upstream.modify(name, data) 147 | self.write(_self_write(status, name, "modify", result)) 148 | 149 | def delete(self, name): 150 | """ 删除 upstream. 151 | 152 | """ 153 | status, result = upstream.delete(name) 154 | self.write(_self_write(status, name, "delet", result)) 155 | 156 | 157 | class DomainsHandler(tornado.web.RequestHandler): 158 | 159 | def post(self): 160 | """ 增加 域名. 161 | 162 | """ 163 | product = json.loads(self.get_argument('product')) 164 | _type = json.loads(self.get_argument('type')) 165 | idc = json.loads(self.get_argument('idc')) 166 | name = json.loads(self.get_argument('name')) 167 | server_names = json.loads(self.get_argument('server_names')) 168 | upstream_node = json.loads(self.get_argument('upstream_node')) 169 | log_name = json.loads(self.get_argument('log_name')) 170 | log_format = json.loads(self.get_argument('log_format')) 171 | 172 | status = domains.add(product, _type, idc, name, 173 | server_names, log_name, 174 | log_format, upstream_node) 175 | self.write(_self_write(status, name, "add")) -------------------------------------------------------------------------------- /restart.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export PYTHONPATH=. 4 | 5 | ps -ef |grep main_service.py |grep -v grep |awk '{print $2}' |xargs sudo kill 6 | 7 | nohup python web/main_service.py & 8 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export PYTHONPATH=. 4 | 5 | nohup python web/main_service.py & 6 | -------------------------------------------------------------------------------- /stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ps -ef |grep main_service.py |grep -v grep |awk '{print $2}' |xargs sudo kill 4 | -------------------------------------------------------------------------------- /web/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex1528/nginx_conf/db661fb231deedf4c337aecb93e02e519daec5d3/web/__init__.py -------------------------------------------------------------------------------- /web/const.py: -------------------------------------------------------------------------------- 1 | #-*- coding: utf-8 -*- 2 | 3 | 4 | # 绑定 IP 和 端口. 5 | BIND_IP = "0.0.0.0" 6 | BIND_PORT = "8084" 7 | 8 | # 日志路径. 9 | LOG_DIR = "./logs/" 10 | LOG_FILE = "nginx_conf.log" 11 | 12 | # REDIS 信息. 13 | REDIS_HOST = "" 14 | REDIS_PORT = 6379 15 | REDIS_DB_NGINX = "0" 16 | 17 | # Nginx 信息. 18 | NGINX_CONF_DIR = "/home/work/nginx_cfg/" # Nginx git 项目配置文件. 19 | NGINX_TMP_STORAGE_DIR = "nginx_conf/" # 临时存储 Nginx 配置主目录. 20 | NGINX_TEMPLATE_DIR = "nginx/template/" 21 | NGINX_UPSTREAM_TEMPLATE_FILE = "upstream.conf" 22 | NGINX_SERVER_TEMPLATE_FILE = "server.conf" 23 | 24 | # Nginx ssl 证书文件, ssl 证书放在 git 中不安全, 这里从 NGINX_SSL_ORIGIN_DIR 25 | # 拷贝到 NGINX_CONF_DIR 目录下的 NGINX_SSL_DEST_DIR 目录里. 26 | NGINX_SSL_ORIGIN_DIR = "" 27 | NGINX_SSL_DEST_DIR = "" 28 | 29 | # Nginx upstream nodes 信息放在数据库里. 30 | MYSQL_USER = "" 31 | MYSQL_PASSWD = "" 32 | MYSQL_HOST = "" 33 | MYSQL_PORT = 3306 34 | MYSQL_DATABASE = "" 35 | MYSQL_TABLE = "" 36 | -------------------------------------------------------------------------------- /web/main_service.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*- coding: utf-8 -*- 3 | 4 | 5 | import socket 6 | import sys 7 | import os 8 | 9 | import ujson as json 10 | 11 | import tornado.ioloop 12 | import tornado.web 13 | import tornado.httpserver 14 | import tornado.options 15 | import tornado.auth 16 | import tornado.web 17 | import tornado.escape 18 | import tornado.netutil 19 | 20 | from web.const import BIND_IP 21 | from web.const import BIND_PORT 22 | from nginx.web import service as nginx_service 23 | 24 | 25 | class Application(tornado.web.Application): 26 | 27 | def __init__(self): 28 | handlers = [ 29 | #(r"/api/v1/nginx/?", nginx_service.WstypeHander), 30 | (r"/api/v1/nginx/conf/?", nginx_service.ConfHandler), 31 | (r"/api/v1/nginx/branches/?", nginx_service.BranchesHandler), 32 | (r"/api/v1/nginx/upstreams/([^/]+)/?", nginx_service.UpstreamHandler), 33 | (r"/api/v1/nginx/upstreams/?", nginx_service.UpstreamsHandler), 34 | (r"/api/v1/nginx/domains/?", nginx_service.DomainsHandler) 35 | ] 36 | 37 | settings = {} 38 | 39 | tornado.web.Application.__init__(self, handlers, **settings) 40 | 41 | 42 | def main(): 43 | application = Application() 44 | 45 | sockets = tornado.netutil.bind_sockets( 46 | BIND_PORT, address=BIND_IP, family=socket.AF_INET) 47 | tornado.process.fork_processes(0) 48 | 49 | http_server = tornado.httpserver.HTTPServer(application, xheaders=True) 50 | http_server.add_sockets(sockets) 51 | tornado.ioloop.IOLoop.instance().start() 52 | 53 | 54 | if __name__ == "__main__": 55 | main() 56 | --------------------------------------------------------------------------------