├── .gitignore ├── ALI.py ├── AWS.py ├── KSC.py ├── README-en.md ├── README.md ├── cloud2falcon.py ├── config.yml ├── info.png ├── log.py ├── logging.yml ├── multiCloud.py ├── requirements.txt └── templates ├── ali-connect ├── ali-connect-max ├── ali-eip ├── ali-eip-max ├── ali-nat ├── ali-oss ├── aws-connect ├── aws-lb ├── aws-nat ├── aws-s3 ├── ksc-connect ├── ksc-connect-max ├── ksc-eip ├── ksc-eip-max ├── ksc-elb ├── ksc-elb-max ├── ksc-nat └── ksc-nat-max /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | __pycache__/ 3 | *.pyc 4 | log/* 5 | venv/ 6 | test.py -------------------------------------------------------------------------------- /ALI.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import datetime 4 | import oss2 5 | import time 6 | from aliyunsdkcore.request import CommonRequest 7 | from aliyunsdkcore.client import AcsClient 8 | from aliyunsdkslb.request.v20140515 import DescribeLoadBalancersRequest 9 | from aliyunsdkvpc.request.v20160428 import DescribeEipAddressesRequest 10 | from aliyunsdkvpc.request.v20160428 import DescribeNatGatewaysRequest 11 | from cloud2falcon import PERIOD 12 | 13 | 14 | def chunks(l, n): 15 | for i in range(0, len(l), n): 16 | yield l[i: i + n] 17 | 18 | 19 | def get_metric_data(period, namespace, name, id_list, 20 | metricname, ak, sk, region): 21 | metric_data = [] 22 | if metricname == 'BandWidth': 23 | ts = time.time() 24 | t = int(ts) 25 | for id in id_list: 26 | v = int(id['BandWidth']) * 1024 * 1024 27 | if v == 0: 28 | continue 29 | data = {"id": id['l'], "ip": id['d'], "region": region['site'], "metric": metricname, 30 | "time": t, "value": v} 31 | metric_data.append(data) 32 | for instance_id in list(chunks(id_list, 50)): 33 | data = get_metric_data_50( 34 | period, 35 | namespace, 36 | name, 37 | instance_id, 38 | metricname, 39 | ak, 40 | sk, 41 | region) 42 | metric_data += data 43 | return metric_data 44 | 45 | 46 | def get_metric_data_50(period, namespace, name, id_list, 47 | metricname, ak, sk, region): 48 | metric_data = [] 49 | clt = AcsClient(ak, sk, region['name']) 50 | request = CommonRequest() 51 | request.set_accept_format('json') 52 | request.set_domain('metrics.cn-hangzhou.aliyuncs.com') 53 | request.set_method('POST') 54 | request.set_version('2018-03-08') 55 | request.set_action_name('QueryMetricList') 56 | request.add_query_param('Metric', metricname) 57 | request.add_query_param('Period', period) 58 | request.add_query_param('Project', name) 59 | request.add_query_param( 60 | 'StartTime', 61 | datetime.datetime.now() - 62 | datetime.timedelta( 63 | minutes=PERIOD)) 64 | request.add_query_param('EndTime', datetime.datetime.now()) 65 | list = [] 66 | for instance_id in id_list: 67 | if (name == "acs_nat_gateway" and metricname == "SnatConnection") \ 68 | or name == "acs_publicip"\ 69 | or name == "acs_express_connect": 70 | list.append({'instanceId': str(instance_id['d'])}) 71 | elif name == "acs_oss": 72 | list.append({'BucketName': str(instance_id['d'])}) 73 | else: 74 | list.append({'instanceId': str(instance_id['l'])}) 75 | 76 | request.add_query_param('Dimensions', list) 77 | response = clt.do_action_with_exception(request) 78 | response1 = str(response) 79 | response1 = json.loads(response1) 80 | try: 81 | data = response1['Datapoints'] 82 | data1 = json.loads(data) 83 | except BaseException: 84 | logging.debug("no data responce: " + metricname) 85 | return metric_data 86 | 87 | if name == "acs_slb_dashboard": 88 | for record in data1: 89 | timestamp = int(record['timestamp'] / 1000) 90 | data = {"id": record['instanceId'], "ip": record['vip'], "region": region['site'], "metric": metricname, 91 | "time": timestamp, "value": record['Average']} 92 | metric_data.append(data) 93 | elif name == "acs_nat_gateway" and metricname == "SnatConnection": 94 | for record in data1: 95 | timestamp = int(record['timestamp'] / 1000) 96 | data = {"id": record['instanceId'], "ip": "", "region": region['site'], "metric": metricname, 97 | "time": timestamp, "value": record['Maximum']} 98 | metric_data.append(data) 99 | elif name == "acs_publicip": 100 | for record in data1: 101 | timestamp = int(record['timestamp'] / 1000) 102 | data = {"id": instance_id['l'], "ip": record['ip'], "region": region['site'], "metric": metricname, 103 | "time": timestamp, "value": record['value']} 104 | metric_data.append(data) 105 | elif name == "acs_oss": 106 | for record in data1: 107 | timestamp = int(record['timestamp'] / 1000) 108 | data = {"id": record['BucketName'], "ip": '', "region": region['site'], "metric": metricname, 109 | "time": timestamp, "value": record[metricname]} 110 | metric_data.append(data) 111 | else: 112 | for record in data1: 113 | timestamp = int(record['timestamp'] / 1000) 114 | data = {"id": record['instanceId'], "ip": '', "region": region['site'], "metric": metricname, 115 | "time": timestamp, "value": record['Value']} 116 | metric_data.append(data) 117 | 118 | metric_data.sort(key=lambda x: x["time"]) 119 | return metric_data 120 | 121 | 122 | def get_id(resource, ak, sk, region): 123 | if resource == "ELB": 124 | return elb(ak, sk, region) 125 | elif resource == "EIP": 126 | return eip(ak, sk, region) 127 | elif resource == "NAT": 128 | return nat(ak, sk, region) 129 | elif resource == "connect": 130 | return connect(ak, sk, region) 131 | elif resource == "oss": 132 | return oss(ak, sk, region) 133 | 134 | 135 | def elb(ak, sk, region): 136 | id_list = [] 137 | client = AcsClient(ak, sk, region) 138 | request = DescribeLoadBalancersRequest.DescribeLoadBalancersRequest() 139 | request.set_accept_format('json') 140 | response = client.do_action_with_exception(request) 141 | response1 = str(response) 142 | response1 = json.loads(response1) 143 | for record in response1['LoadBalancers']['LoadBalancer']: 144 | id_list.append({"l": record['LoadBalancerId'], "d": record['Address']}) 145 | return id_list 146 | 147 | 148 | def eip(ak, sk, region): 149 | id_list = [] 150 | client = AcsClient(ak, sk, region) 151 | request = DescribeEipAddressesRequest.DescribeEipAddressesRequest() 152 | request.set_accept_format('json') 153 | response = client.do_action_with_exception(request) 154 | response1 = str(response) 155 | response1 = json.loads(response1) 156 | id = [] 157 | for record in response1['EipAddresses']['EipAddress']: 158 | id_list.append({"l": record['AllocationId'], 159 | "d": record['IpAddress'], 160 | "BandWidth": record['Bandwidth']}) 161 | return id_list 162 | 163 | 164 | def nat(ak, sk, region): 165 | id_list = [] 166 | client = AcsClient(ak, sk, region) 167 | request = DescribeNatGatewaysRequest.DescribeNatGatewaysRequest() 168 | request.set_accept_format('json') 169 | response = client.do_action_with_exception(request) 170 | response1 = str(response) 171 | response1 = json.loads(response1) 172 | for record in response1['NatGateways']['NatGateway']: 173 | for band in record['BandwidthPackageIds']['BandwidthPackageId']: 174 | id_list.append({"l": band, "d": record['NatGatewayId']}) 175 | return id_list 176 | 177 | 178 | def connect(ak, sk, region): 179 | id_list = [] 180 | client = AcsClient(ak, sk, region) 181 | request = CommonRequest() 182 | request.set_accept_format('json') 183 | request.set_domain('vpc.aliyuncs.com') 184 | request.set_method('POST') 185 | request.set_version('2016-04-28') 186 | request.set_action_name('DescribeRouterInterfaces') 187 | request.add_query_param('PageSize', '50') 188 | response = client.do_action_with_exception(request) 189 | response1 = str(response) 190 | response1 = json.loads(response1) 191 | for record in response1['RouterInterfaceSet']['RouterInterfaceType']: 192 | id_list.append({"l": record['OppositeInterfaceId'], 193 | "d": record['OppositeInterfaceId'], 194 | "BandWidth": record['Bandwidth']}) 195 | return id_list 196 | 197 | 198 | def oss(ak, sk, region): 199 | id_list = [] 200 | auth = oss2.Auth(ak, sk) 201 | service = oss2.Service(auth, 'http://oss-cn-hangzhou.aliyuncs.com') 202 | for b in oss2.BucketIterator(service): 203 | id_list.append({"l": "", "d": b.name}) 204 | return id_list 205 | -------------------------------------------------------------------------------- /AWS.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import datetime 3 | import time 4 | import pytz 5 | from cloud2falcon import PERIOD 6 | 7 | 8 | def get_metric_data(period, namespace, name, id_list, 9 | metricname, ak, sk, region): 10 | metric_data = [] 11 | for instance_id in id_list: 12 | client = boto3.client('cloudwatch', 13 | region_name=region['name'], 14 | aws_access_key_id=ak, 15 | aws_secret_access_key=sk 16 | ) 17 | response = client.get_metric_data( 18 | MetricDataQueries=[ 19 | { 20 | 'Id': 'm1', 21 | 'MetricStat': { 22 | 'Metric': { 23 | 'Namespace': namespace, 24 | 'MetricName': metricname, 25 | 'Dimensions': [ 26 | { 27 | 'Name': name, 28 | 'Value': instance_id['l'] 29 | }, 30 | ] 31 | }, 32 | 'Period': period, 33 | 'Stat': 'SampleCount', 34 | 'Unit': 'Count' 35 | }, 36 | 'ReturnData': True 37 | } 38 | ], 39 | StartTime=datetime.datetime.utcnow() - datetime.timedelta(minutes=PERIOD), 40 | EndTime=datetime.datetime.utcnow() 41 | ) 42 | r = response['MetricDataResults'][0] 43 | for j in range(len(r['Values'])): 44 | t = r['Timestamps'][j].astimezone(pytz.timezone('Asia/Shanghai')) 45 | t = int(time.mktime(t.timetuple())) 46 | data = {"id": instance_id['l'], "ip": instance_id['d'], "region": region['site'], "metric": metricname, 47 | "time": t, "value": r['Values'][j]} 48 | metric_data.append(data) 49 | 50 | metric_data.sort(key=lambda x: x["time"]) 51 | return metric_data 52 | 53 | 54 | def get_id(resource, ak, sk, region): 55 | if resource == "ELB": 56 | return elb(ak, sk, region) 57 | elif resource == "NATGateway": 58 | return nat(ak, sk, region) 59 | elif resource == "DX": 60 | return connect(ak, sk, region) 61 | elif resource == "S3": 62 | return s3(ak, sk, region) 63 | 64 | 65 | def elb(ak, sk, region): 66 | id_list = [] 67 | client = boto3.client('elb', 68 | region_name=region, 69 | aws_access_key_id=ak, 70 | aws_secret_access_key=sk 71 | ) 72 | response = client.describe_load_balancers() 73 | for record in response['LoadBalancerDescriptions']: 74 | id_list.append({"l": record['LoadBalancerName'], "d": record['DNSName']}) 75 | return id_list 76 | 77 | 78 | def nat(ak, sk, region): 79 | id_list = [] 80 | client = boto3.client('ec2', 81 | region_name=region, 82 | aws_access_key_id=ak, 83 | aws_secret_access_key=sk 84 | ) 85 | response = client.describe_nat_gateways() 86 | for record in response['NatGateways']: 87 | id_list.append({"l": record['NatGatewayId'], "d": ''}) 88 | return id_list 89 | 90 | 91 | def connect(ak, sk, region): 92 | id_list = [] 93 | client = boto3.client('directconnect', 94 | region_name=region, 95 | aws_access_key_id=ak, 96 | aws_secret_access_key=sk 97 | ) 98 | response = client.describe_connections() 99 | for record in response['connections']: 100 | id_list.append({"l": record['connectionId'], "d": record['bandwidth']}) 101 | return id_list 102 | 103 | 104 | def s3(ak, sk, region): 105 | id_list = [] 106 | client = boto3.client('s3', 107 | region_name=region, 108 | aws_access_key_id=ak, 109 | aws_secret_access_key=sk 110 | ) 111 | response = client.list_buckets() 112 | for record in response['Buckets']: 113 | id_list.append({"l": record['Name'], "d": ''}) 114 | return id_list 115 | -------------------------------------------------------------------------------- /KSC.py: -------------------------------------------------------------------------------- 1 | from kscore.session import get_session 2 | import json 3 | import time 4 | import logging 5 | from datetime import datetime, timedelta 6 | from cloud2falcon import PERIOD 7 | 8 | 9 | def get_one_metric(namespace, region, metricname, period, id): 10 | s = get_session() 11 | client = s.create_client("monitor", region, use_ssl=True) 12 | now = datetime.now() 13 | start = datetime.now() - timedelta(minutes=PERIOD) 14 | ISOFORMAT = "%Y-%m-%dT%XZ" 15 | m = client.get_metric_statistics( 16 | InstanceID=id, 17 | Namespace=namespace, 18 | MetricName=metricname, 19 | StartTime=start.strftime(ISOFORMAT), 20 | EndTime=now.strftime(ISOFORMAT), 21 | Period='60', 22 | Aggregate='Average' 23 | ) 24 | return json.dumps(m, sort_keys=True, indent=4) 25 | 26 | 27 | def get_metric_data(period, namespace, name, id_list, 28 | metricname, ak, sk, region): 29 | metric_data = [] 30 | ISOFORMAT = "%Y-%m-%dT%XZ" 31 | for id in id_list: 32 | if metricname == 'BandWidth': 33 | ts = time.time() 34 | t = int(ts) 35 | v = id['BandWidth'] * 1000 * 1000 36 | data = {"id": id['l'], "ip": id['d'], "region": region['site'], "metric": metricname, 37 | "time": t, "value": v} 38 | metric_data.append(data) 39 | continue 40 | response = json.loads( 41 | get_one_metric( 42 | name, 43 | region['name'], 44 | metricname, 45 | period, 46 | id['l'])) 47 | try: 48 | metric_list = response['getMetricStatisticsResult']['datapoints']['member'] 49 | for metric in metric_list: 50 | ts = time.strptime(metric['timestamp'], ISOFORMAT) 51 | t = int(time.mktime(ts)) 52 | data = {"id": id['l'], "ip": id['d'], "region": region['site'], "metric": metricname, 53 | "time": t, "value": metric['average']} 54 | metric_data.append(data) 55 | except BaseException: 56 | logging.error('responce from ksc error') 57 | metric_data.sort(key=lambda x: x["time"]) 58 | return metric_data 59 | 60 | 61 | def get_id(resource, ak, sk, region): 62 | if resource == "elb": 63 | return elb(ak, sk, region) 64 | elif resource == "eip": 65 | return eip(ak, sk, region) 66 | elif resource == "nat": 67 | return nat(ak, sk, region) 68 | elif resource == "connect": 69 | return connect(ak, sk, region) 70 | 71 | 72 | def elb(ak, sk, region): 73 | id_list = [] 74 | s = get_session() 75 | region = region 76 | eipClient = s.create_client("eip", region, use_ssl=True) 77 | allEips = eipClient.describe_addresses( 78 | **{'Filter.1.Name': 'instance-type', 'Filter.1.Value.1': 'Slb'}) 79 | for item in allEips['AddressesSet']: 80 | id_list.append({"l": item['InstanceId'], 81 | "d": item['PublicIp'], 82 | "BandWidth": item['BandWidth']}) 83 | return id_list 84 | 85 | 86 | def eip(ak, sk, region): 87 | id_list = [] 88 | s = get_session() 89 | region = region 90 | eipClient = s.create_client("eip", region, use_ssl=True) 91 | allEips = eipClient.describe_addresses( 92 | **{'Filter.1.Name': 'instance-type', 'Filter.1.Value.1': 'Ipfwd'}) 93 | for item in allEips['AddressesSet']: 94 | id_list.append({"l": item['AllocationId'], 95 | "d": item['PublicIp'], 96 | "BandWidth": item['BandWidth']}) 97 | return id_list 98 | 99 | 100 | def nat(ak, sk, region): 101 | s = get_session() 102 | id_list = [] 103 | region = region 104 | vpcClient = s.create_client("vpc", region, use_ssl=True) 105 | allVpcs = vpcClient.describe_nats() 106 | for item in allVpcs['NatSet']: 107 | id_list.append({"l": item['NatId'], 108 | "d": item['NatName'], 109 | "BandWidth": item['BandWidth']}) 110 | return id_list 111 | 112 | 113 | def connect(ak, sk, region): 114 | s = get_session() 115 | id_list = [] 116 | region = region 117 | vpcClient = s.create_client("vpc", region, use_ssl=True) 118 | allConnect = vpcClient.describe_direct_connect_gateways() 119 | for item in allConnect['DirectConnectGatewaySet']: 120 | id_list.append({"l": item['DirectConnectGatewayId'], 121 | "d": item['DirectConnectGatewayName'], 122 | "BandWidth": item['BandWidth']}) 123 | return id_list 124 | -------------------------------------------------------------------------------- /README-en.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | The main purpose of this project is Getting public cloud monitor metric data (eg: ELB, EIP, NAT Gateway, connections, S3) 3 | to falcon. We can check the monitor metric data which can't get by monitor agent installed in the machine but avaiable on the cloud console at any time 4 | 5 | # Prerequisite 6 | * git >= 1.7.5 7 | * python2.X 8 | * develop in ubuntu16, test in centos7.3 9 | 10 | 11 | For getting KSC data, please read: 12 | https://github.com/KscSDK/ksc-sdk-python 13 | 14 | # Getting Started 15 | ```buildoutcfg 16 | 1. git clone http:https://github.com/open-falcon/cloud-mon 17 | 2. pip install -r requirements.txt 18 | 3. config your config.yml as following guide 19 | 4. run : 'python cloud2falcon.py' 20 | 5. You may encount user follow control when get lots of metric data, please connect to cloud provider solve it. 21 | ``` 22 | 23 | ## Description： 24 | - It will get all metric data per minute in 10 minutes by default, change it by yourself 25 | - You can set linux cronjob for running it every 10 minutes 26 | - step and period of time according to demand and cloud user flow control. The example is my best practices 27 | - support: AWS, Alibaba cloud, kingsoft cloud 28 | - config.yml 29 | 30 | ## example: 31 | 32 | falcon_url: 'http://127.0.0.1:1988/v1/push' 33 | metric: 'cloud2falcon' 34 | step: 60 35 | period: 13 36 | cloud: 37 | - c_type: AWS 38 | resource: NATGateway 39 | name: 'NatGatewayId' 40 | to_falcon_template: 'aws-nat' 41 | ak: 'your ak' 42 | sk: 'your sk' 43 | region: [{"name": 'ap-southeast-1', "site": 'sgpaws'}] 44 | metric_list: ['ActiveConnectionCount', 'PacketsDropCount'] 45 | 46 | 47 | ##　adds on： 48 | 49 | | name | explanation | 50 | | ------ | ------ | 51 | |falcon_url |falcon url , set localhost url with falcon-agent installed | 52 | | metric | common config | 53 | | step | step of time | 54 | | period | period of time to get data once| 55 | | cloud |common config, required| 56 | | - c_type |cloud type（AWS, ALI, KSC）| 57 | | name | the name to get resouce instance id | 58 | | resource | resouce type, accrodding to cloud console | 59 | | to_falcon_template | the name in the templates folders files| 60 | | ak　 | access key | 61 | | sk　 | secrect key | 62 | | region | region json list。 name is console region name， site is named by yourself | 63 | | metric_list | metrics list provided by cloud provider's resouces list | 64 | 65 | 66 | ## cloud provider's resouces list 67 | - kingsoft cloud： https://docs.ksyun.com/documents/42 68 | - Alibaba cloud： https://help.aliyun.com/document_detail/28619.html?spm=a2c4g.11174283.6.672.3f5b8f4fcIKe96 69 | - AWS: https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CW_Support_For_AWS.html 70 | 71 | # Architecture 72 |
![arch](http://git.n.xiaomi.com/liuwenjia/cloud2falcon/raw/master/info.png)
73 | 74 | # Q&A 75 | Any issue or question is welcome, Please feel free to open github issues :) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 该项目用于取出公有云的非机器级别的监控数据（ELB, EIP, NAT网关，专线，S3）， 推送到falcon，有利于实时查看。 3 | 主要解决利用监控工具在机器上安装agent,取不到控制台上非机器级别的监控数据，但是大家又非常关心的一些资源类型的指标 4 | 5 | 6 | # Prerequisite 7 | * git >= 1.7.5 8 | * python2.X 9 | * 该代码在ubuntu16, centos7.3上面顺利运行没问题。其他平台没有测试，理论上只有对2.7的依赖 10 | 11 | 12 | 使用金山云的，请按照官网配置 13 | https://github.com/KscSDK/ksc-sdk-python 14 | 15 | # Getting Started 16 | ```buildoutcfg 17 | 1. git clone http:https://github.com/open-falcon/cloud-mon 18 | 2. pip install -r requirements.txt 19 | # 若还有其他的pip确实的情况，请自行手动安装 20 | 3. 编写配置文件config.yml , 编写说明见下. 代码里面的配置文件，可以之间添加上你自己的ak, sk就可以顺利运行。 21 | 4. python cloud2falcon.py 启动程序 22 | 5. 如果你们云控制台需要取的数据较多，可能遇到各家云厂商的流控问题，到时候请和各家云厂商沟通。 23 | ``` 24 | 25 | ## 说明： 26 | - 代码默认跑一次取十分钟之内的数据，每分钟一个点，可以自行修改代码的配置 27 | - 定时启动可以利用linux cronjob配置每十分钟跑一次 28 | - 关于多久跑一次，每次取的时间段，和数据的颗粒度。需要自己根据需求和云厂商的流控限制来合理设置 29 | 目前例子给出的值是满足我们需求，并且经过云厂商提升流控的结果。实战推荐配置。 30 | - 目前支持的云： AWS 阿里云 金山云 31 | - 由于s3类型比较特殊，每天只有一个点的数据，代码单独在s3分支 32 | - config.yml 33 | 34 | ## example: 35 | 36 | falcon_url: 'http://127.0.0.1:1988/v1/push' 37 | metric: 'cloud2falcon' 38 | step: 60 39 | period: 13 40 | cloud: 41 | - c_type: AWS 42 | resource: NATGateway 43 | name: 'NatGatewayId' 44 | to_falcon_template: 'aws-nat' 45 | ak: 'your ak' 46 | sk: 'your sk' 47 | region: [{"name": 'ap-southeast-1', "site": 'sgpaws'}] 48 | metric_list: ['ActiveConnectionCount', 'PacketsDropCount'] 49 | 50 | 51 | ##　解释： 52 | 53 | | 配置项 | 解释 | 54 | | ------ | ------ | 55 | |falcon_url |falcon地址，一般装了falcon agent， 就可以直接配置本地 | 56 | | metric | 通用配置 | 57 | | step | 每次取数的时间间隔，以秒为单位 | 58 | | period | 取当前时间多久之前的数据，分钟为单位| 59 | | cloud |通用配置， 下面的某块按照云的类型和资源类型可以随意追加| 60 | | - c_type |云类型（AWS, ALI, KSC） | 61 | | name | 获取资源id的时候的名字 | 62 | | resource | 资源类型（取决于控制台上面的名字) | 63 | | to_falcon_template | 推送到falcon时的模板(文件夹templates 下面的文件名字，可以自己定义)| 64 | | ak　 | 调用云资源时的ak| 65 | | sk　 | 调用云资源时的sk| 66 | | region | 所有的region json列表。 name是指的是控制台的region的名字， site是自己给那个区域取的名字| 67 | | metric_list | 想要取的指标列表，详细参照各家云厂商文档| 68 | 69 | ## 云厂商的资源列表 70 | - 金山云： https://docs.ksyun.com/documents/42 71 | - 阿里云： https://help.aliyun.com/document_detail/28619.html?spm=a2c4g.11174283.6.672.3f5b8f4fcIKe96 72 | - AWS: https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CW_Support_For_AWS.html 73 | 74 | # Architecture 75 |
![arch](http://git.n.xiaomi.com/liuwenjia/cloud2falcon/raw/master/info.png)
76 | 77 | # Q&A 78 | Any issue or question is welcome, Please feel free to open github issues :) -------------------------------------------------------------------------------- /cloud2falcon.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | import log 5 | import yaml 6 | import threading 7 | import jinja2 8 | import requests 9 | import json 10 | import logging 11 | 12 | from multiCloud import get_id_list, get_metric_data 13 | 14 | 15 | with open('config.yml', 'r') as ymlfile: 16 | # 考虑声明在start函数内?? 只在cloud2falcon中调用。 17 | cfg = yaml.load(ymlfile) 18 | PERIOD = cfg['period'] 19 | 20 | 21 | def render_without_request(template_name, **context): 22 | """ 23 | usage same as flask.render_template: 24 | 25 | render_without_request('template.html', var1='foo', var2='bar') 26 | """ 27 | env = jinja2.Environment( 28 | loader=jinja2.PackageLoader('cloud2falcon', 'templates') 29 | ) 30 | template = env.get_template(template_name) 31 | return template.render(**context) 32 | 33 | 34 | def send_to_falcon(json_model, template_name): 35 | payload = render_without_request(template_name, metrics=json_model) 36 | # print payload 37 | data1 = json.loads(payload) 38 | # add timeout time 39 | r = requests.post(cfg['falcon_url'], data=json.dumps(data1), timeout=3) 40 | if r.status_code != 200: 41 | logging.error("send to falcon failed", r.json()) 42 | 43 | 44 | def peach_send_to_falcon(namespace, name, instance_id, 45 | metric, c_type, ak, sk, region, tempalte): 46 | metric_data = get_metric_data( 47 | cfg['step'], 48 | namespace, 49 | name, 50 | instance_id, 51 | metric, 52 | c_type, 53 | ak, 54 | sk, 55 | region) 56 | if metric_data: 57 | send_to_falcon(metric_data, tempalte) 58 | 59 | 60 | def get_metric_json(resource): 61 | sub_threads = [] 62 | for region in resource['region']: 63 | instance_id = get_id_list( 64 | resource['c_type'], 65 | resource['resource'], 66 | resource['ak'], 67 | resource['sk'], 68 | region['name']) 69 | for metric in resource['metric_list']: 70 | logging.info( 71 | 'process start for ' + 72 | metric + 73 | " " + 74 | resource['c_type']) 75 | namespace = resource['c_type'] + "/" + resource['resource'] 76 | t = threading.Thread( 77 | target=peach_send_to_falcon, 78 | args=( 79 | namespace, 80 | resource['name'], 81 | instance_id, 82 | metric, 83 | resource['c_type'], 84 | resource['ak'], 85 | resource['sk'], 86 | region, 87 | resource['to_falcon_template'] 88 | )) 89 | sub_threads.append(t) 90 | 91 | for i in range(len(sub_threads)): 92 | sub_threads[i].setDaemon(True) 93 | sub_threads[i].start() 94 | 95 | for i in range(len(sub_threads)): 96 | sub_threads[i].join(600) 97 | 98 | 99 | if __name__ == "__main__": 100 | # 需要修改文件名，来配合falcon plugin的运行机制 101 | log.setup_logging("logging.yml") 102 | threads = [] 103 | for res in cfg['cloud']: 104 | logging.info('start main process to get config') 105 | # multiple process 106 | t = threading.Thread(target=get_metric_json, args=(res, )) 107 | threads.append(t) 108 | for i in range(len(threads)): 109 | threads[i].start() 110 | -------------------------------------------------------------------------------- /config.yml: -------------------------------------------------------------------------------- 1 | falcon_url: 'http://127.0.0.1:1988/v1/push' 2 | metric: 'cloud2falcon' 3 | step: 60 4 | period: 13 5 | cloud: 6 | - c_type: KSC 7 | resource: elb 8 | name: 'SLB' 9 | to_falcon_template: 'ksc-elb' 10 | ak: 'your access key' 11 | sk: 'your secret key' 12 | region: [{'name': 'cn-beijing-6', 'site': 'ksybj'}] 13 | metric_list: ['slb.bps.in','slb.bps.out'] 14 | 15 | - c_type: KSC 16 | resource: elb 17 | name: 'SLB' 18 | to_falcon_template: 'ksc-elb-max' 19 | ak: 'your access key' 20 | sk: 'your secret key' 21 | region: [{'name': 'cn-beijing-6', 'site': 'ksybj'}] 22 | metric_list: ['BandWidth'] 23 | 24 | - c_type: ALI 25 | resource: connect 26 | name: 'acs_express_connect' 27 | to_falcon_template: 'ali-connect' 28 | ak: 'your access key' 29 | sk: 'your secret key' 30 | region: [{"name": 'ap-southeast-5', "site": 'jktali'}] 31 | metric_list: ['IntranetRX','ReceiveBandwidth'] 32 | 33 | - c_type: AWS 34 | resource: NATGateway 35 | name: 'NatGatewayId' 36 | to_falcon_template: 'aws-nat' 37 | ak: 'your access key' 38 | sk: 'your secret key' 39 | region: [{"name": 'ap-southeast-1', "site": 'sgpaws'}] 40 | metric_list: ['ActiveConnectionCount', 'PacketsDropCount', 'ErrorPortAllocation'] 41 | -------------------------------------------------------------------------------- /info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/open-falcon/cloud-mon/1da4073a60a086d8c8552517f8d41238cc11dcdd/info.png -------------------------------------------------------------------------------- /log.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python2 2 | # -*- coding: utf-8 -*- 3 | # by Laura 2018/05/31 4 | 5 | import yaml 6 | import logging.config 7 | import os 8 | 9 | 10 | def setup_logging(default_path="logging.yml", 11 | default_level=logging.INFO, env_key="LOG_CFG"): 12 | path = default_path 13 | value = os.getenv(env_key, None) 14 | if value: 15 | path = value 16 | if os.path.exists(path): 17 | with open(path, "r") as f: 18 | config = yaml.load(f) 19 | try: 20 | logging.config.dictConfig(config) 21 | except BaseException: 22 | os.makedirs('log/') 23 | logging.config.dictConfig(config) 24 | else: 25 | logging.basicConfig(level=default_level) 26 | 27 | 28 | def func(): 29 | logging.info("start func") 30 | logging.info("exec func") 31 | logging.info("end func") 32 | -------------------------------------------------------------------------------- /logging.yml: -------------------------------------------------------------------------------- 1 | version: 1 2 | disable_existing_loggers: False 3 | formatters: 4 | simple: 5 | format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s" 6 | handlers: 7 | console: 8 | class: logging.StreamHandler 9 | level: INFO 10 | formatter: simple 11 | stream: ext://sys.stdout 12 | info_file_handler: 13 | class: logging.handlers.RotatingFileHandler 14 | level: INFO 15 | formatter: simple 16 | filename: log/info.log 17 | maxBytes: 10485760 18 | backupCount: 20 19 | encoding: utf8 20 | error_file_handler: 21 | class: logging.handlers.RotatingFileHandler 22 | level: ERROR 23 | formatter: simple 24 | filename: log/errors.log 25 | maxBytes: 10485760 26 | backupCount: 20 27 | encoding: utf8 28 | loggers: 29 | my_module: 30 | level: INFO 31 | handlers: [console,info_file_handler,error_file_handler] 32 | propagate: no 33 | ## change for log level 34 | root: 35 | level: INFO 36 | handlers: [console,info_file_handler,error_file_handler] -------------------------------------------------------------------------------- /multiCloud.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | 3 | 4 | def get_id_list(c_type, resource, ak, sk, region): 5 | cloud_resource = importlib.import_module(c_type) 6 | return cloud_resource.get_id(resource, ak, sk, region) 7 | 8 | 9 | def get_metric_data(period, namespace, name, instance_id, 10 | metricname, c_type, ak, sk, region): 11 | cloud_resource = importlib.import_module(c_type) 12 | return cloud_resource.get_metric_data( 13 | period, namespace, name, instance_id, metricname, ak, sk, region) 14 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pyyaml 2 | flask 3 | requests 4 | xmljson 5 | thrift 6 | boto3 7 | pytz 8 | aliyun-python-sdk-core 9 | aliyun-python-sdk-slb 10 | httplib2 11 | aliyun-python-sdk-cms 12 | aliyun-python-sdk-vpc 13 | oss2 14 | jinja2 -------------------------------------------------------------------------------- /templates/ali-connect: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "connect-{{ metric.region}}-{{ metric.id }}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, id={{ metric.id }}", 7 | "step": 60, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/ali-connect-max: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "connect-{{ metric.region}}-{{ metric.id }}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, id={{ metric.id }}", 7 | "step": 600, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/ali-eip: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "eip-{{ metric.region}}-{{ metric.ip }}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, vip={{ metric.ip }}, id={{ metric.id }}", 7 | "step": 60, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/ali-eip-max: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "eip-{{ metric.region}}-{{ metric.ip }}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, vip={{ metric.ip }}, id={{ metric.id }}", 7 | "step": 600, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/ali-nat: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "NAT-{{ metric.region}}-{{ metric.id }}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, bandwidth={{ metric.ip }}, nat_id={{ metric.id }}", 7 | "step": 60, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/ali-oss: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "s3-{{ metric.region}}-{{ metric.id }}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, id={{ metric.id }}", 7 | "step": 60, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/aws-connect: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "connect-{{ metric.region}}-{{ metric.id }}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, id={{ metric.id }}, bandwith={{ metric.ip}}", 7 | "step": 60, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/aws-lb: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "elb-{{ metric.region}}-{{ metric.ip }}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, vip={{ metric.ip }}, id={{ metric.id }}", 7 | "step": 60, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/aws-nat: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "NAT-{{ metric.region}}-{{ metric.id }}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, id={{ metric.id }}", 7 | "step": 60, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/aws-s3: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "s3-{{ metric.region}}-{{ metric.id }}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, budketName={{ metric.id }}, size={{ metric.ip}}", 7 | "step": 60, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/ksc-connect: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "connect-{{ metric.region}}-{{ metric.id }}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, id={{ metric.id }}, bandwith={{ metric.ip}}", 7 | "step": 60, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/ksc-connect-max: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "connect-{{ metric.region}}-{{ metric.id }}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, id={{ metric.id }}, bandwith={{ metric.ip}}", 7 | "step": 600, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/ksc-eip: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "eip-{{ metric.region}}-{{ metric.ip }}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, vip={{ metric.ip }}, id={{ metric.id }}", 7 | "step": 60, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/ksc-eip-max: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "eip-{{ metric.region}}-{{ metric.ip }}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, vip={{ metric.ip }}, id={{ metric.id }}", 7 | "step": 600, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/ksc-elb: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "elb-{{ metric.region}}-{{ metric.ip }}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, vip={{ metric.ip }}, id={{ metric.id }}", 7 | "step": 60, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/ksc-elb-max: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "elb-{{ metric.region}}-{{ metric.ip }}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, vip={{ metric.ip }}, id={{ metric.id }}", 7 | "step": 600, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/ksc-nat: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "NAT-{{ metric.region}}-{{ metric.ip}}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, Nat_ip={{ metric.ip }}, nat_id={{ metric.id }}", 7 | "step": 60, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] -------------------------------------------------------------------------------- /templates/ksc-nat-max: -------------------------------------------------------------------------------- 1 | [{% for metric in metrics %} 2 | { 3 | "endpoint": "NAT-{{ metric.region}}-{{ metric.ip}}", 4 | "metric": "{{ metric.metric }}", 5 | "counterType": "GAUGE", 6 | "tags": "metric={{ metric.metric }}, Nat_ip={{ metric.ip }}, nat_id={{ metric.id }}", 7 | "step": 600, 8 | "timestamp": {{ metric.time }}, 9 | "value": {{ metric.value }} 10 | }{% if not loop.last%},{% endif %}{% endfor %} 11 | ] --------------------------------------------------------------------------------