├── .github
└── ISSUE_TEMPLATE
│ ├── 1bug.yaml
│ ├── 2feature.yaml
│ ├── 3consultation.yaml
│ └── config.yml
├── .gitignore
├── LICENSE
├── README.md
├── README_en.md
├── auto_discover
├── README.md
└── plugin_mysql.py
├── docs
└── images
│ ├── wechat.jpg
│ └── wechat.png
├── messenger
└── README.md
├── netTopology
├── .gitignore
├── README.md
├── discover
│ ├── collector.go
│ ├── entry.go
│ ├── layer2.go
│ ├── layer3.go
│ ├── master.go
│ ├── models.go
│ └── var.go
├── go.mod
├── go.sum
├── img.png
├── internal
│ ├── common.go
│ ├── node_type.go
│ ├── oids.go
│ ├── types.go
│ └── utils.go
├── lldp.md
└── main.go
├── secret
├── Pipfile
├── README.md
├── README_en.md
├── inner.py
└── vault.py
└── zabbix2tsdb
├── go.mod
├── go.sum
├── readme.md
└── zabbix.go
/.github/ISSUE_TEMPLATE/1bug.yaml:
--------------------------------------------------------------------------------
1 | name: Bug Report
2 | description: File a bug report
3 | title: "[Bug]: "
4 | labels: ["☢️ bug"]
5 | assignees:
6 | - Selina316
7 | body:
8 | - type: markdown
9 | attributes:
10 | value: |
11 | Thanks for taking the time to fill out this bug report!
12 | - type: input
13 | id: contact
14 | attributes:
15 | label: Contact Details
16 | description: How can we get in touch with you if we need more info?
17 | placeholder: ex. email@example.com
18 | validations:
19 | required: false
20 | - type: dropdown
21 | id: aspects
22 | attributes:
23 | label: This bug is related to UI or API?
24 | multiple: true
25 | options:
26 | - UI
27 | - API
28 | - type: textarea
29 | id: happened
30 | attributes:
31 | label: What happened?
32 | description: Also tell us, what did you expect to happen?
33 | placeholder: Tell us what you see!
34 | value: "A bug happened!"
35 | validations:
36 | required: true
37 | - type: input
38 | id: version
39 | attributes:
40 | label: Version
41 | description: What version of our software are you running?
42 | value: "newest"
43 | validations:
44 | required: true
45 | - type: dropdown
46 | id: browsers
47 | attributes:
48 | label: What browsers are you seeing the problem on?
49 | multiple: true
50 | options:
51 | - Firefox
52 | - Chrome
53 | - Safari
54 | - Microsoft Edge
55 | - type: textarea
56 | id: logs
57 | attributes:
58 | label: Relevant log output
59 | description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
60 | render: shell
61 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/2feature.yaml:
--------------------------------------------------------------------------------
1 | name: Feature wanted
2 | description: A new feature would be good
3 | title: "[Feature]: "
4 | labels: ["✏️ feature"]
5 | assignees:
6 | - pycook
7 | body:
8 | - type: markdown
9 | attributes:
10 | value: |
11 | Thank you for your feature suggestion; we will evaluate it carefully!
12 | - type: input
13 | id: contact
14 | attributes:
15 | label: Contact Details
16 | description: How can we get in touch with you if we need more info?
17 | placeholder: ex. email@example.com
18 | validations:
19 | required: false
20 | - type: dropdown
21 | id: aspects
22 | attributes:
23 | label: feature is related to UI or API aspects?
24 | multiple: true
25 | options:
26 | - UI
27 | - API
28 | - type: textarea
29 | id: feature
30 | attributes:
31 | label: What is your advice?
32 | description: Also tell us, what did you expect to happen?
33 | placeholder: Tell us what you want!
34 | value: "everyone wants this feature!"
35 | validations:
36 | required: true
37 | - type: input
38 | id: version
39 | attributes:
40 | label: Version
41 | description: What version of our software are you running?
42 | value: "newest"
43 | validations:
44 | required: true
45 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/3consultation.yaml:
--------------------------------------------------------------------------------
1 | name: Help wanted
2 | description: I have a question
3 | title: "[help wanted]: "
4 | labels: ["help wanted"]
5 | assignees:
6 | - ivonGwy
7 | body:
8 | - type: markdown
9 | attributes:
10 | value: |
11 | Please tell us what's you need!
12 | - type: input
13 | id: contact
14 | attributes:
15 | label: Contact Details
16 | description: How can we get in touch with you if we need more info?
17 | placeholder: ex. email@example.com
18 | validations:
19 | required: false
20 | - type: textarea
21 | id: question
22 | attributes:
23 | label: What is your question?
24 | description: Also tell us, how can we help?
25 | placeholder: Tell us what you need!
26 | value: "i have a question!"
27 | validations:
28 | required: true
29 | - type: input
30 | id: version
31 | attributes:
32 | label: Version
33 | description: What version of our software are you running?
34 | value: "newest"
35 | validations:
36 | required: true
37 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/config.yml:
--------------------------------------------------------------------------------
1 | blank_issues_enabled: false
2 | contact_links:
3 | - name: veops official website
4 | url: https://veops.cn/#hero
5 | about: you can contact us here.
6 |
7 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # If you prefer the allow list template instead of the deny list, see community template:
2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
3 | #
4 | # Binaries for programs and plugins
5 | *.exe
6 | *.exe~
7 | *.dll
8 | *.so
9 | *.dylib
10 |
11 | # Test binary, built with `go test -c`
12 | *.test
13 |
14 | # Output of the go coverage tool, specifically when used with LiteIDE
15 | *.out
16 |
17 | # Dependency directories (remove the comment below to include it)
18 | # vendor/
19 |
20 | # Go workspace file
21 | go.work
22 | .idea
23 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [yyyy] [name of copyright owner]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | [English](README_en.md)
2 | # ops-tools
3 |
4 | 运维过程中一些常用的实践和代码，将一些运维中的小工具不断加入进来，避免重复造轮子
5 |
6 | ### 独立组件
7 |
8 | #### golang
9 | [zabbix2tsdb](zabbix2tsdb/readme.md) 将zabbix的监控数据转换为prometheus格式输出
10 |
11 | [netTopology](netTopology) 基于SNMP协议自动发现局域网网络拓扑关系,输出拓扑数据结构
12 |
13 | [messenger](https://github.com/veops/messenger) 简单易用的消息发送服务
14 |
15 |
16 | #### python
17 | [secret](secret/README.md) 封装好的敏感数据存储工具, 自实现或者对接vault
18 |
19 | [auto_discover](auto_discover/README.md) 实例自动发现，主要用于cmdb自发现插件中
20 |
21 |
22 | ### TODO
23 | [netTopology]() 基于现有输出的结构化数据绘制网络拓扑图
24 |
25 |
26 | ---
27 | _**欢迎关注公众号(维易科技OneOps)，关注后可加入微信群，进行产品和技术交流。**_
28 |
29 |
30 | 
31 |
32 |
33 |
34 |
35 |
--------------------------------------------------------------------------------
/README_en.md:
--------------------------------------------------------------------------------
1 | [简体中文](README.md)
2 | # ops-tools
3 |
4 | Some common practices and code used in the operation and maintenance process.
5 | To avoid reinventing the wheel, strive to continuously incorporate some small
6 | tools used in operation and maintenance
7 |
8 | ### Individual components
9 |
10 | [zabbix2tsdb](zabbix2tsdb/readme.md) Convert monitoring data in Zabbix to Prometheus format for output.
11 |
12 | [netTopology](netTopology) Automatically discover the local area network (LAN) topology
13 | based on the SNMP protocol and output the topology data structure
14 |
15 | [messenger](messenger/README.md) simple and useful message sending server
16 |
17 | [secret](secret/README.md) Packaged sensitive data storage tool， inner method or Integrating with Vault
18 |
19 | [auto_discover](auto_discover/README.md) Instance auto-discovery, primarily used in CMDB auto-discovery plugins.
20 |
21 |
22 | ### TODO
23 | [netTopology]() Draw a network topology diagram based on the existing structured data output.
24 |
25 | ---
26 |
27 | _**Stay connected with us through our official WeChat account. Click to contact us and join our WeChat/QQ operations and maintenance groups to get more product and industry-related information.**_
28 |
29 |
30 |
31 |
--------------------------------------------------------------------------------
/auto_discover/README.md:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/veops/ops-tools/c8912d0e51de78cfed487f57553ccf3f6b4d394e/auto_discover/README.md
--------------------------------------------------------------------------------
/auto_discover/plugin_mysql.py:
--------------------------------------------------------------------------------
1 | # -*- coding:utf-8 -*-
2 |
3 | from __future__ import unicode_literals
4 | import importlib
5 | import json
6 | import os
7 | import socket
8 | import subprocess
9 | import sys
10 | import threading
11 | import time
12 |
13 | try:
14 | import configparser
15 | import ipaddress
16 | import psutil
17 | import tempfile
18 | except:
19 | pass
20 |
21 | global_scan_ip = "10.0.2.15" # Identify on which device the subnet scanning is performed.
22 | cidrs = ["192.168.20.8/30"] # Subnet to be scanned.
23 | global_ports_range = "3306-3310,3320" # ports to be scanned. such as "3306-3310,3320"
24 |
25 | paths = ["/etc/my.cnf", "/etc/mysql/my.cnf"]
26 | threading_number = 10
27 | pip_modules = ["configparser", "ipaddress", "psutil", "tempfile"]
28 | pip_index_url = 'https://pypi.douban.com/simple'
29 |
30 |
31 | class Module:
32 |
33 | def __init__(self, modules=None, index_url=None):
34 | self.modules = modules
35 | self.index_url = pip_index_url
36 | if not modules:
37 | self.modules = pip_modules
38 | if not index_url:
39 | self.index_url = pip_index_url
40 |
41 | def check_install_modules(self):
42 | for v in self.modules:
43 | self.install_missing_module(v)
44 |
45 | def install_missing_module(self, module_name):
46 | try:
47 | importlib.import_module(module_name)
48 | except ImportError:
49 | try:
50 | subprocess.check_output(["pip", "--version"])
51 | except subprocess.CalledProcessError:
52 | print("pips has not been installed, try to install...")
53 | subprocess.check_call([sys.executable, "-m", "ensurepip"])
54 |
55 | try:
56 | subprocess.check_call([sys.executable, "-m", "pip", "install", '--index-url',
57 | self.index_url, module_name])
58 | print("module '{}' has been installed successfully.".format(module_name))
59 | except Exception as e:
60 | print("Failed to install module '{}': {}".format(module_name, str(e)))
61 |
62 |
63 | class Mysql:
64 |
65 | def __init__(self):
66 | pass
67 |
68 | @classmethod
69 | def parse_version(cls, sock):
70 | sock.sendall(b"\x00\x00\x00\x0a\x03\x00\x00\x00\x1b")
71 | data = sock.recv(1024)
72 | for encoding in ["latin-1", "utf-8", "ascii"]:
73 | try:
74 | d = data[5:].decode(encoding)
75 | if "MYSQL" in d.upper():
76 | return data[5:].decode(encoding).split("\x00")[0]
77 | return ""
78 | except socket.error:
79 | continue
80 | return ""
81 |
82 |
83 | class Scan:
84 |
85 | def __init__(self, cidrs, port_range, paths=None, g_scan_ip=None):
86 | self.cidrs = cidrs
87 | self.ports = Utils.parse_port_range(port_range)
88 | self.local_ip = Utils.get_local_ip()
89 | if not paths:
90 | self.config_paths = ['/etc/mysql/my.cnf']
91 | else:
92 | self.config_paths = paths
93 | self.global_scan_ip = g_scan_ip
94 |
95 | def get_from_ip_range(self):
96 | instances = []
97 | if self.global_scan_ip != self.local_ip:
98 | return instances
99 |
100 | semaphore = threading.Semaphore(threading_number)
101 |
102 | def worker(scan_ip, s_port):
103 | r1 = self.scan_port(scan_ip, s_port)
104 | if r1:
105 | instances.append(r1)
106 | semaphore.release()
107 |
108 | threads = []
109 | for cidr in self.cidrs:
110 | ip_network = ipaddress.ip_network(cidr)
111 |
112 | for ip in ip_network.hosts():
113 | ip = str(ip)
114 |
115 | for port in self.ports:
116 | semaphore.acquire()
117 |
118 | t = threading.Thread(target=worker, args=(ip, port))
119 | t.daemon = True
120 | threads.append(t)
121 | t.start()
122 |
123 | for i in threads:
124 | i.join()
125 |
126 | return instances
127 |
128 | @classmethod
129 | def scan_port(cls, ip, port):
130 | try:
131 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
132 | sock.settimeout(5)
133 |
134 | if sock.connect_ex((ip, port)) == 0:
135 | version = Mysql.parse_version(sock)
136 | if version:
137 | return ip, port, version
138 |
139 | sock.close()
140 | except socket.error:
141 | return None
142 | return None
143 |
144 | @classmethod
145 | def local_listening_ports(cls):
146 | listening_ports = []
147 | for conn in psutil.net_connections():
148 | if conn.status == 'LISTEN':
149 | listening_ports.append(conn.laddr.port)
150 | return listening_ports
151 |
152 | def get_local_listening_ports(self):
153 | listening_ports = self.local_listening_ports()
154 | instances = []
155 |
156 | def worker(scan_ip, scan_port):
157 | r1 = self.scan_port(scan_ip, scan_port)
158 | if r1:
159 | instances.append(r1)
160 | semaphore.release()
161 |
162 | semaphore = threading.Semaphore(threading_number)
163 | threads = []
164 | for port in listening_ports:
165 | semaphore.acquire()
166 |
167 | t = threading.Thread(target=worker, args=(self.local_ip, port))
168 | t.daemon = True
169 | threads.append(t)
170 | t.start()
171 |
172 | for i in threads:
173 | i.join()
174 |
175 | return instances
176 |
177 | # this function will deprecate in the future
178 | def get_from_config(self):
179 | instances = []
180 | if self.local_ip == "":
181 | return instances
182 | for path in self.config_paths:
183 | config = configparser.ConfigParser()
184 | config.read(path)
185 |
186 | if 'mysqld' in config:
187 | port = config['mysqld'].get("port", "3306")
188 | pid_file = config['mysqld'].get("pid-file")
189 |
190 | pid = Utils.get_pid(pid_file)
191 | if pid:
192 | ok, name = Utils.is_pid_running(pid)
193 | if ok and name == "mysqld":
194 | instances.append((self.local_ip, port, ""))
195 |
196 | return instances
197 |
198 | def gets(self):
199 | instances = self.get_from_ip_range()
200 |
201 | local_instance = self.get_local_listening_ports()
202 | if local_instance:
203 | instances.extend(local_instance)
204 |
205 | config_instances = self.get_from_config()
206 | if config_instances:
207 | instances.extend(config_instances)
208 | return Utils.deduplicate(instances)
209 |
210 | class Cache:
211 |
212 | def __init__(self, temp_file=None, duration=None):
213 | if not temp_file:
214 | self.temp_file = os.path.join(tempfile.gettempdir(), "auto_discover_mysql_result.json")
215 | else:
216 | self.temp_file = temp_file
217 | if duration and isinstance(duration, int) and duration > 3600:
218 | self.duration = duration
219 | else:
220 | self.duration = 3600
221 |
222 | @classmethod
223 | def convert_data(cls, data):
224 | data = {
225 | "create_at": int(time.time()),
226 | "results": data,
227 | }
228 | return data
229 |
230 | def out_date(self, data):
231 | if isinstance(data, dict) and time.time() - data.get("create_at", 0) > self.duration:
232 | return True
233 | return False
234 |
235 | def read(self):
236 | try:
237 | with open(self.temp_file, 'r') as temp_file:
238 | json_data = json.load(temp_file)
239 | except:
240 | json_data = {}
241 | return json_data
242 |
243 | def write(self, data):
244 | with open(self.temp_file, "w") as temp_file:
245 | json.dump(data, temp_file)
246 |
247 |
248 | class Utils:
249 |
250 | def __init__(self):
251 | pass
252 |
253 | @classmethod
254 | def parse_port_range(cls, port_range):
255 | ports = []
256 | ranges = port_range.split(",")
257 | for r in ranges:
258 | if "-" in r:
259 | start, end = r.split("-")
260 | try:
261 | start = int(start)
262 | end = int(end)
263 | ports.extend(range(start, end + 1))
264 | except ValueError:
265 | print("invalid port:", r)
266 | else:
267 | try:
268 | port = int(r)
269 | ports.append(port)
270 | except ValueError:
271 | print("invalid port:", r)
272 | return ports
273 |
274 | @classmethod
275 | def get_local_ip(cls):
276 | interfaces = psutil.net_if_addrs()
277 | for interface in interfaces:
278 | addresses = interfaces[interface]
279 | for address in addresses:
280 | if address.family == socket.AF_INET and \
281 | not address.address.startswith('127.') and \
282 | not address.address.startswith('172.'):
283 | return address.address
284 | return ""
285 |
286 | @classmethod
287 | def is_pid_running(cls, pid):
288 | if not pid:
289 | return False, ""
290 | try:
291 | process = psutil.Process(pid)
292 | return process.is_running(), process.name()
293 | except psutil.NoSuchProcess:
294 | return False, ""
295 |
296 | @classmethod
297 | def get_pid(cls, pid_file_path):
298 | try:
299 | with open(pid_file_path, "r") as file:
300 | pid = file.read().strip()
301 | return int(pid)
302 | except FileNotFoundError:
303 | return None
304 | except IOError as e:
305 | return None
306 |
307 | @classmethod
308 | def deduplicate(cls, data):
309 | unique_data = {}
310 | for item in data:
311 | key = (item[0], item[1])
312 | if item[2] != "":
313 | unique_data[key] = item
314 | return list(unique_data.values())
315 |
316 | @classmethod
317 | def convert_input(cls):
318 | if len(sys.argv) != 5:
319 | return
320 |
321 | c_cidrs = list(set(sys.argv[1].split(",")))
322 |
323 | c_ports = [3306]
324 | for item in sys.argv[2].split(","):
325 | try:
326 | c_ports.append(int(item))
327 | except ValueError:
328 | continue
329 | c_ports = list(set(c_ports))
330 |
331 | c_paths = list(set(sys.argv[3].split(",")))
332 |
333 | return c_cidrs, c_ports, c_paths, sys.argv[4]
334 |
335 |
336 | class AutoDiscovery(object):
337 |
338 | @property
339 | def unique_key(self):
340 | return "mysql_name"
341 |
342 | @staticmethod
343 | def attributes():
344 | """
345 | :return: 返回属性字段列表, 列表项是(名称, 类型, 描述), 名称必须是英文
346 | 类型: String Integer Float Date DateTime Time JSON
347 | """
348 | return [
349 | ("mysql_name", "String", "实例名称"),
350 | ("ip", "String", "ip"),
351 | ("port", "Integer", "端口"),
352 | ("version", "String", "版本信息")
353 | ]
354 |
355 | @staticmethod
356 | def run():
357 | """
358 | 执行入口, 返回采集的属性值
359 | :return: 返回一个列表, 列表项是字典, 字典key是属性名称, value是属性值
360 | 例如:
361 | return [dict(ci_type="server", private_ip="192.168.1.1")]
362 | """
363 | Module().check_install_modules()
364 | data = Cache().read()
365 | if not data or Cache().out_date(data):
366 | all_instances = Scan(cidrs, global_ports_range, paths, g_scan_ip=global_scan_ip).gets()
367 | results = []
368 | for instance in all_instances:
369 | ip, port, version = instance
370 | results.append(dict(mysql_name="{0}-{1}".format(ip, port), ip=ip, port=port, version=version))
371 | data = Cache().convert_data(results)
372 | Cache().write(data)
373 | return data.get("results", [])
374 |
375 |
376 | if __name__ == "__main__":
377 | result = AutoDiscovery().run()
378 | if isinstance(result, list):
379 | print("AutoDiscovery::Result::{}".format(json.dumps(result)))
380 | else:
381 | print("ERROR: 采集返回必须是列表")
382 |
--------------------------------------------------------------------------------
/docs/images/wechat.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/veops/ops-tools/c8912d0e51de78cfed487f57553ccf3f6b4d394e/docs/images/wechat.jpg
--------------------------------------------------------------------------------
/docs/images/wechat.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/veops/ops-tools/c8912d0e51de78cfed487f57553ccf3f6b4d394e/docs/images/wechat.png
--------------------------------------------------------------------------------
/messenger/README.md:
--------------------------------------------------------------------------------
1 | # Messenger
2 |
3 | Messenger已迁移至新的仓库👇
4 |
5 | **[https://github.com/veops/messenger](https://github.com/veops/messenger)**
--------------------------------------------------------------------------------
/netTopology/.gitignore:
--------------------------------------------------------------------------------
1 | vendor
2 | vendor/*
3 |
--------------------------------------------------------------------------------
/netTopology/README.md:
--------------------------------------------------------------------------------
1 | ### 描述
2 | > 基于snmp自动发现网络拓扑结构，并生成网络拓扑数据结构，输出结果主要包含nodes，links,即节点、连线，并定时进行每个端口的流量、错误数、广播包
3 | > 丢包信息采集
4 |
5 | ### 运行
6 | ```shell
7 | go build .
8 | ./netTopology
9 | ```
10 |
11 | ### 输出结果
12 | ```json
13 | {
14 | "nodes":[
15 | {
16 | "name":"10.172.135.254",
17 | "alias":"",
18 | "type":"router",
19 | "x":0,
20 | "y":0,
21 | "ips":[
22 | "10.172.135.254"
23 | ],
24 | "locals":[
25 | "10.172.135.0/255.255.255.0",
26 | "10.172.135.255/255.255.255.255"
27 | ],
28 | "des_ips":{
29 | "GigabitEthernet0/0/0":[
30 | "10.172.135.1"
31 | ],
32 | "GigabitEthernet0/0/6":[
33 | "58.33.167.73"
34 | ]
35 | }
36 | }
37 | ],
38 | "links":[
39 | {
40 | "node_from":"GigabitEthernet0/0/1",
41 | "node_to":"",
42 | "node_from_name":"192.168.1.2",
43 | "node_to_name":"192.168.13",
44 | "value":0,
45 | "unit":"",
46 | "level":0,
47 | "if_index":"",
48 | "traffic_load":0,
49 | "port":"",
50 | "options":null
51 | }
52 | ],
53 | "location":false,
54 | "graph_id":"id1"
55 | }
56 | ```
57 |
58 | ### 要做
59 | > 后面将把前端展示的代码，以及结合手动编辑架构结合的功能加入进来，下图是一张基于本代码生成的数据进行后续前端展示的效果图
60 |
61 | 
--------------------------------------------------------------------------------
/netTopology/discover/collector.go:
--------------------------------------------------------------------------------
1 | // Package discover /*
2 | package discover
3 |
4 | import (
5 | "errors"
6 | "fmt"
7 | "math/big"
8 | "strings"
9 | "time"
10 |
11 | gsnmp "github.com/gosnmp/gosnmp"
12 | "github.com/patrickmn/go-cache"
13 | . "netTopology/internal"
14 | )
15 |
16 | type PortDetail struct {
17 | Port string
18 | Index string
19 | Name string
20 | }
21 |
22 | type IfCollectorConfig struct {
23 | Oid string
24 | Alias string
25 | Order int
26 | Group string `json:"-"`
27 | }
28 |
29 | // PortsDetail map[string]map[string]PortDetail
30 | // map[ip]map[port]PortDetail
31 | var (
32 | PortsDetail = cache.New(time.Minute, time.Minute)
33 | IfCollectors = map[string]IfCollectorConfig{
34 | "net_if_in_octets": {OLibrary.IfHCInOctets, "输入带宽", 1, "带宽"},
35 | "net_if_out_octets": {OLibrary.IfHCOutOctets, "输出带宽", 2, "带宽"},
36 | //"net_if_in_ucast_pkts": {OLibrary.IfInUcastPkts, "输入非广播包数", 3, "非广播包数"},
37 | //"net_if_out_ucast_pkts": {OLibrary.IfOutUcastPkts, "输出非广播包数", 4, "非广播包数"},
38 | "net_if_in_nucast_pkts": {OLibrary.IfInNUcastPkts, "输入广播包数", 5, "广播包数"},
39 | "net_if_out_nucast_pkts": {OLibrary.IfOutNUcastPkts, "输出广播包数", 6, "广播包数"},
40 | "net_if_in_discards": {OLibrary.IfInDiscards, "输入包丢弃数", 7, "包丢弃数"},
41 | "net_if_out_discards": {OLibrary.IfOutDiscards, "输出包丢弃数", 8, "包丢弃数"},
42 | "net_if_in_errors": {OLibrary.IfInErrors, "输入包错误数", 9, "包错误数"},
43 | "net_if_out_errors": {OLibrary.IfOutErrors, "输出包错误数", 10, "包错误数"},
44 | "net_if_highspeed": {OLibrary.IfHighSpeed, "最大带宽", 11, "最大带宽"},
45 | "net_if_oper_status": {OLibrary.IfOperStatus, "接口配置状态", 12, "接口配置状态"},
46 | "net_if_admin_status": {OLibrary.IfOperStatus, "接口工作状态", 12, "接口工作状态"},
47 | //ifAdminStatus是指配置状态。
48 | //ifOperStatus是指实际工作状态
49 | }
50 | )
51 |
52 | func GetIfDetails(target string, snmpClient *gsnmp.GoSNMP) (data map[string]PortDetail, err error) {
53 | if v, ok := PortsDetail.Get(target); ok {
54 | data = v.(map[string]PortDetail)
55 | return
56 | }
57 | data = map[string]PortDetail{}
58 | if snmpClient == nil {
59 | snmpClient, err = NewSnmp(target)
60 | if err != nil {
61 | err = fmt.Errorf("new snmp client for %s failed %s", target, err.Error())
62 | return
63 | } else {
64 | defer snmpClient.Conn.Close()
65 | }
66 | }
67 |
68 | res1, er := FetchItems("walk", target, []string{OLibrary.IfDes}, snmpClient)
69 | if er != nil {
70 | err = errors.New(StringConcat("fetch item failed for ", target, OLibrary.IfDes))
71 | return
72 | }
73 | for _, v := range res1 {
74 | port := strings.TrimPrefix(v.Oid, fmt.Sprintf("%s.", OLibrary.IfDes))
75 | data[port] = PortDetail{Name: v.DataValue.(string), Port: port}
76 | }
77 | res2, er := FetchItems("walk", target, []string{OLibrary.IfIndex}, snmpClient)
78 | if er != nil {
79 | err = errors.New(StringConcat("fetch item failed for ", target, OLibrary.IfDes))
80 | return
81 | }
82 | for _, v := range res2 {
83 | port := strings.TrimPrefix(v.Oid, fmt.Sprintf("%s.", OLibrary.IfIndex))
84 | if v1, ok := data[port]; ok {
85 | v1.Index = fmt.Sprintf("%s", v.DataValue)
86 | data[port] = v1
87 | }
88 | }
89 | PortsDetail.SetDefault(target, data)
90 | return
91 | }
92 |
93 | func ifStatistics(target string) (data map[string]map[string]int64, err error) {
94 | snmpClient, er := NewSnmp(target)
95 | if er != nil {
96 | e := fmt.Errorf("new snmp client for %s failed %s", target, er.Error())
97 | return nil, e
98 | } else {
99 | defer snmpClient.Conn.Close()
100 | }
101 |
102 | data = map[string]map[string]int64{}
103 | port := ""
104 | for k, v := range IfCollectors {
105 | res, er := FetchItems("walk", target, []string{v.Oid}, snmpClient, 2)
106 | if er != nil {
107 | err = er
108 | return
109 | }
110 | data[k] = map[string]int64{}
111 | for _, v1 := range res {
112 | port = strings.TrimPrefix(v1.Oid, fmt.Sprintf("%s.", v.Oid))
113 | data[k][port] = v1.DataValue.(*big.Int).Int64()
114 | }
115 | }
116 | return data, nil
117 | }
118 |
--------------------------------------------------------------------------------
/netTopology/discover/entry.go:
--------------------------------------------------------------------------------
1 | package discover
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "os"
7 | "time"
8 |
9 | "github.com/pkg/errors"
10 | "github.com/sirupsen/logrus"
11 |
12 | . "netTopology/internal"
13 | )
14 |
15 | const (
16 | RedisKeyDiscoverLinks = "Net::Discover::Links"
17 | RedisKeyCompleteLinks = "Net::Complete::Links"
18 | )
19 |
20 | type CollectorConfig struct {
21 | Id string `json:"id"`
22 | Name string
23 | Gateway string `json:"gateway"`
24 | Nodes []*NodeSnmp
25 | }
26 |
27 | type Discover interface {
28 | Collectors(gateway string) ([]*CollectorConfig, error)
29 | Topology(config *CollectorConfig) (*TopologyGraph, error)
30 | UploadRecord(data *TopologyGraph) error
31 | UploadTsRecord(target string, data map[string]map[string]int64)
32 | Nodes() []string
33 | }
34 |
35 | func tidyGraph(graph *TopologyGraph, portMapIfIndex map[string]map[string]int64,
36 | portMapDes map[string]map[string]string) {
37 | r := map[string]StpInfo{}
38 | NodeLinks.Range(func(key, value any) bool {
39 | v := value.(StpInfo)
40 | r[key.(string)] = v
41 | return true
42 | })
43 | var links []*Link
44 | for _, v := range r {
45 | nodeFrom := v.Port
46 | if v1, ok := portMapDes[v.AgentIp]; ok {
47 | if v2, ok := v1[v.Port]; ok {
48 | nodeFrom = v2
49 | }
50 | } else if v1, ok := portMapIfIndex[v.AgentIp]; ok {
51 | if v2, ok := v1[v.Port]; ok {
52 | nodeFrom = fmt.Sprintf("%v", v2)
53 | }
54 | }
55 | gh := &Link{
56 | NodeFromName: v.AgentIp,
57 | NodeFrom: nodeFrom,
58 | NodeToName: v.DesignAddress,
59 | }
60 | if v1, ok := r[v.DesignAddress]; ok {
61 | if v2, _, er := IpIdentGet(v1.AgentIp, nil); er == nil {
62 | gh.NodeToName = v2
63 | } else {
64 | gh.NodeToName = v1.AgentIp
65 | }
66 | } else {
67 | logrus.Warn("unmatched mac address", v.DesignAddress)
68 | }
69 | links = append(links, gh)
70 | }
71 | graph.Links = links
72 | var ips []string
73 | for index, v := range graph.Nodes {
74 | if v1, ok := NodeInfo.Load(v.Name); ok {
75 | v2 := v1.(map[int64]*IfIndexInfo)
76 | d := map[string][]string{}
77 | for _, v3 := range v2 {
78 | ips = []string{}
79 | for _, v4 := range v3.ToIps {
80 | v4 = ToHardwareAddr(v4)
81 | if ip, ok := MacIps.Load(v4); ok {
82 | ips = append(ips, ip.([]string)...)
83 | } else {
84 | ips = append(ips, v4)
85 | }
86 | }
87 | d[v3.Des] = ips
88 | }
89 | graph.Nodes[index].DesIps = d
90 | }
91 | }
92 | }
93 |
94 | //
95 | //func upsertConfigGraph(data *models.ConfigGraphLink) {
96 | // data.LinkId = internal.MD5(strings.Join([]string{data.Name, data.DeviceInterface, data.To}, "-"))
97 | // data.Action = 3
98 | // count, er := data.Count(db.ConfigGraphLinkCol, bson.M{"link_id": data.LinkId, "graph_id": data.GraphId})
99 | // if er != nil {
100 | // _ = level.Error(g.Logger).Log("module", "net", "msg", er.Error())
101 | // return
102 | // }
103 | // if count == 0 {
104 | // _, er = data.Add(db.ConfigGraphLinkCol, data)
105 | // if er != nil {
106 | // _ = level.Error(g.Logger).Log("module", "net", "msg", er.Error())
107 | // }
108 | // return
109 | // }
110 | // return
111 | //}
112 | //
113 | //func updateLinks(links []*Link, graphId string) {
114 | // for _, v := range links {
115 | // t := &models.ConfigGraphLink{
116 | // LinkNode: &models.LinkNode{
117 | // Name: v.NodeFromName,
118 | // DeviceInterface: v.NodeFrom,
119 | // },
120 | // To: v.NodeToName,
121 | // GraphId: graphId,
122 | // }
123 | // upsertConfigGraph(t)
124 | // }
125 | //}
126 |
127 | func start(md Discover, ident string) error {
128 | logrus.Info("start discover network topology")
129 | collectors, er := md.Collectors(ident)
130 | if er != nil {
131 | return er
132 | }
133 |
134 | for _, v := range collectors {
135 | logrus.Info("start collect from ", v.Id)
136 | graph, er1 := md.Topology(v)
137 | if er1 != nil {
138 | er = errors.Wrap(er, er1.Error())
139 | continue
140 | }
141 | er1 = md.UploadRecord(graph)
142 | if er != nil {
143 | er = errors.Wrap(er, er1.Error())
144 | }
145 | }
146 | logrus.Info("end discover network topology")
147 | logrus.Info("the last auto discover topology graph is as follows")
148 | logrus.Info("---------------------------------------------------")
149 | TG.Range(func(key, value any) bool {
150 | v := value.(*TopologyGraph)
151 | s, _ := json.MarshalIndent(v, "", " ")
152 | logrus.Info(string(s))
153 | return true
154 | })
155 | logrus.Info("---------------------------------------------------")
156 | return nil
157 | }
158 |
159 | func NetworkMonitorInit(cancel <-chan os.Signal, md Discover) {
160 | go func() {
161 | //time.Sleep(time.Duration(rand.New(rand.NewSource(time.Now().Unix())).Intn(24)) * time.Hour)
162 | er := start(md, "")
163 | if er != nil {
164 | logrus.Error(er)
165 | }
166 | ticker := time.NewTicker(time.Hour * 24)
167 | for {
168 | select {
169 | case <-cancel:
170 | return
171 | case <-ticker.C:
172 | er := start(md, "")
173 | if er != nil {
174 | logrus.Error(er)
175 | }
176 | }
177 | }
178 | }()
179 | // 网络流量每分钟更新一次
180 | go func() {
181 | nodes := md.Nodes()
182 | for _, node := range nodes {
183 | go func(node string) {
184 | data, err := ifStatistics(node)
185 | if err != nil {
186 | logrus.Error(err)
187 | }
188 | md.UploadTsRecord(node, data)
189 | }(node)
190 | }
191 | //ticker := time.NewTicker(time.Second * 10)
192 | ticker := time.NewTicker(time.Minute)
193 | for {
194 | select {
195 | case <-cancel:
196 | return
197 | case <-ticker.C:
198 | nodes := md.Nodes()
199 | for _, node := range nodes {
200 | go func(node string) {
201 | data, err := ifStatistics(node)
202 | if err != nil {
203 | logrus.Error(err)
204 | }
205 | md.UploadTsRecord(node, data)
206 | }(node)
207 | }
208 | }
209 | }
210 | }()
211 | }
212 |
213 | //
214 | //func UniversalAction(action string, req string) (resp, nextAction string, err error) {
215 | // switch action {
216 | // case "Net.UploadRecord":
217 | // data := TopologyGraph{}
218 | // er := json.Unmarshal([]byte(req), &data)
219 | // if er != nil {
220 | // logrus.Error(er)
221 | // return
222 | // }
223 | // md := MasterDiscover("master")
224 | // er = md.UploadRecord(&data)
225 | // return "", "", md.UploadRecord(&data)
226 | // case "Net.Collectors":
227 | // data := map[string]string{}
228 | // md := MasterDiscover("master")
229 | // if ident, ok := data["ident"]; ok {
230 | // d, er := md.Collectors(ident)
231 | // if er == nil {
232 | // s, er := json.Marshal(d)
233 | // if er == nil {
234 | // return string(s), "", nil
235 | // }
236 | // }
237 | // if er != nil {
238 | // logrus.Error(er)
239 | // return
240 | // }
241 | // }
242 | // }
243 | // return "", "", nil
244 | //}
245 |
--------------------------------------------------------------------------------
/netTopology/discover/layer2.go:
--------------------------------------------------------------------------------
1 | package discover
2 |
3 | import (
4 | "fmt"
5 | "github.com/sirupsen/logrus"
6 | "math/big"
7 | "strconv"
8 | "strings"
9 | "sync"
10 |
11 | gsnmp "github.com/gosnmp/gosnmp"
12 |
13 | . "netTopology/internal"
14 | )
15 |
16 | type IfIndexInfo struct {
17 | IfIndex int64
18 | // ToIps对接的所有IP
19 | ToIps []string
20 | // Neighbor 对接的IP
21 | Neighbor string
22 | Port string
23 | // Des 接口别名
24 | Des string
25 | }
26 |
27 | type Subnet struct {
28 | Name string `json:"name"`
29 | // Router 网关路由器
30 | Router string `json:"router"`
31 | // Routers 子网内的交换机
32 | Routers map[string]string `json:"routers"`
33 | // Switches 子网内的交换机
34 | Switches map[string]string `json:"switches"`
35 | // Hosts 子网内的主机
36 | Hosts map[string]string `json:"hosts"`
37 | }
38 |
39 | var (
40 | IfIndexMapIpRecord = sync.Map{}
41 | NodeInfo = sync.Map{}
42 | NodeLinks = sync.Map{}
43 | commonLocker = sync.RWMutex{}
44 | //ifIndexPortMap = sync.Map{}
45 | )
46 |
47 | type Subnets struct {
48 | sync.RWMutex
49 | M map[string]*Subnet
50 | }
51 |
52 | func (s *Subnets) Get(name string) *Subnet {
53 | s.RLock()
54 | defer s.RUnlock()
55 | return s.M[name]
56 | }
57 |
58 | func (s *Subnets) Add(fieldType NodeType, name, value string) {
59 | s.Lock()
60 | defer s.Unlock()
61 | if _, ok := s.M[name]; !ok {
62 | s.M[name] = &Subnet{
63 | Name: name,
64 | Switches: map[string]string{},
65 | Hosts: map[string]string{},
66 | Routers: map[string]string{},
67 | }
68 | }
69 | switch fieldType {
70 | case NTRouter:
71 | s.M[name].Router = value
72 | s.M[name].Routers[value] = ""
73 | case NTSwitch:
74 | s.M[name].Switches[value] = ""
75 | case NTServer:
76 | s.M[name].Hosts[value] = ""
77 | }
78 | }
79 |
80 | func (s *Subnets) Clear() {
81 | s.Lock()
82 | defer s.Unlock()
83 | s.M = map[string]*Subnet{}
84 | }
85 |
86 | func (s *Subnets) Range(f func(k, v any) bool) {
87 | s.RLock()
88 | defer s.RUnlock()
89 | for k, v := range s.M {
90 | if !f(k, v) {
91 | break
92 | }
93 | }
94 | }
95 |
96 | func ScanTotalSubnetsAndNeighbor(graph *TopologyGraph) (map[string]map[string]int64, map[string]map[string]string) {
97 | SubnetsVar.Clear()
98 | IfIndexMapIpRecord = sync.Map{}
99 | // step1 扫描出各个子网内的交换机、路由器，以及整个arp表
100 | {
101 | wg := sync.WaitGroup{}
102 | ch := make(chan struct{}, 1000)
103 | Routes.Range(func(key, value any) bool {
104 | for _, objs := range [][]string{value.(*Arp).Subnets, value.(*Arp).PeerIps} {
105 | for _, v := range objs {
106 | if _, ok := SubnetsScanned.LoadOrStore(v, struct{}{}); ok {
107 | continue
108 | }
109 | ips, er := CidrHosts(v)
110 | if er != nil {
111 | logrus.Warn(er)
112 | continue
113 | }
114 | for _, v1 := range ips {
115 | if !ValidInnerIp(v1) {
116 | logrus.Info("invalid intranet ips", v1)
117 | break
118 | }
119 | wg.Add(1)
120 | ch <- struct{}{}
121 | go func(ip, cidr string, wg *sync.WaitGroup) {
122 | defer func() {
123 | wg.Done()
124 | <-ch
125 | }()
126 | nt := GetNodeType(ip)
127 | SubnetsVar.Add(nt, cidr, ip)
128 | switch nt {
129 | case NTSwitch, NTRouter:
130 | _, _ = ScanArp(ip, nil)
131 | Topology3SyncWait.Add(1)
132 | L3Topology(ip, graph, false)
133 | Topology3SyncWait.Wait()
134 | }
135 | }(v1, v, &wg)
136 | }
137 | }
138 | }
139 | return true
140 | })
141 | wg.Wait()
142 | }
143 | logrus.Debug("end scan arp......")
144 |
145 | // step2 针对子网的route,switch 获取接口映射
146 | logrus.Info("start scan links")
147 | portMapIfIndex := map[string]map[string]int64{}
148 | portMapDes := map[string]map[string]string{}
149 | var f1 = func(ip string) {
150 | ident, r, er := IfIndexMapIp(ip, nil)
151 | if er != nil || len(r) == 0 {
152 | return
153 | }
154 | portMapIfIndex[ident] = map[string]int64{}
155 | portMapDes[ident] = map[string]string{}
156 | for ifIndex, v1 := range r {
157 | portMapIfIndex[ident][v1.Port] = ifIndex
158 | portMapDes[ident][v1.Port] = v1.Des
159 | }
160 | NodeInfo.Store(ident, r)
161 | NodeLink(ident, nil)
162 | }
163 | {
164 | SubnetsVar.Range(func(k, v any) bool {
165 | for ip := range v.(*Subnet).Routers {
166 | f1(ip)
167 | }
168 | for ip := range v.(*Subnet).Switches {
169 | f1(ip)
170 | }
171 | return true
172 | })
173 | }
174 | r := map[string]StpInfo{}
175 | NodeLinks.Range(func(key, value any) bool {
176 | v := value.(StpInfo)
177 | r[key.(string)] = v
178 | return true
179 | })
180 | var links []*Link
181 | for _, v := range r {
182 | nodeFrom := v.Port
183 | if v1, ok := portMapDes[v.AgentIp]; ok {
184 | if v2, ok := v1[v.Port]; ok {
185 | nodeFrom = v2
186 | }
187 | } else if v1, ok := portMapIfIndex[v.AgentIp]; ok {
188 | if v2, ok := v1[v.Port]; ok {
189 | nodeFrom = fmt.Sprintf("%v", v2)
190 | }
191 | }
192 | gh := &Link{
193 | NodeFromName: v.AgentIp,
194 | NodeFrom: nodeFrom,
195 | NodeToName: v.DesignAddress,
196 | }
197 | if v1, ok := r[v.DesignAddress]; ok {
198 | if v2, _, er := IpIdentGet(v1.AgentIp, nil); er == nil {
199 | gh.NodeToName = v2
200 | } else {
201 | gh.NodeToName = v1.AgentIp
202 | }
203 | } else {
204 | logrus.Warn("unmatched mac address", v.DesignAddress)
205 | }
206 | links = append(links, gh)
207 | }
208 | graph.Links = links
209 | //// mac to ips
210 | //for i, v := range graph.Nodes {
211 | // for i1, v1 := range v.DesIps {
212 | // ips := []string{}
213 | // for _, v2 := range v1 {
214 | // if v3, ok := MacIps.Load(v2); ok {
215 | // ips = append(ips, v3.([]string)...)
216 | // }
217 | // }
218 | // ips = utils.RmDupElement(ips)
219 | // sort.Strings(ips)
220 | // graph.Nodes[i].DesIps[i1] = ips
221 | // fmt.Println("ips:", ips)
222 | // }
223 | //}
224 | return portMapIfIndex, portMapDes
225 | }
226 |
227 | func ifIndexMapIpSwitch(node string, data map[int64]*IfIndexInfo, snmp *gsnmp.GoSNMP) {
228 | macIfIndex := map[string]int64{}
229 | portIfIndex := map[string]int64{}
230 | ifIndexPort := map[int64]string{}
231 | ifIndexDes := map[int64]string{}
232 | // 物理端口对应接口
233 | r0, er0 := FetchItems("walk", node, []string{OLibrary.Dot1dBasePortIfIndex}, snmp)
234 |
235 | if er0 == nil {
236 | // mac对应物理端口
237 | r1, er1 := FetchItems("walk", node, []string{OLibrary.Dot1dTpFdbPort}, snmp)
238 | if er1 == nil {
239 |
240 | for _, v := range r0 {
241 | tmp := strings.Split(v.Oid, ".")
242 | p := tmp[len(tmp)-1]
243 | portIfIndex[p] = v.DataValue.(*big.Int).Int64()
244 | ifIndexPort[v.DataValue.(*big.Int).Int64()] = p
245 | }
246 | for _, v := range r1 {
247 | macIfIndex[strings.ReplaceAll(strings.TrimPrefix(v.Oid, OLibrary.Dot1dTpFdbPort+"."), ".", " ")] =
248 | portIfIndex[fmt.Sprintf("%v", v.DataValue)]
249 | }
250 | }
251 | }
252 | r2, er2 := FetchItems("walk", node, []string{OLibrary.IfDes}, snmp)
253 | if er2 == nil {
254 | for _, v := range r2 {
255 | index, er := strconv.ParseInt(strings.TrimPrefix(v.Oid, OLibrary.IfDes+"."), 10, 64)
256 | if er == nil {
257 | ifIndexDes[index] = fmt.Sprintf("%v", v.DataValue)
258 | }
259 | }
260 | }
261 | for index, port := range ifIndexPort {
262 | data[index] = &IfIndexInfo{
263 | IfIndex: index,
264 | Port: port,
265 | Des: ifIndexDes[index],
266 | }
267 | }
268 | for k, v := range macIfIndex {
269 | if _, ok := data[v]; ok {
270 | data[v].ToIps = append(data[v].ToIps, k)
271 | }
272 | }
273 | }
274 |
275 | func ifIndexMapIpRoute(node string, data map[int64]*IfIndexInfo, snmp *gsnmp.GoSNMP) {
276 | var (
277 | types = map[string]int64{}
278 | nextHops = map[string]string{}
279 | ifIndexDes = map[int64]string{}
280 | )
281 | r, er := FetchItems("walk", node, []string{OLibrary.IfDes}, snmp)
282 | if er == nil {
283 | for _, v := range r {
284 | index, er := strconv.ParseInt(strings.TrimPrefix(v.Oid, OLibrary.IfDes+"."), 10, 64)
285 | if er == nil {
286 | ifIndexDes[index] = fmt.Sprintf("%v", v.DataValue)
287 | }
288 | }
289 | }
290 | r0, er0 := FetchItems("walk", node, []string{OLibrary.IpRouteType}, snmp)
291 | if er0 == nil {
292 | r1, er1 := FetchItems("walk", node, []string{OLibrary.IpRouteNextHop}, snmp)
293 | if er1 == nil {
294 | r2, er2 := FetchItems("walk", node, []string{OLibrary.IpRouteIfIndex}, snmp)
295 | if er2 == nil {
296 | for _, v := range r0 {
297 | types[strings.TrimPrefix(v.Oid, OLibrary.IpRouteType)] = v.DataValue.(*big.Int).Int64()
298 | }
299 | for _, v := range r1 {
300 | nextHops[strings.TrimPrefix(v.Oid, OLibrary.IpRouteNextHop)] = v.DataValue.(string)
301 | }
302 | for _, v := range r2 {
303 | k := strings.TrimPrefix(v.Oid, OLibrary.IpRouteIfIndex)
304 | ifIndex := v.DataValue.(*big.Int).Int64()
305 | if v1, ok := types[k]; ok && v1 == 4 {
306 | if _, ok1 := data[ifIndex]; ok1 {
307 | if !StringInSlice(nextHops[k], data[ifIndex].ToIps) {
308 | data[ifIndex].ToIps = append(data[ifIndex].ToIps, nextHops[k])
309 | }
310 | } else {
311 | data[ifIndex] = &IfIndexInfo{
312 | IfIndex: ifIndex,
313 | ToIps: []string{nextHops[k]},
314 | Port: fmt.Sprintf("%v", ifIndex),
315 | Des: ifIndexDes[ifIndex],
316 | }
317 | }
318 | }
319 | }
320 | }
321 | }
322 |
323 | }
324 | }
325 |
326 | func IfIndexMapIp(node string, snmp *gsnmp.GoSNMP) (ident string, data map[int64]*IfIndexInfo, err error) {
327 | data = map[int64]*IfIndexInfo{}
328 | if snmp == nil {
329 | snmp, er := NewSnmp(node)
330 | if er != nil {
331 | err = er
332 | return
333 | } else {
334 | defer snmp.Conn.Close()
335 | }
336 | }
337 | ident, _, err = IpIdentGet(node, snmp)
338 | if err != nil {
339 | return
340 | }
341 | if _, ok := IfIndexMapIpRecord.LoadOrStore(ident, struct{}{}); ok {
342 | return ident, data, nil
343 | }
344 | // 对于交换机
345 | ifIndexMapIpSwitch(ident, data, snmp)
346 | // 对于有路由的情况
347 | if len(data) == 0 {
348 | ifIndexMapIpRoute(ident, data, snmp)
349 | }
350 | for k := range data {
351 | data[k].ToIps = RmDupElement(data[k].ToIps)
352 | for _, v1 := range data[k].ToIps {
353 | if v2, ok := IpNodeTypeMaps.Get(v1); ok {
354 | switch v2.(NodeType) {
355 | case NTSwitch, NTRouter:
356 | if ip, _, er := IpIdentGet(v1, nil); er == nil && ip != "" && ip != node {
357 | data[k].Neighbor = ip
358 | goto Loop
359 | }
360 | }
361 | }
362 | }
363 | Loop:
364 | continue
365 | }
366 | return
367 | }
368 |
369 | // ScanArp 获取mac，ip的映射关系
370 | func ScanArp(ipAddr string, snmp *gsnmp.GoSNMP) (macIpsMap map[string][]string, er error) {
371 | if v, ok := IpIdent.Get(ipAddr); ok {
372 | ipAddr = v.(string)
373 | }
374 | if _, ok := ScanArpFinished.LoadOrStore(ipAddr, false); ok {
375 | return
376 | }
377 | if snmp == nil {
378 | snmp, er = NewSnmp(ipAddr)
379 | if er != nil {
380 | return
381 | }
382 | }
383 | arpSnmp, err := FetchItems("walk", ipAddr, []string{OLibrary.IpNetToMediaPhysAddress}, snmp)
384 | if err != nil {
385 | er = err
386 | return
387 | }
388 | r1, err := FetchItems("walk", ipAddr, []string{OLibrary.Dot1dBaseBridgeAddress}, snmp)
389 | if err != nil {
390 | return
391 | }
392 | macIpsMap = make(map[string][]string)
393 | //macIpsMap = make(map[interface{}][]string)
394 |
395 | for _, v := range arpSnmp {
396 | tmp := strings.Split(v.Oid, ".")
397 | if len(tmp) < 5 {
398 | continue
399 | }
400 | //h := HardwareAddr(v.Raw)
401 | h := ToHardwareAddr(v.Raw)
402 | if h == "" {
403 | continue
404 | }
405 | macIpsMap[h] = append(macIpsMap[h], strings.Join(tmp[len(tmp)-4:], "."))
406 | //if _, ok := macIpsMap[h]; ok {
407 | // macIpsMap[h] = append(macIpsMap[h], strings.Join(tmp[len(tmp)-4:], "."))
408 | //} else {
409 | // macIpsMap[h] = []string{strings.Join(tmp[len(tmp)-4:], ".")}
410 | //}
411 | }
412 | for _, v := range r1 {
413 | //h := HardwareAddr(v.Raw)
414 | h := ToHardwareAddr(v.Raw)
415 | macIpsMap[h] = append(macIpsMap[h], ipAddr)
416 | //if _, ok := macIpsMap[h]; ok {
417 | // macIpsMap[h] = append(macIpsMap[h], ipAddr)
418 | //} else {
419 | // macIpsMap[h] = []string{ipAddr}
420 | //}
421 | }
422 |
423 | for k, v := range macIpsMap {
424 | MacIpsSet(k, v)
425 | for _, i := range v {
426 | //TotalIps.Store(i, struct{}{})
427 | IpMacAddress.Store(i, k)
428 | }
429 | }
430 | return
431 | }
432 |
433 | func MacIpsSet(k, v any) {
434 | commonLocker.Lock()
435 | defer commonLocker.Unlock()
436 | var tmp []string
437 | if v1, ok := MacIps.Load(k); ok {
438 | tmp = v1.([]string)
439 | tmp = append(tmp, v.([]string)...)
440 | } else {
441 | tmp = v.([]string)
442 | }
443 | MacIps.Store(k, RmDupElement(tmp))
444 |
445 | }
446 |
447 | type StpInfo struct {
448 | AgentIp string
449 | AgentAddress string
450 | Port string
451 | PortState int64
452 | DesignAddress string
453 | DesignPort string
454 | }
455 |
456 | func NodeLink(node string, snmp *gsnmp.GoSNMP) {
457 | if _, ok := NodeLinkFinished.LoadOrStore(node, false); ok {
458 | return
459 | }
460 |
461 | r1, err := FetchItems("walk", node, []string{OLibrary.Dot1dBaseBridgeAddress}, snmp)
462 | if err != nil {
463 | return
464 | }
465 | r2, err := FetchItems("walk", node, []string{OLibrary.Dot1dStpPortDesignatedBridge}, snmp)
466 | if err != nil {
467 | return
468 | }
469 | r3, err := FetchItems("walk", node, []string{OLibrary.Dot1dStpPortState}, snmp)
470 | if err != nil {
471 | return
472 | }
473 | r4, err := FetchItems("walk", node, []string{OLibrary.Dot1dStpPort}, snmp)
474 | if err != nil {
475 | return
476 | }
477 | r5, err := FetchItems("walk", node, []string{OLibrary.Dot1dStpPortDesignatedPort}, snmp)
478 | if err != nil {
479 | return
480 | }
481 |
482 | selfMacAddress := ""
483 | for _, v := range r1 {
484 | //selfMacAddress = HardwareAddr(v.Raw)
485 | selfMacAddress = ToHardwareAddr(v.Raw)
486 | }
487 | var states = map[string]int64{}
488 | for _, v := range r3 {
489 | states[strings.TrimPrefix(v.Oid, OLibrary.Dot1dStpPortState)] = v.DataValue.(*big.Int).Int64()
490 | }
491 | var Ports = map[string]string{}
492 | for _, v := range r4 {
493 | Ports[strings.TrimPrefix(v.Oid, OLibrary.Dot1dStpPort)] = fmt.Sprintf("%v", v.Raw)
494 | }
495 | var dPorts = map[string]string{}
496 | for _, v := range r5 {
497 | dPorts[strings.TrimPrefix(v.Oid, OLibrary.Dot1dStpPortDesignatedPort)] = fmt.Sprintf("%v", v.Raw)
498 | }
499 |
500 | for _, v := range r2 {
501 | if ToHardwareAddr(v.Raw)[6:] == selfMacAddress {
502 | continue
503 | }
504 | index := strings.TrimPrefix(v.Oid, OLibrary.Dot1dStpPortDesignatedBridge)
505 |
506 | if v1, ok := states[strings.TrimPrefix(v.Oid, OLibrary.Dot1dStpPortDesignatedBridge)]; ok && v1 == 5 {
507 | data := StpInfo{
508 | AgentIp: node,
509 | AgentAddress: selfMacAddress,
510 | Port: Ports[index],
511 | PortState: states[index],
512 | DesignAddress: ToHardwareAddr(v.Raw)[6:],
513 | DesignPort: dPorts[index],
514 | }
515 | NodeLinks.Store(selfMacAddress, data)
516 | //fmt.Println("store:::", node, v.Raw, HardwareAddr(v.Raw)[6:], selfMacAddress)
517 | }
518 | break
519 | }
520 | }
521 |
--------------------------------------------------------------------------------
/netTopology/discover/layer3.go:
--------------------------------------------------------------------------------
1 | package discover
2 |
3 | import (
4 | "errors"
5 | "github.com/sirupsen/logrus"
6 | "math/big"
7 | "sort"
8 | //"sort"
9 | "strings"
10 | //"time"
11 |
12 | gsnmp "github.com/gosnmp/gosnmp"
13 |
14 | . "netTopology/internal"
15 | )
16 |
17 | func Layer3IpLinks(target string, snmpClient *gsnmp.GoSNMP) (horizontal, vertical map[string]int64, err error) {
18 | horizontal = map[string]int64{}
19 | vertical = map[string]int64{}
20 | routeType, er := FetchItems("walk", target, []string{OLibrary.IpRouteType}, snmpClient)
21 | if er != nil {
22 | err = errors.New(StringConcat("fetch item failed for ", target, OLibrary.IpRouteType))
23 | return
24 | }
25 | routeNextHop, er := FetchItems("walk", target, []string{OLibrary.IpRouteNextHop}, snmpClient)
26 | if er != nil {
27 | err = errors.New(StringConcat("fetch item failed for ", target, OLibrary.IpRouteNextHop))
28 | return
29 | }
30 | routeIfIndex, er := FetchItems("walk", target, []string{OLibrary.IpRouteIfIndex}, snmpClient)
31 | if er != nil {
32 | err = errors.New(StringConcat("fetch item failed for ", target, OLibrary.IpRouteIfIndex))
33 | return
34 | }
35 | nextHop := map[string]string{}
36 | for _, v := range routeNextHop {
37 | nextHop[strings.TrimPrefix(v.Oid, OLibrary.IpRouteNextHop+".")] = v.DataValue.(string)
38 | }
39 | ifIndex := map[string]int64{}
40 | for _, v := range routeIfIndex {
41 | ifIndex[strings.TrimPrefix(v.Oid, OLibrary.IpRouteIfIndex+".")] = v.DataValue.(*big.Int).Int64()
42 | }
43 | var key string
44 | for _, v := range routeType {
45 | key = strings.TrimPrefix(v.Oid, OLibrary.IpRouteType+".")
46 | // 有时候下一跳指向本地地址，或者防火墙virtual-if*; virtual-if0为根防火墙
47 | if v1, ok := IpIdent.Get(nextHop[key]); ok && v1.(string) == target {
48 | continue
49 | }
50 | if v.DataValue.(*big.Int).Int64() == 3 {
51 | vertical[key] = ifIndex[key]
52 | } else if v.DataValue.(*big.Int).Int64() == 4 {
53 | horizontal[key] = ifIndex[key]
54 | }
55 | }
56 |
57 | newHorizontal := map[string]int64{}
58 | newVertical := map[string]int64{}
59 |
60 | for k, v := range vertical {
61 | v1 := Final(k, nextHop)
62 | if v1 != k && v1 != "127.0.0.1" && v1 != "127.0.0.0" {
63 | newVertical[v1] = v
64 | }
65 | }
66 |
67 | for k, v := range horizontal {
68 | v1 := Final(k, nextHop)
69 | if v1 != k && v1 != "127.0.0.1" && v1 != "127.0.0.0" {
70 | newHorizontal[v1] = v
71 | }
72 | }
73 | vertical = newVertical
74 | horizontal = newHorizontal
75 | return
76 | }
77 |
78 | // L3Topology
79 | // 访问当前路由器的路由表,对每个路由表项
80 | // ipRouteType等于 indirect(4)，将路由表中ipRouteNextHop的内容不重复地加入邻居列表中
81 | // ipRouteType等于 direct(3)，把ipRouteDest和 ipRouteMask不重复地放到本地相连子网队列中。
82 | func L3Topology(target string, graph *TopologyGraph, recursive bool) {
83 | defer Topology3SyncWait.Done()
84 | if !ValidInnerIp(target) {
85 | logrus.Warnf("invalid intranet ip for %s", target)
86 | return
87 | }
88 | if _, ok := L3Nodes.Load(target); ok {
89 | return
90 | }
91 | if v, ok := IpIdent.Get(target); ok {
92 | target = v.(string)
93 | }
94 | snmpClient, er := NewSnmp(target)
95 | if er != nil {
96 | logrus.Infof("New snmp client for %s failed %s", target, er.Error())
97 | return
98 | } else {
99 | defer snmpClient.Conn.Close()
100 | }
101 | var (
102 | localIps, peerIps []string
103 | )
104 | arp := DiscoverIps(target, snmpClient)
105 | if arp == nil || arp.Target == "" {
106 | logrus.Info("get empty ip for ", target)
107 | return
108 | }
109 | target = arp.Target
110 | IpEntOfDevice.Store(target, arp.SelfIps)
111 | _ = peerIps
112 | if target == "" {
113 | logrus.Info("get self ip failed for ", target)
114 | return
115 | }
116 |
117 | horizontal, vertical, er := Layer3IpLinks(target, snmpClient)
118 | if er != nil {
119 | logrus.Warnf("Layer3IpLinks %s failed: %v", target, er)
120 | return
121 | }
122 | for k := range vertical {
123 | localIps = append(localIps, k)
124 | }
125 | {
126 | if _, ok := L3Nodes.LoadOrStore(target, false); !ok {
127 | topology3Lock.Lock()
128 | graph.Nodes = append(graph.Nodes, &Node{
129 | Name: target,
130 | Type: GetNodeType(target).String(),
131 | Locals: arp.Subnets,
132 | Ips: arp.SelfIps,
133 | })
134 | topology3Lock.Unlock()
135 | }
136 | }
137 | for ip, ifIndex := range horizontal {
138 | Topology3SyncWait.Add(1)
139 | go func(ip string, ifIndex int64) {
140 | snmpCli, er := NewSnmp(ip)
141 | if er != nil {
142 | logrus.Infof("New snmp client[%s] failed %s", ip, er.Error())
143 | Topology3SyncWait.Done()
144 | return
145 | } else {
146 | defer snmpCli.Conn.Close()
147 | }
148 | tmpArp := DiscoverIps(ip, snmpCli)
149 | if tmpArp.Target == "" {
150 | Topology3SyncWait.Done()
151 | return
152 | }
153 | if _, ok := L3Nodes.LoadOrStore(tmpArp.Target, false); !ok {
154 | topology3Lock.Lock()
155 | graph.Nodes = append(graph.Nodes, &Node{
156 | Name: tmpArp.Target,
157 | Type: GetNodeType(ip).String(),
158 | Locals: tmpArp.Subnets,
159 | Ips: tmpArp.SelfIps,
160 | })
161 | topology3Lock.Unlock()
162 | if recursive {
163 | L3Topology(tmpArp.Target, graph, recursive)
164 | } else {
165 | Topology3SyncWait.Done()
166 | }
167 | } else {
168 | Topology3SyncWait.Done()
169 | }
170 | }(ip, ifIndex)
171 | }
172 | }
173 |
174 | // SubnetIps 获取本机IP、连接的子网
175 | func SubnetIps(target string, snmpClient *gsnmp.GoSNMP) (subnetIps, subnetMask []string, er error) {
176 | r1, err := FetchItems("walk", target, []string{OLibrary.IpAdEntAddr}, snmpClient)
177 | if err != nil {
178 | er = err
179 | return
180 | }
181 | r2, err := FetchItems("walk", target, []string{OLibrary.IpAdEntNetMask}, snmpClient)
182 | if err != nil {
183 | er = err
184 | return
185 | }
186 | var (
187 | ips map[string]string
188 | obj string
189 | )
190 | for _, v := range r1 {
191 | obj = strings.TrimPrefix(v.Oid, OLibrary.IpAdEntAddr)
192 | ips[obj] = v.DataValue.(string)
193 | }
194 |
195 | for _, v := range r2 {
196 | obj = strings.TrimPrefix(v.Oid, OLibrary.IpAdEntNetMask)
197 | if ips[obj] != "" && ips[obj] != "127.0.0.1" && ips[obj] != "127.0.0.0" {
198 | subnetIps = append(subnetIps, ips[obj])
199 | subnetMask = append(subnetMask, v.DataValue.(string))
200 | }
201 | }
202 | return
203 | }
204 |
205 | // DiscoverIps 发现网络IP以及自身IP
206 | // localIps 本机ip列表； peerIps 对端ip列表
207 | func DiscoverIps(target string, snmpClient *gsnmp.GoSNMP) (result *Arp) {
208 | logrus.Info("DiscoverIps for ARP ", target)
209 | result = &Arp{}
210 | if v, ok := DiscoverIpsCache.Get(target); ok {
211 | return v.(*Arp)
212 | }
213 | defer func() {
214 | logrus.Info("DiscoverIps for ARP end: ", target)
215 | DiscoverIpsCache.SetDefault(target, result)
216 | }()
217 | r0, er := FetchItems("walk", target, []string{OLibrary.IpAdEntAddr}, snmpClient)
218 | if er != nil {
219 | logrus.Warnf("fetch %s %s failed %v", target, OLibrary.IpAdEntAddr, er)
220 | return
221 | }
222 | //r1, er := FetchItems("walk", target, []string{OLibrary.IpNetToMediaNetAddress}, snmpClient)
223 | //if er != nil {
224 | // fmt.Println(er)
225 | // return
226 | //}
227 | //r2, er := FetchItems("walk", target, []string{OLibrary.IpNetToMediaType}, snmpClient)
228 | //if er != nil {
229 | // fmt.Println(er)
230 | // return
231 | //}
232 | r3, er := FetchItems("walk", target, []string{OLibrary.IpRouteDest}, snmpClient)
233 | if er != nil {
234 | logrus.Warnf("fetch %s %s failed %v", target, OLibrary.IpRouteDest, er)
235 | return
236 | }
237 | r4, er := FetchItems("walk", target, []string{OLibrary.IpAdEntNetMask}, snmpClient)
238 | if er != nil {
239 | logrus.Warnf("fetch %s %s failed %v", target, OLibrary.IpAdEntNetMask, er)
240 | return
241 | }
242 | r5, er := FetchItems("walk", target, []string{OLibrary.IpRouteType}, snmpClient)
243 | if er != nil {
244 | logrus.Warnf("fetch %s %s failed %v", target, OLibrary.IpRouteType, er)
245 | return
246 | }
247 | r6, er := FetchItems("walk", target, []string{OLibrary.IpRouteMask}, snmpClient)
248 | if er != nil {
249 | logrus.Warnf("fetch %s %s failed %v", target, OLibrary.IpRouteMask, er)
250 | return
251 | }
252 | var (
253 | //ips = map[string]string{}
254 | masks = map[string]string{}
255 | //obj string
256 | )
257 | {
258 | for _, v := range r4 {
259 | masks[strings.TrimPrefix(v.Oid, OLibrary.IpAdEntNetMask)] = v.DataValue.(string)
260 | }
261 | sort.Slice(r0, func(i, j int) bool {
262 | return InetAtoi(r0[i].DataValue.(string)) < InetAtoi(r0[j].DataValue.(string))
263 | })
264 | for _, v := range r0 {
265 | if v.DataValue.(string) == "127.0.0.1" ||
266 | v.DataValue.(string) == "0.0.0.0" ||
267 | v.DataValue.(string) == "127.0.0.0" {
268 | continue
269 | }
270 | //ip := StringConcat(v.DataValue.(string), "/", masks[strings.TrimPrefix(v.Oid, OLibrary.IpAdEntAddr)])
271 | result.SelfIps = append(result.SelfIps, v.DataValue.(string))
272 | //if !StringInSlice(ip, result.Subnets) {
273 | // result.Subnets = append(result.Subnets, ip)
274 | //}
275 | }
276 | for _, v := range result.SelfIps {
277 | v = strings.Split(v, "/")[0]
278 | if _, er := NewSnmp(v); er == nil {
279 | result.Target = v
280 | break
281 | } else if PingHost(v) {
282 | result.Target = v
283 | continue
284 | }
285 | }
286 | for _, v := range result.SelfIps {
287 | v = strings.Split(v, "/")[0]
288 | IpIdent.SetDefault(v, result.Target)
289 | }
290 | }
291 | //{
292 | // for _, v := range r1 {
293 | // obj = strings.TrimPrefix(v.Oid, OLibrary.IpNetToMediaNetAddress)
294 | // ips[obj] = v.DataValue.(string)
295 | // }
296 | // for _, v := range r2 {
297 | // obj = strings.TrimPrefix(v.Oid, OLibrary.IpNetToMediaType)
298 | // switch v.DataValue.(*big.Int).String() {
299 | // case "3":
300 | // result.PeerIps = append(result.PeerIps, ips[obj])
301 | // case "4", "1":
302 | // if ips[obj] != "" && ips[obj] != "127.0.0.1" && ips[obj] != "127.0.0.0" {
303 | // ip := StringConcat(ips[obj], "/", masks["."+strings.SplitN(obj, ".", 3)[2]])
304 | // if !StringInSlice(ip, result.Subnets) {
305 | // result.Subnets = append(result.Subnets, ip)
306 | // }
307 | // }
308 | // }
309 | // }
310 | //}
311 | {
312 | dest := map[string]string{}
313 | routeType := map[string]int64{}
314 | for _, v := range r3 {
315 | dest[strings.TrimPrefix(v.Oid, OLibrary.IpRouteDest)] = v.DataValue.(string)
316 | }
317 | for _, v := range r5 {
318 | routeType[strings.TrimPrefix(v.Oid, OLibrary.IpRouteType)] = v.DataValue.(*big.Int).Int64()
319 | }
320 | for _, v := range r6 {
321 | key := strings.TrimPrefix(v.Oid, OLibrary.IpRouteMask)
322 | if v1, ok := routeType[key]; ok {
323 | if v1 == int64(3) {
324 | if !strings.HasPrefix(dest[key], "127.") &&
325 | dest[key] != "0.0.0.0" &&
326 | !StringInSlice(dest[key], result.SelfIps) {
327 | result.Subnets = append(result.Subnets, StringConcat(dest[key], "/", v.DataValue.(string)))
328 | }
329 | } else if v1 == int64(4) {
330 | result.PeerIps = append(result.PeerIps, StringConcat(dest[key], "/", v.DataValue.(string)))
331 | }
332 | }
333 | }
334 | }
335 | Routes.Store(result.Target, result)
336 | return
337 | }
338 |
--------------------------------------------------------------------------------
/netTopology/discover/master.go:
--------------------------------------------------------------------------------
1 | // Package discover /*
2 | package discover
3 |
4 | import (
5 | "github.com/sirupsen/logrus"
6 | "math"
7 | "sort"
8 | "time"
9 |
10 | "github.com/jackpal/gateway"
11 | "github.com/shopspring/decimal"
12 |
13 | . "netTopology/internal"
14 | )
15 |
16 | type MasterDiscover string
17 |
18 | func init() {
19 | go consumerMetrics()
20 | }
21 |
22 | // Collectors agents for collect and community config
23 | func (h *MasterDiscover) Collectors(ident string) (data []*CollectorConfig, er error) {
24 | var nodes []*NodeSnmp
25 | nodes = append(nodes, &NodeSnmp{"", "public", "2c"})
26 | data = append(data, &CollectorConfig{
27 | Id: "id1",
28 | Gateway: "",
29 | Nodes: nodes,
30 | Name: "test",
31 | })
32 | return data, nil
33 | //var resp []*GraphSnmp
34 | //q := map[string]interface{}{"enable": true}
35 | //if ident != "" {
36 | // q["one_agent"] = ident
37 | //}
38 | //_, er = (&GraphSnmp{}).List(db.SnmpTopologyCol, q, int64(1), int64(1000), int64(1000), &resp)
39 | //if er != nil {
40 | // return
41 | //}
42 | //for _, v := range resp {
43 | // var nodes []*NodeSnmp
44 | // for _, v1 := range v.Nodes {
45 | // nodes = append(nodes, &NodeSnmp{v1.Ip, v1.Community, v1.Version})
46 | // }
47 | // data = append(data, &CollectorConfig{
48 | // Id: v.Id.Hex(),
49 | // Gateway: v.Gateway,
50 | // Nodes: nodes,
51 | // Name: v.Name,
52 | // })
53 | //}
54 | //return
55 | }
56 |
57 | // UploadRecord save graph on disk
58 | func (h *MasterDiscover) UploadRecord(graph *TopologyGraph) error {
59 | //r, er := json.Marshal(graph)
60 | //if er != nil {
61 | // return er
62 | //} else {
63 | // _, err := utils.RedisAction("HSET", RedisKeyDiscoverLinks, graph.GraphId, r)
64 | // if err != nil {
65 | // return err
66 | // }
67 | //}
68 | ////
69 | //updateLinks(graph.Links, graph.GraphId)
70 | TG.Store(graph.GraphId, graph)
71 | return nil
72 | }
73 |
74 | func (h *MasterDiscover) Topology(config *CollectorConfig) (*TopologyGraph, error) {
75 | Init()
76 | graph := &TopologyGraph{GraphId: config.Id}
77 |
78 | if config.Gateway == "" {
79 | myGateway, er := gateway.DiscoverGateway()
80 | if er != nil {
81 | return graph, nil
82 | }
83 | config.Gateway = myGateway.String()
84 | }
85 |
86 | Topology3SyncWait.Add(1)
87 | for _, v := range config.Nodes {
88 | CommunityMap.SetDefault(v.Ip, v.Community)
89 | SnmpVersionMap.SetDefault(v.Ip, v.Version)
90 | }
91 | graph.Name = config.Name
92 | L3Topology(config.Gateway, graph, true)
93 | Topology3SyncWait.Wait()
94 | portMapIfIndex, portMapDes := ScanTotalSubnetsAndNeighbor(graph)
95 | tidyGraph(graph, portMapIfIndex, portMapDes)
96 | return graph, nil
97 | }
98 |
99 | func (h *MasterDiscover) Nodes() []string {
100 | //var (
101 | // res []string
102 | // res1 = map[string]string{}
103 | // res2 = map[string]string{}
104 | //)
105 | //err := utils.ScanMap(RedisKeyCompleteLinks, res1)
106 | //if err != nil {
107 | // return res
108 | //}
109 | //nodes := map[string]struct{}{}
110 | //for _, v := range res1 {
111 | // data1 := TopologyGraphV1{}
112 | // er := json.Unmarshal([]byte(v), &data1)
113 | // if er != nil {
114 | // _ = level.Warn(g.Logger).Log("module", "net", "msg", er.Error())
115 | // continue
116 | // }
117 | // for _, v1 := range data1.Nodes {
118 | // nodes[v1.Name] = struct{}{}
119 | // }
120 | //}
121 | //err = utils.ScanMap(RedisKeyDiscoverLinks, res2)
122 | //if err != nil {
123 | // return res
124 | //}
125 | //for _, v := range res2 {
126 | // data2 := TopologyGraph{}
127 | // er := json.Unmarshal([]byte(v), &data2)
128 | // if er != nil {
129 | // _ = level.Warn(g.Logger).Log("module", "net", "msg", er.Error())
130 | // continue
131 | // }
132 | // for _, v2 := range data2.Nodes {
133 | // nodes[v2.Name] = struct{}{}
134 | // }
135 | //}
136 |
137 | // -------simple-----
138 | nodes := map[string]struct{}{}
139 | TG.Range(func(key, value any) bool {
140 | for _, v := range value.(*TopologyGraph).Nodes {
141 | nodes[v.Name] = struct{}{}
142 | }
143 | return true
144 | })
145 | var res []string
146 | for k := range nodes {
147 | res = append(res, k)
148 | }
149 | sort.Strings(res)
150 | return res
151 | }
152 |
153 | func (h *MasterDiscover) UploadTsRecord(target string, data map[string]map[string]int64) {
154 | //var metrics []*pb.Metric
155 | ifDetail, er := GetIfDetails(target, nil)
156 | if er != nil {
157 | logrus.Error(er)
158 | return
159 | }
160 | var value float64
161 | t := time.Now().Unix()
162 |
163 | for metric, v := range data {
164 | for port, val := range v {
165 | if !math.IsInf(float64(val), 0) && !math.IsNaN(float64(val)) {
166 | value, _ = decimal.NewFromFloat(float64(val)).Round(2).Float64()
167 | } else {
168 | //fmt.Printf("invalid value %v for [%s %s %s]\n", val, node, valueType, direction)
169 | value = 0
170 | }
171 | if v2, ok := ifDetail[port]; ok {
172 | LocalMetrics <- &Metric{
173 | Metric: metric,
174 | Type: "counter",
175 | Tags: map[string]string{
176 | "host": target,
177 | "if": v2.Index,
178 | "port": v2.Port,
179 | "name": v2.Name,
180 | },
181 | Timestamp: t,
182 | Value: value,
183 | }
184 | }
185 | }
186 | }
187 | }
188 |
189 | func consumerMetrics() {
190 | for m := range LocalMetrics {
191 | logrus.Info(m)
192 | }
193 | }
194 |
195 | //func updateLinks(links []*Link, graphId string) {
196 | // for _, v := range links {
197 | // t := &ConfigGraphLink{
198 | // LinkNode: &LinkNode{
199 | // Name: v.NodeFromName,
200 | // DeviceInterface: v.NodeFrom,
201 | // },
202 | // To: v.NodeToName,
203 | // GraphId: graphId,
204 | // }
205 | // upsertConfigGraph(t)
206 | // }
207 | //}
208 |
209 | //
210 | //func upsertConfigGraph(data *ConfigGraphLink) {
211 | // data.LinkId = MD5(strings.Join([]string{data.Name, data.DeviceInterface, data.To}, "-"))
212 | // //data.Action = 3
213 | // //count, er := data.Count(db.ConfigGraphLinkCol, bson.M{"link_id": data.LinkId, "graph_id": data.GraphId})
214 | // //if er != nil {
215 | // // _ = level.Error(g.Logger).Log("module", "net", "msg", er.Error())
216 | // // return
217 | // //}
218 | // //if count == 0 {
219 | // // _, er = data.Add(db.ConfigGraphLinkCol, data)
220 | // // if er != nil {
221 | // // _ = level.Error(g.Logger).Log("module", "net", "msg", er.Error())
222 | // // }
223 | // // return
224 | // //}
225 | // return
226 | //}
227 |
--------------------------------------------------------------------------------
/netTopology/discover/models.go:
--------------------------------------------------------------------------------
1 | // Package models /*
2 | package discover
3 |
4 | import (
5 | "go.mongodb.org/mongo-driver/bson/primitive"
6 | "time"
7 | )
8 |
9 | type NodeSnmp struct {
10 | Ip string `json:"ip" bson:"ip"`
11 | Community string `json:"community" bson:"community"`
12 | Version string `json:"version" bson:"version"`
13 | }
14 |
15 | type GraphSnmp struct {
16 | BaseModel `bson:",inline"`
17 | Id primitive.ObjectID `json:"id" bson:"_id,omitempty"`
18 | Name string `json:"name" bson:"name"`
19 | Gateway string `json:"gateway" bson:"gateway"`
20 | Nodes []*NodeSnmp `json:"nodes" bson:"nodes"`
21 | Enable bool `json:"enable" bson:"enable"`
22 | OneAgent string `json:"one_agent" bson:"one_agent"`
23 | }
24 |
25 | type LinkNode struct {
26 | Name string `json:"name" bson:"name"`
27 | DeviceInterface string `json:"device_interface" bson:"device_interface"`
28 | // HighSpeed 单位 Mbps
29 | HighSpeed int `json:"high_speed" bson:"high_speed"`
30 | }
31 |
32 | type BaseModel struct {
33 | CreateTime time.Time `json:"create_time" bson:"create_time"`
34 | UpdateTime time.Time `json:"update_time" bson:"update_time"`
35 | Creator string `json:"creator" bson:"creator"`
36 | }
37 |
38 | type ConfigGraphLink struct {
39 | *BaseModel `bson:",inline"`
40 | *LinkNode `bson:",inline"`
41 | Id primitive.ObjectID `json:"-" bson:"_id,omitempty"`
42 | GraphId string `json:"-" bson:"graph_id"`
43 | // Name 自动生成，根据from,to的名称和type构成
44 | LinkId string `json:"link_id" bson:"link_id"`
45 | To string `json:"to" bson:"to"`
46 | Comment string `json:"comment" bson:"comment"`
47 | // Action 表示改链接的状态
48 | // 0: 人工删除， 1:人工确认或者添加 3:自动发现
49 | Action uint8 `json:"action" bson:"action"`
50 | }
51 |
52 | //type ConfigGraphNode struct {
53 | // BaseModel
54 | // Id primitive.ObjectID `json:"id" bson:"_id,omitempty"`
55 | // Name string `json:"name" bson:"name"`
56 | // DeviceInterfaces []*LinkNode `json:"device_interfaces" bson:"device_interfaces"`
57 | //}
58 |
59 | type Metric struct {
60 | Metric string `protobuf:"bytes,1,opt,name=metric,proto3" json:"metric,omitempty"`
61 | Value float64 `protobuf:"fixed64,2,opt,name=value,proto3" json:"value,omitempty"`
62 | Timestamp int64 `protobuf:"varint,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
63 | Tags map[string]string `protobuf:"bytes,4,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
64 | Ns string `protobuf:"bytes,5,opt,name=ns,proto3" json:"ns,omitempty"`
65 | Type string `protobuf:"bytes,6,opt,name=type,proto3" json:"type,omitempty"`
66 | }
67 |
--------------------------------------------------------------------------------
/netTopology/discover/var.go:
--------------------------------------------------------------------------------
1 | package discover
2 |
3 | import (
4 | "sync"
5 | "time"
6 |
7 | "github.com/patrickmn/go-cache"
8 |
9 | . "netTopology/internal"
10 | )
11 |
12 | //type Node struct {
13 | // // Name must be unique
14 | // Name string `json:"name"`
15 | // // Alias
16 | // Alias string `json:"alias"`
17 | // // Type switch/router/server/internet/firewall/unknown
18 | // Type string `json:"type"`
19 | // // X location x
20 | // X int `json:"x"`
21 | // // Y location y
22 | // Y int `json:"y"`
23 | // // Ips 本机IP
24 | // Ips []string `json:"ips"`
25 | // // Locals 直连ip
26 | // Locals []string `json:"locals"`
27 | // // discoverIps 间接连接ip
28 | // discoverIps []string
29 | // Options interface{} `json:"options"`
30 | // Status string `json:"status"`
31 | //}
32 |
33 | // name2: Router_symbol_(96)
34 | // #bandwidth: 10 # 可以覆盖默认的link里的设置
35 | // #width: 5
36 | // #copy: true # 如果设置，即使拓扑里没这个元素，也会画出来
37 | // hostname: Router # 设置监控项采集的节点
38 | // itemin: net.if.in["eth0"] # 设置监控项的key
39 | // itemout: net.if.out["eth0"] # 设置监控项的key
40 |
41 | type Link struct {
42 | NodeFrom string `json:"node_from"`
43 | NodeTo string `json:"node_to"`
44 | NodeFromName string `json:"node_from_name"`
45 | NodeToName string `json:"node_to_name"`
46 | Value float64 `json:"value"`
47 | Unit string `json:"unit"`
48 | Level uint8 `json:"level"`
49 | IfIndex string `json:"if_index"`
50 | TrafficLoad float64 `json:"traffic_load"`
51 | // Port 物理接口
52 | Port string `json:"port"`
53 | Options interface{} `json:"options"`
54 | }
55 |
56 | type TopologyGraph struct {
57 | Name string `json:"name"`
58 | Nodes []*Node `json:"nodes"`
59 | Links []*Link `json:"links"`
60 | Location bool `json:"location"`
61 | GraphId string `json:"graph_id"`
62 | }
63 |
64 | type Arp struct {
65 | Target string
66 | // SelfIps 自身IP
67 | SelfIps []string
68 | // Subnets 连接的子网
69 | Subnets []string
70 | // LocalIps 子网ip
71 | LocalIps []string
72 | // PeerIps 间接ip列表,即平行连接ip列表，如交换机的nextHop
73 | PeerIps []string
74 | }
75 |
76 | const ()
77 |
78 | var (
79 | topology3Lock = sync.RWMutex{}
80 |
81 | IpScanWaiting = make(chan string, 10000)
82 | // IpScannedRecord 记录Ip是否扫描
83 | IpScannedRecord = sync.Map{}
84 | // IpEntOfDevice 设备的包含的IP列表
85 | IpEntOfDevice = sync.Map{}
86 | Topology3SyncWait = sync.WaitGroup{}
87 | // NetDevices 存放发现的
88 | NetDevices = make(chan string, 10000)
89 | DiscoverIpsCache = cache.New(time.Minute*5, time.Minute)
90 |
91 | //TargetStatus = cache.New(time.Hour, time.Minute*30)
92 | //CommunityMap = cache.New(time.Hour*24, time.Hour)
93 |
94 | MacIps = sync.Map{}
95 | L3Nodes = sync.Map{}
96 | L3Links = sync.Map{}
97 | Routes = sync.Map{}
98 | L2Switches = sync.Map{}
99 | IpMacAddress = sync.Map{}
100 | L2IpScanned = sync.Map{}
101 |
102 | SubnetsVar = &Subnets{M: map[string]*Subnet{}}
103 | // NodeLinkFinished 是否完成NodeLink
104 | NodeLinkFinished = sync.Map{}
105 | ScanArpFinished = sync.Map{}
106 | SubnetsScanned = sync.Map{}
107 | TotalIps = sync.Map{}
108 | LocalMetrics = make(chan *Metric, 1000)
109 | // TG map[string]*TopologyGraph{}
110 | TG = sync.Map{}
111 | )
112 |
113 | func Init() {
114 | IpScanWaiting = make(chan string, 10000)
115 | IpScannedRecord = sync.Map{}
116 | IpEntOfDevice = sync.Map{}
117 | Topology3SyncWait = sync.WaitGroup{}
118 | NetDevices = make(chan string, 10000)
119 | DiscoverIpsCache = cache.New(time.Minute*5, time.Minute)
120 | MacIps = sync.Map{}
121 | L3Nodes = sync.Map{}
122 | L3Links = sync.Map{}
123 | Routes = sync.Map{}
124 | IpMacAddress = sync.Map{}
125 | NodeLinkFinished = sync.Map{}
126 | ScanArpFinished = sync.Map{}
127 | SubnetsScanned = sync.Map{}
128 | TotalIps = sync.Map{}
129 |
130 | ResultCache = cache.New(time.Hour, time.Minute*15)
131 | TargetStatus = cache.New(time.Hour, time.Minute*30)
132 | //CommunityMap = cache.New(time.Hour*24, time.Hour)
133 | //IpIdent = sync.Map{}
134 | IpNodeTypeMaps = cache.New(time.Hour*24*7, time.Hour)
135 | }
136 |
--------------------------------------------------------------------------------
/netTopology/go.mod:
--------------------------------------------------------------------------------
1 | module netTopology
2 |
3 | go 1.19
4 |
5 | require (
6 | github.com/dustin/go-humanize v1.0.1
7 | github.com/gin-gonic/gin v1.9.1
8 | github.com/go-kit/kit v0.12.0
9 | github.com/go-kit/log v0.2.1
10 | github.com/go-ping/ping v1.1.0
11 | github.com/gomodule/redigo v1.8.9
12 | github.com/gosnmp/gosnmp v1.35.0
13 | github.com/jackpal/gateway v1.0.10
14 | github.com/mitchellh/hashstructure/v2 v2.0.2
15 | github.com/patrickmn/go-cache v2.1.0+incompatible
16 | github.com/pkg/errors v0.9.1
17 | github.com/shopspring/decimal v1.3.1
18 | github.com/sirupsen/logrus v1.8.1
19 | go.mongodb.org/mongo-driver v1.12.1
20 | golang.org/x/sync v0.3.0
21 | )
22 |
23 | require (
24 | github.com/bytedance/sonic v1.9.1 // indirect
25 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
26 | github.com/gabriel-vasile/mimetype v1.4.2 // indirect
27 | github.com/gin-contrib/sse v0.1.0 // indirect
28 | github.com/go-logfmt/logfmt v0.5.1 // indirect
29 | github.com/go-playground/locales v0.14.1 // indirect
30 | github.com/go-playground/universal-translator v0.18.1 // indirect
31 | github.com/go-playground/validator/v10 v10.14.0 // indirect
32 | github.com/goccy/go-json v0.10.2 // indirect
33 | github.com/google/uuid v1.2.0 // indirect
34 | github.com/json-iterator/go v1.1.12 // indirect
35 | github.com/klauspost/cpuid/v2 v2.2.4 // indirect
36 | github.com/leodido/go-urn v1.2.4 // indirect
37 | github.com/mattn/go-isatty v0.0.19 // indirect
38 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
39 | github.com/modern-go/reflect2 v1.0.2 // indirect
40 | github.com/pelletier/go-toml/v2 v2.0.8 // indirect
41 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
42 | github.com/ugorji/go/codec v1.2.11 // indirect
43 | golang.org/x/arch v0.3.0 // indirect
44 | golang.org/x/crypto v0.9.0 // indirect
45 | golang.org/x/net v0.10.0 // indirect
46 | golang.org/x/sys v0.8.0 // indirect
47 | golang.org/x/text v0.9.0 // indirect
48 | google.golang.org/protobuf v1.30.0 // indirect
49 | gopkg.in/yaml.v3 v3.0.1 // indirect
50 | )
51 |
--------------------------------------------------------------------------------
/netTopology/go.sum:
--------------------------------------------------------------------------------
1 | github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
2 | github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
3 | github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
4 | github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
5 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
6 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
10 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
11 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
12 | github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
13 | github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
14 | github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
15 | github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
16 | github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
17 | github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
18 | github.com/go-kit/kit v0.12.0 h1:e4o3o3IsBfAKQh5Qbbiqyfu97Ku7jrO/JbohvztANh4=
19 | github.com/go-kit/kit v0.12.0/go.mod h1:lHd+EkCZPIwYItmGDDRdhinkzX2A1sj+M9biaEaizzs=
20 | github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
21 | github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
22 | github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
23 | github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
24 | github.com/go-ping/ping v1.1.0 h1:3MCGhVX4fyEUuhsfwPrsEdQw6xspHkv5zHsiSoDFZYw=
25 | github.com/go-ping/ping v1.1.0/go.mod h1:xIFjORFzTxqIV/tDVGO4eDy/bLuSyawEeojSm3GfRGk=
26 | github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
27 | github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
28 | github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
29 | github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
30 | github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
31 | github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js=
32 | github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
33 | github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
34 | github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
35 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
36 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
37 | github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
38 | github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
39 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
40 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
41 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
42 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
43 | github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
44 | github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
45 | github.com/gosnmp/gosnmp v1.35.0 h1:EuWWNPxTCdAUx2/NbQcSa3WdNxjzpy4Phv57b4MWpJM=
46 | github.com/gosnmp/gosnmp v1.35.0/go.mod h1:2AvKZ3n9aEl5TJEo/fFmf/FGO4Nj4cVeEc5yuk88CYc=
47 | github.com/jackpal/gateway v1.0.10 h1:7g3fDo4Cd3RnTu6PzAfw6poO4Y81uNxrxFQFsBFSzJM=
48 | github.com/jackpal/gateway v1.0.10/go.mod h1:+uPBgIllrbkwYCAoDkGSZbjvpre/bGYAFCYIcrH+LHs=
49 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
50 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
51 | github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
52 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
53 | github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
54 | github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
55 | github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
56 | github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
57 | github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
58 | github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
59 | github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
60 | github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
61 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
62 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
63 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
64 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
65 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
66 | github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
67 | github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
68 | github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
69 | github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
70 | github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
71 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
72 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
73 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
74 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
75 | github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
76 | github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
77 | github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
78 | github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
79 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
80 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
81 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
82 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
83 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
84 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
85 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
86 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
87 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
88 | github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
89 | github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
90 | github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
91 | github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
92 | github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
93 | github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
94 | github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
95 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
96 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
97 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
98 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
99 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
100 | go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE=
101 | go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ=
102 | golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
103 | golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
104 | golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
105 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
106 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
107 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
108 | golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
109 | golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
110 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
111 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
112 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
113 | golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
114 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
115 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
116 | golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
117 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
118 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
119 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
120 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
121 | golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
122 | golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
123 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
124 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
125 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
126 | golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
127 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
128 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
129 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
130 | golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
131 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
132 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
133 | golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
134 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
135 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
136 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
137 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
138 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
139 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
140 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
141 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
142 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
143 | golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
144 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
145 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
146 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
147 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
148 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
149 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
150 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
151 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
152 | google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
153 | google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
154 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
155 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
156 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
157 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
158 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
159 | rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
160 |
--------------------------------------------------------------------------------
/netTopology/img.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/veops/ops-tools/c8912d0e51de78cfed487f57553ccf3f6b4d394e/netTopology/img.png
--------------------------------------------------------------------------------
/netTopology/internal/common.go:
--------------------------------------------------------------------------------
1 | package internal
2 |
3 | import (
4 | "fmt"
5 | "github.com/go-ping/ping"
6 | gsnmp "github.com/gosnmp/gosnmp"
7 | "math/big"
8 | "net"
9 | "strconv"
10 | "strings"
11 | "sync"
12 | "sync/atomic"
13 | "time"
14 | )
15 |
16 | var (
17 | // CidrSpan 默认最大网络号
18 | CidrSpan = 20
19 | // SpecialCIDRs 对于超过4094位的需要单独指定
20 | // 如 {"1.1.1.1/19":""}
21 | SpecialCIDRs = sync.Map{}
22 | )
23 |
24 | func InetItoa(ip int64) string {
25 | return fmt.Sprintf("%d.%d.%d.%d",
26 | byte(ip>>24), byte(ip>>16), byte(ip>>8), byte(ip))
27 | }
28 |
29 | func InetAtoi(ip string) int64 {
30 | ret := big.NewInt(0)
31 | ret.SetBytes(net.ParseIP(strings.TrimSpace(ip)).To4())
32 | return ret.Int64()
33 | }
34 |
35 | func PingHost(hostName string) bool {
36 | p, err := ping.NewPinger(hostName)
37 | if err != nil {
38 | return false
39 | }
40 | p.Timeout = time.Second
41 | p.SetPrivileged(true)
42 | p.Count = 3
43 | err = p.Run() // Blocks until finished.
44 | if err != nil {
45 | return false
46 | }
47 | return p.Statistics().PacketsRecv > 0
48 | }
49 |
50 | // PortScan 扫描ip端口
51 | // @params ignorePort bool 是否忽略端口返回,此种情况只是为了在icmp协议禁用之后,判断主机是否在线
52 | // @return available bool 是否在线
53 | // @return ports []int 在线端口列表
54 | func PortScan(ipStr string, ignorePort bool, scanPorts []int) (available bool, ports []int) {
55 | var wg sync.WaitGroup
56 | var ch = make(chan struct{}, 50)
57 | var lock sync.Mutex
58 | var chClosed atomic.Bool
59 | chClosed.Store(false)
60 | for j := 1; j <= 65535; j++ {
61 | if len(scanPorts) > 0 && !InSlice(j, scanPorts) {
62 | continue
63 | }
64 | wg.Add(1)
65 | if chClosed.Load() {
66 | wg.Done()
67 | break
68 | }
69 | ch <- struct{}{}
70 | go func(i int) {
71 | defer wg.Done()
72 | <-ch
73 | if chClosed.Load() {
74 | return
75 | }
76 | var address = fmt.Sprintf("%s:%d", ipStr, i)
77 | conn, err := net.DialTimeout("tcp", address, time.Second*10)
78 | if err != nil {
79 | return
80 | }
81 | conn.Close()
82 | if ignorePort {
83 | chClosed.Store(true)
84 | } else {
85 | lock.Lock()
86 | ports = append(ports, i)
87 | lock.Unlock()
88 | }
89 | }(j)
90 | }
91 | wg.Wait()
92 | return
93 | }
94 |
95 | func InSlice(a int, list []int) bool {
96 | for _, b := range list {
97 | if b == a {
98 | return true
99 | }
100 | }
101 | return false
102 | }
103 |
104 | func StringInSlice(a string, list []string) bool {
105 | for _, b := range list {
106 | if b == a {
107 | return true
108 | }
109 | }
110 | return false
111 | }
112 |
113 | func StringConcat(str ...string) string {
114 | var builder strings.Builder
115 | for _, v := range str {
116 | builder.WriteString(v)
117 | }
118 | return builder.String()
119 | }
120 |
121 | func inc(ip net.IP) {
122 | for j := len(ip) - 1; j >= 0; j-- {
123 | ip[j]++
124 | if ip[j] > 0 {
125 | break
126 | }
127 | }
128 | }
129 |
130 | func containsCIDR(cr1, cr2 string) bool {
131 | _, a, _ := net.ParseCIDR(cr1)
132 | _, b, _ := net.ParseCIDR(cr2)
133 | ones1, _ := a.Mask.Size()
134 | ones2, _ := b.Mask.Size()
135 | return ones1 <= ones2 && a.Contains(b.IP)
136 | }
137 |
138 | // acceptCidr 当获取一个网段时,网段太大，扫描负担很重，因此默认值采集最小255.255.16.0，即网络号至少20位是网络号
139 | // 如有特殊需求，需明确指定cidr范围
140 | // cidr: 192.168.1.1/24
141 | func validtCidr(cidr string) bool {
142 | //tmp := strings.Split(cidr, "/")
143 | //var sz int
144 | //// 1.1.1.1/255.255.255.0
145 | //if len(tmp) == 2 && len(tmp[1]) > 2 {
146 | // adr := net.ParseIP(tmp[1]).To4()
147 | // sz, _ = net.IPv4Mask(adr[0], adr[1], adr[2], adr[3]).Size()
148 | // if sz >= CidrSpan {
149 | // return true
150 | // } else {
151 | // cidr = fmt.Sprintf("%s/%d", tmp[0], sz)
152 | // }
153 | //}
154 | _, ipNet, _ := net.ParseCIDR(cidr)
155 | if m, _ := ipNet.Mask.Size(); m >= CidrSpan {
156 | return true
157 | }
158 | var valid bool
159 | SpecialCIDRs.Range(func(key, value any) bool {
160 | if containsCIDR(key.(string), cidr) {
161 | valid = true
162 | return false
163 | }
164 | return true
165 | })
166 | return valid
167 | }
168 |
169 | // CidrHosts Convert Cidr Address To Hosts
170 | // newAddr : "192.168.1.0/24" or "192.168.1.0/255.255.255.0"
171 | // 防止返回过多ip,此处限制最多返回一个B段
172 | func CidrHosts(netAddr string) ([]string, error) {
173 | tmp := strings.Split(netAddr, "/")
174 | if len(tmp) == 2 && len(tmp[1]) > 2 {
175 | adr := net.ParseIP(tmp[1]).To4()
176 | sz, _ := net.IPv4Mask(adr[0], adr[1], adr[2], adr[3]).Size()
177 | netAddr = fmt.Sprintf("%s/%d", tmp[0], sz)
178 | }
179 | if !validtCidr(netAddr) {
180 | return nil, fmt.Errorf("ip range is too large for %s, skipped", netAddr)
181 | }
182 | ipAddr, ipv4Net, err := net.ParseCIDR(netAddr)
183 | if err != nil {
184 | return nil, err
185 | }
186 | var ips []string
187 | for ip := ipAddr.Mask(ipv4Net.Mask); ipv4Net.Contains(ip); inc(ip) {
188 | ips = append(ips, ip.String())
189 | if len(ips) > 65535 {
190 | break
191 | }
192 | }
193 | //mask := binary.BigEndian.Uint32(ipv4Net.Mask)
194 | //start := binary.BigEndian.Uint32(ipv4Net.IP)
195 | //finish := (start & mask) | (mask ^ 0xffffffff)
196 | //var hosts []string
197 | //tmp1 := 0
198 | //for i := start + 1; i <= finish-1; i++ {
199 | // if tmp1 > 100 {
200 | // fmt.Println("break", finish, mask, start, mask, start&mask, mask^0xffffffff)
201 | // break
202 | // }
203 | // ip := make(net.IP, 4)
204 | // binary.BigEndian.PutUint32(ip, i)
205 | // hosts = append(hosts, ip.String())
206 | // tmp1++
207 | //}
208 | if len(ips) > 2 {
209 | return ips[1 : len(ips)-1], nil
210 | }
211 | return []string{}, nil
212 | }
213 |
214 | // MacAddressConvert
215 | // from: 11.21.31.41.51.61 --> b:15:1f:29:33:3d
216 | func MacAddressConvert(from string) string {
217 | //.1.3.6.1.2.1.4.22.1.2
218 | v := strings.Split(from, ".")
219 | var s []string
220 | for _, v1 := range v {
221 | if v1 == "" {
222 | continue
223 | }
224 | d, er := strconv.Atoi(v1)
225 | if er != nil {
226 | return from
227 | }
228 | s = append(s, fmt.Sprintf("%x", d))
229 | }
230 | return strings.Join(s, ":")
231 | }
232 |
233 | func ValidInnerIp(ip string) bool {
234 | //
235 | validInnerIps := []string{
236 | // 10.0.0.0/8
237 | "10.0.0.0-10.255.255.255",
238 | // 172.16.0.0/12
239 | "172.16.0.0-172.31.255.255",
240 | // 192.168.0.0/16
241 | "192.168.0.0-192.168.255.255",
242 | // 198.18.0.0/15 用于测试两个独立子网的网间通信
243 | "198.18.0.0-198.19.255.255",
244 | // 198.51.100.0/24
245 | "198.51.100.0-198.51.100.255",
246 | //203.0.113.0/24
247 | "203.0.113.0-203.0.113.255",
248 | }
249 | for _, v := range validInnerIps {
250 | ipSlice := strings.Split(v, `-`)
251 | if len(ipSlice) < 0 {
252 | continue
253 | }
254 | if InetAtoi(ip) >= InetAtoi(ipSlice[0]) && InetAtoi(ip) <= InetAtoi(ipSlice[1]) {
255 | return true
256 | }
257 | }
258 | var valid bool
259 | SpecialCIDRs.Range(func(key, value any) bool {
260 | t := net.ParseIP(ip)
261 | if t != nil {
262 | _, a, _ := net.ParseCIDR(key.(string))
263 | if a.Contains(t) {
264 | valid = true
265 | return false
266 | }
267 | }
268 | return true
269 | })
270 | return valid
271 | }
272 |
273 | func HardwareAddr(addr interface{}) string {
274 | switch address := addr.(type) {
275 | case string:
276 | return address
277 | case []byte:
278 | return net.HardwareAddr(address).String()
279 | }
280 | return ""
281 | }
282 |
283 | func ToHardwareAddr(val interface{}) string {
284 | switch value := val.(type) {
285 | case string:
286 | tmp := strings.Split(value, " ")
287 | if len(tmp) == 6 {
288 | n := []string{}
289 | for _, v := range tmp {
290 | v1, er := strconv.ParseUint(v, 10, 64)
291 | if er != nil {
292 | return val.(string)
293 | }
294 | n = append(n, fmt.Sprintf("%x", v1))
295 | }
296 | return strings.Join(n, ":")
297 | } else {
298 | return val.(string)
299 | }
300 | case []byte:
301 | n := []string{}
302 | for _, v := range value {
303 | n = append(n, fmt.Sprintf("%x", v))
304 | }
305 | return strings.Join(n, ":")
306 | }
307 | return ""
308 |
309 | }
310 |
311 | func RmDupElement(arg []string) []string {
312 | tmp := map[string]int{}
313 | for _, el := range arg {
314 | if _, ok := tmp[el]; !ok {
315 | tmp[el] = 1
316 | }
317 | }
318 | var result []string
319 | for k := range tmp {
320 | result = append(result, k)
321 | }
322 | return result
323 | }
324 |
325 | // Final
326 | // {"a":"b", "b":"c", "c":"d"}, final("a") wil get "d"
327 | func Final(key string, data map[string]string) string {
328 | if v, ok := data[key]; ok {
329 | if key == v {
330 | return key
331 | }
332 | return Final(v, data)
333 | } else {
334 | return key
335 | }
336 | }
337 |
338 | //func macAddress(from string) string {
339 | // var tmp []string
340 | // for _, v1 := range strings.Split(from, " ") {
341 | // t, er := strconv.ParseUint(v1, 10, 8)
342 | // if er != nil {
343 | // break
344 | // }
345 | // tmp = append(tmp, fmt.Sprintf("%x", t))
346 | // }
347 | // if len(tmp) < 6 {
348 | // return from
349 | // }
350 | // tmp = tmp[len(tmp)-6:]
351 | // return strings.Join(tmp, ":")
352 | //}
353 |
354 | //func SnmpWalk(snmp *gsnmp.GoSNMP, action string, oids ...string) (res []*SnmpResp, err error) {
355 | // if len(oids) == 0 {
356 | // return res, fmt.Errorf("oid is none")
357 | // }
358 | // var result []gsnmp.SnmpPDU
359 | // if action == "get" {
360 | // r, er := snmp.Get(oids)
361 | // if er != nil {
362 | // err = er
363 | // return
364 | // }
365 | // result = r.Variables
366 | // } else {
367 | // result, err = snmp.BulkWalkAll(oids[0])
368 | // if err != nil {
369 | // return
370 | // }
371 | // }
372 | // for _, v := range result {
373 | // tmp := SnmpResp{}
374 | // tmp.Oid = v.Name
375 | // switch v.Type {
376 | // case gsnmp.IPAddress:
377 | // tmp.DataType = "string"
378 | // tmp.DataValue = fmt.Sprintf("%v", v.Value)
379 | // case gsnmp.OctetString:
380 | // tmp.DataType = "string"
381 | // value := ""
382 | // // HEX-STRING
383 | // if strings.Contains(strconv.Quote(string(v.Value.([]byte))), "\\x") {
384 | // for i := 0; i < len(v.Value.([]byte)); i++ {
385 | // value += fmt.Sprintf("%v", v.Value.([]byte)[i])
386 | // if i != (len(v.Value.([]byte)) - 1) {
387 | // value += " "
388 | // }
389 | // }
390 | // tmp.DataValue = value
391 | // } else {
392 | // value = string(v.Value.([]byte))
393 | // tmp.DataValue = value
394 | // }
395 | // default:
396 | // tmp.DataType = "integer"
397 | // tmp.DataValue = gsnmp.ToBigInt(v.Value)
398 | // //tmp.DataValue = fmt.Sprintf("%d", v.Value)
399 | // }
400 | // tmp.Raw = v.Value
401 | // res = append(res, &tmp)
402 | // }
403 | // return res, nil
404 | //
405 | //}
406 |
407 | func SnmpWalk(snmp *gsnmp.GoSNMP, action string, oids ...string) (res []*SnmpResp, err error) {
408 | if len(oids) == 0 {
409 | return res, fmt.Errorf("oid is none")
410 | }
411 | var result []gsnmp.SnmpPDU
412 | if action == "get" {
413 | r, er := snmp.Get(oids)
414 | if er != nil {
415 | err = er
416 | return
417 | }
418 | result = r.Variables
419 | } else if action == "first" {
420 | err = snmp.BulkWalk(oids[0], func(dataUnit gsnmp.SnmpPDU) error {
421 | if dataUnit.Value != nil {
422 | result = append(result, dataUnit)
423 | return fmt.Errorf("%s", "skip")
424 | }
425 | return nil
426 | })
427 | if err != nil {
428 | return
429 | }
430 | } else if action == "firstNotNone" {
431 | err = snmp.BulkWalk(oids[0], func(dataUnit gsnmp.SnmpPDU) error {
432 | if dataUnit.Value != nil && dataUnit.Type == gsnmp.OctetString && len(dataUnit.Value.([]byte)) != 0 {
433 | result = append(result, dataUnit)
434 | return fmt.Errorf("%s", "skip")
435 | }
436 | return nil
437 | })
438 | if err != nil {
439 | return
440 | }
441 | } else {
442 | result, err = snmp.BulkWalkAll(oids[0])
443 | if err != nil {
444 | return
445 | }
446 | }
447 | for _, v := range result {
448 | tmp := parseResult(v)
449 | tmp.Raw = v.Value
450 | res = append(res, &tmp)
451 | }
452 | return res, nil
453 | }
454 |
455 | func parseResult(v gsnmp.SnmpPDU) SnmpResp {
456 | tmp := SnmpResp{}
457 | tmp.Oid = v.Name
458 | switch v.Type {
459 | case gsnmp.IPAddress, gsnmp.ObjectIdentifier:
460 | tmp.DataType = "string"
461 | tmp.DataValue = fmt.Sprintf("%v", v.Value)
462 | case gsnmp.OctetString:
463 | tmp.DataType = "string"
464 | value := ""
465 | // HEX-STRING
466 | if strings.Contains(strconv.Quote(string(v.Value.([]byte))), "\\x") {
467 | for i := 0; i < len(v.Value.([]byte)); i++ {
468 | value += fmt.Sprintf("%v", v.Value.([]byte)[i])
469 | if i != (len(v.Value.([]byte)) - 1) {
470 | value += " "
471 | }
472 | }
473 | tmp.DataValue = value
474 | } else {
475 | value = string(v.Value.([]byte))
476 | tmp.DataValue = value
477 | }
478 | default:
479 | tmp.DataType = "integer"
480 | tmp.DataValue = gsnmp.ToBigInt(v.Value)
481 | //tmp.DataValue = fmt.Sprintf("%d", v.Value)
482 | }
483 | return tmp
484 | }
485 |
--------------------------------------------------------------------------------
/netTopology/internal/node_type.go:
--------------------------------------------------------------------------------
1 | package internal
2 |
3 | type NodeType int
4 |
5 | const (
6 | NTServer NodeType = iota
7 | NTSwitch
8 | NTRouter
9 | NTInternet
10 | NTOffline
11 | NTOther
12 | )
13 |
14 | func (nt NodeType) String() string {
15 | switch nt {
16 | case NTServer:
17 | return "server"
18 | case NTSwitch:
19 | return "switch"
20 | case NTRouter:
21 | return "router"
22 | case NTInternet:
23 | return "internet"
24 | case NTOffline:
25 | return "offline"
26 | default:
27 | return "other"
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/netTopology/internal/oids.go:
--------------------------------------------------------------------------------
1 | // Package netTopology /*
2 | package internal
3 |
4 | // Sys 设备信息, Get 信息
5 | type Sys struct {
6 | // SysName 主机名
7 | SysName string
8 | // SysDesc 产商信息
9 | SysDesc string
10 | // SysUpTimeInstance 监控时间
11 | SysUpTimeInstance string
12 | // SysLocation 设备位置
13 | SysLocation string
14 | }
15 |
16 | // Dot1dBridge mac地址表,即mac与端口的映射关系
17 | type Dot1dBridge struct {
18 | // Dot1dTpFdbAddress 桥接转发表, mac地址
19 | // 在这里，需要用到Community String Indexing，思科设备的转发表对每个vlan是不一样的。这时候共同体名为community@vlank号，
20 | // 如：public@1,public@2,public@3等,默认取public@1地的转发表。此时需在对每个vlan的转发表进行查找，直到找到ip对应的mac地址
21 | Dot1dTpFdbAddress string
22 | // Dot1dTpFdbPort 转发端口, 对应物理端口 Port Number
23 | Dot1dTpFdbPort string
24 | }
25 |
26 | type Dot1dStp struct {
27 | // Dot1dStpPortTable
28 | Dot1dStpPortTable string
29 | // Dot1dStpPortDesignatedBridge 对应端口所连接的另一台二层设备的BridgeID, 与dot1dBaseBridgeAddress对应
30 | Dot1dStpPortDesignatedBridge string
31 | // Dot1dStpPortDesignatedPort 该二层设备的某个端口连接了另一台二层设备的哪个桥接端口
32 | Dot1dStpPortDesignatedPort string
33 | // Dot1dStpRootCost 从该网桥到根的路径成本
34 | // 两台交换机之间的连接信息是由其值较大(离root较远)的那一台交换机产生的,另一台则以自身信息作为默认
35 | // 值越大，在生成树种的层级越低，两台交换机的连接关系由层级较低的交换机确定
36 | Dot1dStpRootCost string
37 | // Dot1dStpPort 二层代理设备各个端口的桥接端口号/包含生成树协议管理信息的端口的端口号
38 | Dot1dStpPort string
39 | // Dot1dStpPortState 表明端口的工作状态
40 | //1: disabled; 2:blocking(冗余链路); 3:listening; 4:learning; 5:forwarding; 6:broken
41 | // spanning-tree技术把冗余角色的备用线路设置成forwarding(5)以外的其他状态
42 | Dot1dStpPortState string
43 | }
44 |
45 | type Dot1dBase struct {
46 | // Dot1dBasePort 物理端口号
47 | Dot1dBasePort string
48 | // Dot1dBasePortIfIndex 端口对应的接口索引
49 | Dot1dBasePortIfIndex string
50 | // Dot1dBaseBridgeAddress 代表设备自身MAC地址 .1.3.6.1.2.1.17.1.1
51 | Dot1dBaseBridgeAddress string
52 | }
53 |
54 | // IpNetToMedia 该表是一张IP地址转换表，主要是将IP地址映射为物理地址;该表的索引是ipNetToMediaIfIndex和ipNetToMediaNetAddress
55 | // 通过其可以发现与该设备相连的其他设备的ip地址和物理地址的映射以及映射方式，从而发现一些设备的连接情况
56 | type IpNetToMedia struct {
57 | // ipNetToMediaIfIndex 表示此表项对应的有效接口的索引值, 该值所指定的接口与IF-MIB中的ifIndex值所指定接口相同
58 | IpNetToMediaIfIndex string
59 | // IpNetToMediaPhysAddress arp表. 表示依据媒介而定的物理地址, 即ip与mac地址的映射关系
60 | IpNetToMediaPhysAddress string
61 | // IpNetToMediaNetAddress arp表中 mac对应IP, 和本机在一个子网的ip。 表示这个依据媒介而定的物理地址对应的IP地址。
62 | IpNetToMediaNetAddress string
63 | // ipNetToMediaType other(1),invalid(2) dynamic(3):动态，即对端设备的ip;static(4):静态,代表本机的Ip
64 | IpNetToMediaType string
65 | }
66 |
67 | type Ip struct {
68 | //ipForwarding = ""
69 | // IpForwarding 是否具有路由功能， 1: 有路由功能 2: 无路由功能
70 | IpForwarding string
71 | // PrintMib 是否为打印机，如不为空，则为打印机
72 | PrintMib string
73 | }
74 |
75 | type IpRoute struct {
76 | // IpRouteDest 路由表(目标网络)
77 | IpRouteDest string
78 | // IpRouteIfIndex 路由表（出接口索引 唯一标识本地接口的索引值,通过该接口可以到达该路由的下一站）
79 | IpRouteIfIndex string
80 | // IpRouteNextHop 显示这条路由下一跳的IP地址. 当路由与广播媒介接口绑定时，该节点的值为接口上代理的IP地址
81 | IpRouteNextHop string
82 | // IpRouteType路由类型
83 | // 3(direct):直接路由,表明目标网络或者目标主机与该路由器直接相连
84 | // 4(indirect):间接路由,表明通往目的网络或者目的主机的的路径上还要经过其他路由器
85 | // 直连路由：直接连接到路由器端口的网段，该信息由路由器自动生成；
86 | // 非直连路由：不是直接连接到路由器端口的网段，此记录需要手动添加或使用动态路由生成。
87 | IpRouteType string
88 | IpRouteMask string
89 | }
90 |
91 | type If struct {
92 | // IfName interface 名称
93 | IfName string
94 | IfIndex string
95 | // 网卡描述
96 | IfDes string
97 | // IfHighSpeed 带宽限制
98 | IfHighSpeed string
99 | IfHCInOctets string
100 | IfHCOutOctets string
101 | IfInDiscards string
102 | IfOutDiscards string
103 |
104 | // IfOutUcastPkts 输出非广播包数
105 | IfOutUcastPkts string
106 | IfInUcastPkts string
107 | // ifOutNUcastPkts 广播包和多点发送包计数，速率可以识别广播风暴
108 | IfOutNUcastPkts string
109 | IfInNUcastPkts string
110 | IfOutErrors string
111 | IfInErrors string
112 | // IfAdminStatus 接口的配置状态; up(1),down(2),testing(3)
113 | IfAdminStatus string
114 | // IfOperStatus 接口的当前工作状态up(1),down(2),testing(3)
115 | IfOperStatus string
116 | }
117 |
118 | // IpAdEnt ipAddrTable 该表主要是用来保存IP地址信息，如IP地址、子网掩码等; 该表的索引是ipAdEntAddr。
119 | type IpAdEnt struct {
120 | // IpAdEntAddr 显示这个表项的地址信息所属的IP地址
121 | IpAdEntAddr string
122 | // IpAdEntIfIndex 唯一标识该表项所应用的接口的索引值
123 | IpAdEntIfIndex string
124 | // IpAdEntNetMask 显示该IP地址的子网掩码
125 | IpAdEntNetMask string
126 | }
127 |
128 | type EntPhys struct {
129 | // entPhysicalDescr 设备各模块描述
130 | EntPhysicalDescr string
131 | // EntPhysicalName 设备模块物理名称
132 | EntPhysicalName string
133 | // EntPhysicalSerialNum 设备各模块序列号
134 | EntPhysicalSerialNum string
135 | // EntPhysicalMfgName 设备各模块产商
136 | EntPhysicalMfgName string
137 | }
138 |
139 | type Neighbor struct {
140 | // Neighbor LLDP邻居信息 1.0.8802.1.1.2.1.4
141 | Neighbor string
142 | }
143 |
144 | type AtEntry struct {
145 | AtPhysAddress string
146 | }
147 |
148 | type OidLibrary struct {
149 | Sys
150 | Dot1dBridge
151 | Dot1dBase
152 | IpNetToMedia
153 | Ip
154 | IpRoute
155 | If
156 | IpAdEnt
157 | Dot1dStp
158 | EntPhys
159 | }
160 |
161 | var (
162 | OLibrary = &OidLibrary{
163 | Sys{
164 | SysName: ".1.3.6.1.2.1.1.5.",
165 | SysDesc: ".1.3.6.1.2.1.1.1",
166 | SysUpTimeInstance: ".1.3.6.1.2.1.1.3",
167 | SysLocation: ".1.3.6.1.2.1.1.6",
168 | },
169 | Dot1dBridge{
170 | Dot1dTpFdbAddress: ".1.3.6.1.2.1.17.4.3.1.1",
171 | Dot1dTpFdbPort: ".1.3.6.1.2.1.17.4.3.1.2",
172 | },
173 | Dot1dBase{
174 | Dot1dBaseBridgeAddress: ".1.3.6.1.2.1.17.1.1.0",
175 | Dot1dBasePort: ".1.3.6.1.2.1.17.1.4.1.1",
176 | Dot1dBasePortIfIndex: ".1.3.6.1.2.1.17.1.4.1.2",
177 | },
178 | IpNetToMedia{
179 | IpNetToMediaIfIndex: ".1.3.6.1.2.1.4.22.1.1",
180 | IpNetToMediaPhysAddress: ".1.3.6.1.2.1.4.22.1.2",
181 | IpNetToMediaNetAddress: ".1.3.6.1.2.1.4.22.1.3",
182 | IpNetToMediaType: ".1.3.6.1.2.1.4.22.1.4",
183 | },
184 | Ip{
185 | IpForwarding: ".1.3.6.1.2.1.4.1",
186 | PrintMib: ".1.3.6.1.2.1.43",
187 | },
188 | IpRoute{
189 | IpRouteDest: ".1.3.6.1.2.1.4.21.1.1",
190 | IpRouteIfIndex: ".1.3.6.1.2.1.4.21.1.2",
191 | IpRouteNextHop: ".1.3.6.1.2.1.4.21.1.7",
192 | IpRouteType: ".1.3.6.1.2.1.4.21.1.8",
193 | IpRouteMask: ".1.3.6.1.2.1.4.21.1.11",
194 | },
195 | If{
196 | IfName: ".1.3.6.1.2.1.31.1.1.1.1",
197 | IfIndex: ".1.3.6.1.2.1.2.2.1.1",
198 | IfDes: ".1.3.6.1.2.1.2.2.1.2",
199 | IfAdminStatus: ".1.3.6.1.2.1.2.2.1.7",
200 | IfOperStatus: ".1.3.6.1.2.1.2.2.1.8",
201 | IfInUcastPkts: ".1.3.6.1.2.1.2.2.1.11",
202 | IfInNUcastPkts: ".1.3.6.1.2.1.2.2.1.12",
203 | IfInDiscards: ".1.3.6.1.2.1.2.2.1.13",
204 | IfInErrors: ".1.3.6.1.2.1.2.2.1.14",
205 | IfOutUcastPkts: ".1.3.6.1.2.1.2.2.1.17",
206 | IfOutNUcastPkts: ".1.3.6.1.2.1.2.2.1.18",
207 | IfOutDiscards: ".1.3.6.1.2.1.2.2.1.19",
208 | IfOutErrors: ".1.3.6.1.2.1.2.2.1.20",
209 |
210 | IfHighSpeed: ".1.3.6.1.2.1.31.1.1.1.15",
211 | IfHCInOctets: ".1.3.6.1.2.1.31.1.1.1.6",
212 | IfHCOutOctets: ".1.3.6.1.2.1.31.1.1.1.10",
213 |
214 | //ifAdminStatus
215 | //
216 | //
217 | //
218 | //用于配置接口的状态（可读写）up(1),down(2),testing(3)（见表2）
219 | //
220 | //ifOperStatus
221 | //
222 | //
223 | //
224 | //提供接口的当前工作状态up(1),down(2),testing(3)
225 | //
226 | //（见表2）
227 | },
228 | IpAdEnt{
229 | IpAdEntAddr: ".1.3.6.1.2.1.4.20.1.1",
230 | IpAdEntIfIndex: ".1.3.6.1.2.1.4.20.1.2",
231 | IpAdEntNetMask: ".1.3.6.1.2.1.4.20.1.3",
232 | },
233 | Dot1dStp{
234 | Dot1dStpRootCost: ".1.3.6.1.2.1.17.2.6",
235 | Dot1dStpPortTable: ".1.3.6.1.2.1.17.2.15",
236 | Dot1dStpPort: ".1.3.6.1.2.1.17.2.15.1.1",
237 | Dot1dStpPortState: ".1.3.6.1.2.1.17.2.15.1.3",
238 | Dot1dStpPortDesignatedBridge: ".1.3.6.1.2.1.17.2.15.1.8",
239 | Dot1dStpPortDesignatedPort: ".1.3.6.1.2.1.17.2.15.1.9",
240 | },
241 | EntPhys{
242 | EntPhysicalDescr: ".1.3.6.1.2.1.47.1.1.1.1.2",
243 | EntPhysicalName: ".1.3.6.1.2.1.47.1.1.1.1.7",
244 | EntPhysicalSerialNum: ".1.3.6.1.2.1.47.1.1.1.1.11",
245 | EntPhysicalMfgName: ".1.3.6.1.2.1.47.1.1.1.1.12",
246 | },
247 | }
248 | )
249 |
--------------------------------------------------------------------------------
/netTopology/internal/types.go:
--------------------------------------------------------------------------------
1 | // Package internal /*
2 | package internal
3 |
4 | //import "serinus/pkg/server/models"
5 |
6 | type LinkValue struct {
7 | From string `json:"from"`
8 | To string `json:"to"`
9 | Key string `json:"key"`
10 | Value interface{} `json:"value"`
11 | Unit string `json:"unit"`
12 | Alias string `json:"alias"`
13 | Speed int `json:"speed"`
14 | Status interface{} `json:"status"`
15 | }
16 |
17 | //type LinkV1 struct {
18 | // // Name 自动生成，根据from,to的名称和type构成
19 | // LinkId string `json:"link_id"`
20 | // Node1 *models.LinkNode `json:"node1"`
21 | // Node2 *models.LinkNode `json:"node2"`
22 | // Values []*LinkValue `json:"values"`
23 | // Options interface{} `json:"options"`
24 | // Status string `json:"status"`
25 | //}
26 |
27 | type NodeStatus struct {
28 | Status string `json:"status"`
29 | Value string `json:"value"`
30 | Metric string `json:"metric"`
31 | Alias string `json:"alias"`
32 | }
33 |
34 | type Node struct {
35 | // Name must be unique
36 | Name string `json:"name"`
37 | // Alias
38 | Alias string `json:"alias"`
39 | // Type switch/router/server/internet/firewall/unknown
40 | Type string `json:"type"`
41 | // X location x
42 | X int `json:"x"`
43 | // Y location y
44 | Y int `json:"y"`
45 | // Ips 本机IP
46 | Ips []string `json:"ips"`
47 | // Locals 直连ip
48 | Locals []string `json:"locals"`
49 | // discoverIps 间接连接ip
50 | Options interface{} `json:"options"`
51 | Status string `json:"status"`
52 | IfIndexIps map[int64][]string `json:"if_index_ips"`
53 | DesIps map[string][]string `json:"des_ips"`
54 | }
55 |
56 | //type TopologyGraphV1 struct {
57 | // Name string `json:"name"`
58 | // Nodes []*Node `json:"nodes"`
59 | // Links []*LinkV1 `json:"links"`
60 | // Location bool `json:"location"`
61 | // Illustrate interface{} `json:"illustrate"`
62 | // StatusDetails map[string]map[string]map[string]*NodeStatus `json:"status_details"`
63 | //}
64 |
--------------------------------------------------------------------------------
/netTopology/internal/utils.go:
--------------------------------------------------------------------------------
1 | package internal
2 |
3 | import (
4 | "crypto/md5"
5 | "encoding/hex"
6 | "fmt"
7 | gsnmp "github.com/gosnmp/gosnmp"
8 | "github.com/patrickmn/go-cache"
9 | "github.com/sirupsen/logrus"
10 | "io"
11 | "sort"
12 | "strings"
13 | "time"
14 | )
15 |
16 | const (
17 | defaultCommunity = "public"
18 | defaultCommunityV2 = "publicv2"
19 | )
20 |
21 | var (
22 | TargetStatus = cache.New(time.Minute, time.Minute)
23 | CommunityMap = cache.New(time.Hour*24, time.Hour)
24 | SnmpVersionMap = cache.New(time.Hour*24, time.Hour)
25 | ResultCache = cache.New(time.Hour, time.Minute*15)
26 | IpNodeTypeMaps = cache.New(time.Hour*24*7, time.Hour)
27 | // IpIdent ip映射，当一个设备有多个ip时,统一为一个IP采集数据
28 | IpIdent = cache.New(time.Hour*24*7, time.Hour*24)
29 | )
30 |
31 | type SnmpResp struct {
32 | Domain string `json:"domain"`
33 | Oid string `json:"oid"`
34 | Key string `json:"key"`
35 | DataType string `json:"data_type"`
36 | DataValue interface{} `json:"data_value"`
37 | Raw interface{} `json:"raw"`
38 | }
39 |
40 | func NewSnmp(target string) (*gsnmp.GoSNMP, error) {
41 | if v, ok := TargetStatus.Get(target); ok {
42 | if !v.(bool) {
43 | return nil, fmt.Errorf("connect error")
44 | }
45 | }
46 | if !ValidInnerIp(target) {
47 | return nil, fmt.Errorf("invalid intranet ip for %s", target)
48 | }
49 | c := defaultCommunityV2
50 | if v, ok := CommunityMap.Get(target); ok {
51 | c = v.(string)
52 | }
53 | gVersion := gsnmp.Version2c
54 | if v, ok := SnmpVersionMap.Get(target); ok {
55 | switch v {
56 | case "1":
57 | gVersion = gsnmp.Version1
58 | case "3":
59 | // TODO
60 | gVersion = gsnmp.Version3
61 | }
62 | }
63 | s := &gsnmp.GoSNMP{
64 | Target: target, // eg:"172.18.0.2"
65 | Port: 161,
66 | Community: c,
67 | Version: gVersion,
68 | Timeout: time.Second * 3,
69 | Retries: 1,
70 | Transport: "udp",
71 | ExponentialTimeout: false,
72 | }
73 | er := s.Connect()
74 | defer func() {
75 | if er != nil {
76 | logrus.Warnf("try new snmp %s failed %v", target, er)
77 | } else {
78 | logrus.Infof("try new snmp %s success", target)
79 | }
80 | }()
81 | if er == nil {
82 | if _, er1 := SnmpWalk(s, "walk", OLibrary.IpForwarding); er1 != nil {
83 | er = er1
84 | }
85 | }
86 | if er != nil {
87 | s.Community = func(c string) string {
88 | if c == defaultCommunity {
89 | return defaultCommunityV2
90 | } else {
91 | return defaultCommunity
92 | }
93 | }(c)
94 | er = s.Connect()
95 | if er == nil {
96 | if _, er1 := SnmpWalk(s, "walk", OLibrary.IpForwarding); er1 != nil {
97 | er = er1
98 | }
99 | }
100 | if er != nil {
101 | TargetStatus.SetDefault(target, false)
102 | return s, er
103 | }
104 | }
105 | s.Timeout = time.Second * 15
106 | s.Retries = 1
107 | CommunityMap.SetDefault(target, s.Community)
108 | TargetStatus.SetDefault(target, true)
109 | return s, nil
110 | }
111 |
112 | func FetchItems(action, target string, oids []string, snmp *gsnmp.GoSNMP, args ...interface{}) (res []*SnmpResp, err error) {
113 | if len(oids) == 0 {
114 | return res, fmt.Errorf("%s", "oid is none")
115 | }
116 | key := fmt.Sprintf("%s-%s-%s", action, target, strings.Join(oids, "-"))
117 | if tmp, exist := ResultCache.Get(key); exist {
118 | return tmp.([]*SnmpResp), nil
119 | }
120 | if snmp == nil {
121 | snmp, err = NewSnmp(target)
122 | if err != nil {
123 | return
124 | }
125 | defer snmp.Conn.Close()
126 | }
127 |
128 | res, err = SnmpWalk(snmp, action, oids...)
129 | cacheTime := -2
130 | if len(args) > 0 {
131 | cacheTime = args[0].(int)
132 | }
133 | if err == nil {
134 | if cacheTime == -2 {
135 | if strings.HasSuffix(key, OLibrary.IpNetToMediaPhysAddress) {
136 | ResultCache.Set(key, res, time.Minute*30)
137 | } else {
138 | ResultCache.Set(key, res, time.Second*10)
139 | }
140 | } else if cacheTime > 0 {
141 | ResultCache.Set(key, res, time.Second*time.Duration(cacheTime))
142 | } else {
143 | ResultCache.SetDefault(key, res)
144 | }
145 | }
146 | return
147 | }
148 |
149 | func SnmpCheck(hostName string, oids ...string) (string, bool) {
150 | cli, err := NewSnmp(hostName)
151 | if err != nil {
152 | //fmt.Println("snmp check failed:", hostName, err)
153 | return "", false
154 | }
155 | cli.Timeout = time.Second
156 | defer cli.Conn.Close()
157 | if len(oids) == 0 {
158 | res, er := SnmpWalk(cli, "walk", OLibrary.IpForwarding)
159 | if er != nil {
160 | return "", false
161 | } else if len(res) > 0 {
162 | return fmt.Sprintf("%v", res[0].DataValue), true
163 | } else {
164 | return "", true
165 | }
166 | } else {
167 | _, er := cli.Get(oids)
168 | return "", er == nil
169 | }
170 | }
171 |
172 | func GetNodeType(node string) NodeType {
173 | if nt, ok := IpNodeTypeMaps.Get(node); ok {
174 | return nt.(NodeType)
175 | }
176 | nt := NTOther
177 | defer func() {
178 | IpNodeTypeMaps.SetDefault(node, nt)
179 | }()
180 | // 只有mac地址，无法通过snmp获取数据，暂且标记为server
181 | if len(strings.Split(node, ".")) != 4 {
182 | nt = NTServer
183 | return nt
184 | }
185 | if v, supportSnmp := SnmpCheck(node); supportSnmp {
186 | if v == "2" {
187 | nt = NTServer
188 | } else {
189 | if _, ok := SnmpCheck(node, fmt.Sprintf("%s.0.0.0.0", OLibrary.IpRouteIfIndex)); ok {
190 | nt = NTRouter
191 | } else if _, ok := SnmpCheck(node, OLibrary.Dot1dBaseBridgeAddress); ok {
192 | nt = NTSwitch
193 | }
194 | }
195 | } else {
196 | if PingHost(node) {
197 | nt = NTServer
198 | }
199 | }
200 | return nt
201 | }
202 |
203 | // IpIdentGet 针对非服务器，获取标识ip,
204 | func IpIdentGet(node string, snmp *gsnmp.GoSNMP) (ip string, ips []string, err error) {
205 | ips = []string{}
206 | if v, ok := IpIdent.Get(node); ok {
207 | return v.(string), ips, nil
208 | }
209 | if snmp == nil {
210 | snmp, er := NewSnmp(node)
211 | if er != nil {
212 | err = er
213 | return
214 | } else {
215 | defer snmp.Conn.Close()
216 | }
217 | }
218 | r0, er := FetchItems("walk", node, []string{OLibrary.IpAdEntAddr}, snmp)
219 | if er != nil {
220 | err = er
221 | return
222 | }
223 | sort.Slice(r0, func(i, j int) bool {
224 | return InetAtoi(r0[i].DataValue.(string)) < InetAtoi(r0[j].DataValue.(string))
225 | })
226 | for _, v := range r0 {
227 | if v.DataValue.(string) == "127.0.0.1" ||
228 | v.DataValue.(string) == "0.0.0.0" ||
229 | v.DataValue.(string) == "127.0.0.0" {
230 | continue
231 | }
232 | ips = append(ips, v.DataValue.(string))
233 | if _, er := NewSnmp(v.DataValue.(string)); er == nil {
234 | IpIdent.SetDefault(node, v.DataValue.(string))
235 | ip = v.DataValue.(string)
236 | break
237 | } else if PingHost(v.DataValue.(string)) {
238 | IpIdent.SetDefault(node, v.DataValue.(string))
239 | ip = v.DataValue.(string)
240 | continue
241 | }
242 | }
243 | return
244 | }
245 |
246 | func MD5(str string) string {
247 | h := md5.New()
248 | _, _ = io.WriteString(h, str)
249 | return hex.EncodeToString(h.Sum(nil))
250 | }
251 |
--------------------------------------------------------------------------------
/netTopology/lldp.md:
--------------------------------------------------------------------------------
1 | ### SNMP 获取LLDP信息前提
2 | - 目标开启LLDP功能
3 | ```
4 | 在系统视图下执行lldp enable命令全局使能LLDP功能，缺省情况下，接口下的LLDP功能与全局下的LLDP功能状态保持一致。
5 | ```
6 |
7 | - 开启MIB视图包含LLDP-MIB
8 |
9 | ```
10 | 执行snmp-agent mib-view included iso-view iso命令创建包含所有MIB节点的MIB视图iso-view。
11 | 执行snmp-agent community { read | write } community-name mib-view iso-view命令配置MIB视图iso-view为网管使用的MIB视图。
12 | 执行snmp-agent sys-info version all命令配置交换机启用所有SNMP版本。
13 | ```
--------------------------------------------------------------------------------
/netTopology/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "os"
5 | "os/signal"
6 | "syscall"
7 |
8 | "github.com/sirupsen/logrus"
9 |
10 | "netTopology/discover"
11 | . "netTopology/internal"
12 | )
13 |
14 | func init() {
15 | // add your snmp communities, default is "public" or "publicv2"
16 | communities := map[string]string{}
17 | for k, v := range communities {
18 | CommunityMap.SetDefault(k, v)
19 | }
20 | }
21 |
22 | func main() {
23 | logrus.Info("start...")
24 | term := make(chan os.Signal, 1)
25 | signal.Notify(term, os.Interrupt, syscall.SIGTERM)
26 | md := discover.MasterDiscover("master")
27 | discover.NetworkMonitorInit(term, &md)
28 | <-term
29 | logrus.Info("Received SIGTERM, exiting ...")
30 | }
31 |
--------------------------------------------------------------------------------
/secret/Pipfile:
--------------------------------------------------------------------------------
1 | [[source]]
2 | url = "https://pypi.org/simple"
3 | verify_ssl = true
4 | name = "pypi"
5 |
6 | [packages]
7 |
8 | [dev-packages]
9 |
10 | [requires]
11 | python_version = "2.7"
12 |
--------------------------------------------------------------------------------
/secret/README.md:
--------------------------------------------------------------------------------
1 | [English](README_en.md)
2 | ### 描述
3 | > 在我们的日常项目中，经常会遇得到一些敏感数据的存储问题，比如用户的密码，各种登录认证的token等等，
4 | > 如何安全的存储这些敏感信息十分重要。
5 |
6 | > 本工具是基于python实现的存储敏感数据方案，主要负责数据的加解密以及密钥维护等，
7 | > 包括inner和vault两个方法，你可以在项目中直接使用这些代码来进行敏感信息的处理，
8 | > 减少不必要的研究和开发。
9 |
10 | ### 两种方法
11 | #### inner
12 | > `inner.py` 脚本， 该方案最终使用中提供一个加密函数和一个解密函数函数，通过直接传递值进行加解密操作。
13 | > 用户在进行写入数据之前通过调用加密函数对数据加密，获取到加密数据以后，用户再根据自己场景进行存储。
14 |
15 | #### 使用示例
16 | ```python
17 | km = KeyManage()
18 |
19 | # init
20 | res1, ok1 = km.init()
21 | # Example
22 | t_plaintext = b"Hello, World!" # The plaintext to encrypt
23 | c = InnerCrypt()
24 | t_ciphertext, s1 = c.encrypt(t_plaintext)
25 | print("Ciphertext:", t_ciphertext)
26 | decrypted_plaintext, s2 = c.decrypt(t_ciphertext)
27 | print(decrypted_plaintext)
28 | print(s2)
29 | print("Decrypted plaintext:", decrypted_plaintext)
30 | print(".......")
31 | ```
32 |
33 | #### vault
34 | > 本功能使用前提是需要首先部署第三方密码管理软件vault或者已经有了vault的服务，
35 | > 将敏感信息直接存储在第三方密码管理软件上，当前程序只需要配置服务地址以及令牌，
36 | > 然后执行增删改查的方法即可。
37 |
38 | ##### 使用前提
39 | 1. 启动了vault服务，创建一个kv密码引擎，同时启动一个transit密码引擎，如创建一个mount_path为cmdb的kv,同事启动一个transit引擎；也可以选择使用执行enable_secrets_engine来自动创建。
40 | 2. 程序侧配置vault的地址，以及调用api的X-Vault-Token
41 |
42 | ##### 使用示例
43 | ```python
44 | _base_url = "http://localhost:8200"
45 | _token = "your token"
46 | # Example
47 | sdk = VaultClient(_base_url, _token, mount_path="cmdb")
48 |
49 | _data = {"key1": "value1", "key2": "value2", "key3": "value3"}
50 | _path = "test001"
51 | _data = sdk.update(_path, _data, overwrite=True, encrypt=True)
52 | _data = sdk.read(_path, decrypt=True)
53 | ```
54 |
55 | ### 其他使用场景
56 | > 如果您想在flask项目中使用它，可以参考开源项目[veops/cmdb](http://github.com/veops/cmdb)中
57 | > 关于密码部分的使用，包括flask command 和api中对相应的功能的调用
58 |
--------------------------------------------------------------------------------
/secret/README_en.md:
--------------------------------------------------------------------------------
1 | [中文](README.md)
2 | ### Description
3 | > It is crucial to securely store these sensitive information in our daily projects,
4 | > such as user passwords, various login authentication tokens, and so on. Ensuring
5 | > the secure storage of these sensitive information is of utmost importance.
6 |
7 | ### Methods
8 | #### inner
9 | > the `inner.py` script. The final solution provides an encryption function and
10 | > a decryption function, which allow users to perform encryption and decryption
11 | > operations by directly passing the values.
12 |
13 | > Before writing data, users can call the encryption function to encrypt the data.
14 | > After obtaining the encrypted data, users can store it according to their own scenarios.
15 |
16 | ##### Example
17 | ```python
18 | km = KeyManage()
19 |
20 | # init
21 | res1, ok1 = km.init()
22 | # Example
23 | t_plaintext = b"Hello, World!" # The plaintext to encrypt
24 | c = InnerCrypt()
25 | t_ciphertext, s1 = c.encrypt(t_plaintext)
26 | print("Ciphertext:", t_ciphertext)
27 | decrypted_plaintext, s2 = c.decrypt(t_ciphertext)
28 | print(decrypted_plaintext)
29 | print(s2)
30 | print("Decrypted plaintext:", decrypted_plaintext)
31 | print(".......")
32 | ```
33 |
34 | #### vault
35 | > The prerequisite for using this feature is to deploy a third-party
36 | > password management software like Vault or have an existing Vault service.
37 | > Sensitive information is stored directly in the third-party password
38 | > management software.
39 |
40 | > The current program only needs to configure the service address and token,
41 | > and then execute the methods for CRUD operations.
42 |
43 | ##### Prerequisites for usage
44 | 1. If you have started the Vault service, create a KV secret engine with a mount path of "cmdb" and start a Transit secret engine. Alternatively, you can choose to use the enable_secrets_engine command to automatically create them.
45 | 2. On the program side, configure the Vault address and provide the X-Vault-Token for API calls.
46 |
47 | ##### Example
48 | ```python
49 | _base_url = "http://localhost:8200"
50 | _token = "your token"
51 | # Example
52 | sdk = VaultClient(_base_url, _token, mount_path="cmdb")
53 |
54 | _data = {"key1": "value1", "key2": "value2", "key3": "value3"}
55 | _path = "test001"
56 | _data = sdk.update(_path, _data, overwrite=True, encrypt=True)
57 | _data = sdk.read(_path, decrypt=True)
58 | ```
59 |
60 | ### Other Use cases
61 | > If you want to use it in a Flask project, you can refer to the usage of the password
62 | > section in the open-source project [veops/cmdb](http://github.com/veops/cmdb),
63 | > including the use of Flask commands and API calls for the corresponding functionality.
64 |
--------------------------------------------------------------------------------
/secret/inner.py:
--------------------------------------------------------------------------------
1 | from base64 import b64encode, b64decode
2 | from colorama import Back
3 | from colorama import Fore
4 | from colorama import Style
5 | from colorama import init as colorama_init
6 | from cryptography.hazmat.backends import default_backend
7 | from cryptography.hazmat.primitives.ciphers import Cipher
8 | from cryptography.hazmat.primitives.ciphers import algorithms
9 | from cryptography.hazmat.primitives.ciphers import modes
10 | from cryptography.hazmat.primitives.ciphers.aead import AESGCM
11 | from cryptography.hazmat.primitives import hashes
12 | from cryptography.hazmat.primitives import padding
13 | from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
14 |
15 | import os
16 | import secrets
17 | import sys
18 | from Cryptodome.Protocol.SecretSharing import Shamir
19 |
20 | # global_root_key just for test here
21 | global_root_key = ""
22 | global_encrypt_key = ""
23 | global_iv_length = 16
24 | global_key_shares = 5 # Number of generated key shares
25 | global_key_threshold = 3 # Minimum number of shares required to rebuild the key
26 | global_shares = []
27 |
28 | backend_root_key_name = "root_key"
29 | backend_encrypt_key_name = "encrypt_key"
30 | backend_root_key_salt_name = "root_key_salt"
31 | backend_encrypt_key_salt_name = "encrypt_key_salt"
32 | success = "success"
33 | seal_status = True
34 | cache = {}
35 |
36 |
37 | def string_to_bytes(value):
38 | if isinstance(value, bytes):
39 | return value
40 | if sys.version_info.major == 2:
41 | byte_string = value
42 | else:
43 | byte_string = value.encode("utf-8")
44 | return byte_string
45 |
46 |
47 | class cache_backend:
48 | def __init__(self):
49 | pass
50 |
51 | @classmethod
52 | def get(cls, key):
53 | global cache
54 | return cache.get(key)
55 |
56 | @classmethod
57 | def add(cls, key, value):
58 | cache[key] = value
59 | return success, True
60 |
61 | @classmethod
62 | def update(cls, key, value):
63 | cache[key] = value
64 | return success, True
65 |
66 |
67 | class Backend:
68 | def __init__(self, backend=None):
69 | if not backend:
70 | self.backend = cache_backend
71 | else:
72 | self.backend = backend
73 |
74 | def get(self, key):
75 | return self.backend.get(key)
76 |
77 | def add(self, key, value):
78 | return self.backend.add(key, value)
79 |
80 | def update(self, key, value):
81 | return self.backend.update(key, value)
82 |
83 |
84 | class KeyManage:
85 |
86 | def __init__(self, trigger=None):
87 | self.trigger = trigger
88 | pass
89 |
90 | @classmethod
91 | def hash_root_key(cls, value):
92 | algorithm = hashes.SHA256()
93 | salt = Backend().get(backend_root_key_salt_name)
94 | if not salt:
95 | salt = secrets.token_hex(16)
96 | msg, ok = Backend().add(backend_root_key_salt_name, salt)
97 | if not ok:
98 | return msg, ok
99 | kdf = PBKDF2HMAC(
100 | algorithm=algorithm,
101 | length=32,
102 | salt=string_to_bytes(salt),
103 | iterations=100000,
104 | )
105 | key = kdf.derive(string_to_bytes(value))
106 | return b64encode(key).decode('utf-8'), True
107 |
108 | @classmethod
109 | def generate_encrypt_key(cls, key):
110 | algorithm = hashes.SHA256()
111 | salt = Backend().get(backend_encrypt_key_salt_name)
112 | if not salt:
113 | salt = secrets.token_hex(32)
114 | kdf = PBKDF2HMAC(
115 | algorithm=algorithm,
116 | length=32,
117 | salt=string_to_bytes(salt),
118 | iterations=100000,
119 | backend=default_backend()
120 | )
121 | key = kdf.derive(string_to_bytes(key))
122 | msg, ok = Backend().add(backend_encrypt_key_salt_name, salt)
123 | if ok:
124 | return b64encode(key).decode('utf-8'), ok
125 | else:
126 | return msg, ok
127 |
128 | @classmethod
129 | def generate_keys(cls, secret):
130 | shares = Shamir.split(global_key_threshold, global_key_shares, secret)
131 | new_shares = []
132 | for share in shares:
133 | t = [i for i in share[1]] + [ord(i) for i in "{:0>2}".format(share[0])]
134 | new_shares.append(b64encode(bytes(t)))
135 | return new_shares
136 |
137 | def auth_root_secret(self, root_key):
138 | root_key_hash, ok = self.hash_root_key(b64encode(root_key))
139 | if not ok:
140 | return {
141 | "message": root_key_hash,
142 | "status": "failed"
143 | }
144 | backend_root_key_hash = Backend().get(backend_root_key_name)
145 | if not backend_root_key_hash:
146 | return {
147 | "message": "should init firstly",
148 | "status": "failed"
149 | }
150 | elif backend_root_key_hash != root_key_hash:
151 | return {
152 | "message": "invalid unseal keys",
153 | "status": "failed"
154 | }
155 | encrypt_key_aes = Backend().get(backend_encrypt_key_name)
156 | if not encrypt_key_aes:
157 | return {
158 | "message": "encrypt key is empty",
159 | "status": "failed"
160 | }
161 | global global_encrypt_key
162 | global global_root_key
163 | global global_shares
164 | global_encrypt_key = InnerCrypt.aes_decrypt(string_to_bytes(root_key), encrypt_key_aes)
165 | global_root_key = root_key
166 | global_shares = []
167 | return {
168 | "message": success,
169 | "status": success
170 | }
171 |
172 | def unseal(self, key):
173 | if not self.is_seal():
174 | return {
175 | "message": "current status is unseal, skip",
176 | "status": "skip"
177 | }
178 | global global_shares, global_root_key, global_encrypt_key
179 | t = [i for i in b64decode(key)]
180 | global_shares.append((int("".join([chr(i) for i in t[-2:]])), bytes(t[:-2])))
181 | if len(global_shares) >= global_key_threshold:
182 | recovered_secret = Shamir.combine(global_shares[:global_key_threshold])
183 | return self.auth_root_secret(recovered_secret)
184 | else:
185 | return {
186 | "Process": "{0}/{1}".format(len(global_shares), global_key_threshold),
187 | "message": "waiting for inputting other unseal key",
188 | "status": "waiting"
189 | }
190 |
191 | def generate_unseal_keys(self):
192 | info = Backend().get(backend_root_key_name)
193 | if info:
194 | return "already exist", [], False
195 | secret = AESGCM.generate_key(128)
196 | shares = self.generate_keys(secret)
197 | return b64encode(secret), shares, True
198 |
199 | def init(self):
200 | """
201 | init the master key, unseal key and store in backend
202 | :return:
203 | """
204 | root_key = Backend().get(backend_root_key_name)
205 | if root_key:
206 | return {"message": "already init, skip"}, False
207 | else:
208 | root_key, shares, status = self.generate_unseal_keys()
209 | if not status:
210 | return {"message": root_key}, False
211 | # hash root key and store in backend
212 | root_key_hash, ok = self.hash_root_key(root_key)
213 | if not ok:
214 | return {"message": root_key_hash}, False
215 | msg, ok = Backend().add(backend_root_key_name, root_key_hash)
216 | if not ok:
217 | return {"message": msg}, False
218 | # generate encrypt key from root_key and store in backend
219 | encrypt_key, ok = self.generate_encrypt_key(root_key)
220 | if not ok:
221 | return {"message": encrypt_key}
222 | encrypt_key_aes = InnerCrypt.aes_encrypt(root_key, encrypt_key)
223 | msg, ok = Backend().add(backend_encrypt_key_name, encrypt_key_aes)
224 | if not ok:
225 | return {"message": msg}, False
226 | #
227 | global global_root_key, global_encrypt_key
228 | global_root_key = root_key
229 | global_encrypt_key = encrypt_key
230 | self.print_token(shares, root_token=root_key)
231 | return {"message": "OK",
232 | "details": {
233 | "root_token": root_key,
234 | "seal_tokens": shares,
235 | }}, True
236 |
237 | def auto_unseal(self):
238 | if not self.trigger:
239 | return "trigger config is empty, skip", False
240 | if self.trigger.startswith("http"):
241 | pass
242 | # TODO
243 | elif len(self.trigger.strip()) == 24:
244 | res = self.auth_root_secret(self.trigger)
245 | if res.get("status") == success:
246 | return success, True
247 | else:
248 | return res.get("message"), False
249 | else:
250 | return "trigger config is invalid, skip", False
251 |
252 | def seal(self, root_key):
253 | root_key_hash, ok = self.hash_root_key(b64encode(root_key))
254 | if not ok:
255 | return root_key_hash, ok
256 | backend_root_key_hash = Backend().get(backend_root_key_name)
257 | if not backend_root_key_hash:
258 | return "not init, seal skip", False
259 | else:
260 | global global_root_key
261 | global global_encrypt_key
262 | global_root_key = ""
263 | global_encrypt_key = ""
264 | return success, True
265 |
266 | @classmethod
267 | def is_seal(cls):
268 | """
269 | If there is no initialization or the root key is inconsistent, it is considered to be in a sealed state.
270 | :return:
271 | """
272 | root_key = Backend().get(backend_root_key_name)
273 | if root_key == "" or root_key != global_root_key:
274 | return "invalid root key", True
275 | return "", False
276 |
277 | @classmethod
278 | def print_token(cls, shares, root_token):
279 | """
280 | data: {"message": "OK",
281 | "details": {
282 | "root_token": root_key,
283 | "seal_tokens": shares,
284 | }}
285 | """
286 | colorama_init()
287 | print(Style.BRIGHT, "Please be sure to store the Unseal Key in a secure location and avoid losing it."
288 | " The Unseal Key is required to unseal the system every time when it restarts."
289 | " Successful unsealing is necessary to enable the password feature.")
290 | for i, v in enumerate(shares):
291 | print(
292 | Fore.RED + Back.LIGHTRED_EX + "unseal token " + str(i + 1) + ": " + v.decode("utf-8") + Style.RESET_ALL)
293 | print()
294 | print(Fore.GREEN + "root token: " + root_token.decode("utf-8") + Style.RESET_ALL)
295 |
296 |
297 | class InnerCrypt:
298 | def __init__(self, trigger=None):
299 | self.encrypt_key = b64decode(global_encrypt_key.encode("utf-8"))
300 |
301 | def encrypt(self, plaintext):
302 | """
303 | encrypt method contain aes currently
304 | """
305 | return self.aes_encrypt(self.encrypt_key, plaintext)
306 |
307 | def decrypt(self, ciphertext):
308 | """
309 | decrypt method contain aes currently
310 | """
311 | return self.aes_decrypt(self.encrypt_key, ciphertext)
312 |
313 | @classmethod
314 | def aes_encrypt(cls, key, plaintext):
315 | if isinstance(plaintext, str):
316 | plaintext = string_to_bytes(plaintext)
317 | iv = os.urandom(global_iv_length)
318 | try:
319 | cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())
320 | encryptor = cipher.encryptor()
321 | v_padder = padding.PKCS7(algorithms.AES.block_size).padder()
322 | padded_plaintext = v_padder.update(plaintext) + v_padder.finalize()
323 | ciphertext = encryptor.update(padded_plaintext) + encryptor.finalize()
324 | return b64encode(iv + ciphertext).decode("utf-8"), True
325 | except Exception as e:
326 | return str(e), False
327 |
328 | @classmethod
329 | def aes_decrypt(cls, key, ciphertext):
330 | try:
331 | s = b64decode(ciphertext.encode("utf-8"))
332 | iv = s[:global_iv_length]
333 | ciphertext = s[global_iv_length:]
334 | cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())
335 | decrypter = cipher.decryptor()
336 | decrypted_padded_plaintext = decrypter.update(ciphertext) + decrypter.finalize()
337 | unpadder = padding.PKCS7(algorithms.AES.block_size).unpadder()
338 | plaintext = unpadder.update(decrypted_padded_plaintext) + unpadder.finalize()
339 | return plaintext.decode('utf-8'), True
340 | except Exception as e:
341 | return str(e), False
342 |
343 |
344 |
345 | if __name__ == "__main__":
346 |
347 | km = KeyManage()
348 |
349 | # init
350 | res1, ok1 = km.init()
351 | # Example
352 | t_plaintext = b"Hello, World!" # The plaintext to encrypt
353 | c = InnerCrypt()
354 | t_ciphertext, s1 = c.encrypt(t_plaintext)
355 | print("-"*30)
356 | print("Ciphertext:", t_ciphertext)
357 | decrypted_plaintext, s2 = c.decrypt(t_ciphertext)
358 | print("Decrypted plaintext:", decrypted_plaintext)
359 |
--------------------------------------------------------------------------------
/secret/vault.py:
--------------------------------------------------------------------------------
1 | from base64 import b64decode
2 | from base64 import b64encode
3 |
4 | import hvac
5 |
6 |
7 | class VaultClient:
8 | def __init__(self, base_url, token, mount_path='cmdb'):
9 | self.client = hvac.Client(url=base_url, token=token)
10 | self.mount_path = mount_path
11 |
12 | def create_app_role(self, role_name, policies):
13 | resp = self.client.create_approle(role_name, policies=policies)
14 |
15 | return resp == 200
16 |
17 | def delete_app_role(self, role_name):
18 | resp = self.client.delete_approle(role_name)
19 |
20 | return resp == 204
21 |
22 | def update_app_role_policies(self, role_name, policies):
23 | resp = self.client.update_approle_role(role_name, policies=policies)
24 |
25 | return resp == 204
26 |
27 | def get_app_role(self, role_name):
28 | resp = self.client.get_approle(role_name)
29 | resp.json()
30 | if resp.status_code == 200:
31 | return resp.json
32 | else:
33 | return {}
34 |
35 | def enable_secrets_engine(self):
36 | resp = self.client.sys.enable_secrets_engine('kv', path=self.mount_path)
37 | resp_01 = self.client.sys.enable_secrets_engine('transit')
38 |
39 | if resp.status_code == 200 and resp_01.status_code == 200:
40 | return resp.json
41 | else:
42 | return {}
43 |
44 | def encrypt(self, plaintext):
45 | response = self.client.secrets.transit.encrypt_data(name='transit-key', plaintext=plaintext)
46 | ciphertext = response['data']['ciphertext']
47 |
48 | return ciphertext
49 |
50 | # decrypt data
51 | def decrypt(self, ciphertext):
52 | response = self.client.secrets.transit.decrypt_data(name='transit-key', ciphertext=ciphertext)
53 | plaintext = response['data']['plaintext']
54 |
55 | return plaintext
56 |
57 | def write(self, path, data, encrypt=None):
58 | if encrypt:
59 | for k, v in data.items():
60 | data[k] = self.encrypt(self.encode_base64(v))
61 | response = self.client.secrets.kv.v2.create_or_update_secret(
62 | path=path,
63 | secret=data,
64 | mount_point=self.mount_path
65 | )
66 |
67 | return response
68 |
69 | # read data
70 | def read(self, path, decrypt=True):
71 | try:
72 | response = self.client.secrets.kv.v2.read_secret_version(
73 | path=path, raise_on_deleted_version=False, mount_point=self.mount_path
74 | )
75 | except Exception as e:
76 | return str(e), False
77 | data = response['data']['data']
78 | if decrypt:
79 | try:
80 | for k, v in data.items():
81 | data[k] = self.decode_base64(self.decrypt(v))
82 | except:
83 | return data, True
84 |
85 | return data, True
86 |
87 | # update data
88 | def update(self, path, data, overwrite=True, encrypt=True):
89 | if encrypt:
90 | for k, v in data.items():
91 | data[k] = self.encrypt(self.encode_base64(v))
92 | if overwrite:
93 | response = self.client.secrets.kv.v2.create_or_update_secret(
94 | path=path,
95 | secret=data,
96 | mount_point=self.mount_path
97 | )
98 | else:
99 | response = self.client.secrets.kv.v2.patch(path=path, secret=data, mount_point=self.mount_path)
100 |
101 | return response
102 |
103 | # delete data
104 | def delete(self, path):
105 | response = self.client.secrets.kv.v2.delete_metadata_and_all_versions(
106 | path=path,
107 | mount_point=self.mount_path
108 | )
109 |
110 | return response
111 |
112 | # Base64 encode
113 | @classmethod
114 | def encode_base64(cls, data):
115 | encoded_bytes = b64encode(data.encode())
116 | encoded_string = encoded_bytes.decode()
117 |
118 | return encoded_string
119 |
120 | # Base64 decode
121 | @classmethod
122 | def decode_base64(cls, encoded_string):
123 | decoded_bytes = b64decode(encoded_string)
124 | decoded_string = decoded_bytes.decode()
125 |
126 | return decoded_string
127 |
128 |
129 | if __name__ == "__main__":
130 | _base_url = "http://localhost:8300"
131 | _token = "hvs.LiCunLUksihFxCLlHsWvHTWo"
132 |
133 | _path = "test001"
134 | # Example
135 | sdk = VaultClient(_base_url, _token, mount_path="test")
136 | sdk.enable_secrets_engine()
137 | # sdk.enable_secrets_engine()
138 | _data = {"key1": "value1", "key2": "value2", "key3": "value3"}
139 | _data = sdk.update(_path, _data, overwrite=True, encrypt=True)
140 | print(_data)
141 | _data = sdk.read(_path, decrypt=True)
142 | print(_data)
143 |
144 |
--------------------------------------------------------------------------------
/zabbix2tsdb/go.mod:
--------------------------------------------------------------------------------
1 | module zabbix
2 |
3 | go 1.17
4 |
5 | require github.com/spf13/pflag v1.0.5
6 |
--------------------------------------------------------------------------------
/zabbix2tsdb/go.sum:
--------------------------------------------------------------------------------
1 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
2 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
3 |
--------------------------------------------------------------------------------
/zabbix2tsdb/readme.md:
--------------------------------------------------------------------------------
1 | tools for converting zabbix data to tsdb format, the new output contain metric,labels,value and timestamp
2 | # output
3 | ## influxdb
4 | ```shell
5 | sys_cpu_idle,core=cpu1,env=prod {V}=12.34 1690892492000000000
6 | ```
7 |
8 | ## prometheus
9 | > output as follows
10 | ```shell
11 | system_cpu_idle{core="cpu1",env="prod"} 12.34 1690892492000000000
12 | ```
13 |
14 | # how to use
15 | ```shell
16 | -k, --acceptKeys strings 需要同步的Key,通配符匹配(*). 例如:如需要同步'system.'开头的key_,则配置'system.*' (default [system.*])
17 | -a, --address string zabbix api请求地址。为防止带宽占用以及网络延迟问题，我们强烈推荐将节点选择在与zabbix api
18 | 服务在同一台机器上。如 http://127.0.0.1:8080/api_jsonrpc.php,可以写完整地址,也可以直接省去后缀(api_jsonrpc.php),
19 | 如写http://127.0.0.1:8080
20 | -v, --apiVersion string api版本 (default "2.0")
21 | -c, --cluster string zabbix集群名称, 当采集多个zabbix集群,且不同集群存在相同的主机名(ip),可以避免数据混乱 (default "0")
22 | -f, --dataFormat string data format that you want to convert to, you can choose 'prometheus' or 'influxdb', (default "influxdb")
23 | -g, --groups strings 需要同步的group分组 (default [Linux servers,Zabbix servers,Virtual machines])
24 | -i, --interval int 同步时间间隔,单位秒. 防止对zabbix服务器造成太大压力,系统允许的最小时间间隔为30秒 (default 60)
25 | -p, --password string 允许通过api访问数据的用户对应的密码,推荐使用环境变量 (default "zabbix")
26 | -u, --user string 允许通过api访问数据的用户名, 推荐使用环境变量 (default "Admin")
27 | ```
--------------------------------------------------------------------------------
/zabbix2tsdb/zabbix.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bytes"
5 | "context"
6 | "encoding/json"
7 | "fmt"
8 | "io"
9 | "io/ioutil"
10 | "math"
11 | "net/http"
12 | "os"
13 | "os/signal"
14 | "regexp"
15 | "strings"
16 | "sync"
17 | "syscall"
18 | "time"
19 |
20 | "github.com/spf13/pflag"
21 | )
22 |
23 | const (
24 | MaxIdleConnections int = 20
25 | RequestTimeout int = 30
26 | ApiSuffix = "api_jsonrpc.php"
27 | defaultKey = "{V}"
28 | )
29 |
30 | func Init() {
31 | KeyPattern = regexp.MustCompile(`^([0-9a-zA-Z_.]*)\\[(.*)\\]$`)
32 | ReplMetricPattern = regexp.MustCompile(`([^a-zA-Z0-9:_]+?)`)
33 | ReplParamsPattern = regexp.MustCompile(`([," =]+?)`)
34 | //for _, v := range zabbixConfig.AccetpKeys{
35 | // if p, er := regexp.Compile(v); er == nil{
36 | // AcceptKeysPattern = append(AcceptKeysPattern, p)
37 | // }
38 | //}
39 | zabbixConfig.Cluster = ReplParamsPattern.ReplaceAllString(zabbixConfig.Cluster, "_")
40 | address := zabbixConfig.Address
41 | if !strings.HasSuffix(address, ApiSuffix) {
42 | zabbixConfig.Address = fmt.Sprintf("%s/%s", strings.TrimSuffix(address, "/"), ApiSuffix)
43 | }
44 | zApi = NewZabbixClient()
45 | signal.Notify(signals, os.Interrupt, syscall.SIGTERM)
46 | }
47 |
48 | type Host struct {
49 | Locker sync.RWMutex
50 | Ids []string
51 | }
52 |
53 | type Config struct {
54 | Groups map[string]struct{}
55 | Cluster string
56 | Address string
57 | User string
58 | Password string
59 | ApiVersion string
60 | AccetpKeys []string
61 | Interval int64
62 | DataFormat string
63 | }
64 |
65 | var (
66 | zabbixConfig = Config{Groups: map[string]struct{}{}}
67 | zApi *ZabbixApi
68 | signals = make(chan os.Signal, 1)
69 | // hostIdHost 通过item.get获取到hostId之后，需要查找对应的host
70 | hostIdHost sync.Map
71 | // globalHostIds 之所以使用map，是因为多个groupId可能包含相同的hostId
72 | //globalHostIds sync.Map
73 | hosts = Host{Locker: sync.RWMutex{}}
74 | groupNameId sync.Map
75 |
76 | // KeyPattern key_中提取metric,params
77 | KeyPattern *regexp.Regexp
78 | // ReplMetricPattern metric名称转换
79 | ReplMetricPattern *regexp.Regexp
80 | // ReplParamsPattern 用来转换params,hostName
81 | ReplParamsPattern *regexp.Regexp
82 | AcceptKeysPattern []*regexp.Regexp
83 |
84 | ConnectError = fmt.Errorf("connectError")
85 |
86 | // hostQueryBatch 批量查询host采集项时，控制一次性查询量,防止数据过多
87 | hostQueryBatch int = 150
88 | )
89 |
90 | func (h *Host) Set(hostIds map[string]struct{}) {
91 | tmp := make(map[string]struct{})
92 | h.Locker.Lock()
93 | defer h.Locker.Unlock()
94 | // delete expired hostId
95 | for i := 0; i < len(h.Ids); i++ {
96 | if _, ok := hostIds[h.Ids[i]]; !ok {
97 | h.Ids = append(h.Ids[:i], h.Ids[i+1:]...)
98 | i--
99 | } else {
100 | tmp[h.Ids[i]] = struct{}{}
101 | }
102 | }
103 | // add host
104 | for hostId := range hostIds {
105 | if _, ok := tmp[hostId]; !ok {
106 | h.Ids = append(h.Ids, hostId)
107 | }
108 | }
109 | }
110 |
111 | // createHTTPClient for connection re-use
112 | func createHTTPClient() *http.Client {
113 | client := &http.Client{
114 | Transport: &http.Transport{
115 | MaxConnsPerHost: MaxIdleConnections,
116 | },
117 | Timeout: time.Duration(RequestTimeout) * time.Second,
118 | }
119 | return client
120 | }
121 |
122 | type ZabbixApi struct {
123 | User string
124 | Password string
125 | Address string
126 | ApiVersion string
127 | Auth string
128 | Client *http.Client
129 | }
130 |
131 | func NewZabbixClient() *ZabbixApi {
132 | c := createHTTPClient()
133 | return &ZabbixApi{
134 | User: zabbixConfig.User,
135 | Password: zabbixConfig.Password,
136 | Address: zabbixConfig.Address,
137 | ApiVersion: zabbixConfig.ApiVersion,
138 | Client: c,
139 | }
140 | }
141 |
142 | func (z *ZabbixApi) Login() error {
143 | r, er := json.Marshal(map[string]interface{}{
144 | "jsonrpc": z.ApiVersion,
145 | "method": "user.login",
146 | "params": map[string]string{
147 | "user": z.User,
148 | "password": z.Password,
149 | },
150 | "id": 1,
151 | })
152 | if er != nil {
153 | return er
154 | }
155 | body := bytes.NewBuffer(r)
156 | req, er := http.NewRequest("GET", z.Address, body)
157 | if er != nil {
158 | return er
159 | }
160 | req.Header.Set("Content-Type", "application/json-rpc")
161 | resp, er := z.Client.Do(req)
162 | if er != nil {
163 | return ConnectError
164 | }
165 | defer resp.Body.Close()
166 | result, er := io.ReadAll(resp.Body)
167 | if er != nil {
168 | return er
169 | }
170 | respData := make(map[string]interface{})
171 | er = json.Unmarshal(result, &respData)
172 | if er != nil {
173 | return er
174 | }
175 | if auth, ok := respData["result"]; ok {
176 | z.Auth = auth.(string)
177 | }
178 | if z.Auth == "" {
179 | return fmt.Errorf("获取认证秘钥为空")
180 | }
181 | return nil
182 | }
183 |
184 | func (z *ZabbixApi) GroupIds() error {
185 | r, _ := json.Marshal(map[string]interface{}{
186 | "jsonrpc": z.ApiVersion,
187 | "method": "hostgroup.get",
188 | "params": map[string]interface{}{
189 | "output": []string{"name", "groupid"},
190 | },
191 | "auth": z.Auth,
192 | "id": 1,
193 | })
194 | body := bytes.NewBuffer(r)
195 | req, er := http.NewRequest("GET", z.Address, body)
196 | if er != nil {
197 | return er
198 | }
199 | req.Header.Set("Content-Type", "application/json-rpc")
200 | resp, er := z.Client.Do(req)
201 | if er != nil {
202 | return ConnectError
203 | }
204 | defer resp.Body.Close()
205 | result, er := io.ReadAll(resp.Body)
206 | if er != nil {
207 | return er
208 | }
209 | respData := make(map[string]interface{})
210 | er = json.Unmarshal(result, &respData)
211 | if er != nil {
212 | return er
213 | }
214 | if result, ok := respData["result"]; ok {
215 | for _, v := range result.([]interface{}) {
216 | groupNameId.Store(v.(map[string]interface{})["name"].(string),
217 | v.(map[string]interface{})["groupid"].(string))
218 | }
219 | }
220 | return nil
221 | }
222 |
223 | func (z *ZabbixApi) HostIds(groupIds []string) error {
224 | r, er := json.Marshal(map[string]interface{}{
225 | "jsonrpc": z.ApiVersion,
226 | "method": "host.get",
227 | "params": map[string]interface{}{
228 | "output": []string{"hostid", "host"},
229 | //"output": "extend",
230 | "groupids": groupIds,
231 | },
232 | "auth": z.Auth,
233 | "id": 1,
234 | })
235 | if er != nil {
236 | return er
237 | }
238 | body := bytes.NewBuffer(r)
239 | req, er := http.NewRequest("GET", z.Address, body)
240 | if er != nil {
241 | return er
242 | }
243 | req.Header.Set("Content-Type", "application/json-rpc")
244 | resp, er := z.Client.Do(req)
245 | if er != nil {
246 | return er
247 | }
248 | defer resp.Body.Close()
249 | result, er := ioutil.ReadAll(resp.Body)
250 | if er != nil {
251 | return er
252 | }
253 | respData := make(map[string]interface{})
254 | er = json.Unmarshal(result, &respData)
255 | if er != nil {
256 | return er
257 | }
258 | tmp := map[string]struct{}{}
259 | if result, ok := respData["result"]; ok {
260 | for _, v := range result.([]interface{}) {
261 | if hostId, ok := v.(map[string]interface{})["hostid"]; ok {
262 | tmp[hostId.(string)] = struct{}{}
263 | if host, ok := v.(map[string]interface{})["host"]; ok {
264 | if _, ok := hostIdHost.Load(host); !ok {
265 | hostIdHost.Store(hostId, host)
266 | }
267 | }
268 | }
269 | }
270 | }
271 | hosts.Set(tmp)
272 | return nil
273 | }
274 |
275 | func (z *ZabbixApi) SmartItems(hostIds []string) {
276 | var hostIdsGroups [][]string
277 | groupNumber := int(math.Ceil(float64(len(hostIds)) / float64(hostQueryBatch)))
278 | for i := 0; i < groupNumber; i++ {
279 | startIndex := i * hostQueryBatch
280 | endIndex := (i + 1) * hostQueryBatch
281 | if endIndex > len(hostIds) {
282 | endIndex = len(hostIds)
283 | }
284 | hostIdsGroups = append(hostIdsGroups, hostIds[startIndex:endIndex])
285 | }
286 | for _, subHostIds := range hostIdsGroups {
287 | er := z.Items(subHostIds)
288 | if er != nil {
289 | if er == ConnectError {
290 | _ = z.Login()
291 | }
292 | }
293 | }
294 |
295 | }
296 |
297 | func (z *ZabbixApi) Items(hostIds []string) error {
298 | r, er := json.Marshal(map[string]interface{}{
299 | "jsonrpc": z.ApiVersion,
300 | "method": "item.get",
301 | "params": map[string]interface{}{
302 | //"output": []string{"hostid"},
303 | "output": []string{"key_", "hostid", "lastvalue", "lastclock", "value_type"},
304 | "hostids": hostIds,
305 | "search": map[string]interface{}{"key_": zabbixConfig.AccetpKeys},
306 | "searchWildcardsEnabled": true,
307 | "searchByAny": true,
308 | },
309 | "auth": z.Auth,
310 | "id": 1,
311 | })
312 | if er != nil {
313 | return er
314 | }
315 | body := bytes.NewBuffer(r)
316 | req, er := http.NewRequest("GET", z.Address, body)
317 | if er != nil {
318 | return er
319 | }
320 | req.Header.Set("Content-Type", "application/json-rpc")
321 | resp, er := z.Client.Do(req)
322 | if er != nil {
323 | return ConnectError
324 | }
325 | defer resp.Body.Close()
326 | result, er := io.ReadAll(resp.Body)
327 | if er != nil {
328 | return er
329 | }
330 | respData := make(map[string]interface{})
331 | er = json.Unmarshal(result, &respData)
332 | if er != nil {
333 | return er
334 | }
335 | if result, ok := respData["result"]; ok {
336 | for _, v := range result.([]interface{}) {
337 | er := processMetric(v.(map[string]interface{}))
338 | if er != nil {
339 | _, _ = fmt.Fprintln(os.Stderr, er.Error())
340 | }
341 | }
342 | }
343 | return nil
344 | }
345 |
346 | //func isAcceptedKey(key string) bool {
347 | // for _, p := range AcceptKeysPattern {
348 | // if p.MatchString(key) {
349 | // return true
350 | // }
351 | // }
352 | // return false
353 | //}
354 |
355 | func processMetric(item map[string]interface{}) error {
356 | dataStr := ""
357 | // only numeric/float values can convert to prometheus
358 | //dataStr := "%s,c=%s,__endpoint__=%s,params=%s {val}=%s %s000000000"
359 | if item["value_type"] == "0" || item["value_type"].(string) == "3" {
360 | if _, ok := item["key_"]; !ok {
361 | return fmt.Errorf("no key_")
362 | }
363 | // step 1: metric
364 | //if !isAcceptedKey(item["key_"].(string)){
365 | // return nil
366 | //}
367 | res := KeyPattern.FindStringSubmatch(item["key_"].(string))
368 | if len(res) > 1 {
369 | dataStr = ReplMetricPattern.ReplaceAllString(res[1], "_")
370 | } else {
371 | dataStr = ReplMetricPattern.ReplaceAllString(item["key_"].(string), "_")
372 | }
373 | // step 2 tags: zabbix cluster
374 | dataStr = fmt.Sprintf("%s,c=%s", dataStr, zabbixConfig.Cluster)
375 | // step3 tags: hostName
376 | if hostName, ok := hostIdHost.Load(item["hostid"].(string)); ok {
377 | dataStr = fmt.Sprintf("%s,__endpoint__=%s",
378 | dataStr, ReplParamsPattern.ReplaceAllString(hostName.(string), "_"))
379 | }
380 | // step 4 tags: params
381 | if len(res) >= 2 && res[0] != "" {
382 | dataStr = fmt.Sprintf("%s,p=%s", dataStr, ReplParamsPattern.ReplaceAllString(res[2], "\\$1"))
383 | }
384 | // step5 处理值
385 | if _, ok := item["lastvalue"]; !ok {
386 | return nil
387 | }
388 | dataStr = fmt.Sprintf("%s %s=%v", dataStr, defaultKey, item["lastvalue"])
389 | // step5 处理时间
390 | if _, ok := item["lastclock"]; ok && len(item["lastclock"].(string)) == 10 {
391 | dataStr = fmt.Sprintf(" %s %v000000000", dataStr, item["lastclock"])
392 | } else {
393 | return nil
394 | }
395 | }
396 | if dataStr != "" {
397 | if zabbixConfig.DataFormat == "prometheus" {
398 | s := convertInfluxToPrometheus(dataStr)
399 | for _, v := range s {
400 | fmt.Println(v)
401 | }
402 | } else {
403 | fmt.Println(dataStr)
404 | }
405 | }
406 | return nil
407 | }
408 |
409 | func convertInfluxToPrometheus(influxData string) []string {
410 | var data []string
411 | splitData := strings.Split(influxData, " ")
412 | measurementTagSet := strings.Split(splitData[0], ",")
413 | measurement := measurementTagSet[0]
414 | tagSet := measurementTagSet[1:]
415 |
416 | fieldSet := strings.Split(splitData[1], ",")
417 | var timestamp string
418 | if len(splitData) == 3 {
419 | timestamp = splitData[2]
420 | }
421 |
422 | labelSet := make([]string, len(tagSet))
423 | var t []string
424 | for i, tag := range tagSet {
425 | t = strings.SplitN(tag, "=", 2)
426 | if len(t) == 2 {
427 | labelSet[i] = fmt.Sprintf("%s=\"%s\"", t[0], t[1])
428 | }
429 | }
430 | for _, val := range fieldSet {
431 | t = strings.SplitN(val, "=", 2)
432 | if len(t) == 2 {
433 | if t[0] != defaultKey {
434 | measurement = fmt.Sprintf(`%s_%s`, measurement, t[0])
435 | }
436 | data = append(data, fmt.Sprintf(`%s{%s} %s %s`, measurement, strings.Join(labelSet, ","), t[1], timestamp))
437 | }
438 | }
439 | return data
440 | }
441 |
442 | func configParse() {
443 | //var groups []string
444 | pflag.StringVarP(&zabbixConfig.Cluster, "cluster", "c", "0",
445 | "zabbix集群名称, 当采集多个zabbix集群,且不同集群存在相同的主机名(ip),可以避免数据混乱")
446 | pflag.StringVarP(&zabbixConfig.Address, "address", "a", "",
447 | fmt.Sprintf(`zabbix api请求地址。为防止带宽占用以及网络延迟问题，我们强烈推荐将节点选择在与zabbix api
448 | 服务在同一台机器上。如 http://127.0.0.1:8080/%[1]s,可以写完整地址，也可以直接省去后缀(%[1]s)，
449 | 如写http://127.0.0.1:8080`, ApiSuffix))
450 | pflag.StringVarP(&zabbixConfig.User, "user", "u", "Admin", "允许通过api访问数据的用户名, 推荐使用环境变量")
451 | pflag.StringVarP(&zabbixConfig.Password, "password", "p", "zabbix", "允许通过api访问数据的用户对应的密码,推荐使用环境变量")
452 | groups := pflag.StringSliceP("groups", "g",
453 | []string{"Linux servers", "Zabbix servers", "Virtual machines"}, "需要同步的group分组")
454 | pflag.StringVarP(&zabbixConfig.ApiVersion, "apiVersion", "v", "2.0", "api版本")
455 | //pflag.StringSliceVarP(&zabbixConfig.AccetpKeys, "acceptKeys", "k",
456 | // []string{"system\\..*"}, "需要同步的Key,正则匹配")
457 | pflag.StringSliceVarP(&zabbixConfig.AccetpKeys, "acceptKeys", "k",
458 | []string{"system.*"}, "需要同步的Key,通配符匹配(*). 例如:如需要同步'system.'开头的key_,则配置'system.*'")
459 | pflag.Int64VarP(&zabbixConfig.Interval, "interval", "i", int64(60),
460 | "同步时间间隔,单位秒. 防止对zabbix服务器造成太大压力,系统允许的最小时间间隔为30秒")
461 | pflag.StringVarP(&zabbixConfig.DataFormat, "dataFormat", "f", "influxdb",
462 | "data format that you want to convert to, you can choose 'prometheus' or 'influxdb', default is influxdb")
463 | pflag.Parse()
464 | if zabbixConfig.Address == "" {
465 | _, _ = fmt.Fprintln(os.Stderr, "address 不能为空,我们推荐防止带宽占用，将节点选择在与zabbix api 服务在同一台机器上")
466 | os.Exit(0)
467 | }
468 | if len(*groups) > 0 {
469 | for _, group := range *groups {
470 | zabbixConfig.Groups[group] = struct{}{}
471 | }
472 | }
473 | if zabbixConfig.Interval < 30 {
474 | zabbixConfig.Interval = 30
475 | }
476 | if s := os.Getenv("password"); s != "" {
477 | zabbixConfig.Password = s
478 | }
479 | if s := os.Getenv("user"); s != "" {
480 | zabbixConfig.User = s
481 | }
482 | }
483 |
484 | func updateGroupAndHost() {
485 | er := zApi.GroupIds()
486 | if er != nil {
487 | //_, _ = fmt.Fprintln(os.Stderr, fmt.Sprintf("update groups failed:%s", er))
488 | _, _ = fmt.Fprintf(os.Stderr, "update groups failed:%s\n", er)
489 | // login again
490 | er := zApi.Login()
491 | if er != nil {
492 | _, _ = fmt.Fprintf(os.Stderr, "login failed:%s\n", er)
493 | }
494 | }
495 | groupIds := []string{}
496 | for groupName := range zabbixConfig.Groups {
497 | if groupId, ok := groupNameId.Load(groupName); ok {
498 | groupIds = append(groupIds, groupId.(string))
499 | }
500 | }
501 | if len(groupIds) == 0 {
502 | return
503 | }
504 | er = zApi.HostIds(groupIds)
505 | if er != nil {
506 | _, _ = fmt.Fprintf(os.Stderr, "update hostid failed:%s\n", er)
507 | // login again
508 | er := zApi.Login()
509 | if er != nil {
510 | _, _ = fmt.Fprintf(os.Stderr, "login failed:%s\n", er)
511 | }
512 | }
513 | }
514 |
515 | func GetItems() {
516 | if len(hosts.Ids) == 0 {
517 | return
518 | }
519 | zApi.SmartItems(hosts.Ids)
520 | }
521 |
522 | func Loop(ctx context.Context) {
523 | updateGroupAndHost()
524 | GetItems()
525 | ticker := time.NewTicker(time.Minute * 5)
526 | itemTicker := time.NewTicker(time.Second * time.Duration(zabbixConfig.Interval))
527 | for {
528 | select {
529 | case <-ticker.C:
530 | updateGroupAndHost()
531 | case <-itemTicker.C:
532 | GetItems()
533 | case <-signals:
534 | return
535 | case <-ctx.Done():
536 | return
537 | }
538 | }
539 | }
540 |
541 | func main() {
542 | configParse()
543 | Init()
544 | er := zApi.Login()
545 | if er != nil {
546 | _, _ = fmt.Fprintf(os.Stderr, "login failed:%s\n", er)
547 | os.Exit(1)
548 | }
549 | Loop(context.Background())
550 | }
551 |
--------------------------------------------------------------------------------