├── .idea
    ├── misc.xml
    ├── modules.xml
    ├── mysql_audit_me.iml
    ├── vcs.xml
    └── workspace.xml
├── README.md
├── lib
    ├── __init__.py
    ├── db.py
    ├── log.py
    ├── mysql_protocol.py
    └── packet_op.py
├── log
    └── 123.txt
└── tcp_audit.py


/.idea/misc.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="JavaScriptSettings">
4 |     <option name="languageLevel" value="ES6" />
5 |   </component>
6 |   <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6" project-jdk-type="Python SDK" />
7 | </project>


--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectModuleManager">
4 |     <modules>
5 |       <module fileurl="file://$PROJECT_DIR$/.idea/mysql_audit_me.iml" filepath="$PROJECT_DIR$/.idea/mysql_audit_me.iml" />
6 |       <module fileurl="file://$USER_HOME$/py_test/redis_audit/.idea/redis_audit.iml" filepath="$USER_HOME$/py_test/redis_audit/.idea/redis_audit.iml" />
7 |     </modules>
8 |   </component>
9 | </project>


--------------------------------------------------------------------------------
/.idea/mysql_audit_me.iml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <module type="PYTHON_MODULE" version="4">
 3 |   <component name="NewModuleRootManager">
 4 |     <content url="file://$MODULE_DIR$" />
 5 |     <orderEntry type="inheritedJdk" />
 6 |     <orderEntry type="sourceFolder" forTests="false" />
 7 |   </component>
 8 |   <component name="TestRunnerService">
 9 |     <option name="projectConfiguration" value="Nosetests" />
10 |     <option name="PROJECT_TEST_RUNNER" value="Nosetests" />
11 |   </component>
12 | </module>


--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="VcsDirectoryMappings">
4 |     <mapping directory="$PROJECT_DIR$" vcs="Git" />
5 |   </component>
6 | </project>


--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project version="4">
  3 |   <component name="ChangeListManager">
  4 |     <list default="true" id="917c670b-0773-48ca-bdd1-70e62a939640" name="Default Changelist" comment="">
  5 |       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
  6 |     </list>
  7 |     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
  8 |     <option name="SHOW_DIALOG" value="false" />
  9 |     <option name="HIGHLIGHT_CONFLICTS" value="true" />
 10 |     <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
 11 |     <option name="LAST_RESOLUTION" value="IGNORE" />
 12 |   </component>
 13 |   <component name="FUSProjectUsageTrigger">
 14 |     <session id="1929403949">
 15 |       <usages-collector id="statistics.lifecycle.project">
 16 |         <counts>
 17 |           <entry key="project.closed" value="3" />
 18 |           <entry key="project.open.time.0" value="3" />
 19 |           <entry key="project.opened" value="3" />
 20 |         </counts>
 21 |       </usages-collector>
 22 |       <usages-collector id="statistics.file.extensions.open">
 23 |         <counts>
 24 |           <entry key="py" value="3" />
 25 |         </counts>
 26 |       </usages-collector>
 27 |       <usages-collector id="statistics.file.types.open">
 28 |         <counts>
 29 |           <entry key="Python" value="3" />
 30 |         </counts>
 31 |       </usages-collector>
 32 |       <usages-collector id="statistics.file.extensions.edit">
 33 |         <counts>
 34 |           <entry key="py" value="28" />
 35 |         </counts>
 36 |       </usages-collector>
 37 |       <usages-collector id="statistics.file.types.edit">
 38 |         <counts>
 39 |           <entry key="Python" value="28" />
 40 |         </counts>
 41 |       </usages-collector>
 42 |     </session>
 43 |   </component>
 44 |   <component name="FileEditorManager">
 45 |     <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
 46 |       <file pinned="false" current-in-tab="false">
 47 |         <entry file="file://$PROJECT_DIR$/lib/packet_op.py">
 48 |           <provider selected="true" editor-type-id="text-editor">
 49 |             <state relative-caret-position="391">
 50 |               <caret line="215" column="64" selection-start-line="215" selection-start-column="64" selection-end-line="215" selection-end-column="64" />
 51 |               <folding>
 52 |                 <element signature="e#79#106#0" expanded="true" />
 53 |               </folding>
 54 |             </state>
 55 |           </provider>
 56 |         </entry>
 57 |       </file>
 58 |       <file pinned="false" current-in-tab="false">
 59 |         <entry file="file://$PROJECT_DIR$/lib/mysql_protocol.py">
 60 |           <provider selected="true" editor-type-id="text-editor">
 61 |             <state relative-caret-position="-3" />
 62 |           </provider>
 63 |         </entry>
 64 |       </file>
 65 |       <file pinned="false" current-in-tab="true">
 66 |         <entry file="file://$PROJECT_DIR$/lib/log.py">
 67 |           <provider selected="true" editor-type-id="text-editor" />
 68 |         </entry>
 69 |       </file>
 70 |     </leaf>
 71 |   </component>
 72 |   <component name="FindInProjectRecents">
 73 |     <findStrings>
 74 |       <find>self._logging.info</find>
 75 |     </findStrings>
 76 |   </component>
 77 |   <component name="Git.Settings">
 78 |     <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
 79 |   </component>
 80 |   <component name="IdeDocumentHistory">
 81 |     <option name="CHANGED_PATHS">
 82 |       <list>
 83 |         <option value="$PROJECT_DIR$/lib/packet_op.py" />
 84 |       </list>
 85 |     </option>
 86 |   </component>
 87 |   <component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
 88 |   <component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
 89 |   <component name="JsGulpfileManager">
 90 |     <detection-done>true</detection-done>
 91 |     <sorting>DEFINITION_ORDER</sorting>
 92 |   </component>
 93 |   <component name="ProjectFrameBounds" fullScreen="true">
 94 |     <option name="width" value="1792" />
 95 |     <option name="height" value="1120" />
 96 |   </component>
 97 |   <component name="ProjectView">
 98 |     <navigator proportions="" version="1">
 99 |       <foldersAlwaysOnTop value="true" />
100 |     </navigator>
101 |     <panes>
102 |       <pane id="ProjectPane">
103 |         <subPane>
104 |           <expand>
105 |             <path>
106 |               <item name="mysql_audit_me" type="b2602c69:ProjectViewProjectNode" />
107 |               <item name="mysql_audit_me" type="462c0819:PsiDirectoryNode" />
108 |             </path>
109 |             <path>
110 |               <item name="mysql_audit_me" type="b2602c69:ProjectViewProjectNode" />
111 |               <item name="mysql_audit_me" type="462c0819:PsiDirectoryNode" />
112 |               <item name="lib" type="462c0819:PsiDirectoryNode" />
113 |             </path>
114 |           </expand>
115 |           <select />
116 |         </subPane>
117 |       </pane>
118 |       <pane id="Scope" />
119 |     </panes>
120 |   </component>
121 |   <component name="PropertiesComponent">
122 |     <property name="WebServerToolWindowFactoryState" value="false" />
123 |     <property name="last_opened_file_path" value="$PROJECT_DIR$" />
124 |   </component>
125 |   <component name="RunDashboard">
126 |     <option name="ruleStates">
127 |       <list>
128 |         <RuleState>
129 |           <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
130 |         </RuleState>
131 |         <RuleState>
132 |           <option name="name" value="StatusDashboardGroupingRule" />
133 |         </RuleState>
134 |       </list>
135 |     </option>
136 |   </component>
137 |   <component name="SvnConfiguration">
138 |     <configuration />
139 |   </component>
140 |   <component name="TaskManager">
141 |     <task active="true" id="Default" summary="Default task">
142 |       <changelist id="917c670b-0773-48ca-bdd1-70e62a939640" name="Default Changelist" comment="" />
143 |       <created>1586312571945</created>
144 |       <option name="number" value="Default" />
145 |       <option name="presentableId" value="Default" />
146 |       <updated>1586312571945</updated>
147 |     </task>
148 |     <servers />
149 |   </component>
150 |   <component name="ToolWindowManager">
151 |     <frame x="0" y="0" width="1792" height="1120" extended-state="0" />
152 |     <editor active="true" />
153 |     <layout>
154 |       <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.25542858" />
155 |       <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
156 |       <window_info id="Favorites" order="2" side_tool="true" />
157 |       <window_info anchor="bottom" id="Message" order="0" />
158 |       <window_info anchor="bottom" id="Find" order="1" />
159 |       <window_info anchor="bottom" id="Run" order="2" />
160 |       <window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
161 |       <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
162 |       <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
163 |       <window_info anchor="bottom" id="TODO" order="6" />
164 |       <window_info anchor="bottom" id="Docker" order="7" show_stripe_button="false" />
165 |       <window_info anchor="bottom" id="Version Control" order="8" show_stripe_button="false" />
166 |       <window_info anchor="bottom" id="Database Changes" order="9" show_stripe_button="false" />
167 |       <window_info anchor="bottom" id="Event Log" order="10" side_tool="true" />
168 |       <window_info anchor="bottom" id="Terminal" order="11" />
169 |       <window_info anchor="bottom" id="Python Console" order="12" />
170 |       <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
171 |       <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
172 |       <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
173 |       <window_info anchor="right" id="SciView" order="3" />
174 |       <window_info anchor="right" id="Database" order="4" />
175 |     </layout>
176 |   </component>
177 |   <component name="TypeScriptGeneratedFilesManager">
178 |     <option name="version" value="1" />
179 |   </component>
180 |   <component name="VcsContentAnnotationSettings">
181 |     <option name="myLimit" value="2678400000" />
182 |   </component>
183 |   <component name="editorHistoryManager">
184 |     <entry file="file://$PROJECT_DIR$/lib/packet_op.py">
185 |       <provider selected="true" editor-type-id="text-editor">
186 |         <state relative-caret-position="391">
187 |           <caret line="215" column="64" selection-start-line="215" selection-start-column="64" selection-end-line="215" selection-end-column="64" />
188 |           <folding>
189 |             <element signature="e#79#106#0" expanded="true" />
190 |           </folding>
191 |         </state>
192 |       </provider>
193 |     </entry>
194 |     <entry file="file://$PROJECT_DIR$/lib/mysql_protocol.py">
195 |       <provider selected="true" editor-type-id="text-editor">
196 |         <state relative-caret-position="-3" />
197 |       </provider>
198 |     </entry>
199 |     <entry file="file://$PROJECT_DIR$/lib/log.py">
200 |       <provider selected="true" editor-type-id="text-editor" />
201 |     </entry>
202 |   </component>
203 | </project>


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | 
 2 | # mysql_audit  
 3 |   
 4 | 该小工具通过实时获取数据包的方式，分析并解析出请求的sql语句、执行用户、执行状态和执行时间，insert和update语句对数据做了格式化操作，update语句的where条件保留了数据，select、delete语句未做格式化，可以放于应用端、中间件层、mysql服务器上，在不影响mysql本身性能的情况下获取所有语句的基本情况，可做审计作用，兼容py2、py3，mysql5.7/8.0测试正常，5.6及以下版本未进行测试  ，可直接写入clickhouse
 5 |   
 6 |   
 7 |   
 8 | # 实现原理  
 9 |   
10 |  1. 通过pypcap获取数据包，利用dpkt进行解包  
11 |  2. 通过mysql协议发包回报过程组装session  
12 |  3. 执行时间利用session开始结束时间进行计算(pypcap在py2中返回的时间戳只精确到十毫秒，所以在代码中使用当前时间做计算，会存在偏差)  
13 |  4. 执行操作的用户名获取方式分为两种，如果是新建连接通过解包获取，如果是已经存在的长连接，通过后端数据库中processlist获取(前提是工具所获取的数据流后端就是mysql才能使用)  
14 |   
15 | ## 执行方式：  
16 |   
17 | python tcp_audit.py -h 可以获取参数介绍  
18 |   
19 | ## 示例：  
20 |   
21 |  ### 比如我在中间件层对后面3306端口的数据流进行监听：  
22 | 
23 |      python tcp_audit.py -e eth0 -p 3306 -t src -u username -P password 
24 | 
25 |  ### 如果是对本地端口进行监听，比如我们中间件层端口为6001：   
26 |    
27 | 
28 |     `python tcp_audit.py -e eth0 -p 6001 -t des` 
29 | 
30 |   （这里未提供用户名密码，因为中间件改变了来源信息，而我监听的是应用到中间件这层的数据流，所以不能直接获取链接所使用的用户，只能使用默认的解包获取）  
31 |   
32 | ## 获取内容打印如下：  
33 |   
34 | 
35 |      2019-08-06 08:52:22,984  INFO  log.py : INFO  source_host: 10.1.11.59 source_port: 59272 destination_host: 10.1.1.46 destination_port: 3306 user_name: test01 sql: INSERT INTO proxy_heart_beat.tb_heartbeat (p_id, p_ts) VALUES('?', '?') values: None execute_time:0.0001  status:#42000INSERT, UPDATE command denied to user 'test01'@'10.1.11.59' for table 'tb_heartbeat'  
36 | 
37 | ## 特别提醒：  
38 |   
39 |  1. 如果数据流非常大，该工具会使用不小的cpu时间  
40 |  2. 日志保存在log目录，且默认10分钟切割一次，保留一个小时的数据，如有需要可自行修改log.py中的配置  
41 |  3. 默认以日志的方式记录，如果指定了--ckhost参数将直接存入clickhouse，需首先在clickhouse中创建指定的库表：
42 | 
43 |     CREATE table mysql_audit.mysql_audit_info(   source_host String,   source_port UInt64,   destination_host String,   destination_port UInt64,   user_name String,   db String, sql String,   reponse_value String,   execute_time Float64,   response_status String,   event_date DateTime) ENGINE=MergeTree()   PARTITION BY toYYYYMMDD(event_date)   ORDER BY (source_host, source_port, event_date)   TTL event_date + INTERVAL 5 DAY   SETTINGS index_granularity=8192,enable_mixed_granularity_parts=1;
44 | 
45 |   
46 | ## 需求的包：  
47 |   
48 | dpkt、psutil、pypcap、pymysql 、clickhouse-driver
49 |   
50 | 


--------------------------------------------------------------------------------
/lib/__init__.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- encoding: utf-8 -*-
3 | '''
4 | @author: Great God
5 | '''


--------------------------------------------------------------------------------
/lib/db.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- encoding: utf-8 -*-
 3 | '''
 4 | @author: xiao cai niao
 5 | '''
 6 | import pymysql,traceback
 7 | 
 8 | class db:
 9 |     def __init__(self,**kwargs):
10 |         self.host = kwargs['host']
11 |         self.port = kwargs['port']
12 |         self.user = kwargs['user']
13 |         self.passwd = kwargs['passwd']
14 | 
15 |     def get(self,host,port):
16 |         try:
17 |             self.conn = pymysql.connect(host=self.host,port=self.port,user=self.user,passwd=self.passwd,cursorclass=pymysql.cursors.DictCursor)
18 |             self.cur = self.conn.cursor()
19 |             sql = 'select `user`,`db` from information_schema.processlist where host=%s;'
20 |             self.cur.execute(sql,'{}:{}'.format(host,port))
21 |             result = self.cur.fetchall()
22 |             if result:
23 |                 return result[0]['user'],result[0]['db']
24 |             else:
25 |                 return None,None
26 |         except:
27 |             print(traceback.format_exc())
28 |             return None,None
29 | 
30 |     def close(self):
31 |         try:
32 |             self.cur.close()
33 |             self.conn.close()
34 |         except:
35 |             pass


--------------------------------------------------------------------------------
/lib/log.py:
--------------------------------------------------------------------------------
 1 | # -*- encoding: utf-8 -*-
 2 | '''
 3 | @author: xiao cai niao
 4 | '''
 5 | import logging
 6 | import logging.handlers
 7 | 
 8 | 
 9 | class Logging:
10 |     def __init__(self):
11 |         self.logger = logging.getLogger("tcp_audit")
12 |         self.logger.setLevel(logging.DEBUG)
13 |         self.rf_handler = logging.handlers.TimedRotatingFileHandler(filename="log/tcp_audit.log", when='M', interval=10, \
14 |                                                                     backupCount=6)
15 |         self.rf_handler.setFormatter(
16 |             logging.Formatter("%(asctime)s  %(levelname)s  %(filename)s : %(levelname)s  %(message)s"))
17 | 
18 |         self.logger.addHandler(self.rf_handler)
19 | 
20 | 
21 |     def info(self, msg):
22 |         self.logger.info(msg)
23 | 
24 |     def warning(self, msg):
25 |         self.logger.warning(msg)
26 | 
27 |     def error(self, msg):
28 |         self.logger.error(msg)
29 | 
30 |     def debug(self, msg):
31 |         self.logger.debug(msg)
32 | 
33 |     def close(self):
34 |         self.logger.removeHandler(self.rf_handler)
35 | 


--------------------------------------------------------------------------------
/lib/mysql_protocol.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- encoding: utf-8 -*-
  3 | '''
  4 | @author: xiao cai niao
  5 | '''
  6 | import struct,sys
  7 | 
  8 | class mysql_packet(object):
  9 |     def __init__(self, **kwargs):
 10 |         self.data = None
 11 |         self.offset = 0
 12 |         self._ip = kwargs['_ip']
 13 |         self._type = kwargs['_type']    #src : This machine is the initiator  des: This machine is the receiver
 14 | 
 15 |         self.client_packet_type = {
 16 |             0x03: self.COM_QUERY,
 17 |             0x01: self.COM_QUIT,
 18 |             0x02: self.COM_INIT_DB,
 19 |             0x04: self.COM_FIELD_LIST,
 20 |             0x07: self.COM_PREFRESH,
 21 |             0x08: self.COM_STATISTICS,
 22 |             0x0A: self.COM_PROCESS_INFO,
 23 |             0x0C: self.COM_PROCESS_KILL,
 24 |             0x0D: self.COM_DEBUG,
 25 |             0x0E: self.COM_PING,
 26 |             0x11: self.COM_CHANGE_USER,
 27 |             0x1F: self.COM_RESET_CONNECTION,
 28 |             0x1A: self.COM_SET_OPTION,
 29 |             0x16: self.COM_STMT_PREPARE,
 30 |             0x17: self.COM_STMT_EXECUTE,
 31 |             0x19: self.COM_STMT_CLOSE,
 32 |             0x1A: self.COM_STMT_RESET,
 33 |             0x18: self.COM_STMT_SEND_LONG_DATA
 34 |         }
 35 | 
 36 |     def COM_QUERY(self):
 37 |         """
 38 |         Type	    Name	Description
 39 |         int<1>	    command	0x03: COM_QUERY
 40 |         string<EOF>	query	the text of the SQL query to execute
 41 |         """
 42 |         return self.data[self.offset:].decode("utf8","ignore"),['OK_Packet','ERR_Packet','Text_Resultest']
 43 | 
 44 |     def COM_QUIT(self):
 45 |         """
 46 |         Type	Name	    Description
 47 |         int<1>	command	    0x01: COM_QUIT
 48 | 
 49 |         Server closes the connection or returns ERR_Packet.
 50 |         """
 51 |         return 'COM_QUIT',[]
 52 | 
 53 |     def COM_INIT_DB(self):
 54 |         """
 55 |         Type	    Name	    Description
 56 |         int<1>	    command	    0x02: COM_INIT_DB
 57 |         string<EOF>	schema name	name of the schema to change to
 58 | 
 59 |         server return:
 60 |             OK_Packet on success
 61 |             ERR_Packet on error
 62 |         """
 63 |         return self.data[self.offset:].decode("utf8","ignore"),['OK_Packet','ERR_Packet']
 64 | 
 65 |     def COM_FIELD_LIST(self):
 66 |         """
 67 |         As of MySQL 5.7.11, COM_FIELD_LIST is deprecated and will be removed in a future version of MySQL.
 68 |         Instead, use COM_QUERY to execute a SHOW COLUMNS statement
 69 | 
 70 |         Type	    Name	    Description
 71 |         int<1>	    command	    0x04: COM_FIELD_LIST
 72 |         string<NUL>	table	    the name of the table to return column information for (in the current database for the connection)
 73 |         string<EOF>	wildcard	field wildcard
 74 |         """
 75 |         return self.data[self.offset:].decode("utf8","ignore"),[]
 76 | 
 77 |     def COM_PREFRESH(self):
 78 |         """
 79 |         As of MySQL 5.7.11, COM_REFRESH is deprecated and will be removed in a future version of MySQL. Instead,
 80 |         use COM_QUERY to execute a FLUSH statement
 81 | 
 82 |         Type	Name	    Description
 83 |         int<1>	command	    0x07: COM_REFRESH
 84 |         int<1>	sub_command	A bitmask of sub-systems to refresh. A combination of the first 8 bits of COM_REFRESH Flags
 85 | 
 86 |         server return:
 87 |             ERR_Packet or OK_Packet
 88 |         """
 89 |         return 'COM_PREFRESH',['OK_Packet','ERR_Packet']
 90 | 
 91 |     def COM_STATISTICS(self):
 92 |         """
 93 |         Get a human readable string of some internal status vars
 94 | 
 95 |         Type	Name	Description
 96 |         int<1>	command	0x08: COM_STATISTICS
 97 | 
 98 |         server return:
 99 |             elther a string<EOF>
100 |             https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_dt_strings.html#sect_protocol_basic_dt_string_eof
101 |         """
102 |         return 'COM_STATISTICS',[]
103 | 
104 |     def COM_PROCESS_INFO(self):
105 |         """
106 |         As of 5.7.11 COM_PROCESS_INFO is deprecated in favor of COM_QUERY with SHOW PROCESSLIST
107 | 
108 |         Type	Name	Description
109 |         int<1>	command	0x0A: COM_PROCESS_INFO
110 | 
111 |         server return:
112 |             Text Resultset or a ERR_Packet
113 |         """
114 |         return 'COM_PROCESS_INFO',['ERR_Packet','Text_Resultest']
115 | 
116 |     def COM_PROCESS_KILL(self):
117 |         """
118 |         As of MySQL 5.7.11, COM_PROCESS_KILL is deprecated and will be removed in a future version of MySQL. Instead,
119 |         use COM_QUERY and a KILL command
120 | 
121 |         Type	Name	        Description
122 |         int<1>	command	        0x0C: COM_PROCESS_KILL
123 |         int<4>	connection_id	The connection to kill
124 | 
125 |         server return:
126 |             ERR_Packet or OK_Packet
127 |         """
128 | 
129 |         return 'COM_PROCESS_KILL',['OK_Packet','ERR_Packet']
130 | 
131 |     def COM_DEBUG(self):
132 |         """
133 |         Dump debug info to server's stdout
134 | 
135 |         COM_DEBUG triggers a dump on internal debug info to stdout of the mysql-server.
136 | 
137 |         The SUPER_ACL privilege is required for this operation
138 | 
139 |         Type	Name	Description
140 |         int<1>	command	0x0D: COM_DEBUG
141 | 
142 |         server return:
143 |             ERR_Packet or OK_Packet
144 |         """
145 | 
146 |         return 'COM_DEBUG',['OK_Packet','ERR_Packet']
147 | 
148 |     def COM_PING(self):
149 |         """
150 |         Check if the server is alive
151 | 
152 |         Type	Name	Description
153 |         int<1>	command	0x0E: COM_PING
154 | 
155 |         server return:
156 |             OK_Packet
157 |         """
158 | 
159 |         return 'COM_PING',['OK_Packet']
160 | 
161 |     def COM_CHANGE_USER(self):
162 |         """
163 |         Changes the user of the current connection.
164 | 
165 |         Also and resets the following connection state:
166 | 
167 |         user variables
168 |         temporary tables
169 |         prepared statements
170 |         ... and others
171 |         It is going through the same states as the Initial Handshake
172 | 
173 |         Type	Name	    Description
174 |         int<1>	command	    0x11: COM_CHANGE_USER
175 |         .........................................
176 |         https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_change_user.html
177 | 
178 |         server return:
179 |             Protocol::AuthSwitchRequest: or ERR_Packet
180 |         """
181 | 
182 |         return 'COM_CHANGE_USER',['ERR_Packet','Handshake_Packet']
183 | 
184 |     def COM_RESET_CONNECTION(self):
185 |         """
186 |         Resets the session state
187 | 
188 |         A more lightweightt version of COM_CHANGE_USER that does about the same to clean up the session state, but:
189 | 
190 |         it does not re-authenticate (and do the extra client/server exchange for that)
191 |         it does not close the connection
192 | 
193 | 
194 |         Type	Name	    Description
195 |         int<1>	command	    0x1F: COM_RESET_CONNECTION
196 | 
197 |         server restun:
198 |             OK_Packet
199 |         """
200 | 
201 |         return 'COM_RESET_CONNECTION',['OK_Packet']
202 | 
203 |     def COM_SET_OPTION(self):
204 |         """
205 |         Sets options for the current connection
206 | 
207 |         COM_SET_OPTION enables and disables server capabilities for the current connection.
208 | 
209 | 
210 |         Type	Name	            Description
211 |         int<1>	status	            [0x1A] COM_SET_OPTION
212 |         int<2>	option_operation	One of enum_mysql_set_option
213 | 
214 |         server return:
215 |             OK_Packet on success, ERR_Packet otherwise.
216 |         """
217 |         return 'COM_SET_OPTION',['OK_Packet','ERR_Packet']
218 | 
219 |     def COM_STMT_PREPARE(self):
220 |         """
221 |         Creates a prepared statement for the passed query string
222 | 
223 |         Type	    Name	        Description
224 |         int<1>	    command	        0x16: COM_STMT_PREPARE
225 |         string<EOF>	query	        The query to prepare
226 | 
227 |         server return:
228 |             COM_STMT_PREPARE_OK on success, ERR_Packet otherwise
229 |         """
230 |         return self.data[self.offset:].decode("utf8","ignore"),['OK_Packet','ERR_Packet']
231 | 
232 |     def COM_STMT_EXECUTE(self):
233 |         """
234 |         COM_STMT_EXECUTE asks the server to execute a prepared statement as identified by statement_id
235 | 
236 |         Type	Name	        Description
237 |         int<1>	status	        [0x17] COM_STMT_EXECUTE
238 |         int<4>	statement_id	ID of the prepared statement to execute
239 |         int<1>	flags	        Flags. See enum_cursor_type
240 |         ...................................................
241 |         see: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_execute.html
242 | 
243 |         server return:
244 |             COM_STMT_EXECUTE Response
245 |             see: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_execute_response.html
246 | 
247 |         :returns
248 |             statement_id
249 |         """
250 | 
251 |         return 'COM_STMT_EXECUTE',[]
252 | 
253 |     def COM_STMT_FETCH(self):
254 |         """
255 |         Fetches the requested amount of rows from a resultset produced by COM_STMT_EXECUTE
256 | 
257 |         Type	Name	        Description
258 |         int<1>	status	        0x18
259 |         int<4>	statement_id	ID of the prepared statement to close
260 |         int<4>	num_rows	    max number of rows to return
261 | 
262 |         server return:
263 |             Multi-Resultset : https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_command_phase_sp.html#sect_protocol_command_phase_sp_multi_resultset
264 |             ERR_Packet
265 | 
266 |         :returns
267 |             statement_id
268 |         """
269 | 
270 |         return 'COM_STMT_FETCH',['ERR_Packet','Text_Resultest']
271 | 
272 | 
273 |     def COM_STMT_CLOSE(self):
274 |         """
275 |         COM_STMT_CLOSE deallocates a prepared statement.
276 | 
277 |         No response packet is sent back to the client.
278 | 
279 |         Type	Name	        Description
280 |         int<1>	status	        [0x19] COM_STMT_CLOSE
281 |         int<4>	statement_id	ID of the prepared statement to close
282 |         """
283 |         return 'COM_STMT_CLOSE',[]
284 | 
285 |     def COM_STMT_RESET(self):
286 |         """
287 |         COM_STMT_RESET resets the data of a prepared statement which was accumulated with COM_STMT_SEND_LONG_DATA commands
288 |         and closes the cursor if it was opened with COM_STMT_EXECUTE.
289 | 
290 |         The server will send a OK_Packet if the statement could be reset, a ERR_Packet if not.
291 | 
292 |         server return:
293 |             OK_Packet or a ERR_Packet
294 | 
295 |         Type	Name	        Description
296 |         int<1>	status	        [0x1A] COM_STMT_RESET
297 |         int<4>	statement_id	ID of the prepared statement to reset
298 |         """
299 | 
300 |         return 'COM_STMT_RESET',['OK_Packet','ERR_Packet']
301 | 
302 |     def COM_STMT_SEND_LONG_DATA(self):
303 |         """
304 |         Sends the data for a parameter.
305 | 
306 |         Repeating to send it, appends the data to the parameter.
307 | 
308 |         No response is sent back to the client
309 | 
310 |         Type	    Name	        Description
311 |         int<1>	    status	        [0x18] COM_STMT_SEND_LONG_DATA
312 |         int<4>	    statement_id	ID of the statement
313 |         int<2>	    param_id	    The parameter to supply data to
314 |         binary<var>	data	        The actual payload to send
315 | 
316 |         """
317 | 
318 |         return 'COM_STMT_SEND_LONG_DATA',[]
319 | 
320 |     def Connection_Packets(self, capability_flags=None):
321 |         """
322 |         see :
323 |             https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase.html
324 |         """
325 |         client_plugin_auth_lenenc_client_data = 1<<21
326 |         secure_connection = 1<<15
327 |         client_connect_with_db = 9
328 |         db_name = ''
329 |         self.offset = 36
330 |         _s_end = self.data.find(b'\0', self.offset)
331 |         user_name = self.data[self.offset:_s_end].decode("utf8","ignore")
332 |         self.offset = _s_end + 1;
333 | 
334 |         if capability_flags & client_plugin_auth_lenenc_client_data or capability_flags & secure_connection:
335 |             passwd_len = struct.unpack('B',self.data[self.offset:self.offset+1])[0]
336 |             self.offset += passwd_len + 1
337 |         else:
338 |             _s_end = self.data.find(b'\0', self.offset)
339 |             self.offset = _s_end + 1;
340 | 
341 |         if capability_flags & client_connect_with_db:
342 |             _s_end = self.data.find(b'\0', self.offset)
343 |             db_name = self.data[self.offset:_s_end].decode("utf8","ignore")
344 |             if 'mysql_native_password' in db_name or 'caching_sha2_password' in db_name:
345 |                 db_name = ''
346 | 
347 |         return user_name,db_name,['Handshake_Packet']
348 | 
349 |     def Handshake_Packet(self):
350 |         """
351 |         Initial Handshake Packet
352 | 
353 |         When the client connects to the server the server sends a handshake packet to the client.
354 |         Depending on the server version and configuration options different variants of the initial packet are sent
355 | 
356 |         Protocol::HandshakeV9:  0x09
357 |         Protocol::HandshakeV10: 0x10
358 |         """
359 |         _s_end = self.data.find(b'\0', self.offset)
360 |         server_version = self.data[self.offset:_s_end].decode("utf8","ignore")
361 |         self.offset  = _s_end + 1 + 4 + 8 + 1
362 |         capability_flags_1 = struct.unpack('H', self.data[self.offset:self.offset + 2])[0]
363 |         self.offset += 5
364 |         capability_flags_2 = struct.unpack('H', self.data[self.offset:self.offset + 2])[0]
365 |         capability_flags = capability_flags_2 << 16 | capability_flags_1
366 | 
367 |         return server_version,'create connection',capability_flags
368 | 
369 |     def OK_Packet(self):
370 |         """
371 |         An OK packet is sent from the server to the client to signal successful completion of a command.
372 | 
373 |         As of MySQL 5.7.5, OK packes are also used to indicate EOF, and EOF packets are deprecated
374 | 
375 |         Type	        Name	            Description
376 |         int<1>	        header	            0x00 or 0xFE the OK packet header
377 |         int<lenenc>	    affected_rows	    affected rows
378 |         int<lenenc>	    last_insert_id	    last insert-id
379 |         ..................................................
380 |         see: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_ok_packet.html
381 | 
382 |         :return:
383 |         """
384 | 
385 |         return 'OK_Packet','OK',None
386 | 
387 |     def ERR_Packet(self):
388 |         """
389 |         This packet signals that an error occurred.
390 | 
391 |         It contains a SQL state value if CLIENT_PROTOCOL_41 is enabled
392 | 
393 |         Type	    Name	            Description
394 |         int<1>	    header	            0xFF ERR packet header
395 |         int<2>	    error_code	        error-code
396 |         if capabilities & CLIENT_PROTOCOL_41 {
397 |         string[1]	sql_state_marker	# marker of the SQL state
398 |         string[5]	sql_state	        SQL state
399 |         }
400 |         string<EOF>	error_message	    human readable error message
401 | 
402 |         see:  https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_err_packet.html
403 |         """
404 |         return 'ERR_Packet',self.data[self.offset+2:].decode("utf8","ignore"),None
405 | 
406 |     def EOF_Packet(self):
407 |         """
408 |         If CLIENT_PROTOCOL_41 is enabled, the EOF packet contains a warning count and status flags.
409 | 
410 |         In the MySQL client/server protocol, the EOF_Packet and OK_Packet packets serve the same purpose,
411 |         to mark the end of a query execution result. Due to changes in MySQL 5.7 in the OK_Packet packets (such as session state tracking),
412 |         and to avoid repeating the changes in the EOF_Packet packet, the OK_Packet is deprecated as of MySQL 5.7.5
413 | 
414 |         Type	Name	Description
415 |         int<1>	header	0xFE EOF packet header
416 |         ...........................
417 |         """
418 |         return 'EOF_Packet','EOF',None
419 | 
420 |     def Text_Resultest(self):
421 |         """
422 |         A Text Resultset is a possible COM_QUERY Response.
423 | 
424 |         It is made up of 2 parts:
425 | 
426 |         the column definitions (a.k.a. the metadata)
427 |         the actual rows
428 | 
429 |         see: https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response_text_resultset.html
430 | 
431 |         """
432 |         return 'Text_Resultest','Result',None
433 | 
434 |     def check_(self,packet_header,response_header):
435 |         """
436 | 
437 |         :param packet_header:
438 |         :return:
439 |         """
440 | 
441 | 
442 |     def check_server_response(self,packet_header):
443 |         if self.packet_palyload == 1:
444 |             return self.Text_Resultest()
445 |         elif packet_header == 0x00 and self.packet_palyload >= 7:
446 |             return self.OK_Packet()
447 |         elif packet_header == 0xfe and self.packet_palyload <= 9:
448 |             return self.EOF_Packet()
449 |         elif packet_header == 0xff:
450 |             return self.ERR_Packet()
451 |         elif packet_header in (0x09,0x0a):
452 |             return self.Handshake_Packet()
453 |         else:
454 |             return self.Text_Resultest()
455 | 
456 |     def Unpacking(self,data,srchost,srcport,dsthost,dstport,all_session_users):
457 |         """
458 |         unpack packet
459 |         :return:
460 |         """
461 |         self.data = data
462 |         session = None
463 |         packet_response = None
464 |         client_packet_text = None
465 |         response_status = None
466 |         response_type = []
467 |         db_name = None
468 |         capability_flags = None
469 | 
470 |         self.unpacke_value()
471 |         if self._type == 'src':
472 |             if srchost == self._ip:
473 |                 '''client packet'''
474 |                 session = str([srchost,srcport,dsthost,dstport])
475 |                 if session in all_session_users and all_session_users[session]['pre'] and self.packet_seq_id and self.packet_seq_id-1==all_session_users[session]['seq_id']:
476 |                     client_packet_text, db_name, response_type = self.Connection_Packets(all_session_users[session]['capability_flags'])
477 |                 elif self.packet_header in self.client_packet_type:
478 |                     client_packet_text,response_type = self.client_packet_type[self.packet_header]()
479 |             else:
480 |                 '''server response'''
481 |                 session = str([dsthost, dstport, srchost, srcport])
482 |                 if any([self.packet_palyload,self.packet_seq_id,self.packet_header]):
483 |                     packet_response,response_status, capability_flags = self.check_server_response(self.packet_header)
484 | 
485 |         elif self._type == 'des':
486 |             if srchost == self._ip:
487 |                 '''server response'''
488 |                 session = str([dsthost, dstport, srchost, srcport])
489 |                 if any([self.packet_palyload,self.packet_seq_id,self.packet_header]):
490 |                     packet_response,response_status, capability_flags = self.check_server_response(self.packet_header)
491 |             else:
492 |                 '''client packet'''
493 |                 session = str([srchost, srcport,dsthost, dstport])
494 |                 if session in all_session_users and all_session_users[session]['pre'] and self.packet_seq_id and self.packet_seq_id-1==all_session_users[session]['seq_id']:
495 |                     client_packet_text, db_name, response_type = self.Connection_Packets(all_session_users[session]['capability_flags'])
496 |                 elif self.packet_header in self.client_packet_type:
497 |                     client_packet_text,response_type = self.client_packet_type[self.packet_header]()
498 | 
499 |         return session,packet_response,client_packet_text,self.packet_header,self.packet_seq_id,response_type,response_status,db_name,capability_flags
500 | 
501 | 
502 |     def unpacke_value(self):
503 |         if sys.version_info < (3, 0):
504 |             try:
505 |                 self.offset = 0
506 |                 self.packet_palyload = struct.unpack('B',self.data[2])[0] << 16 | \
507 |                                        struct.unpack('B',self.data[1])[0] << 8 | \
508 |                                        struct.unpack('B',self.data[0])[0]
509 |                 self.offset += 3
510 |                 self.packet_seq_id = struct.unpack('B',self.data[self.offset])[0]
511 |                 self.offset += 1
512 |                 self.packet_header = struct.unpack('B',self.data[self.offset])[0]
513 |                 self.offset += 1
514 |             except:
515 |                 self.packet_palyload, self.packet_seq_id, self.packet_header = None, None, None
516 |         else:
517 |             try:
518 |                 self.offset = 0
519 |                 self.packet_palyload = self.data[2] << 16 | self.data[1] << 8 | self.data[0]
520 |                 self.offset += 3
521 |                 self.packet_seq_id = self.data[self.offset]
522 |                 self.offset += 1
523 |                 self.packet_header = self.data[self.offset]
524 |                 self.offset += 1
525 |             except:
526 |                 self.packet_palyload, self.packet_seq_id, self.packet_header = None, None, None


--------------------------------------------------------------------------------
/lib/packet_op.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- encoding: utf-8 -*-
  3 | '''
  4 | @author: xiao cai niao
  5 | '''
  6 | from lib.log import Logging
  7 | import socket,psutil,dpkt
  8 | from dpkt.compat import compat_ord
  9 | import time,threading
 10 | from lib.mysql_protocol import mysql_packet
 11 | from lib.db import db
 12 | from clickhouse_driver import connect
 13 | import json,traceback
 14 | 
 15 | class Op_packet:
 16 |     def __init__(self,**kwargs):
 17 |         self.kwargs = kwargs
 18 |         self.queue = kwargs['queue']
 19 |         self._type = kwargs['_type']
 20 |         self.ckhost = kwargs['ckhost'] if 'ckhost' in kwargs else None
 21 |         self.many = kwargs['many'] if 'many' in kwargs else 1000
 22 |         self.mysql_user = kwargs['user'] if 'user' in kwargs else None
 23 |         self.mysql_passwd = kwargs['passwd'] if 'passwd' in kwargs else None
 24 |         if self.mysql_user:
 25 |             if self.mysql_passwd:
 26 |                 pass
 27 |             else:
 28 |                 print('Mysql connection information needs to be set at the same time')
 29 |                 import sys
 30 |                 sys.exit()
 31 | 
 32 |         self.all_session_users = {}
 33 |         self.get_user_list = {}
 34 | 
 35 |         self.op_list = []       #用于写入ck的数据临时存放，达到要求批量写入
 36 |         self.op_num = 0         #统计条数
 37 | 
 38 |     def __get_netcard(self):
 39 |         '''get ip address'''
 40 |         info = psutil.net_if_addrs()
 41 |         for k, v in info.items():
 42 |             if k == self.kwargs['eth']:
 43 |                 for item in v:
 44 |                     if item[0] == 2 and not item[1] == '127.0.0.1' and ':' not in k:
 45 |                         netcard_info = item[1]
 46 |         return netcard_info
 47 | 
 48 |     def mac_addr(self,address):
 49 |         """Convert a MAC address to a readable/printable string
 50 | 
 51 |            Args:
 52 |                address (str): a MAC address in hex form (e.g. '\x01\x02\x03\x04\x05\x06')
 53 |            Returns:
 54 |                str: Printable/readable MAC address
 55 |         """
 56 |         return ':'.join('%02x' % compat_ord(b) for b in address)
 57 | 
 58 |     def inet_to_str(self,inet):
 59 |         """Convert inet object to a string
 60 | 
 61 |             Args:
 62 |                 inet (inet struct): inet network address
 63 |             Returns:
 64 |                 str: Printable/readable IP address
 65 |         """
 66 |         # First try ipv4 and then ipv6
 67 |         try:
 68 |             return socket.inet_ntop(socket.AF_INET, inet)
 69 |         except ValueError:
 70 |             return socket.inet_ntop(socket.AF_INET6, inet)
 71 | 
 72 |     def find_str(self,_str):
 73 |         str_list = []
 74 |         vv = 0
 75 |         while 1:
 76 |             v_i = _str.find(',', vv)
 77 |             if v_i == -1:
 78 |                 str_list.append('?')
 79 |                 break
 80 |             else:
 81 |                 str_list.append('?')
 82 |                 vv = v_i + 1
 83 |         return str_list
 84 | 
 85 |     def set_str(self,_str):
 86 |         str_list = _str.strip().split(' ')
 87 |         set_str = ''
 88 |         t = None
 89 |         _tmp_str = ''
 90 |         for set_value in str_list:
 91 |             if t:
 92 |                 if set_value == str_list[-1]:
 93 |                     set_str += '?'
 94 |                 else:
 95 |                     set_str += '?,'
 96 |                 t = None
 97 |                 continue
 98 |             if set_value == '=':
 99 |                 set_str += _tmp_str + set_value
100 |                 t = True
101 |                 continue
102 |             _tmp_str = set_value
103 | 
104 |         return set_str
105 | 
106 |     def conn_maintain(self):
107 |         while 1:
108 |             if self.get_user_list:
109 |                 __get_list = self.get_user_list.copy()
110 |                 for session in __get_list:
111 |                     self.get_user_info(*__get_list[session])
112 |                     del self.get_user_list[session]
113 | 
114 |             _idle_timeout_session = []
115 |             if self.all_session_users:
116 |                 __all_session_users = self.all_session_users.copy()
117 |                 for session in __all_session_users:
118 |                     _cur_time = time.time()
119 |                     if int(_cur_time - __all_session_users[session]['date']) > 300:
120 |                         _idle_timeout_session.append(session)
121 | 
122 |             for session in _idle_timeout_session:
123 |                 del self.all_session_users[session]
124 | 
125 |             time.sleep(0.1)
126 | 
127 |     def sql_parser(self,sql):
128 |         """Format sql
129 |             Args:
130 |                 sql: Captured sql statement
131 |             Returns:
132 |                 list: [sql,[values,]] If it is an insert statement, the returned data is empty.
133 |         """
134 |         sql = sql.strip('\n').strip()
135 |         if sql.startswith('insert') or sql.startswith('INSERT'):
136 |             k = sql.index('(')
137 |             v = sql.index(')')
138 |             v_str = tuple(self.find_str(sql[k:v + 1]))
139 |             try:
140 |                 index = sql.index('values')
141 |             except:
142 |                 index = sql.index('VALUES')
143 | 
144 |             return sql[:index + 6] + str(v_str), ''
145 | 
146 | 
147 |         elif sql.startswith('update') or sql.startswith('UPDATE'):
148 |             try:
149 |                 set_index = sql.index('set')
150 |             except:
151 |                 set_index = sql.index('SET')
152 | 
153 |             try:
154 |                 where_index = sql.index('where')
155 |             except:
156 |                 try:
157 |                     where_index = sql.index('WHERE')
158 |                 except:
159 |                     where_index = ''
160 |             sql_start = sql[:set_index + 4]
161 |             if where_index:
162 |                 sql_end = sql[where_index - 1:]
163 |             else:
164 |                 sql_end = ''
165 |             _set_str = self.set_str(sql[set_index + 4:where_index])
166 |             return sql_start + _set_str + sql_end, ''
167 | 
168 | 
169 |         else:
170 |             return sql, ''
171 | 
172 |     def check_packet_type(self,response):
173 |         respons_status = {
174 |             'Text_Resultest': 1,
175 |             'EOF_Packet': 1,
176 |             'ERR_Packet': 0,
177 |             'OK_Packet': 1,
178 |             'Handshake_Packet': 1
179 |         }
180 | 
181 |         return respons_status[response]
182 | 
183 |     def create_conn(self,session,client_packet_text,packet_seq_id,type,response_type,response_status,db_name):
184 |         """
185 | 
186 |         :param session:
187 |         :param client_packet_text:
188 |         :param packet_seq_id:
189 |         :param type:
190 |         :param response_type:
191 |         :param response_status:
192 |         :return:
193 |         """
194 |         if self.all_session_users[session]['status']:
195 |             pass
196 |         else:
197 |             if type == 'client':
198 |                 if session in self.all_session_users and self.all_session_users[session]['pre']:
199 |                     if packet_seq_id - 1 == self.all_session_users[session]['seq_id']:
200 |                         self.all_session_users[session]['pre'] = False
201 |                         self.all_session_users[session]['user'] = client_packet_text
202 |                         self.all_session_users[session]['seq_id'] = packet_seq_id
203 |                         self.all_session_users[session]['db'] = db_name
204 |                     else:
205 |                         del  self.all_session_users[session]
206 |                     # self.create_conn(session,client_packet_text)
207 |                 elif session in self.all_session_users and not self.all_session_users[session]['status']:
208 |                     if packet_seq_id - 1 == self.all_session_users[session]['seq_id']:
209 |                         self.all_session_users[session]['seq_id'] = packet_seq_id
210 |                     else:
211 |                         del self.all_session_users[session]
212 |             elif type == 'response':
213 |                 if session in self.all_session_users and response_type in ('OK_Packet','ERR_Packet'):
214 |                     if packet_seq_id - 1 == self.all_session_users[session]['seq_id']:
215 |                         self.all_session_users[session]['status'] = True
216 |                         self.all_session_users[session]['date'] = time.time()
217 |                         _session = eval(session)
218 |                         jsons = {'source_host': _session[0], 'source_port': _session[1],
219 |                                  'destination_host': _session[2], 'destination_port': _session[3],
220 |                                  'user_name': self.all_session_users[session]['user'], 'sql': 'create connection',
221 |                                  'db': self.all_session_users[session]['db'],
222 |                                  'reponse_value': '',
223 |                                  'execute_time': 0,
224 |                                  'response_status': response_status, 'event_date': int(time.time())}
225 |                         if self.ckhost:
226 |                             self.ck_insert(jsons)
227 |                         else:
228 |                             self._logging.info(msg=json.dumps(jsons))
229 |                         # self._logging.info(msg=
230 |                         #               'source_host: {} source_port: {} destination_host: {} destination_port: {} user_name: {} sql: {} values: {} '
231 |                         #               'execute_time:{}  status:{}'.format(_session[0], _session[1], _session[2],
232 |                         #                                                   _session[3],
233 |                         #                                                   self.all_session_users[session]['user'],
234 |                         #                                                   'create connection', None,
235 |                         #                                                   None,response_status))
236 |                         if response_type == 'ERR_Packet':
237 |                             del self.all_session_users[session]
238 |                     else:
239 |                         del self.all_session_users[session]
240 | 
241 |                 elif session in self.all_session_users:
242 |                     if packet_seq_id - 1 == self.all_session_users[session]['seq_id']:
243 |                         self.all_session_users[session]['seq_id'] = packet_seq_id
244 |                     else:
245 |                         del self.all_session_users[session]
246 | 
247 |     def get_user_info(self,host,port,mysql_host,mysql_port,session):
248 |         """select user_name from mysql instance"""
249 |         if self.mysql_user:
250 |             _kwargs = {'host':mysql_host,'port':mysql_port,'user':self.mysql_user,'passwd':self.mysql_passwd}
251 |             dd = db(**_kwargs)
252 |             user_name,db_name = dd.get(host,port)
253 |             if db_name in ('null' , 'Null') or db_name is None:
254 |                 db_name = ''
255 |             if user_name:
256 |                 self.all_session_users[session] = {'status':True,'user':user_name,'pre':False,'db':db_name,'date':time.time()}
257 |             dd.close()
258 |             return user_name
259 |         else:
260 |             return ''
261 | 
262 | 
263 | 
264 | 
265 |     def an_packet(self):
266 |         _ip = self.__get_netcard()
267 |         _mysql_packet_op = mysql_packet(**dict({'_type':self._type},**{'_ip':_ip}))
268 |         session_status = {}
269 |         self._logging = Logging()
270 | 
271 |         t = threading.Thread(target=self.conn_maintain,args=())
272 |         t.start()
273 | 
274 |         while 1:
275 |             if not self.queue.empty():
276 |                 buf,_cur_time = self.queue.get()
277 |                 eth = dpkt.ethernet.Ethernet(buf)
278 | 
279 |                 if not isinstance(eth.data, dpkt.ip.IP):
280 |                     self._logging.error(msg='Non IP Packet type not supported %s\n' % eth.data.__class__.__name__)
281 |                     continue
282 | 
283 |                 ip = eth.data
284 | 
285 |                 if isinstance(ip.data, dpkt.tcp.TCP):
286 |                     tcp = ip.data
287 |                     src_host,dst_host = self.inet_to_str(ip.src),self.inet_to_str(ip.dst)
288 |                     session, packet_response, client_packet_text, packet_header, packet_seq_id,response_type,response_status, db_name, capability_flags=_mysql_packet_op.Unpacking(
289 |                                                                                 data=tcp.data,srchost=src_host,
290 |                                                         srcport=tcp.sport,dsthost=dst_host,dstport=tcp.dport,
291 |                         all_session_users=self.all_session_users)
292 |                     if packet_response and packet_response in ('COM_PROCESS_KILL', 'COM_QUIT'):
293 |                         """close connection"""
294 |                         if session in self.all_session_users:
295 |                             del self.all_session_users[session]
296 | 
297 | 
298 |                     if client_packet_text:
299 |                         if session in self.all_session_users:
300 |                             self.create_conn(session,client_packet_text,packet_seq_id,'client',response_type,response_status, db_name)
301 | 
302 |                         if packet_header == 0x16:
303 |                             session_status[session] = {'start_time': _cur_time, 'request_text': client_packet_text,
304 |                                                        'request_header': packet_header, 'seq_id': packet_seq_id,
305 |                                                        'response_type': response_type,'com_pre':True}
306 | 
307 |                         elif packet_header == 0x17 and session in session_status and 'com_pre' in session_status[session]:
308 |                             del session_status[session]['com_pre']
309 |                             continue
310 | 
311 |                         elif packet_header in (0x01, 0x18):
312 |                             session_status[session] = {'start_time': _cur_time, 'request_text': client_packet_text,
313 |                                                        'request_header': packet_header, 'seq_id': packet_seq_id,
314 |                                                        'response_type': response_type,'end_time':_cur_time,
315 |                                                        'status':1,'response_status':''}
316 |                         elif packet_header == 0x19:
317 |                             continue
318 |                         else:
319 |                             session_status[session] = {'start_time': _cur_time, 'request_text': client_packet_text,
320 |                                                        'request_header': packet_header, 'seq_id': packet_seq_id,
321 |                                                        'response_type': response_type}
322 | 
323 |                         if session in self.all_session_users and self.all_session_users[session]['status']:
324 |                             session_status[session]['user_name'] = self.all_session_users[session]['user']
325 |                             session_status[session]['db'] = self.all_session_users[session]['db']
326 |                             self.all_session_users[session]['date'] = _cur_time
327 |                         elif session not in self.all_session_users:
328 |                             # session_status[session]['user_name'] = self.get_user_info(host=src_host,port=tcp.sport,
329 |                             #                                                           mysql_host=dst_host,
330 |                             #                                                           mysql_port=tcp.dport,
331 |                             #                                                           session=session)
332 |                             session_status[session]['user_name'] = ''
333 |                             session_status[session]['db'] = ''
334 |                             if session not in self.get_user_list and packet_header not in (0x01, 0x19, 0x18) and any([self.mysql_user,self.mysql_passwd]):
335 |                                 self.get_user_list[session]=[src_host,tcp.sport,dst_host,tcp.dport,session]
336 | 
337 | 
338 |                     elif packet_response:
339 |                         if packet_header and packet_header in (0x09, 0x0a):
340 |                             """connection"""
341 |                             self.all_session_users[session] = {'pre': True, 'user': '','db': '','capability_flags':capability_flags,
342 |                                                                'server_version': packet_response,
343 |                                                                'seq_id': packet_seq_id, 'status': False,'date':_cur_time}
344 |                             continue
345 |                         if session in self.all_session_users:
346 |                             self.create_conn(session,client_packet_text,packet_seq_id,'response',packet_response,response_status, '')
347 | 
348 |                         if session in session_status :
349 |                             if packet_response in session_status[session]['response_type']:
350 |                                 if packet_seq_id - 1 == session_status[session]['seq_id'] and 'com_pre' not in session_status[session]:
351 |                                     session_status[session]['end_time'] = _cur_time
352 |                                     session_status[session]['status'] = self.check_packet_type(packet_response)
353 |                                     session_status[session]['response_status'] = response_status
354 |                                 elif packet_seq_id - 1 != session_status[session]['seq_id']:
355 |                                     del session_status[session]
356 |                             else:
357 |                                 del session_status[session]
358 | 
359 |                     elif session in self.all_session_users and not self.all_session_users[session]['status'] and packet_seq_id:
360 |                         if packet_seq_id - 1 == self.all_session_users[session]['seq_id']:
361 |                             self.all_session_users[session]['seq_id'] = packet_seq_id
362 |                     else:
363 |                         if session in session_status:
364 |                             del session_status[session]
365 | 
366 |                 del_session = []
367 |                 for session in session_status:
368 |                     if 'status' in session_status[session]:
369 |                         execute_time = float('%.4f' % (session_status[session]['end_time'] - session_status[session]['start_time']))
370 |                         if session_status[session]['request_header'] == 0x03:
371 |                             sql, values = self.sql_parser(session_status[session]['request_text'])
372 |                         else:
373 |                             sql, values = session_status[session]['request_text'],''
374 |                         _session = eval(session)
375 |                         #try:
376 |                         jsons = {'source_host':_session[0],'source_port':_session[1],'destination_host':_session[2],'destination_port':_session[3],
377 |                                  'user_name':session_status[session]['user_name'],'sql':sql, 'db': session_status[session]['db'],'reponse_value':values,'execute_time':execute_time,
378 |                                  'response_status':session_status[session]['response_status'], 'event_date':int(_cur_time)}
379 |                         if self.ckhost:
380 |                             self.ck_insert(jsons)
381 |                         else:
382 |                             self._logging.info(msg=json.dumps(jsons))
383 |                         # self._logging.info(msg=
384 |                         #     'source_host: {} source_port: {} destination_host: {} destination_port: {} user_name: {} sql: {} values: {} '
385 |                         #     'execute_time:{}  status:{}'.format(_session[0], _session[1], _session[2],_session[3],
386 |                         #                                         session_status[session]['user_name'],
387 |                         #                                         sql, values,
388 |                         #                                         execute_time,
389 |                         #                                         session_status[session]['response_status']))
390 |                         # except:
391 |                         #     print(traceback.format_exc())
392 |                         del_session.append(session)
393 | 
394 |                 for session in del_session:
395 |                     del session_status[session]
396 | 
397 |             else:
398 |                 time.sleep(0.01)
399 | 
400 | 
401 |     def ck_insert(self, jsons):
402 |         '''
403 |         必须先在clickhouse创建表
404 | 
405 |         CREATE table mysql_audit.mysql_audit_info(
406 |         source_host String,
407 |         source_port UInt64,
408 |         destination_host String,
409 |         destination_port UInt64,
410 |         user_name String,
411 |         sql String,
412 |         reponse_value String,
413 |         execute_time Float64,
414 |         response_status String,
415 |         event_date DateTime)
416 |         ENGINE=MergeTree()
417 |         PARTITION BY toYYYYMMDD(event_date)
418 |         ORDER BY (source_host, source_port, event_date)
419 |         TTL event_date + INTERVAL 5 DAY
420 |         SETTINGS index_granularity=8192,enable_mixed_granularity_parts=1;
421 |         :param jsons:
422 |         :return:
423 |         '''
424 |         self.op_list.append(jsons)
425 |         self.op_num += 1
426 |         if self.op_num >= self.many:
427 |             try:
428 |                 ck_url = 'clickhouse://{}'.format(self.ckhost)
429 |                 conn = connect(ck_url)
430 |                 cursor = conn.cursor()
431 |                 cursor.executemany('insert into mysql_audit.mysql_audit_info(source_host,source_port,destination_host,destination_port,user_name,'
432 |                                    'sql,db,reponse_value,execute_time,response_status,event_date) values',self.op_list)
433 |             except:
434 |                 print(traceback.format_exc())
435 |             self.op_num = 0
436 |             self.op_list = []
437 | 
438 | 


--------------------------------------------------------------------------------
/log/123.txt:
--------------------------------------------------------------------------------
1 | 123
2 | 


--------------------------------------------------------------------------------
/tcp_audit.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | '''
  3 | @Author  : xiao cai niao
  4 | '''
  5 | 
  6 | import pcap
  7 | import sys
  8 | import getopt
  9 | import time
 10 | from lib.packet_op import Op_packet
 11 | from lib.log import Logging
 12 | from multiprocessing import Process
 13 | from multiprocessing import Queue
 14 | 
 15 | my_queue = Queue(1024)
 16 | 
 17 | 
 18 | 
 19 | def print_packets(**kwargs):
 20 |     """Print out information about each packet in a pcap
 21 | 
 22 |        Args:
 23 |            pcap: dpkt pcap reader object (dpkt.pcap.Reader)
 24 |     """
 25 | 
 26 |     t = ThreadDump(**dict(kwargs,**{'queue':my_queue}))
 27 |     t.start()
 28 | 
 29 |     _pcap = pcap.pcap(name=kwargs['eth'], promisc=True, immediate=True, timeout_ms=5000)
 30 |     _pcap.setfilter("tcp port {}".format(kwargs['port']))
 31 | 
 32 |     _logging = Logging()
 33 |     for timestamp, buf in _pcap:
 34 |         if sys.version_info < (3, 0):
 35 |             _time = time.time()
 36 |         else:
 37 |             _time = timestamp
 38 |         if append_data((buf,_time)):
 39 |             pass
 40 |         else:
 41 |             _logging.error('queue is full!!!!!!!')
 42 |             sys.exit()
 43 | 
 44 | 
 45 | def append_data(data):
 46 |     for i in range(100):
 47 |         if my_queue.full():
 48 |             time.sleep(1)
 49 |             continue
 50 |         my_queue.put(data)
 51 |         return True
 52 |     else:
 53 |         return False
 54 | 
 55 | 
 56 | 
 57 | 
 58 | 
 59 | 
 60 | 
 61 | 
 62 | class ThreadDump(Process):
 63 |     def __init__(self,**kwargs):
 64 |         super(ThreadDump,self).__init__()
 65 |         self.kwargs = kwargs
 66 |     def run(self):
 67 |         Op_packet(**self.kwargs).an_packet()
 68 | 
 69 | 
 70 | def Usage():
 71 |     __usage__ = """
 72 |     	Usage:
 73 |     	Options:
 74 |       		-h [--help] : print help message
 75 |       		-p [--port] : tcp port 
 76 |       		-e [--eth]  : network card
 77 |       		-t [--type] : define whether the local address is the sender or the receiver [src/des]
 78 |       		-u [--user] : remote mysql node username
 79 |       		-P [--passwd] : mysql password
 80 |       		--ckhost : clickhouse host
 81 |       		--many :  execute many number, default 1000
 82 |     	    """
 83 |     print(__usage__)
 84 | 
 85 | 
 86 | def main(argv):
 87 |     _argv = {}
 88 |     try:
 89 |         opts, args = getopt.getopt(argv[1:], 'hp:e:t:u:P:',
 90 |                                    ['help', 'port=', 'eth=','type=','user=','passwd=','ckhost=','many='])
 91 |     except getopt.GetoptError as err:
 92 |         print(str(err))
 93 |         Usage()
 94 |         sys.exit(2)
 95 |     for o, a in opts:
 96 |         if o in ('-h', '--help'):
 97 |             Usage()
 98 |             sys.exit(1)
 99 |         elif o in ('-p', '--port'):
100 |             _argv['port'] = int(a)
101 |         elif o in ('-e','--eth'):
102 |             _argv['eth'] = a
103 |         elif o in ('-t','--type'):
104 |             _argv['_type'] = a
105 |         elif o in ('-u','--user'):
106 |             _argv['user'] = a
107 |         elif o in ('-P','--passwd'):
108 |             _argv['passwd'] = a
109 |         elif o == '--ckhost':
110 |             _argv['ckhost'] = a
111 |         elif o == '--many':
112 |             _argv['many'] = int(a)
113 |         else:
114 |             print('unhandled option')
115 |             Usage()
116 |             sys.exit(3)
117 | 
118 |     print_packets(**_argv)
119 | 
120 | 
121 | if __name__ == "__main__":
122 |     main(sys.argv)
123 | 
124 | 
125 | 
126 | 


--------------------------------------------------------------------------------