├── .idea
    ├── .name
    ├── encodings.xml
    ├── misc.xml
    ├── modules.xml
    ├── rulecat.iml
    ├── vcs.xml
    └── workspace.xml
├── README.md
├── app
    ├── app.go
    ├── customfuc.go
    └── engine.go
├── doc
    ├── README.md
    └── rules_tpl
    │   ├── 1.yml
    │   ├── 2.yml
    │   ├── 3.yml
    │   ├── 4.yml
    │   └── 5.yml
├── etc
    ├── config.yml
    ├── nids-flow_rules
    │   ├── 1.yml
    │   ├── 2.yml
    │   ├── 3.yml
    │   └── 4.yml
    ├── topic_tpl2_rules
    │   ├── 1.yml
    │   ├── 2.yml
    │   ├── 3.yml
    │   └── 4.yml
    └── topic_tpl_rules
    │   ├── 1.yml
    │   ├── 2.yml
    │   ├── 3.yml
    │   └── 4.yml
├── go.mod
├── go.sum
├── img.png
├── main.go
├── test
    ├── cache.go
    ├── engine.go
    ├── json.go
    └── yaml.go
└── utils
    ├── cache
        └── cache.go
    ├── email
        └── email.go
    ├── es
        ├── elastic6.go
        ├── elastic7.go
        └── es.go
    ├── json
        └── json.go
    ├── kafka
        ├── consumer.go
        └── producer.go
    ├── log
        └── log.go
    ├── utils.go
    └── workerpool
        └── workerpool.go


/.idea/.name:
--------------------------------------------------------------------------------
1 | rulecat


--------------------------------------------------------------------------------
/.idea/encodings.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="Encoding" addBOMForNewFiles="with NO BOM" />
4 | </project>


--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="JavaScriptSettings">
4 |     <option name="languageLevel" value="ES6" />
5 |   </component>
6 | </project>


--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="ProjectModuleManager">
4 |     <modules>
5 |       <module fileurl="file://$PROJECT_DIR$/.idea/rulecat.iml" filepath="$PROJECT_DIR$/.idea/rulecat.iml" />
6 |     </modules>
7 |   </component>
8 | </project>


--------------------------------------------------------------------------------
/.idea/rulecat.iml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <module type="WEB_MODULE" version="4">
3 |   <component name="Go" enabled="true" />
4 |   <component name="NewModuleRootManager">
5 |     <content url="file://$MODULE_DIR$" />
6 |     <orderEntry type="inheritedJdk" />
7 |     <orderEntry type="sourceFolder" forTests="false" />
8 |   </component>
9 | </module>


--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project version="4">
3 |   <component name="VcsDirectoryMappings">
4 |     <mapping directory="$PROJECT_DIR$" vcs="Git" />
5 |   </component>
6 | </project>


--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project version="4">
  3 |   <component name="AutoImportSettings">
  4 |     <option name="autoReloadType" value="ALL" />
  5 |   </component>
  6 |   <component name="ChangeListManager">
  7 |     <list default="true" id="7325faa0-9f1a-47b5-8b13-02f1652b2baa" name="Default Changelist" comment="">
  8 |       <change afterPath="$PROJECT_DIR$/img.png" afterDir="false" />
  9 |       <change beforePath="$PROJECT_DIR$/.idea/rulecat.iml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/rulecat.iml" afterDir="false" />
 10 |       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
 11 |       <change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
 12 |       <change beforePath="$PROJECT_DIR$/app/app.go" beforeDir="false" afterPath="$PROJECT_DIR$/app/app.go" afterDir="false" />
 13 |       <change beforePath="$PROJECT_DIR$/doc/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/doc/README.md" afterDir="false" />
 14 |       <change beforePath="$PROJECT_DIR$/etc/config.yml" beforeDir="false" afterPath="$PROJECT_DIR$/etc/config.yml" afterDir="false" />
 15 |       <change beforePath="$PROJECT_DIR$/etc/rules/conn/1.yml" beforeDir="false" />
 16 |       <change beforePath="$PROJECT_DIR$/etc/rules/conn/2.yml" beforeDir="false" afterPath="$PROJECT_DIR$/doc/rules_tpl/2.yml" afterDir="false" />
 17 |       <change beforePath="$PROJECT_DIR$/etc/rules/conn/3.yml" beforeDir="false" />
 18 |       <change beforePath="$PROJECT_DIR$/etc/rules/conn/4.yml" beforeDir="false" />
 19 |       <change beforePath="$PROJECT_DIR$/etc/rules/dns/1.yml" beforeDir="false" afterPath="$PROJECT_DIR$/doc/rules_tpl/1.yml" afterDir="false" />
 20 |       <change beforePath="$PROJECT_DIR$/etc/rules/dns/2.yml" beforeDir="false" afterPath="$PROJECT_DIR$/etc/rules/2.yml" afterDir="false" />
 21 |       <change beforePath="$PROJECT_DIR$/etc/rules/dns/3.yml" beforeDir="false" afterPath="$PROJECT_DIR$/doc/rules_tpl/3.yml" afterDir="false" />
 22 |       <change beforePath="$PROJECT_DIR$/etc/rules/dns/4.yml" beforeDir="false" afterPath="$PROJECT_DIR$/doc/rules_tpl/4.yml" afterDir="false" />
 23 |       <change beforePath="$PROJECT_DIR$/etc/rules/http/1.yml" beforeDir="false" afterPath="$PROJECT_DIR$/etc/rules/1.yml" afterDir="false" />
 24 |       <change beforePath="$PROJECT_DIR$/etc/rules/http/2.yml" beforeDir="false" />
 25 |       <change beforePath="$PROJECT_DIR$/etc/rules/http/3.yml" beforeDir="false" afterPath="$PROJECT_DIR$/etc/rules/3.yml" afterDir="false" />
 26 |       <change beforePath="$PROJECT_DIR$/etc/rules/http/4.yml" beforeDir="false" afterPath="$PROJECT_DIR$/etc/rules/4.yml" afterDir="false" />
 27 |       <change beforePath="$PROJECT_DIR$/etc/rules/icmp/1.yml" beforeDir="false" />
 28 |       <change beforePath="$PROJECT_DIR$/etc/rules/icmp/2.yml" beforeDir="false" />
 29 |       <change beforePath="$PROJECT_DIR$/etc/rules/icmp/3.yml" beforeDir="false" />
 30 |       <change beforePath="$PROJECT_DIR$/etc/rules/icmp/4.yml" beforeDir="false" />
 31 |       <change beforePath="$PROJECT_DIR$/etc/rules/mongodb/1.yml" beforeDir="false" />
 32 |       <change beforePath="$PROJECT_DIR$/etc/rules/mongodb/2.yml" beforeDir="false" />
 33 |       <change beforePath="$PROJECT_DIR$/etc/rules/mongodb/3.yml" beforeDir="false" />
 34 |       <change beforePath="$PROJECT_DIR$/etc/rules/mongodb/4.yml" beforeDir="false" />
 35 |       <change beforePath="$PROJECT_DIR$/etc/rules/mysql/1.yml" beforeDir="false" />
 36 |       <change beforePath="$PROJECT_DIR$/etc/rules/mysql/2.yml" beforeDir="false" />
 37 |       <change beforePath="$PROJECT_DIR$/etc/rules/mysql/3.yml" beforeDir="false" />
 38 |       <change beforePath="$PROJECT_DIR$/etc/rules/mysql/4.yml" beforeDir="false" />
 39 |       <change beforePath="$PROJECT_DIR$/etc/rules/redis/1.yml" beforeDir="false" />
 40 |       <change beforePath="$PROJECT_DIR$/etc/rules/redis/2.yml" beforeDir="false" />
 41 |       <change beforePath="$PROJECT_DIR$/etc/rules/redis/3.yml" beforeDir="false" />
 42 |       <change beforePath="$PROJECT_DIR$/etc/rules/redis/4.yml" beforeDir="false" />
 43 |       <change beforePath="$PROJECT_DIR$/etc/rules/ssh/1.yml" beforeDir="false" />
 44 |       <change beforePath="$PROJECT_DIR$/etc/rules/ssh/2.yml" beforeDir="false" />
 45 |       <change beforePath="$PROJECT_DIR$/etc/rules/ssh/3.yml" beforeDir="false" />
 46 |       <change beforePath="$PROJECT_DIR$/etc/rules/ssh/4.yml" beforeDir="false" />
 47 |       <change beforePath="$PROJECT_DIR$/go.mod" beforeDir="false" afterPath="$PROJECT_DIR$/go.mod" afterDir="false" />
 48 |       <change beforePath="$PROJECT_DIR$/go.sum" beforeDir="false" afterPath="$PROJECT_DIR$/go.sum" afterDir="false" />
 49 |       <change beforePath="$PROJECT_DIR$/main.go" beforeDir="false" afterPath="$PROJECT_DIR$/main.go" afterDir="false" />
 50 |       <change beforePath="$PROJECT_DIR$/utils/log/log.go" beforeDir="false" afterPath="$PROJECT_DIR$/utils/log/log.go" afterDir="false" />
 51 |       <change beforePath="$PROJECT_DIR$/utils/mq/rabbitmq.go" beforeDir="false" />
 52 |       <change beforePath="$PROJECT_DIR$/utils/redis/redis.go" beforeDir="false" />
 53 |       <change beforePath="$PROJECT_DIR$/utils/utils.go" beforeDir="false" afterPath="$PROJECT_DIR$/utils/utils.go" afterDir="false" />
 54 |       <change beforePath="$PROJECT_DIR$/utils/workerpool/workerpool.go" beforeDir="false" afterPath="$PROJECT_DIR$/utils/workerpool/workerpool.go" afterDir="false" />
 55 |     </list>
 56 |     <option name="SHOW_DIALOG" value="false" />
 57 |     <option name="HIGHLIGHT_CONFLICTS" value="true" />
 58 |     <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
 59 |     <option name="LAST_RESOLUTION" value="IGNORE" />
 60 |   </component>
 61 |   <component name="FileTemplateManagerImpl">
 62 |     <option name="RECENT_TEMPLATES">
 63 |       <list>
 64 |         <option value="Go File" />
 65 |       </list>
 66 |     </option>
 67 |   </component>
 68 |   <component name="FindInProjectRecents">
 69 |     <findStrings>
 70 |       <find>pipeline</find>
 71 |       <find>ClusterName</find>
 72 |       <find>New</find>
 73 |       <find>- xxx</find>
 74 |       <find>False</find>
 75 |       <find>ignorecase: False</find>
 76 |       <find>rule</find>
 77 |       <find>rule_engine_by_go</find>
 78 |     </findStrings>
 79 |     <replaceStrings>
 80 |       <replace>:</replace>
 81 |       <replace>- njcx91@tom.com</replace>
 82 |       <replace>ignorecase: false</replace>
 83 |       <replace>rulecat</replace>
 84 |     </replaceStrings>
 85 |   </component>
 86 |   <component name="GOROOT" url="file://$PROJECT_DIR$/../go/go1.23.0" />
 87 |   <component name="Git.Settings">
 88 |     <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
 89 |   </component>
 90 |   <component name="GoDepProject">
 91 |     <exec-path>$PROJECT_DIR$/../../bin/dep</exec-path>
 92 |   </component>
 93 |   <component name="GoLibraries">
 94 |     <option name="indexEntireGoPath" value="false" />
 95 |   </component>
 96 |   <component name="IdeDocumentHistory">
 97 |     <option name="CHANGED_PATHS">
 98 |       <list>
 99 |         <option value="$PROJECT_DIR$/utils/yaml/types.go" />
100 |         <option value="$PROJECT_DIR$/test/yaml.go" />
101 |         <option value="$PROJECT_DIR$/etc/rules/http/2.yml" />
102 |         <option value="$PROJECT_DIR$/etc/rules/http/3.yml" />
103 |         <option value="$PROJECT_DIR$/etc/rules/http/4.yml" />
104 |         <option value="$PROJECT_DIR$/etc/rules/icmp/1.yml" />
105 |         <option value="$PROJECT_DIR$/etc/rules/icmp/2.yml" />
106 |         <option value="$PROJECT_DIR$/etc/rules/icmp/3.yml" />
107 |         <option value="$PROJECT_DIR$/etc/rules/icmp/4.yml" />
108 |         <option value="$PROJECT_DIR$/etc/rules/mongodb/1.yml" />
109 |         <option value="$PROJECT_DIR$/etc/rules/mongodb/2.yml" />
110 |         <option value="$PROJECT_DIR$/etc/rules/mongodb/3.yml" />
111 |         <option value="$PROJECT_DIR$/etc/rules/mongodb/4.yml" />
112 |         <option value="$PROJECT_DIR$/etc/rules/mysql/1.yml" />
113 |         <option value="$PROJECT_DIR$/etc/rules/mysql/2.yml" />
114 |         <option value="$PROJECT_DIR$/etc/rules/mysql/3.yml" />
115 |         <option value="$PROJECT_DIR$/etc/rules/mysql/4.yml" />
116 |         <option value="$PROJECT_DIR$/etc/rules/redis/1.yml" />
117 |         <option value="$PROJECT_DIR$/etc/rules/redis/2.yml" />
118 |         <option value="$PROJECT_DIR$/etc/rules/redis/3.yml" />
119 |         <option value="$PROJECT_DIR$/etc/rules/redis/4.yml" />
120 |         <option value="$PROJECT_DIR$/etc/rules/ssh/1.yml" />
121 |         <option value="$PROJECT_DIR$/etc/rules/ssh/2.yml" />
122 |         <option value="$PROJECT_DIR$/etc/rules/ssh/3.yml" />
123 |         <option value="$PROJECT_DIR$/etc/rules/ssh/4.yml" />
124 |         <option value="$PROJECT_DIR$/utils/kafka/producer.go" />
125 |         <option value="$PROJECT_DIR$/test/es.go" />
126 |         <option value="$PROJECT_DIR$/utils/es/elastic6.go" />
127 |         <option value="$PROJECT_DIR$/utils/es/elastic7.go" />
128 |         <option value="$PROJECT_DIR$/utils/es/es.go" />
129 |         <option value="$PROJECT_DIR$/app/customfuc.go" />
130 |         <option value="$PROJECT_DIR$/app/customf/dfuc.go" />
131 |         <option value="$PROJECT_DIR$/utils/email/email.go" />
132 |         <option value="$PROJECT_DIR$/etc/rules/conn/2.yml" />
133 |         <option value="$PROJECT_DIR$/etc/rules/dns/1.yml" />
134 |         <option value="$PROJECT_DIR$/etc/rules/dns/2.yml" />
135 |         <option value="$PROJECT_DIR$/etc/rules/dns/3.yml" />
136 |         <option value="$PROJECT_DIR$/etc/rules/dns/4.yml" />
137 |         <option value="$PROJECT_DIR$/etc/rules/http/1.yml" />
138 |         <option value="$PROJECT_DIR$/etc/rules/conn/3.yml" />
139 |         <option value="$PROJECT_DIR$/etc/rules/conn/4.yml" />
140 |         <option value="$PROJECT_DIR$/app/customf/cfuc.go" />
141 |         <option value="$PROJECT_DIR$/etc/rules/conn/1.yml" />
142 |         <option value="$PROJECT_DIR$/utils/utils.go" />
143 |         <option value="$PROJECT_DIR$/logs/nids.log" />
144 |         <option value="$PROJECT_DIR$/go.mod" />
145 |         <option value="$PROJECT_DIR$/main.go" />
146 |         <option value="$PROJECT_DIR$/app/engine.go" />
147 |         <option value="$PROJECT_DIR$/etc/config.yml" />
148 |         <option value="$PROJECT_DIR$/app/app.go" />
149 |         <option value="$PROJECT_DIR$/README.md" />
150 |       </list>
151 |     </option>
152 |   </component>
153 |   <component name="MacroExpansionManager">
154 |     <option name="directoryName" value="igjdQUeJ" />
155 |   </component>
156 |   <component name="MarkdownSettingsMigration">
157 |     <option name="stateVersion" value="1" />
158 |   </component>
159 |   <component name="ProjectFrameBounds" fullScreen="true">
160 |     <option name="x" value="-1920" />
161 |     <option name="y" value="23" />
162 |     <option name="width" value="1920" />
163 |     <option name="height" value="973" />
164 |   </component>
165 |   <component name="ProjectId" id="2seGsDhqqKbc0Op8UDG876Yjx7a" />
166 |   <component name="ProjectLevelVcsManager" settingsEditedManually="true">
167 |     <ConfirmationsSetting value="2" id="Add" />
168 |   </component>
169 |   <component name="ProjectView">
170 |     <navigator proportions="" version="1">
171 |       <foldersAlwaysOnTop value="true" />
172 |     </navigator>
173 |     <panes>
174 |       <pane id="Scope" />
175 |       <pane id="ProjectPane">
176 |         <subPane>
177 |           <expand>
178 |             <path>
179 |               <item name="rulecat" type="b2602c69:ProjectViewProjectNode" />
180 |               <item name="RuleCat" type="462c0819:PsiDirectoryNode" />
181 |             </path>
182 |             <path>
183 |               <item name="rulecat" type="b2602c69:ProjectViewProjectNode" />
184 |               <item name="RuleCat" type="462c0819:PsiDirectoryNode" />
185 |               <item name="app" type="462c0819:PsiDirectoryNode" />
186 |             </path>
187 |             <path>
188 |               <item name="rulecat" type="b2602c69:ProjectViewProjectNode" />
189 |               <item name="RuleCat" type="462c0819:PsiDirectoryNode" />
190 |               <item name="app" type="462c0819:PsiDirectoryNode" />
191 |               <item name="customf" type="462c0819:PsiDirectoryNode" />
192 |             </path>
193 |             <path>
194 |               <item name="rulecat" type="b2602c69:ProjectViewProjectNode" />
195 |               <item name="RuleCat" type="462c0819:PsiDirectoryNode" />
196 |               <item name="etc" type="462c0819:PsiDirectoryNode" />
197 |             </path>
198 |             <path>
199 |               <item name="rulecat" type="b2602c69:ProjectViewProjectNode" />
200 |               <item name="RuleCat" type="462c0819:PsiDirectoryNode" />
201 |               <item name="etc" type="462c0819:PsiDirectoryNode" />
202 |               <item name="rules" type="462c0819:PsiDirectoryNode" />
203 |             </path>
204 |             <path>
205 |               <item name="rulecat" type="b2602c69:ProjectViewProjectNode" />
206 |               <item name="RuleCat" type="462c0819:PsiDirectoryNode" />
207 |               <item name="etc" type="462c0819:PsiDirectoryNode" />
208 |               <item name="rules" type="462c0819:PsiDirectoryNode" />
209 |               <item name="conn" type="462c0819:PsiDirectoryNode" />
210 |             </path>
211 |             <path>
212 |               <item name="rulecat" type="b2602c69:ProjectViewProjectNode" />
213 |               <item name="RuleCat" type="462c0819:PsiDirectoryNode" />
214 |               <item name="utils" type="462c0819:PsiDirectoryNode" />
215 |             </path>
216 |             <path>
217 |               <item name="rulecat" type="b2602c69:ProjectViewProjectNode" />
218 |               <item name="RuleCat" type="462c0819:PsiDirectoryNode" />
219 |               <item name="utils" type="462c0819:PsiDirectoryNode" />
220 |               <item name="email" type="462c0819:PsiDirectoryNode" />
221 |             </path>
222 |           </expand>
223 |           <select />
224 |         </subPane>
225 |       </pane>
226 |     </panes>
227 |   </component>
228 |   <component name="ProjectViewState">
229 |     <option name="hideEmptyMiddlePackages" value="true" />
230 |     <option name="showLibraryContents" value="true" />
231 |   </component>
232 |   <component name="PropertiesComponent"><![CDATA[{
233 |   "keyToString": {
234 |     "RunOnceActivity.go.formatter.settings.were.checked": "true",
235 |     "RunOnceActivity.go.migrated.go.modules.settings": "true",
236 |     "RunOnceActivity.go.modules.go.list.on.any.changes.was.set": "true",
237 |     "WebServerToolWindowFactoryState": "false",
238 |     "go.import.settings.migrated": "true",
239 |     "go.sdk.automatically.set": "true",
240 |     "last_opened_file_path": "/Users/njcx/RuleCat/etc",
241 |     "node.js.detected.package.eslint": "true",
242 |     "node.js.selected.package.eslint": "(autodetect)",
243 |     "nodejs_package_manager_path": "npm",
244 |     "vue.rearranger.settings.migration": "true"
245 |   }
246 | }]]></component>
247 |   <component name="RecentsManager">
248 |     <key name="CopyFile.RECENT_KEYS">
249 |       <recent name="$PROJECT_DIR$/etc" />
250 |       <recent name="$PROJECT_DIR$/utils" />
251 |       <recent name="$PROJECT_DIR$/test" />
252 |       <recent name="$PROJECT_DIR$/etc/rules/ssh" />
253 |       <recent name="$PROJECT_DIR$/etc/rules/redis" />
254 |     </key>
255 |     <key name="MoveFile.RECENT_KEYS">
256 |       <recent name="$PROJECT_DIR$/doc/rules_tpl" />
257 |       <recent name="$PROJECT_DIR$/doc" />
258 |       <recent name="$PROJECT_DIR$/app/customf" />
259 |       <recent name="$PROJECT_DIR$/etc" />
260 |       <recent name="$PROJECT_DIR$" />
261 |     </key>
262 |   </component>
263 |   <component name="RunManager" selected="Go Build.go build main.go">
264 |     <configuration default="true" type="GoApplicationRunConfiguration" factoryName="Go Application">
265 |       <module name="rulecat" />
266 |       <working_directory value="$PROJECT_DIR$/" />
267 |       <go_parameters value="-i" />
268 |       <kind value="FILE" />
269 |       <package value="rule_engine_by_go" />
270 |       <directory value="$PROJECT_DIR$/" />
271 |       <filePath value="$PROJECT_DIR$/" />
272 |       <method v="2" />
273 |     </configuration>
274 |     <configuration name="go build es.go" type="GoApplicationRunConfiguration" factoryName="Go Application" temporary="true" nameIsGenerated="true">
275 |       <module name="rulecat" />
276 |       <working_directory value="$PROJECT_DIR$/" />
277 |       <kind value="FILE" />
278 |       <directory value="$PROJECT_DIR$/" />
279 |       <filePath value="$PROJECT_DIR$/test/es.go" />
280 |       <method v="2" />
281 |     </configuration>
282 |     <configuration name="go build kafkap.go" type="GoApplicationRunConfiguration" factoryName="Go Application" temporary="true" nameIsGenerated="true">
283 |       <module name="rulecat" />
284 |       <working_directory value="$PROJECT_DIR$/" />
285 |       <kind value="FILE" />
286 |       <package value="rule_engine_by_go" />
287 |       <directory value="$PROJECT_DIR$/" />
288 |       <filePath value="$PROJECT_DIR$/test/kafkap.go" />
289 |       <method v="2" />
290 |     </configuration>
291 |     <configuration name="go build mail.go" type="GoApplicationRunConfiguration" factoryName="Go Application" temporary="true" nameIsGenerated="true">
292 |       <module name="rulecat" />
293 |       <working_directory value="$PROJECT_DIR$/" />
294 |       <kind value="FILE" />
295 |       <directory value="$PROJECT_DIR$/" />
296 |       <filePath value="$PROJECT_DIR$/test/mail.go" />
297 |       <method v="2" />
298 |     </configuration>
299 |     <configuration name="go build main.go" type="GoApplicationRunConfiguration" factoryName="Go Application" temporary="true" nameIsGenerated="true">
300 |       <module name="rulecat" />
301 |       <working_directory value="$PROJECT_DIR$/" />
302 |       <kind value="FILE" />
303 |       <directory value="$PROJECT_DIR$/" />
304 |       <filePath value="$PROJECT_DIR$/main.go" />
305 |       <method v="2" />
306 |     </configuration>
307 |     <configuration name="go build yaml.go" type="GoApplicationRunConfiguration" factoryName="Go Application" temporary="true" nameIsGenerated="true">
308 |       <module name="rulecat" />
309 |       <working_directory value="$PROJECT_DIR$/" />
310 |       <kind value="FILE" />
311 |       <package value="rule_engine_by_go" />
312 |       <directory value="$PROJECT_DIR$/" />
313 |       <filePath value="$PROJECT_DIR$/test/yaml.go" />
314 |       <method v="2" />
315 |     </configuration>
316 |     <configuration default="true" type="GoTestRunConfiguration" factoryName="Go Test">
317 |       <working_directory value="$PROJECT_DIR$" />
318 |       <go_parameters value="-i" />
319 |       <kind value="DIRECTORY" />
320 |       <package value="rule_engine_by_go" />
321 |       <directory value="$PROJECT_DIR$" />
322 |       <filePath value="$PROJECT_DIR$" />
323 |       <framework value="gotest" />
324 |       <method v="2" />
325 |     </configuration>
326 |     <list>
327 |       <item itemvalue="Go Build.go build es.go" />
328 |       <item itemvalue="Go Build.go build kafkap.go" />
329 |       <item itemvalue="Go Build.go build mail.go" />
330 |       <item itemvalue="Go Build.go build main.go" />
331 |       <item itemvalue="Go Build.go build yaml.go" />
332 |     </list>
333 |     <recent_temporary>
334 |       <list>
335 |         <item itemvalue="Go Build.go build main.go" />
336 |         <item itemvalue="Go Build.go build es.go" />
337 |         <item itemvalue="Go Build.go build yaml.go" />
338 |         <item itemvalue="Go Build.go build kafkap.go" />
339 |         <item itemvalue="Go Build.go build mail.go" />
340 |       </list>
341 |     </recent_temporary>
342 |   </component>
343 |   <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
344 |   <component name="ToolWindowManager">
345 |     <frame x="-1920" y="0" width="1920" height="1080" extended-state="0" />
346 |     <editor active="true" />
347 |     <layout>
348 |       <window_info active="true" content_ui="combo" id="Project" order="0" sideWeight="0.87482804" visible="true" weight="0.20340788" />
349 |       <window_info id="Structure" order="1" sideWeight="0.12517194" side_tool="true" visible="true" weight="0.20340788" />
350 |       <window_info id="Favorites" order="2" side_tool="true" />
351 |       <window_info anchor="bottom" id="Message" order="0" />
352 |       <window_info anchor="bottom" id="Find" order="1" visible="true" weight="0.12673268" />
353 |       <window_info anchor="bottom" id="Run" order="2" visible="true" weight="0.280198" />
354 |       <window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
355 |       <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
356 |       <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
357 |       <window_info anchor="bottom" id="TODO" order="6" />
358 |       <window_info anchor="bottom" id="Docker" order="7" show_stripe_button="false" />
359 |       <window_info anchor="bottom" id="Database Changes" order="8" />
360 |       <window_info anchor="bottom" id="Version Control" order="9" weight="0.033663366" />
361 |       <window_info anchor="bottom" id="Terminal" order="10" weight="0.10700389" />
362 |       <window_info anchor="bottom" id="Event Log" order="11" side_tool="true" />
363 |       <window_info anchor="right" id="Cargo" />
364 |       <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
365 |       <window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
366 |       <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
367 |       <window_info anchor="right" id="Database" order="3" />
368 |       <window_info anchor="right" id="Translate" order="4" />
369 |       <window_info anchor="right" id="Word Book" order="5" side_tool="true" />
370 |     </layout>
371 |   </component>
372 |   <component name="TypeScriptGeneratedFilesManager">
373 |     <option name="version" value="3" />
374 |   </component>
375 |   <component name="VcsManagerConfiguration">
376 |     <MESSAGE value="nm" />
377 |     <MESSAGE value="Default Changelist" />
378 |     <MESSAGE value="add json log out" />
379 |     <MESSAGE value="ADD README" />
380 |     <option name="LAST_COMMIT_MESSAGE" value="ADD README" />
381 |   </component>
382 |   <component name="VgoProject">
383 |     <settings-migrated>true</settings-migrated>
384 |   </component>
385 |   <component name="editorHistoryManager">
386 |     <entry file="file:///usr/local/Cellar/go/1.16.2/libexec/src/html/template/template.go">
387 |       <provider selected="true" editor-type-id="text-editor">
388 |         <state relative-caret-position="257">
389 |           <caret line="119" column="19" selection-start-line="119" selection-start-column="19" selection-end-line="119" selection-end-column="19" />
390 |         </state>
391 |       </provider>
392 |     </entry>
393 |     <entry file="file:///usr/local/Cellar/go/1.16.2/libexec/src/sync/map.go">
394 |       <provider selected="true" editor-type-id="text-editor">
395 |         <state relative-caret-position="-5907">
396 |           <caret line="26" column="5" selection-start-line="26" selection-start-column="5" selection-end-line="26" selection-end-column="5" />
397 |         </state>
398 |       </provider>
399 |     </entry>
400 |     <entry file="file://$PROJECT_DIR$/doc/README.md">
401 |       <provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
402 |         <state split_layout="SPLIT">
403 |           <first_editor />
404 |           <second_editor />
405 |         </state>
406 |       </provider>
407 |     </entry>
408 |     <entry file="file:///usr/local/Cellar/go/1.16.2/libexec/src/html/template/content.go">
409 |       <provider selected="true" editor-type-id="text-editor">
410 |         <state relative-caret-position="-3585">
411 |           <caret line="6" column="7" lean-forward="true" selection-start-line="6" selection-start-column="7" selection-end-line="6" selection-end-column="7" />
412 |           <folding>
413 |             <element signature="e#178#206#0" expanded="true" />
414 |           </folding>
415 |         </state>
416 |       </provider>
417 |     </entry>
418 |     <entry file="file://$PROJECT_DIR$/utils/redis/redis.go">
419 |       <provider selected="true" editor-type-id="text-editor" />
420 |     </entry>
421 |     <entry file="file://$PROJECT_DIR$/utils/mq/rabbitmq.go">
422 |       <provider selected="true" editor-type-id="text-editor" />
423 |     </entry>
424 |     <entry file="file://$PROJECT_DIR$/utils/es/es.go">
425 |       <provider selected="true" editor-type-id="text-editor">
426 |         <state relative-caret-position="315">
427 |           <caret line="134" lean-forward="true" selection-start-line="134" selection-end-line="134" />
428 |           <folding>
429 |             <element signature="e#12#47#0" expanded="true" />
430 |           </folding>
431 |         </state>
432 |       </provider>
433 |     </entry>
434 |     <entry file="file://$PROJECT_DIR$/test/re.go">
435 |       <provider selected="true" editor-type-id="text-editor">
436 |         <state relative-caret-position="-2229" />
437 |       </provider>
438 |     </entry>
439 |     <entry file="file://$PROJECT_DIR$/test/re2.go">
440 |       <provider selected="true" editor-type-id="text-editor" />
441 |     </entry>
442 |     <entry file="file://$PROJECT_DIR$/test/redis.go">
443 |       <provider selected="true" editor-type-id="text-editor">
444 |         <state relative-caret-position="-3540" />
445 |       </provider>
446 |     </entry>
447 |     <entry file="file://$PROJECT_DIR$/etc/rules/dns/1.yml">
448 |       <provider selected="true" editor-type-id="text-editor">
449 |         <state relative-caret-position="253">
450 |           <caret line="11" column="21" selection-start-line="11" selection-start-column="21" selection-end-line="11" selection-end-column="21" />
451 |         </state>
452 |       </provider>
453 |     </entry>
454 |     <entry file="file://$PROJECT_DIR$/etc/rules/dns/2.yml">
455 |       <provider selected="true" editor-type-id="text-editor">
456 |         <state relative-caret-position="253">
457 |           <caret line="11" column="21" selection-start-line="11" selection-start-column="21" selection-end-line="11" selection-end-column="21" />
458 |         </state>
459 |       </provider>
460 |     </entry>
461 |     <entry file="file://$PROJECT_DIR$/etc/rules/dns/3.yml">
462 |       <provider selected="true" editor-type-id="text-editor">
463 |         <state relative-caret-position="253">
464 |           <caret line="11" column="21" selection-start-line="11" selection-start-column="21" selection-end-line="11" selection-end-column="21" />
465 |         </state>
466 |       </provider>
467 |     </entry>
468 |     <entry file="file://$PROJECT_DIR$/etc/rules/dns/4.yml">
469 |       <provider selected="true" editor-type-id="text-editor">
470 |         <state relative-caret-position="253">
471 |           <caret line="11" column="21" selection-start-line="11" selection-start-column="21" selection-end-line="11" selection-end-column="21" />
472 |         </state>
473 |       </provider>
474 |     </entry>
475 |     <entry file="file://$PROJECT_DIR$/etc/rules/http/1.yml">
476 |       <provider selected="true" editor-type-id="text-editor">
477 |         <state relative-caret-position="253">
478 |           <caret line="11" column="6" selection-start-line="11" selection-end-line="11" selection-end-column="21" />
479 |         </state>
480 |       </provider>
481 |     </entry>
482 |     <entry file="file://$PROJECT_DIR$/etc/rules/http/3.yml">
483 |       <provider selected="true" editor-type-id="text-editor" />
484 |     </entry>
485 |     <entry file="file:///usr/local/Cellar/go/1.16.2/libexec/src/bufio/bufio.go">
486 |       <provider selected="true" editor-type-id="text-editor">
487 |         <state relative-caret-position="256">
488 |           <caret line="702" column="17" selection-start-line="702" selection-start-column="17" selection-end-line="702" selection-end-column="17" />
489 |         </state>
490 |       </provider>
491 |     </entry>
492 |     <entry file="file://$PROJECT_DIR$/etc/rules/conn/2.yml">
493 |       <provider selected="true" editor-type-id="text-editor">
494 |         <state relative-caret-position="253">
495 |           <caret line="11" column="21" selection-start-line="11" selection-start-column="21" selection-end-line="11" selection-end-column="21" />
496 |         </state>
497 |       </provider>
498 |     </entry>
499 |     <entry file="file://$PROJECT_DIR$/etc/rules/conn/3.yml">
500 |       <provider selected="true" editor-type-id="text-editor">
501 |         <state relative-caret-position="-108">
502 |           <caret column="10" selection-start-column="10" selection-end-column="10" />
503 |         </state>
504 |       </provider>
505 |     </entry>
506 |     <entry file="file://$PROJECT_DIR$/etc/rules/conn/4.yml">
507 |       <provider selected="true" editor-type-id="text-editor">
508 |         <state relative-caret-position="-131">
509 |           <caret column="10" selection-start-column="10" selection-end-column="10" />
510 |         </state>
511 |       </provider>
512 |     </entry>
513 |     <entry file="file://$PROJECT_DIR$/utils/kafka/producer.go">
514 |       <provider selected="true" editor-type-id="text-editor">
515 |         <state relative-caret-position="40">
516 |           <caret line="34" column="17" selection-start-line="34" selection-start-column="17" selection-end-line="34" selection-end-column="17" />
517 |         </state>
518 |       </provider>
519 |     </entry>
520 |     <entry file="file://$PROJECT_DIR$/utils/es/elastic7.go">
521 |       <provider selected="true" editor-type-id="text-editor">
522 |         <state relative-caret-position="811">
523 |           <caret line="104" column="23" selection-start-line="99" selection-end-line="129" />
524 |           <folding>
525 |             <element signature="e#12#117#0" expanded="true" />
526 |           </folding>
527 |         </state>
528 |       </provider>
529 |     </entry>
530 |     <entry file="file://$PROJECT_DIR$/utils/json/json.go">
531 |       <provider selected="true" editor-type-id="text-editor">
532 |         <state relative-caret-position="-18286">
533 |           <caret line="5" column="9" selection-start-line="5" selection-start-column="5" selection-end-line="5" selection-end-column="23" />
534 |         </state>
535 |       </provider>
536 |     </entry>
537 |     <entry file="file://$PROJECT_DIR$/test/cache.go">
538 |       <provider selected="true" editor-type-id="text-editor">
539 |         <state relative-caret-position="46">
540 |           <caret line="2" column="7" selection-start-line="2" selection-start-column="7" selection-end-line="2" selection-end-column="7" />
541 |           <folding>
542 |             <element signature="e#14#62#0" expanded="true" />
543 |           </folding>
544 |         </state>
545 |       </provider>
546 |     </entry>
547 |     <entry file="file://$PROJECT_DIR$/test/engine.go">
548 |       <provider selected="true" editor-type-id="text-editor" />
549 |     </entry>
550 |     <entry file="file://$PROJECT_DIR$/test/es.go">
551 |       <provider selected="true" editor-type-id="text-editor">
552 |         <state relative-caret-position="46">
553 |           <caret line="2" column="7" lean-forward="true" selection-start-line="2" selection-start-column="7" selection-end-line="2" selection-end-column="7" />
554 |           <folding>
555 |             <element signature="e#14#46#0" expanded="true" />
556 |           </folding>
557 |         </state>
558 |       </provider>
559 |     </entry>
560 |     <entry file="file://$PROJECT_DIR$/test/json.go">
561 |       <provider selected="true" editor-type-id="text-editor" />
562 |     </entry>
563 |     <entry file="file://$PROJECT_DIR$/test/kafkaC.go">
564 |       <provider selected="true" editor-type-id="text-editor">
565 |         <state relative-caret-position="46">
566 |           <caret line="2" column="7" lean-forward="true" selection-start-line="2" selection-start-column="7" selection-end-line="2" selection-end-column="7" />
567 |           <folding>
568 |             <element signature="e#14#54#0" expanded="true" />
569 |           </folding>
570 |         </state>
571 |       </provider>
572 |     </entry>
573 |     <entry file="file://$PROJECT_DIR$/test/mail.go">
574 |       <provider selected="true" editor-type-id="text-editor" />
575 |     </entry>
576 |     <entry file="file://$PROJECT_DIR$/test/mq.go">
577 |       <provider selected="true" editor-type-id="text-editor" />
578 |     </entry>
579 |     <entry file="file://$PROJECT_DIR$/test/mq1.go">
580 |       <provider selected="true" editor-type-id="text-editor" />
581 |     </entry>
582 |     <entry file="file://$PROJECT_DIR$/test/workerpool.go">
583 |       <provider selected="true" editor-type-id="text-editor">
584 |         <state relative-caret-position="46">
585 |           <caret line="2" column="7" selection-start-line="2" selection-start-column="7" selection-end-line="2" selection-end-column="7" />
586 |           <folding>
587 |             <element signature="e#14#97#0" expanded="true" />
588 |           </folding>
589 |         </state>
590 |       </provider>
591 |     </entry>
592 |     <entry file="file://$PROJECT_DIR$/test/yaml.go">
593 |       <provider selected="true" editor-type-id="text-editor">
594 |         <state relative-caret-position="46">
595 |           <caret line="2" column="7" selection-start-line="2" selection-start-column="7" selection-end-line="2" selection-end-column="7" />
596 |           <folding>
597 |             <element signature="e#14#109#0" expanded="true" />
598 |           </folding>
599 |         </state>
600 |       </provider>
601 |     </entry>
602 |     <entry file="file://$PROJECT_DIR$/utils/es/elastic6.go">
603 |       <provider selected="true" editor-type-id="text-editor">
604 |         <state relative-caret-position="52">
605 |           <caret line="121" column="35" selection-start-line="121" selection-start-column="35" selection-end-line="121" selection-end-column="35" />
606 |         </state>
607 |       </provider>
608 |     </entry>
609 |     <entry file="file://$PROJECT_DIR$/utils/log/log.go">
610 |       <provider selected="true" editor-type-id="text-editor">
611 |         <state relative-caret-position="46">
612 |           <caret line="2" column="7" lean-forward="true" selection-start-line="2" selection-start-column="7" selection-end-line="2" selection-end-column="7" />
613 |           <folding>
614 |             <element signature="e#13#59#0" expanded="true" />
615 |           </folding>
616 |         </state>
617 |       </provider>
618 |     </entry>
619 |     <entry file="file://$PROJECT_DIR$/utils/workerpool/workerpool.go">
620 |       <provider selected="true" editor-type-id="text-editor">
621 |         <state relative-caret-position="46">
622 |           <caret line="2" column="10" lean-forward="true" selection-start-line="2" selection-start-column="7" selection-end-line="2" selection-end-column="23" />
623 |         </state>
624 |       </provider>
625 |     </entry>
626 |     <entry file="file://$PROJECT_DIR$/utils/utils.go">
627 |       <provider selected="true" editor-type-id="text-editor">
628 |         <state relative-caret-position="2553">
629 |           <caret line="111" column="5" selection-start-line="111" selection-start-column="5" selection-end-line="111" selection-end-column="5" />
630 |           <folding>
631 |             <element signature="e#15#159#0" expanded="true" />
632 |           </folding>
633 |         </state>
634 |       </provider>
635 |     </entry>
636 |     <entry file="file://$PROJECT_DIR$/go.sum">
637 |       <provider selected="true" editor-type-id="text-editor" />
638 |     </entry>
639 |     <entry file="file://$PROJECT_DIR$/go.mod">
640 |       <provider selected="true" editor-type-id="text-editor">
641 |         <state>
642 |           <caret column="10" selection-start-column="7" selection-end-column="14" />
643 |         </state>
644 |       </provider>
645 |     </entry>
646 |     <entry file="file://$PROJECT_DIR$/etc/rules/conn/1.yml">
647 |       <provider selected="true" editor-type-id="text-editor">
648 |         <state relative-caret-position="184">
649 |           <caret line="8" selection-start-line="8" selection-end-line="19" selection-end-column="13" />
650 |         </state>
651 |       </provider>
652 |     </entry>
653 |     <entry file="file://$PROJECT_DIR$/app/customf/cfuc.go">
654 |       <provider selected="true" editor-type-id="text-editor">
655 |         <state relative-caret-position="391">
656 |           <caret line="17" selection-start-line="17" selection-end-line="17" />
657 |         </state>
658 |       </provider>
659 |     </entry>
660 |     <entry file="file://$PROJECT_DIR$/test/kafkap.go">
661 |       <provider selected="true" editor-type-id="text-editor">
662 |         <state relative-caret-position="46">
663 |           <caret line="2" column="7" selection-start-line="2" selection-start-column="7" selection-end-line="2" selection-end-column="7" />
664 |           <folding>
665 |             <element signature="e#14#54#0" expanded="true" />
666 |           </folding>
667 |         </state>
668 |       </provider>
669 |     </entry>
670 |     <entry file="file://$PROJECT_DIR$/utils/cache/cache.go">
671 |       <provider selected="true" editor-type-id="text-editor">
672 |         <state relative-caret-position="966">
673 |           <caret line="50" column="16" selection-start-line="50" selection-start-column="16" selection-end-line="50" selection-end-column="16" />
674 |         </state>
675 |       </provider>
676 |     </entry>
677 |     <entry file="file://$PROJECT_DIR$/logs/nids.log">
678 |       <provider selected="true" editor-type-id="text-editor" />
679 |     </entry>
680 |     <entry file="file://$PROJECT_DIR$/utils/kafka/consumer.go">
681 |       <provider selected="true" editor-type-id="text-editor">
682 |         <state relative-caret-position="92">
683 |           <caret line="9" selection-start-line="9" selection-end-line="16" selection-end-column="1" />
684 |           <folding>
685 |             <element signature="e#15#105#0" expanded="true" />
686 |           </folding>
687 |         </state>
688 |       </provider>
689 |     </entry>
690 |     <entry file="file://$PROJECT_DIR$/main.go">
691 |       <provider selected="true" editor-type-id="text-editor">
692 |         <state relative-caret-position="138">
693 |           <caret line="6" column="14" selection-start-line="6" selection-start-column="14" selection-end-line="6" selection-end-column="14" />
694 |         </state>
695 |       </provider>
696 |     </entry>
697 |     <entry file="file://$PROJECT_DIR$/app/engine.go">
698 |       <provider selected="true" editor-type-id="text-editor">
699 |         <state relative-caret-position="46">
700 |           <caret line="2" column="7" selection-start-line="2" selection-start-column="7" selection-end-line="2" selection-end-column="7" />
701 |           <folding>
702 |             <element signature="e#13#263#0" expanded="true" />
703 |           </folding>
704 |         </state>
705 |       </provider>
706 |     </entry>
707 |     <entry file="file://$PROJECT_DIR$/app/app.go">
708 |       <provider selected="true" editor-type-id="text-editor">
709 |         <state relative-caret-position="-700">
710 |           <caret line="110" selection-start-line="110" selection-end-line="110" />
711 |           <folding>
712 |             <element signature="e#13#199#0" expanded="true" />
713 |           </folding>
714 |         </state>
715 |       </provider>
716 |     </entry>
717 |     <entry file="file://$PROJECT_DIR$/etc/config.yml">
718 |       <provider selected="true" editor-type-id="text-editor">
719 |         <state relative-caret-position="-1022">
720 |           <caret line="5" column="6" lean-forward="true" selection-end-line="39" />
721 |         </state>
722 |       </provider>
723 |     </entry>
724 |     <entry file="file://$PROJECT_DIR$/utils/email/email.go">
725 |       <provider selected="true" editor-type-id="text-editor">
726 |         <state relative-caret-position="414">
727 |           <caret line="18" column="5" selection-start-line="18" selection-start-column="5" selection-end-line="18" selection-end-column="5" />
728 |           <folding>
729 |             <element signature="e#15#96#0" expanded="true" />
730 |           </folding>
731 |         </state>
732 |       </provider>
733 |     </entry>
734 |     <entry file="file://$PROJECT_DIR$/README.md">
735 |       <provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
736 |         <state split_layout="SPLIT">
737 |           <first_editor relative-caret-position="-142">
738 |             <caret line="119" column="22" selection-start-line="119" selection-start-column="22" selection-end-line="119" selection-end-column="22" />
739 |           </first_editor>
740 |           <second_editor />
741 |         </state>
742 |       </provider>
743 |     </entry>
744 |   </component>
745 | </project>


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # RuleCat
 2 | 
 3 | 
 4 | GO开发而成，用于NIDS HIDS 分析的规则引擎，使用WorkerPool 高性能检测，支持正则、子串、等于，支持多字段 "和" "或" 检测， 支持频率检测,
 5 | 支持自定义函数检测,自定义函数可以满足几乎所有数据类型的检测
 6 | 
 7 | 
 8 | 
 9 | 输入：
10 | 
11 | Kafka(Json)
12 | 
13 | 
14 | 输出： 
15 | 
16 | E-Mail，ES， Kafka，Json 文件
17 | 
18 | 
19 | 
20 | ![示例](img.png)
21 | 
22 | 
23 | 
24 | 


--------------------------------------------------------------------------------
/app/app.go:
--------------------------------------------------------------------------------
  1 | package app
  2 | 
  3 | import (
  4 | 	"bytes"
  5 | 	"gopkg.in/yaml.v2"
  6 | 	"html/template"
  7 | 	"os"
  8 | 	"path/filepath"
  9 | 	"rulecat/utils"
 10 | 	"rulecat/utils/email"
 11 | 	"rulecat/utils/es"
 12 | 	"rulecat/utils/kafka"
 13 | 	log2 "rulecat/utils/log"
 14 | )
 15 | 
 16 | var (
 17 | 	configFile  []byte
 18 | 	emailSender *email.EmailConf
 19 | 	kafkaP      *kafka.DataProducer
 20 | 	ConfigG     Config
 21 | 	esSvc       *es.ElasticSearchService
 22 | )
 23 | 
 24 | type Input struct {
 25 | 	Kafka struct {
 26 | 		Enabled bool     `yaml:"enabled"`
 27 | 		Server  []string `yaml:"server"`
 28 | 		GroupId string   `yaml:"group_id"`
 29 | 		User    string   `yaml:"user"`
 30 | 		Passwd  string   `yaml:"passwd"`
 31 | 	}
 32 | }
 33 | 
 34 | type Output struct {
 35 | 	Es struct {
 36 | 		Enabled bool     `yaml:"enabled"`
 37 | 		Server  []string `yaml:"es_host"`
 38 | 		Version int      `yaml:"version"`
 39 | 		User    string   `yaml:"user"`
 40 | 		Passwd  string   `yaml:"passwd"`
 41 | 	}
 42 | 	Kafka struct {
 43 | 		Enabled bool     `yaml:"enabled"`
 44 | 		Server  []string `yaml:"server"`
 45 | 		Topic   string   `yaml:"topic"`
 46 | 		GroupId string   `yaml:"group_id"`
 47 | 		User    string   `yaml:"user"`
 48 | 		Passwd  string   `yaml:"passwd"`
 49 | 	}
 50 | 
 51 | 	Json struct {
 52 | 		Enabled bool   `yaml:"enabled"`
 53 | 		Path    string `yaml:"path"`
 54 | 		Name    string `yaml:"name"`
 55 | 	}
 56 | 	Email struct {
 57 | 		Enabled       bool   `yaml:"enabled"`
 58 | 		EmailHost     string `yaml:"email_host"`
 59 | 		EmailSmtpPort int    `yaml:"email_smtp_port"`
 60 | 		EmailFrom     string `yaml:"email_from"`
 61 | 		EmailUserName string `yaml:"email_username"`
 62 | 		EmailPwd      string `yaml:"email_pwd"`
 63 | 	}
 64 | }
 65 | 
 66 | type Config struct {
 67 | 	Name   string `yaml:"name"`
 68 | 	Env    string `yaml:"env"`
 69 | 	InPut  Input  `yaml:"input"`
 70 | 	OutPut Output `yaml:"output"`
 71 | }
 72 | 
 73 | func init() {
 74 | 	var err error
 75 | 	utils.ServerBanner()
 76 | 	configFile, err = os.ReadFile(utils.GetCurrentPath() + "/etc/config.yml")
 77 | 	if err != nil {
 78 | 		log2.Error.Fatalf("Get yml file err %v ", err)
 79 | 	}
 80 | 	err = yaml.Unmarshal(configFile, &ConfigG)
 81 | 	if err != nil {
 82 | 		log2.Error.Fatalf("Unmarshal yml file err: %v ", err)
 83 | 	}
 84 | 
 85 | 	if ConfigG.OutPut.Email.Enabled {
 86 | 		emailSender, err = email.New(ConfigG.OutPut.Email.EmailHost, ConfigG.OutPut.Email.EmailSmtpPort,
 87 | 			ConfigG.OutPut.Email.EmailFrom,
 88 | 			ConfigG.OutPut.Email.EmailUserName, ConfigG.OutPut.Email.EmailPwd)
 89 | 		if err != nil {
 90 | 			log2.Error.Fatalf("Create emailSender err: %v ", err)
 91 | 		}
 92 | 	}
 93 | 	if ConfigG.OutPut.Kafka.Enabled {
 94 | 		kafkaP = kafka.InitKafkaProducer(ConfigG.OutPut.Kafka.Server,
 95 | 			ConfigG.OutPut.Kafka.GroupId, ConfigG.OutPut.Kafka.Topic,
 96 | 			ConfigG.OutPut.Kafka.User, ConfigG.OutPut.Kafka.Passwd)
 97 | 
 98 | 	}
 99 | 
100 | 	if ConfigG.OutPut.Es.Enabled {
101 | 		esConf := es.ElasticConfig{Url: ConfigG.OutPut.Es.Server,
102 | 			User:   ConfigG.OutPut.Es.User,
103 | 			Secret: ConfigG.OutPut.Es.Passwd,
104 | 			Sniff:  new(bool)}
105 | 		esSvc, err = es.CreateElasticSearchService(esConf, ConfigG.OutPut.Es.Version)
106 | 		if err != nil {
107 | 			log2.Error.Fatalf("Create elastic search service err: %v ", err)
108 | 		}
109 | 	}
110 | }
111 | 
112 | func SendMail(data []byte) {
113 | 	if ConfigG.OutPut.Email.Enabled {
114 | 		tmp := `<html>
115 | 		<head>
116 | 			<meta charset="utf-8">
117 | 			<style>
118 | 				body {
119 | 					font-family: Arial, sans-serif;
120 | 					margin: 0;
121 | 					padding: 20px;
122 | 					background-color: #f4f4f9;
123 | 					color: #333;
124 | 				}
125 | 				h3 {
126 | 					margin: 10px 0;
127 | 					font-weight: bold;
128 | 				}
129 | 				pre {
130 | 					background-color: #f9f9f9;
131 | 					border: 1px solid #ddd;
132 | 					padding: 10px;
133 | 					border-radius: 5px;
134 | 					white-space: pre-wrap;
135 | 					word-wrap: break-word;
136 | 					margin-top: 10px;
137 | 				}
138 | 				.container {
139 | 					max-width: 800px;
140 | 					margin: 0 auto;
141 | 					background-color: #fff;
142 | 					padding: 20px;
143 | 					border-radius: 8px;
144 | 					box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
145 | 				}
146 | 				.header {
147 | 					text-align: center;
148 | 					margin-bottom: 20px;
149 | 				}
150 | 				.header h2 {
151 | 					margin: 0;
152 | 					font-size: 24px;
153 | 					color: #333;
154 | 				}
155 | 				.section {
156 | 					margin-bottom: 20px;
157 | 				}
158 | 				.section h3 {
159 | 					margin-top: 0;
160 | 				}
161 | 			</style>
162 | 		</head>
163 | 		<body>
164 | 			<div class="container">
165 | 				<div class="header">
166 | 					<h2>Alert Notification</h2>
167 | 				</div>
168 | 				<div class="section">
169 | 					<h3>rule name: {{.Data_.rule_name}}</h3>
170 | 					<h3>rule id: {{.Data_.rule_id}}</h3>
171 | 					<h3>rule tag: {{.Data_.rule_tag}}</h3>
172 | 					<h3>rule type: {{.Data_.rule_type}}</h3>
173 | 					<h3>threat level: {{.Data_.threat_level}}</h3>
174 | 				</div>
175 | 				<div class="section">
176 | 					<h4>data:</h4>
177 | 					<pre id="out_pre">{{.Json}}</pre>
178 | 				</div>
179 | 			</div>
180 | 		</body>
181 | 		</html>`
182 | 
183 | 		type Args struct {
184 | 			Data_ map[string]interface{}
185 | 			Json  template.HTML
186 | 		}
187 | 
188 | 		Data := make(map[string]interface{})
189 | 		_ = Json1.Unmarshal(data, &Data)
190 | 		Data1 := Args{Data_: Data, Json: template.HTML(utils.FormatJson(Data["data"]))}
191 | 		t := template.Must(template.New("mail").Parse(tmp))
192 | 		var tpl bytes.Buffer
193 | 		err := t.Execute(&tpl, Data1)
194 | 		if err != nil {
195 | 			log2.Error.Printf("Email template parse err: %v ", err)
196 | 			err = nil
197 | 		}
198 | 		var eAddr []string
199 | 		for _, v := range Data["e-mail"].([]interface{}) {
200 | 			eAddr = append(eAddr, v.(string))
201 | 		}
202 | 		to := email.ToSomeBody{To: eAddr, Cc: eAddr}
203 | 		err = emailSender.SendEmail(&to, "Alert-"+Data["rule_name"].(string), tpl.String())
204 | 		if err != nil {
205 | 			log2.Error.Printf("Email send err: %v ", err)
206 | 		}
207 | 	}
208 | 
209 | }
210 | 
211 | func SendKafka(message []byte) {
212 | 
213 | 	if ConfigG.OutPut.Kafka.Enabled {
214 | 		err := kafkaP.AddMessage(message)
215 | 		if err != nil {
216 | 			log2.Error.Printf("Kafka message Delivery err: %v ", err)
217 | 		}
218 | 	}
219 | }
220 | 
221 | func SendEs(typeName string, namespace string, sinkData string) {
222 | 
223 | 	if ConfigG.OutPut.Es.Enabled {
224 | 		err := esSvc.AddBodyString(typeName, namespace, sinkData)
225 | 		if err != nil {
226 | 			log2.Error.Printf("Send es message err: %v ", err)
227 | 		}
228 | 	}
229 | }
230 | 
231 | func SendJson(message []byte) {
232 | 	if ConfigG.OutPut.Json.Enabled {
233 | 		filePath := filepath.Join(ConfigG.OutPut.Json.Path, ConfigG.OutPut.Json.Name)
234 | 
235 | 		dir := filepath.Dir(filePath)
236 | 		err := os.MkdirAll(dir, 0755)
237 | 		if err != nil {
238 | 			log2.Error.Printf("Failed to create directory: %v", err)
239 | 			return
240 | 		}
241 | 		err = utils.WriteFile(filePath, string(message)+"\n")
242 | 		if err != nil {
243 | 			log2.Error.Printf("Failed to write to file %s: %v", filePath, err)
244 | 			return
245 | 		}
246 | 
247 | 		log2.Info.Printf("Successfully wrote message to file %s", filePath)
248 | 	}
249 | }
250 | 


--------------------------------------------------------------------------------
/app/customfuc.go:
--------------------------------------------------------------------------------
  1 | package app
  2 | 
  3 | import (
  4 | 	"github.com/go-redis/redis"
  5 | 	log2 "rulecat/utils/log"
  6 | 	"sync"
  7 | )
  8 | 
  9 | /*
 10 | 
 11 | 引入的配置，在init函数里面初始化即可
 12 | 规则添加方式如下：
 13 | 
 14 |   - field: conn.dip
 15 |     type: customf
 16 |     rule: CheckIP
 17 | 
 18 | 然后，在handleMap里面注册一下就可以，
 19 | RegisterHandler("CheckIP", CheckIP)
 20 | 
 21 | 
 22 | 
 23 | */
 24 | 
 25 | var (
 26 | 	RedisClientIP  *redis.Client
 27 | 	RedisClientDNS *redis.Client
 28 | 	HandleMap      = make(map[string]func(interface{}) (bool, map[string]string))
 29 | 	mu             sync.RWMutex
 30 | )
 31 | 
 32 | func init() {
 33 | 
 34 | 	RedisClientIP = redis.NewClient(&redis.Options{
 35 | 		Addr:     "",
 36 | 		Password: "",
 37 | 		DB:       0,
 38 | 	})
 39 | 
 40 | 	RedisClientDNS = redis.NewClient(&redis.Options{
 41 | 		Addr:     "",
 42 | 		Password: "",
 43 | 		DB:       1,
 44 | 	})
 45 | 
 46 | 	RegisterHandler("CheckIP", CheckIP)
 47 | 	RegisterHandler("CheckDNS", CheckDNS)
 48 | 	RegisterHandler("CheckProto", CheckProto)
 49 | }
 50 | 
 51 | func RegisterHandler(name string, handler func(interface{}) (bool, map[string]string)) {
 52 | 	mu.Lock()
 53 | 	defer mu.Unlock()
 54 | 	HandleMap[name] = handler
 55 | }
 56 | 
 57 | func CheckIP(ip interface{}) (bool, map[string]string) {
 58 | 	ipStr, ok := ip.(string)
 59 | 	if !ok {
 60 | 		return false, nil
 61 | 	}
 62 | 
 63 | 	val2, err := RedisClientIP.Get(ipStr).Result()
 64 | 	if err != nil {
 65 | 		if err == redis.Nil {
 66 | 			return false, nil
 67 | 		}
 68 | 		log2.Error.Println("Error fetching IP from Redis:", err)
 69 | 		return false, nil
 70 | 	}
 71 | 
 72 | 	if val2 != "" {
 73 | 		return true, map[string]string{
 74 | 			"ip_tag": val2,
 75 | 		}
 76 | 	}
 77 | 	return false, nil
 78 | }
 79 | 
 80 | func CheckDNS(dns interface{}) (bool, map[string]string) {
 81 | 	dnsStr, ok := dns.(string)
 82 | 	if !ok {
 83 | 		return false, nil
 84 | 	}
 85 | 
 86 | 	val2, err := RedisClientDNS.Get(dnsStr).Result()
 87 | 	if err != nil {
 88 | 		if err == redis.Nil {
 89 | 			return false, nil
 90 | 		}
 91 | 		log2.Error.Println("Error fetching DNS from Redis:", err)
 92 | 		return false, nil
 93 | 	}
 94 | 
 95 | 	if val2 != "" {
 96 | 		return true, map[string]string{
 97 | 			"dns_tag": val2,
 98 | 		}
 99 | 	}
100 | 	return false, nil
101 | }
102 | 
103 | func CheckProto(proto interface{}) (bool, map[string]string) {
104 | 
105 | 	protocolStr, ok := proto.(string)
106 | 	if !ok {
107 | 		return false, nil
108 | 	}
109 | 	if protocolStr == "UDP" {
110 | 		return true, map[string]string{
111 | 			"pro_tag": protocolStr,
112 | 		}
113 | 	}
114 | 	return false, nil
115 | 
116 | }
117 | 


--------------------------------------------------------------------------------
/app/engine.go:
--------------------------------------------------------------------------------
  1 | package app
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	jsoniter "github.com/json-iterator/go"
  6 | 	"gopkg.in/yaml.v2"
  7 | 	"os"
  8 | 	"regexp"
  9 | 	"rulecat/utils"
 10 | 	"rulecat/utils/cache"
 11 | 	"rulecat/utils/json"
 12 | 	log2 "rulecat/utils/log"
 13 | 	"rulecat/utils/workerpool"
 14 | 	"strconv"
 15 | 	"strings"
 16 | 	"sync"
 17 | 	"time"
 18 | )
 19 | 
 20 | var Json1 = jsoniter.ConfigCompatibleWithStandardLibrary
 21 | var Tc = cache.New(60*60*time.Second, 1*time.Second)
 22 | var RuleList []map[string]interface{}
 23 | 
 24 | type engine struct {
 25 | 	RuleType string
 26 | 	InPutC   chan string
 27 | 	OutPutC  chan *sync.Map
 28 | 	Json     string
 29 | }
 30 | 
 31 | func NewEngine(ruleType string) *engine {
 32 | 	return &engine{RuleType: ruleType, InPutC: make(chan string, 48), OutPutC: make(chan *sync.Map, 48)}
 33 | }
 34 | 
 35 | func (e *engine) ReadRules() {
 36 | 	var rulesPath []string
 37 | 	rulesListPath, err := utils.GetAllFile(utils.GetCurrentPath()+"/etc/"+e.RuleType+"_rules", rulesPath)
 38 | 	if err != nil {
 39 | 		log2.Error.Fatalf("Get rule file dir err %s  %v ", rulesListPath, err)
 40 | 	}
 41 | 	for _, ruleFile := range rulesListPath {
 42 | 		rule, err := os.ReadFile(ruleFile)
 43 | 		if err != nil {
 44 | 			log2.Error.Fatalf("Get rule file err  %s %v ", rule, err)
 45 | 		}
 46 | 		m := make(map[string]interface{})
 47 | 		err = yaml.Unmarshal(rule, &m)
 48 | 		if err != nil {
 49 | 			log2.Error.Fatalf("Unmarshal rule file err  %s %v ", rule, err)
 50 | 		}
 51 | 		RuleList = append(RuleList, m)
 52 | 	}
 53 | 	log2.Info.Printf("Rule file have been load %s ", e.RuleType)
 54 | 
 55 | }
 56 | 
 57 | func (e *engine) ResCheck(threadNum int, outPut chan *sync.Map) {
 58 | 	p := workerpool.NewWorkerPool(threadNum)
 59 | 	p.Run()
 60 | 	go func() {
 61 | 		for {
 62 | 			sc := &engine{Json: <-e.InPutC, OutPutC: outPut}
 63 | 			p.JobQueue <- sc
 64 | 		}
 65 | 	}()
 66 | }
 67 | 
 68 | func (e *engine) Do() error {
 69 | 	RuleCheckFuc(e.Json, RuleList, e.OutPutC)
 70 | 	return nil
 71 | }
 72 | 
 73 | func RuleCheckFuc(s string, r []map[string]interface{}, outPut chan *sync.Map) {
 74 | 	for _, rule := range r {
 75 | 		if rule["state"] != "enable" {
 76 | 			continue
 77 | 		}
 78 | 		detectList, infoMap := checkDetectList(s, rule)
 79 | 		handleRuleType(s, rule, detectList, infoMap, outPut)
 80 | 	}
 81 | }
 82 | 
 83 | func interfaceToString(value interface{}) string {
 84 | 	return fmt.Sprintf("%v", value)
 85 | }
 86 | 
 87 | func isArray(v interface{}) bool {
 88 | 	switch v.(type) {
 89 | 	case []int, []string, []float64:
 90 | 		return true
 91 | 	default:
 92 | 		return false
 93 | 	}
 94 | }
 95 | 
 96 | func convertToStringSlice(v interface{}) ([]string, error) {
 97 | 	switch v := v.(type) {
 98 | 	case []int:
 99 | 		result := make([]string, len(v))
100 | 		for i, val := range v {
101 | 			result[i] = strconv.Itoa(val)
102 | 		}
103 | 		return result, nil
104 | 	case []string:
105 | 		return v, nil
106 | 	case []float64:
107 | 		result := make([]string, len(v))
108 | 		for i, val := range v {
109 | 			result[i] = strconv.FormatFloat(val, 'f', -1, 64)
110 | 		}
111 | 		return result, nil
112 | 	default:
113 | 		return nil, fmt.Errorf("unsupported type: %T", v)
114 | 	}
115 | }
116 | 
117 | func isInList(list []string, target string) bool {
118 | 	for _, item := range list {
119 | 		if item == target {
120 | 			return true
121 | 		}
122 | 	}
123 | 	return false
124 | }
125 | 
126 | func ConvertMap(inputMap map[interface{}]interface{}) (map[string]interface{}, error) {
127 | 	convertedMap := make(map[string]interface{})
128 | 	for k, v := range inputMap {
129 | 		keyStr, ok := k.(string)
130 | 		if !ok {
131 | 			return nil, fmt.Errorf("key %v is not a string", k)
132 | 		}
133 | 		switch vTyped := v.(type) {
134 | 		case map[interface{}]interface{}:
135 | 			nestedMap, err := ConvertMap(vTyped)
136 | 			if err != nil {
137 | 				return nil, err
138 | 			}
139 | 			convertedMap[keyStr] = nestedMap
140 | 		case []interface{}:
141 | 			convertedSlice := make([]interface{}, len(vTyped))
142 | 			for i, item := range vTyped {
143 | 				if subMap, ok := item.(map[interface{}]interface{}); ok {
144 | 					nestedMap, err := ConvertMap(subMap)
145 | 					if err != nil {
146 | 						return nil, err
147 | 					}
148 | 					convertedSlice[i] = nestedMap
149 | 				} else {
150 | 					convertedSlice[i] = item
151 | 				}
152 | 			}
153 | 			convertedMap[keyStr] = convertedSlice
154 | 		default:
155 | 			convertedMap[keyStr] = vTyped
156 | 		}
157 | 	}
158 | 	return convertedMap, nil
159 | }
160 | 
161 | func convertToMapSlice(slice []interface{}) []map[string]interface{} {
162 | 	result := make([]map[string]interface{}, 0, len(slice))
163 | 	for _, item := range slice {
164 | 		if m, ok := item.(map[interface{}]interface{}); ok {
165 | 			convertedMap, err := ConvertMap(m)
166 | 			if err != nil {
167 | 				fmt.Printf("Skipping invalid element: %v (type: %T) due to error: %v\n", item, item, err)
168 | 				continue
169 | 			}
170 | 			result = append(result, convertedMap)
171 | 		} else {
172 | 			fmt.Printf("Skipping invalid element: %v (type: %T)\n", item, item)
173 | 		}
174 | 	}
175 | 	return result
176 | }
177 | 
178 | func checkDetectList(s string, rule map[string]interface{}) (int, map[string]string) {
179 | 	detectList := 0
180 | 	infoMap := make(map[string]string)
181 | 	for _, detectItem := range convertToMapSlice(rule["detect_list"].([]interface{})) {
182 | 		field := detectItem["field"].(string)
183 | 		value := json.Get(s, field).String()
184 | 		switch detectItem["type"].(string) {
185 | 		case "equal":
186 | 			if value == interfaceToString(detectItem["rule"]) {
187 | 				detectList++
188 | 			}
189 | 		case "re":
190 | 			pattern := detectItem["rule"].(string)
191 | 			if detectItem["ignore-case"].(bool) {
192 | 				match, _ := regexp.MatchString(`(?i)`+pattern, value)
193 | 				if match {
194 | 					detectList++
195 | 				}
196 | 			} else {
197 | 				match, _ := regexp.MatchString(pattern, value)
198 | 				if match {
199 | 					detectList++
200 | 				}
201 | 			}
202 | 		case "in":
203 | 			if isArray(detectItem["rule"]) {
204 | 				listTmp, err := convertToStringSlice(detectItem["rule"])
205 | 				if err != nil {
206 | 					continue
207 | 				}
208 | 				if isInList(listTmp, value) {
209 | 					detectList++
210 | 				}
211 | 			} else {
212 | 				if strings.Contains(value, detectItem["rule"].(string)) {
213 | 					detectList++
214 | 				}
215 | 			}
216 | 		case "customf":
217 | 			handelFuc := HandleMap[detectItem["rule"].(string)]
218 | 			successHit, tmpMap := handelFuc(value)
219 | 			if successHit {
220 | 				detectList++
221 | 			}
222 | 			infoMap = tmpMap
223 | 		}
224 | 	}
225 | 	return detectList, infoMap
226 | }
227 | 
228 | func handleRuleType(s string, rule map[string]interface{}, detectList int, infoMap map[string]string, outPut chan *sync.Map) {
229 | 	ruleType := rule["rule_type"].(string)
230 | 	detectListCount := len(rule["detect_list"].([]interface{}))
231 | 	switch ruleType {
232 | 	case "and":
233 | 		if detectListCount == detectList {
234 | 			sendResult(s, rule, infoMap, outPut)
235 | 		}
236 | 	case "or":
237 | 		if detectList > 0 {
238 | 			sendResult(s, rule, infoMap, outPut)
239 | 		}
240 | 	case "frequency_and", "frequency_or":
241 | 		if detectListCount == detectList || (ruleType == "frequency_or" && detectList > 0) {
242 | 			key := rule["key"].(string)
243 | 			handleFrequency(key, rule, s, infoMap, outPut)
244 | 		}
245 | 	}
246 | }
247 | 
248 | func sendResult(s string, rule map[string]interface{}, infoMap map[string]string, outPut chan *sync.Map) {
249 | 	var tmpMap map[string]interface{}
250 | 	_ = Json1.Unmarshal([]byte(s), &tmpMap)
251 | 	sMap, _ := utils.MapToSMap(rule)
252 | 	sMap.Store("data", tmpMap)
253 | 	sMap.Store("extra_message", infoMap)
254 | 	outPut <- sMap
255 | }
256 | 
257 | func handleFrequency(key string, rule map[string]interface{}, s string, infoMap map[string]string, outPut chan *sync.Map) {
258 | 	FrequencyMap, _ := ConvertMap(rule["time_interval"].(map[interface{}]interface{}))
259 | 	times := FrequencyMap["times"].(int)
260 | 	second := FrequencyMap["second"].(int)
261 | 	value, found := Tc.Get(key)
262 | 	if found {
263 | 		if value.(int) >= times {
264 | 			sendResult(s, rule, infoMap, outPut)
265 | 		} else {
266 | 			Tc.Increment(key, 1)
267 | 		}
268 | 	} else {
269 | 		Tc.Set(key, 1, time.Duration(second)*time.Second)
270 | 	}
271 | }
272 | 


--------------------------------------------------------------------------------
/doc/README.md:
--------------------------------------------------------------------------------
  1 | 规则编写
  2 | 
  3 | 
  4 | ```go
  5 | 
  6 | state:     enable                        //  规则状态 enable   disable 
  7 | rule_id :  sqli_get_01                   //  规则ID
  8 | rule_tag:  sqli                          //  规则标签
  9 | rule_name: sqli_get_select               //  规则名
 10 | 
 11 | rule_type:  or                           //  or 类型规则代表，detect_list里面命中任何一条，算命中
 12 | rule_type:  and                          //  and 类型规则代表，detect_list里面命中所有规则，算命中
 13 | rule_type:  frequency_or                 //  frequency_or 类型规则代表，detect_list里面命中任何一条，且以key计数，单位时间内达到计数值上限算命中
 14 | rule_type:  frequency_and                //  frequency_and 类型规则代表，detect_list里面命中所有规则，且以key计数，单位时间内达到计数值上限算命中
 15 | 
 16 | 
 17 | detect_list:
 18 | 
 19 |   - field : conn.conn_state              //  字段
 20 |     type: re                             //  正则
 21 |     rule: S0                             //  具体规则 
 22 | 	ignore-case: false                   //  是否忽略大小写
 23 | 
 24 |   - field : conn.proto                   //  字段
 25 |     type: equal                          //  等于
 26 |     rule : tcp                           //  具体规则
 27 | 
 28 |   - field : conn.conn_state              // 字段
 29 |     type: in                             // 判断是否为子串, 是否在list里面
 30 |     rule : S0                            // 具体规则
 31 |  
 32 |   - field: conn.ip                       // 字段
 33 |     type: customf                        // 自定义函数
 34 |     rule: CheckIP                        // 自定义函数名 
 35 | 
 36 | 
 37 | key : conn.id\.orig_h                     // 只有frequency 类型的有，以此字段对应数据为key计数
 38 | 
 39 | time_interval:                            // 只有frequency 类型的有，代表 10s内出现 10次
 40 |    second: 10                  
 41 |    times: 10
 42 | 
 43 | 
 44 | threat_level : high                       // 威胁等级
 45 | auth : njcx86                             // 作者
 46 | info : about sql injection attack         // 注释
 47 | 
 48 | e-mail:                                   // 告警发送的邮箱
 49 |     - 868726@gmail.com
 50 |     - njcx91@tom.com
 51 | 
 52 | 
 53 | ```
 54 | 
 55 | 
 56 | 取字段对应的方式如下：
 57 | ```go
 58 | 
 59 | {
 60 |   "name": {"first": "Tom", "last": "Anderson"},
 61 |   "age":37,
 62 |   "children": ["Sara","Alex","Jack"],
 63 |   "fav.movie": "Deer Hunter",
 64 |   "friends": [
 65 |     {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]},
 66 |     {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]},
 67 |     {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]}
 68 |   ]
 69 | }
 70 | 
 71 | ```
 72 | 
 73 | ```go
 74 | "name.last"          >> "Anderson"
 75 | "age"                >> 37
 76 | "children"           >> ["Sara","Alex","Jack"]
 77 | "children.#"         >> 3
 78 | "children.1"         >> "Alex"
 79 | "child*.2"           >> "Jack"
 80 | "c?ildren.0"         >> "Sara"
 81 | "fav\.movie"         >> "Deer Hunter"
 82 | "friends.#.first"    >> ["Dale","Roger","Jane"]
 83 | "friends.1.last"     >> "Craig"
 84 | 
 85 | ```
 86 | 
 87 | 
 88 | 
 89 | 
 90 | 目前支持四种输出：
 91 | 
 92 | ```bash
 93 | 
 94 | output:
 95 |   es:
 96 |     enabled : true
 97 |     es_host : ["http://10.210.228.22:9201", "http://10.210.228.52:9201"]
 98 |     version : 7
 99 |     user : "elastic"
100 |     passwd : "elastic"
101 | 
102 |   kafka:
103 |     enabled : true
104 |     server : ["10.210.228.22:9092"]
105 |     user : "producer"
106 |     passwd : "producer"
107 |     topic: test22
108 |     group_id: test4
109 | 
110 | 
111 |   email:
112 |     enabled: false
113 |     email_host: smtp.qq.com
114 |     email_smtp_port: 465
115 |     email_from: 123456@qq.com
116 |     email_username: 123456
117 |     email_pwd: 123456
118 | 
119 | 
120 |   json:
121 |     enabled : true
122 |     path : /Users/njcx/RuleCat/logs/
123 |     name : alert.log
124 | 
125 | ```


--------------------------------------------------------------------------------
/doc/rules_tpl/1.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : sqli_get_01
 3 | rule_tag: sqli
 4 | rule_name: sqli_get_select
 5 | rule_type: and
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field : name.first
10 |     type: re
11 |     rule: J
12 |     ignore-case: false
13 | 
14 |   - field : name.first
15 |     type: equal
16 |     rule : Janet
17 | 
18 |   - field : name.name
19 |     type: in
20 |     rule : Prichard
21 | 
22 | 
23 | threat_level : high
24 | auth : njcx
25 | info : about sql injection attack
26 | 
27 | e-mail:
28 |     - njcx91@tom.com
29 |     - njcx91@tom.com
30 | 
31 | 


--------------------------------------------------------------------------------
/doc/rules_tpl/2.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : sqli_get_01
 3 | rule_tag: sqli
 4 | rule_name: sqli_get_select
 5 | rule_type: or
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field : network.ip
10 |     type: re
11 |     rule: xxx
12 |     ignore-case: false
13 | 
14 |   - field : network.ip
15 |     type: equal
16 |     rule : xxx
17 | 
18 |   - field : network.ip,
19 |     type: in
20 |     rule : xxx
21 | 
22 | 
23 | threat_level : high
24 | auth : njcx
25 | info : about sql injection attack
26 | 
27 | e-mail:
28 |     - njcx91@tom.com
29 |     - njcx91@tom.com
30 | 
31 | 


--------------------------------------------------------------------------------
/doc/rules_tpl/3.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : sqli_get_01
 3 | rule_tag: sqli
 4 | rule_name: sqli_get_select
 5 | rule_type:  frequency_or   # frequency_and
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field : network.ip
10 |     type: re
11 |     rule: xxx
12 |     ignore-case: false
13 | 
14 |   - field : network.ip
15 |     type: equal
16 |     rule : xxx
17 | 
18 |   - field : network.ip,
19 |     type: in
20 |     rule : xxx
21 | 
22 | key : network.ip
23 | 
24 | time_interval:
25 |    second: 10
26 |    times: 100
27 | 
28 | 
29 | threat_level : high
30 | auth : njcx
31 | info : about sql injection attack
32 | 
33 | e-mail:
34 |     - njcx91@tom.com
35 |     - njcx91@tom.com
36 | 


--------------------------------------------------------------------------------
/doc/rules_tpl/4.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : sqli_get_01
 3 | rule_tag: sqli
 4 | rule_name: sqli_get_select
 5 | rule_type:  frequency_and   # frequency_and
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field : network.ip
10 |     type: re
11 |     rule: xxx
12 |     ignore-case: false
13 | 
14 |   - field : network.ip
15 |     type: equal
16 |     rule : xxx
17 | 
18 |   - field : network.ip,
19 |     type: in
20 |     rule : xxx
21 | 
22 | key : network.ip
23 | 
24 | time_interval:
25 |    second: 10
26 |    times: 100
27 | 
28 | 
29 | threat_level : high
30 | auth : njcx
31 | info : about sql injection attack
32 | 
33 | e-mail:
34 |     - njcx91@tom.com
35 |     - njcx91@tom.com
36 | 
37 | 


--------------------------------------------------------------------------------
/doc/rules_tpl/5.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : DNS命中威胁情报_ID_01
 3 | rule_tag: dns-threat
 4 | rule_name: DNS命中威胁情报
 5 | rule_type: or
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field: dns.rrname
10 |     type: customf
11 |     rule: CheckDNS
12 | 
13 | 
14 | threat_level : high
15 | auth : njcx
16 | info : about sql injection attack
17 | 
18 | e-mail:
19 |   - njcx91@tom.com
20 |   - njcx91@tom.com


--------------------------------------------------------------------------------
/etc/config.yml:
--------------------------------------------------------------------------------
 1 | 
 2 | name: rule_engine
 3 | 
 4 | env: dev  # 环境 dev demo prod
 5 | 
 6 | 
 7 | input:
 8 |   kafka:
 9 |     enabled : true
10 |     server : ["10.210.228.22:9092"]
11 |     user : "user"
12 |     passwd : "kafka_passwd"
13 |     group_id: kafka_input_group_id
14 | 
15 | 
16 | output:
17 |   es:
18 |     enabled : false
19 |     es_host : ["http://10.210.228.22:9201", "http://10.210.228.52:9201"]
20 |     version : 7
21 |     user : "elastic"
22 |     passwd : "es_passwd"
23 | 
24 |   kafka:
25 |     enabled : false
26 |     server : ["10.210.228.22:9092"]
27 |     user : "user"
28 |     passwd : "kafka_passwd"
29 |     topic: test22
30 |     group_id: output_1
31 | 
32 | 
33 |   email:
34 |     enabled: false
35 |     email_host: smtp.qq.com
36 |     email_smtp_port: 465
37 |     email_from: 123456@qq.com
38 |     email_username: 123456
39 |     email_pwd: 123456
40 | 
41 | 
42 |   json:
43 |     enabled : true
44 |     path : /Users/njcx/RuleCat/logs/
45 |     name : alert.log
46 | 
47 | 
48 | 
49 | 
50 | 
51 | 
52 | 


--------------------------------------------------------------------------------
/etc/nids-flow_rules/1.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : sqli_get_01
 3 | rule_tag: sqli
 4 | rule_name: sqli_get_select
 5 | rule_type: and
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field : name.first
10 |     type: re
11 |     rule: J
12 |     ignore-case: false
13 | 
14 |   - field : name.first
15 |     type: equal
16 |     rule : Janet
17 | 
18 |   - field : name.name
19 |     type: in
20 |     rule : Prichard
21 | 
22 | 
23 | threat_level : high
24 | auth : njcx
25 | info : about sql injection attack
26 | 
27 | e-mail:
28 |     - njcx91@tom.com
29 |     - njcx91@tom.com
30 | 
31 | 


--------------------------------------------------------------------------------
/etc/nids-flow_rules/2.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : sqli_get_01
 3 | rule_tag: sqli
 4 | rule_name: sqli_get_select
 5 | rule_type: or
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field : network.ip
10 |     type: re
11 |     rule: xxx
12 |     ignore-case: false
13 | 
14 |   - field : proto
15 |     type: equal
16 |     rule : xxx
17 | 
18 |   - field : network.ip,
19 |     type: in
20 |     rule : xxx
21 | 
22 |   - field: proto
23 |     type: customf
24 |     rule: CheckProto
25 | 
26 | 
27 | threat_level : high
28 | auth : njcx
29 | info : about sql injection attack
30 | 
31 | e-mail:
32 |     - njcx91@tom.com
33 |     - njcx91@tom.com
34 | 
35 | 


--------------------------------------------------------------------------------
/etc/nids-flow_rules/3.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : sqli_get_01
 3 | rule_tag: sqli
 4 | rule_name: sqli_get_select
 5 | rule_type:  frequency_or   # frequency_and
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field : network.ip
10 |     type: re
11 |     rule: xxx
12 |     ignore-case: false
13 | 
14 |   - field : proto
15 |     type: equal
16 |     rule : ICMP
17 | 
18 |   - field : network.ip,
19 |     type: in
20 |     rule : xxx
21 | 
22 | key : proto
23 | 
24 | time_interval:
25 |    second: 100
26 |    times: 500
27 | 
28 | 
29 | threat_level : high
30 | auth : njcx
31 | info : about sql injection attack
32 | 
33 | e-mail:
34 |     - njcx91@tom.com
35 |     - njcx91@tom.com
36 | 


--------------------------------------------------------------------------------
/etc/nids-flow_rules/4.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : sqli_get_01
 3 | rule_tag: sqli
 4 | rule_name: sqli_get_select
 5 | rule_type:  frequency_and   # frequency_and
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field : network.ip
10 |     type: re
11 |     rule: xxx
12 |     ignore-case: false
13 | 
14 |   - field : proto
15 |     type: equal
16 |     rule : ICMP
17 | 
18 |   - field : network.ip,
19 |     type: in
20 |     rule : xxx
21 | 
22 | key : network.ip
23 | 
24 | time_interval:
25 |    second: 100
26 |    times: 100
27 | 
28 | 
29 | threat_level : high
30 | auth : njcx
31 | info : about sql injection attack
32 | 
33 | e-mail:
34 |     - njcx91@tom.com
35 |     - njcx91@tom.com
36 | 
37 | 


--------------------------------------------------------------------------------
/etc/topic_tpl2_rules/1.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : sqli_get_01
 3 | rule_tag: sqli
 4 | rule_name: sqli_get_select
 5 | rule_type: and
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field : name.first
10 |     type: re
11 |     rule: J
12 |     ignore-case: false
13 | 
14 |   - field : name.first
15 |     type: equal
16 |     rule : Janet
17 | 
18 |   - field : name.name
19 |     type: in
20 |     rule : Prichard
21 | 
22 | 
23 | threat_level : high
24 | auth : njcx
25 | info : about sql injection attack
26 | 
27 | e-mail:
28 |     - njcx91@tom.com
29 |     - njcx91@tom.com
30 | 
31 | 


--------------------------------------------------------------------------------
/etc/topic_tpl2_rules/2.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : sqli_get_01
 3 | rule_tag: sqli
 4 | rule_name: sqli_get_select
 5 | rule_type: or
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field : network.ip
10 |     type: re
11 |     rule: xxx
12 |     ignore-case: false
13 | 
14 |   - field : network.ip
15 |     type: equal
16 |     rule : xxx
17 | 
18 |   - field : network.ip,
19 |     type: in
20 |     rule : xxx
21 | 
22 | 
23 | threat_level : high
24 | auth : njcx
25 | info : about sql injection attack
26 | 
27 | e-mail:
28 |     - njcx91@tom.com
29 |     - njcx91@tom.com
30 | 
31 | 


--------------------------------------------------------------------------------
/etc/topic_tpl2_rules/3.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : sqli_get_01
 3 | rule_tag: sqli
 4 | rule_name: sqli_get_select
 5 | rule_type:  frequency_or   # frequency_and
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field : network.ip
10 |     type: re
11 |     rule: xxx
12 |     ignore-case: false
13 | 
14 |   - field : network.ip
15 |     type: equal
16 |     rule : xxx
17 | 
18 |   - field : network.ip,
19 |     type: in
20 |     rule : xxx
21 | 
22 | key : network.ip
23 | 
24 | time_interval:
25 |    second: 10
26 |    times: 100
27 | 
28 | 
29 | threat_level : high
30 | auth : njcx
31 | info : about sql injection attack
32 | 
33 | e-mail:
34 |     - njcx91@tom.com
35 |     - njcx91@tom.com
36 | 


--------------------------------------------------------------------------------
/etc/topic_tpl2_rules/4.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : sqli_get_01
 3 | rule_tag: sqli
 4 | rule_name: sqli_get_select
 5 | rule_type:  frequency_and   # frequency_and
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field : network.ip
10 |     type: re
11 |     rule: xxx
12 |     ignore-case: false
13 | 
14 |   - field : network.ip
15 |     type: equal
16 |     rule : xxx
17 | 
18 |   - field : network.ip,
19 |     type: in
20 |     rule : xxx
21 | 
22 | key : network.ip
23 | 
24 | time_interval:
25 |    second: 10
26 |    times: 100
27 | 
28 | 
29 | threat_level : high
30 | auth : njcx
31 | info : about sql injection attack
32 | 
33 | e-mail:
34 |     - njcx91@tom.com
35 |     - njcx91@tom.com
36 | 
37 | 


--------------------------------------------------------------------------------
/etc/topic_tpl_rules/1.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : sqli_get_01
 3 | rule_tag: sqli
 4 | rule_name: sqli_get_select
 5 | rule_type: and
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field : name.first
10 |     type: re
11 |     rule: J
12 |     ignore-case: false
13 | 
14 |   - field : name.first
15 |     type: equal
16 |     rule : Janet
17 | 
18 |   - field : name.name
19 |     type: in
20 |     rule : Prichard
21 | 
22 | 
23 | threat_level : high
24 | auth : njcx
25 | info : about sql injection attack
26 | 
27 | e-mail:
28 |     - njcx91@tom.com
29 |     - njcx91@tom.com
30 | 
31 | 


--------------------------------------------------------------------------------
/etc/topic_tpl_rules/2.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : sqli_get_01
 3 | rule_tag: sqli
 4 | rule_name: sqli_get_select
 5 | rule_type: or
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field : network.ip
10 |     type: re
11 |     rule: xxx
12 |     ignore-case: false
13 | 
14 |   - field : network.ip
15 |     type: equal
16 |     rule : xxx
17 | 
18 |   - field : network.ip,
19 |     type: in
20 |     rule : xxx
21 | 
22 | 
23 | threat_level : high
24 | auth : njcx
25 | info : about sql injection attack
26 | 
27 | e-mail:
28 |     - njcx91@tom.com
29 |     - njcx91@tom.com
30 | 
31 | 


--------------------------------------------------------------------------------
/etc/topic_tpl_rules/3.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : sqli_get_01
 3 | rule_tag: sqli
 4 | rule_name: sqli_get_select
 5 | rule_type:  frequency_or   # frequency_and
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field : network.ip
10 |     type: re
11 |     rule: xxx
12 |     ignore-case: false
13 | 
14 |   - field : network.ip
15 |     type: equal
16 |     rule : xxx
17 | 
18 |   - field : network.ip,
19 |     type: in
20 |     rule : xxx
21 | 
22 | key : network.ip
23 | 
24 | time_interval:
25 |    second: 10
26 |    times: 100
27 | 
28 | 
29 | threat_level : high
30 | auth : njcx
31 | info : about sql injection attack
32 | 
33 | e-mail:
34 |     - njcx91@tom.com
35 |     - njcx91@tom.com
36 | 


--------------------------------------------------------------------------------
/etc/topic_tpl_rules/4.yml:
--------------------------------------------------------------------------------
 1 | state: enable
 2 | rule_id : sqli_get_01
 3 | rule_tag: sqli
 4 | rule_name: sqli_get_select
 5 | rule_type:  frequency_and   # frequency_and
 6 | 
 7 | detect_list:
 8 | 
 9 |   - field : network.ip
10 |     type: re
11 |     rule: xxx
12 |     ignore-case: false
13 | 
14 |   - field : network.ip
15 |     type: equal
16 |     rule : xxx
17 | 
18 |   - field : network.ip,
19 |     type: in
20 |     rule : xxx
21 | 
22 | key : network.ip
23 | 
24 | time_interval:
25 |    second: 10
26 |    times: 100
27 | 
28 | 
29 | threat_level : high
30 | auth : njcx
31 | info : about sql injection attack
32 | 
33 | e-mail:
34 |     - njcx91@tom.com
35 |     - njcx91@tom.com
36 | 
37 | 


--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
 1 | module rulecat
 2 | 
 3 | go 1.22.0
 4 | 
 5 | toolchain go1.23.0
 6 | 
 7 | require (
 8 | 	github.com/confluentinc/confluent-kafka-go v1.4.2
 9 | 	github.com/dimiro1/banner v1.1.0
10 | 	github.com/flier/gohs v1.0.0
11 | 	github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df
12 | 	github.com/go-redis/redis v6.15.9+incompatible
13 | 	github.com/google/gopacket v1.1.14
14 | 	github.com/json-iterator/go v1.1.10
15 | 	github.com/olivere/elastic/v7 v7.0.23
16 | 	github.com/pborman/uuid v1.2.1
17 | 	github.com/streadway/amqp v1.1.0
18 | 	github.com/tidwall/match v1.0.3
19 | 	github.com/tidwall/pretty v1.1.0
20 | 	gopkg.in/olivere/elastic.v6 v6.2.35
21 | 	gopkg.in/yaml.v2 v2.4.0
22 | )
23 | 
24 | require (
25 | 	github.com/common-nighthawk/go-figure v0.0.0-20200609044655-c4b36f998cf2 // indirect
26 | 	github.com/google/go-cmp v0.6.0 // indirect
27 | 	github.com/google/uuid v1.2.0 // indirect
28 | 	github.com/hashicorp/errwrap v1.1.0 // indirect
29 | 	github.com/hashicorp/go-multierror v1.1.0 // indirect
30 | 	github.com/josharian/intern v1.0.0 // indirect
31 | 	github.com/mailru/easyjson v0.7.7 // indirect
32 | 	github.com/mattn/go-isatty v0.0.10 // indirect
33 | 	github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
34 | 	github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
35 | 	github.com/olivere/elastic v6.2.35+incompatible // indirect
36 | 	github.com/onsi/ginkgo v1.16.5 // indirect
37 | 	github.com/onsi/gomega v1.36.2 // indirect
38 | 	github.com/pkg/errors v0.9.1 // indirect
39 | 	github.com/stretchr/testify v1.8.4 // indirect
40 | 	golang.org/x/sys v0.28.0 // indirect
41 | 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
42 | 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect
43 | )
44 | 


--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
  1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
  2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
  3 | github.com/aws/aws-sdk-go v1.38.3/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
  4 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
  5 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
  6 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
  7 | github.com/common-nighthawk/go-figure v0.0.0-20200609044655-c4b36f998cf2 h1:tjT4Jp4gxECvsJcYpAMtW2I3YqzBTPuB67OejxXs86s=
  8 | github.com/common-nighthawk/go-figure v0.0.0-20200609044655-c4b36f998cf2/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w=
  9 | github.com/confluentinc/confluent-kafka-go v1.4.2 h1:13EK9RTujF7lVkvHQ5Hbu6bM+Yfrq8L0MkJNnjHSd4Q=
 10 | github.com/confluentinc/confluent-kafka-go v1.4.2/go.mod h1:u2zNLny2xq+5rWeTQjFHbDzzNuba4P1vo31r9r4uAdg=
 11 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 12 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 13 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 14 | github.com/dimiro1/banner v1.1.0 h1:TSfy+FsPIIGLzaMPOt52KrEed/omwFO1P15VA8PMUh0=
 15 | github.com/dimiro1/banner v1.1.0/go.mod h1:tbL318TJiUaHxOUNN+jnlvFSgsh/RX7iJaQrGgOiTco=
 16 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 17 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 18 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 19 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 20 | github.com/flier/gohs v1.0.0 h1:Q0mmufGWTigzKb140WmJ0+k3EGAf335Qgv/pz5SOPvU=
 21 | github.com/flier/gohs v1.0.0/go.mod h1:Jlg6A1xXSMhPorF74/LkYHkCHZ87Txi8CqIHHyIKgKg=
 22 | github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 23 | github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 24 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 25 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 26 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 27 | github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df h1:Bao6dhmbTA1KFVxmJ6nBoMuOJit2yjEgLJpIMYpop0E=
 28 | github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df/go.mod h1:GJr+FCSXshIwgHBtLglIg9M2l2kQSi6QjVAngtzI08Y=
 29 | github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
 30 | github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 31 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 32 | github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 33 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 34 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 35 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 36 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 37 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 38 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 39 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 40 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 41 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 42 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 43 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 44 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 45 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 46 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 47 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 48 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 49 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 50 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 51 | github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 52 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 53 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 54 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 55 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 56 | github.com/google/gopacket v1.1.14 h1:1+TEhSu8Mh154ZBVjyd1Nt2Bb7cnyOeE3GQyb1WGLqI=
 57 | github.com/google/gopacket v1.1.14/go.mod h1:UCLx9mCmAwsVbn6qQl1WIEt2SO7Nd2fD0th1TBAsqBw=
 58 | github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 59 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 60 | github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
 61 | github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 62 | github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c h1:16eHWuMGvCjSfgRJKqIzapE78onvvTbdi1rMkU00lZw=
 63 | github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 64 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 65 | github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 66 | github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 67 | github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 68 | github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
 69 | github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
 70 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 71 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 72 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 73 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 74 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 75 | github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 76 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 77 | github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
 78 | github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 79 | github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 80 | github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 81 | github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 82 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 83 | github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10=
 84 | github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 85 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
 86 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 87 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
 88 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 89 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 90 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 91 | github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 92 | github.com/olivere/elastic v6.2.35+incompatible h1:MMklYDy2ySi01s123CB2WLBuDMzFX4qhFcA5tKWJPgM=
 93 | github.com/olivere/elastic v6.2.35+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
 94 | github.com/olivere/elastic/v7 v7.0.23 h1:b7tjMogDMhf2CisGI+L02LXLVa0ZyE82Z15XfW1e8t8=
 95 | github.com/olivere/elastic/v7 v7.0.23/go.mod h1:OuWmD2DiuYhddWegBKPWQuelVKBLrW0fa/VUYgxuGTY=
 96 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 97 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 98 | github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 99 | github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
100 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
101 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
102 | github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8=
103 | github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY=
104 | github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
105 | github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
106 | github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
107 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
108 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
109 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
110 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
111 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
112 | github.com/smartystreets/assertions v0.0.0-20180820201707-7c9eb446e3cf/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
113 | github.com/smartystreets/assertions v1.1.1 h1:T/YLemO5Yp7KPzS+lVtu+WsHn8yoSwTfItdAd1r3cck=
114 | github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
115 | github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
116 | github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a h1:JSvGDIbmil4Ui/dDdFBExb7/cmkNjyX5F97oglmvCDo=
117 | github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
118 | github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
119 | github.com/streadway/amqp v1.1.0 h1:py12iX8XSyI7aN/3dUT8DFIDJazNJsVJdxNVEpnQTZM=
120 | github.com/streadway/amqp v1.1.0/go.mod h1:WYSrTEYHOXHd0nwFeUXAe2G2hRnQT+deZJJf88uS9Bg=
121 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
122 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
123 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
124 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
125 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
126 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
127 | github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
128 | github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
129 | github.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=
130 | github.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
131 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
132 | go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
133 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
134 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
135 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
136 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
137 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
138 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
139 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
140 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
141 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
142 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
143 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
144 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
145 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
146 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
147 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
148 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
149 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
150 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
151 | golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
152 | golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
153 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
154 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
155 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
156 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
157 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
158 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
159 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
160 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
161 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
162 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
163 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
164 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
165 | golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
166 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
167 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
168 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
169 | golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
170 | golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
171 | golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
172 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
173 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
174 | golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
175 | golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
176 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
177 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
178 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
179 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
180 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
181 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
182 | golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
183 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
184 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
185 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
186 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
187 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
188 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
189 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
190 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
191 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
192 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
193 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
194 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
195 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
196 | google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
197 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
198 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
199 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
200 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
201 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
202 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
203 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
204 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
205 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
206 | gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
207 | gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
208 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
209 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
210 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
211 | gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
212 | gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
213 | gopkg.in/olivere/elastic.v6 v6.2.35 h1:/5dJ0UzM231DGl9eDYOdDgv8yCFzAHQVPMo69rnylks=
214 | gopkg.in/olivere/elastic.v6 v6.2.35/go.mod h1:2cTT8Z+/LcArSWpCgvZqBgt3VOqXiy7v00w12Lz8bd4=
215 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
216 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
217 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
218 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
219 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
220 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
221 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
222 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
223 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
224 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
225 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
226 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
227 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
228 | 


--------------------------------------------------------------------------------
/img.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/njcx/RuleCat/2c13f2d38e539959d2a932154ba9a97b445a3b87/img.png


--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"os"
  6 | 	"os/signal"
  7 | 	"rulecat/app"
  8 | 	"rulecat/utils"
  9 | 	"rulecat/utils/kafka"
 10 | 	log2 "rulecat/utils/log"
 11 | 	"sync"
 12 | 	"syscall"
 13 | 	"time"
 14 | )
 15 | 
 16 | func main() {
 17 | 	topic := [...]string{"topic_tpl", "topic_tpl2"}
 18 | 	outPut := make(chan *sync.Map, 48)
 19 | 	var wg sync.WaitGroup
 20 | 	ctx, cancel := context.WithCancel(context.Background())
 21 | 	defer cancel()
 22 | 
 23 | 	kafkaConsumers := make([]*kafka.Consumer, len(topic))
 24 | 	defer func() {
 25 | 		for _, kafkaC := range kafkaConsumers {
 26 | 			if kafkaC != nil {
 27 | 				kafkaC.Close()
 28 | 			}
 29 | 		}
 30 | 	}()
 31 | 
 32 | 	for i, topicItem := range topic {
 33 | 
 34 | 		kafkaC := kafka.InitKakfaConsumer(app.ConfigG.InPut.Kafka.Server, app.ConfigG.InPut.Kafka.GroupId,
 35 | 			[]string{topicItem}, app.ConfigG.InPut.Kafka.User, app.ConfigG.InPut.Kafka.Passwd)
 36 | 		kafkaConsumers[i] = kafkaC
 37 | 		if err := kafkaC.Open(); err != nil {
 38 | 			log2.Error.Printf("Failed to open Kafka consumer for topic %s: %v", topicItem, err)
 39 | 			wg.Done()
 40 | 			continue
 41 | 		}
 42 | 		e := app.NewEngine(topicItem)
 43 | 		e.ReadRules()
 44 | 		wg.Add(7)
 45 | 
 46 | 		for j := 0; j < 5; j++ {
 47 | 			go func(j int) {
 48 | 				defer wg.Done()
 49 | 				for {
 50 | 					select {
 51 | 					case <-ctx.Done():
 52 | 						return
 53 | 					default:
 54 | 						message := <-kafkaC.Message
 55 | 						e.InPutC <- string(message.Value)
 56 | 					}
 57 | 				}
 58 | 			}(j)
 59 | 		}
 60 | 		go func() {
 61 | 			defer wg.Done()
 62 | 			e.ResCheck(128, outPut)
 63 | 
 64 | 		}()
 65 | 		go func() {
 66 | 			defer wg.Done()
 67 | 			for {
 68 | 				select {
 69 | 				case <-ctx.Done():
 70 | 					return
 71 | 				case dataStr := <-outPut:
 72 | 
 73 | 					JsonByte, _ := utils.MarshalSMapToJSON(dataStr)
 74 | 					app.SendKafka(JsonByte)
 75 | 					app.SendEs("_doc", "index_tpl", string(JsonByte))
 76 | 					app.SendMail(JsonByte)
 77 | 					app.SendJson(JsonByte)
 78 | 				}
 79 | 			}
 80 | 		}()
 81 | 
 82 | 	}
 83 | 	interrupt := make(chan os.Signal, 2)
 84 | 	signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)
 85 | 
 86 | 	defer func() {
 87 | 		signal.Stop(interrupt)
 88 | 	}()
 89 | 
 90 | 	for {
 91 | 		select {
 92 | 		case killSignal := <-interrupt:
 93 | 			log2.Info.Printf("Main app got signal: %v", killSignal)
 94 | 			log2.Info.Printf("Main app is shutting down due to signal: %v", killSignal)
 95 | 			cancel()
 96 | 			timeout := time.After(5 * time.Second)
 97 | 			done := make(chan struct{})
 98 | 			go func() {
 99 | 				wg.Wait()
100 | 				close(done)
101 | 			}()
102 | 			select {
103 | 			case <-done:
104 | 				return
105 | 			case <-timeout:
106 | 				log2.Warning.Println("Shutdown timeout exceeded, forcing exit.")
107 | 				return
108 | 			}
109 | 		}
110 | 	}
111 | }
112 | 


--------------------------------------------------------------------------------
/test/cache.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"log"
 5 | 	"rulecat/utils/cache"
 6 | 	"time"
 7 | )
 8 | 
 9 | func main() {
10 | 
11 | 	tc := cache.New(60*time.Second, 5*time.Second)
12 | 
13 | 	tc.Set("a", 1, cache.DefaultExpiration)
14 | 	tc.Set("b", "b", cache.DefaultExpiration)
15 | 	tc.Set("c", 3.5, cache.DefaultExpiration)
16 | 
17 | 	tc.Set("c", 3, 10*time.Second)
18 | 	tc.Set("d", 4, 10*time.Second)
19 | 
20 | 	tc.Increment("d", 2)
21 | 
22 | 	value, found := tc.Get("b")
23 | 	if found {
24 | 		log.Println("found:", value)
25 | 	} else {
26 | 		log.Println("not found")
27 | 	}
28 | 
29 | 	value, found = tc.Get("c")
30 | 	if found {
31 | 		log.Println("found:", value)
32 | 	} else {
33 | 		log.Println("not found")
34 | 	}
35 | 
36 | 	time.Sleep(60 * time.Second)
37 | 	log.Println("sleep 60s...")
38 | 	value, found = tc.Get("d")
39 | 	if found {
40 | 		log.Println("found:", value)
41 | 	} else {
42 | 		log.Println("not found")
43 | 	}
44 | 
45 | }
46 | 


--------------------------------------------------------------------------------
/test/engine.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"rulecat/app"
 6 | 	"rulecat/utils/kafka"
 7 | 	"time"
 8 | )
 9 | 
10 | func main() {
11 | 
12 | 	topic := [...]string{"conn", "ssh", "redis", "mysql", "mongodb", "icmp", "dns", "http"}
13 | 
14 | 	outPut := make(chan string, 48)
15 | 	for _, topicItem := range topic {
16 | 
17 | 		kafkaC := kafka.InitKakfaConsumer([]string{"172.21.129.2:9092"}, "test1", []string{"nids-" + topicItem})
18 | 		kafkaC.Open()
19 | 
20 | 		e := app.NewEngine(topicItem)
21 | 		e.ReadRules()
22 | 
23 | 		for i := 0; i <= 5; i++ {
24 | 			go func() {
25 | 				for {
26 | 					message := <-kafkaC.Message
27 | 					e.InPutC <- string(message.Value)
28 | 				}
29 | 			}()
30 | 		}
31 | 		go e.ResCheck(128, outPut)
32 | 		go func() {
33 | 			for {
34 | 				fmt.Println(<-outPut)
35 | 			}
36 | 		}()
37 | 
38 | 	}
39 | 	time.Sleep(1000000000 * time.Second)
40 | }
41 | 


--------------------------------------------------------------------------------
/test/json.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import "rulecat/utils/json"
 4 | 
 5 | const json_ = `{"name":{"first":"Janet","name":"Prichard"},"age":47}`
 6 | 
 7 | func main() {
 8 | 	value := json.Get(json_, "name.last\\.name")
 9 | 	println(value.String())
10 | }
11 | 


--------------------------------------------------------------------------------
/test/yaml.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"io/ioutil"
 6 | 	"rulecat/utils"
 7 | 
 8 | 	log2 "rulecat/utils/log"
 9 | 
10 | 	"gopkg.in/yaml.v2"
11 | )
12 | 
13 | type Input struct {
14 | 	Kafka struct {
15 | 		Enabled bool     `yaml:"enabled"`
16 | 		Server  []string `yaml:"server"`
17 | 		Topic   string   `yaml:"topic"`
18 | 		GroupId string   `yaml:"group_id"`
19 | 	}
20 | }
21 | 
22 | type Output struct {
23 | 	Es struct {
24 | 		Enabled bool     `yaml:"enabled"`
25 | 		Server  []string `yaml:"es_host"`
26 | 	}
27 | 	Kafka struct {
28 | 		Enabled bool     `yaml:"enabled"`
29 | 		Server  []string `yaml:"server"`
30 | 		Topic   string   `yaml:"topic"`
31 | 		GroupId string   `yaml:"group_id"`
32 | 	}
33 | 	Json struct {
34 | 		Enabled bool   `yaml:"enabled"`
35 | 		Path    string `yaml:"path"`
36 | 	}
37 | 	Email struct {
38 | 		Enabled       bool   `yaml:"enabled"`
39 | 		EmailHost     string `yaml:"email_host"`
40 | 		EmailSmtpPort string `yaml:"email_smtp_port"`
41 | 		EmailFrom     string `yaml:"email_from"`
42 | 		EmailUserName string `yaml:"email_username"`
43 | 		EmailPwd      string `yaml:"email_pwd"`
44 | 	}
45 | }
46 | 
47 | type Config struct {
48 | 	Name   string `yaml:"name"`
49 | 	Env    string `yaml:"env"`
50 | 	InPut  Input  `yaml:"input"`
51 | 	OutPut Output `yaml:"output"`
52 | }
53 | 
54 | func main() {
55 | 
56 | 	var err error
57 | 	configFile, err := ioutil.ReadFile(utils.GetCurrentPath() + "/etc/config.yml")
58 | 	if err != nil {
59 | 		log2.Error.Fatalf("Get yml file err %v ", err)
60 | 
61 | 	}
62 | 
63 | 	var _config *Config
64 | 	err = yaml.Unmarshal(configFile, &_config)
65 | 
66 | 	if err != nil {
67 | 		fmt.Println(err.Error())
68 | 	}
69 | 	fmt.Printf("config.app: %#v\n", _config)
70 | 
71 | 	fmt.Println(_config.Name)
72 | 	fmt.Println(_config.Env)
73 | 
74 | 	fmt.Println(_config.InPut)
75 | 	fmt.Println(_config.OutPut)
76 | 
77 | }
78 | 


--------------------------------------------------------------------------------
/utils/cache/cache.go:
--------------------------------------------------------------------------------
   1 | package cache
   2 | 
   3 | import (
   4 | 	"encoding/gob"
   5 | 	"fmt"
   6 | 	"io"
   7 | 	"os"
   8 | 	"runtime"
   9 | 	"sync"
  10 | 	"time"
  11 | )
  12 | 
  13 | type Item struct {
  14 | 	Object     interface{}
  15 | 	Expiration int64
  16 | }
  17 | 
  18 | // Returns true if the item has expired.
  19 | func (item Item) Expired() bool {
  20 | 	if item.Expiration == 0 {
  21 | 		return false
  22 | 	}
  23 | 	return time.Now().UnixNano() > item.Expiration
  24 | }
  25 | 
  26 | const (
  27 | 	// For use with functions that take an expiration time.
  28 | 	NoExpiration time.Duration = -1
  29 | 	// For use with functions that take an expiration time. Equivalent to
  30 | 	// passing in the same expiration duration as was given to New() or
  31 | 	// NewFrom() when the cache was created (e.g. 5 minutes.)
  32 | 	DefaultExpiration time.Duration = 0
  33 | )
  34 | 
  35 | type Cache struct {
  36 | 	*cache
  37 | 	// If this is confusing, see the comment at the bottom of New()
  38 | }
  39 | 
  40 | type cache struct {
  41 | 	defaultExpiration time.Duration
  42 | 	items             map[string]Item
  43 | 	mu                sync.RWMutex
  44 | 	onEvicted         func(string, interface{})
  45 | 	janitor           *janitor
  46 | }
  47 | 
  48 | // Add an item to the cache, replacing any existing item. If the duration is 0
  49 | // (DefaultExpiration), the cache's default expiration time is used. If it is -1
  50 | // (NoExpiration), the item never expires.
  51 | func (c *cache) Set(k string, x interface{}, d time.Duration) {
  52 | 	// "Inlining" of set
  53 | 	var e int64
  54 | 	if d == DefaultExpiration {
  55 | 		d = c.defaultExpiration
  56 | 	}
  57 | 	if d > 0 {
  58 | 		e = time.Now().Add(d).UnixNano()
  59 | 	}
  60 | 	c.mu.Lock()
  61 | 	c.items[k] = Item{
  62 | 		Object:     x,
  63 | 		Expiration: e,
  64 | 	}
  65 | 	// TODO: Calls to mu.Unlock are currently not deferred because defer
  66 | 	// adds ~200 ns (as of go1.)
  67 | 	c.mu.Unlock()
  68 | }
  69 | 
  70 | func (c *cache) set(k string, x interface{}, d time.Duration) {
  71 | 	var e int64
  72 | 	if d == DefaultExpiration {
  73 | 		d = c.defaultExpiration
  74 | 	}
  75 | 	if d > 0 {
  76 | 		e = time.Now().Add(d).UnixNano()
  77 | 	}
  78 | 	c.items[k] = Item{
  79 | 		Object:     x,
  80 | 		Expiration: e,
  81 | 	}
  82 | }
  83 | 
  84 | // Add an item to the cache, replacing any existing item, using the default
  85 | // expiration.
  86 | func (c *cache) SetDefault(k string, x interface{}) {
  87 | 	c.Set(k, x, DefaultExpiration)
  88 | }
  89 | 
  90 | // Add an item to the cache only if an item doesn't already exist for the given
  91 | // key, or if the existing item has expired. Returns an error otherwise.
  92 | func (c *cache) Add(k string, x interface{}, d time.Duration) error {
  93 | 	c.mu.Lock()
  94 | 	_, found := c.get(k)
  95 | 	if found {
  96 | 		c.mu.Unlock()
  97 | 		return fmt.Errorf("Item %s already exists", k)
  98 | 	}
  99 | 	c.set(k, x, d)
 100 | 	c.mu.Unlock()
 101 | 	return nil
 102 | }
 103 | 
 104 | // Set a new value for the cache key only if it already exists, and the existing
 105 | // item hasn't expired. Returns an error otherwise.
 106 | func (c *cache) Replace(k string, x interface{}, d time.Duration) error {
 107 | 	c.mu.Lock()
 108 | 	_, found := c.get(k)
 109 | 	if !found {
 110 | 		c.mu.Unlock()
 111 | 		return fmt.Errorf("Item %s doesn't exist", k)
 112 | 	}
 113 | 	c.set(k, x, d)
 114 | 	c.mu.Unlock()
 115 | 	return nil
 116 | }
 117 | 
 118 | // Get an item from the cache. Returns the item or nil, and a bool indicating
 119 | // whether the key was found.
 120 | func (c *cache) Get(k string) (interface{}, bool) {
 121 | 	c.mu.RLock()
 122 | 	// "Inlining" of get and Expired
 123 | 	item, found := c.items[k]
 124 | 	if !found {
 125 | 		c.mu.RUnlock()
 126 | 		return nil, false
 127 | 	}
 128 | 	if item.Expiration > 0 {
 129 | 		if time.Now().UnixNano() > item.Expiration {
 130 | 			c.mu.RUnlock()
 131 | 			return nil, false
 132 | 		}
 133 | 	}
 134 | 	c.mu.RUnlock()
 135 | 	return item.Object, true
 136 | }
 137 | 
 138 | // GetWithExpiration returns an item and its expiration time from the cache.
 139 | // It returns the item or nil, the expiration time if one is set (if the item
 140 | // never expires a zero value for time.Time is returned), and a bool indicating
 141 | // whether the key was found.
 142 | func (c *cache) GetWithExpiration(k string) (interface{}, time.Time, bool) {
 143 | 	c.mu.RLock()
 144 | 	// "Inlining" of get and Expired
 145 | 	item, found := c.items[k]
 146 | 	if !found {
 147 | 		c.mu.RUnlock()
 148 | 		return nil, time.Time{}, false
 149 | 	}
 150 | 
 151 | 	if item.Expiration > 0 {
 152 | 		if time.Now().UnixNano() > item.Expiration {
 153 | 			c.mu.RUnlock()
 154 | 			return nil, time.Time{}, false
 155 | 		}
 156 | 
 157 | 		// Return the item and the expiration time
 158 | 		c.mu.RUnlock()
 159 | 		return item.Object, time.Unix(0, item.Expiration), true
 160 | 	}
 161 | 
 162 | 	// If expiration <= 0 (i.e. no expiration time set) then return the item
 163 | 	// and a zeroed time.Time
 164 | 	c.mu.RUnlock()
 165 | 	return item.Object, time.Time{}, true
 166 | }
 167 | 
 168 | func (c *cache) get(k string) (interface{}, bool) {
 169 | 	item, found := c.items[k]
 170 | 	if !found {
 171 | 		return nil, false
 172 | 	}
 173 | 	// "Inlining" of Expired
 174 | 	if item.Expiration > 0 {
 175 | 		if time.Now().UnixNano() > item.Expiration {
 176 | 			return nil, false
 177 | 		}
 178 | 	}
 179 | 	return item.Object, true
 180 | }
 181 | 
 182 | // Increment an item of type int, int8, int16, int32, int64, uintptr, uint,
 183 | // uint8, uint32, or uint64, float32 or float64 by n. Returns an error if the
 184 | // item's value is not an integer, if it was not found, or if it is not
 185 | // possible to increment it by n. To retrieve the incremented value, use one
 186 | // of the specialized methods, e.g. IncrementInt64.
 187 | func (c *cache) Increment(k string, n int64) error {
 188 | 	c.mu.Lock()
 189 | 	v, found := c.items[k]
 190 | 	if !found || v.Expired() {
 191 | 		c.mu.Unlock()
 192 | 		return fmt.Errorf("Item %s not found", k)
 193 | 	}
 194 | 	switch v.Object.(type) {
 195 | 	case int:
 196 | 		v.Object = v.Object.(int) + int(n)
 197 | 	case int8:
 198 | 		v.Object = v.Object.(int8) + int8(n)
 199 | 	case int16:
 200 | 		v.Object = v.Object.(int16) + int16(n)
 201 | 	case int32:
 202 | 		v.Object = v.Object.(int32) + int32(n)
 203 | 	case int64:
 204 | 		v.Object = v.Object.(int64) + n
 205 | 	case uint:
 206 | 		v.Object = v.Object.(uint) + uint(n)
 207 | 	case uintptr:
 208 | 		v.Object = v.Object.(uintptr) + uintptr(n)
 209 | 	case uint8:
 210 | 		v.Object = v.Object.(uint8) + uint8(n)
 211 | 	case uint16:
 212 | 		v.Object = v.Object.(uint16) + uint16(n)
 213 | 	case uint32:
 214 | 		v.Object = v.Object.(uint32) + uint32(n)
 215 | 	case uint64:
 216 | 		v.Object = v.Object.(uint64) + uint64(n)
 217 | 	case float32:
 218 | 		v.Object = v.Object.(float32) + float32(n)
 219 | 	case float64:
 220 | 		v.Object = v.Object.(float64) + float64(n)
 221 | 	default:
 222 | 		c.mu.Unlock()
 223 | 		return fmt.Errorf("The value for %s is not an integer", k)
 224 | 	}
 225 | 	c.items[k] = v
 226 | 	c.mu.Unlock()
 227 | 	return nil
 228 | }
 229 | 
 230 | // Increment an item of type float32 or float64 by n. Returns an error if the
 231 | // item's value is not floating point, if it was not found, or if it is not
 232 | // possible to increment it by n. Pass a negative number to decrement the
 233 | // value. To retrieve the incremented value, use one of the specialized methods,
 234 | // e.g. IncrementFloat64.
 235 | func (c *cache) IncrementFloat(k string, n float64) error {
 236 | 	c.mu.Lock()
 237 | 	v, found := c.items[k]
 238 | 	if !found || v.Expired() {
 239 | 		c.mu.Unlock()
 240 | 		return fmt.Errorf("Item %s not found", k)
 241 | 	}
 242 | 	switch v.Object.(type) {
 243 | 	case float32:
 244 | 		v.Object = v.Object.(float32) + float32(n)
 245 | 	case float64:
 246 | 		v.Object = v.Object.(float64) + n
 247 | 	default:
 248 | 		c.mu.Unlock()
 249 | 		return fmt.Errorf("The value for %s does not have type float32 or float64", k)
 250 | 	}
 251 | 	c.items[k] = v
 252 | 	c.mu.Unlock()
 253 | 	return nil
 254 | }
 255 | 
 256 | // Increment an item of type int by n. Returns an error if the item's value is
 257 | // not an int, or if it was not found. If there is no error, the incremented
 258 | // value is returned.
 259 | func (c *cache) IncrementInt(k string, n int) (int, error) {
 260 | 	c.mu.Lock()
 261 | 	v, found := c.items[k]
 262 | 	if !found || v.Expired() {
 263 | 		c.mu.Unlock()
 264 | 		return 0, fmt.Errorf("Item %s not found", k)
 265 | 	}
 266 | 	rv, ok := v.Object.(int)
 267 | 	if !ok {
 268 | 		c.mu.Unlock()
 269 | 		return 0, fmt.Errorf("The value for %s is not an int", k)
 270 | 	}
 271 | 	nv := rv + n
 272 | 	v.Object = nv
 273 | 	c.items[k] = v
 274 | 	c.mu.Unlock()
 275 | 	return nv, nil
 276 | }
 277 | 
 278 | // Increment an item of type int8 by n. Returns an error if the item's value is
 279 | // not an int8, or if it was not found. If there is no error, the incremented
 280 | // value is returned.
 281 | func (c *cache) IncrementInt8(k string, n int8) (int8, error) {
 282 | 	c.mu.Lock()
 283 | 	v, found := c.items[k]
 284 | 	if !found || v.Expired() {
 285 | 		c.mu.Unlock()
 286 | 		return 0, fmt.Errorf("Item %s not found", k)
 287 | 	}
 288 | 	rv, ok := v.Object.(int8)
 289 | 	if !ok {
 290 | 		c.mu.Unlock()
 291 | 		return 0, fmt.Errorf("The value for %s is not an int8", k)
 292 | 	}
 293 | 	nv := rv + n
 294 | 	v.Object = nv
 295 | 	c.items[k] = v
 296 | 	c.mu.Unlock()
 297 | 	return nv, nil
 298 | }
 299 | 
 300 | // Increment an item of type int16 by n. Returns an error if the item's value is
 301 | // not an int16, or if it was not found. If there is no error, the incremented
 302 | // value is returned.
 303 | func (c *cache) IncrementInt16(k string, n int16) (int16, error) {
 304 | 	c.mu.Lock()
 305 | 	v, found := c.items[k]
 306 | 	if !found || v.Expired() {
 307 | 		c.mu.Unlock()
 308 | 		return 0, fmt.Errorf("Item %s not found", k)
 309 | 	}
 310 | 	rv, ok := v.Object.(int16)
 311 | 	if !ok {
 312 | 		c.mu.Unlock()
 313 | 		return 0, fmt.Errorf("The value for %s is not an int16", k)
 314 | 	}
 315 | 	nv := rv + n
 316 | 	v.Object = nv
 317 | 	c.items[k] = v
 318 | 	c.mu.Unlock()
 319 | 	return nv, nil
 320 | }
 321 | 
 322 | // Increment an item of type int32 by n. Returns an error if the item's value is
 323 | // not an int32, or if it was not found. If there is no error, the incremented
 324 | // value is returned.
 325 | func (c *cache) IncrementInt32(k string, n int32) (int32, error) {
 326 | 	c.mu.Lock()
 327 | 	v, found := c.items[k]
 328 | 	if !found || v.Expired() {
 329 | 		c.mu.Unlock()
 330 | 		return 0, fmt.Errorf("Item %s not found", k)
 331 | 	}
 332 | 	rv, ok := v.Object.(int32)
 333 | 	if !ok {
 334 | 		c.mu.Unlock()
 335 | 		return 0, fmt.Errorf("The value for %s is not an int32", k)
 336 | 	}
 337 | 	nv := rv + n
 338 | 	v.Object = nv
 339 | 	c.items[k] = v
 340 | 	c.mu.Unlock()
 341 | 	return nv, nil
 342 | }
 343 | 
 344 | // Increment an item of type int64 by n. Returns an error if the item's value is
 345 | // not an int64, or if it was not found. If there is no error, the incremented
 346 | // value is returned.
 347 | func (c *cache) IncrementInt64(k string, n int64) (int64, error) {
 348 | 	c.mu.Lock()
 349 | 	v, found := c.items[k]
 350 | 	if !found || v.Expired() {
 351 | 		c.mu.Unlock()
 352 | 		return 0, fmt.Errorf("Item %s not found", k)
 353 | 	}
 354 | 	rv, ok := v.Object.(int64)
 355 | 	if !ok {
 356 | 		c.mu.Unlock()
 357 | 		return 0, fmt.Errorf("The value for %s is not an int64", k)
 358 | 	}
 359 | 	nv := rv + n
 360 | 	v.Object = nv
 361 | 	c.items[k] = v
 362 | 	c.mu.Unlock()
 363 | 	return nv, nil
 364 | }
 365 | 
 366 | // Increment an item of type uint by n. Returns an error if the item's value is
 367 | // not an uint, or if it was not found. If there is no error, the incremented
 368 | // value is returned.
 369 | func (c *cache) IncrementUint(k string, n uint) (uint, error) {
 370 | 	c.mu.Lock()
 371 | 	v, found := c.items[k]
 372 | 	if !found || v.Expired() {
 373 | 		c.mu.Unlock()
 374 | 		return 0, fmt.Errorf("Item %s not found", k)
 375 | 	}
 376 | 	rv, ok := v.Object.(uint)
 377 | 	if !ok {
 378 | 		c.mu.Unlock()
 379 | 		return 0, fmt.Errorf("The value for %s is not an uint", k)
 380 | 	}
 381 | 	nv := rv + n
 382 | 	v.Object = nv
 383 | 	c.items[k] = v
 384 | 	c.mu.Unlock()
 385 | 	return nv, nil
 386 | }
 387 | 
 388 | // Increment an item of type uintptr by n. Returns an error if the item's value
 389 | // is not an uintptr, or if it was not found. If there is no error, the
 390 | // incremented value is returned.
 391 | func (c *cache) IncrementUintptr(k string, n uintptr) (uintptr, error) {
 392 | 	c.mu.Lock()
 393 | 	v, found := c.items[k]
 394 | 	if !found || v.Expired() {
 395 | 		c.mu.Unlock()
 396 | 		return 0, fmt.Errorf("Item %s not found", k)
 397 | 	}
 398 | 	rv, ok := v.Object.(uintptr)
 399 | 	if !ok {
 400 | 		c.mu.Unlock()
 401 | 		return 0, fmt.Errorf("The value for %s is not an uintptr", k)
 402 | 	}
 403 | 	nv := rv + n
 404 | 	v.Object = nv
 405 | 	c.items[k] = v
 406 | 	c.mu.Unlock()
 407 | 	return nv, nil
 408 | }
 409 | 
 410 | // Increment an item of type uint8 by n. Returns an error if the item's value
 411 | // is not an uint8, or if it was not found. If there is no error, the
 412 | // incremented value is returned.
 413 | func (c *cache) IncrementUint8(k string, n uint8) (uint8, error) {
 414 | 	c.mu.Lock()
 415 | 	v, found := c.items[k]
 416 | 	if !found || v.Expired() {
 417 | 		c.mu.Unlock()
 418 | 		return 0, fmt.Errorf("Item %s not found", k)
 419 | 	}
 420 | 	rv, ok := v.Object.(uint8)
 421 | 	if !ok {
 422 | 		c.mu.Unlock()
 423 | 		return 0, fmt.Errorf("The value for %s is not an uint8", k)
 424 | 	}
 425 | 	nv := rv + n
 426 | 	v.Object = nv
 427 | 	c.items[k] = v
 428 | 	c.mu.Unlock()
 429 | 	return nv, nil
 430 | }
 431 | 
 432 | // Increment an item of type uint16 by n. Returns an error if the item's value
 433 | // is not an uint16, or if it was not found. If there is no error, the
 434 | // incremented value is returned.
 435 | func (c *cache) IncrementUint16(k string, n uint16) (uint16, error) {
 436 | 	c.mu.Lock()
 437 | 	v, found := c.items[k]
 438 | 	if !found || v.Expired() {
 439 | 		c.mu.Unlock()
 440 | 		return 0, fmt.Errorf("Item %s not found", k)
 441 | 	}
 442 | 	rv, ok := v.Object.(uint16)
 443 | 	if !ok {
 444 | 		c.mu.Unlock()
 445 | 		return 0, fmt.Errorf("The value for %s is not an uint16", k)
 446 | 	}
 447 | 	nv := rv + n
 448 | 	v.Object = nv
 449 | 	c.items[k] = v
 450 | 	c.mu.Unlock()
 451 | 	return nv, nil
 452 | }
 453 | 
 454 | // Increment an item of type uint32 by n. Returns an error if the item's value
 455 | // is not an uint32, or if it was not found. If there is no error, the
 456 | // incremented value is returned.
 457 | func (c *cache) IncrementUint32(k string, n uint32) (uint32, error) {
 458 | 	c.mu.Lock()
 459 | 	v, found := c.items[k]
 460 | 	if !found || v.Expired() {
 461 | 		c.mu.Unlock()
 462 | 		return 0, fmt.Errorf("Item %s not found", k)
 463 | 	}
 464 | 	rv, ok := v.Object.(uint32)
 465 | 	if !ok {
 466 | 		c.mu.Unlock()
 467 | 		return 0, fmt.Errorf("The value for %s is not an uint32", k)
 468 | 	}
 469 | 	nv := rv + n
 470 | 	v.Object = nv
 471 | 	c.items[k] = v
 472 | 	c.mu.Unlock()
 473 | 	return nv, nil
 474 | }
 475 | 
 476 | // Increment an item of type uint64 by n. Returns an error if the item's value
 477 | // is not an uint64, or if it was not found. If there is no error, the
 478 | // incremented value is returned.
 479 | func (c *cache) IncrementUint64(k string, n uint64) (uint64, error) {
 480 | 	c.mu.Lock()
 481 | 	v, found := c.items[k]
 482 | 	if !found || v.Expired() {
 483 | 		c.mu.Unlock()
 484 | 		return 0, fmt.Errorf("Item %s not found", k)
 485 | 	}
 486 | 	rv, ok := v.Object.(uint64)
 487 | 	if !ok {
 488 | 		c.mu.Unlock()
 489 | 		return 0, fmt.Errorf("The value for %s is not an uint64", k)
 490 | 	}
 491 | 	nv := rv + n
 492 | 	v.Object = nv
 493 | 	c.items[k] = v
 494 | 	c.mu.Unlock()
 495 | 	return nv, nil
 496 | }
 497 | 
 498 | // Increment an item of type float32 by n. Returns an error if the item's value
 499 | // is not an float32, or if it was not found. If there is no error, the
 500 | // incremented value is returned.
 501 | func (c *cache) IncrementFloat32(k string, n float32) (float32, error) {
 502 | 	c.mu.Lock()
 503 | 	v, found := c.items[k]
 504 | 	if !found || v.Expired() {
 505 | 		c.mu.Unlock()
 506 | 		return 0, fmt.Errorf("Item %s not found", k)
 507 | 	}
 508 | 	rv, ok := v.Object.(float32)
 509 | 	if !ok {
 510 | 		c.mu.Unlock()
 511 | 		return 0, fmt.Errorf("The value for %s is not an float32", k)
 512 | 	}
 513 | 	nv := rv + n
 514 | 	v.Object = nv
 515 | 	c.items[k] = v
 516 | 	c.mu.Unlock()
 517 | 	return nv, nil
 518 | }
 519 | 
 520 | // Increment an item of type float64 by n. Returns an error if the item's value
 521 | // is not an float64, or if it was not found. If there is no error, the
 522 | // incremented value is returned.
 523 | func (c *cache) IncrementFloat64(k string, n float64) (float64, error) {
 524 | 	c.mu.Lock()
 525 | 	v, found := c.items[k]
 526 | 	if !found || v.Expired() {
 527 | 		c.mu.Unlock()
 528 | 		return 0, fmt.Errorf("Item %s not found", k)
 529 | 	}
 530 | 	rv, ok := v.Object.(float64)
 531 | 	if !ok {
 532 | 		c.mu.Unlock()
 533 | 		return 0, fmt.Errorf("The value for %s is not an float64", k)
 534 | 	}
 535 | 	nv := rv + n
 536 | 	v.Object = nv
 537 | 	c.items[k] = v
 538 | 	c.mu.Unlock()
 539 | 	return nv, nil
 540 | }
 541 | 
 542 | // Decrement an item of type int, int8, int16, int32, int64, uintptr, uint,
 543 | // uint8, uint32, or uint64, float32 or float64 by n. Returns an error if the
 544 | // item's value is not an integer, if it was not found, or if it is not
 545 | // possible to decrement it by n. To retrieve the decremented value, use one
 546 | // of the specialized methods, e.g. DecrementInt64.
 547 | func (c *cache) Decrement(k string, n int64) error {
 548 | 	// TODO: Implement Increment and Decrement more cleanly.
 549 | 	// (Cannot do Increment(k, n*-1) for uints.)
 550 | 	c.mu.Lock()
 551 | 	v, found := c.items[k]
 552 | 	if !found || v.Expired() {
 553 | 		c.mu.Unlock()
 554 | 		return fmt.Errorf("Item not found")
 555 | 	}
 556 | 	switch v.Object.(type) {
 557 | 	case int:
 558 | 		v.Object = v.Object.(int) - int(n)
 559 | 	case int8:
 560 | 		v.Object = v.Object.(int8) - int8(n)
 561 | 	case int16:
 562 | 		v.Object = v.Object.(int16) - int16(n)
 563 | 	case int32:
 564 | 		v.Object = v.Object.(int32) - int32(n)
 565 | 	case int64:
 566 | 		v.Object = v.Object.(int64) - n
 567 | 	case uint:
 568 | 		v.Object = v.Object.(uint) - uint(n)
 569 | 	case uintptr:
 570 | 		v.Object = v.Object.(uintptr) - uintptr(n)
 571 | 	case uint8:
 572 | 		v.Object = v.Object.(uint8) - uint8(n)
 573 | 	case uint16:
 574 | 		v.Object = v.Object.(uint16) - uint16(n)
 575 | 	case uint32:
 576 | 		v.Object = v.Object.(uint32) - uint32(n)
 577 | 	case uint64:
 578 | 		v.Object = v.Object.(uint64) - uint64(n)
 579 | 	case float32:
 580 | 		v.Object = v.Object.(float32) - float32(n)
 581 | 	case float64:
 582 | 		v.Object = v.Object.(float64) - float64(n)
 583 | 	default:
 584 | 		c.mu.Unlock()
 585 | 		return fmt.Errorf("The value for %s is not an integer", k)
 586 | 	}
 587 | 	c.items[k] = v
 588 | 	c.mu.Unlock()
 589 | 	return nil
 590 | }
 591 | 
 592 | // Decrement an item of type float32 or float64 by n. Returns an error if the
 593 | // item's value is not floating point, if it was not found, or if it is not
 594 | // possible to decrement it by n. Pass a negative number to decrement the
 595 | // value. To retrieve the decremented value, use one of the specialized methods,
 596 | // e.g. DecrementFloat64.
 597 | func (c *cache) DecrementFloat(k string, n float64) error {
 598 | 	c.mu.Lock()
 599 | 	v, found := c.items[k]
 600 | 	if !found || v.Expired() {
 601 | 		c.mu.Unlock()
 602 | 		return fmt.Errorf("Item %s not found", k)
 603 | 	}
 604 | 	switch v.Object.(type) {
 605 | 	case float32:
 606 | 		v.Object = v.Object.(float32) - float32(n)
 607 | 	case float64:
 608 | 		v.Object = v.Object.(float64) - n
 609 | 	default:
 610 | 		c.mu.Unlock()
 611 | 		return fmt.Errorf("The value for %s does not have type float32 or float64", k)
 612 | 	}
 613 | 	c.items[k] = v
 614 | 	c.mu.Unlock()
 615 | 	return nil
 616 | }
 617 | 
 618 | // Decrement an item of type int by n. Returns an error if the item's value is
 619 | // not an int, or if it was not found. If there is no error, the decremented
 620 | // value is returned.
 621 | func (c *cache) DecrementInt(k string, n int) (int, error) {
 622 | 	c.mu.Lock()
 623 | 	v, found := c.items[k]
 624 | 	if !found || v.Expired() {
 625 | 		c.mu.Unlock()
 626 | 		return 0, fmt.Errorf("Item %s not found", k)
 627 | 	}
 628 | 	rv, ok := v.Object.(int)
 629 | 	if !ok {
 630 | 		c.mu.Unlock()
 631 | 		return 0, fmt.Errorf("The value for %s is not an int", k)
 632 | 	}
 633 | 	nv := rv - n
 634 | 	v.Object = nv
 635 | 	c.items[k] = v
 636 | 	c.mu.Unlock()
 637 | 	return nv, nil
 638 | }
 639 | 
 640 | // Decrement an item of type int8 by n. Returns an error if the item's value is
 641 | // not an int8, or if it was not found. If there is no error, the decremented
 642 | // value is returned.
 643 | func (c *cache) DecrementInt8(k string, n int8) (int8, error) {
 644 | 	c.mu.Lock()
 645 | 	v, found := c.items[k]
 646 | 	if !found || v.Expired() {
 647 | 		c.mu.Unlock()
 648 | 		return 0, fmt.Errorf("Item %s not found", k)
 649 | 	}
 650 | 	rv, ok := v.Object.(int8)
 651 | 	if !ok {
 652 | 		c.mu.Unlock()
 653 | 		return 0, fmt.Errorf("The value for %s is not an int8", k)
 654 | 	}
 655 | 	nv := rv - n
 656 | 	v.Object = nv
 657 | 	c.items[k] = v
 658 | 	c.mu.Unlock()
 659 | 	return nv, nil
 660 | }
 661 | 
 662 | // Decrement an item of type int16 by n. Returns an error if the item's value is
 663 | // not an int16, or if it was not found. If there is no error, the decremented
 664 | // value is returned.
 665 | func (c *cache) DecrementInt16(k string, n int16) (int16, error) {
 666 | 	c.mu.Lock()
 667 | 	v, found := c.items[k]
 668 | 	if !found || v.Expired() {
 669 | 		c.mu.Unlock()
 670 | 		return 0, fmt.Errorf("Item %s not found", k)
 671 | 	}
 672 | 	rv, ok := v.Object.(int16)
 673 | 	if !ok {
 674 | 		c.mu.Unlock()
 675 | 		return 0, fmt.Errorf("The value for %s is not an int16", k)
 676 | 	}
 677 | 	nv := rv - n
 678 | 	v.Object = nv
 679 | 	c.items[k] = v
 680 | 	c.mu.Unlock()
 681 | 	return nv, nil
 682 | }
 683 | 
 684 | // Decrement an item of type int32 by n. Returns an error if the item's value is
 685 | // not an int32, or if it was not found. If there is no error, the decremented
 686 | // value is returned.
 687 | func (c *cache) DecrementInt32(k string, n int32) (int32, error) {
 688 | 	c.mu.Lock()
 689 | 	v, found := c.items[k]
 690 | 	if !found || v.Expired() {
 691 | 		c.mu.Unlock()
 692 | 		return 0, fmt.Errorf("Item %s not found", k)
 693 | 	}
 694 | 	rv, ok := v.Object.(int32)
 695 | 	if !ok {
 696 | 		c.mu.Unlock()
 697 | 		return 0, fmt.Errorf("The value for %s is not an int32", k)
 698 | 	}
 699 | 	nv := rv - n
 700 | 	v.Object = nv
 701 | 	c.items[k] = v
 702 | 	c.mu.Unlock()
 703 | 	return nv, nil
 704 | }
 705 | 
 706 | // Decrement an item of type int64 by n. Returns an error if the item's value is
 707 | // not an int64, or if it was not found. If there is no error, the decremented
 708 | // value is returned.
 709 | func (c *cache) DecrementInt64(k string, n int64) (int64, error) {
 710 | 	c.mu.Lock()
 711 | 	v, found := c.items[k]
 712 | 	if !found || v.Expired() {
 713 | 		c.mu.Unlock()
 714 | 		return 0, fmt.Errorf("Item %s not found", k)
 715 | 	}
 716 | 	rv, ok := v.Object.(int64)
 717 | 	if !ok {
 718 | 		c.mu.Unlock()
 719 | 		return 0, fmt.Errorf("The value for %s is not an int64", k)
 720 | 	}
 721 | 	nv := rv - n
 722 | 	v.Object = nv
 723 | 	c.items[k] = v
 724 | 	c.mu.Unlock()
 725 | 	return nv, nil
 726 | }
 727 | 
 728 | // Decrement an item of type uint by n. Returns an error if the item's value is
 729 | // not an uint, or if it was not found. If there is no error, the decremented
 730 | // value is returned.
 731 | func (c *cache) DecrementUint(k string, n uint) (uint, error) {
 732 | 	c.mu.Lock()
 733 | 	v, found := c.items[k]
 734 | 	if !found || v.Expired() {
 735 | 		c.mu.Unlock()
 736 | 		return 0, fmt.Errorf("Item %s not found", k)
 737 | 	}
 738 | 	rv, ok := v.Object.(uint)
 739 | 	if !ok {
 740 | 		c.mu.Unlock()
 741 | 		return 0, fmt.Errorf("The value for %s is not an uint", k)
 742 | 	}
 743 | 	nv := rv - n
 744 | 	v.Object = nv
 745 | 	c.items[k] = v
 746 | 	c.mu.Unlock()
 747 | 	return nv, nil
 748 | }
 749 | 
 750 | // Decrement an item of type uintptr by n. Returns an error if the item's value
 751 | // is not an uintptr, or if it was not found. If there is no error, the
 752 | // decremented value is returned.
 753 | func (c *cache) DecrementUintptr(k string, n uintptr) (uintptr, error) {
 754 | 	c.mu.Lock()
 755 | 	v, found := c.items[k]
 756 | 	if !found || v.Expired() {
 757 | 		c.mu.Unlock()
 758 | 		return 0, fmt.Errorf("Item %s not found", k)
 759 | 	}
 760 | 	rv, ok := v.Object.(uintptr)
 761 | 	if !ok {
 762 | 		c.mu.Unlock()
 763 | 		return 0, fmt.Errorf("The value for %s is not an uintptr", k)
 764 | 	}
 765 | 	nv := rv - n
 766 | 	v.Object = nv
 767 | 	c.items[k] = v
 768 | 	c.mu.Unlock()
 769 | 	return nv, nil
 770 | }
 771 | 
 772 | // Decrement an item of type uint8 by n. Returns an error if the item's value is
 773 | // not an uint8, or if it was not found. If there is no error, the decremented
 774 | // value is returned.
 775 | func (c *cache) DecrementUint8(k string, n uint8) (uint8, error) {
 776 | 	c.mu.Lock()
 777 | 	v, found := c.items[k]
 778 | 	if !found || v.Expired() {
 779 | 		c.mu.Unlock()
 780 | 		return 0, fmt.Errorf("Item %s not found", k)
 781 | 	}
 782 | 	rv, ok := v.Object.(uint8)
 783 | 	if !ok {
 784 | 		c.mu.Unlock()
 785 | 		return 0, fmt.Errorf("The value for %s is not an uint8", k)
 786 | 	}
 787 | 	nv := rv - n
 788 | 	v.Object = nv
 789 | 	c.items[k] = v
 790 | 	c.mu.Unlock()
 791 | 	return nv, nil
 792 | }
 793 | 
 794 | // Decrement an item of type uint16 by n. Returns an error if the item's value
 795 | // is not an uint16, or if it was not found. If there is no error, the
 796 | // decremented value is returned.
 797 | func (c *cache) DecrementUint16(k string, n uint16) (uint16, error) {
 798 | 	c.mu.Lock()
 799 | 	v, found := c.items[k]
 800 | 	if !found || v.Expired() {
 801 | 		c.mu.Unlock()
 802 | 		return 0, fmt.Errorf("Item %s not found", k)
 803 | 	}
 804 | 	rv, ok := v.Object.(uint16)
 805 | 	if !ok {
 806 | 		c.mu.Unlock()
 807 | 		return 0, fmt.Errorf("The value for %s is not an uint16", k)
 808 | 	}
 809 | 	nv := rv - n
 810 | 	v.Object = nv
 811 | 	c.items[k] = v
 812 | 	c.mu.Unlock()
 813 | 	return nv, nil
 814 | }
 815 | 
 816 | // Decrement an item of type uint32 by n. Returns an error if the item's value
 817 | // is not an uint32, or if it was not found. If there is no error, the
 818 | // decremented value is returned.
 819 | func (c *cache) DecrementUint32(k string, n uint32) (uint32, error) {
 820 | 	c.mu.Lock()
 821 | 	v, found := c.items[k]
 822 | 	if !found || v.Expired() {
 823 | 		c.mu.Unlock()
 824 | 		return 0, fmt.Errorf("Item %s not found", k)
 825 | 	}
 826 | 	rv, ok := v.Object.(uint32)
 827 | 	if !ok {
 828 | 		c.mu.Unlock()
 829 | 		return 0, fmt.Errorf("The value for %s is not an uint32", k)
 830 | 	}
 831 | 	nv := rv - n
 832 | 	v.Object = nv
 833 | 	c.items[k] = v
 834 | 	c.mu.Unlock()
 835 | 	return nv, nil
 836 | }
 837 | 
 838 | // Decrement an item of type uint64 by n. Returns an error if the item's value
 839 | // is not an uint64, or if it was not found. If there is no error, the
 840 | // decremented value is returned.
 841 | func (c *cache) DecrementUint64(k string, n uint64) (uint64, error) {
 842 | 	c.mu.Lock()
 843 | 	v, found := c.items[k]
 844 | 	if !found || v.Expired() {
 845 | 		c.mu.Unlock()
 846 | 		return 0, fmt.Errorf("Item %s not found", k)
 847 | 	}
 848 | 	rv, ok := v.Object.(uint64)
 849 | 	if !ok {
 850 | 		c.mu.Unlock()
 851 | 		return 0, fmt.Errorf("The value for %s is not an uint64", k)
 852 | 	}
 853 | 	nv := rv - n
 854 | 	v.Object = nv
 855 | 	c.items[k] = v
 856 | 	c.mu.Unlock()
 857 | 	return nv, nil
 858 | }
 859 | 
 860 | // Decrement an item of type float32 by n. Returns an error if the item's value
 861 | // is not an float32, or if it was not found. If there is no error, the
 862 | // decremented value is returned.
 863 | func (c *cache) DecrementFloat32(k string, n float32) (float32, error) {
 864 | 	c.mu.Lock()
 865 | 	v, found := c.items[k]
 866 | 	if !found || v.Expired() {
 867 | 		c.mu.Unlock()
 868 | 		return 0, fmt.Errorf("Item %s not found", k)
 869 | 	}
 870 | 	rv, ok := v.Object.(float32)
 871 | 	if !ok {
 872 | 		c.mu.Unlock()
 873 | 		return 0, fmt.Errorf("The value for %s is not an float32", k)
 874 | 	}
 875 | 	nv := rv - n
 876 | 	v.Object = nv
 877 | 	c.items[k] = v
 878 | 	c.mu.Unlock()
 879 | 	return nv, nil
 880 | }
 881 | 
 882 | // Decrement an item of type float64 by n. Returns an error if the item's value
 883 | // is not an float64, or if it was not found. If there is no error, the
 884 | // decremented value is returned.
 885 | func (c *cache) DecrementFloat64(k string, n float64) (float64, error) {
 886 | 	c.mu.Lock()
 887 | 	v, found := c.items[k]
 888 | 	if !found || v.Expired() {
 889 | 		c.mu.Unlock()
 890 | 		return 0, fmt.Errorf("Item %s not found", k)
 891 | 	}
 892 | 	rv, ok := v.Object.(float64)
 893 | 	if !ok {
 894 | 		c.mu.Unlock()
 895 | 		return 0, fmt.Errorf("The value for %s is not an float64", k)
 896 | 	}
 897 | 	nv := rv - n
 898 | 	v.Object = nv
 899 | 	c.items[k] = v
 900 | 	c.mu.Unlock()
 901 | 	return nv, nil
 902 | }
 903 | 
 904 | // Delete an item from the cache. Does nothing if the key is not in the cache.
 905 | func (c *cache) Delete(k string) {
 906 | 	c.mu.Lock()
 907 | 	v, evicted := c.delete(k)
 908 | 	c.mu.Unlock()
 909 | 	if evicted {
 910 | 		c.onEvicted(k, v)
 911 | 	}
 912 | }
 913 | 
 914 | func (c *cache) delete(k string) (interface{}, bool) {
 915 | 	if c.onEvicted != nil {
 916 | 		if v, found := c.items[k]; found {
 917 | 			delete(c.items, k)
 918 | 			return v.Object, true
 919 | 		}
 920 | 	}
 921 | 	delete(c.items, k)
 922 | 	return nil, false
 923 | }
 924 | 
 925 | type keyAndValue struct {
 926 | 	key   string
 927 | 	value interface{}
 928 | }
 929 | 
 930 | // Delete all expired items from the cache.
 931 | func (c *cache) DeleteExpired() {
 932 | 	var evictedItems []keyAndValue
 933 | 	now := time.Now().UnixNano()
 934 | 	c.mu.Lock()
 935 | 	for k, v := range c.items {
 936 | 		// "Inlining" of expired
 937 | 		if v.Expiration > 0 && now > v.Expiration {
 938 | 			ov, evicted := c.delete(k)
 939 | 			if evicted {
 940 | 				evictedItems = append(evictedItems, keyAndValue{k, ov})
 941 | 			}
 942 | 		}
 943 | 	}
 944 | 	c.mu.Unlock()
 945 | 	for _, v := range evictedItems {
 946 | 		c.onEvicted(v.key, v.value)
 947 | 	}
 948 | }
 949 | 
 950 | // Sets an (optional) function that is called with the key and value when an
 951 | // item is evicted from the cache. (Including when it is deleted manually, but
 952 | // not when it is overwritten.) Set to nil to disable.
 953 | func (c *cache) OnEvicted(f func(string, interface{})) {
 954 | 	c.mu.Lock()
 955 | 	c.onEvicted = f
 956 | 	c.mu.Unlock()
 957 | }
 958 | 
 959 | // Write the cache's items (using Gob) to an io.Writer.
 960 | //
 961 | // NOTE: This method is deprecated in favor of c.Items() and NewFrom() (see the
 962 | // documentation for NewFrom().)
 963 | func (c *cache) Save(w io.Writer) (err error) {
 964 | 	enc := gob.NewEncoder(w)
 965 | 	defer func() {
 966 | 		if x := recover(); x != nil {
 967 | 			err = fmt.Errorf("Error registering item types with Gob library")
 968 | 		}
 969 | 	}()
 970 | 	c.mu.RLock()
 971 | 	defer c.mu.RUnlock()
 972 | 	for _, v := range c.items {
 973 | 		gob.Register(v.Object)
 974 | 	}
 975 | 	err = enc.Encode(&c.items)
 976 | 	return
 977 | }
 978 | 
 979 | // Save the cache's items to the given filename, creating the file if it
 980 | // doesn't exist, and overwriting it if it does.
 981 | //
 982 | // NOTE: This method is deprecated in favor of c.Items() and NewFrom() (see the
 983 | // documentation for NewFrom().)
 984 | func (c *cache) SaveFile(fname string) error {
 985 | 	fp, err := os.Create(fname)
 986 | 	if err != nil {
 987 | 		return err
 988 | 	}
 989 | 	err = c.Save(fp)
 990 | 	if err != nil {
 991 | 		fp.Close()
 992 | 		return err
 993 | 	}
 994 | 	return fp.Close()
 995 | }
 996 | 
 997 | // Add (Gob-serialized) cache items from an io.Reader, excluding any items with
 998 | // keys that already exist (and haven't expired) in the current cache.
 999 | //
1000 | // NOTE: This method is deprecated in favor of c.Items() and NewFrom() (see the
1001 | // documentation for NewFrom().)
1002 | func (c *cache) Load(r io.Reader) error {
1003 | 	dec := gob.NewDecoder(r)
1004 | 	items := map[string]Item{}
1005 | 	err := dec.Decode(&items)
1006 | 	if err == nil {
1007 | 		c.mu.Lock()
1008 | 		defer c.mu.Unlock()
1009 | 		for k, v := range items {
1010 | 			ov, found := c.items[k]
1011 | 			if !found || ov.Expired() {
1012 | 				c.items[k] = v
1013 | 			}
1014 | 		}
1015 | 	}
1016 | 	return err
1017 | }
1018 | 
1019 | // Load and add cache items from the given filename, excluding any items with
1020 | // keys that already exist in the current cache.
1021 | //
1022 | // NOTE: This method is deprecated in favor of c.Items() and NewFrom() (see the
1023 | // documentation for NewFrom().)
1024 | func (c *cache) LoadFile(fname string) error {
1025 | 	fp, err := os.Open(fname)
1026 | 	if err != nil {
1027 | 		return err
1028 | 	}
1029 | 	err = c.Load(fp)
1030 | 	if err != nil {
1031 | 		fp.Close()
1032 | 		return err
1033 | 	}
1034 | 	return fp.Close()
1035 | }
1036 | 
1037 | // Copies all unexpired items in the cache into a new map and returns it.
1038 | func (c *cache) Items() map[string]Item {
1039 | 	c.mu.RLock()
1040 | 	defer c.mu.RUnlock()
1041 | 	m := make(map[string]Item, len(c.items))
1042 | 	now := time.Now().UnixNano()
1043 | 	for k, v := range c.items {
1044 | 		// "Inlining" of Expired
1045 | 		if v.Expiration > 0 {
1046 | 			if now > v.Expiration {
1047 | 				continue
1048 | 			}
1049 | 		}
1050 | 		m[k] = v
1051 | 	}
1052 | 	return m
1053 | }
1054 | 
1055 | // Returns the number of items in the cache. This may include items that have
1056 | // expired, but have not yet been cleaned up.
1057 | func (c *cache) ItemCount() int {
1058 | 	c.mu.RLock()
1059 | 	n := len(c.items)
1060 | 	c.mu.RUnlock()
1061 | 	return n
1062 | }
1063 | 
1064 | // Delete all items from the cache.
1065 | func (c *cache) Flush() {
1066 | 	c.mu.Lock()
1067 | 	c.items = map[string]Item{}
1068 | 	c.mu.Unlock()
1069 | }
1070 | 
1071 | type janitor struct {
1072 | 	Interval time.Duration
1073 | 	stop     chan bool
1074 | }
1075 | 
1076 | func (j *janitor) Run(c *cache) {
1077 | 	ticker := time.NewTicker(j.Interval)
1078 | 	for {
1079 | 		select {
1080 | 		case <-ticker.C:
1081 | 			c.DeleteExpired()
1082 | 		case <-j.stop:
1083 | 			ticker.Stop()
1084 | 			return
1085 | 		}
1086 | 	}
1087 | }
1088 | 
1089 | func stopJanitor(c *Cache) {
1090 | 	c.janitor.stop <- true
1091 | }
1092 | 
1093 | func runJanitor(c *cache, ci time.Duration) {
1094 | 	j := &janitor{
1095 | 		Interval: ci,
1096 | 		stop:     make(chan bool),
1097 | 	}
1098 | 	c.janitor = j
1099 | 	go j.Run(c)
1100 | }
1101 | 
1102 | func newCache(de time.Duration, m map[string]Item) *cache {
1103 | 	if de == 0 {
1104 | 		de = -1
1105 | 	}
1106 | 	c := &cache{
1107 | 		defaultExpiration: de,
1108 | 		items:             m,
1109 | 	}
1110 | 	return c
1111 | }
1112 | 
1113 | func newCacheWithJanitor(de time.Duration, ci time.Duration, m map[string]Item) *Cache {
1114 | 	c := newCache(de, m)
1115 | 	// This trick ensures that the janitor goroutine (which--granted it
1116 | 	// was enabled--is running DeleteExpired on c forever) does not keep
1117 | 	// the returned C object from being garbage collected. When it is
1118 | 	// garbage collected, the finalizer stops the janitor goroutine, after
1119 | 	// which c can be collected.
1120 | 	C := &Cache{c}
1121 | 	if ci > 0 {
1122 | 		runJanitor(c, ci)
1123 | 		runtime.SetFinalizer(C, stopJanitor)
1124 | 	}
1125 | 	return C
1126 | }
1127 | 
1128 | // Return a new cache with a given default expiration duration and cleanup
1129 | // interval. If the expiration duration is less than one (or NoExpiration),
1130 | // the items in the cache never expire (by default), and must be deleted
1131 | // manually. If the cleanup interval is less than one, expired items are not
1132 | // deleted from the cache before calling c.DeleteExpired().
1133 | func New(defaultExpiration, cleanupInterval time.Duration) *Cache {
1134 | 	items := make(map[string]Item)
1135 | 	return newCacheWithJanitor(defaultExpiration, cleanupInterval, items)
1136 | }
1137 | 
1138 | // Return a new cache with a given default expiration duration and cleanup
1139 | // interval. If the expiration duration is less than one (or NoExpiration),
1140 | // the items in the cache never expire (by default), and must be deleted
1141 | // manually. If the cleanup interval is less than one, expired items are not
1142 | // deleted from the cache before calling c.DeleteExpired().
1143 | //
1144 | // NewFrom() also accepts an items map which will serve as the underlying map
1145 | // for the cache. This is useful for starting from a deserialized cache
1146 | // (serialized using e.g. gob.Encode() on c.Items()), or passing in e.g.
1147 | // make(map[string]Item, 500) to improve startup performance when the cache
1148 | // is expected to reach a certain minimum size.
1149 | //
1150 | // Only the cache's methods synchronize access to this map, so it is not
1151 | // recommended to keep any references to the map around after creating a cache.
1152 | // If need be, the map can be accessed at a later point using c.Items() (subject
1153 | // to the same caveat.)
1154 | //
1155 | // Note regarding serialization: When using e.g. gob, make sure to
1156 | // gob.Register() the individual types stored in the cache before encoding a
1157 | // map retrieved with c.Items(), and to register those same types before
1158 | // decoding a blob containing an items map.
1159 | func NewFrom(defaultExpiration, cleanupInterval time.Duration, items map[string]Item) *Cache {
1160 | 	return newCacheWithJanitor(defaultExpiration, cleanupInterval, items)
1161 | }
1162 | 


--------------------------------------------------------------------------------
/utils/email/email.go:
--------------------------------------------------------------------------------
 1 | package email
 2 | 
 3 | import (
 4 | 	"crypto/tls"
 5 | 	"fmt"
 6 | 	"github.com/go-gomail/gomail"
 7 | 	"regexp"
 8 | 	log2 "rulecat/utils/log"
 9 | )
10 | 
11 | type EmailConf struct {
12 | 	from string
13 | 	d    *gomail.Dialer
14 | }
15 | 
16 | type ToSomeBody struct {
17 | 	To []string
18 | 	Cc []string
19 | }
20 | 
21 | func New(ServerHost string, ServerPort int, EmailFrom string, EmailUser string, Passwd string) (*EmailConf, error) {
22 | 	ec := &EmailConf{}
23 | 	d := gomail.NewDialer(ServerHost, ServerPort, EmailUser, Passwd)
24 | 	d.TLSConfig = &tls.Config{InsecureSkipVerify: true}
25 | 	_, err := d.Dial()
26 | 	if err != nil {
27 | 		log2.Error.Println(err)
28 | 		return nil, err
29 | 	}
30 | 	ec.d = d
31 | 	ec.from = EmailFrom
32 | 	return ec, nil
33 | }
34 | 
35 | func isValidEmail(email string) bool {
36 | 	emailRegex := regexp.MustCompile(`^[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}$`)
37 | 	return emailRegex.MatchString(email)
38 | }
39 | 
40 | func (ec *EmailConf) SendEmail(to *ToSomeBody, Subject string, content string) error {
41 | 	if !isValidEmail(ec.from) {
42 | 		return fmt.Errorf("invalid email address in From field: %s", ec.from)
43 | 	}
44 | 
45 | 	for _, addr := range to.To {
46 | 		if !isValidEmail(addr) {
47 | 			return fmt.Errorf("invalid email address in To field: %s", addr)
48 | 		}
49 | 	}
50 | 
51 | 	for _, addr := range to.Cc {
52 | 		if !isValidEmail(addr) {
53 | 			return fmt.Errorf("invalid email address in Cc field: %s", addr)
54 | 		}
55 | 	}
56 | 
57 | 	m := gomail.NewMessage()
58 | 	m.SetHeader("From", ec.from)
59 | 	m.SetHeader("To", to.To...)
60 | 	m.SetHeader("Cc", to.Cc...)
61 | 	m.SetHeader("Subject", Subject)
62 | 	m.SetBody("text/html", content)
63 | 
64 | 	if err := ec.d.DialAndSend(m); err != nil {
65 | 		log2.Error.Println(err)
66 | 		return err
67 | 	}
68 | 	return nil
69 | }
70 | 


--------------------------------------------------------------------------------
/utils/es/elastic6.go:
--------------------------------------------------------------------------------
  1 | package es
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"fmt"
  6 | 	"github.com/pborman/uuid"
  7 | 	elastic6 "gopkg.in/olivere/elastic.v6"
  8 | 	log2 "rulecat/utils/log"
  9 | 	"time"
 10 | )
 11 | 
 12 | type Elastic6Wrapper struct {
 13 | 	client        *elastic6.Client
 14 | 	pipeline      string
 15 | 	bulkProcessor *elastic6.BulkProcessor
 16 | }
 17 | 
 18 | func NewEsClient6(config ElasticConfig, bulkWorkers int, pipeline string) (*Elastic6Wrapper, error) {
 19 | 	var startupFns []elastic6.ClientOptionFunc
 20 | 
 21 | 	if len(config.Url) > 0 {
 22 | 		startupFns = append(startupFns, elastic6.SetURL(config.Url...))
 23 | 	}
 24 | 
 25 | 	if config.User != "" && config.Secret != "" {
 26 | 		startupFns = append(startupFns, elastic6.SetBasicAuth(config.User, config.Secret))
 27 | 	}
 28 | 
 29 | 	if config.MaxRetries != nil {
 30 | 		startupFns = append(startupFns, elastic6.SetMaxRetries(*config.MaxRetries))
 31 | 	}
 32 | 
 33 | 	if config.Sniff != nil {
 34 | 		startupFns = append(startupFns, elastic6.SetSniff(*config.Sniff))
 35 | 	}
 36 | 
 37 | 	client, err := elastic6.NewClient(startupFns...)
 38 | 	if err != nil {
 39 | 		return nil, fmt.Errorf("Failed to an ElasticSearch Client: %v", err)
 40 | 	}
 41 | 	bps, err := client.BulkProcessor().
 42 | 		Name("ElasticSearchWorker").
 43 | 		Workers(bulkWorkers).
 44 | 		After(bulkAfterCBV6).
 45 | 		BulkActions(1000).               // commit if # requests >= 1000
 46 | 		BulkSize(2 << 20).               // commit if size of requests >= 2 MB
 47 | 		FlushInterval(10 * time.Second). // commit every 10s
 48 | 		Do(context.Background())
 49 | 	if err != nil {
 50 | 		return nil, fmt.Errorf("Failed to an ElasticSearch Bulk Processor: %v", err)
 51 | 	}
 52 | 
 53 | 	return &Elastic6Wrapper{client: client, bulkProcessor: bps, pipeline: pipeline}, nil
 54 | }
 55 | 
 56 | func (es *Elastic6Wrapper) IndexExists(indices ...string) (bool, error) {
 57 | 	return es.client.IndexExists(indices...).Do(context.Background())
 58 | }
 59 | 
 60 | func (es *Elastic6Wrapper) CreateIndex(name string) (bool, error) {
 61 | 	res, err := es.client.CreateIndex(name).Do(context.Background())
 62 | 	if err != nil {
 63 | 		return false, err
 64 | 	}
 65 | 	return res.Acknowledged, err
 66 | }
 67 | 
 68 | func (es *Elastic6Wrapper) getAliases(index string) (*elastic6.AliasesResult, error) {
 69 | 	return es.client.Aliases().Index(index).Do(context.Background())
 70 | }
 71 | 
 72 | func (es *Elastic6Wrapper) AddAlias(index string, alias string) (bool, error) {
 73 | 	res, err := es.client.Alias().Add(index, alias).Do(context.Background())
 74 | 	if err != nil {
 75 | 		return false, err
 76 | 	}
 77 | 	return res.Acknowledged, err
 78 | }
 79 | 
 80 | func (es *Elastic6Wrapper) HasAlias(indexName string, aliasName string) (bool, error) {
 81 | 	aliases, err := es.getAliases(indexName)
 82 | 	if err != nil {
 83 | 		return false, err
 84 | 	}
 85 | 	return aliases.Indices[indexName].HasAlias(aliasName), nil
 86 | }
 87 | 
 88 | func (es *Elastic6Wrapper) AddBulkReq(index, typeName string, data interface{}) error {
 89 | 	req := elastic6.NewBulkIndexRequest().
 90 | 		Index(index).
 91 | 		Type(typeName).
 92 | 		Id(uuid.NewUUID().String()).
 93 | 		Doc(data)
 94 | 	if es.pipeline != "" {
 95 | 		req.Pipeline(es.pipeline)
 96 | 	}
 97 | 
 98 | 	es.bulkProcessor.Add(req)
 99 | 	return nil
100 | }
101 | 
102 | func (es *Elastic6Wrapper) FlushBulk() error {
103 | 	return es.bulkProcessor.Flush()
104 | }
105 | 
106 | func (es *Elastic6Wrapper) AddBodyJson(index, typeName string, data interface{}) error {
107 | 	put1, err := es.client.Index().
108 | 		Index(index).
109 | 		Type(typeName).
110 | 		Id(uuid.NewUUID().String()).
111 | 		BodyJson(data).
112 | 		Do(context.Background())
113 | 	if err != nil {
114 | 		log2.Error.Println("Failed to indexed data: %v", err)
115 | 		return err
116 | 	}
117 | 	log2.Info.Printf("Indexed data %s to index %s, type %s\n", put1.Id, put1.Index, put1.Type)
118 | 	return nil
119 | }
120 | 
121 | func (es *Elastic6Wrapper) AddBodyString(index, typeName string, data string) error {
122 | 	put2, err := es.client.Index().
123 | 		Index(index).
124 | 		Type(typeName).
125 | 		Id(uuid.NewUUID().String()).
126 | 		BodyString(data).
127 | 		Do(context.Background())
128 | 	if err != nil {
129 | 		log2.Error.Println("Failed to indexed data: %v", err)
130 | 		return err
131 | 	}
132 | 	log2.Info.Printf("Indexed data %s to index %s, type %s\n", put2.Id, put2.Index, put2.Type)
133 | 	return nil
134 | }
135 | 
136 | func bulkAfterCBV6(_ int64, _ []elastic6.BulkableRequest, response *elastic6.BulkResponse, err error) {
137 | 	if err != nil {
138 | 		log2.Error.Println("Failed to execute bulk operation to ElasticSearch: %v", err)
139 | 	}
140 | 
141 | 	if response.Errors {
142 | 		for _, list := range response.Items {
143 | 			for name, itm := range list {
144 | 				if itm.Error != nil {
145 | 					log2.Error.Println("Failed to execute bulk operation to ElasticSearch on %s: %v", name, itm.Error)
146 | 				}
147 | 			}
148 | 		}
149 | 	}
150 | }
151 | 


--------------------------------------------------------------------------------
/utils/es/elastic7.go:
--------------------------------------------------------------------------------
  1 | package es
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"fmt"
  6 | 	elastic7 "github.com/olivere/elastic/v7"
  7 | 	"github.com/pborman/uuid"
  8 | 	log2 "rulecat/utils/log"
  9 | 	"time"
 10 | )
 11 | 
 12 | type Elastic7Wrapper struct {
 13 | 	client        *elastic7.Client
 14 | 	pipeline      string
 15 | 	bulkProcessor *elastic7.BulkProcessor
 16 | }
 17 | 
 18 | func NewEsClient7(config ElasticConfig, bulkWorkers int, pipeline string) (*Elastic7Wrapper, error) {
 19 | 	var startupFns []elastic7.ClientOptionFunc
 20 | 	if len(config.Url) > 0 {
 21 | 		startupFns = append(startupFns, elastic7.SetURL(config.Url...))
 22 | 	}
 23 | 	if config.User != "" && config.Secret != "" {
 24 | 		startupFns = append(startupFns, elastic7.SetBasicAuth(config.User, config.Secret))
 25 | 	}
 26 | 	if config.MaxRetries != nil {
 27 | 		startupFns = append(startupFns, elastic7.SetMaxRetries(*config.MaxRetries))
 28 | 	}
 29 | 	if config.Sniff != nil {
 30 | 		startupFns = append(startupFns, elastic7.SetSniff(*config.Sniff))
 31 | 	}
 32 | 	client, err := elastic7.NewClient(startupFns...)
 33 | 	if err != nil {
 34 | 		return nil, fmt.Errorf("failed to an ElasticSearch Client: %v", err)
 35 | 	}
 36 | 	bps, err := client.BulkProcessor().
 37 | 		Name("ElasticSearchWorker").
 38 | 		Workers(bulkWorkers).
 39 | 		After(bulkAfterCBV7).
 40 | 		BulkActions(1000).               // commit if # requests >= 1000
 41 | 		BulkSize(2 << 20).               // commit if size of requests >= 2 MB
 42 | 		FlushInterval(10 * time.Second). // commit every 10s
 43 | 		Do(context.Background())
 44 | 	if err != nil {
 45 | 		return nil, fmt.Errorf("failed to an ElasticSearch Bulk Processor: %v", err)
 46 | 	}
 47 | 
 48 | 	return &Elastic7Wrapper{client: client, bulkProcessor: bps, pipeline: pipeline}, nil
 49 | }
 50 | 
 51 | func (es *Elastic7Wrapper) IndexExists(indices ...string) (bool, error) {
 52 | 	return es.client.IndexExists(indices...).Do(context.Background())
 53 | }
 54 | 
 55 | func (es *Elastic7Wrapper) CreateIndex(name string) (bool, error) {
 56 | 	res, err := es.client.CreateIndex(name).Do(context.Background())
 57 | 	if err != nil {
 58 | 		return false, err
 59 | 	}
 60 | 	return res.Acknowledged, err
 61 | }
 62 | 
 63 | func (es *Elastic7Wrapper) getAliases(index string) (*elastic7.AliasesResult, error) {
 64 | 	return es.client.Aliases().Index(index).Do(context.Background())
 65 | }
 66 | 
 67 | func (es *Elastic7Wrapper) AddAlias(index string, alias string) (bool, error) {
 68 | 	res, err := es.client.Alias().Add(index, alias).Do(context.Background())
 69 | 	if err != nil {
 70 | 		return false, err
 71 | 	}
 72 | 	return res.Acknowledged, err
 73 | }
 74 | 
 75 | func (es *Elastic7Wrapper) HasAlias(indexName string, aliasName string) (bool, error) {
 76 | 	aliases, err := es.getAliases(indexName)
 77 | 	if err != nil {
 78 | 		return false, err
 79 | 	}
 80 | 	return aliases.Indices[indexName].HasAlias(aliasName), nil
 81 | }
 82 | 
 83 | func (es *Elastic7Wrapper) AddBulkReq(index, typeName string, data interface{}) error {
 84 | 	req := elastic7.NewBulkIndexRequest().
 85 | 		Index(index).
 86 | 		Type(typeName).
 87 | 		Id(uuid.NewUUID().String()).
 88 | 		Doc(data)
 89 | 	if es.pipeline != "" {
 90 | 		req.Pipeline(es.pipeline)
 91 | 	}
 92 | 	es.bulkProcessor.Add(req)
 93 | 	return nil
 94 | }
 95 | 
 96 | func (es *Elastic7Wrapper) FlushBulk() error {
 97 | 	return es.bulkProcessor.Flush()
 98 | }
 99 | 
100 | func (es *Elastic7Wrapper) AddBodyJson(index, typeName string, data interface{}) error {
101 | 	put1, err := es.client.Index().
102 | 		Index(index).
103 | 		Type(typeName).
104 | 		Id(uuid.NewUUID().String()).
105 | 		BodyJson(data).
106 | 		Do(context.Background())
107 | 	if err != nil {
108 | 		log2.Error.Println("Failed to indexed data: %v", err)
109 | 		return err
110 | 	}
111 | 	log2.Info.Printf("Indexed data %s to index %s, type %s\n", put1.Id, put1.Index, put1.Type)
112 | 	return nil
113 | }
114 | 
115 | func (es *Elastic7Wrapper) AddBodyString(index, typeName string, data string) error {
116 | 	put2, err := es.client.Index().
117 | 		Index(index).
118 | 		Type(typeName).
119 | 		Id(uuid.NewUUID().String()).
120 | 		BodyString(data).
121 | 		Do(context.Background())
122 | 	if err != nil {
123 | 		log2.Error.Println("Failed to indexed data: %v", err)
124 | 		return err
125 | 	}
126 | 	log2.Info.Printf("Indexed data %s to index %s, type %s\n", put2.Id, put2.Index, put2.Type)
127 | 	return nil
128 | }
129 | 
130 | func bulkAfterCBV7(_ int64, _ []elastic7.BulkableRequest, response *elastic7.BulkResponse, err error) {
131 | 	if err != nil {
132 | 		fmt.Println("Failed to execute bulk operation to ElasticSearch: %v", err)
133 | 	}
134 | 
135 | 	if response.Errors {
136 | 		for _, list := range response.Items {
137 | 			for name, itm := range list {
138 | 				if itm.Error != nil {
139 | 					log2.Error.Println("Failed to execute bulk operation to ElasticSearch on %s: %v", name, itm.Error)
140 | 				}
141 | 			}
142 | 		}
143 | 	}
144 | }
145 | 


--------------------------------------------------------------------------------
/utils/es/es.go:
--------------------------------------------------------------------------------
  1 | package es
  2 | 
  3 | import (
  4 | 	"errors"
  5 | 	"fmt"
  6 | 	log2 "rulecat/utils/log"
  7 | 	"time"
  8 | )
  9 | 
 10 | const (
 11 | 	ESIndex = "index"
 12 | )
 13 | 
 14 | type UnsupportedVersion struct{}
 15 | 
 16 | func (UnsupportedVersion) Error() string {
 17 | 	return "Unsupported ElasticSearch Client Version"
 18 | }
 19 | 
 20 | type elasticWrapper interface {
 21 | 	IndexExists(indices ...string) (bool, error)
 22 | 	CreateIndex(name string) (bool, error)
 23 | 	AddAlias(index string, alias string) (bool, error)
 24 | 	HasAlias(index string, alias string) (bool, error)
 25 | 	AddBulkReq(index, typeName string, data interface{}) error
 26 | 	FlushBulk() error
 27 | 	AddBodyJson(index, typeName string, data interface{}) error
 28 | 	AddBodyString(index, typeName string, data string) error
 29 | }
 30 | 
 31 | type ElasticConfig struct {
 32 | 	Url        []string
 33 | 	User       string
 34 | 	Secret     string
 35 | 	MaxRetries *int
 36 | 	Sniff      *bool
 37 | }
 38 | 
 39 | type ElasticSearchService struct {
 40 | 	EsClient  elasticWrapper
 41 | 	baseIndex string
 42 | }
 43 | 
 44 | func (esSvc *ElasticSearchService) Index(typeName string, namespace string) (string, error) {
 45 | 	date := time.Now()
 46 | 	dateStr := date.Format("2006.01.02")
 47 | 
 48 | 	indexName := fmt.Sprintf("%s-%s", esSvc.baseIndex, dateStr)
 49 | 	if len(namespace) > 0 {
 50 | 		indexName = fmt.Sprintf("%s-%s", namespace, dateStr)
 51 | 	}
 52 | 
 53 | 	if typeName == "" {
 54 | 		return indexName, errors.New("TypeName  nil")
 55 | 	}
 56 | 	// Use the IndexExists service to check if a specified index exists.
 57 | 	exists, err := esSvc.EsClient.IndexExists(indexName)
 58 | 	if err != nil {
 59 | 		return indexName, err
 60 | 	}
 61 | 	if !exists {
 62 | 		ack, err := esSvc.EsClient.CreateIndex(indexName)
 63 | 		if err != nil {
 64 | 			return indexName, err
 65 | 		}
 66 | 		if !ack {
 67 | 			return indexName, errors.New("Failed to acknoledge index creation")
 68 | 		}
 69 | 	}
 70 | 
 71 | 	aliasName := esSvc.IndexAlias(typeName)
 72 | 	hasAlias, err := esSvc.EsClient.HasAlias(indexName, aliasName)
 73 | 	if err != nil {
 74 | 		return indexName, err
 75 | 	}
 76 | 
 77 | 	if !hasAlias {
 78 | 		ack, err := esSvc.EsClient.AddAlias(indexName, esSvc.IndexAlias(typeName))
 79 | 		if err != nil {
 80 | 			return indexName, err
 81 | 		}
 82 | 
 83 | 		if !ack {
 84 | 			return indexName, errors.New("Failed to acknoledge index alias creation")
 85 | 		}
 86 | 	}
 87 | 
 88 | 	return indexName, nil
 89 | }
 90 | 
 91 | func (esSvc *ElasticSearchService) IndexAlias(typeName string) string {
 92 | 	return fmt.Sprintf("%s-%s", esSvc.baseIndex, typeName)
 93 | }
 94 | 
 95 | func (esSvc *ElasticSearchService) FlushData() error {
 96 | 	return esSvc.EsClient.FlushBulk()
 97 | }
 98 | 
 99 | // SaveDataIntoES save metrics and events to ES by using ES client
100 | func (esSvc *ElasticSearchService) SaveData(typeName string, namespace string, sinkData []interface{}) error {
101 | 	indexName, err := esSvc.Index(typeName, namespace)
102 | 	if err != nil {
103 | 		log2.Error.Println(err)
104 | 		return err
105 | 	}
106 | 	for _, data := range sinkData {
107 | 		esSvc.EsClient.AddBulkReq(indexName, typeName, data)
108 | 	}
109 | 	return nil
110 | }
111 | 
112 | func (esSvc *ElasticSearchService) AddBodyJson(typeName, namespace string, sinkData interface{}) error {
113 | 	indexName, err := esSvc.Index(typeName, namespace)
114 | 	if err != nil {
115 | 		log2.Error.Println(err)
116 | 		return err
117 | 	}
118 | 	return esSvc.EsClient.AddBodyJson(indexName, typeName, sinkData)
119 | }
120 | 
121 | func (esSvc *ElasticSearchService) AddBodyString(typeName, namespace string, sinkData string) error {
122 | 	indexName, err := esSvc.Index(typeName, namespace)
123 | 	if err != nil {
124 | 		log2.Error.Println(err)
125 | 		return err
126 | 	}
127 | 	return esSvc.EsClient.AddBodyString(indexName, typeName, sinkData)
128 | }
129 | 
130 | func CreateElasticSearchService(config ElasticConfig, version int) (*ElasticSearchService, error) {
131 | 	var esSvc ElasticSearchService
132 | 	esSvc.baseIndex = ESIndex
133 | 	var err error
134 | 	bulkWorkers := 5
135 | 	pipeline := ""
136 | 
137 | 	switch version {
138 | 	case 6:
139 | 		esSvc.EsClient, err = NewEsClient6(config, bulkWorkers, pipeline)
140 | 	case 7:
141 | 		esSvc.EsClient, err = NewEsClient7(config, bulkWorkers, pipeline)
142 | 	default:
143 | 		return nil, UnsupportedVersion{}
144 | 	}
145 | 	if err != nil {
146 | 		return nil, fmt.Errorf("Failed to create ElasticSearch client: %v", err)
147 | 	}
148 | 
149 | 	return &esSvc, nil
150 | }
151 | 


--------------------------------------------------------------------------------
/utils/kafka/consumer.go:
--------------------------------------------------------------------------------
  1 | package kafka
  2 | 
  3 | import (
  4 | 	"errors"
  5 | 	"github.com/confluentinc/confluent-kafka-go/kafka"
  6 | 	log2 "rulecat/utils/log"
  7 | 	"strings"
  8 | )
  9 | 
 10 | type Consumer struct {
 11 | 	kafkaConsumer *kafka.Consumer
 12 | 	Message       chan *kafka.Message
 13 | 	IsOpen        bool
 14 | 	address       []string
 15 | 	group         string
 16 | 	topic         []string
 17 | 	user          string
 18 | 	password      string
 19 | }
 20 | 
 21 | func createConsumerCluster(kafkaAddrs []string, kafkaGroup string, user string, password string) *kafka.Consumer {
 22 | 	config := kafka.ConfigMap{
 23 | 		"bootstrap.servers":       strings.Join(kafkaAddrs, ","),
 24 | 		"group.id":                kafkaGroup,
 25 | 		"enable.auto.commit":      true,
 26 | 		"auto.commit.interval.ms": 1000,
 27 | 		"session.timeout.ms":      30000,
 28 | 		"socket.keepalive.enable": true,
 29 | 		"sasl.mechanisms":         "PLAIN",
 30 | 		"security.protocol":       "SASL_PLAINTEXT",
 31 | 		"sasl.username":           user,
 32 | 		"sasl.password":           password,
 33 | 	}
 34 | 	c, err := kafka.NewConsumer(&config)
 35 | 	if err != nil {
 36 | 		log2.Error.Fatal(err)
 37 | 	}
 38 | 	return c
 39 | }
 40 | 
 41 | func InitKakfaConsumer(kafkaAddrs []string, kafkaGroup string, topic []string, user string, password string) *Consumer {
 42 | 
 43 | 	c := new(Consumer)
 44 | 	c.address = kafkaAddrs
 45 | 	c.group = kafkaGroup
 46 | 	c.topic = topic
 47 | 	c.kafkaConsumer = createConsumerCluster(kafkaAddrs, kafkaGroup, user, password)
 48 | 	return c
 49 | }
 50 | 
 51 | func (c *Consumer) MarkOffset(msg *kafka.Message) {
 52 | 	if c.IsOpen == false {
 53 | 		return
 54 | 	}
 55 | 	c.kafkaConsumer.CommitMessage(msg)
 56 | }
 57 | 
 58 | func (c *Consumer) runPooler() {
 59 | 	for c.IsOpen == true {
 60 | 		ev := c.kafkaConsumer.Poll(100)
 61 | 		if ev == nil {
 62 | 			continue
 63 | 		}
 64 | 		switch msg := ev.(type) {
 65 | 		case *kafka.Message:
 66 | 			if strings.HasPrefix(*msg.TopicPartition.Topic, "_") == true {
 67 | 				continue
 68 | 			} else {
 69 | 				c.Message <- msg
 70 | 			}
 71 | 		case kafka.Error:
 72 | 			log2.Error.Printf("%% Error: %v\n", msg)
 73 | 			c.IsOpen = false
 74 | 			var count = 0
 75 | 			for c.kafkaConsumer.Poll(10) != nil && count < 100 {
 76 | 				count++
 77 | 			}
 78 | 			if count == 100 {
 79 | 				log2.Error.Fatalln("Error: Cannot drain pool to close consumer, hard stop.")
 80 | 			}
 81 | 			c.kafkaConsumer.Close()
 82 | 			c.kafkaConsumer = createConsumerCluster(c.address, c.group, c.user, c.password)
 83 | 			err := c.kafkaConsumer.SubscribeTopics(c.topic, nil)
 84 | 			if err != nil {
 85 | 				log2.Error.Fatalln("Fatal, cannot recover from", err)
 86 | 			}
 87 | 			c.IsOpen = true
 88 | 			log2.Error.Printf("Recovered, resuming listening.")
 89 | 		default:
 90 | 			// do nothing, ignore the message.
 91 | 		}
 92 | 	}
 93 | }
 94 | 
 95 | func (c *Consumer) Refresh() error {
 96 | 	if c.IsOpen == false {
 97 | 		return nil
 98 | 	}
 99 | 	err := c.kafkaConsumer.SubscribeTopics(c.topic, nil)
100 | 	if err != nil {
101 | 		log2.Error.Println("Error listening to  topic", err)
102 | 	}
103 | 	return err
104 | }
105 | 
106 | func (c *Consumer) Open() error {
107 | 	if c.IsOpen == true {
108 | 		return errors.New("Unable to open consumer, its already open.")
109 | 	}
110 | 	if c.address == nil {
111 | 		return errors.New("invalid address")
112 | 	}
113 | 	if c.group == "" {
114 | 		return errors.New("invalid group")
115 | 	}
116 | 	err := c.kafkaConsumer.SubscribeTopics(c.topic, nil)
117 | 	if err != nil {
118 | 		log2.Error.Println("Error listening to topics", err)
119 | 	}
120 | 	c.Message = make(chan *kafka.Message)
121 | 	c.IsOpen = true
122 | 	go c.runPooler()
123 | 	return nil
124 | }
125 | 
126 | func (c *Consumer) Close() {
127 | 	if c.IsOpen == false {
128 | 		return
129 | 	}
130 | 	c.IsOpen = false
131 | 	c.kafkaConsumer.Close()
132 | }
133 | 


--------------------------------------------------------------------------------
/utils/kafka/producer.go:
--------------------------------------------------------------------------------
  1 | package kafka
  2 | 
  3 | import (
  4 | 	"errors"
  5 | 	"github.com/confluentinc/confluent-kafka-go/kafka"
  6 | 	"github.com/json-iterator/go"
  7 | 	"log"
  8 | 	log2 "rulecat/utils/log"
  9 | 	"strings"
 10 | )
 11 | 
 12 | var json = jsoniter.ConfigCompatibleWithStandardLibrary
 13 | 
 14 | type DataProducer struct {
 15 | 	IsOpen   bool
 16 | 	address  []string
 17 | 	group    string
 18 | 	topic    string
 19 | 	user     string
 20 | 	password string
 21 | 
 22 | 	producer *kafka.Producer
 23 | }
 24 | 
 25 | func CreateProducer(kafkaAddrs []string, kafkaGroup string, user string, password string) *kafka.Producer {
 26 | 	c, err := kafka.NewProducer(&kafka.ConfigMap{
 27 | 		"bootstrap.servers":  strings.Join(kafkaAddrs, ","),
 28 | 		"group.id":           kafkaGroup,
 29 | 		"session.timeout.ms": 30000,
 30 | 		"sasl.mechanisms":    "PLAIN",
 31 | 		"security.protocol":  "SASL_PLAINTEXT",
 32 | 		"sasl.username":      user,
 33 | 		"sasl.password":      password,
 34 | 	})
 35 | 	if err != nil {
 36 | 		log.Fatal(err)
 37 | 	}
 38 | 	return c
 39 | }
 40 | 
 41 | func (pd *DataProducer) AddMessage(message []byte) error {
 42 | 	var err error
 43 | 	go func() {
 44 | 		for e := range pd.producer.Events() {
 45 | 			switch ev := e.(type) {
 46 | 			case *kafka.Message:
 47 | 				if ev.TopicPartition.Error != nil {
 48 | 					log2.Error.Printf("Kafka Delivery failed: %v\n", ev.TopicPartition)
 49 | 				} else {
 50 | 					log2.Info.Printf("Kafka Delivered message to %v\n", ev.TopicPartition)
 51 | 				}
 52 | 			}
 53 | 		}
 54 | 	}()
 55 | 
 56 | 	err = pd.producer.Produce(&kafka.Message{
 57 | 		TopicPartition: kafka.TopicPartition{Topic: &pd.topic, Partition: kafka.PartitionAny},
 58 | 		Value:          message,
 59 | 		Headers:        []kafka.Header{},
 60 | 	}, nil)
 61 | 
 62 | 	pd.producer.Flush(10 * 1000)
 63 | 	return err
 64 | 
 65 | }
 66 | 
 67 | func InitKafkaProducer(kafkaAddrs []string, kafkaGroup string, topic string, user string, password string) *DataProducer {
 68 | 
 69 | 	pd := new(DataProducer)
 70 | 	pd.address = kafkaAddrs
 71 | 	pd.group = kafkaGroup
 72 | 	pd.topic = topic
 73 | 
 74 | 	pd.user = user
 75 | 	pd.password = password
 76 | 	pd.Open()
 77 | 	return pd
 78 | }
 79 | 
 80 | func (pd *DataProducer) Open() error {
 81 | 	if pd.IsOpen == true {
 82 | 		return errors.New("Unable to open log consumer, its already open.")
 83 | 	}
 84 | 	if pd.address == nil || len(pd.address) == 0 {
 85 | 		return errors.New("invalid address")
 86 | 	}
 87 | 	if pd.group == "" {
 88 | 		return errors.New("invalid group")
 89 | 	}
 90 | 	pd.producer = CreateProducer(pd.address, pd.group, pd.user, pd.password)
 91 | 	pd.IsOpen = true
 92 | 	return nil
 93 | }
 94 | 
 95 | func (pd *DataProducer) Close() {
 96 | 	if pd.IsOpen == false {
 97 | 		return
 98 | 	}
 99 | 
100 | 	pd.producer.Close()
101 | 	pd.IsOpen = false
102 | }
103 | 


--------------------------------------------------------------------------------
/utils/log/log.go:
--------------------------------------------------------------------------------
 1 | package log
 2 | 
 3 | import (
 4 | 	"io"
 5 | 	"log"
 6 | 	"os"
 7 | 	"path/filepath"
 8 | 	"strings"
 9 | )
10 | 
11 | var (
12 | 	Info    *log.Logger
13 | 	Warning *log.Logger
14 | 	Error   *log.Logger
15 | )
16 | 
17 | func init() {
18 | 
19 | 	dir, _ := os.Getwd()
20 | 	currentPath := strings.Replace(dir, "\\", "/", -1)
21 | 	logFilePath := filepath.Join(currentPath, "logs", "rulecat.log")
22 | 	logDir := filepath.Dir(logFilePath)
23 | 
24 | 	if _, err := os.Stat(logDir); os.IsNotExist(err) {
25 | 		if err := os.MkdirAll(logDir, 0755); err != nil {
26 | 			log.Fatalf("Create log dir err: %v", err)
27 | 		}
28 | 	}
29 | 
30 | 	errFile, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
31 | 	if err != nil {
32 | 		log.Fatalf("Open log file err: %v", err)
33 | 	}
34 | 
35 | 	Info = log.New(io.MultiWriter(os.Stdout, errFile), "Info:", log.Ldate|log.Ltime|log.Lshortfile)
36 | 	Warning = log.New(io.MultiWriter(os.Stdout, errFile), "Warning:", log.Ldate|log.Ltime|log.Lshortfile)
37 | 	Error = log.New(io.MultiWriter(os.Stderr, errFile), "Error:", log.Ldate|log.Ltime|log.Lshortfile)
38 | 
39 | }
40 | 


--------------------------------------------------------------------------------
/utils/utils.go:
--------------------------------------------------------------------------------
  1 | package utils
  2 | 
  3 | import (
  4 | 	"bytes"
  5 | 	Json "encoding/json"
  6 | 	"fmt"
  7 | 	"github.com/dimiro1/banner"
  8 | 	"github.com/json-iterator/go"
  9 | 	"os"
 10 | 	log2 "rulecat/utils/log"
 11 | 	"strings"
 12 | 	"sync"
 13 | )
 14 | 
 15 | var json = jsoniter.ConfigCompatibleWithStandardLibrary
 16 | 
 17 | func GetCurrentPath() string {
 18 | 	dir, err := os.Getwd()
 19 | 	if err != nil {
 20 | 		log2.Error.Fatalln(err)
 21 | 	}
 22 | 	return strings.Replace(dir, "\\", "/", -1)
 23 | }
 24 | 
 25 | func GetAllFile(pathname string, s []string) ([]string, error) {
 26 | 	rd, err := os.ReadDir(pathname)
 27 | 	if err != nil {
 28 | 		log2.Error.Println("read dir fail:", err)
 29 | 		return s, err
 30 | 	}
 31 | 	for _, fi := range rd {
 32 | 		if fi.IsDir() {
 33 | 			fullDir := pathname + "/" + fi.Name()
 34 | 			s, err = GetAllFile(fullDir, s)
 35 | 			if err != nil {
 36 | 				log2.Error.Println("read dir fail:", err)
 37 | 				return s, err
 38 | 			}
 39 | 		} else {
 40 | 			fullName := pathname + "/" + fi.Name()
 41 | 			s = append(s, fullName)
 42 | 		}
 43 | 	}
 44 | 	return s, nil
 45 | }
 46 | 
 47 | func ConvertToStringMap(m interface{}) interface{} {
 48 | 	switch x := m.(type) {
 49 | 	case map[interface{}]interface{}:
 50 | 		newMap := map[string]interface{}{}
 51 | 		for k, v := range x {
 52 | 			newMap[fmt.Sprint(k)] = ConvertToStringMap(v)
 53 | 		}
 54 | 		return newMap
 55 | 	case map[string]interface{}:
 56 | 		newMap := map[string]interface{}{}
 57 | 		for k, v := range x {
 58 | 			newMap[k] = ConvertToStringMap(v)
 59 | 		}
 60 | 		return newMap
 61 | 	case []interface{}:
 62 | 		newSlice := make([]interface{}, len(x))
 63 | 		for i, v := range x {
 64 | 			newSlice[i] = ConvertToStringMap(v)
 65 | 		}
 66 | 		return newSlice
 67 | 	default:
 68 | 		return x
 69 | 	}
 70 | }
 71 | 
 72 | func MarshalSMapToJSON(m *sync.Map) ([]byte, error) {
 73 | 
 74 | 	if m == nil {
 75 | 		return nil, fmt.Errorf("sync.Map is nil")
 76 | 	}
 77 | 
 78 | 	tmpMap := make(map[interface{}]interface{})
 79 | 	m.Range(func(k, v interface{}) bool {
 80 | 		if k == nil || v == nil {
 81 | 			return true
 82 | 		}
 83 | 		tmpMap[k] = v
 84 | 		return true
 85 | 	})
 86 | 
 87 | 	convertedMap := ConvertToStringMap(tmpMap)
 88 | 	data, err := Json.Marshal(convertedMap)
 89 | 	if err != nil {
 90 | 		return nil, fmt.Errorf("failed to marshal map to JSON: %w", err)
 91 | 	}
 92 | 	return data, nil
 93 | }
 94 | 
 95 | func MapToSMap(tmpMap map[string]interface{}) (*sync.Map, error) {
 96 | 
 97 | 	m := &sync.Map{}
 98 | 
 99 | 	for key, value := range tmpMap {
100 | 		m.Store(key, value)
101 | 	}
102 | 	return m, nil
103 | }
104 | 
105 | func SMapToMap(tmpMap *sync.Map) map[interface{}]interface{} {
106 | 
107 | 	var m = make(map[interface{}]interface{})
108 | 	tmpMap.Range(func(k, v interface{}) bool {
109 | 		m[k] = v
110 | 		return true
111 | 	})
112 | 	return m
113 | }
114 | 
115 | func FormatJson(data interface{}) string {
116 | 	var out bytes.Buffer
117 | 	data_, _ := Json.Marshal(data)
118 | 	Json.Indent(&out, data_, "", "    ")
119 | 	return out.String()
120 | }
121 | 
122 | func WriteFile(path string, str string) error {
123 | 	_, fileExists := IsFile(path)
124 | 
125 | 	var f *os.File
126 | 	var err error
127 | 
128 | 	if fileExists {
129 | 		f, err = os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0644)
130 | 	} else {
131 | 		f, err = os.Create(path)
132 | 	}
133 | 
134 | 	if err != nil {
135 | 		log2.Error.Printf("Failed to open/create file: %v", err)
136 | 		return err
137 | 	}
138 | 	defer func() {
139 | 		if cerr := f.Close(); cerr != nil {
140 | 			log2.Error.Printf("Failed to close file: %v", cerr)
141 | 		}
142 | 	}()
143 | 
144 | 	_, err = f.WriteString(str)
145 | 	if err != nil {
146 | 		log2.Error.Printf("Failed to write to file: %v", err)
147 | 		return err
148 | 	}
149 | 
150 | 	return nil
151 | }
152 | 
153 | func IsExists(path string) (os.FileInfo, bool) {
154 | 	f, err := os.Stat(path)
155 | 	return f, err == nil || os.IsExist(err)
156 | }
157 | 
158 | func IsFile(path string) (os.FileInfo, bool) {
159 | 	f, flag := IsExists(path)
160 | 	return f, flag && !f.IsDir()
161 | }
162 | 
163 | func ServerBanner() {
164 | 	template := `
165 | {{ .Title "RuleCat" "" 4 }}
166 | GoVersion: {{ .GoVersion }}
167 | GOOS: {{ .GOOS }}
168 | GOARCH: {{ .GOARCH }}
169 | NumCPU: {{ .NumCPU }}
170 | Now: {{ .Now "2006-01-02 15:04:05" }}
171 | 	
172 | `
173 | 	banner.InitString(os.Stdout, true, true, template)
174 | }
175 | 


--------------------------------------------------------------------------------
/utils/workerpool/workerpool.go:
--------------------------------------------------------------------------------
 1 | package workerpool
 2 | 
 3 | type Job interface {
 4 | 	Do() error
 5 | }
 6 | 
 7 | type Worker struct {
 8 | 	JobQueue chan Job
 9 | }
10 | 
11 | func NewWorker() Worker {
12 | 	return Worker{
13 | 		JobQueue: make(chan Job)}
14 | }
15 | 
16 | func (w Worker) Run(wq chan chan Job) {
17 | 	go func() {
18 | 		for {
19 | 			wq <- w.JobQueue
20 | 
21 | 			select {
22 | 			case job := <-w.JobQueue:
23 | 				job.Do()
24 | 			}
25 | 		}
26 | 	}()
27 | }
28 | 
29 | type WorkerPool struct {
30 | 	workerlen   int
31 | 	JobQueue    chan Job
32 | 	WorkerQueue chan chan Job
33 | }
34 | 
35 | func NewWorkerPool(workerlen int) *WorkerPool {
36 | 	return &WorkerPool{
37 | 		workerlen:   workerlen,
38 | 		JobQueue:    make(chan Job),
39 | 		WorkerQueue: make(chan chan Job, workerlen),
40 | 	}
41 | }
42 | 
43 | func (wp *WorkerPool) Run() {
44 | 
45 | 	for i := 0; i < wp.workerlen; i++ {
46 | 		worker := NewWorker()
47 | 		worker.Run(wp.WorkerQueue)
48 | 	}
49 | 	go func() {
50 | 		for {
51 | 			select {
52 | 			case job := <-wp.JobQueue:
53 | 				worker := <-wp.WorkerQueue
54 | 				worker <- job
55 | 			}
56 | 		}
57 | 	}()
58 | }
59 | 


--------------------------------------------------------------------------------