├── .gitignore ├── LICENSE ├── README.md ├── Vagrantfile ├── docker-compose.yml ├── docs ├── hints_and_tips.md ├── images │ ├── techlog_events.png │ └── techlog_events_correlations.png └── server_call.md ├── filebeat ├── configs │ └── onectechlog.yml ├── filebeat.yml └── techlog.template.json ├── install-pipeline.sh └── techlog-pipeline.json /.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant/ 2 | *.log -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This program is free software: you can redistribute it and/or modify 2 | it under the terms of the GNU General Public License as published by 3 | the Free Software Foundation, either version 3 of the License, or 4 | (at your option) any later version. 5 | 6 | This program is distributed in the hope that it will be useful, 7 | but WITHOUT ANY WARRANTY; without even the implied warranty of 8 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 9 | GNU General Public License for more details. 10 | 11 | You should have received a copy of the GNU General Public License 12 | along with this program. If not, see . -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Технологический журнал и EBK 2 | 3 | В репозитории собраны различные настройки для работы с технологическим журналом на стеке EBK. 4 | 5 | Для запуска ВМ с Elasticsearch и Kibana, в каталоге репозитория нужно выполнить команду: 6 | 7 | ``` 8 | vagrant up 9 | ``` 10 | 11 | После запуска виртуальной машины в ней будут запущены docker конейнеры с Elasticsearch (Opendistro for Elasticsearch от Amazon) и Kibana. 12 | Запуск контейнеров выполняется с помощью docker-compose и файла [docker-compose.yml](./docker-compose.yml). 13 | 14 | Сервисы будут доступны по следующему адресу: 15 | 16 | - 192.168.33.30:5601 - Kibana 17 | - 192.168.33.30:9200 - Elasticsearch 18 | 19 | Для изменения адресов и портов необходимо выполнить согласованные изменения в файлах: 20 | 21 | - [docker-compose.yml](./docker-compose.yml) 22 | - [Vagrantfile](./Vagrantfile) 23 | - [install-pipeline.sh](./install-pipeline.sh) 24 | - [filebeat.yml](./filebeat/filebeat.yml) 25 | 26 | Скрипт [install-pipeline.sh](./install-pipeline.sh) установит в Elasticsearch pipeline для разбора записей лога технологического журнала. 27 | Для выполнения установки нужно запустить команду: 28 | 29 | ``` 30 | bash install-pipeline.sh 31 | ``` 32 | 33 | Pipeline в формате json находится в файле [techlog-pipeline.json](./techlog-pipeline.json). 34 | 35 | В каталоге `filebeat` расположены конфигурационные файлы для настройки filebeat под работу с логами технологического журнала. 36 | После установки filebeat и до его запуска, нужно сделать следующие замены файлов в каталоге установки: 37 | 38 | - исходный файл `filebeat.yml` заменить файлом [filebeat.yml](./filebeat/filebeat.yml) (при необходимости исправить адрес Elasticsearch и реквизиты авторизации). 39 | - скопировать файл [techlog.template.json](./filebeat/techlog.template.json) в каталог установки filebeat (это json-файл шаблона индекса Elasticsearch для хранения логов технологического журнала, filebeat самостоятельно загрузит его в Elasticsearch). 40 | - скопировать каталог [filebeat/configs](./filebeat/configs) в каталог установки filebeat (в каталоге configs хранятся конфигурационные файлы, которые указывают filebeat откуда собирать файлы логов, как определять многострочные записи в файлах логов, добавляют информацию о часовом поясе и типу лога). 41 | 42 | Дистрибутив Elasticsearch от Amazon работает только с filebeat-oss, который можно скачать по адресу: 43 | 44 | [filebeat-oss](https://www.elastic.co/downloads/beats/filebeat-oss) 45 | 46 | Дополнительная информация по анализу технологического журнала: 47 | 48 | - [Путешествие серверного вызова в событиях технологического журнала](./docs/server_call.md) 49 | - [Советы и подсказки](./docs/hints_and_tips.md) -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | Vagrant.configure("2") do |config| 2 | 3 | config.vm.box = "centos/7" 4 | 5 | config.vm.network "forwarded_port", guest: 9200, host: 9200 6 | config.vm.network "forwarded_port", guest: 9600, host: 9600 7 | config.vm.network "forwarded_port", guest: 5601, host: 5601 8 | 9 | config.vm.network "private_network", ip: "192.168.33.30" 10 | 11 | config.vm.provider "virtualbox" do |vb| 12 | vb.memory = "4096" 13 | vb.cpus = 2 14 | end 15 | 16 | config.vm.provision :docker 17 | config.vm.provision :docker_compose, yml: "/vagrant/docker-compose.yml", rebuild: true, run: "always" 18 | 19 | config.vm.provision :shell, inline: <<-SHELL 20 | usermod -aG docker vagrant 21 | SHELL 22 | 23 | end 24 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | techlog-node1: 4 | image: amazon/opendistro-for-elasticsearch:1.4.0 5 | container_name: techlog-node1 6 | environment: 7 | - discovery.type=single-node 8 | - cluster.name=techlog-cluster 9 | - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping 10 | - "ES_JAVA_OPTS=-Xms2024m -Xmx2024m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM 11 | ulimits: 12 | memlock: 13 | soft: -1 14 | hard: -1 15 | volumes: 16 | - techlog-data1:/usr/share/elasticsearch/data 17 | ports: 18 | - 9200:9200 19 | - 9600:9600 # required for Performance Analyzer 20 | networks: 21 | - techlog-net 22 | kibana: 23 | image: amazon/opendistro-for-elasticsearch-kibana:1.4.0 24 | container_name: techlog-kibana 25 | ports: 26 | - 5601:5601 27 | expose: 28 | - "5601" 29 | environment: 30 | ELASTICSEARCH_URL: https://192.168.33.30:9200 31 | ELASTICSEARCH_HOSTS: https://192.168.33.30:9200 32 | networks: 33 | - techlog-net 34 | 35 | volumes: 36 | techlog-data1: 37 | 38 | networks: 39 | techlog-net: -------------------------------------------------------------------------------- /docs/hints_and_tips.md: -------------------------------------------------------------------------------- 1 | ## Советы и подсказки 2 | 3 | ### CALL / SCALL IName=ISeanceContextStorage 4 | 5 | - `attachSeanceIB` - назначение сеанса соединению с информационной базой (делается в начале вызова). 6 | - `detachSeanceIB` - отмена назначения сеанса соединению с информационной базой (делается при окончании вызова). 7 | - `seanceParametersPresave` - сохранение измененных за вызов данных сеанса. 8 | - `seanceParametersCommit` - подтверждение сохранения. 9 | - `setCallID` - пометка номера вызова и запрета повтора. 10 | 11 | ### В данной транзакции уже происходили ошибки ... 12 | 13 | Расследование подобной ошибки может быть очень сложным и утомительным. 14 | Чтобы найти место в котором происходит отметка транзакции как не восстановимой, можно собрать события SDBL с фильтром `Func=setRollbackOnly`. 15 | 16 | ### Эскалации управляемых блокировок и ошибки. 17 | 18 | Эскалация управляемых блокировок отмечается в событии TLOCK. 19 | Такие события можно найти в технологическом журнале по запросу ealsticsearch: 20 | 21 | ``` 22 | +event:TLOCK +"escalating=true" 23 | ``` 24 | 25 | Ошибки при установке управляемых блокировок можно найти по запросу: 26 | 27 | ``` 28 | +event:EXCP +"Exception=TLockException" 29 | ``` -------------------------------------------------------------------------------- /docs/images/techlog_events.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maxstarkov/techlog-es/90257baba32f91b94f2d6494fc11cca023ab86b7/docs/images/techlog_events.png -------------------------------------------------------------------------------- /docs/images/techlog_events_correlations.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/maxstarkov/techlog-es/90257baba32f91b94f2d6494fc11cca023ab86b7/docs/images/techlog_events_correlations.png -------------------------------------------------------------------------------- /docs/server_call.md: -------------------------------------------------------------------------------- 1 | ## Серверный вызов и события технологического журнала 2 | 3 | Путешествие серверного вызова с клиента до СУБД можно представить схемой: 4 | 5 | ![techlog_events](./images/techlog_events.png) 6 | 7 | Взаимосвязь событий технологического журнала через их свойства можно представить схемой: 8 | 9 | ![techlog_events_correlations](./images/techlog_events_correlations.png) -------------------------------------------------------------------------------- /filebeat/configs/onectechlog.yml: -------------------------------------------------------------------------------- 1 | - type: log 2 | enabled: true 3 | paths: 4 | - D:\techlog-arch\*\*\*\*\*\*\*.log 5 | 6 | fields: 7 | log_type: all 8 | log_timezone: Asia/Yekaterinburg 9 | 10 | multiline.pattern: ^\d{2}:\d{2}\.\d{6} 11 | multiline.negate: true 12 | multiline.match: after -------------------------------------------------------------------------------- /filebeat/filebeat.yml: -------------------------------------------------------------------------------- 1 | filebeat.config.inputs: 2 | 3 | enable: true 4 | path: configs\*.yml 5 | reload.enabled: true 6 | reload.period: 10s 7 | 8 | max_procs: 2 9 | 10 | #output.console: 11 | # enabled: true 12 | 13 | setup.template.name: "techlog" 14 | setup.template.pattern: "techlog-*" 15 | setup.template.overwrite: true 16 | setup.template.json.enabled: true 17 | setup.template.json.path: "techlog.template.json" 18 | setup.template.json.name: "techlog" 19 | 20 | setup.ilm.enabled: false 21 | 22 | output.elasticsearch: 23 | hosts: ["https://192.168.33.30:9200"] 24 | username: "admin" 25 | password: "admin" 26 | pipeline: techlog 27 | index: "techlog-%{[fields.log_type]}-%{+yyyy.MM.dd}" 28 | ssl.verification_mode: none -------------------------------------------------------------------------------- /filebeat/techlog.template.json: -------------------------------------------------------------------------------- 1 | { 2 | "order": 0, 3 | "index_patterns": [ 4 | "techlog-*" 5 | ], 6 | "settings": { 7 | "index": { 8 | "codec": "best_compression", 9 | "refresh_interval": "10s", 10 | "number_of_shards": "1", 11 | "translog": { 12 | "durability": "async" 13 | }, 14 | "number_of_replicas": "0", 15 | "query": { 16 | "default_field": "message" 17 | } 18 | } 19 | }, 20 | "mappings": { 21 | "dynamic_templates": [ 22 | { 23 | "strings_as_keyword": { 24 | "mapping": { 25 | "type": "keyword", 26 | "ignore_above": 8191 27 | }, 28 | "match_mapping_type": "string" 29 | } 30 | }, 31 | { 32 | "indexed_longs": { 33 | "mapping": { 34 | "type": "long" 35 | }, 36 | "match_mapping_type": "long" 37 | } 38 | } 39 | ], 40 | "properties": { 41 | "@timestamp": { 42 | "type": "date" 43 | }, 44 | "message": { 45 | "type": "text", 46 | "norms": false 47 | }, 48 | "duration": { 49 | "type": "long" 50 | }, 51 | "Rows": { 52 | "type": "long" 53 | }, 54 | "RowsAffected": { 55 | "type": "long" 56 | }, 57 | "Memory": { 58 | "type": "long" 59 | }, 60 | "MemoryPeak": { 61 | "type": "long" 62 | }, 63 | "InBytes": { 64 | "type": "long" 65 | }, 66 | "OutBytes": { 67 | "type": "long" 68 | }, 69 | "CpuTime": { 70 | "type": "long" 71 | }, 72 | "offset": { 73 | "type": "long" 74 | }, 75 | "avgResponseTime": { 76 | "type": "long" 77 | }, 78 | "maxResponseTime": { 79 | "type": "long" 80 | }, 81 | "packetsTimedOut": { 82 | "type": "long" 83 | } 84 | } 85 | }, 86 | "aliases": {} 87 | } 88 | -------------------------------------------------------------------------------- /install-pipeline.sh: -------------------------------------------------------------------------------- 1 | curl -XPUT https://192.168.33.30:9200/_ingest/pipeline/techlog -v -u admin:admin --insecure -H 'Content-Type: application/json' -d @techlog-pipeline.json 2 | -------------------------------------------------------------------------------- /techlog-pipeline.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "techlog pipeline common", 3 | "processors": [ 4 | { 5 | "grok": { 6 | "field": "log.file.path", 7 | "patterns": [ 8 | "\\\\%{USER:onectechlog.logname}\\\\%{USER:onectechlog.process}_%{INT:onectechlog.pid}\\\\%{INT}.log" 9 | ] 10 | } 11 | }, 12 | { 13 | "grok": { 14 | "field": "message", 15 | "patterns": [ 16 | "%{INT:_ingest.tempmm}:%{BASE10NUM:_ingest.tempss}-%{INT:duration},(%{WORD:event})?,%{INT:level}" 17 | ] 18 | } 19 | }, 20 | { 21 | "grok": { 22 | "field": "log.file.path", 23 | "patterns": [ 24 | "%{INT:_ingest.tempyymmddhh}.log" 25 | ] 26 | } 27 | }, 28 | { 29 | "set": { 30 | "field": "_ingest.tempdate", 31 | "value": "{{_ingest.tempyymmddhh}}{{_ingest.tempmm}}{{_ingest.tempss}}" 32 | } 33 | }, 34 | { 35 | "date": { 36 | "field": "_ingest.tempdate", 37 | "target_field": "@timestamp", 38 | "formats": [ 39 | "yyMMddHHmmss.SSSSSS" 40 | ], 41 | "timezone": "{{fields.log_timezone}}" 42 | } 43 | }, 44 | { 45 | "set": { 46 | "field": "LogTimestamp", 47 | "value": "{{_ingest.tempdate}}" 48 | } 49 | }, 50 | { 51 | "set": { 52 | "field": "_id", 53 | "value": "{{_ingest.tempdate}}-{{_source.log.offset}}" 54 | } 55 | }, 56 | { 57 | "grok": { 58 | "field": "message", 59 | "patterns": [ 60 | "process=%{DATA:process}," 61 | ], 62 | "on_failure": [ 63 | { 64 | "set": { 65 | "field": "process", 66 | "value": "" 67 | } 68 | } 69 | ] 70 | } 71 | }, 72 | { 73 | "grok": { 74 | "field": "message", 75 | "patterns": [ 76 | "p:processName=%{DATA:pprocessName}," 77 | ], 78 | "on_failure": [ 79 | { 80 | "set": { 81 | "field": "pprocessName", 82 | "value": "" 83 | } 84 | } 85 | ] 86 | } 87 | }, 88 | { 89 | "grok": { 90 | "field": "message", 91 | "patterns": [ 92 | "t:applicationName=%{DATA:tapplicationName}," 93 | ], 94 | "on_failure": [ 95 | { 96 | "set": { 97 | "field": "tapplicationName", 98 | "value": "" 99 | } 100 | } 101 | ] 102 | } 103 | }, 104 | { 105 | "grok": { 106 | "field": "message", 107 | "patterns": [ 108 | "t:computerName=%{DATA:tcomputerName}," 109 | ], 110 | "on_failure": [ 111 | { 112 | "set": { 113 | "field": "tcomputerName", 114 | "value": "" 115 | } 116 | } 117 | ] 118 | } 119 | }, 120 | { 121 | "grok": { 122 | "field": "message", 123 | "patterns": [ 124 | "SessionID=%{DATA:SessionID}(,|\\()" 125 | ], 126 | "on_failure": [ 127 | { 128 | "set": { 129 | "field": "SessionID", 130 | "value": "" 131 | } 132 | } 133 | ] 134 | } 135 | }, 136 | { 137 | "grok": { 138 | "field": "message", 139 | "patterns": [ 140 | "Usr=%{DATA:Usr}," 141 | ], 142 | "on_failure": [ 143 | { 144 | "set": { 145 | "field": "Usr", 146 | "value": "" 147 | } 148 | } 149 | ] 150 | } 151 | }, 152 | { 153 | "grok": { 154 | "field": "message", 155 | "patterns": [ 156 | "Exception=%{DATA:Exception}," 157 | ], 158 | "on_failure": [ 159 | { 160 | "set": { 161 | "field": "Exception", 162 | "value": "" 163 | } 164 | } 165 | ] 166 | } 167 | }, 168 | { 169 | "grok": { 170 | "field": "message", 171 | "patterns": [ 172 | "RetExcp=(%{onecstr:RetExcp},|%{onecstr:RetExcp}|%{DATA:RetExcp},|%{GREEDYDATA:RetExcp})" 173 | ], 174 | "pattern_definitions": { 175 | "onecstr": "('(?m)(.*)')|(\"(?m)(.*)\")" 176 | }, 177 | "on_failure": [ 178 | { 179 | "set": { 180 | "field": "RetExcp", 181 | "value": "" 182 | } 183 | } 184 | ] 185 | } 186 | }, 187 | { 188 | "grok": { 189 | "field": "message", 190 | "patterns": [ 191 | "Descr=(%{onecstr:Descr},|%{onecstr:Descr}|%{DATA:Descr},|%{GREEDYDATA:Descr})" 192 | ], 193 | "pattern_definitions": { 194 | "onecstr": "('(?m)(.*)')|(\"(?m)(.*)\")" 195 | }, 196 | "on_failure": [ 197 | { 198 | "set": { 199 | "field": "Descr", 200 | "value": "" 201 | } 202 | } 203 | ] 204 | } 205 | }, 206 | { 207 | "grok": { 208 | "field": "message", 209 | "patterns": [ 210 | "Context=(%{onecstr:Context}|%{DATA:Context},|%{GREEDYDATA:Context})" 211 | ], 212 | "pattern_definitions": { 213 | "onecstr": "('(?m)(.*)')|(\"(?m)(.*)\")" 214 | }, 215 | "on_failure": [ 216 | { 217 | "set": { 218 | "field": "Context", 219 | "value": "" 220 | } 221 | } 222 | ] 223 | } 224 | }, 225 | { 226 | "grok": { 227 | "field": "message", 228 | "patterns": [ 229 | "Context=(%{firstline:FirstLineContext}|%{DATA:FirstLineContext},|%{GREEDYDATA:FirstLineContext})" 230 | ], 231 | "pattern_definitions": { 232 | "firstline": "('|\")\n?(.*)" 233 | }, 234 | "on_failure": [ 235 | { 236 | "set": { 237 | "field": "FirstLineContext", 238 | "value": "" 239 | } 240 | } 241 | ] 242 | } 243 | }, 244 | { 245 | "gsub": { 246 | "field": "FirstLineContext", 247 | "pattern": "('|\"|\n)", 248 | "replacement": "" 249 | } 250 | }, 251 | { 252 | "grok": { 253 | "field": "message", 254 | "patterns": [ 255 | "Txt=(%{QS:Txt}|%{DATA:Txt},|%{GREEDYDATA:Txt})" 256 | ], 257 | "on_failure": [ 258 | { 259 | "set": { 260 | "field": "Txt", 261 | "value": "" 262 | } 263 | } 264 | ] 265 | } 266 | }, 267 | { 268 | "grok": { 269 | "field": "message", 270 | "patterns": [ 271 | "Sql=(%{QS:Sql}|%{DATA:Sql},|%{GREEDYDATA:Sql})" 272 | ], 273 | "on_failure": [ 274 | { 275 | "set": { 276 | "field": "Sql", 277 | "value": "" 278 | } 279 | } 280 | ] 281 | } 282 | }, 283 | { 284 | "grok": { 285 | "field": "Sql", 286 | "patterns": [ 287 | "%{SqlParameters:SqlParameters}" 288 | ], 289 | "pattern_definitions": { 290 | "SqlParameters": "(p_.*(\n))+" 291 | }, 292 | "on_failure": [ 293 | { 294 | "set": { 295 | "field": "SqlParameters", 296 | "value": "" 297 | } 298 | } 299 | ] 300 | } 301 | }, 302 | { 303 | "gsub": { 304 | "field": "Sql", 305 | "pattern": "p_.*(\n)", 306 | "replacement": "" 307 | } 308 | }, 309 | { 310 | "gsub": { 311 | "field": "Sql", 312 | "pattern": "#tt[0-9]*", 313 | "replacement": "TEMPTABLE" 314 | } 315 | }, 316 | { 317 | "grok": { 318 | "field": "message", 319 | "patterns": [ 320 | "Rows=(%{DATA:Rows},|%{GREEDYDATA:Rows})" 321 | ], 322 | "on_failure": [ 323 | { 324 | "set": { 325 | "field": "Rows", 326 | "value": 0 327 | } 328 | } 329 | ] 330 | } 331 | }, 332 | { 333 | "grok": { 334 | "field": "message", 335 | "patterns": [ 336 | "RowsAffected=(%{DATA:RowsAffected},|%{GREEDYDATA:RowsAffected})" 337 | ], 338 | "on_failure": [ 339 | { 340 | "set": { 341 | "field": "RowsAffected", 342 | "value": 0 343 | } 344 | } 345 | ] 346 | } 347 | }, 348 | { 349 | "grok": { 350 | "field": "message", 351 | "patterns": [ 352 | "Prm=(%{QS:Prm}|%{DATA:Prm},|%{GREEDYDATA:Prm})" 353 | ], 354 | "on_failure": [ 355 | { 356 | "set": { 357 | "field": "Prm", 358 | "value": "" 359 | } 360 | } 361 | ] 362 | } 363 | }, 364 | { 365 | "grok": { 366 | "field": "message", 367 | "patterns": [ 368 | "Interface=(%{DATA:Interface},|%{GREEDYDATA:Interface})" 369 | ], 370 | "on_failure": [ 371 | { 372 | "set": { 373 | "field": "Interface", 374 | "value": "" 375 | } 376 | } 377 | ] 378 | } 379 | }, 380 | { 381 | "grok": { 382 | "field": "message", 383 | "patterns": [ 384 | "IName=(%{DATA:IName},|%{GREEDYDATA:IName})" 385 | ], 386 | "on_failure": [ 387 | { 388 | "set": { 389 | "field": "IName", 390 | "value": "" 391 | } 392 | } 393 | ] 394 | } 395 | }, 396 | { 397 | "grok": { 398 | "field": "message", 399 | "patterns": [ 400 | "Method=(%{DATA:Method},|%{GREEDYDATA:Method})" 401 | ], 402 | "on_failure": [ 403 | { 404 | "set": { 405 | "field": "Method", 406 | "value": "" 407 | } 408 | } 409 | ] 410 | } 411 | }, 412 | { 413 | "grok": { 414 | "field": "message", 415 | "patterns": [ 416 | "CallID=(%{DATA:CallID},|%{GREEDYDATA:CallID})" 417 | ], 418 | "on_failure": [ 419 | { 420 | "set": { 421 | "field": "CallID", 422 | "value": "" 423 | } 424 | } 425 | ] 426 | } 427 | }, 428 | { 429 | "grok": { 430 | "field": "message", 431 | "patterns": [ 432 | "MName=(%{DATA:MName},|%{GREEDYDATA:MName})" 433 | ], 434 | "on_failure": [ 435 | { 436 | "set": { 437 | "field": "MName", 438 | "value": "" 439 | } 440 | } 441 | ] 442 | } 443 | }, 444 | { 445 | "grok": { 446 | "field": "message", 447 | "patterns": [ 448 | "Memory=(%{DATA:Memory},|%{GREEDYDATA:Memory})" 449 | ], 450 | "on_failure": [ 451 | { 452 | "set": { 453 | "field": "Memory", 454 | "value": 0 455 | } 456 | } 457 | ] 458 | } 459 | }, 460 | { 461 | "grok": { 462 | "field": "message", 463 | "patterns": [ 464 | "MemoryPeak=(%{DATA:MemoryPeak},|%{GREEDYDATA:MemoryPeak})" 465 | ], 466 | "on_failure": [ 467 | { 468 | "set": { 469 | "field": "MemoryPeak", 470 | "value": 0 471 | } 472 | } 473 | ] 474 | } 475 | }, 476 | { 477 | "grok": { 478 | "field": "message", 479 | "patterns": [ 480 | "InBytes=(%{DATA:InBytes},|%{GREEDYDATA:InBytes})" 481 | ], 482 | "on_failure": [ 483 | { 484 | "set": { 485 | "field": "InBytes", 486 | "value": 0 487 | } 488 | } 489 | ] 490 | } 491 | }, 492 | { 493 | "grok": { 494 | "field": "message", 495 | "patterns": [ 496 | "OutBytes=(%{DATA:OutBytes},|%{GREEDYDATA:OutBytes})" 497 | ], 498 | "on_failure": [ 499 | { 500 | "set": { 501 | "field": "OutBytes", 502 | "value": 0 503 | } 504 | } 505 | ] 506 | } 507 | }, 508 | { 509 | "grok": { 510 | "field": "message", 511 | "patterns": [ 512 | "CpuTime=(%{DATA:CpuTime},|%{GREEDYDATA:CpuTime})" 513 | ], 514 | "on_failure": [ 515 | { 516 | "set": { 517 | "field": "CpuTime", 518 | "value": 0 519 | } 520 | } 521 | ] 522 | } 523 | }, 524 | { 525 | "grok": { 526 | "field": "message", 527 | "patterns": [ 528 | "Func=(%{QS:Func}|%{DATA:Func},|%{GREEDYDATA:Func})" 529 | ], 530 | "on_failure": [ 531 | { 532 | "set": { 533 | "field": "Func", 534 | "value": "" 535 | } 536 | } 537 | ] 538 | } 539 | }, 540 | { 541 | "grok": { 542 | "field": "message", 543 | "patterns": [ 544 | "Module=(%{DATA:Module},|%{GREEDYDATA:Module})" 545 | ], 546 | "on_failure": [ 547 | { 548 | "set": { 549 | "field": "Module", 550 | "value": "" 551 | } 552 | } 553 | ] 554 | } 555 | }, 556 | { 557 | "grok": { 558 | "field": "message", 559 | "patterns": [ 560 | "Regions=(%{QS:Regions}|%{onecstr:Regions}|%{DATA:Regions},|%{GREEDYDATA:Regions})" 561 | ], 562 | "pattern_definitions": { 563 | "onecstr": "('(?m)(.*)')|(\"(?m)(.*)\")" 564 | }, 565 | "on_failure": [ 566 | { 567 | "set": { 568 | "field": "Regions", 569 | "value": "" 570 | } 571 | } 572 | ] 573 | } 574 | }, 575 | { 576 | "grok": { 577 | "field": "message", 578 | "patterns": [ 579 | "Locks=(%{QS:Locks}|%{onecstr:Locks}|%{DATA:Locks},|%{GREEDYDATA:Locks})" 580 | ], 581 | "pattern_definitions": { 582 | "onecstr": "('(?m)(.*)')|(\"(?m)(.*)\")" 583 | }, 584 | "on_failure": [ 585 | { 586 | "set": { 587 | "field": "Locks", 588 | "value": "" 589 | } 590 | } 591 | ] 592 | } 593 | }, 594 | { 595 | "grok": { 596 | "field": "message", 597 | "patterns": [ 598 | "WaitConnections=(%{INT:WaitConnections}|%{QS:WaitConnections})" 599 | ], 600 | "on_failure": [ 601 | { 602 | "set": { 603 | "field": "WaitConnections", 604 | "value": "" 605 | } 606 | } 607 | ] 608 | } 609 | }, 610 | { 611 | "grok": { 612 | "field": "message", 613 | "patterns": [ 614 | "DeadlockConnectionIntersections=(%{QS:DeadlockConnectionIntersections}|%{DATA:DeadlockConnectionIntersections},|%{GREEDYDATA:DeadlockConnectionIntersections})" 615 | ], 616 | "pattern_definitions": { 617 | "onecstr": "('(?m)(.*)')|(\"(?m)(.*)\")" 618 | }, 619 | "on_failure": [ 620 | { 621 | "set": { 622 | "field": "DeadlockConnectionIntersections", 623 | "value": "" 624 | } 625 | } 626 | ] 627 | } 628 | }, 629 | { 630 | "grok": { 631 | "field": "message", 632 | "patterns": [ 633 | "address=%{HOSTPORT:address}.*avgResponseTime=%{INT:avgResponseTime},maxResponseTime=%{INT:maxResponseTime},packetsTimedOut=%{INT:packetsTimedOut}" 634 | ], 635 | "on_failure": [ 636 | { 637 | "set": { 638 | "field": "address", 639 | "value": "" 640 | } 641 | }, 642 | { 643 | "set": { 644 | "field": "avgResponseTime", 645 | "value": 0 646 | } 647 | }, 648 | { 649 | "set": { 650 | "field": "maxResponseTime", 651 | "value": 0 652 | } 653 | }, 654 | { 655 | "set": { 656 | "field": "packetsTimedOut", 657 | "value": 0 658 | } 659 | } 660 | ] 661 | } 662 | } 663 | ], 664 | "on_failure": [ 665 | { 666 | "set": { 667 | "field": "error", 668 | "value": "{{_ingest.on_failure_message}}" 669 | } 670 | } 671 | ] 672 | } --------------------------------------------------------------------------------