├── .gitignore ├── README.md ├── config ├── transforms │ ├── CALL.vrl │ ├── DBMSSQL.vrl │ ├── EXCP.vrl │ ├── EXCPCNTX.vrl │ ├── LIC.vrl │ ├── QERR.vrl │ ├── SCALL.vrl │ ├── SDBL.vrl │ ├── TDEADLOCK.vrl │ ├── TLOCK.vrl │ ├── TTIMEOUT.vrl │ └── parseLog.vrl └── vector.toml ├── docker-compose.yaml.example ├── sql_scripts ├── create_call.sql ├── create_dbmssql.sql ├── create_excp.sql ├── create_excpcntx.sql ├── create_qerr.sql ├── create_scall.sql ├── create_sdbl.sql ├── create_tdeadlock.sql ├── create_tlock.sql └── create_ttimeout.sql └── techJournal.env.example /.gitignore: -------------------------------------------------------------------------------- 1 | /tmp/** 2 | /**/*.log 3 | *.env 4 | docker-compose.yaml 5 | fileProgress/input_logs/checkpoints.json 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # **Vector_TechJournal.** Парсинг технологического журнала 1С 2 | 3 | 4 | - [Назначение](#назначение) 5 | - [Возможности](#возможности) 6 | - [Реализация](#реализация) 7 | - [Запуск](#запуск) 8 | - [План реализации](#планы) 9 | - [Лицензия](#лицензия) 10 | 11 | ## Назначение 12 | 13 | Данный проект создан для того чтобы искоренить боль от использования технологического журнала. 14 | 15 | Использование ТЖ должно свестись к трем простым действиям. 16 | - Настроил ТЖ 17 | - Запустил сборщик 18 | - Приступил к анализу. 19 | 20 | ### Возможности 21 | 22 | - Vector_TechJournal - это конфигурация Vector для разбора и трансформации файлов технологического журнала 23 | - Предоставляет возможности чтения файлов ТЖ 24 | - В планах реализация разбора всех известных событий ТЖ. 25 | - Выгрузка результатов парсинга в различных форматах 26 | - В файлы в различных форматах 27 | - В в консоль 28 | - В **Clickhouse** 29 | - В **Loki** и другие аггрегаторы логов 30 | - Возможность масштабирования решения в случае большего объема логов 31 | 32 | 33 | ## Реализация 34 | 35 | Продукт представлен в виде docker-compose файла и пригоден для запуска в любой среде поддерживающей контейнеры, а так же файлов конфигурации для него. 36 | Конфигурация обработки каждого отдельного вида событий находится в **[./config/transformations/](/config/transformations/)\.vrl** 37 | 38 | 39 | 40 | ## Запуск 41 | 42 | Для запуска разбора технологического журнала достаточно: 43 | - Подготовить базу данных clickhouse 44 | - Создать базу 45 | - Создать таблицы скриптами из папки [sql_scripts](/sql_scripts) 46 | - Переименовать [docker-compose.yaml.example](/docker-compose.yaml.example) в **docker-compose.yaml**, и [techJournal.env.example](/techJournal.env.example) в **techJournal.env** 47 | - Отредактировать файл techJournal.env 48 | - указать необходимые события которые вы планируете хранить 49 | - Задать параметры авторизации в Clickhouse 50 | - В файле [docker-compose.yaml](/docker-compose.yaml.example) указать в Volume параметры подключения к папке с логами. 51 | - docker-compose up -d 52 | 53 | ## Планы 54 | - Дописать разбор самых востребованых событий. 55 | - Отладить и оптимизировать разбор. 56 | - Добавить словари для "Очеловечивания" информации 57 | - Реализовать фронтенд для удобного просмотра и анализа ТЖ 58 | 59 | 60 | --- 61 | 62 | ## Лицензия 63 | 64 | Distributed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html) 65 | -------------------------------------------------------------------------------- /config/transforms/CALL.vrl: -------------------------------------------------------------------------------- 1 | #Распарсим контент для события CALL 2 | CallID, err = parse_regex(.Content, r'CallID=(?P\d*)', numeric_groups: false) 3 | . = merge(., CallID) 4 | Process, err = parse_regex(.Content, r'process=(?P[^,]*)', numeric_groups: false) 5 | . = merge(., Process) 6 | ProcessName, err = parse_regex(.Content, r'p:processName=(?P[^,]*)', numeric_groups: false) 7 | . = merge(., ProcessName) 8 | OSThread, err = parse_regex(.Content, r'OSThread=(?P\d*)', numeric_groups: false) 9 | . = merge(., OSThread) 10 | ClientID, err = parse_regex(.Content, r'(C|t:c)lientID=(?P\d*)', numeric_groups: false) 11 | . = merge(., ClientID) 12 | ApplicationName, err = parse_regex(.Content, r't:applicationName=(?P[^,]*)', numeric_groups: false) 13 | . = merge(., ApplicationName) 14 | ComputerName, err = parse_regex(.Content, r't:computerName=(?P[^,]*)', numeric_groups: false) 15 | . = merge(., ComputerName) 16 | CallWait, err = parse_regex(.Content, r'callWait=(?P\d*)', numeric_groups: false) 17 | . = merge(., ClientID) 18 | First, err = parse_regex(.Content, r'first=(?P[^,]*)', numeric_groups: false) 19 | . = merge(., First) 20 | User, err = parse_regex(.Content, r'Usr=(?P[^,]*)', numeric_groups: false) 21 | . = merge(., User) 22 | SessionID, err = parse_regex(.Content, r'SessionID=(?P\d*)', numeric_groups: false) 23 | . = merge(., SessionID) 24 | Function, err = parse_regex(.Content, r'Func=(?P[^,]*)', numeric_groups: false) 25 | . = merge(., Function) 26 | Module, err = parse_regex(.Content, r'Module=(?P[^,]*)', numeric_groups: false) 27 | . = merge(., Module) 28 | Method, err = parse_regex(.Content, r'Method=(?P[^,]*)', numeric_groups: false) 29 | . = merge(., Method) 30 | Report, err = parse_regex(.Content, r'Report=(?P[^,]*)', numeric_groups: false) 31 | . = merge(., Report) 32 | IName, err = parse_regex(.Content, r'IName=(?P[^,]*)', numeric_groups: false) 33 | . = merge(., IName) 34 | MName, err = parse_regex(.Content, r'MName=(?P[^,]*)', numeric_groups: false) 35 | . = merge(., MName) 36 | Memory, err = parse_regex(.Content, r'Memory=(?P\d*)', numeric_groups: false) 37 | . = merge(., Memory) 38 | MemoryPeak, err = parse_regex(.Content, r'MemoryPeak=(?P\d*)', numeric_groups: false) 39 | . = merge(., MemoryPeak) 40 | InBytes, err = parse_regex(.Content, r'InBytes=(?P\d*)', numeric_groups: false) 41 | . = merge(., InBytes) 42 | OutBytes, err = parse_regex(.Content, r'OutBytes=(?P\d*)', numeric_groups: false) 43 | . = merge(., OutBytes) 44 | CpuTime, err = parse_regex(.Content, r'CpuTime=(?P\d*)', numeric_groups: false) 45 | . = merge(., CpuTime) 46 | Context, err = parse_regex(.Content, r'Context=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|$|[0-5]{1}\d:[0-5]{1}\d.\d{6})', numeric_groups: false) 47 | . = merge(., Context) -------------------------------------------------------------------------------- /config/transforms/DBMSSQL.vrl: -------------------------------------------------------------------------------- 1 | #Распарсим контент для события DBMSSQL 2 | ProcessName, err = parse_regex(.Content, r'p:processName=(?P[^,]*)', numeric_groups: false) 3 | . = merge(., ProcessName) 4 | ComputerName, err = parse_regex(.Content, r't:computerName=(?P[^,]*)', numeric_groups: false) 5 | . = merge(., ComputerName) 6 | ConnectID, err = parse_regex(.Content, r't:connectID=(?P\d*)', numeric_groups: false) 7 | . = merge(., ConnectID) 8 | User, err = parse_regex(.Content, r'Usr=(?P[^,]*)', numeric_groups: false) 9 | . = merge(., User) 10 | dbpid, err = parse_regex(.Content, r'dbpid=(?P\d*)', numeric_groups: false) 11 | . = merge(., dbpid) 12 | Sql, err = parse_regex(.Content, r'Sql=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|$|[0-5]{1}\d:[0-5]{1}\d.\d{6})', numeric_groups: false) 13 | . = merge(., Sql) 14 | planSQLText, err = parse_regex(.Content, r'planSQLText=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|$|[0-5]{1}\d:[0-5]{1}\d.\d{6})', numeric_groups: false) 15 | . = merge(., planSQLText) 16 | Context, err = parse_regex(.Content, r'Context=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|$|[0-5]{1}\d:[0-5]{1}\d.\d{6})', numeric_groups: false) 17 | . = merge(., Context) 18 | 19 | 20 | .err = err -------------------------------------------------------------------------------- /config/transforms/EXCP.vrl: -------------------------------------------------------------------------------- 1 | #Распарсим контент для события EXCP 2 | #Изначально все должно было парсится за один прогон... но что то регулярно идет не так. Оставлю регулярку, на случай если кто-то отладит ее 3 | #(process=(?P[^,]*))?(,p:processName=(?P[^,]*))?(,p:processName=(?P[^,]*))?(,OSThread=(?P\d*))?(,(C|t\:c)lientID=(?P\d*))?(,t:applicationName=(?P[^,]*))?(,t:computerName=(?P[^,]*))?(,t:connectID=(?P\d*))?(,SessionID=(?P\d*))?(,Usr=(?P[^,]*))?(,DBMS=(?P[^,]*))?(,DataBase=(?P[^,]*))?(,dbpid=(?P\d*))?(,Exception=(?P[^,]*))?,Descr=(\\"|\')(?P(.|\n|\r)*)(\\"|\') 4 | Process, err = parse_regex(.Content, r'process=(?P[^,]*)', numeric_groups: false) 5 | . = merge(., Process) 6 | ProcessName, err = parse_regex(.Content, r'p:processName=(?P[^,]*)', numeric_groups: false) 7 | . = merge(., ProcessName) 8 | OSThread, err = parse_regex(.Content, r'OSThread=(?P\d*)', numeric_groups: false) 9 | . = merge(., OSThread) 10 | ClientID, err = parse_regex(.Content, r'(C|t:c)lientID=(?P\d*)', numeric_groups: false) 11 | . = merge(., ClientID) 12 | ApplicationName, err = parse_regex(.Content, r't:applicationName=(?P[^,]*)', numeric_groups: false) 13 | . = merge(., ApplicationName) 14 | ComputerName, err = parse_regex(.Content, r't:computerName=(?P[^,]*)', numeric_groups: false) 15 | . = merge(., ComputerName) 16 | ConnectID, err = parse_regex(.Content, r't:connectID=(?P\d*)', numeric_groups: false) 17 | . = merge(., ConnectID) 18 | SessionID, err = parse_regex(.Content, r'SessionID=(?P\d*)', numeric_groups: false) 19 | . = merge(., SessionID) 20 | User, err = parse_regex(.Content, r'Usr=(?P[^,]*)', numeric_groups: false) 21 | . = merge(., User) 22 | DBMS, err = parse_regex(.Content, r'DBMS=(?P[^,]*)', numeric_groups: false) 23 | . = merge(., DBMS) 24 | DataBase, err = parse_regex(.Content, r'DataBase=(?P[^,]*)', numeric_groups: false) 25 | . = merge(., DataBase) 26 | dbpid, err = parse_regex(.Content, r'dbpid=(?P\d*)', numeric_groups: false) 27 | . = merge(., dbpid) 28 | Exception, err = parse_regex(.Content, r'Exception=(?P[^,]*)', numeric_groups: false) 29 | . = merge(., Exception) 30 | Description, err = parse_regex(.Content, r'Descr=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|$|[0-5]{1}\d:[0-5]{1}\d.\d{6})', numeric_groups: false) 31 | . = merge(., Description) 32 | Context, err = parse_regex(.Content, r'Context=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|$|[0-5]{1}\d:[0-5]{1}\d.\d{6})', numeric_groups: false) 33 | . = merge(., Context) 34 | 35 | 36 | .err = err -------------------------------------------------------------------------------- /config/transforms/EXCPCNTX.vrl: -------------------------------------------------------------------------------- 1 | #Распарсим контент для события EXCPCNTX 2 | ClientComputerName, err = parse_regex(.Content, r'ClientComputerName=(?P[^,]*)', numeric_groups: false) 3 | . = merge(., ClientComputerName) 4 | ServerComputerName, err = parse_regex(.Content, r'ServerComputerName=(?P[^,]*)', numeric_groups: false) 5 | . = merge(., ServerComputerName) 6 | UserName, err = parse_regex(.Content, r'UserName=(?P[^,]*)', numeric_groups: false) 7 | . = merge(., UserName) 8 | ConnectString, err = parse_regex(.Content, r'ConnectString=(?P[^,]*)', numeric_groups: false) 9 | . = merge(., ConnectString) 10 | SrcName, err = parse_regex(.Content, r'SrcName=(?P[^,]*)', numeric_groups: false) 11 | . = merge(., SrcName) 12 | Process, err = parse_regex(.Content, r'process=(?P[^,]*)', numeric_groups: false) 13 | . = merge(., Process) 14 | ProcessName, err = parse_regex(.Content, r'p:processName=(?P[^,]*)', numeric_groups: false) 15 | . = merge(., ProcessName) 16 | OSThread, err = parse_regex(.Content, r'OSThread=(?P\d*)', numeric_groups: false) 17 | . = merge(., OSThread) 18 | ClientID, err = parse_regex(.Content, r'(C|t:c)lientID=(?P\d*)', numeric_groups: false) 19 | . = merge(., ClientID) 20 | ApplicationName, err = parse_regex(.Content, r't:applicationName=(?P[^,]*)', numeric_groups: false) 21 | . = merge(., ApplicationName) 22 | ComputerName, err = parse_regex(.Content, r't:computerName=(?P[^,]*)', numeric_groups: false) 23 | . = merge(., ComputerName) 24 | ConnectID, err = parse_regex(.Content, r't:connectID=(?P\d*)', numeric_groups: false) 25 | . = merge(., ConnectID) 26 | SessionID, err = parse_regex(.Content, r'SessionID=(?P\d*)', numeric_groups: false) 27 | . = merge(., SessionID) 28 | User, err = parse_regex(.Content, r'Usr=(?P[^,]*)', numeric_groups: false) 29 | . = merge(., User) 30 | DBMS, err = parse_regex(.Content, r'DBMS=(?P[^,]*)', numeric_groups: false) 31 | . = merge(., DBMS) 32 | DataBase, err = parse_regex(.Content, r'DataBase=(?P[^,]*)', numeric_groups: false) 33 | . = merge(., DataBase) 34 | Trans, err = parse_regex(.Content, r'Trans=(?P[^,]*)', numeric_groups: false) 35 | . = merge(., Trans) 36 | Function, err = parse_regex(.Content, r'Func=(?P[^,]*)', numeric_groups: false) 37 | . = merge(., Function) 38 | callWait, err = parse_regex(.Content, r'callWait=(?P[^,]*)', numeric_groups: false) 39 | . = merge(., callWait) 40 | first, err = parse_regex(.Content, r'first=(?P[^,]*)', numeric_groups: false) 41 | . = merge(., first) 42 | Module, err = parse_regex(.Content, r'Module=(?P[^,]*)', numeric_groups: false) 43 | . = merge(., Module) 44 | Method, err = parse_regex(.Content, r'Method=(?P[^,]*)', numeric_groups: false) 45 | . = merge(., Method) 46 | Sdbl, err = parse_regex(.Content, r'Sdbl=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|[0-5]{1}\d:[0-5]{1}\d.\d{6}|$)', numeric_groups: false) 47 | . = merge(., Sdbl) 48 | CallID, err = parse_regex(.Content, r'CallID=(?P[^,]*)', numeric_groups: false) 49 | . = merge(., CallID) 50 | Context, err = parse_regex(.Content, r'Context=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|[0-5]{1}\d:[0-5]{1}\d.\d{6}|$)', numeric_groups: false) 51 | . = merge(., Context) 52 | 53 | 54 | .err = err -------------------------------------------------------------------------------- /config/transforms/LIC.vrl: -------------------------------------------------------------------------------- 1 | #Распарсим контент для события EXCPCNTX 2 | Process, err = parse_regex(.Content, r'process=(?P[^,]*)', numeric_groups: false) 3 | . = merge(., Process) 4 | ProcessName, err = parse_regex(.Content, r'p:processName=(?P[^,]*)', numeric_groups: false) 5 | . = merge(., ProcessName) 6 | OSThread, err = parse_regex(.Content, r'OSThread=(?P\d*)', numeric_groups: false) 7 | . = merge(., OSThread) 8 | ClientID, err = parse_regex(.Content, r'(C|t:c)lientID=(?P\d*)', numeric_groups: false) 9 | . = merge(., ClientID) 10 | ApplicationName, err = parse_regex(.Content, r't:applicationName=(?P[^,]*)', numeric_groups: false) 11 | . = merge(., ApplicationName) 12 | ComputerName, err = parse_regex(.Content, r't:computerName=(?P[^,]*)', numeric_groups: false) 13 | . = merge(., ComputerName) 14 | ConnectID, err = parse_regex(.Content, r't:connectID=(?P\d*)', numeric_groups: false) 15 | . = merge(., ConnectID) 16 | SessionID, err = parse_regex(.Content, r'SessionID=(?P\d*)', numeric_groups: false) 17 | . = merge(., SessionID) 18 | Function, err = parse_regex(.Content, r'Func=(?P[^,]*)', numeric_groups: false) 19 | . = merge(., Function) 20 | res, err = parse_regex(.Content, r'res=(?P[^,]*)', numeric_groups: false) 21 | . = merge(., res) 22 | txt, err = parse_regex(.Content, r'txt=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|$|[0-5]{1}\d:[0-5]{1}\d.\d{6})', numeric_groups: false) 23 | . = merge(., Context) 24 | 25 | 26 | .err = err -------------------------------------------------------------------------------- /config/transforms/QERR.vrl: -------------------------------------------------------------------------------- 1 | #Распарсим контент для события EXCPCNTX 2 | Process, err = parse_regex(.Content, r'process=(?P[^,]*)', numeric_groups: false) 3 | . = merge(., Process) 4 | ProcessName, err = parse_regex(.Content, r'p:processName=(?P[^,]*)', numeric_groups: false) 5 | . = merge(., ProcessName) 6 | OSThread, err = parse_regex(.Content, r'OSThread=(?P\d*)', numeric_groups: false) 7 | . = merge(., OSThread) 8 | ClientID, err = parse_regex(.Content, r'(C|t:c)lientID=(?P\d*)', numeric_groups: false) 9 | . = merge(., ClientID) 10 | ApplicationName, err = parse_regex(.Content, r't:applicationName=(?P[^,]*)', numeric_groups: false) 11 | . = merge(., ApplicationName) 12 | ComputerName, err = parse_regex(.Content, r't:computerName=(?P[^,]*)', numeric_groups: false) 13 | . = merge(., ComputerName) 14 | ConnectID, err = parse_regex(.Content, r't:connectID=(?P\d*)', numeric_groups: false) 15 | . = merge(., ConnectID) 16 | SessionID, err = parse_regex(.Content, r'SessionID=(?P\d*)', numeric_groups: false) 17 | . = merge(., SessionID) 18 | User, err = parse_regex(.Content, r'Usr=(?P[^,]*)', numeric_groups: false) 19 | . = merge(., User) 20 | Description, err = parse_regex(.Content, r'Descr=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|$|[0-5]{1}\d:[0-5]{1}\d.\d{6})', numeric_groups: false) 21 | . = merge(., Description) 22 | Query, err = parse_regex(.Content, r'Query=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|$|[0-5]{1}\d:[0-5]{1}\d.\d{6})', numeric_groups: false) 23 | . = merge(., Query) 24 | 25 | .err = err -------------------------------------------------------------------------------- /config/transforms/SCALL.vrl: -------------------------------------------------------------------------------- 1 | #Распарсим контент для события SCALL 2 | CallID, err = parse_regex(.Content, r'CallID=(?P\d*)', numeric_groups: false) 3 | . = merge(., CallID) 4 | Process, err = parse_regex(.Content, r'process=(?P[^,]*)', numeric_groups: false) 5 | . = merge(., Process) 6 | ProcessName, err = parse_regex(.Content, r'p:processName=(?P[^,]*)', numeric_groups: false) 7 | . = merge(., ProcessName) 8 | OSThread, err = parse_regex(.Content, r'OSThread=(?P\d*)', numeric_groups: false) 9 | . = merge(., OSThread) 10 | ClientID, err = parse_regex(.Content, r'(C|t:c)lientID=(?P\d*)', numeric_groups: false) 11 | . = merge(., ClientID) 12 | ApplicationName, err = parse_regex(.Content, r't:applicationName=(?P[^,]*)', numeric_groups: false) 13 | . = merge(., ApplicationName) 14 | ComputerName, err = parse_regex(.Content, r't:computerName=(?P[^,]*)', numeric_groups: false) 15 | . = merge(., ComputerName) 16 | User, err = parse_regex(.Content, r'Usr=(?P[^,]*)', numeric_groups: false) 17 | . = merge(., User) 18 | SessionID, err = parse_regex(.Content, r'SessionID=(?P\d*)', numeric_groups: false) 19 | . = merge(., SessionID) 20 | Method, err = parse_regex(.Content, r'Method=(?P[^,]*)', numeric_groups: false) 21 | . = merge(., Method) 22 | IName, err = parse_regex(.Content, r'IName=(?P[^,]*)', numeric_groups: false) 23 | . = merge(., IName) 24 | MName, err = parse_regex(.Content, r'MName=(?P[^,]*)', numeric_groups: false) 25 | . = merge(., MName) 26 | Interface, err = parse_regex(.Content, r'Interface=(?P[^,]*)', numeric_groups: false) 27 | . = merge(., Interface) 28 | DstClientID, err = parse_regex(.Content, r'DstClientID=(?P\d*)', numeric_groups: false) 29 | . = merge(., DstClientID) 30 | Context, err = parse_regex(.Content, r'Context=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|$|[0-5]{1}\d:[0-5]{1}\d.\d{6})', numeric_groups: false) 31 | . = merge(., Context) -------------------------------------------------------------------------------- /config/transforms/SDBL.vrl: -------------------------------------------------------------------------------- 1 | #Распарсим контент для события SDBL 2 | ProcessName, err = parse_regex(.Content, r'p:processName=(?P[^,]*)', numeric_groups: false) 3 | . = merge(., ProcessName) 4 | ComputerName, err = parse_regex(.Content, r't:computerName=(?P[^,]*)', numeric_groups: false) 5 | . = merge(., ComputerName) 6 | ConnectID, err = parse_regex(.Content, r't:connectID=(?P\d*)', numeric_groups: false) 7 | . = merge(., ConnectID) 8 | User, err = parse_regex(.Content, r'Usr=(?P[^,]*)', numeric_groups: false) 9 | . = merge(., User) 10 | Context, err = parse_regex(.Content, r'Context=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|$|[0-5]{1}\d:[0-5]{1}\d.\d{6})', numeric_groups: false) 11 | . = merge(., Context) 12 | 13 | 14 | .err = err -------------------------------------------------------------------------------- /config/transforms/TDEADLOCK.vrl: -------------------------------------------------------------------------------- 1 | #Распарсим контент для события TTIMEOUT 2 | Process, err = parse_regex(.Content, r'process=(?P[^,]*)', numeric_groups: false) 3 | . = merge(., Process) 4 | ProcessName, err = parse_regex(.Content, r'p:processName=(?P[^,]*)', numeric_groups: false) 5 | . = merge(., ProcessName) 6 | OSThread, err = parse_regex(.Content, r'OSThread=(?P\d*)', numeric_groups: false) 7 | . = merge(., OSThread) 8 | ClientID, err = parse_regex(.Content, r'(C|t:c)lientID=(?P\d*)', numeric_groups: false) 9 | . = merge(., ClientID) 10 | ApplicationName, err = parse_regex(.Content, r't:applicationName=(?P[^,]*)', numeric_groups: false) 11 | . = merge(., ApplicationName) 12 | ComputerName, err = parse_regex(.Content, r't:computerName=(?P[^,]*)', numeric_groups: false) 13 | . = merge(., ComputerName) 14 | ConnectID, err = parse_regex(.Content, r't:connectID=(?P\d*)', numeric_groups: false) 15 | . = merge(., ConnectID) 16 | SessionID, err = parse_regex(.Content, r'SessionID=(?P\d*)', numeric_groups: false) 17 | . = merge(., SessionID) 18 | User, err = parse_regex(.Content, r'Usr=(?P[^,]*)', numeric_groups: false) 19 | . = merge(., User) 20 | AppID, err = parse_regex(.Content, r'AppID=(?P[^,]*)', numeric_groups: false) 21 | . = merge(., AppID) 22 | DBMS, err = parse_regex(.Content, r'DBMS=(?P[^,]*)', numeric_groups: false) 23 | . = merge(., DBMS) 24 | DataBase, err = parse_regex(.Content, r'DataBase=(?P[^,]*)', numeric_groups: false) 25 | . = merge(., DataBase) 26 | DeadlockConnectionIntersections, err = parse_regex(.Content, r'DeadlockConnectionIntersections=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|$|[0-5]{1}\d:[0-5]{1}\d.\d{6})', numeric_groups: false) 27 | . = merge(., DeadlockConnectionIntersections) 28 | Durationus, err = parse_regex(.Content, r'(D|d)urationus=(?P\d*)', numeric_groups: false) 29 | . = merge(., Durationus) 30 | Context, err = parse_regex(.Content, r'Context=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|$|[0-5]{1}\d:[0-5]{1}\d.\d{6})', numeric_groups: false) 31 | . = merge(., Context) 32 | 33 | 34 | .err = err -------------------------------------------------------------------------------- /config/transforms/TLOCK.vrl: -------------------------------------------------------------------------------- 1 | #Распарсим контент для события TLOCK 2 | Process, err = parse_regex(.Content, r'process=(?P[^,]*)', numeric_groups: false) 3 | . = merge(., Process) 4 | ProcessName, err = parse_regex(.Content, r'p:processName=(?P[^,]*)', numeric_groups: false) 5 | . = merge(., ProcessName) 6 | OSThread, err = parse_regex(.Content, r'OSThread=(?P\d*)', numeric_groups: false) 7 | . = merge(., OSThread) 8 | ClientID, err = parse_regex(.Content, r'(C|t:c)lientID=(?P\d*)', numeric_groups: false) 9 | . = merge(., ClientID) 10 | ApplicationName, err = parse_regex(.Content, r't:applicationName=(?P[^,]*)', numeric_groups: false) 11 | . = merge(., ApplicationName) 12 | ComputerName, err = parse_regex(.Content, r't:computerName=(?P[^,]*)', numeric_groups: false) 13 | . = merge(., ComputerName) 14 | ConnectID, err = parse_regex(.Content, r't:connectID=(?P\d*)', numeric_groups: false) 15 | . = merge(., ConnectID) 16 | SessionID, err = parse_regex(.Content, r'SessionID=(?P\d*)', numeric_groups: false) 17 | . = merge(., SessionID) 18 | User, err = parse_regex(.Content, r'Usr=(?P[^,]*)', numeric_groups: false) 19 | . = merge(., User) 20 | AppID, err = parse_regex(.Content, r'AppID=(?P[^,]*)', numeric_groups: false) 21 | . = merge(., AppID) 22 | DBMS, err = parse_regex(.Content, r'DBMS=(?P[^,]*)', numeric_groups: false) 23 | . = merge(., DBMS) 24 | DataBase, err = parse_regex(.Content, r'DataBase=(?P[^,]*)', numeric_groups: false) 25 | . = merge(., DataBase) 26 | Regions, err = parse_regex(.Content, r'Regions=(?P[^,]*)', numeric_groups: false) 27 | . = merge(., Regions) 28 | Locks, err = parse_regex(.Content, r'Locks=(?P[^,]*)', numeric_groups: false) 29 | . = merge(., Locks) 30 | WaitConnections, err = parse_regex(.Content, r'WaitConnections=(?P\d*)', numeric_groups: false) 31 | . = merge(., WaitConnections) 32 | Escalating, err = parse_regex(.Content, r'(E|e)scalating=(?P[^,]*)', numeric_groups: false) 33 | . = merge(., Escalating) 34 | Durationus, err = parse_regex(.Content, r'(D|d)urationus=(?P\d*)', numeric_groups: false) 35 | . = merge(., Durationus) 36 | Exception, err = parse_regex(.Content, r'Exception=(?P[^,]*)', numeric_groups: false) 37 | . = merge(., Exception) 38 | Description, err = parse_regex(.Content, r'Descr=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|$|[0-5]{1}\d:[0-5]{1}\d.\d{6})', numeric_groups: false) 39 | . = merge(., Description) 40 | Context, err = parse_regex(.Content, r'Context=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|$|[0-5]{1}\d:[0-5]{1}\d.\d{6})', numeric_groups: false) 41 | . = merge(., Context) 42 | 43 | 44 | .err = err -------------------------------------------------------------------------------- /config/transforms/TTIMEOUT.vrl: -------------------------------------------------------------------------------- 1 | #Распарсим контент для события TTIMEOUT 2 | Process, err = parse_regex(.Content, r'process=(?P[^,]*)', numeric_groups: false) 3 | . = merge(., Process) 4 | ProcessName, err = parse_regex(.Content, r'p:processName=(?P[^,]*)', numeric_groups: false) 5 | . = merge(., ProcessName) 6 | OSThread, err = parse_regex(.Content, r'OSThread=(?P\d*)', numeric_groups: false) 7 | . = merge(., OSThread) 8 | ClientID, err = parse_regex(.Content, r'(C|t:c)lientID=(?P\d*)', numeric_groups: false) 9 | . = merge(., ClientID) 10 | ApplicationName, err = parse_regex(.Content, r't:applicationName=(?P[^,]*)', numeric_groups: false) 11 | . = merge(., ApplicationName) 12 | ComputerName, err = parse_regex(.Content, r't:computerName=(?P[^,]*)', numeric_groups: false) 13 | . = merge(., ComputerName) 14 | ConnectID, err = parse_regex(.Content, r't:connectID=(?P\d*)', numeric_groups: false) 15 | . = merge(., ConnectID) 16 | SessionID, err = parse_regex(.Content, r'SessionID=(?P\d*)', numeric_groups: false) 17 | . = merge(., SessionID) 18 | User, err = parse_regex(.Content, r'Usr=(?P[^,]*)', numeric_groups: false) 19 | . = merge(., User) 20 | AppID, err = parse_regex(.Content, r'AppID=(?P[^,]*)', numeric_groups: false) 21 | . = merge(., AppID) 22 | DBMS, err = parse_regex(.Content, r'DBMS=(?P[^,]*)', numeric_groups: false) 23 | . = merge(., DBMS) 24 | DataBase, err = parse_regex(.Content, r'DataBase=(?P[^,]*)', numeric_groups: false) 25 | . = merge(., DataBase) 26 | WaitConnections, err = parse_regex(.Content, r'WaitConnections=(?P\d*)', numeric_groups: false) 27 | . = merge(., WaitConnections) 28 | Durationus, err = parse_regex(.Content, r'(D|d)urationus=(?P\d*)', numeric_groups: false) 29 | . = merge(., Durationus) 30 | Context, err = parse_regex(.Content, r'Context=(?P(.|\n|\r)*?)(,[a-zA-Z]+=|$|[0-5]{1}\d:[0-5]{1}\d.\d{6})', numeric_groups: false) 31 | . = merge(., Context) 32 | 33 | 34 | .err = err -------------------------------------------------------------------------------- /config/transforms/parseLog.vrl: -------------------------------------------------------------------------------- 1 | 2 | #Выделим основные свойства для фильтрации 3 | structured, err = parse_regex(.message, r'(?P\d{2}:\d{2}.\d{6})-(?P\d+),(?P[a-zA-Z]+),(?P\d{1}),(?P(.|\n|\r)*)', numeric_groups: false) 4 | . = merge(., structured) 5 | 6 | #Дата события - это имя файла в формате "ГГММДДЧЧ"+Часть из записи"ММ:СС.миллисекунды" 7 | date, DateTimeErr = parse_regex(.file,r'(?P(\d)+).log', numeric_groups: false) 8 | if DateTimeErr==null { 9 | .DateTime, errEventTimestamp = format_timestamp(parse_timestamp!((to_string(date.Date) + to_string(.DateTime)), "%y%m%d%H%M:%S.%6f"), "%F %T%.6f") 10 | } 11 | # Получим PID процесса, и зафиксируем его. 12 | PID, DateTimeErr = parse_regex(.file,r'/(.*)_(?P\d+)/\d*.log', numeric_groups: false) 13 | . = merge(., PID) 14 | 15 | .err = err 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /config/vector.toml: -------------------------------------------------------------------------------- 1 | # Чтение из файлов. 2 | [sources.input_logs] 3 | type = "file" 4 | include = ['''/var/log/logtj/**/*.log'''] 5 | data_dir = '''/var/lib/vector''' 6 | fingerprint.strategy = "device_and_inode" 7 | multiline.timeout_ms = 1000 8 | multiline.mode = "halt_before" 9 | multiline.start_pattern = '' 10 | multiline.condition_pattern = '[0-5]{1}\d:[0-5]{1}\d.\d{6}' 11 | 12 | # Базовая трансформация, выделение вида события, длительности и тд. 13 | [transforms.remap_logs] 14 | inputs = ["input_logs"] 15 | type = "remap" 16 | file = "/etc/vector/transforms/parseLog.vrl" 17 | 18 | # Если мы получили событие Context Добавим его к основному событию. 19 | [transforms.add_context_event] 20 | inputs = ["remap_logs"] 21 | type = "reduce" 22 | starts_when = { type = "vrl", source = '''.Event != "Context"''' } 23 | 24 | [transforms.add_context_event.merge_strategies] 25 | DateTime = "discard" 26 | duration = "discard" 27 | Event = "discard" 28 | Level = "discard" 29 | ProcessID = "discard" 30 | Content = "concat_newline" 31 | message = "concat_newline" 32 | file = "discard" 33 | 34 | 35 | # Отфильтруем только требуемые события 36 | [transforms.filter_EVENT_LIST] 37 | inputs = ["add_context_event"] 38 | type = "filter" 39 | condition = '''contains(${EVENT_LIST}, to_string!(.Event))''' 40 | 41 | #################################### СЕКЦИЯ СОБЫТИЙ ########################################################## 42 | # Каждое событие ТЖ обрабатывается отдельно # 43 | # Для обработки необходимо создать фильтр по событию и ремап со ссылкой на vrl файл c описанием транформации # 44 | ############################################################################################################## 45 | 46 | # Событие EXCP 47 | [transforms.filter_EXCP] 48 | inputs = ["filter_EVENT_LIST"] 49 | type = "filter" 50 | condition = '.Event == "EXCP"' 51 | 52 | [transforms.EXCP] 53 | inputs = ["filter_EXCP"] 54 | type = "remap" 55 | file = "/etc/vector/transforms/EXCP.vrl" 56 | 57 | # Событие EXCPCNTX 58 | [transforms.filter_EXCPCNTX] 59 | inputs = ["filter_EVENT_LIST"] 60 | type = "filter" 61 | condition = '.Event == "EXCPCNTX"' 62 | 63 | [transforms.EXCPCNTX] 64 | inputs = ["filter_EXCPCNTX"] 65 | type = "remap" 66 | file = "/etc/vector/transforms/EXCPCNTX.vrl" 67 | 68 | # Событие DBMSSQL 69 | [transforms.filter_DBMSSQL] 70 | inputs = ["filter_EVENT_LIST"] 71 | type = "filter" 72 | condition = '.Event == "DBMSSQL"' 73 | 74 | [transforms.DBMSSQL] 75 | inputs = ["filter_DBMSSQL"] 76 | type = "remap" 77 | file = "/etc/vector/transforms/DBMSSQL.vrl" 78 | 79 | # Событие SDBL 80 | [transforms.filter_SDBL] 81 | inputs = ["filter_EVENT_LIST"] 82 | type = "filter" 83 | condition = '.Event == "SDBL"' 84 | 85 | [transforms.SDBL] 86 | inputs = ["filter_SDBL"] 87 | type = "remap" 88 | file = "/etc/vector/transforms/SDBL.vrl" 89 | 90 | # Событие TLOCK 91 | [transforms.filter_TLOCK] 92 | inputs = ["filter_EVENT_LIST"] 93 | type = "filter" 94 | condition = '.Event == "TLOCK"' 95 | 96 | [transforms.TLOCK] 97 | inputs = ["filter_TLOCK"] 98 | type = "remap" 99 | file = "/etc/vector/transforms/TLOCK.vrl" 100 | 101 | # Событие TTIMEOUT 102 | [transforms.filter_TTIMEOUT] 103 | inputs = ["filter_EVENT_LIST"] 104 | type = "filter" 105 | condition = '.Event == "TTIMEOUT"' 106 | 107 | [transforms.TTIMEOUT] 108 | inputs = ["filter_TTIMEOUT"] 109 | type = "remap" 110 | file = "/etc/vector/transforms/TTIMEOUT.vrl" 111 | 112 | # Событие TDEADLOCK 113 | [transforms.filter_TDEADLOCK] 114 | inputs = ["filter_EVENT_LIST"] 115 | type = "filter" 116 | condition = '.Event == "TDEADLOCK"' 117 | 118 | [transforms.TDEADLOCK] 119 | inputs = ["filter_TDEADLOCK"] 120 | type = "remap" 121 | file = "/etc/vector/transforms/TDEADLOCK.vrl" 122 | 123 | # Событие QERR 124 | [transforms.filter_QERR] 125 | inputs = ["filter_EVENT_LIST"] 126 | type = "filter" 127 | condition = '.Event == "QERR"' 128 | 129 | [transforms.QERR] 130 | inputs = ["filter_QERR"] 131 | type = "remap" 132 | file = "/etc/vector/transforms/QERR.vrl" 133 | 134 | # Событие CALL 135 | [transforms.filter_CALL] 136 | inputs = ["filter_EVENT_LIST"] 137 | type = "filter" 138 | condition = '.Event == "CALL"' 139 | 140 | [transforms.CALL] 141 | inputs = ["filter_CALL"] 142 | type = "remap" 143 | file = "/etc/vector/transforms/CALL.vrl" 144 | 145 | # Событие SCALL 146 | [transforms.filter_SCALL] 147 | inputs = ["filter_EVENT_LIST"] 148 | type = "filter" 149 | condition = '.Event == "SCALL"' 150 | 151 | [transforms.SCALL] 152 | inputs = ["filter_SCALL"] 153 | type = "remap" 154 | file = "/etc/vector/transforms/SCALL.vrl" 155 | 156 | ######################## СЕКЦИЯ ВЫВОДА ДАННЫХ ################################### 157 | # В данной секции описываются все используемые методы вывода обработанных логов # 158 | ################################################################################# 159 | 160 | ##################################### 161 | # Вывод результата в БД clickhouse # 162 | ##################################### 163 | 164 | # Событие EXCP 165 | [sinks.cl_excp] 166 | type = "clickhouse" 167 | inputs = ["EXCP"] 168 | endpoint = '''${CH_SERVER}''' 169 | auth.strategy = "basic" 170 | auth.user = '''${CH_USER}''' 171 | auth.password = '''${CH_PASSWORD}''' 172 | database = '''${CH_DATABASE}''' 173 | table = '''excp''' 174 | skip_unknown_fields = true 175 | batch.max_events = 10000 176 | batch.timeout_secs = 5 177 | acknowledgements.enabled = true 178 | 179 | # Событие EXCPCNTX 180 | [sinks.cl_EXCPCNTX] 181 | type = "clickhouse" 182 | inputs = ["EXCPCNTX"] 183 | endpoint = '''${CH_SERVER}''' 184 | auth.strategy = "basic" 185 | auth.user = '''${CH_USER}''' 186 | auth.password = '''${CH_PASSWORD}''' 187 | database = '''${CH_DATABASE}''' 188 | table = '''excpcntx''' 189 | skip_unknown_fields = true 190 | batch.max_events = 10000 191 | batch.timeout_secs = 5 192 | acknowledgements.enabled = true 193 | 194 | # Событие DBMSSQL 195 | [sinks.cl_DBMSSQL] 196 | type = "clickhouse" 197 | inputs = ["DBMSSQL"] 198 | endpoint = '''${CH_SERVER}''' 199 | auth.strategy = "basic" 200 | auth.user = '''${CH_USER}''' 201 | auth.password = '''${CH_PASSWORD}''' 202 | database = '''${CH_DATABASE}''' 203 | table = '''dbmssql''' 204 | skip_unknown_fields = true 205 | batch.max_events = 10000 206 | batch.timeout_secs = 5 207 | acknowledgements.enabled = true 208 | 209 | # Событие SDBL 210 | [sinks.cl_SDBL] 211 | type = "clickhouse" 212 | inputs = ["SDBL"] 213 | endpoint = '''${CH_SERVER}''' 214 | auth.strategy = "basic" 215 | auth.user = '''${CH_USER}''' 216 | auth.password = '''${CH_PASSWORD}''' 217 | database = '''${CH_DATABASE}''' 218 | table = '''sdbl''' 219 | skip_unknown_fields = true 220 | batch.max_events = 10000 221 | batch.timeout_secs = 5 222 | acknowledgements.enabled = true 223 | 224 | # Событие TLOCK 225 | [sinks.cl_TLOCK] 226 | type = "clickhouse" 227 | inputs = ["TLOCK"] 228 | endpoint = '''${CH_SERVER}''' 229 | auth.strategy = "basic" 230 | auth.user = '''${CH_USER}''' 231 | auth.password = '''${CH_PASSWORD}''' 232 | database = '''${CH_DATABASE}''' 233 | table = '''tlock''' 234 | skip_unknown_fields = true 235 | batch.max_events = 10000 236 | batch.timeout_secs = 5 237 | acknowledgements.enabled = true 238 | 239 | # Событие TTIMEOUT 240 | [sinks.cl_TTIMEOUT] 241 | type = "clickhouse" 242 | inputs = ["TTIMEOUT"] 243 | endpoint = '''${CH_SERVER}''' 244 | auth.strategy = "basic" 245 | auth.user = '''${CH_USER}''' 246 | auth.password = '''${CH_PASSWORD}''' 247 | database = '''${CH_DATABASE}''' 248 | table = '''ttimeout''' 249 | skip_unknown_fields = true 250 | batch.max_events = 10000 251 | batch.timeout_secs = 5 252 | acknowledgements.enabled = true 253 | 254 | # Событие TDEADLOCK 255 | [sinks.cl_TDEADLOCK] 256 | type = "clickhouse" 257 | inputs = ["TDEADLOCK"] 258 | endpoint = '''${CH_SERVER}''' 259 | auth.strategy = "basic" 260 | auth.user = '''${CH_USER}''' 261 | auth.password = '''${CH_PASSWORD}''' 262 | database = '''${CH_DATABASE}''' 263 | table = '''tdeadlock''' 264 | skip_unknown_fields = true 265 | batch.max_events = 10000 266 | batch.timeout_secs = 5 267 | acknowledgements.enabled = true 268 | 269 | # Событие QERR 270 | [sinks.cl_QERR] 271 | type = "clickhouse" 272 | inputs = ["QERR"] 273 | endpoint = '''${CH_SERVER}''' 274 | auth.strategy = "basic" 275 | auth.user = '''${CH_USER}''' 276 | auth.password = '''${CH_PASSWORD}''' 277 | database = '''${CH_DATABASE}''' 278 | table = '''qerr''' 279 | skip_unknown_fields = true 280 | batch.max_events = 10000 281 | batch.timeout_secs = 5 282 | acknowledgements.enabled = true 283 | 284 | # Событие CALL 285 | [sinks.cl_CALL] 286 | type = "clickhouse" 287 | inputs = ["CALL"] 288 | endpoint = '''${CH_SERVER}''' 289 | auth.strategy = "basic" 290 | auth.user = '''${CH_USER}''' 291 | auth.password = '''${CH_PASSWORD}''' 292 | database = '''${CH_DATABASE}''' 293 | table = '''call''' 294 | skip_unknown_fields = true 295 | batch.max_events = 10000 296 | batch.timeout_secs = 5 297 | acknowledgements.enabled = true 298 | 299 | # Событие SCALL 300 | [sinks.cl_CALL] 301 | type = "clickhouse" 302 | inputs = ["SCALL"] 303 | endpoint = '''${CH_SERVER}''' 304 | auth.strategy = "basic" 305 | auth.user = '''${CH_USER}''' 306 | auth.password = '''${CH_PASSWORD}''' 307 | database = '''${CH_DATABASE}''' 308 | table = '''scall''' 309 | skip_unknown_fields = true 310 | batch.max_events = 10000 311 | batch.timeout_secs = 5 312 | acknowledgements.enabled = true 313 | 314 | 315 | ################################################## 316 | # Пример вывода результата в файл, в формате JSON# 317 | ################################################## 318 | 319 | # [sinks.EXCP_json_out] 320 | # type = "file" 321 | # inputs = [ "EXCP" ] 322 | # compression = "none" 323 | # path = "/tmp/EXCP/%Y-%m-%d %H:%M.json" 324 | 325 | 326 | # [sinks.EXCP_json_out.encoding] 327 | # codec = "json" 328 | 329 | # [sinks.EXCPCNTX_json_out] 330 | # type = "file" 331 | # inputs = [ "EXCPCNTX" ] 332 | # compression = "none" 333 | # path = "/tmp/EXCPCNTX/%Y-%m-%d %H:%M.json" 334 | 335 | 336 | # [sinks.EXCPCNTX_json_out.encoding] 337 | # codec = "json" 338 | 339 | # [sinks.DBMSSQL_json_out] 340 | # type = "file" 341 | # inputs = [ "DBMSSQL" ] 342 | # compression = "none" 343 | # path = "/tmp/DBMSSQL/%Y-%m-%d %H:%M.json" 344 | 345 | 346 | # [sinks.DBMSSQL_json_out.encoding] 347 | # codec = "json" 348 | 349 | # [sinks.SDBL_json_out] 350 | # type = "file" 351 | # inputs = [ "SDBL" ] 352 | # compression = "none" 353 | # path = "/tmp/SDBL/%Y-%m-%d %H:%M.json" 354 | 355 | 356 | # [sinks.SDBL_json_out.encoding] 357 | # codec = "json" 358 | 359 | # [sinks.QERR_json_out] 360 | # type = "file" 361 | # inputs = [ "QERR" ] 362 | # compression = "none" 363 | # path = "/tmp/QERR/%Y-%m-%d %H:%M.json" 364 | 365 | 366 | # [sinks.QERR_json_out.encoding] 367 | # codec = "json" -------------------------------------------------------------------------------- /docker-compose.yaml.example: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | vector: 5 | image: timberio/vector:latest-alpine 6 | env_file: 7 | - ./techJournal.env 8 | hostname: vector_techJournal 9 | container_name: vector_techJournal 10 | restart: always 11 | volumes: 12 | - ./config:/etc/vector/ 13 | - ./fileProgress:/var/lib/vector 14 | - ./logs:/var/log/logtj 15 | # - tjLogs:/var/log/logtj 16 | - ./tmp:/tmp 17 | ports: 18 | - 8686:8686 19 | 20 | # volumes: 21 | # tjLogs: 22 | # driver: local 23 | # driver_opts: 24 | # type: cifs 25 | # device: //{server}/tjlogs 26 | # o: "username={User},password={Password},domain=mfg,vers=3.0,addr={server},file_mode=0777,dir_mode=0777" -------------------------------------------------------------------------------- /sql_scripts/create_call.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE .`call`(`DateTime` DateTime64(6,'Europe/Moscow'), 2 | `duration` Int32, 3 | `Level` Int32, 4 | `Process` String, 5 | `ProcessID` Int32, 6 | `ProcessName` String, 7 | `OSThread` Int32, 8 | `ClientID` Int32, 9 | `ApplicationName` String, 10 | `ComputerName` String, 11 | `CallWait` Int32, 12 | `First` String, 13 | `SessionID` Int32, 14 | `User` String, 15 | `Function` String, 16 | `Module` String, 17 | `Method` String, 18 | `Report` String, 19 | `IName` String, 20 | `MName` String, 21 | `Interface` String, 22 | `DstClientID` Int32, 23 | `CallID` Int32, 24 | `Memory` Int32, 25 | `MemoryPeak` Int32, 26 | `InBytes` Int32, 27 | `OutBytes` Int32, 28 | `CpuTime` Int32, 29 | `Context` String, 30 | `message` String, 31 | `file` String) ENGINE = MergeTree() PARTITION BY toYYYYMM(DateTime) 32 | ORDER BY (DateTime); -------------------------------------------------------------------------------- /sql_scripts/create_dbmssql.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE .dbmssql(`DateTime` DateTime64(6,'Europe/Moscow'), 2 | `duration` Int32, 3 | `ProcessID` Int32, 4 | `Level` Int32, 5 | `ProcessName` String, 6 | `ConnectID` Int32, 7 | `User` String, 8 | `dbpid` Int32, 9 | -- `Sql` String, здоровенные куски текста 10 | -- `planSQLText` String, здоровенные куски текста 11 | `Context` String, 12 | `message` String, 13 | `file` String) ENGINE = MergeTree() PARTITION BY toYYYYMM(DateTime) 14 | ORDER BY (DateTime); -------------------------------------------------------------------------------- /sql_scripts/create_excp.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE .excp(`DateTime` DateTime64(6,'Europe/Moscow'), 2 | `duration` Int32, 3 | `Level` Int32, 4 | `Process` String, 5 | `ProcessID` Int32, 6 | `ProcessName` String, 7 | `OSThread` Int32, 8 | `ClientID` Int32, 9 | `ApplicationName` String, 10 | `ComputerName` String, 11 | `ConnectID` Int32, 12 | `SessionID` Int32, 13 | `User` String, 14 | `DBMS` String, 15 | `DataBase` String, 16 | `dbpid` Int32, 17 | `Exception` String, 18 | `Description` String, 19 | `Context` String, 20 | `message` String, 21 | `file` String) ENGINE = MergeTree() PARTITION BY toYYYYMM(DateTime) 22 | ORDER BY (DateTime); -------------------------------------------------------------------------------- /sql_scripts/create_excpcntx.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE .excpcntx(`DateTime` DateTime64(6,'Europe/Moscow'), 2 | `duration` Int32, 3 | `Level` Int32, 4 | `ClientComputerName` String, 5 | `ServerComputerName` String, 6 | `UserName` String, 7 | `ConnectString` String, 8 | `SrcName` String, 9 | `Process` String, 10 | `ProcessID` Int32, 11 | `ProcessName` String, 12 | `OSThread` Int32, 13 | `ClientID` Int32, 14 | `ApplicationName` String, 15 | `ComputerName` String, 16 | `ConnectID` Int32, 17 | `SessionID` Int32, 18 | `User` String, 19 | `DBMS` String, 20 | `DataBase` String, 21 | `Trans` String, 22 | `Function` String, 23 | `callWait` String, 24 | `first` String, 25 | `Module` String, 26 | `Method` String, 27 | `Sdbl` String, 28 | `CallID` Int32, 29 | `Context` String, 30 | `message` String, 31 | `file` String) ENGINE = MergeTree() PARTITION BY toYYYYMM(DateTime) 32 | ORDER BY (DateTime); -------------------------------------------------------------------------------- /sql_scripts/create_qerr.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE .qerr(`DateTime` DateTime64(6,'Europe/Moscow'), 2 | `duration` Int32, 3 | `Level` Int32, 4 | `Process` String, 5 | `ProcessID` Int32, 6 | `ProcessName` String, 7 | `OSThread` Int32, 8 | `ClientID` Int32, 9 | `ApplicationName` String, 10 | `ComputerName` String, 11 | `ConnectID` Int32, 12 | `SessionID` Int32, 13 | `User` String, 14 | `Query` String, 15 | `Description` String, 16 | `message` String, 17 | `file` String) ENGINE = MergeTree() PARTITION BY toYYYYMM(DateTime) 18 | ORDER BY (DateTime); -------------------------------------------------------------------------------- /sql_scripts/create_scall.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE .scall(`DateTime` DateTime64(6,'Europe/Moscow'), 2 | `duration` Int32, 3 | `Level` Int32, 4 | `Process` String, 5 | `ProcessID` Int32, 6 | `ProcessName` String, 7 | `OSThread` Int32, 8 | `ClientID` Int32, 9 | `ApplicationName` String, 10 | `ComputerName` String, 11 | `SessionID` Int32, 12 | `User` String, 13 | `Method` String, 14 | `IName` String, 15 | `MName` String, 16 | `Interface` String, 17 | `DstClientID` Int32, 18 | `CallID` Int32, 19 | `Context` String, 20 | `message` String, 21 | `file` String) ENGINE = MergeTree() PARTITION BY toYYYYMM(DateTime) 22 | ORDER BY (DateTime); -------------------------------------------------------------------------------- /sql_scripts/create_sdbl.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE .sdbl(`DateTime` DateTime64(6,'Europe/Moscow'), 2 | `duration` Int32, 3 | `ProcessID` Int32, 4 | `Level` Int32, 5 | `ProcessName` String, 6 | `ComputerName` String, 7 | `ConnectID` Int32, 8 | `User` String, 9 | `Context` String, 10 | `message` String, 11 | `file` String) ENGINE = MergeTree() PARTITION BY toYYYYMM(DateTime) 12 | ORDER BY (DateTime); -------------------------------------------------------------------------------- /sql_scripts/create_tdeadlock.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE .tdeadlock(`DateTime` DateTime64(6,'Europe/Moscow'), 2 | `duration` Int32, 3 | `Level` Int32, 4 | `Process` String, 5 | `ProcessID` Int32, 6 | `ProcessName` String, 7 | `OSThread` Int32, 8 | `ClientID` Int32, 9 | `ApplicationName` String, 10 | `ComputerName` String, 11 | `ConnectID` Int32, 12 | `SessionID` Int32, 13 | `User` String, 14 | `AppID` String, 15 | `DBMS` String, 16 | `DataBase` String, 17 | `Durationus` Int32, 18 | `DeadlockConnectionIntersections` String, 19 | `Context` String, 20 | `message` String, 21 | `file` String) ENGINE = MergeTree() PARTITION BY toYYYYMM(DateTime) 22 | ORDER BY (DateTime); -------------------------------------------------------------------------------- /sql_scripts/create_tlock.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE .tlock(`DateTime` DateTime64(6,'Europe/Moscow'), 2 | `duration` Int32, 3 | `Level` Int32, 4 | `Process` String, 5 | `ProcessID` Int32, 6 | `ProcessName` String, 7 | `OSThread` Int32, 8 | `ClientID` Int32, 9 | `ApplicationName` String, 10 | `ComputerName` String, 11 | `ConnectID` Int32, 12 | `SessionID` Int32, 13 | `User` String, 14 | `AppID` String, 15 | `DBMS` String, 16 | `DataBase` String, 17 | `Regions` String, 18 | `Locks` String, 19 | `WaitConnections` Int32, 20 | `Escalating` String, 21 | `Durationus` Int32, 22 | `Exception` String, 23 | `Description` String, 24 | `Context` String, 25 | `message` String, 26 | `file` String) ENGINE = MergeTree() PARTITION BY toYYYYMM(DateTime) 27 | ORDER BY (DateTime); -------------------------------------------------------------------------------- /sql_scripts/create_ttimeout.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE .ttimeout(`DateTime` DateTime64(6,'Europe/Moscow'), 2 | `duration` Int32, 3 | `Level` Int32, 4 | `Process` String, 5 | `ProcessID` Int32, 6 | `ProcessName` String, 7 | `OSThread` Int32, 8 | `ClientID` Int32, 9 | `ApplicationName` String, 10 | `ComputerName` String, 11 | `ConnectID` Int32, 12 | `SessionID` Int32, 13 | `User` String, 14 | `AppID` String, 15 | `DBMS` String, 16 | `DataBase` String, 17 | `WaitConnections` Int32, 18 | `Durationus` Int32, 19 | `Context` String, 20 | `message` String, 21 | `file` String) ENGINE = MergeTree() PARTITION BY toYYYYMM(DateTime) 22 | ORDER BY (DateTime); -------------------------------------------------------------------------------- /techJournal.env.example: -------------------------------------------------------------------------------- 1 | #Установка переменных окружения для ClickHouse 2 | CH_SERVER=192.168.1.53:8123 3 | CH_USER=default 4 | CH_PASSWORD= 5 | CH_DATABASE=tjournal 6 | 7 | # Список событий которые планируется выгружать. 8 | EVENT_LIST="EXCP,EXCPCNTX,TLOCK,TTIMEOUT,SDBL,QERR" 9 | --------------------------------------------------------------------------------