├── IPLom_parser.py
├── README.md
├── anomaly_detection_benchmark.py
├── anomaly_labels.csv
├── log_parsed
    └── HDFS.log_structured.csv
├── loglizer
    ├── __pycache__
    │   ├── dataloader.cpython-37.pyc
    │   ├── preprocessing.cpython-37.pyc
    │   ├── utils.cpython-36.pyc
    │   └── utils.cpython-37.pyc
    ├── dataloader.py
    ├── models
    │   ├── InvariantsMiner.py
    │   ├── LogClustering.py
    │   ├── PCA.py
    │   └── __pycache__
    │   │   ├── InvariantsMiner.cpython-37.pyc
    │   │   ├── LogClustering.cpython-37.pyc
    │   │   ├── PCA.cpython-36.pyc
    │   │   └── PCA.cpython-37.pyc
    ├── preprocessing.py
    └── utils.py
├── logparser
    ├── IPLoM
    │   ├── IPLoM.py
    │   ├── README.md
    │   ├── __init__.py
    │   └── __pycache__
    │   │   ├── IPLoM.cpython-36.pyc
    │   │   ├── IPLoM.cpython-37.pyc
    │   │   ├── __init__.cpython-36.pyc
    │   │   └── __init__.cpython-37.pyc
    ├── __init__.py
    ├── __pycache__
    │   ├── __init__.cpython-36.pyc
    │   └── __init__.cpython-37.pyc
    ├── logmatch
    │   ├── __init__.py
    │   ├── __pycache__
    │   │   ├── __init__.cpython-36.pyc
    │   │   └── regexmatch.cpython-36.pyc
    │   └── regexmatch.py
    └── utils
    │   ├── __init__.py
    │   ├── __pycache__
    │       ├── __init__.cpython-36.pyc
    │       ├── __init__.cpython-37.pyc
    │       ├── evaluator.cpython-36.pyc
    │       ├── evaluator.cpython-37.pyc
    │       ├── logloader.cpython-36.pyc
    │       └── logloader.cpython-37.pyc
    │   ├── evaluator.py
    │   └── logloader.py
└── plot_anomaly_detection_charts.py


/IPLom_parser.py:
--------------------------------------------------------------------------------
 1 | from logparser import IPLoM
 2 | 
 3 | input_dir = 'log/' # The input directory of log file
 4 | output_dir = 'parsed_log/'  # The output directory of parsing results
 5 | log_file = 'HDFS.log'  # The input log file name
 6 | log_format = '<Date> <Time> <Pid> <Level> <Component>: <Content>'  # HDFS log format
 7 | maxEventLen = 120  # The maximal token number of log messages (default: 200)
 8 | step2Support = 0  # The minimal support for creating a new partition (default: 0)
 9 | CT = 0.35  # The cluster goodness threshold (default: 0.35)
10 | lowerBound = 0.25  # The lower bound distance (default: 0.25)
11 | upperBound = 0.9  # The upper bound distance (default: 0.9)
12 | regex = []  # Regular expression list for optional preprocessing (default: [])
13 | 
14 | parser = IPLoM.LogParser(log_format=log_format, indir=input_dir, outdir=output_dir,
15 |                          maxEventLen=maxEventLen, step2Support=step2Support, CT=CT,
16 |                          lowerBound=lowerBound, upperBound=upperBound, rex=regex)
17 | parser.parse(log_file)


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Log-Analysis-for-Anomaly-Detection
 2 | 
 3 | The process of log analysis for anomaly detection involves four main steps: <strong>log collection</strong>, <strong>log parsing</strong>, <strong>feature extraction</strong>, and <strong>anomaly detection</strong>.  
 4 | To run the whole anomaly detection pipeline follow the below steps:  
 5 | - create a "log" folder and put the log file in it.
 6 | - run "IPLom_parser.py" to parse the log file.
 7 | - run "anomaly_detection_benchmark.py" to detect anomalies in the data.
 8 | - run "plot_anomaly_detection_charts.py" to plot the charts relative to the metrics of the detection results.  
 9 | 
10 | Project report:
11 | - https://davideliu.com/2019/10/26/log-analysis-for-anomaly-detection/
12 | 
13 | References:
14 | - https://github.com/logpai
15 | - http://jmzhu.logpai.com/pub/slhe_issre2016.pdf
16 | 
17 | 


--------------------------------------------------------------------------------
/anomaly_detection_benchmark.py:
--------------------------------------------------------------------------------
 1 | import pandas as pd
 2 | from loglizer.models.PCA import PCA
 3 | from loglizer.models.LogClustering import LogClustering
 4 | from loglizer.models.InvariantsMiner import InvariantsMiner
 5 | from loglizer import dataloader, preprocessing
 6 | import timeit
 7 | 
 8 | models = ['PCA', 'InvariantsMiner', 'LogClustering']  # anomaly detection models
 9 | struct_log = 'log_parsed/HDFS.log_structured.csv'  # dataset
10 | labels = 'anomaly_labels.csv'  # labels used for testing
11 | 
12 | if __name__ == '__main__':
13 |     # load the dataset
14 |     (x_tr, y_train), (x_te, y_test) = dataloader.load_HDFS(struct_log,
15 |                                                            label_file=labels,
16 |                                                            window='session',
17 |                                                            train_ratio=0.7,
18 |                                                            split_type='uniform')
19 |     benchmark_results = []
20 |     feature_extractor = preprocessing.FeatureExtractor()
21 | 
22 |     # train each model
23 |     for _model in models:
24 |         print('Evaluating {} on HDFS:'.format(_model))
25 | 
26 |         if _model == 'PCA':
27 |             x_train = feature_extractor.fit_transform(x_tr, term_weighting='tf-idf',
28 |                                                       normalization='zero-mean')
29 |             model = PCA()
30 |             model.fit(x_train)
31 | 
32 |         elif _model == 'InvariantsMiner':
33 |             feature_extractor = preprocessing.FeatureExtractor()
34 |             x_train = feature_extractor.fit_transform(x_tr)
35 |             model = InvariantsMiner(epsilon=0.5)
36 |             model.fit(x_train)
37 | 
38 |         elif _model == 'LogClustering':
39 |             feature_extractor = preprocessing.FeatureExtractor()
40 |             x_train = feature_extractor.fit_transform(x_tr, term_weighting='tf-idf')
41 |             model = LogClustering(max_dist=0.3, anomaly_threshold=0.3)
42 |             model.fit(x_train[y_train == 0, :])  # Use only normal samples for training"""
43 | 
44 |         # extract feature matrix
45 |         x_test = feature_extractor.transform(x_te)
46 | 
47 |         print('Train accuracy:')
48 |         start = timeit.default_timer()
49 |         precision, recall, f1 = model.evaluate(x_train, y_train)
50 |         stop = timeit.default_timer()
51 |         runtime = stop - start
52 |         benchmark_results.append([_model + '-train', precision, recall, f1, runtime])
53 | 
54 |         print('Test accuracy:')
55 |         start = timeit.default_timer()
56 |         precision, recall, f1 = model.evaluate(x_test, y_test)
57 |         stop = timeit.default_timer()
58 |         runtime = stop - start
59 |         benchmark_results.append([_model + '-test', precision, recall, f1, runtime])
60 | 
61 |     # store results as csv file
62 |     pd.DataFrame(benchmark_results, columns=['Model', 'Precision', 'Recall', 'F1', 'Runtime']) \
63 |         .to_csv('anomaly_detection_results/benchmark_results.csv', index=False)
64 | 


--------------------------------------------------------------------------------
/loglizer/__pycache__/dataloader.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/loglizer/__pycache__/dataloader.cpython-37.pyc


--------------------------------------------------------------------------------
/loglizer/__pycache__/preprocessing.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/loglizer/__pycache__/preprocessing.cpython-37.pyc


--------------------------------------------------------------------------------
/loglizer/__pycache__/utils.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/loglizer/__pycache__/utils.cpython-36.pyc


--------------------------------------------------------------------------------
/loglizer/__pycache__/utils.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/loglizer/__pycache__/utils.cpython-37.pyc


--------------------------------------------------------------------------------
/loglizer/dataloader.py:
--------------------------------------------------------------------------------
  1 | """
  2 | The interface to load log datasets. The datasets currently supported include
  3 | HDFS and BGL.
  4 | Authors:
  5 |     LogPAI Team
  6 | """
  7 | 
  8 | import pandas as pd
  9 | import os
 10 | import numpy as np
 11 | import re
 12 | import sys
 13 | from sklearn.utils import shuffle
 14 | from collections import OrderedDict
 15 | 
 16 | def _split_data(x_data, y_data=None, train_ratio=0, split_type='uniform'):
 17 |     if split_type == 'uniform' and y_data is not None:
 18 |         pos_idx = y_data > 0
 19 |         x_pos = x_data[pos_idx]
 20 |         y_pos = y_data[pos_idx]
 21 |         x_neg = x_data[~pos_idx]
 22 |         y_neg = y_data[~pos_idx]
 23 |         train_pos = int(train_ratio * x_pos.shape[0])
 24 |         train_neg = int(train_ratio * x_neg.shape[0])
 25 |         x_train = np.hstack([x_pos[0:train_pos], x_neg[0:train_neg]])
 26 |         y_train = np.hstack([y_pos[0:train_pos], y_neg[0:train_neg]])
 27 |         x_test = np.hstack([x_pos[train_pos:], x_neg[train_neg:]])
 28 |         y_test = np.hstack([y_pos[train_pos:], y_neg[train_neg:]])
 29 |     elif split_type == 'sequential':
 30 |         num_train = int(train_ratio * x_data.shape[0])
 31 |         x_train = x_data[0:num_train]
 32 |         x_test = x_data[num_train:]
 33 |         if y_data is None:
 34 |             y_train = None
 35 |             y_test = None
 36 |         else:
 37 |             y_train = y_data[0:num_train]
 38 |             y_test = y_data[num_train:]
 39 |     # Random shuffle
 40 |     indexes = shuffle(np.arange(x_train.shape[0]))
 41 |     x_train = x_train[indexes]
 42 |     if y_train is not None:
 43 |         y_train = y_train[indexes]
 44 |     return (x_train, y_train), (x_test, y_test)
 45 | 
 46 | def load_HDFS(log_file, label_file=None, window='session', train_ratio=0.5, split_type='sequential',
 47 |     save_csv=False):
 48 |     """ Load HDFS structured log into train and test log_data
 49 |     Arguments
 50 |     ---------
 51 |         log_file: str, the file path of structured log.
 52 |         label_file: str, the file path of anomaly labels, None for unlabeled log_data
 53 |         window: str, the window options including `session` (default).
 54 |         train_ratio: float, the ratio of training log_data for train/test split.
 55 |         split_type: `uniform` or `sequential`, which determines how to split dataset. `uniform` means
 56 |             to split positive samples and negative samples equally when setting label_file. `sequential`
 57 |             means to split the log_data sequentially without label_file. That is, the first part is for training,
 58 |             while the second part is for testing.
 59 |     Returns
 60 |     -------
 61 |         (x_train, y_train): the training log_data
 62 |         (x_test, y_test): the testing log_data
 63 |     """
 64 | 
 65 |     print('====== Input log_data summary ======')
 66 | 
 67 |     if log_file.endswith('.npz'):
 68 |         # Split training and validation set in a class-uniform way
 69 |         data = np.load(log_file)
 70 |         x_data = data['x_data']
 71 |         y_data = data['y_data']
 72 |         (x_train, y_train), (x_test, y_test) = _split_data(x_data, y_data, train_ratio, split_type)
 73 | 
 74 |     elif log_file.endswith('.csv'):
 75 |         assert window == 'session', "Only window=session is supported for HDFS dataset."
 76 |         struct_log = pd.read_csv(log_file, engine='c', na_filter=False, memory_map=True)
 77 |         data_dict = OrderedDict()
 78 |         for idx, row in struct_log.iterrows():
 79 |             blkId_list = re.findall(r'(blk_-?\d+)', row['Content'])
 80 |             blkId_set = set(blkId_list)
 81 |             for blk_Id in blkId_set:
 82 |                 if not blk_Id in data_dict:
 83 |                     data_dict[blk_Id] = []
 84 |                 data_dict[blk_Id].append(row['EventId'])
 85 |         data_df = pd.DataFrame(list(data_dict.items()), columns=['BlockId', 'EventSequence'])
 86 | 
 87 |         if label_file:
 88 |             # Split training and validation set in a class-uniform way
 89 |             label_data = pd.read_csv(label_file, engine='c', na_filter=False, memory_map=True)
 90 |             label_data = label_data.set_index('BlockId')
 91 |             label_dict = label_data['Label'].to_dict()
 92 |             data_df['Label'] = data_df['BlockId'].apply(lambda x: 1 if label_dict[x] == 'Anomaly' else 0)
 93 | 
 94 |             # Split train and test log_data
 95 |             (x_train, y_train), (x_test, y_test) = _split_data(data_df['EventSequence'].values,
 96 |                 data_df['Label'].values, train_ratio, split_type)
 97 | 
 98 |         if save_csv:
 99 |             data_df.to_csv('data_instances.csv', index=False)
100 | 
101 |         if label_file is None:
102 |             if split_type == 'uniform':
103 |                 split_type = 'sequential'
104 |                 print('Warning: Only split_type=sequential is supported \
105 |                 if label_file=None.'.format(split_type))
106 |             # Split training and validation set sequentially
107 |             x_data = data_df['EventSequence'].values
108 |             (x_train, _), (x_test, _) = _split_data(x_data, train_ratio=train_ratio, split_type=split_type)
109 |             print('Total: {} instances, train: {} instances, test: {} instances'.format(
110 |                   x_data.shape[0], x_train.shape[0], x_test.shape[0]))
111 |             return (x_train, None), (x_test, None)
112 |     else:
113 |         raise NotImplementedError('load_HDFS() only support csv and npz files!')
114 | 
115 |     num_train = x_train.shape[0]
116 |     num_test = x_test.shape[0]
117 |     num_total = num_train + num_test
118 |     num_train_pos = sum(y_train)
119 |     num_test_pos = sum(y_test)
120 |     num_pos = num_train_pos + num_test_pos
121 | 
122 |     print('Total: {} instances, {} anomaly, {} normal' \
123 |           .format(num_total, num_pos, num_total - num_pos))
124 |     print('Train: {} instances, {} anomaly, {} normal' \
125 |           .format(num_train, num_train_pos, num_train - num_train_pos))
126 |     print('Test: {} instances, {} anomaly, {} normal\n' \
127 |           .format(num_test, num_test_pos, num_test - num_test_pos))
128 | 
129 |     return (x_train, y_train), (x_test, y_test)
130 | 
131 | 
132 | def load_BGL(log_file, label_file=None, window='sliding', time_interval=60, stepping_size=60,
133 |              train_ratio=0.8):
134 |     """  TODO
135 |     """
136 | 
137 | 
138 | def bgl_preprocess_data(para, raw_data, event_mapping_data):
139 |     """ split logs into sliding windows, built an event count matrix and get the corresponding label
140 |     Args:
141 |     --------
142 |     para: the parameters dictionary
143 |     raw_data: list of (label, time)
144 |     event_mapping_data: a list of event index, where each row index indicates a corresponding log
145 |     Returns:
146 |     --------
147 |     event_count_matrix: event count matrix, where each row is an instance (log sequence vector)
148 |     labels: a list of labels, 1 represents anomaly
149 |     """
150 | 
151 |     # create the directory for saving the sliding windows (start_index, end_index), which can be directly loaded in future running
152 |     if not os.path.exists(para['save_path']):
153 |         os.mkdir(para['save_path'])
154 |     log_size = raw_data.shape[0]
155 |     sliding_file_path = para['save_path']+'sliding_'+str(para['window_size'])+'h_'+str(para['step_size'])+'h.csv'
156 | 
157 |     #=============divide into sliding windows=========#
158 |     start_end_index_list = [] # list of tuples, tuple contains two number, which represent the start and end of sliding time window
159 |     label_data, time_data = raw_data[:,0], raw_data[:, 1]
160 |     if not os.path.exists(sliding_file_path):
161 |         # split into sliding window
162 |         start_time = time_data[0]
163 |         start_index = 0
164 |         end_index = 0
165 | 
166 |         # get the first start, end index, end time
167 |         for cur_time in time_data:
168 |             if  cur_time < start_time + para['window_size']*3600:
169 |                 end_index += 1
170 |                 end_time = cur_time
171 |             else:
172 |                 start_end_pair=tuple((start_index,end_index))
173 |                 start_end_index_list.append(start_end_pair)
174 |                 break
175 |         # move the start and end index until next sliding window
176 |         while end_index < log_size:
177 |             start_time = start_time + para['step_size']*3600
178 |             end_time = end_time + para['step_size']*3600
179 |             for i in range(start_index,end_index):
180 |                 if time_data[i] < start_time:
181 |                     i+=1
182 |                 else:
183 |                     break
184 |             for j in range(end_index, log_size):
185 |                 if time_data[j] < end_time:
186 |                     j+=1
187 |                 else:
188 |                     break
189 |             start_index = i
190 |             end_index = j
191 |             start_end_pair = tuple((start_index, end_index))
192 |             start_end_index_list.append(start_end_pair)
193 |         inst_number = len(start_end_index_list)
194 |         print('there are %d instances (sliding windows) in this dataset\n'%inst_number)
195 |         np.savetxt(sliding_file_path,start_end_index_list,delimiter=',',fmt='%d')
196 |     else:
197 |         print('Loading start_end_index_list from file')
198 |         start_end_index_list = pd.read_csv(sliding_file_path, header=None).values
199 |         inst_number = len(start_end_index_list)
200 |         print('there are %d instances (sliding windows) in this dataset' % inst_number)
201 | 
202 |     # get all the log indexes in each time window by ranging from start_index to end_index
203 |     expanded_indexes_list=[]
204 |     for t in range(inst_number):
205 |         index_list = []
206 |         expanded_indexes_list.append(index_list)
207 |     for i in range(inst_number):
208 |         start_index = start_end_index_list[i][0]
209 |         end_index = start_end_index_list[i][1]
210 |         for l in range(start_index, end_index):
211 |             expanded_indexes_list[i].append(l)
212 | 
213 |     event_mapping_data = [row[0] for row in event_mapping_data]
214 |     event_num = len(list(set(event_mapping_data)))
215 |     print('There are %d log events'%event_num)
216 | 
217 |     #=============get labels and event count of each sliding window =========#
218 |     labels = []
219 |     event_count_matrix = np.zeros((inst_number,event_num))
220 |     for j in range(inst_number):
221 |         label = 0   #0 represent success, 1 represent failure
222 |         for k in expanded_indexes_list[j]:
223 |             event_index = event_mapping_data[k]
224 |             event_count_matrix[j, event_index] += 1
225 |             if label_data[k]:
226 |                 label = 1
227 |                 continue
228 |         labels.append(label)
229 |     assert inst_number == len(labels)
230 |     print("Among all instances, %d are anomalies"%sum(labels))
231 |     assert event_count_matrix.shape[0] == len(labels)
232 |     return event_count_matrix, labels


--------------------------------------------------------------------------------
/loglizer/models/InvariantsMiner.py:
--------------------------------------------------------------------------------
  1 | """
  2 | The implementation of Invariants Mining model for anomaly detection.
  3 | Authors:
  4 |     LogPAI Team
  5 | Reference:
  6 |     [1] Jian-Guang Lou, Qiang Fu, Shengqi Yang, Ye Xu, Jiang Li. Mining Invariants
  7 |         from Console Logs for System Problem Detection. USENIX Annual Technical
  8 |         Conference (ATC), 2010.
  9 | """
 10 | 
 11 | import numpy as np
 12 | from itertools import combinations
 13 | from ..utils import metrics
 14 | 
 15 | 
 16 | class InvariantsMiner(object):
 17 | 
 18 |     def __init__(self, percentage=0.98, epsilon=0.5, longest_invarant=None, scale_list=[1, 2, 3]):
 19 |         """ The Invariants Mining model for anomaly detection
 20 |         Attributes
 21 |         ----------
 22 |             percentage: float, percentage of samples satisfying the condition that |X_j * V_i| < epsilon
 23 |             epsilon: float, the threshold for estimating the invariant space
 24 |             longest_invarant: int, the specified maximal length of invariant, default to None. Stop
 25 |                 searching when the invariant length is larger than longest_invarant.
 26 |             scale_list: list, the list used to scale the theta of float into integer
 27 |             invariants_dict: dict, dictionary of invariants where key is the selected columns
 28 |                 and value is the weights the of invariant
 29 |         """
 30 |         self.percentage = percentage
 31 |         self.epsilon = epsilon
 32 |         self.longest_invarant = longest_invarant
 33 |         self.scale_list = scale_list
 34 |         self.invariants_dict = None
 35 | 
 36 |     def fit(self, X):
 37 |         """
 38 |         Arguments
 39 |         ---------
 40 |             X: ndarray, the event count matrix of shape num_instances-by-num_events
 41 |         """
 42 |         print('====== Model summary ======')
 43 |         invar_dim = self._estimate_invarant_space(X)
 44 |         self._invariants_search(X, invar_dim)
 45 | 
 46 |     def predict(self, X):
 47 |         """ Predict anomalies with mined invariants
 48 |         Arguments
 49 |         ---------
 50 |             X: the input event count matrix
 51 |         Returns
 52 |         -------
 53 |             y_pred: ndarray, the predicted label vector of shape (num_instances,)
 54 |         """
 55 | 
 56 |         y_sum = np.zeros(X.shape[0])
 57 |         for cols, theta in self.invariants_dict.items():
 58 |             y_sum += np.fabs(np.dot(X[:, cols], np.array(theta)))
 59 |         y_pred = (y_sum > 1e-6).astype(int)
 60 |         return y_pred
 61 | 
 62 |     def evaluate(self, X, y_true):
 63 |         print('====== Evaluation summary ======')
 64 |         y_pred = self.predict(X)
 65 |         precision, recall, f1 = metrics(y_pred, y_true)
 66 |         print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n'.format(precision, recall, f1))
 67 |         return precision, recall, f1
 68 | 
 69 |     def _estimate_invarant_space(self, X):
 70 |         """ Estimate the dimension of invariant space using SVD decomposition
 71 |         Arguments
 72 |         ---------
 73 |             X: ndarray, the event count matrix of shape num_instances-by-num_events
 74 |             percentage: float, percentage of samples satisfying the condition that |X_j * V_i| < epsilon
 75 |             epsilon: float, the threshold for estimating the invariant space
 76 |         Returns
 77 |         -------
 78 |             r: the dimension of invariant space
 79 |         """
 80 |         covariance_matrix = np.dot(X.T, X)
 81 |         U, sigma, V = np.linalg.svd(covariance_matrix)  # SVD decomposition
 82 |         # Start from the right most column of matrix V, sigular values are in ascending order
 83 |         num_instances, num_events = X.shape
 84 |         r = 0
 85 |         for i in range(num_events - 1, -1, -1):
 86 |             zero_count = sum(abs(np.dot(X, U[:, i])) < self.epsilon)
 87 |             if zero_count / float(num_instances) < self.percentage:
 88 |                 break
 89 |             r += 1
 90 |         print('Invariant space dimension: {}'.format(r))
 91 | 
 92 |         return r
 93 | 
 94 |     def _invariants_search(self, X, r):
 95 |         """ Mine invariant relationships from X
 96 |         Arguments
 97 |         ---------
 98 |             X: ndarray, the event count matrix of shape num_instances-by-num_events
 99 |             r: the dimension of invariant space
100 |         """
101 | 
102 |         num_instances, num_events = X.shape
103 |         invariants_dict = dict()  # save the mined Invariants(value) and its corresponding columns(key)
104 |         search_space = []  # only invariant candidates in this list are valid
105 | 
106 |         # invariant of only one column (all zero columns)
107 |         init_cols = sorted([[item] for item in range(num_events)])
108 |         for col in init_cols:
109 |             search_space.append(col)
110 |         init_col_list = init_cols[:]
111 |         for col in init_cols:
112 |             if np.count_nonzero(X[:, col]) == 0:
113 |                 invariants_dict[tuple(col)] = [1]
114 |                 search_space.remove(col)
115 |                 init_col_list.remove(col)
116 | 
117 |         item_list = init_col_list
118 |         length = 2
119 |         FLAG_break_loop = False
120 |         # check invariant of more columns
121 |         while len(item_list) != 0:
122 |             if self.longest_invarant and len(item_list[0]) >= self.longest_invarant:
123 |                 break
124 |             joined_item_list = self._join_set(item_list, length)  # generate new invariant candidates
125 |             for items in joined_item_list:
126 |                 if self._check_candi_valid(items, length, search_space):
127 |                     search_space.append(items)
128 |             item_list = []
129 |             for item in joined_item_list:
130 |                 if tuple(item) in invariants_dict:
131 |                     continue
132 |                 if item not in search_space:
133 |                     continue
134 |                 if not self._check_candi_valid(tuple(item), length, search_space) and length > 2:
135 |                     search_space.remove(item)
136 |                     continue  # an item must be superset of all other subitems in searchSpace, else skip
137 |                 validity, scaled_theta = self._check_invar_validity(X, item)
138 |                 if validity:
139 |                     self._prune(invariants_dict.keys(), set(item), search_space)
140 |                     invariants_dict[tuple(item)] = scaled_theta.tolist()
141 |                     search_space.remove(item)
142 |                 else:
143 |                     item_list.append(item)
144 |                 if len(invariants_dict) >= r:
145 |                     FLAG_break_loop = True
146 |                     break
147 |             if FLAG_break_loop:
148 |                 break
149 |             length += 1
150 |         print('Mined {} invariants: {}\n'.format(len(invariants_dict), invariants_dict))
151 |         self.invariants_dict = invariants_dict
152 | 
153 |     def _compute_eigenvector(self, X):
154 |         """ calculate the smallest eigenvalue and corresponding eigenvector (theta in the paper)
155 |             for a given sub_matrix
156 |         Arguments
157 |         ---------
158 |             X: the event count matrix (each row is a log sequence vector, each column represents an event)
159 |         Returns
160 |         -------
161 |             min_vec: the eigenvector of corresponding minimum eigen value
162 |             FLAG_contain_zero: whether the min_vec contains zero (very small value)
163 |         """
164 | 
165 |         FLAG_contain_zero = False
166 |         count_zero = 0
167 |         dot_result = np.dot(X.T, X)
168 |         U, S, V = np.linalg.svd(dot_result)
169 |         min_vec = U[:, -1]
170 |         count_zero = sum(np.fabs(min_vec) < 1e-6)
171 |         if count_zero != 0:
172 |             FLAG_contain_zero = True
173 |         return min_vec, FLAG_contain_zero
174 | 
175 |     def _check_invar_validity(self, X, selected_columns):
176 |         """ scale the eigenvector of float number into integer, and check whether the scaled number is valid
177 |         Arguments
178 |         ---------
179 |             X: the event count matrix (each row is a log sequence vector, each column represents an event)
180 |             selected_columns: select columns from all column list
181 |         Returns
182 |         -------
183 |             validity: whether the selected columns is valid
184 |             scaled_theta: the scaled theta vector
185 |         """
186 | 
187 |         sub_matrix = X[:, selected_columns]
188 |         inst_num = X.shape[0]
189 |         validity = False
190 |         min_theta, FLAG_contain_zero = self._compute_eigenvector(sub_matrix)
191 |         abs_min_theta = [np.fabs(it) for it in min_theta]
192 |         if FLAG_contain_zero:
193 |             return validity, []
194 |         else:
195 |             for i in self.scale_list:
196 |                 min_index = np.argmin(abs_min_theta)
197 |                 scale = float(i) / min_theta[min_index]
198 |                 scaled_theta = np.array([round(item * scale) for item in min_theta])
199 |                 scaled_theta[min_index] = i
200 |                 scaled_theta = scaled_theta.T
201 |                 if 0 in np.fabs(scaled_theta):
202 |                     continue
203 |                 dot_submat_theta = np.dot(sub_matrix, scaled_theta)
204 |                 count_zero = 0
205 |                 for j in dot_submat_theta:
206 |                     if np.fabs(j) < 1e-8:
207 |                         count_zero += 1
208 |                 if count_zero >= self.percentage * inst_num:
209 |                     validity = True
210 |                     # print('A valid invariant is found: ',scaled_theta, selected_columns)
211 |                     break
212 |             return validity, scaled_theta
213 | 
214 |     def _prune(self, valid_cols, new_item_set, search_space):
215 |         """ prune invalid combination of columns
216 |         Arguments
217 |         ---------
218 |             valid_cols: existing valid column list
219 |             new_item_set: item set to be merged
220 |             search_space: the search space that stores possible candidates
221 |         """
222 | 
223 |         if len(valid_cols) == 0:
224 |             return
225 |         for se in valid_cols:
226 |             intersection = set(se) & new_item_set
227 |             if len(intersection) == 0:
228 |                 continue
229 |             union = set(se) | new_item_set
230 |             for item in list(intersection):
231 |                 diff = sorted(list(union - set([item])))
232 |                 if diff in search_space:
233 |                     search_space.remove(diff)
234 | 
235 |     def _join_set(self, item_list, length):
236 |         """ Join a set with itself and returns the n-element (length) itemsets
237 |         Arguments
238 |         ---------
239 |             item_list: current list of columns
240 |             length: generate new items of length
241 |         Returns
242 |         -------
243 |             return_list: list of items of length-element
244 |         """
245 | 
246 |         set_len = len(item_list)
247 |         return_list = []
248 |         for i in range(set_len):
249 |             for j in range(i + 1, set_len):
250 |                 i_set = set(item_list[i])
251 |                 j_set = set(item_list[j])
252 |                 if len(i_set.union(j_set)) == length:
253 |                     joined = sorted(list(i_set.union(j_set)))
254 |                     if joined not in return_list:
255 |                         return_list.append(joined)
256 |         return_list = sorted(return_list)
257 |         return return_list
258 | 
259 |     def _check_candi_valid(self, item, length, search_space):
260 |         """ check whether an item's subitems are in searchspace
261 |         Arguments
262 |         ---------
263 |             item: item to be checked
264 |             length: the length of item
265 |             search_space: the search space that stores possible candidates
266 |         Returns
267 |         -------
268 |             True or False
269 |         """
270 | 
271 |         for subItem in combinations(item, length - 1):
272 |             if sorted(list(subItem)) not in search_space:
273 |                 return False
274 |         return True


--------------------------------------------------------------------------------
/loglizer/models/LogClustering.py:
--------------------------------------------------------------------------------
  1 | """
  2 | The implementation of Log Clustering model for anomaly detection.
  3 | Authors:
  4 |     LogPAI Team
  5 | Reference:
  6 |     [1] Qingwei Lin, Hongyu Zhang, Jian-Guang Lou, Yu Zhang, Xuewei Chen. Log Clustering
  7 |         based Problem Identification for Online Service Systems. International Conference
  8 |         on Software Engineering (ICSE), 2016.
  9 | """
 10 | 
 11 | import numpy as np
 12 | import pprint
 13 | from scipy.special import expit
 14 | from numpy import linalg as LA
 15 | from scipy.cluster.hierarchy import linkage, fcluster
 16 | from scipy.spatial.distance import pdist, squareform
 17 | from ..utils import metrics
 18 | 
 19 | 
 20 | class LogClustering(object):
 21 | 
 22 |     def __init__(self, max_dist=0.3, anomaly_threshold=0.3, mode='online', num_bootstrap_samples=1000):
 23 |         """
 24 |         Attributes
 25 |         ----------
 26 |             max_dist: float, the threshold to stop the clustering process
 27 |             anomaly_threshold: float, the threshold for anomaly detection
 28 |             mode: str, 'offline' or 'online' mode for clustering
 29 |             num_bootstrap_samples: int, online clustering starts with a bootstraping process, which
 30 |                 determines the initial cluster representatives offline using a subset of samples
 31 |             representatives: ndarray, the representative samples of clusters, of shape
 32 |                 num_clusters-by-num_events
 33 |             cluster_size_dict: dict, the size of each cluster, used to update representatives online
 34 |         """
 35 |         self.max_dist = max_dist
 36 |         self.anomaly_threshold = anomaly_threshold
 37 |         self.mode = mode
 38 |         self.num_bootstrap_samples = num_bootstrap_samples
 39 |         self.representatives = list()
 40 |         self.cluster_size_dict = dict()
 41 | 
 42 |     def fit(self, X):
 43 |         print('====== Model summary ======')
 44 |         if self.mode == 'offline':
 45 |             # The offline mode can process about 10K samples only due to huge memory consumption.
 46 |             self._offline_clustering(X)
 47 |         elif self.mode == 'online':
 48 |             # Bootstrapping phase
 49 |             if self.num_bootstrap_samples > 0:
 50 |                 X_bootstrap = X[0:self.num_bootstrap_samples, :]
 51 |                 self._offline_clustering(X_bootstrap)
 52 |             # Online learning phase
 53 |             if X.shape[0] > self.num_bootstrap_samples:
 54 |                 self._online_clustering(X)
 55 | 
 56 |     def predict(self, X):
 57 |         y_pred = np.zeros(X.shape[0])
 58 |         for i in range(X.shape[0]):
 59 |             min_dist, min_index = self._get_min_cluster_dist(X[i, :])
 60 |             if min_dist > self.anomaly_threshold:
 61 |                 y_pred[i] = 1
 62 |         return y_pred
 63 | 
 64 |     def evaluate(self, X, y_true):
 65 |         print('====== Evaluation summary ======')
 66 |         y_pred = self.predict(X)
 67 |         precision, recall, f1 = metrics(y_pred, y_true)
 68 |         print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n' \
 69 |               .format(precision, recall, f1))
 70 |         return precision, recall, f1
 71 | 
 72 |     def _offline_clustering(self, X):
 73 |         print('Starting offline clustering...')
 74 |         p_dist = pdist(X, metric=self._distance_metric)
 75 |         Z = linkage(p_dist, 'complete')
 76 |         cluster_index = fcluster(Z, self.max_dist, criterion='distance')
 77 |         self._extract_representatives(X, cluster_index)
 78 |         print('Processed {} instances.'.format(X.shape[0]))
 79 |         print('Found {} clusters offline.\n'.format(len(self.representatives)))
 80 |         # print('The representive vectors are:')
 81 |         # pprint.pprint(self.representatives.tolist())
 82 | 
 83 |     def _extract_representatives(self, X, cluster_index):
 84 |         num_clusters = len(set(cluster_index))
 85 |         for clu in range(num_clusters):
 86 |             clu_idx = np.argwhere(cluster_index == clu + 1)[:, 0]
 87 |             self.cluster_size_dict[clu] = clu_idx.shape[0]
 88 |             repre_center = np.average(X[clu_idx, :], axis=0)
 89 |             self.representatives.append(repre_center)
 90 | 
 91 |     def _online_clustering(self, X):
 92 |         print("Starting online clustering...")
 93 |         for i in range(self.num_bootstrap_samples, X.shape[0]):
 94 |             if (i + 1) % 2000 == 0:
 95 |                 print('Processed {} instances.'.format(i + 1))
 96 |             instance_vec = X[i, :]
 97 |             if len(self.representatives) > 0:
 98 |                 min_dist, clu_id = self._get_min_cluster_dist(instance_vec)
 99 |                 if min_dist <= self.max_dist:
100 |                     self.cluster_size_dict[clu_id] += 1
101 |                     self.representatives[clu_id] = self.representatives[clu_id] \
102 |                                                  + (instance_vec - self.representatives[clu_id]) \
103 |                                                  / self.cluster_size_dict[clu_id]
104 |                     continue
105 |             self.cluster_size_dict[len(self.representatives)] = 1
106 |             self.representatives.append(instance_vec)
107 |         print('Processed {} instances.'.format(X.shape[0]))
108 |         print('Found {} clusters online.\n'.format(len(self.representatives)))
109 |         # print('The representive vectors are:')
110 |         # pprint.pprint(self.representatives.tolist())
111 | 
112 |     def _distance_metric(self, x1, x2):
113 |         norm= LA.norm(x1) * LA.norm(x2)
114 |         distance = 1 - np.dot(x1, x2) / (norm + 1e-8)
115 |         if distance < 1e-8:
116 |             distance = 0
117 |         return distance
118 | 
119 |     def _get_min_cluster_dist(self, instance_vec):
120 |         min_index = -1
121 |         min_dist = float('inf')
122 |         for i in range(len(self.representatives)):
123 |             cluster_rep = self.representatives[i]
124 |             dist = self._distance_metric(instance_vec, cluster_rep)
125 |             if dist < 1e-8:
126 |                 min_dist = 0
127 |                 min_index = i
128 |                 break
129 |             elif dist < min_dist:
130 |                 min_dist = dist
131 |                 min_index = i
132 |         return min_dist, min_index


--------------------------------------------------------------------------------
/loglizer/models/PCA.py:
--------------------------------------------------------------------------------
  1 | """
  2 | The implementation of PCA model for anomaly detection.
  3 | Authors:
  4 |     LogPAI Team
  5 | Reference:
  6 |     [1] Wei Xu, Ling Huang, Armando Fox, David Patterson, Michael I. Jordan.
  7 |         Large-Scale System Problems Detection by Mining Console Logs. ACM
  8 |         Symposium on Operating Systems Principles (SOSP), 2009.
  9 | """
 10 | 
 11 | import numpy as np
 12 | from ..utils import metrics
 13 | 
 14 | class PCA(object):
 15 | 
 16 |     def __init__(self, n_components=0.95, threshold=None, c_alpha=3.2905):
 17 |         """ The PCA model for anomaly detection
 18 |         Attributes
 19 |         ----------
 20 |             proj_C: The projection matrix for projecting feature vector to abnormal space
 21 |             n_components: float/int, number of principal compnents or the variance ratio they cover
 22 |             threshold: float, the anomaly detection threshold. When setting to None, the threshold
 23 |                 is automatically caculated using Q-statistics
 24 |             c_alpha: float, the c_alpha parameter for caculating anomaly detection threshold using
 25 |                 Q-statistics. The following is lookup table for c_alpha:
 26 |                 c_alpha = 1.7507; # alpha = 0.08
 27 |                 c_alpha = 1.9600; # alpha = 0.05
 28 |                 c_alpha = 2.5758; # alpha = 0.01
 29 |                 c_alpha = 2.807; # alpha = 0.005
 30 |                 c_alpha = 2.9677;  # alpha = 0.003
 31 |                 c_alpha = 3.2905;  # alpha = 0.001
 32 |                 c_alpha = 3.4808;  # alpha = 0.0005
 33 |                 c_alpha = 3.8906;  # alpha = 0.0001
 34 |                 c_alpha = 4.4172;  # alpha = 0.00001
 35 |         """
 36 | 
 37 |         self.proj_C = None
 38 |         self.components = None
 39 |         self.n_components = n_components
 40 |         self.threshold = threshold
 41 |         self.c_alpha = c_alpha
 42 | 
 43 | 
 44 |     def fit(self, X):
 45 |         """
 46 |         Auguments
 47 |         ---------
 48 |             X: ndarray, the event count matrix of shape num_instances-by-num_events
 49 |         """
 50 | 
 51 |         print('====== Model summary ======')
 52 |         num_instances, num_events = X.shape
 53 |         X_cov = np.dot(X.T, X) / float(num_instances)
 54 |         U, sigma, V = np.linalg.svd(X_cov)
 55 |         n_components = self.n_components
 56 |         if n_components < 1:
 57 |             total_variance = np.sum(sigma)
 58 |             variance = 0
 59 |             for i in range(num_events):
 60 |                 variance += sigma[i]
 61 |                 if variance / total_variance >= n_components:
 62 |                     break
 63 |             n_components = i + 1
 64 | 
 65 |         P = U[:, :n_components]
 66 |         I = np.identity(num_events, int)
 67 |         self.components = P
 68 |         self.proj_C = I - np.dot(P, P.T)
 69 |         print('n_components: {}'.format(n_components))
 70 |         print('Project matrix shape: {}-by-{}'.format(self.proj_C.shape[0], self.proj_C.shape[1]))
 71 | 
 72 |         if not self.threshold:
 73 |             # Calculate threshold using Q-statistic. Information can be found at:
 74 |             # http://conferences.sigcomm.org/sigcomm/2004/papers/p405-lakhina111.pdf
 75 |             phi = np.zeros(3)
 76 |             for i in range(3):
 77 |                 for j in range(n_components, num_events):
 78 |                     phi[i] += np.power(sigma[j], i + 1)
 79 |             h0 = 1.0 - 2 * phi[0] * phi[2] / (3.0 * phi[1] * phi[1])
 80 |             self.threshold = phi[0] * np.power(self.c_alpha * np.sqrt(2 * phi[1] * h0 * h0) / phi[0]
 81 |                                                + 1.0 + phi[1] * h0 * (h0 - 1) / (phi[0] * phi[0]),
 82 |                                                1.0 / h0)
 83 |         print('SPE threshold: {}\n'.format(self.threshold))
 84 | 
 85 |     def predict(self, X):
 86 |         assert self.proj_C is not None, 'PCA model needs to be trained before prediction.'
 87 |         y_pred = np.zeros(X.shape[0])
 88 |         for i in range(X.shape[0]):
 89 |             y_a = np.dot(self.proj_C, X[i, :])
 90 |             SPE = np.dot(y_a, y_a)
 91 |             if SPE > self.threshold:
 92 |                 y_pred[i] = 1
 93 |         return y_pred
 94 | 
 95 |     def evaluate(self, X, y_true):
 96 |         print('====== Evaluation summary ======')
 97 |         y_pred = self.predict(X)
 98 |         precision, recall, f1 = metrics(y_pred, y_true)
 99 |         print('Precision: {:.3f}, recall: {:.3f}, F1-measure: {:.3f}\n'.format(precision, recall, f1))
100 |         return precision, recall, f1


--------------------------------------------------------------------------------
/loglizer/models/__pycache__/InvariantsMiner.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/loglizer/models/__pycache__/InvariantsMiner.cpython-37.pyc


--------------------------------------------------------------------------------
/loglizer/models/__pycache__/LogClustering.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/loglizer/models/__pycache__/LogClustering.cpython-37.pyc


--------------------------------------------------------------------------------
/loglizer/models/__pycache__/PCA.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/loglizer/models/__pycache__/PCA.cpython-36.pyc


--------------------------------------------------------------------------------
/loglizer/models/__pycache__/PCA.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/loglizer/models/__pycache__/PCA.cpython-37.pyc


--------------------------------------------------------------------------------
/loglizer/preprocessing.py:
--------------------------------------------------------------------------------
  1 | """
  2 | The interface for log_data preprocessing.
  3 | Authors:
  4 |     LogPAI Team
  5 | """
  6 | 
  7 | import pandas as pd
  8 | import os
  9 | import numpy as np
 10 | import re
 11 | import sys
 12 | from collections import Counter
 13 | from scipy.special import expit
 14 | from itertools import compress
 15 | 
 16 | 
 17 | class FeatureExtractor(object):
 18 | 
 19 |     def __init__(self):
 20 |         self.idf_vec = None
 21 |         self.mean_vec = None
 22 |         self.events = None
 23 |         self.term_weighting = None
 24 |         self.normalization = None
 25 |         self.oov = None
 26 | 
 27 |     def fit_transform(self, X_seq, term_weighting=None, normalization=None, oov=False, min_count=1):
 28 |         """ Fit and transform the log_data matrix
 29 |         Arguments
 30 |         ---------
 31 |             X_seq: ndarray, log sequences matrix
 32 |             term_weighting: None or `tf-idf`
 33 |             normalization: None or `zero-mean`
 34 |             oov: bool, whether to use OOV event
 35 |             min_count: int, the minimal occurrence of events (default 0), only valid when oov=True.
 36 |         Returns
 37 |         -------
 38 |             X_new: The transformed log_data matrix
 39 |         """
 40 |         print('====== Transformed train log_data summary ======')
 41 |         self.term_weighting = term_weighting
 42 |         self.normalization = normalization
 43 |         self.oov = oov
 44 | 
 45 |         X_counts = []
 46 |         for i in range(X_seq.shape[0]):
 47 |             event_counts = Counter(X_seq[i])
 48 |             X_counts.append(event_counts)
 49 |         X_df = pd.DataFrame(X_counts)
 50 |         X_df = X_df.fillna(0)
 51 |         self.events = X_df.columns
 52 |         X = X_df.values
 53 |         if self.oov:
 54 |             oov_vec = np.zeros(X.shape[0])
 55 |             if min_count > 1:
 56 |                 idx = np.sum(X > 0, axis=0) >= min_count
 57 |                 oov_vec = np.sum(X[:, ~idx] > 0, axis=1)
 58 |                 X = X[:, idx]
 59 |                 self.events = np.array(X_df.columns)[idx].tolist()
 60 |             X = np.hstack([X, oov_vec.reshape(X.shape[0], 1)])
 61 | 
 62 |         num_instance, num_event = X.shape
 63 |         if self.term_weighting == 'tf-idf':
 64 |             df_vec = np.sum(X > 0, axis=0)
 65 |             self.idf_vec = np.log(num_instance / (df_vec + 1e-8))
 66 |             idf_matrix = X * np.tile(self.idf_vec, (num_instance, 1))
 67 |             X = idf_matrix
 68 |         if self.normalization == 'zero-mean':
 69 |             mean_vec = X.mean(axis=0)
 70 |             self.mean_vec = mean_vec.reshape(1, num_event)
 71 |             X = X - np.tile(self.mean_vec, (num_instance, 1))
 72 |         elif self.normalization == 'sigmoid':
 73 |             X[X != 0] = expit(X[X != 0])
 74 |         X_new = X
 75 | 
 76 |         print('Train log_data shape: {}-by-{}\n'.format(X_new.shape[0], X_new.shape[1]))
 77 |         return X_new
 78 | 
 79 |     def transform(self, X_seq):
 80 |         """ Transform the log_data matrix with trained parameters
 81 |         Arguments
 82 |         ---------
 83 |             X: log sequences matrix
 84 |             term_weighting: None or `tf-idf`
 85 |         Returns
 86 |         -------
 87 |             X_new: The transformed log_data matrix
 88 |         """
 89 |         print('====== Transformed test log_data summary ======')
 90 |         X_counts = []
 91 |         for i in range(X_seq.shape[0]):
 92 |             event_counts = Counter(X_seq[i])
 93 |             X_counts.append(event_counts)
 94 |         X_df = pd.DataFrame(X_counts)
 95 |         X_df = X_df.fillna(0)
 96 |         empty_events = set(self.events) - set(X_df.columns)
 97 |         for event in empty_events:
 98 |             X_df[event] = [0] * len(X_df)
 99 |         X = X_df[self.events].values
100 |         if self.oov:
101 |             oov_vec = np.sum(X_df[X_df.columns.difference(self.events)].values > 0, axis=1)
102 |             X = np.hstack([X, oov_vec.reshape(X.shape[0], 1)])
103 | 
104 |         num_instance, num_event = X.shape
105 |         if self.term_weighting == 'tf-idf':
106 |             idf_matrix = X * np.tile(self.idf_vec, (num_instance, 1))
107 |             X = idf_matrix
108 |         if self.normalization == 'zero-mean':
109 |             X = X - np.tile(self.mean_vec, (num_instance, 1))
110 |         elif self.normalization == 'sigmoid':
111 |             X[X != 0] = expit(X[X != 0])
112 |         X_new = X
113 | 
114 |         print('Test log_data shape: {}-by-{}\n'.format(X_new.shape[0], X_new.shape[1]))
115 | 
116 |         return X_new


--------------------------------------------------------------------------------
/loglizer/utils.py:
--------------------------------------------------------------------------------
 1 | """
 2 | The utility functions of loglizer
 3 | Authors:
 4 |     LogPAI Team
 5 | """
 6 | 
 7 | from sklearn.metrics import precision_recall_fscore_support
 8 | import numpy as np
 9 | 
10 | 
11 | def metrics(y_pred, y_true):
12 |     """ Calucate evaluation metrics for precision, recall, and f1.
13 |     Arguments
14 |     ---------
15 |         y_pred: ndarry, the predicted result list
16 |         y_true: ndarray, the ground truth label list
17 |     Returns
18 |     -------
19 |         precision: float, precision value
20 |         recall: float, recall value
21 |         f1: float, f1 measure value
22 |     """
23 |     precision, recall, f1, _ = precision_recall_fscore_support(y_true, y_pred, average='binary')
24 |     return precision, recall, f1


--------------------------------------------------------------------------------
/logparser/IPLoM/IPLoM.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Description : This file implements the IPLoM algorithm for log parsing
  3 | Author      : LogPAI team
  4 | License     : MIT
  5 | """
  6 | 
  7 | import copy
  8 | import sys
  9 | from datetime import datetime
 10 | import os
 11 | import gc
 12 | import re
 13 | import pandas as pd
 14 | import hashlib
 15 | import string
 16 | 
 17 | class Partition:
 18 |     """ Wrap around the logs and the step number
 19 |     """
 20 |     def __init__(self, stepNo, numOfLogs=0, lenOfLogs=0):
 21 |         self.logLL = []
 22 |         self.stepNo = stepNo
 23 |         self.valid = True
 24 |         self.numOfLogs = numOfLogs
 25 |         self.lenOfLogs = lenOfLogs
 26 | 
 27 | 
 28 | class Event:
 29 |     def __init__(self, eventStr):
 30 |         self.eventStr = eventStr
 31 |         self.eventId = hashlib.md5(' '.join(eventStr).encode('utf-8')).hexdigest()[0:8]
 32 |         self.eventCount = 0
 33 | 
 34 | 
 35 | class Para:
 36 |     """ Para class
 37 | 
 38 |     Attributes
 39 |     ----------
 40 |         maxEventLen : the length of the longest log/event, which is used in step 1 to split logs into partitions 
 41 |             according to their length
 42 |         path : the path of the input file
 43 |         step2Support : the support threshold to create a new partition, partitions which contains less than 
 44 |             step2Support logs will not go through step 2
 45 |         PST : Partition support ratio threshold
 46 |         CT : Cluster goodness threshold used in DetermineP1P2 in step3. If the columns with unique term more 
 47 |             than CT, we skip step 3
 48 |     """
 49 | 
 50 |     def __init__(self, log_format, indir, outdir, maxEventLen, step2Support, PST, CT, lowerBound,
 51 |                  upperBound, rex):
 52 |         self.maxEventLen = maxEventLen
 53 |         self.path = indir
 54 |         self.savePath = outdir
 55 |         self.step2Support = step2Support
 56 |         self.PST = PST
 57 |         self.CT = CT
 58 |         self.lowerBound = lowerBound
 59 |         self.upperBound = upperBound
 60 |         self.rex = rex
 61 |         self.logformat = log_format
 62 | 
 63 | class LogParser:
 64 |     def __init__(self, log_format, indir='../logs/', outdir='./result/',
 65 |                  maxEventLen=200, step2Support=0, PST=0,
 66 |                  CT=0.35, lowerBound=0.25, upperBound=0.9,
 67 |                  rex=[], keep_para=True):
 68 | 
 69 |         self.para = Para(log_format=log_format, indir=indir, outdir=outdir, maxEventLen=maxEventLen, step2Support=step2Support,
 70 |                          PST=PST, CT=CT, lowerBound=lowerBound, upperBound=upperBound, rex=rex)
 71 |         self.partitionsL = []
 72 |         self.eventsL = []
 73 |         self.output = []
 74 |         self.keep_para = keep_para
 75 | 
 76 |         if not os.path.exists(self.para.savePath):
 77 |             os.makedirs(self.para.savePath)
 78 | 
 79 |         # Initialize some partitions which contain logs with different length
 80 |         for logLen in range(self.para.maxEventLen + 1):
 81 |             self.partitionsL.append(Partition(stepNo=1, numOfLogs=0, lenOfLogs=logLen))
 82 | 
 83 |     def parse(self, logname):
 84 |         print('Parsing file: ' + os.path.join(self.para.path, logname))
 85 |         self.logname = logname
 86 |         starttime = datetime.now()
 87 |         self.Step1()
 88 |         self.Step2()
 89 |         self.Step3()
 90 |         self.Step4()
 91 |         self.getOutput()
 92 |         self.WriteEventToFile()
 93 |         print('Parsing done. [Time taken: {!s}]'.format(datetime.now() - starttime))
 94 |         return datetime.now() - starttime
 95 | 
 96 |     def Step1(self):
 97 |         headers, regex = self.generate_logformat_regex(self.para.logformat)
 98 |         self.df_log = self.log_to_dataframe(os.path.join(self.para.path, self.logname), regex, headers, self.para.logformat)
 99 |         lineCount = 1
100 |         for idx, line in self.df_log.iterrows():
101 |             line = line['Content']
102 |             # If line is empty, skip
103 |             if line.strip() == "":
104 |                 continue
105 | 
106 |             if self.para.rex:
107 |                 for currentRex in self.para.rex:
108 |                     line = re.sub(currentRex, '', line)
109 | 
110 |             wordSeq = list(filter(lambda x: x != '', re.split(r'[\s=:,]', line)))
111 |             if not wordSeq:
112 |                 wordSeq = [' ']
113 | 
114 |             # Generate terms list, with ID in the end
115 |             wordSeq.append(str(lineCount))
116 |             lineCount += 1
117 | 
118 |             # Add current log to the corresponding partition
119 |             self.partitionsL[len(wordSeq) - 1].logLL.append(wordSeq)
120 |             self.partitionsL[len(wordSeq) - 1].numOfLogs += 1
121 | 
122 |         for partition in self.partitionsL:
123 |             if partition.numOfLogs == 0:
124 |                 partition.valid = False
125 | 
126 |             elif self.para.PST != 0 and 1.0 * partition.numOfLogs / lineCount < self.para.PST:
127 |                 for logL in partition.logLL:
128 |                     self.partitionsL[0].logLL.append(logL)
129 |                     self.partitionsL[0].numOfLogs += 1
130 |                 partition.valid = False
131 | 
132 |     def Step2(self):
133 | 
134 |         for partition in self.partitionsL:
135 | 
136 |             if not partition.valid:
137 |                 continue
138 | 
139 |             if partition.numOfLogs <= self.para.step2Support:
140 |                 continue
141 | 
142 |             # Avoid going through newly generated partitions
143 |             if partition.stepNo == 2:
144 |                 break
145 | 
146 |             # For each column, create a set to hold the unique tokens in that column. 
147 |             # And finally, calculate the number of the unique tokens in each column
148 |             uniqueTokensCountLS = []
149 |             for columnIdx in range(partition.lenOfLogs):
150 |                 uniqueTokensCountLS.append(set())
151 | 
152 |             for logL in partition.logLL:
153 |                 for columnIdx in range(partition.lenOfLogs):
154 |                     uniqueTokensCountLS[columnIdx].add(logL[columnIdx])
155 | 
156 |             # Find the column with minimum unique tokens
157 |             minColumnIdx = 0
158 |             minColumnCount = len(uniqueTokensCountLS[0])
159 | 
160 |             for columnIdx in range(partition.lenOfLogs):
161 |                 if minColumnCount > len(uniqueTokensCountLS[columnIdx]):
162 |                     minColumnCount = len(uniqueTokensCountLS[columnIdx])
163 |                     minColumnIdx = columnIdx
164 | 
165 |             # If there is one column with one unique term, do not split this partition
166 |             if minColumnCount == 1:
167 |                 continue
168 | 
169 |             # From split-token to log list
170 |             logDLL = {}
171 |             for logL in partition.logLL:
172 |                 if logL[minColumnIdx] not in logDLL:
173 |                     logDLL[logL[minColumnIdx]] = []
174 |                 logDLL[logL[minColumnIdx]].append(logL)
175 | 
176 |             for key in logDLL:
177 |                 if self.para.PST != 0 and 1.0 * len(logDLL[key]) / partition.numOfLogs < self.para.PST:
178 |                     self.partitionsL[0].logLL += logDLL[key]
179 |                     self.partitionsL[0].numOfLogs += len(logDLL[key])
180 |                 else:
181 |                     newPartition = Partition(stepNo=2, numOfLogs=len(logDLL[key]), lenOfLogs=partition.lenOfLogs)
182 |                     newPartition.logLL = logDLL[key]
183 |                     self.partitionsL.append(newPartition)
184 | 
185 |             partition.valid = False
186 | 
187 |     def Step3(self):
188 | 
189 |         for partition in self.partitionsL:
190 | 
191 |             if not partition.valid:
192 |                 continue
193 | 
194 |             if partition.stepNo == 3:
195 |                 break
196 | 
197 |             # Find two columns that my cause split in this step
198 |             p1, p2 = self.DetermineP1P2(partition)
199 | 
200 |             if p1 == -1 or p2 == -1:
201 |                 continue
202 | 
203 |             try:
204 | 
205 |                 p1Set = set()
206 |                 p2Set = set()
207 |                 mapRelation1DS = {}
208 |                 mapRelation2DS = {}
209 | 
210 |                 # Construct token sets for p1 and p2, dictionary to record the mapping relations between p1 and p2
211 |                 for logL in partition.logLL:
212 |                     p1Set.add(logL[p1])
213 |                     p2Set.add(logL[p2])
214 | 
215 |                     if (logL[p1] == logL[p2]):
216 |                         print ("Warning: p1 may be equal to p2")
217 | 
218 |                     if logL[p1] not in mapRelation1DS:
219 |                         mapRelation1DS[logL[p1]] = set()
220 |                     mapRelation1DS[logL[p1]].add(logL[p2])
221 | 
222 |                     if logL[p2] not in mapRelation2DS:
223 |                         mapRelation2DS[logL[p2]] = set()
224 |                     mapRelation2DS[logL[p2]].add(logL[p1])
225 | 
226 |                 # Construct sets to record the tokens in 1-1, 1-M, M-1 relationships, the left-tokens in p1Set & p2Set 
227 |                 # are in M-M relationships
228 |                 oneToOneS = set()
229 |                 oneToMP1D = {}
230 |                 oneToMP2D = {}
231 | 
232 |                 # select 1-1 and 1-M relationships
233 |                 for p1Token in p1Set:
234 |                     if len(mapRelation1DS[p1Token]) == 1:
235 |                         if len(mapRelation2DS[list(mapRelation1DS[p1Token])[0]]) == 1:
236 |                             oneToOneS.add(p1Token)
237 | 
238 |                     else:
239 |                         isOneToM = True
240 | 
241 |                         for p2Token in mapRelation1DS[p1Token]:
242 |                             if len(mapRelation2DS[p2Token]) != 1:
243 |                                 isOneToM = False
244 |                                 break
245 |                         if isOneToM:
246 |                             oneToMP1D[p1Token] = 0
247 | 
248 |                 # delete the tokens which are picked to 1-1 and 1-M relationships from p1Set, so that the left are M-M
249 |                 for deleteToken in oneToOneS:
250 |                     p1Set.remove(deleteToken)
251 |                     p2Set.remove(list(mapRelation1DS[deleteToken])[0])
252 | 
253 |                 for deleteToken in oneToMP1D:
254 |                     for deleteTokenP2 in mapRelation1DS[deleteToken]:
255 |                         p2Set.remove(deleteTokenP2)
256 |                     p1Set.remove(deleteToken)
257 | 
258 |                 # select M-1 relationships
259 |                 for p2Token in p2Set:
260 |                     if len(mapRelation2DS[p2Token]) != 1:
261 |                         isOneToM = True
262 |                         for p1Token in mapRelation2DS[p2Token]:
263 |                             if len(mapRelation1DS[p1Token]) != 1:
264 |                                 isOneToM = False
265 |                                 break
266 |                         if isOneToM:
267 |                             oneToMP2D[p2Token] = 0
268 | 
269 |                 # delete the tokens which are picked to M-1 relationships from p2Set, so that the left are M-M
270 |                 for deleteToken in oneToMP2D:
271 |                     p2Set.remove(deleteToken)
272 |                     for deleteTokenP1 in mapRelation2DS[deleteToken]:
273 |                         p1Set.remove(deleteTokenP1)
274 | 
275 |                 # calculate the #Lines_that_match_S
276 |                 for logL in partition.logLL:
277 |                     if logL[p1] in oneToMP1D:
278 |                         oneToMP1D[logL[p1]] += 1
279 | 
280 |                     if logL[p2] in oneToMP2D:
281 |                         oneToMP2D[logL[p2]] += 1
282 | 
283 |             except KeyError as er:
284 |                 print (er)
285 |                 print ('error: ' + str(p1) + '\t' + str(p2))
286 | 
287 |             newPartitionsD = {}
288 |             if partition.stepNo == 2:
289 |                 newPartitionsD["dumpKeyforMMrelationInStep2__"] = Partition(stepNo=3, numOfLogs=0,
290 |                                                                             lenOfLogs=partition.lenOfLogs)
291 |             # Split partition
292 |             for logL in partition.logLL:
293 |                 # If is 1-1
294 |                 if logL[p1] in oneToOneS:
295 |                     if logL[p1] not in newPartitionsD:
296 |                         newPartitionsD[logL[p1]] = Partition(stepNo=3, numOfLogs=0, lenOfLogs=partition.lenOfLogs)
297 |                     newPartitionsD[logL[p1]].logLL.append(logL)
298 |                     newPartitionsD[logL[p1]].numOfLogs += 1
299 | 
300 |                 # This part can be improved. The split_rank can be calculated once.
301 |                 # If is 1-M
302 |                 elif logL[p1] in oneToMP1D:
303 |                     split_rank = self.Get_Rank_Posistion(len(mapRelation1DS[logL[p1]]), oneToMP1D[logL[p1]], True)
304 |                     if split_rank == 1:
305 |                         if logL[p1] not in newPartitionsD:
306 |                             newPartitionsD[logL[p1]] = Partition(stepNo=3, numOfLogs=0, lenOfLogs=partition.lenOfLogs)
307 |                         newPartitionsD[logL[p1]].logLL.append(logL)
308 |                         newPartitionsD[logL[p1]].numOfLogs += 1
309 |                     else:
310 |                         if logL[p2] not in newPartitionsD:
311 |                             newPartitionsD[logL[p2]] = Partition(stepNo=3, numOfLogs=0, lenOfLogs=partition.lenOfLogs)
312 |                         newPartitionsD[logL[p2]].logLL.append(logL)
313 |                         newPartitionsD[logL[p2]].numOfLogs += 1
314 | 
315 |                 # If is M-1
316 |                 elif logL[p2] in oneToMP2D:
317 |                     split_rank = self.Get_Rank_Posistion(len(mapRelation2DS[logL[p2]]), oneToMP2D[logL[p2]], False)
318 |                     if split_rank == 1:
319 |                         if logL[p1] not in newPartitionsD:
320 |                             newPartitionsD[logL[p1]] = Partition(stepNo=3, numOfLogs=0, lenOfLogs=partition.lenOfLogs)
321 |                         newPartitionsD[logL[p1]].logLL.append(logL)
322 |                         newPartitionsD[logL[p1]].numOfLogs += 1
323 |                     else:
324 |                         if logL[p2] not in newPartitionsD:
325 |                             newPartitionsD[logL[p2]] = Partition(stepNo=3, numOfLogs=0, lenOfLogs=partition.lenOfLogs)
326 |                         newPartitionsD[logL[p2]].logLL.append(logL)
327 |                         newPartitionsD[logL[p2]].numOfLogs += 1
328 | 
329 |                 # M-M
330 |                 else:
331 |                     if partition.stepNo == 2:
332 |                         newPartitionsD["dumpKeyforMMrelationInStep2__"].logLL.append(logL)
333 |                         newPartitionsD["dumpKeyforMMrelationInStep2__"].numOfLogs += 1
334 |                     else:
335 |                         if len(p1Set) < len(p2Set):
336 |                             if logL[p1] not in newPartitionsD:
337 |                                 newPartitionsD[logL[p1]] = Partition(stepNo=3, numOfLogs=0,
338 |                                                                      lenOfLogs=partition.lenOfLogs)
339 |                             newPartitionsD[logL[p1]].logLL.append(logL)
340 |                             newPartitionsD[logL[p1]].numOfLogs += 1
341 |                         else:
342 |                             if logL[p2] not in newPartitionsD:
343 |                                 newPartitionsD[logL[p2]] = Partition(stepNo=3, numOfLogs=0,
344 |                                                                      lenOfLogs=partition.lenOfLogs)
345 |                             newPartitionsD[logL[p2]].logLL.append(logL)
346 |                             newPartitionsD[logL[p2]].numOfLogs += 1
347 | 
348 |             if "dumpKeyforMMrelationInStep2__" in newPartitionsD and newPartitionsD[
349 |                 "dumpKeyforMMrelationInStep2__"].numOfLogs == 0:
350 |                 newPartitionsD["dumpKeyforMMrelationInStep2__"].valid = False
351 |             # Add all the new partitions to collection
352 |             for key in newPartitionsD:
353 |                 if self.para.PST != 0 and 1.0 * newPartitionsD[key].numOfLogs / partition.numOfLogs < self.para.PST:
354 |                     self.partitionsL[0].logLL += newPartitionsD[key].logLL
355 |                     self.partitionsL[0].numOfLogs += newPartitionsD[key].numOfLogs
356 |                 else:
357 |                     self.partitionsL.append(newPartitionsD[key])
358 | 
359 |             partition.valid = False
360 | 
361 |     def Step4(self):
362 |         self.partitionsL[0].valid = False
363 |         if self.para.PST == 0 and self.partitionsL[0].numOfLogs != 0:
364 |             event = Event(['Outlier'])
365 |             event.eventCount = self.partitionsL[0].numOfLogs
366 |             self.eventsL.append(event)
367 | 
368 |             for logL in self.partitionsL[0].logLL:
369 |                 logL.append(str(event.eventId))
370 | 
371 |         for partition in self.partitionsL:
372 |             if not partition.valid:
373 |                 continue
374 | 
375 |             if partition.numOfLogs == 0:
376 |                 print (str(partition.stepNo) + '\t')
377 | 
378 |             uniqueTokensCountLS = []
379 |             for columnIdx in range(partition.lenOfLogs):
380 |                 uniqueTokensCountLS.append(set())
381 | 
382 |             for logL in partition.logLL:
383 |                 for columnIdx in range(partition.lenOfLogs):
384 |                     uniqueTokensCountLS[columnIdx].add(logL[columnIdx])
385 | 
386 |             e = copy.deepcopy(partition.logLL[0])[:partition.lenOfLogs]
387 | 
388 |             for columnIdx in range(partition.lenOfLogs):
389 |                 if len(uniqueTokensCountLS[columnIdx]) == 1:
390 |                     continue
391 |                 else:
392 |                     e[columnIdx] = '<*>'
393 | 
394 |             event = Event(e)
395 |             event.eventCount = partition.numOfLogs
396 | 
397 |             self.eventsL.append(event)
398 | 
399 |             for logL in partition.logLL:
400 |                 logL.append(str(event.eventId))
401 | 
402 |     def getOutput(self):
403 |         if self.para.PST == 0 and self.partitionsL[0].numOfLogs != 0:
404 |             for logL in self.partitionsL[0].logLL:
405 |                 self.output.append(logL[-2:] + logL[:-2])
406 |         for partition in self.partitionsL:
407 |             if not partition.valid:
408 |                 continue
409 |             for logL in partition.logLL:
410 |                 self.output.append(logL[-2:] + logL[:-2])
411 | 
412 |     def WriteEventToFile(self):
413 |         eventID_template = {event.eventId : ' '.join(event.eventStr) for event in self.eventsL}
414 |         eventList = [[event.eventId, ' '.join(event.eventStr), event.eventCount] for event in self.eventsL]
415 |         eventDf = pd.DataFrame(eventList, columns=['EventId', 'EventTemplate', 'Occurrences'])
416 |         eventDf.to_csv(os.path.join(self.para.savePath, self.logname + '_templates.csv'), index=False)
417 | 
418 |         self.output.sort(key=lambda x: int(x[0]))
419 |         self.df_log['EventId'] = [str(logL[1]) for logL in self.output]
420 |         self.df_log['EventTemplate'] = [eventID_template[logL[1]] for logL in self.output]
421 |         if self.keep_para:
422 |             self.df_log["ParameterList"] = self.df_log.apply(self.get_parameter_list, axis=1) 
423 |         self.df_log.to_csv(os.path.join(self.para.savePath, self.logname + '_structured.csv'), index=False)
424 | 
425 |     """
426 |     For 1-M and M-1 mappings, you need to decide whether M side are constants or variables. This method is to decide which side to split
427 | 
428 |     cardOfS           : The number of unique values in this set
429 |     Lines_that_match_S: The number of lines that have these values
430 |     one_m             : If the mapping is 1-M, this value is True. Otherwise, False
431 |     """
432 | 
433 |     def Get_Rank_Posistion(self, cardOfS, Lines_that_match_S, one_m):
434 |         try:
435 |             distance = 1.0 * cardOfS / Lines_that_match_S
436 |         except ZeroDivisionError as er1:
437 |             print (er1)
438 |             print ("cardOfS: " + str(cardOfS) + '\t' + 'Lines_that_match_S: ' + str(Lines_that_match_S))
439 | 
440 |         if distance <= self.para.lowerBound:
441 |             if one_m:
442 |                 split_rank = 2
443 |             else:
444 |                 split_rank = 1
445 |         elif distance >= self.para.upperBound:
446 |             if one_m:
447 |                 split_rank = 1
448 |             else:
449 |                 split_rank = 2
450 |         else:
451 |             if one_m:
452 |                 split_rank = 1
453 |             else:
454 |                 split_rank = 2
455 | 
456 |         return split_rank
457 | 
458 |     def DetermineP1P2(self, partition):
459 |         if partition.lenOfLogs > 2:
460 |             count_1 = 0
461 | 
462 |             uniqueTokensCountLS = []
463 |             for columnIdx in range(partition.lenOfLogs):
464 |                 uniqueTokensCountLS.append(set())
465 | 
466 |             for logL in partition.logLL:
467 |                 for columnIdx in range(partition.lenOfLogs):
468 |                     uniqueTokensCountLS[columnIdx].add(logL[columnIdx])
469 | 
470 |             # Count how many columns have only one unique term
471 |             for columnIdx in range(partition.lenOfLogs):
472 |                 if len(uniqueTokensCountLS[columnIdx]) == 1:
473 |                     count_1 += 1
474 | 
475 |             # If the columns with unique term more than a threshold, we return (-1, -1) to skip step 3
476 |             GC = 1.0 * count_1 / partition.lenOfLogs
477 | 
478 |             if GC < self.para.CT:
479 |                 return self.Get_Mapping_Position(partition, uniqueTokensCountLS)
480 |             else:
481 |                 return (-1, -1)
482 | 
483 | 
484 |         elif partition.lenOfLogs == 2:
485 |             return (0, 1)
486 |         else:
487 |             return (-1, -1)
488 | 
489 |     def Get_Mapping_Position(self, partition, uniqueTokensCountLS):
490 |         p1 = p2 = -1
491 | 
492 |         # Caculate #unqiueterms in each column, and record how many column with each #uniqueterms
493 |         numOfUniqueTokensD = {}
494 |         for columnIdx in range(partition.lenOfLogs):
495 |             if len(uniqueTokensCountLS[columnIdx]) not in numOfUniqueTokensD:
496 |                 numOfUniqueTokensD[len(uniqueTokensCountLS[columnIdx])] = 0
497 |             numOfUniqueTokensD[len(uniqueTokensCountLS[columnIdx])] += 1
498 | 
499 |         if partition.stepNo == 2:
500 | 
501 |             # Find the largest card and second largest card
502 |             maxIdx = secondMaxIdx = -1
503 |             maxCount = secondMaxCount = 0
504 |             for key in numOfUniqueTokensD:
505 |                 if key == 1:
506 |                     continue
507 |                 if numOfUniqueTokensD[key] > maxCount:
508 |                     secondMaxIdx = maxIdx
509 |                     secondMaxCount = maxCount
510 |                     maxIdx = key
511 |                     maxCount = numOfUniqueTokensD[key]
512 |                 elif numOfUniqueTokensD[key] > secondMaxCount and numOfUniqueTokensD[key] != maxCount:
513 |                     secondMaxIdx = key
514 |                     secondMaxCount = numOfUniqueTokensD[key]
515 | 
516 |             # If the frequency of the freq_card>1 then
517 |             if maxCount > 1:
518 |                 for columnIdx in range(partition.lenOfLogs):
519 |                     if len(uniqueTokensCountLS[columnIdx]) == maxIdx:
520 |                         if p1 == -1:
521 |                             p1 = columnIdx
522 |                         else:
523 |                             p2 = columnIdx
524 |                             break
525 | 
526 | #                 for columnIdx in range(partition.lenOfLogs):
527 | #                     if p2 != -1:
528 | #                         break
529 | #                     if numOfUniqueTokensD[len(uniqueTokensCountLS[columnIdx])] == secondMaxCount:
530 | #                         p2 = columnIdx
531 | #                         break
532 | 
533 |             # If the frequency of the freq_card==1 then
534 |             else:
535 |                 for columnIdx in range(partition.lenOfLogs):
536 |                     if len(uniqueTokensCountLS[columnIdx]) == maxIdx:
537 |                         p1 = columnIdx
538 |                         break
539 | 
540 |                 for columnIdx in range(partition.lenOfLogs):
541 |                     if len(uniqueTokensCountLS[columnIdx]) == secondMaxIdx:
542 |                         p2 = columnIdx
543 |                         break
544 | 
545 |             if p1 == -1 or p2 == -1:
546 |                 return (-1, -1)
547 |             else:
548 |                 return (p1, p2)
549 | 
550 |         # If it is from step 1
551 |         else:
552 |             minIdx = secondMinIdx = -1
553 |             minCount = secondMinCount = sys.maxsize
554 |             for key in numOfUniqueTokensD:
555 |                 if numOfUniqueTokensD[key] < minCount:
556 |                     secondMinIdx = minIdx
557 |                     secondMinCount = minCount
558 |                     minIdx = key
559 |                     minCount = numOfUniqueTokensD[key]
560 |                 elif numOfUniqueTokensD[key] < secondMinCount and numOfUniqueTokensD[key] != minCount:
561 |                     secondMinIdx = key
562 |                     secondMinCount = numOfUniqueTokensD[key]
563 | 
564 |             for columnIdx in range(len(uniqueTokensCountLS)):
565 |                 if numOfUniqueTokensD[len(uniqueTokensCountLS[columnIdx])] == minCount:
566 |                     if p1 == -1:
567 |                         p1 = columnIdx
568 |                         break
569 | 
570 |             for columnIdx in range(len(uniqueTokensCountLS)):
571 |                 if numOfUniqueTokensD[len(uniqueTokensCountLS[columnIdx])] == secondMinCount:
572 |                     p2 = columnIdx
573 |                     break
574 | 
575 |             return (p1, p2)
576 | 
577 |     def PrintPartitions(self):
578 |         for idx in range(len(self.partitionsL)):
579 |             print ('Partition {}:(from step {})    Valid:{}'.format(idx, self.partitionsL[idx].stepNo,
580 |                                                                     self.partitionsL[idx].valid))
581 | 
582 |             for log in self.partitionsL[idx].logLL:
583 |                 print (log)
584 | 
585 |     def PrintEventStats(self):
586 |         for event in self.eventsL:
587 |             if event.eventCount > 1:
588 |                 print (str(event.eventId) + '\t' + str(event.eventCount))
589 |                 print (event.eventStr)
590 | 
591 |     def log_to_dataframe(self, log_file, regex, headers, logformat):
592 |         """ Function to transform log file to dataframe 
593 |         """
594 |         log_messages = []
595 |         linecount = 0
596 |         with open(log_file, 'r') as fin:
597 |             for line in fin.readlines():
598 |                 try:
599 |                     match = regex.search(line.strip())
600 |                     message = [match.group(header) for header in headers]
601 |                     log_messages.append(message)
602 |                     linecount += 1
603 |                 except Exception as e:
604 |                     pass
605 |         logdf = pd.DataFrame(log_messages, columns=headers)
606 |         logdf.insert(0, 'LineId', None)
607 |         logdf['LineId'] = [i + 1 for i in range(linecount)]
608 |         return logdf
609 | 
610 |     def generate_logformat_regex(self, logformat):
611 |         """ Function to generate regular expression to split log messages
612 |         """
613 |         headers = []
614 |         splitters = re.split(r'(<[^<>]+>)', logformat)
615 |         regex = ''
616 |         for k in range(len(splitters)):
617 |             if k % 2 == 0:
618 |                 splitter = re.sub(' +', '\s+', splitters[k])
619 |                 regex += splitter
620 |             else:
621 |                 header = splitters[k].strip('<').strip('>')
622 |                 regex += '(?P<%s>.*?)' % header
623 |                 headers.append(header)
624 |         regex = re.compile('^' + regex + '$')
625 |         return headers, regex
626 | 
627 |     def get_parameter_list(self, row):
628 |         template_regex = re.sub(r"\s<.{1,5}>\s", "<*>", row["EventTemplate"])
629 |         if "<*>" not in template_regex: return []
630 |         template_regex = re.sub(r'([^A-Za-z0-9])', r'\\\1', template_regex)
631 |         template_regex = re.sub(r'\\ +', r'[^A-Za-z0-9]+', template_regex)
632 |         template_regex = "^" + template_regex.replace("\<\*\>", "(.*?)") + "$"
633 |         parameter_list = re.findall(template_regex, row["Content"])
634 |         parameter_list = parameter_list[0] if parameter_list else ()
635 |         parameter_list = list(parameter_list) if isinstance(parameter_list, tuple) else [parameter_list]
636 |         parameter_list = [para.strip(string.punctuation).strip(' ') for para in parameter_list]
637 |         return parameter_list
638 | 


--------------------------------------------------------------------------------
/logparser/IPLoM/README.md:
--------------------------------------------------------------------------------
 1 | # IPLoM - Iterative Partitioning Log Mining
 2 | 
 3 | IPLoM is one of the state-of-the-art algorithms for log parsing. It leverages the unique characteristics of log messages for iterative log partitioning, which thus enables efficient message type extraction. Since the original open-source implementation is not available anymore, we re-implement the algorithm using Python with a nice interface provided.
 4 | 
 5 | Read more information about IPLoM from the following papers:
 6 | 
 7 | + Adetokunbo Makanju, A. Nur Zincir-Heywood, Evangelos E. Milios. [Clustering Event Logs Using Iterative Partitioning](https://web.cs.dal.ca/~makanju/publications/paper/kdd09.pdf), *ACM SIGKDD Conference on Knowledge Discovery and Data Mining (KDD)*, 2009.
 8 | 
 9 | + Adetokunbo Makanju, A. Nur Zincir-Heywood, Evangelos E. Milios. [A Lightweight Algorithm for Message Type Extraction in System Application Logs](http://ieeexplore.ieee.org/abstract/document/5936060/), *IEEE Transactions on Knowledge and Data Engineering (TKDE)*, 2012.
10 | 
11 | 
12 | 


--------------------------------------------------------------------------------
/logparser/IPLoM/__init__.py:
--------------------------------------------------------------------------------
1 | from .IPLoM import *
2 | 


--------------------------------------------------------------------------------
/logparser/IPLoM/__pycache__/IPLoM.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/logparser/IPLoM/__pycache__/IPLoM.cpython-36.pyc


--------------------------------------------------------------------------------
/logparser/IPLoM/__pycache__/IPLoM.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/logparser/IPLoM/__pycache__/IPLoM.cpython-37.pyc


--------------------------------------------------------------------------------
/logparser/IPLoM/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/logparser/IPLoM/__pycache__/__init__.cpython-36.pyc


--------------------------------------------------------------------------------
/logparser/IPLoM/__pycache__/__init__.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/logparser/IPLoM/__pycache__/__init__.cpython-37.pyc


--------------------------------------------------------------------------------
/logparser/__init__.py:
--------------------------------------------------------------------------------
1 | from .utils import evaluator


--------------------------------------------------------------------------------
/logparser/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/logparser/__pycache__/__init__.cpython-36.pyc


--------------------------------------------------------------------------------
/logparser/__pycache__/__init__.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/logparser/__pycache__/__init__.cpython-37.pyc


--------------------------------------------------------------------------------
/logparser/logmatch/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/logparser/logmatch/__init__.py


--------------------------------------------------------------------------------
/logparser/logmatch/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/logparser/logmatch/__pycache__/__init__.cpython-36.pyc


--------------------------------------------------------------------------------
/logparser/logmatch/__pycache__/regexmatch.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/logparser/logmatch/__pycache__/regexmatch.cpython-36.pyc


--------------------------------------------------------------------------------
/logparser/logmatch/regexmatch.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2018 The LogPAI Team (https://github.com/logpai).
  2 | #
  3 | # Licensed under the MIT License:
  4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 
  5 | # of this software and associated documentation files (the "Software"), to deal
  6 | # in the Software without restriction, including without limitation the rights 
  7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 
  8 | # copies of the Software, and to permit persons to whom the Software is 
  9 | # furnished to do so, subject to the following conditions:
 10 | #
 11 | # The above copyright notice and this permission notice shall be included in 
 12 | # all copies or substantial portions of the Software.
 13 | #
 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 
 20 | # SOFTWARE.
 21 | # =============================================================================
 22 | """ 
 23 | This file implements the regular expression based algorithm for log 
 24 | template matching. The algorithm is described in the following paper:
 25 | 
 26 | [1] Jieming Zhu, Jingyang Liu, Pinjia He, Zibin Zheng, Michael R. Lyu. 
 27 |     "Real-Time Log Event Matching at Scale", XXX, 2018.
 28 | """
 29 | 
 30 | from ..utils import logloader
 31 | from collections import defaultdict, Counter, OrderedDict
 32 | import re
 33 | import pandas as pd
 34 | import os
 35 | from datetime import datetime
 36 | import multiprocessing as mp
 37 | import itertools
 38 | import hashlib
 39 | import numpy as np
 40 | 
 41 | 
 42 | class PatternMatch(object):
 43 | 
 44 |     def __init__(self, outdir='./result/', n_workers=1, optimized=False, logformat=None):
 45 |         self.outdir = outdir
 46 |         if not os.path.exists(outdir):
 47 |             os.makedirs(outdir) # Make the result directory
 48 |         self.template_match_dict = defaultdict(dict)
 49 |         self.template_freq_dict = Counter()
 50 |         self.logformat = logformat
 51 |         self.n_workers = n_workers
 52 |         self.optimized = optimized
 53 | 
 54 |     def add_event_template(self, event_template, event_Id=None):
 55 |         if not event_Id:
 56 |             event_Id = self._generate_hash_eventId(event_template)
 57 |         if self.optimized:
 58 |             start_token = event_template.split(' ')[0]
 59 |             if re.search(r'<.*?>', start_token):
 60 |                 start_token = '<*>'
 61 |             self.template_match_dict[start_token][self._generate_template_regex(event_template)] = (event_Id, event_template)
 62 |         else:
 63 |             self.template_match_dict[self._generate_template_regex(event_template)] = (event_Id, event_template)
 64 | 
 65 |     def _generate_template_regex(self, template):
 66 |         template = re.sub(r'(<\*>\s?){2,}', '<*>', template)
 67 |         regex = re.sub(r'([^A-Za-z0-9])', r'\\\1', template)
 68 |         regex = regex.replace('\<\*\>', '(.*?)')
 69 |         regex = regex.replace('\<NUM\>', '(([\-|\+]?\d+)|(0[Xx][a-fA-F\d]+))')
 70 |         regex = regex.replace('\<IP\>', '((\d+\.){3}\d+)')
 71 |         regex = '^' + regex + '$'
 72 |         return regex
 73 | 
 74 |     def match_event(self, event_list):
 75 |         match_list = []
 76 |         paras = []
 77 |         if self.n_workers == 1:
 78 |             results = match_fn(event_list, self.template_match_dict, self.optimized)
 79 |         else:
 80 |             pool = mp.Pool(processes=self.n_workers)
 81 |             chunk_size = len(event_list) / self.n_workers + 1
 82 |             result_chunks = [pool.apply_async(match_fn, args=(event_list[i:i + chunk_size], self.template_match_dict, self.optimized))\
 83 |                              for i in xrange(0, len(event_list), chunk_size)]
 84 |             pool.close()
 85 |             pool.join()
 86 |             results = list(itertools.chain(*[result.get() for result in result_chunks]))
 87 |         for event, parameter_list in results:
 88 |             self.template_freq_dict[event] += 1
 89 |             paras.append(parameter_list)
 90 |             match_list.append(event)
 91 |         return match_list, paras
 92 | 
 93 |     def read_template_from_csv(self, template_filepath):
 94 |         template_dataframe = pd.read_csv(template_filepath)
 95 |         for idx, row in template_dataframe.iterrows():
 96 |             event_Id = row['EventId']
 97 |             event_template = row['EventTemplate']
 98 |             self.add_event_template(event_template, event_Id)
 99 | 
100 | 
101 |     def match(self, log_filepath, template_filepath):
102 |         print('Processing log file: {}'.format(log_filepath))
103 |         start_time = datetime.now()
104 |         loader = logloader.LogLoader(self.logformat, self.n_workers)
105 |         self.read_template_from_csv(template_filepath)
106 |         log_dataframe = loader.load_to_dataframe(log_filepath)
107 |         print('Matching event templates...')
108 |         match_list, paras = self.match_event(log_dataframe['Content'].tolist())
109 |         log_dataframe = pd.concat([log_dataframe, pd.DataFrame(match_list, columns=['EventId', 'EventTemplate'])], axis=1)
110 |         log_dataframe['ParameterList'] = paras
111 |         self._dump_match_result(os.path.basename(log_filepath), log_dataframe)
112 |         match_rate = sum(log_dataframe['EventId'] != 'NONE') / float(len(log_dataframe))
113 |         print('Matching done, matching rate: {:.1%} [Time taken: {!s}]'.format(match_rate, datetime.now() - start_time))
114 |         return log_dataframe
115 | 
116 |     def _dump_match_result(self, log_filename, log_dataframe):
117 |         log_dataframe.to_csv(os.path.join(self.outdir, log_filename + '_structured.csv'), index=False)
118 |         template_freq_list = [[eventId, template, freq] for (eventId, template), freq in self.template_freq_dict.iteritems()]
119 |         template_freq_df = pd.DataFrame(template_freq_list, columns=['EventId', 'EventTemplate', 'Occurrences'])
120 |         template_freq_df.to_csv(os.path.join(self.outdir, log_filename + '_templates.csv'), index=False)
121 | 
122 |     def _generate_hash_eventId(self, template_str):
123 |         return hashlib.md5(template_str.encode('utf-8')).hexdigest()[0:8]
124 | 
125 |     def _get_parameter_list(self, row):
126 |         template_regex = re.sub(r'([^A-Za-z0-9])', r'\\\1', row["EventTemplate"])
127 |         template_regex = "^" + template_regex.replace("\<\*\>", "(.*?)") + "$"
128 |         parameter_list = re.findall(template_regex, row["Content"])
129 |         parameter_list = parameter_list[0] if parameter_list else ()
130 |         parameter_list = list(parameter_list) if isinstance(parameter_list, tuple) else [parameter_list]
131 |         return parameter_list
132 | 
133 | def match_fn(event_list, template_match_dict, optimized=True):
134 |     print("Worker {} start matching {} lines.".format(os.getpid(), len(event_list)))
135 |     match_list = [regex_match(event_content, template_match_dict, optimized)
136 |                   for event_content in event_list]
137 |     return match_list
138 | 
139 | def regex_match(msg, template_match_dict, optimized):
140 |     matched_event = None
141 |     template_freq_dict = Counter()
142 |     match_dict = template_match_dict
143 |     parameter_list = []
144 |     if optimized:
145 |         start_token = msg.split(' ')[0]
146 |         if start_token in template_match_dict:
147 |             match_dict = template_match_dict[start_token]
148 |             if len(match_dict) > 1:
149 |                 match_dict = OrderedDict(sorted(match_dict.items(), 
150 |                      key=lambda x: (len(x[1][1]), -x[1][1].count('<*>')), reverse=True))
151 |             for regex, event in match_dict.iteritems():
152 |                 parameter_list = re.findall(regex, msg.strip())
153 |                 if parameter_list:
154 |                     matched_event = event
155 |                     break    
156 |     
157 |     if not matched_event:
158 |         if optimized:
159 |             match_dict = template_match_dict['<*>']
160 |         if len(match_dict) > 1:
161 |             match_dict = OrderedDict(sorted(match_dict.items(), 
162 |                  key=lambda x: (len(x[1][1]), -x[1][1].count('<*>')), reverse=True))
163 |         for regex, event in match_dict.iteritems():
164 |             parameter_list = re.findall(regex, msg.strip())
165 |             if parameter_list:
166 |                 matched_event = event
167 |                 break    
168 | 
169 |     if not matched_event:
170 |         matched_event = ('NONE', 'NONE')
171 |     if parameter_list:
172 |         parameter_list = list(parameter_list[0])
173 |     return matched_event, parameter_list
174 | 
175 | 


--------------------------------------------------------------------------------
/logparser/utils/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/logparser/utils/__init__.py


--------------------------------------------------------------------------------
/logparser/utils/__pycache__/__init__.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/logparser/utils/__pycache__/__init__.cpython-36.pyc


--------------------------------------------------------------------------------
/logparser/utils/__pycache__/__init__.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/logparser/utils/__pycache__/__init__.cpython-37.pyc


--------------------------------------------------------------------------------
/logparser/utils/__pycache__/evaluator.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/logparser/utils/__pycache__/evaluator.cpython-36.pyc


--------------------------------------------------------------------------------
/logparser/utils/__pycache__/evaluator.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/logparser/utils/__pycache__/evaluator.cpython-37.pyc


--------------------------------------------------------------------------------
/logparser/utils/__pycache__/logloader.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/logparser/utils/__pycache__/logloader.cpython-36.pyc


--------------------------------------------------------------------------------
/logparser/utils/__pycache__/logloader.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/davide97l/Log-Analysis-for-Anomaly-Detection/10e5d706bdc798c44cbc508b7d9fcc2e15fbfda5/logparser/utils/__pycache__/logloader.cpython-37.pyc


--------------------------------------------------------------------------------
/logparser/utils/evaluator.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Description : This file implements the function to evaluation accuracy of log parsing
  3 | Author      : LogPAI team
  4 | License     : MIT
  5 | """
  6 | 
  7 | import sys
  8 | import pandas as pd
  9 | from collections import defaultdict
 10 | #import scipy.misc as comb
 11 | from scipy.special import comb
 12 | 
 13 | 
 14 | def evaluate(groundtruth, parsedresult):
 15 |     """ Evaluation function to benchmark log parsing accuracy
 16 |     
 17 |     Arguments
 18 |     ---------
 19 |         groundtruth : str
 20 |             file path of groundtruth structured csv file 
 21 |         parsedresult : str
 22 |             file path of parsed structured csv file
 23 | 
 24 |     Returns
 25 |     -------
 26 |         f_measure : float
 27 |         accuracy : float
 28 |     """ 
 29 |     df_groundtruth = pd.read_csv(groundtruth)
 30 |     df_parsedlog = pd.read_csv(parsedresult)
 31 |     # Remove invalid groundtruth event Ids
 32 |     null_logids = df_groundtruth[~df_groundtruth['EventId'].isnull()].index
 33 |     df_groundtruth = df_groundtruth.loc[null_logids]
 34 |     df_parsedlog = df_parsedlog.loc[null_logids]
 35 |     (precision, recall, f_measure, accuracy) = get_accuracy(df_groundtruth['EventId'], df_parsedlog['EventId'])
 36 |     print('Precision: %.4f, Recall: %.4f, F1_measure: %.4f, Parsing_Accuracy: %.4f'%(precision, recall, f_measure, accuracy))
 37 |     return f_measure, accuracy
 38 | 
 39 | def get_accuracy(series_groundtruth, series_parsedlog, debug=False):
 40 |     """ Compute accuracy metrics between log parsing results and ground truth
 41 |     
 42 |     Arguments
 43 |     ---------
 44 |         series_groundtruth : pandas.Series
 45 |             A sequence of groundtruth event Ids
 46 |         series_parsedlog : pandas.Series
 47 |             A sequence of parsed event Ids
 48 |         debug : bool, default False
 49 |             print error log messages when set to True
 50 | 
 51 |     Returns
 52 |     -------
 53 |         precision : float
 54 |         recall : float
 55 |         f_measure : float
 56 |         accuracy : float
 57 |     """
 58 |     series_groundtruth_valuecounts = series_groundtruth.value_counts()
 59 |     real_pairs = 0
 60 |     for count in series_groundtruth_valuecounts:
 61 |         if count > 1:
 62 |             real_pairs += comb(count, 2)
 63 | 
 64 |     series_parsedlog_valuecounts = series_parsedlog.value_counts()
 65 |     parsed_pairs = 0
 66 |     for count in series_parsedlog_valuecounts:
 67 |         if count > 1:
 68 |             parsed_pairs += comb(count, 2)
 69 | 
 70 |     accurate_pairs = 0
 71 |     accurate_events = 0 # determine how many lines are correctly parsed
 72 |     for parsed_eventId in series_parsedlog_valuecounts.index:
 73 |         logIds = series_parsedlog[series_parsedlog == parsed_eventId].index
 74 |         series_groundtruth_logId_valuecounts = series_groundtruth[logIds].value_counts()
 75 |         error_eventIds = (parsed_eventId, series_groundtruth_logId_valuecounts.index.tolist())
 76 |         error = True
 77 |         if series_groundtruth_logId_valuecounts.size == 1:
 78 |             groundtruth_eventId = series_groundtruth_logId_valuecounts.index[0]
 79 |             if logIds.size == series_groundtruth[series_groundtruth == groundtruth_eventId].size:
 80 |                 accurate_events += logIds.size
 81 |                 error = False
 82 |         if error and debug:
 83 |             print('(parsed_eventId, groundtruth_eventId) =', error_eventIds, 'failed', logIds.size, 'messages')
 84 |         for count in series_groundtruth_logId_valuecounts:
 85 |             if count > 1:
 86 |                 accurate_pairs += comb(count, 2)
 87 | 
 88 |     precision = float(accurate_pairs) / parsed_pairs
 89 |     recall = float(accurate_pairs) / real_pairs
 90 |     f_measure = 2 * precision * recall / (precision + recall)
 91 |     accuracy = float(accurate_events) / series_groundtruth.size
 92 |     return precision, recall, f_measure, accuracy
 93 | 
 94 | 
 95 | 
 96 | 
 97 | 
 98 | 
 99 | 
100 | 


--------------------------------------------------------------------------------
/logparser/utils/logloader.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2018 The LogPAI Team (https://github.com/logpai).
  2 | #
  3 | # Licensed under the MIT License:
  4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 
  5 | # of this software and associated documentation files (the "Software"), to deal
  6 | # in the Software without restriction, including without limitation the rights 
  7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 
  8 | # copies of the Software, and to permit persons to whom the Software is 
  9 | # furnished to do so, subject to the following conditions:
 10 | #
 11 | # The above copyright notice and this permission notice shall be included in 
 12 | # all copies or substantial portions of the Software.
 13 | #
 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 
 20 | # SOFTWARE.
 21 | # =============================================================================
 22 | """ This file implements the formating interface to load log file to dataframe
 23 | """
 24 | 
 25 | import sys
 26 | import pandas as pd
 27 | import re
 28 | import multiprocessing as mp
 29 | from itertools import groupby, count, chain
 30 | import numpy as np
 31 | 
 32 | class LogLoader(object):
 33 | 
 34 |     def __init__(self, logformat, n_workers=1):
 35 |         if not logformat:
 36 |             raise RuntimeError('Logformat is required!')
 37 |         self.logformat = logformat.strip()
 38 |         self.headers, self.regex = self._generate_logformat_regex(self.logformat)
 39 |         self.n_workers = n_workers
 40 | 
 41 |     def load_to_dataframe(self, log_filepath):
 42 |         """ Function to transform log file to dataframe 
 43 |         """
 44 |         print('Loading log messages to dataframe...')
 45 |         lines = []
 46 |         with open(log_filepath, 'r') as fid:
 47 |             lines = fid.readlines()
 48 |         
 49 |         log_messages = []
 50 |         if self.n_workers == 1: 
 51 |             log_messages = formalize_message(enumerate(lines), self.regex, self.headers)
 52 |         else:
 53 |             chunk_size = np.ceil(len(lines) / float(self.n_workers))
 54 |             chunks = groupby(enumerate(lines), key=lambda k, line=count(): next(line)//chunk_size)
 55 |             log_chunks = [list(chunk) for _, chunk in chunks]
 56 |             print('Read %d log chunks in parallel'%len(log_chunks))
 57 |             pool = mp.Pool(processes=self.n_workers)
 58 |             result_chunks = [pool.apply_async(formalize_message, args=(chunk, self.regex, self.headers))
 59 |                              for chunk in log_chunks]
 60 |             pool.close()
 61 |             pool.join()
 62 |             log_messages = list(chain(*[result.get() for result in result_chunks]))
 63 | 
 64 |         if not log_messages:
 65 |             raise RuntimeError('Logformat error or log file is empty!')
 66 |         log_dataframe = pd.DataFrame(log_messages, columns=['LineId'] + self.headers)
 67 |         success_rate = len(log_messages) / float(len(lines))
 68 |         print('Loading {} messages done, loading rate: {:.1%}'.format(len(log_messages), success_rate))
 69 |         return log_dataframe
 70 | 
 71 |     def _generate_logformat_regex(self, logformat):
 72 |         """ Function to generate regular expression to split log messages
 73 |         """
 74 |         headers = []
 75 |         splitters = re.split(r'(<[^<>]+>)', logformat)
 76 |         regex = ''
 77 |         for k in range(len(splitters)):
 78 |             if k % 2 == 0:
 79 |                 splitter = re.sub(' +', '\s+', splitters[k])
 80 |                 regex += splitter
 81 |             else:
 82 |                 header = splitters[k].strip('<').strip('>')
 83 |                 regex += '(?P<%s>.*?)' % header
 84 |                 headers.append(header)
 85 |         regex = re.compile('^' + regex + '$')
 86 |         return headers, regex
 87 | 
 88 | 
 89 | def formalize_message(enumerated_lines, regex, headers):
 90 |     log_messages = []
 91 |     for line_count, line in enumerated_lines:
 92 |         line = line.strip()
 93 |         if not line:
 94 |             continue
 95 |         line = re.sub(r'[^\x00-\x7F]+', '<N/ASCII>', line)
 96 |         try:
 97 |             match = regex.search(line)
 98 |             message = [match.group(header) for header in headers]
 99 |             message.insert(0, line_count + 1)
100 |             log_messages.append(message)
101 |         except Exception as e:
102 |             pass
103 |     return log_messages


--------------------------------------------------------------------------------
/plot_anomaly_detection_charts.py:
--------------------------------------------------------------------------------
 1 | import pandas as pd
 2 | import matplotlib.pyplot as plt
 3 | 
 4 | res = pd.read_csv("anomaly_detection_results/benchmark_results.csv")
 5 | res.set_index('Model', inplace=True)  # set 'Model' columns as index
 6 | 
 7 | models = ['PCA', 'InvariantsMiner', 'LogClustering']
 8 | metrics = ['Precision', 'Recall', 'F1', 'Runtime']
 9 | 
10 | 
11 | def autolabel(ax, rects):
12 |     """Attach a text label above each bar in *rects*, displaying its height."""
13 |     for rect in rects:
14 |         height = rect.get_height()
15 |         ax.annotate('{0:.2f}'.format(height),
16 |                     xy=(rect.get_x() + rect.get_width() / 2, height),
17 |                     ha='center', va='bottom')
18 | 
19 | 
20 | # plot bar chart of precision, recall, F1 and runtime for each model
21 | for m in metrics:
22 |     m_train = []
23 |     m_test = []
24 |     for mod in models:
25 |         m_train.append(float(res.loc[mod + '-train', m]))
26 |         m_test.append(float(res.loc[mod + '-test', m]))
27 |         if m == 'Runtime':  # convert seconds to ms
28 |             m_train[-1] = int(m_train[-1] * 1000)
29 |             m_test[-1] = int(m_test[-1] * 1000)
30 | 
31 |     fig, ax = plt.subplots()
32 |     rect = ax.bar(models, m_train)
33 |     autolabel(ax, rect)
34 |     plt.ylabel(m)
35 |     if m == 'Runtime':
36 |         plt.ylabel(m + ' (ms)')
37 |     plt.xlabel('Models')
38 |     plt.title(m + ' Comparison Training Set')
39 |     plt.savefig("charts/" + m + "_anomaly_detection_results_train")
40 | 
41 |     fig, ax = plt.subplots()
42 |     rect = ax.bar(models, m_test)
43 |     autolabel(ax, rect)
44 |     plt.ylabel(m)
45 |     if m == 'Runtime':
46 |         plt.ylabel(m + ' (ms)')
47 |     plt.xlabel('Models')
48 |     plt.title(m + ' Comparison Test Set')
49 |     plt.savefig("charts/" + m + "_anomaly_detection_results_test")
50 | 
51 | plt.show()


--------------------------------------------------------------------------------