174 | Click for high resolution examples
175 |
176 |
177 | Normal - Empty university corridors (on different floors)
181 |
182 |
183 |
184 |
185 | Box - Cardboard boxes placed in front/near of the robot
187 |
188 |
189 |
190 |
191 | Cable - Various cables layed on the floor around and in front of the robot
193 |
194 |
195 |
196 |
197 | Debris - Various debris
199 |
200 |
201 |
202 |
203 | Defects - Defects of the robot
205 |
206 |
207 |
208 |
209 | Door - Open doors where doors should be closed
211 |
212 |
213 |
214 |
215 | Human - Human presence
217 |
218 |
219 |
220 |
221 | Clutter - Chairs, tables and furniture moved around the corridor
223 |
224 |
225 |
226 |
227 | Foam - Foam placed on the floor
229 |
230 |
231 |
232 |
233 | Sawdust - Sawdust placed on the floor
235 |
236 |
237 |
238 |
239 | Cellophane - Cellophane foil stretched between walls
241 |
242 |
243 |
244 |
245 | Floor - Fake flooring different than original floor
247 |
248 |
249 |
250 |
251 | Screws - Small screws and bolts placed in front of the robot
253 |
254 |
255 |
256 |
257 | Water - Water puddle in front of robot
259 |
260 |
261 |
262 |
263 | Cones - Multiple orange cones placed in the corridor
265 |
266 |
267 |
268 |
269 | Hanging cables - Cables hanging from the ceiling
271 |
272 |
273 |
274 |
--------------------------------------------------------------------------------
/code/Latest/WRN/train_oe_wrn.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | from collections import defaultdict
3 | from datetime import datetime
4 | from typing import List
5 |
6 | from models.wrn_oe.wrn import WideResNet
7 | from utils.variables_util import combined_labels_to_names
8 | from utils.metrics_util import compute_pr_aucs_single_loss, compute_roc_aucs_single_loss
9 | import pandas as pd
10 | import torch
11 | import torch.backends.cudnn as cudnn
12 | import torchvision.transforms as trn
13 | from rich.console import Console
14 | from tqdm import tqdm
15 |
16 | from models.wrn_oe.wrn_model_functions import set_wrn_class_parameters, \
17 | cosine_annealing, train_wrn, val_wrn, best_epoch_saver, test_wrn, oe_wrn_artifact_saver
18 | from oe_wrn_dataset import OEWRNImagesDataset, OEWRNImagesTestset, \
19 | OEWRNImagesOutlierset
20 | from utils.check_create_folder import check_create_folder
21 |
22 | console = Console()
23 |
24 |
25 | def main():
26 | parser = argparse.ArgumentParser(description='Trains a WRT with OE',
27 | formatter_class=argparse.ArgumentDefaultsHelpFormatter)
28 | parser.add_argument('--model', '-m', type=str, default='wrn',
29 | choices=['allconv', 'wrn'], help='Choose architecture.')
30 | # parser.add_argument('--calibration', '-c', action='store_true',
31 | # help='Train a model to be used for calibration. This holds out some data for validation.')
32 | # Optimization options
33 | parser.add_argument('--epochs', '-e', type=int, default=100, help='Number of epochs to train.')
34 | parser.add_argument('--learning_rate', '-lr', type=float, default=0.1, help='The initial learning rate.')
35 | parser.add_argument('--batch_size', '-b', type=int, default=128, help='Batch size.')
36 | parser.add_argument('--oe_batch_size', type=int, default=256, help='Batch size.')
37 | parser.add_argument('--test_bs', type=int, default=200)
38 | parser.add_argument('--momentum', type=float, default=0.9, help='Momentum.')
39 | parser.add_argument('--decay', '-d', type=float, default=0.0005, help='Weight decay (L2 penalty).')
40 | # WRN Architecture
41 | parser.add_argument('--layers', default=16, type=int, help='total number of layers')
42 | parser.add_argument('--widen-factor', default=4, type=int, help='widen factor')
43 | parser.add_argument('--droprate', default=0.3, type=float, help='dropout probability')
44 | # Acceleration
45 | parser.add_argument('--ngpu', type=int, default=1, help='0 = CPU.')
46 | # parser.add_argument('--prefetch', type=int, default=4, help='Pre-fetching threads.')
47 | parser.add_argument('--workers', type=int, default=4, help='Pre-fetching threads.')
48 | # IDSIA params
49 | parser.add_argument('--gpu_number',
50 | '-g', type=int, default=0)
51 | parser.add_argument('--root_path',
52 | '-r', type=str, default=".")
53 | param = parser.parse_args()
54 |
55 | mean = [0.485, 0.456, 0.406]
56 | std = [0.229, 0.224, 0.225]
57 | train_transform = trn.Compose([trn.RandomHorizontalFlip(), trn.RandomCrop(64, padding=8),
58 | trn.Normalize(mean, std)])
59 | test_transform = trn.Compose([trn.Normalize(mean, std)])
60 |
61 | num_classes = 3 # the classes are the envs: -1, 1long, 1short
62 |
63 | console.log(f'Using the following params:{param}')
64 | time_string = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
65 |
66 | if param.gpu_number == -1:
67 | cuda_gpu = f"cpu"
68 | else:
69 | cuda_gpu = f"cuda:{param.gpu_number}"
70 | # (
71 | # folder_prefix,
72 | # train_set_path,
73 | # outliers_set_path,
74 | # validation_set_path,
75 | # test_set_path,
76 | # test_set_labels_csv,
77 | # outliers_set_labels_csv,
78 | # train_set_labels_csv,
79 | # validation_set_labels_csv
80 | # ) = oe_wrn_model_paths_init(param)
81 | root_path = param.root_path
82 |
83 | train_set_path = f"{root_path}/data/train_set"
84 | outliers_set_path = f"{root_path}/data/outlier_set"
85 | validation_set_path = f"{root_path}/data/validation_set"
86 | test_set_path = f"{root_path}/data/test_set"
87 | test_set_labels_csv = f"{root_path}/data/metadata/frames_labels.csv"
88 | outliers_set_labels_csv = f"{root_path}/data/metadata/outliers_frames_labels.csv"
89 | train_set_labels_csv = f"{root_path}/data/metadata/train_frames_envs.csv"
90 | validation_set_labels_csv = f"{root_path}/data/metadata/validation_frames_envs.csv"
91 |
92 | model_key = f"WRN_{time_string}"
93 | save_folder = f"{root_path}/data/wrn/saves/{model_key}"
94 | check_create_folder(save_folder)
95 |
96 | device = torch.device(cuda_gpu if torch.cuda.is_available() else "cpu")
97 | print(device)
98 | (
99 | batch_size,
100 | epochs,
101 | lr,
102 | ml_flow_run_id,
103 | workers,
104 | momentum,
105 | decay,
106 | layers,
107 | widen_factor,
108 | droprate,
109 | outlier_batch_size,
110 | analysis_number,
111 | split_date,
112 | subset_size
113 | ) = set_wrn_class_parameters(
114 | param
115 | )
116 |
117 | model = WideResNet(layers, num_classes, widen_factor, dropRate=droprate)
118 | model.to(device)
119 | cudnn.benchmark = True # fire on all cylinders
120 |
121 | # DATA INIT
122 | train_set = OEWRNImagesDataset(train_set_path, train_transform, train_set_labels_csv)
123 | train_loader_in = torch.utils.data.DataLoader(
124 | train_set,
125 | batch_size=batch_size, shuffle=True,
126 | num_workers=workers, pin_memory=True)
127 | outliers_set_required_size = outlier_batch_size * len(train_loader_in)
128 | outliers_set = OEWRNImagesOutlierset(
129 | files_path=outliers_set_path,
130 | label_csv_path=outliers_set_labels_csv,
131 | transform=test_transform,
132 | required_dataset_size=outliers_set_required_size,
133 | )
134 | train_loader_out = torch.utils.data.DataLoader(
135 | outliers_set,
136 | batch_size=outlier_batch_size, shuffle=True,
137 | num_workers=workers, pin_memory=True)
138 | val_set = OEWRNImagesDataset(validation_set_path, test_transform, validation_set_labels_csv)
139 | val_loader = torch.utils.data.DataLoader(
140 | val_set,
141 | batch_size=batch_size, shuffle=False,
142 | num_workers=workers, pin_memory=True)
143 | test_set = OEWRNImagesTestset(test_set_path, label_csv_path=test_set_labels_csv, transform=test_transform)
144 | test_loader = torch.utils.data.DataLoader(
145 | test_set,
146 | batch_size=1, shuffle=False,
147 | num_workers=workers, pin_memory=True)
148 | print('Beginning Training\n')
149 | optimizer = torch.optim.SGD(model.parameters(), lr, momentum=momentum, weight_decay=decay, nesterov=True)
150 | scheduler = torch.optim.lr_scheduler.LambdaLR(
151 | optimizer,
152 | lr_lambda=lambda step: cosine_annealing(
153 | step,
154 | epochs * len(train_loader_in),
155 | 1, # since lr_lambda computes multiplicative factor
156 | 1e-6 / lr))
157 | # Main loop
158 | etqdm = tqdm(range(epochs), total=epochs, postfix="Training")
159 | metrics = defaultdict(list)
160 | best_loss = None
161 | best_epoch = 0
162 | best_model_path = None
163 | for epoch in etqdm:
164 | model, train_loss = train_wrn(model, train_loader_out, train_loader_in, scheduler, optimizer, device=device)
165 | metrics[f'train_loss'].append(train_loss)
166 | val_loss = val_wrn(model, val_loader, device)
167 | metrics[f'val_loss'].append(val_loss)
168 | if best_loss is None:
169 | best_epoch, best_loss, best_model_path = best_epoch_saver(epoch, model, param,
170 | save_folder, val_loss, model_key)
171 | if val_loss < best_loss:
172 | best_epoch, best_loss, best_model_path = best_epoch_saver(epoch, model, param,
173 | save_folder, val_loss, model_key)
174 |
175 | etqdm.set_description(
176 | f"Train loss: {train_loss:.3f} | Val loss: {val_loss:.3f} | best model @ epoch {best_epoch}")
177 |
178 | console.log("Training Completed, Testing Started")
179 |
180 | df_dict = test_wrn(best_model_path, model, test_loader, device, )
181 | test_set_df = pd.DataFrame.from_dict(df_dict)
182 | test_set_df["label"] = pd.to_numeric(test_set_df["label"])
183 |
184 | auc_computation_and_logging(test_set_df)
185 | list_of_labels_in_test_set = list(set(test_set.labels))
186 |
187 | # csv row building
188 | metrics_dict = {}
189 | for k in list_of_labels_in_test_set:
190 | v = combined_labels_to_names[k]
191 | class_metrics_dict = per_label_metrics(test_set_df, k)
192 | if k == 0:
193 | v = "all"
194 | metrics_dict[f"{v}_ok_mean_loss"] = class_metrics_dict["ok_mean_loss"]
195 | metrics_dict[f"{v}_an_mean_loss"] = class_metrics_dict["an_mean_loss"]
196 | metrics_dict[f"{v}_roc_auc"] = class_metrics_dict["roc_auc"]
197 | metrics_dict[f"{v}_pr_auc"] = class_metrics_dict["pr_auc"]
198 |
199 | csv_row = {
200 | **{"model_key": model_key,
201 | },
202 | **metrics_dict,
203 | }
204 |
205 | # SAVE STUFF
206 | console.log("Testing Completed")
207 | console.log("Creating and saving Artifacts")
208 | artifacts_path = save_folder + "/artifacts/"
209 | oe_wrn_artifact_saver(
210 | batch_size,
211 | epochs,
212 | lr,
213 | momentum,
214 | decay,
215 | layers,
216 | widen_factor,
217 | droprate,
218 | metrics,
219 | test_set_df,
220 | model,
221 | artifacts_path,
222 | param=param,
223 | csv_row=csv_row,
224 | csv_key=model_key,
225 | best_model_path=best_model_path,
226 | )
227 |
228 | console.log(f"Script completed, artifacts located at {save_folder}.")
229 |
230 |
231 | def auc_computation_and_logging(test_df):
232 | labels = [0 if el == 0 else 1 for el in test_df["label"].values]
233 | metrics_dict = compute_model_metrics(
234 | labels,
235 | test_df
236 | )
237 | print(f'test_set_ok_mean_loss = {metrics_dict["ok_mean_loss"]}\n'
238 | f'test_set_an_mean_loss = {metrics_dict["an_mean_loss"]}\n'
239 | f'test_set_roc_auc = {metrics_dict["roc_auc"]}\n'
240 | f'test_set_pr_auc = {metrics_dict["pr_auc"]}\n'
241 | )
242 |
243 |
244 | def per_label_metrics(df, label_key):
245 | if label_key == 0:
246 | label_unique_values = [0 if el == 0 else 1 for el in df["label"].values]
247 | return_dict = compute_model_metrics(
248 | label_unique_values,
249 | df,
250 | )
251 | else:
252 | df_anomaly = df[df.label.isin([0, label_key])]
253 | label_unique_values = [0 if el == 0 else 1 for el in df_anomaly["label"].values]
254 | return_dict = compute_model_metrics(
255 | label_unique_values,
256 | df_anomaly,
257 | )
258 | return return_dict
259 |
260 |
261 | def compute_model_metrics(
262 | labels: List[int],
263 | df_losses: pd.DataFrame,
264 | ):
265 | y_true = labels
266 | losses = df_losses["loss"].values
267 | pr_auc = compute_pr_aucs_single_loss(y_true, losses)
268 | roc_auc = compute_roc_aucs_single_loss(y_true, losses)
269 | an_mean_loss = df_losses[df_losses["label"] != 0]["loss"].values.mean()
270 | ok_mean_loss = df_losses[df_losses["label"] == 0]["loss"].values.mean()
271 | # composing return dict
272 | return_dict = {
273 | "an_mean_loss": an_mean_loss,
274 | "ok_mean_loss": ok_mean_loss,
275 | "roc_auc": roc_auc,
276 | "pr_auc": pr_auc,
277 | }
278 | return return_dict
279 |
280 |
281 | if __name__ == '__main__':
282 | main()
283 |
--------------------------------------------------------------------------------
/code/Latest/models/real_nvp/real_nvp_model_functions.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | import logging
3 | from collections import defaultdict
4 |
5 | import numpy as np
6 | import torch
7 | from torch import linalg as LA
8 | from tqdm import tqdm
9 |
10 | from utils.check_create_folder import check_create_folder
11 | from models.real_nvp import real_nvp
12 |
13 | logging.basicConfig(level=logging.INFO)
14 |
15 |
16 | # USED
17 | def build_network(embedding_size, coupling_topology, n_layers, mask_type, batch_norm=True):
18 | """Builds the neural network."""
19 | model = real_nvp.LinearRNVP(input_dim=embedding_size,
20 | coupling_topology=coupling_topology,
21 | flow_n=n_layers,
22 | batch_norm=batch_norm,
23 | mask_type=mask_type,
24 | conditioning_size=None,
25 | use_permutation=False,
26 | single_function=False)
27 |
28 | return model
29 |
30 |
31 | # USED
32 | def epoch_loop_oe_normal_rnvp(device: torch.device,
33 | epochs: int,
34 | model: real_nvp,
35 | optimizer,
36 | scheduler,
37 | train_loader: torch.utils.data.DataLoader,
38 | val_loader: torch.utils.data.DataLoader,
39 | save_folder: str,
40 | model_key: str,
41 | ):
42 | # Epoch Loop
43 | logging.info('Training started')
44 |
45 | etqdm = tqdm(range(epochs), total=epochs, postfix="Training")
46 | best_loss = None
47 | best_epoch = 0
48 | best_model_path = None
49 | metrics = defaultdict(list)
50 | for epoch in etqdm:
51 | model, train_loss, train_log_prob, train_log_det_j = trainer(
52 | device, model, optimizer, train_loader
53 | )
54 |
55 | metrics[f'train_loss'].append(train_loss)
56 | val_loss, val_log_prob, val_log_det_j = validator(val_loader, model, device)
57 | scheduler.step(val_loss)
58 | metrics[f'val_loss'].append(val_loss)
59 | if best_loss is None:
60 | best_epoch, best_loss, best_model_path = best_epoch_saver(epoch, model,
61 | save_folder, val_loss, model_key)
62 | if val_loss < best_loss:
63 | best_epoch, best_loss, best_model_path = best_epoch_saver(epoch, model,
64 | save_folder, val_loss, model_key)
65 |
66 | etqdm.set_description(
67 | f"Train loss: {train_loss:.3f} | Val loss: {val_loss:.3f} | best model @ epoch {best_epoch}")
68 | logging.info('Finished training.')
69 |
70 | return metrics, best_model_path
71 |
72 |
73 | # USED
74 | def epoch_loop_oe_with_oe_rnvp(device: torch.device,
75 | epochs: int,
76 | model: real_nvp,
77 | optimizer,
78 | scheduler,
79 | normal_train_loader: torch.utils.data.DataLoader,
80 | anomalies_train_loader: torch.utils.data.DataLoader,
81 | val_loader: torch.utils.data.DataLoader,
82 | save_folder: str,
83 | etn_key: str,
84 | gamma: float,
85 | lambda_p: float,
86 | ):
87 | # Epoch Loop
88 | logging.info('Training started')
89 |
90 | etqdm = tqdm(range(epochs), total=epochs, postfix="Training")
91 | best_loss = None
92 | best_epoch = 0
93 | best_model_path = None
94 | metrics = defaultdict(list)
95 | for epoch in etqdm:
96 | model, train_loss, train_log_prob, train_log_det_j, oe_loss_component_list, ok_l2_mean, an_l2_mean = trainer_oe(
97 | device,
98 | model,
99 | optimizer,
100 | normal_train_loader,
101 | anomalies_train_loader,
102 | gamma=gamma,
103 | lambda_p=lambda_p
104 | )
105 | metrics[f'train_loss'].append(train_loss)
106 |
107 | val_loss, val_log_prob, val_log_det_j = validator(val_loader, model, device)
108 | scheduler.step(val_loss)
109 |
110 | metrics[f'val_loss'].append(val_loss)
111 | if best_loss is None:
112 | best_epoch, best_loss, best_model_path = best_epoch_saver(epoch, model,
113 | save_folder, val_loss, etn_key)
114 | if val_loss < best_loss:
115 | best_epoch, best_loss, best_model_path = best_epoch_saver(epoch, model,
116 | save_folder, val_loss, etn_key)
117 |
118 | etqdm.set_description(
119 | f"Train loss: {train_loss:.3f} | Val loss: {val_loss:.3f} | best model @ epoch {best_epoch}")
120 | logging.info('Finished training.')
121 |
122 | return metrics, best_model_path
123 |
124 |
125 | # USED
126 | def best_epoch_saver(epoch, model, save_folder, val_loss, etn_key):
127 | best_loss = val_loss
128 | best_epoch = epoch
129 | checkpoint_folder = save_folder + f'/checkpoints/'
130 | check_create_folder(checkpoint_folder)
131 | best_model_path = checkpoint_folder + f'model_{etn_key}_epoch_{epoch}.pth'
132 | torch.save(model.state_dict(), best_model_path)
133 | logging.debug(f"Checkpoin model epoch {epoch},saved {best_model_path}")
134 | return best_epoch, best_loss, best_model_path
135 |
136 |
137 | # USED
138 | def set_parameters(
139 | param: argparse.ArgumentParser,
140 | ):
141 | epochs = 500
142 | batch_size = param.batch_size
143 | embedding_size = 128
144 | workers = param.num_workers
145 | lr = 0.001
146 | max_patience = int(np.log2(epochs)) + 2
147 | coupling_topology = [128]
148 | num_layers = 4
149 | mask_type = 'odds'
150 |
151 | return batch_size, embedding_size, epochs, mask_type, num_layers, lr, coupling_topology, workers, max_patience
152 |
153 |
154 | # USED
155 | def set_parameters_rnvp_inference(
156 | param: argparse.ArgumentParser,
157 | ):
158 | embedding_size = 128
159 | workers = param.num_workers
160 | coupling_topology = [128]
161 | num_layers = 4
162 | mask_type = 'odds'
163 | layer_1_ft = param.first_layer_size
164 | layer_2_ft = layer_1_ft * 2
165 | layer_3_ft = layer_1_ft * 2 * 2
166 | layer_4_ft = layer_1_ft * 2 * 2 * 2
167 | input_channel = 3
168 | image_size = (64, 64)
169 | widths_ = [
170 | input_channel,
171 | layer_1_ft,
172 | layer_2_ft,
173 | layer_3_ft,
174 | layer_4_ft,
175 | ]
176 | return embedding_size, mask_type, num_layers, coupling_topology, workers, widths_, image_size
177 |
178 |
179 | # USED
180 | def set_oe_parameters(
181 | param: argparse.ArgumentParser,
182 | ):
183 | epochs = 500
184 | batch_size = param.batch_size
185 | embedding_size = 128
186 | workers = param.num_workers
187 | lr = 0.001
188 | max_patience = int(np.log2(epochs)) + 2
189 | coupling_topology = [128]
190 | num_layers = 4
191 | mask_type = 'odds'
192 | outlier_batch_size = int(batch_size * 0.1)
193 | gamma = 100
194 | lambda_p = 1
195 |
196 | return batch_size, embedding_size, epochs, mask_type, num_layers, lr, coupling_topology, workers, max_patience, gamma, lambda_p, outlier_batch_size
197 |
198 |
199 | # USED
200 | def model_tester(best_model_path, model, device, test_loader):
201 | model.load_state_dict(torch.load(best_model_path, map_location=device))
202 | model.to(device)
203 | model.eval()
204 | df_dict = {'z': [], 'label': [], 'loss': [], 'log_prob': [], 'log_det_J': [], 'l2_norm_of_z': []}
205 | with torch.no_grad():
206 | for data in tqdm(test_loader, total=len(test_loader), postfix="Running Test Set inference"):
207 | inputs, label, _ = data
208 | inputs = inputs.to(device)
209 | outputs = model(inputs)
210 | z, log_det_J = outputs
211 | log_prob_z = model.log_prob(z)
212 | log_prob_z_mean = log_prob_z.mean()
213 | log_det_j_mean = log_det_J.mean()
214 | loss = - log_prob_z_mean - log_det_j_mean
215 | logging.debug(f"TEST: inputs.shape={inputs.shape}\n"
216 | f" outputs[0].shape={outputs[0].shape}\n"
217 | f" loss {loss} = - {log_prob_z_mean} - {log_det_j_mean}")
218 | df_dict["z"].append(z.detach().squeeze(0).cpu().numpy())
219 | df_dict["label"].append(label.item())
220 | df_dict["loss"].append(loss.detach().cpu().item())
221 | df_dict["log_prob"].append(log_prob_z.detach().cpu().item())
222 | df_dict["log_det_J"].append(log_det_J.detach().cpu().item())
223 | df_dict["l2_norm_of_z"].append(LA.norm(z).detach().cpu().item())
224 | return df_dict
225 |
226 |
227 | # USED
228 | def validator(val_loader, model, device):
229 | with torch.no_grad():
230 | model.eval()
231 | losses = []
232 | log_prob_mean_list = []
233 | log_det_j_list = []
234 | for data in val_loader:
235 | inputs = data[0]
236 | inputs = inputs.to(device)
237 | outputs = model(inputs)
238 | z, log_det_J = outputs
239 | log_prob_z = model.log_prob(z)
240 | log_prob_z_mean = log_prob_z.mean()
241 | log_det_j_mean = log_det_J.mean()
242 | log_prob_mean_list.append(log_prob_z_mean.item())
243 | log_det_j_list.append(log_det_j_mean.item())
244 | loss = - log_prob_z_mean - log_det_j_mean
245 | logging.debug(f"VAL: inputs.shape={inputs.shape}\n"
246 | f" outputs[0].shape={outputs[0].shape}\n"
247 | f" loss {loss} = - {log_prob_z_mean} - {log_det_j_mean}")
248 | losses.append(loss.detach().cpu().item())
249 | return np.mean(losses), np.mean(log_prob_mean_list), np.mean(log_det_j_list)
250 |
251 |
252 | # USED
253 | def trainer_oe(device, model, optimizer, normal_train_loader, anomalies_train_loader, gamma, lambda_p):
254 | model.train()
255 | losses = []
256 | log_prob_z_mean_list = []
257 | log_det_j_mean_list = []
258 | oe_loss_component_list = []
259 | ok_l2norm_list = []
260 | an_l2norm_list = []
261 | for ok_data, an_data in zip(normal_train_loader, anomalies_train_loader):
262 | # normal data
263 | ok_inputs, _ = ok_data
264 | ok_inputs = ok_inputs.to(device)
265 | optimizer.zero_grad()
266 | ok_outputs = model(ok_inputs)
267 | ok_z, ok_log_det_J = ok_outputs
268 | ok_log_prob_z = model.log_prob(ok_z)
269 | ok_log_prob_z_mean = ok_log_prob_z.mean()
270 | ok_log_det_j_mean = ok_log_det_J.mean()
271 |
272 | an_inputs, _, _ = an_data
273 | an_inputs = an_inputs.to(device)
274 |
275 | model.eval()
276 | an_outputs = model(an_inputs)
277 | model.train()
278 |
279 | an_z, an_log_det_J = an_outputs
280 | an_log_prob_z = model.log_prob(an_z)
281 | maxs = torch.maximum(torch.zeros((len(an_log_prob_z)), device=device),
282 | gamma
283 | +
284 | (- ok_log_prob_z[:len(an_log_prob_z)] - ok_log_det_J[:len(an_log_det_J)])
285 | -
286 | (- an_log_prob_z - an_log_det_J))
287 | oe_loss_component = maxs.mean()
288 | log_prob_z_mean_list.append(ok_log_prob_z_mean.item())
289 | log_det_j_mean_list.append(ok_log_det_j_mean.item())
290 | oe_loss_component_list.append(oe_loss_component.item())
291 | loss = - ok_log_prob_z_mean - ok_log_det_j_mean + lambda_p * oe_loss_component
292 | logging.debug(f"TRAIN: ok_inputs.shape={ok_inputs.shape}\n"
293 | f" ok_outputs[0].shape={ok_outputs[0].shape}\n"
294 | f" ok_log_prob_z = {ok_log_prob_z}\n"
295 | f" an_log_prob_z = {an_log_prob_z}\n"
296 | f" maxs = {maxs}\n"
297 | f" loss {loss} = - {ok_log_prob_z_mean} - {ok_log_det_j_mean} + {lambda_p} * {oe_loss_component}\n")
298 | loss.backward()
299 | optimizer.step()
300 | losses.append(loss.item())
301 | ok_l2norm_list += LA.norm(ok_z, dim=1).detach().cpu().tolist()
302 | an_l2norm_list += LA.norm(an_z, dim=1).detach().cpu().tolist()
303 | return model, np.mean(losses), np.mean(log_prob_z_mean_list), np.mean(log_det_j_mean_list), np.mean(
304 | oe_loss_component_list), np.mean(ok_l2norm_list), np.mean(an_l2norm_list)
305 |
306 |
307 | # USED
308 | def trainer(device, model, optimizer, train_loader):
309 | model.train()
310 | losses = []
311 | log_prob_z_mean_list = []
312 | log_det_j_mean_list = []
313 | for data in train_loader:
314 | inputs, _ = data
315 | inputs = inputs.to(device)
316 |
317 | optimizer.zero_grad()
318 | outputs = model(inputs)
319 | z, log_det_J = outputs
320 | log_prob_z = model.log_prob(z)
321 | log_prob_z_mean = log_prob_z.mean()
322 | log_det_j_mean = log_det_J.mean()
323 | log_prob_z_mean_list.append(log_prob_z_mean.item())
324 | log_det_j_mean_list.append(log_det_j_mean.item())
325 | loss = - log_prob_z_mean - log_det_j_mean
326 | logging.debug(f"TRAIN: inputs.shape={inputs.shape}\n"
327 | f" outputs[0].shape={outputs[0].shape}\n"
328 | f" loss {loss} = - {log_prob_z_mean} - {log_det_j_mean}")
329 | loss.backward()
330 | optimizer.step()
331 | losses.append(loss.item())
332 |
333 | return model, np.mean(losses), np.mean(log_prob_z_mean_list), np.mean(log_det_j_mean_list)
334 |
--------------------------------------------------------------------------------
/code/OLD_CODE/run.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | from datetime import datetime
3 |
4 | import albumentations as A
5 |
6 | import mlflow
7 | import pandas as pd
8 | import torch
9 | from rich.console import Console
10 | from torch.optim.lr_scheduler import ReduceLROnPlateau
11 | from torch.utils.data import Dataset
12 |
13 | from paper_code_release.model.autoencoder import AE
14 | from paper_code_release.model.uniformed_model_functions import set_model_and_train_parameters_patches, epoch_loop_patches, \
15 | test_loop_df_rows
16 | from paper_code_release.paper_utils.artifacts_util import uniformed_model_artifact_saver
17 | from paper_code_release.dataset import UniformedPatchesDataset, UniformedPatchesTestset
18 |
19 | from paper_code_release.paper_utils.check_create_folder import check_create_folder
20 | from paper_code_release.paper_utils.init_utils import uniformed_model_paths_init
21 | from paper_code_release.paper_utils.losses_util import losses_list
22 | from paper_code_release.paper_utils.metrics_util import compute_uniformed_model_metrics
23 | from paper_code_release.paper_utils.variables_util import dataset_names, available_scale_levels, datasets_labels_names, scaled_image_shapes
24 |
25 | console = Console()
26 |
27 |
28 | def params_parser():
29 | parser = argparse.ArgumentParser()
30 | parser.add_argument("--max_epochs",
31 | '-e', type=int, default=15)
32 | parser.add_argument("--batch_size",
33 | "-b", type=int, default=320)
34 | parser.add_argument("--bottleneck",
35 | "-n", type=int, default=128)
36 | parser.add_argument("--num_workers",
37 | '-w', type=int, default=4)
38 | parser.add_argument('--gpu_number',
39 | '-g', type=int, default=0)
40 | parser.add_argument('--learning_rate',
41 | '-l', type=float, default=1e-3)
42 | parser.add_argument('--use_ml_flow',
43 | '-m', type=int, default=1)
44 | parser.add_argument('--first_layer_size',
45 | '-f', type=int, default=128)
46 | parser.add_argument('--input_channels',
47 | '-i', type=int, default=3)
48 | parser.add_argument('--id_optimized_loss',
49 | '-o', type=int, default=0)
50 | parser.add_argument('--render_video',
51 | '-v', type=int, default=0)
52 | parser.add_argument('--patience_thres',
53 | '-p', type=float, default=0.001)
54 | parser.add_argument('--dataset',
55 | '-d', type=int, default=1)
56 | parser.add_argument('--scale_level',
57 | '-s', type=int, default=2)
58 | parser.add_argument('--test_patches_number',
59 | '-t', type=int, default=250)
60 | param = parser.parse_args()
61 | return param
62 |
63 |
64 | def per_label_metrics(df, label_key):
65 | if label_key == 0:
66 | label_unique_values = [0 if el[0] == 0 else 1 for el in
67 | df[["frame_id", "frame_label"]].groupby("frame_id")["frame_label"].unique().values]
68 | return_dict = compute_uniformed_model_metrics(
69 | label_unique_values,
70 | losses_list,
71 | df[["frame_id", "mse_loss", "mae_loss"]],
72 | stdev=True,
73 | )
74 | else:
75 | df_anomaly = df[df["frame_label"].isin([0, label_key])]
76 | label_unique_values = [0 if el[0] == 0 else 1 for el in
77 | df_anomaly[["frame_id", "frame_label"]].groupby("frame_id")[
78 | "frame_label"].unique().values]
79 |
80 | return_dict = compute_uniformed_model_metrics(
81 | label_unique_values,
82 | losses_list,
83 | df_anomaly[["frame_id", "mse_loss", "mae_loss"]],
84 | stdev=True,
85 | )
86 | return return_dict
87 |
88 |
89 | def main():
90 | """
91 | if param.gpu_number = -1 it uses the cpu
92 | Returns:
93 | """
94 | # MODEL INIT
95 | ml_flow_run_id = None
96 | param = params_parser()
97 | console.log(f'Using the following params:{param}')
98 | time_string = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
99 | if param.scale_level not in available_scale_levels.keys():
100 | raise ValueError(
101 | f"Oops! That was no valid value for flag --scale_level. It has to be one of {available_scale_levels}")
102 |
103 | if param.dataset not in dataset_names.keys():
104 | raise ValueError(f"Oops! That was no valid value for flag --dataset. It has to be be one of {dataset_names}")
105 |
106 | if param.use_ml_flow:
107 | mlflow.set_tracking_uri("http://localhost:9999")
108 | mlflow.set_experiment("ICRA 2022 experiments")
109 | mlflow.start_run()
110 | artifact_uri = mlflow.get_artifact_uri()
111 | console.log(f"artifact uri {artifact_uri}")
112 |
113 | patch_shape = (64, 64)
114 | image_shape = scaled_image_shapes[param.scale_level]
115 |
116 | model_save_folder_prefix, qualitative_paths, test_path, test_labels_csv, train_path, val_path, noise_path = uniformed_model_paths_init(
117 | param)
118 | if param.gpu_number == "-1":
119 | cuda_gpu = f"cpu"
120 | else:
121 | cuda_gpu = f"cuda:{param.gpu_number}"
122 | device = torch.device(cuda_gpu if torch.cuda.is_available() else "cpu")
123 |
124 | batch_size, bottleneck, epochs, input_channel, layer_1_ft, lr, max_patience, ml_flow_run_id, widths, workers = set_model_and_train_parameters_patches(
125 | device, ml_flow_run_id, param, patch_shape)
126 |
127 | train_patch_num = 1
128 |
129 | model = AE(widths, image_shape=patch_shape, bottleneck_size=bottleneck)
130 | model.to(device)
131 | if param.use_ml_flow:
132 | # noinspection PyUnboundLocalVariable
133 | model_save_folder = artifact_uri
134 | else:
135 | model_save_folder = model_save_folder_prefix + f'/saves/dataset_{dataset_names[param.dataset]}_B_{bottleneck}_F_{layer_1_ft}_S_{param.scale_level}_{time_string}'
136 |
137 | # AUG INIT
138 |
139 | composed_transform = A.Compose(
140 | [
141 | A.transforms.HorizontalFlip(p=0.5),
142 | A.transforms.RandomBrightnessContrast(brightness_limit=0.1, contrast_limit=0.1, p=0.5),
143 | A.RandomSizedCrop(min_max_height=[50, image_shape[0]], height=image_shape[0],
144 | width=image_shape[1], p=0.5),
145 | A.Rotate(limit=10, p=0.5),
146 | ]
147 | )
148 |
149 | # DATA INIT
150 | train_set = UniformedPatchesDataset(train_path,
151 | patch_shape,
152 | max_patches=train_patch_num,
153 | aug_flag=True,
154 | transform=composed_transform,
155 | noise_path=noise_path,
156 | noise_flag=True,
157 | )
158 | train_loader = torch.utils.data.DataLoader(
159 | train_set,
160 | batch_size=batch_size,
161 | shuffle=True,
162 | num_workers=workers,
163 | pin_memory=True,
164 | drop_last=False
165 | )
166 | with torch.no_grad():
167 | train_sample_batch = next(iter(train_loader))
168 | train_sample_batch = train_sample_batch.cpu()
169 | train_sample_batch_shape = train_sample_batch.shape
170 | train_sample_batch = train_sample_batch.view(train_sample_batch_shape[0] * train_sample_batch_shape[1],
171 | train_sample_batch_shape[2],
172 | train_sample_batch_shape[3],
173 | train_sample_batch_shape[4])
174 |
175 | val_set = UniformedPatchesDataset(
176 | val_path,
177 | patch_shape,
178 | max_patches=train_patch_num,
179 | aug_flag=False,
180 | transform=None,
181 | )
182 | val_loader = torch.utils.data.DataLoader(
183 | val_set,
184 | batch_size=1,
185 | shuffle=False,
186 | num_workers=workers,
187 | pin_memory=True
188 | )
189 |
190 | # Optimizer
191 | optimizer = torch.optim.Adam(model.parameters(), lr=lr)
192 | scheduler = ReduceLROnPlateau(optimizer, 'min', patience=max_patience, verbose=True)
193 |
194 | # TRAIN
195 | metrics = epoch_loop_patches(device, epochs, model, optimizer, param, scheduler, train_loader, val_loader)
196 |
197 | # TEST
198 | console.log("Training Completed, Testing Started")
199 |
200 | test_set = UniformedPatchesTestset(
201 | test_path,
202 | patch_shape=patch_shape,
203 | label_csv=test_labels_csv,
204 | max_patches=param.test_patches_number,
205 | )
206 | test_loader = torch.utils.data.DataLoader(
207 | test_set,
208 | batch_size=1,
209 | shuffle=False,
210 | num_workers=workers,
211 | pin_memory=True,
212 | drop_last=False
213 | )
214 | with torch.no_grad():
215 | test_sample_batch = next(iter(test_loader))["patches"]
216 | test_sample_batch = test_sample_batch.cpu()
217 | test_sample_batch_shape = test_sample_batch.shape
218 | test_sample_batch = test_sample_batch.view(test_sample_batch_shape[0] * test_sample_batch_shape[1],
219 | test_sample_batch_shape[2],
220 | test_sample_batch_shape[3],
221 | test_sample_batch_shape[4])
222 | df_dict = test_loop_df_rows(device, model, test_loader)
223 | test_set_df = pd.DataFrame.from_dict(df_dict)
224 |
225 | # Compute AUC fro ml flow logging
226 |
227 | uniformed_auc_computation_and_logging(test_set_df, param)
228 |
229 | # csv row building
230 | metrics_dict = {}
231 | for k, v in datasets_labels_names[param.dataset].items():
232 | auc_dict = per_label_metrics(test_set_df, k)
233 | if k == 0:
234 | v = "all"
235 | metrics_dict[f"{v}_mean_roc_auc_mse"] = auc_dict["mean_roc_auc"]["mse"]
236 | metrics_dict[f"{v}_mean_roc_auc_mae"] = auc_dict["mean_roc_auc"]["mae"]
237 | metrics_dict[f"{v}_mean_pr_auc_mse"] = auc_dict["mean_pr_auc"]["mse"]
238 | metrics_dict[f"{v}_mean_pr_auc_mae"] = auc_dict["mean_pr_auc"]["mae"]
239 | metrics_dict[f"{v}_q99_roc_auc_mse"] = auc_dict["q99_roc_auc"]["mse"]
240 | metrics_dict[f"{v}_q99_roc_auc_mae"] = auc_dict["q99_roc_auc"]["mae"]
241 | metrics_dict[f"{v}_q99_pr_auc_mse"] = auc_dict["q99_pr_auc"]["mse"]
242 | metrics_dict[f"{v}_q99_pr_auc_mae"] = auc_dict["q99_pr_auc"]["mae"]
243 | metrics_dict[f"{v}_std_roc_auc_mse"] = auc_dict["std_roc_auc"]["mse"]
244 | metrics_dict[f"{v}_std_roc_auc_mae"] = auc_dict["std_roc_auc"]["mae"]
245 | metrics_dict[f"{v}_std_pr_auc_mse"] = auc_dict["std_pr_auc"]["mse"]
246 | metrics_dict[f"{v}_std_pr_auc_mae"] = auc_dict["std_pr_auc"]["mae"]
247 |
248 | bfs_key = f"B{bottleneck}F{layer_1_ft}S{available_scale_levels[param.scale_level]}"
249 | csv_row = {
250 | **{"BFS": bfs_key,
251 | "dataset": param.dataset,
252 | "bottleneck": bottleneck,
253 | "first layer size": layer_1_ft,
254 | "scale level": available_scale_levels[param.scale_level],
255 | },
256 | **metrics_dict,
257 | }
258 | # SAVE STUFF
259 | console.log("Testing Completed")
260 | check_create_folder(model_save_folder)
261 | console.log("Creating and saving Artifacts")
262 | artifacts_path = model_save_folder + "/artifacts/"
263 | uniformed_model_artifact_saver(
264 | batch_size,
265 | bottleneck,
266 | epochs,
267 | layer_1_ft,
268 | lr,
269 | metrics,
270 | test_set_df,
271 | losses_list,
272 | model,
273 | input_channel,
274 | patch_shape,
275 | artifacts_path,
276 | ml_flow_run_id,
277 | param.id_optimized_loss,
278 | param=param,
279 | csv_row=csv_row,
280 | csv_key=f"d_{param.dataset}_{bfs_key}",
281 | train_sample_batch=train_sample_batch,
282 | test_sample_batch=test_sample_batch,
283 | )
284 | console.log(f"Script completed, artifacts located at {model_save_folder}."
285 | f" Videos and plots are skipped because still in todo ")
286 |
287 |
288 | def uniformed_auc_computation_and_logging(test_df, param):
289 | label_unique_values = [0 if el[0] == 0 else 1 for el in
290 | test_df[["frame_id", "frame_label"]].groupby("frame_id")["frame_label"].unique().values]
291 | metrics_dict = compute_uniformed_model_metrics(
292 | label_unique_values,
293 | losses_list, test_df[["frame_id", "mse_loss", "mae_loss"]])
294 | if param.use_ml_flow:
295 | mlflow.log_metric(f'test_set_q99_roc_auc_{losses_list[0]}', metrics_dict["q99_roc_auc"][losses_list[0]])
296 | mlflow.log_metric(f'test_set_q99_roc_auc_{losses_list[1]}', metrics_dict["q99_roc_auc"][losses_list[1]])
297 | mlflow.log_metric(f'test_set_mean_roc_auc_{losses_list[0]}', metrics_dict["mean_roc_auc"][losses_list[0]])
298 | mlflow.log_metric(f'test_set_mean_roc_auc_{losses_list[1]}', metrics_dict["mean_roc_auc"][losses_list[1]])
299 | mlflow.log_metric(f'test_set_q99_pr_auc_{losses_list[0]}', metrics_dict["q99_pr_auc"][losses_list[0]])
300 | mlflow.log_metric(f'test_set_q99_pr_auc_{losses_list[1]}', metrics_dict["q99_pr_auc"][losses_list[1]])
301 | mlflow.log_metric(f'test_set_mean_pr_auc_{losses_list[0]}', metrics_dict["mean_pr_auc"][losses_list[0]])
302 | mlflow.log_metric(f'test_set_mean_pr_auc_{losses_list[1]}', metrics_dict["mean_pr_auc"][losses_list[1]])
303 | else:
304 | print(f'test_set_q99_roc_auc_{losses_list[0]} = {metrics_dict["q99_roc_auc"][losses_list[0]]}\n'
305 | f'test_set_q99_roc_auc_{losses_list[1]} = {metrics_dict["q99_roc_auc"][losses_list[1]]}\n'
306 | f'test_set_mean_roc_auc_{losses_list[0]} = {metrics_dict["mean_roc_auc"][losses_list[0]]}\n'
307 | f'test_set_mean_roc_auc_{losses_list[1]} = {metrics_dict["mean_roc_auc"][losses_list[1]]}\n'
308 | f'test_set_q99_pr_auc_{losses_list[0]} = {metrics_dict["q99_pr_auc"][losses_list[0]]}\n'
309 | f'test_set_q99_pr_auc_{losses_list[1]} = {metrics_dict["q99_pr_auc"][losses_list[1]]}\n'
310 | f'test_set_mean_pr_auc_{losses_list[0]} = {metrics_dict["mean_pr_auc"][losses_list[0]]}\n'
311 | f'test_set_mean_pr_auc_{losses_list[1]} = {metrics_dict["mean_pr_auc"][losses_list[1]]}\n'
312 | )
313 |
314 |
315 | if __name__ == "__main__":
316 | # with torch.autograd.detect_anomaly():
317 | # main()
318 | main()
319 |
--------------------------------------------------------------------------------
91 | 
125 | 
130 | 
136 | 
142 | 
154 | 
161 | 
167 | 
178 | 
179 | 
180 | 
186 | 
192 | 
198 | 
204 | 
210 | 
216 | 
222 | 
228 | 
234 | 
240 | 
246 | 
252 | 
258 | 
264 | 
270 |