├── .gitignore ├── README.md ├── DNFNet ├── Competitions │ ├── EyeMovements │ │ └── CompetitionConfig.py │ ├── House │ │ └── CompetitionConfig.py │ ├── SantanderTransaction │ │ └── CompetitionConfig.py │ ├── Gas │ │ └── CompetitionConfig.py │ ├── RobotNavigation │ │ └── CompetitionConfig.py │ ├── GesturePhase │ │ └── CompetitionConfig.py │ └── Otto │ │ └── CompetitionConfig.py ├── XGBModelHandler.py ├── DNFNetModels │ ├── model100.py │ ├── model2.py │ ├── model101.py │ ├── model3.py │ ├── model6.py │ ├── model5.py │ ├── model4.py │ ├── model1.py │ └── DNFNetComponents.py ├── FeatureSelectionSynExp.py ├── AblationStudy.py ├── CompetitionsHandler.py └── ModelHandler.py ├── Utils ├── metrics.py ├── NumpyGenerator.py ├── experiment_utils.py ├── file_utils.py └── grid_search_utils.py └── data └── data_utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DisjunctiveNormalFormNet 2 | 3 | **Dependencies:** 4 | 5 | 1. Python 3.6 6 | 2. ternsorflow-gpu 1.12.0 7 | 3. xgboost 0.90 8 | 4. numpy 1.16.2 9 | 5. scipy 1.2.1 10 | 6. pandas 0.24.2 11 | -------------------------------------------------------------------------------- /DNFNet/Competitions/EyeMovements/CompetitionConfig.py: -------------------------------------------------------------------------------- 1 | from Utils.metrics import multiclass_log_loss 2 | 3 | config = { 4 | 'input_dim': 26, 5 | 'output_dim': 3, 6 | 'translate_label_to_one_hot': True, 7 | 'csv': 'PATH_TO_DATA/data/OpenML/eye_movement/eye_movement.csv', 8 | 9 | 'XGB_objective': 'multi:softmax', 10 | } 11 | 12 | score_config = { 13 | 'score_metric': multiclass_log_loss, 14 | 'score_increases': False, 15 | 'XGB_eval_metric': 'mlogloss', 16 | } 17 | 18 | 19 | def get_configs(): 20 | return config, score_config 21 | 22 | 23 | def dataset_handler(df): 24 | df = df.drop('lineNo', axis=1) 25 | return df 26 | 27 | -------------------------------------------------------------------------------- /DNFNet/Competitions/House/CompetitionConfig.py: -------------------------------------------------------------------------------- 1 | from sklearn.metrics import roc_auc_score 2 | 3 | config = { 4 | 'input_dim': 16, 5 | 'output_dim': 1, 6 | 'translate_label_to_one_hot': False, 7 | 'csv': 'PATH_TO_DATA/data/OpenML/House/house.csv', 8 | 9 | 'XGB_objective': "binary:logistic", 10 | } 11 | 12 | score_config = { 13 | 'score_metric': roc_auc_score, 14 | 'score_increases': True, 15 | 'XGB_eval_metric': 'auc', 16 | } 17 | 18 | 19 | def get_configs(): 20 | return config, score_config 21 | 22 | 23 | def dataset_handler(df): 24 | map_labels = { 25 | "N": 0, 26 | "P": 1, 27 | } 28 | df = df.replace({'binaryClass': map_labels}) 29 | return df 30 | -------------------------------------------------------------------------------- /DNFNet/Competitions/SantanderTransaction/CompetitionConfig.py: -------------------------------------------------------------------------------- 1 | from sklearn.metrics import roc_auc_score 2 | 3 | config = { 4 | 'input_dim': 200, 5 | 'output_dim': 1, 6 | 'translate_label_to_one_hot': False, 7 | 'csv': 'PATH_TO_DATA/data/Santander_transaction/train.csv', 8 | 9 | 'XGB_objective': "binary:logistic", 10 | } 11 | 12 | score_config = { 13 | 'score_metric': roc_auc_score, 14 | 'score_increases': True, 15 | 'XGB_eval_metric': 'auc', 16 | } 17 | 18 | 19 | def get_configs(): 20 | return config, score_config 21 | 22 | 23 | def dataset_handler(df): 24 | df = df.drop('ID_code', axis=1) 25 | col = df.columns.tolist() 26 | col = col[1:] + [col[0]] 27 | df = df[col] 28 | return df -------------------------------------------------------------------------------- /DNFNet/Competitions/Gas/CompetitionConfig.py: -------------------------------------------------------------------------------- 1 | from Utils.metrics import multiclass_log_loss 2 | 3 | config = { 4 | 'input_dim': 129, 5 | 'output_dim': 6, 6 | 'translate_label_to_one_hot': True, 7 | 'csv': 'PATH_TO_DATA/data/OpenML/Gas/gas.csv', 8 | 9 | 'XGB_objective': 'multi:softmax', 10 | } 11 | 12 | score_config = { 13 | 'score_metric': multiclass_log_loss, 14 | 'score_increases': False, 15 | 'XGB_eval_metric': 'mlogloss', 16 | } 17 | 18 | 19 | def get_configs(): 20 | return config, score_config 21 | 22 | 23 | def dataset_handler(df): 24 | map_labels = { 25 | 1: 0, 26 | 2: 1, 27 | 3: 2, 28 | 4: 3, 29 | 5: 4, 30 | 6: 5 31 | } 32 | df['Class'] = df['Class'].map(map_labels) 33 | return df -------------------------------------------------------------------------------- /DNFNet/Competitions/RobotNavigation/CompetitionConfig.py: -------------------------------------------------------------------------------- 1 | from Utils.metrics import multiclass_log_loss 2 | 3 | config = { 4 | 'input_dim': 24, 5 | 'output_dim': 4, 6 | 'translate_label_to_one_hot': True, 7 | 'csv': 'PATH_TO_DATA/data/OpenML/RobotNavigation/RobotNavigation.csv', 8 | 9 | 'XGB_objective': 'multi:softmax', 10 | } 11 | 12 | score_config = { 13 | 'score_metric': multiclass_log_loss, 14 | 'score_increases': False, 15 | 'XGB_eval_metric': 'mlogloss', 16 | } 17 | 18 | 19 | def get_configs(): 20 | return config, score_config 21 | 22 | 23 | def dataset_handler(df): 24 | map_labels = { 25 | 1: 0, 26 | 2: 1, 27 | 3: 2, 28 | 4: 3, 29 | } 30 | df['Class'] = df['Class'].map(map_labels) 31 | return df -------------------------------------------------------------------------------- /DNFNet/Competitions/GesturePhase/CompetitionConfig.py: -------------------------------------------------------------------------------- 1 | from Utils.metrics import multiclass_log_loss 2 | 3 | config = { 4 | 'input_dim': 32, 5 | 'output_dim': 5, 6 | 'translate_label_to_one_hot': True, 7 | 'csv': 'PATH_TO_DATA/data/OpenML/gesture_phase/gesture_phase.csv', 8 | 9 | 'XGB_objective': 'multi:softmax', 10 | } 11 | 12 | score_config = { 13 | 'score_metric': multiclass_log_loss, 14 | 'score_increases': False, 15 | 'XGB_eval_metric': 'mlogloss', 16 | } 17 | 18 | 19 | def get_configs(): 20 | return config, score_config 21 | 22 | 23 | def dataset_handler(df): 24 | map_labels = { 25 | "'D'": 0, 26 | "'P'": 1, 27 | "'S'": 2, 28 | "'H'": 3, 29 | "'R'": 4, 30 | } 31 | df = df.replace({'Phase': map_labels}) 32 | return df 33 | -------------------------------------------------------------------------------- /DNFNet/Competitions/Otto/CompetitionConfig.py: -------------------------------------------------------------------------------- 1 | from Utils.metrics import multiclass_log_loss 2 | 3 | config = { 4 | 'input_dim': 93, 5 | 'output_dim': 9, 6 | 'translate_label_to_one_hot': True, 7 | 'csv': 'PATH_TO_DATA/data/Otto/train.csv', 8 | 9 | 'XGB_objective': 'multi:softmax', 10 | } 11 | 12 | score_config = { 13 | 'score_metric': multiclass_log_loss, 14 | 'score_increases': False, 15 | 'XGB_eval_metric': 'mlogloss', 16 | } 17 | 18 | 19 | def get_configs(): 20 | return config, score_config 21 | 22 | 23 | def dataset_handler(df): 24 | map_labels = { 25 | 'Class_1': 0, 26 | 'Class_2': 1, 27 | 'Class_3': 2, 28 | 'Class_4': 3, 29 | 'Class_5': 4, 30 | 'Class_6': 5, 31 | 'Class_7': 6, 32 | 'Class_8': 7, 33 | 'Class_9': 8, 34 | } 35 | df = df.replace({'target': map_labels}) 36 | df = df.drop('id', axis=1) 37 | return df -------------------------------------------------------------------------------- /Utils/metrics.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def multiclass_log_loss(y_true, y_pred, eps=1e-15): 5 | "based on this post: https://www.kaggle.com/c/predict-closed-questions-on-stack-overflow/discussion/2644" 6 | 7 | 8 | """Multi class version of Logarithmic Loss metric. 9 | https://www.kaggle.com/wiki/MultiClassLogLoss 10 | 11 | idea from this post: 12 | http://www.kaggle.com/c/emc-data-science/forums/t/2149/is-anyone-noticing-difference-betwen-validation-and-leaderboard-error/12209#post12209 13 | 14 | Parameters 15 | ---------- 16 | y_true : array, shape = [n_samples, n_classes] one hot vectors 17 | y_pred : array, shape = [n_samples, n_classes] 18 | 19 | Returns 20 | ------- 21 | loss : float 22 | """ 23 | predictions = np.clip(y_pred, eps, 1 - eps) 24 | 25 | # normalize row sums to 1 26 | predictions /= predictions.sum(axis=1)[:, np.newaxis] 27 | 28 | # actual = np.zeros(y_pred.shape) 29 | # rows = actual.shape[0] 30 | # actual[np.arange(rows), y_true.astype(int)] = 1 31 | n_samples = y_true.shape[0] 32 | vsota = np.sum(y_true * np.log(predictions)) 33 | return (-1.0 / n_samples) * vsota 34 | -------------------------------------------------------------------------------- /DNFNet/XGBModelHandler.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import tensorflow as tf 4 | from xgboost import XGBClassifier 5 | 6 | 7 | class XGBModelHandler: 8 | @staticmethod 9 | def train_and_test(config, data, score_config): 10 | print('train XGBoost') 11 | os.environ["CUDA_VISIBLE_DEVICES"] = config['GPU'] 12 | score_metric = score_config['score_metric'] 13 | eval_metric = score_config['XGB_eval_metric'] 14 | np.random.seed(seed=config['random_seed']) 15 | 16 | model = XGBClassifier(learning_rate=config['XGB_learning_rate'], 17 | n_estimators=config['XGB_n_estimators'], 18 | colsample_bytree=config['XGB_colsample_bytree'], 19 | subsample=config['XGB_subsample'], 20 | max_depth=config['XGB_max_depth'], 21 | objective=config['XGB_objective'], 22 | random_state=config['random_seed'], 23 | gpu_id=0, 24 | predictor='gpu_predictor', 25 | tree_method='gpu_hist') 26 | 27 | eval_set = [(data['X_val'], data['Y_val'])] 28 | trained_model = model.fit(data['X_train'], data['Y_train'], eval_set=eval_set, eval_metric=eval_metric, verbose=False, early_stopping_rounds=50) 29 | 30 | if config['output_dim'] > 1: 31 | y_val = tf.keras.utils.to_categorical(data['Y_val'], num_classes=config['output_dim']) 32 | y_test = tf.keras.utils.to_categorical(data['Y_test'], num_classes=config['output_dim']) 33 | else: 34 | y_val = data['Y_val'] 35 | y_test = data['Y_test'] 36 | 37 | y_val_pred = trained_model.predict_proba(data['X_val']) 38 | if config['output_dim'] == 1: 39 | y_val_pred = y_val_pred[:, 1] 40 | validation_score = score_metric(y_val, y_val_pred) 41 | 42 | y_test_pred = trained_model.predict_proba(data['X_test']) 43 | if config['output_dim'] == 1: 44 | y_test_pred = y_test_pred[:, 1] 45 | test_score = score_metric(y_test, y_test_pred) 46 | 47 | n_epochs = trained_model.get_booster().best_iteration 48 | 49 | return {'test_score': test_score, 'validation_score': validation_score, 'n_epochs': n_epochs} 50 | -------------------------------------------------------------------------------- /Utils/NumpyGenerator.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | 4 | 5 | class NumpyGenerator(object): 6 | def __init__(self, x, y, output_dim, batch_size, scale=None, translate_label_to_one_hot=True, shuffle=True, copy_dataset=True): 7 | self.batch_size = batch_size 8 | self.scale = scale 9 | self.output_dim = output_dim 10 | self.shuffle = shuffle 11 | self.translate_label_to_one_hot = translate_label_to_one_hot 12 | self.index = 0 13 | self.epochs = 0 14 | self.dataset_size = x.shape[0] 15 | self.indices = np.arange(x.shape[0]) 16 | 17 | if copy_dataset: 18 | self.x = np.copy(x) 19 | self.y = np.copy(y) 20 | else: 21 | self.x = x 22 | self.y = y 23 | 24 | if self.shuffle: 25 | self._shuffle_dataset() 26 | 27 | def _get_values_by_range(self, begin, end): 28 | batch_ix = self.indices[begin: end] 29 | x_batch = self.x[batch_ix].copy() 30 | y_batch = self.y[batch_ix].copy() 31 | 32 | if self.translate_label_to_one_hot: 33 | y_batch = tf.keras.utils.to_categorical(y_batch, num_classes=self.output_dim) 34 | elif self.output_dim == 1: 35 | y_batch = np.expand_dims(y_batch, axis=1) 36 | 37 | if self.scale is not None: 38 | x_batch = self.scale(x_batch) 39 | 40 | x_batch = x_batch.astype('float32') 41 | return x_batch, y_batch 42 | 43 | def _shuffle_dataset(self): 44 | np.random.shuffle(self.indices) 45 | 46 | def _build_feed_dict(self, x, y): 47 | feed_dict = dict() 48 | feed_dict['x'] = x 49 | feed_dict['y'] = y 50 | return feed_dict 51 | 52 | def reset(self): 53 | self.index = 0 54 | self.epochs = 0 55 | 56 | def get_epochs(self): 57 | return self.epochs 58 | 59 | def get_dataset_size(self): 60 | return self.dataset_size 61 | 62 | def get_batch(self): 63 | batch_size = self.batch_size 64 | if self.index + batch_size < self.dataset_size: 65 | batch_features, batch_labels = self._get_values_by_range(self.index, self.index + batch_size) 66 | self.index += batch_size 67 | else: 68 | batch_features, batch_labels = self._get_values_by_range(self.index, self.dataset_size) 69 | self.index = 0 70 | self.epochs += 1 71 | if self.shuffle: 72 | self._shuffle_dataset() 73 | return self._build_feed_dict(batch_features, batch_labels) -------------------------------------------------------------------------------- /DNFNet/DNFNetModels/model100.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | 4 | def fetch_placeholders(placeholders): 5 | x = placeholders['x'] 6 | y = placeholders['y'] 7 | lr = placeholders['lr'] 8 | dropout_rate = placeholders['dropout_rate'] 9 | is_training = placeholders['is_training'] 10 | return x, y, lr, dropout_rate, is_training 11 | 12 | 13 | ########################################################################################### 14 | # FCN 15 | ########################################################################################### 16 | class MNN(object): 17 | def __init__(self, config): 18 | self.config = config 19 | self.loaded_input_masks = None 20 | 21 | def build(self, placeholders, n_labels, phase): 22 | x, y, lr, _, is_training = fetch_placeholders(placeholders) 23 | 24 | net_description = '#'*50 + '\n' 25 | out = x 26 | for layer_width in self.config['FCN_layers']: 27 | in_shape = out.get_shape()[1].value 28 | net_description += '{} -> {} \n'.format(in_shape, layer_width) 29 | 30 | out = tf.layers.dense(out, layer_width, activation=tf.nn.relu, kernel_regularizer=tf.contrib.layers.l2_regularizer(self.config['FCN_l2_lambda']), kernel_initializer=tf.contrib.layers.xavier_initializer()) 31 | out = tf.layers.dropout(inputs=out, rate=self.config['dropout_rate'], training=is_training) 32 | 33 | in_shape = out.get_shape()[1].value 34 | net_description += '{} -> {} \n'.format(in_shape, n_labels) 35 | net_description += '#' * 50 + '\n' 36 | print(net_description) 37 | 38 | output = tf.layers.dense(out, n_labels, activation=None, kernel_initializer=tf.contrib.layers.xavier_initializer()) 39 | 40 | with tf.variable_scope('loss'): 41 | if n_labels > 1: 42 | loss = tf.losses.softmax_cross_entropy(onehot_labels=y, logits=output) 43 | output = tf.nn.softmax(output) 44 | else: 45 | loss = tf.losses.sigmoid_cross_entropy(multi_class_labels=y, logits=output) 46 | output = tf.nn.sigmoid(output) 47 | 48 | loss += tf.losses.get_regularization_loss() 49 | 50 | if phase == 'train': 51 | print('Build optimizer') 52 | optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss) 53 | else: 54 | optimizer = None 55 | 56 | build_dict = {'optimizer': optimizer, 'loss': loss, 'output': output, 'features': None, 'input_masks': None} 57 | return build_dict 58 | 59 | -------------------------------------------------------------------------------- /DNFNet/DNFNetModels/model2.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from DNFNet.DNFNetModels.DNFNetComponents import compute_total_number_of_literals, compute_total_number_of_conjunctions 3 | 4 | 5 | def fetch_placeholders(placeholders): 6 | x = placeholders['x'] 7 | y = placeholders['y'] 8 | lr = placeholders['lr'] 9 | return x, y, lr 10 | 11 | 12 | ########################################################################################### 13 | # Fully trained FCN 14 | ########################################################################################### 15 | class MNN(object): 16 | def __init__(self, config): 17 | self.config = config 18 | self.loaded_input_masks = None 19 | 20 | def build(self, placeholders, n_labels, phase): 21 | x, y, lr = fetch_placeholders(placeholders) 22 | 23 | n_conjunctions_arr = self.config['n_conjunctions_arr'] 24 | conjunctions_depth_arr = self.config['conjunctions_depth_arr'] 25 | n_formulas = self.config['n_formulas'] 26 | total_number_of_literals = compute_total_number_of_literals(n_formulas, n_conjunctions_arr, conjunctions_depth_arr) 27 | total_number_of_conjunctions = compute_total_number_of_conjunctions(n_formulas, n_conjunctions_arr) 28 | 29 | out = tf.layers.dense(x, total_number_of_literals, activation=tf.tanh, kernel_initializer=tf.contrib.layers.xavier_initializer()) 30 | out = tf.layers.dense(out, total_number_of_conjunctions, activation=tf.tanh, kernel_initializer=tf.contrib.layers.xavier_initializer()) 31 | out = tf.layers.dense(out, n_formulas, activation=tf.tanh, kernel_initializer=tf.contrib.layers.xavier_initializer()) 32 | output = tf.layers.dense(out, n_labels, activation=None, kernel_initializer=tf.contrib.layers.xavier_initializer()) 33 | 34 | with tf.variable_scope('loss'): 35 | if n_labels > 1: 36 | loss = tf.losses.softmax_cross_entropy(onehot_labels=y, logits=output) 37 | output = tf.nn.softmax(output) 38 | else: 39 | loss = tf.losses.sigmoid_cross_entropy(multi_class_labels=y, logits=output) 40 | output = tf.nn.sigmoid(output) 41 | 42 | loss += tf.losses.get_regularization_loss() 43 | 44 | if phase == 'train': 45 | print('Build optimizer') 46 | optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss) 47 | else: 48 | optimizer = None 49 | 50 | build_dict = {'optimizer': optimizer, 'loss': loss, 'output': output, 'features': None, 'input_masks': None} 51 | return build_dict 52 | 53 | -------------------------------------------------------------------------------- /Utils/experiment_utils.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | import os 3 | import copy 4 | import numpy as np 5 | from shutil import copyfile 6 | import pandas as pd 7 | from Utils.file_utils import create_dir, write_dict_to_file, read_dict_from_file, delete_file 8 | 9 | 10 | def write_config(config, run_dir): 11 | d = copy.deepcopy(config) 12 | if 'loc_dict' in d: 13 | del d['loc_dict'] 14 | if 'mask' in d: 15 | d['mask'] = list(d['mask']) 16 | write_dict_to_file(d, run_dir) 17 | 18 | 19 | def create_experiment_directory(config, copy_model=True, return_sub_dirs=False): 20 | experiment_dir = config['experiments_dir'] + '/{}'.format(config['experiment_number']) 21 | weights_dir = os.path.join(experiment_dir, "weights") 22 | logs_dir = os.path.join(experiment_dir, "logs") 23 | create_dir(experiment_dir) 24 | create_dir(weights_dir) 25 | create_dir(logs_dir) 26 | write_config(config, experiment_dir + '/configuration.json') 27 | if copy_model: 28 | copyfile(config['models_dir'] + '/model{}.py'.format(config['model_number']), experiment_dir + '/model.py') 29 | 30 | if return_sub_dirs: 31 | return experiment_dir, weights_dir, logs_dir 32 | else: 33 | return experiment_dir 34 | 35 | 36 | def create_model(config, models_module_name): 37 | full_class_name = "{}.model{}".format(models_module_name, config['model_number']) 38 | module = importlib.import_module(full_class_name) 39 | model_class = getattr(module, "MNN") 40 | return model_class(config) 41 | 42 | 43 | def read_config(config_path): 44 | config = read_dict_from_file(config_path) 45 | return config 46 | 47 | 48 | def load_config_from_experiment_dir(config): 49 | experiment_dir = config['experiments_dir'] + '/{}'.format(config['experiment_number']) 50 | loaded_config = read_config(experiment_dir + '/configuration.json') 51 | return loaded_config, experiment_dir 52 | 53 | 54 | def load_config(experiment_number, experiments_dir): 55 | experiment_dir = experiments_dir + '/{}'.format(experiment_number) 56 | loaded_config = read_config(experiment_dir + '/configuration.json') 57 | return loaded_config 58 | 59 | 60 | def create_all_FCN_layers_with_fixed_width(depth_arr, width_arr): 61 | layers = [] 62 | for d in depth_arr: 63 | for w in width_arr: 64 | layers.append([w] * d) 65 | return layers 66 | 67 | 68 | def create_all_FCN_layers_with_factor_2_reduction(depth_arr, width_arr): 69 | layers = [] 70 | for initial_w in width_arr: 71 | for d in depth_arr: 72 | net = [] 73 | for i in range(d): 74 | net.append(int(initial_w / (2**i))) 75 | layers.append(net) 76 | return layers 77 | 78 | 79 | def create_all_FCN_layers_grid(depth_arr, width_arr): 80 | depth_arr_copy = copy.deepcopy(depth_arr) 81 | if 1 in depth_arr_copy: 82 | depth_arr_copy.remove(1) 83 | return create_all_FCN_layers_with_fixed_width(depth_arr_copy, width_arr) + create_all_FCN_layers_with_factor_2_reduction(depth_arr, width_arr) 84 | -------------------------------------------------------------------------------- /Utils/file_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import errno 3 | import scipy.io 4 | import shutil 5 | import json 6 | from os.path import isfile, join, isdir 7 | 8 | 9 | def dir_to_file_list(path_to_dir): 10 | return [path_to_dir + '/' + f for f in os.listdir(path_to_dir) if isfile(join(path_to_dir, f))] 11 | 12 | 13 | def dir_to_file_names_list(path_to_dir): 14 | return [f for f in os.listdir(path_to_dir) if isfile(join(path_to_dir, f))] 15 | 16 | 17 | def dir_to_file_list_with_ext(path_to_dir, ext): 18 | return list(filter(lambda x: x.endswith(ext), dir_to_file_list(path_to_dir))) 19 | 20 | 21 | def dir_to_subdir_list(path_to_dir): 22 | return [path_to_dir + '/' + f for f in os.listdir(path_to_dir) if isdir(join(path_to_dir, f))] 23 | 24 | 25 | def create_dir(path): 26 | if os.path.exists(path): 27 | return 28 | 29 | if not os.path.isdir(path): 30 | try: 31 | os.makedirs(path) 32 | except OSError as e: 33 | if e.errno != errno.EEXIST: 34 | raise 35 | 36 | 37 | def delete_dir(path): 38 | if os.path.exists(path) and os.path.isdir(path): 39 | shutil.rmtree(path) 40 | 41 | 42 | def delete_file(path): 43 | try: 44 | if os.path.exists(path): 45 | os.remove(path) 46 | except OSError as e: 47 | if e.errno != errno.ENOENT: 48 | raise 49 | 50 | 51 | def get_file_name(path): 52 | path = path.replace('\\', '/') 53 | return path.split('/')[-1] 54 | 55 | 56 | def get_file_name_without_ext(path): 57 | name_with_ext = get_file_name(path) 58 | return name_with_ext.split(".")[0] 59 | 60 | 61 | def read_list_from_file(file_path): 62 | with open(file_path, "r") as file: 63 | l = [] 64 | for line in file: 65 | l.append(line.rstrip()) 66 | return l 67 | 68 | 69 | def file_to_path_list(file_path): 70 | with open(file_path, "r") as file: 71 | l = [] 72 | for line in file: 73 | l.append(line.rstrip()) 74 | return l 75 | 76 | 77 | def write_list_to_file(data_list, file_path): 78 | with open(file_path, "a") as file: 79 | for item in data_list: 80 | file.write(str(item) + "\n") 81 | 82 | 83 | def write_dict_to_file(data_dict, file_path): 84 | with open(file_path, 'w') as file: 85 | file.write(json.dumps(data_dict, sort_keys=False, indent=4, separators=(',', ': '))) 86 | 87 | 88 | def read_dict_from_file(file_path): 89 | with open(file_path) as f: 90 | return json.load(f) 91 | 92 | 93 | def sort_files(file_list): 94 | file_list = [(int(get_file_name_without_ext(x)), x) for x in file_list] 95 | file_list = sorted(file_list, key=lambda x: x[0]) 96 | file_list = [x[1] for x in file_list] 97 | return file_list 98 | 99 | 100 | def read_mat(file): 101 | return scipy.io.loadmat(file) 102 | 103 | 104 | def write_mat(file, mat): 105 | ''' 106 | :param mat: dictionary of names and arrays 107 | ''' 108 | return scipy.io.savemat(file, mat) -------------------------------------------------------------------------------- /DNFNet/FeatureSelectionSynExp.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | from DNFNet.ModelHandler import ModelHandler 4 | from Utils.file_utils import create_dir 5 | from Utils.grid_search_utils import cross_validation 6 | from data.data_utils import Ground_Truth_Mask_Generation 7 | from sklearn.metrics import accuracy_score 8 | 9 | 10 | def accuracy_wrapper(y_true, y_pred): 11 | y_pred_01 = np.zeros_like(y_true) 12 | y_pred_01[y_pred > 0.5] = 1 13 | return accuracy_score(y_true, y_pred_01) 14 | 15 | 16 | config = { 17 | 'model_number': 101, 18 | 'model_name': 'FeatureSelectionSynExp', 19 | 20 | 'output_dim': 1, 21 | 'translate_label_to_one_hot': False, 22 | 23 | 'initial_lr': 1e-3, 24 | 'lr_decay_factor': 0.5, 25 | 'lr_patience': 10, 26 | 'min_lr': 1e-6, 27 | 28 | 'early_stopping_patience': 30, 29 | 'epochs': 1000, 30 | 'batch_size': 256, 31 | 32 | 'apply_standardization': True, 33 | 34 | 'save_weights': True, 35 | 'starting_epoch_to_save': 0, 36 | 'models_module_name': 'DNFNet.DNFNetModels', 37 | 'models_dir': './DNFNetModels', 38 | } 39 | 40 | score_config = { 41 | 'score_metric': accuracy_wrapper, 42 | 'score_increases': True, 43 | } 44 | 45 | batch_size = 256 46 | FCN_grid = {'batch_size': [batch_size]} 47 | FCN_with_oracle_mask_grid = {'batch_size': [batch_size]} 48 | FCN_with_feature_selection_grid = {'batch_size': [batch_size], 'elastic_net_beta': [1.3, 1., 0.7, 0.4]} 49 | 50 | grid_map = {'FCN': FCN_grid, 51 | 'FCN_with_oracle_mask': FCN_with_oracle_mask_grid, 52 | 'FCN_with_feature_selection': FCN_with_feature_selection_grid} 53 | 54 | if __name__ == '__main__': 55 | base_dir = 'path/to/base/dir' 56 | model_type_arr = ['FCN', 'FCN_with_oracle_mask', 'FCN_with_feature_selection'] 57 | syn_names = ['Syn1', 'Syn2', 'Syn3', 'Syn4', 'Syn5', 'Syn6'] 58 | d_arr = [11, 50, 100, 150, 200, 250, 300] 59 | folds = [0, 1, 2, 3, 4] 60 | gpus_list = ['0'] 61 | seeds_arr = [1, 2, 3] 62 | 63 | for syn in syn_names: 64 | for model_type in model_type_arr: 65 | for d in d_arr: 66 | experiment_name = 'exp_{}_model_{}_d_{}'.format(syn, model_type, d) 67 | config['experiments_dir'] = '{}/FeatureSelectionSynExp/experiments/{}'.format(base_dir, experiment_name) 68 | config['csv'] = 'data/FeatureSelectionSynExp/{}_{}/data.csv'.format(syn, d) 69 | config['input_dim'] = d 70 | config['mask'] = Ground_Truth_Mask_Generation(d, syn) 71 | config['model_type'] = model_type 72 | 73 | output_dir = os.path.join(config['experiments_dir'], 'grid_search') 74 | create_dir(config['experiments_dir']) 75 | 76 | cross_validation(config, ModelHandler, score_config, 77 | folds=folds, 78 | gpus_list=gpus_list, 79 | grid_params=grid_map[model_type], 80 | output_dir=output_dir, 81 | seeds_arr=seeds_arr) 82 | -------------------------------------------------------------------------------- /DNFNet/DNFNetModels/model101.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from DNFNet.DNFNetModels.DNFNetComponents import binary_threshold 3 | 4 | 5 | def fetch_placeholders(placeholders): 6 | x = placeholders['x'] 7 | y = placeholders['y'] 8 | lr = placeholders['lr'] 9 | return x, y, lr 10 | 11 | 12 | ########################################################################################### 13 | # FCN - syn experiment 14 | ########################################################################################### 15 | class MNN(object): 16 | def __init__(self, config): 17 | self.config = config 18 | 19 | def build(self, placeholders, n_labels, phase): 20 | x, y, lr = fetch_placeholders(placeholders) 21 | mask = self.config['mask'] 22 | elastic_net_reg = None 23 | 24 | if self.config['model_type'] == 'FCN_with_oracle_mask': 25 | m = tf.constant(mask, dtype='float32') 26 | x_i = tf.multiply(x, m) 27 | elif self.config['model_type'] == 'FCN_with_feature_selection': 28 | binary_threshold_eps = 1. 29 | feature_selector = tf.get_variable('feature_selector', [self.config['input_dim']], initializer=tf.initializers.constant(binary_threshold_eps + 0.5)) 30 | feature_selector_01 = binary_threshold(feature_selector, eps=binary_threshold_eps) 31 | x_i = tf.multiply(feature_selector_01, x) 32 | 33 | elastic_net_alpha = tf.get_variable('elastic_net_alpha', initializer=tf.constant(value=0.)) 34 | l2 = tf.abs(tf.div(tf.square(tf.norm(feature_selector, ord=2)), self.config['input_dim']) - self.config['elastic_net_beta'] * binary_threshold_eps**2) 35 | l1 = tf.abs(tf.div(tf.norm(feature_selector, ord=1), self.config['input_dim']) - self.config['elastic_net_beta'] * binary_threshold_eps) 36 | elastic_net_reg = l2 * ((1 - tf.nn.sigmoid(elastic_net_alpha)) / 2) + l1 * tf.nn.sigmoid(elastic_net_alpha) 37 | elif self.config['model_type'] == 'FCN': 38 | x_i = x 39 | else: 40 | raise Exception('ERROR: in model_type') 41 | 42 | out = tf.layers.dense(x_i, 64, activation=tf.nn.relu, kernel_initializer=tf.contrib.layers.xavier_initializer()) 43 | out = tf.layers.dense(out, 32, activation=tf.nn.relu, kernel_initializer=tf.contrib.layers.xavier_initializer()) 44 | output = tf.layers.dense(out, n_labels, activation=None, kernel_initializer=tf.contrib.layers.xavier_initializer()) 45 | 46 | with tf.variable_scope('loss'): 47 | loss = tf.losses.sigmoid_cross_entropy(multi_class_labels=y, logits=output) 48 | output = tf.nn.sigmoid(output) 49 | loss += tf.losses.get_regularization_loss() 50 | 51 | if elastic_net_reg is not None: 52 | loss += elastic_net_reg 53 | 54 | if phase == 'train': 55 | print('Build optimizer') 56 | optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss) 57 | else: 58 | optimizer = None 59 | 60 | build_dict = {'optimizer': optimizer, 'loss': loss, 'output': output, 'features': None} 61 | return build_dict 62 | 63 | -------------------------------------------------------------------------------- /DNFNet/DNFNetModels/model3.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from DNFNet.DNFNetModels.DNFNetComponents import compute_total_number_of_literals, compute_total_number_of_conjunctions, denseNDArrayToSparseTensor, create_conjunctions_indicator_matrix, create_formulas_indicator_matrix, \ 4 | and_operator, or_operator, tf_matmul_sparse_dense 5 | 6 | 7 | def fetch_placeholders(placeholders): 8 | x = placeholders['x'] 9 | y = placeholders['y'] 10 | lr = placeholders['lr'] 11 | return x, y, lr 12 | 13 | 14 | ########################################################################################### 15 | # DNF structure 16 | ########################################################################################### 17 | class MNN(object): 18 | def __init__(self, config): 19 | self.config = config 20 | self.loaded_input_masks = None 21 | 22 | def build(self, placeholders, n_labels, phase): 23 | x, y, lr = fetch_placeholders(placeholders) 24 | 25 | input_dim = self.config['input_dim'] 26 | n_conjunctions_arr = self.config['n_conjunctions_arr'] 27 | conjunctions_depth_arr = self.config['conjunctions_depth_arr'] 28 | n_formulas = self.config['n_formulas'] 29 | 30 | total_number_of_literals = compute_total_number_of_literals(n_formulas, n_conjunctions_arr, conjunctions_depth_arr) 31 | total_number_of_conjunctions = compute_total_number_of_conjunctions(n_formulas, n_conjunctions_arr) 32 | conjunctions_indicator_matrix = create_conjunctions_indicator_matrix(total_number_of_literals, total_number_of_conjunctions, conjunctions_depth_arr) 33 | formulas_indicator_matrix = create_formulas_indicator_matrix(n_formulas, n_conjunctions_arr) 34 | 35 | conjunctions_indicator_sparse_matrix = denseNDArrayToSparseTensor(conjunctions_indicator_matrix) 36 | formulas_indicator_sparse_matrix = denseNDArrayToSparseTensor(formulas_indicator_matrix) 37 | and_bias = np.sum(conjunctions_indicator_matrix, axis=0) 38 | or_bias = np.sum(formulas_indicator_matrix, axis=0) 39 | 40 | bias = tf.get_variable('literals_bias', [total_number_of_literals], initializer=tf.zeros_initializer()) 41 | weight = tf.get_variable('literals_weights', [input_dim, total_number_of_literals], initializer=tf.contrib.layers.xavier_initializer()) 42 | 43 | out_literals = tf.tanh(tf.add(tf.matmul(x, weight), bias)) 44 | out_conjunctions = and_operator(tf_matmul_sparse_dense(conjunctions_indicator_sparse_matrix, out_literals), d=and_bias) 45 | out_DNNFs = or_operator(tf_matmul_sparse_dense(formulas_indicator_sparse_matrix, out_conjunctions), d=or_bias) 46 | out_DNNFs = tf.reshape(out_DNNFs, shape=(-1, n_formulas)) 47 | 48 | output = tf.layers.dense(out_DNNFs, n_labels, activation=None, kernel_initializer=tf.contrib.layers.xavier_initializer()) 49 | 50 | with tf.variable_scope('loss'): 51 | if n_labels > 1: 52 | loss = tf.losses.softmax_cross_entropy(onehot_labels=y, logits=output) 53 | output = tf.nn.softmax(output) 54 | else: 55 | loss = tf.losses.sigmoid_cross_entropy(multi_class_labels=y, logits=output) 56 | output = tf.nn.sigmoid(output) 57 | 58 | loss += tf.losses.get_regularization_loss() 59 | 60 | if phase == 'train': 61 | print('Build optimizer') 62 | optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss) 63 | else: 64 | optimizer = None 65 | 66 | build_dict = {'optimizer': optimizer, 'loss': loss, 'output': output, 'features': None, 'input_masks': None} 67 | return build_dict 68 | 69 | -------------------------------------------------------------------------------- /DNFNet/DNFNetModels/model6.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from DNFNet.DNFNetModels.DNFNetComponents import compute_total_number_of_literals, compute_total_number_of_conjunctions, compute_n_literals_per_formula, \ 3 | feature_selection, broadcast_exp 4 | 5 | 6 | def fetch_placeholders(placeholders): 7 | x = placeholders['x'] 8 | y = placeholders['y'] 9 | lr = placeholders['lr'] 10 | return x, y, lr 11 | 12 | 13 | ########################################################################################### 14 | # FCN with feature selection and localization 15 | ########################################################################################### 16 | class MNN(object): 17 | def __init__(self, config): 18 | self.config = config 19 | self.loaded_input_masks = None 20 | 21 | def build(self, placeholders, n_labels, phase): 22 | x, y, lr = fetch_placeholders(placeholders) 23 | 24 | input_dim = self.config['input_dim'] 25 | n_conjunctions_arr = self.config['n_conjunctions_arr'] 26 | conjunctions_depth_arr = self.config['conjunctions_depth_arr'] 27 | n_formulas = self.config['n_formulas'] 28 | 29 | total_number_of_literals = compute_total_number_of_literals(n_formulas, n_conjunctions_arr, conjunctions_depth_arr) 30 | total_number_of_conjunctions = compute_total_number_of_conjunctions(n_formulas, n_conjunctions_arr) 31 | n_literals_per_formula_arr = compute_n_literals_per_formula(n_conjunctions_arr, conjunctions_depth_arr) 32 | 33 | learnable_binary_mask, literals_random_mask, elastic_net_reg = feature_selection(input_dim, self.config['keep_feature_prob_arr'], n_literals_per_formula_arr, n_formulas, self.config['elastic_net_beta']) 34 | 35 | bias = tf.get_variable('literals_bias', [total_number_of_literals], initializer=tf.zeros_initializer()) 36 | weight = tf.get_variable('literals_weights', [input_dim, total_number_of_literals], initializer=tf.contrib.layers.xavier_initializer()) 37 | weight = tf.multiply(weight, tf.constant(literals_random_mask)) 38 | 39 | out = tf.tanh(tf.add(tf.matmul(x, tf.multiply(weight, learnable_binary_mask)), bias)) 40 | out = tf.layers.dense(out, total_number_of_conjunctions, activation=tf.tanh, kernel_initializer=tf.contrib.layers.xavier_initializer()) 41 | out = tf.layers.dense(out, n_formulas, activation=tf.tanh, kernel_initializer=tf.contrib.layers.xavier_initializer()) 42 | 43 | loc = broadcast_exp(x, n_formulas, self.config['input_dim']) 44 | temperature = tf.get_variable('temperature', initializer=tf.constant(value=2.)) 45 | loc = tf.nn.softmax(tf.sigmoid(temperature) * loc) 46 | out = tf.multiply(out, loc) 47 | 48 | output = tf.layers.dense(out, n_labels, activation=None, kernel_initializer=tf.contrib.layers.xavier_initializer()) 49 | 50 | with tf.variable_scope('loss'): 51 | if n_labels > 1: 52 | loss = tf.losses.softmax_cross_entropy(onehot_labels=y, logits=output) 53 | output = tf.nn.softmax(output) 54 | else: 55 | loss = tf.losses.sigmoid_cross_entropy(multi_class_labels=y, logits=output) 56 | output = tf.nn.sigmoid(output) 57 | 58 | loss += elastic_net_reg 59 | loss += tf.losses.get_regularization_loss() 60 | 61 | if phase == 'train': 62 | print('Build optimizer') 63 | optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss) 64 | else: 65 | optimizer = None 66 | 67 | build_dict = {'optimizer': optimizer, 'loss': loss, 'output': output, 'features': None, 'input_masks': None} 68 | return build_dict 69 | 70 | -------------------------------------------------------------------------------- /DNFNet/DNFNetModels/model5.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from DNFNet.DNFNetModels.DNFNetComponents import compute_total_number_of_literals, compute_total_number_of_conjunctions, compute_n_literals_per_formula,\ 4 | denseNDArrayToSparseTensor, create_conjunctions_indicator_matrix, create_formulas_indicator_matrix, feature_selection, broadcast_orthogonal_constraint,\ 5 | and_operator, or_operator, broadcast_exp, tf_matmul_sparse_dense 6 | 7 | 8 | def fetch_placeholders(placeholders): 9 | x = placeholders['x'] 10 | y = placeholders['y'] 11 | lr = placeholders['lr'] 12 | return x, y, lr 13 | 14 | 15 | ########################################################################################### 16 | # DNF structure with localization 17 | ########################################################################################### 18 | class MNN(object): 19 | def __init__(self, config): 20 | self.config = config 21 | self.loaded_input_masks = None 22 | 23 | def build(self, placeholders, n_labels, phase): 24 | x, y, lr = fetch_placeholders(placeholders) 25 | 26 | input_dim = self.config['input_dim'] 27 | n_conjunctions_arr = self.config['n_conjunctions_arr'] 28 | conjunctions_depth_arr = self.config['conjunctions_depth_arr'] 29 | n_formulas = self.config['n_formulas'] 30 | 31 | total_number_of_literals = compute_total_number_of_literals(n_formulas, n_conjunctions_arr, conjunctions_depth_arr) 32 | total_number_of_conjunctions = compute_total_number_of_conjunctions(n_formulas, n_conjunctions_arr) 33 | conjunctions_indicator_matrix = create_conjunctions_indicator_matrix(total_number_of_literals, total_number_of_conjunctions, conjunctions_depth_arr) 34 | formulas_indicator_matrix = create_formulas_indicator_matrix(n_formulas, n_conjunctions_arr) 35 | 36 | conjunctions_indicator_sparse_matrix = denseNDArrayToSparseTensor(conjunctions_indicator_matrix) 37 | formulas_indicator_sparse_matrix = denseNDArrayToSparseTensor(formulas_indicator_matrix) 38 | and_bias = np.sum(conjunctions_indicator_matrix, axis=0) 39 | or_bias = np.sum(formulas_indicator_matrix, axis=0) 40 | 41 | bias = tf.get_variable('literals_bias', [total_number_of_literals], initializer=tf.zeros_initializer()) 42 | weight = tf.get_variable('literals_weights', [input_dim, total_number_of_literals], initializer=tf.contrib.layers.xavier_initializer()) 43 | 44 | out_literals = tf.tanh(tf.add(tf.matmul(x, weight), bias)) 45 | out_conjunctions = and_operator(tf_matmul_sparse_dense(conjunctions_indicator_sparse_matrix, out_literals), d=and_bias) 46 | out_DNNFs = or_operator(tf_matmul_sparse_dense(formulas_indicator_sparse_matrix, out_conjunctions), d=or_bias) 47 | out_DNNFs = tf.reshape(out_DNNFs, shape=(-1, n_formulas)) 48 | 49 | loc = broadcast_exp(x, n_formulas, self.config['input_dim']) 50 | temperature = tf.get_variable('temperature', initializer=tf.constant(value=2.)) 51 | loc = tf.nn.softmax(tf.sigmoid(temperature) * loc) 52 | out_DNNFs = tf.multiply(out_DNNFs, loc) 53 | 54 | output = tf.layers.dense(out_DNNFs, n_labels, activation=None, kernel_initializer=tf.contrib.layers.xavier_initializer()) 55 | 56 | with tf.variable_scope('loss'): 57 | if n_labels > 1: 58 | loss = tf.losses.softmax_cross_entropy(onehot_labels=y, logits=output) 59 | output = tf.nn.softmax(output) 60 | else: 61 | loss = tf.losses.sigmoid_cross_entropy(multi_class_labels=y, logits=output) 62 | output = tf.nn.sigmoid(output) 63 | 64 | loss += tf.losses.get_regularization_loss() 65 | 66 | if phase == 'train': 67 | print('Build optimizer') 68 | optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss) 69 | else: 70 | optimizer = None 71 | 72 | build_dict = {'optimizer': optimizer, 'loss': loss, 'output': output, 'features': None, 'input_masks': None} 73 | return build_dict 74 | 75 | -------------------------------------------------------------------------------- /DNFNet/AblationStudy.py: -------------------------------------------------------------------------------- 1 | import os 2 | from DNFNet.CompetitionsHandler import CompetitionsHandler, shared_config 3 | from DNFNet.ModelHandler import ModelHandler 4 | from Utils.file_utils import create_dir 5 | from Utils.grid_search_utils import cross_validation 6 | 7 | 8 | # model2 - fully_trained_FCN 9 | grid_params_exp_1 = { 10 | 'n_formulas': [128, 512], # 2048 is excluded due to out of memory 11 | } 12 | 13 | # model 3 - DNF_structure_only 14 | grid_params_exp_2 = { 15 | 'n_formulas': [128, 512, 2048], 16 | } 17 | 18 | # model 4 - DNF with feature selection 19 | grid_params_exp_3 = { 20 | 'n_formulas': [128, 512, 2048], 21 | 'elastic_net_beta': [1.6, 1.3, 1., 0.7, 0.4, 0.1], 22 | } 23 | 24 | # model 1 - complete DNF-Net 25 | grid_params_exp_4 = { 26 | 'n_formulas': [128, 512, 2048], 27 | 'elastic_net_beta': [1.6, 1.3, 1., 0.7, 0.4, 0.1], 28 | } 29 | 30 | # model 5 - DNF structure with localization (without feature selection) 31 | grid_params_exp_5 = { 32 | 'n_formulas': [128, 512, 2048], 33 | } 34 | 35 | # exp 6 - without localization - equal to exp 4 36 | 37 | # model 6 - FCN with localization and feature selection 38 | grid_params_exp_7 = { 39 | 'n_formulas': [128, 512], # 2048 is excluded due to out of memory 40 | 'elastic_net_beta': [1.6, 1.3, 1., 0.7, 0.4, 0.1], 41 | } 42 | 43 | 44 | if __name__ == '__main__': 45 | base_dir = 'path/to/base/dir' 46 | model = CompetitionsHandler.DNFNET 47 | folds = [0, 1, 2, 3, 4] 48 | gpus_list = ['0'] 49 | seeds_arr = [1, 2, 3] 50 | 51 | ablation_study_configs = [ 52 | {'exp_name': 'ablation_exp_1_fully_trained_FCN', 53 | 'model_number': 2, 54 | 'grid': grid_params_exp_1}, 55 | 56 | {'exp_name': 'ablation_exp_2_DNF_structure', 57 | 'model_number': 3, 58 | 'grid': grid_params_exp_2}, 59 | 60 | {'exp_name': 'ablation_exp_03_DNF_feature_selection', 61 | 'model_number': 4, 62 | 'grid': grid_params_exp_3}, 63 | 64 | {'exp_name': 'ablation_exp_04_DNF_structure_feature_selection_localization', 65 | 'model_number': 1, 66 | 'grid': grid_params_exp_4}, 67 | 68 | {'exp_name': 'ablation_exp_05_DNF-Net_without_feature_selection', 69 | 'model_number': 5, 70 | 'grid': grid_params_exp_5}, 71 | 72 | {'exp_name': 'ablation_exp_06_DNF-Net_without_DNF_structure', 73 | 'model_number': 6, 74 | 'grid': grid_params_exp_7} 75 | ] 76 | 77 | for ablation_conf in ablation_study_configs: 78 | model_number = ablation_conf['model_number'] 79 | grid_params = ablation_conf['grid'] 80 | experiment_name = ablation_conf['exp_name'] 81 | 82 | for competition_name in [CompetitionsHandler.GESTURE_PHASE, CompetitionsHandler.EYE_MOVEMENTS, CompetitionsHandler.GAS]: 83 | shared_config['experiments_dir'] = '{}/{}Competitions/{}/experiments/{}'.format(base_dir, model, competition_name, experiment_name) 84 | output_dir = os.path.join(shared_config['experiments_dir'], 'grid_search') 85 | create_dir(shared_config['experiments_dir']) 86 | 87 | shared_config['model_number'] = model_number 88 | shared_config['model_name'] = model 89 | shared_config['competition_name'] = competition_name 90 | config, score_config = CompetitionsHandler.get_configs(competition_name) 91 | config = CompetitionsHandler.merge_configs(shared_config, config) 92 | cross_validation(config, ModelHandler, score_config, 93 | folds=folds, 94 | gpus_list=gpus_list, 95 | grid_params=grid_params, 96 | output_dir=output_dir, 97 | seeds_arr=seeds_arr) -------------------------------------------------------------------------------- /DNFNet/DNFNetModels/model4.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from DNFNet.DNFNetModels.DNFNetComponents import compute_total_number_of_literals, compute_total_number_of_conjunctions, compute_n_literals_per_formula,\ 4 | denseNDArrayToSparseTensor, create_conjunctions_indicator_matrix, create_formulas_indicator_matrix, feature_selection, and_operator, or_operator, tf_matmul_sparse_dense 5 | 6 | 7 | def fetch_placeholders(placeholders): 8 | x = placeholders['x'] 9 | y = placeholders['y'] 10 | lr = placeholders['lr'] 11 | return x, y, lr 12 | 13 | 14 | ########################################################################################### 15 | # DNF structure with feature selection 16 | ########################################################################################### 17 | class MNN(object): 18 | def __init__(self, config): 19 | self.config = config 20 | self.loaded_input_masks = None 21 | 22 | def build(self, placeholders, n_labels, phase): 23 | x, y, lr = fetch_placeholders(placeholders) 24 | 25 | input_dim = self.config['input_dim'] 26 | n_conjunctions_arr = self.config['n_conjunctions_arr'] 27 | conjunctions_depth_arr = self.config['conjunctions_depth_arr'] 28 | n_formulas = self.config['n_formulas'] 29 | 30 | total_number_of_literals = compute_total_number_of_literals(n_formulas, n_conjunctions_arr, conjunctions_depth_arr) 31 | total_number_of_conjunctions = compute_total_number_of_conjunctions(n_formulas, n_conjunctions_arr) 32 | n_literals_per_formula_arr = compute_n_literals_per_formula(n_conjunctions_arr, conjunctions_depth_arr) 33 | conjunctions_indicator_matrix = create_conjunctions_indicator_matrix(total_number_of_literals, total_number_of_conjunctions, conjunctions_depth_arr) 34 | formulas_indicator_matrix = create_formulas_indicator_matrix(n_formulas, n_conjunctions_arr) 35 | 36 | conjunctions_indicator_sparse_matrix = denseNDArrayToSparseTensor(conjunctions_indicator_matrix) 37 | formulas_indicator_sparse_matrix = denseNDArrayToSparseTensor(formulas_indicator_matrix) 38 | and_bias = np.sum(conjunctions_indicator_matrix, axis=0) 39 | or_bias = np.sum(formulas_indicator_matrix, axis=0) 40 | learnable_binary_mask, literals_random_mask, elastic_net_reg = feature_selection(input_dim, self.config['keep_feature_prob_arr'], n_literals_per_formula_arr, n_formulas, self.config['elastic_net_beta']) 41 | 42 | bias = tf.get_variable('literals_bias', [total_number_of_literals], initializer=tf.zeros_initializer()) 43 | weight = tf.get_variable('literals_weights', [input_dim, total_number_of_literals], initializer=tf.contrib.layers.xavier_initializer()) 44 | weight = tf.multiply(weight, tf.constant(literals_random_mask)) 45 | 46 | out_literals = tf.tanh(tf.add(tf.matmul(x, tf.multiply(weight, learnable_binary_mask)), bias)) 47 | out_conjunctions = and_operator(tf_matmul_sparse_dense(conjunctions_indicator_sparse_matrix, out_literals), d=and_bias) 48 | out_DNNFs = or_operator(tf_matmul_sparse_dense(formulas_indicator_sparse_matrix, out_conjunctions), d=or_bias) 49 | out_DNNFs = tf.reshape(out_DNNFs, shape=(-1, n_formulas)) 50 | 51 | output = tf.layers.dense(out_DNNFs, n_labels, activation=None, kernel_initializer=tf.contrib.layers.xavier_initializer()) 52 | 53 | with tf.variable_scope('loss'): 54 | if n_labels > 1: 55 | loss = tf.losses.softmax_cross_entropy(onehot_labels=y, logits=output) 56 | output = tf.nn.softmax(output) 57 | else: 58 | loss = tf.losses.sigmoid_cross_entropy(multi_class_labels=y, logits=output) 59 | output = tf.nn.sigmoid(output) 60 | 61 | loss += elastic_net_reg 62 | loss += tf.losses.get_regularization_loss() 63 | 64 | if phase == 'train': 65 | print('Build optimizer') 66 | optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss) 67 | else: 68 | optimizer = None 69 | 70 | build_dict = {'optimizer': optimizer, 'loss': loss, 'output': output, 'features': None, 'input_masks': None} 71 | return build_dict 72 | 73 | -------------------------------------------------------------------------------- /DNFNet/DNFNetModels/model1.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from DNFNet.DNFNetModels.DNFNetComponents import compute_total_number_of_literals, compute_total_number_of_conjunctions, compute_n_literals_per_formula,\ 4 | denseNDArrayToSparseTensor, create_conjunctions_indicator_matrix, create_formulas_indicator_matrix, feature_selection, and_operator, or_operator, broadcast_exp, tf_matmul_sparse_dense 5 | 6 | 7 | def fetch_placeholders(placeholders): 8 | x = placeholders['x'] 9 | y = placeholders['y'] 10 | lr = placeholders['lr'] 11 | return x, y, lr 12 | 13 | 14 | ########################################################################################### 15 | # complete DNF-Net: DNF structure with feature selection and localization 16 | ########################################################################################### 17 | class MNN(object): 18 | def __init__(self, config): 19 | self.config = config 20 | self.loaded_input_masks = None 21 | 22 | def build(self, placeholders, n_labels, phase): 23 | x, y, lr = fetch_placeholders(placeholders) 24 | 25 | input_dim = self.config['input_dim'] 26 | n_conjunctions_arr = self.config['n_conjunctions_arr'] 27 | conjunctions_depth_arr = self.config['conjunctions_depth_arr'] 28 | n_formulas = self.config['n_formulas'] 29 | 30 | total_number_of_literals = compute_total_number_of_literals(n_formulas, n_conjunctions_arr, conjunctions_depth_arr) 31 | total_number_of_conjunctions = compute_total_number_of_conjunctions(n_formulas, n_conjunctions_arr) 32 | n_literals_per_formula_arr = compute_n_literals_per_formula(n_conjunctions_arr, conjunctions_depth_arr) 33 | conjunctions_indicator_matrix = create_conjunctions_indicator_matrix(total_number_of_literals, total_number_of_conjunctions, conjunctions_depth_arr) 34 | formulas_indicator_matrix = create_formulas_indicator_matrix(n_formulas, n_conjunctions_arr) 35 | 36 | conjunctions_indicator_sparse_matrix = denseNDArrayToSparseTensor(conjunctions_indicator_matrix) 37 | formulas_indicator_sparse_matrix = denseNDArrayToSparseTensor(formulas_indicator_matrix) 38 | 39 | and_bias = np.sum(conjunctions_indicator_matrix, axis=0) 40 | or_bias = np.sum(formulas_indicator_matrix, axis=0) 41 | 42 | learnable_binary_mask, literals_random_mask, elastic_net_reg = feature_selection(input_dim, self.config['keep_feature_prob_arr'], n_literals_per_formula_arr, n_formulas, self.config['elastic_net_beta']) 43 | bias = tf.get_variable('literals_bias', [total_number_of_literals], initializer=tf.zeros_initializer()) 44 | weight = tf.get_variable('literals_weights', [input_dim, total_number_of_literals], initializer=tf.contrib.layers.xavier_initializer()) 45 | weight = tf.multiply(weight, tf.constant(literals_random_mask)) 46 | 47 | out_literals = tf.tanh(tf.add(tf.matmul(x, tf.multiply(weight, learnable_binary_mask)), bias)) 48 | out_conjunctions = and_operator(tf_matmul_sparse_dense(conjunctions_indicator_sparse_matrix, out_literals), d=and_bias) 49 | out_DNNFs = or_operator(tf_matmul_sparse_dense(formulas_indicator_sparse_matrix, out_conjunctions), d=or_bias) 50 | out_DNNFs = tf.reshape(out_DNNFs, shape=(-1, n_formulas)) 51 | 52 | loc = broadcast_exp(x, n_formulas, self.config['input_dim']) 53 | temperature = tf.get_variable('temperature', initializer=tf.constant(value=2.)) 54 | loc = tf.nn.softmax(tf.sigmoid(temperature) * loc) 55 | out_DNNFs = tf.multiply(out_DNNFs, loc) 56 | output = tf.layers.dense(out_DNNFs, n_labels, activation=None, kernel_initializer=tf.contrib.layers.xavier_initializer()) 57 | 58 | with tf.variable_scope('loss'): 59 | if n_labels > 1: 60 | loss = tf.losses.softmax_cross_entropy(onehot_labels=y, logits=output) 61 | output = tf.nn.softmax(output) 62 | else: 63 | loss = tf.losses.sigmoid_cross_entropy(multi_class_labels=y, logits=output) 64 | output = tf.nn.sigmoid(output) 65 | 66 | loss += elastic_net_reg 67 | loss += tf.losses.get_regularization_loss() 68 | 69 | if phase == 'train': 70 | print('Build optimizer') 71 | optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss) 72 | else: 73 | optimizer = None 74 | 75 | build_dict = {'optimizer': optimizer, 'loss': loss, 'output': output, 'features': None, 'input_masks': None} 76 | return build_dict 77 | 78 | -------------------------------------------------------------------------------- /DNFNet/CompetitionsHandler.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import os 3 | 4 | from DNFNet.Competitions.EyeMovements.CompetitionConfig import get_configs as get_eye_movements_configs 5 | from DNFNet.Competitions.Gas.CompetitionConfig import get_configs as get_gas_configs 6 | from DNFNet.Competitions.GesturePhase.CompetitionConfig import get_configs as get_gesture_phase_configs 7 | from DNFNet.Competitions.Otto.CompetitionConfig import get_configs as get_otto_configs 8 | from DNFNet.Competitions.SantanderTransaction.CompetitionConfig import get_configs as get_santander_transaction_configs 9 | from DNFNet.Competitions.House.CompetitionConfig import get_configs as get_house_configs 10 | from DNFNet.Competitions.RobotNavigation.CompetitionConfig import get_configs as get_robot_navigation_configs 11 | 12 | 13 | from DNFNet.ModelHandler import ModelHandler 14 | from DNFNet.XGBModelHandler import XGBModelHandler 15 | 16 | from Utils.experiment_utils import create_all_FCN_layers_grid 17 | from Utils.file_utils import create_dir 18 | from Utils.grid_search_utils import cross_validation 19 | 20 | 21 | class CompetitionsHandler: 22 | # Competitions 23 | GAS = 'Gas' 24 | OTTO = 'Otto' 25 | EYE_MOVEMENTS = 'EyeMovements' 26 | GESTURE_PHASE = 'GesturePhase' 27 | SANTANDER_TRANSACTION = 'SantanderTransaction' 28 | HOUSE = 'House' 29 | ROBOT_NAVIGATION = 'RobotNavigation' 30 | 31 | # Models 32 | DNFNET = 'DNFNet' 33 | FCN = 'FCN' 34 | XGB = 'XGB' 35 | 36 | @staticmethod 37 | def get_configs(competition_name): 38 | if competition_name == CompetitionsHandler.EYE_MOVEMENTS: 39 | return get_eye_movements_configs() 40 | elif competition_name == CompetitionsHandler.GAS: 41 | return get_gas_configs() 42 | elif competition_name == CompetitionsHandler.GESTURE_PHASE: 43 | return get_gesture_phase_configs() 44 | elif competition_name == CompetitionsHandler.OTTO: 45 | return get_otto_configs() 46 | elif competition_name == CompetitionsHandler.SANTANDER_TRANSACTION: 47 | return get_santander_transaction_configs() 48 | elif competition_name == CompetitionsHandler.HOUSE: 49 | return get_house_configs() 50 | elif competition_name == CompetitionsHandler.ROBOT_NAVIGATION: 51 | return get_robot_navigation_configs() 52 | 53 | 54 | @staticmethod 55 | def merge_configs(config_dst, config_src): 56 | config_dst = copy.deepcopy(config_dst) 57 | config_src = copy.deepcopy(config_src) 58 | for key, value in config_src.items(): 59 | config_dst[key] = value 60 | return config_dst 61 | 62 | 63 | DNFNet_grid_params = { 64 | 'n_formulas': [3072, 2048, 1024, 512, 256, 128, 64], 65 | 'orthogonal_lambda': [0.], 66 | 'elastic_net_beta': [1.6, 1.3, 1., 0.7, 0.4, 0.1], 67 | } 68 | 69 | FCN_grid_params = { 70 | 'FCN_layers': create_all_FCN_layers_grid(depth_arr=[1, 2, 3, 4, 5, 6], width_arr=[128, 256, 512, 1024, 2048]), 71 | 'FCN_l2_lambda': [1e-2, 1e-4, 1e-6, 1e-8, 0.], 72 | 'dropout_rate': [0., 0.25, 0.5, 0.75], 73 | 'initial_lr': [5e-2, 5e-3, 5e-4] 74 | } 75 | 76 | XGB_grid_params = { 77 | 'XGB_n_estimators': [2500], 78 | 'XGB_learning_rate': [0.001, 0.005, 0.01, 0.05, 0.1, 0.5], 79 | 'XGB_max_depth': [2, 3, 4, 5, 7, 9, 11, 13, 15], 80 | 'XGB_colsample_bytree': [0.25, 0.5, 0.75, 1.], 81 | 'XGB_subsample': [0.25, 0.5, 0.75, 1.] 82 | } 83 | 84 | shared_config = { 85 | 'n_conjunctions_arr': [6, 9, 12, 15], 86 | 'conjunctions_depth_arr': [2, 4, 6], 87 | 'keep_feature_prob_arr': [0.1, 0.3, 0.5, 0.7, 0.9], 88 | 89 | 'initial_lr': 5e-2, 90 | 'lr_decay_factor': 0.5, 91 | 'lr_patience': 10, 92 | 'min_lr': 1e-6, 93 | 94 | 'early_stopping_patience': 30, 95 | 'epochs': 1000, 96 | 'batch_size': 2048, 97 | 98 | 'apply_standardization': True, 99 | 100 | 'save_weights': True, 101 | 'starting_epoch_to_save': 0, 102 | 'models_module_name': 'DNFNet.DNFNetModels', 103 | 'models_dir': './DNFNetModels', 104 | } 105 | 106 | 107 | if __name__ == '__main__': 108 | competition_name_list = [ 109 | CompetitionsHandler.SANTANDER_TRANSACTION, 110 | CompetitionsHandler.OTTO, 111 | CompetitionsHandler.GESTURE_PHASE, 112 | CompetitionsHandler.EYE_MOVEMENTS, 113 | CompetitionsHandler.GAS, 114 | CompetitionsHandler.HOUSE, 115 | CompetitionsHandler.ROBOT_NAVIGATION 116 | ] 117 | 118 | base_dir = 'path/to/base/dir' 119 | model = CompetitionsHandler.FCN 120 | experiment_name = 'exp_comparative_evaluation' 121 | folds = [0, 1, 2, 3, 4] 122 | gpus_list = ['0'] 123 | seeds_arr = [1, 2, 3] 124 | 125 | for competition_name in competition_name_list: 126 | shared_config['experiments_dir'] = '{}/{}Competitions/{}/experiments/{}'.format(base_dir, model, competition_name, experiment_name) 127 | output_dir = os.path.join(shared_config['experiments_dir'], 'grid_search') 128 | create_dir(shared_config['experiments_dir']) 129 | shared_config['competition_name'] = competition_name 130 | shared_config['model_name'] = model 131 | 132 | print(competition_name) 133 | if model == CompetitionsHandler.DNFNET: 134 | shared_config['model_number'] = 1 135 | config, score_config = CompetitionsHandler.get_configs(competition_name) 136 | config = CompetitionsHandler.merge_configs(shared_config, config) 137 | cross_validation(config, ModelHandler, score_config, 138 | folds=folds, 139 | gpus_list=gpus_list, 140 | grid_params=DNFNet_grid_params, 141 | output_dir=output_dir, 142 | seeds_arr=seeds_arr) 143 | 144 | elif model == CompetitionsHandler.FCN: 145 | shared_config['model_number'] = 100 146 | config, score_config = CompetitionsHandler.get_configs(competition_name) 147 | config = CompetitionsHandler.merge_configs(shared_config, config) 148 | cross_validation(config, ModelHandler, score_config, 149 | folds=folds, 150 | gpus_list=gpus_list, 151 | grid_params=FCN_grid_params, 152 | output_dir=output_dir, 153 | seeds_arr=seeds_arr) 154 | 155 | elif model == CompetitionsHandler.XGB: 156 | config, score_config = CompetitionsHandler.get_configs(competition_name) 157 | config = CompetitionsHandler.merge_configs(shared_config, config) 158 | cross_validation(config, XGBModelHandler, score_config, 159 | folds=folds, 160 | gpus_list=gpus_list, 161 | grid_params=XGB_grid_params, 162 | output_dir=output_dir, 163 | seeds_arr=seeds_arr) 164 | -------------------------------------------------------------------------------- /DNFNet/DNFNetModels/DNFNetComponents.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | 5 | ################################################################################################################################################################ 6 | # Soft binary gates 7 | ################################################################################################################################################################ 8 | def and_operator(x, d): 9 | out = tf.add(x, -d + 1.5) 10 | out = tf.tanh(out) 11 | return out 12 | 13 | 14 | def or_operator(x, d): 15 | out = tf.add(x, d - 1.5) 16 | out = tf.tanh(out) 17 | return out 18 | 19 | 20 | ################################################################################################################################################################ 21 | # Localization 22 | ################################################################################################################################################################ 23 | def broadcast_exp(x, n_formulas, input_dim): 24 | mu = tf.get_variable('exp_mu', [n_formulas, input_dim], initializer=tf.initializers.random_normal()) 25 | sigma = tf.get_variable('exp_sigma', [1, n_formulas, input_dim], initializer=tf.initializers.random_normal()) 26 | diff = tf.expand_dims(x, axis=1) - tf.expand_dims(mu, axis=0) 27 | loc = tf.exp(-1 * tf.norm(tf.multiply(diff, sigma), axis=-1)) 28 | return loc 29 | 30 | 31 | ################################################################################################################################################################ 32 | # Feature Selection 33 | ################################################################################################################################################################ 34 | def binary_threshold(x, eps=0.1): 35 | x = tf.abs(x) - eps 36 | return 0.5*binary_activation(x) + 0.5 37 | 38 | 39 | def binary_activation(x): 40 | forward = tf.sign(x) 41 | backward = tf.tanh(x) 42 | return backward + tf.stop_gradient(forward - backward) 43 | 44 | 45 | def feature_selection(input_dim, keep_feature_prob_arr, n_literals_per_formula_arr, n_formulas, elastic_net_beta): 46 | binary_threshold_eps = 1 47 | literals_random_mask, formulas_random_mask = create_random_mask(input_dim, keep_feature_prob_arr, n_literals_per_formula_arr, n_formulas) 48 | n_effective_features = np.sum(formulas_random_mask, axis=0) 49 | formulas_random_mask = tf.constant(formulas_random_mask) 50 | ext_matrix = extension_matrix(n_formulas, n_literals_per_formula_arr) 51 | 52 | elastic_net_alpha = tf.get_variable('elastic_net_alpha', initializer=tf.constant(value=0.)) 53 | learnable_mask = tf.get_variable('learnable_mask', [input_dim, n_formulas], initializer=tf.initializers.constant(binary_threshold_eps + 0.5)) 54 | 55 | learnable_mask_01 = binary_threshold(learnable_mask, eps=binary_threshold_eps) 56 | 57 | l2_square_norm_selected = tf.diag_part(tf.matmul(tf.transpose(tf.square(learnable_mask)), formulas_random_mask)) 58 | l1_norm_selected = tf.diag_part(tf.matmul(tf.transpose(tf.abs(learnable_mask)), formulas_random_mask)) 59 | 60 | l2 = tf.abs(tf.div(l2_square_norm_selected, n_effective_features) - elastic_net_beta * binary_threshold_eps ** 2) 61 | l1 = tf.abs(tf.div(l1_norm_selected, n_effective_features) - elastic_net_beta * binary_threshold_eps) 62 | elastic_net_reg = tf.reduce_mean((l2 * ((1 - tf.nn.sigmoid(elastic_net_alpha)) / 2) + l1 * tf.nn.sigmoid(elastic_net_alpha))) 63 | learnable_binary_mask = tf_matmul_sparse_dense(ext_matrix, learnable_mask_01) 64 | return learnable_binary_mask, literals_random_mask, elastic_net_reg 65 | 66 | 67 | def create_random_mask(input_dim, keep_feature_prob_arr, n_literals_per_formula_arr, n_formulas): 68 | literals_random_mask = [] 69 | formulas_random_mask = [] 70 | p_index = 0 71 | for i in range(n_formulas): 72 | if i % len(n_literals_per_formula_arr) == 0 and i != 0: 73 | p_index = (p_index + 1) % len(keep_feature_prob_arr) 74 | 75 | p_i = keep_feature_prob_arr[p_index] 76 | mask = np.random.choice([1., 0.], input_dim, p=[p_i, 1 - p_i]) 77 | while np.sum(mask) == 0: 78 | mask = np.random.choice([1., 0.], input_dim, p=[p_i, 1 - p_i]) 79 | n_literals_in_formula = n_literals_per_formula_arr[i % len(n_literals_per_formula_arr)] 80 | formulas_random_mask.append(np.copy(mask)) 81 | for _ in range(n_literals_in_formula): 82 | literals_random_mask.append(np.copy(mask)) 83 | return np.array(literals_random_mask).T.astype('float32'), np.array(formulas_random_mask).T.astype('float32') 84 | 85 | 86 | def extension_matrix(n_formulas, n_literals_per_formula_arr): 87 | formula_index = 0 88 | mat = [] 89 | for i in range(n_formulas): 90 | n_nodes_in_tree = n_literals_per_formula_arr[i % len(n_literals_per_formula_arr)] 91 | v = np.zeros(n_formulas) 92 | v[formula_index] = 1. 93 | for _ in range(n_nodes_in_tree): 94 | mat.append(v) 95 | formula_index += 1 96 | return denseNDArrayToSparseTensor(np.array(mat).T.astype('float32')) 97 | 98 | 99 | ################################################################################################################################################################ 100 | # DNF structure 101 | ################################################################################################################################################################ 102 | def create_conjunctions_indicator_matrix(total_number_of_literals, total_number_of_conjunctions, conjunctions_depth_arr): 103 | n_different_depth = len(conjunctions_depth_arr) 104 | n_literals_in_group = np.sum(conjunctions_depth_arr) 105 | result = [] 106 | for i in range(total_number_of_conjunctions // n_different_depth): 107 | s = 0 108 | for d in conjunctions_depth_arr: 109 | b = np.zeros(total_number_of_literals).astype('bool') 110 | b[i * n_literals_in_group + s: i * n_literals_in_group + s + d] = True 111 | s += d 112 | result.append(b) 113 | return np.array(result).T 114 | 115 | 116 | def create_formulas_indicator_matrix(n_formulas, n_conjunctions_arr): 117 | result = [] 118 | total_number_of_conjunctions = compute_total_number_of_conjunctions(n_formulas, n_conjunctions_arr) 119 | 120 | base = 0 121 | for i in range(n_formulas): 122 | n_conjunctions = n_conjunctions_arr[i % len(n_conjunctions_arr)] 123 | b = np.zeros(total_number_of_conjunctions).astype('bool') 124 | b[base: base + n_conjunctions] = True 125 | result.append(b) 126 | base += n_conjunctions 127 | return np.array(result).T 128 | 129 | 130 | def compute_total_number_of_literals(n_formulas, n_conjunctions_arr, conjunctions_depth_arr): 131 | return (np.sum(n_conjunctions_arr) * np.sum(conjunctions_depth_arr) * n_formulas) // (len(conjunctions_depth_arr) * len(n_conjunctions_arr)) 132 | 133 | 134 | def compute_total_number_of_conjunctions(n_formulas, n_conjunctions_arr): 135 | return (n_formulas // len(n_conjunctions_arr)) * np.sum(n_conjunctions_arr) 136 | 137 | 138 | def compute_n_literals_per_formula(n_conjunctions_arr, conjunctions_depth_arr): 139 | n_literals_per_formula_arr = [] 140 | for n_conjunctions in n_conjunctions_arr: 141 | n_literals_per_formula_arr.append((n_conjunctions // len(conjunctions_depth_arr)) * np.sum(conjunctions_depth_arr)) 142 | return n_literals_per_formula_arr 143 | 144 | 145 | ################################################################################################################################################################ 146 | # General functions 147 | ################################################################################################################################################################ 148 | def denseNDArrayToSparseTensor(arr): 149 | if arr.dtype == bool: 150 | idx = np.where(arr != False) 151 | else: 152 | idx = np.where(arr != 0.0) 153 | 154 | if arr.dtype == bool: 155 | idx = np.vstack(idx).T 156 | return tf.SparseTensor(idx, np.ones(idx.shape[0]).astype('float32'), arr.shape) 157 | else: 158 | return tf.SparseTensor(np.vstack(idx).T, arr[idx], arr.shape) 159 | 160 | 161 | def tf_matmul_sparse_dense(sparse_A, dense_B): 162 | return tf.transpose(tf.sparse.matmul(tf.sparse.transpose(sparse_A), tf.transpose(dense_B))) 163 | -------------------------------------------------------------------------------- /Utils/grid_search_utils.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import os 3 | import time 4 | from multiprocessing import Queue, Process, Manager 5 | 6 | import numpy as np 7 | import pandas as pd 8 | from Utils.file_utils import delete_file, create_dir 9 | from data.data_utils import read_train_val_test_folds 10 | from scipy.stats import sem 11 | 12 | 13 | def is_nan(x): 14 | return x is np.nan or x != x 15 | 16 | 17 | def get_best_val_result(all_results, is_increasing): 18 | best_val_score = np.NINF if is_increasing else np.Inf 19 | best_val_experiment = None 20 | correspond_test_score = None 21 | for res in all_results: 22 | if res['validation_score'] is None: 23 | continue 24 | 25 | if is_increasing: 26 | if res['validation_score'] > best_val_score: 27 | best_val_score = res['validation_score'] 28 | correspond_test_score = res['test_score'] 29 | best_val_experiment = res['experiment_number'] 30 | else: 31 | if res['validation_score'] < best_val_score: 32 | best_val_score = res['validation_score'] 33 | correspond_test_score = res['test_score'] 34 | best_val_experiment = res['experiment_number'] 35 | return best_val_score, best_val_experiment, correspond_test_score 36 | 37 | 38 | def training_worker(model_handler, config, train_args, return_dict): 39 | res = model_handler.train_and_test(config, **train_args) 40 | return_dict['res'] = res 41 | 42 | 43 | def worker(model_handler, config, train_args, input_queue, output_queue, gpu_index): 44 | config['GPU'] = gpu_index 45 | 46 | while not input_queue.empty(): 47 | try: 48 | c = input_queue.get_nowait() 49 | except Exception as e: 50 | if input_queue.empty(): 51 | print('Done, no more configurations') 52 | exit(-1) 53 | else: 54 | continue 55 | 56 | for key, value in c.items(): 57 | config[key] = value 58 | 59 | if config['model_name'] == 'DNFNet': 60 | if config['competition_name'] == 'Gas': 61 | if config['n_formulas'] == 3072: 62 | config['batch_size'] = 1024 63 | else: 64 | config['batch_size'] = 2048 65 | if config['competition_name'] == 'SantanderTransaction': 66 | if config['n_formulas'] == 3072 or config['n_formulas'] == 2048: 67 | config['batch_size'] = 1024 68 | else: 69 | config['batch_size'] = 2048 70 | 71 | manager = Manager() 72 | return_dict = manager.dict() 73 | p = Process(target=training_worker, args=(model_handler, config, train_args, return_dict)) 74 | p.start() 75 | p.join() 76 | res = return_dict['res'] 77 | output_queue.put({'res': res, 'config': c}) 78 | 79 | print('Done, no more configurations') 80 | exit(-1) 81 | 82 | 83 | def some_worker_is_alive(worker_list): 84 | for p in worker_list: 85 | if p.is_alive(): 86 | return True 87 | return False 88 | 89 | 90 | def create_all_permutations(grid_params, permutations): 91 | if len(grid_params) == 0: 92 | return permutations 93 | key = next(iter(grid_params)) 94 | param_list = grid_params[key] 95 | del grid_params[key] 96 | new_permutations = [] 97 | for p in permutations: 98 | for val in param_list: 99 | new_p = p.copy() 100 | new_p[key] = val 101 | new_permutations.append(new_p) 102 | return create_all_permutations(grid_params, new_permutations) 103 | 104 | 105 | def validate_results(df): 106 | for _, res in df.iterrows(): 107 | if is_nan(res['validation_score']) or is_nan(res['test_score']): 108 | return False 109 | return True 110 | 111 | 112 | def distributed_grid_search(GPUs, model_handler, config, grid_parmas, name, train_args, output_dir): 113 | all_configurations = create_all_permutations(grid_parmas, [{}]) 114 | input_queue = Queue(maxsize=len(all_configurations) + 1) 115 | output_queue = Queue(maxsize=len(all_configurations) + 1) 116 | process_list = [] 117 | 118 | exp_i = config['experiment_number'] 119 | for c in all_configurations: 120 | c['experiment_number'] = exp_i 121 | input_queue.put(c) 122 | exp_i += 1 123 | 124 | for i in range(len(GPUs)): 125 | p = Process(target=worker, args=(model_handler, config.copy(), train_args, input_queue, output_queue, GPUs[i])) 126 | time.sleep(1) # for queue issues 127 | p.start() 128 | process_list.append(p) 129 | 130 | all_results = [] 131 | csv_lines = [] 132 | i = 1 133 | cols = list(all_configurations[0].keys()) 134 | while not output_queue.empty() or (len(all_results) != len(all_configurations) and some_worker_is_alive(process_list)): 135 | res = output_queue.get() 136 | res['res']['experiment_number'] = res['config']['experiment_number'] 137 | all_results.append(res['res']) 138 | res_i = dict() 139 | for key, value in res['res'].items(): 140 | res_i[key] = value 141 | for key, value in res['config'].items(): 142 | res_i[key] = value 143 | csv_lines.append(res_i) 144 | df = pd.DataFrame(csv_lines) 145 | df.to_csv('{}/{}_{}.csv'.format(output_dir, name, i), columns=cols + list(res['res'].keys())) 146 | delete_file('{}/{}_{}.csv'.format(output_dir, name, i - 1)) 147 | i += 1 148 | 149 | for p in process_list: 150 | p.join() 151 | 152 | is_valid = validate_results(df) 153 | return all_results, is_valid 154 | 155 | 156 | def model_handler_params(data, score_config): 157 | return {'data': data, 'score_config': score_config} 158 | 159 | 160 | def write_cv_results(all_test_scores, all_val_scores, all_test_experiments, output_dir, name=''): 161 | print('all test scores:') 162 | test_res_csv_rows = [] 163 | for test_score, val_score, exp_number in zip(all_test_scores, all_val_scores, all_test_experiments): 164 | print('experiment number: {}, test score: {}'.format(exp_number, test_score)) 165 | test_res_csv_rows.append({ 166 | 'experiment': exp_number, 167 | 'test_score': test_score, 168 | 'val_score': val_score, 169 | 'mean test': None, 170 | 'sem test': None, 171 | 'mean val': None, 172 | 'sem val': None, 173 | }) 174 | 175 | test_scores_mean = np.mean(all_test_scores) 176 | test_scores_sem = sem(all_test_scores) 177 | print('mean of test scores: {}'.format(test_scores_mean)) 178 | print('sem: {}'.format(test_scores_sem)) 179 | 180 | test_res_csv_rows.append({ 181 | 'experiment': None, 182 | 'test_score': None, 183 | 'val_score': None, 184 | 'mean test': test_scores_mean, 185 | 'sem test': test_scores_sem, 186 | 'mean val': np.mean(all_val_scores), 187 | 'sem val': sem(all_val_scores), 188 | }) 189 | 190 | df = pd.DataFrame(test_res_csv_rows) 191 | df.to_csv(os.path.join(output_dir, 'test_scores{}.csv'.format(name))) 192 | 193 | 194 | def cross_validation(config, model_handler, score_config, folds, gpus_list, grid_params, seeds_arr, output_dir='./grid_search'): 195 | create_dir(output_dir) 196 | 197 | if 'experiment_number' not in config: 198 | config['experiment_number'] = 0 199 | 200 | valid_list = [] 201 | for seed in seeds_arr: 202 | all_test_scores = [] 203 | all_val_scores = [] 204 | all_test_experiments = [] 205 | for k in folds: 206 | config['random_seed'] = seed 207 | data = read_train_val_test_folds(config['csv'], k, apply_standardization=config['apply_standardization']) 208 | 209 | all_results, is_valid = distributed_grid_search(gpus_list, model_handler, copy.deepcopy(config), 210 | grid_parmas=copy.deepcopy(grid_params), 211 | name='iter_{}_seed_{}_conf'.format(k, seed), 212 | train_args=model_handler_params(data, score_config), 213 | output_dir=output_dir) 214 | config['experiment_number'] += len(all_results) 215 | 216 | best_val_score, best_val_experiment, correspond_test_score = get_best_val_result(all_results, score_config['score_increases']) 217 | all_test_scores.append(correspond_test_score) 218 | all_val_scores.append(best_val_score) 219 | all_test_experiments.append(best_val_experiment) 220 | valid_list.append({'fold': k, 'seed': seed, 'is valid': is_valid}) 221 | 222 | write_cv_results(all_test_scores, all_val_scores, all_test_experiments, output_dir, name='_{}'.format(str(seed))) 223 | 224 | for valid_desc in valid_list: 225 | print(valid_desc) 226 | -------------------------------------------------------------------------------- /data/data_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pandas as pd 3 | import numpy as np 4 | from DNFNet.Competitions.Otto.CompetitionConfig import dataset_handler as otto_dataset_handler 5 | from DNFNet.Competitions.SantanderTransaction.CompetitionConfig import dataset_handler as santander_transaction_dataset_handler 6 | from DNFNet.Competitions.EyeMovements.CompetitionConfig import dataset_handler as eye_movements_dataset_handler 7 | from DNFNet.Competitions.GesturePhase.CompetitionConfig import dataset_handler as gesture_phase_dataset_handler 8 | from DNFNet.Competitions.Gas.CompetitionConfig import dataset_handler as gas_dataset_handler 9 | from DNFNet.Competitions.House.CompetitionConfig import dataset_handler as house_dataset_handler 10 | from DNFNet.Competitions.RobotNavigation.CompetitionConfig import dataset_handler as robot_dataset_handler 11 | 12 | from Utils.file_utils import create_dir 13 | from sklearn.model_selection import StratifiedKFold, train_test_split 14 | 15 | 16 | def read_train_val_test_folds(csv_path, fold_idx, apply_standardization=True): 17 | fold_dir = os.path.join(os.path.dirname(csv_path), 'cross_validation', 'fold_{}'.format(fold_idx)) 18 | train_df = pd.read_csv(os.path.join(fold_dir, 'train.csv')) 19 | val_df = pd.read_csv(os.path.join(fold_dir, 'val.csv')) 20 | test_df = pd.read_csv(os.path.join(fold_dir, 'test.csv')) 21 | 22 | X_train = train_df.iloc[:, :-1].values 23 | Y_train = train_df.iloc[:, -1].values 24 | X_val = val_df.iloc[:, :-1].values 25 | Y_val = val_df.iloc[:, -1].values 26 | X_test = test_df.iloc[:, :-1].values 27 | Y_test = test_df.iloc[:, -1].values 28 | 29 | if apply_standardization: 30 | mean = np.mean(X_train, axis=0) 31 | std = np.std(X_train, axis=0) 32 | X_train = (X_train - mean) / std 33 | X_val = (X_val - mean) / std 34 | X_test = (X_test - mean) / std 35 | 36 | data = dict() 37 | data['X_train'] = X_train.astype('float32') 38 | data['X_val'] = X_val.astype('float32') 39 | data['X_test'] = X_test.astype('float32') 40 | data['Y_train'] = Y_train.astype('float32') 41 | data['Y_val'] = Y_val.astype('float32') 42 | data['Y_test'] = Y_test.astype('float32') 43 | return data 44 | 45 | 46 | class DatasetHandler: 47 | # Competitions 48 | GAS = 'Gas' 49 | OTTO = 'Otto' 50 | EYE_MOVEMENTS = 'EyeMovements' 51 | GESTURE_PHASE = 'GesturePhase' 52 | SANTANDER_TRANSACTION = 'SantanderTransaction' 53 | HOUSE = 'House' 54 | ROBOT_NAVIGATION = 'RobotNavigation' 55 | 56 | 57 | @staticmethod 58 | def get_dataset_handler(competition_name): 59 | if competition_name == DatasetHandler.OTTO: 60 | return otto_dataset_handler 61 | elif competition_name == DatasetHandler.SANTANDER_TRANSACTION: 62 | return santander_transaction_dataset_handler 63 | elif competition_name == DatasetHandler.EYE_MOVEMENTS: 64 | return eye_movements_dataset_handler 65 | elif competition_name == DatasetHandler.GESTURE_PHASE: 66 | return gesture_phase_dataset_handler 67 | elif competition_name == DatasetHandler.GAS: 68 | return gas_dataset_handler 69 | elif competition_name == DatasetHandler.HOUSE: 70 | return house_dataset_handler 71 | elif competition_name == DatasetHandler.ROBOT_NAVIGATION: 72 | return robot_dataset_handler 73 | 74 | 75 | def create_dataset_partitions(csv_path, dataset_name=None, train_proportion=0.7, k_folds=5, seed=1): 76 | print(csv_path) 77 | np.random.seed(seed=seed) 78 | output_dir = os.path.dirname(csv_path) + '/cross_validation' 79 | create_dir(output_dir) 80 | create_dir(output_dir + '/StratifiedKFold') 81 | create_dir(output_dir + '/seed_{}'.format(seed)) 82 | 83 | df = pd.read_csv(csv_path) 84 | if dataset_name is not None: 85 | dataset_handler = DatasetHandler.get_dataset_handler(dataset_name) 86 | df = dataset_handler(df) 87 | df = df.sample(frac=1, random_state=seed).reset_index(drop=True) 88 | y = df.iloc[:, -1].values 89 | kf = StratifiedKFold(n_splits=k_folds, shuffle=True, random_state=seed) 90 | val_proportion = 1 - (1. / k_folds) - train_proportion 91 | for i, (train_val_idx, test_idx) in enumerate(kf.split(df, y)): 92 | df_i = df.copy() 93 | test_df = df_i.iloc[test_idx] 94 | train_val_df = df_i.iloc[train_val_idx] 95 | y_train_val = train_val_df.iloc[:, -1].values 96 | train_df, val_df, _, _ = train_test_split(train_val_df, y_train_val, random_state=seed, stratify=y_train_val, 97 | test_size=val_proportion / (val_proportion + train_proportion)) 98 | 99 | fold_dir = os.path.join(output_dir, 'fold_{}'.format(i)) 100 | create_dir(fold_dir) 101 | train_df.to_csv(os.path.join(fold_dir, 'train.csv'), index=False) 102 | val_df.to_csv(os.path.join(fold_dir, 'val.csv'), index=False) 103 | test_df.to_csv(os.path.join(fold_dir, 'test.csv'), index=False) 104 | 105 | 106 | ################################################################################################################################################################################################ 107 | # Feature selection - syn data generation 108 | ################################################################################################################################################################################################ 109 | ''' 110 | Based on the work of 'Jinsung Yoon': https://github.com/jsyoon0823/INVASE/blob/master/data_generation.py 111 | 112 | 113 | Written by Jinsung Yoon 114 | INVASE: Instance-wise Variable Selection using Neural Networks Implementation on Synthetic Datasets 115 | Reference: J. Yoon, J. Jordon, M. van der Schaar, "IINVASE: Instance-wise Variable Selection using Neural Networks," International Conference on Learning Representations (ICLR), 2019. 116 | Paper Link: https://openreview.net/forum?id=BJg_roAcK7 117 | Contact: jsyoon0823@g.ucla.edu 118 | --------------------------------------------------- 119 | Generating Synthetic Data for Synthetic Examples 120 | There are 6 Synthetic Datasets 121 | X ~ N(0,I) where d = 100 122 | Y = 1/(1+logit) 123 | - Syn1: logit = exp(X1 * X2) 124 | - Syn2: logit = exp(X3^2 + X4^2 + X5^2 + X6^2 -4) 125 | - Syn3: logit = -10 sin(2 * X7) + 2|X8| + X9 + exp(-X10) - 2.4 126 | - Syn4: If X11 < 0, Syn1, X11 >= Syn2 127 | - Syn5: If X11 < 0, Syn1, X11 >= Syn3 128 | - Syn6: If X11 < 0, Syn2, X11 >= Syn3 129 | ''' 130 | 131 | 132 | # %% Basic Label Generation (Syn1, Syn2, Syn3) 133 | def Basic_Label_Generation(X, data_type): 134 | # number of samples 135 | n = len(X[:, 0]) 136 | 137 | # Logit computation 138 | # 1. Syn1 139 | if data_type == 'Syn1': 140 | logit = np.exp(X[:, 0] * X[:, 1]) 141 | 142 | # 2. Syn2 143 | elif data_type == 'Syn2': 144 | logit = np.exp(np.sum(X[:, 2:6] ** 2, axis=1) - 4.0) 145 | 146 | # 3. Syn3 147 | elif data_type == 'Syn3': 148 | logit = np.exp(-10 * np.sin(0.2 * X[:, 6]) + abs(X[:, 7]) + X[:, 8] + np.exp(-X[:, 9]) - 2.4) 149 | 150 | # P(Y=1|X) & P(Y=0|X) 151 | prob_1 = np.reshape((1 / (1 + logit)), [n, 1]) 152 | prob_0 = np.reshape((logit / (1 + logit)), [n, 1]) 153 | 154 | # Probability output 155 | prob_y = np.concatenate((prob_0, prob_1), axis=1) 156 | 157 | # Sampling from the probability 158 | y = np.zeros([n, 2]) 159 | y[:, 0] = np.reshape(np.random.binomial(1, prob_0), [n, ]) 160 | y[:, 1] = 1 - y[:, 0] 161 | 162 | return y[:, 1] 163 | 164 | 165 | # %% Complex Label Generation (Syn4, Syn5, Syn6) 166 | def Complex_Label_Generation(X, data_type): 167 | # number of samples 168 | n = len(X[:, 0]) 169 | 170 | # Logit generation 171 | # 1. Syn4 172 | if data_type == 'Syn4': 173 | logit1 = np.exp(X[:, 0] * X[:, 1]) 174 | logit2 = np.exp(np.sum(X[:, 2:6] ** 2, axis=1) - 4.0) 175 | 176 | # 2. Syn5 177 | elif data_type == 'Syn5': 178 | logit1 = np.exp(X[:, 0] * X[:, 1]) 179 | logit2 = np.exp(-10 * np.sin(0.2 * X[:, 6]) + abs(X[:, 7]) + X[:, 8] + np.exp(-X[:, 9]) - 2.4) 180 | 181 | # 3. Syn6 182 | elif data_type == 'Syn6': 183 | logit1 = np.exp(np.sum(X[:, 2:6] ** 2, axis=1) - 4.0) 184 | logit2 = np.exp(-10 * np.sin(0.2 * X[:, 6]) + abs(X[:, 7]) + X[:, 8] + np.exp(-X[:, 9]) - 2.4) 185 | 186 | # Based on X[:,10], combine two logits 187 | idx1 = (X[:, 10] < 0) * 1 188 | idx2 = (X[:, 10] >= 0) * 1 189 | 190 | logit = logit1 * idx1 + logit2 * idx2 191 | 192 | # P(Y=1|X) & P(Y=0|X) 193 | prob_1 = np.reshape((1 / (1 + logit)), [n, 1]) 194 | prob_0 = np.reshape((logit / (1 + logit)), [n, 1]) 195 | 196 | # Probability output 197 | prob_y = np.concatenate((prob_0, prob_1), axis=1) 198 | 199 | # Sampling from the probability 200 | y = np.zeros([n, 2]) 201 | y[:, 0] = np.reshape(np.random.binomial(1, prob_0), [n, ]) 202 | y[:, 1] = 1 - y[:, 0] 203 | 204 | return y[:, 1] 205 | 206 | 207 | # %% Ground truth Variable Importance 208 | def Ground_Truth_Mask_Generation(n_features, data_type): 209 | # mask initialization 210 | m = np.zeros(n_features) 211 | 212 | # For each data_type 213 | # Simple 214 | if data_type in ['Syn1', 'Syn2', 'Syn3']: 215 | if data_type == 'Syn1': 216 | m[:2] = 1 217 | elif data_type == 'Syn2': 218 | m[2:6] = 1 219 | elif data_type == 'Syn3': 220 | m[6:10] = 1 221 | 222 | # Complex 223 | if data_type in ['Syn4', 'Syn5', 'Syn6']: 224 | if data_type == 'Syn4': 225 | m[:2] = 1 226 | m[2:6] = 1 227 | elif data_type == 'Syn5': 228 | m[:2] = 1 229 | m[6:10] = 1 230 | elif data_type == 'Syn6': 231 | m[2:6] = 1 232 | m[6:10] = 1 233 | m[10] = 1 234 | return m 235 | 236 | 237 | # %% Generate X and Y 238 | def generate_data(n=10000, d=11, data_type='Syn1', seed=1, output_dir='.'): 239 | """ 240 | :param n: Number of samples 241 | :param d: input dimension 242 | :param data_type: the name of the syn dataset 243 | :param seed: random seed for numpy 244 | """ 245 | 246 | np.random.seed(seed) 247 | 248 | # X generation 249 | X = np.random.randn(n, d) 250 | 251 | # Y generation 252 | if data_type in ['Syn1', 'Syn2', 'Syn3']: 253 | Y = Basic_Label_Generation(X, data_type) 254 | 255 | elif data_type in ['Syn4', 'Syn5', 'Syn6']: 256 | Y = Complex_Label_Generation(X, data_type) 257 | 258 | data = np.concatenate([X, np.expand_dims(Y, axis=1)], axis=1) 259 | output_dir = os.path.join(output_dir, '{}_{}'.format(data_type, str(d))) 260 | create_dir(output_dir) 261 | output_path = os.path.join(output_dir, 'data.csv') 262 | pd.DataFrame(data=data).to_csv(output_path, index=False) 263 | create_dataset_partitions(output_path) 264 | 265 | 266 | if __name__ == '__main__': 267 | create_dataset_partitions('PATH_TO_DATA/data/Otto/train.csv', dataset_name=DatasetHandler.OTTO) 268 | create_dataset_partitions('PATH_TO_DATA/data/SantanderTransaction/train.csv', dataset_name=DatasetHandler.SANTANDER_TRANSACTION) 269 | create_dataset_partitions('PATH_TO_DATA/data/OpenML/EyeMovements/EyeMovements.csv', dataset_name=DatasetHandler.EYE_MOVEMENTS) 270 | create_dataset_partitions('PATH_TO_DATA/data/OpenML/GesturePhase/GesturePhase.csv', dataset_name=DatasetHandler.GESTURE_PHASE) 271 | create_dataset_partitions('PATH_TO_DATA/data/OpenML/Gas/Gas.csv', dataset_name=DatasetHandler.GAS) 272 | create_dataset_partitions('PATH_TO_DATA/data/OpenML/House/house.csv', dataset_name=DatasetHandler.HOUSE) 273 | create_dataset_partitions('PATH_TO_DATA/data/OpenML/RobotNavigation/RobotNavigation.csv', dataset_name=DatasetHandler.ROBOT_NAVIGATION) 274 | 275 | syn_names = ['Syn1', 'Syn2', 'Syn3', 'Syn4', 'Syn5', 'Syn6'] 276 | d_arr = [11, 50, 100, 150, 200, 250, 300] 277 | for syn in syn_names: 278 | for d in d_arr: 279 | generate_data(d=d, data_type=syn, output_dir='FeatureSelectionSynExp') -------------------------------------------------------------------------------- /DNFNet/ModelHandler.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | 4 | import numpy as np 5 | import tensorflow as tf 6 | from Utils.NumpyGenerator import NumpyGenerator 7 | from Utils.experiment_utils import create_model, create_experiment_directory 8 | 9 | 10 | def get_if_exists(dict, key): 11 | if key in dict: 12 | return dict[key] 13 | return None 14 | 15 | 16 | def score_comparator_wrapper(score_increases): 17 | if score_increases: 18 | return lambda a, b: a > b 19 | else: 20 | return lambda a, b: a < b 21 | 22 | 23 | class EarlyStopping(object): 24 | def __init__(self, patience, score_increases, monitor='val_score'): 25 | if monitor == 'val_loss': 26 | score_increases = False 27 | 28 | self.patience = patience 29 | self.best_test_score = np.NINF if score_increases else np.inf 30 | self.score_comparator = score_comparator_wrapper(score_increases) 31 | self.score_not_improved_counter = 0 32 | self.monitor = monitor 33 | 34 | def _early_stop(self, score): 35 | if self.score_comparator(score, self.best_test_score): 36 | self.best_test_score = score 37 | self.score_not_improved_counter = 0 38 | return False 39 | 40 | self.score_not_improved_counter += 1 41 | if self.score_not_improved_counter >= self.patience: 42 | print("Early Stopping") 43 | return True 44 | return False 45 | 46 | def pre_training(self, model): 47 | pass 48 | 49 | def epoch_end(self, model): 50 | if self.monitor == 'val_score': 51 | model.early_stopping = self._early_stop(model.val_score) 52 | elif self.monitor == 'val_loss': 53 | model.early_stopping = self._early_stop(model.val_loss) 54 | 55 | 56 | class ReduceLRonPlateau(object): 57 | def __init__(self, initilal_lr, factor, patience, min_lr, monitor='val_loss'): 58 | self.lr = initilal_lr 59 | self.factor = factor 60 | self.patience = patience 61 | self.min_lr = min_lr 62 | self.min_loss = np.inf 63 | self.loss_not_reducing_counter = 0 64 | self.monitor = monitor 65 | 66 | def compute_lr(self, loss): 67 | if loss < self.min_loss: 68 | self.min_loss = loss 69 | self.loss_not_reducing_counter = 0 70 | return self.lr 71 | 72 | self.loss_not_reducing_counter += 1 73 | if self.loss_not_reducing_counter >= self.patience: 74 | self.loss_not_reducing_counter = 0 75 | self.lr = self.lr * self.factor 76 | if self.lr < self.min_lr: 77 | self.lr = self.min_lr 78 | 79 | print("Learning rate: {}".format(self.lr)) 80 | return self.lr 81 | 82 | def pre_training(self, model): 83 | model.current_lr = self.lr 84 | 85 | def epoch_end(self, model): 86 | if self.monitor == 'val_loss': 87 | model.current_lr = self.compute_lr(model.val_loss) 88 | elif self.monitor == 'train_loss': 89 | model.current_lr = self.compute_lr(model.train_loss) 90 | else: 91 | print('ReduceLRonPlateau: {} is not supported'.format(self.monitor)) 92 | 93 | 94 | class ModelHandler: 95 | def __init__(self, config, model, callbacks, target_dir, logs_dir=None): 96 | self.model = model 97 | self.config = config 98 | self.target_dir = target_dir 99 | self.logs_dir = logs_dir 100 | self.callbacks = callbacks 101 | self.n_labels = self.config['output_dim'] 102 | self.sess = None 103 | self.current_lr = self.config['initial_lr'] 104 | self.early_stopping = None 105 | self.val_loss = None 106 | self.val_score = None 107 | self.train_loss = None 108 | 109 | # placeholders 110 | self.x = None 111 | self.y = None 112 | self.y_weights = None 113 | self.lr = None 114 | self.dropout_rate = None 115 | self.is_training = None 116 | 117 | # returned vars from the model 118 | self.output = None 119 | self.features = None 120 | self.loss = None 121 | self.optimizer = None 122 | self.labels = None 123 | 124 | def define_placeholders(self): 125 | self.x = tf.placeholder(tf.float32, shape=[None, self.config['input_dim']], name="x") 126 | self.y = tf.placeholder(tf.float32, shape=[None, self.config['output_dim']], name="y") 127 | self.y_weights = tf.placeholder(tf.float32, shape=[None, 1], name="y_weights") 128 | self.lr = tf.placeholder(tf.float32, name="lr") 129 | self.dropout_rate = tf.placeholder(tf.float32, name="dropout_rate") 130 | self.is_training = tf.placeholder(tf.bool, name="is_training") 131 | placeholders = { 132 | 'x': self.x, 133 | 'y': self.y, 134 | 'y_weights': self.y_weights, 135 | 'lr': self.lr, 136 | 'dropout_rate': self.dropout_rate, 137 | 'is_training': self.is_training, 138 | } 139 | return placeholders 140 | 141 | def build_graph(self, load_weights=False, phase='train'): 142 | placeholders = self.define_placeholders() 143 | build_dict = self.model.build(placeholders, n_labels=self.n_labels, phase=phase) 144 | self.optimizer = get_if_exists(build_dict, 'optimizer') 145 | self.loss = get_if_exists(build_dict, 'loss') 146 | self.output = get_if_exists(build_dict, 'output') 147 | self.features = get_if_exists(build_dict, 'features') 148 | self.labels = get_if_exists(build_dict, 'labels') 149 | 150 | self.sess = tf.Session() 151 | 152 | if load_weights: 153 | print('Loading weights') 154 | saver = tf.train.Saver(tf.global_variables()) 155 | saver.restore(self.sess, self.target_dir + '/model_weights.ckpt') 156 | 157 | def _translate_feed_dict(self, str_feed_dict): 158 | return {getattr(self, filed_name): value for filed_name, value in str_feed_dict.items()} 159 | 160 | def callbacks_handler(self, op): 161 | for callback in self.callbacks: 162 | if op == 'pre_training': 163 | callback.pre_training(self) 164 | elif op == 'epoch_end': 165 | callback.epoch_end(self) 166 | 167 | def train(self, train_generator, val_generator, score_metric, score_increases): 168 | self.sess.run(tf.global_variables_initializer()) 169 | saver = tf.train.Saver(tf.global_variables()) 170 | score_comparator = score_comparator_wrapper(score_increases) 171 | best_val_score = np.NINF if score_increases else np.inf 172 | self.early_stopping = False 173 | self.callbacks_handler(op='pre_training') 174 | 175 | try: 176 | for epoch in range(self.config['epochs']): 177 | 178 | train_loss_sum = 0 179 | while train_generator.get_epochs() <= epoch: 180 | str_feed_dict = train_generator.get_batch() 181 | feed_dict = self._translate_feed_dict(str_feed_dict) 182 | feed_dict[self.lr] = self.current_lr 183 | feed_dict[self.is_training] = True 184 | _, l = self.sess.run([self.optimizer, self.loss], feed_dict=feed_dict) 185 | train_loss_sum += l * str_feed_dict['x'].shape[0] 186 | self.train_loss = train_loss_sum / train_generator.get_dataset_size() 187 | 188 | val_loss_sum = 0 189 | y_true = None 190 | y_pred = None 191 | while val_generator.get_epochs() <= epoch: 192 | str_feed_dict = val_generator.get_batch() 193 | feed_dict = self._translate_feed_dict(str_feed_dict) 194 | feed_dict[self.lr] = self.current_lr 195 | feed_dict[self.is_training] = False 196 | l, pred = self.sess.run([self.loss, self.output], feed_dict=feed_dict) 197 | val_loss_sum += l * str_feed_dict['x'].shape[0] 198 | y_batch = str_feed_dict['y'] 199 | y_true = y_batch if y_true is None else np.concatenate([y_true, y_batch]) 200 | y_pred = pred if y_pred is None else np.concatenate([y_pred, pred]) 201 | self.val_loss = val_loss_sum / val_generator.get_dataset_size() 202 | self.val_score = score_metric(y_true, y_pred) 203 | assert val_generator.get_dataset_size() == y_true.shape[0] 204 | 205 | print("Epoch: {0:}, loss: {1:.6f}, val loss: {2:.6f}, score: {3:.6f}".format(epoch, self.train_loss, self.val_loss, self.val_score)) 206 | if score_comparator(self.val_score, best_val_score): 207 | print("Val score improved from {} to {}".format(best_val_score, self.val_score)) 208 | best_val_score = self.val_score 209 | if self.config['save_weights'] and epoch >= self.config['starting_epoch_to_save']: 210 | print('saving model weights.') 211 | saver.save(self.sess, os.path.join(self.target_dir, "model_weights.ckpt")) 212 | 213 | self.callbacks_handler(op='epoch_end') 214 | 215 | if self.early_stopping: 216 | break 217 | 218 | print("Best validation score: {}".format(best_val_score)) 219 | return best_val_score, epoch 220 | 221 | except tf.errors.ResourceExhaustedError: 222 | return None, epoch 223 | 224 | def test(self, test_generator): 225 | test_generator.reset() 226 | y = None 227 | y_pred = None 228 | while test_generator.get_epochs() < 1: 229 | str_feed_dict = test_generator.get_batch() 230 | feed_dict = self._translate_feed_dict(str_feed_dict) 231 | feed_dict[self.is_training] = False 232 | pred = self.sess.run(self.output, feed_dict=feed_dict) 233 | y_batch = str_feed_dict['y'] 234 | y = y_batch if y is None else np.concatenate([y, y_batch]) 235 | y_pred = pred if y_pred is None else np.concatenate([y_pred, pred]) 236 | 237 | assert test_generator.get_dataset_size() == y.shape[0] 238 | 239 | return y, y_pred 240 | 241 | @staticmethod 242 | def train_and_test(config, data, score_config): 243 | print('train {}'.format(config['model_name'])) 244 | os.environ["CUDA_VISIBLE_DEVICES"] = config['GPU'] 245 | tf.reset_default_graph() 246 | tf.random.set_random_seed(seed=config['random_seed']) 247 | np.random.seed(seed=config['random_seed']) 248 | score_metric = score_config['score_metric'] 249 | 250 | experiment_dir, weights_dir, logs_dir = create_experiment_directory(config, return_sub_dirs=True) 251 | model = create_model(config, models_module_name=config['models_module_name']) 252 | train_generator = NumpyGenerator(data['X_train'], data['Y_train'], config['output_dim'], config['batch_size'], translate_label_to_one_hot=config['translate_label_to_one_hot'], copy_dataset=False) 253 | val_generator = NumpyGenerator(data['X_val'], data['Y_val'], config['output_dim'], config['batch_size'], translate_label_to_one_hot=config['translate_label_to_one_hot'], copy_dataset=False) 254 | test_generator = NumpyGenerator(data['X_test'], data['Y_test'], config['output_dim'], config['batch_size'], translate_label_to_one_hot=config['translate_label_to_one_hot'], copy_dataset=False) 255 | 256 | early_stopping = EarlyStopping(patience=config['early_stopping_patience'], score_increases=score_config['score_increases'], monitor='val_score') 257 | lr_scheduler = ReduceLRonPlateau(initilal_lr=config['initial_lr'], factor=config['lr_decay_factor'], patience=config['lr_patience'], min_lr=config['min_lr'], monitor='train_loss') 258 | 259 | model_handler = ModelHandler(config=config, model=model, callbacks=[lr_scheduler, early_stopping], target_dir=weights_dir, logs_dir=logs_dir) 260 | model_handler.build_graph(phase='train') 261 | val_score, epoch = model_handler.train(train_generator, val_generator, score_metric=score_metric, score_increases=score_config['score_increases']) 262 | 263 | assert os.path.exists(model_handler.target_dir + '/model_weights.ckpt.meta') 264 | 265 | if os.path.exists(model_handler.target_dir + '/model_weights.ckpt.meta'): 266 | print('Loading weights') 267 | saver = tf.train.Saver(tf.global_variables()) 268 | saver.restore(model_handler.sess, model_handler.target_dir + '/model_weights.ckpt') 269 | 270 | y_true, y_pred = model_handler.test(test_generator) 271 | test_score = score_metric(y_true, y_pred) 272 | print('Test score: {}'.format(test_score)) 273 | 274 | model_handler.sess.close() 275 | shutil.rmtree(weights_dir, ignore_errors=True) 276 | return {'test_score': test_score, 'validation_score': val_score, 'n_epochs': epoch} 277 | --------------------------------------------------------------------------------