├── LICENSE ├── README.md ├── dataloaders ├── __pycache__ │ ├── dataset_loader.cpython-37.pyc │ ├── dataset_loader.cpython-38.pyc │ └── mnist_loader.cpython-38.pyc ├── adult_loader.py ├── adult_process.py ├── dataset_loader.py └── mnist_loader.py ├── metrics ├── Accuracy.py ├── Average.py ├── BCR.py ├── CV.py ├── CalibrationNeg.py ├── CalibrationPos.py ├── DIAvgAll.py ├── DIBinary.py ├── Diff.py ├── EqOppo_fn_diff.py ├── EqOppo_fn_ratio.py ├── EqOppo_fp_diff.py ├── EqOppo_fp_ratio.py ├── FNR.py ├── FPR.py ├── FilterSensitive.py ├── MCC.py ├── Metric.py ├── README.md ├── Ratio.py ├── SensitiveMetric.py ├── TNR.py ├── TPR.py ├── __init__.py ├── __pycache__ │ ├── Accuracy.cpython-36.pyc │ ├── Accuracy.cpython-37.pyc │ ├── Accuracy.cpython-38.pyc │ ├── Average.cpython-36.pyc │ ├── Average.cpython-37.pyc │ ├── Average.cpython-38.pyc │ ├── BCR.cpython-36.pyc │ ├── BCR.cpython-37.pyc │ ├── BCR.cpython-38.pyc │ ├── CV.cpython-36.pyc │ ├── CV.cpython-37.pyc │ ├── CV.cpython-38.pyc │ ├── CalibrationNeg.cpython-36.pyc │ ├── CalibrationNeg.cpython-37.pyc │ ├── CalibrationNeg.cpython-38.pyc │ ├── CalibrationPos.cpython-36.pyc │ ├── CalibrationPos.cpython-37.pyc │ ├── CalibrationPos.cpython-38.pyc │ ├── DIAvgAll.cpython-36.pyc │ ├── DIAvgAll.cpython-37.pyc │ ├── DIAvgAll.cpython-38.pyc │ ├── DIBinary.cpython-36.pyc │ ├── DIBinary.cpython-37.pyc │ ├── DIBinary.cpython-38.pyc │ ├── Diff.cpython-36.pyc │ ├── Diff.cpython-37.pyc │ ├── Diff.cpython-38.pyc │ ├── EqOppo_fn_diff.cpython-36.pyc │ ├── EqOppo_fn_diff.cpython-37.pyc │ ├── EqOppo_fn_diff.cpython-38.pyc │ ├── EqOppo_fn_ratio.cpython-36.pyc │ ├── EqOppo_fn_ratio.cpython-37.pyc │ ├── EqOppo_fn_ratio.cpython-38.pyc │ ├── EqOppo_fp_diff.cpython-36.pyc │ ├── EqOppo_fp_diff.cpython-37.pyc │ ├── EqOppo_fp_diff.cpython-38.pyc │ ├── EqOppo_fp_ratio.cpython-36.pyc │ ├── EqOppo_fp_ratio.cpython-37.pyc │ ├── EqOppo_fp_ratio.cpython-38.pyc │ ├── FNR.cpython-36.pyc │ ├── FNR.cpython-37.pyc │ ├── FNR.cpython-38.pyc │ ├── FPR.cpython-36.pyc │ ├── FPR.cpython-37.pyc │ ├── FPR.cpython-38.pyc │ ├── FilterSensitive.cpython-36.pyc │ ├── FilterSensitive.cpython-37.pyc │ ├── FilterSensitive.cpython-38.pyc │ ├── MCC.cpython-36.pyc │ ├── MCC.cpython-37.pyc │ ├── MCC.cpython-38.pyc │ ├── Metric.cpython-36.pyc │ ├── Metric.cpython-37.pyc │ ├── Metric.cpython-38.pyc │ ├── Ratio.cpython-36.pyc │ ├── Ratio.cpython-37.pyc │ ├── Ratio.cpython-38.pyc │ ├── SensitiveMetric.cpython-36.pyc │ ├── SensitiveMetric.cpython-37.pyc │ ├── SensitiveMetric.cpython-38.pyc │ ├── TNR.cpython-36.pyc │ ├── TNR.cpython-37.pyc │ ├── TNR.cpython-38.pyc │ ├── TPR.cpython-36.pyc │ ├── TPR.cpython-37.pyc │ ├── TPR.cpython-38.pyc │ ├── __init__.cpython-36.pyc │ ├── __init__.cpython-37.pyc │ ├── __init__.cpython-38.pyc │ ├── list.cpython-36.pyc │ ├── list.cpython-37.pyc │ ├── list.cpython-38.pyc │ ├── utils.cpython-36.pyc │ ├── utils.cpython-37.pyc │ └── utils.cpython-38.pyc ├── list.py └── utils.py ├── models ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-37.pyc │ ├── __init__.cpython-38.pyc │ ├── model_cfair.cpython-37.pyc │ ├── model_cfair.cpython-38.pyc │ ├── model_conv.cpython-37.pyc │ ├── model_conv.cpython-38.pyc │ ├── model_ffvae.cpython-37.pyc │ ├── model_ffvae.cpython-38.pyc │ ├── model_laftr.cpython-37.pyc │ ├── model_laftr.cpython-38.pyc │ ├── model_mlp.cpython-37.pyc │ ├── model_mlp.cpython-38.pyc │ ├── model_mubal.cpython-37.pyc │ └── model_mubal.cpython-38.pyc ├── ablations │ ├── __init__.py │ ├── model_ffvae_cfair.py │ └── model_ffvae_laftr.py ├── model_cfair.py ├── model_conv.py ├── model_ffvae.py ├── model_laftr.py └── model_mlp.py ├── requirements.txt ├── scripts ├── Adult │ └── settings12 │ │ ├── cfair_multiple_launcher.py │ │ ├── ffvae_multiple_launcher.py │ │ ├── laftr_multiple_launcher.py │ │ └── mlp_multiple_launcher.py ├── CIMNIST │ ├── settings12 │ │ ├── cfair_multiple_launcher.py │ │ ├── ffvae_multiple_launcher.py │ │ ├── laftr_multiple_launcher.py │ │ └── mlp_conv_multiple_launcher.py │ └── settings34 │ │ ├── cfair_multiple_launcher.py │ │ ├── ffvae_multiple_launcher.py │ │ ├── laftr_multiple_launcher.py │ │ └── mlp_conv_multiple_launcher.py ├── ablations │ ├── dataset_replication │ │ ├── conv_multiple_launcher.py │ │ └── laftr_multiple_launcher.py │ └── ffvae_ablations │ │ ├── ffvae_cfair_multiple_launcher.py │ │ └── ffvae_laftr_multiple_launcher.py ├── plots_tables │ ├── csvs │ │ ├── adult.csv │ │ ├── mnist.csv │ │ └── parse_csv.py │ ├── plots │ │ ├── barplots_adult.py │ │ ├── barplots_mnist.py │ │ ├── heatmap_corr.py │ │ ├── heatmap_std_adult.py │ │ └── heatmap_std_mnist.py │ └── tables │ │ └── get_tables.py ├── slurm_launcher.sh └── wandb_init.json ├── train ├── __pycache__ │ └── train_mlp.cpython-38.pyc ├── execute.py └── train.py └── utils ├── __init__.py ├── __pycache__ ├── __init__.cpython-37.pyc ├── log_utils.cpython-37.pyc └── options.cpython-37.pyc ├── early_stopping.py ├── log_utils.py ├── media ├── metrics.png ├── results_adult.PNG └── results_mnist.PNG ├── options.py └── utils.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Charan Reddy 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /dataloaders/__pycache__/dataset_loader.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/dataloaders/__pycache__/dataset_loader.cpython-37.pyc -------------------------------------------------------------------------------- /dataloaders/__pycache__/dataset_loader.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/dataloaders/__pycache__/dataset_loader.cpython-38.pyc -------------------------------------------------------------------------------- /dataloaders/__pycache__/mnist_loader.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/dataloaders/__pycache__/mnist_loader.cpython-38.pyc -------------------------------------------------------------------------------- /dataloaders/adult_loader.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import pandas as pd 4 | import torch 5 | from torch.utils.data import Dataset 6 | from dataloaders.adult_process import get_adult_data 7 | 8 | 9 | class AdultDataset(Dataset): 10 | """ 11 | The UCI Adult dataset. 12 | """ 13 | 14 | def __init__(self, root_dir, phase, tar_attr, priv_attr, clr_ratio): 15 | self.tar_attr = tar_attr 16 | self.priv_attr = priv_attr 17 | 18 | self.data = get_adult_data(tar_attr, priv_attr, clr_ratio) 19 | if phase not in ["train", "val", "test"]: 20 | raise NotImplementedError 21 | 22 | if phase == "train": 23 | self.X = self.data[f"x_train"][self.data["train_inds"]] 24 | self.Y = self.data[f"y_train"][self.data["train_inds"]] 25 | self.A = self.data[f"attr_train"][self.data["train_inds"]] 26 | elif phase == "val": 27 | self.X = self.data[f"x_train"][self.data["valid_inds"]] 28 | self.Y = self.data[f"y_train"][self.data["valid_inds"]] 29 | self.A = self.data[f"attr_train"][self.data["valid_inds"]] 30 | elif phase == "test": 31 | self.X = self.data[f"x_test"] 32 | self.Y = self.data[f"y_test"] 33 | self.A = self.data[f"attr_test"] 34 | else: 35 | raise Exception("Wrong phase") 36 | 37 | self.input_shape = self.X.shape 38 | self.num_samples = self.input_shape[0] 39 | self.xdim = self.X.shape[1] 40 | self.ydim = 1 41 | self.adim = 1 42 | 43 | def __len__(self): 44 | return len(self.X) 45 | 46 | def __getitem__(self, idx): 47 | if self.ydim == 1 and len(self.Y.shape) == 2: # binary classification 48 | return ( 49 | torch.from_numpy(self.X[idx]).float(), 50 | torch.from_numpy(self.Y[idx]), 51 | torch.from_numpy(self.A[idx]), 52 | ) 53 | raise NotImplementedError 54 | 55 | def onehot_2_int(self, ts): 56 | if len(ts.shape) == 2: 57 | return torch.argmax(ts, dim=1) 58 | if len(ts.shape) == 1: 59 | return torch.argmax(ts, dim=0) 60 | raise NotImplementedError 61 | 62 | def get_A_proportions(self): 63 | """for catergorical attribute""" 64 | assert len(self.A.shape) == 2 65 | num_class = self.A.shape[1] 66 | 67 | A_label = np.argmax(self.A, axis=1) 68 | A_proportions = [] 69 | for cls_idx in range(num_class): 70 | A_proportion = np.sum(cls_idx == A_label) 71 | A_proportions.append(A_proportion) 72 | A_proportions = [a_prop * 1.0 / len(A_label) for a_prop in A_proportions] 73 | return A_proportions 74 | 75 | def get_Y_proportions(self): 76 | """for catergorical attribute""" 77 | assert len(self.Y.shape) == 2 78 | num_class = self.Y.shape[1] 79 | 80 | Y_label = np.argmax(self.Y, axis=1) 81 | Y_proportions = [] 82 | for cls_idx in range(num_class): 83 | Y_proportion = np.sum(cls_idx == Y_label) 84 | Y_proportions.append(Y_proportion) 85 | Y_proportions = [y_prop * 1.0 / len(Y_label) for y_prop in Y_proportions] 86 | return Y_proportions 87 | 88 | def get_AY_proportions(self): 89 | """for catergorical attributes""" 90 | assert len(self.Y.shape) == len(self.A.shape) == 2 91 | A_num_class = self.A.shape[1] 92 | Y_num_class = self.Y.shape[1] 93 | A_label = np.argmax(self.A, axis=1) 94 | Y_label = np.argmax(self.Y, axis=1) 95 | AY_proportions = [] 96 | for A_cls_idx in range(A_num_class): 97 | Y_proportions = [] 98 | for Y_cls_idx in range(Y_num_class): 99 | AY_proprtion = np.sum( 100 | np.logical_and(Y_cls_idx == Y_label, A_cls_idx == A_label) 101 | ) 102 | Y_proportions.append(AY_proprtion) 103 | Y_proportions = [y_prop * 1.0 / len(Y_label) for y_prop in Y_proportions] 104 | AY_proportions.append(Y_proportions) 105 | return AY_proportions 106 | -------------------------------------------------------------------------------- /metrics/Accuracy.py: -------------------------------------------------------------------------------- 1 | from metrics.Metric import Metric 2 | from sklearn.metrics import accuracy_score 3 | 4 | 5 | class Accuracy(Metric): 6 | """returns the fraction of correctly classified samples (float)""" 7 | 8 | def __init__(self): 9 | Metric.__init__(self) 10 | self.name = "accuracy" 11 | 12 | def calc( 13 | self, 14 | actual, 15 | predicted, 16 | dict_of_sensitive_lists, 17 | single_sensitive_name, 18 | unprotected_vals, 19 | positive_pred, 20 | ): 21 | 22 | return accuracy_score(actual, predicted) 23 | -------------------------------------------------------------------------------- /metrics/Average.py: -------------------------------------------------------------------------------- 1 | from metrics.Metric import Metric 2 | 3 | 4 | class Average(Metric): 5 | """ 6 | Takes the average (mean) of a given list of metrics. Assumes that if the total over all 7 | metrics is 0, the returned result should be 1. 8 | """ 9 | 10 | def __init__(self, metrics_list, name): 11 | Metric.__init__(self) 12 | self.name = name 13 | self.metrics = metrics_list 14 | 15 | def calc( 16 | self, 17 | actual, 18 | predicted, 19 | dict_of_sensitive_lists, 20 | single_sensitive_name, 21 | unprotected_vals, 22 | positive_pred, 23 | ): 24 | 25 | total = 0.0 26 | for metric in self.metrics: 27 | result = metric.calc( 28 | actual, 29 | predicted, 30 | dict_of_sensitive_lists, 31 | single_sensitive_name, 32 | unprotected_vals, 33 | positive_pred, 34 | ) 35 | if result != None: 36 | total += result 37 | 38 | if total == 0.0: 39 | return 1.0 40 | 41 | return total / len(self.metrics) 42 | 43 | def is_better_than(self, val1, val2): 44 | return self.metrics[0].is_better_than(val1, val2) 45 | -------------------------------------------------------------------------------- /metrics/BCR.py: -------------------------------------------------------------------------------- 1 | from metrics.Metric import Metric 2 | from metrics.TNR import TNR 3 | from metrics.TPR import TPR 4 | 5 | 6 | class BCR(Metric): 7 | def __init__(self): 8 | Metric.__init__(self) 9 | self.name = "BCR" 10 | 11 | def calc( 12 | self, 13 | actual, 14 | predicted, 15 | dict_of_sensitive_lists, 16 | single_sensitive_name, 17 | unprotected_vals, 18 | positive_pred, 19 | ): 20 | tnr = TNR() 21 | tnr_val = tnr.calc( 22 | actual, 23 | predicted, 24 | dict_of_sensitive_lists, 25 | single_sensitive_name, 26 | unprotected_vals, 27 | positive_pred, 28 | ) 29 | tpr = TPR() 30 | tpr_val = tpr.calc( 31 | actual, 32 | predicted, 33 | dict_of_sensitive_lists, 34 | single_sensitive_name, 35 | unprotected_vals, 36 | positive_pred, 37 | ) 38 | bcr = (tpr_val + tnr_val) / 2.0 39 | return bcr 40 | -------------------------------------------------------------------------------- /metrics/CV.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import numpy 3 | import math 4 | 5 | from metrics.utils import calc_pos_protected_percents 6 | from metrics.Metric import Metric 7 | 8 | 9 | class CV(Metric): 10 | def __init__(self): 11 | Metric.__init__(self) 12 | self.name = "CV" 13 | 14 | def calc( 15 | self, 16 | actual, 17 | predicted, 18 | dict_of_sensitive_lists, 19 | single_sensitive_name, 20 | unprotected_vals, 21 | positive_pred, 22 | ): 23 | sensitive = dict_of_sensitive_lists[single_sensitive_name] 24 | unprotected_pos_percent, protected_pos_percent = calc_pos_protected_percents( 25 | predicted, sensitive, unprotected_vals, positive_pred 26 | ) 27 | CV = unprotected_pos_percent - protected_pos_percent 28 | return 1.0 - abs(CV) 29 | 30 | def is_better_than(self, val1, val2): 31 | dist1 = math.fabs(1.0 - val1) 32 | dist2 = math.fabs(1.0 - val2) 33 | return dist1 < dist2 34 | -------------------------------------------------------------------------------- /metrics/CalibrationNeg.py: -------------------------------------------------------------------------------- 1 | from metrics.Metric import Metric 2 | 3 | 4 | class CalibrationNeg(Metric): 5 | def __init__(self): 6 | Metric.__init__(self) 7 | self.name = "calibrationNeg" 8 | 9 | def calc( 10 | self, 11 | actual, 12 | predicted, 13 | dict_of_sensitive_lists, 14 | single_sensitive_name, 15 | unprotected_vals, 16 | positive_pred, 17 | ): 18 | total_pred_negative = 0.0 19 | act_correct = 0.0 20 | for act, pred in zip(actual, predicted): 21 | if pred != positive_pred: 22 | total_pred_negative += 1 23 | if act == positive_pred: 24 | act_correct += 1 25 | if act_correct == 0.0 and total_pred_negative == 0.0: 26 | return 1.0 27 | if total_pred_negative == 0.0: 28 | return 0.0 29 | return act_correct / total_pred_negative 30 | -------------------------------------------------------------------------------- /metrics/CalibrationPos.py: -------------------------------------------------------------------------------- 1 | from metrics.Metric import Metric 2 | 3 | 4 | class CalibrationPos(Metric): 5 | def __init__(self): 6 | Metric.__init__(self) 7 | self.name = "calibrationPos" 8 | 9 | def calc( 10 | self, 11 | actual, 12 | predicted, 13 | dict_of_sensitive_lists, 14 | single_sensitive_name, 15 | unprotected_vals, 16 | positive_pred, 17 | ): 18 | total_pred_positive = 0.0 19 | act_correct = 0.0 20 | for act, pred in zip(actual, predicted): 21 | if pred == positive_pred: 22 | total_pred_positive += 1 23 | if act == positive_pred: 24 | act_correct += 1 25 | if act_correct == 0.0 and total_pred_positive == 0.0: 26 | return 1.0 27 | if total_pred_positive == 0.0: 28 | return 0.0 29 | return act_correct / total_pred_positive 30 | -------------------------------------------------------------------------------- /metrics/DIAvgAll.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | from metrics.utils import calc_prob_class_given_sensitive 4 | from metrics.Metric import Metric 5 | 6 | 7 | class DIAvgAll(Metric): 8 | """ 9 | This metric calculates disparate imapct in the sense of the 80% rule before the 80% 10 | threshold is applied. This is described as DI in: https://arxiv.org/abs/1412.3756 11 | If there are no positive protected classifications, 0.0 is returned. 12 | 13 | If there are multiple protected classes, the average DI over all groups is returned. 14 | """ 15 | 16 | def __init__(self): 17 | Metric.__init__(self) 18 | self.name = "DIavgall" 19 | 20 | def calc( 21 | self, 22 | actual, 23 | predicted, 24 | dict_of_sensitive_lists, 25 | single_sensitive_name, 26 | unprotected_vals, 27 | positive_pred, 28 | ): 29 | sensitive = dict_of_sensitive_lists[single_sensitive_name] 30 | sensitive_values = list(set(sensitive)) 31 | 32 | if len(sensitive_values) <= 1: 33 | print( 34 | "ERROR: Attempted to calculate DI without enough sensitive values:" 35 | + str(sensitive_values) 36 | ) 37 | return 1.0 38 | 39 | # this list should only have one item in it 40 | single_unprotected = [ 41 | val for val in sensitive_values if val in unprotected_vals 42 | ][0] 43 | unprotected_prob = calc_prob_class_given_sensitive( 44 | predicted, sensitive, positive_pred, single_unprotected 45 | ) 46 | sensitive_values.remove(single_unprotected) 47 | total = 0.0 48 | for sens in sensitive_values: 49 | pos_prob = calc_prob_class_given_sensitive( 50 | predicted, sensitive, positive_pred, sens 51 | ) 52 | DI = 0.0 53 | if unprotected_prob > 0: 54 | DI = pos_prob / unprotected_prob 55 | if unprotected_prob == 0.0 and pos_prob == 0.0: 56 | DI = 1.0 57 | total += DI 58 | 59 | if total == 0.0: 60 | return 1.0 61 | 62 | return total / len(sensitive_values) 63 | 64 | def is_better_than(self, val1, val2): 65 | dist1 = math.fabs(1.0 - val1) 66 | dist2 = math.fabs(1.0 - val2) 67 | return dist1 < dist2 68 | -------------------------------------------------------------------------------- /metrics/DIBinary.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | from metrics.utils import calc_pos_protected_percents 4 | from metrics.Metric import Metric 5 | 6 | 7 | class DIBinary(Metric): 8 | """ 9 | This metric calculates disparate imapct in the sense of the 80% rule before the 80% 10 | threshold is applied. This is described as DI in: https://arxiv.org/abs/1412.3756 11 | If there are no positive protected classifications, 0.0 is returned. 12 | 13 | Multiple protected classes are treated as one large group, so that this compares the privileged 14 | class to all non-privileged classes as a group. 15 | """ 16 | 17 | def __init__(self): 18 | Metric.__init__(self) 19 | self.name = "DIbinary" 20 | 21 | def calc( 22 | self, 23 | actual, 24 | predicted, 25 | dict_of_sensitive_lists, 26 | single_sensitive_name, 27 | unprotected_vals, 28 | positive_pred, 29 | ): 30 | sensitive = dict_of_sensitive_lists[single_sensitive_name] 31 | unprotected_pos_percent, protected_pos_percent = calc_pos_protected_percents( 32 | predicted, sensitive, unprotected_vals, positive_pred 33 | ) 34 | DI = 0.0 35 | if unprotected_pos_percent > 0: 36 | DI = protected_pos_percent / unprotected_pos_percent 37 | if unprotected_pos_percent == 0.0 and protected_pos_percent == 0.0: 38 | DI = 1.0 39 | return DI 40 | 41 | def is_better_than(self, val1, val2): 42 | dist1 = math.fabs(1.0 - val1) 43 | dist2 = math.fabs(1.0 - val2) 44 | return dist1 < dist2 45 | -------------------------------------------------------------------------------- /metrics/Diff.py: -------------------------------------------------------------------------------- 1 | import math 2 | from metrics.Metric import Metric 3 | 4 | 5 | class Diff(Metric): 6 | def __init__(self, metric1, metric2): 7 | Metric.__init__(self) 8 | self.metric1 = metric1 9 | self.metric2 = metric2 10 | self.name = "diff:" + self.metric1.get_name() + "to" + self.metric2.get_name() 11 | 12 | def calc( 13 | self, 14 | actual, 15 | predicted, 16 | dict_of_sensitive_lists, 17 | single_sensitive_name, 18 | unprotected_vals, 19 | positive_pred, 20 | ): 21 | m1 = self.metric1.calc( 22 | actual, 23 | predicted, 24 | dict_of_sensitive_lists, 25 | single_sensitive_name, 26 | unprotected_vals, 27 | positive_pred, 28 | ) 29 | m2 = self.metric2.calc( 30 | actual, 31 | predicted, 32 | dict_of_sensitive_lists, 33 | single_sensitive_name, 34 | unprotected_vals, 35 | positive_pred, 36 | ) 37 | 38 | if m1 is None or m2 is None: 39 | return None 40 | 41 | diff = m1 - m2 42 | return 1.0 - abs(diff) 43 | 44 | def is_better_than(self, val1, val2): 45 | """ 46 | Assumes that 1.0 is the goal value. 47 | """ 48 | dist1 = math.fabs(1.0 - val1) 49 | dist2 = math.fabs(1.0 - val2) 50 | return dist1 < dist2 51 | -------------------------------------------------------------------------------- /metrics/EqOppo_fn_diff.py: -------------------------------------------------------------------------------- 1 | """ Equal opportunity - Protected and unprotected False negative difference""" 2 | import math 3 | import sys 4 | import numpy 5 | 6 | from metrics.utils import calc_fp_fn 7 | from metrics.Metric import Metric 8 | 9 | 10 | class EqOppo_fn_diff(Metric): 11 | def __init__(self): 12 | Metric.__init__(self) 13 | self.name = "EqOppo_fn_diff" 14 | 15 | def calc( 16 | self, 17 | actual, 18 | predicted, 19 | dict_of_sensitive_lists, 20 | single_sensitive_name, 21 | unprotected_vals, 22 | positive_pred, 23 | ): 24 | sensitive = dict_of_sensitive_lists[single_sensitive_name] 25 | fp_unprotected, fp_protected, fn_protected, fn_unprotected = calc_fp_fn( 26 | actual, predicted, sensitive, unprotected_vals, positive_pred 27 | ) 28 | 29 | fn_diff = math.fabs(fn_protected - fn_unprotected) 30 | 31 | return fn_diff 32 | -------------------------------------------------------------------------------- /metrics/EqOppo_fn_ratio.py: -------------------------------------------------------------------------------- 1 | """Equal opportunity - Protected and unprotected False negative ratio""" 2 | import math 3 | import sys 4 | import numpy 5 | 6 | from metrics.utils import calc_fp_fn 7 | from metrics.Metric import Metric 8 | 9 | 10 | class EqOppo_fn_ratio(Metric): 11 | def __init__(self): 12 | Metric.__init__(self) 13 | self.name = "EqOppo_fn_ratio" 14 | 15 | def calc( 16 | self, 17 | actual, 18 | predicted, 19 | dict_of_sensitive_lists, 20 | single_sensitive_name, 21 | unprotected_vals, 22 | positive_pred, 23 | ): 24 | sensitive = dict_of_sensitive_lists[single_sensitive_name] 25 | fp_unprotected, fp_protected, fn_protected, fn_unprotected = calc_fp_fn( 26 | actual, predicted, sensitive, unprotected_vals, positive_pred 27 | ) 28 | fn_ratio = 0.0 29 | if fn_unprotected > 0: 30 | fn_ratio = fn_protected / fn_unprotected 31 | if fn_unprotected == 0.0 and fn_protected == 0.0: 32 | fn_ratio = 1.0 33 | return fn_ratio 34 | -------------------------------------------------------------------------------- /metrics/EqOppo_fp_diff.py: -------------------------------------------------------------------------------- 1 | """ Equal opportunity - Protected and unprotected False postives difference """ 2 | import math 3 | import sys 4 | import numpy 5 | 6 | from metrics.utils import calc_fp_fn 7 | from metrics.Metric import Metric 8 | 9 | 10 | class EqOppo_fp_diff(Metric): 11 | def __init__(self): 12 | Metric.__init__(self) 13 | self.name = "EqOppo_fp_diff" 14 | 15 | def calc( 16 | self, 17 | actual, 18 | predicted, 19 | dict_of_sensitive_lists, 20 | single_sensitive_name, 21 | unprotected_vals, 22 | positive_pred, 23 | ): 24 | sensitive = dict_of_sensitive_lists[single_sensitive_name] 25 | fp_unprotected, fp_protected, fn_protected, fn_unprotected = calc_fp_fn( 26 | actual, predicted, sensitive, unprotected_vals, positive_pred 27 | ) 28 | 29 | fp_diff = math.fabs(fp_protected - fp_unprotected) 30 | 31 | return fp_diff 32 | -------------------------------------------------------------------------------- /metrics/EqOppo_fp_ratio.py: -------------------------------------------------------------------------------- 1 | """ Equal opportunity - Protected and unprotected False postives ratio""" 2 | import math 3 | import sys 4 | import numpy 5 | 6 | from metrics.utils import calc_fp_fn 7 | from metrics.Metric import Metric 8 | 9 | 10 | class EqOppo_fp_ratio(Metric): 11 | def __init__(self): 12 | Metric.__init__(self) 13 | self.name = "EqOppo_fp_ratio" 14 | 15 | def calc( 16 | self, 17 | actual, 18 | predicted, 19 | dict_of_sensitive_lists, 20 | single_sensitive_name, 21 | unprotected_vals, 22 | positive_pred, 23 | ): 24 | sensitive = dict_of_sensitive_lists[single_sensitive_name] 25 | 26 | fp_unprotected, fp_protected, fn_protected, fn_unprotected = calc_fp_fn( 27 | actual, predicted, sensitive, unprotected_vals, positive_pred 28 | ) 29 | fp_ratio = 0.0 30 | if fp_unprotected > 0: 31 | fp_ratio = fp_protected / fp_unprotected 32 | if fp_unprotected == 0.0 and fp_protected == 0.0: 33 | fp_ratio = 1.0 34 | return fp_ratio 35 | -------------------------------------------------------------------------------- /metrics/FNR.py: -------------------------------------------------------------------------------- 1 | from metrics.Metric import Metric 2 | from metrics.TPR import TPR 3 | 4 | 5 | class FNR(Metric): 6 | def __init__(self): 7 | Metric.__init__(self) 8 | self.name = "FNR" 9 | 10 | def calc( 11 | self, 12 | actual, 13 | predicted, 14 | dict_of_sensitive_lists, 15 | single_sensitive_name, 16 | unprotected_vals, 17 | positive_pred, 18 | ): 19 | tpr = TPR() 20 | tpr_val = tpr.calc( 21 | actual, 22 | predicted, 23 | dict_of_sensitive_lists, 24 | single_sensitive_name, 25 | unprotected_vals, 26 | positive_pred, 27 | ) 28 | return 1 - tpr_val 29 | -------------------------------------------------------------------------------- /metrics/FPR.py: -------------------------------------------------------------------------------- 1 | from metrics.Metric import Metric 2 | from metrics.TNR import TNR 3 | 4 | 5 | class FPR(Metric): 6 | def __init__(self): 7 | Metric.__init__(self) 8 | self.name = "FPR" 9 | 10 | def calc( 11 | self, 12 | actual, 13 | predicted, 14 | dict_of_sensitive_lists, 15 | single_sensitive_name, 16 | unprotected_vals, 17 | positive_pred, 18 | ): 19 | tnr = TNR() 20 | tnr_val = tnr.calc( 21 | actual, 22 | predicted, 23 | dict_of_sensitive_lists, 24 | single_sensitive_name, 25 | unprotected_vals, 26 | positive_pred, 27 | ) 28 | return 1 - tnr_val 29 | -------------------------------------------------------------------------------- /metrics/FilterSensitive.py: -------------------------------------------------------------------------------- 1 | from metrics.Metric import Metric 2 | 3 | 4 | class FilterSensitive(Metric): 5 | def __init__(self, metric): 6 | Metric.__init__(self) 7 | self.metric = metric 8 | self.name = metric.get_name() 9 | 10 | def calc( 11 | self, 12 | actual, 13 | predicted, 14 | dict_of_sensitive_lists, 15 | single_sensitive_name, 16 | unprotected_vals, 17 | positive_pred, 18 | ): 19 | 20 | sensitive = dict_of_sensitive_lists[self.sensitive_for_metric] 21 | actual_sens = [ 22 | act for act, sens in zip(actual, sensitive) if sens == self.sensitive_filter 23 | ] 24 | predicted_sens = [ 25 | pred 26 | for pred, sens in zip(predicted, sensitive) 27 | if sens == self.sensitive_filter 28 | ] 29 | sensitive_sens = [sens for sens in sensitive if sens == self.sensitive_filter] 30 | 31 | filtered_dict = {} 32 | for sens_val in dict_of_sensitive_lists: 33 | other_sensitive = dict_of_sensitive_lists[sens_val] 34 | filtered = [ 35 | s 36 | for s, sens in zip(other_sensitive, sensitive) 37 | if sens == self.sensitive_filter 38 | ] 39 | filtered_dict[sens_val] = filtered 40 | 41 | if len(actual_sens) < 1: 42 | return None 43 | 44 | return self.metric.calc( 45 | actual_sens, 46 | predicted_sens, 47 | filtered_dict, 48 | single_sensitive_name, 49 | unprotected_vals, 50 | positive_pred, 51 | ) 52 | 53 | def set_sensitive_to_filter(self, sensitive_name, sensitive_val): 54 | """ 55 | Sets the specific sensitive value to filter based on. The given metric will be 56 | calculated only with respect to the actual and predicted values that have this sensitive 57 | value as part of that item. 58 | 59 | sensitive_name sensitive attribute name (e.g., 'race') 60 | sensitive_val specific sensitive value (e.g., 'white') 61 | """ 62 | self.name += str(sensitive_val) 63 | self.sensitive_filter = sensitive_val 64 | self.sensitive_for_metric = sensitive_name 65 | -------------------------------------------------------------------------------- /metrics/MCC.py: -------------------------------------------------------------------------------- 1 | from metrics.Metric import Metric 2 | from sklearn.metrics import matthews_corrcoef 3 | 4 | 5 | class MCC(Metric): 6 | def __init__(self): 7 | Metric.__init__(self) 8 | self.name = "MCC" 9 | 10 | def calc( 11 | self, 12 | actual, 13 | predicted, 14 | dict_of_sensitive_lists, 15 | single_sensitive_name, 16 | unprotected_vals, 17 | positive_pred, 18 | ): 19 | return matthews_corrcoef(actual, predicted) 20 | -------------------------------------------------------------------------------- /metrics/Metric.py: -------------------------------------------------------------------------------- 1 | class Metric: 2 | def __init__(self): 3 | self.name = ( 4 | "Name not implemented" ## This should be replaced in implemented metrics. 5 | ) 6 | self.iter_counter = 0 7 | 8 | def __iter__(self): 9 | self.iter_counter = 0 10 | return self 11 | 12 | def __next__(self): 13 | self.iter_counter += 1 14 | if self.iter_counter > 1: 15 | raise StopIteration 16 | return self 17 | 18 | def calc( 19 | self, 20 | actual, 21 | predicted, 22 | dict_of_sensitive_lists, 23 | single_sensitive_name, 24 | unprotected_vals, 25 | positive_pred, 26 | ): 27 | """ 28 | actual a list of the actual results on the test set 29 | predicted a list of the predicted results 30 | dict_of_sensitive_lsits dict mapping sensitive attr names to list of sensitive vals 31 | single_sensitive_name sensitive name (dict key) for the sensitive attr being 32 | focused on by this run of the algorithm 33 | unprotected_vals a list of the unprotected values for all sensitive attrs 34 | positive_pred the positive value of the prediction task. 35 | 36 | returns the calculated result for this metric 37 | 38 | The actual and predicted results and the sensitive attribute lists in the dict should have 39 | the same length (the length of the test set). 40 | 41 | If there is an error and the metric can not be calculated (e.g., no data is passed in), the 42 | metric returns None. 43 | """ 44 | raise NotImplementedError("calc() in Metric is not implemented") 45 | 46 | def get_name(self): 47 | """ 48 | Returns a name for the metric. This will be used as the key for a dictionary and will 49 | also be printed to the final output file. 50 | """ 51 | return self.name 52 | 53 | def is_better_than(self, val1, val2): 54 | """ 55 | Compares the two given values that were calculated by this metric and returns true if 56 | val1 is better than val2, false otherwise. 57 | """ 58 | return val1 > val2 59 | 60 | def expand_per_dataset(self, privileged_vals, sensitive_dict): 61 | """ 62 | !Changed to work with MNIST variants! 63 | 64 | Optionally allows the expansion of the metric into a returned list of metrics based on the 65 | dataset, e.g., where there is one metric per sensitive attribute given, and a dictionary 66 | mapping sensitive attributes to all seen sensitive values from the data. 67 | """ 68 | return self 69 | -------------------------------------------------------------------------------- /metrics/README.md: -------------------------------------------------------------------------------- 1 | |Metric | Comparing_fn | Ideal Value | 2 | |------------------------------|-----------------|-------------------| 3 | |accuracy|`Metric.is_better_than`|HIGHER is better| 4 | |TPR|`Metric.is_better_than`|HIGHER is better| 5 | |TNR|`Metric.is_better_than`|HIGHER is better| 6 | |BCR|`Metric.is_better_than`|HIGHER is better| 7 | |MCC|`Metric.is_better_than`|HIGHER is better| 8 | |DIbinary|`DIBinary.is_better_than`|Closer to 1| 9 | |DIavgall|`DIAvgAll.is_better_than`|Closer to 1| 10 | |CV|`CV.is_better_than`|Closer to 1| 11 | |0-accuracy|`Metric.is_better_than`|HIGHER is better| 12 | |1-accuracy|`Metric.is_better_than`|HIGHER is better| 13 | |sensitiveattr-accuracy|[Average]`Metric.is_better_than`|HIGHER is better| 14 | |1-accuracy_over_0-accuracy|`Ratio.is_better_than`|Closer to 1| 15 | |sensitiveattr-accuracyRatio|[Average]`Ratio.is_better_than`|Closer to 1| 16 | |diff:0-accuracyto1-accuracy|`Diff.is_better_than`|Closer to 1| 17 | |sensitiveattr-accuracyDiff|[Average]`Diff.is_better_than`|Closer to 1| 18 | |0-TPR|`Metric.is_better_than`|HIGHER is better| 19 | |1-TPR|`Metric.is_better_than`|HIGHER is better| 20 | |sensitiveattr-TPR|[Average]`Metric.is_better_than`|HIGHER is better| 21 | |1-TPR_over_0-TPR|`Ratio.is_better_than`|Closer to 1| 22 | |sensitiveattr-TPRRatio|[Average]`Ratio.is_better_than`|Closer to 1| 23 | |diff:0-TPRto1-TPR|`Diff.is_better_than`|Closer to 1| 24 | |sensitiveattr-TPRDiff|[Average]`Diff.is_better_than`|Closer to 1| 25 | |0-TNR|`Metric.is_better_than`|HIGHER is better| 26 | |1-TNR|`Metric.is_better_than`|HIGHER is better| 27 | |sensitiveattr-TNR|[Average]`Metric.is_better_than`|HIGHER is better| 28 | |1-TNR_over_0-TNR|`Ratio.is_better_than`|Closer to 1| 29 | |sensitiveattr-TNRRatio|[Average]`Ratio.is_better_than`|Closer to 1| 30 | |diff:0-TNRto1-TNR|`Diff.is_better_than`|Closer to 1| 31 | |sensitiveattr-TNRDiff|[Average]`Diff.is_better_than`|Closer to 1| 32 | |0-FPR|`Metric.is_better_than`|HIGHER is better| 33 | |1-FPR|`Metric.is_better_than`|HIGHER is better| 34 | |sensitiveattr-FPR|[Average]`Metric.is_better_than`|HIGHER is better| 35 | |1-FPR_over_0-FPR|`Ratio.is_better_than`|Closer to 1| 36 | |sensitiveattr-FPRRatio|[Average]`Ratio.is_better_than`|Closer to 1| 37 | |diff:0-FPRto1-FPR|`Diff.is_better_than`|Closer to 1| 38 | |sensitiveattr-FPRDiff|[Average]`Diff.is_better_than`|Closer to 1| 39 | |0-FNR|`Metric.is_better_than`|HIGHER is better| 40 | |1-FNR|`Metric.is_better_than`|HIGHER is better| 41 | |sensitiveattr-FNR|[Average]`Metric.is_better_than`|HIGHER is better| 42 | |1-FNR_over_0-FNR|`Ratio.is_better_than`|Closer to 1| 43 | |sensitiveattr-FNRRatio|[Average]`Ratio.is_better_than`|Closer to 1| 44 | |diff:0-FNRto1-FNR|`Diff.is_better_than`|Closer to 1| 45 | |sensitiveattr-FNRDiff|[Average]`Diff.is_better_than`|Closer to 1| 46 | |0-calibrationPos|`Metric.is_better_than`|HIGHER is better| 47 | |1-calibrationPos|`Metric.is_better_than`|HIGHER is better| 48 | |sensitiveattr-calibrationPos|[Average]`Metric.is_better_than`|HIGHER is better| 49 | |1-calibrationPos_over_0-calibrationPos|`Ratio.is_better_than`|Closer to 1| 50 | |sensitiveattr-calibrationPosRatio|[Average]`Ratio.is_better_than`|Closer to 1| 51 | |diff:0-calibrationPosto1-calibrationPos|`Diff.is_better_than`|Closer to 1| 52 | |sensitiveattr-calibrationPosDiff|[Average]`Diff.is_better_than`|Closer to 1| 53 | |0-calibrationNeg|`Metric.is_better_than`|HIGHER is better| 54 | |1-calibrationNeg|`Metric.is_better_than`|HIGHER is better| 55 | |sensitiveattr-calibrationNeg|[Average]`Metric.is_better_than`|HIGHER is better| 56 | |1-calibrationNeg_over_0-calibrationNeg|`Ratio.is_better_than`|Closer to 1| 57 | |sensitiveattr-calibrationNegRatio|[Average]`Ratio.is_better_than`|Closer to 1| 58 | |diff:0-calibrationNegto1-calibrationNeg|`Diff.is_better_than`|Closer to 1| 59 | |sensitiveattr-calibrationNegDiff|[Average]`Diff.is_better_than`|Closer to 1| 60 | |EqOppo_fn_diff|`Metric.is_better_than`|HIGHER is better| 61 | |EqOppo_fn_ratio|`Metric.is_better_than`|HIGHER is better| 62 | |EqOppo_fp_diff|`Metric.is_better_than`|HIGHER is better| 63 | |EqOppo_fp_ratio|`Metric.is_better_than`|HIGHER is better| 64 | -------------------------------------------------------------------------------- /metrics/Ratio.py: -------------------------------------------------------------------------------- 1 | import math 2 | from metrics.Metric import Metric 3 | 4 | 5 | class Ratio(Metric): 6 | def __init__(self, metric_numerator, metric_denominator): 7 | Metric.__init__(self) 8 | self.numerator = metric_numerator 9 | self.denominator = metric_denominator 10 | self.name = self.numerator.get_name() + "_over_" + self.denominator.get_name() 11 | 12 | def calc( 13 | self, 14 | actual, 15 | predicted, 16 | dict_of_sensitive_lists, 17 | single_sensitive_name, 18 | unprotected_vals, 19 | positive_pred, 20 | ): 21 | num = self.numerator.calc( 22 | actual, 23 | predicted, 24 | dict_of_sensitive_lists, 25 | single_sensitive_name, 26 | unprotected_vals, 27 | positive_pred, 28 | ) 29 | den = self.denominator.calc( 30 | actual, 31 | predicted, 32 | dict_of_sensitive_lists, 33 | single_sensitive_name, 34 | unprotected_vals, 35 | positive_pred, 36 | ) 37 | 38 | if num is None or den is None: 39 | return None 40 | 41 | if num == 0.0 and den == 0.0: 42 | return 1.0 43 | if den == 0.0: 44 | return 0.0 45 | return num / den 46 | 47 | def is_better_than(self, val1, val2): 48 | """ 49 | Assumes that the goal ratio is 1.0. 50 | """ 51 | dist1 = math.fabs(1.0 - val1) 52 | dist2 = math.fabs(1.0 - val2) 53 | return dist1 < dist2 54 | -------------------------------------------------------------------------------- /metrics/SensitiveMetric.py: -------------------------------------------------------------------------------- 1 | from metrics.Average import Average 2 | from metrics.Diff import Diff 3 | from metrics.FilterSensitive import FilterSensitive 4 | from metrics.Metric import Metric 5 | from metrics.Ratio import Ratio 6 | 7 | 8 | class SensitiveMetric(Metric): 9 | """ 10 | Takes the given metric and creates a version of it that is conditioned on a sensitive 11 | attribute. 12 | 13 | For example, for SensitiveMetric(Accuracy), this measure takes the average accuracy per 14 | sensitive value. It is unweighted in the sense that each sensitive value's accuracy is treated 15 | equally in the average. This measure is designed to catch the scenario when misclassifying all 16 | Native-Americans but having high accuracy (say, 100%) on everyone else causes an algorithm to 17 | have 98% accuracy because Native-Americans make up about 2% of the U.S. population. In this 18 | scenario, assuming the listed sensitive values were Native-American and not-Native-American, 19 | this metric would return 1, 0, and 0.5. Given more than two sensitive values, it will return 20 | the average over all of the per-value accuracies. It will also return the ratios with respect 21 | to the privileged value on this metric, the average of that, the differece with respect to the 22 | privileged value, and the average of that as well. 23 | """ 24 | 25 | def __init__(self, metric_class): 26 | Metric.__init__(self) 27 | self.metric = metric_class 28 | self.name = ( 29 | self.metric().get_name() 30 | ) # to be modified as this metric is expanded 31 | 32 | def calc( 33 | self, 34 | actual, 35 | predicted, 36 | dict_of_sensitive_lists, 37 | single_sensitive_name, 38 | unprotected_vals, 39 | positive_pred, 40 | ): 41 | sfilter = FilterSensitive(self.metric()) 42 | sfilter.set_sensitive_to_filter(self.sensitive_attr, self.sensitive_val) 43 | return sfilter.calc( 44 | actual, 45 | predicted, 46 | dict_of_sensitive_lists, 47 | single_sensitive_name, 48 | unprotected_vals, 49 | positive_pred, 50 | ) 51 | 52 | def expand_per_dataset(self, priviledged_vals, sensitive_dict): 53 | objects_list = [] 54 | for sensitive in sensitive_dict.keys(): 55 | objects_list += self.make_metric_objects( 56 | sensitive, sensitive_dict, priviledged_vals[sensitive] 57 | ) 58 | return objects_list 59 | 60 | def make_metric_objects(self, sensitive_name, sensitive_values, privileged_val): 61 | objs_list = [] 62 | ratios_list = [] 63 | diff_list = [] 64 | for val in sensitive_values[sensitive_name]: 65 | # adding sensitive variants of the given metric to the objects list 66 | objs_list.append(self.make_sensitive_obj(sensitive_name, val)) 67 | 68 | # adding ratio of sensitive variants of the given metric to the ratios list 69 | if val != privileged_val: 70 | ratios_list.append( 71 | self.make_ratio_obj(sensitive_name, val, privileged_val) 72 | ) 73 | 74 | # adding diff of sensitive variants of given metric to the diff list 75 | if val != privileged_val: 76 | diff_list.append( 77 | self.make_diff_obj(sensitive_name, val, privileged_val) 78 | ) 79 | avg = Average(objs_list, sensitive_name + "-" + self.metric().get_name()) 80 | ratio_avg = Average( 81 | ratios_list, sensitive_name + "-" + self.metric().get_name() + "Ratio" 82 | ) 83 | diff_avg = Average( 84 | diff_list, sensitive_name + "-" + self.metric().get_name() + "Diff" 85 | ) 86 | return objs_list + [avg] + ratios_list + [ratio_avg] + diff_list + [diff_avg] 87 | 88 | def make_sensitive_obj(self, sensitive_attr, sensitive_val): 89 | obj = self.__class__(self.metric) 90 | obj.set_sensitive_to_filter(sensitive_attr, sensitive_val) 91 | return obj 92 | 93 | def make_ratio_obj(self, sensitive_attr, sensitive_val, privileged): 94 | privileged_metric = self.make_sensitive_obj(sensitive_attr, privileged) 95 | unprivileged_metric = self.make_sensitive_obj(sensitive_attr, sensitive_val) 96 | return Ratio(unprivileged_metric, privileged_metric) 97 | 98 | def make_diff_obj(self, sensitive_attr, sensitive_val, privileged): 99 | privileged_metric = self.make_sensitive_obj(sensitive_attr, privileged) 100 | unprivileged_metric = self.make_sensitive_obj(sensitive_attr, sensitive_val) 101 | return Diff(privileged_metric, unprivileged_metric) 102 | 103 | def set_sensitive_to_filter(self, sensitive_name, sensitive_val): 104 | """ 105 | Set the attribute and value to filter, i.e., to calculate this metric for. 106 | """ 107 | self.sensitive_attr = sensitive_name 108 | self.sensitive_val = sensitive_val 109 | self.name = str(sensitive_val) + "-" + self.name 110 | -------------------------------------------------------------------------------- /metrics/TNR.py: -------------------------------------------------------------------------------- 1 | from metrics.Metric import Metric 2 | from sklearn.metrics import confusion_matrix 3 | 4 | 5 | class TNR(Metric): 6 | def __init__(self): 7 | Metric.__init__(self) 8 | self.name = "TNR" 9 | 10 | def calc( 11 | self, 12 | actual, 13 | predicted, 14 | dict_of_sensitive_lists, 15 | single_sensitive_name, 16 | unprotected_vals, 17 | positive_pred, 18 | ): 19 | classes = list(set(actual)) 20 | matrix = confusion_matrix(actual, predicted, labels=classes) 21 | # matrix[i][j] is the number of observations with actual class i that were predicted as j 22 | TN = 0.0 23 | allN = 0.0 24 | for i in range(0, len(classes)): 25 | trueval = classes[i] 26 | if trueval == positive_pred: 27 | continue 28 | for j in range(0, len(classes)): 29 | allN += matrix[i][j] 30 | predval = classes[j] 31 | if trueval == predval: 32 | TN += matrix[i][j] 33 | 34 | if allN == 0.0: 35 | return 1.0 36 | 37 | return TN / allN 38 | -------------------------------------------------------------------------------- /metrics/TPR.py: -------------------------------------------------------------------------------- 1 | from metrics.Metric import Metric 2 | from sklearn.metrics import recall_score 3 | 4 | 5 | class TPR(Metric): 6 | """ 7 | Returns the true positive rate (aka recall) for the predictions. Assumes binary 8 | classification. 9 | """ 10 | 11 | def __init__(self): 12 | Metric.__init__(self) 13 | self.name = "TPR" 14 | 15 | def calc( 16 | self, 17 | actual, 18 | predicted, 19 | dict_of_sensitive_lists, 20 | single_sensitive_name, 21 | unprotected_vals, 22 | positive_pred, 23 | ): 24 | return recall_score( 25 | actual, 26 | predicted, 27 | pos_label=positive_pred, 28 | average="binary", 29 | zero_division=1, 30 | ) 31 | -------------------------------------------------------------------------------- /metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__init__.py -------------------------------------------------------------------------------- /metrics/__pycache__/Accuracy.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/Accuracy.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/Accuracy.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/Accuracy.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/Accuracy.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/Accuracy.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/Average.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/Average.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/Average.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/Average.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/Average.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/Average.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/BCR.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/BCR.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/BCR.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/BCR.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/BCR.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/BCR.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/CV.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/CV.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/CV.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/CV.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/CV.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/CV.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/CalibrationNeg.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/CalibrationNeg.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/CalibrationNeg.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/CalibrationNeg.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/CalibrationNeg.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/CalibrationNeg.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/CalibrationPos.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/CalibrationPos.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/CalibrationPos.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/CalibrationPos.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/CalibrationPos.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/CalibrationPos.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/DIAvgAll.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/DIAvgAll.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/DIAvgAll.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/DIAvgAll.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/DIAvgAll.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/DIAvgAll.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/DIBinary.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/DIBinary.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/DIBinary.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/DIBinary.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/DIBinary.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/DIBinary.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/Diff.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/Diff.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/Diff.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/Diff.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/Diff.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/Diff.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/EqOppo_fn_diff.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/EqOppo_fn_diff.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/EqOppo_fn_diff.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/EqOppo_fn_diff.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/EqOppo_fn_diff.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/EqOppo_fn_diff.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/EqOppo_fn_ratio.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/EqOppo_fn_ratio.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/EqOppo_fn_ratio.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/EqOppo_fn_ratio.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/EqOppo_fn_ratio.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/EqOppo_fn_ratio.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/EqOppo_fp_diff.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/EqOppo_fp_diff.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/EqOppo_fp_diff.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/EqOppo_fp_diff.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/EqOppo_fp_diff.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/EqOppo_fp_diff.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/EqOppo_fp_ratio.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/EqOppo_fp_ratio.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/EqOppo_fp_ratio.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/EqOppo_fp_ratio.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/EqOppo_fp_ratio.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/EqOppo_fp_ratio.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/FNR.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/FNR.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/FNR.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/FNR.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/FNR.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/FNR.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/FPR.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/FPR.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/FPR.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/FPR.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/FPR.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/FPR.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/FilterSensitive.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/FilterSensitive.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/FilterSensitive.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/FilterSensitive.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/FilterSensitive.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/FilterSensitive.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/MCC.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/MCC.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/MCC.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/MCC.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/MCC.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/MCC.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/Metric.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/Metric.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/Metric.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/Metric.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/Metric.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/Metric.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/Ratio.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/Ratio.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/Ratio.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/Ratio.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/Ratio.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/Ratio.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/SensitiveMetric.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/SensitiveMetric.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/SensitiveMetric.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/SensitiveMetric.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/SensitiveMetric.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/SensitiveMetric.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/TNR.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/TNR.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/TNR.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/TNR.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/TNR.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/TNR.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/TPR.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/TPR.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/TPR.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/TPR.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/TPR.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/TPR.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/__init__.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/__init__.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/list.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/list.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/list.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/list.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/list.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/list.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/utils.cpython-36.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/utils.cpython-37.pyc -------------------------------------------------------------------------------- /metrics/__pycache__/utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/metrics/__pycache__/utils.cpython-38.pyc -------------------------------------------------------------------------------- /metrics/list.py: -------------------------------------------------------------------------------- 1 | from argparse import Namespace 2 | import numpy as np 3 | 4 | from metrics.Accuracy import Accuracy 5 | from metrics.BCR import BCR 6 | from metrics.CalibrationNeg import CalibrationNeg 7 | from metrics.CalibrationPos import CalibrationPos 8 | from metrics.CV import CV 9 | from metrics.DIAvgAll import DIAvgAll 10 | from metrics.DIBinary import DIBinary 11 | from metrics.EqOppo_fn_diff import EqOppo_fn_diff 12 | from metrics.EqOppo_fn_ratio import EqOppo_fn_ratio 13 | from metrics.EqOppo_fp_diff import EqOppo_fp_diff 14 | from metrics.EqOppo_fp_ratio import EqOppo_fp_ratio 15 | from metrics.FNR import FNR 16 | from metrics.FPR import FPR 17 | from metrics.MCC import MCC 18 | from metrics.SensitiveMetric import SensitiveMetric 19 | from metrics.TNR import TNR 20 | from metrics.TPR import TPR 21 | 22 | ACC_METRICS = [ 23 | Accuracy(), 24 | TPR(), 25 | TNR(), 26 | BCR(), 27 | MCC(), 28 | ] 29 | 30 | FAIRNESS_METRICS = [ 31 | DIBinary(), 32 | DIAvgAll(), 33 | CV(), 34 | SensitiveMetric(Accuracy), 35 | SensitiveMetric(TPR), 36 | SensitiveMetric(TNR), 37 | SensitiveMetric(FPR), 38 | SensitiveMetric(FNR), 39 | SensitiveMetric(CalibrationPos), 40 | SensitiveMetric(CalibrationNeg), 41 | EqOppo_fn_diff(), # TODO: ~4.9k? 42 | EqOppo_fn_ratio(), 43 | EqOppo_fp_diff(), # TODO: ~40? 44 | EqOppo_fp_ratio(), 45 | ] 46 | 47 | METRICS = ACC_METRICS + FAIRNESS_METRICS 48 | 49 | 50 | def add_metric(metric): 51 | METRICS.append(metric) 52 | 53 | 54 | def get_metrics(privileged_vals, sensitive_dict): 55 | """ 56 | Takes a dataset object and a dictionary mapping sensitive attributes to a list of the sensitive 57 | values seen in the data. Returns an expanded list of metrics based on the base METRICS. 58 | """ 59 | metrics = [] 60 | for metric in METRICS: 61 | metrics += metric.expand_per_dataset(privileged_vals, sensitive_dict) 62 | return metrics 63 | 64 | 65 | def calculate_metrics(input_dict): 66 | """ 67 | example: ricci dataset 68 | Race Class other_attrs 69 | W 1 1 70 | B 1 0 71 | H 0 1 72 | single_sensitive = 'Race' 73 | all_sensitive_attributes = ['Race'] 74 | sensitive_dict = {'Race': [0, 1]} from dataset -> can define new metrics 75 | sensitive_dict = {'Race': ['H', 'B', 'W']} 76 | dict_sensitive_lists = {'Race': [1, 0, 0, 1, 0, 1, 0, 1,...]} 77 | privileged_vals or unprotected_vals = ['W'] or [1] 78 | """ 79 | inps = Namespace(**input_dict) 80 | results = dict() 81 | for mtr in get_metrics(inps.privileged_vals, inps.sensitive_dict): 82 | mtr_value = mtr.calc( 83 | inps.label, 84 | inps.predicted, 85 | {"sensitiveattr": inps.sensitiveattr}, 86 | "sensitiveattr", 87 | [inps.privileged_vals["sensitiveattr"]], # TODO: this is messy! Fix please. 88 | inps.positive_pred, 89 | ) 90 | results[mtr.get_name()] = mtr_value 91 | return results 92 | 93 | 94 | class CostTemplate: 95 | def is_better_than(self, val1, val2): 96 | return val1 < val2 97 | -------------------------------------------------------------------------------- /metrics/utils.py: -------------------------------------------------------------------------------- 1 | def calc_pos_protected_percents(predicted, sensitive, unprotected_vals, positive_pred): 2 | """ 3 | Returns P(C=YES|sensitive=privileged) and P(C=YES|sensitive=not privileged) in that order where 4 | C is the predicited classification and where all not privileged values are considered 5 | equivalent. Assumes that predicted and sensitive have the same lengths. 6 | """ 7 | unprotected_positive = 0.0 8 | unprotected_negative = 0.0 9 | protected_positive = 0.0 10 | protected_negative = 0.0 11 | for i in range(0, len(predicted)): 12 | protected_val = sensitive[i] 13 | predicted_val = predicted[i] 14 | if protected_val in unprotected_vals: 15 | if str(predicted_val) == str(positive_pred): 16 | unprotected_positive += 1 17 | else: 18 | unprotected_negative += 1 19 | else: 20 | if str(predicted_val) == str(positive_pred): 21 | protected_positive += 1 22 | else: 23 | protected_negative += 1 24 | 25 | protected_pos_percent = 0.0 26 | if protected_positive + protected_negative > 0: 27 | protected_pos_percent = protected_positive / ( 28 | protected_positive + protected_negative 29 | ) 30 | unprotected_pos_percent = 0.0 31 | if unprotected_positive + unprotected_negative > 0: 32 | unprotected_pos_percent = unprotected_positive / ( 33 | unprotected_positive + unprotected_negative 34 | ) 35 | 36 | return unprotected_pos_percent, protected_pos_percent 37 | 38 | 39 | def calc_prob_class_given_sensitive( 40 | predicted, sensitive, predicted_goal, sensitive_goal 41 | ): 42 | """ 43 | Returns P(predicted = predicted_goal | sensitive = sensitive_goal). Assumes that predicted 44 | and sensitive have the same length. If there are no attributes matching the given 45 | sensitive_goal, this will error. 46 | """ 47 | match_count = 0.0 48 | total = 0.0 49 | for sens, pred in zip(sensitive, predicted): 50 | if str(sens) == str(sensitive_goal): 51 | total += 1 52 | if str(pred) == str(predicted_goal): 53 | match_count += 1 54 | 55 | return match_count / total 56 | 57 | 58 | def calc_fp_fn(actual, predicted, sensitive, unprotected_vals, positive_pred): 59 | """ 60 | Returns False positive and false negative for protected and unprotected group. 61 | """ 62 | unprotected_negative = 0.0 63 | protected_positive = 0.0 64 | protected_negative = 0.0 65 | fp_protected = 0.0 66 | fp_unprotected = 0.0 67 | fn_protected = 0.0 68 | fn_unprotected = 0.0 69 | fp_diff = 0.0 70 | for i in range(0, len(predicted)): 71 | protected_val = sensitive[i] 72 | predicted_val = predicted[i] 73 | actual_val = actual[i] 74 | if protected_val in unprotected_vals: 75 | if (str(predicted_val) == str(positive_pred)) & ( 76 | str(actual_val) != str(predicted_val) 77 | ): 78 | fp_unprotected += 1 79 | elif (str(predicted_val) != str(positive_pred)) & ( 80 | str(actual_val) == str(predicted_val) 81 | ): 82 | fn_unprotected += 1 83 | else: 84 | if (str(predicted_val) == str(positive_pred)) & ( 85 | str(actual_val) != str(predicted_val) 86 | ): 87 | fp_protected += 1 88 | elif (str(predicted_val) != str(positive_pred)) & ( 89 | str(actual_val) == str(predicted_val) 90 | ): 91 | fn_protected += 1 92 | return fp_unprotected, fp_protected, fn_protected, fn_unprotected 93 | -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/models/__init__.py -------------------------------------------------------------------------------- /models/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/models/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /models/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/models/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /models/__pycache__/model_cfair.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/models/__pycache__/model_cfair.cpython-37.pyc -------------------------------------------------------------------------------- /models/__pycache__/model_cfair.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/models/__pycache__/model_cfair.cpython-38.pyc -------------------------------------------------------------------------------- /models/__pycache__/model_conv.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/models/__pycache__/model_conv.cpython-37.pyc -------------------------------------------------------------------------------- /models/__pycache__/model_conv.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/models/__pycache__/model_conv.cpython-38.pyc -------------------------------------------------------------------------------- /models/__pycache__/model_ffvae.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/models/__pycache__/model_ffvae.cpython-37.pyc -------------------------------------------------------------------------------- /models/__pycache__/model_ffvae.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/models/__pycache__/model_ffvae.cpython-38.pyc -------------------------------------------------------------------------------- /models/__pycache__/model_laftr.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/models/__pycache__/model_laftr.cpython-37.pyc -------------------------------------------------------------------------------- /models/__pycache__/model_laftr.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/models/__pycache__/model_laftr.cpython-38.pyc -------------------------------------------------------------------------------- /models/__pycache__/model_mlp.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/models/__pycache__/model_mlp.cpython-37.pyc -------------------------------------------------------------------------------- /models/__pycache__/model_mlp.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/models/__pycache__/model_mlp.cpython-38.pyc -------------------------------------------------------------------------------- /models/__pycache__/model_mubal.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/models/__pycache__/model_mubal.cpython-37.pyc -------------------------------------------------------------------------------- /models/__pycache__/model_mubal.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/models/__pycache__/model_mubal.cpython-38.pyc -------------------------------------------------------------------------------- /models/ablations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/models/ablations/__init__.py -------------------------------------------------------------------------------- /models/model_cfair.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | from torch.autograd import Function 8 | from models.model_mlp import MLP 9 | 10 | class GradReverse(Function): 11 | """ 12 | Implement the gradient reversal layer for the convenience of domain adaptation neural network. 13 | The forward part is the identity function while the backward part is the negative function. 14 | """ 15 | 16 | @staticmethod 17 | def forward(ctx, x): 18 | return x.view_as(x) 19 | 20 | @staticmethod 21 | def backward(ctx, grad_output): 22 | return grad_output.neg() 23 | 24 | 25 | def grad_reverse(x): 26 | return GradReverse.apply(x) 27 | 28 | 29 | """Module for CFAIR network 30 | Parameters 31 | ---------- 32 | args: ArgumentParser 33 | Contains all model and shared input arguments 34 | """ 35 | class CFairNet(nn.Module): 36 | """Initializes CFAIR network: MLP encoder, MLP classifier, 2 MLP discriminators""" 37 | def __init__(self, args): 38 | super(CFairNet, self).__init__() 39 | self.input_dim = args.input_dim 40 | self.num_classes = args.num_classes 41 | 42 | self.enc_neurons = [self.input_dim] + args.edepth * [args.ewidths] + [args.zdim] 43 | self.encoder = MLP(self.enc_neurons) 44 | 45 | self.class_neurons = ( 46 | [args.zdim] + args.cdepth * [args.cwidths] + [args.num_classes] 47 | ) 48 | self.classifier = MLP(self.class_neurons) 49 | 50 | self.adv_neurons = ( 51 | [args.zdim] + args.adepth * [args.awidths] + [args.num_groups] 52 | ) 53 | self.discriminator = nn.ModuleList( 54 | [MLP(self.adv_neurons) for _ in range(self.num_classes)] 55 | ) 56 | 57 | self.adv_coeff = args.adv_coeff 58 | self.device = args.device 59 | self.data = args.data 60 | 61 | self.optimizer = self.get_optimizer() 62 | 63 | @staticmethod 64 | def build_model(args): 65 | """ Builds CFAIR network """ 66 | model = CFairNet(args) 67 | return model 68 | 69 | def all_params(self): 70 | """Returns all model parameters""" 71 | return ( 72 | list(self.encoder.parameters()) 73 | + list(self.classifier.parameters()) 74 | + list(self.discriminator.parameters()) 75 | ) 76 | 77 | def get_optimizer(self): 78 | """Returns an optimizer""" 79 | if self.data == "adult": 80 | optim = torch.optim.Adadelta(self.all_params()) 81 | else: 82 | optim = torch.optim.Adam(self.all_params()) 83 | return optim 84 | 85 | def forward(self, inputs, labels, attrs, mode="train", A_wts=None, Y_wts=None, AY_wts=None, reweight_target_tensor=None, reweight_attr_tensors=None): 86 | """Computes forward pass through encoder , 87 | Computes backward pass on the target function""" 88 | h_relu = inputs 89 | 90 | h_relu = self.encoder(h_relu) 91 | 92 | # Classification probabilities. 93 | pre_softmax = self.classifier(h_relu) 94 | logprobs = F.log_softmax(pre_softmax, dim=1) 95 | 96 | main_cost = F.nll_loss( 97 | logprobs, 98 | labels, 99 | weight=reweight_target_tensor.type(torch.FloatTensor).to(self.device), 100 | ) 101 | 102 | # Adversary component. 103 | c_losses = [] 104 | h_relu = grad_reverse(h_relu) 105 | for j in range(self.num_classes): 106 | idx = labels == j 107 | c_h_relu = h_relu[idx] 108 | c_cls = F.log_softmax(self.discriminator[j](c_h_relu), dim=1) 109 | c_losses.append(c_cls) 110 | 111 | adv_cost = torch.mean( 112 | torch.stack( 113 | [ 114 | F.nll_loss( 115 | c_losses[j], 116 | attrs[labels == j], 117 | weight=reweight_attr_tensors[j] 118 | .type(torch.FloatTensor) 119 | .to(self.device), 120 | ) 121 | for j in range(self.num_classes) 122 | ] 123 | ) 124 | ) 125 | 126 | main_cost += self.adv_coeff * adv_cost 127 | 128 | # This would be the target of optimization 129 | cost_dict = dict( 130 | main_cost=main_cost, 131 | ) 132 | 133 | if mode == "train": 134 | self.optimizer.zero_grad() 135 | main_cost.backward() 136 | torch.nn.utils.clip_grad_norm_(self.all_params(), 5.0) 137 | self.optimizer.step() 138 | 139 | return pre_softmax, cost_dict 140 | -------------------------------------------------------------------------------- /models/model_conv.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | import torch.utils.data 8 | from models.model_mlp import MLP 9 | 10 | 11 | """Module for a single Convolutional Neural Network (CNN) unit 12 | Parameters 13 | ---------- 14 | args: ArgumentParser 15 | Contains all model and shared input arguments 16 | """ 17 | class CNN(nn.Module): 18 | # Input to ConvVAE is resized to 32 * 32 * 3, each pixel has 3 float values 19 | # between 0, 1 to represent each of RGB channels 20 | def __init__(self, args): 21 | """Initializes CNN class""" 22 | super(CNN, self).__init__() 23 | 24 | self.z_dim = args.zdim 25 | self.batch_size = args.batch_size 26 | 27 | self.enc = nn.Sequential( 28 | nn.Conv2d(3, 32, 4, 2, 1), 29 | nn.LeakyReLU(True), 30 | nn.Conv2d(32, 64, 4, 2, 1), 31 | nn.LeakyReLU(True), 32 | nn.Conv2d(64, 64, 4, 2, 1), 33 | nn.LeakyReLU(True), 34 | nn.Conv2d(64, 256, 4, 1), 35 | nn.LeakyReLU(True), 36 | nn.Conv2d(256, self.z_dim, 1) 37 | ) 38 | 39 | self.weight_init() 40 | 41 | def weight_init(self, mode="normal"): 42 | """Initializes weights of CNN parameters""" 43 | for block in self._modules: 44 | for m in self._modules[block]: 45 | if type(m) == nn.Conv2d: 46 | torch.nn.init.xavier_uniform_(m.weight) 47 | 48 | def encode(self, x): 49 | """Encodes an CNN input""" 50 | return self.enc(x).squeeze() 51 | 52 | 53 | """Module for CNN network 54 | Parameters 55 | ---------- 56 | args: ArgumentParser 57 | Contains all model and shared input arguments 58 | """ 59 | class ConvNet(nn.Module): 60 | """Initializes CNN network and an MLP classifier""" 61 | def __init__(self, args): 62 | super(ConvNet, self).__init__() 63 | self.input_dim = args.input_dim 64 | self.sensattr = args.sensattr 65 | 66 | self.cnn = CNN(args) 67 | 68 | self.class_neurons = ( 69 | [args.zdim] + args.cdepth * [args.cwidths] + [args.num_classes] 70 | ) 71 | self.linear_layer = MLP(self.class_neurons) 72 | 73 | self.zdim = args.zdim 74 | self.n_sens = 1 75 | self.sens_idx = list(range(self.n_sens)) 76 | self.nonsens_idx = [ 77 | i for i in range(int(self.zdim / 2)) if i not in self.sens_idx 78 | ] 79 | self.count = 0 80 | 81 | self.optimizer = self.get_optimizer() 82 | 83 | @staticmethod 84 | def build_model(args): 85 | """ Builds ConvNet class """ 86 | model = ConvNet(args) 87 | return model 88 | 89 | def all_params(self): 90 | """ Returns all parameters of the encoder CNN, classifier MLP """ 91 | return list(self.cnn.parameters()) + list(self.linear_layer.parameters()) 92 | 93 | def get_optimizer(self): 94 | """Returns optimizer over all parameters of the network""" 95 | return torch.optim.Adam(self.all_params()) 96 | 97 | def forward(self, inputs, labels, attrs, mode="train", A_wts=None, Y_wts=None, AY_wts=None, reweight_target_tensor=None, reweight_attr_tensors=None): 98 | """Computes forward pass through encoder and classifier, 99 | Computes backward pass on the target function""" 100 | # Make inputs between 0, 1 101 | x = (inputs + 1) / 2 102 | 103 | # encode 104 | h_relu = self.cnn.encode(x) 105 | 106 | # classification mlp 107 | pre_softmax = self.linear_layer(h_relu) 108 | 109 | log_softmax = F.log_softmax(pre_softmax, dim=1) 110 | 111 | main_cost = F.nll_loss(log_softmax, labels) 112 | 113 | # This would be the target of optimization 114 | cost_dict = dict( 115 | main_cost=main_cost, 116 | ) 117 | if mode == "train": 118 | self.optimizer.zero_grad() 119 | main_cost.backward() 120 | torch.nn.utils.clip_grad_norm_(self.all_params(), 5.0) 121 | self.optimizer.step() 122 | 123 | return pre_softmax, cost_dict 124 | -------------------------------------------------------------------------------- /models/model_laftr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import torch 5 | import torch.nn as nn 6 | from models.model_mlp import MLP 7 | 8 | """Module for LAFTR network 9 | Parameters 10 | ---------- 11 | args: ArgumentParser 12 | Contains all model and shared input arguments 13 | """ 14 | class LaftrNet(nn.Module): 15 | def __init__(self, args): 16 | """Initializes LAFTR network: MLP encoder, MLP classifier, MLP discriminator""" 17 | super(LaftrNet, self).__init__() 18 | 19 | self.enc_neurons = [args.input_dim] + args.edepth * [args.ewidths] + [args.zdim] 20 | self.class_neurons = ( 21 | [args.zdim] + args.cdepth * [args.cwidths] + [args.num_classes - 1] 22 | ) 23 | if args.arch != "laftr-dp": 24 | self.adv_neurons = ( 25 | [args.zdim + args.num_classes - 1] 26 | + args.adepth * [args.awidths] 27 | + [args.num_groups - 1] 28 | ) 29 | else: 30 | self.adv_neurons = ( 31 | [args.zdim] + args.adepth * [args.awidths] + [args.num_groups - 1] 32 | ) 33 | 34 | self.encoder = MLP(self.enc_neurons) 35 | self.classifier = MLP(self.class_neurons) 36 | self.discriminator = MLP(self.adv_neurons) 37 | 38 | self.device = args.device 39 | self.arch = args.arch 40 | self.fair_coeff = args.fair_coeff 41 | self.class_coeff = 1.0 42 | self.aud_steps = args.aud_steps 43 | 44 | self.optimizer, self.optimizer_disc = self.get_optimizer() 45 | 46 | @staticmethod 47 | def build_model(args): 48 | """ Builds LAFTR network """ 49 | model = LaftrNet(args) 50 | return model 51 | 52 | def get_optimizer(self): 53 | """Returns an optimizer for each network""" 54 | optimizer = torch.optim.Adam(self.predict_params()) 55 | optimizer_disc = torch.optim.Adam(self.audit_params()) 56 | return optimizer, optimizer_disc 57 | 58 | def predict_params(self): 59 | """Returns encoder and classifier parameters""" 60 | return list(self.classifier.parameters()) + list(self.encoder.parameters()) 61 | 62 | def audit_params(self): 63 | """Returns discriminator parameters""" 64 | return self.discriminator.parameters() 65 | 66 | def l1_loss(self, y, y_logits): 67 | """Returns l1 loss""" 68 | y_hat = torch.sigmoid(y_logits) 69 | return torch.squeeze(torch.abs(y - y_hat)) 70 | 71 | def ce_loss(self, y, y_logits, eps=1e-8): 72 | """Returns cross entropy loss""" 73 | y_hat = torch.sigmoid(y_logits) 74 | return -torch.sum( 75 | y * torch.log(y_hat + eps) + (1 - y) * torch.log(1 - y_hat + eps) 76 | ) 77 | 78 | def get_weighted_aud_loss(self, L, X, Y, A, A_wts, Y_wts, AY_wts): 79 | """Returns weighted discriminator loss""" 80 | if self.arch == "laftr-dp": 81 | A0_wt = A_wts[0] 82 | A1_wt = A_wts[1] 83 | wts = A0_wt * (1 - A) + A1_wt * A 84 | wtd_L = L * torch.squeeze(wts) 85 | elif ( 86 | self.arch == "laftr-eqodd" 87 | or self.arch == "laftr-eqopp0" 88 | or self.arch == "laftr-eqopp1" 89 | ): 90 | A0_Y0_wt = AY_wts[0][0] 91 | A0_Y1_wt = AY_wts[0][1] 92 | A1_Y0_wt = AY_wts[1][0] 93 | A1_Y1_wt = AY_wts[1][1] 94 | 95 | if self.arch == "laftr-eqodd": 96 | wts = ( 97 | A0_Y0_wt * (1 - A) * (1 - Y) 98 | + A0_Y1_wt * (1 - A) * (Y) 99 | + A1_Y0_wt * (A) * (1 - Y) 100 | + A1_Y1_wt * (A) * (Y) 101 | ) 102 | elif self.arch == "laftr-eqopp0": 103 | wts = A0_Y0_wt * (1 - A) * (1 - Y) + A1_Y0_wt * (A) * (1 - Y) 104 | elif self.arch == "laftr-eqopp1": 105 | wts = A0_Y1_wt * (1 - A) * (Y) + A1_Y1_wt * (A) * (Y) 106 | 107 | wtd_L = L * torch.squeeze(wts) 108 | else: 109 | raise Exception("Wrong model name") 110 | exit(0) 111 | 112 | return wtd_L 113 | 114 | def forward(self, X, Y, A, mode="train", A_wts=None, Y_wts=None, AY_wts=None, reweight_target_tensor=None, reweight_attr_tensors=None): 115 | """Computes forward pass through encoder , 116 | Computes backward pass on the target function""" 117 | Z = self.encoder(X) 118 | Y_logits = torch.squeeze(self.classifier(Z)) 119 | 120 | if self.arch != "laftr-dp": 121 | Z = torch.cat( 122 | [Z, torch.unsqueeze(Y.type(torch.FloatTensor), 1).to(self.device)], 123 | axis=1, 124 | ) 125 | class_loss = self.class_coeff * torch.mean(self.ce_loss(Y, Y_logits)) 126 | 127 | # discriminator loss 128 | A_logits = torch.squeeze(self.discriminator(Z)) 129 | aud_loss = -self.fair_coeff * self.l1_loss(A, A_logits) 130 | weighted_aud_loss = self.get_weighted_aud_loss( 131 | aud_loss, X, Y, A, A_wts, Y_wts, AY_wts 132 | ) 133 | weighted_aud_loss = torch.mean(weighted_aud_loss) 134 | 135 | loss = class_loss + weighted_aud_loss 136 | 137 | torch.autograd.set_detect_anomaly(True) 138 | 139 | # train (encoder, classifier), discriminator alternatively 140 | if mode == "train": 141 | self.optimizer.zero_grad() 142 | loss.backward(retain_graph=True) 143 | torch.nn.utils.clip_grad_norm_(self.predict_params(), 5.0) 144 | self.optimizer.step() 145 | 146 | for i in range(self.aud_steps): 147 | self.optimizer_disc.zero_grad() 148 | if i != self.aud_steps - 1: 149 | loss.backward(retain_graph=True) 150 | else: 151 | loss.backward() 152 | torch.nn.utils.clip_grad_norm_(self.audit_params(), 5.0) 153 | self.optimizer_disc.step() 154 | 155 | cost_dict = dict( 156 | main_cost=loss, 157 | ) 158 | 159 | logits = torch.sigmoid(Y_logits) 160 | 161 | return logits, cost_dict 162 | -------------------------------------------------------------------------------- /models/model_mlp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | 8 | 9 | """Module for a single Multi Layer Perception (MLP) unit 10 | Parameters 11 | ---------- 12 | num_neurons: list, dtype: int 13 | Number of neurons in each layer 14 | activ: string, default: "leakyrelu" 15 | Activation function 16 | """ 17 | class MLP(nn.Module): 18 | def __init__(self, num_neurons, activ="leakyrelu"): 19 | """Initializes MLP unit""" 20 | super(MLP, self).__init__() 21 | self.num_neurons = num_neurons 22 | self.num_layers = len(self.num_neurons) - 1 23 | self.hiddens = nn.ModuleList( 24 | [ 25 | nn.Linear(self.num_neurons[i], self.num_neurons[i + 1]) 26 | for i in range(self.num_layers) 27 | ] 28 | ) 29 | for hidden in self.hiddens: 30 | torch.nn.init.xavier_uniform_(hidden.weight) 31 | self.activ = activ 32 | 33 | def forward(self, inputs): 34 | """Computes forward pass through the model""" 35 | L = inputs 36 | for hidden in self.hiddens: 37 | L = hidden(L) 38 | if self.activ == "softplus": 39 | L = F.softplus(L) 40 | elif self.activ == "sigmoid": 41 | L = F.sigmoid(L) 42 | elif self.activ == "relu": 43 | L = F.relu(L) 44 | elif self.activ == "leakyrelu": 45 | L = F.leaky_relu(L) 46 | elif self.activ == "None": 47 | pass 48 | else: 49 | raise Exception("bad activation function") 50 | return L 51 | 52 | def freeze(self): 53 | """Stops gradient computation through MLP parameters""" 54 | for para in self.parameters(): 55 | para.requires_grad = False 56 | 57 | def activate(self): 58 | """Activates gradient computation through MLP parameters""" 59 | for para in self.parameters(): 60 | para.requires_grad = True 61 | 62 | 63 | 64 | """Module for MLP network 65 | Parameters 66 | ---------- 67 | args: ArgumentParser 68 | Contains all model and shared input arguments 69 | """ 70 | class MLPNet(nn.Module): 71 | def __init__(self, args): 72 | super(MLPNet, self).__init__() 73 | self.input_dim = args.input_dim 74 | 75 | self.enc_neurons = [self.input_dim] + args.edepth * [args.ewidths] + [args.zdim] 76 | self.encoder = MLP(self.enc_neurons) 77 | 78 | self.class_neurons = ( 79 | [args.zdim] + args.cdepth * [args.cwidths] + [args.num_classes] 80 | ) 81 | self.classifier = MLP(self.class_neurons) 82 | 83 | # Parameter of the final softmax classification layer. 84 | self.num_classes = args.num_classes 85 | 86 | self.optimizer = self.get_optimizer() 87 | 88 | @staticmethod 89 | def build_model(args): 90 | """ Builds MLPNet class """ 91 | model = MLPNet(args) 92 | return model 93 | 94 | def all_params(self): 95 | """ Returns all parameters of the MLP model """ 96 | return list(self.encoder.parameters()) + list(self.classifier.parameters()) 97 | 98 | def get_optimizer(self): 99 | """Returns optimizer over all parameters of MLP""" 100 | return torch.optim.Adam(self.all_params()) 101 | 102 | def forward(self, inputs, labels, attrs, mode="train", A_wts=None, Y_wts=None, AY_wts=None, reweight_target_tensor=None, reweight_attr_tensors=None): 103 | """Computes forward pass through encoder and classifier, 104 | Computes backward pass on the target function""" 105 | h_relu = inputs 106 | 107 | # encoder mlp 108 | h_relu = self.encoder(h_relu) 109 | 110 | # classification mlp 111 | pre_softmax = self.classifier(h_relu) 112 | 113 | log_softmax = F.log_softmax(pre_softmax, dim=1) 114 | 115 | main_cost = F.nll_loss(log_softmax, labels) 116 | 117 | # target of optimization 118 | cost_dict = dict( 119 | main_cost=main_cost, 120 | ) 121 | if mode == "train": 122 | self.optimizer.zero_grad() 123 | main_cost.backward() 124 | torch.nn.utils.clip_grad_norm_(self.all_params(), 5.0) 125 | self.optimizer.step() 126 | 127 | return pre_softmax, cost_dict 128 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch 2 | torchvision 3 | pandas 4 | requests 5 | sklearn 6 | tensorboardX 7 | tensorboard 8 | tensorflow 9 | scikit-image 10 | autopep8 11 | wandb 12 | matplotlib 13 | scikit-learn 14 | fire 15 | python-git 16 | -------------------------------------------------------------------------------- /scripts/Adult/settings12/cfair_multiple_launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | For reproducing Setting 1, 2 experiments with Cfair model on Adult dataset 3 | """ 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import itertools 8 | import subprocess 9 | import re 10 | import git 11 | import os 12 | import hashlib 13 | 14 | dataset = ['adult'] 15 | seeds = [3, 4, 5] 16 | clr_ratios = [(0.5, 0.5), (0.01, 0.01), (0.66, 0.33), (0.1, 0.1), (0.06, 0.36)] 17 | sensitiveattrs = ['age'] 18 | pos_ratios = [(0.5, 0.5)] 19 | 20 | archs = ['cfair', 'cfair-eo'] 21 | 22 | fair_coeffs = [1.0] 23 | aud_steps = [2] 24 | adv_coeffs = [0.1, 1.0, 10.0, 100.0, 1000.0] 25 | gammas = [0.1] 26 | alphas = [10] 27 | 28 | widths = [32] 29 | edepths = [2] 30 | ewidths = [32] 31 | adepths = [2] 32 | awidths = [32] 33 | cdepths = [2] 34 | cwidths = [32] 35 | zdims = [16] 36 | 37 | data_dir = ['./data/'] 38 | out_dir = ['/scratch/charanr/fairness-project/output/'] 39 | replicate = [1] 40 | num_epochs = [150] 41 | patiences = [5] 42 | 43 | all_experiments = list(itertools.product(data_dir, out_dir, 44 | dataset, 45 | clr_ratios, pos_ratios, 46 | sensitiveattrs, 47 | edepths, ewidths, adepths, 48 | awidths, cdepths, cwidths, 49 | zdims, 50 | seeds, archs, 51 | fair_coeffs, aud_steps, 52 | adv_coeffs, gammas, alphas, 53 | replicate, num_epochs, patiences)) 54 | 55 | 56 | print(f"Total number of jobs = {len(all_experiments)}") 57 | for idx, exp in enumerate(all_experiments): 58 | 59 | MAIN_CMD = f"./scripts/slurm_launcher.sh" \ 60 | f" --ddir {exp[0]}"\ 61 | f" --odir {exp[1]}"\ 62 | f" --dsetname {exp[2]}"\ 63 | f" --beta_1 {exp[3][0]}"\ 64 | f" --beta_2 {exp[3][1]}"\ 65 | f" --egr {exp[4][0]}"\ 66 | f" --ogr {exp[4][1]}"\ 67 | f" --sattr {exp[5]}"\ 68 | f" --edpt {exp[6]}"\ 69 | f" --ewd {exp[7]}"\ 70 | f" --adpt {exp[8]}"\ 71 | f" --awd {exp[9]}"\ 72 | f" --cdpt {exp[10]}"\ 73 | f" --cwd {exp[11]}"\ 74 | f" --zdim {exp[12]}"\ 75 | f" --seed {exp[13]}"\ 76 | f" --arch {exp[14]}"\ 77 | f" --fair_coeff {exp[15]}"\ 78 | f" --aud_steps {exp[16]}"\ 79 | f" --adv_coeff {exp[17]}"\ 80 | f" --gamma {exp[18]}"\ 81 | f" --alpha {exp[19]}"\ 82 | f" --replicate {exp[20]}"\ 83 | f" --num_epochs {exp[21]}"\ 84 | f" --ptnc {exp[22]}"\ 85 | 86 | hashed_id = int(hashlib.sha1(MAIN_CMD.encode( 87 | 'utf-8')).hexdigest(), 16) % (10 ** 8) 88 | MAIN_CMD = f"{MAIN_CMD} --wdbn deepfairness-{hashed_id}" 89 | 90 | print("###################################") 91 | print(f"EXPERIMENT HASHED ID = {hashed_id}") 92 | print(f"COMMAND RUNNING = \n {MAIN_CMD} \n") 93 | print("###################################") 94 | CMD = MAIN_CMD.split(" ") 95 | 96 | process = subprocess.Popen(CMD, stdout=subprocess.PIPE) 97 | 98 | out, err = process.communicate() 99 | 100 | print(out) 101 | -------------------------------------------------------------------------------- /scripts/Adult/settings12/ffvae_multiple_launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | For reproducing Setting 1, 2 experiments with Ffvae model on Adult dataset 3 | """ 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import itertools 8 | import subprocess 9 | import re 10 | import git 11 | import os 12 | import hashlib 13 | 14 | dataset = ['adult'] 15 | seeds = [3, 4, 5] 16 | clr_ratios = [(0.5, 0.5), (0.01, 0.01), (0.66, 0.33), (0.1, 0.1), (0.06, 0.36)] 17 | sensitiveattrs = ['age'] 18 | pos_ratios = [(0.5, 0.5)] 19 | 20 | archs = ['ffvae'] 21 | 22 | fair_coeffs = [1.0] 23 | aud_steps = [2] 24 | adv_coeffs = [0.1] 25 | gammas = [10, 50, 100] 26 | alphas = [10, 100, 1000] 27 | 28 | widths = [32] 29 | edepths = [2] 30 | ewidths = [32] 31 | adepths = [2] 32 | awidths = [32] 33 | cdepths = [2] 34 | cwidths = [32] 35 | zdims = [16] 36 | 37 | data_dir = ['./data/'] 38 | out_dir = ['/scratch/charanr/fairness-project/output/'] 39 | replicate = [1] 40 | num_epochs = [150] 41 | patiences = [5] 42 | 43 | all_experiments = list(itertools.product(data_dir, out_dir, 44 | dataset, 45 | clr_ratios, pos_ratios, 46 | sensitiveattrs, 47 | edepths, ewidths, adepths, 48 | awidths, cdepths, cwidths, 49 | zdims, 50 | seeds, archs, 51 | fair_coeffs, aud_steps, 52 | adv_coeffs, gammas, alphas, 53 | replicate, num_epochs, patiences)) 54 | 55 | 56 | print(f"Total number of jobs = {len(all_experiments)}") 57 | for idx, exp in enumerate(all_experiments): 58 | 59 | MAIN_CMD = f"./scripts/slurm_launcher.sh" \ 60 | f" --ddir {exp[0]}"\ 61 | f" --odir {exp[1]}"\ 62 | f" --dsetname {exp[2]}"\ 63 | f" --beta_1 {exp[3][0]}"\ 64 | f" --beta_2 {exp[3][1]}"\ 65 | f" --egr {exp[4][0]}"\ 66 | f" --ogr {exp[4][1]}"\ 67 | f" --sattr {exp[5]}"\ 68 | f" --edpt {exp[6]}"\ 69 | f" --ewd {exp[7]}"\ 70 | f" --adpt {exp[8]}"\ 71 | f" --awd {exp[9]}"\ 72 | f" --cdpt {exp[10]}"\ 73 | f" --cwd {exp[11]}"\ 74 | f" --zdim {exp[12]}"\ 75 | f" --seed {exp[13]}"\ 76 | f" --arch {exp[14]}"\ 77 | f" --fair_coeff {exp[15]}"\ 78 | f" --aud_steps {exp[16]}"\ 79 | f" --adv_coeff {exp[17]}"\ 80 | f" --gamma {exp[18]}"\ 81 | f" --alpha {exp[19]}"\ 82 | f" --replicate {exp[20]}"\ 83 | f" --num_epochs {exp[21]}"\ 84 | f" --ptnc {exp[22]}"\ 85 | 86 | hashed_id = int(hashlib.sha1(MAIN_CMD.encode( 87 | 'utf-8')).hexdigest(), 16) % (10 ** 8) 88 | MAIN_CMD = f"{MAIN_CMD} --wdbn deepfairness-{hashed_id}" 89 | 90 | print("###################################") 91 | print(f"EXPERIMENT HASHED ID = {hashed_id}") 92 | print(f"COMMAND RUNNING = \n {MAIN_CMD} \n") 93 | print("###################################") 94 | CMD = MAIN_CMD.split(" ") 95 | 96 | process = subprocess.Popen(CMD, stdout=subprocess.PIPE) 97 | 98 | out, err = process.communicate() 99 | 100 | print(out) 101 | -------------------------------------------------------------------------------- /scripts/Adult/settings12/laftr_multiple_launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | For reproducing Setting 1, 2 experiments with Laftr model on Adult dataset 3 | """ 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import itertools 8 | import subprocess 9 | import re 10 | import git 11 | import os 12 | import hashlib 13 | 14 | dataset = ['adult'] 15 | seeds = [3, 4, 5] 16 | clr_ratios = [(0.5, 0.5), (0.01, 0.01), (0.66, 0.33), (0.1, 0.1), (0.06, 0.36)] 17 | sensitiveattrs = ['age'] 18 | pos_ratios = [(0.5, 0.5)] 19 | 20 | archs = ['laftr-dp', 'laftr-eqopp1', 'laftr-eqopp0', 'laftr-eqodd'] 21 | 22 | fair_coeffs = [0.1, 0.5, 1.0, 2.0, 4.0] 23 | aud_steps = [2] 24 | adv_coeffs = [0.1] 25 | gammas = [0.1] 26 | alphas = [10] 27 | 28 | widths = [32] 29 | edepths = [2] 30 | ewidths = [32] 31 | adepths = [2] 32 | awidths = [32] 33 | cdepths = [2] 34 | cwidths = [32] 35 | zdims = [16] 36 | 37 | data_dir = ['./data/'] 38 | out_dir = ['/scratch/charanr/fairness-project/output/'] 39 | replicate = [1] 40 | num_epochs = [150] 41 | patiences = [5] 42 | 43 | all_experiments = list(itertools.product(data_dir, out_dir, 44 | dataset, 45 | clr_ratios, pos_ratios, 46 | sensitiveattrs, 47 | edepths, ewidths, adepths, 48 | awidths, cdepths, cwidths, 49 | zdims, 50 | seeds, archs, 51 | fair_coeffs, aud_steps, 52 | adv_coeffs, gammas, alphas, 53 | replicate, num_epochs, patiences)) 54 | 55 | 56 | print(f"Total number of jobs = {len(all_experiments)}") 57 | for idx, exp in enumerate(all_experiments): 58 | 59 | MAIN_CMD = f"./scripts/slurm_launcher.sh" \ 60 | f" --ddir {exp[0]}"\ 61 | f" --odir {exp[1]}"\ 62 | f" --dsetname {exp[2]}"\ 63 | f" --beta_1 {exp[3][0]}"\ 64 | f" --beta_2 {exp[3][1]}"\ 65 | f" --egr {exp[4][0]}"\ 66 | f" --ogr {exp[4][1]}"\ 67 | f" --sattr {exp[5]}"\ 68 | f" --edpt {exp[6]}"\ 69 | f" --ewd {exp[7]}"\ 70 | f" --adpt {exp[8]}"\ 71 | f" --awd {exp[9]}"\ 72 | f" --cdpt {exp[10]}"\ 73 | f" --cwd {exp[11]}"\ 74 | f" --zdim {exp[12]}"\ 75 | f" --seed {exp[13]}"\ 76 | f" --arch {exp[14]}"\ 77 | f" --fair_coeff {exp[15]}"\ 78 | f" --aud_steps {exp[16]}"\ 79 | f" --adv_coeff {exp[17]}"\ 80 | f" --gamma {exp[18]}"\ 81 | f" --alpha {exp[19]}"\ 82 | f" --replicate {exp[20]}"\ 83 | f" --num_epochs {exp[21]}"\ 84 | f" --ptnc {exp[22]}"\ 85 | 86 | hashed_id = int(hashlib.sha1(MAIN_CMD.encode( 87 | 'utf-8')).hexdigest(), 16) % (10 ** 8) 88 | MAIN_CMD = f"{MAIN_CMD} --wdbn deepfairness-{hashed_id}" 89 | 90 | print("###################################") 91 | print(f"EXPERIMENT HASHED ID = {hashed_id}") 92 | print(f"COMMAND RUNNING = \n {MAIN_CMD} \n") 93 | print("###################################") 94 | CMD = MAIN_CMD.split(" ") 95 | 96 | process = subprocess.Popen(CMD, stdout=subprocess.PIPE) 97 | 98 | out, err = process.communicate() 99 | 100 | print(out) 101 | -------------------------------------------------------------------------------- /scripts/Adult/settings12/mlp_multiple_launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | For reproducing Setting 1, 2 experiments with Mlp model on Adult dataset 3 | """ 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import itertools 8 | import subprocess 9 | import re 10 | import git 11 | import os 12 | import hashlib 13 | 14 | dataset = ['adult'] 15 | seeds = [3, 4, 5] 16 | clr_ratios = [(0.5, 0.5), (0.01, 0.01), (0.66, 0.33), (0.1, 0.1), (0.06, 0.36)] 17 | sensitiveattrs = ['age'] 18 | pos_ratios = [(0.5, 0.5)] 19 | 20 | archs = ['mlp'] 21 | 22 | fair_coeffs = [0.1] 23 | aud_steps = [2] 24 | adv_coeffs = [0.1] 25 | gammas = [0.1] 26 | alphas = [10] 27 | 28 | widths = [32] 29 | edepths = [2] 30 | ewidths = [32] 31 | adepths = [2] 32 | awidths = [32] 33 | cdepths = [2] 34 | cwidths = [32] 35 | zdims = [16] 36 | 37 | data_dir = ['./data/'] 38 | out_dir = ['/scratch/charanr/fairness-project/output/'] 39 | replicate = [1] 40 | num_epochs = [150] 41 | patiences = [5] 42 | 43 | all_experiments = list(itertools.product(data_dir, out_dir, 44 | dataset, 45 | clr_ratios, pos_ratios, 46 | sensitiveattrs, 47 | edepths, ewidths, adepths, 48 | awidths, cdepths, cwidths, 49 | zdims, 50 | seeds, archs, 51 | fair_coeffs, aud_steps, 52 | adv_coeffs, gammas, alphas, 53 | replicate, num_epochs, patiences)) 54 | 55 | 56 | print(f"Total number of jobs = {len(all_experiments)}") 57 | for idx, exp in enumerate(all_experiments): 58 | 59 | MAIN_CMD = f"./scripts/slurm_launcher.sh" \ 60 | f" --ddir {exp[0]}"\ 61 | f" --odir {exp[1]}"\ 62 | f" --dsetname {exp[2]}"\ 63 | f" --beta_1 {exp[3][0]}"\ 64 | f" --beta_2 {exp[3][1]}"\ 65 | f" --egr {exp[4][0]}"\ 66 | f" --ogr {exp[4][1]}"\ 67 | f" --sattr {exp[5]}"\ 68 | f" --edpt {exp[6]}"\ 69 | f" --ewd {exp[7]}"\ 70 | f" --adpt {exp[8]}"\ 71 | f" --awd {exp[9]}"\ 72 | f" --cdpt {exp[10]}"\ 73 | f" --cwd {exp[11]}"\ 74 | f" --zdim {exp[12]}"\ 75 | f" --seed {exp[13]}"\ 76 | f" --arch {exp[14]}"\ 77 | f" --fair_coeff {exp[15]}"\ 78 | f" --aud_steps {exp[16]}"\ 79 | f" --adv_coeff {exp[17]}"\ 80 | f" --gamma {exp[18]}"\ 81 | f" --alpha {exp[19]}"\ 82 | f" --replicate {exp[20]}"\ 83 | f" --num_epochs {exp[21]}"\ 84 | f" --ptnc {exp[22]}"\ 85 | 86 | hashed_id = int(hashlib.sha1(MAIN_CMD.encode( 87 | 'utf-8')).hexdigest(), 16) % (10 ** 8) 88 | MAIN_CMD = f"{MAIN_CMD} --wdbn deepfairness-{hashed_id}" 89 | 90 | print("###################################") 91 | print(f"EXPERIMENT HASHED ID = {hashed_id}") 92 | print(f"COMMAND RUNNING = \n {MAIN_CMD} \n") 93 | print("###################################") 94 | CMD = MAIN_CMD.split(" ") 95 | 96 | process = subprocess.Popen(CMD, stdout=subprocess.PIPE) 97 | 98 | out, err = process.communicate() 99 | 100 | print(out) 101 | -------------------------------------------------------------------------------- /scripts/CIMNIST/settings12/cfair_multiple_launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | For reproducing Setting 1, 2 experiments with Cfair model on CI-MNIST dataset 3 | """ 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import itertools 8 | import subprocess 9 | import re 10 | import git 11 | import os 12 | import hashlib 13 | 14 | dataset = ['clr-mnist'] 15 | seeds = [3, 4, 5] 16 | clr_ratios = [(0.1, 0.1), (0.01, 0.01), (0.001, 0.001), 17 | (0.1, 0.9), (0.01, 0.99), (0.5, 0.5)] 18 | sensitiveattrs = ['bck'] 19 | pos_ratios = [(0.5, 0.5)] 20 | 21 | archs = ['cfair'] 22 | 23 | fair_coeffs = [1.0] 24 | aud_steps = [2] 25 | adv_coeffs = [0.1, 1.0, 10.0, 100.0, 1000.0] 26 | gammas = [0.1] 27 | alphas = [10] 28 | 29 | widths = [32] 30 | edepths = [2] 31 | ewidths = [32] 32 | adepths = [2] 33 | awidths = [32] 34 | cdepths = [2] 35 | cwidths = [32] 36 | zdims = [16] 37 | 38 | data_dir = ['./data/'] 39 | out_dir = ['/scratch/charanr/fairness-project/output/'] 40 | replicate = [1] 41 | num_epochs = [150] 42 | patiences = [5] 43 | 44 | all_experiments = list(itertools.product(data_dir, out_dir, 45 | dataset, 46 | clr_ratios, pos_ratios, 47 | sensitiveattrs, 48 | edepths, ewidths, adepths, 49 | awidths, cdepths, cwidths, 50 | zdims, 51 | seeds, archs, 52 | fair_coeffs, aud_steps, 53 | adv_coeffs, gammas, alphas, 54 | replicate, num_epochs, patiences)) 55 | 56 | 57 | print(f"Total number of jobs = {len(all_experiments)}") 58 | for idx, exp in enumerate(all_experiments): 59 | 60 | MAIN_CMD = f"./scripts/slurm_launcher.sh" \ 61 | f" --ddir {exp[0]}"\ 62 | f" --odir {exp[1]}"\ 63 | f" --dsetname {exp[2]}"\ 64 | f" --beta_1 {exp[3][0]}"\ 65 | f" --beta_2 {exp[3][1]}"\ 66 | f" --egr {exp[4][0]}"\ 67 | f" --ogr {exp[4][1]}"\ 68 | f" --sattr {exp[5]}"\ 69 | f" --edpt {exp[6]}"\ 70 | f" --ewd {exp[7]}"\ 71 | f" --adpt {exp[8]}"\ 72 | f" --awd {exp[9]}"\ 73 | f" --cdpt {exp[10]}"\ 74 | f" --cwd {exp[11]}"\ 75 | f" --zdim {exp[12]}"\ 76 | f" --seed {exp[13]}"\ 77 | f" --arch {exp[14]}"\ 78 | f" --fair_coeff {exp[15]}"\ 79 | f" --aud_steps {exp[16]}"\ 80 | f" --adv_coeff {exp[17]}"\ 81 | f" --gamma {exp[18]}"\ 82 | f" --alpha {exp[19]}"\ 83 | f" --replicate {exp[20]}"\ 84 | f" --num_epochs {exp[21]}"\ 85 | f" --ptnc {exp[22]}"\ 86 | 87 | hashed_id = int(hashlib.sha1(MAIN_CMD.encode( 88 | 'utf-8')).hexdigest(), 16) % (10 ** 8) 89 | MAIN_CMD = f"{MAIN_CMD} --wdbn deepfairness-{hashed_id}" 90 | 91 | print("###################################") 92 | print(f"EXPERIMENT HASHED ID = {hashed_id}") 93 | print(f"COMMAND RUNNING = \n {MAIN_CMD} \n") 94 | print("###################################") 95 | CMD = MAIN_CMD.split(" ") 96 | 97 | process = subprocess.Popen(CMD, stdout=subprocess.PIPE) 98 | 99 | out, err = process.communicate() 100 | 101 | print(out) 102 | -------------------------------------------------------------------------------- /scripts/CIMNIST/settings12/ffvae_multiple_launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | For reproducing Setting 1, 2 experiments with Ffvae model on CI-MNIST dataset 3 | """ 4 | 5 | 6 | import pandas as pd 7 | import numpy as np 8 | import itertools 9 | import subprocess 10 | import re 11 | import git 12 | import os 13 | import hashlib 14 | 15 | dataset = ['clr-mnist'] 16 | seeds = [3, 4, 5] 17 | clr_ratios = [(0.1, 0.1), (0.01, 0.01), (0.001, 0.001), 18 | (0.1, 0.9), (0.01, 0.99), (0.5, 0.5)] 19 | sensitiveattrs = ['bck'] 20 | pos_ratios = [(0.5, 0.5)] 21 | 22 | archs = ['ffvae'] 23 | 24 | fair_coeffs = [1.0] 25 | aud_steps = [2] 26 | adv_coeffs = [1.0] 27 | gammas = [10, 50, 100] 28 | alphas = [10, 100, 1000] 29 | 30 | widths = [32] 31 | edepths = [2] 32 | ewidths = [32] 33 | adepths = [2] 34 | awidths = [32] 35 | cdepths = [2] 36 | cwidths = [32] 37 | zdims = [16] 38 | 39 | data_dir = ['./data/'] 40 | out_dir = ['/scratch/charanr/fairness-project/output/'] 41 | replicate = [1] 42 | num_epochs = [150] 43 | patiences = [5] 44 | 45 | all_experiments = list(itertools.product(data_dir, out_dir, 46 | dataset, 47 | clr_ratios, pos_ratios, 48 | sensitiveattrs, 49 | edepths, ewidths, adepths, 50 | awidths, cdepths, cwidths, 51 | zdims, 52 | seeds, archs, 53 | fair_coeffs, aud_steps, 54 | adv_coeffs, gammas, alphas, 55 | replicate, num_epochs, patiences)) 56 | 57 | 58 | print(f"Total number of jobs = {len(all_experiments)}") 59 | for idx, exp in enumerate(all_experiments): 60 | 61 | MAIN_CMD = f"./scripts/slurm_launcher.sh" \ 62 | f" --ddir {exp[0]}"\ 63 | f" --odir {exp[1]}"\ 64 | f" --dsetname {exp[2]}"\ 65 | f" --beta_1 {exp[3][0]}"\ 66 | f" --beta_2 {exp[3][1]}"\ 67 | f" --egr {exp[4][0]}"\ 68 | f" --ogr {exp[4][1]}"\ 69 | f" --sattr {exp[5]}"\ 70 | f" --edpt {exp[6]}"\ 71 | f" --ewd {exp[7]}"\ 72 | f" --adpt {exp[8]}"\ 73 | f" --awd {exp[9]}"\ 74 | f" --cdpt {exp[10]}"\ 75 | f" --cwd {exp[11]}"\ 76 | f" --zdim {exp[12]}"\ 77 | f" --seed {exp[13]}"\ 78 | f" --arch {exp[14]}"\ 79 | f" --fair_coeff {exp[15]}"\ 80 | f" --aud_steps {exp[16]}"\ 81 | f" --adv_coeff {exp[17]}"\ 82 | f" --gamma {exp[18]}"\ 83 | f" --alpha {exp[19]}"\ 84 | f" --replicate {exp[20]}"\ 85 | f" --num_epochs {exp[21]}"\ 86 | f" --ptnc {exp[22]}"\ 87 | 88 | hashed_id = int(hashlib.sha1(MAIN_CMD.encode( 89 | 'utf-8')).hexdigest(), 16) % (10 ** 8) 90 | MAIN_CMD = f"{MAIN_CMD} --wdbn deepfairness-{hashed_id}" 91 | 92 | print("###################################") 93 | print(f"EXPERIMENT HASHED ID = {hashed_id}") 94 | print(f"COMMAND RUNNING = \n {MAIN_CMD} \n") 95 | print("###################################") 96 | CMD = MAIN_CMD.split(" ") 97 | 98 | process = subprocess.Popen(CMD, stdout=subprocess.PIPE) 99 | 100 | out, err = process.communicate() 101 | 102 | print(out) 103 | -------------------------------------------------------------------------------- /scripts/CIMNIST/settings12/laftr_multiple_launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | For reproducing Setting 1, 2 experiments with Laftr model on CI-MNIST dataset 3 | """ 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import itertools 8 | import subprocess 9 | import re 10 | import git 11 | import os 12 | import hashlib 13 | 14 | dataset = ['clr-mnist'] 15 | seeds = [3, 4, 5] 16 | clr_ratios = [(0.1, 0.1), (0.01, 0.01), (0.001, 0.001), 17 | (0.1, 0.9), (0.01, 0.99), (0.5, 0.5)] 18 | sensitiveattrs = ['bck'] 19 | pos_ratios = [(0.5, 0.5)] 20 | 21 | archs = ['laftr-dp', 'laftr-eqopp0', 'laftr-eqopp1', 'laftr-eqodd'] 22 | 23 | fair_coeffs = [0.1, 0.5, 1.0, 2.0, 4.0] 24 | aud_steps = [2] 25 | adv_coeffs = [1.0] 26 | gammas = [10] 27 | alphas = [10] 28 | 29 | widths = [32] 30 | edepths = [2] 31 | ewidths = [32] 32 | adepths = [2] 33 | awidths = [32] 34 | cdepths = [2] 35 | cwidths = [32] 36 | zdims = [16] 37 | 38 | data_dir = ['./data/'] 39 | out_dir = ['/scratch/charanr/fairness-project/output/'] 40 | replicate = [1] 41 | num_epochs = [150] 42 | patiences = [5] 43 | 44 | all_experiments = list(itertools.product(data_dir, out_dir, 45 | dataset, 46 | clr_ratios, pos_ratios, 47 | sensitiveattrs, 48 | edepths, ewidths, adepths, 49 | awidths, cdepths, cwidths, 50 | zdims, 51 | seeds, archs, 52 | fair_coeffs, aud_steps, 53 | adv_coeffs, gammas, alphas, 54 | replicate, num_epochs, patiences)) 55 | 56 | 57 | print(f"Total number of jobs = {len(all_experiments)}") 58 | for idx, exp in enumerate(all_experiments): 59 | 60 | MAIN_CMD = f"./scripts/slurm_launcher.sh" \ 61 | f" --ddir {exp[0]}"\ 62 | f" --odir {exp[1]}"\ 63 | f" --dsetname {exp[2]}"\ 64 | f" --beta_1 {exp[3][0]}"\ 65 | f" --beta_2 {exp[3][1]}"\ 66 | f" --egr {exp[4][0]}"\ 67 | f" --ogr {exp[4][1]}"\ 68 | f" --sattr {exp[5]}"\ 69 | f" --edpt {exp[6]}"\ 70 | f" --ewd {exp[7]}"\ 71 | f" --adpt {exp[8]}"\ 72 | f" --awd {exp[9]}"\ 73 | f" --cdpt {exp[10]}"\ 74 | f" --cwd {exp[11]}"\ 75 | f" --zdim {exp[12]}"\ 76 | f" --seed {exp[13]}"\ 77 | f" --arch {exp[14]}"\ 78 | f" --fair_coeff {exp[15]}"\ 79 | f" --aud_steps {exp[16]}"\ 80 | f" --adv_coeff {exp[17]}"\ 81 | f" --gamma {exp[18]}"\ 82 | f" --alpha {exp[19]}"\ 83 | f" --replicate {exp[20]}"\ 84 | f" --num_epochs {exp[21]}"\ 85 | f" --ptnc {exp[22]}"\ 86 | 87 | hashed_id = int(hashlib.sha1(MAIN_CMD.encode( 88 | 'utf-8')).hexdigest(), 16) % (10 ** 8) 89 | MAIN_CMD = f"{MAIN_CMD} --wdbn deepfairness-{hashed_id}" 90 | 91 | print("###################################") 92 | print(f"EXPERIMENT HASHED ID = {hashed_id}") 93 | print(f"COMMAND RUNNING = \n {MAIN_CMD} \n") 94 | print("###################################") 95 | CMD = MAIN_CMD.split(" ") 96 | 97 | process = subprocess.Popen(CMD, stdout=subprocess.PIPE) 98 | 99 | out, err = process.communicate() 100 | 101 | print(out) 102 | -------------------------------------------------------------------------------- /scripts/CIMNIST/settings12/mlp_conv_multiple_launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | For reproducing Setting 1, 2 experiments with Cnn, Mlp models on CI-MNIST dataset 3 | """ 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import itertools 8 | import subprocess 9 | import re 10 | import git 11 | import os 12 | import hashlib 13 | 14 | dataset = ['clr-mnist'] 15 | seeds = [3, 4, 5] 16 | clr_ratios = [(0.1, 0.1), (0.01, 0.01), (0.001, 0.001), 17 | (0.1, 0.9), (0.01, 0.99), (0.5, 0.5)] 18 | sensitiveattrs = ['bck'] 19 | pos_ratios = [(0.5, 0.5)] 20 | 21 | archs = ['mlp', 'conv'] 22 | 23 | fair_coeffs = [0.1] 24 | aud_steps = [2] 25 | adv_coeffs = [1.0] 26 | gammas = [10] 27 | alphas = [10] 28 | 29 | widths = [32] 30 | edepths = [2] 31 | ewidths = [32] 32 | adepths = [2] 33 | awidths = [32] 34 | cdepths = [2] 35 | cwidths = [32] 36 | zdims = [16] 37 | 38 | data_dir = ['./data/'] 39 | out_dir = ['/scratch/charanr/fairness-project/output/'] 40 | replicate = [1] 41 | num_epochs = [150] 42 | patiences = [5] 43 | 44 | all_experiments = list(itertools.product(data_dir, out_dir, 45 | dataset, 46 | clr_ratios, pos_ratios, 47 | sensitiveattrs, 48 | edepths, ewidths, adepths, 49 | awidths, cdepths, cwidths, 50 | zdims, 51 | seeds, archs, 52 | fair_coeffs, aud_steps, 53 | adv_coeffs, gammas, alphas, 54 | replicate, num_epochs, patiences)) 55 | 56 | 57 | print(f"Total number of jobs = {len(all_experiments)}") 58 | for idx, exp in enumerate(all_experiments): 59 | 60 | MAIN_CMD = f"./scripts/slurm_launcher.sh" \ 61 | f" --ddir {exp[0]}"\ 62 | f" --odir {exp[1]}"\ 63 | f" --dsetname {exp[2]}"\ 64 | f" --beta_1 {exp[3][0]}"\ 65 | f" --beta_2 {exp[3][1]}"\ 66 | f" --egr {exp[4][0]}"\ 67 | f" --ogr {exp[4][1]}"\ 68 | f" --sattr {exp[5]}"\ 69 | f" --edpt {exp[6]}"\ 70 | f" --ewd {exp[7]}"\ 71 | f" --adpt {exp[8]}"\ 72 | f" --awd {exp[9]}"\ 73 | f" --cdpt {exp[10]}"\ 74 | f" --cwd {exp[11]}"\ 75 | f" --zdim {exp[12]}"\ 76 | f" --seed {exp[13]}"\ 77 | f" --arch {exp[14]}"\ 78 | f" --fair_coeff {exp[15]}"\ 79 | f" --aud_steps {exp[16]}"\ 80 | f" --adv_coeff {exp[17]}"\ 81 | f" --gamma {exp[18]}"\ 82 | f" --alpha {exp[19]}"\ 83 | f" --replicate {exp[20]}"\ 84 | f" --num_epochs {exp[21]}"\ 85 | f" --ptnc {exp[22]}"\ 86 | 87 | hashed_id = int(hashlib.sha1(MAIN_CMD.encode( 88 | 'utf-8')).hexdigest(), 16) % (10 ** 8) 89 | MAIN_CMD = f"{MAIN_CMD} --wdbn deepfairness-{hashed_id}" 90 | 91 | print("###################################") 92 | print(f"EXPERIMENT HASHED ID = {hashed_id}") 93 | print(f"COMMAND RUNNING = \n {MAIN_CMD} \n") 94 | print("###################################") 95 | CMD = MAIN_CMD.split(" ") 96 | 97 | process = subprocess.Popen(CMD, stdout=subprocess.PIPE) 98 | 99 | out, err = process.communicate() 100 | 101 | print(out) 102 | -------------------------------------------------------------------------------- /scripts/CIMNIST/settings34/cfair_multiple_launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | For reproducing Setting 3, 4 experiments with Cfair model on CI-MNIST dataset 3 | """ 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import itertools 8 | import subprocess 9 | import re 10 | import git 11 | import os 12 | import hashlib 13 | 14 | dataset = ['clr-mnist'] 15 | seeds = [3, 4, 5] 16 | clr_ratios = [(0.5, 0.5)] 17 | sensitiveattrs = ['bck', 'color_gy'] 18 | pos_ratios = [(0.5, 0.5), (0.75, 0.25), (0.9, 0.1)] 19 | 20 | archs = ['cfair'] 21 | 22 | fair_coeffs = [1.0] 23 | aud_steps = [2] 24 | adv_coeffs = [0.1, 1.0, 10.0, 100.0, 1000.0] 25 | gammas = [0.1] 26 | alphas = [10] 27 | 28 | widths = [32] 29 | edepths = [2] 30 | ewidths = [32] 31 | adepths = [2] 32 | awidths = [32] 33 | cdepths = [2] 34 | cwidths = [32] 35 | zdims = [16] 36 | 37 | data_dir = ['./data/'] 38 | out_dir = ['/scratch/charanr/fairness-project/output/'] 39 | replicate = [1] 40 | num_epochs = [150] 41 | patiences = [5] 42 | 43 | all_experiments = list(itertools.product(data_dir, out_dir, 44 | dataset, 45 | clr_ratios, pos_ratios, 46 | sensitiveattrs, 47 | edepths, ewidths, adepths, 48 | awidths, cdepths, cwidths, 49 | zdims, 50 | seeds, archs, 51 | fair_coeffs, aud_steps, 52 | adv_coeffs, gammas, alphas, 53 | replicate, num_epochs, patiences)) 54 | 55 | 56 | print(f"Total number of jobs = {len(all_experiments)}") 57 | for idx, exp in enumerate(all_experiments): 58 | 59 | MAIN_CMD = f"./scripts/slurm_launcher.sh" \ 60 | f" --ddir {exp[0]}"\ 61 | f" --odir {exp[1]}"\ 62 | f" --dsetname {exp[2]}"\ 63 | f" --beta_1 {exp[3][0]}"\ 64 | f" --beta_2 {exp[3][1]}"\ 65 | f" --egr {exp[4][0]}"\ 66 | f" --ogr {exp[4][1]}"\ 67 | f" --sattr {exp[5]}"\ 68 | f" --edpt {exp[6]}"\ 69 | f" --ewd {exp[7]}"\ 70 | f" --adpt {exp[8]}"\ 71 | f" --awd {exp[9]}"\ 72 | f" --cdpt {exp[10]}"\ 73 | f" --cwd {exp[11]}"\ 74 | f" --zdim {exp[12]}"\ 75 | f" --seed {exp[13]}"\ 76 | f" --arch {exp[14]}"\ 77 | f" --fair_coeff {exp[15]}"\ 78 | f" --aud_steps {exp[16]}"\ 79 | f" --adv_coeff {exp[17]}"\ 80 | f" --gamma {exp[18]}"\ 81 | f" --alpha {exp[19]}"\ 82 | f" --replicate {exp[20]}"\ 83 | f" --num_epochs {exp[21]}"\ 84 | f" --ptnc {exp[22]}"\ 85 | 86 | hashed_id = int(hashlib.sha1(MAIN_CMD.encode( 87 | 'utf-8')).hexdigest(), 16) % (10 ** 8) 88 | MAIN_CMD = f"{MAIN_CMD} --wdbn deepfairness-{hashed_id}" 89 | 90 | print("###################################") 91 | print(f"EXPERIMENT HASHED ID = {hashed_id}") 92 | print(f"COMMAND RUNNING = \n {MAIN_CMD} \n") 93 | print("###################################") 94 | CMD = MAIN_CMD.split(" ") 95 | 96 | process = subprocess.Popen(CMD, stdout=subprocess.PIPE) 97 | 98 | out, err = process.communicate() 99 | 100 | print(out) 101 | -------------------------------------------------------------------------------- /scripts/CIMNIST/settings34/ffvae_multiple_launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | For reproducing Setting 3, 4 experiments with Ffvae model on CI-MNIST dataset 3 | """ 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import itertools 8 | import subprocess 9 | import re 10 | import git 11 | import os 12 | import hashlib 13 | 14 | dataset = ['clr-mnist'] 15 | seeds = [3, 4, 5] 16 | clr_ratios = [(0.5, 0.5)] 17 | sensitiveattrs = ['bck', 'color_gy'] 18 | pos_ratios = [(0.5, 0.5), (0.75, 0.25), (0.9, 0.1)] 19 | 20 | archs = ['ffvae'] 21 | 22 | fair_coeffs = [1.0] 23 | aud_steps = [2] 24 | adv_coeffs = [0.1] 25 | gammas = [10, 50, 100] 26 | alphas = [10, 100, 1000] 27 | 28 | widths = [32] 29 | edepths = [2] 30 | ewidths = [32] 31 | adepths = [2] 32 | awidths = [32] 33 | cdepths = [2] 34 | cwidths = [32] 35 | zdims = [16] 36 | 37 | data_dir = ['./data/'] 38 | out_dir = ['/scratch/charanr/fairness-project/output/'] 39 | replicate = [1] 40 | num_epochs = [150] 41 | patiences = [5] 42 | 43 | all_experiments = list(itertools.product(data_dir, out_dir, 44 | dataset, 45 | clr_ratios, pos_ratios, 46 | sensitiveattrs, 47 | edepths, ewidths, adepths, 48 | awidths, cdepths, cwidths, 49 | zdims, 50 | seeds, archs, 51 | fair_coeffs, aud_steps, 52 | adv_coeffs, gammas, alphas, 53 | replicate, num_epochs, patiences)) 54 | 55 | 56 | print(f"Total number of jobs = {len(all_experiments)}") 57 | for idx, exp in enumerate(all_experiments): 58 | 59 | MAIN_CMD = f"./scripts/slurm_launcher.sh" \ 60 | f" --ddir {exp[0]}"\ 61 | f" --odir {exp[1]}"\ 62 | f" --dsetname {exp[2]}"\ 63 | f" --beta_1 {exp[3][0]}"\ 64 | f" --beta_2 {exp[3][1]}"\ 65 | f" --egr {exp[4][0]}"\ 66 | f" --ogr {exp[4][1]}"\ 67 | f" --sattr {exp[5]}"\ 68 | f" --edpt {exp[6]}"\ 69 | f" --ewd {exp[7]}"\ 70 | f" --adpt {exp[8]}"\ 71 | f" --awd {exp[9]}"\ 72 | f" --cdpt {exp[10]}"\ 73 | f" --cwd {exp[11]}"\ 74 | f" --zdim {exp[12]}"\ 75 | f" --seed {exp[13]}"\ 76 | f" --arch {exp[14]}"\ 77 | f" --fair_coeff {exp[15]}"\ 78 | f" --aud_steps {exp[16]}"\ 79 | f" --adv_coeff {exp[17]}"\ 80 | f" --gamma {exp[18]}"\ 81 | f" --alpha {exp[19]}"\ 82 | f" --replicate {exp[20]}"\ 83 | f" --num_epochs {exp[21]}"\ 84 | f" --ptnc {exp[22]}"\ 85 | 86 | hashed_id = int(hashlib.sha1(MAIN_CMD.encode( 87 | 'utf-8')).hexdigest(), 16) % (10 ** 8) 88 | MAIN_CMD = f"{MAIN_CMD} --wdbn deepfairness-{hashed_id}" 89 | 90 | print("###################################") 91 | print(f"EXPERIMENT HASHED ID = {hashed_id}") 92 | print(f"COMMAND RUNNING = \n {MAIN_CMD} \n") 93 | print("###################################") 94 | CMD = MAIN_CMD.split(" ") 95 | 96 | process = subprocess.Popen(CMD, stdout=subprocess.PIPE) 97 | 98 | out, err = process.communicate() 99 | 100 | print(out) 101 | -------------------------------------------------------------------------------- /scripts/CIMNIST/settings34/laftr_multiple_launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | For reproducing Setting 3, 4 experiments with Laftr model on CI-MNIST dataset 3 | """ 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import itertools 8 | import subprocess 9 | import re 10 | import git 11 | import os 12 | import hashlib 13 | 14 | dataset = ['clr-mnist'] 15 | seeds = [3, 4, 5] 16 | clr_ratios = [(0.5, 0.5)] 17 | sensitiveattrs = ['bck', 'color_gy'] 18 | pos_ratios = [(0.5, 0.5), (0.75, 0.25), (0.9, 0.1)] 19 | 20 | archs = ['laftr-dp', 'laftr-eqopp0', 'laftr-eqopp1', 'laftr-eqodd'] 21 | 22 | fair_coeffs = [0.1, 0.5, 1.0, 2.0, 4.0] 23 | aud_steps = [2] 24 | adv_coeffs = [0.1] 25 | gammas = [10] 26 | alphas = [10] 27 | 28 | widths = [32] 29 | edepths = [2] 30 | ewidths = [32] 31 | adepths = [2] 32 | awidths = [32] 33 | cdepths = [2] 34 | cwidths = [32] 35 | zdims = [16] 36 | 37 | data_dir = ['./data/'] 38 | out_dir = ['/scratch/charanr/fairness-project/output/'] 39 | replicate = [1] 40 | num_epochs = [150] 41 | patiences = [5] 42 | 43 | all_experiments = list(itertools.product(data_dir, out_dir, 44 | dataset, 45 | clr_ratios, pos_ratios, 46 | sensitiveattrs, 47 | edepths, ewidths, adepths, 48 | awidths, cdepths, cwidths, 49 | zdims, 50 | seeds, archs, 51 | fair_coeffs, aud_steps, 52 | adv_coeffs, gammas, alphas, 53 | replicate, num_epochs, patiences)) 54 | 55 | 56 | print(f"Total number of jobs = {len(all_experiments)}") 57 | for idx, exp in enumerate(all_experiments): 58 | 59 | MAIN_CMD = f"./scripts/slurm_launcher.sh" \ 60 | f" --ddir {exp[0]}"\ 61 | f" --odir {exp[1]}"\ 62 | f" --dsetname {exp[2]}"\ 63 | f" --beta_1 {exp[3][0]}"\ 64 | f" --beta_2 {exp[3][1]}"\ 65 | f" --egr {exp[4][0]}"\ 66 | f" --ogr {exp[4][1]}"\ 67 | f" --sattr {exp[5]}"\ 68 | f" --edpt {exp[6]}"\ 69 | f" --ewd {exp[7]}"\ 70 | f" --adpt {exp[8]}"\ 71 | f" --awd {exp[9]}"\ 72 | f" --cdpt {exp[10]}"\ 73 | f" --cwd {exp[11]}"\ 74 | f" --zdim {exp[12]}"\ 75 | f" --seed {exp[13]}"\ 76 | f" --arch {exp[14]}"\ 77 | f" --fair_coeff {exp[15]}"\ 78 | f" --aud_steps {exp[16]}"\ 79 | f" --adv_coeff {exp[17]}"\ 80 | f" --gamma {exp[18]}"\ 81 | f" --alpha {exp[19]}"\ 82 | f" --replicate {exp[20]}"\ 83 | f" --num_epochs {exp[21]}"\ 84 | f" --ptnc {exp[22]}"\ 85 | 86 | hashed_id = int(hashlib.sha1(MAIN_CMD.encode( 87 | 'utf-8')).hexdigest(), 16) % (10 ** 8) 88 | MAIN_CMD = f"{MAIN_CMD} --wdbn deepfairness-{hashed_id}" 89 | 90 | print("###################################") 91 | print(f"EXPERIMENT HASHED ID = {hashed_id}") 92 | print(f"COMMAND RUNNING = \n {MAIN_CMD} \n") 93 | print("###################################") 94 | CMD = MAIN_CMD.split(" ") 95 | 96 | process = subprocess.Popen(CMD, stdout=subprocess.PIPE) 97 | 98 | out, err = process.communicate() 99 | 100 | print(out) 101 | -------------------------------------------------------------------------------- /scripts/CIMNIST/settings34/mlp_conv_multiple_launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | For reproducing Setting 3, 4 experiments with Cnn, Mlp models on CI-MNIST dataset 3 | """ 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import itertools 8 | import subprocess 9 | import re 10 | import git 11 | import os 12 | import hashlib 13 | 14 | dataset = ['clr-mnist'] 15 | seeds = [3, 4, 5] 16 | clr_ratios = [(0.5, 0.5)] 17 | sensitiveattrs = ['bck', 'color_gy'] 18 | pos_ratios = [(0.5, 0.5), (0.75, 0.25), (0.9, 0.1)] 19 | 20 | archs = ['mlp', 'conv'] 21 | 22 | fair_coeffs = [0.1] 23 | aud_steps = [2] 24 | adv_coeffs = [0.1] 25 | gammas = [10] 26 | alphas = [10] 27 | 28 | widths = [32] 29 | edepths = [2] 30 | ewidths = [32] 31 | adepths = [2] 32 | awidths = [32] 33 | cdepths = [2] 34 | cwidths = [32] 35 | zdims = [16] 36 | 37 | data_dir = ['./data/'] 38 | out_dir = ['/scratch/charanr/fairness-project/output/'] 39 | replicate = [1] 40 | num_epochs = [150] 41 | patiences = [5] 42 | 43 | all_experiments = list(itertools.product(data_dir, out_dir, 44 | dataset, 45 | clr_ratios, pos_ratios, 46 | sensitiveattrs, 47 | edepths, ewidths, adepths, 48 | awidths, cdepths, cwidths, 49 | zdims, 50 | seeds, archs, 51 | fair_coeffs, aud_steps, 52 | adv_coeffs, gammas, alphas, 53 | replicate, num_epochs, patiences)) 54 | 55 | 56 | print(f"Total number of jobs = {len(all_experiments)}") 57 | for idx, exp in enumerate(all_experiments): 58 | 59 | MAIN_CMD = f"./scripts/slurm_launcher.sh" \ 60 | f" --ddir {exp[0]}"\ 61 | f" --odir {exp[1]}"\ 62 | f" --dsetname {exp[2]}"\ 63 | f" --beta_1 {exp[3][0]}"\ 64 | f" --beta_2 {exp[3][1]}"\ 65 | f" --egr {exp[4][0]}"\ 66 | f" --ogr {exp[4][1]}"\ 67 | f" --sattr {exp[5]}"\ 68 | f" --edpt {exp[6]}"\ 69 | f" --ewd {exp[7]}"\ 70 | f" --adpt {exp[8]}"\ 71 | f" --awd {exp[9]}"\ 72 | f" --cdpt {exp[10]}"\ 73 | f" --cwd {exp[11]}"\ 74 | f" --zdim {exp[12]}"\ 75 | f" --seed {exp[13]}"\ 76 | f" --arch {exp[14]}"\ 77 | f" --fair_coeff {exp[15]}"\ 78 | f" --aud_steps {exp[16]}"\ 79 | f" --adv_coeff {exp[17]}"\ 80 | f" --gamma {exp[18]}"\ 81 | f" --alpha {exp[19]}"\ 82 | f" --replicate {exp[20]}"\ 83 | f" --num_epochs {exp[21]}"\ 84 | f" --ptnc {exp[22]}"\ 85 | 86 | hashed_id = int(hashlib.sha1(MAIN_CMD.encode( 87 | 'utf-8')).hexdigest(), 16) % (10 ** 8) 88 | MAIN_CMD = f"{MAIN_CMD} --wdbn deepfairness-{hashed_id}" 89 | 90 | print("###################################") 91 | print(f"EXPERIMENT HASHED ID = {hashed_id}") 92 | print(f"COMMAND RUNNING = \n {MAIN_CMD} \n") 93 | print("###################################") 94 | CMD = MAIN_CMD.split(" ") 95 | 96 | process = subprocess.Popen(CMD, stdout=subprocess.PIPE) 97 | 98 | out, err = process.communicate() 99 | 100 | print(out) 101 | -------------------------------------------------------------------------------- /scripts/ablations/dataset_replication/conv_multiple_launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | For reproducing dataset replication ablation experiments in Tables 54, 55 with Cnn model on CI-MNIST dataset 3 | """ 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import itertools 8 | import subprocess 9 | import re 10 | import git 11 | import os 12 | import hashlib 13 | 14 | dataset = ['clr-mnist'] 15 | seeds = [3, 4, 5] 16 | clr_ratios = [(0.001, 0.001)] 17 | sensitiveattrs = ['bck'] 18 | pos_ratios = [(0.5, 0.5)] 19 | replicate = [1, 10, 100, 1000] 20 | 21 | archs = ['conv'] 22 | 23 | fair_coeffs = [0.1] 24 | aud_steps = [2] 25 | adv_coeffs = [1.0] 26 | gammas = [10] 27 | alphas = [10] 28 | 29 | widths = [32] 30 | edepths = [2] 31 | ewidths = [32] 32 | adepths = [2] 33 | awidths = [32] 34 | cdepths = [2] 35 | cwidths = [32] 36 | zdims = [16] 37 | 38 | data_dir = ['./data/'] 39 | out_dir = ['/scratch/charanr/fairness-project/output/'] 40 | num_epochs = [150] 41 | patiences = [5] 42 | 43 | all_experiments = list(itertools.product(data_dir, out_dir, 44 | dataset, 45 | clr_ratios, pos_ratios, 46 | sensitiveattrs, 47 | edepths, ewidths, adepths, 48 | awidths, cdepths, cwidths, 49 | zdims, 50 | seeds, archs, 51 | fair_coeffs, aud_steps, 52 | adv_coeffs, gammas, alphas, 53 | replicate, num_epochs, patiences)) 54 | 55 | 56 | print(f"Total number of jobs = {len(all_experiments)}") 57 | for idx, exp in enumerate(all_experiments): 58 | 59 | MAIN_CMD = f"./scripts/slurm_launcher.sh" \ 60 | f" --ddir {exp[0]}"\ 61 | f" --odir {exp[1]}"\ 62 | f" --dsetname {exp[2]}"\ 63 | f" --beta_1 {exp[3][0]}"\ 64 | f" --beta_2 {exp[3][1]}"\ 65 | f" --egr {exp[4][0]}"\ 66 | f" --ogr {exp[4][1]}"\ 67 | f" --sattr {exp[5]}"\ 68 | f" --edpt {exp[6]}"\ 69 | f" --ewd {exp[7]}"\ 70 | f" --adpt {exp[8]}"\ 71 | f" --awd {exp[9]}"\ 72 | f" --cdpt {exp[10]}"\ 73 | f" --cwd {exp[11]}"\ 74 | f" --zdim {exp[12]}"\ 75 | f" --seed {exp[13]}"\ 76 | f" --arch {exp[14]}"\ 77 | f" --fair_coeff {exp[15]}"\ 78 | f" --aud_steps {exp[16]}"\ 79 | f" --adv_coeff {exp[17]}"\ 80 | f" --gamma {exp[18]}"\ 81 | f" --alpha {exp[19]}"\ 82 | f" --replicate {exp[20]}"\ 83 | f" --num_epochs {exp[21]}"\ 84 | f" --ptnc {exp[22]}"\ 85 | 86 | hashed_id = int(hashlib.sha1(MAIN_CMD.encode( 87 | 'utf-8')).hexdigest(), 16) % (10 ** 8) 88 | MAIN_CMD = f"{MAIN_CMD} --wdbn deepfairness-{hashed_id}" 89 | 90 | print("###################################") 91 | print(f"EXPERIMENT HASHED ID = {hashed_id}") 92 | print(f"COMMAND RUNNING = \n {MAIN_CMD} \n") 93 | print("###################################") 94 | CMD = MAIN_CMD.split(" ") 95 | 96 | process = subprocess.Popen(CMD, stdout=subprocess.PIPE) 97 | 98 | out, err = process.communicate() 99 | 100 | print(out) 101 | -------------------------------------------------------------------------------- /scripts/ablations/dataset_replication/laftr_multiple_launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | For reproducing dataset replication ablation experiments in Tables 54, 55 with Laftr-EqOpp0 model on CI-MNIST dataset 3 | """ 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import itertools 8 | import subprocess 9 | import re 10 | import git 11 | import os 12 | import hashlib 13 | 14 | dataset = ['clr-mnist'] 15 | seeds = [3, 4, 5] 16 | clr_ratios = [(0.001, 0.001)] 17 | sensitiveattrs = ['bck'] 18 | pos_ratios = [(0.5, 0.5)] 19 | replicate = [1, 10, 100, 1000] 20 | 21 | archs = ['laftr-eqopp0'] 22 | 23 | fair_coeffs = [0.1, 0.5, 1.0, 2.0, 4.0] 24 | aud_steps = [2] 25 | adv_coeffs = [1.0] 26 | gammas = [10] 27 | alphas = [10] 28 | 29 | widths = [32] 30 | edepths = [2] 31 | ewidths = [32] 32 | adepths = [2] 33 | awidths = [32] 34 | cdepths = [2] 35 | cwidths = [32] 36 | zdims = [16] 37 | 38 | data_dir = ['./data/'] 39 | out_dir = ['/scratch/charanr/fairness-project/output/'] 40 | num_epochs = [150] 41 | patiences = [5] 42 | 43 | all_experiments = list(itertools.product(data_dir, out_dir, 44 | dataset, 45 | clr_ratios, pos_ratios, 46 | sensitiveattrs, 47 | edepths, ewidths, adepths, 48 | awidths, cdepths, cwidths, 49 | zdims, 50 | seeds, archs, 51 | fair_coeffs, aud_steps, 52 | adv_coeffs, gammas, alphas, 53 | replicate, num_epochs, patiences)) 54 | 55 | 56 | print(f"Total number of jobs = {len(all_experiments)}") 57 | for idx, exp in enumerate(all_experiments): 58 | 59 | MAIN_CMD = f"./scripts/slurm_launcher.sh" \ 60 | f" --ddir {exp[0]}"\ 61 | f" --odir {exp[1]}"\ 62 | f" --dsetname {exp[2]}"\ 63 | f" --beta_1 {exp[3][0]}"\ 64 | f" --beta_2 {exp[3][1]}"\ 65 | f" --egr {exp[4][0]}"\ 66 | f" --ogr {exp[4][1]}"\ 67 | f" --sattr {exp[5]}"\ 68 | f" --edpt {exp[6]}"\ 69 | f" --ewd {exp[7]}"\ 70 | f" --adpt {exp[8]}"\ 71 | f" --awd {exp[9]}"\ 72 | f" --cdpt {exp[10]}"\ 73 | f" --cwd {exp[11]}"\ 74 | f" --zdim {exp[12]}"\ 75 | f" --seed {exp[13]}"\ 76 | f" --arch {exp[14]}"\ 77 | f" --fair_coeff {exp[15]}"\ 78 | f" --aud_steps {exp[16]}"\ 79 | f" --adv_coeff {exp[17]}"\ 80 | f" --gamma {exp[18]}"\ 81 | f" --alpha {exp[19]}"\ 82 | f" --replicate {exp[20]}"\ 83 | f" --num_epochs {exp[21]}"\ 84 | f" --ptnc {exp[22]}"\ 85 | 86 | hashed_id = int(hashlib.sha1(MAIN_CMD.encode( 87 | 'utf-8')).hexdigest(), 16) % (10 ** 8) 88 | MAIN_CMD = f"{MAIN_CMD} --wdbn deepfairness-{hashed_id}" 89 | 90 | print("###################################") 91 | print(f"EXPERIMENT HASHED ID = {hashed_id}") 92 | print(f"COMMAND RUNNING = \n {MAIN_CMD} \n") 93 | print("###################################") 94 | CMD = MAIN_CMD.split(" ") 95 | 96 | process = subprocess.Popen(CMD, stdout=subprocess.PIPE) 97 | 98 | out, err = process.communicate() 99 | 100 | print(out) 101 | -------------------------------------------------------------------------------- /scripts/ablations/ffvae_ablations/ffvae_cfair_multiple_launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | For reproducing ablation experiments in Tables 56, 57 with Ffvae model on Adult dataset 3 | """ 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import itertools 8 | import subprocess 9 | import re 10 | import git 11 | import os 12 | import hashlib 13 | 14 | dataset = ['adult'] 15 | seeds = [3, 4, 5] 16 | clr_ratios = [(0.5, 0.5), (0.01, 0.01), (0.66, 0.33), (0.1, 0.1), (0.06, 0.36)] 17 | sensitiveattrs = ['age'] 18 | pos_ratios = [(0.5, 0.5)] 19 | 20 | archs = ['ffvae_cfair'] 21 | 22 | fair_coeffs = [0.1] 23 | aud_steps = [2] 24 | adv_coeffs = [0.1, 1.0, 10.0, 100.0, 1000.0] 25 | gammas = [10, 50, 100] 26 | alphas = [10, 100, 1000] 27 | 28 | widths = [32] 29 | edepths = [2] 30 | ewidths = [32] 31 | adepths = [2] 32 | awidths = [32] 33 | cdepths = [2] 34 | cwidths = [32] 35 | zdims = [16] 36 | 37 | data_dir = ['./data/'] 38 | out_dir = ['/scratch/charanr/fairness-project/output/'] 39 | replicate = [1] 40 | num_epochs = [150] 41 | patiences = [5] 42 | 43 | all_experiments = list(itertools.product(data_dir, out_dir, 44 | dataset, 45 | clr_ratios, pos_ratios, 46 | sensitiveattrs, 47 | edepths, ewidths, adepths, 48 | awidths, cdepths, cwidths, 49 | zdims, 50 | seeds, archs, 51 | fair_coeffs, aud_steps, 52 | adv_coeffs, gammas, alphas, 53 | replicate, num_epochs, patiences)) 54 | 55 | 56 | print(f"Total number of jobs = {len(all_experiments)}") 57 | for idx, exp in enumerate(all_experiments): 58 | 59 | MAIN_CMD = f"./scripts/slurm_launcher.sh" \ 60 | f" --ddir {exp[0]}"\ 61 | f" --odir {exp[1]}"\ 62 | f" --dsetname {exp[2]}"\ 63 | f" --beta_1 {exp[3][0]}"\ 64 | f" --beta_2 {exp[3][1]}"\ 65 | f" --egr {exp[4][0]}"\ 66 | f" --ogr {exp[4][1]}"\ 67 | f" --sattr {exp[5]}"\ 68 | f" --edpt {exp[6]}"\ 69 | f" --ewd {exp[7]}"\ 70 | f" --adpt {exp[8]}"\ 71 | f" --awd {exp[9]}"\ 72 | f" --cdpt {exp[10]}"\ 73 | f" --cwd {exp[11]}"\ 74 | f" --zdim {exp[12]}"\ 75 | f" --seed {exp[13]}"\ 76 | f" --arch {exp[14]}"\ 77 | f" --fair_coeff {exp[15]}"\ 78 | f" --aud_steps {exp[16]}"\ 79 | f" --adv_coeff {exp[17]}"\ 80 | f" --gamma {exp[18]}"\ 81 | f" --alpha {exp[19]}"\ 82 | f" --replicate {exp[20]}"\ 83 | f" --num_epochs {exp[21]}"\ 84 | f" --ptnc {exp[22]}"\ 85 | 86 | hashed_id = int(hashlib.sha1(MAIN_CMD.encode( 87 | 'utf-8')).hexdigest(), 16) % (10 ** 8) 88 | MAIN_CMD = f"{MAIN_CMD} --wdbn deepfairness-{hashed_id}" 89 | 90 | print("###################################") 91 | print(f"EXPERIMENT HASHED ID = {hashed_id}") 92 | print(f"COMMAND RUNNING = \n {MAIN_CMD} \n") 93 | print("###################################") 94 | CMD = MAIN_CMD.split(" ") 95 | 96 | process = subprocess.Popen(CMD, stdout=subprocess.PIPE) 97 | 98 | out, err = process.communicate() 99 | 100 | print(out) 101 | -------------------------------------------------------------------------------- /scripts/ablations/ffvae_ablations/ffvae_laftr_multiple_launcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | For reproducing ablation experiments in Tables 58, 59 with Ffvae model on Adult dataset 3 | """ 4 | 5 | import pandas as pd 6 | import numpy as np 7 | import itertools 8 | import subprocess 9 | import re 10 | import git 11 | import os 12 | import hashlib 13 | 14 | dataset = ['adult'] 15 | seeds = [3, 4, 5] 16 | clr_ratios = [(0.5, 0.5), (0.01, 0.01), (0.66, 0.33), (0.1, 0.1), (0.06, 0.36)] 17 | sensitiveattrs = ['age'] 18 | pos_ratios = [(0.5, 0.5)] 19 | 20 | archs = ['ffvae_laftr'] 21 | 22 | fair_coeffs = [0.1, 0.5, 1.0, 2.0, 4.0] 23 | aud_steps = [2] 24 | adv_coeffs = [0.1] 25 | gammas = [10, 50, 100] 26 | alphas = [10, 100, 1000] 27 | 28 | widths = [32] 29 | edepths = [2] 30 | ewidths = [32] 31 | adepths = [2] 32 | awidths = [32] 33 | cdepths = [2] 34 | cwidths = [32] 35 | zdims = [16] 36 | 37 | data_dir = ['./data/'] 38 | out_dir = ['/scratch/charanr/fairness-project/output/'] 39 | replicate = [1] 40 | num_epochs = [150] 41 | patiences = [5] 42 | 43 | all_experiments = list(itertools.product(data_dir, out_dir, 44 | dataset, 45 | clr_ratios, pos_ratios, 46 | sensitiveattrs, 47 | edepths, ewidths, adepths, 48 | awidths, cdepths, cwidths, 49 | zdims, 50 | seeds, archs, 51 | fair_coeffs, aud_steps, 52 | adv_coeffs, gammas, alphas, 53 | replicate, num_epochs, patiences)) 54 | 55 | 56 | print(f"Total number of jobs = {len(all_experiments)}") 57 | for idx, exp in enumerate(all_experiments): 58 | 59 | MAIN_CMD = f"./scripts/slurm_launcher.sh" \ 60 | f" --ddir {exp[0]}"\ 61 | f" --odir {exp[1]}"\ 62 | f" --dsetname {exp[2]}"\ 63 | f" --beta_1 {exp[3][0]}"\ 64 | f" --beta_2 {exp[3][1]}"\ 65 | f" --egr {exp[4][0]}"\ 66 | f" --ogr {exp[4][1]}"\ 67 | f" --sattr {exp[5]}"\ 68 | f" --edpt {exp[6]}"\ 69 | f" --ewd {exp[7]}"\ 70 | f" --adpt {exp[8]}"\ 71 | f" --awd {exp[9]}"\ 72 | f" --cdpt {exp[10]}"\ 73 | f" --cwd {exp[11]}"\ 74 | f" --zdim {exp[12]}"\ 75 | f" --seed {exp[13]}"\ 76 | f" --arch {exp[14]}"\ 77 | f" --fair_coeff {exp[15]}"\ 78 | f" --aud_steps {exp[16]}"\ 79 | f" --adv_coeff {exp[17]}"\ 80 | f" --gamma {exp[18]}"\ 81 | f" --alpha {exp[19]}"\ 82 | f" --replicate {exp[20]}"\ 83 | f" --num_epochs {exp[21]}"\ 84 | f" --ptnc {exp[22]}"\ 85 | 86 | hashed_id = int(hashlib.sha1(MAIN_CMD.encode( 87 | 'utf-8')).hexdigest(), 16) % (10 ** 8) 88 | MAIN_CMD = f"{MAIN_CMD} --wdbn deepfairness-{hashed_id}" 89 | 90 | print("###################################") 91 | print(f"EXPERIMENT HASHED ID = {hashed_id}") 92 | print(f"COMMAND RUNNING = \n {MAIN_CMD} \n") 93 | print("###################################") 94 | CMD = MAIN_CMD.split(" ") 95 | 96 | process = subprocess.Popen(CMD, stdout=subprocess.PIPE) 97 | 98 | out, err = process.communicate() 99 | 100 | print(out) 101 | -------------------------------------------------------------------------------- /scripts/plots_tables/csvs/parse_csv.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import os 3 | 4 | if not os.path.exists("./scripts/plots_tables/csvs/processed_csvs/"): 5 | os.makedirs("./scripts/plots_tables/csvs/processed_csvs/") 6 | 7 | csv_folder = './scripts/plots_tables/csvs/' 8 | out_csv_folder = './scripts/plots_tables/csvs/processed_csvs/' 9 | 10 | def filter_table(data_table, metric_cols, all_grouping_cols): 11 | for k, v in metric_cols.items(): 12 | data_table[v] = data_table[k] 13 | data_table[v] = 1 - (data_table[v] - 1).abs() 14 | 15 | return data_table[list(metric_cols.values()) + all_grouping_cols] 16 | 17 | def group_and_agg(tbl, mean_grouping_cols, metric_cols): 18 | grouped_tbl = tbl.groupby(mean_grouping_cols) 19 | 20 | avg_tbl = grouped_tbl[list(metric_cols.values())].agg(['mean']) 21 | avg_tbl.columns = ["_".join(x) for x in avg_tbl.columns.tolist()] 22 | avg_tbl.dropna() 23 | avg_tbl = avg_tbl.reset_index() 24 | 25 | std_tbl = grouped_tbl[list(metric_cols.values())].agg(['std']) 26 | std_tbl.columns = ["_".join(x) for x in std_tbl.columns.tolist()] 27 | std_tbl.dropna() 28 | std_tbl = std_tbl.reset_index() 29 | return grouped_tbl, avg_tbl, std_tbl 30 | 31 | 32 | def group_and_max(tbl, max_grouping_cols, metric_cols): 33 | grouped_tbl = tbl.groupby(max_grouping_cols) 34 | col_list = [s + "_mean" for s in list(metric_cols.values())] 35 | avg_tbl = grouped_tbl[col_list].agg(['max']) 36 | avg_tbl.columns = ["_".join(x) for x in avg_tbl.columns.tolist()] 37 | avg_tbl.dropna() 38 | avg_tbl = avg_tbl.reset_index() 39 | return grouped_tbl, avg_tbl 40 | 41 | 42 | for dataset in ['adult', 'mnist']: 43 | if dataset == 'adult': 44 | archs = ['mlp', 'laftr', 'cfair', 'ffvae'] 45 | else: 46 | archs = ['mlp', 'conv', 'laftr', 'cfair', 'ffvae'] 47 | 48 | for arch in archs: 49 | 50 | df_filename = os.path.join(csv_folder, dataset + '.csv') 51 | df = pd.read_csv(df_filename) 52 | 53 | if arch == "laftr": 54 | df = df[(df['arch'].isin( 55 | ["laftr-eqopp0", "laftr-eqopp1", "laftr-eqodd", "laftr-dp"]))] 56 | elif arch == "cfair": 57 | df = df[(df['arch'].isin(["cfair", "cfair-eo"]))] 58 | else: 59 | df = df[(df['arch'] == arch)] 60 | 61 | mean_filename = os.path.join(out_csv_folder, dataset + '-' + arch + '-mean_over_seed.csv') 62 | mean_max_filename = os.path.join(out_csv_folder, dataset + '-' + arch + '-mean_over_seed_max_over_hp.csv') 63 | table_num = 4 64 | 65 | if arch == "cfair": 66 | hyps = ['adv_coeff'] 67 | elif arch == "laftr": 68 | hyps = ['fair_coeff'] 69 | elif arch == "ffvae": 70 | hyps = ['gamma', 'alpha'] 71 | else: 72 | hyps = [] 73 | 74 | all_grouping_cols = ['clr_ratio', 75 | 'sensattr', 'arch', 'seed'] + hyps 76 | mean_grouping_cols = ['clr_ratio', 'sensattr', 'arch'] + hyps 77 | max_grouping_cols = ['clr_ratio', 'sensattr', 'arch'] 78 | 79 | metric_cols = {'TEST/DP': 'DP', 'TEST/0-accuracy': '0-acc', 80 | 'TEST/1-accuracy': '1-acc', 'TEST/EqOp(y=0)': 'EqOp0', 81 | 'TEST/EqOp(y=1)': 'EqOp1'} 82 | 83 | if dataset == "mnist": 84 | all_grouping_cols = all_grouping_cols + ['g_y_ratio'] 85 | mean_grouping_cols = mean_grouping_cols + ['g_y_ratio'] 86 | max_grouping_cols = max_grouping_cols + ['g_y_ratio'] 87 | 88 | metric_cols = {'es/DP/test': 'DP', 'es/0-accuracy/test': '0-acc', 89 | 'es/1-accuracy/test': '1-acc', 'es/EqOp(y=0)/test': 'EqOp0', 90 | 'es/EqOp(y=1)/test': 'EqOp1'} 91 | 92 | print(df.shape) 93 | table_df = filter_table(df, metric_cols, all_grouping_cols) 94 | print(table_df.shape) 95 | 96 | (grouped_tbl_mean, avg_tbl, std_tbl) = group_and_agg(table_df, mean_grouping_cols, metric_cols) 97 | print(avg_tbl.shape) 98 | avg_tbl.to_csv(mean_filename, index=False) 99 | 100 | (grouped_tbl_max, max_tbl) = group_and_max(avg_tbl, max_grouping_cols, metric_cols) 101 | print(max_tbl.shape) 102 | 103 | with pd.option_context('display.max_rows', None, 'display.max_columns', None): 104 | for k, v in metric_cols.items(): 105 | pp = std_tbl.loc[grouped_tbl_max[v+"_mean"].idxmax()] 106 | max_tbl.index = pp.index 107 | new_v = v + '_std' + '_corresp' 108 | max_tbl[new_v] = pp[v + '_std'] 109 | 110 | max_tbl['table_type'] = table_num 111 | max_tbl = max_tbl[['table_type'] + max_tbl.columns.tolist()[:-1]] 112 | print(max_tbl.shape) 113 | 114 | max_tbl.to_csv(mean_max_filename, index=False) 115 | print(max_tbl.columns) 116 | -------------------------------------------------------------------------------- /scripts/plots_tables/plots/barplots_adult.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | import pandas as pd 4 | import os 5 | 6 | adult_csv_folder = './scripts/plots_tables/csvs/processed_csvs/' 7 | out_plots_folder = './scripts/plots_tables/plots/adult_barplots/' 8 | if not os.path.exists(out_plots_folder): 9 | os.makedirs(out_plots_folder) 10 | 11 | 12 | # get specific rows from the wandb csv file 13 | def parse_csv(tab, table_type, clr_ratio, sensattr, metric_names, l): 14 | return tab[(tab['table_type'] == table_type) & 15 | (tab['clr_ratio'] == clr_ratio) & 16 | (tab['sensattr'] == sensattr)][metric_names[l]].values 17 | 18 | # load processed files (mean taken over seeds and maximum metrics over hyperparameters) 19 | mlp_csv = os.path.join(adult_csv_folder, 'adult-mlp-mean_over_seed_max_over_hp.csv') 20 | laftar_csv = pd.read_csv(os.path.join(adult_csv_folder, 'adult-laftr-mean_over_seed_max_over_hp.csv')) 21 | cfair_mod_csv = pd.read_csv(os.path.join(adult_csv_folder, 'adult-cfair-mean_over_seed_max_over_hp.csv')) 22 | ffvae_csv = os.path.join(adult_csv_folder, 'adult-ffvae-mean_over_seed_max_over_hp.csv') 23 | 24 | laftar_dp_csv = laftar_csv[laftar_csv["arch"] == "laftr-dp"] 25 | laftar_eqodd_csv = laftar_csv[laftar_csv["arch"] == "laftr-eqodd"] 26 | laftar_eqopp0_csv = laftar_csv[laftar_csv["arch"] == "laftr-eqopp0"] 27 | laftar_eqopp1_csv = laftar_csv[laftar_csv["arch"] == "laftr-eqopp1"] 28 | cfair_csv = cfair_mod_csv[cfair_mod_csv["arch"] == "cfair"] 29 | cfair_eo_csv = cfair_mod_csv[cfair_mod_csv["arch"] == "cfair-eo"] 30 | 31 | # labels, metrics for plotting 32 | labels = ['acc', 'p-acc', 'up-acc', 'DP', 'EqOpp0', 'EqOpp1', 'EqOdd'] 33 | mean_metric_names = ['acc_mean_max', '1-acc_mean_max', '0-acc_mean_max', 'DP_mean_max', 'EqOp0_mean_max', 'EqOp1_mean_max', 'EqOdd_mean_max'] 34 | std_metric_names = ['acc_std_corresp', '1-acc_std_corresp', '0-acc_std_corresp', 'DP_std_corresp', 'EqOp0_std_corresp', 'EqOp1_std_corresp', 'EqOdd_std_corresp'] 35 | clr = ['deepskyblue', 'khaki', 'lightcoral', 'mediumseagreen', 'lightslategrey', 'plum', 'turquoise', 'cornflowerblue'] 36 | clr_ratios = ["(0.5, 0.5)", "(0.1, 0.1)", "(0.01, 0.01)", "(0.66, 0.33)", "(0.33, 0.33)", "(0.06, 0.36)"] 37 | table_type = 4 # fixed value 38 | 39 | # load csvs into an array 40 | csvs = [] 41 | csvs.append(pd.read_csv(mlp_csv)) 42 | csvs.append(laftar_dp_csv) 43 | csvs.append(laftar_eqopp0_csv) 44 | csvs.append(laftar_eqopp1_csv) 45 | csvs.append(laftar_eqodd_csv) 46 | csvs.append(cfair_csv) 47 | csvs.append(cfair_eo_csv) 48 | csvs.append(pd.read_csv(ffvae_csv)) 49 | 50 | # generate plot for each color ratio 51 | for j in range(7): 52 | if j<=5: 53 | clr_ratio = clr_ratios[j] 54 | sensattr = "age" 55 | else: 56 | clr_ratio = "(0.5, 0.5)" 57 | sensattr = "sex" 58 | 59 | # parse for metric values (length of bar plots) 60 | values = [] 61 | val = [] 62 | for k in range(8): 63 | for l in range(7): 64 | tab = csvs[k] 65 | if l == 0: 66 | first_val = parse_csv(tab, table_type, clr_ratio, sensattr, mean_metric_names, 1) 67 | second_val = parse_csv(tab, table_type, clr_ratio, sensattr, mean_metric_names, 2) 68 | avg_val = (first_val + second_val)/2 69 | val.append(avg_val) 70 | elif l == 6: 71 | first_val = parse_csv(tab, table_type, clr_ratio, sensattr, mean_metric_names, 4) 72 | second_val = parse_csv(tab, table_type, clr_ratio, sensattr, mean_metric_names, 5) 73 | avg_val = (first_val + second_val)/2 74 | val.append(avg_val) 75 | else: 76 | val.append(parse_csv(tab, table_type, clr_ratio, sensattr, mean_metric_names, l)) 77 | val = [l.tolist() for l in val] 78 | values.append(val) 79 | val = [] 80 | values = np.asarray(values).squeeze() 81 | 82 | # parse for standard deviation metric values (error bars of bar plots) 83 | std_values = [] 84 | std_val = [] 85 | for k in range(8): 86 | for l in range(7): 87 | tab = csvs[k] 88 | if l == 0: 89 | first_val = parse_csv(tab, table_type, clr_ratio, sensattr, std_metric_names, 1) 90 | second_val = parse_csv(tab, table_type, clr_ratio, sensattr, std_metric_names, 2) 91 | avg_val = (first_val + second_val)/2 92 | std_val.append(avg_val) 93 | elif l == 6: 94 | first_val = parse_csv(tab, table_type, clr_ratio, sensattr, std_metric_names, 4) 95 | second_val = parse_csv(tab, table_type, clr_ratio, sensattr, std_metric_names, 5) 96 | avg_val = (first_val + second_val)/2 97 | std_val.append(avg_val) 98 | else: 99 | std_val.append(parse_csv(tab, table_type, clr_ratio, sensattr, std_metric_names, l)) 100 | std_val = [l.tolist() for l in std_val] 101 | std_values.append(std_val) 102 | std_val = [] 103 | std_values = np.asarray(std_values).squeeze() 104 | 105 | # barplot code 106 | x = np.arange(len(labels)) 107 | width = 0.6 108 | fig, ax = plt.subplots(figsize=(15,10)) 109 | rects1 = ax.bar(x - 3*width/6, values[0], width/6, yerr=std_values[0], ecolor='gray', capsize=3, label='Mlp', color=clr[0]) 110 | rects2 = ax.bar(x - 2*width/6, values[1], width/6, yerr=std_values[1], ecolor='gray', capsize=3, label='Laftr-DP', color=clr[1]) 111 | rects3 = ax.bar(x - 1*width/6, values[2], width/6, yerr=std_values[2], ecolor='gray', capsize=3, label='Laftr-EqOpp0', color=clr[2]) 112 | rects4 = ax.bar(x, values[3], width/6, yerr=std_values[3], ecolor='gray', capsize=3, label='Laftr-EqOpp1', color=clr[3]) 113 | rects5 = ax.bar(x + 1*width/6, values[4], width/6, yerr=std_values[4], ecolor='gray', capsize=3, label='Laftr-EqOdd', color=clr[4]) 114 | rects6 = ax.bar(x + 2*width/6, values[5], width/6, yerr=std_values[5], ecolor='gray', capsize=3, label='Cfair', color=clr[5]) 115 | rects7 = ax.bar(x + 3*width/6, values[6], width/6, yerr=std_values[6], ecolor='gray', capsize=3, label='Cfair-EO', color=clr[6]) 116 | rects8 = ax.bar(x + 4*width/6, values[7], width/6, yerr=std_values[7], ecolor='gray', capsize=3, label='Ffvae', color=clr[7]) 117 | 118 | ax.set_ylabel('metric value', fontsize=25) 119 | ax.set_xticks(x) 120 | ax.set_xticklabels(labels, fontsize=25) 121 | ax.legend(bbox_to_anchor=(0.5, 1.17), loc='upper center', ncol=4, prop={'size': 22}, frameon=False) 122 | plt.margins(0) 123 | plt.gcf().subplots_adjust(bottom=0.2) 124 | ax.yaxis.grid() 125 | ax.set_axisbelow(True) 126 | ax.set_ylim([0.45,1.05]) 127 | 128 | # labelling and saving plots 129 | if j<=5: 130 | label_name = "Age: (u-elg, u-inelg) = " + clr_ratios[j] 131 | else: 132 | label_name = "Sex: (u-elg, u-inelg) = " + "(0.15, 0.38)" 133 | 134 | if j <= 12: 135 | plt.savefig(out_plots_folder + str(label_name) + ".png", bbox_inches = 'tight', pad_inches = 0) 136 | -------------------------------------------------------------------------------- /scripts/plots_tables/plots/barplots_mnist.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | import pandas as pd 4 | import os 5 | 6 | mnist_csv_folder = './scripts/plots_tables/csvs/processed_csvs/' 7 | out_plots_folder = './scripts/plots_tables/plots/mnist_barplots/' 8 | if not os.path.exists(out_plots_folder): 9 | os.makedirs(out_plots_folder) 10 | 11 | # get specific rows from the wandb csv file 12 | def parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, metric_names, l): 13 | return tab[(tab['table_type'] == table_type) & 14 | (tab['clr_ratio'] == clr_ratio) & 15 | (tab['g_y_ratio'] == g_y_ratio) & 16 | (tab['sensattr'] == sensattr)][metric_names[l]].values 17 | 18 | # load processed files (mean taken over seeds and maximum metrics over hyperparameters) 19 | mlp_csv = os.path.join(mnist_csv_folder, 'mnist-mlp-mean_over_seed_max_over_hp.csv') 20 | laftar_csv = pd.read_csv(os.path.join(mnist_csv_folder, 'mnist-laftr-mean_over_seed_max_over_hp.csv')) 21 | cfair_mod_csv = pd.read_csv(os.path.join(mnist_csv_folder, 'mnist-cfair-mean_over_seed_max_over_hp.csv')) 22 | ffvae_csv = os.path.join(mnist_csv_folder, 'mnist-ffvae-mean_over_seed_max_over_hp.csv') 23 | conv_csv = os.path.join(mnist_csv_folder, 'mnist-conv-mean_over_seed_max_over_hp.csv') 24 | 25 | laftar_dp_csv = laftar_csv[laftar_csv["arch"] == "laftr-dp"] 26 | laftar_eqodd_csv = laftar_csv[laftar_csv["arch"] == "laftr-eqodd"] 27 | laftar_eqopp0_csv = laftar_csv[laftar_csv["arch"] == "laftr-eqopp0"] 28 | laftar_eqopp1_csv = laftar_csv[laftar_csv["arch"] == "laftr-eqopp1"] 29 | cfair_csv = cfair_mod_csv[cfair_mod_csv["arch"] == "cfair"] 30 | 31 | # labels, metrics for plotting 32 | labels = ['acc', 'p-acc', 'up-acc', 'DP', 'EqOpp0', 'EqOpp1', 'EqOdd'] 33 | mean_metric_names = ['acc_mean_max', '0-acc_mean_max', '1-acc_mean_max', 'DP_mean_max', 'EqOp0_mean_max', 'EqOp1_mean_max', 'EqOdd_mean_max'] 34 | std_metric_names = ['acc_std_corresp', '0-acc_std_corresp', '1-acc_std_corresp', 'DP_std_corresp', 'EqOp0_std_corresp', 'EqOp1_std_corresp', 'EqOdd_std_corresp'] 35 | clr = ['deepskyblue', 'khaki', 'lightcoral', 'mediumseagreen', 'lightslategrey', 'plum', 'turquoise', 'cornflowerblue'] 36 | clr_ratios = ["(0.5, 0.5)", "(0.1, 0.1)", "(0.01, 0.01)", "(0.001, 0.001)", "(0.1, 0.9)", "(0.01, 0.99)"] 37 | g_y_ratios = ["(0.5, 0.5)", "(0.75, 0.25)", "(0.9, 0.1)"] 38 | table_type = 4 39 | sensattr = "bck" 40 | 41 | # load csvs into an array 42 | csvs = [] 43 | csvs.append(pd.read_csv(mlp_csv)) 44 | csvs.append(pd.read_csv(conv_csv)) 45 | csvs.append(laftar_dp_csv) 46 | csvs.append(laftar_eqopp0_csv) 47 | csvs.append(laftar_eqopp1_csv) 48 | csvs.append(laftar_eqodd_csv) 49 | csvs.append(cfair_csv) 50 | csvs.append(pd.read_csv(ffvae_csv)) 51 | 52 | # generate plot for each color ratio, pos_ratio (g_y_ratio) 53 | for j in range(12): 54 | if j<=5: 55 | clr_ratio = clr_ratios[j] 56 | g_y_ratio = "(0.5, 0.5)" 57 | 58 | elif 6 <= j <= 8: 59 | clr_ratio = "(0.5, 0.5)" 60 | g_y_ratio = g_y_ratios[j-6] 61 | sensattr = "bck" 62 | 63 | elif 9 <= j <= 11: 64 | clr_ratio = "(0.5, 0.5)" 65 | g_y_ratio = g_y_ratios[j-9] 66 | sensattr = "color_gy" 67 | 68 | # parse for metric values (length of bar plots) 69 | values = [] 70 | val = [] 71 | for k in range(8): 72 | for l in range(7): 73 | tab = csvs[k] 74 | if l == 0: 75 | first_val = parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, mean_metric_names, 1) 76 | second_val = parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, mean_metric_names, 2) 77 | avg_val = (first_val + second_val)/2 78 | val.append(avg_val) 79 | elif l == 6: 80 | first_val = parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, mean_metric_names, 4) 81 | second_val = parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, mean_metric_names, 5) 82 | avg_val = (first_val + second_val)/2 83 | val.append(avg_val) 84 | else: 85 | val.append(parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, mean_metric_names, l)) 86 | val = [l.tolist() for l in val] 87 | values.append(val) 88 | val = [] 89 | values = np.asarray(values).squeeze() 90 | 91 | # parse for standard deviation metric values (error bars of bar plots) 92 | std_values = [] 93 | std_val = [] 94 | for k in range(8): 95 | for l in range(7): 96 | tab = csvs[k] 97 | if l == 0: 98 | first_val = parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, std_metric_names, 1) 99 | second_val = parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, std_metric_names, 2) 100 | avg_val = (first_val + second_val)/2 101 | std_val.append(avg_val) 102 | elif l == 6: 103 | first_val = parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, std_metric_names, 4) 104 | second_val = parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, std_metric_names, 5) 105 | avg_val = (first_val + second_val)/2 106 | std_val.append(avg_val) 107 | else: 108 | std_val.append(parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, std_metric_names, l)) 109 | std_val = [l.tolist() for l in std_val] 110 | std_values.append(std_val) 111 | std_val = [] 112 | std_values = np.asarray(std_values).squeeze() 113 | 114 | # barplot code 115 | x = np.arange(len(labels)) 116 | width = 0.6 117 | fig, ax = plt.subplots(figsize=(15,10)) 118 | rects1 = ax.bar(x - 3*width/6, values[0], width/6, yerr=std_values[0], ecolor='gray', capsize=3, label='Mlp', color=clr[0]) 119 | rects1 = ax.bar(x - 2*width/6, values[1], width/6, yerr=std_values[1], ecolor='gray', capsize=3, label='Cnn', color=clr[1]) 120 | rects2 = ax.bar(x - 1*width/6, values[2], width/6, yerr=std_values[2], ecolor='gray', capsize=3, label='Laftr-DP', color=clr[2]) 121 | rects3 = ax.bar(x - 0*width/6, values[3], width/6, yerr=std_values[3], ecolor='gray', capsize=3, label='Laftr-EqOpp0', color=clr[3]) 122 | rects4 = ax.bar(x + 1*width/6, values[4], width/6, yerr=std_values[4], ecolor='gray', capsize=3, label='Laftr-EqOpp1', color=clr[4]) 123 | rects5 = ax.bar(x + 2*width/6, values[5], width/6, yerr=std_values[5], ecolor='gray', capsize=3, label='Laftr-EqOdd', color=clr[5]) 124 | rects6 = ax.bar(x + 3*width/6, values[6], width/6, yerr=std_values[6], ecolor='gray', capsize=3, label='Cfair', color=clr[6]) 125 | rects8 = ax.bar(x + 4*width/6, values[7], width/6, yerr=std_values[7], ecolor='gray', capsize=3, label='Ffvae', color=clr[7]) 126 | 127 | ax.set_ylabel('metric value', fontsize=25) 128 | ax.set_xticks(x) 129 | ax.set_xticklabels(labels, fontsize=25) 130 | ax.legend(bbox_to_anchor=(0.5, 1.17), loc='upper center', ncol=4, prop={'size': 22}, frameon=False) 131 | plt.margins(0) 132 | plt.gcf().subplots_adjust(bottom=0.2) 133 | ax.yaxis.grid() 134 | ax.set_axisbelow(True) 135 | ax.set_ylim([0,1.05]) 136 | 137 | # labelling and saving plots 138 | if j<=5: 139 | label_name = "Sens: bck (u-elg, u-inelg) = " + clr_ratios[j] 140 | elif 6 <=j <=8: 141 | label_name = "Sens: bck (left-elg, left-inelg) = " + g_y_ratios[j-6] 142 | elif 9 <=j <=11: 143 | label_name = "Sens: pos (left-elg, left-inelg) = " + g_y_ratios[j-9] 144 | 145 | if j <= 12: 146 | plt.savefig(out_plots_folder + str(label_name) + ".png", bbox_inches = 'tight', pad_inches = 0) 147 | -------------------------------------------------------------------------------- /scripts/plots_tables/plots/heatmap_corr.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import matplotlib.pyplot as plt 4 | import os 5 | 6 | adult_csv = "./scripts/plots_tables/csvs/adult.csv" 7 | mnist_csv = "./scripts/plots_tables/csvs/mnist.csv" 8 | out_plots_folder = './scripts/plots_tables/plots/corrheatmaps/' 9 | if not os.path.exists(out_plots_folder): 10 | os.makedirs(out_plots_folder) 11 | 12 | # compute acc, EqOdd metrics from other metrics 13 | METRICS = { 14 | "acc": lambda x: (x['p-acc'] + x['up-acc']) / 2, 15 | "p-acc": lambda x: x['p-acc'], 16 | "up-acc": lambda x: x['up-acc'], 17 | "DP": lambda x: x["DP"], 18 | "EqOpp0": lambda x: x["EqOpp0"], 19 | "EqOpp1": lambda x: x["EqOpp1"], 20 | "EqOdd": lambda x: (x["EqOpp0"] + x["EqOpp1"]) / 2, 21 | } 22 | 23 | # column names (keys) in the csv generated from wandb and their corresponding short names (values) 24 | MNIST_CSV_COLS = { 25 | 'beta_1': 'be', 26 | 'beta_2': 'bo', 27 | 'egr': 'le', 28 | 'ogr': 'lo', 29 | 'es/0-accuracy/test': 'p-acc', 30 | 'es/1-accuracy/test': 'up-acc', 31 | 'es/DP/test': 'DP', 32 | 'es/EqOp(y=0)/test': 'EqOpp0', 33 | 'es/EqOp(y=1)/test': 'EqOpp1' 34 | } 35 | 36 | ADULT_CSV_COLS = { 37 | 'beta_1': 'be', 38 | 'beta_2': 'bo', 39 | 'egr': 'le', 40 | 'ogr': 'lo', 41 | 'TEST/0-accuracy': 'p-acc', 42 | 'TEST/1-accuracy': 'up-acc', 43 | 'TEST/DP': 'DP', 44 | 'TEST/EqOp(y=0)': 'EqOpp0', 45 | 'TEST/EqOp(y=1)': 'EqOpp1' 46 | } 47 | new_metric_cols = ["acc", "EqOdd"] 48 | metric_cols = list(METRICS.keys()) 49 | data_features = ['be', 'bo', 'le', 'lo'] 50 | 51 | # change column names, calculate all metrics 52 | def preprocess_data(df, csv_cols): 53 | for k, v in csv_cols.items(): 54 | df[v] = df[k] 55 | 56 | # take reference as 0.5 for be, bo, le, lo values 57 | for data_col in data_features: 58 | df[data_col] = abs(0.5 - df[data_col]) 59 | 60 | for k, func in METRICS.items(): 61 | df.loc[:, k] = df.apply(func, axis=1) 62 | 63 | return df[list(csv_cols.values()) + new_metric_cols] 64 | #return df[data_features + metric_cols] 65 | 66 | # plot heatmaps given correlation values 67 | def plot_data(corr_data, plot_filename): 68 | if 'adult' in plot_filename: 69 | label_names = ['Mlp', 'Laftr-DP', 'Laftr-EqOpp0', 70 | 'Laftr-EqOpp1', 'Laftr-EqOdd', 'Cfair', 'Cfair-EO', 'Ffvae'] 71 | else: 72 | label_names = ['Mlp', 'Cnn', 'Laftr-DP', 'Laftr-EqOpp0', 73 | 'Laftr-EqOpp1', 'Laftr-EqOdd', 'Cfair', 'Ffvae'] 74 | 75 | z = abs(corr_data) 76 | 77 | fig, ax = plt.subplots() 78 | im = ax.imshow(z, cmap="OrRd") 79 | im.set_clim(vmin=0.3, vmax=1.3) 80 | plt.colorbar(im) 81 | ax.set_xticks(np.arange(len(metric_cols))) 82 | ax.set_yticks(np.arange(len(label_names))) 83 | ax.set_xticklabels(metric_cols) 84 | ax.set_yticklabels(label_names) 85 | plt.setp(ax.get_xticklabels(), rotation=45, 86 | ha="right", rotation_mode="anchor") 87 | plt.gcf().subplots_adjust(top=0.15) 88 | 89 | for p in range(len(label_names)): 90 | for q in range(len(metric_cols)): 91 | text = ax.text(q, p, ("%.2f" % z[p, q]).lstrip('0'), 92 | ha="center", va="center", color="k") 93 | 94 | fig.tight_layout() 95 | plt.savefig(plot_filename, bbox_inches='tight', pad_inches=0) 96 | 97 | 98 | 99 | adult_data = pd.read_csv(adult_csv, sep=',') 100 | mnist_data = pd.read_csv(mnist_csv, sep=',') 101 | 102 | adult_setting1 = [(0.5, 0.5), (0.1, 0.1), (0.01, 0.01)] 103 | adult_setting2 = [(0.5, 0.5), (0.66, 0.33), (0.06, 0.36)] 104 | mnist_setting1 = [(0.5, 0.5), (0.1, 0.1), (0.01, 0.01), (0.001, 0.001)] 105 | mnist_setting2 = [(0.5, 0.5), (0.1, 0.9), (0.01, 0.99)] 106 | mnist_setting3 = [(0.5, 0.5), (0.75, 0.25), (0.9, 0.1)] 107 | mnist_setting4 = [(0.5, 0.5), (0.75, 0.25), (0.9, 0.1)] 108 | 109 | 110 | data = [] 111 | data.append(adult_data[(adult_data['sensattr'] == "age") & (adult_data['beta_1'].isin([ 112 | x[0] for x in adult_setting1])) & (adult_data['beta_2'].isin([x[1] for x in adult_setting1]))]) 113 | data.append(adult_data[(adult_data['sensattr'] == "age") & (adult_data['beta_1'].isin([ 114 | x[0] for x in adult_setting2])) & (adult_data['beta_2'].isin([x[1] for x in adult_setting2]))]) 115 | data.append(mnist_data[(mnist_data['sensattr'] == "bck") & (mnist_data['beta_1'].isin([ 116 | x[0] for x in mnist_setting1])) & (mnist_data['beta_2'].isin([x[1] for x in mnist_setting1]))]) 117 | data.append(mnist_data[(mnist_data['sensattr'] == "bck") & (mnist_data['beta_1'].isin([ 118 | x[0] for x in mnist_setting2])) & (mnist_data['beta_2'].isin([x[1] for x in mnist_setting2]))]) 119 | data.append(mnist_data[(mnist_data['sensattr'] == "bck") & (mnist_data['egr'].isin([ 120 | x[0] for x in mnist_setting3])) & (mnist_data['ogr'].isin([x[1] for x in mnist_setting3]))]) 121 | data.append(mnist_data[(mnist_data['sensattr'] == "color_gy") & (mnist_data['egr'].isin([ 122 | x[0] for x in mnist_setting4])) & (mnist_data['ogr'].isin([x[1] for x in mnist_setting4]))]) 123 | 124 | settings = ['adult1', 'adult2', 'mnist1', 'mnist2', 'mnist3', 'mnist4'] 125 | for k in range(len(settings)): 126 | full_z = np.zeros((8, 7)) 127 | 128 | if 'adult' in settings[k]: 129 | prjs = ['mlp', 'laftr-dp', 'laftr-eqopp0', 'laftr-eqopp1', 130 | 'laftr-eqodd', 'cfair', 'cfair-eo', 'ffvae'] 131 | csv_cols = ADULT_CSV_COLS 132 | else: 133 | prjs = ['mlp', 'conv', 'laftr-dp', 'laftr-eqopp0', 134 | 'laftr-eqopp1', 'laftr-eqodd', 'cfair', 'ffvae'] 135 | csv_cols = MNIST_CSV_COLS 136 | 137 | for i in range(len(prjs)): 138 | prj = prjs[i] 139 | 140 | # parse csv 141 | df = data[k] 142 | df = df[(df['arch'] == prj)] 143 | data_processed = preprocess_data(df, csv_cols) 144 | plot_dataset = data_processed[data_features + metric_cols] 145 | 146 | # get correlation between dataset and metric columns 147 | corr_dataset = plot_dataset.corr(method='spearman') 148 | corr_out = corr_dataset.loc[data_features, metric_cols] 149 | 150 | # settings 1, 2 in paper 151 | if "1" in settings[k] or "2" in settings[k]: 152 | # use p=0 for be, p=1 for bo 153 | p = 0 154 | else: # for settings 3, 4 in paper 155 | # use p=2 for le, p=3 for lo 156 | p = 2 157 | 158 | # correlation values of all models in each file 159 | full_z[i] = corr_out.values[p, :7] 160 | 161 | # settings 1, 2 in paper 162 | if "1" in settings[k] or "2" in settings[k]: 163 | # use p=0 for be, p=1 for bo 164 | p = 1 165 | else: # for settings 3, 4 in paper 166 | # use p=2 for le, p=3 for lo 167 | p = 3 168 | 169 | # correlation values of all models in each file 170 | full_z[i] += corr_out.values[p, :7] 171 | 172 | plot_filename = out_plots_folder + "corr_" + settings[k] + ".png" 173 | plot_data(full_z/2, plot_filename) 174 | -------------------------------------------------------------------------------- /scripts/plots_tables/plots/heatmap_std_adult.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | import pandas as pd 4 | import os 5 | 6 | adult_csv_folder = './scripts/plots_tables/csvs/processed_csvs/' 7 | out_plots_folder = './scripts/plots_tables/plots/adult_stdheatmaps/' 8 | if not os.path.exists(out_plots_folder): 9 | os.makedirs(out_plots_folder) 10 | 11 | # get specific rows from the wandb csv file 12 | def parse_csv(tab, table_type, clr_ratio, sensattr, metric_names, l): 13 | return tab[(tab['table_type'] == table_type) & 14 | (tab['clr_ratio'] == clr_ratio) & 15 | (tab['sensattr'] == sensattr)][metric_names[l]].values 16 | 17 | 18 | # load processed files (mean taken over seeds and maximum metrics over hyperparameters) 19 | mlp_csv = os.path.join(adult_csv_folder, 'adult-mlp-mean_over_seed_max_over_hp.csv') 20 | laftar_csv = pd.read_csv(os.path.join(adult_csv_folder, 'adult-laftr-mean_over_seed_max_over_hp.csv')) 21 | cfair_mod_csv = pd.read_csv(os.path.join(adult_csv_folder, 'adult-cfair-mean_over_seed_max_over_hp.csv')) 22 | ffvae_csv = os.path.join(adult_csv_folder, 'adult-ffvae-mean_over_seed_max_over_hp.csv') 23 | 24 | laftar_dp_csv = laftar_csv[laftar_csv["arch"] == "laftr-dp"] 25 | laftar_eqodd_csv = laftar_csv[laftar_csv["arch"] == "laftr-eqodd"] 26 | laftar_eqopp0_csv = laftar_csv[laftar_csv["arch"] == "laftr-eqopp0"] 27 | laftar_eqopp1_csv = laftar_csv[laftar_csv["arch"] == "laftr-eqopp1"] 28 | cfair_csv = cfair_mod_csv[cfair_mod_csv["arch"] == "cfair"] 29 | cfair_eo_csv = cfair_mod_csv[cfair_mod_csv["arch"] == "cfair-eo"] 30 | 31 | # labels, metrics for plotting 32 | models = ['Mlp', 'Laftr-DP', 'Laftr-EqOpp0', 'Laftr-EqOpp1', 'Laftr-EqOdd', 'Cfair', 'Cfair-EO', 'Ffvae'] 33 | labels = ['acc', 'p-acc', 'up-acc', 'DP', 'EqOpp0', 'EqOpp1', 'EqOdd'] 34 | mean_metric_names = ['acc_mean_max', '1-acc_mean_max', '0-acc_mean_max', 'DP_mean_max', 'EqOp0_mean_max', 'EqOp1_mean_max', 'EqOdd_mean_max'] 35 | std_metric_names = ['acc_std_corresp', '1-acc_std_corresp', '0-acc_std_corresp', 'DP_std_corresp', 'EqOp0_std_corresp', 'EqOp1_std_corresp', 'EqOdd_std_corresp'] 36 | clr = ['deepskyblue', 'khaki', 'lightcoral', 'mediumseagreen', 'lightslategrey', 'plum', 'turquoise'] 37 | clr_ratios = ["(0.5, 0.5)", "(0.1, 0.1)", "(0.01, 0.01)", "(0.66, 0.33)", "(0.33, 0.33)", "(0.06, 0.36)"] 38 | table_type = 4 39 | sensattr = "age" 40 | 41 | # load csvs into an array 42 | csvs = [] 43 | csvs.append(pd.read_csv(mlp_csv)) 44 | csvs.append(laftar_dp_csv) 45 | csvs.append(laftar_eqopp0_csv) 46 | csvs.append(laftar_eqopp1_csv) 47 | csvs.append(laftar_eqodd_csv) 48 | csvs.append(cfair_csv) 49 | csvs.append(cfair_eo_csv) 50 | csvs.append(pd.read_csv(ffvae_csv)) 51 | 52 | 53 | # generate plot for each color ratio 54 | for j in range(7): 55 | if 0 <= j <= 5: 56 | clr_ratio = clr_ratios[j] 57 | sensattr = "age" 58 | elif j == 6: 59 | clr_ratio = "(0.5, 0.5)" 60 | sensattr = "sex" 61 | 62 | # parse for standard deviation metric values 63 | std_values = [] 64 | std_val = [] 65 | for k in range(8): 66 | for l in range(7): 67 | tab = csvs[k] 68 | if l == 0: 69 | first_val = parse_csv(tab, table_type, clr_ratio, sensattr, std_metric_names, 1) 70 | second_val = parse_csv(tab, table_type, clr_ratio, sensattr, std_metric_names, 2) 71 | avg_val = (first_val + second_val)/2 72 | std_val.append(avg_val) 73 | elif l == 6: 74 | first_val = parse_csv(tab, table_type, clr_ratio, sensattr, std_metric_names, 4) 75 | second_val = parse_csv(tab, table_type, clr_ratio, sensattr, std_metric_names, 5) 76 | avg_val = (first_val + second_val)/2 77 | std_val.append(avg_val) 78 | else: 79 | std_val.append(parse_csv(tab, table_type, clr_ratio, sensattr, std_metric_names, l)) 80 | std_values.append(std_val) 81 | std_val = [] 82 | std_values = np.asarray(std_values).squeeze() 83 | 84 | # heatmap code 85 | x = np.arange(len(labels)) 86 | width = 0.6 87 | fig, ax = plt.subplots() 88 | std_values = np.round(np.nan_to_num(std_values), 2) 89 | im = ax.imshow(std_values) 90 | im.set_clim(vmin=0.0, vmax=0.1) 91 | plt.colorbar(im) 92 | ax.set_xticks(np.arange(len(labels))) 93 | ax.set_yticks(np.arange(len(models))) 94 | ax.set_xticklabels(labels) 95 | ax.set_yticklabels(models) 96 | plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor") 97 | plt.gcf().subplots_adjust(top=0.15) 98 | 99 | for p in range(len(models)): 100 | for q in range(len(labels)): 101 | text = ax.text(q, p, ("%.2f" % std_values[p, q]).lstrip('0'), 102 | ha="center", va="center", color="w") 103 | 104 | fig.tight_layout() 105 | 106 | # labelling and saving plots 107 | if j == 0: 108 | label_name = "age-ratio (0.5, 0.5)" 109 | elif j == 1: 110 | label_name = "age-ratio (0.1, 0.1)" 111 | elif j == 2: 112 | label_name = "age-ratio (0.01, 0.01)" 113 | elif j == 3: 114 | label_name = "age-ratio (0.66, 0.33)" 115 | elif j == 4: 116 | label_name = "age-ratio (0.33, 0.33)" 117 | elif j == 5: 118 | label_name = "age-ratio (0.06, 0.36)" 119 | elif j == 6: 120 | label_name = "sex-ratio (0.15, 0.38)" 121 | 122 | ax.set_title(label_name) 123 | 124 | if j <= 6: 125 | plt.savefig(out_plots_folder + str(label_name) + ".png", bbox_inches = 'tight', pad_inches = 0) -------------------------------------------------------------------------------- /scripts/plots_tables/plots/heatmap_std_mnist.py: -------------------------------------------------------------------------------- 1 | import matplotlib 2 | import matplotlib.pyplot as plt 3 | import numpy as np 4 | import pandas as pd 5 | import os 6 | 7 | mnist_csv_folder = './scripts/plots_tables/csvs/processed_csvs/' 8 | out_plots_folder = './scripts/plots_tables/plots/mnist_stdheatmaps/' 9 | if not os.path.exists(out_plots_folder): 10 | os.makedirs(out_plots_folder) 11 | 12 | # get specific rows from the wandb csv file 13 | def parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, metric_names, l): 14 | return tab[(tab['table_type'] == table_type) & 15 | (tab['clr_ratio'] == clr_ratio) & 16 | (tab['g_y_ratio'] == g_y_ratio) & 17 | (tab['sensattr'] == sensattr)][metric_names[l]].values 18 | 19 | # load processed files (mean taken over seeds and maximum metrics over hyperparameters) 20 | mlp_csv = os.path.join(mnist_csv_folder, 'mnist-mlp-mean_over_seed_max_over_hp.csv') 21 | laftar_csv = pd.read_csv(os.path.join(mnist_csv_folder, 'mnist-laftr-mean_over_seed_max_over_hp.csv')) 22 | cfair_mod_csv = pd.read_csv(os.path.join(mnist_csv_folder, 'mnist-cfair-mean_over_seed_max_over_hp.csv')) 23 | ffvae_csv = os.path.join(mnist_csv_folder, 'mnist-ffvae-mean_over_seed_max_over_hp.csv') 24 | conv_csv = os.path.join(mnist_csv_folder, 'mnist-conv-mean_over_seed_max_over_hp.csv') 25 | 26 | laftar_dp_csv = laftar_csv[laftar_csv["arch"] == "laftr-dp"] 27 | laftar_eqodd_csv = laftar_csv[laftar_csv["arch"] == "laftr-eqodd"] 28 | laftar_eqopp0_csv = laftar_csv[laftar_csv["arch"] == "laftr-eqopp0"] 29 | laftar_eqopp1_csv = laftar_csv[laftar_csv["arch"] == "laftr-eqopp1"] 30 | cfair_csv = cfair_mod_csv[cfair_mod_csv["arch"] == "cfair"] 31 | 32 | # labels, metrics for plotting 33 | models = ['Mlp', 'Cnn', 'Laftr-DP', 'Laftr-EqOpp0', 'Laftr-EqOpp1', 'Laftr-EqOdd', 'Cfair', 'Ffvae'] 34 | labels = ['acc', 'p-acc', 'up-acc', 'DP', 'EqOpp0', 'EqOpp1', 'EqOdd'] 35 | mean_metric_names = ['acc_mean_max', '0-acc_mean_max', '1-acc_mean_max', 'DP_mean_max', 'EqOp0_mean_max', 'EqOp1_mean_max', 'EqOdd_mean_max'] 36 | std_metric_names = ['acc_std_corresp', '0-acc_std_corresp', '1-acc_std_corresp', 'DP_std_corresp', 'EqOp0_std_corresp', 'EqOp1_std_corresp', 'EqOdd_std_corresp'] 37 | clr = ['deepskyblue', 'khaki', 'lightcoral', 'mediumseagreen', 'lightslategrey', 'plum', 'turquoise'] 38 | clr_ratios = ["(0.5, 0.5)", "(0.1, 0.1)", "(0.01, 0.01)", "(0.001, 0.001)", "(0.1, 0.9)", "(0.01, 0.99)"] 39 | g_y_ratios = ["(0.5, 0.5)", "(0.75, 0.25)", "(0.9, 0.1)"] 40 | table_type = 4 41 | sensattr = "bck" 42 | clr_ratio = "(0.5, 0.5)" 43 | g_y_ratio = "(0.5, 0.5)" 44 | 45 | # load csvs into an array 46 | csvs = [] 47 | csvs.append(pd.read_csv(mlp_csv)) 48 | csvs.append(pd.read_csv(conv_csv)) 49 | csvs.append(laftar_dp_csv) 50 | csvs.append(laftar_eqopp0_csv) 51 | csvs.append(laftar_eqopp1_csv) 52 | csvs.append(laftar_eqodd_csv) 53 | csvs.append(cfair_csv) 54 | csvs.append(pd.read_csv(ffvae_csv)) 55 | 56 | # generate plot for each color ratio, pos_ratio (g_y_ratio) 57 | for j in range(12): 58 | if 0 <= j <= 5: 59 | clr_ratio = clr_ratios[j] 60 | g_y_ratio = "(0.5, 0.5)" 61 | sensattr = "bck" 62 | elif 6 <= j <= 8: 63 | clr_ratio = "(0.5, 0.5)" 64 | g_y_ratio = g_y_ratios[j-6] 65 | sensattr = "bck" 66 | elif 9 <= j <= 11: 67 | clr_ratio = "(0.5, 0.5)" 68 | g_y_ratio = g_y_ratios[j-9] 69 | sensattr = "color_gy" 70 | 71 | # parse for standard deviation metric values 72 | std_values = [] 73 | std_val = [] 74 | for k in range(8): 75 | for l in range(7): 76 | tab = csvs[k] 77 | 78 | if l == 0: 79 | first_val = parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, std_metric_names, 1) 80 | second_val = parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, std_metric_names, 2) 81 | avg_val = (first_val + second_val)/2 82 | std_val.append(avg_val) 83 | elif l == 6: 84 | first_val = parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, std_metric_names, 4) 85 | second_val = parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, std_metric_names, 5) 86 | avg_val = (first_val + second_val)/2 87 | std_val.append(avg_val) 88 | else: 89 | std_val.append(parse_csv(tab, table_type, clr_ratio, g_y_ratio, sensattr, std_metric_names, l)) 90 | std_values.append(std_val) 91 | std_val = [] 92 | 93 | # heatmap code 94 | std_values = np.asarray(std_values).squeeze() 95 | x = np.arange(len(labels)) 96 | width = 0.6 97 | fig, ax = plt.subplots() 98 | std_values = np.round(np.nan_to_num(std_values), 2) 99 | im = ax.imshow(std_values) 100 | im.set_clim(vmin=0.0, vmax=0.2) 101 | plt.colorbar(im) 102 | ax.set_xticks(np.arange(len(labels))) 103 | ax.set_yticks(np.arange(len(models))) 104 | ax.set_xticklabels(labels) 105 | ax.set_yticklabels(models) 106 | plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor") 107 | plt.gcf().subplots_adjust(top=0.15) 108 | 109 | for p in range(len(models)): 110 | for q in range(len(labels)): 111 | text = ax.text(q, p, ("%.2f" % std_values[p, q]).lstrip('0'), 112 | ha="center", va="center", color="w") 113 | 114 | fig.tight_layout() 115 | 116 | # labelling and saving plots 117 | if j == 0: 118 | label_name = "clr-ratio (0.5, 0.5)" 119 | elif j == 1: 120 | label_name = "clr-ratio (0.1, 0.1)" 121 | elif j == 2: 122 | label_name = "clr-ratio (0.01, 0.01)" 123 | elif j == 3: 124 | label_name = "clr-ratio (0.001, 0.001)" 125 | elif j == 4: 126 | label_name = "clr-ratio (0.1, 0.9)" 127 | elif j == 5: 128 | label_name = "clr-ratio (0.01, 0.99)" 129 | elif j == 6: 130 | label_name = "pos_ratio (0.5, 0.5), sensattr=bck" 131 | elif j == 7: 132 | label_name = "pos_ratio (0.75, 0.25), sensattr=bck" 133 | elif j == 8: 134 | label_name = "pos_ratio (0.9, 0.1), sensattr=bck" 135 | elif j == 9: 136 | label_name = "pos_ratio (0.5, 0.5), sensattr=pos" 137 | elif j == 10: 138 | label_name = "pos_ratio (0.75, 0.25), sensattr=pos" 139 | elif j == 11: 140 | label_name = "pos_ratio (0.9, 0.1), sensattr=pos" 141 | 142 | ax.set_title(label_name) 143 | 144 | if j <= 11: 145 | plt.savefig(out_plots_folder + str(label_name) + ".png", bbox_inches = 'tight', pad_inches = 0) -------------------------------------------------------------------------------- /scripts/plots_tables/tables/get_tables.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import os 3 | 4 | csv_folder = './scripts/plots_tables/csvs/processed_csvs/' 5 | 6 | def get_mnist_tables(): 7 | 8 | for arch in ['mlp', 'conv', 'laftr-dp', 'laftr-eqopp1', 'laftr-eqopp0', 'laftr-eqodd', 'cfair', 'ffvae']: 9 | 10 | print("Dataset: Mnist, Model: ", arch) 11 | 12 | if 'laftr' in arch: 13 | model = 'laftr' 14 | elif 'cfair' in arch: 15 | model = 'cfair' 16 | else: 17 | model = arch 18 | 19 | filename = os.path.join(csv_folder, 'mnist-' + model + '-mean_over_seed_max_over_hp.csv') 20 | df = pd.read_csv(filename) 21 | df = df[df['arch'] == arch] 22 | 23 | columns = {'DP_mean_max': 'DP_mean', '0-acc_mean_max': '0-acc_mean', 24 | '1-acc_mean_max': '1-acc_mean', 'EqOp0_mean_max': 'EqOp0_mean', 25 | 'EqOp1_mean_max': 'EqOp1_mean'} 26 | 27 | for k,v in columns.items(): 28 | df[v] = df[k].round(decimals=2) 29 | 30 | grouped_df = df.groupby(['table_type', 'clr_ratio', 'g_y_ratio', 'sensattr', 'arch']) 31 | 32 | grouped_df = grouped_df[list(columns.values())].agg(tuple).reset_index() 33 | 34 | grouped_df["OverleafRow"] = grouped_df["0-acc_mean"].astype(str) + " & " + grouped_df["1-acc_mean"].astype(str) + " & " + grouped_df["DP_mean"].astype(str) + " & " + grouped_df["EqOp0_mean"].astype(str) + " & " + grouped_df["EqOp1_mean"].astype(str) 35 | 36 | clr_ratios_b = ["(0.5, 0.5)", "(0.1, 0.1)", "(0.01, 0.01)", "(0.001, 0.001)"] 37 | clr_ratios_ub = ["(0.5, 0.5)", "(0.1, 0.9)", "(0.01, 0.99)"] 38 | clr_ratio = "(0.5, 0.5)" 39 | 40 | g_y_ratios = ["(0.5, 0.5)", "(0.75, 0.25)", "(0.9, 0.1)"] 41 | g_y_ratio = "(0.5, 0.5)" 42 | 43 | print("Setting 1") 44 | for i in clr_ratios_b: 45 | for idx, row in grouped_df[grouped_df["table_type"] == 4].iterrows(): 46 | if row["sensattr"] == "bck" and row["arch"] == arch: 47 | if row["clr_ratio"] == i and row["g_y_ratio"] == g_y_ratio: 48 | acc_avg = round((row["0-acc_mean"][0] + row["1-acc_mean"][0])/2, 2) 49 | eqodd_avg = round((row["EqOp0_mean"][0] + row["EqOp1_mean"][0])/2, 2) 50 | mytable = row["OverleafRow"].maketrans({"(": "", ")": "", ",": ""}) 51 | str_out = str(acc_avg) + " & " + row["OverleafRow"].translate(mytable) + " & " + str(eqodd_avg) 52 | print(row["clr_ratio"] + " & " + str_out + "\cr \hline") 53 | 54 | print("\n\n") 55 | 56 | print("Setting 2") 57 | for i in clr_ratios_ub: 58 | for idx, row in grouped_df[grouped_df["table_type"] == 4].iterrows(): 59 | if row["sensattr"] == "bck" and row["arch"] == arch: 60 | if row["clr_ratio"] == i and row["g_y_ratio"] == g_y_ratio: 61 | acc_avg = round((row["0-acc_mean"][0] + row["1-acc_mean"][0])/2, 2) 62 | eqodd_avg = round((row["EqOp0_mean"][0] + row["EqOp1_mean"][0])/2, 2) 63 | mytable = row["OverleafRow"].maketrans({"(": "", ")": "", ",": ""}) 64 | str_out = str(acc_avg) + " & " + row["OverleafRow"].translate(mytable) + " & " + str(eqodd_avg) 65 | print(row["clr_ratio"] + " & " + str_out + "\cr \hline") 66 | 67 | print("\n\n") 68 | 69 | print("Setting 3") 70 | for i in g_y_ratios: 71 | for idx, row in grouped_df[grouped_df["table_type"] == 4].iterrows(): 72 | if row["sensattr"] == "bck" and row["arch"] == arch: 73 | if row["clr_ratio"] == clr_ratio and row["g_y_ratio"] == i: 74 | acc_avg = round((row["0-acc_mean"][0] + row["1-acc_mean"][0])/2, 2) 75 | eqodd_avg = round((row["EqOp0_mean"][0] + row["EqOp1_mean"][0])/2, 2) 76 | mytable = row["OverleafRow"].maketrans({"(": "", ")": "", ",": ""}) 77 | str_out = str(acc_avg) + " & " + row["OverleafRow"].translate(mytable) + " & " + str(eqodd_avg) 78 | print(row["g_y_ratio"] + " & " + str_out + "\cr \hline") 79 | 80 | print("\n\n") 81 | 82 | print("Setting 4") 83 | for i in g_y_ratios: 84 | for idx, row in grouped_df[grouped_df["table_type"] == 4].iterrows(): 85 | if row["sensattr"] == "color_gy" and row["arch"] == arch: 86 | if row["clr_ratio"] == clr_ratio and row["g_y_ratio"] == i: 87 | acc_avg = round((row["0-acc_mean"][0] + row["1-acc_mean"][0])/2, 2) 88 | eqodd_avg = round((row["EqOp0_mean"][0] + row["EqOp1_mean"][0])/2, 2) 89 | mytable = row["OverleafRow"].maketrans({"(": "", ")": "", ",": ""}) 90 | str_out = str(acc_avg) + " & " + row["OverleafRow"].translate(mytable) + " & " + str(eqodd_avg) 91 | print(row["g_y_ratio"] + " & " + str_out + "\cr \hline") 92 | 93 | print("\n\n") 94 | 95 | def get_adult_tables(): 96 | 97 | for arch in ['mlp', 'laftr-dp', 'laftr-eqopp1', 'laftr-eqopp0', 'laftr-eqodd', 'cfair', 'cfair-eo', 'ffvae']: 98 | 99 | print("Dataset: Adult, Model: ", arch) 100 | 101 | if 'laftr' in arch: 102 | model = 'laftr' 103 | elif 'cfair' in arch: 104 | model = 'cfair' 105 | else: 106 | model = arch 107 | 108 | filename = os.path.join(csv_folder, 'adult-' + model + '-mean_over_seed_max_over_hp.csv') 109 | df = pd.read_csv(filename) 110 | df = df[df['arch'] == arch] 111 | 112 | columns = {'DP_mean_max': 'DP_mean', '0-acc_mean_max': '0-acc_mean', 113 | '1-acc_mean_max': '1-acc_mean', 'EqOp0_mean_max': 'EqOp0_mean', 114 | 'EqOp1_mean_max': 'EqOp1_mean'} 115 | 116 | for k,v in columns.items(): 117 | df[v] = df[k].round(decimals=2) 118 | 119 | grouped_df = df.groupby(['table_type', 'clr_ratio', 'sensattr', 'arch']) 120 | 121 | grouped_df = grouped_df[list(columns.values())].agg(tuple).reset_index() 122 | 123 | grouped_df["OverleafRow"] = grouped_df["1-acc_mean"].astype(str) + " & " + grouped_df["0-acc_mean"].astype(str) + " & " + grouped_df["DP_mean"].astype(str) + " & " + grouped_df["EqOp0_mean"].astype(str) + " & " + grouped_df["EqOp1_mean"].astype(str) 124 | 125 | clr_ratios_b = ["(0.5, 0.5)", "(0.1, 0.1)", "(0.01, 0.01)"] 126 | clr_ratios_ub = ["(0.5, 0.5)", "(0.66, 0.33)", "(0.06, 0.36)"] 127 | 128 | print("Setting 1") 129 | for i in clr_ratios_b: 130 | for idx, row in grouped_df[grouped_df["table_type"] == 4].iterrows(): 131 | if row["sensattr"] == "age" and row["arch"] == arch: 132 | if row["clr_ratio"] == i: 133 | acc_avg = round((row["0-acc_mean"][0] + row["1-acc_mean"][0])/2, 2) 134 | eqodd_avg = round((row["EqOp0_mean"][0] + row["EqOp1_mean"][0])/2, 2) 135 | mytable = row["OverleafRow"].maketrans({"(": "", ")": "", ",": ""}) 136 | str_out = str(acc_avg) + " & " + row["OverleafRow"].translate(mytable) + " & " + str(eqodd_avg) 137 | print(row["clr_ratio"] + " & " + str_out + " \\\ \hline") 138 | 139 | print("\n\n") 140 | 141 | print("Setting 2") 142 | for i in clr_ratios_ub: 143 | for idx, row in grouped_df[grouped_df["table_type"] == 4].iterrows(): 144 | if row["sensattr"] == "age" and row["arch"] == arch: 145 | if row["clr_ratio"] == i: 146 | acc_avg = round((row["0-acc_mean"][0] + row["1-acc_mean"][0])/2, 2) 147 | eqodd_avg = round((row["EqOp0_mean"][0] + row["EqOp1_mean"][0])/2, 2) 148 | mytable = row["OverleafRow"].maketrans({"(": "", ")": "", ",": ""}) 149 | str_out = str(acc_avg) + " & " + row["OverleafRow"].translate(mytable) + " & " + str(eqodd_avg) 150 | print(row["clr_ratio"] + " & " + str_out + " \\\ \hline") 151 | 152 | print("\n\n") 153 | 154 | get_mnist_tables() 155 | get_adult_tables() -------------------------------------------------------------------------------- /scripts/slurm_launcher.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # saner programming env: these switches turn some bugs into errors 4 | set -o errexit -o pipefail -o noclobber -o nounset 5 | 6 | # -allow a command to fail with !’s side effect on errexit 7 | # -use return value from ${PIPESTATUS[0]}, because ! hosed $? 8 | ! getopt --test > /dev/null 9 | if [[ ${PIPESTATUS[0]} -ne 4 ]]; then 10 | echo 'I’m sorry, `getopt --test` failed in this environment.' 11 | exit 1 12 | fi 13 | 14 | OPTIONS=d: 15 | LONGOPTS=ddir:,odir:,dsetname:,beta_1:,beta_2:,egr:,ogr:,sattr:,edpt:,ewd:,adpt:,awd:,cdpt:,cwd:,zdim:,seed:,arch:,fair_coeff:,aud_steps:,adv_coeff:,gamma:,alpha:,replicate:,num_epochs:,ptnc:,wdbn: 16 | 17 | # -regarding ! and PIPESTATUS see above 18 | # -temporarily store output to be able to check for errors 19 | # -activate quoting/enhanced mode (e.g. by writing out “--options”) 20 | # -pass arguments only via -- "$@" to separate them correctly 21 | 22 | ! PARSED=$(getopt --options=$OPTIONS --longoptions=$LONGOPTS --name "$0" -- "$@") 23 | 24 | # ! PARSED=$(getopt --longoptions=$LONGOPTS --name "$0" -- "$@") 25 | if [[ ${PIPESTATUS[0]} -ne 0 ]]; then 26 | # e.g. return value is 1 27 | # then getopt has complained about wrong arguments to stdout 28 | exit 2 29 | fi 30 | # read getopt’s output this way to handle the quoting right: 31 | eval set -- "$PARSED" 32 | 33 | 34 | # Below are just default args, 35 | # args needed to run are taken from launcher.py files in scripts/ folder 36 | 37 | # FOLDERS 38 | ddir="./data" 39 | odir="/scratch/charanr/fairness-project/output/" 40 | mkdir -p $odir $ddir 41 | 42 | # DATASET ARGS 43 | dsetname="clr-mnist" 44 | beta_1="0.5" 45 | beta_2="0.5" 46 | egr=0.5 47 | ogr=0.5 48 | sattr="bck" 49 | 50 | # ARCHITECTURE ARGS 51 | edpt=32 52 | ewd=2 53 | adpt=32 54 | awd=2 55 | cdpt=32 56 | cwd=2 57 | zdim=16 58 | seed=3 59 | 60 | # MODEL ARGS 61 | arch="ffvae" 62 | fair_coeff=0.1 63 | aud_steps=2 64 | adv_coeff=0.1 65 | gamma=10 66 | alpha=10 67 | 68 | # OTHERS 69 | replicate=1 70 | num_epochs=150 71 | ptnc=5 72 | wdbn="deepfairness-0" 73 | exp_name="slurm-demo" 74 | 75 | 76 | while true; do 77 | case "$1" in 78 | --ddir) 79 | ddir="$2" 80 | shift 2 81 | ;; 82 | --odir) 83 | odir="$2" 84 | mkdir -p $odir 85 | shift 2 86 | ;; 87 | --dsetname) 88 | dsetname="$2" 89 | shift 2 90 | ;; 91 | --beta_1) 92 | beta_1="$2" 93 | shift 2 94 | ;; 95 | --beta_2) 96 | beta_2="$2" 97 | shift 2 98 | ;; 99 | --egr) 100 | egr="$2" 101 | shift 2 102 | ;; 103 | --ogr) 104 | ogr="$2" 105 | shift 2 106 | ;; 107 | --sattr) 108 | sattr="$2" 109 | shift 2 110 | ;; 111 | --edpt) 112 | edpt="$2" 113 | shift 2 114 | ;; 115 | --ewd) 116 | ewd="$2" 117 | shift 2 118 | ;; 119 | --adpt) 120 | adpt="$2" 121 | shift 2 122 | ;; 123 | --awd) 124 | awd="$2" 125 | shift 2 126 | ;; 127 | --cdpt) 128 | cdpt="$2" 129 | shift 2 130 | ;; 131 | --cwd) 132 | cwd="$2" 133 | shift 2 134 | ;; 135 | --zdim) 136 | zdim="$2" 137 | shift 2 138 | ;; 139 | --seed) 140 | seed="$2" 141 | shift 2 142 | ;; 143 | --arch) 144 | arch="$2" 145 | shift 2 146 | ;; 147 | --fair_coeff) 148 | fair_coeff="$2" 149 | shift 2 150 | ;; 151 | --aud_steps) 152 | aud_steps="$2" 153 | shift 2 154 | ;; 155 | --adv_coeff) 156 | adv_coeff="$2" 157 | shift 2 158 | ;; 159 | --gamma) 160 | gamma="$2" 161 | shift 2 162 | ;; 163 | --alpha) 164 | alpha="$2" 165 | shift 2 166 | ;; 167 | --replicate) 168 | replicate="$2" 169 | shift 2 170 | ;; 171 | --num_epochs) 172 | num_epochs="$2" 173 | shift 2 174 | ;; 175 | --ptnc) 176 | ptnc="$2" 177 | shift 2 178 | ;; 179 | --wdbn) 180 | wdbn="$2" 181 | shift 2 182 | ;; 183 | --exp_name) 184 | exp_name="$2" 185 | shift 2 186 | ;; 187 | -d) 188 | dummy_var="$2" 189 | shift 2 190 | ;; 191 | --) 192 | shift 193 | break 194 | ;; 195 | *) 196 | echo "Programming error" 197 | exit 3 198 | ;; 199 | esac 200 | done 201 | 202 | echo "Starting training" 203 | set -o xtrace 204 | python -u train/execute.py \ 205 | --data-dir $ddir\ 206 | --output-dir $odir\ 207 | --data $dsetname\ 208 | --beta_1 $beta_1\ 209 | --beta_2 $beta_2\ 210 | --egr $egr\ 211 | --ogr $ogr\ 212 | --sensattr $sattr\ 213 | --edepth $edpt\ 214 | --ewidths $ewd\ 215 | --adepth $adpt\ 216 | --awidths $awd\ 217 | --cdepth $cdpt\ 218 | --cwidths $cwd\ 219 | --zdim $zdim\ 220 | --seed $seed\ 221 | --arch $arch\ 222 | --fair_coeff $fair_coeff\ 223 | --aud_steps $aud_steps\ 224 | --adv_coeff $adv_coeff\ 225 | --gamma $gamma\ 226 | --alpha $alpha\ 227 | --replicate $replicate\ 228 | --epochs $num_epochs\ 229 | --patience $ptnc\ 230 | --wandb_name $wdbn\ 231 | --name $exp_name\ 232 | --ifwandb True 233 | 234 | set +o xtrace 235 | -------------------------------------------------------------------------------- /scripts/wandb_init.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "deepfairness", 3 | "project" : "deep-fairness-project", 4 | "dir": "/scratch/charanr/fairness-project/output/wandb", 5 | "entity": "charanreddymila", 6 | "allow_val_change": true, 7 | "sync_tensorboard": true, 8 | "tags": ["all"] 9 | } 10 | -------------------------------------------------------------------------------- /train/__pycache__/train_mlp.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/train/__pycache__/train_mlp.cpython-38.pyc -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/utils/__init__.py -------------------------------------------------------------------------------- /utils/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/utils/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /utils/__pycache__/log_utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/utils/__pycache__/log_utils.cpython-37.pyc -------------------------------------------------------------------------------- /utils/__pycache__/options.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/utils/__pycache__/options.cpython-37.pyc -------------------------------------------------------------------------------- /utils/early_stopping.py: -------------------------------------------------------------------------------- 1 | import json 2 | import numpy as np 3 | 4 | class EarlyStopping: 5 | def __init__(self, patience, name, is_better_fn): 6 | self.patience = patience 7 | self.name = 'main_cost/avg' 8 | self.is_better_fn = is_better_fn 9 | self.metric_class_name = is_better_fn.__self__.__class__.__name__ 10 | self.best = None # best VALIDATION 11 | self.best_call_counter = 0 # best VALIDATION epoch 12 | self.best_chpt = None # address to best VALIDATION checkpoint, if provided 13 | self.corresponding_test = None # TEST value for the best VALIDATION 14 | self.should_stop = False 15 | self.patience_counter = 0 16 | self.call_counter = 0 17 | self.anynan = False 18 | self.min_delta = 0.05 19 | 20 | def reset_patience(self): 21 | self.patience_counter = 0 22 | 23 | def reduce_patience(self): 24 | self.patience_counter += 1 25 | if self.patience_counter >= self.patience: 26 | self.should_stop = True 27 | 28 | def __call__(self, vlog, tlog, chpt_str=''): 29 | if self.should_stop: 30 | return 31 | if np.isnan(vlog[self.name]): 32 | self.anynan = True 33 | self.reduce_patience() 34 | return 35 | if self.best is None: # keep separate from next condition 36 | self.best = vlog[self.name] 37 | self.best_call_counter = self.call_counter 38 | self.best_chpt = chpt_str 39 | self.corresponding_test = tlog 40 | self.corresponding_valid = vlog 41 | self.reset_patience() 42 | elif self.is_better_fn(vlog[self.name] + self.min_delta, self.best): 43 | self.best = vlog[self.name] 44 | self.best_call_counter = self.call_counter 45 | self.best_chpt = chpt_str 46 | self.corresponding_test = tlog 47 | self.corresponding_valid = vlog 48 | self.reset_patience() 49 | else: 50 | self.reduce_patience() 51 | self.call_counter += 1 52 | print('Patience count: ', self.patience_counter) 53 | 54 | def save(self, _file): 55 | with open(_file, 'w') as f: 56 | f.write(json.dumps(self.get_status(), indent=4)) 57 | 58 | def get_status(self): 59 | return dict( 60 | name=self.name, 61 | best=self.best, 62 | best_call_counter=self.best_call_counter, 63 | best_chpt=self.best_chpt, 64 | corresponding_test=self.corresponding_test, 65 | corresponding_valid=self.corresponding_valid, 66 | should_stop=self.should_stop, 67 | patience_counter=self.patience_counter, 68 | call_counter=self.call_counter, 69 | anynan=self.anynan, 70 | metric_class_name=self.metric_class_name, 71 | ) 72 | 73 | 74 | 75 | class EarlyStoppingVAE: 76 | def __init__(self, patience=3, min_delta1 = 1, min_delta2 = 0.1): 77 | self.patience = patience 78 | self.min_delta1 = min_delta1 79 | self.min_delta2 = min_delta2 80 | 81 | self.patience_cnt1 = 0 82 | self.prev_loss_val1 = 200000 83 | 84 | self.patience_cnt2 = 0 85 | self.prev_loss_val2 = 200000 86 | 87 | def stop(self, loss_val1, loss_val2): 88 | if(self.prev_loss_val1 - loss_val1>self.min_delta1): 89 | self.patience_cnt1 = 0 90 | self.prev_loss_val1 = loss_val1 91 | else: 92 | self.patience_cnt1 += 1 93 | 94 | if(self.prev_loss_val2 - loss_val2>self.min_delta2): 95 | self.patience_cnt2 = 0 96 | self.prev_loss_val2 = loss_val2 97 | else: 98 | self.patience_cnt2 += 1 99 | 100 | print('Patience count1, count2: ', self.patience_cnt1, self.patience_cnt2) 101 | if self.patience_cnt1 > self.patience and self.patience_cnt2 > self.patience : 102 | return True 103 | else: 104 | return False 105 | -------------------------------------------------------------------------------- /utils/log_utils.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | from typing import Dict 3 | import threading 4 | import time 5 | import os 6 | import json 7 | from tensorboardX import SummaryWriter 8 | from pandas import DataFrame 9 | from collections import defaultdict 10 | import argparse 11 | import torch 12 | 13 | 14 | class AverageMeterSet: 15 | """Computes average values of metrics""" 16 | def __init__(self): 17 | self.meters = {} 18 | 19 | def __getitem__(self, key): 20 | return self.meters[key] 21 | 22 | def update_dict(self, name_val_dict, n=1): 23 | for name, val in name_val_dict.items(): 24 | self.update(name, val, n) 25 | 26 | def update(self, name, value, n=1): 27 | if not name in self.meters: 28 | self.meters[name] = AverageMeter() 29 | self.meters[name].update(value, n) 30 | 31 | def reset(self): 32 | for meter in self.meters.values(): 33 | meter.reset() 34 | 35 | def values(self, postfix=""): 36 | return {name + postfix: meter.val for name, meter in self.meters.items()} 37 | 38 | def averages(self, postfix="/avg"): 39 | return {name + postfix: meter.avg for name, meter in self.meters.items()} 40 | 41 | def sums(self, postfix="/sum"): 42 | return {name + postfix: meter.sum for name, meter in self.meters.items()} 43 | 44 | def counts(self, postfix="/count"): 45 | return {name + postfix: meter.count for name, meter in self.meters.items()} 46 | 47 | 48 | class AverageMeter: 49 | """Computes and stores the average and current value""" 50 | def __init__(self): 51 | self.reset() 52 | 53 | def reset(self): 54 | self.val = 0 55 | self.avg = 0 56 | self.sum = 0 57 | self.count = 0 58 | 59 | def update(self, val, n=1): 60 | self.val = val 61 | self.sum += val * n 62 | self.count += n 63 | self.avg = self.sum / self.count 64 | 65 | def __format__(self, format): 66 | return "{self.val:{format}} ({self.avg:{format}})".format( 67 | self=self, format=format 68 | ) 69 | 70 | 71 | class TrainLog: 72 | """Saves training logs in Pandas msgpacks""" 73 | INCREMENTAL_UPDATE_TIME = 300 74 | 75 | def __init__(self, directory, name, wandb=None, init_tb=False): 76 | self.name = name 77 | self.log_file_path = "{}/{}.msgpack".format(directory, name) 78 | self._log = defaultdict(dict) 79 | self._log_lock = threading.RLock() 80 | self._last_update_time = time.time() - self.INCREMENTAL_UPDATE_TIME 81 | self._summary_writer = None 82 | self._meter_set = AverageMeterSet() 83 | if init_tb: 84 | self._summary_writer = SummaryWriter(directory) 85 | self._tb_fields = None 86 | self.wandb = wandb 87 | 88 | def set_tb_fields(self, tbf): 89 | self._tb_fields = set(tbf) 90 | 91 | def is_tb_key(self, key): 92 | if not self._tb_fields: 93 | return True 94 | for name in self._tb_fields: 95 | if name in key: 96 | return True 97 | return False 98 | 99 | def get_summary_writer(self): 100 | return self._summary_writer 101 | 102 | def set_summary_writer(self, tbw): 103 | self._summary_writer = tbw 104 | 105 | def record_single(self, step, column, value): 106 | self._record(step, {column: value}) 107 | 108 | def record(self, step, col_val_dict, tbstep=None): 109 | if not tbstep: 110 | tbstep = step 111 | if self._summary_writer: 112 | for name, value in col_val_dict.items(): 113 | if self.is_tb_key(name): 114 | self._summary_writer.add_scalar( 115 | "{}/{}".format(self.name, name), value, tbstep 116 | ) 117 | self._record(step, col_val_dict) 118 | 119 | def save(self): 120 | df = self._as_dataframe() 121 | df.to_pickle(self.log_file_path, compression="gzip") 122 | 123 | def _record(self, step, col_val_dict): 124 | with self._log_lock: 125 | self._log[step].update(col_val_dict) 126 | if time.time() - self._last_update_time >= self.INCREMENTAL_UPDATE_TIME: 127 | self._last_update_time = time.time() 128 | self.save() 129 | if self._summary_writer: 130 | self._summary_writer.file_writer.flush() 131 | 132 | def _as_dataframe(self): 133 | with self._log_lock: 134 | return DataFrame.from_dict(self._log, orient="index") 135 | 136 | def update_config(self, config_dict): 137 | if self.wandb is not None: 138 | self.wandb.config.update(config_dict) 139 | 140 | def log(self, metrics, commit=True): 141 | if self.wandb is not None: 142 | self.wandb.log(metrics, commit=commit) 143 | 144 | def add_to_wandb_summary(self, updt: Dict): 145 | if self.wandb is not None: 146 | for k, v in updt.items(): 147 | self.wandb.run.summary[k] = v 148 | 149 | def save_matplot(self, fig, title): 150 | if self.wandb is not None: 151 | print(f"Sending figure titled '{title}' to wandb") 152 | self.wandb.log({title: self.wandb.Image(fig)}) 153 | 154 | def send_files(self, files): 155 | if self.wandb is not None: 156 | print(f"Sending {files} to wandb") 157 | self.wandb.save(files) 158 | 159 | 160 | def save_dict2json(_dict, _file): 161 | """saves json to dict""" 162 | with open(_file, "w") as f: 163 | f.write(json.dumps(_dict, indent=4)) 164 | 165 | 166 | def load_json2dict(_file): 167 | """loads json to dict""" 168 | with open(_file, "r") as f: 169 | return json.load(f) 170 | 171 | 172 | def save_args(args, address): 173 | """save args""" 174 | with open(address, "w") as f: 175 | json.dump(args.__dict__, f, indent=4) 176 | 177 | 178 | def load_args(address): 179 | """load args""" 180 | args = argparse.Namespace() 181 | with open(address, "r") as f: 182 | args.__dict__ = json.load(f) 183 | return args 184 | 185 | 186 | def save_torch_model(args, epoch, model, path, early_stop=None): 187 | """saves torch models""" 188 | if args.no_save_model: 189 | return 190 | torch.save( 191 | { 192 | "args": args, 193 | "epoch": epoch, 194 | "model_state_dict": model.state_dict(), 195 | "early_stop": early_stop, 196 | }, 197 | path, 198 | ) 199 | print("Model saved here: ", path) 200 | 201 | 202 | def copytree(src, dst, symlinks=False, ignore=None): 203 | """copy source, destination files""" 204 | for item in os.listdir(src): 205 | s = os.path.join(src, item) 206 | d = os.path.join(dst, item) 207 | if os.path.isdir(s): 208 | shutil.copytree(s, d, symlinks, ignore) 209 | else: 210 | try: 211 | shutil.copy2(src, dst) 212 | except shutil.SameFileError: 213 | print(f"{src} is the same file as {dst}, skipping.") 214 | # code when Exception occur 215 | pass 216 | 217 | 218 | def append_metric_type(type_name, metric_dict): 219 | """metric appending""" 220 | updated_metric_dict = {} 221 | for k, v in metric_dict.items(): 222 | if type_name not in k: 223 | updated_metric_dict[f"{type_name}/{k}"] = v 224 | else: 225 | updated_metric_dict[k] = v 226 | 227 | return updated_metric_dict 228 | -------------------------------------------------------------------------------- /utils/media/metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/utils/media/metrics.png -------------------------------------------------------------------------------- /utils/media/results_adult.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/utils/media/results_adult.PNG -------------------------------------------------------------------------------- /utils/media/results_mnist.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/charan223/FairDeepLearning/2d8486b7b6f982dc74e22f225229ae40cde3ee9f/utils/media/results_mnist.PNG -------------------------------------------------------------------------------- /utils/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn.functional as F 4 | from itertools import repeat 5 | import collections 6 | import math 7 | import matplotlib 8 | matplotlib.use('Agg') 9 | from matplotlib import pyplot as plt 10 | 11 | def map_dict_device(arg, device): 12 | return {k: v.to(device) for k, v in arg.items()} 13 | 14 | def add_argument(parser, name, *args, **kwargs): 15 | name = name.strip('-') 16 | l = ['--' + name] 17 | if '-' in name: 18 | l.append('--' + name.replace('-', '_')) 19 | if '_' in name: 20 | l.append('--' + name.replace('_', '-')) 21 | parser.add_argument(*l, *args, **kwargs) 22 | 23 | def train_precompute(args, device, dataset): 24 | """computing sample numbers, base ratios of datasets for Cfair model""" 25 | if args.data == "clr-mnist": 26 | train_target_attrs, train_target_labels = ( 27 | dataset.train_loader.att, 28 | dataset.train_loader.eo_lbl, 29 | ) 30 | else: 31 | train_target_attrs, train_target_labels = ( 32 | dataset.train_loader.A, 33 | dataset.train_loader.Y, 34 | ) 35 | train_idx = train_target_attrs == 0 36 | train_base_0, train_base_1 = np.mean(train_target_labels[train_idx]), np.mean( 37 | train_target_labels[~train_idx] 38 | ) 39 | train_y_1 = np.mean(train_target_labels) 40 | 41 | reweight_target_tensor = None 42 | # For reweighing purpose. 43 | if args.arch == "cfair": 44 | reweight_target_tensor = torch.tensor( 45 | [1.0 / (1.0 - train_y_1), 1.0 / train_y_1] 46 | ).to(device) 47 | elif args.arch == "cfair-eo": 48 | reweight_target_tensor = torch.tensor([1.0, 1.0]).to(device) 49 | 50 | reweight_attr_0_tensor = torch.tensor( 51 | [1.0 / (1.0 - train_base_0), 1.0 / (train_base_0)] 52 | ).to(device) 53 | reweight_attr_1_tensor = torch.tensor( 54 | [1.0 / (1.0 - train_base_1), 1.0 / (train_base_1)] 55 | ).to(device) 56 | reweight_attr_tensors = [reweight_attr_0_tensor, reweight_attr_1_tensor] 57 | 58 | print("Average value of A = {}".format(np.mean(train_target_attrs))) 59 | print("A: sensitive attribute = 0/1") 60 | return reweight_target_tensor, reweight_attr_tensors 61 | --------------------------------------------------------------------------------