├── images └── noisetypes.png ├── noisylabels ├── cifar10 │ ├── logits.npy │ ├── uniform_5_ytrain.npy │ ├── uniform_15_ytrain.npy │ ├── uniform_25_ytrain.npy │ ├── uniform_35_ytrain.npy │ ├── uniform_45_ytrain.npy │ ├── uniform_55_ytrain.npy │ ├── uniform_65_ytrain.npy │ ├── uniform_75_ytrain.npy │ ├── uniform_85_ytrain.npy │ ├── class-dependent_15_ytrain.npy │ ├── class-dependent_25_ytrain.npy │ ├── class-dependent_35_ytrain.npy │ ├── class-dependent_45_ytrain.npy │ ├── class-dependent_55_ytrain.npy │ ├── class-dependent_5_ytrain.npy │ ├── class-dependent_65_ytrain.npy │ ├── class-dependent_75_ytrain.npy │ ├── class-dependent_85_ytrain.npy │ ├── feature-dependent_5_ytrain.npy │ ├── feature-dependent_15_ytrain.npy │ ├── feature-dependent_25_ytrain.npy │ ├── feature-dependent_35_ytrain.npy │ ├── feature-dependent_45_ytrain.npy │ ├── feature-dependent_55_ytrain.npy │ ├── feature-dependent_65_ytrain.npy │ ├── feature-dependent_75_ytrain.npy │ ├── feature-dependent_85_ytrain.npy │ ├── locally-concentrated_15_ytrain.npy │ ├── locally-concentrated_25_ytrain.npy │ ├── locally-concentrated_35_ytrain.npy │ ├── locally-concentrated_45_ytrain.npy │ ├── locally-concentrated_55_ytrain.npy │ ├── locally-concentrated_5_ytrain.npy │ ├── locally-concentrated_65_ytrain.npy │ ├── locally-concentrated_75_ytrain.npy │ └── locally-concentrated_85_ytrain.npy ├── mnist │ ├── logits.npy │ ├── uniform_15_ytrain.npy │ ├── uniform_25_ytrain.npy │ ├── uniform_35_ytrain.npy │ ├── uniform_45_ytrain.npy │ ├── uniform_55_ytrain.npy │ ├── uniform_5_ytrain.npy │ ├── uniform_65_ytrain.npy │ ├── uniform_75_ytrain.npy │ ├── uniform_85_ytrain.npy │ ├── class-dependent_15_ytrain.npy │ ├── class-dependent_25_ytrain.npy │ ├── class-dependent_35_ytrain.npy │ ├── class-dependent_45_ytrain.npy │ ├── class-dependent_55_ytrain.npy │ ├── class-dependent_5_ytrain.npy │ ├── class-dependent_65_ytrain.npy │ ├── class-dependent_75_ytrain.npy │ ├── class-dependent_85_ytrain.npy │ ├── feature-dependent_15_ytrain.npy │ ├── feature-dependent_25_ytrain.npy │ ├── feature-dependent_35_ytrain.npy │ ├── feature-dependent_45_ytrain.npy │ ├── feature-dependent_55_ytrain.npy │ ├── feature-dependent_5_ytrain.npy │ ├── feature-dependent_65_ytrain.npy │ ├── feature-dependent_75_ytrain.npy │ ├── feature-dependent_85_ytrain.npy │ ├── locally-concentrated_15_ytrain.npy │ ├── locally-concentrated_25_ytrain.npy │ ├── locally-concentrated_35_ytrain.npy │ ├── locally-concentrated_45_ytrain.npy │ ├── locally-concentrated_55_ytrain.npy │ ├── locally-concentrated_5_ytrain.npy │ ├── locally-concentrated_65_ytrain.npy │ ├── locally-concentrated_75_ytrain.npy │ └── locally-concentrated_85_ytrain.npy ├── cifar100 │ ├── logits.npy │ ├── uniform_15_ytrain.npy │ ├── uniform_25_ytrain.npy │ ├── uniform_35_ytrain.npy │ ├── uniform_45_ytrain.npy │ ├── uniform_55_ytrain.npy │ ├── uniform_5_ytrain.npy │ ├── uniform_65_ytrain.npy │ ├── uniform_75_ytrain.npy │ ├── uniform_85_ytrain.npy │ ├── class-dependent_15_ytrain.npy │ ├── class-dependent_25_ytrain.npy │ ├── class-dependent_35_ytrain.npy │ ├── class-dependent_45_ytrain.npy │ ├── class-dependent_55_ytrain.npy │ ├── class-dependent_5_ytrain.npy │ ├── class-dependent_65_ytrain.npy │ ├── class-dependent_75_ytrain.npy │ ├── class-dependent_85_ytrain.npy │ ├── feature-dependent_15_ytrain.npy │ ├── feature-dependent_25_ytrain.npy │ ├── feature-dependent_35_ytrain.npy │ ├── feature-dependent_45_ytrain.npy │ ├── feature-dependent_55_ytrain.npy │ ├── feature-dependent_5_ytrain.npy │ ├── feature-dependent_65_ytrain.npy │ ├── feature-dependent_75_ytrain.npy │ ├── feature-dependent_85_ytrain.npy │ ├── locally-concentrated_15_ytrain.npy │ ├── locally-concentrated_25_ytrain.npy │ ├── locally-concentrated_35_ytrain.npy │ ├── locally-concentrated_45_ytrain.npy │ ├── locally-concentrated_55_ytrain.npy │ ├── locally-concentrated_5_ytrain.npy │ ├── locally-concentrated_65_ytrain.npy │ ├── locally-concentrated_75_ytrain.npy │ └── locally-concentrated_85_ytrain.npy └── mnist_fashion │ ├── logits.npy │ ├── uniform_15_ytrain.npy │ ├── uniform_25_ytrain.npy │ ├── uniform_35_ytrain.npy │ ├── uniform_45_ytrain.npy │ ├── uniform_55_ytrain.npy │ ├── uniform_5_ytrain.npy │ ├── uniform_65_ytrain.npy │ ├── uniform_75_ytrain.npy │ ├── uniform_85_ytrain.npy │ ├── class-dependent_15_ytrain.npy │ ├── class-dependent_25_ytrain.npy │ ├── class-dependent_35_ytrain.npy │ ├── class-dependent_45_ytrain.npy │ ├── class-dependent_55_ytrain.npy │ ├── class-dependent_5_ytrain.npy │ ├── class-dependent_65_ytrain.npy │ ├── class-dependent_75_ytrain.npy │ ├── class-dependent_85_ytrain.npy │ ├── feature-dependent_15_ytrain.npy │ ├── feature-dependent_25_ytrain.npy │ ├── feature-dependent_35_ytrain.npy │ ├── feature-dependent_45_ytrain.npy │ ├── feature-dependent_55_ytrain.npy │ ├── feature-dependent_5_ytrain.npy │ ├── feature-dependent_65_ytrain.npy │ ├── feature-dependent_75_ytrain.npy │ ├── feature-dependent_85_ytrain.npy │ ├── locally-concentrated_15_ytrain.npy │ ├── locally-concentrated_25_ytrain.npy │ ├── locally-concentrated_35_ytrain.npy │ ├── locally-concentrated_45_ytrain.npy │ ├── locally-concentrated_55_ytrain.npy │ ├── locally-concentrated_5_ytrain.npy │ ├── locally-concentrated_65_ytrain.npy │ ├── locally-concentrated_75_ytrain.npy │ └── locally-concentrated_85_ytrain.npy ├── .gitignore ├── metrics.py ├── README.md ├── callbacks.py ├── noise_xy.py ├── noise.py ├── util.py ├── loss.py ├── models.py ├── dataset.py ├── main.py └── visualizer.py /images/noisetypes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/images/noisetypes.png -------------------------------------------------------------------------------- /noisylabels/cifar10/logits.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/logits.npy -------------------------------------------------------------------------------- /noisylabels/mnist/logits.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/logits.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/logits.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/logits.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/logits.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/logits.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/uniform_5_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/uniform_5_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/uniform_15_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/uniform_15_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/uniform_25_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/uniform_25_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/uniform_35_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/uniform_35_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/uniform_45_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/uniform_45_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/uniform_55_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/uniform_55_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/uniform_5_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/uniform_5_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/uniform_65_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/uniform_65_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/uniform_75_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/uniform_75_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/uniform_85_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/uniform_85_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/uniform_15_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/uniform_15_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/uniform_25_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/uniform_25_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/uniform_35_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/uniform_35_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/uniform_45_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/uniform_45_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/uniform_55_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/uniform_55_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/uniform_65_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/uniform_65_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/uniform_75_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/uniform_75_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/uniform_85_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/uniform_85_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/uniform_15_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/uniform_15_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/uniform_25_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/uniform_25_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/uniform_35_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/uniform_35_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/uniform_45_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/uniform_45_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/uniform_55_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/uniform_55_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/uniform_5_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/uniform_5_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/uniform_65_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/uniform_65_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/uniform_75_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/uniform_75_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/uniform_85_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/uniform_85_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/class-dependent_15_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/class-dependent_15_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/class-dependent_25_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/class-dependent_25_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/class-dependent_35_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/class-dependent_35_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/class-dependent_45_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/class-dependent_45_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/class-dependent_55_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/class-dependent_55_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/class-dependent_5_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/class-dependent_5_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/class-dependent_65_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/class-dependent_65_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/class-dependent_75_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/class-dependent_75_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/class-dependent_85_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/class-dependent_85_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/uniform_15_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/uniform_15_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/uniform_25_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/uniform_25_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/uniform_35_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/uniform_35_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/uniform_45_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/uniform_45_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/uniform_55_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/uniform_55_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/uniform_5_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/uniform_5_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/uniform_65_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/uniform_65_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/uniform_75_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/uniform_75_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/uniform_85_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/uniform_85_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/class-dependent_15_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/class-dependent_15_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/class-dependent_25_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/class-dependent_25_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/class-dependent_35_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/class-dependent_35_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/class-dependent_45_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/class-dependent_45_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/class-dependent_55_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/class-dependent_55_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/class-dependent_5_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/class-dependent_5_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/class-dependent_65_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/class-dependent_65_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/class-dependent_75_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/class-dependent_75_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/class-dependent_85_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/class-dependent_85_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/feature-dependent_5_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/feature-dependent_5_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/class-dependent_15_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/class-dependent_15_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/class-dependent_25_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/class-dependent_25_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/class-dependent_35_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/class-dependent_35_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/class-dependent_45_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/class-dependent_45_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/class-dependent_55_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/class-dependent_55_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/class-dependent_5_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/class-dependent_5_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/class-dependent_65_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/class-dependent_65_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/class-dependent_75_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/class-dependent_75_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/class-dependent_85_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/class-dependent_85_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/feature-dependent_15_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/feature-dependent_15_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/feature-dependent_25_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/feature-dependent_25_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/feature-dependent_35_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/feature-dependent_35_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/feature-dependent_45_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/feature-dependent_45_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/feature-dependent_55_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/feature-dependent_55_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/feature-dependent_5_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/feature-dependent_5_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/feature-dependent_65_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/feature-dependent_65_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/feature-dependent_75_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/feature-dependent_75_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/feature-dependent_85_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/feature-dependent_85_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/feature-dependent_15_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/feature-dependent_15_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/feature-dependent_25_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/feature-dependent_25_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/feature-dependent_35_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/feature-dependent_35_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/feature-dependent_45_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/feature-dependent_45_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/feature-dependent_55_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/feature-dependent_55_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/feature-dependent_65_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/feature-dependent_65_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/feature-dependent_75_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/feature-dependent_75_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/feature-dependent_85_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/feature-dependent_85_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/feature-dependent_15_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/feature-dependent_15_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/feature-dependent_25_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/feature-dependent_25_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/feature-dependent_35_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/feature-dependent_35_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/feature-dependent_45_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/feature-dependent_45_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/feature-dependent_55_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/feature-dependent_55_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/feature-dependent_5_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/feature-dependent_5_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/feature-dependent_65_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/feature-dependent_65_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/feature-dependent_75_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/feature-dependent_75_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/feature-dependent_85_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/feature-dependent_85_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/locally-concentrated_15_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/locally-concentrated_15_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/locally-concentrated_25_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/locally-concentrated_25_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/locally-concentrated_35_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/locally-concentrated_35_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/locally-concentrated_45_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/locally-concentrated_45_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/locally-concentrated_55_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/locally-concentrated_55_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/locally-concentrated_5_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/locally-concentrated_5_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/locally-concentrated_65_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/locally-concentrated_65_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/locally-concentrated_75_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/locally-concentrated_75_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist/locally-concentrated_85_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist/locally-concentrated_85_ytrain.npy -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode/ 2 | *.pyc 3 | gitparams.py 4 | /mnist/ 5 | /mnist_fashion/ 6 | /cifar10 7 | /cifar10_smallnetwork 8 | /cifar100/ 9 | mnist_mlp/ 10 | mnist_fashion_mlp/ 11 | /tmp/ -------------------------------------------------------------------------------- /noisylabels/cifar10/locally-concentrated_15_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/locally-concentrated_15_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/locally-concentrated_25_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/locally-concentrated_25_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/locally-concentrated_35_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/locally-concentrated_35_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/locally-concentrated_45_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/locally-concentrated_45_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/locally-concentrated_55_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/locally-concentrated_55_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/locally-concentrated_5_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/locally-concentrated_5_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/locally-concentrated_65_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/locally-concentrated_65_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/locally-concentrated_75_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/locally-concentrated_75_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar10/locally-concentrated_85_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar10/locally-concentrated_85_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/locally-concentrated_15_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/locally-concentrated_15_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/locally-concentrated_25_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/locally-concentrated_25_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/locally-concentrated_35_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/locally-concentrated_35_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/locally-concentrated_45_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/locally-concentrated_45_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/locally-concentrated_55_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/locally-concentrated_55_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/locally-concentrated_5_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/locally-concentrated_5_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/locally-concentrated_65_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/locally-concentrated_65_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/locally-concentrated_75_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/locally-concentrated_75_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/cifar100/locally-concentrated_85_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/cifar100/locally-concentrated_85_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/class-dependent_15_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/class-dependent_15_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/class-dependent_25_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/class-dependent_25_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/class-dependent_35_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/class-dependent_35_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/class-dependent_45_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/class-dependent_45_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/class-dependent_55_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/class-dependent_55_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/class-dependent_5_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/class-dependent_5_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/class-dependent_65_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/class-dependent_65_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/class-dependent_75_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/class-dependent_75_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/class-dependent_85_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/class-dependent_85_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/feature-dependent_15_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/feature-dependent_15_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/feature-dependent_25_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/feature-dependent_25_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/feature-dependent_35_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/feature-dependent_35_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/feature-dependent_45_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/feature-dependent_45_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/feature-dependent_55_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/feature-dependent_55_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/feature-dependent_5_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/feature-dependent_5_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/feature-dependent_65_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/feature-dependent_65_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/feature-dependent_75_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/feature-dependent_75_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/feature-dependent_85_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/feature-dependent_85_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/locally-concentrated_15_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/locally-concentrated_15_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/locally-concentrated_25_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/locally-concentrated_25_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/locally-concentrated_35_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/locally-concentrated_35_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/locally-concentrated_45_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/locally-concentrated_45_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/locally-concentrated_55_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/locally-concentrated_55_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/locally-concentrated_5_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/locally-concentrated_5_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/locally-concentrated_65_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/locally-concentrated_65_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/locally-concentrated_75_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/locally-concentrated_75_ytrain.npy -------------------------------------------------------------------------------- /noisylabels/mnist_fashion/locally-concentrated_85_ytrain.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gorkemalgan/corrupting_labels_with_distillation/HEAD/noisylabels/mnist_fashion/locally-concentrated_85_ytrain.npy -------------------------------------------------------------------------------- /metrics.py: -------------------------------------------------------------------------------- 1 | import util 2 | import tensorflow as tf 3 | from tensorflow.keras.metrics import categorical_accuracy 4 | 5 | def get_metrics(metric_type): 6 | if metric_type is 'distillation': 7 | return acc 8 | else: 9 | return 'accuracy' 10 | 11 | # For testing use regular output probabilities - without temperature 12 | def acc(y_true, y_pred): 13 | num_classes = util.NUM_CLASS 14 | y_true = y_true[:, :num_classes] 15 | y_pred = y_pred[:, :num_classes] 16 | return categorical_accuracy(y_true, y_pred) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Corrupting Labels with Distillation 2 | Code for paper "Label Noise Types and Their Effects on Learning". 3 | 4 | 4 different types of synthetic label noises are implemented as illustrated in the figure below. 5 | 6 | ![](/images/noisetypes.png) 7 | *Four types of different label noise types. Starting from left: no noise, uniform noise, class-dependent noise, locally-concentrated noise, feature-dependent noise (generated with proposed algorithm)* 8 | 9 | Requirements: 10 | * Tenxerflow 1.x 11 | * numpy 12 | * scikit-learn 13 | * seaborn 14 | 15 | In order to import pre-generated noisy labels: 16 | 17 | ```python 18 | import numpy as np 19 | 20 | dataset='mnist_fashion' # alternatives = 'mnist', 'cifar10', 'cifar100' 21 | noise_type='feature-dependent' # alternatives = 'uniform', 'class-dependent', 'locally-concentrated' 22 | noise_ratio=35 # alternatives = 5,15,25,35,45,55,65,75,85 23 | 24 | y_train_noisy = np.load('./noisylabels/{}/{}_{}_ytrain.npy'.format(dataset,noise_type,noise_ratio)) 25 | ``` 26 | 27 | In order to generate noisy labels for different noise ratio than pre-generated noise ratios 28 | 29 | ```python 30 | import numpy as np 31 | from dataset import get_data 32 | from noise_xy import noise_softmax 33 | 34 | dataset='mnist_fashion' # alternatives = 'mnist', 'cifar10', 'cifar100' 35 | noise_type='feature-dependent' # alternatives = 'uniform', 'class-dependent', 'locally-concentrated' 36 | noise_ratio = 0.35 # can be any number between 0-1 37 | 38 | _dataset = get_data(dataset) 39 | logits = np.load('./noisylabels/{}/logits.npy'.format(dataset)) 40 | 41 | y_train_noisy, probs = noise_softmax(_dataset.x_train, _dataset.y_train_int(), logits, noise_ratio) 42 | _dataset = get_data(dataset, y_noisy=y_train_noisy) 43 | _dataset.get_stats() 44 | x_train, y_train, x_test, y_test = _dataset.get_data() 45 | ``` -------------------------------------------------------------------------------- /callbacks.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from os.path import exists, join, isfile 3 | import tensorflow as tf 4 | from tensorflow.keras.callbacks import Callback, ModelCheckpoint, EarlyStopping, LearningRateScheduler, ReduceLROnPlateau, CSVLogger, TensorBoard 5 | import matplotlib.pyplot as plt 6 | import pandas as pd 7 | from collections import OrderedDict 8 | 9 | import util 10 | from util import VERBOSE, create_folders 11 | 12 | def csv_logger(log_dir, **kwargs): 13 | return CSVLogger(log_dir+'logs.csv', **kwargs) 14 | 15 | def model_checkpoint(log_dir, **kwargs): 16 | logdir = log_dir+'model/' 17 | create_folders(logdir) 18 | return ModelCheckpoint(logdir+"checkpoint.hdf5", **kwargs) 19 | 20 | def early_stop(**kwargs): 21 | return EarlyStopping(**kwargs) 22 | 23 | def learning_rate_scheduler(schedule,verbose=0): 24 | return LearningRateScheduler(schedule,verbose) 25 | 26 | def lr_plateau(**kwargs): 27 | return ReduceLROnPlateau(**kwargs) 28 | 29 | def tensor_board(dataset, log_dir, is_embed=False, **kwargs): 30 | tb_dir = log_dir+'tensorboard/' 31 | create_folders(tb_dir) 32 | with open(join(tb_dir, 'metadata.tsv'), 'w') as f: 33 | np.savetxt(f, dataset.y_test_int()) 34 | if not is_embed: 35 | return TensorBoard(log_dir=tb_dir, **kwargs) 36 | else: 37 | return TensorBoard(log_dir=tb_dir, 38 | embeddings_freq=1, 39 | embeddings_layer_names=['features'], 40 | embeddings_metadata='metadata.tsv', 41 | embeddings_data=dataset.x_test, 42 | **kwargs) 43 | 44 | class MyLogger(Callback): 45 | def __init__(self, log_dir, dataset): 46 | self.keys = None 47 | self.log_dir = log_dir 48 | _,_,self.x_test,self.y_test = dataset.get_data() 49 | super().__init__() 50 | 51 | def on_train_begin(self, logs={}): 52 | self.losses = [] 53 | self.val_losses = [] 54 | self.test_losses = [] 55 | self.acc = [] 56 | self.val_acc = [] 57 | self.test_acc = [] 58 | self.fig = plt.figure() 59 | self.logs = [] 60 | 61 | def on_epoch_end(self, epoch, logs={}): 62 | self.logs.append(logs) 63 | self.losses.append(logs.get('loss')) 64 | self.val_losses.append(logs.get('val_loss')) 65 | self.acc.append(logs.get('acc')) 66 | self.val_acc.append(logs.get('val_acc')) 67 | 68 | loss, acc = self.model.evaluate(self.x_test, self.y_test, verbose=0) 69 | self.test_losses.append(loss) 70 | self.test_acc.append(acc) 71 | 72 | plt.figure(self.fig.number) 73 | 74 | plt.clf() 75 | filename = self.log_dir +'loss.png' 76 | plt.plot(self.losses, label="loss") 77 | plt.plot(self.val_losses, label="val_loss") 78 | plt.plot(self.test_losses, label="test_loss") 79 | plt.legend(loc='best') 80 | plt.xlabel('# iterations') 81 | plt.title('Loss') 82 | plt.savefig(filename) 83 | 84 | plt.clf() 85 | filename = self.log_dir +'accuracy.png' 86 | plt.plot(self.acc, label="acc") 87 | plt.plot(self.val_acc, label="val_acc") 88 | plt.plot(self.test_acc, label="test_acc") 89 | plt.legend(loc='best') 90 | plt.xlabel('# iterations') 91 | plt.title('Accuracy') 92 | plt.savefig(filename) 93 | 94 | # log csv 95 | if self.keys is None: 96 | self.keys = sorted(logs.keys()) 97 | 98 | cols = ['epoch'] 99 | cols.extend([key for key in self.keys]) 100 | 101 | # if csv doesn't exists, initialize it 102 | if not isfile(self.log_dir+'log.csv'): 103 | row_dict = OrderedDict() 104 | row_dict['epoch']= [epoch] 105 | for col in cols[1:]: 106 | row_dict[col] = [logs.get(col)] 107 | row_dict['test_acc'] = [acc] 108 | row_dict['test_loss'] = [loss] 109 | df = pd.DataFrame(row_dict) 110 | else: 111 | df = pd.read_csv(self.log_dir+'log.csv') 112 | row_dict = OrderedDict() 113 | row_dict['epoch']= epoch 114 | for col in cols[1:]: 115 | row_dict[col] = logs.get(col) 116 | row_dict['test_acc'] = acc 117 | row_dict['test_loss'] = loss 118 | df = df.append([row_dict]) 119 | 120 | df.to_csv(self.log_dir+'log.csv', index=False) 121 | 122 | def on_train_end(self, logs=None): 123 | plt.close(self.fig) -------------------------------------------------------------------------------- /noise_xy.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sklearn.cluster import KMeans 3 | from sklearn.utils.multiclass import unique_labels 4 | from util import get_sorted_idx 5 | 6 | def noise_softmax(x_train, y_train_int, probs, noise_ratio): 7 | y_noisy = np.copy(y_train_int) 8 | num_classes = len(unique_labels(y_train_int)) 9 | num_noisy = int(x_train.shape[0]*noise_ratio) 10 | # find maximum noise ratio and change original noise ratio if necessary 11 | non_zeros = np.count_nonzero(probs, axis=1) 12 | num_multiple_guess = np.sum(non_zeros > 1) 13 | # class ids sorted according to their probabilities for each instance shape=(num_samples,num_classes) 14 | prob_preds = np.argsort(probs, axis=1) 15 | # first and second predicted classes for each instance shape=(num_samples) 16 | prob_pred1, prob_pred2 = prob_preds[:,-1], prob_preds[:,-2] 17 | # indices of wrong predictions for first prediction 18 | idx_wrong = np.where(prob_pred1 != y_train_int)[0] 19 | # change mis-predicted instances to their first prediction because it is most similer to that class 20 | if len(idx_wrong) >= num_noisy: 21 | # get the probabilities of first predictions for each sample shape=(num_samples) 22 | prob1 = np.array([probs[i,prob_pred1[i]] for i in range(len(prob_pred1))]) 23 | # sorted list of second prediction probabilities 24 | idx_sorted = np.argsort(prob1) 25 | # sort them according to prob1 26 | idx_wrong2 = get_sorted_idx(prob1, idx_wrong) 27 | # get elements with highest probability on second prediciton because they are closest to other classes 28 | idx2change = idx_wrong2[-num_noisy:] 29 | # change them to their most likely class which is second most probable prediction 30 | y_noisy[idx2change] = prob_pred1[idx2change] 31 | else: 32 | y_noisy[idx_wrong] = prob_pred1[idx_wrong] 33 | # remaining number of elements to be mislabeled 34 | num_noisy_remain = num_noisy - len(idx_wrong) 35 | # get the probabilities of second predictions for each sample shape=(num_samples) 36 | prob2 = np.array([probs[i,prob_pred2[i]] for i in range(len(prob_pred2))]) 37 | # sorted list of second prediction probabilities 38 | idx_sorted = np.argsort(prob2) 39 | # remove already changed indices for wrong first prediction 40 | idx_wrong2 = np.setdiff1d(idx_sorted, idx_wrong) 41 | # sort them according to prob2 42 | idx_wrong2 = get_sorted_idx(prob2, idx_wrong2) 43 | # get elements with highest probability on second prediciton because they are closest to other classes 44 | idx2change = idx_wrong2[-num_noisy_remain:] 45 | # change them to their most likely class which is second most probable prediction 46 | y_noisy[idx2change] = prob_pred2[idx2change] 47 | # get indices where second prediction has zero probability 48 | idx_tmp = np.where(prob2[idx2change] == 0)[0] 49 | idx_prob0 = idx2change[idx_tmp] 50 | assert np.sum(prob2[idx_prob0] != 0) == 0 51 | # since there is no information in second prediction, to prevent all samples with zero probability on second prediction to have same class 52 | # we will choose a random class for that sample 53 | for i in idx_prob0: 54 | classes = np.arange(num_classes) 55 | classes_clipped = np.delete(classes, y_train_int[i]) 56 | y_noisy[i] = np.random.choice(classes_clipped, 1) 57 | 58 | return y_noisy, probs 59 | 60 | def noise_xy_localized(features, y_train_int, noise_ratio): 61 | y_noisy = np.copy(y_train_int) 62 | n_clusters = 20 63 | num_classes = len(unique_labels(y_train_int)) 64 | for i in range(num_classes): 65 | idx = y_train_int == i 66 | kmeans = KMeans(n_clusters=n_clusters, random_state=0).fit(features[idx]) 67 | y_pred = kmeans.labels_ 68 | 69 | # number of samples for clusters 70 | n_samples = [np.sum(y_pred==k) for k in range(n_clusters)] 71 | sorted_idx = np.argsort(n_samples) 72 | # find clusters idx whose sum is equal for noise ratio 73 | n_tobecorrupted = round(np.sum(idx)*noise_ratio) 74 | 75 | for j in range(len(sorted_idx)): 76 | if n_samples[sorted_idx[j]] > n_tobecorrupted: 77 | break 78 | if j > 0: 79 | mid = sorted_idx[j-1] 80 | else: 81 | mid = sorted_idx[0] 82 | idx_class=np.where(idx==True)[0] 83 | idx2change=idx_class[y_pred==mid] 84 | y_noisy[idx2change] = np.random.choice(np.delete(np.arange(num_classes),i),1) 85 | num_corrupted = len(idx2change) 86 | 87 | for k in reversed(range(j-1)): 88 | if n_samples[sorted_idx[k]] + num_corrupted < n_tobecorrupted: 89 | idx2change=idx_class[y_pred==sorted_idx[k]] 90 | y_noisy[idx2change] = np.random.choice(np.delete(np.arange(num_classes),i),1) 91 | num_corrupted += len(idx2change) 92 | 93 | return y_noisy -------------------------------------------------------------------------------- /noise.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from numpy.testing import assert_array_almost_equal 3 | from sklearn.metrics import confusion_matrix 4 | 5 | from noise_xy import noise_softmax, noise_xy_localized 6 | from util import NOISETYPES 7 | 8 | def get_noisy_labels(dataset, noise_type, noise_ratio): 9 | assert noise_type in NOISETYPES, "invalid noise type" 10 | num_classes = dataset.num_classes 11 | probs = None 12 | # if noise ratio is integer, convert it to float 13 | if noise_ratio > 1 and noise_ratio < 100: 14 | noise_ratio = noise_ratio / 100. 15 | # generate confusion matrix for given noise type 16 | if noise_type == 'none': 17 | y_train_noisy = dataset.y_train_int() 18 | y_test_noisy = dataset.y_test_int() 19 | elif noise_type == 'uniform': 20 | P = cm_uniform(num_classes, noise_ratio) 21 | y_train_noisy, _ = noise_cm(dataset.y_train_int(), P) 22 | y_test_noisy, _ = noise_cm(dataset.y_test_int(), P) 23 | elif noise_type == 'class-dependent': 24 | path = '{}/models/teacher/npy/'.format(dataset.name) 25 | test_logits = np.load(path+'test_logits.npy') 26 | P = cm_model_prediction(dataset.x_test, dataset.y_test_int(), test_logits, noise_ratio) 27 | y_train_noisy, _ = noise_cm(dataset.y_train_int(), P) 28 | y_test_noisy, _ = noise_cm(dataset.y_test_int(), P) 29 | elif noise_type == 'feature-dependent': 30 | path = '{}/models/student/npy/'.format(dataset.name) 31 | train_preds = np.load(path+'train_preds.npy') 32 | test_preds = np.load(path+'test_preds.npy') 33 | y_train_noisy, probs = noise_softmax(dataset.x_train, dataset.y_train_int(), train_preds, noise_ratio) 34 | y_test_noisy, _ = noise_softmax(dataset.x_test, dataset.y_test_int(), test_preds, noise_ratio) 35 | elif noise_type == 'locally-concentrated': 36 | path = '{}/models/teacher/npy/'.format(dataset.name) 37 | train_logits = np.load(path+'train_logits.npy') 38 | test_logits = np.load(path+'test_logits.npy') 39 | y_train_noisy = noise_xy_localized(train_logits, dataset.y_train_int(), noise_ratio) 40 | y_test_noisy = noise_xy_localized(test_logits, dataset.y_test_int(), noise_ratio) 41 | 42 | return y_train_noisy, y_test_noisy, probs 43 | 44 | def cm_uniform(num_classes, noise_ratio): 45 | # if noise ratio is integer, convert it to float 46 | if noise_ratio > 1 and noise_ratio < 100: 47 | noise_ratio = noise_ratio / 100. 48 | assert (noise_ratio >= 0.) and (noise_ratio <= 1.) 49 | 50 | P = noise_ratio / (num_classes - 1) * np.ones((num_classes, num_classes)) 51 | np.fill_diagonal(P, (1 - noise_ratio) * np.ones(num_classes)) 52 | 53 | assert_array_almost_equal(P.sum(axis=1), 1, 1) 54 | return P 55 | 56 | def cm_model_prediction(x_test, y_test, logits, noise_ratio): 57 | # if noise ratio is integer, convert it to float 58 | if noise_ratio > 1 and noise_ratio < 100: 59 | noise_ratio = noise_ratio / 100. 60 | 61 | y_pred = np.argmax(logits, axis=1) 62 | 63 | # extract confusion matrix 64 | cm = confusion_matrix(y_test, y_pred) 65 | # set diagonal entries to 0 for now 66 | np.fill_diagonal(cm, 0) 67 | # find probability of each misclassification with avoiding zero division 68 | sums = cm.sum(axis=1) 69 | idx_zeros = np.where(sums == 0)[0] 70 | sums[idx_zeros] = 1 71 | cm = (cm.T / sums).T 72 | # weight them with noise 73 | cm = cm * noise_ratio 74 | # set diagonal entries 75 | np.fill_diagonal(cm, (1-noise_ratio)) 76 | # if noise was with zero probabiilty, set the coresponding class probability to 1 77 | for idx in idx_zeros: 78 | cm[idx,idx] = 1 79 | 80 | assert_array_almost_equal(cm.sum(axis=1), 1, 1) 81 | 82 | return cm 83 | 84 | def noise_cm(y, cm=None): 85 | assert_array_almost_equal(cm.sum(axis=1), 1, 1) 86 | 87 | y_noisy = np.copy(y) 88 | num_classes = cm.shape[0] 89 | 90 | for i in range(num_classes): 91 | # indices of samples belonging to class i 92 | idx = np.where(y == i)[0] 93 | # number of samples belonging to class i 94 | n_samples = len(idx) 95 | for j in range(num_classes): 96 | if i != j: 97 | # number of noisy samples according to confusion matrix 98 | n_noisy = int(n_samples*cm[i,j]) 99 | if n_noisy > 0: 100 | # indices of noisy samples 101 | noisy_idx = np.random.choice(len(idx), n_noisy, replace=False) 102 | # change their classes 103 | y_noisy[idx[noisy_idx]] = j 104 | # update indices 105 | idx = np.delete(idx, noisy_idx) 106 | 107 | return y_noisy, None 108 | -------------------------------------------------------------------------------- /util.py: -------------------------------------------------------------------------------- 1 | import os 2 | from shutil import rmtree 3 | import numpy as np 4 | import tensorflow as tf 5 | from tensorflow.keras.models import model_from_json 6 | from sklearn.utils.multiclass import unique_labels 7 | from sklearn.neighbors.nearest_centroid import NearestCentroid 8 | 9 | VERBOSE = 0 10 | ALPHA = 0.1 11 | TEMPERATURE = 4 12 | RANDOM_SEED = 0 13 | NUM_CLASS = 10 14 | PLOT_NOISE = False 15 | 16 | DATASETS = ['mnist', 'mnist_fashion', 'cifar10', 'cifar100'] 17 | MODELS = ['ce', 'boot_hard', 'boot_soft', 'forward', 'd2l', 'coteaching'] # 'backward' 18 | NOISETYPES = ['uniform', 'class-dependent', 'locally-concentrated', 'feature-dependent'] 19 | 20 | PARAMS = {'mnist':{'epochs': 12, 'batch_size':256, 'patience': 10, 'alpha':0.1, 'temperature':16}, 21 | 'mnist_fashion':{'epochs': 40, 'batch_size':256, 'patience': 10, 'alpha':0.1, 'temperature':16}, 22 | 'cifar10':{'epochs': 100, 'batch_size':128, 'patience': 23, 'alpha':0.3, 'temperature':2}, 23 | 'cifar100':{'epochs': 125, 'batch_size':256, 'patience': 23}} 24 | 25 | def clean_empty_logs(): 26 | paths = ['logs/mnist', 'logs/cifar10'] 27 | for log_dir in paths: 28 | if os.path.isdir(log_dir): 29 | for dirs in os.listdir(log_dir): 30 | path = os.path.join(log_dir,dirs) 31 | if os.path.isdir(path): # 'logs/mnist/nr_45/' 32 | for dirs2 in os.listdir(path): 33 | path2 = os.path.join(path,dirs2) 34 | if len(os.listdir(path2)) < 2: 35 | rmtree(path2) 36 | 37 | def create_folders(*folders): 38 | for folder in folders: 39 | if not os.path.exists(folder): 40 | os.makedirs(folder) 41 | 42 | def delete_folders(*folders): 43 | for folder in folders: 44 | if os.path.isdir(folder): 45 | rmtree(folder) 46 | 47 | def seed_everything(seed=RANDOM_SEED): 48 | np.random.seed(seed=seed) 49 | 50 | def normalize(arr, div=None): 51 | if div is None: 52 | max_val = np.amax(arr) 53 | min_val = np.amin(arr) 54 | div = max_val - min_val 55 | return arr / div 56 | 57 | def save_model(model,path): 58 | model_json = model.to_json() 59 | with open(path+'.json', 'w') as json_file: 60 | json_file.write(model_json) 61 | model.save_weights(path+'.h5') 62 | 63 | def load_model(path): 64 | with open(path+'.json', 'r') as json_file: 65 | loaded_model_json = json_file.read() 66 | loaded_model = model_from_json(loaded_model_json) 67 | loaded_model.load_weights(path+'.h5') 68 | return loaded_model 69 | 70 | def softmax(x): 71 | return np.exp(x)/(np.exp(x).sum()) 72 | 73 | def get_centroids(logits, labels, num_classes=None): 74 | # extract number of classes if not given 75 | if num_classes is None: 76 | num_classes = len(unique_labels(labels)) 77 | # find centroid locations with scikit-learn 78 | clf = NearestCentroid() 79 | clf.fit(logits, labels) 80 | return clf.centroids_ 81 | 82 | def dm_centroid(centroids): 83 | num_classes = centroids.shape[1] 84 | cm = np.zeros((num_classes,num_classes)) 85 | for i in range(num_classes): 86 | diffs = centroids - centroids[i,:] 87 | cm[i,:] = np.linalg.norm(diffs, axis=1) 88 | return cm 89 | 90 | def get_sorted_idx(probs, labels, class_id=None): 91 | ''' 92 | Returns indices of samples beloning to class_id. Indices are sorted according to probs. First one is least confidently class_id 93 | and last one is most confidently class_id. 94 | If class_id is None, then just sorts all samples according to given probability 95 | ''' 96 | # indices of samples which belong to class i 97 | if class_id is None: 98 | idx_i = labels 99 | else: 100 | idx_i = np.where(labels == class_id)[0] 101 | # order according to probabilities of confidence. First one is least likely for class i and last one is most likely 102 | idx_tmp = np.argsort(probs[idx_i]) 103 | idx_sorted = idx_i[idx_tmp] 104 | 105 | # make sure sorted idx indeed belongs to given class 106 | if class_id is not None: 107 | assert np.sum(labels[idx_sorted] == class_id) == len(idx_sorted) 108 | # make sure idx are indeed sorted 109 | assert np.sum(np.diff(probs[idx_sorted])<0) == 0 110 | 111 | return idx_sorted 112 | 113 | def get_sorted_intersect(arr1, arr2, arr_val): 114 | ''' 115 | Returns the intersection of arr1 and arr2 arrays of indices. Then intersection array of indices according to index value in arr_val 116 | ''' 117 | intersects = np.intersect1d(arr1, arr2) 118 | idx_tmp = np.argsort(arr_val[intersects]) 119 | intersects_sorted = intersects[idx_tmp] 120 | 121 | assert len(np.intersect1d(arr1, intersects_sorted)) == len(intersects_sorted) 122 | assert len(np.intersect1d(arr2, intersects_sorted)) == len(intersects_sorted) 123 | assert np.sum(np.diff(arr_val[intersects_sorted])<0) == 0 124 | 125 | return intersects_sorted 126 | 127 | if __name__ == "__main__": 128 | clean_empty_logs() -------------------------------------------------------------------------------- /loss.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | from tensorflow.keras import backend as K 4 | from tensorflow.keras.losses import categorical_crossentropy 5 | from tensorflow.keras.metrics import categorical_accuracy 6 | 7 | from noise import cm_uniform 8 | import util 9 | 10 | def distillation_loss(num_classes, alpha=util.ALPHA): 11 | def loss(y_true, y_pred): 12 | # Extract the one-hot encoded values and the softs separately so that we can create two objective functions 13 | y_true, y_true_softs = y_true[: , :num_classes], y_true[: , num_classes:] 14 | y_pred, y_pred_softs = y_pred[: , :num_classes], y_pred[: , num_classes:] 15 | return alpha*categorical_crossentropy(y_true,y_pred) + (1-alpha)*categorical_crossentropy(y_true_softs, y_pred_softs) 16 | 17 | return loss 18 | 19 | def acc_distillation(num_classes): 20 | def acc(y_true, y_pred): 21 | y_true = y_true[:, :num_classes] 22 | y_pred = y_pred[:, :num_classes] 23 | return categorical_accuracy(y_true, y_pred) 24 | return acc 25 | 26 | # implementation is taken from: https://github.com/xingjunm/dimensionality-driven-learning 27 | def symmetric_cross_entropy(alpha, beta): 28 | """ 29 | Symmetric Cross Entropy: 30 | ICCV2019 "Symmetric Cross Entropy for Robust Learning with Noisy Labels" 31 | https://arxiv.org/abs/1908.06112 32 | """ 33 | def loss(y_true, y_pred): 34 | y_true_1 = y_true 35 | y_pred_1 = y_pred 36 | 37 | y_true_2 = y_true 38 | y_pred_2 = y_pred 39 | 40 | y_pred_1 = tf.clip_by_value(y_pred_1, 1e-7, 1.0) 41 | y_true_2 = tf.clip_by_value(y_true_2, 1e-4, 1.0) 42 | 43 | return alpha*tf.reduce_mean(-tf.reduce_sum(y_true_1 * tf.log(y_pred_1), axis = -1)) + beta*tf.reduce_mean(-tf.reduce_sum(y_pred_2 * tf.log(y_true_2), axis = -1)) 44 | return loss 45 | 46 | # implementation is taken from: https://github.com/xingjunm/dimensionality-driven-learning 47 | def cross_entropy(y_true, y_pred): 48 | return categorical_crossentropy(y_true, y_pred) 49 | 50 | # implementation is taken from: https://github.com/xingjunm/dimensionality-driven-learning 51 | def boot_soft(y_true, y_pred): 52 | """ 53 | 2015 - iclrws - Training deep neural networks on noisy labels with bootstrapping. 54 | https://arxiv.org/abs/1412.6596 55 | 56 | :param y_true: 57 | :param y_pred: 58 | :return: 59 | """ 60 | beta = 0.95 61 | 62 | y_pred /= K.sum(y_pred, axis=-1, keepdims=True) 63 | y_pred = K.clip(y_pred, K.epsilon(), 1.0 - K.epsilon()) 64 | return -K.sum((beta * y_true + (1. - beta) * y_pred) * 65 | K.log(y_pred), axis=-1) 66 | 67 | # implementation is taken from: https://github.com/xingjunm/dimensionality-driven-learning 68 | def boot_hard(y_true, y_pred): 69 | """ 70 | 2015 - iclrws - Training deep neural networks on noisy labels with bootstrapping. 71 | https://arxiv.org/abs/1412.6596 72 | 73 | :param y_true: 74 | :param y_pred: 75 | :return: 76 | """ 77 | beta = 0.8 78 | 79 | y_pred /= K.sum(y_pred, axis=-1, keepdims=True) 80 | y_pred = K.clip(y_pred, K.epsilon(), 1.0 - K.epsilon()) 81 | pred_labels = K.one_hot(K.argmax(y_pred, 1), num_classes=K.shape(y_true)[1]) 82 | return -K.sum((beta * y_true + (1. - beta) * pred_labels) * 83 | K.log(y_pred), axis=-1) 84 | 85 | # implementation is taken from: https://github.com/xingjunm/dimensionality-driven-learning 86 | def forward(P): 87 | """ 88 | Making Deep Neural Networks Robust to Label Noise: a Loss Correction Approach 89 | CVPR17 https://arxiv.org/abs/1609.03683 90 | :param P: noise model, a noisy label transition probability matrix 91 | :return: 92 | """ 93 | P = K.constant(P) 94 | 95 | def loss(y_true, y_pred): 96 | y_pred /= K.sum(y_pred, axis=-1, keepdims=True) 97 | y_pred = K.clip(y_pred, K.epsilon(), 1.0 - K.epsilon()) 98 | return -K.sum(y_true * K.log(K.dot(y_pred, P)), axis=-1) 99 | 100 | return loss 101 | 102 | # implementation is taken from: https://github.com/xingjunm/dimensionality-driven-learning 103 | def backward(P): 104 | """ 105 | Making Deep Neural Networks Robust to Label Noise: a Loss Correction Approach 106 | CVPR17 https://arxiv.org/abs/1609.03683 107 | :param P: noise model, a noisy label transition probability matrix 108 | :return: 109 | """ 110 | P_inv = K.constant(np.linalg.inv(P)) 111 | 112 | def loss(y_true, y_pred): 113 | y_pred /= K.sum(y_pred, axis=-1, keepdims=True) 114 | y_pred = K.clip(y_pred, K.epsilon(), 1.0 - K.epsilon()) 115 | return -K.sum(K.dot(y_true, P_inv) * K.log(y_pred), axis=-1) 116 | 117 | return loss 118 | 119 | # implementation is taken from: https://github.com/xingjunm/dimensionality-driven-learning 120 | def lid(logits, k=20): 121 | """ 122 | Calculate LID for each data point in the array. 123 | 124 | :param logits: 125 | :param k: 126 | :return: 127 | """ 128 | batch_size = tf.shape(logits)[0] 129 | # n_samples = logits.get_shape().as_list() 130 | # calculate pairwise distance 131 | r = tf.reduce_sum(logits * logits, 1) 132 | # turn r into column vector 133 | r1 = tf.reshape(r, [-1, 1]) 134 | D = r1 - 2 * tf.matmul(logits, tf.transpose(logits)) + tf.transpose(r1) + \ 135 | tf.ones([batch_size, batch_size]) 136 | 137 | # find the k nearest neighbor 138 | D1 = -tf.sqrt(D) 139 | D2, _ = tf.nn.top_k(D1, k=k, sorted=True) 140 | D3 = -D2[:, 1:] # skip the x-to-x distance 0 by using [,1:] 141 | 142 | m = tf.transpose(tf.multiply(tf.transpose(D3), 1.0 / D3[:, -1])) 143 | v_log = tf.reduce_sum(tf.log(m + K.epsilon()), axis=1) # to avoid nan 144 | lids = -k / v_log 145 | 146 | return lids 147 | 148 | # implementation is taken from: https://github.com/xingjunm/dimensionality-driven-learning 149 | def lid_paced_loss(alpha=1.0): 150 | """TO_DO 151 | Class wise lid pace learning, targeting classwise asymetric label noise. 152 | 153 | Args: 154 | alpha: lid based adjustment paramter: this needs real-time update. 155 | Returns: 156 | Loss tensor of type float. 157 | """ 158 | if alpha == 1.0: 159 | return symmetric_cross_entropy(alpha=0.1, beta=1.0) 160 | else: 161 | def loss(y_true, y_pred): 162 | pred_labels = K.one_hot(K.argmax(y_pred, 1), num_classes=K.shape(y_true)[1]) 163 | y_new = alpha * y_true + (1. - alpha) * pred_labels 164 | y_pred /= K.sum(y_pred, axis=-1, keepdims=True) 165 | y_pred = K.clip(y_pred, K.epsilon(), 1.0 - K.epsilon()) 166 | return -K.sum(y_new * K.log(y_pred), axis=-1) 167 | 168 | return loss 169 | -------------------------------------------------------------------------------- /models.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import tensorflow as tf 3 | import tensorflow.keras.backend as K 4 | from tensorflow.keras.models import Model, Sequential 5 | from tensorflow.keras.regularizers import l2 6 | from tensorflow.keras.layers import Input, Conv2D, Dense, MaxPooling2D, Flatten, Activation, BatchNormalization, Dropout 7 | from tensorflow.keras.losses import categorical_crossentropy 8 | from tensorflow.keras.metrics import categorical_accuracy 9 | from tensorflow.keras.optimizers import Adadelta, Adam, SGD 10 | from tensorflow.keras import regularizers 11 | 12 | from loss import cross_entropy, forward, backward, boot_hard, boot_soft, lid_paced_loss, distillation_loss, acc_distillation 13 | from noise import cm_uniform 14 | from util import NUM_CLASS, DATASETS 15 | 16 | def get_model_architecture(dataset, is_dropout=False): 17 | assert dataset.name in DATASETS, "dataset must be one of: mnist, mnist_fashion, cifar10, cifar100" 18 | 19 | num_classes = dataset.num_classes 20 | img_shape = dataset.img_shape 21 | img_input = Input(shape=img_shape) 22 | 23 | if dataset.name == 'mnist' or dataset.name == 'mnist_fashion': 24 | 25 | # architecture from: https://keras.io/examples/mnist_cnn/ 26 | x = Conv2D(32, kernel_size=(3, 3))(img_input) 27 | x = Activation('relu')(x) 28 | x = Conv2D(64, (3, 3))(x) 29 | x = Activation('relu')(x) 30 | x = MaxPooling2D(pool_size=(2, 2), name='pool1')(x) 31 | if is_dropout: 32 | x = Dropout(0.25)(x) 33 | x = Flatten()(x) 34 | x = Dense(128)(x) 35 | x = Activation('relu')(x) 36 | if is_dropout: 37 | x = Dropout(0.5)(x) 38 | x = Dense(num_classes, name='features')(x) 39 | x = Activation('softmax')(x) 40 | # Create model 41 | model = Model(img_input, x) 42 | 43 | elif dataset.name == 'cifar100' or dataset.name == 'cifar10': 44 | # taken from: https://github.com/geifmany/cifar-vgg/tree/e7d4bd4807d15631177a2fafabb5497d0e4be3ba 45 | model = Sequential() 46 | weight_decay = 0.0005 47 | 48 | model.add(Conv2D(64, (3, 3), padding='same', 49 | input_shape=img_shape,kernel_regularizer=regularizers.l2(weight_decay))) 50 | model.add(Activation('relu')) 51 | model.add(BatchNormalization()) 52 | if is_dropout: 53 | model.add(Dropout(0.3)) 54 | 55 | model.add(Conv2D(64, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 56 | model.add(Activation('relu')) 57 | model.add(BatchNormalization()) 58 | 59 | model.add(MaxPooling2D(pool_size=(2, 2))) 60 | 61 | model.add(Conv2D(128, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 62 | model.add(Activation('relu')) 63 | model.add(BatchNormalization()) 64 | if is_dropout: 65 | model.add(Dropout(0.4)) 66 | 67 | model.add(Conv2D(128, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 68 | model.add(Activation('relu')) 69 | model.add(BatchNormalization()) 70 | 71 | model.add(MaxPooling2D(pool_size=(2, 2))) 72 | 73 | model.add(Conv2D(256, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 74 | model.add(Activation('relu')) 75 | model.add(BatchNormalization()) 76 | if is_dropout: 77 | model.add(Dropout(0.4)) 78 | 79 | model.add(Conv2D(256, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 80 | model.add(Activation('relu')) 81 | model.add(BatchNormalization()) 82 | if is_dropout: 83 | model.add(Dropout(0.4)) 84 | 85 | model.add(Conv2D(256, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 86 | model.add(Activation('relu')) 87 | model.add(BatchNormalization()) 88 | 89 | model.add(MaxPooling2D(pool_size=(2, 2))) 90 | 91 | 92 | model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 93 | model.add(Activation('relu')) 94 | model.add(BatchNormalization()) 95 | if is_dropout: 96 | model.add(Dropout(0.4)) 97 | 98 | model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 99 | model.add(Activation('relu')) 100 | model.add(BatchNormalization()) 101 | if is_dropout: 102 | model.add(Dropout(0.4)) 103 | 104 | model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 105 | model.add(Activation('relu')) 106 | model.add(BatchNormalization()) 107 | 108 | model.add(MaxPooling2D(pool_size=(2, 2))) 109 | 110 | 111 | model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 112 | model.add(Activation('relu')) 113 | model.add(BatchNormalization()) 114 | if is_dropout: 115 | model.add(Dropout(0.4)) 116 | 117 | model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 118 | model.add(Activation('relu')) 119 | model.add(BatchNormalization()) 120 | if is_dropout: 121 | model.add(Dropout(0.4)) 122 | 123 | model.add(Conv2D(512, (3, 3), padding='same',kernel_regularizer=regularizers.l2(weight_decay))) 124 | model.add(Activation('relu')) 125 | model.add(BatchNormalization()) 126 | 127 | model.add(MaxPooling2D(pool_size=(2, 2))) 128 | if is_dropout: 129 | model.add(Dropout(0.5)) 130 | 131 | model.add(Flatten()) 132 | model.add(Dense(512,kernel_regularizer=regularizers.l2(weight_decay))) 133 | model.add(Activation('relu')) 134 | model.add(BatchNormalization()) 135 | 136 | if is_dropout: 137 | model.add(Dropout(0.5)) 138 | model.add(Dense(num_classes, name='features')) 139 | model.add(Activation('softmax')) 140 | 141 | ''' 142 | x = Conv2D(32, (3, 3), padding='same')(img_input) 143 | x = Activation('relu')(x) 144 | 145 | x = Conv2D(32, (3, 3))(x) 146 | x = Activation('relu')(x) 147 | x = MaxPooling2D(pool_size=(2, 2))(x) 148 | if is_dropout: 149 | x = Dropout(0.25)(x) 150 | x = Conv2D(64, (3, 3), padding='same')(x) 151 | x = Activation('relu')(x) 152 | x = Conv2D(64, (3, 3))(x) 153 | x = Activation('relu')(x) 154 | x = MaxPooling2D(pool_size=(2, 2))(x) 155 | if is_dropout: 156 | x = Dropout(0.25)(x) 157 | x = Flatten()(x) 158 | x = Dense(512)(x) 159 | x = Activation('relu')(x) 160 | if is_dropout: 161 | x = Dropout(0.5)(x) 162 | x = Dense(num_classes, name='features')(x) 163 | x = Activation('softmax')(x) 164 | # Create model 165 | model = Model(img_input, x) 166 | ''' 167 | 168 | #model.summary() 169 | return model 170 | 171 | def get_model(dataset, model_name='ce', cm=None, **kwargs): 172 | # model architecture 173 | model = get_model_architecture(dataset, **kwargs) 174 | optimizer = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True) 175 | # loss 176 | if model_name == 'forward': 177 | assert cm is not None 178 | model.compile(loss=forward(cm), optimizer=optimizer, metrics=['accuracy']) 179 | elif model_name == 'backward': 180 | assert cm is not None 181 | model.compile(loss=backward(cm), optimizer=optimizer, metrics=['accuracy']) 182 | elif model_name == 'boot_hard': 183 | model.compile(loss=boot_hard, optimizer=optimizer, metrics=['accuracy']) 184 | elif model_name == 'boot_soft': 185 | model.compile(loss=boot_soft, optimizer=optimizer, metrics=['accuracy']) 186 | elif model_name == 'd2l': 187 | model.compile(loss=lid_paced_loss(), optimizer=optimizer, metrics=['accuracy']) 188 | elif model_name == 'ce' or model_name == 'coteaching': 189 | model.compile(loss=cross_entropy, optimizer=optimizer, metrics=['accuracy']) 190 | elif model_name == 'distillation': 191 | model.compile(loss=distillation_loss(dataset.num_classes), optimizer=optimizer, metrics=[acc_distillation(dataset.num_classes)]) 192 | 193 | model._name = model_name 194 | return model 195 | 196 | def compile_model(model, loss=None, optimizer=None, metrics=['accuracy']): 197 | if loss is None: 198 | loss = cross_entropy 199 | if optimizer is None: 200 | optimizer=SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True) 201 | 202 | return model.compile(loss=loss, optimizer=optimizer, metrics=metrics) 203 | -------------------------------------------------------------------------------- /dataset.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | import gzip 3 | import multiprocessing as mp 4 | from subprocess import call 5 | import warnings 6 | import numpy as np 7 | import scipy.io as sio 8 | import numpy as np 9 | import matplotlib.pyplot as plt 10 | import tensorflow as tf 11 | import tensorflow.keras.backend as K 12 | from tensorflow.keras.datasets import mnist, cifar10, cifar100 13 | from tensorflow.keras.preprocessing.image import ImageDataGenerator 14 | from tensorflow.keras.utils import to_categorical 15 | from sklearn.metrics import confusion_matrix 16 | from sklearn.model_selection import train_test_split 17 | from sklearn.utils import shuffle 18 | 19 | from visualizer import plot_matrix 20 | from util import normalize, create_folders, DATASETS, RANDOM_SEED 21 | 22 | import urllib 23 | 24 | try: 25 | from urllib.error import URLError 26 | from urllib.request import urlretrieve 27 | except ImportError: 28 | from urllib2 import URLError 29 | from urllib import urlretrieve 30 | 31 | MNIST_RESOURCES = ['train-images-idx3-ubyte.gz', 'train-labels-idx1-ubyte.gz', 't10k-images-idx3-ubyte.gz', 't10k-labels-idx1-ubyte.gz'] 32 | 33 | def report_download_progress(chunk_number, chunk_size, file_size): 34 | if file_size != -1: 35 | percent = min(1, (chunk_number * chunk_size) / file_size) 36 | bar = '#' * int(64 * percent) 37 | sys.stdout.write('\r0% |{:<64}| {}%'.format(bar, int(percent * 100))) 38 | 39 | def download(destination_path, urlbase, resources): 40 | for resource in resources: 41 | path = os.path.join(os.getcwd(),'{}{}'.format(destination_path,resource)) 42 | url = '{}{}'.format(urlbase,resource) 43 | if not os.path.exists(path): 44 | print('Downloading {} ...'.format(url)) 45 | try: 46 | hook = report_download_progress 47 | urlretrieve(url, path, reporthook=hook) 48 | except URLError: 49 | raise RuntimeError('Error downloading resource!') 50 | #else: 51 | # print('{} already exists, skipping ...'.format(path)) 52 | 53 | def load_mnist(path, kind='train'): 54 | """Load MNIST data from `path`""" 55 | labels_path = os.path.join(path, 56 | '%s-labels-idx1-ubyte.gz' 57 | % kind) 58 | images_path = os.path.join(path, 59 | '%s-images-idx3-ubyte.gz' 60 | % kind) 61 | 62 | with gzip.open(labels_path, 'rb') as lbpath: 63 | labels = np.frombuffer(lbpath.read(), dtype=np.uint8, offset=8) 64 | 65 | with gzip.open(images_path, 'rb') as imgpath: 66 | images = np.frombuffer(imgpath.read(), dtype=np.uint8, offset=16).reshape(len(labels), 784) 67 | 68 | return images, labels 69 | 70 | class DatasetCls: 71 | def __init__(self, x_train, y_train, x_test, y_test, y_noisy = None, y_noisy_test = None, 72 | num_classes = None, dataset_name=None, class_names=None): 73 | 74 | self.x_train = x_train.astype('float32') 75 | self.y_train = y_train 76 | self.x_test = x_test.astype('float32') 77 | self.y_test = y_test 78 | self.y_noisy = y_noisy if y_noisy is not None else np.copy(y_train) 79 | self.y_noisy_test = y_noisy_test if y_noisy_test is not None else np.copy(y_test) 80 | self.num_classes = num_classes 81 | self.name = dataset_name 82 | 83 | self.num_train_samples = self.y_train.shape[0] 84 | self.num_test_samples = self.y_test.shape[0] 85 | self.num_samples = self.num_train_samples + self.num_test_samples 86 | self.img_shape = x_train.shape[1:] 87 | self.mean = self.x_train.mean(axis=0) 88 | self.std = np.std(self.x_train) 89 | 90 | self.idx = np.arange(self.y_train.shape[0]) 91 | self.idx_clean = np.nonzero(self.y_noisy == self.y_train)[0] 92 | self.idx_noisy = np.nonzero(self.y_noisy != self.y_train)[0] 93 | 94 | 95 | self.num_noisy_samples = len(self.idx_noisy) 96 | self.num_clean_samples = self.num_train_samples - self.num_noisy_samples 97 | self.noise_ratio = (self.num_noisy_samples*100/self.num_train_samples) 98 | 99 | self._set_num_classes() 100 | 101 | self.int_to_onehot() 102 | 103 | if class_names is not None: 104 | self.class_names = class_names 105 | else: 106 | self.class_namesclass_names = range(self.num_classes) 107 | 108 | def get_data(self): 109 | return self.x_train, self.y_noisy, self.x_test, self.y_test 110 | 111 | def get_stats(self): 112 | assert(self.y_train.shape == self.y_noisy.shape), "Shapes should be equavelent!" 113 | 114 | print('Number of training samples: ', self.num_train_samples) 115 | print('Number of test samples: ', self.num_test_samples) 116 | 117 | str_class_nums = 'Number of samples per class: ' 118 | for i in range(self.num_classes): 119 | str_class_nums += str(self.class_names[i])+'('+str(np.sum(self.y_train_int() == i))+')' 120 | print(str_class_nums) 121 | 122 | print('Number of all samples: ', self.num_samples) 123 | print('Number of noisy samples: ', self.num_noisy_samples) 124 | print('Ratio of noise: ', self.noise_ratio) 125 | print('Image shape: ', self.img_shape) 126 | 127 | def get_cm_train(self): 128 | return confusion_matrix(self.y_train_int(),self.y_noisy_int()) 129 | 130 | def get_cm_test(self): 131 | return confusion_matrix(self.y_test_int(),self.y_noisy_test()) 132 | 133 | def get_percentage(self, percentage): 134 | if percentage < 1 and percentage > 0: 135 | _, x_clean, _, y_clean = train_test_split(self.x_train[self.idx_clean], self.y_train[self.idx_clean], test_size=percentage, random_state=RANDOM_SEED) 136 | _, x_noisy, _, y_noisy = train_test_split(self.x_train[self.idx_noisy], self.y_noisy[self.idx_noisy], test_size=percentage, random_state=RANDOM_SEED) 137 | _, x_noisy_clean, _, y_noisy_clean = train_test_split(self.x_train[self.idx_noisy], self.y_train[self.idx_noisy], test_size=percentage, random_state=RANDOM_SEED) 138 | 139 | assert np.array_equal(x_noisy, x_noisy_clean) 140 | 141 | self.x_train = np.concatenate((x_clean,x_noisy)) 142 | self.y_train = np.concatenate((y_clean,y_noisy_clean)) 143 | self.y_noisy = np.concatenate((y_clean,y_noisy)) 144 | 145 | self.x_train, self.y_train, self.y_noisy = shuffle(self.x_train, self.y_train, self.y_noisy, random_state=RANDOM_SEED) 146 | 147 | y_train_int = np.array([np.where(r==1)[0][0] for r in self.y_train]) 148 | y_noisy_int = np.array([np.where(r==1)[0][0] for r in self.y_noisy]) 149 | 150 | self.idx = np.arange(self.y_train.shape[0]) 151 | self.idx_clean = np.nonzero(y_noisy_int == y_train_int)[0] 152 | self.idx_noisy = np.nonzero(y_noisy_int != y_train_int)[0] 153 | 154 | self.num_train_samples = self.y_train.shape[0] 155 | self.num_test_samples = self.y_test.shape[0] 156 | self.num_samples = self.num_train_samples + self.num_test_samples 157 | self.num_noisy_samples = len(self.idx_noisy) 158 | self.num_clean_samples = self.num_train_samples - self.num_noisy_samples 159 | self.mean = self.x_train.mean(axis=0) 160 | self.std = np.std(self.x_train) 161 | self.noise_ratio = (self.num_noisy_samples*100/self.num_train_samples) 162 | 163 | def flow_train(self,batch_size): 164 | datagen = ImageDataGenerator() 165 | gen = datagen.flow(self.x_train, self.y_train, batch_size) 166 | while True: 167 | xy = gen.next() 168 | yield(xy[0], xy[1]) 169 | 170 | def flow_test(self,batch_size): 171 | datagen = ImageDataGenerator() 172 | gen = datagen.flow(self.x_test, self.y_test, batch_size) 173 | while True: 174 | xy = gen.next() 175 | yield(xy[0], xy[1]) 176 | 177 | def save_cm_train(self,path=None): 178 | if path is None: 179 | path = 'noisylabels/0_{}_{}_cm_train.png'.format(int(self.noise_ratio*100), self.name) 180 | cm = self.get_cm_train() 181 | plot_matrix(cm, self.class_names, title='Noise ratio: {}'.format(self.noise_ratio)) 182 | plt.savefig(path) 183 | plt.close() 184 | 185 | def save_cm_test(self,path=None): 186 | if path is None: 187 | path = 'noisylabels/0_{}_{}_cm_test.png'.format(int(self.noise_ratio*100), self.name) 188 | cm = self.get_cm_test() 189 | plot_matrix(cm, self.class_names, title='Noise ratio: {}'.format(self.noise_ratio)) 190 | plt.savefig(path) 191 | plt.close() 192 | 193 | def int_to_onehot(self): 194 | if len(self.y_train.shape) == 1: 195 | self.y_train = to_categorical(self.y_train, self.num_classes) 196 | if len(self.y_test.shape) == 1: 197 | self.y_test = to_categorical(self.y_test, self.num_classes) 198 | if len(self.y_noisy.shape) == 1: 199 | self.y_noisy = to_categorical(self.y_noisy, self.num_classes) 200 | 201 | def get_labels_int(self): 202 | return self.y_train_int(), self.y_test_int(), self.y_noisy_int() 203 | 204 | def x_train_img(self): 205 | if self.name is 'mnist' or self.name is 'mnist_fashion': 206 | return self.x_train.reshape(self.num_train_samples,28,28) 207 | else: 208 | return self.x_train 209 | 210 | def y_train_int(self): 211 | if not len(self.y_train.shape) == 1: 212 | y_train_int = np.argmax(self.y_train, axis=1) 213 | else: 214 | y_train_int = self.y_train 215 | return y_train_int 216 | 217 | def y_test_int(self): 218 | if not len(self.y_test.shape) == 1: 219 | y_test_int = np.argmax(self.y_test, axis=1) 220 | else: 221 | y_test_int = self.y_test 222 | return y_test_int 223 | 224 | def y_noisy_int(self): 225 | if not len(self.y_noisy.shape) == 1: 226 | y_noisy_int = np.argmax(self.y_noisy, axis=1) 227 | else: 228 | y_noisy_int = self.y_noisy 229 | return y_noisy_int 230 | 231 | def y_noisy_test_int(self): 232 | if not len(self.y_noisy_test.shape) == 1: 233 | y_noisy_test_int = np.argmax(self.y_noisy_test, axis=1) 234 | else: 235 | y_noisy_test_int = self.y_noisy_test 236 | return y_noisy_test_int 237 | 238 | def shuffle(self): 239 | idx_perm = np.random.permutation(self.num_train_samples) 240 | self.x_train, self.y_train = self.x_train[idx_perm], self.y_train[idx_perm] 241 | 242 | def normalize_mean(self): 243 | self.x_train = self.x_train - self.mean 244 | self.x_test = self.x_test - self.mean 245 | 246 | def normalize_std(self): 247 | self.x_train = self.x_train / self.std 248 | self.x_test = self.x_test / self.std 249 | 250 | def _set_num_classes(self): 251 | if self.num_classes == None: 252 | if len(self.y_train.shape) == 1: 253 | self.num_classes = max(self.y_train) - min(self.y_train) + 1 254 | else: 255 | self.num_classes = self.y_train.shape[1] 256 | 257 | 258 | def get_data(dataset_name='mnist', **kwargs): 259 | assert dataset_name in DATASETS, "dataset must be one of: mnist, mnist_fashion, cifar10, cifar100" 260 | if dataset_name == 'mnist': 261 | download('mnist/dataset/', 'http://yann.lecun.com/exdb/mnist/', MNIST_RESOURCES) 262 | x_train, y_train = load_mnist('mnist/dataset/', kind='train') 263 | x_test, y_test = load_mnist('mnist/dataset/', kind='t10k') 264 | 265 | x_train = x_train.reshape(-1, 28, 28, 1) / 255.0 266 | x_test = x_test.reshape(-1, 28, 28, 1) / 255.0 267 | 268 | num_classes = 10 269 | class_names = np.arange(num_classes) 270 | 271 | elif dataset_name == 'mnist_fashion': 272 | download('mnist_fashion/dataset/', 'http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/', MNIST_RESOURCES) 273 | x_train, y_train = load_mnist('mnist_fashion/dataset/', kind='train') 274 | x_test, y_test = load_mnist('mnist_fashion/dataset/', kind='t10k') 275 | 276 | x_train = x_train.reshape(-1, 28, 28, 1) / 255.0 277 | x_test = x_test.reshape(-1, 28, 28, 1) / 255.0 278 | 279 | num_classes = 10 280 | class_names = ['Tshirt_top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle_boot'] 281 | 282 | elif dataset_name == 'cifar10': 283 | (x_train, y_train), (x_test, y_test) = cifar10.load_data() 284 | 285 | x_train = x_train.reshape(-1, 32, 32, 3) / 255.0 286 | x_test = x_test.reshape(-1, 32, 32, 3) / 255.0 287 | 288 | means = x_train.mean(axis=0) 289 | # std = np.std(X_train) 290 | x_train = (x_train - means) # / std 291 | x_test = (x_test - means) # / std 292 | 293 | # they are 2D originally in cifar 294 | y_train = y_train.ravel() 295 | y_test = y_test.ravel() 296 | 297 | num_classes = 10 298 | class_names = ['airplane', 'automobile', 'bird', 'cat', 'dear', 'dog', 'frog', 'horse', 'ship', 'truck'] 299 | 300 | elif dataset_name == 'cifar100': 301 | (x_train, y_train), (x_test, y_test) = cifar100.load_data() 302 | 303 | x_train = x_train.reshape(-1, 32, 32, 3) / 255.0 304 | x_test = x_test.reshape(-1, 32, 32, 3) / 255.0 305 | 306 | means = x_train.mean(axis=0) 307 | # std = np.std(X_train) 308 | x_train = (x_train - means) # / std 309 | x_test = (x_test - means) # / std 310 | 311 | # they are 2D originally in cifar 312 | y_train = y_train.ravel() 313 | y_test = y_test.ravel() 314 | 315 | num_classes = 100 316 | class_names = ['apple', 'aquarium_fish', 'baby', 'bear', 'beaver', 'bed', 'bee', 'beetle', 317 | 'bicycle', 'bottle', 'bowl', 'boy', 'bridge', 'bus', 'butterfly', 'camel', 318 | 'can', 'castle', 'caterpillar', 'cattle', 'chair', 'chimpanzee', 'clock', 319 | 'cloud', 'cockroach', 'couch', 'crab', 'crocodile', 'cup', 'dinosaur', 320 | 'dolphin', 'elephant', 'flatfish', 'forest', 'fox', 'girl', 'hamster', 321 | 'house', 'kangaroo', 'keyboard', 'lamp', 'lawn_mower', 'leopard', 'lion', 322 | 'lizard', 'lobster', 'man', 'maple_tree', 'motorcycle', 'mountain', 'mouse', 323 | 'mushroom', 'oak_tree', 'orange', 'orchid', 'otter', 'palm_tree', 'pear', 324 | 'pickup_truck', 'pine_tree', 'plain', 'plate', 'poppy', 'porcupine', 325 | 'possum', 'rabbit', 'raccoon', 'ray', 'road', 'rocket', 'rose', 326 | 'sea', 'seal', 'shark', 'shrew', 'skunk', 'skyscraper', 'snail', 'snake', 327 | 'spider', 'squirrel', 'streetcar', 'sunflower', 'sweet_pepper', 'table', 328 | 'tank', 'telephone', 'television', 'tiger', 'tractor', 'train', 'trout', 329 | 'tulip', 'turtle', 'wardrobe', 'whale', 'willow_tree', 'wolf', 'woman', 330 | 'worm'] 331 | 332 | else: 333 | return None 334 | 335 | return DatasetCls(x_train, y_train, x_test, y_test, dataset_name=dataset_name, num_classes=num_classes, class_names=class_names, **kwargs) 336 | 337 | 338 | 339 | if __name__ == "__main__": 340 | dataset = get_data('mnist') 341 | dataset.get_stats() 342 | dataset = get_data('mnist_fashion') 343 | dataset.get_stats() 344 | download('mnist/dataset/', 'http://yann.lecun.com/exdb/mnist/', MNIST_RESOURCES) 345 | load_mnist('mnist/dataset/') 346 | np.random.seed(123) -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import os 2 | from os.path import isfile 3 | import shutil 4 | from datetime import datetime 5 | import argparse 6 | import gc 7 | from pathlib import Path 8 | import numpy as np 9 | import scipy.spatial.distance as distance 10 | import matplotlib.pyplot as plt 11 | import pandas as pd 12 | import tensorflow as tf 13 | from tensorflow import keras 14 | import tensorflow.keras.backend as K 15 | from tensorflow.keras.preprocessing.image import ImageDataGenerator 16 | from tensorflow.keras.models import Model 17 | from tensorflow.keras.layers import Activation, Lambda, concatenate, Input, Average 18 | from tensorflow.keras.optimizers import Adam, SGD 19 | from tensorflow.keras.utils import plot_model 20 | from sklearn.metrics import confusion_matrix, accuracy_score 21 | from sklearn.model_selection import StratifiedShuffleSplit, train_test_split 22 | from scipy.special import softmax as softmax_scipy 23 | 24 | from dataset import get_data, DatasetCls 25 | from noise import get_noisy_labels, cm_uniform 26 | from models import get_model, compile_model 27 | from loss import distillation_loss, acc_distillation 28 | from metrics import get_metrics 29 | from callbacks import csv_logger, model_checkpoint, early_stop, learning_rate_scheduler,lr_plateau, tensor_board, MyLogger 30 | from visualizer import plot_cm, plot_dm, plot_matrix, plot_confused_samples, plot_confused_samples2, plot_overall 31 | from util import clean_empty_logs, create_folders, seed_everything 32 | from util import save_model, load_model, softmax, get_centroids, dm_centroid 33 | from util import MODELS, DATASETS, NOISETYPES, PARAMS, RANDOM_SEED, PLOT_NOISE 34 | import util 35 | 36 | logcsv_cols = ['Dataset', 'Noise Type','Model Name', 'Noise Rate', 'Accuracy', 'Loss', 'Similarity'] 37 | 38 | def lr_schedule(epoch): 39 | learning_rate = 0.1 40 | return learning_rate * (0.5 ** (epoch // 20)) 41 | 42 | def train(dataset, model, epochs=50, batch_size=128, log_dir=None, callbacks=[], verbose=1): 43 | # prepare folders 44 | create_folders(log_dir+'model/', log_dir+'npy/') 45 | 46 | # get data 47 | x_train, y_train, x_test, y_test = dataset.get_data() 48 | x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=0.5, random_state=RANDOM_SEED) 49 | 50 | # callbacks 51 | mycallbacks = [] 52 | mycallbacks.extend(callbacks) 53 | if log_dir is not None: 54 | mycallbacks.extend([MyLogger(log_dir, dataset), 55 | learning_rate_scheduler(lr_schedule,verbose=verbose), 56 | lr_plateau(factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6, verbose=verbose) 57 | ]) 58 | plot_model(model, log_dir+'model/model.png') 59 | # image generator 60 | datagen = ImageDataGenerator(width_shift_range=0.1, height_shift_range=0.1) 61 | # train model on clean data 62 | model.fit_generator(datagen.flow(x_train, y_train, 63 | batch_size=batch_size), 64 | steps_per_epoch=dataset.num_train_samples/batch_size, 65 | epochs=epochs, 66 | validation_data=(x_val, y_val), 67 | verbose=verbose, 68 | callbacks=mycallbacks 69 | ) 70 | 71 | loss, acc = model.evaluate(x_test, y_test, verbose=0) 72 | print('val_loss:', loss, '- val_acc:', acc) 73 | 74 | del callbacks 75 | gc.collect() 76 | 77 | return model 78 | 79 | def train_coteaching(dataset, model1, model2, epochs=50, batch_size=128, log_dir=None, forget_rate=0.2, num_graduals=10,exponent=0.2,learning_rate=1e-3,epoch_decay_start=30): 80 | def epoch_stats(dataset, model, epoch, logdir, csv_path=None): 81 | x_train, y_train_noisy, x_test, y_test = dataset.get_data() 82 | y_train_int = dataset.y_noisy_int() 83 | y_test_int = dataset.y_test_int() 84 | clean_index = dataset.idx_clean 85 | noisy_index = dataset.idx_noisy 86 | 87 | if csv_path is not None: 88 | train_loss, train_acc = model.evaluate(x_train, y_train, verbose=0) 89 | test_loss, test_acc = model.evaluate(x_test, y_test, verbose=0) 90 | print("-%s: acc_test: %.4f - acc_mix: %.4f" % (model.name, test_acc, train_acc)) 91 | df = pd.read_csv(log_dir+csv_path) 92 | #row = [{'epoch':epoch, 'acc':acc_mix,'loss':loss_mix, 'val_acc':acc_test, 'val_loss':loss_test,'test_acc':test_acc,'test_loss':test_loss}] 93 | row = [{'epoch':epoch, 'acc':train_acc,'loss':train_loss,'test_acc':test_acc,'test_loss':test_loss}] 94 | df = df.append(row) 95 | df.to_csv(log_dir+csv_path, index=False) 96 | 97 | xticks = np.arange(1, epoch+1, 1.0) 98 | plt.figure() 99 | plt.plot(xticks, df['acc'], label="acc") 100 | #plt.plot(xticks, df['val_acc'], label="val_acc") 101 | plt.plot(xticks, df['test_acc'], label="test_acc") 102 | plt.legend(loc='best') 103 | plt.xlabel('# iterations') 104 | plt.xticks(xticks) 105 | plt.title('Accuracy') 106 | plt.savefig(log_dir+'accuracy_{}.png'.format(model.name)) 107 | 108 | plt.clf() 109 | plt.plot(xticks, df['loss'], label="loss") 110 | #plt.plot(xticks, df['val_loss'], label="val_loss") 111 | plt.plot(xticks, df['test_loss'], label="test_loss") 112 | plt.legend(loc='best') 113 | plt.xlabel('# iterations') 114 | plt.xticks(xticks) 115 | plt.title('Loss') 116 | plt.savefig(log_dir+'loss_{}.png'.format(model.name)) 117 | plt.close() 118 | create_folders(log_dir+'model/', log_dir+'npy/') 119 | # get data 120 | x_train, y_train, x_test, y_test = dataset.get_data() 121 | model1._name = 'model1' 122 | model2._name = 'model2' 123 | 124 | # number of batches in an epoch 125 | num_batch_iter = x_train.shape[0] / batch_size 126 | # calculate forget rates for each epoch (from origianl code) 127 | forget_rates = np.ones(epochs)*forget_rate 128 | forget_rates[:num_graduals] = np.linspace(0, forget_rate**exponent, num_graduals) 129 | # calculate learning rates for each epoch (from origianl code) 130 | learning_rates = [learning_rate] * epochs 131 | for i in range(epoch_decay_start, epochs): 132 | learning_rates[i] = float(epochs - i) / (epochs - epoch_decay_start) * learning_rate 133 | 134 | if log_dir is not None: 135 | #logcsv_cols =['epoch','acc','loss','val_acc','val_loss','test_acc','test_loss'] 136 | logcsv_cols =['epoch','acc','loss','test_acc','test_loss'] 137 | df = pd.DataFrame(columns=logcsv_cols) 138 | df.to_csv(log_dir+'log1.csv', index=False) 139 | df.to_csv(log_dir+'log2.csv', index=False) 140 | 141 | for e in range(1,epochs): 142 | # if learning rate changes, recompile 143 | if (learning_rates[e] != learning_rates[e-1]): 144 | model1.compile(loss='categorical_crossentropy', optimizer=SGD(lr=learning_rates[e], decay=1e-6, momentum=0.9, nesterov=True), metrics=['accuracy']) 145 | model2.compile(loss='categorical_crossentropy', optimizer=SGD(lr=learning_rates[e], decay=1e-6, momentum=0.9, nesterov=True), metrics=['accuracy']) 146 | #model2.compile(loss='categorical_crossentropy', optimizer=Adam(learning_rates[e]), metrics=['accuracy']) 147 | 148 | remember_rate = 1 - forget_rates[e] 149 | print("Epoch: %d/%d; Learning rate: %.7f; n_keep: %d" % (e+1, epochs, learning_rates[e], remember_rate*batch_size)) 150 | 151 | # iterate for each batch in an epoch 152 | for (i, (x_batch, y_batch)) in enumerate(dataset.flow_train(batch_size)): 153 | num_remember = int(remember_rate * len(x_batch)) 154 | 155 | # select samples based on model 1 156 | y_pred = model1.predict_on_batch(x_batch) 157 | cross_entropy = np.sum(-y_batch*np.log(y_pred+1e-8),axis=1) 158 | batch_idx1= np.argsort(cross_entropy)[:num_remember] 159 | 160 | # select samples based on model 2 161 | y_pred = model2.predict_on_batch(x_batch) 162 | cross_entropy = np.sum(-y_batch*np.log(y_pred+1e-8),axis=1) 163 | batch_idx2 = np.argsort(cross_entropy)[:num_remember] 164 | 165 | # training 166 | model1.train_on_batch(x_batch[batch_idx2,:], y_batch[batch_idx2,:]) 167 | model2.train_on_batch(x_batch[batch_idx1,:], y_batch[batch_idx1,:]) 168 | 169 | if i >= num_batch_iter: 170 | break 171 | 172 | epoch_stats(dataset,model1,e,log_dir,'log1.csv') 173 | epoch_stats(dataset,model2,e,log_dir,'log2.csv') 174 | 175 | # choose best model 176 | loss1, acc1 = model1.evaluate(x_test, y_test, verbose=0) 177 | loss2, acc2 = model2.evaluate(x_test, y_test, verbose=0) 178 | if acc1 > acc2: 179 | os.rename(log_dir+'log1.csv', log_dir+'log.csv') 180 | return model1 181 | else: 182 | os.rename(log_dir+'log2.csv', log_dir+'log.csv') 183 | return model2 184 | 185 | def save_model_outputs(model, _dataset, model_path): 186 | npy_path = model_path+'npy/' 187 | create_folders(npy_path, model_path+'model/') 188 | model_soft = Model(model.input, model.get_layer('features').output) 189 | # save softmax predictions 190 | pred = model.predict(_dataset.x_train)[:, :_dataset.num_classes] 191 | pred_int = np.argmax(pred, axis=1) 192 | np.save(npy_path+'train_preds.npy', pred) 193 | np.save(npy_path+'train_preds_int.npy', pred_int) 194 | pred = model.predict(_dataset.x_test)[:, :_dataset.num_classes] 195 | pred_int = np.argmax(pred, axis=1) 196 | np.save(npy_path+'test_preds.npy', pred) 197 | np.save(npy_path+'test_preds_int.npy', pred_int) 198 | # save logits 199 | logits_train = model_soft.predict(_dataset.x_train)[:, :_dataset.num_classes] 200 | logits_test = model_soft.predict(_dataset.x_test)[:, :_dataset.num_classes] 201 | np.save(npy_path+'train_logits.npy', logits_train) 202 | np.save(npy_path+'test_logits.npy', logits_test) 203 | # save confusion matrices 204 | cm_train = plot_cm(model, _dataset.x_train,_dataset.y_train_int(),_dataset.class_names, model_path+'train_cm.png') 205 | cm_test = plot_cm(model, _dataset.x_test,_dataset.y_test_int(),_dataset.class_names, model_path+'test_cm.png') 206 | np.save(npy_path+'train_cm.npy', cm_train) 207 | np.save(npy_path+'test_cm.npy', cm_test) 208 | # save distance matrices 209 | plot_dm(model_soft, _dataset.x_train, _dataset.y_train_int(), _dataset.class_names, model_path+'train_dm.png') 210 | plot_dm(model_soft, _dataset.x_test, _dataset.y_test_int(), _dataset.class_names, model_path+'test_dm.png') 211 | # save model 212 | plot_model(model,model_path+'model/model.png') 213 | save_model(model,model_path+'model/model') 214 | K.clear_session() 215 | 216 | def prep_teacher_model(dataset, verbose): 217 | # train model on noise free data with dropout 218 | model_path = '{}/models/teacher/'.format(dataset) 219 | if not isfile(model_path+'model/model.h5') or not isfile(model_path+'model/model.json'): 220 | print('Main model doesnt exist, training it...') 221 | # train teacher 222 | _dataset = get_data(dataset) 223 | model = get_model(_dataset, is_dropout=True) 224 | model = train(_dataset, model, 225 | PARAMS[dataset]['epochs'], PARAMS[dataset]['batch_size'], 226 | log_dir=model_path, 227 | callbacks=[early_stop(patience=PARAMS[dataset]['patience'], monitor='val_loss', verbose=verbose)], 228 | verbose=verbose) 229 | # save output files 230 | save_model_outputs(model, _dataset, model_path) 231 | 232 | def prep_student(dataset, verbose, alpha, temperature): 233 | # if student is not saved beforehand, train and save it 234 | model_path = '{}/models/student/'.format(dataset) 235 | if not isfile(model_path+'model/model.h5') or not isfile(model_path+'model/model.json'): 236 | print('Student model doesnt exist, training it...') 237 | # load dataset and logits 238 | _dataset = get_data(dataset) 239 | x_train, y_train, x_test, y_test = _dataset.get_data() 240 | train_features = np.load('{}/models/teacher/npy/train_logits.npy'.format(dataset)) 241 | test_features = np.load('{}/models/teacher/npy/test_logits.npy'.format(dataset)) 242 | # normalized output with temperature shape=(num_samples,num_classes) 243 | y_train_soft = softmax(train_features/temperature) 244 | y_test_soft = softmax(test_features/temperature) 245 | # concatenated output labels=(num_samples,2*num_classes) 246 | y_train_new = np.concatenate([y_train, y_train_soft], axis=1) 247 | y_test_new = np.concatenate([y_test, y_test_soft], axis =1) 248 | # build student model 249 | student = get_model(_dataset, 'distillation', is_dropout=True) 250 | # remove softmax 251 | student.layers.pop() 252 | # get features 253 | logits = student.layers[-1].output 254 | # normal softmax output 255 | probs = Activation('softmax')(logits) 256 | # softmax output with temperature 257 | logits_T = Lambda(lambda x: x / temperature)(logits) 258 | probs_T = Activation('softmax')(logits_T) 259 | # concatanete 260 | output = concatenate([probs, probs_T]) 261 | # This is our new student model 262 | student = Model(student.input, output) 263 | compile_model(student, loss=distillation_loss(_dataset.num_classes,alpha),metrics=[acc_distillation(_dataset.num_classes)]) 264 | # create a new dataset with generated data 265 | dataset_s = DatasetCls(x_train,y_train_new,x_test,y_test_new, dataset_name=dataset) 266 | # train student 267 | student = train(dataset_s, student, 268 | PARAMS[dataset]['epochs']*2, PARAMS[dataset]['batch_size'], 269 | log_dir=model_path, 270 | callbacks=[early_stop(patience=PARAMS[dataset]['patience'], monitor='val_loss', verbose=verbose)], 271 | verbose=verbose) 272 | # save output files 273 | save_model_outputs(student, _dataset, model_path) 274 | 275 | K.clear_session() 276 | 277 | def prep_noisylabels(dataset, folders, noise_type, noise_ratio, verbose, alpha,temperature, is_dropout): 278 | # prepare required models first 279 | prep_teacher_model(dataset, verbose) 280 | prep_student(dataset, verbose,alpha,temperature) 281 | 282 | # generate and save corrupted labels for each noise type for given noise ratio 283 | _dataset = get_data(dataset) 284 | 285 | # copy xy model as none for baseline 286 | path = str(Path(folders['logdir']).parent) 287 | if not os.path.isdir(path+'/none/'): 288 | shutil.copytree('{}/models/teacher'.format(dataset), path+'/none/') 289 | 290 | # generate noisy labels 291 | y_train_noisy, y_test_noisy, probs = get_noisy_labels(_dataset, noise_type, noise_ratio) 292 | if PLOT_NOISE: 293 | y_train_clean, y_test_clean = _dataset.y_train_int(), _dataset.y_test_int() 294 | create_folders(folders['noisedir']) 295 | if not isfile(folders['noisedir']+'cmtest.png'): 296 | print('Noise for {} doesnt exist, creating it...'.format(folders['noisedir'])) 297 | # plot confused samples 298 | if probs is not None: 299 | create_folders(folders['noisedir']+'/plots') 300 | np.save(folders['noisedir']+'probs.npy', probs) 301 | _dataset_noisy = get_data(dataset, y_noisy=y_train_noisy, y_noisy_test=y_test_noisy) 302 | plot_confused_samples(probs, _dataset_noisy, path=folders['noisedir']+'plots/') 303 | plot_confused_samples2(probs, _dataset_noisy, path=folders['noisedir']+'plots/') 304 | # save confusion matrix 305 | cm = confusion_matrix(y_train_clean,y_train_noisy) 306 | plot_matrix(cm, _dataset.class_names, title='Noise ratio: {}'.format(noise_ratio)) 307 | plt.savefig(folders['noisedir']+'cmtrain.png') 308 | cm = confusion_matrix(y_test_clean,y_test_noisy) 309 | plot_matrix(cm, _dataset.class_names, title='Noise ratio: {}'.format(noise_ratio)) 310 | plt.savefig(folders['noisedir']+'cmtest.png') 311 | 312 | return y_train_noisy, y_test_noisy 313 | 314 | def postprocess(dataset, model, noise_type, noise_ratio, folders, y_test_noisy): 315 | log_dir = folders['logdir'] 316 | loss, acc = model.evaluate(dataset.x_test, dataset.y_test, verbose=0) 317 | print('loss:', loss, '- acc:', acc) 318 | 319 | # calculate similarity ofgiven confusion matrix and output confusion matrix 320 | pred = model.predict(dataset.x_test) 321 | pred_int = np.argmax(pred, axis=1) 322 | sim = 1 - distance.cosine(pred_int, y_test_noisy) 323 | print('Similarity is',sim) 324 | # plot confusion matrix 325 | plot_cm(model, dataset.x_test,dataset.y_test_int(),dataset.class_names, log_dir+'/cm.png', title='acc({}), similarity({})'.format(round(acc,3),round(sim,2))) 326 | # plot accuracies and losses for all models 327 | base_folder = folders['logbase_nr'] 328 | plot_overall(base_folder) 329 | # save variables 330 | np.save(log_dir+'preds.npy', pred_int) 331 | save_model(model, log_dir+'model/model') 332 | 333 | def main(dataset_name, model_name, epochs, batch_size, noise_type, noise_ratio, verbose=1, alpha=util.ALPHA, temperature=16, is_dropout=False, percentage=1): 334 | K.clear_session() 335 | os.environ['CUDA_VISIBLE_DEVICES'] = '0' 336 | 337 | seed_everything() 338 | # folders to be used 339 | noise_base_path = 'nr_{}'.format(noise_ratio) if not is_dropout else 'nr_{}_do'.format(noise_ratio) 340 | folders = {'logbase': '{}/logs_{}/'.format(dataset_name,percentage), 341 | 'logbase_nr': '{}/logs_{}/{}/{}/'.format(dataset_name,percentage,noise_base_path,model_name), 342 | 'logdir': '{}/logs_{}/{}/{}/{}/'.format(dataset_name,percentage,noise_base_path,model_name, noise_type), 343 | 'modelbase' : '{}/models/'.format(dataset_name), 344 | 'noisebase': '{}/noisylabels/'.format(dataset_name), 345 | 'noisedir': '{}/noisylabels/{}/'.format(dataset_name,noise_type), 346 | 'dataset': '{}/dataset'.format(dataset_name) 347 | } 348 | 349 | # if log file already exis"ts dont run it again 350 | if isfile(folders['logdir']+'model/model.h5') and isfile(folders['logdir']+'model/model.json'): 351 | print('Logs exists, skipping run...') 352 | return 353 | 354 | # clean empty logs if there is any 355 | clean_empty_logs() 356 | # create necessary folders 357 | create_folders(folders['dataset'], folders['logdir']) 358 | # generate noisy labels 359 | y_train_noisy, y_test_noisy = prep_noisylabels(dataset_name, folders, noise_type, noise_ratio, verbose, alpha, temperature, is_dropout) 360 | 361 | # load dataset with noisy labels 362 | dataset = get_data(dataset_name, y_noisy=y_train_noisy, y_noisy_test=y_test_noisy) 363 | dataset.get_percentage(percentage) 364 | 365 | # stats before training 366 | print('Dataset: {}, model: {}, noise_type: {}, noise_ratio: {}, epochs: {}, batch: {} , dropout: {}'.format( 367 | dataset.name, model_name, noise_type, noise_ratio, epochs, batch_size, is_dropout)) 368 | dataset.get_stats() 369 | dataset.save_cm_train(folders['logdir']+'corrupted_data.png') 370 | 371 | 372 | # train model 373 | if model_name == 'coteaching': 374 | model1 = get_model(dataset, model_name, is_dropout=is_dropout) 375 | model2 = get_model(dataset, model_name, is_dropout=is_dropout) 376 | model = train_coteaching(dataset, model1, model2, epochs, batch_size, folders['logdir']) 377 | else: 378 | #cm = np.load('{}/models/xy/npy/test_cm.npy'.format(dataset_name)) 379 | cm = dataset.get_cm_train() 380 | cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] 381 | model = get_model(dataset, model_name, cm, is_dropout=is_dropout) 382 | model = train(dataset, model, epochs, batch_size, folders['logdir'], verbose=verbose) 383 | 384 | # performance analysis 385 | postprocess(dataset, model, noise_type, noise_ratio, folders, y_test_noisy) 386 | K.clear_session() 387 | 388 | if __name__ == "__main__": 389 | parser = argparse.ArgumentParser() 390 | parser.add_argument('-d', '--dataset_name', help="Dataset to use: 'mnist', cifar10'", required=False, type=str, default='mnist_fashion') 391 | parser.add_argument('-m', '--model_name', help="Model name: 'ce', 'forward', 'backward', 'boot_hard', 'boot_soft', 'd2l'.", required=False, type=str, default='ce') 392 | parser.add_argument('-e', '--epochs', help="The number of epochs to train for.", required=False, type=int, default=30) 393 | parser.add_argument('-b', '--batch_size', help="The batch size to use for training.", required=False, type=int, default=128) 394 | parser.add_argument('-n', '--noise_type', help="'none', 'uniform', 'random', 'random_symmetric', 'pairwise', 'model_pred', 'xy'",required=False, type=str, default='feature-dependent') 395 | parser.add_argument('-r', '--noise_ratio', help="The percentage of noisy labels [0, 100].", required=False, type=int, default=35) 396 | parser.add_argument('-t', '--temperature', help="Temeperature for student to be trained", required=False, type=int, default=16) 397 | parser.add_argument('-a', '--alpha', help="Alpha for learning with distillation", required=False, type=float, default=0.1) 398 | parser.add_argument('-v', '--verbose', help="V,erbose one of the following: 0,1,2", required=False, type=int, default=1) 399 | parser.add_argument('-p', '--percentage', help="Percentage of dataset to be used. Between 0-1", required=False, type=float, default=1) 400 | parser.add_argument('--dropout', dest='dropout', action='store_true') 401 | parser.add_argument('--no-dropout', dest='dropout', action='store_false') 402 | 403 | args = parser.parse_args() 404 | main(args.dataset_name, args.model_name, args.epochs, args.batch_size, args.noise_type, args.noise_ratio, 405 | args.verbose, args.alpha, args.temperature, args.dropout, args.percentage) 406 | -------------------------------------------------------------------------------- /visualizer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import pandas as pd 4 | from math import sqrt, ceil 5 | import matplotlib.pyplot as plt 6 | from sklearn.metrics import confusion_matrix 7 | from sklearn.utils.multiclass import unique_labels 8 | from util import get_centroids, dm_centroid, get_sorted_idx, get_sorted_intersect, load_model, NOISETYPES, DATASETS 9 | 10 | def plot(vals, title, xlabel='# iterations', save=True, save_folder='./', show=False): 11 | filename = save_folder + title+'.png' 12 | plt.figure() 13 | for name in vals: 14 | plt.plot(vals[name], label=name) 15 | plt.legend(loc='best') 16 | plt.xlabel(xlabel) 17 | plt.title(title) 18 | if save: 19 | plt.savefig(filename) 20 | if show is not False: 21 | plt.show() 22 | return filename 23 | 24 | def plot_acc_loss(csv_file, save=True, save_folder='./', show=False): 25 | df = pd.read_csv(csv_file) 26 | accs = {'acc': df['acc'], 27 | 'val_acc': df['val_acc']} 28 | losses = {'loss': df['loss'], 29 | 'val_loss': df['val_loss']} 30 | acc_file=plot(accs, 'accuracy', save=save, save_folder=save_folder, show=show) 31 | loss_file=plot(losses, 'loss', save=save, save_folder=save_folder, show=show) 32 | return acc_file, loss_file 33 | 34 | def plot_cm(model, x_test, y_test_int, class_names, path=None, **kwargs): 35 | pred = model.predict(x_test) 36 | pred_int = np.argmax(pred, axis=1) 37 | cm = confusion_matrix(y_test_int,pred_int) 38 | plot_matrix(cm, class_names, **kwargs) 39 | if path is not None: 40 | plt.savefig(path) 41 | plt.close() 42 | return cm 43 | 44 | def plot_dm(model, x_test, y_test_int, class_names, path=None, **kwargs): 45 | logits = model.predict(x_test) 46 | centroids = get_centroids(logits, y_test_int ,len(class_names)) 47 | dm_centroids = dm_centroid(centroids) 48 | plot_matrix(dm_centroids, class_names, **kwargs) 49 | if path is not None: 50 | plt.savefig(path) 51 | plt.close() 52 | return dm_centroids 53 | 54 | def plot_matrix(cm, classes=None, 55 | normalize=True, 56 | title=None, 57 | cmap=plt.cm.Blues): 58 | """ 59 | This function prints and plots the confusion matrix. 60 | Normalization can be applied by setting `normalize=True`. 61 | """ 62 | if not title: 63 | if normalize: 64 | title = 'Normalized confusion matrix' 65 | else: 66 | title = 'Confusion matrix, without normalization' 67 | 68 | # Set classes 69 | if classes is None: 70 | classes = np.arange(cm.shape[0]) 71 | 72 | if normalize: 73 | cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] 74 | 75 | fig, ax = plt.subplots() 76 | im = ax.imshow(cm, interpolation='nearest', cmap=cmap) 77 | ax.figure.colorbar(im, ax=ax) 78 | # We want to show all ticks... 79 | ax.set(xticks=np.arange(cm.shape[1]), 80 | yticks=np.arange(cm.shape[0]), 81 | # ... and label them with the respective list entries 82 | xticklabels=classes, yticklabels=classes, 83 | title=title, 84 | ylabel='True label', 85 | xlabel='Predicted label') 86 | 87 | # Rotate the tick labels and set their alignment. 88 | plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor") 89 | 90 | # Loop over data dimensions and create text annotations. 91 | fmt = '.2f' if normalize else 'd' 92 | thresh = cm.max() / 2. 93 | for i in range(cm.shape[0]): 94 | for j in range(cm.shape[1]): 95 | ax.text(j, i, format(cm[i, j], fmt), 96 | ha="center", va="center", 97 | color="white" if cm[i, j] > thresh else "black") 98 | fig.tight_layout() 99 | 100 | fig.set_size_inches(cm.shape[0]/2, cm.shape[1]/2) 101 | 102 | # correction for matplotlib bug 103 | bottom, top = ax.get_ylim() 104 | ax.set_ylim(bottom+0.5, top-0.5) 105 | 106 | return ax 107 | 108 | def plot_confused_samples(probs, dataset, class_id_true=None, class_id_false=None, path=None, num_plots=100): 109 | y_clean, y_noisy = dataset.y_train_int(), dataset.y_noisy_int() 110 | class_names = dataset.class_names 111 | idx_mislabeled = dataset.idx_noisy 112 | num_plot = ceil(sqrt(num_plots)) 113 | num_plots = num_plot * num_plot 114 | 115 | idx_mislabeled = np.where(y_noisy != y_clean)[0] 116 | if class_id_true is None or class_id_false is None: 117 | prob_false = np.choose(y_noisy[idx_mislabeled], probs[idx_mislabeled].T) 118 | idx_sorted = get_sorted_idx(prob_false, np.arange(len(y_clean[idx_mislabeled]))) 119 | idx_mislabeled_sorted = idx_mislabeled[idx_sorted] 120 | else: 121 | idx_sorted = get_sorted_idx(probs[:,class_id_false], y_clean, class_id_true) 122 | idx_mislabeled_sorted = get_sorted_intersect(idx_sorted, idx_mislabeled, probs[:,class_id_false]) 123 | 124 | fig, axs = plt.subplots(num_plot, num_plot) 125 | fig2, axs2 = plt.subplots(num_plot, num_plot) 126 | 127 | count = 0 128 | for j in range(num_plot): 129 | for k in range(num_plot): 130 | # confident samples for class i 131 | idx = idx_mislabeled_sorted[-count-1] 132 | axs[j, k].imshow(dataset.x_train_img()[idx], cmap='gray') 133 | title = '{}({:.2f}) - {}({:.2f})'.format(class_names[y_clean[idx]], probs[idx,y_clean[idx]], 134 | class_names[y_noisy[idx]], probs[idx,y_noisy[idx]]) 135 | axs[j, k].set_title(title) 136 | # unconfident samples for class i 137 | idx = idx_mislabeled_sorted[num_plots-count] 138 | axs2[j, k].imshow(dataset.x_train_img()[idx], cmap='gray') 139 | title = '{}({:.2f}) - {}({:.2f})'.format(class_names[y_clean[idx]], probs[idx,y_clean[idx]], 140 | class_names[y_noisy[idx]], probs[idx,y_noisy[idx]]) 141 | axs2[j, k].set_title(title) 142 | 143 | count += 1 144 | 145 | fig.set_size_inches(3*num_plot, 3*num_plot) 146 | plt.figure(fig.number) 147 | plt.savefig(path+'mislabeled_conf.png') 148 | plt.close() 149 | fig2.set_size_inches(3*num_plot, 3*num_plot) 150 | plt.figure(fig2.number) 151 | plt.savefig(path+'mislabeled_unconf.png') 152 | plt.close() 153 | 154 | def plot_confused_samples2(probs, dataset, path, num_plots=100): 155 | y_clean, y_noisy = dataset.y_train_int(), dataset.y_noisy_int() 156 | class_names = dataset.class_names 157 | num_plot = ceil(sqrt(num_plots)) 158 | num_plots = num_plot * num_plot 159 | 160 | for i in range(dataset.num_classes): 161 | idx_sorted = get_sorted_idx(probs[:,i], y_clean, i) 162 | fig, axs = plt.subplots(num_plot, num_plot) 163 | fig2, axs2 = plt.subplots(num_plot, num_plot) 164 | count = 0 165 | for j in range(num_plot): 166 | for k in range(num_plot): 167 | # confident samples for class i 168 | idx = idx_sorted[-count-1] 169 | axs[j, k].imshow(dataset.x_train_img()[idx], cmap='gray') 170 | title = '{}({:.2f}) - {}({:.2f})'.format(class_names[y_clean[idx]], probs[idx,y_clean[idx]], 171 | class_names[y_noisy[idx]], probs[idx,y_noisy[idx]]) 172 | axs[j, k].set_title(title) 173 | # unconfident samples for class i 174 | idx = idx_sorted[num_plots-count] 175 | axs2[j, k].imshow(dataset.x_train_img()[idx], cmap='gray') 176 | title = '{}({:.2f}) - {}({:.2f})'.format(class_names[y_clean[idx]], probs[idx,y_clean[idx]], 177 | class_names[y_noisy[idx]], probs[idx,y_noisy[idx]]) 178 | axs2[j, k].set_title(title) 179 | 180 | count += 1 181 | 182 | fig.set_size_inches(3*num_plot, 3*num_plot) 183 | plt.figure(fig.number) 184 | plt.savefig(path+'{}_conf.png'.format(class_names[i])) 185 | plt.close() 186 | fig2.set_size_inches(3*num_plot, 3*num_plot) 187 | plt.figure(fig2.number) 188 | plt.savefig(path+'{}_unconf.png'.format(class_names[i])) 189 | plt.close() 190 | 191 | def plot_overall(base_folder, dirs=None): 192 | acc_train_path = base_folder+'accs_train.png' 193 | acc_val_path = base_folder+'accs_val.png' 194 | acc_test_path = base_folder+'accs_test.png' 195 | loss_train_path = base_folder+'losses_train.png' 196 | loss_val_path = base_folder+'losses_val.png' 197 | loss_test_path = base_folder+'losses_test.png' 198 | 199 | fig_acc_train = plt.figure() 200 | fig_acc_val = plt.figure() 201 | fig_acc_test = plt.figure() 202 | fig_loss_train = plt.figure() 203 | fig_loss_val = plt.figure() 204 | fig_loss_test = plt.figure() 205 | 206 | cols = ['acc', 'val_acc', 'test_acc', 'loss', 'val_loss', 'test_loss'] 207 | titles = ['Accuracy train','Accuracy validation', 'Accuracy test', 'Loss train', 'Loss validation', 'Loss Test'] 208 | figs = [fig_acc_train, fig_acc_val, fig_acc_test, fig_loss_train, fig_loss_val, fig_loss_test] 209 | paths = [acc_train_path, acc_val_path, acc_test_path, loss_train_path, loss_val_path, loss_test_path] 210 | 211 | if dirs is None: 212 | dirs = os.listdir(base_folder) 213 | 214 | for folder in dirs: 215 | log_path = base_folder+folder+'/log.csv' 216 | if os.path.isfile(log_path): 217 | df = pd.read_csv(log_path) 218 | for col, fig in zip(cols, figs): 219 | plt.figure(fig.number) 220 | plt.plot(df[col], label=folder) 221 | 222 | for fig, title, path in zip(figs, titles, paths): 223 | plt.figure(fig.number) 224 | plt.legend(loc='best') 225 | plt.xlabel('# iterations') 226 | plt.title(title) 227 | plt.savefig(path) 228 | plt.close() 229 | 230 | def logs_through_nrs(base_folder, model='ce'): 231 | cols = ['acc', 'val_acc', 'test_acc', 'loss', 'val_loss', 'test_loss'] 232 | 233 | dfs = [] 234 | for col in cols: 235 | dfs.append(pd.DataFrame(columns=['nr'])) 236 | 237 | dirs = os.listdir(base_folder) 238 | 239 | # take only dropout 240 | dirs_do, dirs_no_do = [], [] 241 | for folder in dirs: 242 | if folder != 'logs.csv': 243 | if folder.endswith('_do'): 244 | dirs_do.append(folder) 245 | else: 246 | dirs_no_do.append(folder) 247 | 248 | for folder in dirs_do: 249 | nr_path = base_folder+folder+'/'+model+'/' 250 | nr = int(folder.replace('nr_','').replace('_do','')) 251 | for model_path in os.listdir(nr_path): 252 | log_path = nr_path+model_path+'/log.csv' 253 | if os.path.isfile(log_path): 254 | df_tmp = pd.read_csv(log_path) 255 | for col, df in zip(cols, dfs): 256 | df.at[nr, model_path] = df_tmp.iloc[-1][col]#df.set_value(nr, model_path, df_tmp.iloc[-1][col]) 257 | df.at[nr, 'nr'] = nr#df.set_value(nr, 'nr', nr) 258 | 259 | for df,col in zip(dfs,cols): 260 | df = df.sort_values(by=['nr']) 261 | df.to_csv(base_folder+'logs_{}_{}.csv'.format(model,col), index=False) 262 | 263 | return dfs 264 | 265 | def plot_through_nrs(base_folder, model='ce'): 266 | acc_train_path = base_folder+'accs_train.png' 267 | acc_val_path = base_folder+'accs_val.png' 268 | acc_test_path = base_folder+'accs_test.png' 269 | loss_train_path = base_folder+'losses_train.png' 270 | loss_val_path = base_folder+'losses_val.png' 271 | loss_test_path = base_folder+'losses_test.png' 272 | 273 | fig_acc_train = plt.figure() 274 | fig_acc_val = plt.figure() 275 | fig_acc_test = plt.figure() 276 | fig_loss_train = plt.figure() 277 | fig_loss_val = plt.figure() 278 | fig_loss_test = plt.figure() 279 | 280 | titles = ['Accuracy train','Accuracy validation', 'Accuracy test', 'Loss train', 'Loss validation', 'Loss Test'] 281 | figs = [fig_acc_train, fig_acc_val, fig_acc_test, fig_loss_train, fig_loss_val, fig_loss_test] 282 | paths = [acc_train_path, acc_val_path, acc_test_path, loss_train_path, loss_val_path, loss_test_path] 283 | 284 | dfs = logs_through_nrs(base_folder, model) 285 | 286 | for fig, df in zip(figs, dfs): 287 | for model_name in df.columns: 288 | if model_name != 'nr' and model_name != 'none': 289 | df = df.sort_values(by=['nr']) 290 | plt.figure(fig.number) 291 | plt.plot(df[model_name], label=model_name) 292 | 293 | for fig, title, path in zip(figs, titles, paths): 294 | plt.figure(fig.number) 295 | plt.legend(loc='best') 296 | plt.xlabel('Noise Ratio (%)') 297 | plt.ylabel('Accuracy') 298 | plt.title(title) 299 | plt.savefig(path) 300 | plt.close() 301 | 302 | def logs_through_percentages(base_folder, model='ce', nr=35): 303 | cols = ['acc', 'val_acc', 'test_acc', 'loss', 'val_loss', 'test_loss'] 304 | dirs = os.listdir(base_folder) 305 | 306 | # get log folders 307 | log_dirs = [] 308 | percentages = [] 309 | for folder in dirs: 310 | if folder.startswith('logs_') and not folder.endswith('.csv'): 311 | log_dirs.append(os.path.join(base_folder, folder)+'/') 312 | percentages.append(folder.replace('logs_','')) 313 | 314 | # create csvs for each log 315 | for log_dir in log_dirs: 316 | logs_through_nrs(log_dir,model) 317 | 318 | dfs_orgs = [] 319 | for log_dir in log_dirs: 320 | df_org = pd.read_csv(log_dir+'logs_{}_test_acc.csv'.format(model)) 321 | dfs_orgs.append(df_org) 322 | 323 | columns = list(df_org.columns) 324 | columns[0] = 'percentage' 325 | raw_columns = columns.copy() 326 | raw_columns.remove('percentage') 327 | 328 | dfs = [] 329 | for col in cols: 330 | df = pd.DataFrame(columns=columns) 331 | for i,(df_org,percentage) in enumerate(zip(dfs_orgs,percentages)): 332 | df.set_value(i, 'percentage', percentage) 333 | for nt in raw_columns: 334 | df.set_value(i, nt, df_org.loc[df_org['nr'] == nr, nt].values[0]) 335 | df = df.sort_values(by=['percentage']) 336 | dfs.append(df) 337 | df.to_csv(base_folder+'logs_{}_{}.csv'.format(model,col), index=False) 338 | 339 | return dfs 340 | 341 | def get_svcca(acts1, acts2): 342 | import cca_core 343 | 344 | # flatten convolutional layers 345 | if len(acts1.shape) == 4: 346 | num_datapoints, h, w, channels = acts1.shape 347 | acts1 = acts1.reshape((num_datapoints*h*w, channels)) 348 | num_datapoints, h, w, channels = acts2.shape 349 | acts2 = acts2.reshape((num_datapoints*h*w, channels)) 350 | 351 | # Mean subtract activations 352 | cacts1 = acts1 - np.mean(acts1, axis=1, keepdims=True) 353 | cacts2 = acts2 - np.mean(acts2, axis=1, keepdims=True) 354 | 355 | # Perform SVD 356 | U1, s1, V1 = np.linalg.svd(cacts1, full_matrices=False) 357 | U2, s2, V2 = np.linalg.svd(cacts2, full_matrices=False) 358 | 359 | n = min(20, len(acts1)) 360 | svacts1 = np.dot(s1[:n]*np.eye(n), V1[:n]) 361 | # can also compute as svacts1 = np.dot(U1.T[:20], cacts1) 362 | svacts2 = np.dot(s2[:n]*np.eye(n), V2[:n]) 363 | # can also compute as svacts1 = np.dot(U2.T[:20], cacts2) 364 | 365 | svcca_results = cca_core.get_cca_similarity(svacts1, svacts2, epsilon=1e-10, verbose=False) 366 | return np.mean(svcca_results["cca_coef1"]) 367 | 368 | def plot_cosine_similarities(base_folder, dataset): 369 | from tensorflow.keras.layers import dot, Dropout, Dense, Activation 370 | from tensorflow.keras import backend as K 371 | from tensorflow.keras.models import Model 372 | from dataset import get_data 373 | from models import compile_model 374 | from scipy.spatial.distance import cosine 375 | from sklearn.preprocessing import normalize 376 | 377 | K.clear_session() 378 | dataset = get_data(dataset) 379 | test = dataset.x_test 380 | dirs = os.listdir(base_folder) 381 | 382 | modelDict = {} 383 | for folder in dirs: 384 | if os.path.isdir(base_folder+folder): 385 | model = load_model(base_folder+folder+'/model/model') 386 | compile_model(model) 387 | outputs = [layer.output for layer in model.layers if isinstance(layer, Activation)][1:-1] 388 | outputs.append(model.input) 389 | modelDict[folder] = outputs 390 | 391 | layer_names = [layer.name.split('/',1)[0] for layer in outputs] 392 | 393 | for key in modelDict: 394 | if key != 'none': 395 | similarity = [] 396 | for i in range(len(modelDict['none'])-1): 397 | model_layer_clean = Model(inputs=modelDict['none'][-1], outputs=modelDict['none'][i]) 398 | output_layer_clean = model_layer_clean.predict(test) 399 | model_layer_noisy = Model(inputs=modelDict[key][-1], outputs=modelDict[key][i]) 400 | output_layer_noisy = model_layer_noisy.predict(test) 401 | print('{}__{}__'.format(i,output_layer_noisy.shape)) 402 | 403 | if len(output_layer_clean.shape) == 4: 404 | num_datapoints, h, w, channels = output_layer_clean.shape 405 | output_layer_clean = output_layer_clean.reshape((num_datapoints*h*w, channels)) 406 | output_layer_noisy = output_layer_noisy.reshape((num_datapoints*h*w, channels)) 407 | 408 | sim = get_svcca(np.transpose(output_layer_clean), np.transpose(output_layer_noisy)) 409 | similarity.append(sim) 410 | print('{}{} = {}'.format(key,i,sim)) 411 | del model_layer_clean 412 | del model_layer_noisy 413 | modelDict[key] = similarity 414 | 415 | plt.figure() 416 | plt.xticks(np.arange(len(layer_names)), layer_names, rotation=45) 417 | for key in modelDict: 418 | if key != 'none': 419 | plt.plot(modelDict[key], label=key) 420 | plt.legend(loc='best') 421 | plt.xlabel('Layer number') 422 | plt.title("Similarities of Layers (%)") 423 | plt.show() 424 | plt.savefig(base_folder+'layer_sims.png') 425 | plt.close() 426 | 427 | def plot_csvs(myDict, title='Title', save_path='plot.png', xlabel=None, ylabel=None): 428 | plt.figure() 429 | for key in myDict: 430 | df = pd.read_csv(myDict[key]['path']) 431 | plt.plot(df[myDict[key]['col']], label=myDict[key]['key']) 432 | plt.legend(loc='lower ') 433 | plt.title(title) 434 | if xlabel: 435 | plt.xlabel(xlabel) 436 | if ylabel: 437 | plt.ylabel(ylabel) 438 | #plt.show() 439 | plt.savefig(save_path) 440 | plt.close() 441 | 442 | def merge_images(images_list, save_path='test.jpg',orientation='horizontal'): 443 | import sys 444 | from PIL import Image 445 | 446 | images = [Image.open(x) for x in images_list] 447 | widths, heights = zip(*(i.size for i in images)) 448 | 449 | if orientation=='horizontal': 450 | total_width = sum(widths) 451 | max_height = max(heights) 452 | new_im = Image.new('RGB', (total_width, max_height)) 453 | x_offset = 0 454 | for im in images: 455 | new_im.paste(im, (x_offset,0)) 456 | x_offset += im.size[0] 457 | else: 458 | max_width = max(widths) 459 | total_height = sum(heights) 460 | new_im = Image.new('RGB', (max_width, total_height)) 461 | y_offset = 0 462 | for im in images: 463 | new_im.paste(im, (0,y_offset)) 464 | y_offset += im.size[1] 465 | 466 | new_im.save(save_path) 467 | 468 | def visualize_all(): 469 | dirs = os.listdir('./') 470 | 471 | # Get list of existing logs 472 | base_folders = [] 473 | for folder in dirs: 474 | if folder in DATASETS: 475 | base_folders.append(folder+'/') 476 | 477 | # to be removed 478 | base_folders = ['mnist_fashion'] 479 | for base_folder in base_folders: 480 | # write all log.csv files 481 | logs_through_percentages(base_folder) 482 | 483 | # get list of log folders and percentages 484 | log_dirs, percentages = [], [] 485 | dirs = os.listdir(base_folder) 486 | for folder in dirs: 487 | if folder.startswith('logs_') and os.path.isdir(base_folder+folder): 488 | log_dirs.append(os.path.join(base_folder, folder)+'/') 489 | percentages.append(folder.replace('logs_','')) 490 | # plots through different noise rations 491 | for log_dir in log_dirs: 492 | plot_through_nrs(log_dir) 493 | # plot cosine similarities 494 | 495 | if __name__ == "__main__": 496 | # merge_images(['mnist_fashion_35.png','cifar100_35.png'],save_path='accuracies.png',orientation='vertical') 497 | merge_images(['./cifar100/logs_1/accs_train.png','./cifar100/logs_1/accs_val.png', './cifar100/logs_1/accs_test.png'],save_path='noiseratios.png') 498 | 499 | dataset = 'mnist_fashion' 500 | noise_rate = 35 501 | cols = ['acc', 'val_acc', 'test_acc'] 502 | titles = {'acc':'Train accuracy', 'val_acc':'Validation accuracy', 'test_acc':'Test accuracy'} 503 | imagenames = [] 504 | for dataset in ['mnist_fashion', 'cifar100']: 505 | images_list = [] 506 | for col in cols: 507 | myDict = {'none': {'path': '{}/logs_1/nr_{}_do/ce/none/log.csv'.format(dataset,noise_rate), 'col': col, 'key':'none'}, 508 | 'uniform': {'path': '{}/logs_1/nr_{}_do/ce/uniform/log.csv'.format(dataset,noise_rate), 'col': col, 'key':'uniform'}, 509 | 'class-dependent': {'path': '{}/logs_1/nr_{}_do/ce/class-dependent/log.csv'.format(dataset,noise_rate), 'col': col, 'key':'class-dependent'}, 510 | 'locally-concentrated': {'path': '{}/logs_1/nr_{}_do/ce/locally-concentrated/log.csv'.format(dataset,noise_rate), 'col': col, 'key':'locally-concentrated'}, 511 | 'feature-dependent': {'path': '{}/logs_1/nr_{}_do/ce/feature-dependent/log.csv'.format(dataset,noise_rate), 'col': col, 'key':'feature-dependent'}} 512 | save_path = '{}_{}_{}_.png'.format(dataset,noise_rate,col) 513 | images_list.append(save_path) 514 | plot_csvs(myDict,title='{} for %{} noise rate: {}'.format(titles[col],noise_rate, dataset.upper()), save_path=save_path, xlabel='# epochs', ylabel='Accuracy') 515 | merge_images(images_list,save_path='{}_{}.png'.format(dataset,noise_rate)) 516 | imagenames.append('{}_{}.png'.format(dataset,noise_rate)) 517 | merge_images(imagenames,save_path='accuracies.png',orientation='vertical') 518 | 519 | #plot_cosine_similarities('cifar10/logs_1/nr_45_do/ce/', 'cifar10') 520 | #logs_through_percentages('mnist_fashion/') 521 | #plot_through_nrs('mnist_fashion/logs_1/') 522 | #plot_overall('cifar10/logs/nr_45/ce/') --------------------------------------------------------------------------------