├── README.md ├── caffe ├── __init__.py ├── train.txt ├── train.h5 ├── data2.prototxt ├── solver_template2.prototxt ├── discriminator2.prototxt ├── generator2.prototxt ├── creating_proto.py └── caffe_train.py ├── keras ├── __init__.py ├── analysis │ ├── context.py │ ├── plot_loss.py │ └── simple_generation.py ├── lcd_utils.py ├── EnergyGan.py ├── gan3D.py ├── EnergyGanEmbedding.py ├── EcalEnergyGan.py ├── particle3Dtrain.py ├── EnergyTrainEmbedding.py ├── EnergyTrain.py ├── ParallelTest.py ├── EcalEnergyTrain.py ├── HoroTest.py └── paramScan │ ├── parameter_scan.py │ └── vegan_param3.py ├── neon ├── __init__.py ├── gen_data_norm.pyc ├── temporary_utils.pyc ├── gen_data_norm.py ├── temporary_utils.py ├── gan3D.py ├── energy_dataset.py ├── potential_gan.py └── energy_gan.py ├── analysis ├── __init__.py ├── h5toroot_evc.py ├── analysis3D.py └── plot_root_vec.C ├── Makefile ├── requirements.txt ├── setup.py ├── utils ├── gaussian.py └── data_info.py └── LICENSE /README.md: -------------------------------------------------------------------------------- 1 | # 3Dgan -------------------------------------------------------------------------------- /caffe/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /keras/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /neon/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /analysis/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /caffe/train.txt: -------------------------------------------------------------------------------- 1 | /data/gkhattak/train.h5 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | init: 2 | pip install -r requirements.txt 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow 2 | keras 3 | numpy 4 | sklearn 5 | 6 | -------------------------------------------------------------------------------- /caffe/train.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashishpatel26/3Dgan/master/caffe/train.h5 -------------------------------------------------------------------------------- /neon/gen_data_norm.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashishpatel26/3Dgan/master/neon/gen_data_norm.pyc -------------------------------------------------------------------------------- /neon/temporary_utils.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashishpatel26/3Dgan/master/neon/temporary_utils.pyc -------------------------------------------------------------------------------- /keras/analysis/context.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) 4 | 5 | import EcalEnergyGan 6 | -------------------------------------------------------------------------------- /caffe/data2.prototxt: -------------------------------------------------------------------------------- 1 | layer { 2 | name: "ECAL" 3 | type: "HDF5Data" 4 | top: "ECAL" 5 | top: "TAG" 6 | hdf5_data_param { 7 | source: "train.txt" 8 | batch_size: 100 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /keras/lcd_utils.py: -------------------------------------------------------------------------------- 1 | import numpy 2 | import h5py 3 | 4 | def lcd_3Ddata(): 5 | 6 | f = h5py.File("EGshuffled.h5","r") 7 | data =f.get('ECAL') 8 | dtag =f.get('TAG') 9 | xtr=numpy.array(data) 10 | tag=numpy.array(dtag) 11 | #xtr=xtr[...,numpy.newaxis] 12 | #xtr=numpy.rollaxis(xtr,4,1) 13 | print xtr.shape 14 | 15 | return xtr, tag.astype(bool) 16 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Learn more: https://github.com/kennethreitz/setup.py 4 | 5 | from setuptools import setup, find_packages 6 | 7 | 8 | with open('README.md') as f: 9 | readme = f.read() 10 | 11 | 12 | setup( 13 | name='3Dgan', 14 | description='3D convolutiona GAN for fast calorimeter simulation', 15 | long_description=readme, 16 | author='Sofia Vallecorsa', 17 | author_email='Sofia.Vallecorsa@cern.ch', 18 | url='https://github.com/svalleco/3Dgan', 19 | #license=license, 20 | packages=find_packages(exclude=('docs')) 21 | ) 22 | -------------------------------------------------------------------------------- /neon/gen_data_norm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import scipy as sp 3 | 4 | def osc(x, y, z, kx, ky, kz, cx, cy, cz): 5 | return 0.5 * (kx * (x - cx)**2 + ky * (y - cy)**2 + kz * (z - cz)**2) 6 | 7 | def gen_rhs(size): 8 | features = np.zeros((size, 25, 25, 25, 1)) 9 | labels = np.ones(size) 10 | x = np.linspace(-20., 20., 25) 11 | xx, yy, zz = np.meshgrid(x, x, x) 12 | 13 | 14 | for i in range(size): 15 | cx = -8.0 + 16.0 * np.random.rand() 16 | cy = -8.0 + 16.0 * np.random.rand() 17 | cz = -8.0 + 16.0 * np.random.rand() 18 | kx = 0.16 * np.random.rand() 19 | ky = 0.16 * np.random.rand() 20 | kz = 0.16 * np.random.rand() 21 | dat = osc(xx, yy, zz, kx, ky, kz, cx, cy, cz) 22 | features[i, :, :, :, 0] = np.minimum(dat, 30 * np.ones_like(dat)) 23 | #labels[i] = 0.5 * (np.sqrt(kx) + np.sqrt(ky) + np.sqrt(kz)) 24 | 25 | return features.reshape((size, 25 * 25 * 25)).astype(np.float32), labels.astype(np.float32) 26 | -------------------------------------------------------------------------------- /caffe/solver_template2.prototxt: -------------------------------------------------------------------------------- 1 | net: "@NET@.prototxt" 2 | display: 0 3 | # iteration in caffe = number of batches to be processed 4 | # Training data = 116062 and testing = 12896 and epochs =30 5 | # test_iter specifies how many forward passes the test should carry out. 6 | # we have test batch size 100 => 129 test iterations 7 | #test_iter: 129 8 | # Carry out testing every 1000 training iterations. 9 | #test_interval: 1000 10 | # The maximum number of training iterations for 30 epochs => 116062 * 30/100 11 | max_iter: 34900 12 | #################### solver parameters ###################################### 13 | ##### Keras rmsprop parms used = lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0 14 | type: "RMSProp" 15 | base_lr: 0.001 # lr 16 | momentum:0.0 # not used 17 | rms_decay: 0.9 # rho 18 | weight_decay: 0.0 # decay 19 | lr_policy: "fixed" # 20 | # no need to define epsilon = delta (caffe) = 1e-8 by default 21 | ############################################################################# 22 | solver_mode: GPU 23 | device_id: 0 24 | 25 | -------------------------------------------------------------------------------- /analysis/h5toroot_evc.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import h5py 3 | import sys 4 | import ROOT 5 | from ROOT import TTree, TFile, AddressOf, gROOT, std, vector 6 | gROOT.ProcessLine("#include "); 7 | events = [] 8 | 9 | if len(sys.argv) != 4: 10 | print('Usage: python h5toroot.py [h5 input file name] [root output file name] [max number of events]') 11 | sys.exit() 12 | 13 | ifile = h5py.File(sys.argv[1],'r') 14 | ofile = ROOT.TFile(sys.argv[2],'RECREATE') 15 | maxn = int(sys.argv[3]) 16 | ecalTree = TTree('ecalTree', "ecal Tree") 17 | 18 | #hcal = np.array(ifile['HCAL']) 19 | ecal = np.array(ifile['ECAL']) 20 | #target = np.array(ifile['target']) 21 | 22 | #print('TARGET:', target.shape) 23 | #print('ECAL:', ecal.shape) 24 | #print('HCAL:', hcal.shape) 25 | en = 0 26 | ei = ecal.shape[1] 27 | ej = ecal.shape[2] 28 | ek = ecal.shape[3] 29 | 30 | #hi = hcal.shape[1] 31 | #hj = hcal.shape[2] 32 | #hk = hcal.shape[3] 33 | 34 | #ti = target.shape[1] 35 | #tj = target.shape[2] 36 | 37 | #assert(ecal.shape[0] == hcal.shape[0] == target.shape[0]) 38 | 39 | nevents = ecal.shape[0] 40 | if ecal.shape[0] > maxn: 41 | nevents = maxn 42 | vec_x = ROOT.std.vector(int)() 43 | vec_y = ROOT.std.vector(int)() 44 | vec_z = ROOT.std.vector(int)() 45 | vec_E = ROOT.std.vector(float)() 46 | 47 | nevents = maxn 48 | 49 | 50 | ecalTree.Branch('x',vec_x) 51 | ecalTree.Branch('y',vec_y) 52 | ecalTree.Branch('z',vec_z) 53 | ecalTree.Branch('E',vec_E) 54 | 55 | for e in range(nevents): 56 | 57 | ec = ecal[e] 58 | #hc = hcal[e] 59 | #tg = target[e] 60 | 61 | vec_x.clear() 62 | vec_y.clear() 63 | vec_z.clear() 64 | vec_E.clear() 65 | 66 | for i in range(ei): 67 | for j in range(ej): 68 | for k in range(ek): 69 | 70 | energy = ec[i][j][k] 71 | if energy > 0: 72 | vec_E.push_back(energy) 73 | vec_x.push_back(i) 74 | vec_y.push_back(j) 75 | vec_z.push_back(k) 76 | ecalTree.Fill() 77 | en += 1 78 | 79 | ofile.Write() 80 | ofile.Close() 81 | -------------------------------------------------------------------------------- /neon/temporary_utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import h5py 3 | from sklearn.model_selection import train_test_split 4 | 5 | def temp_3Ddata(): 6 | 7 | f = h5py.File("/Users/svalleco/GAN/data/EGshuffled.h5","r") 8 | data = f.get('ECAL') 9 | #dtag =f.get('TAG') 10 | xtr = np.array(data) 11 | print (xtr.shape) 12 | labels = np.ones(xtr.shape[0]) 13 | #labels =f.get('TAG') 14 | #tag=numpy.array(dtag) 15 | #xtr=xtr[...,numpy.newaxis] 16 | #xtr=numpy.rollaxis(xtr,4,1) 17 | #print xtr.shape 18 | 19 | return xtr.reshape((xtr.shape[0], 25 * 25 * 25)).astype(np.float32), labels.astype(np.float32) 20 | 21 | def get_output(): 22 | f = h5py.File("/Users/svalleco/GAN/data/output_data.h5","r") 23 | data = f.get("dataset_1") 24 | x = np.array(data) 25 | return x.astype(np.float32) 26 | 27 | def make_hdf5iterator_files(): 28 | X, y = temp_3Ddata() 29 | X[X < 1e-6] = 0 30 | mean = np.mean(X, axis=0, keepdims=True) 31 | max_elem = np.max(np.abs(X)) 32 | print(np.max(np.abs(X)),'max abs element') 33 | print(np.min(X),'min element') 34 | X = (X- mean)/max_elem 35 | print(X.shape, 'X shape') 36 | print(np.max(X),'max element after normalisation') 37 | print(np.min(X),'min element after normalisation') 38 | X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.9, random_state=42) 39 | print(X_train.shape, 'X train shape') 40 | print(y_train.shape, 'y train shape') 41 | 42 | gen_backend(backend='cpu', batch_size=100) 43 | 44 | # generate the HDF5 file 45 | datsets = {'train': (X_train, y_train), 46 | 'test': (X_test, y_test)} 47 | 48 | for ky in ['train', 'test']: 49 | df = h5py.File('EGshuffled_%s.h5' % ky, 'w') 50 | 51 | # input images 52 | in_dat = datsets[ky][0] 53 | df.create_dataset('input', data=in_dat) 54 | df['input'].attrs['lshape'] = (1, 25, 25, 25) # (C, H, W) 55 | 56 | target = datsets[ky][1].reshape((-1, 1)) # make it a 2D array 57 | df.create_dataset('output', data=target) 58 | df['output'].attrs['nclass'] = 1 59 | df.close() 60 | -------------------------------------------------------------------------------- /utils/gaussian.py: -------------------------------------------------------------------------------- 1 | # This program generates a dataset of guassian(only along x-axis) with some noise 2 | import matplotlib 3 | import matplotlib.pyplot as plt 4 | plt.switch_backend('Agg') 5 | from matplotlib import cm 6 | import numpy as np 7 | from scipy.stats import norm 8 | import numpy as np 9 | import h5py 10 | from mpl_toolkits.mplot3d import Axes3D 11 | import math 12 | mean1 = 10 13 | sigma1 = 2 14 | nevents =5 # events of one class. Total will be double 15 | mean2 = 10 16 | sigma2 = 4 17 | 18 | # Normalized guassian functions. Both give same value 19 | #def gaussian(x, mu, sig): 20 | # return 1./(math.sqrt(2.*math.pi)*sig)*np.exp(-np.power((x - mu)/sig, 2.)/2) 21 | 22 | def gaussian(x,x0,sigma): 23 | return np.exp(-np.power((x - x0)/sigma, 2.)/2.) 24 | 25 | x = np.arange(0, 25, 1) 26 | y = np.arange(0, 25, 1) 27 | z = np.arange(0, 25, 1) 28 | 29 | Z = np.zeros(shape=(nevents*2, 25, 25, 25)) 30 | T = np.zeros(shape=(nevents*2,1)) 31 | index = 0 32 | for i in range(nevents): 33 | noise = np.random.normal(0, 0.01, 25) 34 | B = gaussian(x, mean1, sigma1)+ noise 35 | B = 80 * B 36 | Z[index,:,12,12]= B 37 | fig=plt.figure(index) 38 | fig=plt.plot(x, B) 39 | plt.title("Training data class1 along x_axis") 40 | plt.xlabel("Cell location along x_axis") 41 | plt.ylabel("Value") 42 | plt.savefig("figs/demo" + str(index) +".png") 43 | noise = np.random.normal(0, 0.01, 25) 44 | B = gaussian(x, mean2, sigma2)+ noise 45 | B = 80 * B 46 | index= index + 1 47 | T[index]= 1 48 | Z[index,:,12,12]= B 49 | fig=plt.figure(index) 50 | fig=plt.plot(x, B) 51 | plt.title("Training data class 2 along x_axis") 52 | plt.xlabel("Cell location along x_axis") 53 | plt.ylabel("Value") 54 | plt.savefig("figs/demo" + str(index) +".png") 55 | index= index+1 56 | print "mean of 1 = %f" %np.mean(Z[:,:,12,12]) 57 | print "sigma of 1= %f" %np.std(Z[:,:,12,12]) 58 | 59 | print(Z.shape) 60 | print(T.shape) 61 | 62 | h5f = h5py.File('data.h5', 'w') 63 | h5f.create_dataset('ECAL', data=Z, dtype='f8') 64 | h5f.create_dataset('TAG', data=T, dtype='i8') 65 | h5f.close() 66 | #fig = plt.figure(nevents1*2) 67 | #ax = fig.add_subplot(111, projection='3d') 68 | #a, b, c, d = Z.nonzero() 69 | #sp = ax.scatter(b, c, d, c=Z[0,b,c,d]) 70 | #plt.colorbar(sp) 71 | #plt.savefig("figs/3d2.png") 72 | -------------------------------------------------------------------------------- /keras/analysis/plot_loss.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | plt.switch_backend('Agg') 4 | 5 | try: 6 | import cPickle as pickle 7 | except ImportError: 8 | import pickle 9 | with open('lossweights.pkl', 'rb') as f: 10 | x = pickle.load(f) 11 | gen_test = np.asarray(x['test']['generator']) 12 | gen_train = np.asarray(x['train']['generator']) 13 | disc_test = np.asarray(x['test']['discriminator']) 14 | disc_train = np.asarray(x['train']['discriminator']) 15 | plt.figure(1) 16 | plt.subplot(221) 17 | plt.title('Training loss for Discriminator') 18 | plt.plot(disc_train[:,0], label='tot') 19 | plt.plot(disc_train[:,1], label='gen') 20 | plt.plot(disc_train[:,2], label='aux') 21 | plt.legend() 22 | plt.subplot(222) 23 | plt.title('\nTraining loss for Generator') 24 | plt.plot(gen_train[:,0], label='tot') 25 | plt.plot(gen_train[:,1], label='gen') 26 | plt.plot(gen_train[:,2], label='aux') 27 | plt.legend() 28 | plt.subplot(223) 29 | plt.title('\nTesting loss for Discriminator') 30 | plt.plot(disc_test[:,0], label='tot') 31 | plt.plot(disc_test[:,1], label='gen') 32 | plt.plot(disc_test[:,2], label='aux') 33 | plt.legend() 34 | plt.subplot(224) 35 | plt.title('\nTesting loss for Generator') 36 | plt.plot(gen_test[:,0], label='tot') 37 | plt.plot(gen_test[:,1], label='gen') 38 | plt.plot(gen_test[:,2], label='aux') 39 | plt.legend() 40 | plt.savefig('losses.pdf') 41 | 42 | plt.figure(2) 43 | plt.title('Training losses for GAN') 44 | plt.plot(disc_train[:,0], label='Disc tot', color='red') 45 | plt.plot(disc_train[:,1], label='Disc gen (Binary Cross Entropy)', color='green') 46 | plt.plot(disc_train[:,2], label='Disc aux (Mean Absolute Error)', color='blue') 47 | plt.plot(gen_train[:,0], label='GAN tot', color='red', linestyle='--') 48 | plt.plot(gen_train[:,1], label='GAN gen (Binary Cross Entropy)', color='green', linestyle='--') 49 | plt.plot(gen_train[:,2], label='GAN aux (Mean Absolute Error)', color='blue', linestyle='--') 50 | plt.legend() 51 | plt.xlabel('Epochs') 52 | plt.ylabel('Loss') 53 | plt.savefig('Combined_losses.pdf') 54 | -------------------------------------------------------------------------------- /keras/EnergyGan.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import h5py 3 | 4 | from h5py import File as HDF5File 5 | import numpy as np 6 | 7 | import keras.backend as K 8 | from keras.layers import (Input, Dense, Reshape, Flatten, Lambda, merge, 9 | Dropout, BatchNormalization, Activation, Embedding) 10 | from keras.layers.advanced_activations import LeakyReLU 11 | from keras.layers.convolutional import (UpSampling3D, Conv3D, ZeroPadding3D, 12 | AveragePooling3D) 13 | 14 | from keras.models import Model, Sequential 15 | 16 | K.set_image_dim_ordering('tf') 17 | 18 | 19 | def discriminator(): 20 | 21 | image = Input(shape=(25, 25, 25, 1)) 22 | 23 | x = Conv3D(32, 5, 5,5, border_mode='same')(image) 24 | x = LeakyReLU()(x) 25 | x = Dropout(0.2)(x) 26 | 27 | x = ZeroPadding3D((2, 2,2))(x) 28 | x = Conv3D(8, 5, 5, 5, border_mode='valid')(x) 29 | x = LeakyReLU()(x) 30 | x = BatchNormalization()(x) 31 | x = Dropout(0.2)(x) 32 | 33 | x = ZeroPadding3D((2, 2, 2))(x) 34 | x = Conv3D(8, 5, 5,5, border_mode='valid')(x) 35 | x = LeakyReLU()(x) 36 | x = BatchNormalization()(x) 37 | x = Dropout(0.2)(x) 38 | 39 | x = ZeroPadding3D((1, 1, 1))(x) 40 | x = Conv3D(8, 5, 5, 5, border_mode='valid')(x) 41 | x = LeakyReLU()(x) 42 | x = BatchNormalization()(x) 43 | x = Dropout(0.2)(x) 44 | 45 | x = AveragePooling3D((2, 2, 2))(x) 46 | h = Flatten()(x) 47 | 48 | dnn = Model(image, h) 49 | 50 | image = Input(shape=(25, 25, 25, 1)) 51 | 52 | dnn_out = dnn(image) 53 | 54 | 55 | fake = Dense(1, activation='sigmoid', name='generation')(dnn_out) 56 | aux = Dense(1, activation='linear', name='auxiliary')(dnn_out) 57 | 58 | Model(input=image, output=[fake, aux]).summary() 59 | return Model(input=image, output=[fake, aux]) 60 | 61 | def generator(latent_size=1024, return_intermediate=False): 62 | 63 | loc = Sequential([ 64 | Dense(64 * 7* 7, input_dim=latent_size), 65 | Reshape((7, 7,8, 8)), 66 | 67 | Conv3D(64, 6, 6, 8, border_mode='same', init='he_uniform'), 68 | LeakyReLU(), 69 | BatchNormalization(), 70 | UpSampling3D(size=(2, 2, 2)), 71 | 72 | ZeroPadding3D((2, 2, 0)), 73 | Conv3D(6, 6, 5, 8, init='he_uniform'), 74 | LeakyReLU(), 75 | BatchNormalization(), 76 | UpSampling3D(size=(2, 2, 3)), 77 | 78 | ZeroPadding3D((1,0,3)), 79 | Conv3D(6, 3, 3, 8, init='he_uniform'), 80 | LeakyReLU(), 81 | Conv3D(1, 2, 2, 2, bias=False, init='glorot_normal'), 82 | Activation('relu') 83 | ]) 84 | 85 | latent = Input(shape=(latent_size, )) 86 | 87 | fake_image = loc(latent) 88 | 89 | Model(input=[latent], output=fake_image).summary() 90 | return Model(input=[latent], output=fake_image) 91 | -------------------------------------------------------------------------------- /keras/gan3D.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import h5py 3 | 4 | from h5py import File as HDF5File 5 | import numpy as np 6 | 7 | import keras.backend as K 8 | from keras.layers import (Input, Dense, Reshape, Flatten, Lambda, merge, 9 | Dropout, BatchNormalization, Activation, Embedding) 10 | from keras.layers.advanced_activations import LeakyReLU 11 | from keras.layers.convolutional import (UpSampling3D, Conv3D, ZeroPadding3D, 12 | AveragePooling3D) 13 | 14 | from keras.models import Model, Sequential 15 | 16 | K.set_image_dim_ordering('tf') 17 | 18 | 19 | def discriminator(): 20 | 21 | image = Input(shape=(25, 25, 25, 1)) 22 | 23 | x = Conv3D(32, 5, 5,5, border_mode='same')(image) 24 | x = LeakyReLU()(x) 25 | x = Dropout(0.2)(x) 26 | 27 | x = ZeroPadding3D((2, 2,2))(x) 28 | x = Conv3D(8, 5, 5, 5, border_mode='valid')(x) 29 | x = LeakyReLU()(x) 30 | x = BatchNormalization()(x) 31 | x = Dropout(0.2)(x) 32 | 33 | x = ZeroPadding3D((2, 2, 2))(x) 34 | x = Conv3D(8, 5, 5,5, border_mode='valid')(x) 35 | x = LeakyReLU()(x) 36 | x = BatchNormalization()(x) 37 | x = Dropout(0.2)(x) 38 | 39 | x = ZeroPadding3D((1, 1, 1))(x) 40 | x = Conv3D(8, 5, 5, 5, border_mode='valid')(x) 41 | x = LeakyReLU()(x) 42 | x = BatchNormalization()(x) 43 | x = Dropout(0.2)(x) 44 | 45 | x = AveragePooling3D((2, 2, 2))(x) 46 | h = Flatten()(x) 47 | 48 | dnn = Model(image, h) 49 | 50 | image = Input(shape=(25, 25, 25, 1)) 51 | 52 | dnn_out = dnn(image) 53 | 54 | 55 | fake = Dense(1, activation='sigmoid', name='generation')(dnn_out) 56 | aux = Dense(1, activation='sigmoid', name='auxiliary')(dnn_out) 57 | 58 | Model(input=image, output=[fake, aux]).summary() 59 | return Model(input=image, output=[fake, aux]) 60 | 61 | def generator(latent_size=200, return_intermediate=False): 62 | 63 | loc = Sequential([ 64 | Dense(64 * 7* 7, input_dim=latent_size), 65 | Reshape((7, 7,8, 8)), 66 | 67 | Conv3D(64, 6, 6, 8, border_mode='same', init='he_uniform'), 68 | LeakyReLU(), 69 | BatchNormalization(), 70 | UpSampling3D(size=(2, 2, 2)), 71 | 72 | ZeroPadding3D((2, 2, 0)), 73 | Conv3D(6, 6, 5, 8, init='he_uniform'), 74 | LeakyReLU(), 75 | BatchNormalization(), 76 | UpSampling3D(size=(2, 2, 3)), 77 | 78 | ZeroPadding3D((1,0,3)), 79 | Conv3D(6, 3, 3, 8, init='he_uniform'), 80 | LeakyReLU(), 81 | Conv3D(1, 2, 2, 2, bias=False, init='glorot_normal'), 82 | Activation('relu') 83 | ]) 84 | 85 | latent = Input(shape=(latent_size, )) 86 | 87 | image_class = Input(shape=(1, ), dtype='int32') 88 | emb = Flatten()(Embedding(2, latent_size, input_length=1, 89 | init='glorot_normal')(image_class)) 90 | 91 | h = merge([latent, emb], mode='mul') 92 | 93 | fake_image = loc(h) 94 | 95 | Model(input=[latent, image_class], output=fake_image).summary() 96 | return Model(input=[latent, image_class], output=fake_image) 97 | 98 | -------------------------------------------------------------------------------- /keras/EnergyGanEmbedding.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import h5py 3 | 4 | from h5py import File as HDF5File 5 | import numpy as np 6 | 7 | import keras.backend as K 8 | from keras.layers import (Input, Dense, Reshape, Flatten, Lambda, merge, 9 | Dropout, BatchNormalization, Activation, Embedding) 10 | from keras.layers.advanced_activations import LeakyReLU 11 | from keras.layers.convolutional import (UpSampling3D, Conv3D, ZeroPadding3D, 12 | AveragePooling3D) 13 | 14 | from keras.models import Model, Sequential 15 | 16 | K.set_image_dim_ordering('tf') 17 | 18 | 19 | def discriminator(): 20 | 21 | image = Input(shape=(25, 25, 25, 1)) 22 | 23 | x = Conv3D(32, 5, 5,5, border_mode='same')(image) 24 | x = LeakyReLU()(x) 25 | x = Dropout(0.2)(x) 26 | 27 | x = ZeroPadding3D((2, 2,2))(x) 28 | x = Conv3D(8, 5, 5, 5, border_mode='valid')(x) 29 | x = LeakyReLU()(x) 30 | x = BatchNormalization()(x) 31 | x = Dropout(0.2)(x) 32 | 33 | x = ZeroPadding3D((2, 2, 2))(x) 34 | x = Conv3D(8, 5, 5,5, border_mode='valid')(x) 35 | x = LeakyReLU()(x) 36 | x = BatchNormalization()(x) 37 | x = Dropout(0.2)(x) 38 | 39 | x = ZeroPadding3D((1, 1, 1))(x) 40 | x = Conv3D(8, 5, 5, 5, border_mode='valid')(x) 41 | x = LeakyReLU()(x) 42 | x = BatchNormalization()(x) 43 | x = Dropout(0.2)(x) 44 | 45 | x = AveragePooling3D((2, 2, 2))(x) 46 | h = Flatten()(x) 47 | 48 | dnn = Model(image, h) 49 | 50 | image = Input(shape=(25, 25, 25, 1)) 51 | 52 | dnn_out = dnn(image) 53 | 54 | 55 | fake = Dense(1, activation='sigmoid', name='generation')(dnn_out) 56 | aux = Dense(1, activation='LeakyReLU', name='auxiliary')(dnn_out) 57 | 58 | Model(input=image, output=[fake, aux]).summary() 59 | return Model(input=image, output=[fake, aux]) 60 | 61 | def generator(latent_size=200, return_intermediate=False): 62 | 63 | loc = Sequential([ 64 | Dense(64 * 7* 7, input_dim=latent_size), 65 | Reshape((7, 7,8, 8)), 66 | 67 | Conv3D(64, 6, 6, 8, border_mode='same', init='he_uniform'), 68 | LeakyReLU(), 69 | BatchNormalization(), 70 | UpSampling3D(size=(2, 2, 2)), 71 | 72 | ZeroPadding3D((2, 2, 0)), 73 | Conv3D(6, 6, 5, 8, init='he_uniform'), 74 | LeakyReLU(), 75 | BatchNormalization(), 76 | UpSampling3D(size=(2, 2, 3)), 77 | 78 | ZeroPadding3D((1,0,3)), 79 | Conv3D(6, 3, 3, 8, init='he_uniform'), 80 | LeakyReLU(), 81 | Conv3D(1, 2, 2, 2, bias=False, init='glorot_normal'), 82 | Activation('relu') 83 | ]) 84 | 85 | latent = Input(shape=(latent_size, )) 86 | 87 | image_class = Input(shape=(1, ), dtype='float32') 88 | emb = Flatten()(Embedding(500, latent_size, input_length=1, 89 | init='glorot_normal')(image_class)) 90 | 91 | h = merge([latent, emb], mode='mul') 92 | 93 | fake_image = loc(h) 94 | 95 | Model(input=[latent, image_class], output=fake_image).summary() 96 | return Model(input=[latent, image_class], output=fake_image) 97 | 98 | -------------------------------------------------------------------------------- /neon/gan3D.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | A GAN using 3D conv layers 4 | """ 5 | import os 6 | from datetime import datetime 7 | from neon.callbacks.callbacks import Callbacks, GANCostCallback 8 | from neon.callbacks.plotting_callbacks import GANPlotCallback 9 | from neon.initializers import Gaussian 10 | from neon.layers import GeneralizedGANCost, Sequential, Conv, Deconv, Dropout, Pooling 11 | from neon.layers.layer import Linear, Reshape 12 | from neon.layers.container import GenerativeAdversarial 13 | from neon.models.model import GAN 14 | from neon.transforms import Rectlin, Logistic, GANCost 15 | from neon.util.argparser import NeonArgparser 16 | from neon.util.persist import ensure_dirs_exist 17 | from neon.layers.layer import Dropout 18 | 19 | # parse the command line arguments 20 | parser = NeonArgparser(__doc__) 21 | parser.add_argument('--kbatch', type=int, default=1, 22 | help='number of data batches per noise batch in training') 23 | args = parser.parse_args() 24 | 25 | # load up the data set 26 | 27 | # setup weight initialization function 28 | init = Gaussian() 29 | 30 | # discriminiator using convolution layers 31 | lrelu = Rectlin(slope=0.1) # leaky relu for discriminator 32 | # sigmoid = Logistic() # sigmoid activation function 33 | conv1 = dict(init=init, batch_norm=False, activation=lrelu) # what's about BatchNorm Layer and batch_norm parameter? 34 | conv2 = dict(init=init, batch_norm=True, activation=lrelu, padding=2) 35 | conv3 = dict(init=init, batch_norm=True, activation=lrelu, padding=1) 36 | D_layers = [Conv((5, 5, 5, 32), **conv1), 37 | Dropout(keep = 0.8), 38 | 39 | Conv((5, 5, 5, 8), **conv2), 40 | Dropout(keep = 0.8), 41 | 42 | Conv((5, 5, 5, 8), **conv2), 43 | Dropout(keep = 0.8), 44 | 45 | Conv((5, 5, 5, 8), **conv3), 46 | Dropout(keep = 0.8), 47 | 48 | Pooling((2, 2, 2)), 49 | # what's about the Flatten Layer? 50 | Linear(1, init=init)] #what's about the activation function? 51 | 52 | 53 | # generator using covolution layers 54 | latent_size = 200 55 | relu = Rectlin(slope=0) # relu for generator 56 | conv4 = dict(init=init, batch_norm=True, activation=lrelu, dilation=dict(dil_h=2, dil_w=2, dil_d=2)) 57 | conv5 = dict(init=init, batch_norm=True, activation=lrelu, padding=dict(pad_h=2, pad_w=2, pad_d=0), dilation=dict(dil_h=2, dil_w=2, dil_d=3)) 58 | conv6 = dict(init=init, batch_norm=False, activation=lrelu, padding=dict(pad_h=1, pad_w=0, pad_d=3)) 59 | G_layers = [Linear(64 * 7 * 7, init=init), # what's about the input volume 60 | Reshape((7, 7, 8, 8)), 61 | Conv((6, 6, 8, 64), **conv4), 62 | Conv((6, 5, 8, 6), **conv5), 63 | Conv((3, 3, 8, 6), **conv6), 64 | Conv((2, 2, 2, 1), init=init, batch_norm=False, activation=relu)] 65 | # what's about Embedding 66 | 67 | layers = GenerativeAdversarial(generator=Sequential(G_layers, name="Generator"), 68 | discriminator=Sequential(D_layers, name="Discriminator")) 69 | 70 | # setup cost function as CrossEntropy 71 | cost = GeneralizedGANCost(costfunc=GANCost(func="modified")) 72 | -------------------------------------------------------------------------------- /keras/analysis/simple_generation.py: -------------------------------------------------------------------------------- 1 | 2 | import argparse 3 | import os, h5py 4 | import numpy as np 5 | import context 6 | #from context import EcalEnergyGan 7 | from EcalEnergyGan import discriminator as build_discriminator 8 | from EcalEnergyGan import generator as build_generator 9 | 10 | def get_parser(): 11 | parser = argparse.ArgumentParser( 12 | description='Generate 3D images', 13 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 14 | ) 15 | parser.add_argument('--nevents', '-n', action='store', type=int, 16 | default='1000', help='Number of events to simulate') 17 | parser.add_argument('--latent-size', action='store', type=int, default=200, 18 | help='size of random N(0, 1) latent space to sample') 19 | parser.add_argument('--nrun', action='store', type=int, default=1, 20 | help='size of random N(0, 1) latent space to sample') 21 | parser.add_argument('--gweights', '-g', action='store', type=str, 22 | default='params_generator_epoch_029.hdf5', help='path to generator weights file') 23 | parser.add_argument('--dweights', '-d', action='store', type=str, 24 | default='params_discriminator_epoch_029.hdf5', help='path to discriminator weights file') 25 | 26 | 27 | return parser 28 | 29 | parser = get_parser() 30 | results = parser.parse_args() 31 | n_events = results.nevents 32 | latent_space = results.latent_size 33 | gen_weights = results.gweights 34 | disc_weights = results.dweights 35 | #gen_weights='/data/svalleco/GAN/weights_gk/params_generator_epoch_029.hdf5' 36 | #disc_weights='/data/svalleco/GAN/weights_gk/params_discriminator_epoch_029.hdf5' 37 | outfile_name = 'generation{0:03d}.hdf5'.format(results.nrun) 38 | #if not ((os.path.isfile(gen_weights)) & (os.path.isfile(disc_weights))): 39 | # download from somewhere 40 | # but for now just load mine 41 | # gen_weights='params_generator_epoch_048.hdf5' 42 | # disc_weights='params_discriminator_epoch_048.hdf5' 43 | 44 | np.random.seed() 45 | g = build_generator(latent_space, return_intermediate=False) 46 | g.load_weights(gen_weights) 47 | 48 | noise = np.random.normal(0, 1, (n_events, latent_space)) 49 | sampled_energies = np.random.uniform(1, 5, (n_events, 1)) 50 | generator_ip = np.multiply(sampled_energies, noise) 51 | 52 | generated_images = g.predict(generator_ip, verbose=False, batch_size=128) 53 | print(generated_images.shape) 54 | def safe_mkdir(path): 55 | ''' 56 | Safe mkdir (i.e., don't create if already exists, 57 | and no violation of race conditions) 58 | ''' 59 | from os import makedirs 60 | from errno import EEXIST 61 | try: 62 | makedirs(path) 63 | except OSError as exception: 64 | if exception.errno != EEXIST: 65 | raise exception 66 | outdir = 'plots' 67 | safe_mkdir(outdir) 68 | 69 | #f = plt.figure(figsize=(6, 6)) 70 | bins = np.linspace(0, 1, 30) 71 | 72 | #_ = plt.hist(isreal, bins=bins, histtype='step', label='GAN', color='green') 73 | 74 | #plt.legend(ncol=2, mode='expand') 75 | #plt.xlabel('P(real)') 76 | #plt.ylabel('Number of events') 77 | #plt.ylim(ymax=4000) 78 | 79 | #plt.tight_layout() 80 | #plt.savefig(os.path.join('..', outdir, 'prob_real.pdf')) 81 | 82 | 83 | 84 | generated_images = np.squeeze(generated_images) 85 | 86 | with h5py.File(outfile_name,'w') as outfile: 87 | outfile.create_dataset('ECAL',data=generated_images) 88 | -------------------------------------------------------------------------------- /neon/energy_dataset.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import h5py 3 | 4 | from neon.data import NervanaDataIterator 5 | 6 | def temp_3Ddata(): 7 | 8 | f = h5py.File("/Users/svalleco/GAN/data/small_test.h5","r") 9 | #f = h5py.File("/Users/svalleco/GAN/data/Ele_v1_1_2.h5","r") 10 | data = f.get('ECAL') 11 | #dtag =f.get('TAG') 12 | xtr = np.array(data) 13 | print (xtr.shape) 14 | #labels = np.ones(xtr.shape[0]) 15 | dtag =f.get('target') 16 | temp = np.array(dtag[:,1]) 17 | print (temp.shape) 18 | aa=np.reshape(xtr, (xtr.shape[0], 25*25*25)) 19 | print (aa.shape) 20 | sumE = np.sum(aa, axis=(1)) 21 | labels=np.stack((np.array(dtag[:,1]),sumE),axis=1) 22 | #xtr=xtr[...,numpy.newaxis] 23 | #xtr=numpy.rollaxis(xtr,4,1) 24 | print labels.shape 25 | 26 | return aa.astype(np.float32), labels.astype(np.float32) 27 | #return xtr.reshape((xtr.shape[0], 25 * 25 * 25)).astype(np.float32), labels.astype(np.float32) 28 | 29 | def get_output(): 30 | f = h5py.File("/Users/svalleco/GAN/data/output_data.h5","r") 31 | data = f.get("dataset_1") 32 | x = np.array(data) 33 | return x.astype(np.float32) 34 | 35 | 36 | class EnergyData(NervanaDataIterator): 37 | def __init__(self, X, Y, lshape): 38 | self.X = X 39 | self.Y = Y 40 | self.shape = lshape 41 | self.start = 0 42 | self.ndata = self.X.shape[0] 43 | self.nfeatures =self.X.shape[1] #Nchannels*W*H*D 44 | self.nyfeatures =self.Y.shape[1] #Nchannels*W*H*D 45 | self.nbatches = int(self.ndata/self.be.bsz) 46 | self.dev_X = self.be.zeros((self.nfeatures, self.be.bsz)) 47 | self.dev_Y = self.be.zeros((self.nyfeatures, self.be.bsz)) # 3 targets: real/fake, primaryE, sumEcal 48 | 49 | def reset(self): 50 | self.start = 0 51 | 52 | def __iter__(self): 53 | # 3. loop through minibatches in the dataset 54 | for index in range(self.start, self.ndata, self.be.bsz): 55 | # 3a. grab the right slice from the numpy arrays 56 | inputs = self.X[index:(index+self.be.bsz),:] 57 | targets = self.Y[index:(index+self.be.bsz),:] 58 | 59 | # The arrays X and Y data are in shape (batch_size, num_features), 60 | # but the iterator needs to return data with shape (num_features, batch_size). 61 | # here we transpose the data, and then store it as a contiguous array. 62 | # numpy arrays need to be contiguous before being loaded onto the GPU. 63 | inputs = np.ascontiguousarray(inputs.T) 64 | targets = np.ascontiguousarray(targets.T) 65 | 66 | # here we test your implementation 67 | # your slice has to have the same shape as the GPU tensors you allocated 68 | assert inputs.shape == self.dev_X.shape, \ 69 | "inputs has shape {}, but dev_X is {}".format(inputs.shape, self.dev_X.shape) 70 | assert targets.shape == self.dev_Y.shape, \ 71 | "targets has shape {}, but dev_Y is {}".format(targets.shape, self.dev_Y.shape) 72 | 73 | # 3b. transfer from numpy arrays to device 74 | # - use the GPU memory buffers allocated previously, 75 | # and call the myTensorBuffer.set() function. 76 | self.dev_X.set(inputs) 77 | self.dev_Y.set(targets) 78 | 79 | # 3c. yield a tuple of the device tensors. 80 | # the first should of shape (num_features, batch_size) 81 | # the second should of shape (4, batch_size) 82 | yield (self.dev_X, self.dev_Y) 83 | -------------------------------------------------------------------------------- /keras/EcalEnergyGan.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import h5py 3 | 4 | from h5py import File as HDF5File 5 | import numpy as np 6 | 7 | import keras.backend as K 8 | from keras.layers import (Input, Dense, Reshape, Flatten, Lambda, merge, 9 | Dropout, BatchNormalization, Activation, Embedding) 10 | from keras.layers.advanced_activations import LeakyReLU 11 | from keras.layers.convolutional import (UpSampling3D, Conv3D, ZeroPadding3D, 12 | AveragePooling3D) 13 | 14 | from keras.models import Model, Sequential 15 | 16 | #K.set_image_dim_ordering('channels_first') 17 | 18 | def ecal_sum(image): 19 | sum = K.sum(image, axis=(2, 3, 4)) 20 | return sum 21 | 22 | 23 | def discriminator(keras_dformat='channels_last'): 24 | print (keras_dformat) 25 | if keras_dformat =='channels_last': 26 | dshape=(25, 25, 25,1) 27 | daxis=(1,2,3) 28 | else: 29 | dshape=(1, 25, 25, 25) 30 | daxis=(2,3,4) 31 | image = Input(shape=dshape) 32 | x = Conv3D(32, (5, 5,5), data_format=keras_dformat, padding='same')(image) 33 | x = LeakyReLU()(x) 34 | x = Dropout(0.2)(x) 35 | 36 | x = ZeroPadding3D((2, 2,2))(x) 37 | x = Conv3D(8, (5, 5, 5), data_format=keras_dformat, padding='valid')(x) 38 | x = LeakyReLU()(x) 39 | x = BatchNormalization()(x) 40 | x = Dropout(0.2)(x) 41 | 42 | x = ZeroPadding3D((2, 2, 2))(x) 43 | x = Conv3D(8, (5, 5,5), data_format=keras_dformat, padding='valid')(x) 44 | x = LeakyReLU()(x) 45 | x = BatchNormalization()(x) 46 | x = Dropout(0.2)(x) 47 | 48 | x = ZeroPadding3D((1, 1, 1))(x) 49 | x = Conv3D(8, (5, 5, 5), data_format=keras_dformat, padding='valid')(x) 50 | x = LeakyReLU()(x) 51 | x = BatchNormalization()(x) 52 | x = Dropout(0.2)(x) 53 | 54 | x = AveragePooling3D((2, 2, 2))(x) 55 | h = Flatten()(x) 56 | 57 | dnn = Model(image, h) 58 | 59 | #image = Input(shape=(25, 25, 25,1)) 60 | 61 | dnn_out = dnn(image) 62 | 63 | 64 | fake = Dense(1, activation='sigmoid', name='generation')(dnn_out) 65 | aux = Dense(1, activation='linear', name='auxiliary')(dnn_out) 66 | ecal = Lambda(lambda x: K.sum(x, axis=daxis))(image) 67 | Model(input=image, output=[fake, aux, ecal]).summary() 68 | return Model(input=image, output=[fake, aux, ecal]) 69 | 70 | def generator(latent_size=200, return_intermediate=False, keras_dformat='channels_last'): 71 | 72 | if keras_dformat =='channels_last': 73 | dim = (7,7,8,8) 74 | else: 75 | dim = (8, 7, 7,8) 76 | 77 | loc = Sequential([ 78 | Dense(64 * 7* 7, input_dim=latent_size), 79 | Reshape(dim), 80 | Conv3D(64, (6, 6, 8), data_format=keras_dformat, padding='same', kernel_initializer='he_uniform'), 81 | LeakyReLU(), 82 | BatchNormalization(), 83 | UpSampling3D(size=(2, 2, 2)), 84 | 85 | ZeroPadding3D((2, 2, 0)), 86 | Conv3D(6, (6, 5, 8), data_format=keras_dformat, kernel_initializer='he_uniform'), 87 | LeakyReLU(), 88 | BatchNormalization(), 89 | UpSampling3D(size=(2, 2, 3)), 90 | 91 | ZeroPadding3D((1,0,3)), 92 | Conv3D(6, (3, 3, 8), data_format=keras_dformat, kernel_initializer='he_uniform'), 93 | LeakyReLU(), 94 | Conv3D(1, (2, 2, 2), data_format=keras_dformat, use_bias=False, kernel_initializer='glorot_normal'), 95 | Activation('relu') 96 | 97 | ]) 98 | 99 | latent = Input(shape=(latent_size, )) 100 | 101 | fake_image = loc(latent) 102 | 103 | Model(input=[latent], output=fake_image).summary() 104 | return Model(input=[latent], output=fake_image) 105 | -------------------------------------------------------------------------------- /analysis/analysis3D.py: -------------------------------------------------------------------------------- 1 | 2 | import argparse 3 | import os, h5py 4 | import numpy as np 5 | 6 | from gan3D import discriminator as build_discriminator 7 | from gan3D import generator as build_generator 8 | 9 | def get_parser(): 10 | parser = argparse.ArgumentParser( 11 | description='Generate 3D images' 12 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 13 | ) 14 | parser.add_argument('--nevents', '-n', action='store', type=int, 15 | default='1000', help='Number of events to simulate') 16 | parser.add_argument('--latent-size', action='store', type=int, default=200, 17 | help='size of random N(0, 1) latent space to sample') 18 | parser.add_argument('--nrun', action='store', type=int, default=0, 19 | help='size of random N(0, 1) latent space to sample') 20 | 21 | 22 | return parser 23 | 24 | parser = get_parser() 25 | results = parser.parse_args() 26 | n_events = results.nevents 27 | latent_space = results.latent_size 28 | 29 | #gen_weights = 'gen_weights.hdf5' 30 | #disc_weights = 'disc_weights.hdf5' 31 | gen_weights='/data/svalleco/GAN/weights_gk/params_generator_epoch_029.hdf5' 32 | disc_weights='/data/svalleco/GAN/weights_gk/params_discriminator_epoch_029.hdf5' 33 | outfile_name = '/data/svalleco/GAN/generatedImages/generation{1:03d}.hdf5'.format(results.nrun) 34 | outfileE_name = '/data/svalleco/GAN/generatedImages/generationE{1:03d}.hdf5'.format(results.nrun) 35 | outfileG_name = '/data/svalleco/GAN/generatedImages/generationG{1:03d}.hdf5'.format(results.nrun) 36 | #if not ((os.path.isfile(gen_weights)) & (os.path.isfile(disc_weights))): 37 | # download from somewhere 38 | # but for now just load mine 39 | # gen_weights='params_generator_epoch_048.hdf5' 40 | # disc_weights='params_discriminator_epoch_048.hdf5' 41 | 42 | np.random.seed() 43 | g = build_generator(latent_space, return_intermediate=False) 44 | g.load_weights(gen_weights) 45 | 46 | noise = np.random.normal(0, 1, (n_events, latent_space)) 47 | sampled_labels = np.random.randint(0, 2, n_events) 48 | 49 | 50 | generated_images = g.predict( 51 | [noise, sampled_labels.reshape(-1, 1)], verbose=False, batch_size=100) 52 | print generated_images.shape 53 | d = build_discriminator() 54 | d.load_weights(disc_weights) 55 | 56 | isreal, aux_out = np.array( 57 | d.predict(generated_images, verbose=False, batch_size=100) 58 | ) 59 | #isreal_pythia, aux_out_pythia = np.array( 60 | # d.predict(np.expand_dims(real_images / 100, -1), verbose=False, batch_size=100) 61 | #) 62 | print aux_out.shape 63 | print isreal.shape 64 | def safe_mkdir(path): 65 | ''' 66 | Safe mkdir (i.e., don't create if already exists, 67 | and no violation of race conditions) 68 | ''' 69 | from os import makedirs 70 | from errno import EEXIST 71 | try: 72 | makedirs(path) 73 | except OSError as exception: 74 | if exception.errno != EEXIST: 75 | raise exception 76 | outdir = 'plots' 77 | safe_mkdir(outdir) 78 | 79 | #f = plt.figure(figsize=(6, 6)) 80 | bins = np.linspace(0, 1, 30) 81 | 82 | #_ = plt.hist(isreal, bins=bins, histtype='step', label='GAN', color='green') 83 | 84 | #plt.legend(ncol=2, mode='expand') 85 | #plt.xlabel('P(real)') 86 | #plt.ylabel('Number of events') 87 | #plt.ylim(ymax=4000) 88 | 89 | #plt.tight_layout() 90 | #plt.savefig(os.path.join('..', outdir, 'prob_real.pdf')) 91 | 92 | 93 | 94 | generated_images = (generated_images) * 100. 95 | generated_images = np.squeeze(generated_images) 96 | np.float64(generated_images) 97 | print generated_images.dtype 98 | fakeHCAL = np.zeros((n_events,5,5,60)) 99 | fakeTARGET = np.ones((n_events,1,5)) 100 | #print isreal 101 | print np.where(aux_out>0.8000) 102 | 103 | with h5py.File(outfile_name,'w') as outfile: 104 | outfile.create_dataset('ECAL',data=generated_images) 105 | outfile.create_dataset('HCAL',data=fakeHCAL) 106 | outfile.create_dataset('target',data=fakeTARGET) 107 | 108 | with h5py.File(outfileE_name,'w') as outfileE: 109 | outfileE.create_dataset('ECAL',data=generated_images[np.where(aux_out>0.8000)[0]]) 110 | outfileE.create_dataset('HCAL',data=fakeHCAL) 111 | outfileE.create_dataset('target',data=fakeTARGET) 112 | 113 | with h5py.File(outfileG_name,'w') as outfileG: 114 | outfileG.create_dataset('ECAL',data=generated_images[np.where(aux_out<0.2000)[0]]) 115 | outfileG.create_dataset('HCAL',data=fakeHCAL) 116 | outfileG.create_dataset('target',data=fakeTARGET) 117 | 118 | -------------------------------------------------------------------------------- /caffe/discriminator2.prototxt: -------------------------------------------------------------------------------- 1 | layer { 2 | name: "ECAL" 3 | type: "Input" 4 | top: "ECAL" 5 | input_param { 6 | shape { 7 | dim: 100 8 | dim: 1 9 | dim: 25 10 | dim: 25 11 | dim: 25 12 | } 13 | } 14 | } 15 | layer { 16 | name: "TAG" 17 | type: "Input" 18 | top: "TAG" 19 | input_param { 20 | shape { 21 | dim: 100 22 | } 23 | } 24 | } 25 | layer { 26 | name: "event" 27 | type: "Input" 28 | top: "event" 29 | input_param { 30 | shape { 31 | dim: 100 32 | dim: 1 33 | } 34 | } 35 | } 36 | layer { 37 | name: "conv1" 38 | type: "Convolution" 39 | bottom: "ECAL" 40 | top: "conv1" 41 | convolution_param { 42 | num_output: 32 43 | pad: 2 44 | kernel_size: 5 45 | weight_filler { 46 | type: "gaussian" 47 | std: 9.99999974738e-05 48 | } 49 | bias_filler { 50 | type: "constant" 51 | value: 0.0 52 | } 53 | engine: CAFFE 54 | } 55 | } 56 | layer { 57 | name: "relu1" 58 | type: "ReLU" 59 | bottom: "conv1" 60 | top: "relu1" 61 | relu_param { 62 | negative_slope: 0.300000011921 63 | engine: CAFFE 64 | } 65 | } 66 | layer { 67 | name: "drpout1" 68 | type: "Dropout" 69 | bottom: "relu1" 70 | top: "drpout1" 71 | dropout_param { 72 | dropout_ratio: 0.20000000298 73 | } 74 | } 75 | layer { 76 | name: "conv2" 77 | type: "Convolution" 78 | bottom: "drpout1" 79 | top: "conv2" 80 | convolution_param { 81 | num_output: 8 82 | pad: 2 83 | kernel_size: 5 84 | weight_filler { 85 | type: "gaussian" 86 | std: 9.99999974738e-05 87 | } 88 | bias_filler { 89 | type: "constant" 90 | value: 0.0 91 | } 92 | engine: CAFFE 93 | } 94 | } 95 | layer { 96 | name: "relu2" 97 | type: "ReLU" 98 | bottom: "conv2" 99 | top: "relu2" 100 | relu_param { 101 | negative_slope: 0.300000011921 102 | engine: CAFFE 103 | } 104 | } 105 | layer { 106 | name: "bn2" 107 | type: "BatchNorm" 108 | bottom: "relu2" 109 | top: "relu2" 110 | } 111 | layer { 112 | name: "drpout2" 113 | type: "Dropout" 114 | bottom: "relu2" 115 | top: "drpout2" 116 | dropout_param { 117 | dropout_ratio: 0.20000000298 118 | } 119 | } 120 | layer { 121 | name: "conv3" 122 | type: "Convolution" 123 | bottom: "drpout2" 124 | top: "conv3" 125 | convolution_param { 126 | num_output: 8 127 | pad: 2 128 | kernel_size: 5 129 | engine: CAFFE 130 | } 131 | } 132 | layer { 133 | name: "relu3" 134 | type: "ReLU" 135 | bottom: "conv3" 136 | top: "relu3" 137 | relu_param { 138 | negative_slope: 0.300000011921 139 | engine: CAFFE 140 | } 141 | } 142 | layer { 143 | name: "bn3" 144 | type: "BatchNorm" 145 | bottom: "relu3" 146 | top: "relu3" 147 | } 148 | layer { 149 | name: "drpout3" 150 | type: "Dropout" 151 | bottom: "relu3" 152 | top: "drpout3" 153 | dropout_param { 154 | dropout_ratio: 0.20000000298 155 | } 156 | } 157 | layer { 158 | name: "conv4" 159 | type: "Convolution" 160 | bottom: "drpout3" 161 | top: "conv4" 162 | convolution_param { 163 | num_output: 8 164 | pad: 1 165 | kernel_size: 5 166 | stride: 2 167 | engine: CAFFE 168 | } 169 | } 170 | layer { 171 | name: "relu4" 172 | type: "ReLU" 173 | bottom: "conv4" 174 | top: "relu4" 175 | relu_param { 176 | negative_slope: 0.300000011921 177 | engine: CAFFE 178 | } 179 | } 180 | layer { 181 | name: "bn4" 182 | type: "BatchNorm" 183 | bottom: "relu4" 184 | top: "relu4" 185 | } 186 | layer { 187 | name: "drpout4" 188 | type: "Dropout" 189 | bottom: "relu4" 190 | top: "drpout4" 191 | dropout_param { 192 | dropout_ratio: 0.20000000298 193 | } 194 | } 195 | layer { 196 | name: "tag" 197 | type: "InnerProduct" 198 | bottom: "drpout4" 199 | top: "tag" 200 | inner_product_param { 201 | num_output: 1 202 | } 203 | } 204 | layer { 205 | name: "tag_loss" 206 | type: "SigmoidCrossEntropyLoss" 207 | bottom: "tag" 208 | bottom: "TAG" 209 | top: "tag_loss" 210 | loss_weight: 0.5 211 | } 212 | layer { 213 | name: "aux" 214 | type: "InnerProduct" 215 | bottom: "drpout4" 216 | top: "aux" 217 | inner_product_param { 218 | num_output: 1 219 | } 220 | } 221 | layer { 222 | name: "aux_loss" 223 | type: "SigmoidCrossEntropyLoss" 224 | bottom: "aux" 225 | bottom: "event" 226 | top: "aux_loss" 227 | loss_weight: 0.5 228 | } 229 | layer { 230 | name: "loss" 231 | type: "Eltwise" 232 | bottom: "tag_loss" 233 | bottom: "aux_loss" 234 | top: "loss" 235 | eltwise_param { 236 | operation: SUM 237 | } 238 | } 239 | -------------------------------------------------------------------------------- /caffe/generator2.prototxt: -------------------------------------------------------------------------------- 1 | layer { 2 | name: "feat" 3 | type: "Input" 4 | top: "feat" 5 | input_param { 6 | shape { 7 | dim: 100 8 | dim: 200 9 | } 10 | } 11 | } 12 | layer { 13 | name: "clas" 14 | type: "Input" 15 | top: "clas" 16 | input_param { 17 | shape { 18 | dim: 100 19 | dim: 1 20 | } 21 | } 22 | } 23 | layer { 24 | name: "embed" 25 | type: "Embed" 26 | bottom: "clas" 27 | top: "embed" 28 | embed_param { 29 | num_output: 200 30 | input_dim: 2 31 | weight_filler { 32 | type: "xavier" 33 | } 34 | } 35 | } 36 | layer { 37 | name: "flat" 38 | type: "Flatten" 39 | bottom: "embed" 40 | top: "flat" 41 | } 42 | layer { 43 | name: "mult" 44 | type: "Eltwise" 45 | bottom: "flat" 46 | bottom: "feat" 47 | top: "mult" 48 | eltwise_param { 49 | operation: PROD 50 | } 51 | } 52 | layer { 53 | name: "Dense" 54 | type: "InnerProduct" 55 | bottom: "mult" 56 | top: "Dense" 57 | inner_product_param { 58 | num_output: 3136 59 | weight_filler { 60 | type: "msra" 61 | } 62 | } 63 | } 64 | layer { 65 | name: "resh" 66 | type: "Reshape" 67 | bottom: "Dense" 68 | top: "resh" 69 | reshape_param { 70 | shape { 71 | dim: 100 72 | dim: 7 73 | dim: 7 74 | dim: 8 75 | dim: 8 76 | } 77 | } 78 | } 79 | layer { 80 | name: "conv5" 81 | type: "Convolution" 82 | bottom: "resh" 83 | top: "conv5" 84 | convolution_param { 85 | num_output: 64 86 | pad: 2 87 | pad: 2 88 | pad: 3 89 | kernel_size: 6 90 | kernel_size: 6 91 | kernel_size: 8 92 | weight_filler { 93 | type: "gaussian" 94 | std: 9.99999974738e-05 95 | } 96 | bias_filler { 97 | type: "constant" 98 | value: 0.0 99 | } 100 | engine: CAFFE 101 | } 102 | } 103 | layer { 104 | name: "relu5" 105 | type: "ReLU" 106 | bottom: "conv5" 107 | top: "relu5" 108 | relu_param { 109 | negative_slope: 0.300000011921 110 | engine: CAFFE 111 | } 112 | } 113 | layer { 114 | name: "bn5" 115 | type: "BatchNorm" 116 | bottom: "relu5" 117 | top: "relu5" 118 | } 119 | layer { 120 | name: "upsmpl5" 121 | type: "Deconvolution" 122 | bottom: "relu5" 123 | top: "upsmpl5" 124 | convolution_param { 125 | num_output: 1 126 | pad: 1 127 | kernel_size: 4 128 | group: 1 129 | stride: 2 130 | } 131 | } 132 | layer { 133 | name: "conv4" 134 | type: "Convolution" 135 | bottom: "upsmpl5" 136 | top: "conv4" 137 | convolution_param { 138 | num_output: 6 139 | pad: 2 140 | pad: 2 141 | pad: 0 142 | kernel_size: 6 143 | kernel_size: 5 144 | kernel_size: 8 145 | weight_filler { 146 | type: "gaussian" 147 | std: 9.99999974738e-05 148 | } 149 | bias_filler { 150 | type: "constant" 151 | value: 0.0 152 | } 153 | engine: CAFFE 154 | } 155 | } 156 | layer { 157 | name: "relu4" 158 | type: "ReLU" 159 | bottom: "conv4" 160 | top: "relu4" 161 | relu_param { 162 | negative_slope: 0.300000011921 163 | engine: CAFFE 164 | } 165 | } 166 | layer { 167 | name: "bn4" 168 | type: "BatchNorm" 169 | bottom: "relu4" 170 | top: "relu4" 171 | } 172 | layer { 173 | name: "upsmpl4" 174 | type: "Deconvolution" 175 | bottom: "relu4" 176 | top: "upsmpl4" 177 | convolution_param { 178 | num_output: 1 179 | pad: 1 180 | kernel_size: 4 181 | kernel_size: 4 182 | kernel_size: 5 183 | group: 1 184 | stride: 2 185 | stride: 2 186 | stride: 3 187 | weight_filler { 188 | type: "gaussian" 189 | std: 9.99999974738e-05 190 | } 191 | bias_filler { 192 | type: "constant" 193 | value: 0.0 194 | } 195 | } 196 | } 197 | layer { 198 | name: "conv3" 199 | type: "Convolution" 200 | bottom: "upsmpl4" 201 | top: "conv3" 202 | convolution_param { 203 | num_output: 6 204 | pad: 1 205 | pad: 0 206 | pad: 3 207 | kernel_size: 3 208 | kernel_size: 3 209 | kernel_size: 8 210 | weight_filler { 211 | type: "gaussian" 212 | std: 9.99999974738e-05 213 | } 214 | bias_filler { 215 | type: "constant" 216 | value: 0.0 217 | } 218 | engine: CAFFE 219 | } 220 | } 221 | layer { 222 | name: "relu3" 223 | type: "ReLU" 224 | bottom: "conv3" 225 | top: "relu3" 226 | relu_param { 227 | negative_slope: 0.300000011921 228 | engine: CAFFE 229 | } 230 | } 231 | layer { 232 | name: "conv2" 233 | type: "Convolution" 234 | bottom: "relu3" 235 | top: "conv2" 236 | convolution_param { 237 | num_output: 1 238 | pad: 2 239 | pad: 0 240 | pad: 3 241 | kernel_size: 2 242 | kernel_size: 2 243 | kernel_size: 2 244 | weight_filler { 245 | type: "gaussian" 246 | std: 9.99999974738e-05 247 | } 248 | bias_filler { 249 | type: "constant" 250 | value: 0.0 251 | } 252 | engine: CAFFE 253 | } 254 | } 255 | layer { 256 | name: "generated" 257 | type: "ReLU" 258 | bottom: "conv2" 259 | top: "generated" 260 | relu_param { 261 | negative_slope: 0.300000011921 262 | engine: CAFFE 263 | } 264 | } 265 | -------------------------------------------------------------------------------- /neon/potential_gan.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime 3 | from neon.callbacks.callbacks import Callbacks, GANCostCallback 4 | #from neon.callbacks.plotting_callbacks import GANPlotCallback 5 | from neon.initializers import Gaussian 6 | from neon.layers import GeneralizedGANCost, Affine, Linear, Sequential, Conv, Deconv, Dropout, Pooling, BatchNorm 7 | from neon.layers.layer import Linear, Reshape 8 | from neon.layers.container import GenerativeAdversarial 9 | from neon.models.model import GAN, Model 10 | from neon.transforms import Rectlin, Logistic, GANCost, Tanh 11 | from neon.util.argparser import NeonArgparser 12 | from neon.util.persist import ensure_dirs_exist 13 | from neon.layers.layer import Dropout 14 | from neon.data.hdf5iterator import HDF5Iterator 15 | from neon.optimizers import GradientDescentMomentum, RMSProp, Adam 16 | from gen_data_norm import gen_rhs 17 | from neon.backends import gen_backend 18 | from temporary_utils import temp_3Ddata 19 | import numpy as np 20 | # import matplotlib.pyplot as plt 21 | import h5py 22 | 23 | # load up the data set 24 | 25 | print 'starting HDF5Iterator' 26 | train_set = HDF5Iterator('EGshuffled_train.h5') 27 | valid_set = HDF5Iterator('EGshuffled_test.h5') 28 | 29 | print 'train_set OK' 30 | #tate=lt.plot(X_train[0, 12]) 31 | #plt.savefigure('data_img.png') 32 | 33 | # setup weight initialization function 34 | init = Gaussian(scale=0.01) 35 | 36 | # discriminiator using convolution layers 37 | lrelu = Rectlin(slope=0.1) # leaky relu for discriminator 38 | # sigmoid = Logistic() # sigmoid activation function 39 | conv1 = dict(init=init, batch_norm=False, activation=lrelu, bias=init) 40 | conv2 = dict(init=init, batch_norm=False, activation=lrelu, padding=2, bias=init) 41 | conv3 = dict(init=init, batch_norm=False, activation=lrelu, padding=1, bias=init) 42 | D_layers = [ 43 | Conv((5, 5, 5, 32), **conv1), 44 | Dropout(keep = 0.8), 45 | Conv((5, 5, 5, 8), **conv2), 46 | BatchNorm(), 47 | Dropout(keep = 0.8), 48 | Conv((5, 5, 5, 8), **conv2), 49 | BatchNorm(), 50 | Dropout(keep = 0.8), 51 | Conv((5, 5, 5, 8), **conv3), 52 | BatchNorm(), 53 | Dropout(keep = 0.8), 54 | Pooling((2, 2, 2)), 55 | Affine(1024, init=init, activation=lrelu), 56 | BatchNorm(), 57 | Affine(1024, init=init, activation=lrelu), 58 | BatchNorm(), 59 | Affine(1, init=init, bias=init, activation=Logistic()) 60 | ] 61 | 62 | # generator using convolution layers 63 | init_gen = Gaussian(scale=0.001) 64 | relu = Rectlin(slope=0) # relu for generator 65 | pad1 = dict(pad_h=2, pad_w=2, pad_d=2) 66 | str1 = dict(str_h=2, str_w=2, str_d=2) 67 | conv1 = dict(init=init_gen, batch_norm=False, activation=lrelu, padding=pad1, strides=str1, bias=init_gen) 68 | pad2 = dict(pad_h=2, pad_w=2, pad_d=2) 69 | str2 = dict(str_h=2, str_w=2, str_d=2) 70 | conv2 = dict(init=init_gen, batch_norm=False, activation=lrelu, padding=pad2, strides=str2, bias=init_gen) 71 | pad3 = dict(pad_h=0, pad_w=0, pad_d=0) 72 | str3 = dict(str_h=1, str_w=1, str_d=1) 73 | conv3 = dict(init=init_gen, batch_norm=False, activation=Tanh(), padding=pad3, strides=str3, bias=init_gen) 74 | G_layers = [ 75 | Affine(1024, init=init_gen, bias=init_gen, activation=relu), 76 | BatchNorm(), 77 | Affine(8 * 7 * 7 * 7, init=init_gen, bias=init_gen), 78 | Reshape((8, 7, 7, 7)), 79 | Deconv((6, 6, 6, 6), **conv1), #14x14x14 80 | BatchNorm(), 81 | # Linear(5 * 14 * 14 * 14, init=init), 82 | # Reshape((5, 14, 14, 14)), 83 | Deconv((5, 5, 5, 64), **conv2), #27x27x27 84 | BatchNorm(), 85 | Conv((3, 3, 3, 1), **conv3) 86 | ] 87 | 88 | layers = GenerativeAdversarial(generator=Sequential(G_layers, name="Generator"), 89 | discriminator=Sequential(D_layers, name="Discriminator")) 90 | print 'layers defined' 91 | # setup optimizer 92 | # optimizer = RMSProp(learning_rate=1e-4, decay_rate=0.9, epsilon=1e-8) 93 | optimizer = GradientDescentMomentum(learning_rate=1e-3, momentum_coef = 0.9) 94 | #optimizer = Adam(learning_rate=1e-3) 95 | 96 | # setup cost function as Binary CrossEntropy 97 | cost = GeneralizedGANCost(costfunc=GANCost(func="wasserstein")) 98 | 99 | nb_epochs = 15 100 | latent_size = 200 101 | inb_classes = 2 102 | nb_test = 100 103 | 104 | # initialize model 105 | noise_dim = (latent_size) 106 | gan = GAN(layers=layers, noise_dim=noise_dim, k=5, wgan_param_clamp=0.9) 107 | 108 | # configure callbacks 109 | callbacks = Callbacks(gan, eval_set=valid_set) 110 | callbacks.add_callback(GANCostCallback()) 111 | #callbacks.add_save_best_state_callback("./best_state.pkl") 112 | 113 | print 'starting training' 114 | # run fit 115 | gan.fit(train_set, num_epochs=nb_epochs, optimizer=optimizer, 116 | cost=cost, callbacks=callbacks) 117 | 118 | # gan.save_params('our_gan.prm') 119 | 120 | x_new = np.random.randn(100, latent_size) 121 | inference_set = HDF5Iterator(x_new, None, nclass=2, lshape=(latent_size)) 122 | my_generator = Model(gan.layers.generator) 123 | my_generator.save_params('our_gen.prm') 124 | my_discriminator = Model(gan.layers.discriminator) 125 | my_discriminator.save_params('our_disc.prm') 126 | test = my_generator.get_outputs(inference_set) 127 | test = np.float32(test*max_elem + mean) 128 | test = test.reshape((100, 25, 25, 25)) 129 | 130 | print(test.shape, 'generator output') 131 | 132 | #plt.plot(test[0, :, 12, :]) 133 | # plt.savefigure('output_img.png') 134 | 135 | h5f = h5py.File('output_data.h5', 'w') 136 | h5f.create_dataset('dataset_1', data=test) 137 | 138 | -------------------------------------------------------------------------------- /caffe/creating_proto.py: -------------------------------------------------------------------------------- 1 | ####### This file creates the prototext for the net and solvers ######################################################################################## 2 | 3 | import sys 4 | caffe_root='/data/gkhattak/caffe/' 5 | sys.path.insert(0, caffe_root+'python') 6 | 7 | import caffe 8 | import numpy as np 9 | import time 10 | import os 11 | import sys 12 | 13 | import h5py 14 | from h5py import File as HDF5File 15 | 16 | from caffe import layers as L, params as P 17 | 18 | latent = 200 # latent vector dimension 19 | img = 25 # image size 20 | batch_size = 100 21 | num_clas = 2 22 | sub_nets = ('generator2', 'discriminator2', 'data2') 23 | 24 | ############ creating the data net ############################# 25 | data = caffe.NetSpec() 26 | data.ECAL, data.TAG = L.HDF5Data(batch_size = batch_size, source = "train.txt", ntop = 2) # train.txt is a text file containing the path to the training data folder 27 | with open('data2.prototxt', 'w') as f: 28 | f.write(str(data.to_proto())) 29 | 30 | ############ creating the generator net ######################## 31 | n = caffe.NetSpec() 32 | n.feat = L.Input(shape=dict(dim=[batch_size, latent])) # random array 33 | n.clas = L.Input(shape=dict(dim=[batch_size,1])) # array with classes 34 | n.embed = L.Embed(n.clas, input_dim=num_clas, num_output=latent, weight_filler=dict(type='xavier')) # class dependant embedding (xavier for glorot_normal in keras) 35 | n.flat = L.Flatten(n.embed) 36 | n.mult = L.Eltwise(n.flat, n.feat, operation=0) # 0 = multiplication mode 37 | n.Dense = L.InnerProduct(n.mult, num_output=7*7*8*8, weight_filler=dict(type='msra')) # 3136 38 | n.resh = L.Reshape(n.Dense, reshape_param ={'shape':{'dim':[100, 7, 7, 8, 8]}}) 39 | n.conv5 = L.Convolution(n.resh, num_output=64, kernel_size= [6, 6, 8], pad=[2, 2, 3], engine=1, weight_filler=dict(type='gaussian', std= 0.0001 ), bias_filler=dict(type='constant', value=0)) # (not working for nd) weight_filler=dict(type='msra') => keras he_uniform 40 | n.relu5 = L.ReLU(n.conv5, negative_slope=0.3, engine=1) 41 | n.bn5 = L.BatchNorm(n.relu5, in_place=True) 42 | n.upsmpl5 = L.Deconvolution(n.bn5, convolution_param=dict(num_output=1, group=1, kernel_size=4, stride = 2, pad=1)) #f=2, kernel_size:{{2*f- f%2}} stride:{{f}} num_output:{{C}} group:{{C}} pad:{{ceil((f-1)/2.)}} (gives error for nd) weight_filler: "bilinear" 43 | n.conv4 = L.Convolution(n.upsmpl5, num_output=6, kernel_size= [6, 5, 8], pad=[2, 2, 0], engine=1, weight_filler=dict(type='gaussian', std= 0.0001 ), bias_filler=dict(type='constant', value=0))# (not working for nd) weight_filler=dict(type='msra') => keras he_uniform 44 | n.relu4 = L.ReLU(n.conv4, negative_slope=0.3, engine=1) 45 | n.bn4 = L.BatchNorm(n.relu4, in_place=True) 46 | n.upsmpl4 = L.Deconvolution(n.bn4, convolution_param=dict(num_output=1, group=1, kernel_size=[4, 4, 5], stride = [2, 2, 3], pad=1, weight_filler=dict(type='gaussian', std= 0.0001 ), bias_filler=dict(type='constant', value=0))) # f = [2, 2, 3] 47 | n.conv3 = L.Convolution(n.upsmpl4, num_output=6, kernel_size= [3, 3, 8], pad=[1, 0, 3], engine=1, weight_filler=dict(type='gaussian', std= 0.0001 ), bias_filler=dict(type='constant', value=0)) # (not working for nd) weight_filler=dict(type='msra') => keras he_uniform 48 | n.relu3 = L.ReLU(n.conv3, negative_slope=0.3, engine=1) 49 | n.conv2 = L.Convolution(n.relu3, num_output=1, kernel_size= [2, 2, 2],pad = [2, 0, 3], engine=1, weight_filler=dict(type='gaussian', std= 0.0001 ), bias_filler=dict(type='constant', value=0)) # (not working for nd) weight_filler=dict(type='xavier') 50 | n.generated = L.ReLU(n.conv2, negative_slope=0.3, engine=1) 51 | #with open('generator2.prototxt', 'w') as f: # for GAN implementation 52 | # f.write('force_backward:true\n') 53 | with open('generator2.prototxt', 'w') as f: 54 | f.write(str(n.to_proto())) 55 | ############ creating the discriminator net ####################### 56 | n = caffe.NetSpec() 57 | n.ECAL = L.Input(shape=dict(dim=[batch_size, 1, img, img, img])) 58 | n.TAG = L.Input(shape=dict(dim=[batch_size, ])) # class 59 | n.event = L.Input(shape=dict(dim=[batch_size, 1])) # If real or gen 60 | n.conv1 = L.Convolution(n.ECAL, num_output=32, kernel_size= 5, pad=2, engine=1, weight_filler=dict(type='gaussian', std= 0.0001 ), bias_filler=dict(type='constant', value=0)) 61 | n.relu1 = L.ReLU(n.conv1, negative_slope=0.3, engine=1) 62 | n.drpout1 = L.Dropout(n.relu1, dropout_ratio= 0.2) 63 | n.conv2 = L.Convolution(n.drpout1, num_output=8, kernel_size= 5, pad=2, engine=1, weight_filler=dict(type='gaussian', std= 0.0001 ), bias_filler=dict(type='constant', value=0)) 64 | n.relu2 = L.ReLU(n.conv2, negative_slope=0.3, engine=1) 65 | n.bn2 = L.BatchNorm(n.relu2, in_place=True) 66 | n.drpout2 = L.Dropout(n.relu2, dropout_ratio= 0.2) 67 | n.conv3 = L.Convolution(n.drpout2, num_output=8, kernel_size= 5, pad=2, engine=1) 68 | n.relu3 = L.ReLU(n.conv3, negative_slope=0.3, engine=1) 69 | n.bn3 = L.BatchNorm(n.relu3, in_place=True) 70 | n.drpout3 = L.Dropout(n.bn3, dropout_ratio= 0.2) 71 | n.conv4 = L.Convolution(n.drpout3, num_output=8, kernel_size= 5, pad=1, stride= 2, engine=1) # used stride instead of average pooling 72 | n.relu4 = L.ReLU(n.conv4, negative_slope=0.3, engine=1) 73 | n.bn4 = L.BatchNorm(n.relu4, in_place=True) 74 | n.drpout4 = L.Dropout(n.bn4, dropout_ratio= 0.2) 75 | #n.pool4 = L.Pooling(n.drpout4, kernel_size=2, stride=1, pool=P.Pooling.AVE) 76 | n.tag = L.InnerProduct(n.drpout4, num_output = 1) 77 | n.tag_loss = L.SigmoidCrossEntropyLoss(n.tag, n.TAG, loss_weight=0.5) 78 | n.aux = L.InnerProduct(n.drpout4, num_output = 1) 79 | n.aux_loss = L.SigmoidCrossEntropyLoss(n.aux, n.event, loss_weight=0.5) 80 | n.loss = L.Eltwise(n.tag_loss, n.aux_loss, operation=1) # 1 = sum mode 81 | 82 | with open('discriminator2.prototxt', 'w') as f: 83 | f.write(str(n.to_proto())) 84 | 85 | 86 | ############################################################################# 87 | 88 | #make solvers 89 | with open ("solver_template3.prototxt", "r") as myfile: 90 | solver_template=myfile.read() 91 | 92 | for curr_net in sub_nets: 93 | with open("solver_%s.prototxt" % curr_net, "w") as myfile: 94 | myfile.write(solver_template.replace('@NET@', curr_net)) 95 | 96 | #initialize the nets 97 | #caffe.set_device(gpu_id) 98 | #caffe.set_mode_gpu() 99 | #generator = caffe.RMSPropSolver('solver_generator2.prototxt') 100 | #discriminator = caffe.RMSPropSolver('solver_discriminator2.prototxt') 101 | #data_reader = caffe.RMSPropSolver('solver_data2.prototxt') 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /keras/particle3Dtrain.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from __future__ import print_function 5 | 6 | from collections import defaultdict 7 | try: 8 | import cPickle as pickle 9 | except ImportError: 10 | import pickle 11 | 12 | import argparse 13 | import os 14 | os.environ['LD_LIBRARY_PATH'] = os.getcwd() 15 | from six.moves import range 16 | import sys 17 | 18 | from h5py import File as HDF5File 19 | import numpy as np 20 | from lcd_utils import lcd_3Ddata 21 | 22 | def bit_flip(x, prob=0.05): 23 | """ flips a int array's values with some probability """ 24 | x = np.array(x) 25 | selection = np.random.uniform(0, 1, x.shape) < prob 26 | x[selection] = 1 * np.logical_not(x[selection]) 27 | return x 28 | 29 | 30 | 31 | if __name__ == '__main__': 32 | 33 | 34 | import keras.backend as K 35 | 36 | K.set_image_dim_ordering('tf') 37 | 38 | from keras.layers import Input 39 | from keras.models import Model 40 | from keras.optimizers import Adadelta, Adam, RMSprop 41 | from keras.utils.generic_utils import Progbar 42 | from sklearn.cross_validation import train_test_split 43 | 44 | import tensorflow as tf 45 | config = tf.ConfigProto(log_device_placement=True) 46 | 47 | from gan3D import generator 48 | from gan3D import discriminator 49 | 50 | g_weights = 'params_generator_epoch_' 51 | d_weights = 'params_discriminator_epoch_' 52 | 53 | nb_epochs = 50 54 | batch_size = 100 55 | latent_size = 200 56 | verbose = 'true' 57 | 58 | nb_classes = 2 59 | 60 | adam_lr = 0.0002 61 | adam_beta_1 = 0.5 62 | 63 | print('[INFO] Building discriminator') 64 | discriminator=discriminator() 65 | discriminator.summary() 66 | discriminator.compile( 67 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 68 | optimizer=RMSprop(), 69 | loss=['binary_crossentropy', 'binary_crossentropy'] 70 | ) 71 | 72 | # build the generator 73 | print('[INFO] Building generator') 74 | generator=generator() 75 | generator.summary() 76 | generator.compile( 77 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 78 | optimizer=RMSprop(), 79 | loss='binary_crossentropy' 80 | ) 81 | 82 | image_class = Input(shape=(1, ), name='combined_aux', dtype='int32') 83 | latent = Input(shape=(latent_size, ), name='combined_z') 84 | 85 | fake = generator([latent, image_class]) 86 | 87 | discriminator.trainable = False 88 | fake, aux = discriminator(fake) 89 | combined = Model( 90 | input=[latent, image_class], 91 | output=[fake, aux], 92 | name='combined_model' 93 | ) 94 | combined.compile( 95 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 96 | optimizer=RMSprop(), 97 | loss=['binary_crossentropy', 'binary_crossentropy'] 98 | ) 99 | 100 | 101 | 102 | X, y = lcd_3Ddata() 103 | # remove unphysical values 104 | X[X < 1e-3] = 0 105 | 106 | X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.9) 107 | 108 | # tensorflow ordering 109 | X_train = np.expand_dims(X_train, axis=-1) 110 | X_test = np.expand_dims(X_test, axis=-1) 111 | 112 | nb_train, nb_test = X_train.shape[0], X_test.shape[0] 113 | 114 | X_train = X_train.astype(np.float32) / 100 115 | X_test = X_test.astype(np.float32) / 100 116 | 117 | train_history = defaultdict(list) 118 | test_history = defaultdict(list) 119 | 120 | for epoch in range(nb_epochs): 121 | print('Epoch {} of {}'.format(epoch + 1, nb_epochs)) 122 | 123 | nb_batches = int(X_train.shape[0] / batch_size) 124 | if verbose: 125 | progress_bar = Progbar(target=nb_batches) 126 | 127 | epoch_gen_loss = [] 128 | epoch_disc_loss = [] 129 | 130 | for index in range(nb_batches): 131 | if verbose: 132 | progress_bar.update(index) 133 | else: 134 | if index % 100 == 0: 135 | print('processed {}/{} batches'.format(index + 1, nb_batches)) 136 | 137 | noise = np.random.normal(0, 1, (batch_size, latent_size)) 138 | 139 | image_batch = X_train[index * batch_size:(index + 1) * batch_size] 140 | label_batch = y_train[index * batch_size:(index + 1) * batch_size] 141 | 142 | sampled_labels = np.random.randint(0, nb_classes, batch_size) 143 | 144 | generated_images = generator.predict( 145 | [noise, sampled_labels.reshape((-1, 1))], verbose=0) 146 | 147 | real_batch_loss = discriminator.train_on_batch( 148 | image_batch, [bit_flip(np.ones(batch_size)), label_batch] 149 | ) 150 | 151 | fake_batch_loss = discriminator.train_on_batch( 152 | generated_images, 153 | [bit_flip(np.zeros(batch_size)), bit_flip(sampled_labels)] 154 | ) 155 | 156 | epoch_disc_loss.append([ 157 | (a + b) / 2 for a, b in zip(real_batch_loss, fake_batch_loss) 158 | ]) 159 | 160 | trick = np.ones(batch_size) 161 | 162 | gen_losses = [] 163 | 164 | for _ in range(2): 165 | noise = np.random.normal(0, 1, (batch_size, latent_size)) 166 | sampled_labels = np.random.randint(0, nb_classes, batch_size) 167 | 168 | gen_losses.append(combined.train_on_batch( 169 | [noise, sampled_labels.reshape((-1, 1))], 170 | [trick, bit_flip(sampled_labels, 0.09)] 171 | )) 172 | 173 | epoch_gen_loss.append([ 174 | (a + b) / 2 for a, b in zip(*gen_losses) 175 | ]) 176 | 177 | print('\nTesting for epoch {}:'.format(epoch + 1)) 178 | 179 | noise = np.random.normal(0, 1, (nb_test, latent_size)) 180 | 181 | sampled_labels = np.random.randint(0, nb_classes, nb_test) 182 | generated_images = generator.predict( 183 | [noise, sampled_labels.reshape((-1, 1))], verbose=False) 184 | 185 | X = np.concatenate((X_test, generated_images)) 186 | y = np.array([1] * nb_test + [0] * nb_test) 187 | aux_y = np.concatenate((y_test, sampled_labels), axis=0) 188 | 189 | discriminator_test_loss = discriminator.evaluate( 190 | X, [y, aux_y], verbose=False, batch_size=batch_size) 191 | 192 | discriminator_train_loss = np.mean(np.array(epoch_disc_loss), axis=0) 193 | 194 | noise = np.random.normal(0, 1, (2 * nb_test, latent_size)) 195 | sampled_labels = np.random.randint(0, nb_classes, 2 * nb_test) 196 | 197 | trick = np.ones(2 * nb_test) 198 | 199 | generator_test_loss = combined.evaluate( 200 | [noise, sampled_labels.reshape((-1, 1))], 201 | [trick, sampled_labels], verbose=False, batch_size=batch_size) 202 | 203 | generator_train_loss = np.mean(np.array(epoch_gen_loss), axis=0) 204 | 205 | train_history['generator'].append(generator_train_loss) 206 | train_history['discriminator'].append(discriminator_train_loss) 207 | 208 | test_history['generator'].append(generator_test_loss) 209 | test_history['discriminator'].append(discriminator_test_loss) 210 | 211 | print('{0:<22s} | {1:4s} | {2:15s} | {3:5s}'.format( 212 | 'component', *discriminator.metrics_names)) 213 | print('-' * 65) 214 | 215 | ROW_FMT = '{0:<22s} | {1:<4.2f} | {2:<15.2f} | {3:<5.2f}' 216 | print(ROW_FMT.format('generator (train)', 217 | *train_history['generator'][-1])) 218 | print(ROW_FMT.format('generator (test)', 219 | *test_history['generator'][-1])) 220 | print(ROW_FMT.format('discriminator (train)', 221 | *train_history['discriminator'][-1])) 222 | print(ROW_FMT.format('discriminator (test)', 223 | *test_history['discriminator'][-1])) 224 | 225 | # save weights every epoch 226 | generator.save_weights('{0}{1:03d}.hdf5'.format(g_weights, epoch), 227 | overwrite=True) 228 | discriminator.save_weights('{0}{1:03d}.hdf5'.format(d_weights, epoch), 229 | overwrite=True) 230 | 231 | pickle.dump({'train': train_history, 'test': test_history}, 232 | open('dcgan-history.pkl', 'wb')) 233 | 234 | -------------------------------------------------------------------------------- /caffe/caffe_train.py: -------------------------------------------------------------------------------- 1 | ######## This file creates the prototext for the net and solvers ######################################################################################## 2 | ######## The dcgan implementation is used from" https://github.com/samson-wang/dcgan.caffe " 3 | import sys 4 | caffe_root='/data/gkhattak/caffe/' 5 | sys.path.insert(0, caffe_root+'python') 6 | 7 | import caffe 8 | import numpy as np 9 | import time 10 | import os 11 | import sys 12 | 13 | import h5py 14 | from h5py import File as HDF5File 15 | 16 | from caffe import layers as L, params as P 17 | 18 | def bit_flip(x, prob=0.05): 19 | """ flips a int array's values with some probability """ 20 | x = np.array(x) 21 | selection = np.random.uniform(0, 1, x.shape) < prob 22 | x[selection] = 1 * np.logical_not(x[selection]) 23 | return x 24 | 25 | if len(sys.argv) == 1: 26 | start_snapshot = 0 27 | batch_size = 100 28 | latent = 200 # latent vector dimension 29 | max_iter = 34900 # maximum number of iterations 30 | display_every = 100 # show losses every so many iterations 31 | snapshot_every = 10000 # snapshot every so many iterations 32 | snapshot_folder = 'snapshots_test' # where to save the snapshots (and load from) 33 | snapshot_at_iter = -1 34 | snapshot_at_iter_file = 'snapshot_at_iter.txt' 35 | gpu_id = 0 36 | num_clas = 2 37 | lr = 0.01 38 | sub_nets = ('generator2', 'discriminator2', 'data2') 39 | 40 | # Manual SGD 41 | base_lr = 0.01 42 | momentum = 0.9 43 | weight_decay = 0.0005 44 | lr_w_mult = 1 45 | lr_b_mult = 0 46 | gamma = 0.1 47 | stepsize = 5000 48 | 49 | #initialize the nets 50 | #caffe.set_device(gpu_id) 51 | #caffe.set_mode_gpu() 52 | generator = caffe.RMSPropSolver('solver_generator2.prototxt') 53 | discriminator = caffe.RMSPropSolver('solver_discriminator2.prototxt') 54 | data_reader = caffe.RMSPropSolver('solver_data2.prototxt') 55 | 56 | momentum_hist ={} 57 | for layer in generator.net.params: 58 | m_w = np.zeros_like(generator.net.params[layer][0].data) 59 | m_b = np.zeros_like(generator.net.params[layer][1].data) 60 | momentum_hist[layer] = [m_w, m_b] 61 | 62 | #load from snapshot 63 | if start_snapshot: 64 | curr_snapshot_folder = snapshot_folder +'/' + str(start_snapshot) 65 | print >> sys.stderr, '\n === Starting from snapshot ' + curr_snapshot_folder + ' ===\n' 66 | generator_caffemodel = curr_snapshot_folder +'/' + 'generator.caffemodel' 67 | if os.path.isfile(generator_caffemodel): 68 | generator.net.copy_from(generator_caffemodel) 69 | else: 70 | raise Exception('File %s does not exist' % generator_caffemodel) 71 | discriminator_caffemodel = curr_snapshot_folder +'/' + 'discriminator.caffemodel' 72 | if os.path.isfile(discriminator_caffemodel): 73 | discriminator.net.copy_from(discriminator_caffemodel) 74 | else: 75 | raise Exception('File %s does not exist' % discriminator_caffemodel) 76 | 77 | 78 | #read weights of losses 79 | #discr_loss_weight = discriminator.net._blob_loss_weights # he used[discriminator.net._blob_names_index['discr_loss']] 80 | 81 | #do training 82 | start = time.time() 83 | #do training 84 | start = time.time() 85 | for it in range(start_snapshot,max_iter): 86 | # read the data 87 | data_reader.net.forward() 88 | 89 | # feed the data to the generator and run it 90 | noise = np.random.normal(0, 1, (batch_size, latent)).astype(np.float32) 91 | sampled_labels = np.random.randint(0, num_clas, batch_size) 92 | generator.net.blobs['feat'].data[...] = noise 93 | generator.net.blobs['clas'].data[...] = sampled_labels.reshape((-1, 1)) 94 | generator.net.forward() 95 | generated_img = generator.net.blobs['generated'].data 96 | 97 | # run the discriminator on real data 98 | discriminator.net.blobs['ECAL'].data[...] = data_reader.net.blobs['ECAL'].data 99 | discriminator.net.blobs['TAG'].data[...] = data_reader.net.blobs['TAG'].data 100 | discriminator.net.blobs['event'].data[...] = bit_flip(np.ones((batch_size,1), dtype='float32')) 101 | discriminator.net.forward() 102 | discr_real_loss = np.copy(discriminator.net.blobs['loss'].data) 103 | # discriminator.increment_iter() 104 | # discriminator.net.clear_param_diffs() 105 | # discriminator.net.backward() 106 | discriminator.step(1) 107 | #discriminator.apply_update() 108 | # run the discriminator on generated data 109 | discriminator.net.blobs['ECAL'].data[...] = generated_img 110 | discriminator.net.blobs['TAG'].data[...] = sampled_labels #.reshape((-1, 1)) 111 | discriminator.net.blobs['event'].data[...] = bit_flip(np.zeros((batch_size,1), dtype='float32')) 112 | discriminator.net.forward() 113 | discr_fake_loss = np.copy(discriminator.net.blobs['loss'].data) 114 | #discriminator.net.backward() 115 | # discriminator.rmsprop_update_gpu() 116 | #discriminator.update() 117 | discriminator.step(1) 118 | 119 | # run the discriminator on generated data with opposite labels, to get the gradient for the generator 120 | generator_loss =0 121 | #generator_loss = {} 122 | for i in range(2): 123 | # feed the data to the generator and run it 124 | noise = np.random.normal(0, 1, (batch_size, latent)).astype(np.float32) 125 | sampled_labels = np.random.randint(0, num_clas, batch_size) 126 | generator.net.blobs['feat'].data[...] = noise 127 | generator.net.blobs['clas'].data[...] = sampled_labels.reshape((-1, 1)) 128 | generator.net.forward() 129 | generated_img = generator.net.blobs['generated'].data 130 | discriminator.net.blobs['event'].data[...] =bit_flip( np.ones((batch_size,1), dtype='float32')) 131 | discriminator.net.forward() 132 | discr_fake_for_generator_loss = np.copy(discriminator.net.blobs['aux_loss'].data) 133 | generator_loss = generator_loss + discr_fake_for_generator_loss 134 | # generator.increment_iter() 135 | generator.net.clear_param_diffs() 136 | discriminator.net.backward() 137 | # Train the generator 138 | generator.net.blobs['generated'].diff[...] = discriminator.net.blobs['ECAL'].diff 139 | generator.net.backward() 140 | #generator.apply_update() 141 | #generator.step(1) 142 | #for layer in generator.net.layers: 143 | # for blob in layer.blobs: 144 | # blob.data[...] -= lr * blob.dif 145 | for layer in generator.net.params: 146 | momentum_hist[layer][0] = momentum_hist[layer][0] * momentum + (generator.net.params[layer][0].diff + weight_decay * 147 | generator.net.params[layer][0].data) * base_lr * lr_w_mult 148 | momentum_hist[layer][1] = momentum_hist[layer][1] * momentum + (generator.net.params[layer][1].diff + weight_decay * 149 | generator.net.params[layer][1].data) * base_lr * lr_b_mult 150 | generator.net.params[layer][0].data[...] -= momentum_hist[layer][0] 151 | generator.net.params[layer][1].data[...] -= momentum_hist[layer][1] 152 | generator.net.params[layer][0].diff[...] *= 0 153 | generator.net.params[layer][1].diff[...] *= 0 154 | base_lr = base_lr * np.power(gamma, (np.floor(it / stepsize))) 155 | 156 | # add by samson 157 | #display 158 | if it % display_every == 0 or it == max_iter: 159 | print >> sys.stderr, "[%s] Iteration %d: %f seconds" % (time.strftime("%c"), it, time.time()-start) 160 | print >> sys.stderr, " discr real loss: %f" % (discr_real_loss) 161 | print >> sys.stderr, " discr fake loss: %f" % (discr_fake_loss) 162 | print >> sys.stderr, " discr fake loss for generator: %f" % (discr_fake_for_generator_loss) 163 | print >> sys.stderr, " generator loss: %f" % (generator_loss) 164 | start = time.time() 165 | if os.path.isfile(snapshot_at_iter_file): 166 | with open (snapshot_at_iter_file, "r") as myfile: 167 | snapshot_at_iter = int(myfile.read()) 168 | 169 | #snapshot 170 | if it % snapshot_every == 0 or it == snapshot_at_iter or it == max_iter: 171 | curr_snapshot_folder = snapshot_folder +'/' + str(it) 172 | print >> sys.stderr, '\n === Saving snapshot to ' + curr_snapshot_folder + ' ===\n' 173 | if not os.path.exists(curr_snapshot_folder): 174 | os.makedirs(curr_snapshot_folder) 175 | generator_caffemodel = curr_snapshot_folder + '/' + 'generator.caffemodel' 176 | generator.net.save(generator_caffemodel) 177 | discriminator_caffemodel = curr_snapshot_folder + '/' + 'discriminator.caffemodel' 178 | discriminator.net.save(discriminator_caffemodel) 179 | -------------------------------------------------------------------------------- /keras/EnergyTrainEmbedding.py: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/bin/env python 3 | # -*- coding: utf-8 -*- 4 | 5 | from __future__ import print_function 6 | 7 | from collections import defaultdict 8 | try: 9 | import cPickle as pickle 10 | except ImportError: 11 | import pickle 12 | import keras 13 | import argparse 14 | import os 15 | os.environ['LD_LIBRARY_PATH'] = os.getcwd() 16 | from six.moves import range 17 | import sys 18 | 19 | import h5py 20 | import numpy as np 21 | from lcd_utils import lcd_3Ddata 22 | 23 | def bit_flip(x, prob=0.05): 24 | """ flips a int array's values with some probability """ 25 | x = np.array(x) 26 | selection = np.random.uniform(0, 1, x.shape) < prob 27 | x[selection] = 1 * np.logical_not(x[selection]) 28 | return x 29 | 30 | 31 | 32 | if __name__ == '__main__': 33 | 34 | 35 | import keras.backend as K 36 | 37 | K.set_image_dim_ordering('tf') 38 | 39 | from keras.layers import Input 40 | from keras.models import Model 41 | from keras.optimizers import Adadelta, Adam, RMSprop 42 | from keras.utils.generic_utils import Progbar 43 | from sklearn.cross_validation import train_test_split 44 | 45 | import tensorflow as tf 46 | config = tf.ConfigProto(log_device_placement=True) 47 | 48 | from EnergyGanEmbedding import generator, discriminator 49 | 50 | generator=generator() 51 | discriminator=discriminator() 52 | 53 | g_weights = 'params_generator_epoch_' 54 | d_weights = 'params_discriminator_epoch_' 55 | 56 | nb_epochs = 50 57 | batch_size = 100 58 | latent_size = 200 59 | verbose = 'true' 60 | 61 | nb_classes = 2 62 | 63 | adam_lr = 0.0002 64 | adam_beta_1 = 0.5 65 | 66 | print('[INFO] Building discriminator') 67 | discriminator.summary() 68 | discriminator.compile( 69 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 70 | optimizer=RMSprop(), 71 | loss=['binary_crossentropy', 'mean_absolute_percentage_error'] 72 | ) 73 | 74 | # build the generator 75 | print('[INFO] Building generator') 76 | generator.summary() 77 | generator.compile( 78 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 79 | optimizer=RMSprop(), 80 | loss='binary_crossentropy' 81 | ) 82 | 83 | image_class = Input(shape=(1, ), name='combined_aux', dtype='float32') 84 | latent = Input(shape=(latent_size, ), name='combined_z') 85 | 86 | fake = generator([latent, image_class]) 87 | 88 | discriminator.trainable = False 89 | fake, aux = discriminator(fake) 90 | combined = Model( 91 | input=[latent, image_class], 92 | output=[fake, aux], 93 | name='combined_model' 94 | ) 95 | combined.compile( 96 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 97 | optimizer=RMSprop(), 98 | loss=['binary_crossentropy', 'mean_absolute_percentage_error'] 99 | ) 100 | 101 | 102 | d=h5py.File("/afs/cern.ch/work/s/ssharan/ElectronEnergyFile.h5",'r') 103 | e=d.get('energy') 104 | X=np.array(d.get('ECAL')) 105 | x=np.array(e[:,0]) 106 | y=(np.array(x[:,1])) 107 | #print(X) 108 | #print('*************************************************************************************') 109 | #print(y) 110 | #print('*************************************************************************************') 111 | 112 | #Y=np.sum(X, axis=(1,2,3)) 113 | #print(Y) 114 | #print('*************************************************************************************') 115 | 116 | 117 | # remove unphysical values 118 | X[X < 1e-3] = 0 119 | 120 | X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.9) 121 | 122 | # tensorflow ordering 123 | X_train =np.array( np.expand_dims(X_train, axis=-1)) 124 | X_test = np.array(np.expand_dims(X_test, axis=-1)) 125 | y_train= np.array(y_train) 126 | y_test=np.array(y_test) 127 | #print(X_train) 128 | #print(X_test) 129 | #print(y_train) 130 | #print(y_test) 131 | #print('*************************************************************************************') 132 | 133 | 134 | nb_train, nb_test = X_train.shape[0], X_test.shape[0] 135 | 136 | X_train = X_train.astype(np.float32) 137 | X_test = X_test.astype(np.float32) 138 | #print(X_train) 139 | #print(X_test) 140 | train_history = defaultdict(list) 141 | test_history = defaultdict(list) 142 | 143 | for epoch in range(nb_epochs): 144 | print('Epoch {} of {}'.format(epoch + 1, nb_epochs)) 145 | 146 | nb_batches = int(X_train.shape[0] / batch_size) 147 | if verbose: 148 | progress_bar = Progbar(target=nb_batches) 149 | 150 | epoch_gen_loss = [] 151 | epoch_disc_loss = [] 152 | 153 | for index in range(nb_batches): 154 | if verbose: 155 | progress_bar.update(index) 156 | else: 157 | if index % 100 == 0: 158 | print('processed {}/{} batches'.format(index + 1, nb_batches)) 159 | 160 | noise = np.random.normal(0, 1, (batch_size, latent_size)) 161 | 162 | image_batch = X_train[index * batch_size:(index + 1) * batch_size] 163 | energy_batch = y_train[index * batch_size:(index + 1) * batch_size] 164 | 165 | sampled_energies = np.random.uniform(1, 500,( batch_size,1 )) 166 | generator_ip = [noise,sampled_energies] 167 | 168 | generated_images = generator.predict(generator_ip, verbose=0) 169 | 170 | # d,,isc_in_real=[image_batch, energy_batch] 171 | # disc_op_real=[np.ones(batch_size), energy_batch] 172 | # disc_in_fake=[generated_images, sampled_energies] 173 | # disc_op_fake=[np.zeros(batch_size),sampled_energies] 174 | # loss_weights=[np.ones(batch_size), 0.05 * np.ones(batch_size)] 175 | 176 | real_batch_loss = discriminator.train_on_batch(image_batch, [bit_flip(np.ones(batch_size)), energy_batch]) 177 | fake_batch_loss = discriminator.train_on_batch(generated_images, [bit_flip(np.zeros(batch_size)),sampled_energies]) 178 | # print(real_batch_loss) 179 | # print(fake_batch_loss) 180 | 181 | # fake_batch_loss = discriminator.train_on_batch(disc_in_fake, disc_op_fake, loss_weights) 182 | 183 | epoch_disc_loss.append([ 184 | (a + b) / 2 for a, b in zip(real_batch_loss, fake_batch_loss) 185 | ]) 186 | 187 | trick = np.ones(batch_size) 188 | 189 | gen_losses = [] 190 | 191 | for _ in range(2): 192 | noise = np.random.normal(0, 1, (batch_size, latent_size)) 193 | sampled_energies = np.random.uniform(1, 500, batch_size) 194 | 195 | gen_losses.append(combined.train_on_batch( 196 | [noise, sampled_energies], 197 | [trick, sampled_energies])) 198 | 199 | epoch_gen_loss.append([ 200 | (a + b) / 2 for a, b in zip(*gen_losses) 201 | ]) 202 | 203 | print('\nTesting for epoch {}:'.format(epoch + 1)) 204 | 205 | noise = np.random.normal(0, 1, (nb_test, latent_size)) 206 | 207 | sampled_energies = np.random.uniform(1, 500, nb_test) 208 | generated_images = generator.predict( 209 | [noise, sampled_energies.reshape((-1, 1))], verbose=False) 210 | 211 | X = np.concatenate((X_test, generated_images)) 212 | y = np.array([1] * nb_test + [0] * nb_test) 213 | aux_y = np.concatenate((y_test, sampled_energies), axis=0) 214 | 215 | discriminator_test_loss = discriminator.evaluate( 216 | X, [y, aux_y], verbose=False, batch_size=batch_size) 217 | 218 | discriminator_train_loss = np.mean(np.array(epoch_disc_loss), axis=0) 219 | 220 | noise = np.random.normal(0, 1, (2 * nb_test, latent_size)) 221 | sampled_energies = np.random.uniform(1, 500, 2 * nb_test) 222 | 223 | trick = np.ones(2 * nb_test) 224 | 225 | generator_test_loss = combined.evaluate( 226 | [noise, sampled_energies.reshape((-1, 1))], 227 | [trick, sampled_energies], verbose=False, batch_size=batch_size) 228 | 229 | generator_train_loss = np.mean(np.array(epoch_gen_loss), axis=0) 230 | 231 | train_history['generator'].append(generator_train_loss) 232 | train_history['discriminator'].append(discriminator_train_loss) 233 | 234 | test_history['generator'].append(generator_test_loss) 235 | test_history['discriminator'].append(discriminator_test_loss) 236 | 237 | print('{0:<22s} | {1:4s} | {2:15s} | {3:5s}'.format( 238 | 'component', *discriminator.metrics_names)) 239 | print('-' * 65) 240 | 241 | ROW_FMT = '{0:<22s} | {1:<4.2f} | {2:<15.2f} | {3:<5.2f}' 242 | print(ROW_FMT.format('generator (train)', 243 | *train_history['generator'][-1])) 244 | print(ROW_FMT.format('generator (test)', 245 | *test_history['generator'][-1])) 246 | print(ROW_FMT.format('discriminator (train)', 247 | *train_history['discriminator'][-1])) 248 | print(ROW_FMT.format('discriminator (test)', 249 | *test_history['discriminator'][-1])) 250 | 251 | # save weights every epoch 252 | generator.save_weights('{0}{1:03d}.hdf5'.format(g_weights, epoch), 253 | overwrite=True) 254 | discriminator.save_weights('{0}{1:03d}.hdf5'.format(d_weights, epoch), 255 | overwrite=True) 256 | 257 | pickle.dump({'train': train_history, 'test': test_history}, 258 | open('dcgan-history.pkl', 'wb')) 259 | -------------------------------------------------------------------------------- /keras/EnergyTrain.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from __future__ import print_function 5 | from collections import defaultdict 6 | try: 7 | import cPickle as pickle 8 | except ImportError: 9 | import pickle 10 | import keras 11 | import argparse 12 | import os 13 | os.environ['LD_LIBRARY_PATH'] = os.getcwd() 14 | from six.moves import range 15 | import sys 16 | 17 | import h5py 18 | import numpy as np 19 | def bit_flip(x, prob=0.05): 20 | """ flips a int array's values with some probability """ 21 | x = np.array(x) 22 | selection = np.random.uniform(0, 1, x.shape) < prob 23 | x[selection] = 1 * np.logical_not(x[selection]) 24 | return x 25 | 26 | if __name__ == '__main__': 27 | 28 | import keras.backend as K 29 | 30 | K.set_image_dim_ordering('tf') 31 | 32 | from keras.layers import Input 33 | from keras.models import Model 34 | from keras.optimizers import Adadelta, Adam, RMSprop 35 | from keras.utils.generic_utils import Progbar 36 | from sklearn.cross_validation import train_test_split 37 | 38 | import tensorflow as tf 39 | config = tf.ConfigProto(log_device_placement=True) 40 | 41 | from ecalvegan import generator 42 | from ecalvegan import discriminator 43 | 44 | g_weights = 'params_generator_epoch_' 45 | d_weights = 'params_discriminator_epoch_' 46 | 47 | nb_epochs = 30 48 | batch_size = 128 49 | latent_size = 200 50 | verbose = 'false' 51 | 52 | generator=generator(latent_size) 53 | discriminator=discriminator() 54 | 55 | nb_classes = 2 56 | 57 | print('[INFO] Building discriminator') 58 | discriminator.summary() 59 | #discriminator.load_weights('veganweights/params_discriminator_epoch_019.hdf5') 60 | discriminator.compile( 61 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 62 | optimizer=RMSprop(), 63 | loss=['binary_crossentropy', 'mean_absolute_percentage_error', 'mean_absolute_percentage_error'], 64 | loss_weights=[6, 0.1, 0.1] 65 | #loss=['binary_crossentropy', 'kullback_leibler_divergence'] 66 | ) 67 | 68 | # build the generator 69 | print('[INFO] Building generator') 70 | generator.summary() 71 | #generator.load_weights('veganweights/params_generator_epoch_019.hdf5') 72 | generator.compile( 73 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 74 | optimizer=RMSprop(), 75 | loss='binary_crossentropy' 76 | ) 77 | 78 | latent = Input(shape=(latent_size, ), name='combined_z') 79 | 80 | fake_image = generator( latent) 81 | 82 | discriminator.trainable = False 83 | fake, aux, ecal = discriminator(fake_image) 84 | combined = Model( 85 | input=[latent], 86 | output=[fake, aux, ecal], 87 | name='combined_model' 88 | ) 89 | combined.compile( 90 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 91 | optimizer=RMSprop(), 92 | loss=['binary_crossentropy', 'mean_absolute_percentage_error', 'mean_absolute_percentage_error'], 93 | loss_weights=[6, 0.1, 0.1] 94 | ) 95 | 96 | 97 | d=h5py.File("/afs/cern.ch/work/g/gkhattak/public/Ele_v1_1_2.h5",'r') 98 | e=d.get('target') 99 | X=np.array(d.get('ECAL')) 100 | y=(np.array(e[:,1])) 101 | print(X.shape) 102 | #print('*************************************************************************************') 103 | print(y) 104 | print('*************************************************************************************') 105 | 106 | #Y=np.sum(X, axis=(1,2,3)) 107 | #print(Y) 108 | #print('*************************************************************************************') 109 | 110 | 111 | # remove unphysical values 112 | X[X < 1e-6] = 0 113 | 114 | X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.9) 115 | 116 | # tensorflow ordering 117 | X_train =np.array(np.expand_dims(X_train, axis=-1)) 118 | X_test = np.array(np.expand_dims(X_test, axis=-1)) 119 | y_train= np.array(y_train)/100 120 | y_test=np.array(y_test)/100 121 | print(X_train.shape) 122 | print(X_test.shape) 123 | print(y_train.shape) 124 | print(y_test.shape) 125 | print('*************************************************************************************') 126 | 127 | 128 | nb_train, nb_test = X_train.shape[0], X_test.shape[0] 129 | 130 | X_train = X_train.astype(np.float32) 131 | X_test = X_test.astype(np.float32) 132 | y_train = y_train.astype(np.float32) 133 | y_test = y_test.astype(np.float32) 134 | ecal_train = np.sum(X_train, axis=(1, 2, 3)) 135 | ecal_test = np.sum(X_test, axis=(1, 2, 3)) 136 | 137 | print(X_train.shape) 138 | print(X_test.shape) 139 | print(ecal_train.shape) 140 | print(ecal_test.shape) 141 | print('*************************************************************************************') 142 | train_history = defaultdict(list) 143 | test_history = defaultdict(list) 144 | 145 | for epoch in range(nb_epochs): 146 | print('Epoch {} of {}'.format(epoch + 1, nb_epochs)) 147 | 148 | nb_batches = int(X_train.shape[0] / batch_size) 149 | if verbose: 150 | progress_bar = Progbar(target=nb_batches) 151 | 152 | epoch_gen_loss = [] 153 | epoch_disc_loss = [] 154 | for index in range(nb_batches): 155 | if verbose: 156 | progress_bar.update(index) 157 | else: 158 | if index % 100 == 0: 159 | print('processed {}/{} batches'.format(index + 1, nb_batches)) 160 | 161 | noise = np.random.normal(0, 1, (batch_size, latent_size)) 162 | 163 | image_batch = X_train[index * batch_size:(index + 1) * batch_size] 164 | energy_batch = y_train[index * batch_size:(index + 1) * batch_size] 165 | ecal_batch = ecal_train[index * batch_size:(index + 1) * batch_size] 166 | 167 | print(image_batch.shape) 168 | print(ecal_batch.shape) 169 | sampled_energies = np.random.uniform(0, 5,( batch_size,1 )) 170 | generator_ip = np.multiply(sampled_energies, noise) 171 | ecal_ip = np.multiply(2, sampled_energies) 172 | generated_images = generator.predict(generator_ip, verbose=0) 173 | 174 | # loss_weights=[np.ones(batch_size), 0.05 * np.ones(batch_size)] 175 | 176 | real_batch_loss = discriminator.train_on_batch(image_batch, [bit_flip(np.ones(batch_size)), energy_batch, ecal_batch]) 177 | fake_batch_loss = discriminator.train_on_batch(generated_images, [bit_flip(np.zeros(batch_size)), sampled_energies, ecal_ip]) 178 | # print(real_batch_loss) 179 | # print(fake_batch_loss) 180 | 181 | # fake_batch_loss = discriminator.train_on_batch(disc_in_fake, disc_op_fake, loss_weights) 182 | 183 | epoch_disc_loss.append([ 184 | (a + b) / 2 for a, b in zip(real_batch_loss, fake_batch_loss) 185 | ]) 186 | 187 | trick = np.ones(batch_size) 188 | 189 | gen_losses = [] 190 | 191 | for _ in range(2): 192 | noise = np.random.normal(0, 1, (batch_size, latent_size)) 193 | sampled_energies = np.random.uniform(0, 5, ( batch_size,1 )) 194 | generator_ip = np.multiply(sampled_energies, noise) 195 | ecal_ip = np.multiply(2, sampled_energies) 196 | 197 | gen_losses.append(combined.train_on_batch( 198 | [generator_ip], 199 | [trick, sampled_energies.reshape((-1, 1)), ecal_ip])) 200 | 201 | epoch_gen_loss.append([ 202 | (a + b) / 2 for a, b in zip(*gen_losses) 203 | ]) 204 | 205 | print('\nTesting for epoch {}:'.format(epoch + 1)) 206 | 207 | noise = np.random.normal(0, 1, (nb_test, latent_size)) 208 | 209 | sampled_energies = np.random.uniform(0, 5, (nb_test, 1)) 210 | generator_ip = np.multiply(sampled_energies, noise) 211 | generated_images = generator.predict(generator_ip, verbose=False) 212 | ecal_ip = np.multiply(2, sampled_energies) 213 | sampled_energies = np.squeeze(sampled_energies, axis=(1,)) 214 | X = np.concatenate((X_test, generated_images)) 215 | y = np.array([1] * nb_test + [0] * nb_test) 216 | ecal = np.concatenate((ecal_test, ecal_ip)) 217 | print(ecal.shape) 218 | print(y_test.shape) 219 | print(sampled_energies.shape) 220 | aux_y = np.concatenate((y_test, sampled_energies), axis=0) 221 | print(aux_y.shape) 222 | discriminator_test_loss = discriminator.evaluate( 223 | X, [y, aux_y, ecal], verbose=False, batch_size=batch_size) 224 | 225 | discriminator_train_loss = np.mean(np.array(epoch_disc_loss), axis=0) 226 | 227 | noise = np.random.normal(0, 1, (2 * nb_test, latent_size)) 228 | sampled_energies = np.random.uniform(1, 5, (2 * nb_test, 1)) 229 | generator_ip = np.multiply(sampled_energies, noise) 230 | ecal_ip = np.multiply(2, sampled_energies) 231 | 232 | trick = np.ones(2 * nb_test) 233 | 234 | generator_test_loss = combined.evaluate(generator_ip, 235 | [trick, sampled_energies.reshape((-1, 1)), ecal_ip], verbose=False, batch_size=batch_size) 236 | 237 | generator_train_loss = np.mean(np.array(epoch_gen_loss), axis=0) 238 | 239 | train_history['generator'].append(generator_train_loss) 240 | train_history['discriminator'].append(discriminator_train_loss) 241 | 242 | test_history['generator'].append(generator_test_loss) 243 | test_history['discriminator'].append(discriminator_test_loss) 244 | 245 | print('{0:<22s} | {1:4s} | {2:15s} | {3:5s}| {4:5s}'.format( 246 | 'component', *discriminator.metrics_names)) 247 | print('-' * 65) 248 | 249 | ROW_FMT = '{0:<22s} | {1:<4.2f} | {2:<15.2f} | {3:<5.2f}| {4:<5.2f}' 250 | print(ROW_FMT.format('generator (train)', 251 | *train_history['generator'][-1])) 252 | print(ROW_FMT.format('generator (test)', 253 | *test_history['generator'][-1])) 254 | print(ROW_FMT.format('discriminator (train)', 255 | *train_history['discriminator'][-1])) 256 | print(ROW_FMT.format('discriminator (test)', 257 | *test_history['discriminator'][-1])) 258 | 259 | # save weights every epoch 260 | generator.save_weights('veganweights/{0}{1:03d}.hdf5'.format(g_weights, epoch), 261 | overwrite=True) 262 | discriminator.save_weights('veganweights/{0}{1:03d}.hdf5'.format(d_weights, epoch), 263 | overwrite=True) 264 | 265 | pickle.dump({'train': train_history, 'test': test_history}, 266 | open('dcgan-history.pkl', 'wb')) 267 | -------------------------------------------------------------------------------- /keras/ParallelTest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from __future__ import print_function 5 | from collections import defaultdict 6 | try: 7 | import cPickle as pickle 8 | except ImportError: 9 | import pickle 10 | import keras 11 | import argparse 12 | import os 13 | os.environ['LD_LIBRARY_PATH'] = os.getcwd() 14 | from six.moves import range 15 | import sys 16 | import tensorflow 17 | from keras.utils.training_utils import multi_gpu_model 18 | import h5py 19 | import numpy as np 20 | def bit_flip(x, prob=0.05): 21 | """ flips a int array's values with some probability """ 22 | x = np.array(x) 23 | selection = np.random.uniform(0, 1, x.shape) < prob 24 | x[selection] = 1 * np.logical_not(x[selection]) 25 | return x 26 | 27 | if __name__ == '__main__': 28 | 29 | import keras.backend as K 30 | 31 | K.set_image_dim_ordering('tf') 32 | 33 | from keras.layers import Input 34 | from keras.models import Model 35 | from keras.optimizers import Adadelta, Adam, RMSprop 36 | from keras.utils.generic_utils import Progbar 37 | from sklearn.cross_validation import train_test_split 38 | 39 | import tensorflow as tf 40 | config = tf.ConfigProto(log_device_placement=True) 41 | 42 | from EcalEnergyGan import generator, discriminator 43 | 44 | g_weights = 'params_generator_epoch_' 45 | d_weights = 'params_discriminator_epoch_' 46 | 47 | nb_epochs = 30 48 | batch_size = 128 49 | latent_size = 200 50 | verbose = 'false' 51 | nb_classes = 2 52 | 53 | 54 | with tf.device('/cpu:0'): 55 | generator=generator(latent_size) 56 | discriminator=discriminator() 57 | 58 | 59 | 60 | 61 | 62 | 63 | print('[INFO] Building discriminator') 64 | #discriminator.summary() 65 | #discriminator.load_weights('veganweights/params_discriminator_epoch_019.hdf5') 66 | parallel_discr = multi_gpu_model(discriminator, gpus=1) 67 | 68 | #discriminator.compile( 69 | parallel_discr.compile( 70 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 71 | optimizer=RMSprop(), 72 | loss=['binary_crossentropy', 'mean_absolute_percentage_error', 'mean_absolute_percentage_error'], 73 | loss_weights=[6, 2, 1] 74 | #loss=['binary_crossentropy', 'kullback_leibler_divergence'] 75 | ) 76 | 77 | # build the generator 78 | print('[INFO] Building generator') 79 | #generator.summary() 80 | #generator.load_weights('veganweights/params_generator_epoch_019.hdf5') 81 | parallel_gener = multi_gpu_model(generator, gpus=1) 82 | #generator.compile( 83 | parallel_gener.compile( 84 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 85 | optimizer=RMSprop(), 86 | loss='binary_crossentropy' 87 | ) 88 | 89 | latent = Input(shape=(latent_size, ), name='combined_z') 90 | 91 | fake_image = generator( latent) 92 | 93 | discriminator.trainable = False 94 | fake, aux, ecal = discriminator(fake_image) 95 | combined = Model( 96 | input=[latent], 97 | output=[fake, aux, ecal], 98 | name='combined_model' 99 | ) 100 | parallel_comb = multi_gpu_model(combined, gpus=1) 101 | parallel_comb.compile( 102 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 103 | optimizer=RMSprop(), 104 | loss=['binary_crossentropy', 'mean_absolute_percentage_error', 'mean_absolute_percentage_error'], 105 | loss_weights=[5, 1, 1] 106 | ) 107 | 108 | 109 | d=h5py.File("/bigdata/shared/LCD/Pions_fullSpectrum/ChPiEscan_1_2.h5",'r') 110 | e=d.get('target') 111 | X=np.array(d.get('ECAL')) 112 | y=(np.array(e[:,1])) 113 | print(X.shape) 114 | #print('*************************************************************************************') 115 | print(y) 116 | print('*************************************************************************************') 117 | 118 | #Y=np.sum(X, axis=(1,2,3)) 119 | #print(Y) 120 | #print('*************************************************************************************') 121 | 122 | 123 | # remove unphysical values 124 | X[X < 1e-6] = 0 125 | 126 | X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.9) 127 | 128 | # tensorflow ordering 129 | X_train =np.array(np.expand_dims(X_train, axis=-1)) 130 | X_test = np.array(np.expand_dims(X_test, axis=-1)) 131 | y_train= np.array(y_train)/100 132 | y_test=np.array(y_test)/100 133 | print(X_train.shape) 134 | print(X_test.shape) 135 | print(y_train.shape) 136 | print(y_test.shape) 137 | print('*************************************************************************************') 138 | 139 | 140 | nb_train, nb_test = X_train.shape[0], X_test.shape[0] 141 | 142 | X_train = X_train.astype(np.float32) 143 | X_test = X_test.astype(np.float32) 144 | y_train = y_train.astype(np.float32) 145 | y_test = y_test.astype(np.float32) 146 | ecal_train = np.sum(X_train, axis=(1, 2, 3)) 147 | ecal_test = np.sum(X_test, axis=(1, 2, 3)) 148 | 149 | print(X_train.shape) 150 | print(X_test.shape) 151 | print(ecal_train.shape) 152 | print(ecal_test.shape) 153 | print('*************************************************************************************') 154 | train_history = defaultdict(list) 155 | test_history = defaultdict(list) 156 | 157 | for epoch in range(nb_epochs): 158 | print('Epoch {} of {}'.format(epoch + 1, nb_epochs)) 159 | 160 | nb_batches = int(X_train.shape[0] / batch_size) 161 | if verbose: 162 | progress_bar = Progbar(target=nb_batches) 163 | 164 | epoch_gen_loss = [] 165 | epoch_disc_loss = [] 166 | for index in range(nb_batches): 167 | if verbose: 168 | progress_bar.update(index) 169 | else: 170 | if index % 100 == 0: 171 | print('processed {}/{} batches'.format(index + 1, nb_batches)) 172 | 173 | noise = np.random.normal(0, 1, (batch_size, latent_size)) 174 | 175 | image_batch = X_train[index * batch_size:(index + 1) * batch_size] 176 | energy_batch = y_train[index * batch_size:(index + 1) * batch_size] 177 | ecal_batch = ecal_train[index * batch_size:(index + 1) * batch_size] 178 | 179 | print(image_batch.shape) 180 | print(ecal_batch.shape) 181 | sampled_energies = np.random.uniform(0, 5,( batch_size,1 )) 182 | generator_ip = np.multiply(sampled_energies, noise) 183 | ecal_ip = np.multiply(2, sampled_energies) 184 | generated_images = generator.predict(generator_ip, verbose=0) 185 | 186 | # loss_weights=[np.ones(batch_size), 0.05 * np.ones(batch_size)] 187 | 188 | real_batch_loss = discriminator.train_on_batch(image_batch, [bit_flip(np.ones(batch_size)), energy_batch, ecal_batch]) 189 | fake_batch_loss = discriminator.train_on_batch(generated_images, [bit_flip(np.zeros(batch_size)), sampled_energies, ecal_ip]) 190 | # print(real_batch_loss) 191 | # print(fake_batch_loss) 192 | 193 | # fake_batch_loss = discriminator.train_on_batch(disc_in_fake, disc_op_fake, loss_weights) 194 | 195 | epoch_disc_loss.append([ 196 | (a + b) / 2 for a, b in zip(real_batch_loss, fake_batch_loss) 197 | ]) 198 | 199 | trick = np.ones(batch_size) 200 | 201 | gen_losses = [] 202 | 203 | for _ in range(2): 204 | noise = np.random.normal(0, 1, (batch_size, latent_size)) 205 | sampled_energies = np.random.uniform(0, 5, ( batch_size,1 )) 206 | generator_ip = np.multiply(sampled_energies, noise) 207 | ecal_ip = np.multiply(2, sampled_energies) 208 | 209 | gen_losses.append(combined.train_on_batch( 210 | [generator_ip], 211 | [trick, sampled_energies.reshape((-1, 1)), ecal_ip])) 212 | 213 | epoch_gen_loss.append([ 214 | (a + b) / 2 for a, b in zip(*gen_losses) 215 | ]) 216 | # 217 | # print('\nTesting for epoch {}:'.format(epoch + 1)) 218 | 219 | # noise = np.random.normal(0, 1, (nb_test, latent_size)) 220 | 221 | # sampled_energies = np.random.uniform(0, 5, (nb_test, 1)) 222 | # generator_ip = np.multiply(sampled_energies, noise) 223 | # generated_images = generator.predict(generator_ip, verbose=False) 224 | # ecal_ip = np.multiply(2, sampled_energies) 225 | # sampled_energies = np.squeeze(sampled_energies, axis=(1,)) 226 | # X = np.concatenate((X_test, generated_images)) 227 | # y = np.array([1] * nb_test + [0] * nb_test) 228 | # ecal = np.concatenate((ecal_test, ecal_ip)) 229 | # print(ecal.shape) 230 | # print(y_test.shape) 231 | # print(sampled_energies.shape) 232 | # aux_y = np.concatenate((y_test, sampled_energies), axis=0) 233 | # print(aux_y.shape) 234 | # discriminator_test_loss = discriminator.evaluate( 235 | # X, [y, aux_y, ecal], verbose=False, batch_size=batch_size) 236 | # 237 | # discriminator_train_loss = np.mean(np.array(epoch_disc_loss), axis=0) 238 | 239 | # noise = np.random.normal(0, 1, (2 * nb_test, latent_size)) 240 | # sampled_energies = np.random.uniform(0, 5, (2 * nb_test, 1)) 241 | # generator_ip = np.multiply(sampled_energies, noise) 242 | # ecal_ip = np.multiply(2, sampled_energies) 243 | # 244 | # trick = np.ones(2 * nb_test) 245 | 246 | # generator_test_loss = combined.evaluate(generator_ip, 247 | # [trick, sampled_energies.reshape((-1, 1)), ecal_ip], verbose=False, batch_size=batch_size) 248 | 249 | # generator_train_loss = np.mean(np.array(epoch_gen_loss), axis=0) 250 | # 251 | # train_history['generator'].append(generator_train_loss) 252 | # train_history['discriminator'].append(discriminator_train_loss) 253 | # 254 | # test_history['generator'].append(generator_test_loss) 255 | # test_history['discriminator'].append(discriminator_test_loss) 256 | 257 | # print('{0:<22s} | {1:4s} | {2:15s} | {3:5s}| {4:5s}'.format( 258 | # 'component', *discriminator.metrics_names)) 259 | # print('-' * 65) 260 | # 261 | # ROW_FMT = '{0:<22s} | {1:<4.2f} | {2:<15.2f} | {3:<5.2f}| {4:<5.2f}' 262 | # print(ROW_FMT.format('generator (train)', 263 | # *train_history['generator'][-1])) 264 | # print(ROW_FMT.format('generator (test)', 265 | # *test_history['generator'][-1])) 266 | # print(ROW_FMT.format('discriminator (train)', 267 | # *train_history['discriminator'][-1])) 268 | # print(ROW_FMT.format('discriminator (test)', 269 | # *test_history['discriminator'][-1])) 270 | 271 | # save weights every epoch 272 | generator.save_weights('weights/{0}{1:03d}.hdf5'.format(g_weights, epoch), 273 | overwrite=True) 274 | discriminator.save_weights('weights/{0}{1:03d}.hdf5'.format(d_weights, epoch), 275 | overwrite=True) 276 | 277 | pickle.dump({'train': train_history, 'test': test_history}, 278 | open('pion-dcgan-history.pkl', 'wb')) 279 | -------------------------------------------------------------------------------- /keras/EcalEnergyTrain.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from __future__ import print_function 5 | from collections import defaultdict 6 | try: 7 | import cPickle as pickle 8 | except ImportError: 9 | import pickle 10 | import keras 11 | import argparse 12 | import os 13 | os.environ['LD_LIBRARY_PATH'] = os.getcwd() 14 | from six.moves import range 15 | import sys 16 | 17 | import h5py 18 | import numpy as np 19 | def bit_flip(x, prob=0.05): 20 | """ flips a int array's values with some probability """ 21 | x = np.array(x) 22 | selection = np.random.uniform(0, 1, x.shape) < prob 23 | x[selection] = 1 * np.logical_not(x[selection]) 24 | return x 25 | 26 | if __name__ == '__main__': 27 | 28 | import keras.backend as K 29 | 30 | #K.set_image_dim_ordering('channels_first') 31 | 32 | from keras.layers import Input 33 | from keras.models import Model 34 | from keras.optimizers import Adadelta, Adam, RMSprop 35 | from keras.utils.generic_utils import Progbar 36 | from sklearn.cross_validation import train_test_split 37 | 38 | import tensorflow as tf 39 | tf.flags.DEFINE_string("d", "/data/svalleco/Ele_v1_1_2.h5", "data file") 40 | tf.flags.DEFINE_integer("bs", 128, "inference batch size") 41 | tf.flags.DEFINE_integer("num_inter_threads", 1, "number of inter_threads") 42 | tf.flags.DEFINE_integer("num_intra_threads", 56, "number of intra_threads") 43 | tf.flags.DEFINE_integer("num_epochs", 2, "number of epochs") 44 | FLAGS = tf.flags.FLAGS 45 | 46 | session_config = tf.ConfigProto(log_device_placement=True, inter_op_parallelism_threads=FLAGS.num_inter_threads, intra_op_parallelism_threads=FLAGS.num_intra_threads) 47 | session = tf.Session(config=session_config) 48 | K.set_session(session) 49 | 50 | from EcalEnergyGan import generator, discriminator 51 | #from EcalEnergyGan_16f import generator, discriminator 52 | 53 | g_weights = 'params_generator_epoch_' 54 | d_weights = 'params_discriminator_epoch_' 55 | keras_dformat = 'channels_first' 56 | nb_epochs = 1 57 | batch_size = 128 58 | 59 | latent_size = 200 60 | verbose = 'false' 61 | 62 | generator=generator(latent_size,keras_dformat=keras_dformat) 63 | discriminator=discriminator(keras_dformat=keras_dformat) 64 | 65 | nb_classes = 2 66 | print (tf.__version__) 67 | print('[INFO] Building discriminator') 68 | discriminator.summary() 69 | discriminator.compile( 70 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 71 | optimizer=RMSprop(), 72 | loss=['binary_crossentropy', 'mean_absolute_percentage_error', 'mean_absolute_percentage_error'], 73 | loss_weights=[6, 0.2, 0.1] 74 | #loss=['binary_crossentropy', 'kullback_leibler_divergence'] 75 | ) 76 | 77 | # build the generator 78 | print('[INFO] Building generator') 79 | generator.summary() 80 | generator.compile( 81 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 82 | optimizer=RMSprop(), 83 | loss='binary_crossentropy' 84 | ) 85 | 86 | latent = Input(shape=(latent_size, ), name='combined_z') 87 | 88 | fake_image = generator( latent) 89 | 90 | discriminator.trainable = False 91 | fake, aux, ecal = discriminator(fake_image) 92 | combined = Model( 93 | input=[latent], 94 | output=[fake, aux, ecal], 95 | name='combined_model' 96 | ) 97 | combined.compile( 98 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 99 | optimizer=RMSprop(), 100 | loss=['binary_crossentropy', 'mean_absolute_percentage_error', 'mean_absolute_percentage_error'], 101 | loss_weights=[6, 0.2, 0.1] 102 | ) 103 | 104 | 105 | d=h5py.File(FLAGS.d,'r') 106 | e=d.get('target') 107 | X=np.array(d.get('ECAL')) 108 | y=(np.array(e[:,1])) 109 | print(X.shape) 110 | #print('*************************************************************************************') 111 | print(y) 112 | print('*************************************************************************************') 113 | 114 | #Y=np.sum(X, axis=(1,2,3)) 115 | #print(Y) 116 | #print('*************************************************************************************') 117 | 118 | 119 | # remove unphysical values 120 | X[X < 1e-6] = 0 121 | 122 | X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.9, test_size=0.1) 123 | 124 | # tensorflow ordering 125 | X_train =np.expand_dims(X_train, axis=-1) 126 | X_test = np.expand_dims(X_test, axis=-1) 127 | 128 | print(X_train.shape) 129 | if keras_dformat !='channels_last': 130 | X_train =np.moveaxis(X_train, -1, 1) 131 | X_test = np.moveaxis(X_test, -1,1) 132 | 133 | y_train= y_train/100 134 | y_test=y_test/100 135 | print(X_train.shape) 136 | print(X_test.shape) 137 | print(y_train.shape) 138 | print(y_test.shape) 139 | print('*************************************************************************************') 140 | 141 | 142 | nb_train, nb_test = X_train.shape[0], X_test.shape[0] 143 | 144 | X_train = X_train.astype(np.float32) 145 | X_test = X_test.astype(np.float32) 146 | y_train = y_train.astype(np.float32) 147 | y_test = y_test.astype(np.float32) 148 | if keras_dformat =='channels_last': 149 | ecal_train = np.sum(X_train, axis=(1, 2, 3)) 150 | ecal_test = np.sum(X_test, axis=(1, 2, 3)) 151 | else: 152 | ecal_train = np.sum(X_train, axis=(2, 3, 4)) 153 | ecal_test = np.sum(X_test, axis=(2, 3, 4)) 154 | 155 | print(X_train.shape) 156 | print(X_test.shape) 157 | print(ecal_train.shape) 158 | print(ecal_test.shape) 159 | print('*************************************************************************************') 160 | train_history = defaultdict(list) 161 | test_history = defaultdict(list) 162 | 163 | for epoch in range(nb_epochs): 164 | print('Epoch {} of {}'.format(epoch + 1, nb_epochs)) 165 | 166 | nb_batches = int(X_train.shape[0] / batch_size) 167 | if verbose: 168 | progress_bar = Progbar(target=nb_batches) 169 | 170 | epoch_gen_loss = [] 171 | epoch_disc_loss = [] 172 | for index in range(nb_batches): 173 | if verbose: 174 | progress_bar.update(index) 175 | else: 176 | if index % 100 == 0: 177 | print('processed {}/{} batches'.format(index + 1, nb_batches)) 178 | 179 | noise = np.random.normal(0, 1, (batch_size, latent_size)) 180 | 181 | image_batch = X_train[index * batch_size:(index + 1) * batch_size] 182 | energy_batch = y_train[index * batch_size:(index + 1) * batch_size] 183 | ecal_batch = ecal_train[index * batch_size:(index + 1) * batch_size] 184 | 185 | print(image_batch.shape) 186 | print(ecal_batch.shape) 187 | sampled_energies = np.random.uniform(1, 5,( batch_size,1 )) 188 | generator_ip = np.multiply(sampled_energies, noise) 189 | ecal_ip = np.multiply(2, sampled_energies) 190 | generated_images = generator.predict(generator_ip, verbose=0) 191 | 192 | # loss_weights=[np.ones(batch_size), 0.05 * np.ones(batch_size)] 193 | 194 | real_batch_loss = discriminator.train_on_batch(image_batch, [bit_flip(np.ones(batch_size)), energy_batch, ecal_batch]) 195 | fake_batch_loss = discriminator.train_on_batch(generated_images, [bit_flip(np.zeros(batch_size)), sampled_energies, ecal_ip]) 196 | # print(real_batch_loss) 197 | # print(fake_batch_loss) 198 | 199 | # fake_batch_loss = discriminator.train_on_batch(disc_in_fake, disc_op_fake, loss_weights) 200 | 201 | epoch_disc_loss.append([ 202 | (a + b) / 2 for a, b in zip(real_batch_loss, fake_batch_loss) 203 | ]) 204 | 205 | trick = np.ones(batch_size) 206 | 207 | gen_losses = [] 208 | 209 | for _ in range(2): 210 | noise = np.random.normal(0, 1, (batch_size, latent_size)) 211 | sampled_energies = np.random.uniform(1, 5, ( batch_size,1 )) 212 | generator_ip = np.multiply(sampled_energies, noise) 213 | ecal_ip = np.multiply(2, sampled_energies) 214 | 215 | gen_losses.append(combined.train_on_batch( 216 | [generator_ip], 217 | [trick, sampled_energies.reshape((-1, 1)), ecal_ip])) 218 | 219 | epoch_gen_loss.append([ 220 | (a + b) / 2 for a, b in zip(*gen_losses) 221 | ]) 222 | 223 | print('\nTesting for epoch {}:'.format(epoch + 1)) 224 | 225 | noise = np.random.normal(0, 1, (nb_test, latent_size)) 226 | 227 | sampled_energies = np.random.uniform(1, 5, (nb_test, 1)) 228 | generator_ip = np.multiply(sampled_energies, noise) 229 | generated_images = generator.predict(generator_ip, verbose=False) 230 | ecal_ip = np.multiply(2, sampled_energies) 231 | sampled_energies = np.squeeze(sampled_energies, axis=(1,)) 232 | X = np.concatenate((X_test, generated_images)) 233 | y = np.array([1] * nb_test + [0] * nb_test) 234 | ecal = np.concatenate((ecal_test, ecal_ip)) 235 | print(ecal.shape) 236 | print(y_test.shape) 237 | print(sampled_energies.shape) 238 | aux_y = np.concatenate((y_test, sampled_energies), axis=0) 239 | print(aux_y.shape) 240 | discriminator_test_loss = discriminator.evaluate( 241 | X, [y, aux_y, ecal], verbose=False, batch_size=batch_size) 242 | 243 | discriminator_train_loss = np.mean(np.array(epoch_disc_loss), axis=0) 244 | 245 | noise = np.random.normal(0, 1, (2 * nb_test, latent_size)) 246 | sampled_energies = np.random.uniform(1, 5, (2 * nb_test, 1)) 247 | generator_ip = np.multiply(sampled_energies, noise) 248 | ecal_ip = np.multiply(2, sampled_energies) 249 | 250 | trick = np.ones(2 * nb_test) 251 | 252 | generator_test_loss = combined.evaluate(generator_ip, 253 | [trick, sampled_energies.reshape((-1, 1)), ecal_ip], verbose=False, batch_size=batch_size) 254 | 255 | generator_train_loss = np.mean(np.array(epoch_gen_loss), axis=0) 256 | 257 | train_history['generator'].append(generator_train_loss) 258 | train_history['discriminator'].append(discriminator_train_loss) 259 | 260 | test_history['generator'].append(generator_test_loss) 261 | test_history['discriminator'].append(discriminator_test_loss) 262 | 263 | print('{0:<22s} | {1:4s} | {2:15s} | {3:5s}| {4:5s}'.format( 264 | 'component', *discriminator.metrics_names)) 265 | print('-' * 65) 266 | 267 | ROW_FMT = '{0:<22s} | {1:<4.2f} | {2:<15.2f} | {3:<5.2f}| {4:<5.2f}' 268 | print(ROW_FMT.format('generator (train)', 269 | *train_history['generator'][-1])) 270 | print(ROW_FMT.format('generator (test)', 271 | *test_history['generator'][-1])) 272 | print(ROW_FMT.format('discriminator (train)', 273 | *train_history['discriminator'][-1])) 274 | print(ROW_FMT.format('discriminator (test)', 275 | *test_history['discriminator'][-1])) 276 | 277 | # save weights every epoch 278 | generator.save_weights('{0}{1:03d}.hdf5'.format(g_weights, epoch), 279 | overwrite=True) 280 | discriminator.save_weights('{0}{1:03d}.hdf5'.format(d_weights, epoch), 281 | overwrite=True) 282 | 283 | pickle.dump({'train': train_history, 'test': test_history}, 284 | open('3dgan-history.pkl', 'wb')) 285 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /analysis/plot_root_vec.C: -------------------------------------------------------------------------------- 1 | #include "TROOT.h" 2 | #include "TFile.h" 3 | #include "TTree.h" 4 | #include "TBrowser.h" 5 | #include "TH2.h" 6 | #include "TRandom.h" 7 | #include 8 | void plot_root_vec() 9 | { 10 | std::vector *E = 0; 11 | std::vector *px =0; 12 | std::vector *py =0; 13 | std::vector *pz =0; 14 | 15 | std::vector *E_g4 = 0; 16 | std::vector *px_g4 =0; 17 | std::vector *py_g4 =0; 18 | std::vector *pz_g4 =0; 19 | 20 | // GANs output 21 | // TFile *f = new TFile("/home/svalleco/GAN/rootFiles/generationEdiscr000.root"); 22 | //TFile *f = new TFile("/home/svalleco/rootfiles/GeneratedEnergyRoot.root"); 23 | TFile *f = new TFile("/home/svalleco/rootfiles/generated100Rootfile.root"); 24 | TTree *t1 = (TTree*)f->Get("ecalTree"); 25 | 26 | t1->SetBranchAddress("x",&px); 27 | t1->SetBranchAddress("y",&py); 28 | t1->SetBranchAddress("z",&pz); 29 | t1->SetBranchAddress("E",&E); 30 | //Geant4 full sim 31 | //TFile *f_g4 = new TFile("/home/svalleco/GAN/test_nv08/testG.root"); 32 | //TFile *f_g4 = new TFile("Geant4.root"); 33 | //TFile *f_g4 = new TFile("/home/svalleco/rootfiles/ElectronEnergyRoot.root"); 34 | TFile *f_g4 = new TFile("/home/svalleco/rootfiles/EG100rootfile.root"); 35 | TTree *t1_g4 = (TTree*)f_g4->Get("ecalTree"); 36 | 37 | t1_g4->SetBranchAddress("x",&px_g4); 38 | t1_g4->SetBranchAddress("y",&py_g4); 39 | t1_g4->SetBranchAddress("z",&pz_g4); 40 | t1_g4->SetBranchAddress("E",&E_g4); 41 | 42 | TH1F *hx = new TH1F("hx","Ex distribution",25,0,25); 43 | TH1F *hy = new TH1F("hy","Ey distribution",25,0,25); 44 | TH1F *hz = new TH1F("hz","Ez distribution",25,0,25); 45 | TH1F *hx_g4 = new TH1F("hx_g4","Ex distribution",25,0,25); 46 | TH1F *hy_g4 = new TH1F("hy_g4","Ey distribution",25,0,25); 47 | TH1F *hz_g4 = new TH1F("hz_g4","Ez distribution",25,0,25); 48 | TH1F *hx_half = new TH1F("hx_half","Ex distribution",25,0,25); 49 | TH1F *hy_half = new TH1F("hy_half","Ey distribution",25,0,25); 50 | TH1F *hz_half = new TH1F("hz_half","Ez distribution",25,0,25); 51 | TH1F *hx_half_g4 = new TH1F("hx_half_g4","Ex distribution",25,0,25); 52 | TH1F *hy_half_g4 = new TH1F("hy_half_g4","Ey distribution",25,0,25); 53 | TH1F *hz_half_g4 = new TH1F("hz_half_g4","Ez distribution",25,0,25); 54 | TH3D *gan = new TH3D("gan","E distribution", 25,0,25,25,0,25,25,0,25); 55 | TH3D *g4 = new TH3D("g4","E distribution", 25,0,25,25,0,25,25,0,25); 56 | hx->Sumw2(); 57 | hy->Sumw2(); 58 | hz->Sumw2(); 59 | hx_g4->Sumw2(); 60 | hy_g4->Sumw2(); 61 | hz_g4->Sumw2(); 62 | hx_half->Sumw2(); 63 | hy_half->Sumw2(); 64 | hz_half->Sumw2(); 65 | hx_half_g4->Sumw2(); 66 | hy_half_g4->Sumw2(); 67 | hz_half_g4->Sumw2(); 68 | 69 | 70 | int SLICE = 9; 71 | TH1F* hx_short[SLICE]; 72 | TH1F* hy_short[SLICE]; 73 | TH1F* hx_g4_short[SLICE]; 74 | TH1F* hy_g4_short[SLICE]; 75 | 76 | for (int i=0;iSumw2(); 93 | hy_short[i]->Sumw2(); 94 | hx_g4_short[i]->Sumw2(); 95 | hy_g4_short[i]->Sumw2(); 96 | } 97 | 98 | int FLOOR = 9; 99 | int CELLX = 5; 100 | int CELLY = 5; 101 | int CELLZ = 5; 102 | TH1F* hE_gan[CELLX][CELLY][CELLZ]; 103 | TH1F* hE_g4[CELLX][CELLX][CELLZ]; 104 | int nEbins = 60; 105 | int low_Ebin = 0; 106 | int high_Ebin = 180; 107 | 108 | for (int i=0;iSumw2(); 122 | hE_g4[i][j][k]->Sumw2(); 123 | } 124 | } 125 | } 126 | 127 | Long64_t nentries = t1->GetEntries(); 128 | cout<<"GAN events : "<GetEntry(i); 131 | cout<<"entries OK"<size()<<" "<size()<<" "<size()<<" "<size()<size();ie++) { 135 | 136 | if (E->at(ie)>0.001) { 137 | hx->Fill(px->at(ie),E->at(ie)); 138 | hy->Fill(py->at(ie),E->at(ie)); 139 | hz->Fill(pz->at(ie),E->at(ie)); 140 | for (int is=0;isFill(px->at(ie),E->at(ie)); 142 | hy_short[is]->Fill(py->at(ie),E->at(ie)); 143 | } 144 | } 145 | if (pz->at(ie)==12 &&E->at(ie)>0.001) { 146 | hx_half->Fill(px->at(ie),E->at(ie)); 147 | hy_half->Fill(py->at(ie),E->at(ie)); 148 | hz_half->Fill(pz->at(ie),E->at(ie)); 149 | } 150 | if (pz->at(ie)>FLOOR && pz->at(ie)<(FLOOR+CELLZ+1) && px->at(ie)>FLOOR && px->at(ie)<(FLOOR+CELLX+1) &&py->at(ie)>FLOOR && py->at(ie)<(FLOOR+CELLY+1) &&E->at(ie)>0.001) { 151 | int idx = px->at(ie) -FLOOR - 1; 152 | int idy = py->at(ie) -FLOOR - 1; 153 | int idz = pz->at(ie) -FLOOR - 1; 154 | hE_gan[idx][idy][idz]->Fill(E->at(ie)); 155 | } 156 | } 157 | 158 | } 159 | 160 | Long64_t nentries_g4 = t1_g4->GetEntries(); 161 | cout<<"G4 entries: "<GetEntry(j); 164 | for (int je = 0;jesize();je++) { 165 | if (E_g4->at(je)>0) { 166 | hx_g4->Fill(px_g4->at(je),E_g4->at(je)); 167 | hy_g4->Fill(py_g4->at(je),E_g4->at(je)); 168 | hz_g4->Fill(pz_g4->at(je),E_g4->at(je)); 169 | for (int i=0;iFill(px_g4->at(je),E_g4->at(je)); 171 | hy_g4_short[i]->Fill(py_g4->at(je),E_g4->at(je)); 172 | } 173 | } 174 | if (pz_g4->at(je)==12 && E_g4->at(je)>0) { 175 | hx_half_g4->Fill(px_g4->at(je),E_g4->at(je)); 176 | hy_half_g4->Fill(py_g4->at(je),E_g4->at(je)); 177 | hz_half_g4->Fill(pz_g4->at(je),E_g4->at(je)); 178 | } 179 | if (pz_g4->at(je)>FLOOR && pz_g4->at(je)<(FLOOR+CELLZ+1) && px_g4->at(je)>FLOOR && px_g4->at(je)<(FLOOR+CELLX+1) &&py_g4->at(je)>FLOOR && py_g4->at(je)<(FLOOR+CELLY+1) &&E_g4->at(je)>0.001) { 180 | int idx = px_g4->at(je) -FLOOR - 1; 181 | int idy = py_g4->at(je) -FLOOR - 1; 182 | int idz = pz_g4->at(je) -FLOOR - 1; 183 | hE_g4[idx][idy][idz]->Fill(E_g4->at(je)); 184 | } 185 | } 186 | } 187 | //hx->Scale(hx_g4->Integral(0,24)/hx->Integral(0,24)); 188 | //hy->Scale(hy_g4->Integral(0,24)/hy->Integral(0,24)); 189 | // hz->Scale(hz_g4->Integral(0,24)/hz->Integral(0,24)); 190 | 191 | 192 | hx->SetLineColor(2); 193 | hy->SetLineColor(2); 194 | hz->SetLineColor(2); 195 | TCanvas *c1=new TCanvas("c1","",200,10,700,500); 196 | c1->cd(); 197 | hx->SetStats(0); 198 | hx->Draw(); 199 | hx_g4->Draw("SAME"); 200 | /* 201 | double probx = hx->Chi2Test(hx_g4,"WWPUF"); 202 | double kolx = hx->KolmogorovTest(hx_g4,"D"); 203 | c1->SaveAs("hx_lin_50epochs.C"); 204 | c1->SaveAs("hx_lin_50epochs.pdf"); 205 | */ 206 | TCanvas *c2=new TCanvas("c2","",200,10,700,500); 207 | c2->cd(); 208 | hy->SetStats(0); 209 | hy->Draw(); 210 | hy_g4->Draw("SAME"); 211 | /* 212 | double proby = hy->Chi2Test(hy_g4,"WWPUF"); 213 | double koly = hy->KolmogorovTest(hy_g4,"D"); 214 | c2->SaveAs("hy_lin_50epochs.C"); 215 | */ 216 | TCanvas *c3=new TCanvas("c3","",200,10,700,500); 217 | c3->cd(); 218 | hz->Draw(); 219 | hz_g4->Draw("SAMES"); 220 | /* 221 | double probz = hz->Chi2Test(hz_g4,"WWPUF"); 222 | double kolz = hz->KolmogorovTest(hz_g4,"D"); 223 | c3->SaveAs("hz_ele.C"); 224 | c3->SaveAs("hz_ele.pdf"); 225 | */ 226 | std::cout<<""<Chi2Test(hx_g4_short[i],"WWp"); 234 | double ppy = hy_short[i]->Chi2Test(hy_g4_short[i],"WWp"); 235 | } 236 | 237 | TCanvas *c4=new TCanvas("c4","",200,10,700,500); 238 | c4->cd(); 239 | double probE = hE_gan[2][2][2]->Chi2Test(hE_g4[2][2][2],"WWPUF"); 240 | double kolE = hE_gan[2][2][2]->KolmogorovTest(hE_g4[2][2][2],"D"); 241 | hE_gan[2][2][2]->Draw(); 242 | hE_g4[2][2][2]->Draw("SAMES"); 243 | 244 | TFile *fE =new TFile("Ehistos_gendiscr000_E.root","RECREATE"); 245 | 246 | TGraphAsymmErrors* Mgr = new TGraphAsymmErrors(); 247 | TGraphAsymmErrors* Sgr = new TGraphAsymmErrors(); 248 | int npoints = 0; 249 | for (int i=0;iSetLineColor(2); 253 | float Mgan = hE_gan[i][j][k]->GetMean(); 254 | float Sgan = hE_gan[i][j][k]->GetStdDev(); 255 | float Mgan_err = hE_gan[i][j][k]->GetMeanError(); 256 | float Sgan_err = hE_gan[i][j][k]->GetStdDevError(); 257 | float Mg4 = hE_g4[i][j][k]->GetMean(); 258 | float Sg4 = hE_g4[i][j][k]->GetStdDev(); 259 | float Mg4_err = hE_g4[i][j][k]->GetMeanError(); 260 | float Sg4_err = hE_g4[i][j][k]->GetStdDevError(); 261 | float Mratio = 0; 262 | float Mratio_err = 0; 263 | if (Mg4 !=0 && Mgan !=0) { 264 | Mratio = Mgan/Mg4; 265 | Mratio_err = Mratio*sqrt((Mgan_err*Mgan_err/(Mgan*Mgan)) + (Mg4_err*Mg4_err/(Mg4*Mg4))); 266 | } 267 | float Sratio = 0; 268 | float Sratio_err = 0; 269 | if (Sg4 !=0 && Sgan !=0) { 270 | Sratio = 1.2*Sgan/Sg4; 271 | Sratio_err = Sratio*sqrt((Sgan_err*Sgan_err/(Sgan*Sgan)) + (Sg4_err*Sg4_err/(Sg4*Sg4))); 272 | } 273 | Mgr->SetPoint(npoints, npoints, Mratio); 274 | Mgr->SetPointEYhigh(npoints,Mratio_err); 275 | Mgr->SetPointEYlow(npoints,Mratio_err); 276 | Sgr->SetPoint(npoints, npoints,Sratio); 277 | Sgr->SetPointEYhigh(npoints,Sratio_err); 278 | Sgr->SetPointEYlow(npoints,Sratio_err); 279 | npoints++; 280 | 281 | hE_gan[i][j][k]->Write(); 282 | hE_g4[i][j][k]->Write(); 283 | 284 | } 285 | TCanvas *c5=new TCanvas("c5","",200,10,700,500); 286 | c5->cd(); 287 | Mgr->SetMarkerColor(2); 288 | Mgr->SetMarkerStyle(21); 289 | Sgr->SetMarkerColor(3); 290 | Sgr->SetMarkerStyle(21); 291 | Mgr->Draw("ALP"); 292 | Mgr->SetName("Mgr"); 293 | c5->SaveAs("Mgr.C"); 294 | TCanvas *c6=new TCanvas("c6","",200,10,700,500); 295 | c6->cd(); 296 | Sgr->Draw("ALP"); 297 | Sgr->SetName("Sgr"); 298 | c6->SaveAs("Sgr.C"); 299 | Mgr->Write(); 300 | Sgr->Write(); 301 | hx->Write(); 302 | hy->Write(); 303 | hz->Write(); 304 | } 305 | -------------------------------------------------------------------------------- /utils/data_info.py: -------------------------------------------------------------------------------- 1 | 2 | import sys 3 | import os 4 | 5 | import h5py 6 | import numpy as np 7 | import matplotlib 8 | import matplotlib.pyplot as plt 9 | import matplotlib.cm as cm 10 | from matplotlib.colors import LogNorm, Normalize 11 | plt.switch_backend('Agg') 12 | data_file=0 13 | num_events= 500 14 | 15 | if data_file== 0: 16 | d=h5py.File("ElectronEnergyFile.h5",'r') 17 | X=np.array(d.get('ECAL')) 18 | e=d.get('energy') 19 | Y=(np.array(e[:,0, 1])) 20 | if data_file== 1: 21 | d=h5py.File("/afs/cern.ch/work/g/gkhattak/public/Ele_v1_1_2.h5",'r') 22 | X=np.array(d.get('ECAL')) 23 | e=d.get('target') 24 | Y=(np.array(e[:,1])) 25 | 26 | # Initialization of parameters 27 | index50 = 0 28 | index100 = 0 29 | index150 = 0 30 | index200 = 0 31 | index300 = 0 32 | index400 = 0 33 | index500 = 0 34 | #Initialization of arrays 35 | events50 = np.zeros((num_events, 25, 25, 25)) 36 | max_pos_50 = np.zeros((num_events, 3)) 37 | events100 = np.zeros((num_events, 25, 25, 25)) 38 | max_pos_100 = np.zeros((num_events, 3)) 39 | events150 = np.zeros((num_events, 25, 25, 25)) 40 | max_pos_150 = np.zeros((num_events, 3)) 41 | events200 = np.zeros((num_events, 25, 25, 25)) 42 | max_pos_200 = np.zeros((num_events, 3)) 43 | events300 = np.zeros((num_events, 25, 25, 25)) 44 | max_pos_300 = np.zeros((num_events, 3)) 45 | events400 = np.zeros((num_events, 25, 25, 25)) 46 | max_pos_400 = np.zeros((num_events, 3)) 47 | events500 = np.zeros((num_events, 25, 25, 25)) 48 | max_pos_500 = np.zeros((num_events, 3)) 49 | sum_50 = np.zeros((num_events, 3, 25)) 50 | sum_100 = np.zeros((num_events, 3, 25)) 51 | sum_150 = np.zeros((num_events, 3, 25)) 52 | sum_200 = np.zeros((num_events, 3, 25)) 53 | sum_300 = np.zeros((num_events, 3, 25)) 54 | sum_400 = np.zeros((num_events, 3, 25)) 55 | sum_500 = np.zeros((num_events, 3, 25)) 56 | 57 | 58 | size_data = int(X.shape[0]) 59 | for i in range(size_data): 60 | if Y[i] > 45 and Y[i] > 55 and index50 < num_events: 61 | events50[index50] = X[i] 62 | index50 = index50 + 1 63 | elif Y[i] > 95 and Y[i] > 105 and index100 < num_events: 64 | events100[index100] = X[i] 65 | index100 = index100 + 1 66 | elif Y[i] > 145 and Y[i] > 155 and index150 < num_events: 67 | events150[index150] = X[i] 68 | index150 = index150 + 1 69 | elif Y[i] > 195 and Y[i] > 205 and index200 < num_events: 70 | events200[index200] = X[i] 71 | index200 = index200 + 1 72 | elif Y[i] > 295 and Y[i] > 305 and index300 < num_events: 73 | events300[index300] = X[i] 74 | index300 = index300 + 1 75 | elif Y[i] > 395 and Y[i] > 405 and index400 < num_events: 76 | events400[index400] = X[i] 77 | index400 = index400 + 1 78 | elif Y[i] > 495 and Y[i] > 505 and index500 < num_events: 79 | events500[index500] = X[i] 80 | index500 = index500 + 1 81 | 82 | for j in range(num_events): 83 | max_pos_50[j] = np.unravel_index(events50[j].argmax(), (25, 25, 25)) 84 | max_pos_100[j] = np.unravel_index(events100[j].argmax(), (25, 25, 25)) 85 | max_pos_150[j] = np.unravel_index(events150[j].argmax(), (25, 25, 25)) 86 | max_pos_200[j] = np.unravel_index(events200[j].argmax(), (25, 25, 25)) 87 | max_pos_300[j] = np.unravel_index(events300[j].argmax(), (25, 25, 25)) 88 | max_pos_400[j] = np.unravel_index(events400[j].argmax(), (25, 25, 25)) 89 | max_pos_500[j] = np.unravel_index(events500[j].argmax(), (25, 25, 25)) 90 | sum_50[j, 0] = np.sum(events50[j], axis=(1,2)) 91 | sum_50[j, 1] = np.sum(events50[j], axis=(0,2)) 92 | sum_50[j, 2] = np.sum(events50[j], axis=(0,1)) 93 | sum_100[j, 0] = np.sum(events100[j], axis=(1,2)) 94 | sum_100[j, 1] = np.sum(events100[j], axis=(0,2)) 95 | sum_100[j, 2] = np.sum(events100[j], axis=(0,1)) 96 | sum_150[j, 0] = np.sum(events150[j], axis=(1,2)) 97 | sum_150[j, 1] = np.sum(events150[j], axis=(0,2)) 98 | sum_150[j, 2] = np.sum(events150[j], axis=(0,1)) 99 | sum_200[j, 0] = np.sum(events200[j], axis=(1,2)) 100 | sum_200[j, 1] = np.sum(events200[j], axis=(0,2)) 101 | sum_200[j, 2] = np.sum(events200[j], axis=(0,1)) 102 | sum_300[j, 0] = np.sum(events300[j], axis=(1,2)) 103 | sum_300[j, 1] = np.sum(events300[j], axis=(0,2)) 104 | sum_300[j, 2] = np.sum(events300[j], axis=(0,1)) 105 | sum_400[j, 0] = np.sum(events400[j], axis=(1,2)) 106 | sum_400[j, 1] = np.sum(events400[j], axis=(0,2)) 107 | sum_400[j, 2] = np.sum(events400[j], axis=(0,1)) 108 | sum_500[j, 0] = np.sum(events500[j], axis=(1,2)) 109 | sum_500[j, 1] = np.sum(events500[j], axis=(0,2)) 110 | sum_500[j, 2] = np.sum(events500[j], axis=(0,1)) 111 | 112 | #### Generate a table to screen 113 | print "Energy\t\t Events\t\tMaximum Value\t\t Maximum loc\t\t\t Mean\t\t\t Minimum\t\t" 114 | print "50 \t\t%d \t\t%f \t\t%s \t\t%f \t\t%f" %(index50, np.amax(events50), str(np.unravel_index(events50.argmax(), (index50, 25, 25, 25))), np.mean(events50), np.amin(events50)) 115 | print "100 \t\t%d \t\t%f \t\t%s \t\t%f \t\t%f" %(index100, np.amax(events100), str(np.unravel_index(events100.argmax(), (index100, 25, 25, 25))), np.mean(events100), np.amin(events100)) 116 | print "150 \t\t%d \t\t%f \t\t%s \t\t%f \t\t%f" %(index150, np.amax(events150), str(np.unravel_index(events150.argmax(), (index150, 25, 25, 25))), np.mean(events150), np.amin(events150)) 117 | print "200 \t\t%d \t\t%f \t\t%s \t\t%f \t\t%f" %(index200, np.amax(events200), str(np.unravel_index(events200.argmax(), (index200, 25, 25, 25))), np.mean(events200), np.amin(events200)) 118 | print "300 \t\t%d \t\t%f \t\t%s \t\t%f \t\t%f" %(index300, np.amax(events300), str(np.unravel_index(events300.argmax(), (index300, 25, 25, 25))), np.mean(events300), np.amin(events300)) 119 | print "400 \t\t%d \t\t%f \t\t%s \t\t%f \t\t%f" %(index400, np.amax(events400), str(np.unravel_index(events400.argmax(), (index400, 25, 25, 25))), np.mean(events400), np.amin(events400)) 120 | print "500 \t\t%d \t\t%f \t\t%s \t\t%f \t\t%f" %(index500, np.amax(events500), str(np.unravel_index(events500.argmax(), (index500, 25, 25, 25))), np.mean(events500), np.amin(events500)) 121 | 122 | #### Make plots 123 | def safe_mkdir(path): 124 | ''' 125 | Safe mkdir (i.e., don't create if already exists, 126 | and no violation of race conditions) 127 | ''' 128 | from os import makedirs 129 | from errno import EEXIST 130 | try: 131 | makedirs(path) 132 | except OSError as exception: 133 | if exception.errno != EEXIST: 134 | raise exception 135 | outdir = 'data_info_plots' 136 | safe_mkdir(outdir) 137 | bins = np.arange(0, 25, 1) 138 | 139 | def plot_max(array, index, out_file, num_fig, energy): 140 | ## Plot the Histogram of Maximum energy deposition location on all axis 141 | plt.figure(num_fig) 142 | plt.subplot(221) 143 | plt.title('X-axis') 144 | plt.hist(array[0:index-1, 0], bins=bins, histtype='step', label= str(energy), normed=True) 145 | plt.legend() 146 | plt.ylabel('Events') 147 | 148 | plt.subplot(222) 149 | plt.title('Y-axis') 150 | plt.hist(array[0:index-1, 1], bins=bins, histtype='step', label=str(energy), normed=True) 151 | plt.legend() 152 | plt.xlabel('Position') 153 | #plt.ylabel('Events') 154 | 155 | plt.subplot(223) 156 | #plt.title('Z-axis') 157 | plt.hist(array[0:index-1, 2], bins=bins, histtype='step', label=str(energy), normed=True) 158 | plt.legend(loc=1) 159 | plt.xlabel('Position') 160 | plt.ylabel('Events') 161 | plt.savefig(os.path.join(outdir, out_file)) 162 | 163 | def plot_energy(array, out_file, num_fig, energy): 164 | ### Plot Histogram of energy deposition along all three axis 165 | plt.figure(num_fig) 166 | plt.subplot(221) 167 | plt.title('X-axis') 168 | plt.hist(array[:, 0].flatten(), bins='auto', histtype='step', label=str(energy)) 169 | plt.legend() 170 | 171 | plt.subplot(222) 172 | plt.title('Y-axis') 173 | plt.hist(array[:, 1].flatten(), bins='auto', histtype='step', label=str(energy)) 174 | plt.legend() 175 | 176 | plt.subplot(223) 177 | plt.hist(array[:, 2].flatten(), bins='auto', histtype='step', label=str(energy)) 178 | plt.legend() 179 | plt.savefig(os.path.join(outdir, out_file)) 180 | 181 | def plot_energy_hist(array, index, out_file, num_fig, energy): 182 | ### Plot total energy deposition cell by cell along x, y, z axis 183 | plt.figure(num_fig) 184 | plt.subplot(221) 185 | plt.title('X-axis') 186 | plt.plot(array[0:index, 0].sum(axis = 0), label=str(energy)) 187 | plt.legend() 188 | 189 | plt.subplot(222) 190 | plt.title('Y-axis') 191 | plt.plot(array[0:index, 1].sum(axis = 0), label=str(energy)) 192 | plt.legend() 193 | 194 | plt.subplot(223) 195 | plt.title('Z-axis') 196 | plt.plot(array[0:index, 2].sum(axis = 0), label=str(energy)) 197 | plt.legend() 198 | plt.savefig(os.path.join(outdir, out_file)) 199 | 200 | def plot_energy_mean(array, index, out_file, num_fig, energy): 201 | ### Plot total energy deposition cell by cell along x, y, z axis 202 | plt.figure(num_fig) 203 | plt.subplot(221) 204 | plt.title('X-axis') 205 | plt.plot(array[0:index, 0].mean(axis = 0), label=str(energy)) 206 | plt.legend() 207 | 208 | plt.subplot(222) 209 | plt.title('Y-axis') 210 | plt.plot(array[0:index, 1].mean(axis = 0), label=str(energy)) 211 | plt.legend() 212 | 213 | plt.subplot(223) 214 | plt.title('Z-axis') 215 | plt.plot(array[0:index, 2].mean(axis = 0), label=str(energy)) 216 | plt.legend() 217 | plt.savefig(os.path.join(outdir, out_file)) 218 | 219 | plot_max(max_pos_50, index50, 'Position_of_max.pdf', 1, 50) 220 | plot_max(max_pos_100, index100,'Position_of_max.pdf', 1, 100) 221 | plot_max(max_pos_150, index150,'Position_of_max.pdf', 1, 150) 222 | plot_max(max_pos_200, index200,'Position_of_max.pdf', 1, 200) 223 | plot_max(max_pos_300, index300,'Position_of_max.pdf', 1, 300) 224 | plot_max(max_pos_400, index400,'Position_of_max.pdf', 1, 400) 225 | plot_max(max_pos_500, index500,'Position_of_max.pdf', 1, 500) 226 | 227 | plot_energy(sum_50, 'Flat_energy_50.pdf', 2, 50) 228 | plot_energy(sum_100, 'Flat_energy_100.pdf', 3, 100) 229 | plot_energy(sum_150, 'Flat_energy_150.pdf', 4, 150) 230 | plot_energy(sum_200, 'Flat_energy_200.pdf', 5, 200) 231 | plot_energy(sum_300, 'Flat_energy_300.pdf', 6, 300) 232 | plot_energy(sum_400, 'Flat_energy_400.pdf', 7, 400) 233 | plot_energy(sum_500, 'Flat_energy_500.pdf', 8, 500) 234 | 235 | plot_energy_hist(sum_50, index50, 'hist_50.pdf', 9, 50) 236 | plot_energy_hist(sum_100, index100, 'hist_100.pdf', 10, 100) 237 | plot_energy_hist(sum_150, index150, 'hist_150.pdf', 11, 150) 238 | plot_energy_hist(sum_200, index200, 'hist_200.pdf', 12, 200) 239 | plot_energy_hist(sum_300, index300, 'hist_300.pdf', 13, 300) 240 | plot_energy_hist(sum_400, index400, 'hist_400.pdf', 14, 400) 241 | plot_energy_hist(sum_500, index500, 'hist_500.pdf', 15, 500) 242 | 243 | plot_energy_hist(sum_50, index50, 'hist_all.pdf', 16, 50) 244 | plot_energy_hist(sum_100, index100, 'hist_all.pdf', 16, 100) 245 | plot_energy_hist(sum_150, index150, 'hist_all.pdf', 16, 150) 246 | plot_energy_hist(sum_200, index200, 'hist_all.pdf', 16, 200) 247 | plot_energy_hist(sum_300, index300, 'hist_all.pdf', 16, 300) 248 | plot_energy_hist(sum_400, index400, 'hist_all.pdf', 16, 400) 249 | plot_energy_hist(sum_500, index500, 'hist_all.pdf', 16, 500) 250 | 251 | plot_energy_mean(sum_50, index50, 'hist_mean_all.pdf', 17, 50) 252 | plot_energy_mean(sum_100, index100, 'hist_mean_all.pdf', 17, 100) 253 | plot_energy_mean(sum_150, index150, 'hist_mean_all.pdf', 17, 150) 254 | plot_energy_mean(sum_200, index200, 'hist_mean_all.pdf', 17, 200) 255 | plot_energy_mean(sum_300, index300, 'hist_mean_all.pdf', 17, 300) 256 | plot_energy_mean(sum_400, index400, 'hist_mean_all.pdf', 17, 400) 257 | plot_energy_mean(sum_500, index500, 'hist_mean_all.pdf', 17, 500) 258 | 259 | plt.figure(18) 260 | plt.title('Energy') 261 | ebins=np.arange(0, 500, 10) 262 | plt.hist(Y, bins=ebins, histtype='step', label='Actual Energy') 263 | plt.savefig(os.path.join(outdir, 'Incoming_energy_histogram.pdf')) 264 | 265 | plt.figure(19) 266 | plt.title('Energy') 267 | plt.hist(np.sum(X, axis=(1, 2, 3)), bins='auto', histtype='step') 268 | plt.savefig(os.path.join(outdir, 'ECAL_histogram.pdf')) 269 | 270 | plt.figure(18) 271 | plt.title('Energy') 272 | plt.hist(np.multiply(0.05, np.sum(X, axis=(1, 2, 3))), bins=ebins, histtype='step', label='Scaled (0.05)ECAL Energy') 273 | plt.legend(loc=8) 274 | plt.savefig(os.path.join(outdir, 'Combined_histogram.pdf')) 275 | -------------------------------------------------------------------------------- /keras/HoroTest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | from __future__ import print_function 5 | from collections import defaultdict 6 | try: 7 | import cPickle as pickle 8 | except ImportError: 9 | import pickle 10 | import keras 11 | import argparse 12 | #import os 13 | #os.environ['LD_LIBRARY_PATH'] = os.getcwd() 14 | from six.moves import range 15 | import sys 16 | #from keras.utils.training_utils import multi_gpu_model 17 | import h5py 18 | import numpy as np 19 | def bit_flip(x, prob=0.05): 20 | """ flips a int array's values with some probability """ 21 | x = np.array(x) 22 | selection = np.random.uniform(0, 1, x.shape) < prob 23 | x[selection] = 1 * np.logical_not(x[selection]) 24 | return x 25 | 26 | if __name__ == '__main__': 27 | 28 | import keras.backend as K 29 | 30 | K.set_image_dim_ordering('tf') 31 | 32 | from keras.layers import Input 33 | from keras.models import Model 34 | from keras.optimizers import Adadelta, Adam, RMSprop 35 | from keras.utils.generic_utils import Progbar 36 | from sklearn.cross_validation import train_test_split 37 | 38 | #import tensorflow as tf 39 | #config = tf.ConfigProto(log_device_placement=True) 40 | 41 | from EcalEnergyGan import generator, discriminator 42 | 43 | import tensorflow as tf 44 | import horovod.keras as hvd 45 | 46 | # Initialize Horovod. 47 | hvd.init() 48 | 49 | # Pin GPU to be used to process local rank (one GPU per process) 50 | config = tf.ConfigProto() 51 | config.gpu_options.allow_growth = True 52 | import time 53 | time.sleep( 10* hvd.local_rank()) 54 | import setGPU 55 | #config.gpu_options.visible_device_list = str(hvd.local_rank()) 56 | tf.Session(config=config) 57 | 58 | 59 | 60 | g_weights = 'params_generator_epoch_' 61 | d_weights = 'params_discriminator_epoch_' 62 | 63 | nb_epochs = 25 64 | batch_size = 128 65 | latent_size = 200 66 | verbose = 'false' 67 | nb_classes = 2 68 | 69 | 70 | generator=generator(latent_size) 71 | discriminator=discriminator() 72 | 73 | 74 | 75 | print('[INFO] Building discriminator') 76 | discriminator.summary() 77 | #discriminator.load_weights('veganweights/params_discriminator_epoch_019.hdf5') 78 | 79 | # Add Horovod Distributed Optimizer. 80 | opt = hvd.DistributedOptimizer(RMSprop()) 81 | 82 | discriminator.compile( 83 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 84 | optimizer=opt, 85 | loss=['binary_crossentropy', 'mean_absolute_percentage_error', 'mean_absolute_percentage_error'], 86 | loss_weights=[6, 2, 1] 87 | #loss=['binary_crossentropy', 'kullback_leibler_divergence'] 88 | ) 89 | 90 | # build the generator 91 | print('[INFO] Building generator') 92 | generator.summary() 93 | #generator.load_weights('veganweights/params_generator_epoch_019.hdf5') 94 | generator.compile( 95 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 96 | optimizer=opt, 97 | loss='binary_crossentropy' 98 | ) 99 | 100 | latent = Input(shape=(latent_size, ), name='combined_z') 101 | 102 | fake_image = generator( latent) 103 | 104 | #fake.name = 'real_fake' 105 | #aux.name = 'primary_energy' 106 | #ecal.name = 'ecal_sum' 107 | discriminator.trainable = False 108 | discriminator.get_layer(name='generation').name='generation_1' 109 | generation, auxiliary, ecal = discriminator(fake_image) 110 | combined = Model( 111 | input=[latent], 112 | output=[generation, auxiliary, ecal], 113 | name='combined_model' 114 | ) 115 | combined.compile( 116 | #optimizer=Adam(lr=adam_lr, beta_1=adam_beta_1), 117 | optimizer=opt, 118 | loss=['binary_crossentropy', 'mean_absolute_percentage_error', 'mean_absolute_percentage_error'], 119 | loss_weights=[5, 1, 1] 120 | ) 121 | 122 | 123 | #d=h5py.File("/bigdata/shared/LCD/Pions_fullSpectrum/ChPiEscan_1_2.h5",'r') 124 | #d=h5py.File("/bigdata/shared/LCD/small_test.h5",'r') 125 | d=h5py.File("/bigdata/shared/LCD/Electrons_fullSpectrum/Ele_v1_1_2.h5",'r') 126 | e=d.get('target') 127 | X=np.array(d.get('ECAL')) 128 | y=(np.array(e[:,1])) 129 | print(X.shape) 130 | #print('*************************************************************************************') 131 | print(y) 132 | print('*************************************************************************************') 133 | 134 | #Y=np.sum(X, axis=(1,2,3)) 135 | #print(Y) 136 | #print('*************************************************************************************') 137 | 138 | 139 | # remove unphysical values 140 | X[X < 1e-6] = 0 141 | 142 | X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.2, test_size=0.1) 143 | 144 | # tensorflow ordering 145 | X_train =np.expand_dims(X_train, axis=-1) 146 | X_test = np.expand_dims(X_test, axis=-1) 147 | y_train= y_train/100 148 | y_test=y_test/100 149 | print(X_train.shape) 150 | print(X_test.shape) 151 | print(y_train.shape) 152 | print(y_test.shape) 153 | print('*************************************************************************************') 154 | 155 | 156 | nb_train, nb_test = X_train.shape[0], X_test.shape[0] 157 | 158 | X_train = X_train.astype(np.float32) 159 | X_test = X_test.astype(np.float32) 160 | y_train = y_train.astype(np.float32) 161 | y_test = y_test.astype(np.float32) 162 | ecal_train = np.sum(X_train, axis=(1, 2, 3)) 163 | ecal_test = np.sum(X_test, axis=(1, 2, 3)) 164 | 165 | print(X_train.shape) 166 | print(X_test.shape) 167 | print(ecal_train.shape) 168 | print(ecal_test.shape) 169 | print('*************************************************************************************') 170 | train_history = defaultdict(list) 171 | test_history = defaultdict(list) 172 | 173 | # Broadcast initial variable states from rank 0 to all other processes. 174 | # This is necessary to ensure consistent initialization of all workers when 175 | # training is started with random weights or restored from a checkpoint. 176 | #callbacks = [ 177 | # 178 | #] 179 | 180 | gcb =hvd.callbacks.BroadcastGlobalVariablesCallback(0) 181 | dcb =hvd.callbacks.BroadcastGlobalVariablesCallback(0) 182 | ccb =hvd.callbacks.BroadcastGlobalVariablesCallback(0) 183 | gcb.set_model( generator ) 184 | dcb.set_model( discriminator ) 185 | ccb.set_model( combined ) 186 | 187 | 188 | gcb.on_train_begin() 189 | dcb.on_train_begin() 190 | ccb.on_train_begin() 191 | 192 | for epoch in range(nb_epochs): 193 | print('Epoch {} of {}'.format(epoch + 1, nb_epochs)) 194 | 195 | nb_batches = int(X_train.shape[0] / batch_size) 196 | if verbose: 197 | progress_bar = Progbar(target=nb_batches) 198 | 199 | epoch_gen_loss = [] 200 | epoch_disc_loss = [] 201 | for index in range(nb_batches): 202 | if verbose: 203 | progress_bar.update(index) 204 | else: 205 | if index % 100 == 0: 206 | print('processed {}/{} batches'.format(index + 1, nb_batches)) 207 | 208 | noise = np.random.normal(0, 1, (batch_size, latent_size)) 209 | 210 | image_batch = X_train[index * batch_size:(index + 1) * batch_size] 211 | energy_batch = y_train[index * batch_size:(index + 1) * batch_size] 212 | ecal_batch = ecal_train[index * batch_size:(index + 1) * batch_size] 213 | 214 | print(image_batch.shape) 215 | print(ecal_batch.shape) 216 | sampled_energies = np.random.uniform(0, 5,( batch_size,1 )) 217 | generator_ip = np.multiply(sampled_energies, noise) 218 | ecal_ip = np.multiply(2, sampled_energies) 219 | generated_images = generator.predict(generator_ip, verbose=0) 220 | 221 | # loss_weights=[np.ones(batch_size), 0.05 * np.ones(batch_size)] 222 | 223 | real_batch_loss = discriminator.train_on_batch(image_batch, [bit_flip(np.ones(batch_size)), energy_batch, ecal_batch]) 224 | fake_batch_loss = discriminator.train_on_batch(generated_images, [bit_flip(np.zeros(batch_size)), sampled_energies, ecal_ip]) 225 | # print(real_batch_loss) 226 | # print(fake_batch_loss) 227 | 228 | # fake_batch_loss = discriminator.train_on_batch(disc_in_fake, disc_op_fake, loss_weights) 229 | 230 | epoch_disc_loss.append([ 231 | (a + b) / 2 for a, b in zip(real_batch_loss, fake_batch_loss) 232 | ]) 233 | 234 | trick = np.ones(batch_size) 235 | 236 | gen_losses = [] 237 | 238 | for _ in range(2): 239 | noise = np.random.normal(0, 1, (batch_size, latent_size)) 240 | sampled_energies = np.random.uniform(0, 5, ( batch_size,1 )) 241 | generator_ip = np.multiply(sampled_energies, noise) 242 | ecal_ip = np.multiply(2, sampled_energies) 243 | 244 | gen_losses.append(combined.train_on_batch( 245 | [generator_ip], 246 | [trick, sampled_energies.reshape((-1, 1)), ecal_ip])) 247 | 248 | epoch_gen_loss.append([ 249 | (a + b) / 2 for a, b in zip(*gen_losses) 250 | ]) 251 | # 252 | # print('\nTesting for epoch {}:'.format(epoch + 1)) 253 | 254 | # noise = np.random.normal(0, 1, (nb_test, latent_size)) 255 | 256 | # sampled_energies = np.random.uniform(0, 5, (nb_test, 1)) 257 | # generator_ip = np.multiply(sampled_energies, noise) 258 | # generated_images = generator.predict(generator_ip, verbose=False) 259 | # ecal_ip = np.multiply(2, sampled_energies) 260 | # sampled_energies = np.squeeze(sampled_energies, axis=(1,)) 261 | # X = np.concatenate((X_test, generated_images)) 262 | # y = np.array([1] * nb_test + [0] * nb_test) 263 | # ecal = np.concatenate((ecal_test, ecal_ip)) 264 | # print(ecal.shape) 265 | # print(y_test.shape) 266 | # print(sampled_energies.shape) 267 | # aux_y = np.concatenate((y_test, sampled_energies), axis=0) 268 | # print(aux_y.shape) 269 | # discriminator_test_loss = discriminator.evaluate( 270 | # X, [y, aux_y, ecal], verbose=False, batch_size=batch_size) 271 | # 272 | # discriminator_train_loss = np.mean(np.array(epoch_disc_loss), axis=0) 273 | 274 | # noise = np.random.normal(0, 1, (2 * nb_test, latent_size)) 275 | # sampled_energies = np.random.uniform(0, 5, (2 * nb_test, 1)) 276 | # generator_ip = np.multiply(sampled_energies, noise) 277 | # ecal_ip = np.multiply(2, sampled_energies) 278 | # 279 | # trick = np.ones(2 * nb_test) 280 | 281 | # generator_test_loss = combined.evaluate(generator_ip, 282 | # [trick, sampled_energies.reshape((-1, 1)), ecal_ip], verbose=False, batch_size=batch_size) 283 | 284 | # generator_train_loss = np.mean(np.array(epoch_gen_loss), axis=0) 285 | # 286 | # train_history['generator'].append(generator_train_loss) 287 | # train_history['discriminator'].append(discriminator_train_loss) 288 | # 289 | # test_history['generator'].append(generator_test_loss) 290 | # test_history['discriminator'].append(discriminator_test_loss) 291 | 292 | # print('{0:<22s} | {1:4s} | {2:15s} | {3:5s}| {4:5s}'.format( 293 | # 'component', *discriminator.metrics_names)) 294 | # print('-' * 65) 295 | # 296 | # ROW_FMT = '{0:<22s} | {1:<4.2f} | {2:<15.2f} | {3:<5.2f}| {4:<5.2f}' 297 | # print(ROW_FMT.format('generator (train)', 298 | # *train_history['generator'][-1])) 299 | # print(ROW_FMT.format('generator (test)', 300 | # *test_history['generator'][-1])) 301 | # print(ROW_FMT.format('discriminator (train)', 302 | # *train_history['discriminator'][-1])) 303 | # print(ROW_FMT.format('discriminator (test)', 304 | # *test_history['discriminator'][-1])) 305 | 306 | # save weights every epoch 307 | ## this needs to done only on one process. overwise each worker is writing it 308 | if hvd.rank()==0: 309 | print ("saving weights of gen") 310 | generator.save_weights('weights/{0}{1:03d}.hdf5'.format(g_weights, epoch), 311 | overwrite=True) 312 | print ("saving weights of disc") 313 | discriminator.save_weights('weights/{0}{1:03d}.hdf5'.format(d_weights, epoch), 314 | overwrite=True) 315 | 316 | pickle.dump({'train': train_history, 'test': test_history}, 317 | open('pion-dcgan-history.pkl', 'wb')) 318 | -------------------------------------------------------------------------------- /keras/paramScan/parameter_scan.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # This file has a function that will take a list of params as input and run training using that. Finally it will run analysis to get a single matric that we will need to optimize 5 | 6 | 7 | from __future__ import print_function 8 | from collections import defaultdict 9 | try: 10 | import cPickle as pickle 11 | except ImportError: 12 | import pickle 13 | 14 | import sys 15 | import h5py 16 | import os 17 | 18 | import numpy as np 19 | 20 | #this does not work so far 21 | #from skopt import gp_minimize 22 | 23 | import keras.backend as K 24 | from keras.layers import (Input, Dense, Reshape, Flatten, Lambda, merge, 25 | Dropout, BatchNormalization, Activation, Embedding) 26 | from keras.layers.advanced_activations import LeakyReLU 27 | from keras.layers.convolutional import (UpSampling3D, Conv3D, ZeroPadding3D, 28 | AveragePooling3D) 29 | 30 | from keras.models import Model, Sequential 31 | from keras.optimizers import Adadelta, Adam, RMSprop 32 | from keras.utils.generic_utils import Progbar 33 | from sklearn.cross_validation import train_test_split 34 | K.set_image_dim_ordering('tf') 35 | import tensorflow as tf 36 | config = tf.ConfigProto(log_device_placement=True) 37 | 38 | # Fuction used to flip random bits for training 39 | def bit_flip(x, prob=0.05): 40 | """ flips a int array's values with some probability """ 41 | x = np.array(x) 42 | selection = np.random.uniform(0, 1, x.shape) < prob 43 | x[selection] = 1 * np.logical_not(x[selection]) 44 | return x 45 | 46 | # Architecture that can have additional parametrized layer 47 | def discriminator(dflag=0, df=8, dx=5, dy=5, dz=5): 48 | 49 | image = Input(shape=(25, 25, 25, 1)) 50 | 51 | x = Conv3D(32, 5, 5, 5, border_mode='same')(image) 52 | x = LeakyReLU()(x) 53 | x = Dropout(0.2)(x) 54 | 55 | x = ZeroPadding3D((2, 2,2))(x) 56 | x = Conv3D(8, 5, 5, 5, border_mode='valid')(x) 57 | x = LeakyReLU()(x) 58 | x = BatchNormalization()(x) 59 | x = Dropout(0.2)(x) 60 | 61 | x = ZeroPadding3D((2, 2, 2))(x) 62 | x = Conv3D(8, 5, 5,5, border_mode='valid')(x) 63 | x = LeakyReLU()(x) 64 | x = BatchNormalization()(x) 65 | x = Dropout(0.2)(x) 66 | 67 | if dflag==1: 68 | x = Conv3D(df, dx, dy, dz, border_mode='same')(x) 69 | x = LeakyReLU()(x) 70 | x = BatchNormalization()(x) 71 | x = Dropout(0.2)(x) 72 | 73 | x = ZeroPadding3D((1, 1, 1))(x) 74 | x = Conv3D(8, 5, 5, 5, border_mode='valid')(x) 75 | x = LeakyReLU()(x) 76 | x = BatchNormalization()(x) 77 | x = Dropout(0.2)(x) 78 | 79 | x = AveragePooling3D((2, 2, 2))(x) 80 | h = Flatten()(x) 81 | 82 | dnn = Model(image, h) 83 | dnn.summary() 84 | 85 | dnn_out = dnn(image) 86 | 87 | fake = Dense(1, activation='sigmoid', name='generation')(dnn_out) 88 | aux = Dense(1, activation='linear', name='auxiliary')(dnn_out) 89 | ecal = Lambda(lambda x: K.sum(x, axis=(1, 2, 3)))(image) 90 | Model(input=image, output=[fake, aux, ecal]) 91 | return Model(input=image, output=[fake, aux, ecal]) 92 | 93 | def generator(latent_size=200, gflag=0, gf=8, gx=5, gy=5, gz=5): 94 | 95 | latent = Input(shape=(latent_size, )) 96 | 97 | x = Dense(64 * 7* 7)(latent) 98 | x = Reshape((7, 7,8, 8))(x) 99 | x = Conv3D(64, 6, 6, 8, border_mode='same', init='he_uniform')(x) 100 | x = LeakyReLU()(x) 101 | x = BatchNormalization()(x) 102 | x = UpSampling3D(size=(2, 2, 2))(x) 103 | 104 | x = ZeroPadding3D((2, 2, 0))(x) 105 | x = Conv3D(6, 6, 5, 8, init='he_uniform')(x) 106 | x = LeakyReLU()(x) 107 | x = BatchNormalization()(x) 108 | x = UpSampling3D(size=(2, 2, 3))(x) 109 | 110 | if gflag==1: 111 | x = Conv3D(gf, gx, gy, gz, init='he_uniform', border_mode='same')(x) 112 | x = LeakyReLU()(x) 113 | x = BatchNormalization()(x) 114 | 115 | x = ZeroPadding3D((1,0,3))(x) 116 | x = Conv3D(6, 3, 3, 8, init='he_uniform')(x) 117 | x = LeakyReLU()(x) 118 | x = Conv3D(1, 2, 2, 2, bias=False, init='glorot_normal')(x) 119 | x = Activation('relu')(x) 120 | 121 | loc = Model(latent, x) 122 | loc.summary() 123 | fake_image = loc(latent) 124 | Model(input=[latent], output=fake_image) 125 | return Model(input=[latent], output=fake_image) 126 | 127 | ## Training Function 128 | def vegantrain(epochs=30, batch_size=128, latent_size=200, gen_weight=6, aux_weight=0.2, ecal_weight=0.1, lr=0.001, rho=0.9, decay=0.0, dflag=0, df= 16, dx=8, dy=8, dz= 8, gflag=0, gf= 16, gx=8, gy=8, gz= 8): 129 | g_weights = 'params_generator_epoch_' 130 | d_weights = 'params_discriminator_epoch_' 131 | 132 | d= discriminator(dflag, df= df, dx=dx, dy=dy, dz= dz) 133 | g= generator(latent_size=latent_size, gflag=gflag, gf=gf, gx=gx, gy=gy, gz=gz) 134 | 135 | print('[INFO] Building discriminator') 136 | d.summary() 137 | d.compile( 138 | optimizer=RMSprop(lr, rho=rho, decay=decay), 139 | loss=['binary_crossentropy', 'mean_absolute_percentage_error', 'mean_absolute_percentage_error'], 140 | loss_weights=[gen_weight, aux_weight, ecal_weight] 141 | ) 142 | 143 | # build the generator 144 | print('[INFO] Building generator') 145 | g.summary() 146 | g.compile( 147 | optimizer=RMSprop(lr=lr, rho=rho, decay=decay), 148 | loss='binary_crossentropy') 149 | 150 | latent = Input(shape=(latent_size, ), name='combined_z') 151 | fake_image = g(latent) 152 | d.trainable = False 153 | fake, aux, ecal = d(fake_image) 154 | combined = Model( 155 | input=[latent], 156 | output=[fake, aux, ecal], 157 | name='combined_model') 158 | combined.compile( 159 | optimizer=RMSprop(lr=lr, rho=rho, decay=decay), 160 | loss=['binary_crossentropy', 'mean_absolute_percentage_error', 'mean_absolute_percentage_error'], 161 | loss_weights=[gen_weight, aux_weight, ecal_weight]) 162 | 163 | #get data for training 164 | f=h5py.File("/afs/cern.ch/work/g/gkhattak/public/Ele_v1_1_2.h5",'r') 165 | e=f.get('target') 166 | X=np.array(f.get('ECAL')) 167 | y=(np.array(e[:,1])) 168 | # remove unphysical values 169 | X[X < 1e-6] = 0 170 | X_train = X 171 | y_train = y 172 | X_train =np.array(np.expand_dims(X_train, axis=-1)) 173 | y_train= np.array(y_train)/100 174 | 175 | nb_train= X_train.shape[0] 176 | X_train = X_train.astype(np.float32) 177 | y_train = y_train.astype(np.float32) 178 | ecal_train = np.sum(X_train, axis=(1, 2, 3)) 179 | 180 | train_history = defaultdict(list) 181 | 182 | for epoch in range(epochs): 183 | 184 | print('Epoch {} of {}'.format(epoch + 1, epochs)) 185 | 186 | nb_batches = int(X_train.shape[0] / batch_size) 187 | 188 | epoch_gen_loss = [] 189 | epoch_disc_loss = [] 190 | 191 | for index in range(nb_batches): 192 | if index % 100 == 0: 193 | print('processed {}/{} batches'.format(index + 1, nb_batches)) 194 | 195 | noise = np.random.normal(0, 1, (batch_size, latent_size)) 196 | 197 | image_batch = X_train[index * batch_size:(index + 1) * batch_size] 198 | energy_batch = y_train[index * batch_size:(index + 1) * batch_size] 199 | ecal_batch = ecal_train[index * batch_size:(index + 1) * batch_size] 200 | 201 | #print(image_batch.shape) 202 | #print(ecal_batch.shape) 203 | sampled_energies = np.random.uniform(0, 5,( batch_size,1 )) 204 | generator_ip = np.multiply(sampled_energies, noise) 205 | ecal_ip = np.multiply(2, sampled_energies) 206 | generated_images = g.predict(generator_ip, verbose=0) 207 | real_batch_loss = d.train_on_batch(image_batch, [bit_flip(np.ones(batch_size)), energy_batch, ecal_batch]) 208 | fake_batch_loss = d.train_on_batch(generated_images, [bit_flip(np.zeros(batch_size)), sampled_energies, ecal_ip]) 209 | 210 | epoch_disc_loss.append([ 211 | (a + b) / 2 for a, b in zip(real_batch_loss, fake_batch_loss) 212 | ]) 213 | 214 | trick = np.ones(batch_size) 215 | 216 | gen_losses = [] 217 | 218 | for _ in range(2): 219 | noise = np.random.normal(0, 1, (batch_size, latent_size)) 220 | sampled_energies = np.random.uniform(0, 5, ( batch_size,1 )) 221 | generator_ip = np.multiply(sampled_energies, noise) 222 | ecal_ip = np.multiply(2, sampled_energies) 223 | gen_losses.append(combined.train_on_batch( 224 | [generator_ip], 225 | [trick, sampled_energies.reshape((-1, 1)), ecal_ip])) 226 | 227 | epoch_gen_loss.append([ 228 | (a + b) / 2 for a, b in zip(*gen_losses) 229 | ]) 230 | 231 | #The testing portion was removed 232 | discriminator_train_loss = np.mean(np.array(epoch_disc_loss), axis=0) 233 | generator_train_loss = np.mean(np.array(epoch_gen_loss), axis=0) 234 | 235 | train_history['generator'].append(generator_train_loss) 236 | train_history['discriminator'].append(discriminator_train_loss) 237 | 238 | print('{0:<22s} | {1:4s} | {2:15s} | {3:5s}| {4:5s}'.format('component', *d.metrics_names)) 239 | print('-' * 65) 240 | 241 | ROW_FMT = '{0:<22s} | {1:<4.2f} | {2:<15.2f} | {3:<5.2f}| {4:<5.2f}' 242 | print(ROW_FMT.format('generator (train)', *train_history['generator'][-1])) 243 | print(ROW_FMT.format('discriminator (train)', *train_history['discriminator'][-1])) 244 | 245 | pickle.dump({'train': train_history},open('dcgan-history.pkl', 'wb')) 246 | 247 | #save weights at last epoch 248 | g.save_weights('gen_weights.hdf5'.format(g_weights, epoch), overwrite=True) 249 | d.save_weights('disc_weights.hdf5'.format(d_weights, epoch), overwrite=True) 250 | 251 | # This function will calculate two errors derived from position of maximum along an axis and the sum of ecal along the axis 252 | def analyse(gen_weights, disc_weights, datafile, latent=200, gflag=0, gf=8, gx=5, gy=5, gz=5, dflag=0, df=8, dx=5, dy=5, dz=5): 253 | print ("Started") 254 | num_events=3000 255 | data=h5py.File(datafile,'r') 256 | X=np.array(data.get('ECAL')) 257 | y=np.array(data.get('target')) 258 | Y=np.expand_dims(y[:,1], axis=-1) 259 | X[X < 1e-6] = 0 260 | print("Data is loaded") 261 | energies=[50, 100, 150, 200, 300, 400, 500] 262 | tolerance = 5 263 | g = generator(latent_size=latent, gflag=0, gf=8, gx=5, gy=5, gz=5) 264 | g.load_weights(gen_weights) 265 | d = discriminator(dflag=0, df=8, dx=5, dy=5, dz=5) 266 | d.load_weights(disc_weights) 267 | 268 | # Initialization of parameters 269 | var = {} 270 | for energy in energies: 271 | var["index" + str(energy)] = 0 272 | var["events_act" + str(energy)] = np.zeros((num_events, 25, 25, 25)) 273 | var["max_pos_act_" + str(energy)] = np.zeros((num_events, 3)) 274 | var["sumact" + str(energy)] = np.zeros((num_events, 3, 25)) 275 | var["energy_sampled" + str(energy)] = np.zeros((num_events, 1)) 276 | var["max_pos_gan_" + str(energy)] = np.zeros((num_events, 3)) 277 | var["sumgan" + str(energy)] = np.zeros((num_events, 3, 25)) 278 | 279 | ## Sorting data in bins 280 | size_data = int(X.shape[0]) 281 | print ("Sorting data") 282 | print(Y[:10]) 283 | for i in range(size_data): 284 | for energy in energies: 285 | if Y[i][0] > energy-tolerance and Y[i][0] < energy+tolerance and var["index" + str(energy)] < num_events: 286 | var["events_act" + str(energy)][var["index" + str(energy)]]= X[i] 287 | var["energy_sampled" + str(energy)][var["index" + str(energy)]] = Y[i]/100 288 | var["index" + str(energy)]= var["index" + str(energy)] + 1 289 | # Generate images 290 | for energy in energies: 291 | noise = np.random.normal(0, 1, (var["index" + str(energy)], latent)) 292 | sampled_labels = var["energy_sampled" + str(energy)] 293 | generator_in = np.multiply(sampled_labels, noise) 294 | generated_images = g.predict(generator_in, verbose=False, batch_size=100) 295 | var["events_gan" + str(energy)]= np.squeeze(generated_images) 296 | var["isreal_gan" + str(energy)], var["energy_gan" + str(energy)], var["ecal_gan"] = np.array(d.predict(generated_images, verbose=False, batch_size=100)) 297 | var["isreal_act" + str(energy)], var["energy_act" + str(energy)], var["ecal_act"] = np.array(d.predict(np.expand_dims(var["events_act" + str(energy)], -1), verbose=False, batch_size=100)) 298 | print(var["events_gan" + str(energy)].shape) 299 | print(var["events_act" + str(energy)].shape) 300 | # calculations 301 | for j in range(num_events): 302 | for energy in energies: 303 | var["max_pos_act_" + str(energy)][j] = np.unravel_index(var["events_act" + str(energy)][j].argmax(), (25, 25, 25)) 304 | var["sumact" + str(energy)][j, 0] = np.sum(var["events_act" + str(energy)][j], axis=(1,2)) 305 | var["sumact" + str(energy)][j, 1] = np.sum(var["events_act" + str(energy)][j], axis=(0,2)) 306 | var["sumact" + str(energy)][j, 2] = np.sum(var["events_act" + str(energy)][j], axis=(0,1)) 307 | var["max_pos_gan_" + str(energy)][j] = np.unravel_index(var["events_gan" + str(energy)][j].argmax(), (25, 25, 25)) 308 | var["sumgan" + str(energy)][j, 0] = np.sum(var["events_gan" + str(energy)][j], axis=(1,2)) 309 | var["sumgan" + str(energy)][j, 1] = np.sum(var["events_gan" + str(energy)][j], axis=(0,2)) 310 | var["sumgan" + str(energy)][j, 2] = np.sum(var["events_gan" + str(energy)][j], axis=(0,1)) 311 | #### Generate Data table to screen 312 | print ("Actual Data") 313 | print ("Energy\t\t Events\t\tMaximum Value\t\t Maximum loc\t\t\t Mean\t\t\t Minimum\t\t") 314 | for energy in energies: 315 | print ("%d \t\t%d \t\t%f \t\t%s \t\t%f \t\t%f" %(energy, var["index" +str(energy)], np.amax(var["events_act" + str(energy)]), str(np.unravel_index(var["events_act" + str(energy)].argmax(), (var["index" + str(energy)], 25, 25, 25))), np.mean(var["events_act" + str(energy)]), np.amin(var["events_act" + str(energy)]))) 316 | 317 | #### Generate GAN table to screen 318 | 319 | print ("Generated Data") 320 | print ("Energy\t\t Events\t\tMaximum Value\t\t Maximum loc\t\t\t Mean\t\t\t Minimum\t\tPosition error\t\t Energy Profile error\t\t") 321 | metricp = 0 322 | metrice = 0 323 | for energy in energies: 324 | var["pos_error"+ str(energy)] = var["max_pos_act_" + str(energy)] - var["max_pos_gan_" + str(energy)] 325 | var["pos_total"+ str(energy)] = np.sum(var["pos_error"+ str(energy)]**2)/num_events 326 | metricp += var["pos_total"+ str(energy)] 327 | var["eprofile_error"+ str(energy)]= var["sumact" + str(energy)] - var["sumgan" + str(energy)] 328 | var["eprofile_total"+ str(energy)]= np.sum(var["eprofile_error"+ str(energy)]**2)/num_events 329 | var["eprofile_total"+ str(energy)]= 400000 * var["eprofile_total"+ str(energy)]/(energy* energy) 330 | metrice += var["eprofile_total"+ str(energy)] 331 | tot = metricp + metrice 332 | for energy in energies: 333 | print ("%d \t\t%d \t\t%f \t\t%s \t\t%f \t\t%f \t\t%f \t\t%f" %(energy, var["index" +str(energy)], np.amax(var["events_gan" + str(energy)]), str(np.unravel_index(var["events_gan" + str(energy)].argmax(), (var["index" + str(energy)], 25, 25, 25))), np.mean(var["events_gan" + str(energy)]), np.amin(var["events_gan" + str(energy)]), var["pos_total"+ str(energy)], var["eprofile_total"+ str(energy)])) 334 | print(" Position Error = %.4f\t Energy Profile Error = %.4f" %(metricp, metrice)) 335 | print(" Total Error = %.4f" %(tot)) 336 | return(tot) 337 | 338 | #Function to return a single value for a network performnace metric. The metric needs to be minimized. 339 | def objective(params): 340 | gen_weights = "gen_weights.hdf5" 341 | disc_weights = "disc_weights.hdf5" 342 | datafile = "/afs/cern.ch/work/g/gkhattak/public/Ele_v1_1_2.h5" 343 | 344 | # Just done to print the parameter setting to screen 345 | epochs, batch_size, latent, gen_weight, aux_weight, ecal_weight, lr, rho, decay, dflag, df, dx, dy, dz, gflag, gf, gx, gy, gz= params 346 | params1= [1*epochs, pow(2,batch_size), latent, gen_weight, aux_weight, ecal_weight, pow(10,lr), rho * 0.1, decay, dflag, df, dx, dy, dz, gflag, gf, gx, gy, gz] 347 | print(len(params1)) 348 | print("epochs= {} batchsize={} Latent space={}\nGeneration loss weight={} Auxilliary loss weight={} ECAL loss weight={}\nLearning rate={} rho={} decay={}\nDiscriminator: extra layer={} filters={} x={} y={} z{\ 349 | }\nGenerator: extra layer={} filters={} x={} y={} z{}\n".format(*params1)) 350 | 351 | vegantrain(1*epochs, pow(2,batch_size), latent, gen_weight, aux_weight, ecal_weight, pow(10,lr), rho * 0.1, decay, dflag, df, dx, dy, dz, gflag, gf, gx, gy, gz) 352 | score = analyse(gen_weights, disc_weights, datafile, latent, dflag, df, dx, dy, dz, gflag, gf, gx, gy, gz) 353 | return score 354 | 355 | def main(): 356 | space = [(3, 5), #epochs x 10 357 | (5, 8), #batch_size power of 2 358 | [256, 512], #latent size 359 | (1, 10), #gen_weight 360 | (0.01, 0.1), #aux_weight 361 | (0.01, 0.1), #ecal_weight 362 | (-8, -1), #lr 363 | (2, 9), #rho 364 | [0, 0.001], #decay 365 | [True,False], # dflag 366 | (4, 64), #df 367 | (2, 16), #dx 368 | (2, 16), #dy 369 | (2, 16), #dz 370 | [True,False],#gflag 371 | (4, 64), #gf 372 | (2, 16), #gx 373 | (2, 16), #gy 374 | (2, 16), #gz 375 | ] 376 | #Instead of scan parameters are provided as lits and objective value is calculated 377 | params1 =[1, 7, 256, 10, 0.1, 0.2, -3, 9 , 0, False, 12, 5, 5, 5, False, 12, 5, 5, 5] 378 | tot1 = objective(params1) 379 | print (" The negative performance metric for first = %.4f"%(tot1)) 380 | 381 | if __name__ == "__main__": 382 | main() 383 | -------------------------------------------------------------------------------- /neon/energy_gan.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime 3 | from neon.callbacks.callbacks import Callbacks, GANCostCallback 4 | #from neon.callbacks.plotting_callbacks import GANPlotCallback 5 | from neon.initializers import Gaussian, Constant 6 | from neon.layers import GeneralizedGANCost, Affine, Linear, Sequential, Conv, Deconv, Dropout, Pooling, BatchNorm, BranchNode, GeneralizedCost 7 | from neon.layers.layer import Linear, Reshape 8 | from neon.layers.container import Tree, Multicost, LayerContainer 9 | from neon.models.model import Model 10 | from neon.transforms import Rectlin, Logistic, GANCost, Tanh, MeanSquared 11 | from neon.util.argparser import NeonArgparser 12 | from neon.util.persist import ensure_dirs_exist 13 | from neon.layers.layer import Dropout 14 | from neon.data.hdf5iterator import HDF5Iterator 15 | from neon.optimizers import GradientDescentMomentum, RMSProp, Adam 16 | from gen_data_norm import gen_rhs 17 | from neon.backends import gen_backend 18 | from energy_dataset import temp_3Ddata, EnergyData 19 | import numpy as np 20 | from sklearn.model_selection import train_test_split 21 | # import matplotlib.pyplot as plt 22 | import h5py 23 | # new definitions 24 | 25 | import logging 26 | main_logger = logging.getLogger('neon') 27 | main_logger.setLevel(10) 28 | 29 | class myGenerativeAdversarial(LayerContainer): 30 | """ 31 | Container for Generative Adversarial Net (GAN). It contains the Generator 32 | and Discriminator stacks as sequential containers. 33 | 34 | Arguments: 35 | layers (list): A list containing two Sequential containers 36 | """ 37 | def __init__(self, generator, discriminator, name=None): 38 | super(LayerContainer, self).__init__(name) 39 | 40 | self.generator = generator 41 | self.discriminator = discriminator 42 | self.layers = self.generator.layers + self.discriminator.layers 43 | 44 | def nested_str(self, level=0): 45 | """ 46 | Utility function for displaying layer info with a given indentation level. 47 | 48 | Arguments: 49 | level (int, optional): indentation level 50 | 51 | Returns: 52 | str: layer info at the given indentation level 53 | """ 54 | padstr = '\n' + ' ' * level 55 | ss = ' ' * level + self.classnm + padstr 56 | ss += ' ' * level + 'Generator:\n' 57 | ss += padstr.join([l.nested_str(level + 1) for l in self.generator.layers]) 58 | ss += '\n' + ' ' * level + 'Discriminator:\n' 59 | ss += padstr.join([l.nested_str(level + 1) for l in self.discriminator.layers]) 60 | return ss 61 | def get_terminal(self): 62 | return self.generator.get_terminal() + self.discriminator.get_terminal() 63 | 64 | 65 | 66 | class myGAN(Model): 67 | """ 68 | Model for Generative Adversarial Networks. 69 | 70 | Arguments: 71 | layers: Generative Adversarial layer container 72 | noise_dim (Tuple): Dimensionality of the noise feeding the generator 73 | noise_type (Str): Noise distribution, 'normal' (default) or 'uniform' 74 | weights_only (bool): set to True if you do not want to recreate layers 75 | and states during deserialization from a serialized model 76 | description. Defaults to False. 77 | name (str): Model name. Defaults to "model" 78 | optimizer (Optimizer): Optimizer object which defines the learning rule for updating 79 | model parameters (i.e., GradientDescentMomentum, Adadelta) 80 | k (int): Number of data batches per noise batch 81 | wgan_param_clamp (float or None): In case of WGAN weight clamp value, None for others 82 | wgan_train_sched (bool): Whether to use the FAIR WGAN training schedule of critics 83 | """ 84 | def __init__(self, layers, noise_dim, dataset, noise_type='normal', weights_only=False, 85 | name="model", optimizer=None, k=1, 86 | wgan_param_clamp=None, wgan_train_sched=False): 87 | self.noise_dim = noise_dim 88 | self.noise_type = noise_type 89 | self.k = k 90 | self.wgan_param_clamp = wgan_param_clamp 91 | self.wgan_train_sched = wgan_train_sched 92 | self.nbatches = 0 93 | self.ndata = 0 94 | super(myGAN, self).__init__(layers, weights_only=weights_only, name=name, 95 | optimizer=optimizer) 96 | 97 | @staticmethod 98 | def clip_param_in_layers(layer_list, abs_bound=None): 99 | """ 100 | Element-wise clip all parameter tensors to between 101 | ``-abs_bound`` and ``+abs_bound`` in a list of layers. 102 | 103 | Arguments: 104 | layer_list (list): List of layers 105 | be (Backend object): Backend in which the tensor resides 106 | abs_bound (float, optional): Value to element-wise clip gradients 107 | or parameters. Defaults to None. 108 | """ 109 | param_list = get_param_list(layer_list) 110 | for (param, grad), states in param_list: 111 | if abs_bound: 112 | param[:] = param.backend.clip(param, -abs(abs_bound), abs(abs_bound)) 113 | 114 | def fill_noise(self, z, normal=True): 115 | """ 116 | Fill z with either uniform or normally distributed random numbers 117 | """ 118 | if normal: 119 | # Note fill_normal is not deterministic 120 | self.be.fill_normal(z) 121 | else: 122 | z[:] = 2 * self.be.rand() - 1. 123 | 124 | def fill_noise_sampledE(self, z, normal=True, minE=0., maxE=5.): 125 | """ 126 | Fill z with either uniform or normally distributed random numbers 127 | """ 128 | if normal: 129 | # Note fill_normal is not deterministic 130 | self.be.fill_normal(z) 131 | else: 132 | z[:] = 2 * self.be.rand() - 1. 133 | z[:] = z[:]*(self.be.rand()*(maxE -minE)+minE) 134 | 135 | def initialize(self, dataset, cost=None): 136 | """ 137 | Propagate shapes through the layers to configure, then allocate space. 138 | 139 | Arguments: 140 | dataset (NervanaDataIterator): Dataset iterator to perform initialization on 141 | cost (Cost): Defines the function which the model is minimizing based 142 | on the output of the last layer and the input labels. 143 | """ 144 | if self.initialized: 145 | return 146 | 147 | # Propagate shapes through the layers to configure 148 | prev_input = dataset 149 | prev_input = self.layers.generator.configure(self.noise_dim) 150 | prev_input = self.layers.discriminator.configure(dataset) 151 | 152 | if cost is not None: 153 | cost.initialize(self.layers) 154 | self.cost = cost 155 | 156 | # Now allocate space 157 | self.layers.generator.allocate(accumulate_updates=False) 158 | self.layers.discriminator.allocate(accumulate_updates=True) 159 | self.layers.allocate_deltas(None) 160 | self.initialized = True 161 | 162 | self.zbuf = self.be.iobuf(self.noise_dim) 163 | self.ybuf = self.be.iobuf((1,)) 164 | self.z0 = self.be.iobuf(self.noise_dim) # a fixed noise buffer for generating images 165 | self.fill_noise_sampledE(self.z0, normal=(self.noise_type == 'normal')) 166 | self.cost_dis = np.empty((1,), dtype=np.float32) 167 | self.current_batch = self.gen_iter = self.last_gen_batch = 0 168 | 169 | def get_k(self, giter): 170 | """ 171 | WGAN training schedule for generator following Arjovsky et al. 2017 172 | 173 | Arguments: 174 | giter (int): Counter for generator iterations 175 | """ 176 | if self.wgan_train_sched and (giter < 25 or giter % 500 == 0): 177 | return 100 178 | else: 179 | return self.k 180 | 181 | def _epoch_fit(self, dataset, callbacks): 182 | """ 183 | Helper function for fit which performs training on a dataset for one epoch. 184 | 185 | Arguments: 186 | dataset (NervanaDataIterator): Dataset iterator to perform fit on 187 | """ 188 | epoch = self.epoch_index 189 | self.total_cost[:] = 0 190 | last_gen_iter = self.gen_iter 191 | z, y_temp = self.zbuf, self.ybuf 192 | 193 | # iterate through minibatches of the dataset 194 | for mb_idx, (x, _) in enumerate(dataset): 195 | callbacks.on_minibatch_begin(epoch, mb_idx) 196 | self.be.begin(Block.minibatch, mb_idx) 197 | 198 | # clip all discriminator parameters to a cube in case of WGAN 199 | if self.wgan_param_clamp: 200 | self.clip_param_in_layers(self.layers.discriminator.layers_to_optimize, 201 | self.wgan_param_clamp) 202 | 203 | # train discriminator on noise 204 | self.fill_noise_sampledE(z, normal=(self.noise_type == 'normal')) 205 | #z = self.z0 206 | Gz = self.fprop_gen(z) 207 | y_noise = self.fprop_dis(Gz) 208 | y_temp[:] = y_noise 209 | delta_noise = self.cost.costfunc.bprop_noise(y_noise) 210 | self.bprop_dis(delta_noise) 211 | self.layers.discriminator.set_acc_on(True) 212 | 213 | # train discriminator on data 214 | y_data = self.fprop_dis(x) 215 | delta_data = self.cost.costfunc.bprop_data(y_data) 216 | self.bprop_dis(delta_data) 217 | self.optimizer.optimize(self.layers.discriminator.layers_to_optimize, epoch=epoch) 218 | self.layers.discriminator.set_acc_on(False) 219 | 220 | # keep GAN cost values for the current minibatch 221 | # abuses get_cost(y,t) using y_noise as the "target" 222 | self.cost_dis[:] = self.cost.get_cost(y_data, y_temp, cost_type='dis') 223 | 224 | # train generator 225 | if self.current_batch == self.last_gen_batch + self.get_k(self.gen_iter): 226 | self.fill_noise_sampledE(z, normal=(self.noise_type == 'normal')) 227 | Gz = self.fprop_gen(z) 228 | y_temp[:] = y_data 229 | y_noise = self.fprop_dis(Gz) 230 | delta_noise = self.cost.costfunc.bprop_generator(y_noise) 231 | delta_dis = self.bprop_dis(delta_noise) 232 | self.bprop_gen(delta_dis) 233 | self.optimizer.optimize(self.layers.generator.layers_to_optimize, epoch=epoch) 234 | # keep GAN cost values for the current minibatch 235 | self.cost_dis[:] = self.cost.get_cost(y_temp, y_noise, cost_type='dis') 236 | # accumulate total cost. 237 | self.total_cost[:] = self.total_cost + self.cost_dis 238 | self.last_gen_batch = self.current_batch 239 | self.gen_iter += 1 240 | 241 | self.be.end(Block.minibatch, mb_idx) 242 | callbacks.on_minibatch_end(epoch, mb_idx) 243 | self.current_batch += 1 244 | 245 | # now we divide total cost by the number of generator iterations, 246 | # so it was never total cost, but sum of averages 247 | # across all the minibatches we trained on the generator 248 | assert self.gen_iter > last_gen_iter, \ 249 | "at least one generator iteration is required for total cost estimation in this epoch" 250 | self.total_cost[:] = self.total_cost / (self.gen_iter - last_gen_iter) 251 | 252 | # Package a batch of data for plotting 253 | self.data_batch, self.noise_batch = x, self.fprop_gen(self.z0) 254 | 255 | def fprop_gen(self, x, inference=False): 256 | """ 257 | fprop the generator layer stack 258 | """ 259 | return self.layers.generator.fprop(x, inference) 260 | 261 | def fprop_dis(self, x, inference=False): 262 | """ 263 | fprop the discriminator layer stack 264 | """ 265 | return self.layers.discriminator.fprop(x, inference) 266 | 267 | def bprop_dis(self, delta): 268 | """ 269 | bprop the discriminator layer stack 270 | """ 271 | return self.layers.discriminator.bprop(delta) 272 | 273 | def bprop_gen(self, delta): 274 | """ 275 | bprop the generator layer stack 276 | """ 277 | return self.layers.generator.bprop(delta) 278 | 279 | 280 | # load up the data set 281 | X, y = temp_3Ddata() 282 | X[X < 1e-6] = 0 283 | #mean = np.mean(X, axis=0, keepdims=True) 284 | #max_elem = np.max(np.abs(X)) 285 | #print(np.max(np.abs(X)),'max abs element') 286 | #print(np.min(X),'min element') 287 | #X = (X- mean)/max_elem 288 | #print(X.shape, 'X shape') 289 | #print(np.max(X),'max element after normalisation') 290 | #print(np.min(X),'min element after normalisation') 291 | X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.1, test_size=0.1, random_state=42) 292 | print(X_train.shape, 'X train shape') 293 | print(y_train.shape, 'y train shape') 294 | 295 | gen_backend(backend='cpu', batch_size=100) 296 | 297 | #X_train.reshape((X_train.shape[0], 25 * 25 * 25)) 298 | 299 | # setup datasets 300 | train_set = EnergyData(X=X_train, Y=y_train, lshape=(1,25,25,25)) 301 | 302 | # grab one iteration from the train_set 303 | iterator = train_set.__iter__() 304 | (X, Y) = iterator.next() 305 | print X # this should be shape (N, 25,25, 25) 306 | print Y # this should be shape (Y1,Y2) of shapes (1)(1) 307 | assert X.is_contiguous 308 | assert Y.is_contiguous 309 | 310 | train_set.reset() 311 | 312 | # generate test set 313 | valid_set =EnergyData(X=X_test, Y=y_test, lshape=(1,25,25,25)) 314 | 315 | 316 | print 'train_set OK' 317 | #tate=lt.plot(X_train[0, 12]) 318 | #plt.savefigure('data_img.png') 319 | 320 | # setup weight initialization function 321 | init = Gaussian(scale=0.01) 322 | 323 | # discriminiator using convolution layers 324 | lrelu = Rectlin(slope=0.1) # leaky relu for discriminator 325 | # sigmoid = Logistic() # sigmoid activation function 326 | conv1 = dict(init=init, batch_norm=False, activation=lrelu, bias=init) 327 | conv2 = dict(init=init, batch_norm=False, activation=lrelu, padding=2, bias=init) 328 | conv3 = dict(init=init, batch_norm=False, activation=lrelu, padding=1, bias=init) 329 | b1 = BranchNode("b1") 330 | b2 = BranchNode("b2") 331 | branch1 = [ 332 | b1, 333 | Conv((5, 5, 5, 32), **conv1), 334 | Dropout(keep = 0.8), 335 | Conv((5, 5, 5, 8), **conv2), 336 | BatchNorm(), 337 | Dropout(keep = 0.8), 338 | Conv((5, 5, 5, 8), **conv2), 339 | BatchNorm(), 340 | Dropout(keep = 0.8), 341 | Conv((5, 5, 5, 8), **conv3), 342 | BatchNorm(), 343 | Dropout(keep = 0.8), 344 | Pooling((2, 2, 2)), 345 | Affine(1024, init=init, activation=lrelu), 346 | BatchNorm(), 347 | Affine(1024, init=init, activation=lrelu), 348 | BatchNorm(), 349 | b2, 350 | Affine(nout=1, init=init, bias=init, activation=Logistic()) 351 | ] #real/fake 352 | branch2 = [b2, 353 | Affine(nout=1, init=init, bias=init, activation=lrelu)] #E primary 354 | branch3 = [b1,Linear(1, init=Constant(val=1.0))] #SUM ECAL 355 | 356 | D_layers = Tree([branch1, branch2, branch3], name="Discriminator") #keep weight between branches equal to 1. for now (alphas=(1.,1.,1.) as by default ) 357 | # generator using convolution layers 358 | init_gen = Gaussian(scale=0.001) 359 | relu = Rectlin(slope=0) # relu for generator 360 | pad1 = dict(pad_h=2, pad_w=2, pad_d=2) 361 | str1 = dict(str_h=2, str_w=2, str_d=2) 362 | conv1 = dict(init=init_gen, batch_norm=False, activation=lrelu, padding=pad1, strides=str1, bias=init_gen) 363 | pad2 = dict(pad_h=2, pad_w=2, pad_d=2) 364 | str2 = dict(str_h=2, str_w=2, str_d=2) 365 | conv2 = dict(init=init_gen, batch_norm=False, activation=lrelu, padding=pad2, strides=str2, bias=init_gen) 366 | pad3 = dict(pad_h=0, pad_w=0, pad_d=0) 367 | str3 = dict(str_h=1, str_w=1, str_d=1) 368 | conv3 = dict(init=init_gen, batch_norm=False, activation=Tanh(), padding=pad3, strides=str3, bias=init_gen) 369 | bg = BranchNode("bg") 370 | branchg = [bg, 371 | Affine(1024, init=init_gen, bias=init_gen, activation=relu), 372 | BatchNorm(), 373 | Affine(8 * 7 * 7 * 7, init=init_gen, bias=init_gen), 374 | Reshape((8, 7, 7, 7)), 375 | Deconv((6, 6, 6, 6), **conv1), #14x14x14 376 | BatchNorm(), 377 | # Linear(5 * 14 * 14 * 14, init=init), 378 | # Reshape((5, 14, 14, 14)), 379 | Deconv((5, 5, 5, 64), **conv2), #27x27x27 380 | BatchNorm(), 381 | Conv((3, 3, 3, 1), **conv3) 382 | ] 383 | 384 | G_layers = Tree([branchg], name="Generator") 385 | 386 | print D_layers 387 | print G_layers 388 | layers = myGenerativeAdversarial(generator=G_layers, discriminator=D_layers) 389 | #discriminator=Sequential(D_layers, name="Discriminator")) 390 | print 'layers defined' 391 | print layers 392 | # setup optimizer 393 | # optimizer = RMSProp(learning_rate=1e-4, decay_rate=0.9, epsilon=1e-8) 394 | optimizer = GradientDescentMomentum(learning_rate=1e-3, momentum_coef = 0.9) 395 | #optimizer = Adam(learning_rate=1e-3) 396 | 397 | # setup cost function as Binary CrossEntropy 398 | #cost = GeneralizedGANCost(costfunc=GANCost(func="wasserstein")) 399 | #cost = GeneralizedGANCost(costfunc=Multicost[GANCost(func="wasserstein"), MeanSquared, MeanSquared]) 400 | cost = Multicost(costs=[GeneralizedGANCost(costfunc=GANCost(func="wasserstein")), GeneralizedCost(costfunc=MeanSquared), GeneralizedCost(costfunc=MeanSquared)]) 401 | nb_epochs = 15 402 | latent_size = 200 403 | # initialize model 404 | noise_dim = (latent_size) 405 | gan = myGAN(layers=layers, noise_dim=noise_dim, dataset=train_set, k=5, wgan_param_clamp=0.9) 406 | # configure callbacks 407 | callbacks = Callbacks(gan, eval_set=valid_set) 408 | callbacks.add_callback(GANCostCallback()) 409 | #callbacks.add_save_best_state_callback("./best_state.pkl") 410 | 411 | print 'starting training' 412 | # run fit 413 | gan.fit(train_set, num_epochs=nb_epochs, optimizer=optimizer, 414 | cost=cost, callbacks=callbacks) 415 | 416 | # gan.save_params('our_gan.prm') 417 | 418 | #x_new = np.random.randn(100, latent_size) 419 | #inference_set = HDF5Iterator(x_new, None, nclass=2, lshape=(latent_size)) 420 | #my_generator = Model(gan.layers.generator) 421 | #my_generator.save_params('our_gen.prm') 422 | #my_discriminator = Model(gan.layers.discriminator) 423 | #my_discriminator.save_params('our_disc.prm') 424 | #test = my_generator.get_outputs(inference_set) 425 | #test = np.float32(test*max_elem + mean) 426 | #test = test.reshape((100, 25, 25, 25)) 427 | 428 | #print(test.shape, 'generator output') 429 | 430 | #plt.plot(test[0, :, 12, :]) 431 | # plt.savefigure('output_img.png') 432 | 433 | #h5f = h5py.File('output_data.h5', 'w') 434 | #h5f.create_dataset('dataset_1', data=test) 435 | 436 | -------------------------------------------------------------------------------- /keras/paramScan/vegan_param3.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # This file has a function that will take a list of params as input and run training using that. Finally it will run analysis to get a single metric that we will need to optimize 5 | 6 | 7 | from __future__ import print_function 8 | from collections import defaultdict 9 | try: 10 | import cPickle as pickle 11 | except ImportError: 12 | import pickle 13 | 14 | import sys 15 | import h5py 16 | import os 17 | 18 | import numpy as np 19 | 20 | #this does not work so far 21 | #from skopt import gp_minimize 22 | 23 | import keras.backend as K 24 | from keras.layers import (Input, Dense, Reshape, Flatten, Lambda, merge, 25 | Dropout, BatchNormalization, Activation, Embedding) 26 | from keras.layers.advanced_activations import LeakyReLU 27 | from keras.layers.convolutional import (UpSampling3D, Conv3D, ZeroPadding3D, 28 | AveragePooling3D) 29 | 30 | from keras.models import Model, Sequential 31 | from keras.optimizers import Adadelta, Adam, RMSprop 32 | from keras.utils.generic_utils import Progbar 33 | from sklearn.cross_validation import train_test_split 34 | K.set_image_dim_ordering('tf') 35 | import tensorflow as tf 36 | config = tf.ConfigProto(log_device_placement=True) 37 | 38 | # Fuction used to flip random bits for training 39 | def bit_flip(x, prob=0.05): 40 | """ flips a int array's values with some probability """ 41 | x = np.array(x) 42 | selection = np.random.uniform(0, 1, x.shape) < prob 43 | x[selection] = 1 * np.logical_not(x[selection]) 44 | return x 45 | 46 | # Architecture that can have additional parametrized layer 47 | def discriminator(dflag=0, df=8, dx=5, dy=5, dz=5): 48 | 49 | image = Input(shape=(25, 25, 25, 1)) 50 | 51 | x = Conv3D(32, 5, 5, 5, border_mode='same')(image) 52 | x = LeakyReLU()(x) 53 | x = Dropout(0.2)(x) 54 | 55 | x = ZeroPadding3D((2, 2,2))(x) 56 | x = Conv3D(8, 5, 5, 5, border_mode='valid')(x) 57 | x = LeakyReLU()(x) 58 | x = BatchNormalization()(x) 59 | x = Dropout(0.2)(x) 60 | 61 | x = ZeroPadding3D((2, 2, 2))(x) 62 | x = Conv3D(8, 5, 5,5, border_mode='valid')(x) 63 | x = LeakyReLU()(x) 64 | x = BatchNormalization()(x) 65 | x = Dropout(0.2)(x) 66 | 67 | if dflag==1: 68 | x = Conv3D(df, dx, dy, dz, border_mode='same')(x) 69 | x = LeakyReLU()(x) 70 | x = BatchNormalization()(x) 71 | x = Dropout(0.2)(x) 72 | 73 | x = ZeroPadding3D((1, 1, 1))(x) 74 | x = Conv3D(8, 5, 5, 5, border_mode='valid')(x) 75 | x = LeakyReLU()(x) 76 | x = BatchNormalization()(x) 77 | x = Dropout(0.2)(x) 78 | 79 | x = AveragePooling3D((2, 2, 2))(x) 80 | h = Flatten()(x) 81 | 82 | dnn = Model(image, h) 83 | dnn.summary() 84 | 85 | dnn_out = dnn(image) 86 | 87 | fake = Dense(1, activation='sigmoid', name='generation')(dnn_out) 88 | aux = Dense(1, activation='linear', name='auxiliary')(dnn_out) 89 | ecal = Lambda(lambda x: K.sum(x, axis=(1, 2, 3)))(image) 90 | Model(input=image, output=[fake, aux, ecal]) 91 | return Model(input=image, output=[fake, aux, ecal]) 92 | 93 | def generator(latent_size=200, gflag=0, gf=8, gx=5, gy=5, gz=5): 94 | 95 | latent = Input(shape=(latent_size, )) 96 | 97 | x = Dense(64 * 7* 7)(latent) 98 | x = Reshape((7, 7,8, 8))(x) 99 | x = Conv3D(64, 6, 6, 8, border_mode='same', init='he_uniform')(x) 100 | x = LeakyReLU()(x) 101 | x = BatchNormalization()(x) 102 | x = UpSampling3D(size=(2, 2, 2))(x) 103 | 104 | x = ZeroPadding3D((2, 2, 0))(x) 105 | x = Conv3D(6, 6, 5, 8, init='he_uniform')(x) 106 | x = LeakyReLU()(x) 107 | x = BatchNormalization()(x) 108 | x = UpSampling3D(size=(2, 2, 3))(x) 109 | 110 | if gflag==1: 111 | x = Conv3D(gf, gx, gy, gz, init='he_uniform', border_mode='same')(x) 112 | x = LeakyReLU()(x) 113 | x = BatchNormalization()(x) 114 | 115 | x = ZeroPadding3D((1,0,3))(x) 116 | x = Conv3D(6, 3, 3, 8, init='he_uniform')(x) 117 | x = LeakyReLU()(x) 118 | x = Conv3D(1, 2, 2, 2, bias=False, init='glorot_normal')(x) 119 | x = Activation('relu')(x) 120 | 121 | loc = Model(latent, x) 122 | loc.summary() 123 | fake_image = loc(latent) 124 | Model(input=[latent], output=fake_image) 125 | return Model(input=[latent], output=fake_image) 126 | 127 | ## Training Function 128 | def vegantrain(epochs=30, batch_size=128, latent_size=200, gen_weight=6, aux_weight=0.2, ecal_weight=0.1, lr=0.001, rho=0.9, decay=0.0, dflag=0, df= 16, dx=8, dy=8, dz= 8, gflag=0, gf= 16, gx=8, gy=8, gz= 8): 129 | g_weights = 'params_generator_epoch_' 130 | d_weights = 'params_discriminator_epoch_' 131 | 132 | d= discriminator(dflag, df= df, dx=dx, dy=dy, dz= dz) 133 | g= generator(latent_size=latent_size, gflag=gflag, gf=gf, gx=gx, gy=gy, gz=gz) 134 | 135 | print('[INFO] Building discriminator') 136 | d.summary() 137 | d.compile( 138 | optimizer=RMSprop(lr, rho=rho, decay=decay), 139 | loss=['binary_crossentropy', 'mean_absolute_percentage_error', 'mean_absolute_percentage_error'], 140 | loss_weights=[gen_weight, aux_weight, ecal_weight] 141 | ) 142 | 143 | # build the generator 144 | print('[INFO] Building generator') 145 | g.summary() 146 | g.compile( 147 | optimizer=RMSprop(lr=lr, rho=rho, decay=decay), 148 | loss='binary_crossentropy') 149 | 150 | latent = Input(shape=(latent_size, ), name='combined_z') 151 | fake_image = g(latent) 152 | d.trainable = False 153 | fake, aux, ecal = d(fake_image) 154 | combined = Model( 155 | input=[latent], 156 | output=[fake, aux, ecal], 157 | name='combined_model') 158 | combined.compile( 159 | optimizer=RMSprop(lr=lr, rho=rho, decay=decay), 160 | loss=['binary_crossentropy', 'mean_absolute_percentage_error', 'mean_absolute_percentage_error'], 161 | loss_weights=[gen_weight, aux_weight, ecal_weight]) 162 | 163 | #get data for training 164 | f=h5py.File("/afs/cern.ch/work/g/gkhattak/public/Ele_v1_1_2.h5",'r') 165 | e=f.get('target') 166 | X=np.array(f.get('ECAL')) 167 | y=(np.array(e[:,1])) 168 | # remove unphysical values 169 | X[X < 1e-6] = 0 170 | X_train = X 171 | y_train = y 172 | X_train =np.array(np.expand_dims(X_train, axis=-1)) 173 | y_train= np.array(y_train)/100 174 | 175 | nb_train= X_train.shape[0] 176 | X_train = X_train.astype(np.float32) 177 | y_train = y_train.astype(np.float32) 178 | ecal_train = np.sum(X_train, axis=(1, 2, 3)) 179 | 180 | train_history = defaultdict(list) 181 | 182 | for epoch in range(epochs): 183 | 184 | print('Epoch {} of {}'.format(epoch + 1, epochs)) 185 | 186 | nb_batches = int(X_train.shape[0] / batch_size) 187 | 188 | epoch_gen_loss = [] 189 | epoch_disc_loss = [] 190 | 191 | for index in range(nb_batches): 192 | if index % 100 == 0: 193 | print('processed {}/{} batches'.format(index + 1, nb_batches)) 194 | 195 | noise = np.random.normal(0, 1, (batch_size, latent_size)) 196 | 197 | image_batch = X_train[index * batch_size:(index + 1) * batch_size] 198 | energy_batch = y_train[index * batch_size:(index + 1) * batch_size] 199 | ecal_batch = ecal_train[index * batch_size:(index + 1) * batch_size] 200 | 201 | #print(image_batch.shape) 202 | #print(ecal_batch.shape) 203 | sampled_energies = np.random.uniform(0, 5,( batch_size,1 )) 204 | generator_ip = np.multiply(sampled_energies, noise) 205 | ecal_ip = np.multiply(2, sampled_energies) 206 | generated_images = g.predict(generator_ip, verbose=0) 207 | real_batch_loss = d.train_on_batch(image_batch, [bit_flip(np.ones(batch_size)), energy_batch, ecal_batch]) 208 | fake_batch_loss = d.train_on_batch(generated_images, [bit_flip(np.zeros(batch_size)), sampled_energies, ecal_ip]) 209 | 210 | epoch_disc_loss.append([ 211 | (a + b) / 2 for a, b in zip(real_batch_loss, fake_batch_loss) 212 | ]) 213 | 214 | trick = np.ones(batch_size) 215 | 216 | gen_losses = [] 217 | 218 | for _ in range(2): 219 | noise = np.random.normal(0, 1, (batch_size, latent_size)) 220 | sampled_energies = np.random.uniform(0, 5, ( batch_size,1 )) 221 | generator_ip = np.multiply(sampled_energies, noise) 222 | ecal_ip = np.multiply(2, sampled_energies) 223 | gen_losses.append(combined.train_on_batch( 224 | [generator_ip], 225 | [trick, sampled_energies.reshape((-1, 1)), ecal_ip])) 226 | 227 | epoch_gen_loss.append([ 228 | (a + b) / 2 for a, b in zip(*gen_losses) 229 | ]) 230 | 231 | #The testing portion may have to be removed but for the time being it is good to have it there 232 | 233 | discriminator_train_loss = np.mean(np.array(epoch_disc_loss), axis=0) 234 | 235 | generator_train_loss = np.mean(np.array(epoch_gen_loss), axis=0) 236 | 237 | train_history['generator'].append(generator_train_loss) 238 | train_history['discriminator'].append(discriminator_train_loss) 239 | 240 | print('{0:<22s} | {1:4s} | {2:15s} | {3:5s}| {4:5s}'.format( 241 | 'component', *d.metrics_names)) 242 | print('-' * 65) 243 | 244 | ROW_FMT = '{0:<22s} | {1:<4.2f} | {2:<15.2f} | {3:<5.2f}| {4:<5.2f}' 245 | print(ROW_FMT.format('generator (train)', *train_history['generator'][-1])) 246 | # print(ROW_FMT.format('generator (test)', *test_history['generator'][-1])) 247 | print(ROW_FMT.format('discriminator (train)', *train_history['discriminator'][-1])) 248 | # print(ROW_FMT.format('discriminator (test)', *test_history['discriminator'][-1])) 249 | 250 | pickle.dump({'train': train_history},open('dcgan-history.pkl', 'wb')) 251 | 252 | #save weights at last epoch 253 | g.save_weights('gen_weights.hdf5'.format(g_weights, epoch), overwrite=True) 254 | d.save_weights('disc_weights.hdf5'.format(d_weights, epoch), overwrite=True) 255 | 256 | # This function will calculate two errors derived from position of maximum along an axis and the sum of ecal along the axis 257 | def analyse(gen_weights, disc_weights, datafile, latent=200, gflag=0, gf=8, gx=5, gy=5, gz=5, dflag=0, df=8, dx=5, dy=5, dz=5): 258 | print ("Started") 259 | num_events=3000 260 | data=h5py.File(datafile,'r') 261 | X=np.array(data.get('ECAL')) 262 | y=np.array(data.get('target')) 263 | Y=np.expand_dims(y[:,1], axis=-1) 264 | X[X < 1e-6] = 0 265 | print("Data is loaded") 266 | energies=[50, 100, 150, 200, 300, 400, 500] 267 | tolerance = 5 268 | g = generator(latent_size=latent, gflag=0, gf=8, gx=5, gy=5, gz=5) 269 | g.load_weights(gen_weights) 270 | d = discriminator(dflag=0, df=8, dx=5, dy=5, dz=5) 271 | d.load_weights(disc_weights) 272 | 273 | # Initialization of parameters 274 | var = {} 275 | for energy in energies: 276 | var["index" + str(energy)] = 0 277 | var["events_act" + str(energy)] = np.zeros((num_events, 25, 25, 25)) 278 | var["max_pos_act_" + str(energy)] = np.zeros((num_events, 3)) 279 | var["sumact" + str(energy)] = np.zeros((num_events, 3, 25)) 280 | var["energy_sampled" + str(energy)] = np.zeros((num_events, 1)) 281 | var["max_pos_gan_" + str(energy)] = np.zeros((num_events, 3)) 282 | var["sumgan" + str(energy)] = np.zeros((num_events, 3, 25)) 283 | 284 | ## Sorting data in bins 285 | size_data = int(X.shape[0]) 286 | print ("Sorting data") 287 | print(Y[:10]) 288 | for i in range(size_data): 289 | for energy in energies: 290 | if Y[i][0] > energy-tolerance and Y[i][0] < energy+tolerance and var["index" + str(energy)] < num_events: 291 | var["events_act" + str(energy)][var["index" + str(energy)]]= X[i] 292 | var["energy_sampled" + str(energy)][var["index" + str(energy)]] = Y[i]/100 293 | var["index" + str(energy)]= var["index" + str(energy)] + 1 294 | # Generate images 295 | for energy in energies: 296 | noise = np.random.normal(0, 1, (var["index" + str(energy)], latent)) 297 | sampled_labels = var["energy_sampled" + str(energy)] 298 | generator_in = np.multiply(sampled_labels, noise) 299 | generated_images = g.predict(generator_in, verbose=False, batch_size=100) 300 | var["events_gan" + str(energy)]= np.squeeze(generated_images) 301 | var["isreal_gan" + str(energy)], var["energy_gan" + str(energy)], var["ecal_gan"] = np.array(d.predict(generated_images, verbose=False, batch_size=100)) 302 | var["isreal_act" + str(energy)], var["energy_act" + str(energy)], var["ecal_act"] = np.array(d.predict(np.expand_dims(var["events_act" + str(energy)], -1), verbose=False, batch_size=100)) 303 | print(var["events_gan" + str(energy)].shape) 304 | print(var["events_act" + str(energy)].shape) 305 | # calculations 306 | for j in range(num_events): 307 | for energy in energies: 308 | var["max_pos_act_" + str(energy)][j] = np.unravel_index(var["events_act" + str(energy)][j].argmax(), (25, 25, 25)) 309 | var["sumact" + str(energy)][j, 0] = np.sum(var["events_act" + str(energy)][j], axis=(1,2)) 310 | var["sumact" + str(energy)][j, 1] = np.sum(var["events_act" + str(energy)][j], axis=(0,2)) 311 | var["sumact" + str(energy)][j, 2] = np.sum(var["events_act" + str(energy)][j], axis=(0,1)) 312 | var["max_pos_gan_" + str(energy)][j] = np.unravel_index(var["events_gan" + str(energy)][j].argmax(), (25, 25, 25)) 313 | var["sumgan" + str(energy)][j, 0] = np.sum(var["events_gan" + str(energy)][j], axis=(1,2)) 314 | var["sumgan" + str(energy)][j, 1] = np.sum(var["events_gan" + str(energy)][j], axis=(0,2)) 315 | var["sumgan" + str(energy)][j, 2] = np.sum(var["events_gan" + str(energy)][j], axis=(0,1)) 316 | #### Generate Data table to screen 317 | print ("Actual Data") 318 | print ("Energy\t\t Events\t\tMaximum Value\t\t Maximum loc\t\t\t Mean\t\t\t Minimum\t\t") 319 | for energy in energies: 320 | print ("%d \t\t%d \t\t%f \t\t%s \t\t%f \t\t%f" %(energy, var["index" +str(energy)], np.amax(var["events_act" + str(energy)]), str(np.unravel_index(var["events_act" + str(energy)].argmax(), (var["index" + str(energy)], 25, 25, 25))), np.mean(var["events_act" + str(energy)]), np.amin(var["events_act" + str(energy)]))) 321 | 322 | #### Generate GAN table to screen 323 | 324 | print ("Generated Data") 325 | print ("Energy\t\t Events\t\tMaximum Value\t\t Maximum loc\t\t\t Mean\t\t\t Minimum\t\tPosition error\t\t Energy Profile error\t\t") 326 | metricp = 0 327 | metrice = 0 328 | for energy in energies: 329 | var["pos_error"+ str(energy)] = var["max_pos_act_" + str(energy)] - var["max_pos_gan_" + str(energy)] 330 | var["pos_total"+ str(energy)] = np.sum(var["pos_error"+ str(energy)]**2)/num_events 331 | metricp += var["pos_total"+ str(energy)] 332 | var["eprofile_error"+ str(energy)]= var["sumact" + str(energy)] - var["sumgan" + str(energy)] 333 | var["eprofile_total"+ str(energy)]= np.sum(var["eprofile_error"+ str(energy)]**2)/num_events 334 | var["eprofile_total"+ str(energy)]= 400000 * var["eprofile_total"+ str(energy)]/(energy* energy) 335 | metrice += var["eprofile_total"+ str(energy)] 336 | tot = metricp + metrice 337 | for energy in energies: 338 | print ("%d \t\t%d \t\t%f \t\t%s \t\t%f \t\t%f \t\t%f \t\t%f" %(energy, var["index" +str(energy)], np.amax(var["events_gan" + str(energy)]), str(np.unravel_index(var["events_gan" + str(energy)].argmax(), (var["index" + str(energy)], 25, 25, 25))), np.mean(var["events_gan" + str(energy)]), np.amin(var["events_gan" + str(energy)]), var["pos_total"+ str(energy)], var["eprofile_total"+ str(energy)])) 339 | print(" Position Error = %.4f\t Energy Profile Error = %.4f" %(metricp, metrice)) 340 | print(" Total Error = %.4f" %(tot)) 341 | return(tot) 342 | 343 | #Function to return a single value for a network performnace metric. The metric needs to be minimized. 344 | def objective(params): 345 | gen_weights = "gen_weights.hdf5" 346 | disc_weights = "disc_weights.hdf5" 347 | datafile = "/afs/cern.ch/work/g/gkhattak/public/Ele_v1_1_2.h5" 348 | 349 | # Just done to print the parameter setting to screen 350 | epochs, batch_size, latent, gen_weight, aux_weight, ecal_weight, lr, rho, decay, dflag, df, dx, dy, dz, gflag, gf, gx, gy, gz= params 351 | params1= [1*epochs, pow(2,batch_size), latent, gen_weight, aux_weight, ecal_weight, pow(10,lr), rho * 0.1, decay, dflag, df, dx, dy, dz, gflag, gf, gx, gy, gz] 352 | print(len(params1)) 353 | print("epochs= {} batchsize={} Latent space={}\nGeneration loss weight={} Auxilliary loss weight={} ECAL loss weight={}\nLearning rate={} rho={} decay={}\nDiscriminator: extra layer={} filters={} x={} y={} z{\ 354 | }\nGenerator: extra layer={} filters={} x={} y={} z{}\n".format(*params1)) 355 | 356 | vegantrain(1*epochs, pow(2,batch_size), latent, gen_weight, aux_weight, ecal_weight, pow(10,lr), rho * 0.1, decay, dflag, df, dx, dy, dz, gflag, gf, gx, gy, gz) 357 | score = analyse(gen_weights, disc_weights, datafile, latent, dflag, df, dx, dy, dz, gflag, gf, gx, gy, gz) 358 | return score 359 | 360 | def main(): 361 | space = [(3, 5), #epochs x 10 362 | (5, 8), #batch_size power of 2 363 | [256, 512], #latent size 364 | (1, 10), #gen_weight 365 | (0.01, 0.1), #aux_weight 366 | (0.01, 0.1), #ecal_weight 367 | (-8, -1), #lr 368 | (2, 9), #rho 369 | [0, 0.001], #decay 370 | [True,False], # dflag 371 | (4, 64), #df 372 | (2, 16), #dx 373 | (2, 16), #dy 374 | (2, 16), #dz 375 | [True,False],#gflag 376 | (4, 64), #gf 377 | (2, 16), #gx 378 | (2, 16), #gy 379 | (2, 16), #gz 380 | ] 381 | datafile = "/afs/cern.ch/work/g/gkhattak/public/Ele_v1_1_2.h5" 382 | gen_weights1="veganweights/params_generator_epoch_019.hdf5" 383 | disc_weights1="veganweights/params_discriminator_epoch_019.hdf5" 384 | gen_weights2="veganweights/params_generator_epoch_029.hdf5" 385 | disc_weights2="veganweights/params_discriminator_epoch_029.hdf5" 386 | gen_weights3="veganweights/params_generator_epoch_039.hdf5" 387 | disc_weights3="veganweights/params_discriminator_epoch_039.hdf5" 388 | 389 | params1 =[1, 7, 256, 10, 0.1, 0.2, -3, 9 , 0, False, 12, 5, 5, 5, False, 12, 5, 5, 5] 390 | #tot1 = objective(params1) 391 | #print (" The negative performance metric for first = %.4f"%(tot1)) 392 | tot1 = analyse(gen_weights1, disc_weights1, datafile) 393 | tot2 = analyse(gen_weights2, disc_weights2, datafile) 394 | tot3 = analyse(gen_weights3, disc_weights3, datafile) 395 | print (" The negative performance metric for first = %.4f"%(tot1)) 396 | print (" The negative performance metric for first = %.4f"%(tot2)) 397 | print (" The negative performance metric for first = %.4f"%(tot3)) 398 | if __name__ == "__main__": 399 | main() 400 | --------------------------------------------------------------------------------