├── .gitattributes ├── README.md ├── .gitignore └── ResNetOptimalModel.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deep Residual Learning for Nonlinear Regression (ResNetRegression) 2 | An optimal deep residual regression model is built for nonlinear regression. 3 | 4 | ## Download 5 | Details could be found at [Deep Residual Learning for Nonlinear Regression](https://doi.org/10.3390/e22020193). 6 | 7 | ## Code 8 | The Python code is provided on this GitHub page. 9 | ``` 10 | ResNetOptimalModel.py 11 | ``` 12 | ## Citation 13 | ``` 14 | Dongwei Chen, Fei Hu, Guokui Nian, Tiantian Yang. 15 | "Deep Residual Learning for Nonlinear Regression." 16 | Entropy 2020, 22, 193. 17 | ``` 18 | ## Contact 19 | Dongwei Chen (dongwei.chen@colostate.edu) 20 | 21 | Tiantian Yang (tyang@uidaho.edu) 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Windows image file caches 2 | Thumbs.db 3 | ehthumbs.db 4 | 5 | # Folder config file 6 | Desktop.ini 7 | 8 | # Recycle Bin used on file shares 9 | $RECYCLE.BIN/ 10 | 11 | # Windows Installer files 12 | *.cab 13 | *.msi 14 | *.msm 15 | *.msp 16 | 17 | # Windows shortcuts 18 | *.lnk 19 | 20 | # ========================= 21 | # Operating System Files 22 | # ========================= 23 | 24 | # OSX 25 | # ========================= 26 | 27 | .DS_Store 28 | .AppleDouble 29 | .LSOverride 30 | 31 | # Thumbnails 32 | ._* 33 | 34 | # Files that might appear in the root of a volume 35 | .DocumentRevisions-V100 36 | .fseventsd 37 | .Spotlight-V100 38 | .TemporaryItems 39 | .Trashes 40 | .VolumeIcon.icns 41 | 42 | # Directories potentially created on remote AFP share 43 | .AppleDB 44 | .AppleDesktop 45 | Network Trash Folder 46 | Temporary Items 47 | .apdisk 48 | -------------------------------------------------------------------------------- /ResNetOptimalModel.py: -------------------------------------------------------------------------------- 1 | """ 2 | ResNet model for regression of Keras. 3 | Optimal model for the paper 4 | 5 | "Chen, D.; Hu, F.; Nian, G.; Yang, T. Deep Residual Learning for Nonlinear Regression. Entropy 2020, 22, 193." 6 | 7 | Depth:28 8 | Width:16 9 | """ 10 | from tensorflow.keras import layers,models 11 | from tensorflow.keras import callbacks 12 | from keras.utils.vis_utils import plot_model 13 | from sklearn.model_selection import train_test_split 14 | from sklearn.preprocessing import MinMaxScaler 15 | import matplotlib.pyplot as plt 16 | import numpy as np 17 | import pandas as pd 18 | import datetime 19 | from sklearn.metrics import mean_squared_error 20 | 21 | 22 | def identity_block(input_tensor,units): 23 | """The identity block is the block that has no conv layer at shortcut. 24 | # Arguments 25 | input_tensor: input tensor 26 | units:output shape 27 | # Returns 28 | Output tensor for the block. 29 | """ 30 | x = layers.Dense(units)(input_tensor) 31 | x = layers.BatchNormalization()(x) 32 | x = layers.Activation('relu')(x) 33 | 34 | x = layers.Dense(units)(x) 35 | x = layers.BatchNormalization()(x) 36 | x = layers.Activation('relu')(x) 37 | 38 | x = layers.Dense(units)(x) 39 | x = layers.BatchNormalization()(x) 40 | 41 | x = layers.add([x, input_tensor]) 42 | x = layers.Activation('relu')(x) 43 | 44 | return x 45 | 46 | def dens_block(input_tensor,units): 47 | """A block that has a dense layer at shortcut. 48 | # Arguments 49 | input_tensor: input tensor 50 | unit: output tensor shape 51 | # Returns 52 | Output tensor for the block. 53 | """ 54 | x = layers.Dense(units)(input_tensor) 55 | x = layers.BatchNormalization()(x) 56 | x = layers.Activation('relu')(x) 57 | 58 | x = layers.Dense(units)(x) 59 | x = layers.BatchNormalization()(x) 60 | x = layers.Activation('relu')(x) 61 | 62 | x = layers.Dense(units)(x) 63 | x = layers.BatchNormalization()(x) 64 | 65 | shortcut = layers.Dense(units)(input_tensor) 66 | shortcut = layers.BatchNormalization()(shortcut) 67 | 68 | x = layers.add([x, shortcut]) 69 | x = layers.Activation('relu')(x) 70 | return x 71 | 72 | 73 | def ResNet50Regression(): 74 | """Instantiates the ResNet50 architecture. 75 | # Arguments 76 | input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) 77 | to use as input for the model. 78 | # Returns 79 | A Keras model instance. 80 | """ 81 | Res_input = layers.Input(shape=(7,)) 82 | 83 | width = 16 84 | 85 | x = dens_block(Res_input,width) 86 | x = identity_block(x,width) 87 | x = identity_block(x,width) 88 | 89 | x = dens_block(x,width) 90 | x = identity_block(x,width) 91 | x = identity_block(x,width) 92 | 93 | x = dens_block(x,width) 94 | x = identity_block(x,width) 95 | x = identity_block(x,width) 96 | 97 | x = layers.BatchNormalization()(x) 98 | x = layers.Dense(1, activation='linear')(x) 99 | model = models.Model(inputs=Res_input, outputs=x) 100 | 101 | return model 102 | 103 | #################################Prepare data#################################### 104 | plt.switch_backend('agg') 105 | #path = "~/pub/dwchen/testData/min4008001200.csv" 106 | path = "~/pub/dwchen/testData/min400800.csv" 107 | dataSet = pd.read_csv(path) 108 | dataSet = np.array(dataSet) 109 | 110 | x = dataSet[:,0:7] 111 | y = dataSet[:,7] 112 | y = y.reshape(-1,1) 113 | 114 | scaler_x = MinMaxScaler() 115 | scaler_y = MinMaxScaler() 116 | scaler_x.fit(x) 117 | xscale = scaler_x.transform(x) 118 | scaler_y.fit(y) 119 | yscale = scaler_y.transform(y) 120 | X_train, X_test, y_train, y_test = train_test_split(xscale, yscale,test_size=0.25) 121 | 122 | ##############################Build Model################################ 123 | model = ResNet50Regression() 124 | 125 | model.compile(loss='mse', optimizer='adam', metrics=['mse']) 126 | model.summary() 127 | 128 | #compute running time 129 | starttime = datetime.datetime.now() 130 | 131 | history = model.fit(X_train, y_train, epochs=50, batch_size=5000, verbose=2, callbacks=[callbacks.EarlyStopping(monitor='val_loss', patience=10,verbose=2, mode='auto')], validation_split=0.1) 132 | #history = model.fit(X_train, y_train, epochs=10, batch_size=60000, verbose=1, validation_split=0.1) 133 | endtime = datetime.datetime.now() 134 | 135 | ##############################Save Model################################# 136 | model.save('OptimalModelDataSet2.h5') 137 | #plot_model(model, to_file='ResnetModel.png') 138 | #from keras.models import load_model 139 | #model.save('my_model.h5') 140 | #model = load_model('my_model.h5') 141 | 142 | #############################Model Predicting################################# 143 | yhat = model.predict(X_test) 144 | 145 | print('The time cost: ') 146 | print(endtime - starttime) 147 | print('The test loss: ') 148 | print(mean_squared_error(yhat,y_test)) 149 | 150 | #invert normalize 151 | yhat = scaler_y.inverse_transform(yhat) 152 | y_test = scaler_y.inverse_transform(y_test) 153 | 154 | 155 | ###############################Visualize Model################################ 156 | # "Loss" 157 | plt.figure() 158 | plt.plot(history.history['loss']) 159 | plt.plot(history.history['val_loss']) 160 | plt.title('Model Loss') 161 | plt.ylabel('Loss') 162 | plt.xlabel('epoch') 163 | plt.legend(['train', 'validation'], loc='upper right') 164 | #plt.show() 165 | plt.savefig('OptimalModelDataSet2.png') 166 | 167 | plt.figure() 168 | plt.plot(y_test[0:4000],'rx') 169 | plt.plot(yhat[0:4000],' go',markerfacecolor='none') 170 | plt.title('Result for ResNet Regression') 171 | plt.ylabel('Y value') 172 | plt.xlabel('Instance') 173 | plt.legend(['Real value', 'Predicted Value'], loc='upper right') 174 | plt.savefig('OptimalModelDataSet2.png') 175 | #plt.show() 176 | 177 | file = open('/data92/pub/dwchen/testData/dataset2.txt','r+') 178 | file.write('predicted ' + 'observed ' + '\n') 179 | for i in range(len(yhat)): 180 | file.write(str(yhat[i][0])+' '+str(y_test[i][0])+'\n') 181 | file.close() 182 | 183 | 184 | 185 | --------------------------------------------------------------------------------