├── IECFONET ├── _Support │ ├── __pycache__ │ │ ├── support_NLSTM.cpython-36.pyc │ │ ├── support_SSA.cpython-36.pyc │ │ ├── support_SWT.cpython-36.pyc │ │ └── support_VMD.cpython-36.pyc │ ├── support_SSA.py │ ├── support_SWT.py │ ├── support_VMD.py │ └── support_NLSTM.py ├── _Part │ └── part_evaluate.py ├── Excecute.py ├── _Model │ └── model_major.py ├── _DATA.py └── _FSOP.py ├── README.md └── requirements.txt /IECFONET/_Support/__pycache__/support_NLSTM.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fairy-09/AUDPNet/HEAD/IECFONET/_Support/__pycache__/support_NLSTM.cpython-36.pyc -------------------------------------------------------------------------------- /IECFONET/_Support/__pycache__/support_SSA.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fairy-09/AUDPNet/HEAD/IECFONET/_Support/__pycache__/support_SSA.cpython-36.pyc -------------------------------------------------------------------------------- /IECFONET/_Support/__pycache__/support_SWT.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fairy-09/AUDPNet/HEAD/IECFONET/_Support/__pycache__/support_SWT.cpython-36.pyc -------------------------------------------------------------------------------- /IECFONET/_Support/__pycache__/support_VMD.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Fairy-09/AUDPNet/HEAD/IECFONET/_Support/__pycache__/support_VMD.cpython-36.pyc -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🔥 AUDPNet 2 | Two traditional machine learning models, one single deep learning model, four ensemble deep learning models and one innovative AUDPNet model for industrial electricity consumption forecasting. 3 | 4 | # 📚 Requirements 5 | Dependent libraries are placed in a file named 'requirements.txt'. 6 | 7 | # 📒 Usage 8 | Run 'Excecute.py' to train. 9 | 10 | # 🌟 Citation 11 | 12 | If you find our repo useful for your research, please consider giving a 🌟 and citing our work below. 13 | 14 | ``` 15 | @article{yang2025high, 16 | title={High-precision short-term industrial energy consumption forecasting via parallel-NN with Adaptive Universal Decomposition}, 17 | author={Yang, Fan and Ge, Shuning and Liu, Jian and Yan, Ke and Gao, Ao and Dong, Yijie and Yang, Man and Zhang, Wei}, 18 | journal={Expert Systems with Applications}, 19 | pages={128366}, 20 | year={2025}, 21 | publisher={Elsevier} 22 | } 23 | ``` 24 | 25 | # 🔐 License 26 | The source code is free for research and education use only. Any commercial use should get formal permission first. 27 | -------------------------------------------------------------------------------- /IECFONET/_Part/part_evaluate.py: -------------------------------------------------------------------------------- 1 | 2 | import math 3 | from sklearn.metrics import mean_squared_error 4 | from sklearn.metrics import mean_absolute_error 5 | from sklearn.metrics import r2_score 6 | 7 | 8 | def MAPE1(true,predict): 9 | 10 | L1 = int(len(true)) 11 | L2 = int(len(predict)) 12 | 13 | if L1 == L2: 14 | 15 | SUM = 0.0 16 | for i in range(L1): 17 | if true[i] == 0: 18 | SUM = abs(predict[i]) + SUM 19 | else: 20 | SUM = abs((true[i] - predict[i]) / true[i]) + SUM 21 | per_SUM = SUM * 100.0 22 | mape = per_SUM / L1 23 | return mape 24 | else: 25 | print("error") 26 | 27 | 28 | def RMSE1(true_data, predict_data): 29 | testY = true_data 30 | testPredict = predict_data 31 | rmse = math.sqrt( mean_squared_error(testY[:], testPredict[:])) 32 | return rmse 33 | 34 | 35 | def MAE1(true_data, predict_data): 36 | testY = true_data 37 | testPredict = predict_data 38 | mae=mean_absolute_error(testY[:], testPredict[:]) 39 | return mae 40 | 41 | 42 | def R2(y_true, y_predict): 43 | score = r2_score(y_true, y_predict) 44 | return score 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /IECFONET/Excecute.py: -------------------------------------------------------------------------------- 1 | from _DATA import Data 2 | from _FSOP import FSOP 3 | 4 | 5 | if __name__ == '__main__': 6 | 7 | startNum = 0 8 | # trainNum = 12 * 700 9 | # testNum = 12 * 100 10 | trainNum = 60 * 24 * 7 11 | testNum = 60 * 24 * 1 12 | # aheadNum = 8 13 | aheadNum = 8 14 | 15 | # if input('Do you want to load & process data? [yes\\no] \n' 16 | # 'If you already have done it, please input \'no\'\n') == 'yes': 17 | if aheadNum==8: 18 | print('The full dataset shall be loaded and processed immediately.') 19 | # for i in range(5): 20 | # Data(i, 1, startNum, trainNum, testNum, aheadNum) 21 | # Data(i, 2, startNum, trainNum, testNum, aheadNum) 22 | # Data(i, 4, startNum, trainNum, testNum, aheadNum) 23 | # Data(i, 6, startNum, trainNum, testNum, aheadNum) 24 | # Data(i, 12, startNum, trainNum, testNum, aheadNum) 25 | 26 | 27 | # for i in range(1): 28 | # Data(i, 1, startNum, trainNum, testNum, aheadNum) 29 | # Data(i, 2, startNum, trainNum, testNum, aheadNum) 30 | # Data(i, 4, startNum, trainNum, testNum, aheadNum) 31 | # Data(i, 6, startNum, trainNum, testNum, aheadNum) 32 | # Data(i, 12, startNum, trainNum, testNum, aheadNum) 33 | else: 34 | print('The data will not be loaded.') 35 | 36 | 37 | # for i in range(5): 38 | # FSOP(8, i, 1) 39 | # FSOP(8, i, 1) 40 | # FSOP(8, i, 2) 41 | # FSOP(8, i, 4) 42 | # FSOP(8, i, 6) 43 | # FSOP(8, i, 12) -------------------------------------------------------------------------------- /IECFONET/_Support/support_SSA.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | # import matplotlib.pyplot as plt 3 | 4 | #data:(9608,1) 5 | def SSA(series, level): 6 | # series = 0 7 | series = series - np.mean(series) # 中心化(非必须) 8 | 9 | # step1 嵌入 10 | windowLen = level # 嵌入窗口长度4 11 | seriesLen = len(series) # 序列长度9608 12 | K = seriesLen - windowLen + 1 #K:(9605,) 13 | X = np.zeros((windowLen, K)) #X:(4,9605) 14 | for i in range(K): 15 | X[:, i] = series[i:i + windowLen] 16 | 17 | # step2: svd分解， U和sigma已经按升序排序 18 | U, sigma, VT = np.linalg.svd(X, full_matrices=False) #U:(4,4) 19 | 20 | for i in range(VT.shape[0]): 21 | VT[i, :] *= sigma[i] 22 | A = VT #A:(4,9605),VT:(4,9605) 23 | 24 | # 重组;重构 25 | rec = np.zeros((windowLen, seriesLen)) 26 | for i in range(windowLen): 27 | for j in range(windowLen - 1): 28 | for m in range(j + 1): 29 | rec[i, j] += A[i, j - m] * U[m, i] 30 | rec[i, j] /= (j + 1) 31 | for j in range(windowLen - 1, seriesLen - windowLen + 1): 32 | for m in range(windowLen): 33 | rec[i, j] += A[i, j - m] * U[m, i] 34 | rec[i, j] /= windowLen 35 | for j in range(seriesLen - windowLen + 1, seriesLen): 36 | for m in range(j - seriesLen + windowLen, windowLen): 37 | rec[i, j] += A[i, j - m] * U[m, i] 38 | rec[i, j] /= (seriesLen - j) 39 | return rec 40 | 41 | # rrr = np.sum(rec, axis=0) # 选择重构的部分，这里选了全部 42 | # 43 | # plt.figure() 44 | # for i in range(10): 45 | # ax = plt.subplot(5, 2, i + 1) 46 | # ax.plot(rec[i, :]) 47 | # 48 | # plt.figure(2) 49 | # plt.plot(series) 50 | # plt.show() -------------------------------------------------------------------------------- /IECFONET/_Support/support_SWT.py: -------------------------------------------------------------------------------- 1 | import pywt 2 | from matplotlib import pyplot as plt 3 | 4 | 5 | # 函数输出如[A3, D3, D2, D1] 6 | def swt_decom(data, wavefunc, lv): 7 | 8 | coeffs_list = [] 9 | data_to_decom = data 10 | A = None 11 | 12 | for i in range(lv): 13 | [(A, D)] = pywt.swt(data_to_decom, wavefunc, level=1, axis=0) 14 | data_to_decom = A 15 | coeffs_list.insert(0, D) 16 | coeffs_list.insert(0, A) 17 | 18 | return coeffs_list 19 | 20 | 21 | # 函数输出如[D3, A3, A2, A1] 22 | def swt_decom_high(data, wavefunc, lv): 23 | coeffs_list = [] 24 | data_to_decom = data 25 | D = None 26 | 27 | for i in range(lv): 28 | [(A, D)] = pywt.swt(data_to_decom, wavefunc, level=1, axis=0) 29 | data_to_decom = D 30 | coeffs_list.insert(0, A) 31 | coeffs_list.insert(0, D) 32 | 33 | return coeffs_list 34 | 35 | 36 | # pywt.iswt函数输入[A, D] 37 | # 该函数输入如[A3, D3, D2, D1] 38 | def iswt_decom(data, wavefunc): 39 | 40 | y = data[0] 41 | for i in range(len(data) - 1): 42 | y = pywt.iswt([(y, data[i+1])], wavefunc) 43 | return y 44 | 45 | 46 | # pywt.iswt函数输入[A, D] 47 | # 该函数输入如[D3, A3, A2, A1] 48 | def iswt_decom_high(data, wavefunc): 49 | y = data[0] 50 | for i in range(len(data) - 1): 51 | y = pywt.iswt([(data[i + 1], y)], wavefunc) 52 | return y 53 | 54 | 55 | # 函数输出如[A3, D3, D2, D1, D3, A3, A2, A1]，前半部分为A1的低频分解，后半部分为D1的高频分解 56 | def swpt_decom_V1(data, wavefunc, lv): 57 | data_to_decom = data 58 | [(A, D)] = pywt.swt(data_to_decom, wavefunc, level=1, axis=0) 59 | coeffs_list_low = swt_decom(A, wavefunc, lv - 1) 60 | coeffs_list_high = swt_decom_high(D, wavefunc, lv - 1) 61 | coeffs_list = coeffs_list_low + coeffs_list_high 62 | return coeffs_list 63 | 64 | 65 | def iswpt_decom_V1(data, wavefunc): 66 | coeffs_list_low = data[:(len(data))//2] 67 | coeffs_list_high = data[:(len(data))//2] 68 | coeffs_low = iswt_decom(coeffs_list_low, wavefunc) 69 | coeffs_high = iswt_decom_high(coeffs_list_high, wavefunc) 70 | original = iswt_decom([coeffs_low, coeffs_high], wavefunc) 71 | return original 72 | 73 | 74 | # 函数输出如[A3, D3, D2, D1, A3, D3, D2, D1]，前半部分为A1的低频分解，后半部分为D1的低频分解 75 | def swpt_decom_V2(data, wavefunc, lv): 76 | data_to_decom = data 77 | [(A, D)] = pywt.swt(data_to_decom, wavefunc, level=1, axis=0) 78 | coeffs_list_low = swt_decom(A, wavefunc, lv - 1) 79 | coeffs_list_high = swt_decom(D, wavefunc, lv - 1) 80 | coeffs_list = coeffs_list_low + coeffs_list_high 81 | return coeffs_list 82 | 83 | 84 | def iswpt_decom_V2(data, wavefunc): 85 | coeffs_list_low = data[:(len(data))//2] 86 | coeffs_list_high = data[:(len(data))//2] 87 | coeffs_low = iswt_decom(coeffs_list_low, wavefunc) 88 | coeffs_high = iswt_decom(coeffs_list_high, wavefunc) 89 | original = iswt_decom([coeffs_low, coeffs_high], wavefunc) 90 | return original 91 | 92 | 93 | 94 | 95 | 96 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | _tflow_select==2.2.0 2 | absl-py==0.12.0 3 | astor== 0.8.1 4 | ca-certificates==2020.12.5 5 | cached-property==1.3.0 6 | certifi==2020.12.5 7 | chardet== 4.0.0 8 | cycler==0.10.0 9 | ewtpy==0.2 10 | freetype==2.10.4 11 | gast==0.2.2 12 | google-pasta== 0.2.0 13 | grpcio==1.37.0 14 | h5py==3.1.0 15 | hdf5==1.10.6 16 | icu==68.1 17 | idna==2.10 18 | importlib-metadata==3.10.1 19 | intel-openmp== 2021.2.0 20 | joblib==0.11 21 | jpeg==9d 22 | keras==2.3.1 23 | keras-applications==1.0.8 24 | keras-preprocessing==1.1.2 25 | kiwisolver== 1.3.1 26 | krb5==1.17.2 27 | libblas==3.9.0 28 | libcblas==3.9.0 29 | libclang==11.1.0 30 | libcurl==7.76.0 31 | libgpuarray== 0.7.6 32 | liblapack==3.9.0 33 | libpng==1.6.37 34 | libprotobuf==3.15.8 35 | libpython==2.0 36 | libssh2==1.9.0 37 | m2w64-binutils==2.25.1 38 | m2w64-bzip2==1.0.6 39 | m2w64-crt-git==5.0.0.4636.2595836 40 | m2w64-gcc==5.3.0 41 | m2w64-gcc-ada==5.3.0 42 | m2w64-gcc-fortran==5.3.0 43 | m2w64-gcc-libgfortran==5.3.0 44 | m2w64-gcc-libs==5.3.0 45 | m2w64-gcc-libs-core==5.3.0 46 | m2w64-gcc-objc==5.3.0 47 | m2w64-gmp==6.1.0 48 | m2w64-headers-git==5.0.0.4636.c0ad18a 49 | m2w64-isl==0.16.1 50 | m2w64-libiconv==1.14 51 | m2w64-libmangle-git==5.0.0.4509.2e5a9a2 52 | m2w64-libwinpthread-git==5.0.0.4634.697f757 53 | m2w64-make==4.1.2351.a80a8b8 54 | m2w64-mpc==1.0.3 55 | m2w64-mpfr==3.1.4 56 | m2w64-pkg-config==0.29.1 57 | m2w64-toolchain==5.3.0 58 | m2w64-tools-git==5.0.0.4592.90b8472 59 | m2w64-windows-default-manifest 6.4 60 | m2w64-winpthreads-git==5.0.0.4634.697f757 61 | m2w64-zlib==1.2.8 62 | mako==1.0.6 63 | markdown==3.3.4 64 | markupsafe==1.0 65 | matplotlib== 3.2.2 66 | matplotlib-base==3.2.2 67 | mkl==2020.4 68 | mkl-service==1.1.2 69 | msys2-conda-epoch==20160418 70 | numpy==1.19.5 71 | openssl ==1.1.1k 72 | opt_einsum==3.3.0 73 | pandas ==0.20.3 74 | pip==9.0.1 75 | protobuf==3.15.8 76 | pygpu==0.7.6 77 | pyhht==0.1.0 78 | pyparsing==2.2.0 79 | pyqt==5.12.3 80 | pyqt-impl==5.12.3 81 | pyqt5-sip==4.19.18 82 | pyqtchart==5.12 83 | pyqtwebengine==5.12.1 84 | pyreadline==2.1 85 | python==3.6.13 86 | python-dateutil==2.6.1 87 | python_abi==3.6 88 | pytz==2017.2 89 | pywavelets ==1.1.1 90 | pyyaml==3.12 91 | qt==5.12.9 92 | requests==2.25.1 93 | scikit-learn==0.24.1 94 | scipy==1.5.3 95 | setuptools==36.4.0 96 | six==1.15.0 97 | sqlite==3.35.4 98 | tensorboard==1.15.0 99 | tensorflow==1.15.0 100 | tensorflow-base ==1.15.0 101 | tensorflow-estimator==2.4.0 102 | termcolor==1.1.0 103 | theano==0.9.0 104 | threadpoolctl==2.1.0 105 | tk==8.6.10 106 | tornado==4.5.2 107 | typing_extensions==3.7.4.3 108 | urllib3== 1.26.4 109 | vc ==14.2 110 | vs2015_runtime==14.28.29325 111 | werkzeug==0.16.1 112 | wheel== 0.29.0 113 | wincertstore== 0.2 114 | wrapt==1.12.1 115 | zipp==3.4.1 116 | zlib==1.2.11 117 | -------------------------------------------------------------------------------- /IECFONET/_Model/model_major.py: -------------------------------------------------------------------------------- 1 | import time 2 | from keras.callbacks import Callback 3 | from keras.models import Input, Model, Sequential #### TCN 4 | from keras.layers import Dense, Activation, Conv1D, LSTM, Dropout, Reshape, Bidirectional, Flatten, Add, Concatenate, MaxPool1D, LeakyReLU, GRU 5 | import keras.backend as K 6 | 7 | from _Support.support_NLSTM import NestedLSTM 8 | 9 | 10 | class TimeHistory(Callback): 11 | def on_train_begin(self, logs={}): 12 | self.times = [] 13 | self.totaltime = time.time() 14 | 15 | def on_train_end(self, logs={}): 16 | self.totaltime = time.time() - self.totaltime 17 | 18 | def on_epoch_begin(self, batch, logs={}): 19 | self.epoch_time_start = time.time() 20 | 21 | def on_epoch_end(self, batch, logs={}): 22 | self.times.append(time.time() - self.epoch_time_start) 23 | 24 | 25 | def build_GRU(timestep): 26 | 27 | batch_size, timesteps, input_dim = None, timestep, 1 28 | i = Input(batch_shape=(batch_size, timesteps, input_dim)) 29 | 30 | x = GRU(64)(i) 31 | x = Dense(16, activation='linear')(x) 32 | o = Dense(1, activation="linear")(x) 33 | 34 | model = Model(inputs=[i], outputs=[o]) 35 | model.compile(optimizer='rmsprop', loss='mse', ) 36 | model.summary() 37 | 38 | return model 39 | 40 | 41 | def build_LSTM(timestep): 42 | 43 | batch_size, timesteps, input_dim = None, timestep, 1 44 | i = Input(batch_shape=(batch_size, timesteps, input_dim)) 45 | 46 | # x = LSTM(64)(i) 47 | x = LSTM(64)(i) 48 | x = Dense(16, activation='linear')(x) 49 | # o = Dense(1, activation="linear")(x) 50 | o = Dense(1, activation="softmax")(x) 51 | 52 | model = Model(inputs=[i], outputs=[o]) 53 | model.compile(optimizer='rmsprop', loss='mse', ) 54 | model.summary() 55 | 56 | return model 57 | 58 | 59 | def build_SLSTM(timestep): 60 | batch_size, timesteps, input_dim = None, timestep, 1 61 | i = Input(batch_shape=(batch_size, timesteps, input_dim)) 62 | 63 | x = Reshape((-1, 1))(i) 64 | x = LSTM(64, return_sequences=True)(x) 65 | x = LSTM(64, return_sequences=True)(x) 66 | x = LSTM(64)(x) 67 | x = Dense(16, activation='linear')(x) 68 | o = Dense(1, activation="linear")(x) 69 | 70 | model = Model(inputs=[i], outputs=[o]) 71 | 72 | model.compile(optimizer='rmsprop', loss='mse', ) 73 | model.summary() 74 | return model 75 | 76 | 77 | def build_BiLSTM(timestep): 78 | 79 | batch_size, timesteps, input_dim = None, timestep, 1 80 | i = Input(batch_shape=(batch_size, timesteps, input_dim)) 81 | 82 | x = Bidirectional(LSTM(64), merge_mode='concat')(i) 83 | x = Dense(16, activation='linear')(x) 84 | o = Dense(1, activation="linear")(x) 85 | 86 | model = Model(inputs=[i], outputs=[o]) 87 | 88 | model.compile(optimizer='Adam', loss='mse', ) 89 | model.summary() 90 | 91 | return model 92 | 93 | 94 | def build_NLSTM(timestep): 95 | 96 | batch_size, timesteps, input_dim = None, timestep, 1 97 | i = Input(batch_shape=(batch_size, timesteps, input_dim)) 98 | 99 | x = NestedLSTM(64, depth=2, dropout=0.0, recurrent_dropout=0.1)(i) 100 | x = Dense(16, activation='linear')(x) 101 | o = Dense(1, activation="linear")(x) 102 | 103 | model = Model(inputs=[i], outputs=[o]) 104 | model.compile(optimizer='rmsprop', loss='mse', ) 105 | model.summary() 106 | 107 | return model 108 | 109 | def build_MC_BiNLSTM(timestep): 110 | 111 | batch_size, timesteps, input_dim = None, timestep, 1 112 | 113 | input_A3 = Input(batch_shape=(batch_size, timesteps, input_dim)) 114 | x0 = Bidirectional(NestedLSTM(64, depth=2, dropout=0.0, recurrent_dropout=0.1))(input_A3) 115 | x0 = Dense(16, activation='linear')(x0) 116 | model0 = Model(inputs=input_A3, outputs=x0) 117 | 118 | input_D1 = Input(batch_shape=(batch_size, timesteps, input_dim)) 119 | x1 = Bidirectional(NestedLSTM(64, depth=2, dropout=0.0, recurrent_dropout=0.1))(input_D1) 120 | x1 = Dense(16, activation='linear')(x1) 121 | model1 = Model(inputs=input_D1, outputs=x1) 122 | 123 | input_D2 = Input(batch_shape=(batch_size, timesteps, input_dim)) 124 | x2 = Bidirectional(NestedLSTM(64, depth=2, dropout=0.0, recurrent_dropout=0.1))(input_D2) 125 | x2 = Dense(16, activation='linear')(x2) 126 | model2 = Model(inputs=input_D2, outputs=x2) 127 | 128 | input_D3 = Input(batch_shape=(batch_size, timesteps, input_dim)) 129 | x3 = Bidirectional(NestedLSTM(64, depth=2, dropout=0.0, recurrent_dropout=0.1))(input_D3) 130 | x3 = Dense(16, activation='linear')(x3) 131 | model3 = Model(inputs=input_D3, outputs=x3) 132 | 133 | # o = Add()([model0.output, model1.output, model2.output, model3.output]) 134 | 135 | combined = Concatenate(axis=1)([model0.output, model1.output, model2.output, model3.output]) 136 | o = Dense(16, activation="linear")(combined) 137 | 138 | o = Dense(1, activation="linear")(o) 139 | 140 | model = Model(inputs=[model0.input, model1.input, model2.input, model3.input], outputs=o) 141 | 142 | model.compile(optimizer='rmsprop', loss='mse') 143 | model.summary() 144 | 145 | return model -------------------------------------------------------------------------------- /IECFONET/_Support/support_VMD.py: -------------------------------------------------------------------------------- 1 | def VMD(signal, K): 2 | # --------------------- 3 | # signal - the time domain signal (1D) to be decomposed 4 | # alpha - the balancing parameter of the data-fidelity constraint 5 | # tau - time-step of the dual ascent ( pick 0 for noise-slack ) 6 | # K - the number of modes to be recovered 7 | # DC - true if the first mode is put and kept at DC (0-freq) 8 | # init - 0 = all omegas start at 0 9 | # 1 = all omegas start uniformly distributed 10 | # 2 = all omegas initialized randomly 11 | # tol - tolerance of convergence criterion; typically around 1e-6 12 | # 13 | # Output: 14 | # ------- 15 | # u - the collection of decomposed modes 16 | # u_hat - spectra of the modes 17 | # omega - estimated mode center-frequencies 18 | # 19 | 20 | alpha = 2000.0 21 | tau = 0 22 | DC = 0 23 | init = 1 24 | tol = 1e-7 25 | 26 | import numpy as np 27 | import math 28 | import matplotlib.pyplot as plt 29 | # Period and sampling frequency of input signal 30 | save_T=len(signal) 31 | fs=1/float(save_T) 32 | 33 | # extend the signal by mirroring 34 | T=save_T 35 | # print(T) 36 | f_mirror=np.zeros(2*T) 37 | # print(f_mirror) 38 | f_mirror[0:int(T/2)]=signal[int(-(T/2)-1)::-1] 39 | # print(f_mirror) 40 | f_mirror[int(T/2):int(3*T/2)]= signal 41 | # print(f_mirror) 42 | f_mirror[int(3*T/2):int(2*T)]=signal[-1:int(-T/2-1):-1] 43 | # print(f_mirror) 44 | f=f_mirror 45 | # print('f_mirror') 46 | # print(f_mirror) 47 | # print('-------') 48 | 49 | # Time Domain 0 to T (of mirrored signal) 50 | T=float(len(f)) 51 | # print(T) 52 | t=np.linspace(1/float(T),1,int(T),endpoint=True) 53 | # print(t) 54 | 55 | # Spectral Domain discretization 56 | freqs=t-0.5-1/T 57 | # print(freqs) 58 | # print('-----') 59 | # Maximum number of iterations (if not converged yet, then it won't anyway) 60 | N=500 61 | 62 | # For future generalizations: individual alpha for each mode 63 | Alpha=alpha*np.ones(K,dtype=complex) 64 | # print(Alpha.shape) 65 | # print(Alpha) 66 | # print('-----') 67 | 68 | # Construct and center f_hat 69 | f_hat=np.fft.fftshift(np.fft.fft(f)) 70 | # print('f_hat') 71 | # print(f_hat.shape) 72 | # print(f_hat) 73 | # print('-----') 74 | f_hat_plus=f_hat 75 | f_hat_plus[0:int(T/2)]=0 76 | # print('f_hat_plus') 77 | # print(f_hat_plus.shape) 78 | # print(f_hat_plus) 79 | # print('-----') 80 | # matrix keeping track of every iterant // could be discarded for mem 81 | u_hat_plus=np.zeros((N,len(freqs),K),dtype=complex) 82 | # print('u_hat_plus') 83 | # print(u_hat_plus.shape) 84 | # print(u_hat_plus) 85 | # print('-----') 86 | 87 | 88 | # Initialization of omega_k 89 | omega_plus=np.zeros((N,K),dtype=complex) 90 | # print('omega_plus') 91 | # print(omega_plus.shape) 92 | # print(omega_plus) 93 | 94 | if (init==1): 95 | for i in range(1,K+1): 96 | omega_plus[0,i-1]=(0.5/K)*(i-1) 97 | elif (init==2): 98 | omega_plus[0,:]=np.sort(math.exp(math.log(fs))+(math.log(0.5)-math.log(fs))*np.random.rand(1,K)) 99 | else: 100 | omega_plus[0,:]=0 101 | 102 | if (DC): 103 | omega_plus[0,0]=0 104 | 105 | # print('omega_plus') 106 | # print(omega_plus.shape) 107 | # print(omega_plus) 108 | 109 | # start with empty dual variables 110 | lamda_hat=np.zeros((N,len(freqs)),dtype=complex) 111 | 112 | # other inits 113 | uDiff=tol+2.2204e-16 #updata step 114 | # print('uDiff') 115 | # print(uDiff) 116 | # print('----') 117 | n=1 #loop counter 118 | sum_uk=0 #accumulator 119 | 120 | T=int(T) 121 | 122 | 123 | # ----------- Main loop for iterative updates 124 | 125 | while uDiff > tol and n 0: 185 | flag_temp.append(1) 186 | elif (data[i + 1] - data[i]) < 0: 187 | flag_temp.append(2) 188 | else: 189 | flag_temp.append(0) 190 | 191 | return flag_temp 192 | 193 | def size_judge(self): 194 | if self.data.shape[0] < self.total_num: 195 | print('This Dataset is not enough for such trainNum & testNum.') 196 | sys.exit(1) 197 | 198 | def data_preprocessing(self): 199 | 200 | dict_save = {} 201 | 202 | targetData = self.data 203 | 204 | targetData = targetData[self.start_num + 1: self.start_num + self.train_num + self.test_num + 1] 205 | targetData = targetData.reshape(-1, 1) 206 | scaler_target = StandardScaler(copy=True, with_mean=True, with_std=True) 207 | dict_save['scaler_target'] = scaler_target 208 | 209 | targetData = scaler_target.fit_transform(targetData) 210 | 211 | data_regular = self.__preprocessing_regular(targetData) 212 | dict_save['data_regular'] = data_regular 213 | data_EMD = self.__preprocessing_EMD(targetData) 214 | dict_save['data_EMD'] = data_EMD 215 | data_SWT = self.__preprocessing_SWT(targetData) 216 | dict_save['data_SWT'] = data_SWT 217 | data_EWT = self.__preprocessing_EWT(targetData) 218 | dict_save['data_EWT'] = data_EWT 219 | data_VMD = self.__preprocessing_VMD(targetData) 220 | dict_save['data_VMD'] = data_VMD 221 | data_SSA = self.__preprocessing_SSA(targetData) 222 | dict_save['data_SSA'] = data_SSA 223 | 224 | y_real = scaler_target.inverse_transform(data_regular[3]) 225 | y_flag = self.__deal_flag(y_real) 226 | dict_save['y_real'] = y_real 227 | dict_save['y_flag'] = y_flag 228 | 229 | dict_save['name'] = self.name_list[self.order] 230 | dict_save['print'] = self.print_list[self.order] 231 | 232 | np.save('saved\\Dataset'+str(self.order+1)+'_interval'+str(self.interval * 5)+'min_ts'+str(self.step_num)+'.npy', dict_save) 233 | 234 | print(self.name_list[self.order] + ' Data with Interval of ' + str(self.interval * 5) + ' minutes Preprocessed and Saved.\n') 235 | 236 | def EXECUTE(self): 237 | self.size_judge() 238 | self.data_preprocessing() 239 | 240 | 241 | def Data(order, interval, startNum, trainNum, testNum, aheadNum): 242 | 243 | # file = 'dataset\\house' + str(order + 1) + '_5min_KWh.csv' 244 | file= 'datasets\\Dataset' + str(order + 1) + '.csv' 245 | 246 | trainNum = trainNum // interval 247 | print("TrainNum:"+str(trainNum)) 248 | 249 | testNum = (testNum // interval) + aheadNum 250 | print("TestNum:"+str(testNum)) 251 | 252 | dataProcessing(file, order, interval, startNum, trainNum, testNum, aheadNum).EXECUTE() 253 | -------------------------------------------------------------------------------- /IECFONET/_FSOP.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from sklearn import tree 4 | from sklearn import ensemble 5 | from sklearn.neural_network import MLPRegressor 6 | from sklearn import svm 7 | 8 | from _Support.support_SWT import * 9 | from _Model.model_major import * 10 | from _Part.part_evaluate import MAE1, RMSE1, MAPE1, R2 11 | 12 | 13 | class mainFunc_FORECASTING: 14 | def __init__(self, step_num, order, interval): 15 | 16 | # self.dict_load = np.load('saved\\dataset_house'+str(order)+'_interval'+str(interval * 5)+'min_ts'+str(step_num)+'.npy', 17 | # allow_pickle=True).item() 18 | self.dict_load = np.load('saved\\Dataset'+str(order+1)+'_interval'+str(interval * 5)+'min_ts'+str(step_num)+'.npy', 19 | allow_pickle=True).item() 20 | self.data_regular = self.dict_load['data_regular'] 21 | self.data_EMD = self.dict_load['data_EMD'] 22 | self.data_SWT = self.dict_load['data_SWT'] 23 | self.data_EWT = self.dict_load['data_EWT'] 24 | self.data_VMD = self.dict_load['data_VMD'] 25 | self.data_SSA = self.dict_load['data_SSA'] 26 | self.y_real = self.dict_load['y_real'] 27 | self.y_flag = self.dict_load['y_flag'] 28 | self.scaler_target = self.dict_load['scaler_target'] 29 | 30 | self.name = self.dict_load['name'] 31 | self.print_name = self.dict_load['print'] 32 | 33 | self.step_num = step_num 34 | self.order = order 35 | self.interval = interval 36 | 37 | self.result_list = [] 38 | self.result_print = '\nEvaluation.' 39 | self.result_save = [] 40 | self.dict_save = {} 41 | 42 | print(self.name + ' Dataset is Ready.\n') 43 | 44 | ############################################################################### 45 | 46 | def __TrainPredict_ML(self, model, data): 47 | x_train = data[0] 48 | y_train = data[1] 49 | x_test = data[2] 50 | 51 | time_start = time.time() 52 | model.fit(x_train, y_train) 53 | time_end = time.time() 54 | train_time = time_end - time_start 55 | 56 | predict = model.predict(x_test).reshape(-1, ) 57 | return predict, train_time 58 | 59 | def __TrainPredict_DL(self, model, data): 60 | x_train = data[0] 61 | y_train = data[1] 62 | x_test = data[2] 63 | 64 | x_train = x_train.reshape(-1, self.step_num, 1) 65 | x_test = x_test.reshape(-1, self.step_num, 1) 66 | 67 | time_callback = TimeHistory() 68 | history = model.fit(x_train, y_train, epochs=16, validation_split=0.05, verbose=1, 69 | shuffle=False, callbacks=[time_callback]) 70 | print("history.history:{}".format(history.history)) 71 | # plt.plot(history.history['loss']) 72 | # plt.plot(history.history['val_loss']) 73 | # plt.title('model train vs validation loss') 74 | # plt.ylabel('loss') 75 | # plt.xlabel('epoch') 76 | # plt.legend(['train', 'validation'], loc='upper right') 77 | # plt.show() 78 | 79 | train_time = time_callback.totaltime 80 | 81 | predict = model.predict(x_test).reshape(-1, ) 82 | return predict, train_time 83 | 84 | ############################################################################### 85 | 86 | def __deal_flag_accuracy(self, predict): 87 | 88 | predict_flag = [] 89 | 90 | for i_flag in range(len(predict) - 1): 91 | if (predict[i_flag + 1] - predict[i_flag]) > 0: 92 | predict_flag.append(1) 93 | elif (predict[i_flag + 1] - predict[i_flag]) < 0: 94 | predict_flag.append(2) 95 | else: 96 | predict_flag.append(0) 97 | 98 | rightCount = 0 99 | for j_flag in range(len(predict_flag)): 100 | if self.y_flag[j_flag] == predict_flag[j_flag]: 101 | rightCount = rightCount + 1 102 | accuracy = rightCount / len(predict_flag) 103 | 104 | return accuracy * 100 105 | 106 | def __evaluate(self, name, name_short, predict, train_time): 107 | 108 | mae = MAE1(self.y_real, predict) 109 | rmse = RMSE1(self.y_real, predict) 110 | mape = MAPE1(self.y_real, predict) 111 | r2 = R2(self.y_real, predict) 112 | 113 | accuracy = self.__deal_flag_accuracy(predict) 114 | 115 | result_print = '\n\nMAE_' + name + ': {}'.format(mae) 116 | result_print += '\nRMSE_' + name + ': {}'.format(rmse) 117 | result_print += '\nMAPE_' + name + ': {}'.format(mape) 118 | result_print += '\nR2_' + name + ': {}'.format(r2) 119 | result_print += '\nACC_' + name + ': {}'.format(accuracy) 120 | result_print += '\nTIME_' + name + ': {}'.format(train_time) 121 | result_csv = [name_short, mae, rmse, mape, r2, accuracy, train_time] 122 | 123 | self.result_print += result_print 124 | self.result_save.append(result_csv) 125 | 126 | def __postpredict(self, predict, name, name_short, train_time): 127 | predict = self.scaler_target.inverse_transform(predict) 128 | self.__evaluate(name, name_short, predict, train_time) 129 | self.result_list.append([[name, name_short], predict]) 130 | # self.dict_save['predict_'+str(name)] = predict 131 | 132 | ############################################################################### 133 | 134 | def FSOP_ML(self, model, name, name_short, data): 135 | 136 | print(name + ' Start.') 137 | 138 | predict, train_time = self.__TrainPredict_ML(model, data) 139 | self.__postpredict(predict, name, name_short, train_time) 140 | 141 | print(name + ' Complete.\n') 142 | 143 | def FSOP_DL(self, model, name, name_short, data): 144 | 145 | print(name + ' Start.') 146 | 147 | predict, train_time = self.__TrainPredict_DL(model, data) 148 | self.__postpredict(predict, name, name_short, train_time) 149 | 150 | K.clear_session() 151 | del model 152 | print(name + ' Complete.\n') 153 | 154 | def FSOP_DL_loop(self, model, name, name_short, data): 155 | 156 | print(name + ' Start.') 157 | 158 | train_time_total = 0 159 | predict_list = [] 160 | for subSignal_index in range(len(data)): 161 | 162 | print(str(name) + '_level ' + str(subSignal_index + 1) + ' Start.') 163 | 164 | predict, train_time = self.__TrainPredict_DL(model, data[subSignal_index]) 165 | 166 | train_time_total = train_time_total + train_time 167 | predict_list.append(predict) 168 | print(str(name) + '_level ' + str(subSignal_index + 1) + ' Complete.\n') 169 | 170 | predict_list = np.array(predict_list).T.tolist() 171 | for j in range(len(predict_list)): 172 | predict_list[j] = sum(predict_list[j]) 173 | predict = np.array(predict_list).reshape(-1, ) 174 | 175 | self.__postpredict(predict, name, name_short, train_time_total) 176 | 177 | K.clear_session() 178 | del model 179 | print(name + ' Complete.\n') 180 | 181 | def FSOP_DL_loop_SWT(self, model, name, name_short, data): 182 | 183 | print(name + ' Start.') 184 | 185 | train_time_total = 0 186 | predict_list = [] 187 | for subSignal_index in range(len(data)): 188 | print(str(name) + '_level ' + str(subSignal_index + 1) + ' Start.') 189 | 190 | predict, train_time = self.__TrainPredict_DL(model, data[subSignal_index]) 191 | 192 | train_time_total = train_time_total + train_time 193 | predict_list.append(predict) 194 | print(str(name) + '_level ' + str(subSignal_index + 1) + ' Complete.\n') 195 | 196 | wavefun = pywt.Wavelet('db1') 197 | predict = iswt_decom(predict_list, wavefun) 198 | 199 | self.__postpredict(predict, name, name_short, train_time_total) 200 | 201 | K.clear_session() 202 | del model 203 | print(name + ' Complete.\n') 204 | 205 | ############################################################################### 206 | 207 | def Decision_Tree(self): 208 | return tree.DecisionTreeRegressor() 209 | 210 | def Random_Forest(self): 211 | return ensemble.RandomForestRegressor(n_estimators=50) 212 | 213 | def SVR(self): 214 | return svm.SVR() 215 | 216 | def MLP(self): 217 | return MLPRegressor(solver='lbfgs', hidden_layer_sizes=(20, 20, 20), random_state=2) 218 | 219 | def LSTM(self): 220 | return build_LSTM(self.step_num) 221 | 222 | def GRU(self): 223 | return build_GRU(self.step_num) 224 | 225 | def NLSTM(self): 226 | return build_NLSTM(self.step_num) 227 | 228 | def BiLSTM(self): 229 | return build_BiLSTM(self.step_num) 230 | 231 | def SLSTM(self): 232 | return build_SLSTM(self.step_num) 233 | 234 | # def MC_BiNLSTM(self): 235 | # return build_MC_BiNLSTM(self.step_num) 236 | 237 | ############################################################################### 238 | 239 | def EXECUTE(self): 240 | 241 | 242 | self.FSOP_ML(self.Random_Forest(), 'RandomForest', 'RF', self.data_regular) 243 | self.FSOP_ML(self.SVR(), 'SVR', 'SVR', self.data_regular) 244 | 245 | 246 | self.FSOP_DL(self.GRU(), 'GRU', 'GRU', self.data_regular) 247 | 248 | self.FSOP_DL_loop(self.SLSTM(), 'EMD-SLSTM', 'EMD-SLSTM', self.data_EMD) 249 | self.FSOP_DL_loop(self.BiLSTM(), 'EMD-BI-LSTM', 'EMD-BI-LSTM', self.data_EMD) 250 | 251 | 252 | 253 | self.FSOP_DL_loop(self.LSTM(), 'EWT-LSTM', 'EWT-LSTM', self.data_EWT) 254 | 255 | 256 | self.FSOP_DL_loop(self.LSTM(), 'VMD-LSTM', 'VMD-LSTM', self.data_VMD) 257 | 258 | self.FSOP_DL_loop(self.LSTM(), 'SSA-PLSTM', 'SSA-PLSTM', self.data_SSA) 259 | self.FSOP_DL_loop_SWT(self.BiLSTM(), 'SWTBL', 'SWTBL', self.data_SWT) 260 | 261 | 262 | self.dict_save['result_list'] = self.result_list 263 | self.dict_save['result_print'] = self.result_print 264 | self.dict_save['result_save'] = self.result_save 265 | self.dict_save['y_real'] = self.y_real 266 | 267 | self.dict_save['name'] = self.name 268 | self.dict_save['print'] = self.print_name 269 | 270 | np.save('saved\\predict_Dataset'+str(self.order+1)+'_interval'+str(self.interval * 5)+'min_ts'+str(self.step_num)+'.npy', self.dict_save) 271 | 272 | print(self.name + 'Data with Interval of ' + str(self.interval * 5) + ' minutes Train and Predict Complete.\n') 273 | return self.dict_save 274 | 275 | 276 | def FSOP(ahead_num, order, interval): 277 | mainFunc_FORECASTING(ahead_num, order, interval).EXECUTE() 278 | -------------------------------------------------------------------------------- /IECFONET/_Support/support_NLSTM.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import warnings 3 | 4 | from keras import backend as K 5 | from keras import activations 6 | from keras import initializers 7 | from keras import regularizers 8 | from keras import constraints 9 | from keras.engine import Layer 10 | from keras.engine import InputSpec 11 | from keras.legacy import interfaces 12 | from keras.layers import RNN 13 | from keras.layers.recurrent import _generate_dropout_mask 14 | from keras.layers import LSTMCell, LSTM 15 | 16 | 17 | class NestedLSTMCell(Layer): 18 | """Nested NestedLSTM Cell class. 19 | 20 | Derived from the paper [Nested LSTMs](https://arxiv.org/abs/1801.10308) 21 | Ref: [Tensorflow implementation](https://github.com/hannw/nlstm) 22 | 23 | # Arguments 24 | units: Positive integer, dimensionality of the output space. 25 | depth: Depth of nesting of the memory component. 26 | activation: Activation function to use 27 | (see [activations](../activations.md)). 28 | If you pass None, no activation is applied 29 | (ie. "linear" activation: `a(x) = x`). 30 | recurrent_activation: Activation function to use 31 | for the recurrent step 32 | (see [activations](../activations.md)). 33 | cell_activation: Activation function of the first cell gate. 34 | Note that in the paper only the first cell_activation is identity. 35 | (see [activations](../activations.md)). 36 | use_bias: Boolean, whether the layer uses a bias vector. 37 | kernel_initializer: Initializer for the `kernel` weights matrix, 38 | used for the linear transformation of the inputs 39 | (see [initializers](../initializers.md)). 40 | recurrent_initializer: Initializer for the `recurrent_kernel` 41 | weights matrix, 42 | used for the linear transformation of the recurrent state 43 | (see [initializers](../initializers.md)). 44 | bias_initializer: Initializer for the bias vector 45 | (see [initializers](../initializers.md)). 46 | unit_forget_bias: Boolean. 47 | If True, add 1 to the bias of the forget gate at initialization. 48 | Setting it to true will also force `bias_initializer="zeros"`. 49 | This is recommended in [Jozefowicz et al.](http://www.jmlr.org/proceedings/papers/v37/jozefowicz15.pdf) 50 | kernel_regularizer: Regularizer function applied to 51 | the `kernel` weights matrix 52 | (see [regularizer](../regularizers.md)). 53 | recurrent_regularizer: Regularizer function applied to 54 | the `recurrent_kernel` weights matrix 55 | (see [regularizer](../regularizers.md)). 56 | bias_regularizer: Regularizer function applied to the bias vector 57 | (see [regularizer](../regularizers.md)). 58 | kernel_constraint: Constraint function applied to 59 | the `kernel` weights matrix 60 | (see [constraints](../constraints.md)). 61 | recurrent_constraint: Constraint function applied to 62 | the `recurrent_kernel` weights matrix 63 | (see [constraints](../constraints.md)). 64 | bias_constraint: Constraint function applied to the bias vector 65 | (see [constraints](../constraints.md)). 66 | dropout: Float between 0 and 1. 67 | Fraction of the units to drop for 68 | the linear transformation of the inputs. 69 | recurrent_dropout: Float between 0 and 1. 70 | Fraction of the units to drop for 71 | the linear transformation of the recurrent state. 72 | implementation: Implementation mode, must be 2. 73 | Mode 1 will structure its operations as a larger number of 74 | smaller dot products and additions, whereas mode 2 will 75 | batch them into fewer, larger operations. These modes will 76 | have different performance profiles on different hardware and 77 | for different applications. 78 | """ 79 | 80 | def __init__(self, units, depth, 81 | activation='tanh', 82 | # activation='linear', 83 | recurrent_activation='sigmoid', 84 | # recurrent_activation='linear', 85 | cell_activation='linear', 86 | # cell_activation='linear', 87 | use_bias=True, 88 | kernel_initializer='glorot_uniform', 89 | recurrent_initializer='orthogonal', 90 | bias_initializer='zeros', 91 | unit_forget_bias=False, 92 | kernel_regularizer=None, 93 | recurrent_regularizer=None, 94 | bias_regularizer=None, 95 | kernel_constraint=None, 96 | recurrent_constraint=None, 97 | bias_constraint=None, 98 | dropout=0., 99 | recurrent_dropout=0., 100 | implementation=2, 101 | **kwargs): 102 | super(NestedLSTMCell, self).__init__(**kwargs) 103 | 104 | if depth < 1: 105 | raise ValueError("`depth` must be at least 1. For better performance, consider using depth > 1.") 106 | 107 | if implementation != 1: 108 | warnings.warn( 109 | "Nested LSTMs only supports implementation 2 for the moment. Defaulting to implementation = 2") 110 | implementation = 2 111 | 112 | self.units = units 113 | self.depth = depth 114 | self.activation = activations.get(activation) 115 | self.recurrent_activation = activations.get(recurrent_activation) 116 | self.cell_activation = activations.get(cell_activation) 117 | self.use_bias = use_bias 118 | 119 | self.kernel_initializer = initializers.get(kernel_initializer) 120 | self.recurrent_initializer = initializers.get(recurrent_initializer) 121 | self.bias_initializer = initializers.get(bias_initializer) 122 | self.unit_forget_bias = unit_forget_bias 123 | 124 | self.kernel_regularizer = regularizers.get(kernel_regularizer) 125 | self.recurrent_regularizer = regularizers.get(recurrent_regularizer) 126 | self.bias_regularizer = regularizers.get(bias_regularizer) 127 | 128 | self.kernel_constraint = constraints.get(kernel_constraint) 129 | self.recurrent_constraint = constraints.get(recurrent_constraint) 130 | self.bias_constraint = constraints.get(bias_constraint) 131 | 132 | self.dropout = min(1., max(0., dropout)) 133 | self.recurrent_dropout = min(1., max(0., recurrent_dropout)) 134 | self.implementation = implementation 135 | self.state_size = tuple([self.units] * (self.depth + 1)) 136 | self._dropout_mask = None 137 | self._nested_recurrent_masks = None 138 | 139 | def build(self, input_shape): 140 | input_dim = input_shape[-1] 141 | self.kernels = [] 142 | self.biases = [] 143 | 144 | for i in range(self.depth): 145 | if i == 0: 146 | input_kernel = self.add_weight(shape=(input_dim, self.units * 4), 147 | name='input_kernel_%d' % (i + 1), 148 | initializer=self.kernel_initializer, 149 | regularizer=self.kernel_regularizer, 150 | constraint=self.kernel_constraint) 151 | hidden_kernel = self.add_weight(shape=(self.units, self.units * 4), 152 | name='kernel_%d' % (i + 1), 153 | initializer=self.recurrent_initializer, 154 | regularizer=self.recurrent_regularizer, 155 | constraint=self.recurrent_constraint) 156 | kernel = K.concatenate([input_kernel, hidden_kernel], axis=0) 157 | else: 158 | kernel = self.add_weight(shape=(self.units * 2, self.units * 4), 159 | name='kernel_%d' % (i + 1), 160 | initializer=self.recurrent_initializer, 161 | regularizer=self.recurrent_regularizer, 162 | constraint=self.recurrent_constraint) 163 | self.kernels.append(kernel) 164 | 165 | if self.use_bias: 166 | if self.unit_forget_bias: 167 | def bias_initializer(_, *args, **kwargs): 168 | return K.concatenate([ 169 | self.bias_initializer((self.units,), *args, **kwargs), 170 | initializers.Ones()((self.units,), *args, **kwargs), 171 | self.bias_initializer((self.units * 2,), *args, **kwargs), 172 | ]) 173 | else: 174 | bias_initializer = self.bias_initializer 175 | 176 | for i in range(self.depth): 177 | bias = self.add_weight(shape=(self.units * 4,), 178 | name='bias_%d' % (i + 1), 179 | initializer=bias_initializer, 180 | regularizer=self.bias_regularizer, 181 | constraint=self.bias_constraint) 182 | self.biases.append(bias) 183 | else: 184 | self.biases = None 185 | 186 | self.built = True 187 | 188 | def call(self, inputs, states, training=None): 189 | if 0 < self.dropout < 1 and self._dropout_mask is None: 190 | self._dropout_mask = _generate_dropout_mask( 191 | K.ones_like(inputs), 192 | self.dropout, 193 | training=training, 194 | count=1) 195 | if (0 < self.recurrent_dropout < 1 and 196 | self._nested_recurrent_masks is None): 197 | _nested_recurrent_mask = _generate_dropout_mask( 198 | K.ones_like(states[0]), 199 | self.recurrent_dropout, 200 | training=training, 201 | count=self.depth) 202 | self._nested_recurrent_masks = _nested_recurrent_mask 203 | 204 | # dropout matrices for input units 205 | dp_mask = self._dropout_mask 206 | # dropout matrices for recurrent units 207 | rec_dp_masks = self._nested_recurrent_masks 208 | 209 | h_tm1 = states[0] # previous memory state 210 | c_tm1 = states[1:self.depth + 1] # previous carry states 211 | 212 | if 0. < self.dropout < 1.: 213 | inputs *= dp_mask[0] 214 | 215 | h, c = self.nested_recurrence(inputs, 216 | hidden_state=h_tm1, 217 | cell_states=c_tm1, 218 | recurrent_masks=rec_dp_masks, 219 | current_depth=0) 220 | 221 | if 0 < self.dropout + self.recurrent_dropout: 222 | if training is None: 223 | h._uses_learning_phase = True 224 | return h, c 225 | 226 | def nested_recurrence(self, inputs, hidden_state, cell_states, recurrent_masks, current_depth): 227 | h_state = hidden_state 228 | c_state = cell_states[current_depth] 229 | 230 | if 0.0 < self.recurrent_dropout <= 1. and recurrent_masks is not None: 231 | hidden_state = h_state * recurrent_masks[current_depth] 232 | 233 | ip = K.concatenate([inputs, hidden_state], axis=-1) 234 | gate_inputs = K.dot(ip, self.kernels[current_depth]) 235 | 236 | if self.use_bias: 237 | gate_inputs = K.bias_add(gate_inputs, self.biases[current_depth]) 238 | 239 | i = gate_inputs[:, :self.units] # input gate 240 | f = gate_inputs[:, self.units * 2: self.units * 3] # forget gate 241 | c = gate_inputs[:, self.units: 2 * self.units] # new input 242 | o = gate_inputs[:, self.units * 3: self.units * 4] # output gate 243 | 244 | inner_hidden = c_state * self.recurrent_activation(f) 245 | 246 | if current_depth == 0: 247 | inner_input = self.recurrent_activation(i) + self.cell_activation(c) 248 | else: 249 | inner_input = self.recurrent_activation(i) + self.activation(c) 250 | 251 | if (current_depth == self.depth - 1): 252 | new_c = inner_hidden + inner_input 253 | new_cs = [new_c] 254 | else: 255 | new_c, new_cs = self.nested_recurrence(inner_input, 256 | hidden_state=inner_hidden, 257 | cell_states=cell_states, 258 | recurrent_masks=recurrent_masks, 259 | current_depth=current_depth + 1) 260 | 261 | new_h = self.activation(new_c) * self.recurrent_activation(o) 262 | new_cs = [new_h] + new_cs 263 | 264 | return new_h, new_cs 265 | 266 | def get_config(self): 267 | config = {'units': self.units, 268 | 'depth': self.depth, 269 | 'activation': activations.serialize(self.activation), 270 | 'recurrent_activation': activations.serialize(self.recurrent_activation), 271 | 'cell_activation': activations.serialize(self.cell_activation), 272 | 'use_bias': self.use_bias, 273 | 'kernel_initializer': initializers.serialize(self.kernel_initializer), 274 | 'recurrent_initializer': initializers.serialize(self.recurrent_initializer), 275 | 'bias_initializer': initializers.serialize(self.bias_initializer), 276 | 'unit_forget_bias': self.unit_forget_bias, 277 | 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), 278 | 'recurrent_regularizer': regularizers.serialize(self.recurrent_regularizer), 279 | 'bias_regularizer': regularizers.serialize(self.bias_regularizer), 280 | 'kernel_constraint': constraints.serialize(self.kernel_constraint), 281 | 'recurrent_constraint': constraints.serialize(self.recurrent_constraint), 282 | 'bias_constraint': constraints.serialize(self.bias_constraint), 283 | 'dropout': self.dropout, 284 | 'recurrent_dropout': self.recurrent_dropout, 285 | 'implementation': self.implementation} 286 | base_config = super(NestedLSTMCell, self).get_config() 287 | return dict(list(base_config.items()) + list(config.items())) 288 | 289 | 290 | class NestedLSTM(RNN): 291 | """Nested Long-Short-Term-Memory layer - [Nested LSTMs](https://arxiv.org/abs/1801.10308). 292 | 293 | # Arguments 294 | units: Positive integer, dimensionality of the output space. 295 | depth: Depth of nesting of the memory component. 296 | activation: Activation function to use 297 | (see [activations](../activations.md)). 298 | If you pass None, no activation is applied 299 | (ie. "linear" activation: `a(x) = x`). 300 | recurrent_activation: Activation function to use 301 | for the recurrent step 302 | (see [activations](../activations.md)). 303 | cell_activation: Activation function of the first cell gate. 304 | Note that in the paper only the first cell_activation is identity. 305 | (see [activations](../activations.md)). 306 | use_bias: Boolean, whether the layer uses a bias vector. 307 | kernel_initializer: Initializer for the `kernel` weights matrix, 308 | used for the linear transformation of the inputs. 309 | (see [initializers](../initializers.md)). 310 | recurrent_initializer: Initializer for the `recurrent_kernel` 311 | weights matrix, 312 | used for the linear transformation of the recurrent state. 313 | (see [initializers](../initializers.md)). 314 | bias_initializer: Initializer for the bias vector 315 | (see [initializers](../initializers.md)). 316 | unit_forget_bias: Boolean. 317 | If True, add 1 to the bias of the forget gate at initialization. 318 | Setting it to true will also force `bias_initializer="zeros"`. 319 | This is recommended in [Jozefowicz et al.](http://www.jmlr.org/proceedings/papers/v37/jozefowicz15.pdf) 320 | kernel_regularizer: Regularizer function applied to 321 | the `kernel` weights matrix 322 | (see [regularizer](../regularizers.md)). 323 | recurrent_regularizer: Regularizer function applied to 324 | the `recurrent_kernel` weights matrix 325 | (see [regularizer](../regularizers.md)). 326 | bias_regularizer: Regularizer function applied to the bias vector 327 | (see [regularizer](../regularizers.md)). 328 | activity_regularizer: Regularizer function applied to 329 | the output of the layer (its "activation"). 330 | (see [regularizer](../regularizers.md)). 331 | kernel_constraint: Constraint function applied to 332 | the `kernel` weights matrix 333 | (see [constraints](../constraints.md)). 334 | recurrent_constraint: Constraint function applied to 335 | the `recurrent_kernel` weights matrix 336 | (see [constraints](../constraints.md)). 337 | bias_constraint: Constraint function applied to the bias vector 338 | (see [constraints](../constraints.md)). 339 | dropout: Float between 0 and 1. 340 | Fraction of the units to drop for 341 | the linear transformation of the inputs. 342 | recurrent_dropout: Float between 0 and 1. 343 | Fraction of the units to drop for 344 | the linear transformation of the recurrent state. 345 | implementation: Implementation mode, either 1 or 2. 346 | Mode 1 will structure its operations as a larger number of 347 | smaller dot products and additions, whereas mode 2 will 348 | batch them into fewer, larger operations. These modes will 349 | have different performance profiles on different hardware and 350 | for different applications. 351 | return_sequences: Boolean. Whether to return the last output. 352 | in the output sequence, or the full sequence. 353 | return_state: Boolean. Whether to return the last state 354 | in addition to the output. 355 | go_backwards: Boolean (default False). 356 | If True, process the input sequence backwards and return the 357 | reversed sequence. 358 | stateful: Boolean (default False). If True, the last state 359 | for each sample at index i in a batch will be used as initial 360 | state for the sample of index i in the following batch. 361 | unroll: Boolean (default False). 362 | If True, the network will be unrolled, 363 | else a symbolic loop will be used. 364 | Unrolling can speed-up a RNN, 365 | although it tends to be more memory-intensive. 366 | Unrolling is only suitable for short sequences. 367 | 368 | # References 369 | - [Long short-term memory](http://www.bioinf.jku.at/publications/older/2604.pdf) (original 1997 paper) 370 | - [Learning to forget: Continual prediction with NestedLSTM](http://www.mitpressjournals.org/doi/pdf/10.1162/089976600300015015) 371 | - [Supervised sequence labeling with recurrent neural networks](http://www.cs.toronto.edu/~graves/preprint.pdf) 372 | - [A Theoretically Grounded Application of Dropout in Recurrent Neural Networks](http://arxiv.org/abs/1512.05287) 373 | - [Nested LSTMs](https://arxiv.org/abs/1801.10308) 374 | """ 375 | 376 | @interfaces.legacy_recurrent_support 377 | def __init__(self, units, depth, 378 | activation='tanh', 379 | # activation='linear', 380 | recurrent_activation='sigmoid', 381 | # recurrent_activation='linear', 382 | cell_activation='linear', 383 | # cell_activation='linear', 384 | use_bias=True, 385 | kernel_initializer='glorot_uniform', 386 | recurrent_initializer='orthogonal', 387 | bias_initializer='zeros', 388 | unit_forget_bias=False, 389 | kernel_regularizer=None, 390 | recurrent_regularizer=None, 391 | bias_regularizer=None, 392 | activity_regularizer=None, 393 | kernel_constraint=None, 394 | recurrent_constraint=None, 395 | bias_constraint=None, 396 | dropout=0., 397 | recurrent_dropout=0., 398 | implementation=1, 399 | return_sequences=False, 400 | return_state=False, 401 | go_backwards=False, 402 | stateful=False, 403 | unroll=False, 404 | **kwargs): 405 | if implementation == 0: 406 | warnings.warn('`implementation=0` has been deprecated, ' 407 | 'and now defaults to `implementation=2`.' 408 | 'Please update your layer call.') 409 | if K.backend() == 'theano': 410 | warnings.warn( 411 | 'RNN dropout is no longer supported with the Theano backend ' 412 | 'due to technical limitations. ' 413 | 'You can either set `dropout` and `recurrent_dropout` to 0, ' 414 | 'or use the TensorFlow backend.') 415 | dropout = 0. 416 | recurrent_dropout = 0. 417 | 418 | cell = NestedLSTMCell(units, depth, 419 | activation=activation, 420 | recurrent_activation=recurrent_activation, 421 | cell_activation=cell_activation, 422 | use_bias=use_bias, 423 | kernel_initializer=kernel_initializer, 424 | recurrent_initializer=recurrent_initializer, 425 | unit_forget_bias=unit_forget_bias, 426 | bias_initializer=bias_initializer, 427 | kernel_regularizer=kernel_regularizer, 428 | recurrent_regularizer=recurrent_regularizer, 429 | bias_regularizer=bias_regularizer, 430 | kernel_constraint=kernel_constraint, 431 | recurrent_constraint=recurrent_constraint, 432 | bias_constraint=bias_constraint, 433 | dropout=dropout, 434 | recurrent_dropout=recurrent_dropout, 435 | implementation=implementation) 436 | super(NestedLSTM, self).__init__(cell, 437 | return_sequences=return_sequences, 438 | return_state=return_state, 439 | go_backwards=go_backwards, 440 | stateful=stateful, 441 | unroll=unroll, 442 | **kwargs) 443 | self.activity_regularizer = regularizers.get(activity_regularizer) 444 | 445 | def call(self, inputs, mask=None, training=None, initial_state=None, constants=None): 446 | self.cell._dropout_mask = None 447 | self.cell._nested_recurrent_masks = None 448 | return super(NestedLSTM, self).call(inputs, 449 | mask=mask, 450 | training=training, 451 | initial_state=initial_state, 452 | constants=constants) 453 | 454 | @property 455 | def units(self): 456 | return self.cell.units 457 | 458 | @property 459 | def depth(self): 460 | return self.cell.depth 461 | 462 | @property 463 | def activation(self): 464 | return self.cell.activation 465 | 466 | @property 467 | def recurrent_activation(self): 468 | return self.cell.recurrent_activation 469 | 470 | @property 471 | def cell_activation(self): 472 | return self.cell.cell_activation 473 | 474 | @property 475 | def use_bias(self): 476 | return self.cell.use_bias 477 | 478 | @property 479 | def kernel_initializer(self): 480 | return self.cell.kernel_initializer 481 | 482 | @property 483 | def recurrent_initializer(self): 484 | return self.cell.recurrent_initializer 485 | 486 | @property 487 | def bias_initializer(self): 488 | return self.cell.bias_initializer 489 | 490 | @property 491 | def unit_forget_bias(self): 492 | return self.cell.unit_forget_bias 493 | 494 | @property 495 | def kernel_regularizer(self): 496 | return self.cell.kernel_regularizer 497 | 498 | @property 499 | def recurrent_regularizer(self): 500 | return self.cell.recurrent_regularizer 501 | 502 | @property 503 | def bias_regularizer(self): 504 | return self.cell.bias_regularizer 505 | 506 | @property 507 | def kernel_constraint(self): 508 | return self.cell.kernel_constraint 509 | 510 | @property 511 | def recurrent_constraint(self): 512 | return self.cell.recurrent_constraint 513 | 514 | @property 515 | def bias_constraint(self): 516 | return self.cell.bias_constraint 517 | 518 | @property 519 | def dropout(self): 520 | return self.cell.dropout 521 | 522 | @property 523 | def recurrent_dropout(self): 524 | return self.cell.recurrent_dropout 525 | 526 | @property 527 | def implementation(self): 528 | return self.cell.implementation 529 | 530 | def get_config(self): 531 | config = {'units': self.units, 532 | 'depth': self.depth, 533 | 'activation': activations.serialize(self.activation), 534 | 'recurrent_activation': activations.serialize(self.recurrent_activation), 535 | 'cell_activation': activations.serialize(self.cell_activation), 536 | 'use_bias': self.use_bias, 537 | 'kernel_initializer': initializers.serialize(self.kernel_initializer), 538 | 'recurrent_initializer': initializers.serialize(self.recurrent_initializer), 539 | 'bias_initializer': initializers.serialize(self.bias_initializer), 540 | 'unit_forget_bias': self.unit_forget_bias, 541 | 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), 542 | 'recurrent_regularizer': regularizers.serialize(self.recurrent_regularizer), 543 | 'bias_regularizer': regularizers.serialize(self.bias_regularizer), 544 | 'activity_regularizer': regularizers.serialize(self.activity_regularizer), 545 | 'kernel_constraint': constraints.serialize(self.kernel_constraint), 546 | 'recurrent_constraint': constraints.serialize(self.recurrent_constraint), 547 | 'bias_constraint': constraints.serialize(self.bias_constraint), 548 | 'dropout': self.dropout, 549 | 'recurrent_dropout': self.recurrent_dropout, 550 | 'implementation': self.implementation} 551 | base_config = super(NestedLSTM, self).get_config() 552 | del base_config['cell'] 553 | return dict(list(base_config.items()) + list(config.items())) 554 | 555 | @classmethod 556 | def from_config(cls, config): 557 | if 'implementation' in config and config['implementation'] == 0: 558 | config['implementation'] = 2 559 | return cls(**config) 560 | --------------------------------------------------------------------------------