├── .gitignore ├── README.md ├── execute_model.py ├── load-forecasting.py ├── train_model.py ├── utils.py └── valohai.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Load Forecasting Example on Valohai 2 | 3 | Blog post: [Valohai Blog - Load Forecasting using Machine Learning][blog]. 4 | 5 | This serves as an example repository for the [Valohai machine learning platform][vh]. [Create a free account](https://app.valohai.com/accounts/signup/) and run your experiments in the cloud immediately. 6 | 7 | [blog]: https://blog.valohai.com/smart-grids-use-machine-learning-to-forecast-load 8 | [vh]: https://valohai.com/ 9 | -------------------------------------------------------------------------------- /execute_model.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | import json 4 | import numpy as np 5 | from math import sqrt 6 | from keras.models import model_from_json 7 | from sklearn.metrics import mean_absolute_error 8 | import argparse 9 | from utils import feature_extraction, split_features 10 | 11 | 12 | def mean_absolute_percentage_error(y_true, y_pred): 13 | y_true, y_pred = np.array(y_true), np.array(y_pred) 14 | return np.mean(np.abs((y_true - y_pred) / y_true)) * 100 15 | 16 | 17 | def main(settings): 18 | features, minima, maxima, scaling_parameter = feature_extraction(settings.dataset_dir) 19 | window = 5 20 | X_train, y_train, X_test, y_test = split_features(features[::-1], window) 21 | print("X_train", X_train.shape) 22 | print("y_train", y_train.shape) 23 | print("X_test", X_test.shape) 24 | print("y_test", y_test.shape) 25 | 26 | # load json and create model 27 | layout_path = glob.glob(os.path.join(settings.model_dir, "*layout.json"))[0] 28 | json_file = open(layout_path, 'r') 29 | loaded_model_json = json_file.read() 30 | json_file.close() 31 | model = model_from_json(loaded_model_json) 32 | 33 | # load weights into new model 34 | weights_path = glob.glob(os.path.join(settings.model_dir, "*weights.h5"))[0] 35 | model.load_weights(weights_path) 36 | print("Loaded model from disk") 37 | 38 | predicted2 = model.predict(X_test) 39 | actual = y_test 40 | predicted2 = (predicted2 * scaling_parameter) + minima 41 | actual = (actual * scaling_parameter) + minima 42 | 43 | mape2 = sqrt(mean_absolute_percentage_error(predicted2, actual)) 44 | mse2 = mean_absolute_error(actual, predicted2) 45 | 46 | print(json.dumps({ 47 | "mape": mape2, 48 | "mse": mse2 49 | })) 50 | 51 | 52 | def cli(): 53 | parser = argparse.ArgumentParser() 54 | parser.add_argument("--dataset_dir", type=str, default="/valohai/inputs/dataset") 55 | parser.add_argument("--model_dir", type=str, default="/valohai/inputs/model") 56 | settings = parser.parse_args() 57 | main(settings) 58 | 59 | 60 | if __name__ == "__main__": 61 | cli() 62 | -------------------------------------------------------------------------------- /load-forecasting.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import pandas as pd 4 | from pandas import datetime 5 | import math, time 6 | import itertools 7 | from sklearn import preprocessing 8 | import datetime 9 | from operator import itemgetter 10 | from sklearn.metrics import mean_squared_error 11 | from math import sqrt 12 | from keras.models import Sequential 13 | from keras.models import model_from_json 14 | from keras.layers.core import Dense, Dropout, Activation 15 | from keras.layers.recurrent import LSTM 16 | from sklearn.metrics import mean_absolute_error 17 | 18 | def get_data(normalized=0): 19 | my_data = pd.read_csv('rte.csv', header=1, error_bad_lines=False) 20 | df = pd.DataFrame(my_data) 21 | df.drop(df.columns[[0, 3, 7, 8]], axis=1, inplace=True) 22 | # df.drop(df.index[1000:]) 23 | return df 24 | 25 | def load_data(my_data, seq_len): 26 | amount_of_features = len(my_data.columns) 27 | data = my_data.as_matrix() 28 | sequence_length = seq_len + 1 29 | result = [] 30 | for index in range(len(data) - sequence_length): 31 | result.append(data[index: index + sequence_length]) 32 | 33 | result = np.array(result) 34 | row = round(0.8 * result.shape[0]) 35 | train = result[:int(row), :] 36 | x_train = train[:, :-1] 37 | y_train = train[:, -1][:,-1] 38 | x_test = result[int(row):, :-1] 39 | y_test = result[int(row):, -1][:,-1] 40 | x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], amount_of_features)) 41 | x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], amount_of_features)) 42 | print(x_train[:3]) 43 | print(y_train[:3]) 44 | return [x_train, y_train, x_test, y_test] 45 | 46 | 47 | df = get_data(0) 48 | # min_max_scaler = preprocessing.MinMaxScaler() 49 | # nomalized = min_max_scaler.fit_transform(df.values) 50 | values = df.values 51 | minima = np.amin(values[:, -1]) 52 | maxima = np.amax(values[:, -1]) 53 | scaling_parameter = maxima - minima 54 | values[:, 4] = (values[:, 4]-minima)/scaling_parameter 55 | values[:, 0] = (values[:, 0]-np.amin(values[:, 0]))/(np.amax(values[:, 0])-np.amin(values[:, 0])) 56 | values[:, 1] = (values[:, 1]-np.amin(values[:, 1]))/(np.amax(values[:, 1])-np.amin(values[:, 1])) 57 | values[:, 2] = (values[:, 2]-np.amin(values[:, 2]))/(np.amax(values[:, 2])-np.amin(values[:, 2])) 58 | values[:, 3] = (values[:, 3]-np.amin(values[:, 3]))/(np.amax(values[:, 3])-np.amin(values[:, 3])) 59 | 60 | df = pd.DataFrame(values) 61 | window = 5 62 | X_train, y_train, X_test, y_test = load_data(df[::-1], window) 63 | print("X_train", X_train.shape) 64 | print("y_train", y_train.shape) 65 | print("X_test", X_test.shape) 66 | print("y_test", y_test.shape) 67 | 68 | def build_model(layers): 69 | model = Sequential() 70 | model.add(LSTM(50,input_shape=(layers[1], layers[0]),return_sequences=False)) 71 | model.add(Dense(1, activation="relu", kernel_initializer="uniform")) 72 | 73 | start = time.time() 74 | model.compile(loss="mse", optimizer="rmsprop",metrics=['accuracy']) 75 | print("Compilation Time : ", time.time() - start) 76 | return model 77 | model = build_model([5,window,1]) 78 | print("----------------") 79 | print(X_train.shape) 80 | model.fit( 81 | X_train, 82 | y_train, 83 | batch_size=1024, 84 | epochs=100, 85 | validation_split=0.2, 86 | verbose=2) 87 | 88 | # serialize model to JSON 89 | model_json = model.to_json() 90 | with open("model2.json", "w") as json_file: 91 | json_file.write(model_json) 92 | # serialize weights to HDF5 93 | model.save_weights("model2.h5") 94 | print("Saved model to disk") 95 | 96 | def build_model2(layers): 97 | d = 0.2 98 | model = Sequential() 99 | model.add(LSTM(128, input_shape=(layers[1], layers[0]), return_sequences=True)) 100 | model.add(Dropout(d)) 101 | model.add(LSTM(64, input_shape=(layers[1], layers[0]), return_sequences=False)) 102 | model.add(Dropout(d)) 103 | model.add(Dense(16, activation="relu", kernel_initializer="uniform")) 104 | model.add(Dense(1, activation="relu", kernel_initializer="uniform")) 105 | model.compile(loss='mse',optimizer='adam',metrics=['accuracy']) 106 | return model 107 | 108 | 109 | model2 = build_model2([5,window,1]) 110 | model2.fit( 111 | X_train, 112 | y_train, 113 | batch_size=1024, 114 | epochs=50, 115 | validation_split=0.2, 116 | verbose=2) 117 | 118 | from matplotlib import pyplot 119 | def mean_absolute_percentage_error(y_true, y_pred): 120 | y_true, y_pred = np.array(y_true), np.array(y_pred) 121 | return np.mean(np.abs((y_true - y_pred) / y_true)) * 100 122 | 123 | 124 | predicted1 = model.predict(X_test) 125 | predicted2 = model2.predict(X_test) 126 | actual = y_test 127 | 128 | 129 | 130 | predicted1 = (predicted1*scaling_parameter)+minima 131 | predicted2 = (predicted2*scaling_parameter)+minima 132 | actual = (actual*scaling_parameter)+minima 133 | 134 | mape1 = sqrt(mean_absolute_percentage_error(predicted1, actual)) 135 | mape2 = sqrt(mean_absolute_percentage_error(predicted2, actual)) 136 | print('Test MAPE1: %.3f' % mape1) 137 | print('Test MAPE2: %.3f' % mape2) 138 | mse1 = mean_absolute_error(actual, predicted1) 139 | mse2 = mean_absolute_error(actual, predicted2) 140 | print('Test MSE1: %.3f' % mse1) 141 | print('Test MSE2: %.3f' % mse2) 142 | 143 | 144 | 145 | pyplot.plot(actual[:24], label='Actual', color="blue") 146 | pyplot.plot(predicted1[:24], label='Model1', color="red") 147 | pyplot.plot(predicted2[:24], label='Model2', color="green") 148 | #pyplot.xlabel("Timestamp") 149 | pyplot.ylabel("Load(MW)") 150 | pyplot.title("Actual Vs Predicted Results") 151 | pyplot.legend() 152 | pyplot.show() 153 | -------------------------------------------------------------------------------- /train_model.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import argparse 4 | from keras.layers.core import Dense, Dropout 5 | from keras.layers.recurrent import LSTM 6 | from keras.models import Sequential 7 | from keras.callbacks import LambdaCallback 8 | from utils import feature_extraction, split_features 9 | 10 | 11 | def build_single_lstm(layers): 12 | model = Sequential() 13 | model.add(LSTM(50, input_shape=(layers[1], layers[0]), return_sequences=False)) 14 | model.add(Dense(1, activation="relu", kernel_initializer="uniform")) 15 | model.compile(loss="mse", optimizer="rmsprop", metrics=["accuracy"]) 16 | return model 17 | 18 | 19 | def build_double_lstm(layers): 20 | dropout = 0.2 21 | model = Sequential() 22 | model.add(LSTM(128, input_shape=(layers[1], layers[0]), return_sequences=True)) 23 | model.add(Dropout(dropout)) 24 | model.add(LSTM(64, input_shape=(layers[1], layers[0]), return_sequences=False)) 25 | model.add(Dropout(dropout)) 26 | model.add(Dense(16, activation="relu", kernel_initializer="uniform")) 27 | model.add(Dense(1, activation="relu", kernel_initializer="uniform")) 28 | model.compile(loss="mse", optimizer="adam", metrics=["accuracy"]) 29 | return model 30 | 31 | 32 | model_architectures = { 33 | "single_lstm": build_single_lstm, 34 | "double_lstm": build_double_lstm, 35 | } 36 | 37 | 38 | def main(settings): 39 | features, minima, maxima, scaling_parameter = feature_extraction(settings.dataset_dir) 40 | window = 5 41 | X_train, y_train, X_test, y_test = split_features(features[::-1], window) 42 | print("X_train", X_train.shape) 43 | print("y_train", y_train.shape) 44 | print("X_test", X_test.shape) 45 | print("y_test", y_test.shape) 46 | 47 | json_logging_callback = LambdaCallback( 48 | on_epoch_end=lambda epoch, logs: print(json.dumps({ 49 | "epoch": epoch, 50 | "loss": logs["loss"], 51 | "acc": logs["acc"], 52 | "val_loss": logs["val_loss"], 53 | "val_acc": logs["val_acc"], 54 | })), 55 | ) 56 | 57 | # figure out which model architecture to use 58 | arch = settings.model_architecture 59 | assert arch in model_architectures, "Unknown model architecture '%s'." % arch 60 | builder = model_architectures[arch] 61 | 62 | # build and train the model 63 | model = builder([len(features.columns), window, 1]) 64 | model.fit( 65 | X_train, 66 | y_train, 67 | batch_size=settings.batch_size, 68 | epochs=settings.epochs, 69 | validation_split=settings.validation_split, 70 | callbacks=[json_logging_callback], 71 | verbose=0) 72 | 73 | # serialize model to JSON 74 | model_json = model.to_json() 75 | with open(os.path.join(settings.output_dir, "model-layout.json"), "w") as json_file: 76 | json_file.write(model_json) 77 | 78 | # serialize weights to HDF5 79 | model.save_weights(os.path.join(settings.output_dir, "model-weights.h5")) 80 | print("Saved model to disk") 81 | 82 | 83 | def cli(): 84 | parser = argparse.ArgumentParser() 85 | parser.add_argument("--epochs", type=int, required=True) 86 | parser.add_argument("--batch_size", type=int, required=True) 87 | parser.add_argument("--validation_split", type=float, required=True) 88 | parser.add_argument("--model_architecture", type=str, required=True, help="'single_lstm' or 'double_lstm'") 89 | parser.add_argument("--dataset_dir", type=str, default="/valohai/inputs/dataset") 90 | parser.add_argument("--output_dir", type=str, default="/valohai/outputs") 91 | settings = parser.parse_args() 92 | main(settings) 93 | 94 | 95 | if __name__ == "__main__": 96 | cli() 97 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | import pandas as pd 4 | import numpy as np 5 | 6 | 7 | def get_dataframe(dataset_dir): 8 | """ 9 | :param dataset_dir: Path to directory containing dataset as a .csv file 10 | :return: Panda DataFrame 11 | """ 12 | 13 | # use the csv first file in the dataset directory 14 | dataset_path = glob.glob(os.path.join(dataset_dir, "*.csv"))[0] 15 | 16 | my_data = pd.read_csv(dataset_path, error_bad_lines=False) 17 | df = pd.DataFrame(my_data) 18 | 19 | column_names = list(df) 20 | if 'Demand' in column_names: 21 | # RTE dataset format 22 | df = df.filter(items=['Day', 'Month', 'Hours', 'Temperature', 'Demand']) 23 | return df, 'rte' 24 | elif 'SYSLoad' in column_names: 25 | # ERCOT dataset format. 26 | df = df.filter(items=['Day', 'Month', 'Minutes', 'SYSLoad']) 27 | return df, 'ercot' 28 | else: 29 | raise Exception('Unknown dataset format with columns: {}'.format(column_names)) 30 | 31 | 32 | def feature_extraction(dataset_dir): 33 | df, dataset_format = get_dataframe(dataset_dir) 34 | 35 | values = df.values 36 | minima = np.amin(values[:, -1]) 37 | maxima = np.amax(values[:, -1]) 38 | scaling_parameter = maxima - minima 39 | 40 | if dataset_format == 'rte': 41 | values[:, 0] = (values[:, 0] - np.amin(values[:, 0])) / (np.amax(values[:, 0]) - np.amin(values[:, 0])) 42 | values[:, 1] = (values[:, 1] - np.amin(values[:, 1])) / (np.amax(values[:, 1]) - np.amin(values[:, 1])) 43 | values[:, 2] = (values[:, 2] - np.amin(values[:, 2])) / (np.amax(values[:, 2]) - np.amin(values[:, 2])) 44 | values[:, 3] = (values[:, 3] - np.amin(values[:, 3])) / (np.amax(values[:, 3]) - np.amin(values[:, 3])) 45 | values[:, 4] = (values[:, 4] - minima) / scaling_parameter 46 | elif dataset_format == 'ercot': 47 | values[:, 0] = (values[:, 0] - np.amin(values[:, 0])) / (np.amax(values[:, 0]) - np.amin(values[:, 0])) 48 | values[:, 1] = (values[:, 1] - np.amin(values[:, 1])) / (np.amax(values[:, 1]) - np.amin(values[:, 1])) 49 | values[:, 2] = (values[:, 2] - np.amin(values[:, 2])) / (np.amax(values[:, 2]) - np.amin(values[:, 2])) 50 | values[:, 3] = (values[:, 3] - minima) / scaling_parameter 51 | 52 | df = pd.DataFrame(values) 53 | return df, minima, maxima, scaling_parameter 54 | 55 | 56 | def split_features(features_data_frame, seq_len): 57 | amount_of_features = len(features_data_frame.columns) 58 | data = features_data_frame.as_matrix() 59 | sequence_length = seq_len + 1 60 | result = [] 61 | for index in range(len(data) - sequence_length): 62 | result.append(data[index: index + sequence_length]) 63 | 64 | result = np.array(result) 65 | row = round(0.8 * result.shape[0]) 66 | train = result[:int(row), :] 67 | x_train = train[:, :-1] 68 | y_train = train[:, -1][:, -1] 69 | x_test = result[int(row):, :-1] 70 | y_test = result[int(row):, -1][:, -1] 71 | x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], amount_of_features)) 72 | x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], amount_of_features)) 73 | return [x_train, y_train, x_test, y_test] 74 | -------------------------------------------------------------------------------- /valohai.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - step: 4 | name: train-model 5 | image: valohai/keras:2.1.3-tensorflow1.4.0-python3.5-cuda8.0-cudnn6-devel-ubuntu14.04 6 | command: 7 | - python train_model.py {parameters} 8 | parameters: 9 | - name: epochs 10 | type: integer 11 | default: 50 12 | - name: batch_size 13 | type: integer 14 | default: 1024 15 | - name: validation_split 16 | type: float 17 | default: 0.2 18 | - name: model_architecture 19 | description: Use single_lstm or double_lstm 20 | type: string 21 | default: single_lstm 22 | inputs: 23 | - name: dataset 24 | default: https://s3-eu-west-1.amazonaws.com/valohai-examples/load-forecasting/rte-dataset.csv 25 | 26 | - step: 27 | name: execute-model 28 | image: valohai/keras:2.1.3-tensorflow1.4.0-python3.5-cuda8.0-cudnn6-devel-ubuntu14.04 29 | command: python execute_model.py 30 | inputs: 31 | - name: dataset 32 | default: https://s3-eu-west-1.amazonaws.com/valohai-examples/load-forecasting/rte-dataset.csv 33 | - name: model 34 | --------------------------------------------------------------------------------