├── .gitignore ├── LICENSE ├── utils ├── gee_images.py ├── tiff_image.py ├── feature_generation.py ├── gee.py └── gee_image_generation.py ├── models ├── lstm.py ├── lstm_multi.py ├── lstm_autoencoder.py ├── convlstm.py ├── lstm_multi_input.py ├── convlstm_multi_input.py └── lgbm_model.py ├── README.md └── experiment.py /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | data/ 3 | 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Leo Muckley 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /utils/gee_images.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | import geopandas as gpd 6 | import rasterio 7 | from rasterstats import zonal_stats 8 | from urllib.parse import urljoin 9 | import pandas as pd 10 | 11 | class GeeImages(): 12 | """ Helper function for adding GEE images. """ 13 | 14 | def __init__(self, filepath, filelist): 15 | self.filepath = filepath 16 | self.filelist = [urljoin(self.filepath, i) for i in filelist] 17 | 18 | 19 | def get_stats(self, df, stats_list, return_stats='mean'): 20 | s = [] 21 | for i in range(len(df)): 22 | s.append(stats_list[i][return_stats]) 23 | 24 | return pd.Series(s) 25 | 26 | 27 | def add_image(self, df, file): 28 | image = rasterio.open(file) 29 | band1 = image.read(1) 30 | affine = image.transform 31 | gdf = gpd.GeoDataFrame(df) 32 | 33 | stat = zonal_stats(gdf, band1, affine=affine, stats=['mean']) 34 | file = file.replace('.tif', '') 35 | file = file.replace(f'{self.filepath}', '') 36 | df[f'{file}'] = self.get_stats(df, stats_list=stat, return_stats='mean') 37 | 38 | # Return new df 39 | return df 40 | 41 | 42 | def add_all_images(self, df): 43 | 44 | for file in self.filelist: 45 | df = self.add_image(df, file) 46 | 47 | return df 48 | 49 | -------------------------------------------------------------------------------- /models/lstm.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import tensorflow as tf 5 | import matplotlib.pyplot as plt 6 | 7 | class LSTMAutoencoder(): 8 | 9 | def __init__(self, epochs, batch_size, loss, metric): 10 | self.num_time_steps = 30 11 | self.num_temporal_feats = 8 12 | self.epochs = epochs 13 | self.batch = batch_size 14 | self.loss = loss 15 | self.metric = metric 16 | self.model = None 17 | self.history = None 18 | self.es = tf.keras.callbacks.EarlyStopping(monitor='val_root_mean_squared_error', 19 | restore_best_weights=True, min_delta=0.0001, patience=10) 20 | 21 | def fit(self, X, y, X_val, y_val): 22 | temporal_input = tf.keras.Input(shape=(self.num_time_steps, self.num_temporal_feats), name="temporal") 23 | lstm_1 = tf.keras.layers.LSTM(32, return_sequences=True, kernel_initializer='glorot_normal')(temporal_input) 24 | lstm_2 = tf.keras.layers.LSTM(16, return_sequences=True, kernel_initializer='glorot_normal')(lstm_1) 25 | lstm_3 = tf.keras.layers.LSTM(1, kernel_initializer='glorot_normal')(lstm_2) 26 | repeat_1 = tf.keras.layers.RepeatVector(32)(lstm_3) 27 | lstm_4 = tf.keras.layers.LSTM(16, return_sequences=True, kernel_initializer='glorot_normal')(repeat_1) 28 | lstm_5 = tf.keras.layers.LSTM(32, kernel_initializer='glorot_normal')(lstm_4) 29 | dense_1 = tf.keras.layers.Dense(1)(lstm_5) 30 | self.model = tf.keras.Model(inputs=temporal_input, outputs=dense_1) 31 | self.model.compile(optimizer='adam', loss=self.loss, metrics=[self.metric]) 32 | self.history = self.model.fit(X, y, epochs=self.epochs, batch_size=self.batch, validation_data=(X_val, y_val), callbacks=[self.es]) 33 | 34 | return self.history 35 | 36 | def predict(self, X_test): 37 | return self.model.predict(X_test, batch_size=self.batch) 38 | 39 | def save(self, filename): 40 | return self.model.save(filename) 41 | 42 | 43 | -------------------------------------------------------------------------------- /utils/tiff_image.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import os 5 | import rasterio as rio 6 | from rasterio.warp import calculate_default_transform, reproject, Resampling 7 | 8 | 9 | class CRS(): 10 | """ Convert TIFF image coordinate system. 11 | 12 | For example, if the existing TIFF image has crs = EPSG:2935 it is possible 13 | to convert this to EPSG:4326 and output the new image. 14 | ------------------------- 15 | # File to be converted 16 | crs = CRS("data/out.tif") 17 | crs.convert() 18 | 19 | # Read converted (4326) file 20 | new_file = rio.open("data/out-4326.tif") 21 | print(new_file.meta) # 4326 22 | ------------------------- 23 | 24 | """ 25 | 26 | def __init__(self, filename): 27 | self.crs = "EPSG:4326" 28 | #self.path = path 29 | self.old_filename = filename 30 | #root_ext = os.path.splitext(filename) 31 | #self.new_filename = root_ext[0] + f"-{self.crs[-4:]}" + root_ext[1] 32 | 33 | def convert(self, new_filename): 34 | with rio.open(self.old_filename) as src: 35 | transform, width, height = calculate_default_transform( 36 | src.crs, self.crs, src.width, src.height, *src.bounds) 37 | kwargs = src.meta.copy() 38 | kwargs.update({ 39 | 'crs': self.crs, 40 | 'transform': transform, 41 | 'width': width, 42 | 'height': height 43 | }) 44 | 45 | with rio.open(new_filename, 'w', **kwargs) as dst: 46 | for i in range(1, src.count + 1): 47 | reproject( 48 | source=rio.band(src, i), 49 | destination=rio.band(dst, i), 50 | src_transform=src.transform, 51 | src_crs=src.crs, 52 | dst_transform=transform, 53 | dst_crs=self.crs, 54 | resampling=Resampling.nearest) 55 | print(f"Converted tiff file to {self.crs}") 56 | return None 57 | 58 | 59 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Multi-Input ConvLSTM for Flood Extent Prediction 2 | TensorFlow implementation of a Multi-Input ConvLSTM, as presented at the *Machine Learning Advances Envrionmental Sciences* workshop at ICPR 2020. 3 | Repo previously titled _malawi-flood-prediction_ 4 | 5 | 6 | 7 | [![MIT](https://img.shields.io/github/license/leomuckley/malawi-flood-prediction?logo=MIT)](/LICENSE) 8 | 9 | 10 | 11 | ### Abstract: 12 | Flooding is among the most destructive natural disasters in the world. As a result of this, there is a great need to develop accurate flood prediction models to prevent flooding ahead of time. However, due to the many factors that are related to predicting flood extent, floods are highly complex to model. Machine learning techniques have shown promise in this space but the current state-of-the-art techniques fail to generalise to other flood events. This study shows that a novel deep learning technique, namely the Multi-Input ConvLSTM, is effective at predicting the level of flood extent and generalising to other types of flood events. 13 | 14 | Link to full [paper](https://link.springer.com/chapter/10.1007/978-3-030-68780-9_8). 15 | 16 | **Example** (see *experiment.py*) 17 | 18 | ``` python 19 | 20 | def run_lstm_exp(model_list, folds=3, seed=SEED): 21 | 22 | tf.random.set_seed(seed) 23 | 24 | exp_dict = defaultdict(list) 25 | for model in model_list: 26 | if model == lmi: 27 | for fold in range(1, folds+1): 28 | print(f"*Training on fold {fold}") 29 | model.fit(X_train_temp, X_train_const, y_train, X_val_temp, X_val_const, y_val) 30 | preds = model.predict(X_test_temp, X_test_const).reshape(len(X_test,)) 31 | preds[preds < 0.0] = 0.0 32 | preds[preds > 1.0] = 1.0 33 | y = y_test.flatten() 34 | res = rmse(preds, y) 35 | exp_dict[model].append(np.round(res, 6)) 36 | else: 37 | for fold in range(1, folds+1): 38 | print(f"*Training on fold {fold}") 39 | model.fit(X_train, y_train, X_val, y_val) 40 | preds = model.predict(X_test).reshape(len(X_test,)) 41 | y = y_test.flatten() 42 | preds[preds < 0.0] = 0.0 43 | preds[preds > 1.0] = 1.0 44 | res = rmse(preds, y) 45 | exp_dict[model].append(np.round(res, 6)) 46 | return exp_dict 47 | 48 | ``` 49 | 50 | -------------------------------------------------------------------------------- /models/lstm_multi.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import tensorflow as tf 5 | import matplotlib.pyplot as plt 6 | 7 | 8 | class LSTMmulti(): 9 | """ 10 | This class represents an Multivariate LSTM model used for the task of predicting 11 | flood extent. 12 | 13 | 14 | Parameters 15 | ---------- 16 | epochs : int 17 | This parameters determines the number passes through the data 18 | that that the model will train. 19 | 20 | batch_size : int 21 | The batch size will determine after how many samples the 22 | model update the weights. 23 | 24 | loss : tf.keras.losses object 25 | Example: tf.keras.loss.Mean_Squared_Error 26 | 27 | metric : tf.keras.metrics object 28 | Example: tf.keras.metrics.Root_Mean_Squared_Error 29 | 30 | callbacks : tf.keras.callbacks object 31 | Example: tf.keras.callbacks.EarlyStopping 32 | 33 | 34 | Examples 35 | -------- 36 | >>> from lstm_multi import LSTMmulti 37 | >>> lmulti = LSTMmulti(epochs, batch_size, loss, metric, callbacks) 38 | >>> lmulti.fit(X, y) 39 | >>> lmulti.predict(X_test) 40 | array([0, 1, 0.5, .........]) 41 | 42 | """ 43 | 44 | 45 | def __init__(self, epochs, batch_size, loss, metric, callbacks): 46 | self.num_time_steps = 30 47 | self.num_temporal_feats = 10 48 | self.epochs = epochs 49 | self.batch = batch_size 50 | self.loss = loss 51 | self.metric = metric 52 | self.model = None 53 | self.history = None 54 | self.es = callbacks 55 | 56 | def fit(self, X, y, X_val, y_val): 57 | """ 58 | This method will apply the fit operation. 59 | 60 | The model will be trained on the training set and the early stopping 61 | method will be applied on the validation set. 62 | 63 | Parameters 64 | ---------- 65 | X : array-lke 66 | The predictor features in the training set. 67 | 68 | y : array-lke 69 | The target feature in the training set. 70 | 71 | X_val : array-lke 72 | The predictor features in the validation set. 73 | 74 | y_val : array-lke 75 | The target feature in the training set. 76 | 77 | """ 78 | # define model where LSTM is also output layer 79 | self.model = tf.keras.Sequential() 80 | self.model.add(tf.keras.layers.LSTM(64, return_sequences=True, input_shape=(self.num_time_steps, self.num_temporal_feats))) 81 | self.model.add(tf.keras.layers.LSTM(32, dropout=0.1, recurrent_dropout=0.1)) 82 | self.model.add(tf.keras.layers.Dense(1)) 83 | self.model.compile(optimizer='adam', loss=self.loss, metrics=[self.metric]) 84 | self.history = self.model.fit(X, y, epochs=self.epochs, batch_size=self.batch, validation_data=(X_val, y_val), callbacks=[self.es]) 85 | 86 | return self.history 87 | 88 | def predict(self, X_test): 89 | """ 90 | This method will apply the predict operation. 91 | 92 | The trained model will be used to predict on the test set. 93 | 94 | Parameters 95 | ---------- 96 | X_test : array-lke 97 | The predictor features in the test set. 98 | """ 99 | return self.model.predict(X_test, batch_size=self.batch) 100 | 101 | def save(self, filename): 102 | """ 103 | This method will save the trained model. 104 | 105 | Parameters 106 | ---------- 107 | filename : str 108 | The name of the file to be saved. 109 | """ 110 | return self.model.save(filename) 111 | 112 | def return_history(self): 113 | """ This method will return the history object of the trained model.""" 114 | return self.history 115 | 116 | 117 | 118 | -------------------------------------------------------------------------------- /models/lstm_autoencoder.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import tensorflow as tf 5 | 6 | class LSTMAutoencoder(): 7 | """ 8 | This class represents an LSTM-Autoencoder model used for the task of predicting 9 | flood extent. 10 | 11 | 12 | Parameters 13 | ---------- 14 | epochs : int 15 | This parameters determines the number passes through the data 16 | that that the model will train. 17 | 18 | batch_size : int 19 | The batch size will determine after how many samples the 20 | model update the weights. 21 | 22 | loss : tf.keras.losses object 23 | Example: tf.keras.loss.Mean_Squared_Error 24 | 25 | metric : tf.keras.metrics object 26 | Example: tf.keras.metrics.Root_Mean_Squared_Error 27 | 28 | callbacks : tf.keras.callbacks object 29 | Example: tf.keras.callbacks.EarlyStopping 30 | 31 | 32 | Examples 33 | -------- 34 | >>> from lstm_autoencoder import LSTMAutoencoder 35 | >>> lauto = LSTMAutoencoder(epochs, batch_size, loss, metric, callbacks) 36 | >>> lauto.fit(X, y) 37 | >>> lauto.predict(X_test) 38 | array([0, 1, 0.5, .........]) 39 | 40 | """ 41 | 42 | 43 | def __init__(self, epochs, batch_size, loss, metric, callbacks): 44 | self.num_time_steps = 30 45 | self.num_temporal_feats = 10 46 | self.epochs = epochs 47 | self.batch = batch_size 48 | self.loss = loss 49 | self.metric = metric 50 | self.model = None 51 | self.history = None 52 | self.es = callbacks 53 | 54 | def fit(self, X, y, X_val, y_val): 55 | """ 56 | This method will apply the fit operation. 57 | 58 | The model will be trained on the training set and the early stopping 59 | method will be applied on the validation set. 60 | 61 | Parameters 62 | ---------- 63 | X : array-lke 64 | The predictor features in the training set. 65 | 66 | y : array-lke 67 | The target feature in the training set. 68 | 69 | X_val : array-lke 70 | The predictor features in the validation set. 71 | 72 | y_val : array-lke 73 | The target feature in the training set. 74 | 75 | """ 76 | 77 | temporal_input = tf.keras.Input(shape=(self.num_time_steps, self.num_temporal_feats), name="temporal") 78 | lstm_1 = tf.keras.layers.LSTM(32, return_sequences=True, kernel_initializer='glorot_normal')(temporal_input) 79 | lstm_2 = tf.keras.layers.LSTM(16, return_sequences=True, kerbenel_initializer='glorot_normal')(lstm_1) 80 | lstm_3 = tf.keras.layers.LSTM(1, kernel_initializer='glorot_normal')(lstm_2) 81 | repeat_1 = tf.keras.layers.RepeatVector(32)(lstm_3) 82 | lstm_4 = tf.keras.layers.LSTM(16, return_sequences=True, kernel_initializer='glorot_normal')(repeat_1) 83 | lstm_5 = tf.keras.layers.LSTM(32, kernel_initializer='glorot_normal')(lstm_4) 84 | dense_1 = tf.keras.layers.Dense(1)(lstm_5) 85 | self.model = tf.keras.Model(inputs=temporal_input, outputs=dense_1) 86 | self.model.compile(optimizer='adam', loss=self.loss, metrics=[self.metric]) 87 | self.history = self.model.fit(X, y, epochs=self.epochs, batch_size=self.batch, validation_data=(X_val, y_val), callbacks=[self.es]) 88 | 89 | return self.history 90 | 91 | def predict(self, X_test): 92 | """ 93 | This method will apply the predict operation. 94 | 95 | The trained model will be used to predict on the test set. 96 | 97 | Parameters 98 | ---------- 99 | X_test : array-lke 100 | The predictor features in the test set. 101 | """ 102 | return self.model.predict(X_test, batch_size=self.batch) 103 | 104 | def save(self, filename): 105 | """ 106 | This method will save the trained model. 107 | 108 | Parameters 109 | ---------- 110 | filename : str 111 | The name of the file to be saved. 112 | """ 113 | return self.model.save(filename) 114 | 115 | def return_history(self): 116 | """ This method will return the history object of the trained model.""" 117 | return self.history 118 | 119 | 120 | 121 | 122 | -------------------------------------------------------------------------------- /models/convlstm.py: -------------------------------------------------------------------------------- 1 | 2 | import tensorflow as tf 3 | from tensorflow.keras.layers import LSTM, RepeatVector, Dense, Flatten, ConvLSTM2D, TimeDistributed 4 | 5 | 6 | class ConvLSTM(): 7 | """ 8 | This class represents an Multivariate ConvLSTM model used for the task of predicting 9 | flood extent. 10 | 11 | 12 | Parameters 13 | ---------- 14 | epochs : int 15 | This parameters determines the number passes through the data 16 | that that the model will train. 17 | 18 | batch_size : int 19 | The batch size will determine after how many samples the 20 | model update the weights. 21 | 22 | loss : tf.keras.losses object 23 | Example: tf.keras.loss.Mean_Squared_Error 24 | 25 | metric : tf.keras.metrics object 26 | Example: tf.keras.metrics.Root_Mean_Squared_Error 27 | 28 | callbacks : tf.keras.callbacks object 29 | Example: tf.keras.callbacks.EarlyStopping 30 | 31 | 32 | Examples 33 | -------- 34 | >>> from lstm_multi import LSTMmulti 35 | >>> lmulti = LSTMmulti(epochs, batch_size, loss, metric, callbacks) 36 | >>> lmulti.fit(X, y) 37 | >>> lmulti.predict(X_test) 38 | array([0, 1, 0.5, .........]) 39 | 40 | """ 41 | 42 | 43 | def __init__(self, epochs, batch_size, loss, metric, callbacks): 44 | self.num_time_steps = 30 45 | self.num_temporal_feats = 10 46 | self.epochs = epochs 47 | self.batch = batch_size 48 | self.loss = loss 49 | self.metric = metric 50 | self.model = None 51 | self.history = None 52 | self.es = callbacks 53 | 54 | def fit(self, X, y, X_val, y_val): 55 | """ 56 | This method will apply the fit operation. 57 | 58 | The model will be trained on the training set and the early stopping 59 | method will be applied on the validation set. 60 | 61 | Parameters 62 | ---------- 63 | X : array-lke 64 | The predictor features in the training set. 65 | 66 | y : array-lke 67 | The target feature in the training set. 68 | 69 | X_val : array-lke 70 | The predictor features in the validation set. 71 | 72 | y_val : array-lke 73 | The target feature in the training set. 74 | 75 | """ 76 | # Convert the input data to tensors 77 | X_train = X.reshape((len(X), self.num_time_steps, self.num_temporal_feats)) 78 | y_train = y.values.reshape((len(y), 1)) 79 | X_val = X_val.reshape((len(X_val), self.num_time_steps, self.num_temporal_feats)) 80 | y_val = y_val.values.reshape((len(y_val), 1)) 81 | train_x = X_train.reshape((X_train.shape[0], 1, 1, self.num_time_steps, self.num_temporal_feats)) 82 | train_y = y_train.reshape((y_train.shape[0], y_train.shape[1], 1)) 83 | val_x = X_val.reshape((X_val.shape[0], 1, 1, self.num_time_steps, self.num_temporal_feats)) 84 | val_y = y_val.reshape((y_val.shape[0], y_val.shape[1], 1)) 85 | 86 | # ConvLstm architecture 87 | model = tf.keras.Sequential() 88 | model.add(ConvLSTM2D(filters=64, kernel_size=(1,3), 89 | input_shape=(1, 1, self.num_time_steps, self.num_temporal_feats))) 90 | model.add(Flatten()) 91 | model.add(RepeatVector(X.shape)) 92 | model.add(LSTM(32, return_sequences=True)) 93 | model.add(LSTM(16, dropout=0.1, recurrent_dropout=0.1)) 94 | model.add(Dense(1)) 95 | model.compile(optimizer='adam', loss=tf.keras.losses.Huber(), metrics=[tf.keras.metrics.RootMeanSquaredError()]) 96 | # fit network 97 | self.history = model.fit(train_x, train_y, epochs=self.epoch, batch_size=self.batch, validation_data=(val_x, val_y), callbacks=[self.es]) 98 | 99 | return self.history 100 | 101 | def predict(self, X_test): 102 | """ 103 | This method will apply the predict operation. 104 | 105 | The trained model will be used to predict on the test set. 106 | 107 | Parameters 108 | ---------- 109 | X_test : array-lke 110 | The predictor features in the test set. 111 | """ 112 | return self.model.predict(X_test, batch_size=self.batch) 113 | 114 | def save(self, filename): 115 | """ 116 | This method will save the trained model. 117 | 118 | Parameters 119 | ---------- 120 | filename : str 121 | The name of the file to be saved. 122 | """ 123 | return self.model.save(filename) 124 | 125 | def return_history(self): 126 | """ This method will return the history object of the trained model.""" 127 | return self.history 128 | 129 | 130 | 131 | -------------------------------------------------------------------------------- /models/lstm_multi_input.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import tensorflow as tf 5 | 6 | 7 | class LSTMmultiInput(): 8 | """ 9 | This class represents a Multi-Input LSTM model used for the task of predicting 10 | flood extent. 11 | 12 | 13 | Parameters 14 | ---------- 15 | epochs : int 16 | This parameters determines the number passes through the data 17 | that that the model will train. 18 | 19 | batch_size : int 20 | The batch size will determine after how many samples the 21 | model update the weights. 22 | 23 | loss : tf.keras.losses object 24 | Example: tf.keras.loss.Mean_Squared_Error 25 | 26 | metric : tf.keras.metrics object 27 | Example: tf.keras.metrics.Root_Mean_Squared_Error 28 | 29 | callbacks : tf.keras.callbacks object 30 | Example: tf.keras.callbacks.EarlyStopping 31 | 32 | 33 | Examples 34 | -------- 35 | >>> from lstm_multi_input import LSTMmultiInput 36 | >>> lmulti = LSTMmultiInput(epochs, batch_size, loss, metric, callbacks) 37 | >>> lmulti.fit(X, y) 38 | >>> lmulti.predict(X_test) 39 | array([0, 1, 0.5, .........]) 40 | """ 41 | 42 | def __init__(self, epochs, batch_size, loss, metric, callbacks): 43 | self.num_time_steps = 30 44 | self.num_temporal_feats = 5 45 | self.num_constant_feats = 5 46 | self.epochs = epochs 47 | self.batch = batch_size 48 | self.loss = loss 49 | self.metric = metric 50 | self.model = None 51 | self.history = None 52 | self.es = callbacks 53 | 54 | def fit(self, X_temp, X_const, y, X_val_temp, X_val_const, y_val): 55 | """ 56 | This method will apply the fit operation to both the constant features 57 | and the temporal features. 58 | 59 | The model will be trained on the training set and the early stopping 60 | method will be applied on the validation set. 61 | 62 | Parameters 63 | ---------- 64 | X_temp : array-lke 65 | The temporal predictor features in the training set. 66 | X_const : array-lke 67 | The constant predictor features in the training set. 68 | y : array-lke 69 | The target feature in the training set. 70 | X_val_temporal : array-lke 71 | The temporal predictor features in the validation set. 72 | X_val_const : array-lke 73 | The constant predictor features in the validation set. 74 | y_val : array-lke 75 | The target feature in the training set. 76 | 77 | """ 78 | 79 | constant_input = tf.keras.Input(shape=(self.num_constant_feats), name="constants") 80 | temporal_input = tf.keras.Input(shape=(self.num_time_steps, self.num_temporal_feats), name="temporal") 81 | lstm_1 = tf.keras.layers.LSTM(64, return_sequences=True)(temporal_input) 82 | lstm_2 = tf.keras.layers.LSTM(32, dropout=0.1, recurrent_dropout=0.1)(lstm_1) 83 | concat_1 = tf.keras.layers.concatenate([constant_input, lstm_2]) 84 | dense_2 = tf.keras.layers.Dense(1)(concat_1) 85 | self.model = tf.keras.Model(inputs=[temporal_input, constant_input], outputs=[dense_2]) 86 | self.model.compile(optimizer='adam', loss=self.loss, metrics=[self.metric]) 87 | self.history = self.model.fit({"constants": X_const, "temporal": X_temp}, y, epochs=self.epochs, callbacks=[self.es], 88 | batch_size=self.batch, validation_data=({"constants": X_val_const, "temporal": X_val_temp}, y_val)) 89 | return self.history 90 | 91 | 92 | def predict(self, X_test_temp, X_test_const): 93 | """ 94 | This method will apply the predict operation. 95 | 96 | The trained model will be used to predict on the test set. 97 | 98 | Parameters 99 | ---------- 100 | X_test_temp : array-lke 101 | The temporal predictor features in the test set. 102 | X_test_const : array-lke 103 | The constant predictor features in the test set. 104 | """ 105 | return self.model.predict({"constants": X_test_const, "temporal": X_test_temp}, batch_size=self.batch) 106 | 107 | def save(self, filename): 108 | """ 109 | This method will save the trained model. 110 | 111 | Parameters 112 | ---------- 113 | filename : str 114 | The name of the file to be saved. 115 | """ 116 | return self.model.save(filename) 117 | 118 | def return_history(self): 119 | """ This method will return the history object of the trained model.""" 120 | return self.history 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /models/convlstm_multi_input.py: -------------------------------------------------------------------------------- 1 | 2 | import tensorflow as tf 3 | from tensorflow.keras.layers import LSTM, RepeatVector, Dense, Flatten, ConvLSTM2D, TimeDistributed 4 | 5 | 6 | class MultiInputConvLSTM(): 7 | """ 8 | This class represents an Multivariate ConvLSTM model used for the task of predicting 9 | flood extent. 10 | 11 | 12 | Parameters 13 | ---------- 14 | epochs : int 15 | This parameters determines the number passes through the data 16 | that that the model will train. 17 | 18 | batch_size : int 19 | The batch size will determine after how many samples the 20 | model update the weights. 21 | 22 | loss : tf.keras.losses object 23 | Example: tf.keras.loss.Mean_Squared_Error 24 | 25 | metric : tf.keras.metrics object 26 | Example: tf.keras.metrics.Root_Mean_Squared_Error 27 | 28 | callbacks : tf.keras.callbacks object 29 | Example: tf.keras.callbacks.EarlyStopping 30 | 31 | 32 | Examples 33 | -------- 34 | >>> from lstm_multi import LSTMmulti 35 | >>> lmulti = LSTMmulti(epochs, batch_size, loss, metric, callbacks) 36 | >>> lmulti.fit(X, y) 37 | >>> lmulti.predict(X_test) 38 | array([0, 1, 0.5, .........]) 39 | 40 | """ 41 | 42 | 43 | def __init__(self, epochs, batch_size, loss, metric, callbacks): 44 | self.num_time_steps = 30 45 | self.num_temporal_feats = 5 46 | self.num_constant_feats = 5 47 | self.epochs = epochs 48 | self.batch = batch_size 49 | self.loss = loss 50 | self.metric = metric 51 | self.model = None 52 | self.history = None 53 | self.es = callbacks 54 | 55 | def fit(self, X_temp, X_const, y, X_val_temp, X_val_const, y_val): 56 | """ 57 | This method will apply the fit operation. 58 | 59 | The model will be trained on the training set and the early stopping 60 | method will be applied on the validation set. 61 | 62 | Parameters 63 | ---------- 64 | X_temp : array-lke 65 | The temporal predictor features in the training set. 66 | X_const : array-lke 67 | The constant predictor features in the training set. 68 | y : array-lke 69 | The target feature in the training set. 70 | X_val_temporal : array-lke 71 | The temporal predictor features in the validation set. 72 | X_val_const : array-lke 73 | The constant predictor features in the validation set. 74 | y_val : array-lke 75 | The target feature in the training set. 76 | 77 | """ 78 | 79 | y = tf.convert_to_tensor(y.values.reshape(len(y), 1)) 80 | y_val = tf.convert_to_tensor(y_val.values.reshape(len(y_val), 1)) 81 | 82 | X_temp = tf.reshape(X_temp, shape=(X_temp.shape[0], 1, 1, self.num_time_steps, self.num_temporal_feats)) 83 | X_val_temp = tf.reshape(X_val_temp, shape=(X_val_temp.shape[0], 1, 1, self.num_time_steps, self.num_temporal_feats)) 84 | 85 | constant_input = tf.keras.Input(shape=(self.num_constant_feats), name="constants") 86 | temporal_input = tf.keras.Input(shape=(1, 1, self.num_time_steps, self.num_temporal_feats), name='temporal') 87 | 88 | lstm_1 = tf.keras.layers.ConvLSTM2D(filters=64, kernel_size=(1,5))(temporal_input) 89 | flat = tf.keras.layers.Flatten()(lstm_1) 90 | repeat = tf.keras.layers.RepeatVector(y.shape[1])(flat) 91 | lstm_2 = tf.keras.layers.LSTM(32, return_sequences=True)(repeat) 92 | lstm_3 = tf.keras.layers.LSTM(16, dropout=0.1, recurrent_dropout=0.1)(lstm_2) 93 | 94 | dense_1 = tf.keras.layers.Dense(16, activation='relu')(constant_input) 95 | flat_2 = tf.keras.layers.Flatten()(dense_1) 96 | #repeat_2 = tf.keras.layers.RepeatVector(y_train.shape[1])(flat_2) 97 | #flat_3 = tf.keras.layers.Flatten()(repeat_2) 98 | concat_1 = tf.keras.layers.concatenate([flat_2, lstm_3]) 99 | dense_2 = tf.keras.layers.Dense(1)(concat_1) 100 | model = tf.keras.Model(inputs=[temporal_input, constant_input], outputs=[dense_2]) 101 | 102 | model.compile(optimizer='adam', loss=tf.keras.losses.Huber(), metrics=[tf.keras.metrics.RootMeanSquaredError()]) 103 | self.history = model.fit({"constants": X_const, "temporal": X_temp}, y, epochs=self.epoch, callbacks=[self.es], 104 | batch_size=self.batch, validation_data=({"constants": X_val_const, "temporal": X_val_temp}, y_val)) 105 | 106 | return self.history 107 | 108 | def predict(self, X_test_temp, X_test_const): 109 | """ 110 | This method will apply the predict operation. 111 | 112 | The trained model will be used to predict on the test set. 113 | 114 | Parameters 115 | ---------- 116 | X_test_temp : array-lke 117 | The temporal predictor features in the test set. 118 | X_test_const : array-lke 119 | The constant predictor features in the test set. 120 | """ 121 | return self.model.predict({"constants": X_test_const, "temporal": X_test_temp}, batch_size=self.batch) 122 | 123 | def save(self, filename): 124 | """ 125 | This method will save the trained model. 126 | 127 | Parameters 128 | ---------- 129 | filename : str 130 | The name of the file to be saved. 131 | """ 132 | return self.model.save(filename) 133 | 134 | def return_history(self): 135 | """ This method will return the history object of the trained model.""" 136 | return self.history 137 | 138 | 139 | 140 | -------------------------------------------------------------------------------- /utils/feature_generation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import pandas as pd 5 | import numpy as np 6 | 7 | import matplotlib.pyplot as plt 8 | import geopandas as gpd 9 | from shapely.geometry import Polygon, Point 10 | from shapely.ops import nearest_points 11 | 12 | from gee_images import GeeImages 13 | 14 | 15 | 16 | def make_grid(xmin, ymin, xmax, ymax): 17 | """ 18 | Function for creating a grid using polygon objects. 19 | 20 | Parameters 21 | ---------- 22 | xmin : float, geo-coordinate 23 | ymin : float, geo-coordinate 24 | xmax : float, geo-coordinate 25 | ymax : float, geo-coordinate 26 | 27 | Returns 28 | ------- 29 | grid : GeoDataFrame 30 | 31 | """ 32 | length = 0.01 33 | wide = 0.01 34 | 35 | cols = list(np.arange(xmin, xmax, wide)) 36 | rows = list(np.arange(ymin, ymax, length)) 37 | rows = rows[::-1] 38 | 39 | polygons = [] 40 | for x in cols: 41 | for y in rows: 42 | polygons.append( Polygon([(x,y), (x+wide, y), (x+wide, y-length), (x, y-length)]) ) 43 | 44 | grid = gpd.GeoDataFrame({'geometry':polygons}) 45 | return grid 46 | 47 | 48 | def plot_grid(flood, grid): 49 | """ 50 | Plot grid over flood image. 51 | 52 | Parameters 53 | ---------- 54 | flood : GeoDataFrame 55 | grid : GeoDataFrame 56 | 57 | """ 58 | ax = grid.plot(color='yellow'); 59 | return flood.plot(ax=ax, color='black', alpha=0.5); 60 | 61 | 62 | def flood_frac_df(gdf, grid): 63 | """ 64 | Function for calculating the fraction of each square in the grid 65 | that contains flooding. Each square conatined in the grid will then 66 | return a value in the interval between 0 and 1. 67 | 68 | Parameters 69 | ---------- 70 | gdf : GeoDataFrame 71 | grid : GeoDataFrame 72 | 73 | Returns 74 | ------- 75 | df : DataFrame 76 | The resuling DataFrame will contain the fraction of flooding, 77 | for each of the original coordinates. 78 | 79 | """ 80 | flood = gpd.sjoin(grid, gdf, how='left') 81 | total_list = [] 82 | for grd in grid.geometry: 83 | total = 0 84 | for fld in gdf.geometry: 85 | if fld.intersects(grd): 86 | flood = (fld.intersection(grd)).area / grd.area 87 | total = total + flood 88 | print(total) 89 | total_list.append(total) 90 | df = pd.DataFrame(data={'target':total_list}) 91 | df['centroid'] = grid.centroid 92 | df['geometry'] = grid.geometry 93 | 94 | return df 95 | 96 | 97 | def plot_flood(df, grid, xmin, xmax, ymin, ymax): 98 | rows = grid.shape[0] 99 | cols = grid.shape[1] 100 | cols = len(list(np.arange(xmin, xmax, 0.01))) 101 | rows = len(list(np.arange(ymin, ymax, 0.01))) 102 | flood_list = df['target'].to_list() 103 | mat = np.array(flood_list).reshape(cols, rows) 104 | plt.imshow(mat.T, cmap='Blues') 105 | return plt.show() 106 | 107 | 108 | 109 | def plot_flood_new(df, grid): 110 | rows = grid.shape[0] 111 | cols = grid.shape[1] 112 | bnds = grid.total_bounds 113 | xmin, ymin, xmax, ymax = bnds[0], bnds[1], bnds[2], bnds[3] 114 | cols = len(list(np.arange(xmin, xmax, 0.01))) 115 | rows = len(list(np.arange(ymin, ymax, 0.01))) 116 | flood_list = df['target'].to_list() 117 | #zeros = list(np.zeros((rows * cols) - len(grid))) 118 | #full_flood = flood_list + zeros 119 | mat = np.array(flood_list).reshape(cols, rows) 120 | plt.imshow(mat.T, cmap='Blues') 121 | return plt.show() 122 | 123 | 124 | def dist_to_water(df, zipfile, geom1_col='geometry'): 125 | # import data from http://www.diva-gis.org/datadown 126 | #zipfile = "zip:///home/leo/Desktop/ml_flood_prediction/data/inland_water/MOZ_wat.zip" 127 | mwi_water = gpd.read_file(zipfile) 128 | water_union = mwi_water.unary_union 129 | dist_list =[] 130 | for index, row in df.iterrows(): 131 | nearest = nearest_points(row[geom1_col], water_union)[1] 132 | dist = row[geom1_col].distance(nearest) 133 | dist_list.append(dist) 134 | 135 | # Return column 136 | return dist_list 137 | 138 | 139 | def copy_col(df_, column, rep): 140 | df = df_.copy() 141 | for i in range(rep, 0, -1): 142 | df[f'{column}_t-{i}'] = df[column] 143 | df = df.drop(column, axis=1) 144 | return df 145 | 146 | 147 | def copy_time_col(df_, column_stem, start, end, step, gap): 148 | df = df_.copy() 149 | for i in range(start, end-gap, step): 150 | for j in range(i+gap, i, -1): 151 | df[f'{column_stem}_t-{j}'] = df[f'{column_stem}_t-{i}'] 152 | return df 153 | 154 | 155 | def copy_time_fw_col(df_, column_stem, start, end, step, gap): 156 | df = df_.copy() 157 | for i in range(start, end, step): 158 | for j in range(i, i-gap, -1): 159 | df[f'{column_stem}_t-{j}'] = df[f'{column_stem}_t-{i}'] 160 | return df 161 | 162 | 163 | 164 | def order_feats(df_): 165 | df = df_.copy() 166 | col = df.columns 167 | tup_list = [(i, np.abs(np.int(i[-2:]))) for i in col if "-" in i] 168 | sort_list = [b for a,b in sorted((tup[1], tup) for tup in tup_list)] 169 | sort_list = list(reversed([i[0] for i in sort_list])) 170 | target = "target" 171 | sort_list.append(target) 172 | return sort_list 173 | 174 | 175 | def write_file(df, header, filename): 176 | df = df[header] 177 | 178 | return df.to_csv(filename, header=header, index=False) 179 | 180 | 181 | def inside_border(grid, country='Mozambique'): 182 | 183 | world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres')) 184 | border = world[world.name == country]['geometry'] 185 | 186 | in_border = [border.contains(i).values[0] for i in grid.geometry] 187 | return grid[in_border] 188 | 189 | 190 | def return_coords(df_): 191 | df = df_.copy() 192 | def get_coords(row): 193 | row['X'] = row['centroid'].x 194 | row['Y'] = row['centroid'].y 195 | return row 196 | return df.apply(get_coords, axis=1) 197 | 198 | 199 | def coords_to_geom(df_): 200 | df = df_.copy() 201 | geom = [Point(df.X[i], df.Y[i]) for i in range(len(df))] 202 | #df['geometry'] = geom 203 | return geom 204 | 205 | -------------------------------------------------------------------------------- /models/lgbm_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import pandas as pd 5 | import numpy as np 6 | 7 | import lightgbm as lgb 8 | from sklearn.model_selection import train_test_split 9 | 10 | 11 | class LGBM(): 12 | """ 13 | This class represents the winnning solution for the Zindi Unicef competition: 14 | www.zindi.com/competitions/ 15 | 16 | This class will apply the winning soltuion in an experimental fashion to test 17 | the accuracy of the model on various train/test sets. 18 | The winnning solution consists of a LGBM model with model averaging. 19 | 20 | This strategy includes prepocessing of the data and feature engineering. The 21 | models will be fit on the training set. Then the trained models will be 22 | applied to the test set to predict future values. 23 | 24 | 25 | Parameters 26 | ---------- 27 | df_train : Pandas DataFrame 28 | The training dataset to be used modelling. This dataframe 29 | includes both the predictor features and the target feature. 30 | 31 | df_test : Pandas DataFrame 32 | The test dataset to be used for testing the LGBM model. This 33 | dataframe includes both the predictor features and the target 34 | feature. 35 | 36 | seed : int, default=400 37 | The parameter that controls the random state of the modelling 38 | process to allow for adequate comparion with other models. 39 | 40 | 41 | Examples 42 | -------- 43 | >>> from lgbm_model import LGBM 44 | >>> lgb = LGBM(training_set, test_set, seed=100) 45 | >>> lgb.fit_predict() 46 | array([0, 0.5, 0.2, 0.001, .........]) 47 | 48 | """ 49 | 50 | def __init__(self, X_train, X_test, seed=400): 51 | self.params = {'learning_rate':0.07,'max_depth':8, 'num_leaves':2*8} 52 | self.n_estimators = 221 53 | self.n_iters = 5 54 | self.df_train = X_train 55 | self.df_test = X_test 56 | self.model = None 57 | self.seed = seed 58 | 59 | def metric(self, predictions, targets): 60 | """ Root Mean Squared Error (RMSE) used for evaluation. """ 61 | return np.sqrt(((predictions - targets) ** 2).mean()) 62 | 63 | def check(self, col): 64 | """ Make unbounded values conform with upper/lower bounds. """ 65 | if col < 0: 66 | return 0 67 | elif col > 1: 68 | return 1 69 | else: 70 | return col 71 | 72 | def preprocess(self): 73 | """ Process and engineer the features in the train/test set. """ 74 | train = self.df_train.copy() 75 | test = self.df_test.copy() 76 | train = train.fillna(0) 77 | test = test.fillna(0) 78 | train['week_9_precip'] = train[train[[i for i in train.columns if 'max_precip' in i]].columns[-7:]].max().max() 79 | train['week_8_precip'] = train[train[[i for i in train.columns if 'max_precip' in i]].columns[-14:-7]].max().max() 80 | train['week_7_precip'] = train[train[[i for i in train.columns if 'max_precip' in i]].columns[-21:-14]].max().max() 81 | train['week_6_precip'] = train[train[[i for i in train.columns if 'max_precip' in i]].columns[:-21]].max().max() 82 | test['week_9_precip'] = test[test[[i for i in test.columns if 'max_precip' in i]].columns[-7:]].max().max() 83 | test['week_8_precip'] = test[test[[i for i in test.columns if 'max_precip' in i]].columns[-14:-7]].max().max() 84 | test['week_7_precip'] = test[test[[i for i in test.columns if 'max_precip' in i]].columns[-21:-14]].max().max() 85 | test['week_6_precip'] = test[test[[i for i in test.columns if 'max_precip' in i]].columns[:-21]].max().max() 86 | train['week_7_precip_']=train['week_7_precip']+train['week_6_precip'] 87 | test['week_7_precip_']=test['week_7_precip']+test['week_6_precip'] 88 | train['week_8_precip_']=train['week_8_precip']+train['week_7_precip'] 89 | test['week_8_precip_']=test['week_8_precip']+test['week_7_precip'] 90 | train['week_9_precip_']=train['week_9_precip']+train['week_8_precip'] 91 | test['week_9_precip_']=test['week_9_precip']+test['week_8_precip'] 92 | train['max_2_weeks']=train[['week_7_precip_','week_8_precip_','week_9_precip_']].apply(np.max,axis=1) 93 | test['max_2_weeks']=test[['week_7_precip_','week_8_precip_','week_9_precip_']].apply(np.max,axis=1) 94 | train['LC_Type1_mode'] = train['land_cover_mode_t-1'].astype(int) 95 | test['LC_Type1_mode'] = test['land_cover_mode_t-1'].astype(int) 96 | train['X'], test['X'] = train['X_t-1'], test['X_t-1'] 97 | train['Y'], test['Y']= train['Y_t-1'], test['Y_t-1'] 98 | train['elevation'] = train['elevation_t-1'] 99 | test['elevation'] = test['elevation_t-1'] 100 | train['soil_major'] = train['soc_mean_t-1'] 101 | test['soil_major'] = test['soc_mean_t-1'] 102 | 103 | def index(col): 104 | l=list(col) 105 | return l.index(max(l)) 106 | 107 | train['max_index'] = train[['week_6_precip', 'week_7_precip', 'week_8_precip','week_9_precip']].apply(index,axis=1) 108 | test['max_index'] = test[['week_6_precip', 'week_7_precip', 'week_8_precip','week_9_precip']].apply(index,axis=1) 109 | train['slope_8_7'] = ((train['week_8_precip']/train['week_7_precip'])>1)*1 110 | test['slope_8_7'] = ((test['week_8_precip']/test['week_7_precip'])>1)*1 111 | train['slope_9_8'] = ((train['week_9_precip']/train['week_8_precip'])>1)*1 112 | test['slope_9_8'] = ((test['week_9_precip']/test['week_8_precip'])>1)*1 113 | cols = ['LC_Type1_mode', 'X', 'Y', 'elevation','week_7_precip', 'week_8_precip', 'week_9_precip', 114 | 'max_2_weeks', 'slope_8_7', 'slope_9_8', 'soil_major', 'target'] 115 | test_cols = ['LC_Type1_mode', 'X', 'Y', 'elevation','week_7_precip', 'week_8_precip', 'week_9_precip', 116 | 'max_2_weeks', 'slope_8_7', 'slope_9_8', 'soil_major'] 117 | 118 | return train[cols], test[test_cols] 119 | 120 | 121 | def fit_predict(self): 122 | """ Fit training set and predict on testing set. """ 123 | train, test = self.preprocess() 124 | preds_buf = [] 125 | X = train.drop(columns=['target']) 126 | y = train['target'] 127 | for _ in range(self.n_iters): 128 | x_train, x_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, 129 | random_state=np.random.randint(0, 100)) 130 | d_train = lgb.Dataset(x_train, label=y_train) 131 | d_valid = lgb.Dataset(x_valid, label=y_valid) 132 | watchlist = [d_valid] 133 | model = lgb.train(self.params, d_train, self.n_estimators, watchlist, verbose_eval=1) 134 | preds = model.predict(self.df_test) 135 | preds_buf.append(preds) 136 | self.model = model 137 | preds1 = pd.Series(np.mean(preds_buf, axis=0)) 138 | preds=preds1.apply(self.check) 139 | preds-=0.08 140 | return preds 141 | 142 | 143 | 144 | -------------------------------------------------------------------------------- /experiment.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import geopandas as gpd 5 | import pandas as pd 6 | import numpy as np 7 | import json 8 | 9 | 10 | from sklearn.model_selection import train_test_split 11 | from sklearn.preprocessing import MinMaxScaler 12 | 13 | import tensorflow as tf 14 | from collections import defaultdict 15 | 16 | from lstm_multi import LSTMmulti 17 | from lstm_autoencoder import LSTMAutoencoder 18 | from lstm_multi_input import LSTMmultiInput 19 | 20 | from lgbm_model import LGBM 21 | 22 | from shapely.geometry import Point, Polygon, MultiPolygon 23 | from gee_images import GeeImages 24 | 25 | 26 | 27 | 28 | # Seed to be used for al models etc. 29 | SEED = 400 30 | 31 | df = pd.read_csv("data/moz_13mar_dropna_df.csv") 32 | df1 = pd.read_csv("data/moz_13mar_dropna_df1.csv") 33 | df2 = pd.read_csv("data/moz_13mar_dropna_df2.csv") 34 | df3 = pd.read_csv("data/moz_13mar_dropna_df3.csv") 35 | 36 | train_df = pd.concat([df, df1]) 37 | train_df = pd.concat([train_df, df2]) 38 | train_df = pd.concat([train_df, df3]) 39 | 40 | train_df = train_df[train_df['land_cover_mode_t-1'] != 17] 41 | 42 | 43 | del df, df1, df2, df3 44 | 45 | test_df = pd.read_csv('data/mwi_13mar_df.csv') 46 | 47 | 48 | def prep_data(df, val=True, multi_input=False, classifier=False): 49 | 50 | feats = [i for i in df.columns if "land_cover" not in i] 51 | feat = [i for i in df[feats].columns if ("X" not in i and "Y" not in i)] 52 | 53 | df = df[feat] 54 | #df = df.fillna(df.mean()) 55 | 56 | X, y = df.drop(columns=['target']), df['target'] 57 | 58 | if val: 59 | X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=SEED) 60 | else: 61 | X_train = X 62 | y_train = y 63 | 64 | if multi_input == False: 65 | scaler = MinMaxScaler(feature_range=(0,1)) 66 | scaler.fit(X) 67 | X_train = scaler.transform(X_train) 68 | X_train = X_train.reshape((len(X_train), 30, 8)) 69 | y_train = y_train.values.reshape((len(y_train), 1)) 70 | if val: 71 | X_val = scaler.transform(X_val) 72 | X_val = X_val.reshape((len(X_val), 30, 8)) 73 | y_val = y_val.values.reshape((len(y_val), 1)) 74 | return X_train, y_train, X_val, y_val 75 | else: 76 | return X_train, y_train 77 | else: 78 | cols = [i for i in df.columns if ("X" not in i and "Y" not in i and "mode" not in i)] 79 | cols = [i for i in cols if ("pred" not in i and "target" not in i)] 80 | cols = [i for i in cols if ("dist" not in i and "slope" not in i and "elevation" not in i)] 81 | cols = [i for i in cols if ("soc" not in i and "slope" not in i and "elevation" not in i)] 82 | constant_feats = train_df[['dist_to_water_t-1', 'slope_t-1', 'elevation_t-1', 'soc_mean_t-1']] 83 | temporal_feats = train_df[cols] 84 | const_scaler = MinMaxScaler(feature_range=(0,1)) 85 | temp_scaler = MinMaxScaler(feature_range=(0,1)) 86 | const_scaler.fit(constant_feats) 87 | temp_scaler.fit(temporal_feats) 88 | X_train_const = X_train[constant_feats.columns] 89 | X_train_const = const_scaler.transform(X_train_const) 90 | X_train_temp = X_train[cols] 91 | X_train_temp = temp_scaler.transform(X_train_temp) 92 | X_train_temp = X_train_temp.reshape((len(X_train), 30, 4)) 93 | if val: 94 | X_val_const = X_val[constant_feats.columns] 95 | X_val_const = const_scaler.transform(X_val_const) 96 | X_val_temp = X_val[cols] 97 | X_val_temp = temp_scaler.transform(X_val_temp) 98 | X_val_temp = X_val_temp.reshape((len(X_val), 30, 4)) 99 | 100 | return X_train_temp, X_train_const, y_train, X_val_temp, X_val_const, y_val 101 | else: 102 | return X_train_temp, X_train_const, y_train 103 | 104 | 105 | def rmse(predictions, targets): 106 | predictions = np.float32(predictions) 107 | targets = np.float32(targets) 108 | return np.sqrt(((predictions - targets) ** 2).mean()) 109 | 110 | 111 | def run_lstm_exp(model_list, folds=3, seed=SEED): 112 | 113 | tf.random.set_seed(seed) 114 | 115 | exp_dict = defaultdict(list) 116 | for model in model_list: 117 | if model == lmi: 118 | for fold in range(1, folds+1): 119 | print(f"*Training on fold {fold}") 120 | model.fit(X_train_temp, X_train_const, y_train, X_val_temp, X_val_const, y_val) 121 | preds = model.predict(X_test_temp, X_test_const).reshape(len(X_test,)) 122 | preds[preds < 0.0] = 0.0 123 | preds[preds > 1.0] = 1.0 124 | y = y_test.flatten() 125 | res = rmse(preds, y) 126 | exp_dict[model].append(np.round(res, 6)) 127 | else: 128 | for fold in range(1, folds+1): 129 | print(f"*Training on fold {fold}") 130 | model.fit(X_train, y_train, X_val, y_val) 131 | preds = model.predict(X_test).reshape(len(X_test,)) 132 | y = y_test.flatten() 133 | preds[preds < 0.0] = 0.0 134 | preds[preds > 1.0] = 1.0 135 | res = rmse(preds, y) 136 | exp_dict[model].append(np.round(res, 6)) 137 | return exp_dict 138 | 139 | 140 | 141 | def run_lgbm_exp(model, folds=1, seed=SEED): 142 | np.random.seed(SEED) 143 | res_list = [] 144 | for fold in range(1, folds+1): 145 | print(f"*Training on fold {fold}") 146 | preds = model.fit_predict(seed=np.random.randint(0, 100)) 147 | #y = y_test.flatten() 148 | #res_list.append(rmse(preds, y)) 149 | return preds 150 | 151 | 152 | def run_experiments(model_list): 153 | 154 | #results = run_lstm_exp(model_list) 155 | #res_dict = {str(k):str(v) for k, v in results.items()} 156 | 157 | lgb = LGBM(train_df, test_df) 158 | #res_dict["lgb"] = str(run_lgbm_exp(lgb)) 159 | p = run_lgbm_exp(lgb) 160 | return p 161 | 162 | 163 | 164 | ## Prepare training data 165 | X_train, y_train, X_val, y_val = prep_data(train_df) 166 | X_train_temp, X_train_const, y_train, X_val_temp, X_val_const, y_val = prep_data(train_df, multi_input=True) 167 | ## Prepare test data 168 | X_test, y_test = prep_data(test_df, val=False) 169 | X_test_temp, X_test_const, _ = prep_data(test_df, val=False, multi_input=True) 170 | 171 | 172 | test_df = test_df.sort_values(['Y_t-1', 'X_t-1'], ascending=[False, True]) 173 | 174 | 175 | lgb = LGBM(train_df, test_df) 176 | #res_dict["lgb"] = str(run_lgbm_exp(lgb)) 177 | p = run_lgbm_exp(lgb) 178 | 179 | 180 | 181 | def plot_flood(res, rows, cols): 182 | 183 | flood_list = list(res) 184 | mat = np.array(flood_list).reshape(cols, rows) 185 | plt.imshow(mat) #, cmap='Blues') 186 | return plt.show() 187 | 188 | test_preds = p 189 | 190 | test_preds[test_preds < 0.0] = 0.0 191 | test_preds[test_preds > 1.0] = 1.0 192 | 193 | final_preds = test_preds#.values.reshape(363, -1) #, order='F')) 194 | plot_flood(final_preds, rows=-1, cols=363) 195 | 196 | #final_preds = pd.Series(test_preds.reshape(len(test_preds),)) 197 | 198 | test_res = rmse(np.float32(final_preds), np.float32(y_test)) 199 | 200 | 201 | plot_flood(y_test, rows=-1, cols=363) 202 | 203 | 204 | plot_flood(p, -1, 363) 205 | 206 | plot_flood(y_test, 363, -1) 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | # Hyper-parameters 215 | epoch = 1000 216 | BATCHES = [2048, 512, 128, 32] 217 | LOSS = tf.keras.losses.Huber() 218 | METRIC = tf.keras.metrics.RootMeanSquaredError() 219 | CALLBACKS = tf.keras.callbacks.EarlyStopping(monitor='val_root_mean_squared_error', 220 | restore_best_weights=True, 221 | min_delta=0.0001, patience=10) 222 | 223 | 224 | 225 | result_list = [] 226 | for batch in BATCHES: 227 | lm = LSTMmulti(epoch, batch, LOSS, METRIC, CALLBACKS) 228 | lmi = LSTMmultiInput(epoch, batch, LOSS, METRIC, CALLBACKS) 229 | la = LSTMAutoencoder(epoch, batch, LOSS, METRIC, CALLBACKS) 230 | model_list = [lm, lmi, la] 231 | result_dict = run_experiments(model_list) 232 | with open(f"data/results/dropna/result-{batch}.json", "w") as outfile: 233 | json.dump(result_dict, outfile) 234 | result_list.append(result_dict) 235 | 236 | 237 | -------------------------------------------------------------------------------- /utils/gee.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | import pandas as pd 6 | import numpy as np 7 | 8 | import matplotlib.pyplot as plt 9 | import geopandas as gpd 10 | from shapely.geometry import Point 11 | 12 | moz_zipfile = "zip:///home/leo/Desktop/ml_flood_prediction/data/floods_13-03-2020/moz_flood.zip" 13 | moz_gdf = gpd.read_file(moz_zipfile) 14 | moz_geo = gpd.GeoDataFrame(moz_gdf) 15 | xmin,ymin,xmax,ymax = 33.50, -20.60, 35.50, -19.00 16 | moz_geo = moz_gdf.cx[xmin:xmax, ymin:ymax] 17 | 18 | 19 | mwi_zipfile = "zip:///home/leo/Desktop/ml_flood_prediction/data/floods_13-03-2020/mwi_flood.zip" 20 | mwi_gdf = gpd.read_file(mwi_zipfile) 21 | mwi_geo = gpd.GeoDataFrame(mwi_gdf) 22 | xmin, ymin, xmax, ymax = 34.26, -16.64, 35.86, -15.21 23 | mwi_geo = mwi_gdf.cx[xmin:xmax, ymin:ymax] 24 | 25 | 26 | ken_zipfile = "zip:///home/leo/Desktop/ml_flood_prediction/data/floods_06-05-2020/ky_flood_vec.zip" 27 | ken_gdf = gpd.read_file(ken_zipfile) 28 | ken_geo = gpd.GeoDataFrame(ken_gdf) 29 | xmin,ymin,xmax,ymax = 38.12, 0.77, 39.50, 2.66 30 | ken_geo = ken_gdf.cx[xmin:xmax, ymin:ymax] 31 | 32 | 33 | import ee 34 | import datetime 35 | 36 | ee.Initialize() 37 | 38 | 39 | 40 | 41 | 42 | def rainfall_images(new_geo, flood_start, folder): 43 | """ Export GEE images for rainfall. 44 | Example: flood_start = '2019-03-13' for mozambique. 45 | """ 46 | 47 | trmm = ee.ImageCollection("TRMM/3B42")# new_shapefile = "mwi_3-18.shp" 48 | 49 | trmm = trmm.select(['precipitation']) 50 | 51 | # setting the Area of Interest (AOI) 52 | mwi_aoi = ee.Geometry.Rectangle(list(new_geo.total_bounds)) 53 | 54 | # Get dates for rainfall a month before flood 55 | base = pd.to_datetime(flood_start) 56 | date_list = [(base - datetime.timedelta(days=x)).strftime('%Y-%m-%d') for x in range(31)] 57 | date_list = list(reversed(date_list)) 58 | dates = [(date_list[num-1], date_list[num]) for num in range(1, 31)] 59 | 60 | # Create a list of all images filtered by date 61 | trmm_files = [trmm.filterDate(dates[i][0], dates[i][1]) for i in range(30)] 62 | 63 | # Assign export tasks for GEE 64 | task_list = [] 65 | count = -30 66 | for trmm in trmm_files: 67 | # Filter by are of interest and return the mean for each image 68 | trmm_aoi = trmm.filterBounds(mwi_aoi) 69 | total = trmm_aoi.reduce(ee.Reducer.sum()) 70 | task = ee.batch.Export.image.toDrive(image=total, 71 | region=mwi_aoi.getInfo()['coordinates'], 72 | description='13-mar', 73 | folder=folder, 74 | fileNamePrefix='total_precip_t' + str(count), 75 | maxPixels=1e12, 76 | scale=30, 77 | crs='EPSG:4326') 78 | task_list.append(task) 79 | count += 1 80 | for i in task_list: 81 | i.start() 82 | 83 | return i.status() 84 | 85 | 86 | 87 | def rainfall_images(new_geo, flood_start, folder): 88 | """ Export GEE images for rainfall. 89 | Example: flood_start = '2019-03-13' for mozambique. 90 | """ 91 | 92 | chirps = ee.ImageCollection("UCSB-CHG/CHIRPS/DAILY")# new_shapefile = "mwi_3-18.shp" 93 | 94 | chirps = chirps.select(['precipitation']) 95 | 96 | # setting the Area of Interest (AOI) 97 | mwi_aoi = ee.Geometry.Rectangle(list(new_geo.total_bounds)) 98 | 99 | # Get dates for rainfall a month before flood 100 | base = pd.to_datetime(flood_start) 101 | date_list = [(base - datetime.timedelta(days=x)).strftime('%Y-%m-%d') for x in range(31)] 102 | date_list = list(reversed(date_list)) 103 | dates = [(date_list[num-1], date_list[num]) for num in range(1, 31)] 104 | 105 | # Create a list of all images filtered by date 106 | chirps_files = [chirps.filterDate(dates[i][0], dates[i][1]) for i in range(30)] 107 | 108 | # Assign export tasks for GEE 109 | task_list = [] 110 | count = -30 111 | for chirps in chirps_files: 112 | # Filter by are of interest and return the mean for each image 113 | chirps_aoi = chirps.filterBounds(mwi_aoi) 114 | total = chirps_aoi.reduce(ee.Reducer.sum()) 115 | task = ee.batch.Export.image.toDrive(image=total, 116 | region=mwi_aoi.getInfo()['coordinates'], 117 | description='13-mar', 118 | folder=folder, 119 | fileNamePrefix='total_precip_t' + str(count), 120 | maxPixels=1e12, 121 | scale=30, 122 | crs='EPSG:4326') 123 | task_list.append(task) 124 | count += 1 125 | for i in task_list: 126 | i.start() 127 | 128 | return i.status() 129 | 130 | 131 | 132 | rainfall_images(moz_geo, flood_start='2019-03-15', folder='moz_floods_2019-03-13') 133 | rainfall_images(mwi_geo, flood_start='2019-03-15', folder='mwi_floods_2019-03-13') 134 | rainfall_images(ken_geo, flood_start='2020-05-08', folder='mwi_floods_2020-05-06') 135 | 136 | 137 | def rainfall_max_images(new_geo, flood_start, folder): 138 | """ Export GEE images for rainfall. 139 | Example: flood_start = '2019-03-13' for mozambique. 140 | """ 141 | 142 | trmm = ee.ImageCollection("TRMM/3B42")# new_shapefile = "mwi_3-18.shp" 143 | 144 | trmm = trmm.select(['precipitation']) 145 | 146 | # setting the Area of Interest (AOI) 147 | mwi_aoi = ee.Geometry.Rectangle(list(new_geo.total_bounds)) 148 | 149 | # Get dates for rainfall a month before flood 150 | base = pd.to_datetime(flood_start) 151 | date_list = [(base - datetime.timedelta(days=x)).strftime('%Y-%m-%d') for x in range(31)] 152 | date_list = list(reversed(date_list)) 153 | dates = [(date_list[num-1], date_list[num]) for num in range(1, 31)] 154 | 155 | # Create a list of all images filtered by date 156 | trmm_files = [trmm.filterDate(dates[i][0], dates[i][1]) for i in range(30)] 157 | 158 | # Assign export tasks for GEE 159 | task_list = [] 160 | count = -30 161 | for trmm in trmm_files: 162 | # Filter by are of interest and return the mean for each image 163 | trmm_aoi = trmm.filterBounds(mwi_aoi) 164 | maximum = trmm_aoi.reduce(ee.Reducer.max()) 165 | task = ee.batch.Export.image.toDrive(image=maximum, 166 | region=mwi_aoi.getInfo()['coordinates'], 167 | description='13-mar', 168 | folder=folder, 169 | fileNamePrefix='max_precip_t' + str(count), 170 | maxPixels=1e12, 171 | scale=30, 172 | crs='EPSG:4326') 173 | task_list.append(task) 174 | count += 1 175 | for i in task_list: 176 | i.start() 177 | 178 | return i.status() 179 | 180 | 181 | def rainfall_max_images(new_geo, flood_start, folder): 182 | """ Export GEE images for rainfall. 183 | Example: flood_start = '2019-03-13' for mozambique. 184 | """ 185 | 186 | chirps = ee.ImageCollection("UCSB-CHG/CHIRPS/DAILY")# new_shapefile = "mwi_3-18.shp" 187 | 188 | chirps = chirps.select(['precipitation']) 189 | 190 | # setting the Area of Interest (AOI) 191 | mwi_aoi = ee.Geometry.Rectangle(list(new_geo.total_bounds)) 192 | 193 | # Get dates for rainfall a month before flood 194 | base = pd.to_datetime(flood_start) 195 | date_list = [(base - datetime.timedelta(days=x)).strftime('%Y-%m-%d') for x in range(31)] 196 | date_list = list(reversed(date_list)) 197 | dates = [(date_list[num-1], date_list[num]) for num in range(1, 31)] 198 | 199 | # Create a list of all images filtered by date 200 | chirps_files = [chirps.filterDate(dates[i][0], dates[i][1]) for i in range(30)] 201 | 202 | # Assign export tasks for GEE 203 | task_list = [] 204 | count = -30 205 | for chirps in chirps_files: 206 | # Filter by are of interest and return the mean for each image 207 | chirps_aoi = chirps_files[20].filterBounds(mwi_aoi) 208 | total = chirps_aoi.reduce(ee.Reducer.max()) 209 | task = ee.batch.Export.image.toDrive(image=total, 210 | region=mwi_aoi.getInfo()['coordinates'], 211 | description='13-mar', 212 | folder=folder, 213 | fileNamePrefix='max_precip_t' + str(count), 214 | maxPixels=1e12, 215 | scale=30, 216 | crs='EPSG:4326') 217 | task_list.append(task) 218 | count += 1 219 | for i in task_list: 220 | i.start() 221 | 222 | return i.status() 223 | 224 | 225 | rainfall_max_images(moz_geo, flood_start='2019-03-15', folder='moz_floods_2019-03-13') 226 | rainfall_max_images(mwi_geo, flood_start='2019-03-15', folder='mwi_floods_2019-03-13') 227 | rainfall_max_images(ken_geo, flood_start='2020-05-08', folder='mwi_floods_2020-05-06') 228 | 229 | def rainfall_std_images(new_geo, flood_start, folder): 230 | """ Export GEE images for rainfall. 231 | Example: flood_start = '2019-03-13' for mozambique. 232 | """ 233 | 234 | chirps = ee.ImageCollection("UCSB-CHG/CHIRPS/DAILY")# new_shapefile = "mwi_3-18.shp" 235 | 236 | chirps = chirps.select(['precipitation']) 237 | 238 | # setting the Area of Interest (AOI) 239 | mwi_aoi = ee.Geometry.Rectangle(list(new_geo.total_bounds)) 240 | 241 | # Get dates for rainfall a month before flood 242 | base = pd.to_datetime(flood_start) 243 | date_list = [(base - datetime.timedelta(days=x)).strftime('%Y-%m-%d') for x in range(31)] 244 | date_list = list(reversed(date_list)) 245 | dates = [(date_list[num-1], date_list[num]) for num in range(1, 31)] 246 | 247 | # Create a list of all images filtered by date 248 | chirps_files = [chirps.filterDate(dates[i][0], dates[i][1]) for i in range(30)] 249 | 250 | # Assign export tasks for GEE 251 | task_list = [] 252 | count = -30 253 | for chirps in chirps_files: 254 | # Filter by are of interest and return the mean for each image 255 | chirps_aoi = chirps.filterBounds(mwi_aoi) 256 | total = chirps_aoi.reduce(ee.Reducer.sum()) 257 | task = ee.batch.Export.image.toDrive(image=total, 258 | region=mwi_aoi.getInfo()['coordinates'], 259 | description='13-mar', 260 | folder=folder, 261 | fileNamePrefix='std_precip_t' + str(count), 262 | maxPixels=1e12, 263 | scale=30, 264 | crs='EPSG:4326') 265 | task_list.append(task) 266 | count += 1 267 | for i in task_list: 268 | i.start() 269 | 270 | return i.status() 271 | 272 | return i.status() 273 | 274 | 275 | rainfall_std_images(moz_geo, flood_start='2019-03-15', folder='moz_floods_2019-03-13') 276 | rainfall_std_images(mwi_geo, flood_start='2019-03-15', folder='mwi_floods_2019-03-13') 277 | rainfall_std_images(ken_geo, flood_start='2020-05-08', folder='mwi_floods_2020-05-06') 278 | 279 | 280 | def smap_images(new_geo, flood_start, folder): 281 | """ Export GEE images for rainfall. 282 | Example: flood_start = '2019-03-13' for mwimabique. 283 | """ 284 | 285 | ssm = ee.ImageCollection("NASA_USDA/HSL/SMAP_soil_moisture") # new_shapefile = "mwi_3-18.shp" 286 | 287 | ssm = ssm.select(['ssm']) 288 | 289 | # setting the Area of Interest (AOI) 290 | mwi_aoi = ee.Geometry.Rectangle(list(new_geo.total_bounds)) 291 | 292 | # Get dates for rainfall a month before flood 293 | base = pd.to_datetime(flood_start) 294 | date_list = [(base - datetime.timedelta(days=x)).strftime('%Y-%m-%d') for x in range(31)] 295 | date_list = list(reversed(date_list)) 296 | dates = [(date_list[num-1], date_list[num]) for num in range(1, 31)] 297 | 298 | # Create a list of all images filtered by date 299 | ssm_files = [ssm.filterDate(dates[i][0], dates[i][1]) for i in range(30)] 300 | 301 | # Assign export tasks for GEE 302 | task_list = [] 303 | count = -30 304 | for ssm in ssm_files: 305 | # Filter by are of interest and return the mean for each image 306 | ssm_aoi = ssm.filterBounds(mwi_aoi) 307 | total = ssm_aoi.reduce(ee.Reducer.sum()) 308 | task = ee.batch.Export.image.toDrive(image=total, 309 | region=mwi_aoi.getInfo()['coordinates'], 310 | description='13-mar', 311 | folder=folder, 312 | fileNamePrefix='ssm_t' + str(count), 313 | maxPixels=1e12, 314 | scale=30, 315 | crs='EPSG:4326') 316 | task_list.append(task) 317 | count += 1 318 | for i in task_list: 319 | i.start() 320 | 321 | return i.status() 322 | 323 | 324 | smap_images(moz_geo, flood_start='2019-03-15', folder='moz_floods_2019-03-13') 325 | smap_images(mwi_geo, flood_start='2019-03-15', folder='mwi_floods_2019-03-13') 326 | smap_images(ken_geo, flood_start='2020-05-08', folder='mwi_floods_2020-05-06') 327 | 328 | 329 | def lc_images(new_geo, flood_start, folder) : 330 | 331 | modis = ee.ImageCollection("MODIS/006/MCD12Q1") # new_shapefile = "mwi_3-18.shp" 332 | modis = modis.select(['LC_Type1']) 333 | 334 | # setting the Area of Interest (AOI) 335 | mwi_aoi = ee.Geometry.Rectangle(list(new_geo.total_bounds)) 336 | modis_aoi = modis.filterBounds(mwi_aoi) 337 | mode = modis_aoi.reduce(ee.Reducer.mode()) 338 | #least_cloudy = ee.Image(modis_aoi.sort('CLOUD_COVER').first()) 339 | task = ee.batch.Export.image.toDrive(image=mode, 340 | region=mwi_aoi.getInfo()['coordinates'], 341 | description='13-mar', 342 | folder=folder, 343 | fileNamePrefix='land_cover_mode', 344 | scale=30, 345 | crs='EPSG:4326') 346 | task.start() 347 | return task.status() 348 | 349 | 350 | 351 | lc_images(moz_geo, flood_start='2019-03-15', folder='moz_floods_2019-03-13') 352 | lc_images(mwi_geo, flood_start='2019-03-15', folder='mwi_floods_2019-03-13') 353 | lc_images(ken_geo, flood_start='2020-05-08', folder='mwi_floods_2020-05-06') 354 | 355 | 356 | def elevation_images(new_geo, flood_start, folder): 357 | srtm = ee.Image("USGS/SRTMGL1_003") 358 | srtm = srtm.select('elevation') 359 | mwi_aoi = ee.Geometry.Rectangle(list(new_geo.total_bounds)) 360 | srtm_aoi = srtm.clip(mwi_aoi) 361 | task = ee.batch.Export.image.toDrive(image=srtm_aoi, 362 | region=mwi_aoi.getInfo()['coordinates'], 363 | description='13-mar', 364 | folder=folder, 365 | fileNamePrefix='elevation', 366 | scale=30, 367 | crs='EPSG:4326') 368 | task.start() 369 | return task.status() 370 | 371 | elevation_images(moz_geo, flood_start='2019-03-15', folder='moz_floods_2019-03-13') 372 | elevation_images(mwi_geo, flood_start='2019-03-15', folder='mwi_floods_2019-03-13') 373 | elevation_images(ken_geo, flood_start='2020-05-08', folder='mwi_floods_2020-05-06') 374 | 375 | 376 | def slope_images(new_geo, flood_start, folder): 377 | srtm = ee.Image("USGS/SRTMGL1_003") 378 | srtm = srtm.select('elevation') 379 | slope = ee.Terrain.slope(srtm) 380 | mwi_aoi = ee.Geometry.Rectangle(list(new_geo.total_bounds)) 381 | srtm_aoi = slope.clip(mwi_aoi) 382 | task = ee.batch.Export.image.toDrive(image=srtm_aoi, 383 | region=mwi_aoi.getInfo()['coordinates'], 384 | description='13-mar', 385 | folder=folder, 386 | fileNamePrefix='slope', 387 | scale=30, 388 | crs='EPSG:4326') 389 | task.start() 390 | return task.status() 391 | 392 | 393 | slope_images(moz_geo, flood_start='2019-03-15', folder='moz_floods_2019-03-13') 394 | slope_images(mwi_geo, flood_start='2019-03-15', folder='mwi_floods_2019-03-13') 395 | slope_images(ken_geo, flood_start='2020-05-08', folder='mwi_floods_2020-05-06') 396 | 397 | 398 | 399 | def ndvi_images(new_geo, flood_start, folder): 400 | """ Export GEE images for rainfall. 401 | Example: flood_start = '2019-03-13' for mwimabique. 402 | """ 403 | ndvi = ee.ImageCollection('MODIS/006/MOD13A2') 404 | # choose dates 405 | ndvi = ndvi.select('NDVI') 406 | # setting the Area of Interest (AOI) 407 | # setting the Area of Interest (AOI) 408 | mwi_aoi = ee.Geometry.Rectangle(list(new_geo.total_bounds)) 409 | 410 | # Get dates for rainfall a month before flood 411 | base = pd.to_datetime(flood_start) 412 | date_list = [(base - datetime.timedelta(days=x)).strftime('%Y-%m-%d') for x in range(31)] 413 | date_list = list(reversed(date_list)) 414 | dates = [(date_list[num-1], date_list[num]) for num in range(1, 31)] 415 | 416 | # Create a list of all images filtered by date 417 | ndvi_files = [ndvi.filterDate(dates[i][0], dates[i][1]) for i in range(30)] 418 | # the least cloudy image 419 | 420 | # Assign export tasks for GEE 421 | task_list = [] 422 | count = -30 423 | for ndvi in ndvi_files: 424 | # Filter by are of interest and return the mean for each image 425 | ndvi_aoi = ndvi.filterBounds(mwi_aoi) 426 | total = ndvi_aoi.reduce(ee.Reducer.sum()) 427 | task = ee.batch.Export.image.toDrive(image=total.float(), 428 | region=mwi_aoi.getInfo()['coordinates'], 429 | description='13-mar', 430 | folder=folder, 431 | fileNamePrefix='ndvi_t' + str(count), 432 | maxPixels=1e12, 433 | scale=30, 434 | crs='EPSG:4326') 435 | task_list.append(task) 436 | count += 1 437 | for i in task_list: 438 | i.start() 439 | 440 | return i.status() 441 | 442 | 443 | 444 | ndvi_images(moz_geo, flood_start='2019-03-15', folder='moz_floods_2019-03-13') 445 | ndvi_images(mwi_geo, flood_start='2019-03-15', folder='mwi_floods_2019-03-13') 446 | ndvi_images(ken_geo, flood_start='2020-05-08', folder='mwi_floods_2020-05-06') 447 | 448 | 449 | #### Get soil organic carbon 450 | from tiff_image import CRS 451 | 452 | moz_crs = CRS("data/moz_soil_organic_carbon.tif") 453 | moz_crs.convert("data/floods_13-03-2020/moz_feats/soc_mean.tif") 454 | 455 | mwi_crs = CRS("data/mwi_soil_organic_carbon.tif") 456 | mwi_crs.convert("data/floods_13-03-2020/mwi_feats/soc_mean.tif") 457 | 458 | 459 | 460 | 461 | 462 | 463 | 464 | 465 | 466 | 467 | 468 | 469 | 470 | 471 | 472 | 473 | -------------------------------------------------------------------------------- /utils/gee_image_generation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import ee 5 | from datetime import datetime as dt 6 | 7 | ee.Initialize() 8 | 9 | """ 10 | This script contains functionality for exporting Google Earth Engine images to 11 | Google Drive folder. 12 | """ 13 | 14 | 15 | ############## Training flood data ###################################### 16 | 17 | ##### == Check if these exports match in the provided datatset == 18 | 19 | 20 | trmm = ee.ImageCollection("TRMM/3B42")# new_shapefile = "mwi_3-18.shp" 21 | 22 | trmm = trmm.select(['precipitation', 'HQprecipitation', 'IRprecipitation']) 23 | 24 | 25 | trmm_1 = trmm.filterDate('2015-03-08','2015-03-15') 26 | 27 | trmm_2 = trmm.filterDate('2015-03-01','2015-03-08') 28 | 29 | trmm_3 = trmm.filterDate('2015-02-22','2015-03-01') 30 | 31 | trmm_4 = trmm.filterDate('2015-02-15','2015-02-22') 32 | 33 | trmm_5 = trmm.filterDate('2015-02-08','2015-02-22') 34 | 35 | trmm_6 = trmm.filterDate('2015-02-01','2015-02-08') 36 | 37 | trmm_7 = trmm.filterDate('2015-01-25','2015-02-01') 38 | 39 | trmm_8 = trmm.filterDate('2015-01-18','2015-01-25') 40 | 41 | trmm_9 = trmm.filterDate('2015-01-11','2015-01-18') 42 | 43 | trmm_10 = trmm.filterDate('2015-01-04','2015-01-11') 44 | 45 | trmm_11 = trmm.filterDate('2014-12-28','2015-01-04') 46 | 47 | trmm_12 = trmm.filterDate('2014-12-21','2014-12-28') 48 | 49 | trmm_13 = trmm.filterDate('2014-12-14','2014-12-21') 50 | 51 | trmm_14 = trmm.filterDate('2014-12-07','2014-12-14') 52 | 53 | trmm_15 = trmm.filterDate('2014-11-30','2014-12-07') 54 | 55 | trmm_16 = trmm.filterDate('2014-11-23','2014-11-30') 56 | 57 | trmm_17 = trmm.filterDate('2014-11-16','2014-11-23') 58 | 59 | # setting the Area of Interest (AOI) 60 | mwi_aoi = ee.Geometry.Rectangle([34.255, -16.645, 35.865, -15.205]) 61 | 62 | #trmm_2_aoi = trmm_2.filterBounds(mwi_aoi) 63 | #std = trmm_2.reduce(ee.Reducer.std()) 64 | 65 | trmm_list = [trmm_1, trmm_2, trmm_3, trmm_4, trmm_5, trmm_6, trmm_7, trmm_8, trmm_9, 66 | trmm_10, trmm_11, trmm_12, trmm_13, trmm_14, trmm_15, trmm_16, trmm_17] 67 | 68 | task_list = [] 69 | count = 1 70 | for trmm in trmm_list: 71 | trmm_aoi = trmm.filterBounds(mwi_aoi) 72 | mean = trmm_aoi.reduce(ee.Reducer.mean()) 73 | 74 | task = ee.batch.Export.image.toDrive(image=mean, 75 | region=mwi_aoi.getInfo()['coordinates'], 76 | description='mwi_precip_for_15-03-15_flood', 77 | folder='mwi_floods', 78 | fileNamePrefix='precip_wk' + str(count) + '-tr', 79 | scale=30, 80 | crs='EPSG:4326') 81 | task_list.append(task) 82 | count += 1 83 | 84 | 85 | for i in task_list: 86 | i.start() 87 | 88 | 89 | 90 | task_list = [] 91 | count = 1 92 | for trmm in trmm_list: 93 | trmm_aoi = trmm.filterBounds(mwi_aoi) 94 | total = trmm_aoi.reduce(ee.Reducer.sum()) 95 | 96 | task = ee.batch.Export.image.toDrive(image=total, 97 | region=mwi_aoi.getInfo()['coordinates'], 98 | description='mwi_precip_for_15-03-15_flood', 99 | folder='mwi_floods', 100 | fileNamePrefix='precip_wk' + str(count) + '_total-tr', 101 | scale=30, 102 | crs='EPSG:4326') 103 | task_list.append(task) 104 | count += 1 105 | 106 | for i in task_list: 107 | i.start() 108 | 109 | task_list = [] 110 | count = 1 111 | for trmm in trmm_list: 112 | trmm_aoi = trmm.filterBounds(mwi_aoi) 113 | var = trmm_aoi.reduce(ee.Reducer.variance()) 114 | 115 | task = ee.batch.Export.image.toDrive(image=var, 116 | region=mwi_aoi.getInfo()['coordinates'], 117 | description='mwi_precip_for_15-03-15_flood', 118 | folder='mwi_floods', 119 | fileNamePrefix='precip_wk' + str(count) + '_var-tr', 120 | scale=30, 121 | crs='EPSG:4326') 122 | task_list.append(task) 123 | count += 1 124 | 125 | for i in task_list: 126 | i.start() 127 | 128 | 129 | smap = ee.ImageCollection("NASA_USDA/HSL/SMAP_soil_moisture") 130 | # choose dates 131 | smap = smap.filterDate('2014-12-04', '2015-01-04') 132 | # Choose band 133 | smap = smap.select(['ssm', 'susm']) 134 | 135 | aoi_bound = smap.filterBounds(mwi_aoi) 136 | 137 | # the least cloudy image 138 | least_cloudy = ee.Image(aoi_bound.sort('CLOUD_COVER').first()) 139 | # how cloudy is it? 140 | #print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 141 | # when was this image taken? 142 | # date = ee.Date(least_cloudy.get('system:time_start')) 143 | # time = date.getInfo()['value']/1000. 144 | # dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 145 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 146 | region=mwi_aoi.getInfo()['coordinates'], 147 | description='train_smap', 148 | folder='train', 149 | fileNamePrefix='train_smap', 150 | scale=30, 151 | crs='EPSG:4326') 152 | task.start() 153 | 154 | 155 | ## Ndvi 156 | 157 | 158 | 159 | 160 | 161 | ndvi = ee.ImageCollection('MODIS/006/MOD13A2') 162 | # choose dates 163 | ndvi = ndvi.filterDate('2014-12-04', '2015-01-04') 164 | ndvi = ndvi.select(['NDVI', 'EVI']) 165 | 166 | # setting the Area of Interest (AOI) 167 | #moz_aoi = ee.Geometry.Rectangle(aoi) 168 | aoi_bound = ndvi.filterBounds(mwi_aoi) 169 | 170 | # the least cloudy image 171 | least_cloudy = ee.Image(aoi_bound.sort('CLOUD_COVER').first()) 172 | # how cloudy is it? 173 | print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 174 | # when was this image taken? 175 | date = ee.Date(least_cloudy.get('system:time_start')) 176 | time = date.getInfo()['value']/1000. 177 | dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 178 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 179 | region=mwi_aoi.getInfo()['coordinates'], 180 | description='train' + '_ndvi', 181 | folder='train', 182 | fileNamePrefix='ndvi', 183 | scale=30, 184 | crs='EPSG:4326') 185 | task.start() 186 | 187 | 188 | 189 | ## Evapotranspiration 190 | 191 | evap = ee.ImageCollection("MODIS/006/MOD16A2") 192 | 193 | # choose dates 194 | evap = evap.filterDate('2014-12-04', '2015-01-04') 195 | evap = evap.select('ET') 196 | 197 | # setting the Area of Interest (AOI) 198 | aoi_bound = evap.filterBounds(mwi_aoi) 199 | 200 | # the least cloudy image 201 | least_cloudy = ee.Image(aoi_bound.sort('CLOUD_COVER').first()) 202 | # how cloudy is it? 203 | print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 204 | # when was this image taken? 205 | date = ee.Date(least_cloudy.get('system:time_start')) 206 | time = date.getInfo()['value']/1000. 207 | dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 208 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 209 | region=mwi_aoi.getInfo()['coordinates'], 210 | description='train' + '_evap', 211 | folder='train', 212 | fileNamePrefix='evap', 213 | scale=30, 214 | crs='EPSG:4326') 215 | task.start() 216 | 217 | 218 | 219 | 220 | 221 | ## climate fetaures 222 | 223 | clim = ee.ImageCollection("ECMWF/ERA5/DAILY") 224 | 225 | # choose dates 226 | clim = clim.filterDate('2015-01-12','2015-01-14') 227 | clim = clim.select(['mean_2m_air_temperature', 'dewpoint_2m_temperature', 228 | 'total_precipitation', 'u_component_of_wind_10m', 229 | 'v_component_of_wind_10m']) 230 | 231 | # setting the Area of Interest (AOI) 232 | aoi_bound = clim.filterBounds(mwi_aoi) 233 | 234 | # the least cloudy image 235 | least_cloudy = ee.Image(aoi_bound.sort('CLOUD_COVER').first()) 236 | # how cloudy is it? 237 | print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 238 | # when was this image taken? 239 | date = ee.Date(least_cloudy.get('system:time_start')) 240 | time = date.getInfo()['value']/1000. 241 | dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 242 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 243 | region=mwi_aoi.getInfo()['coordinates'], 244 | description='train' + '_clim', 245 | folder='train', 246 | fileNamePrefix='clim', 247 | scale=30, 248 | crs='EPSG:4326') 249 | task.start() 250 | 251 | 252 | 253 | 254 | ## Land Surface Temperature 255 | 256 | land_temp = ee.ImageCollection("MODIS/006/MOD11A1") 257 | 258 | 259 | lt_1 = land_temp.filterDate('2015-03-08','2015-03-15') 260 | 261 | lt_2 = land_temp.filterDate('2015-03-01','2015-03-08') 262 | 263 | lt_3 = land_temp.filterDate('2015-02-22','2015-03-01') 264 | 265 | lt_4 = land_temp.filterDate('2015-02-15','2015-02-22') 266 | 267 | lt_5 = land_temp.filterDate('2015-02-08','2015-02-22') 268 | 269 | lt_6 = land_temp.filterDate('2015-02-01','2015-02-08') 270 | 271 | lt_7 = land_temp.filterDate('2015-01-25','2015-02-01') 272 | 273 | lt_8 = land_temp.filterDate('2015-01-18','2015-01-25') 274 | 275 | lt_9 = land_temp.filterDate('2015-01-11','2015-01-18') 276 | 277 | lt_10 = land_temp.filterDate('2015-01-04','2015-01-11') 278 | 279 | lt_11 = land_temp.filterDate('2014-12-28','2015-01-04') 280 | 281 | lt_12 = land_temp.filterDate('2014-12-21','2014-12-28') 282 | 283 | lt_13 = land_temp.filterDate('2014-12-14','2014-12-21') 284 | 285 | lt_14 = land_temp.filterDate('2014-12-07','2014-12-14') 286 | 287 | lt_15 = land_temp.filterDate('2014-11-30','2014-12-07') 288 | 289 | lt_16 = land_temp.filterDate('2014-11-23','2014-11-30') 290 | 291 | lt_17 = land_temp.filterDate('2014-11-16','2014-11-23') 292 | 293 | 294 | 295 | lt_list = [lt_1, lt_2, lt_3, lt_4, lt_5, lt_6, lt_7, lt_8, lt_9, 296 | lt_10, lt_11, lt_12, lt_13, lt_14, lt_15, lt_16, lt_17] 297 | 298 | task_list = [] 299 | count = 1 300 | for lt in lt_list: 301 | lt_aoi = lt.select('LST_Day_1km').filterBounds(mwi_aoi) 302 | mean = lt_aoi.reduce(ee.Reducer.mean()) 303 | 304 | task = ee.batch.Export.image.toDrive(image=mean, 305 | region=mwi_aoi.getInfo()['coordinates'], 306 | description='land_temp_for_15-03-15_flood', 307 | folder='mwi_land_temp', 308 | fileNamePrefix='land_temp_wk' + str(count) + '-tr', 309 | scale=30, 310 | crs='EPSG:4326') 311 | task_list.append(task) 312 | count += 1 313 | 314 | 315 | for i in task_list: 316 | i.start() 317 | 318 | 319 | 320 | 321 | 322 | ############# Test flood data ########################################### 323 | 324 | ##### == Check if these exports match in the provided test set == 325 | 326 | 327 | trmm = ee.ImageCollection("TRMM/3B42")# new_shapefile = "mwi_3-18.shp" 328 | 329 | trmm = trmm.select(['precipitation', 'HQprecipitation', 'IRprecipitation']) 330 | 331 | 332 | trmm_1 = trmm.filterDate('2019-05-12','2019-05-19') 333 | 334 | trmm_2 = trmm.filterDate('2019-05-05','2019-05-12') 335 | 336 | trmm_3 = trmm.filterDate('2019-04-28','2019-05-05') 337 | 338 | trmm_4 = trmm.filterDate('2019-04-21','2019-04-28') 339 | 340 | trmm_5 = trmm.filterDate('2019-04-14','2019-04-21') 341 | 342 | trmm_6 = trmm.filterDate('2019-04-07','2019-04-14') 343 | 344 | trmm_7 = trmm.filterDate('2019-03-31','2019-04-07') 345 | 346 | trmm_8 = trmm.filterDate('2019-03-24','2019-03-31') 347 | 348 | trmm_9 = trmm.filterDate('2019-03-17','2019-03-24') 349 | 350 | trmm_10 = trmm.filterDate('2019-03-10','2019-03-17') 351 | 352 | trmm_11 = trmm.filterDate('2019-03-03','2019-03-10') 353 | 354 | trmm_12 = trmm.filterDate('2019-02-24','2019-03-03') 355 | 356 | trmm_13 = trmm.filterDate('2019-02-17','2019-02-24') 357 | 358 | trmm_14 = trmm.filterDate('2019-02-10','2019-02-17') 359 | 360 | trmm_15 = trmm.filterDate('2019-02-03','2019-02-10') 361 | 362 | trmm_16 = trmm.filterDate('2019-01-27','2019-02-03') 363 | 364 | trmm_17 = trmm.filterDate('2019-01-20','2019-01-27') 365 | 366 | 367 | # setting the Area of Interest (AOI) 368 | mwi_aoi = ee.Geometry.Rectangle([34.255, -16.645, 35.865, -15.205]) 369 | 370 | #trmm_2_aoi = trmm_2.filterBounds(mwi_aoi) 371 | #std = trmm_2.reduce(ee.Reducer.std()) 372 | 373 | trmm_list = [trmm_1, trmm_2, trmm_3, trmm_4, trmm_5, trmm_6, trmm_7, trmm_8, trmm_9, 374 | trmm_10, trmm_11, trmm_12, trmm_13, trmm_14, trmm_15, trmm_16, trmm_17] 375 | task_list = [] 376 | count = 1 377 | for trmm in trmm_list: 378 | trmm_aoi = trmm.filterBounds(mwi_aoi) 379 | total = trmm_aoi.reduce(ee.Reducer.sum()) 380 | 381 | task = ee.batch.Export.image.toDrive(image=total, 382 | region=mwi_aoi.getInfo()['coordinates'], 383 | description='mwi_precip_for_05-19_flood', 384 | folder='my_gdrive_folder', 385 | fileNamePrefix='precip_wk' + str(count) + '_total-te', 386 | scale=30, 387 | crs='EPSG:4326') 388 | task_list.append(task) 389 | count += 1 390 | 391 | for i in task_list: 392 | i.start() 393 | 394 | 395 | 396 | smap = ee.ImageCollection("NASA_USDA/HSL/SMAP_soil_moisture") 397 | # choose dates 398 | smap = smap.filterDate('2019-02-10', '2019-03-10') 399 | # Choose band 400 | smap = smap.select(['ssm', 'susm']) 401 | 402 | aoi_bound = smap.filterBounds(mwi_aoi) 403 | 404 | # the least cloudy image 405 | least_cloudy = ee.Image(aoi_bound.sort('CLOUD_COVER').first()) 406 | # how cloudy is it? 407 | #print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 408 | # when was this image taken? 409 | # date = ee.Date(least_cloudy.get('system:time_start')) 410 | # time = date.getInfo()['value']/1000. 411 | # dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 412 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 413 | region=mwi_aoi.getInfo()['coordinates'], 414 | description='test_smap', 415 | folder='test', 416 | fileNamePrefix='test_smap', 417 | scale=30, 418 | crs='EPSG:4326') 419 | task.start() 420 | 421 | 422 | ## ndvi 423 | 424 | ndvi = ee.ImageCollection('MODIS/006/MOD13A2') 425 | # choose dates 426 | ndvi = ndvi.filterDate('2019-02-10', '2019-03-10') 427 | ndvi = ndvi.select(['NDVI', 'EVI']) 428 | 429 | # setting the Area of Interest (AOI) 430 | #moz_aoi = ee.Geometry.Rectangle(aoi) 431 | aoi_bound = ndvi.filterBounds(mwi_aoi) 432 | 433 | # the least cloudy image 434 | least_cloudy = ee.Image(aoi_bound.sort('CLOUD_COVER').first()) 435 | # how cloudy is it? 436 | print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 437 | # when was this image taken? 438 | date = ee.Date(least_cloudy.get('system:time_start')) 439 | time = date.getInfo()['value']/1000. 440 | dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 441 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 442 | region=mwi_aoi.getInfo()['coordinates'], 443 | description='test' + '_ndvi', 444 | folder='test', 445 | fileNamePrefix='ndvi', 446 | scale=30, 447 | crs='EPSG:4326') 448 | task.start() 449 | 450 | 451 | 452 | ## Evapotranspiration 453 | 454 | evap = ee.ImageCollection("MODIS/006/MOD16A2") 455 | 456 | # choose dates 457 | evap = evap.filterDate('2019-02-10', '2019-03-10') 458 | evap = evap.select('ET') 459 | 460 | # setting the Area of Interest (AOI) 461 | aoi_bound = evap.filterBounds(mwi_aoi) 462 | 463 | # the least cloudy image 464 | least_cloudy = ee.Image(aoi_bound.sort('CLOUD_COVER').first()) 465 | # how cloudy is it? 466 | print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 467 | # when was this image taken? 468 | date = ee.Date(least_cloudy.get('system:time_start')) 469 | time = date.getInfo()['value']/1000. 470 | dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 471 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 472 | region=mwi_aoi.getInfo()['coordinates'], 473 | description='test' + '_evap', 474 | folder='test', 475 | fileNamePrefix='evap', 476 | scale=30, 477 | crs='EPSG:4326') 478 | task.start() 479 | 480 | 481 | ## climate fetaures 482 | 483 | clim = ee.ImageCollection("ECMWF/ERA5/DAILY") 484 | 485 | # choose dates 486 | clim = clim.filterDate('2019-03-13','2019-03-15') 487 | clim = clim.select(['mean_2m_air_temperature', 'dewpoint_2m_temperature', 488 | 'total_precipitation', 'u_component_of_wind_10m', 489 | 'v_component_of_wind_10m']) 490 | 491 | # setting the Area of Interest (AOI) 492 | aoi_bound = clim.filterBounds(mwi_aoi) 493 | 494 | # the least cloudy image 495 | least_cloudy = ee.Image(aoi_bound.sort('CLOUD_COVER').first()) 496 | # how cloudy is it? 497 | print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 498 | # when was this image taken? 499 | date = ee.Date(least_cloudy.get('system:time_start')) 500 | time = date.getInfo()['value']/1000. 501 | dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 502 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 503 | region=mwi_aoi.getInfo()['coordinates'], 504 | description='test' + '_clim', 505 | folder='test', 506 | fileNamePrefix='clim', 507 | scale=30, 508 | crs='EPSG:4326') 509 | task.start() 510 | 511 | 512 | 513 | 514 | 515 | 516 | ## Land Surface Temperature 517 | 518 | land_temp = ee.ImageCollection("MODIS/006/MOD11A1") 519 | 520 | 521 | 522 | lt_1 = lt.filterDate('2019-05-12','2019-05-19') 523 | 524 | lt_2 = lt.filterDate('2019-05-05','2019-05-12') 525 | 526 | lt_3 = lt.filterDate('2019-04-28','2019-05-05') 527 | 528 | lt_4 = lt.filterDate('2019-04-21','2019-04-28') 529 | 530 | lt_5 = lt.filterDate('2019-04-14','2019-04-21') 531 | 532 | lt_6 = lt.filterDate('2019-04-07','2019-04-14') 533 | 534 | lt_7 = lt.filterDate('2019-03-31','2019-04-07') 535 | 536 | lt_8 = lt.filterDate('2019-03-24','2019-03-31') 537 | 538 | lt_9 = lt.filterDate('2019-03-17','2019-03-24') 539 | 540 | lt_10 = lt.filterDate('2019-03-10','2019-03-17') 541 | 542 | lt_11 = lt.filterDate('2019-03-03','2019-03-10') 543 | 544 | lt_12 = lt.filterDate('2019-02-24','2019-03-03') 545 | 546 | lt_13 = lt.filterDate('2019-02-17','2019-02-24') 547 | 548 | lt_14 = lt.filterDate('2019-02-10','2019-02-17') 549 | 550 | lt_15 = lt.filterDate('2019-02-03','2019-02-10') 551 | 552 | lt_16 = lt.filterDate('2019-01-27','2019-02-03') 553 | 554 | lt_17 = lt.filterDate('2019-01-20','2019-01-27') 555 | 556 | 557 | 558 | lt_list = [lt_1, lt_2, lt_3, lt_4, lt_5, lt_6, lt_7, lt_8, lt_9, 559 | lt_10, lt_11, lt_12, lt_13, lt_14, lt_15, lt_16, lt_17] 560 | 561 | task_list = [] 562 | count = 1 563 | for lt in lt_list: 564 | lt_aoi = lt.select('LST_Day_1km').filterBounds(mwi_aoi) 565 | mean = lt_aoi.reduce(ee.Reducer.mean()) 566 | 567 | task = ee.batch.Export.image.toDrive(image=mean, 568 | region=mwi_aoi.getInfo()['coordinates'], 569 | description='land_temp_for_05-19_flood', 570 | folder='mwi_land_temp', 571 | fileNamePrefix='land_temp_wk' + str(count) + '-te', 572 | scale=30, 573 | crs='EPSG:4326') 574 | task_list.append(task) 575 | count += 1 576 | 577 | 578 | for i in task_list: 579 | i.start() 580 | 581 | 582 | 583 | ############################################################################### 584 | ############################################################################### 585 | 586 | # Extract new features for the new flood event data 587 | 588 | ## Functions 589 | 590 | 591 | def begin_trmm_export(aoi, date_list, stat_name, folder): 592 | """ Get precip tif files from GEE """ 593 | if stat_name == 'mean': 594 | stat = ee.Reducer.mean() 595 | if stat_name == 'total': 596 | stat = ee.Reducer.sum() 597 | if stat_name == 'var': 598 | stat = ee.Reducer.variance() 599 | task_list = [] 600 | count = 1 601 | for trmm in trmm_list: 602 | trmm_aoi = trmm.filterBounds(aoi) 603 | task = ee.batch.Export.image.toDrive(image=trmm_aoi.reduce(stat) , 604 | region=mwi_aoi.getInfo()['coordinates'], 605 | description=folder + '_' + stat_name, 606 | folder=folder, 607 | fileNamePrefix='precip_wk-' + str(count) + f'_{stat_name}', 608 | scale=30, 609 | crs='EPSG:4326') 610 | task_list.append(task) 611 | count += 1 612 | for i in task_list: 613 | print(f"Start task {i}") 614 | i.start() 615 | 616 | return task.status() 617 | 618 | 619 | 620 | def begin_lc_export(aoi, start_date, end_date, folder): 621 | """ Get Landcover data. """ 622 | modis = ee.ImageCollection('MODIS/006/MCD12Q1') 623 | # choose dates 624 | modis = modis.filterDate(start_date,end_date) 625 | 626 | # setting the Area of Interest (AOI) 627 | #moz_aoi = ee.Geometry.Rectangle(aoi) 628 | aoi_bound = modis.filterBounds(aoi) 629 | 630 | # the least cloudy image 631 | least_cloudy = ee.Image(aoi_bound.sort('CLOUD_COVER').first()) 632 | # how cloudy is it? 633 | print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 634 | # when was this image taken? 635 | date = ee.Date(least_cloudy.get('system:time_start')) 636 | time = date.getInfo()['value']/1000. 637 | dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 638 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 639 | region=aoi.getInfo()['coordinates'], 640 | description=folder + '_lc', 641 | folder=folder, 642 | fileNamePrefix='lc_type1', 643 | scale=30, 644 | crs='EPSG:4326') 645 | task.start() 646 | return task.status() 647 | 648 | 649 | 650 | def begin_smap_export(aoi, start_date, end_date, folder, dataset=['ssm', 'susm']): 651 | """ Get Soil Moisture. """ 652 | smap = ee.ImageCollection("NASA_USDA/HSL/SMAP_soil_moisture") 653 | # choose dates 654 | smap = smap.filterDate(start_date,end_date) 655 | # Choose band 656 | smap = smap.select(dataset) 657 | aoi_bound = smap.filterBounds(aoi) 658 | 659 | # the least cloudy image 660 | least_cloudy = ee.Image(aoi_bound.sort('CLOUD_COVER').first()) 661 | # how cloudy is it? 662 | print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 663 | # when was this image taken? 664 | date = ee.Date(least_cloudy.get('system:time_start')) 665 | time = date.getInfo()['value']/1000. 666 | dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 667 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 668 | region=aoi.getInfo()['coordinates'], 669 | description=folder + '_smap', 670 | folder=folder, 671 | fileNamePrefix='smap', 672 | maxPixels=1e9, 673 | scale=30, 674 | crs='EPSG:4326') 675 | task.start() 676 | return task.status() 677 | 678 | 679 | def begin_ndvi_export(aoi, start_date, end_date, folder, dataset=['NDVI', 'EVI']): 680 | """ Get Normalized difference vegetation index/enhanced vegetation index.""" 681 | ndvi = ee.ImageCollection('MODIS/006/MOD13A2') 682 | # choose dates 683 | ndvi = ndvi.filterDate(start_date, end_date) 684 | ndvi = ndvi.select(dataset) 685 | 686 | # setting the Area of Interest (AOI) 687 | #moz_aoi = ee.Geometry.Rectangle(aoi) 688 | aoi_bound = ndvi.filterBounds(aoi) 689 | 690 | # the least cloudy image 691 | least_cloudy = ee.Image(aoi_bound.sort('CLOUD_COVER').first()) 692 | # how cloudy is it? 693 | print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 694 | # when was this image taken? 695 | date = ee.Date(least_cloudy.get('system:time_start')) 696 | time = date.getInfo()['value']/1000. 697 | dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 698 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 699 | region=aoi.getInfo()['coordinates'], 700 | description=folder + '_ndvi', 701 | folder=folder, 702 | fileNamePrefix='ndvi', 703 | scale=30, 704 | crs='EPSG:4326') 705 | task.start() 706 | return task.status() 707 | 708 | 709 | def begin_evap_export(aoi, start_date, end_date, folder, dataset=['ET']): 710 | """Get evapotranspiration. """ 711 | evap = ee.ImageCollection("MODIS/006/MOD16A2") 712 | # choose dates 713 | evap = evap.filterDate(start_date, end_date) 714 | evap = evap.select(dataset) 715 | # setting the Area of Interest (AOI) 716 | aoi_bound = evap.filterBounds(aoi) 717 | # the least cloudy image 718 | least_cloudy = ee.Image(aoi_bound.sort('CLOUD_COVER').first()) 719 | # how cloudy is it? 720 | print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 721 | # when was this image taken? 722 | date = ee.Date(least_cloudy.get('system:time_start')) 723 | time = date.getInfo()['value']/1000. 724 | dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 725 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 726 | region=aoi.getInfo()['coordinates'], 727 | description=folder + '_evap', 728 | folder=folder, 729 | fileNamePrefix='evap', 730 | scale=30, 731 | crs='EPSG:4326') 732 | task.start() 733 | return task.status() 734 | 735 | 736 | def begin_clim_export(aoi, start_date, end_date, folder): 737 | """ Get climate features. """ 738 | clim = ee.ImageCollection("ECMWF/ERA5/DAILY") 739 | # choose dates 740 | clim = clim.filterDate(start_date, end_date) 741 | clim = clim.select(['mean_2m_air_temperature', 'dewpoint_2m_temperature', 742 | 'total_precipitation', 'u_component_of_wind_10m', 743 | 'v_component_of_wind_10m']) 744 | # setting the Area of Interest (AOI) 745 | aoi_bound = clim.filterBounds(aoi) 746 | # the least cloudy image 747 | least_cloudy = ee.Image(aoi_bound.sort('CLOUD_COVER').first()) 748 | # how cloudy is it? 749 | print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 750 | # when was this image taken? 751 | date = ee.Date(least_cloudy.get('system:time_start')) 752 | time = date.getInfo()['value']/1000. 753 | dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 754 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 755 | region=aoi.getInfo()['coordinates'], 756 | description=folder + '_clim', 757 | folder=folder, 758 | fileNamePrefix='clim', 759 | maxPixels=1e9, 760 | scale=30, 761 | crs='EPSG:4326') 762 | task.start() 763 | return task.status() 764 | 765 | 766 | 767 | ############################################################################### 768 | 769 | 770 | ###################### Malawi flood 03-18 ####################### 771 | 772 | trmm = ee.ImageCollection("TRMM/3B42")# new_shapefile = "mwi_3-18.shp" 773 | 774 | trmm = trmm.select(['precipitation', 'HQprecipitation', 'IRprecipitation']) 775 | 776 | 777 | trmm_1 = trmm.filterDate('2018-02-27','2018-03-04') 778 | 779 | trmm_2 = trmm.filterDate('2018-02-20','2018-02-27') 780 | 781 | trmm_3 = trmm.filterDate('2018-02-13','2018-02-20') 782 | 783 | trmm_4 = trmm.filterDate('2018-02-05','2018-02-13') 784 | 785 | trmm_5 = trmm.filterDate('2018-01-30','2018-02-05') 786 | 787 | trmm_6 = trmm.filterDate('2018-01-23','2018-01-30') 788 | 789 | trmm_7 = trmm.filterDate('2018-01-14','2018-01-23') 790 | 791 | trmm_8 = trmm.filterDate('2018-01-07','2018-01-14') 792 | 793 | trmm_9 = trmm.filterDate('2017-12-31','2018-01-07') 794 | 795 | trmm_10 = trmm.filterDate('2017-12-24','2017-12-31') 796 | 797 | trmm_11 = trmm.filterDate('2017-12-17','2017-12-24') 798 | 799 | trmm_12 = trmm.filterDate('2017-12-10','2017-12-17') 800 | 801 | trmm_13 = trmm.filterDate('2017-12-03','2017-12-10') 802 | 803 | trmm_14 = trmm.filterDate('2017-11-26','2017-12-03') 804 | 805 | trmm_15 = trmm.filterDate('2017-11-19','2017-11-26') 806 | 807 | trmm_16 = trmm.filterDate('2017-11-12','2017-11-19') 808 | 809 | trmm_17 = trmm.filterDate('2017-11-05','2017-11-12') 810 | 811 | 812 | # setting the Area of Interest (AOI) 813 | mwi_aoi = ee.Geometry.Rectangle([34. , -17.01, 36. , -15.01]) 814 | 815 | 816 | trmm_list = [trmm_1, trmm_2, trmm_3, trmm_4, trmm_5, trmm_6, trmm_7, trmm_8, trmm_9, 817 | trmm_10, trmm_11, trmm_12, trmm_13, trmm_14, trmm_15, trmm_16, trmm_17] 818 | 819 | 820 | # Get precipitations 821 | mwi_precip_mean = begin_trmm_export(mwi_aoi, trmm_list, 'mean', 'mwi_floods_03-18') 822 | mwi_precip_total = begin_trmm_export(mwi_aoi, trmm_list, 'total', 'mwi_floods_03-18') 823 | mwi_precip_var = begin_trmm_export(mwi_aoi, trmm_list, 'var', 'mwi_floods_03-18') 824 | 825 | 826 | # Get landcover 827 | mwi_lc = begin_lc_export(mwi_aoi, '2017-01-01', '2018-01-01', 'mwi_floods_03-18') 828 | 829 | # Get smap 830 | mwi_smap = begin_smap_export(mwi_aoi, '2017-11-24', '2017-12-24', 'mwi_floods_03-18') 831 | 832 | # Get ndvi 833 | mwi_ndvi = begin_ndvi_export(mwi_aoi, '2017-11-24', '2017-12-24', 'mwi_floods_03-18') 834 | 835 | # Get evap 836 | mwi_evap = begin_evap_export(mwi_aoi, '2017-11-24', '2017-12-24', 'mwi_floods_03-18') 837 | 838 | # Get clim 839 | mwi_clip = begin_clim_export(mwi_aoi, '2017-11-24', '2017-12-24', 'mwi_floods_03-18') 840 | 841 | 842 | ################ Mozambique 03-19 ######################################### 843 | 844 | 845 | ## Landdcover 846 | 847 | modis = ee.ImageCollection('MODIS/006/MCD12Q1') 848 | # choose dates 849 | modis = modis.filterDate('2018-01-01','2019-01-01') 850 | 851 | # setting the Area of Interest (AOI) 852 | moz_aoi = ee.Geometry.Rectangle([ 33.69213642, -20.57556134, 35.30819006, -18.97632074]) 853 | 854 | modis_aoi = modis.filterBounds(moz_aoi) 855 | 856 | 857 | # the least cloudy image 858 | least_cloudy = ee.Image(modis.sort('CLOUD_COVER').first()) 859 | # how cloudy is it? 860 | print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 861 | 862 | 863 | # when was this image taken? 864 | date = ee.Date(least_cloudy.get('system:time_start')) 865 | time = date.getInfo()['value']/1000. 866 | dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 867 | 868 | parameters = {'min': 0, 869 | 'max': 1000, 870 | 'dimensions': 512, 871 | 'bands': 'LC_Type1', 872 | 'region': moz_aoi} 873 | 874 | 875 | task_config = { 876 | 'scale': 30, 877 | 'region': moz_aoi['coordinates'][0] 878 | } 879 | 880 | task = ee.batch.Export.image(least_cloudy, description='exportExample', **task_config) 881 | 882 | task.start() 883 | 884 | 885 | 886 | 887 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 888 | region=moz_aoi.getInfo()['coordinates'], 889 | description='my_description', 890 | folder='mozam_03/19', 891 | fileNamePrefix='moz_lc_type1', 892 | scale=30, 893 | crs='EPSG:4326') 894 | task.start() 895 | 896 | 897 | 898 | #### Precip 899 | 900 | trmm = ee.ImageCollection("TRMM/3B42") 901 | trmm = trmm.select(['precipitation', 'HQprecipitation', 'IRprecipitation']) 902 | 903 | 904 | trmm_1 = trmm.filterDate('2019-04-01','2019-04-09') 905 | 906 | trmm_2 = trmm.filterDate('2019-03-24','2019-04-01') 907 | 908 | trmm_3 = trmm.filterDate('2019-03-17','2019-03-24') 909 | 910 | trmm_4 = trmm.filterDate('2019-03-10','2019-03-17') 911 | 912 | trmm_5 = trmm.filterDate('2019-03-03','2019-03-10') 913 | 914 | trmm_6 = trmm.filterDate('2019-02-24','2019-03-03') 915 | 916 | trmm_7 = trmm.filterDate('2019-02-17','2019-02-24') 917 | 918 | trmm_8 = trmm.filterDate('2019-02-10','2019-02-17') 919 | 920 | trmm_9 = trmm.filterDate('2019-02-03','2019-02-10') 921 | 922 | trmm_10 = trmm.filterDate('2019-01-27','2019-02-03') 923 | 924 | trmm_11 = trmm.filterDate('2019-01-20','2019-01-27') 925 | 926 | trmm_12 = trmm.filterDate('2019-01-13','2019-01-20') 927 | 928 | trmm_13 = trmm.filterDate('2019-01-06','2019-01-13') 929 | 930 | trmm_14 = trmm.filterDate('2018-12-30','2019-01-06') 931 | 932 | trmm_15 = trmm.filterDate('2018-12-23','2018-12-30') 933 | 934 | trmm_16 = trmm.filterDate('2018-12-16','2018-12-23') 935 | 936 | trmm_17 = trmm.filterDate('2018-12-09','2018-12-16') 937 | 938 | 939 | 940 | # setting the Area of Interest (AOI) 941 | moz_aoi = ee.Geometry.Rectangle([ 33.69213642, -20.57556134, 35.30819006, -18.97632074]) 942 | 943 | trmm_list = [trmm_1, trmm_2, trmm_3, trmm_4, trmm_5, trmm_6, trmm_7, trmm_8, trmm_9, 944 | trmm_10, trmm_11, trmm_12, trmm_13, trmm_14, trmm_15, trmm_16, trmm_17] 945 | 946 | # Get precipitations 947 | moz_precip_mean = begin_trmm_export(moz_aoi, trmm_list, 'mean', 'moz_floods_03-19') 948 | moz_precip_total = begin_trmm_export(moz_aoi, trmm_list, 'total', 'moz_floods_03-19') 949 | moz_precip_var = begin_trmm_export(moz_aoi, trmm_list, 'var', 'moz_floods_03-19') 950 | 951 | 952 | # Get landcover 953 | moz_lc = begin_lc_export(moz_aoi, '2018-01-01', '2019-01-01', 'moz_floods_03-19') 954 | 955 | # Get smap 956 | moz_smap = begin_smap_export(moz_aoi, '2018-12-27', '2019-01-27', 'moz_floods_03-19') 957 | 958 | # Get ndvi 959 | moz_ndvi = begin_ndvi_export(moz_aoi, '2018-12-27', '2019-01-27', 'moz_floods_03-19') 960 | 961 | # Get evap 962 | moz_evap = begin_evap_export(moz_aoi, '2018-12-27', '2019-01-27', 'moz_floods_03-19') 963 | 964 | # Get clim 965 | moz_clip = begin_clim_export(moz_aoi, '2018-12-27', '2019-01-27', 'moz_floods_03-19') 966 | 967 | 968 | ################ Mozambique 01-19 ######################################### 969 | 970 | 971 | 972 | ### Landdcover 973 | 974 | modis = ee.ImageCollection('MODIS/006/MCD12Q1') 975 | # choose dates 976 | 977 | trmm = ee.ImageCollection("TRMM/3B42") 978 | trmm = trmm.select(['precipitation', 'HQprecipitation', 'IRprecipitation']) 979 | 980 | trmm_1 = trmm.filterDate('2019-02-07','2019-02-14') 981 | 982 | trmm_2 = trmm.filterDate('2019-01-31','2019-02-07') 983 | 984 | trmm_3 = trmm.filterDate('2019-01-24','2019-01-31') 985 | 986 | trmm_4 = trmm.filterDate('2019-01-17','2019-01-24') 987 | 988 | trmm_5 = trmm.filterDate('2019-01-10','2019-01-17') 989 | 990 | trmm_6 = trmm.filterDate('2019-01-03','2019-01-10') 991 | 992 | trmm_7 = trmm.filterDate('2018-12-27','2019-01-03') 993 | 994 | trmm_8 = trmm.filterDate('2018-12-20','2018-12-27') 995 | 996 | trmm_9 = trmm.filterDate('2018-12-13','2018-12-20') 997 | 998 | trmm_10 = trmm.filterDate('2018-12-06','2018-12-13') 999 | 1000 | trmm_11 = trmm.filterDate('2018-11-30','2018-12-06') 1001 | 1002 | trmm_12 = trmm.filterDate('2018-11-23','2018-11-30') 1003 | 1004 | trmm_13 = trmm.filterDate('2018-11-16','2018-11-23') 1005 | 1006 | trmm_14 = trmm.filterDate('2018-11-09','2018-11-16') 1007 | 1008 | trmm_15 = trmm.filterDate('2018-11-02','2018-11-09') 1009 | 1010 | trmm_16 = trmm.filterDate('2018-10-25','2018-11-02') 1011 | 1012 | trmm_17 = trmm.filterDate('2018-10-18','2018-10-25') 1013 | 1014 | 1015 | 1016 | # setting the Area of Interest (AOI) 1017 | moz_aoi = ee.Geometry.Rectangle([ 32.855, -19.485, 37.115, -15.625]) 1018 | 1019 | trmm_list = [trmm_1, trmm_2, trmm_3, trmm_4, trmm_5, trmm_6, trmm_7, trmm_8, trmm_9, 1020 | trmm_10, trmm_11, trmm_12, trmm_13, trmm_14, trmm_15, trmm_16, trmm_17] 1021 | 1022 | # Get precipitations 1023 | moz_precip_mean = begin_trmm_export(moz_aoi, trmm_list, 'mean', 'moz_floods_01-19') 1024 | moz_precip_total = begin_trmm_export(moz_aoi, trmm_list, 'total', 'moz_floods_01-19') 1025 | moz_precip_var = begin_trmm_export(moz_aoi, trmm_list, 'var', 'moz_floods_01-19') 1026 | 1027 | 1028 | # Get landcover 1029 | moz_lc = begin_lc_export(moz_aoi, '2018-01-01', '2019-01-01', 'moz_floods_01-19') 1030 | 1031 | # Get smap 1032 | moz_smap = begin_smap_export(moz_aoi, '2018-11-06', '2018-12-06', 'moz_floods_01-19') 1033 | 1034 | 1035 | 1036 | smap = ee.ImageCollection("NASA_USDA/HSL/SMAP_soil_moisture") 1037 | # choose dates 1038 | smap = smap.filterDate('2018-11-06', '2018-12-06') 1039 | # Choose band 1040 | smap = smap.select(['ssm', 'susm']) 1041 | 1042 | aoi_bound = smap.filterBounds(moz_aoi) 1043 | 1044 | # the least cloudy image 1045 | least_cloudy = ee.Image(aoi_bound.sort('CLOUD_COVER').first()) 1046 | # how cloudy is it? 1047 | print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 1048 | # when was this image taken? 1049 | date = ee.Date(least_cloudy.get('system:time_start')) 1050 | time = date.getInfo()['value']/1000. 1051 | dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 1052 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 1053 | region=moz_aoi.getInfo()['coordinates'], 1054 | description='moz_floods_01' + '_smap', 1055 | folder='moz_floods_01', 1056 | fileNamePrefix='smap', 1057 | maxPixels=1e5, 1058 | scale=30, 1059 | crs='EPSG:4326') 1060 | task.start() 1061 | 1062 | 1063 | 1064 | # Get NDVI 1065 | moz_ndvi = begin_ndvi_export(moz_aoi, '2018-11-06', '2018-12-06', 'moz_floods_01-19') 1066 | 1067 | 1068 | 1069 | 1070 | 1071 | 1072 | ndvi = ee.ImageCollection('MODIS/006/MOD13A2') 1073 | # choose dates 1074 | ndvi = ndvi.filterDate('2018-11-06', '2018-12-06') 1075 | ndvi = ndvi.select(['NDVI', 'EVI']) 1076 | 1077 | # setting the Area of Interest (AOI) 1078 | #moz_aoi = ee.Geometry.Rectangle(aoi) 1079 | aoi_bound = ndvi.filterBounds(moz_aoi) 1080 | 1081 | # the least cloudy image 1082 | least_cloudy = ee.Image(aoi_bound.sort('CLOUD_COVER').first()) 1083 | # how cloudy is it? 1084 | print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 1085 | # when was this image taken? 1086 | date = ee.Date(least_cloudy.get('system:time_start')) 1087 | time = date.getInfo()['value']/1000. 1088 | dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 1089 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 1090 | region=moz_aoi.getInfo()['coordinates'], 1091 | description='moz_floods_01-19' + '_ndvi', 1092 | folder='moz_floods_01-19', 1093 | fileNamePrefix='ndvi', 1094 | maxPixels=1e9, 1095 | scale=30, 1096 | crs='EPSG:4326') 1097 | task.start() 1098 | 1099 | 1100 | 1101 | 1102 | # Get evap 1103 | moz_evap = begin_evap_export(moz_aoi, '2018-11-06', '2018-12-06', 'moz_floods_01-19') 1104 | 1105 | 1106 | evap = ee.ImageCollection("MODIS/006/MOD16A2") 1107 | 1108 | # choose dates 1109 | evap = evap.filterDate('2018-11-06', '2018-12-06') 1110 | evap = evap.select('ET') 1111 | 1112 | # setting the Area of Interest (AOI) 1113 | aoi_bound = evap.filterBounds(moz_aoi) 1114 | 1115 | # the least cloudy image 1116 | least_cloudy = ee.Image(aoi_bound.sort('CLOUD_COVER').first()) 1117 | # how cloudy is it? 1118 | print('Cloud Cover (%):', least_cloudy.get('CLOUD_COVER').getInfo()) 1119 | # when was this image taken? 1120 | date = ee.Date(least_cloudy.get('system:time_start')) 1121 | time = date.getInfo()['value']/1000. 1122 | dt.utcfromtimestamp(time).strftime('%Y-%m-%d %H:%M:%S') 1123 | task = ee.batch.Export.image.toDrive(image=least_cloudy, 1124 | region=moz_aoi.getInfo()['coordinates'], 1125 | description='moz_floods_01-19' + '_evap', 1126 | folder='moz_floods_01-19', 1127 | fileNamePrefix='evap', 1128 | maxPixels=1e9, 1129 | scale=30, 1130 | crs='EPSG:4326') 1131 | task.start() 1132 | 1133 | 1134 | 1135 | # Get clim 1136 | moz_clip = begin_clim_export(moz_aoi, '2018-11-06', '2018-12-06', 'moz_floods_01-19') 1137 | 1138 | 1139 | ###################### Malawi 01-19 #################################### 1140 | 1141 | 1142 | 1143 | ### Landdcover 1144 | 1145 | modis = ee.ImageCollection('MODIS/006/MCD12Q1') 1146 | 1147 | # setting the Area of Interest (AOI) 1148 | mwi_aoi = ee.Geometry.Rectangle([ 34.325, -17.035, 35.805, -15.335]) 1149 | 1150 | 1151 | trmm = ee.ImageCollection("TRMM/3B42") 1152 | trmm = trmm.select(['precipitation', 'HQprecipitation', 'IRprecipitation']) 1153 | 1154 | trmm_1 = trmm.filterDate('2019-02-07','2019-02-14') 1155 | 1156 | trmm_2 = trmm.filterDate('2019-01-31','2019-02-07') 1157 | 1158 | trmm_3 = trmm.filterDate('2019-01-24','2019-01-31') 1159 | 1160 | trmm_4 = trmm.filterDate('2019-01-17','2019-01-24') 1161 | 1162 | trmm_5 = trmm.filterDate('2019-01-10','2019-01-17') 1163 | 1164 | trmm_6 = trmm.filterDate('2019-01-03','2019-01-10') 1165 | 1166 | trmm_7 = trmm.filterDate('2018-12-27','2019-01-03') 1167 | 1168 | trmm_8 = trmm.filterDate('2018-12-20','2018-12-27') 1169 | 1170 | trmm_9 = trmm.filterDate('2018-12-13','2018-12-20') 1171 | 1172 | trmm_10 = trmm.filterDate('2018-12-06','2018-12-13') 1173 | 1174 | trmm_11 = trmm.filterDate('2018-11-30','2018-12-06') 1175 | 1176 | trmm_12 = trmm.filterDate('2018-11-23','2018-11-30') 1177 | 1178 | trmm_13 = trmm.filterDate('2018-11-16','2018-11-23') 1179 | 1180 | trmm_14 = trmm.filterDate('2018-11-09','2018-11-16') 1181 | 1182 | trmm_15 = trmm.filterDate('2018-11-02','2018-11-09') 1183 | 1184 | trmm_16 = trmm.filterDate('2018-10-25','2018-11-02') 1185 | 1186 | trmm_17 = trmm.filterDate('2018-10-18','2018-10-25') 1187 | 1188 | 1189 | trmm_list = [trmm_1, trmm_2, trmm_3, trmm_4, trmm_5, trmm_6, trmm_7, trmm_8, trmm_9, 1190 | trmm_10, trmm_11, trmm_12, trmm_13, trmm_14, trmm_15, trmm_16, trmm_17] 1191 | 1192 | # Get precipitations 1193 | mwi_precip_mean = begin_trmm_export(mwi_aoi, trmm_list, 'mean', 'mwi_floods_01-19') 1194 | mwi_precip_total = begin_trmm_export(mwi_aoi, trmm_list, 'total', 'mwi_floods_01-19') 1195 | mwi_precip_var = begin_trmm_export(mwi_aoi, trmm_list, 'var', 'mwi_floods_01-19') 1196 | 1197 | 1198 | # Get landcover 1199 | mwi_lc = begin_lc_export(mwi_aoi, '2018-01-01', '2019-01-01', 'mwi_floods_01-19') 1200 | 1201 | # Get smap 1202 | mwi_smap = begin_smap_export(mwi_aoi, '2018-11-06', '2018-12-06', 'mwi_floods_01-19') 1203 | 1204 | mwi_ndvi = begin_ndvi_export(mwi_aoi, '2018-11-06', '2018-12-06', 'mwi_floods_01-19') 1205 | 1206 | 1207 | # Get evap 1208 | mwi_evap = begin_evap_export(mwi_aoi, '2018-11-06', '2018-12-06', 'mwi_floods_01-19') 1209 | 1210 | # Get clim 1211 | mwi_clim = begin_clim_export(mwi_aoi, '2018-11-06', '2018-12-06', 'mwi_floods_01-19') 1212 | 1213 | 1214 | ################# Malawi 03-19 (test) ####################################### 1215 | 1216 | 1217 | trmm = ee.ImageCollection("TRMM/3B42") 1218 | trmm = trmm.select(['precipitation', 'HQprecipitation', 'IRprecipitation']) 1219 | 1220 | 1221 | trmm_1 = trmm.filterDate('2019-04-01','2019-04-09') 1222 | 1223 | trmm_2 = trmm.filterDate('2019-03-24','2019-04-01') 1224 | 1225 | trmm_3 = trmm.filterDate('2019-03-17','2019-03-24') 1226 | 1227 | trmm_4 = trmm.filterDate('2019-03-10','2019-03-17') 1228 | 1229 | trmm_5 = trmm.filterDate('2019-03-03','2019-03-10') 1230 | 1231 | trmm_6 = trmm.filterDate('2019-02-24','2019-03-03') 1232 | 1233 | trmm_7 = trmm.filterDate('2019-02-17','2019-02-24') 1234 | 1235 | trmm_8 = trmm.filterDate('2019-02-10','2019-02-17') 1236 | 1237 | trmm_9 = trmm.filterDate('2019-02-03','2019-02-10') 1238 | 1239 | trmm_10 = trmm.filterDate('2019-01-27','2019-02-03') 1240 | 1241 | trmm_11 = trmm.filterDate('2019-01-20','2019-01-27') 1242 | 1243 | trmm_12 = trmm.filterDate('2019-01-13','2019-01-20') 1244 | 1245 | trmm_13 = trmm.filterDate('2019-01-06','2019-01-13') 1246 | 1247 | trmm_14 = trmm.filterDate('2018-12-30','2019-01-06') 1248 | 1249 | trmm_15 = trmm.filterDate('2018-12-23','2018-12-30') 1250 | 1251 | trmm_16 = trmm.filterDate('2018-12-16','2018-12-23') 1252 | 1253 | trmm_17 = trmm.filterDate('2018-12-09','2018-12-16') 1254 | 1255 | 1256 | 1257 | # setting the Area of Interest (AOI) 1258 | mwi_aoi = ee.Geometry.Rectangle([ 34.325, -17.035, 35.805, -15.335]) 1259 | 1260 | trmm_list = [trmm_1, trmm_2, trmm_3, trmm_4, trmm_5, trmm_6, trmm_7, trmm_8, trmm_9, 1261 | trmm_10, trmm_11, trmm_12, trmm_13, trmm_14, trmm_15, trmm_16, trmm_17] 1262 | 1263 | # Get precipitations 1264 | mwi_precip_mean = begin_trmm_export(mwi_aoi, trmm_list, 'mean', 'mwifloods_03-19') 1265 | mwi_precip_total = begin_trmm_export(mwi_aoi, trmm_list, 'total', 'mwi_floods_03-19') 1266 | mwi_precip_var = begin_trmm_export(mwi_aoi, trmm_list, 'var', 'mwi_floods_03-19') 1267 | 1268 | 1269 | # Get landcover 1270 | mwi_lc = begin_lc_export(mwi_aoi, '2018-01-01', '2019-01-01', 'mwi_floods_03-19') 1271 | 1272 | # Get smap 1273 | mwi_smap = begin_smap_export(mwi_aoi, '2018-12-27', '2019-01-27', 'mwi_floods_03-19') 1274 | 1275 | mwi_ndvi = begin_ndvi_export(mwi_aoi, '2018-12-27', '2019-01-27', 'mwi_floods_03-19') 1276 | 1277 | mwi_evap = begin_evap_export(mwi_aoi, '2018-12-27', '2019-01-27', 'mwi_floods_03-19') 1278 | 1279 | # Get clim 1280 | mwi_clim = begin_clim_export(mwi_aoi, '2018-12-27', '2019-01-27', 'mwi_floods_03-19') 1281 | 1282 | 1283 | 1284 | 1285 | 1286 | 1287 | --------------------------------------------------------------------------------