├── .gitignore ├── requirements.txt ├── collect_weather.py ├── preprocess.py ├── train_test_dnn.py ├── train_test.py ├── weather.py ├── README.md └── Collect Weather Data API.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints/ 2 | *-working.ipynb 3 | .vscode/ 4 | tf_wx_model/ 5 | *.csv 6 | *.pkl 7 | *.pxi 8 | __pycache__ -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==0.7.0 2 | asn1crypto==0.24.0 3 | astor==0.7.1 4 | astroid==2.1.0 5 | certifi==2018.11.29 6 | cffi==1.11.5 7 | chardet==3.0.4 8 | colorama==0.4.1 9 | cryptography==2.5 10 | gast==0.2.2 11 | grpcio==1.16.1 12 | h5py==2.9.0 13 | idna==2.8 14 | isort==4.3.4 15 | Keras-Applications==1.0.6 16 | Keras-Preprocessing==1.0.5 17 | lazy-object-proxy==1.3.1 18 | Markdown==3.0.1 19 | mccabe==0.6.1 20 | mkl-fft==1.0.10 21 | mkl-random==1.0.2 22 | numpy==1.15.4 23 | pandas==0.24.1 24 | patsy==0.5.1 25 | protobuf==3.6.1 26 | psutil==5.5.0 27 | pycparser==2.19 28 | pylint==2.2.2 29 | pyOpenSSL==19.0.0 30 | PyPrind==2.11.2 31 | pyreadline==2.1 32 | PySocks==1.6.8 33 | python-dateutil==2.7.5 34 | pytz==2018.9 35 | requests==2.21.0 36 | scikit-learn==0.20.2 37 | scipy==1.2.0 38 | six==1.12.0 39 | statsmodels==0.9.0 40 | tensorboard==1.12.2 41 | tensorflow==1.12.2 42 | termcolor==1.1.0 43 | typed-ast==1.1.0 44 | urllib3==1.24.2 45 | Werkzeug==0.15.3 46 | win-inet-pton==1.0.1 47 | wincertstore==0.2 48 | wrapt==1.11.1 49 | -------------------------------------------------------------------------------- /collect_weather.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import pickle 3 | from datetime import timedelta 4 | 5 | from weather import API_KEY, BASE_URL, extract_weather_data, get_target_date 6 | 7 | filename1 = 'records_pt1.pkl' 8 | filename2 = 'records_pt2.pkl' 9 | 10 | if os.path.isfile(filename2): 11 | print('1000 records already collected from Dark Sky API') 12 | 13 | elif os.path.isfile(filename1): 14 | with open(filename1, 'rb') as fp: 15 | records = pickle.load(fp) 16 | 17 | target_date = records[-1][0] + timedelta(days=1) 18 | 19 | records += extract_weather_data(BASE_URL, API_KEY, target_date, 500) 20 | 21 | records_length = len(records) 22 | print(f'{records_length} records collected from Dark Sky API') 23 | 24 | with open(filename2, 'wb') as f: 25 | pickle.dump(records, f) 26 | 27 | print(f'Weather records from day 2 saved to {filename2}.') 28 | 29 | else: 30 | target_date = get_target_date() 31 | 32 | records = extract_weather_data(BASE_URL, API_KEY, target_date, 500) 33 | 34 | records_length = len(records) 35 | print(f'{records_length} records collected from Dark Sky API') 36 | 37 | with open(filename1, 'wb') as f: 38 | pickle.dump(records, f) 39 | 40 | print(f'Weather records from day 1 saved to {filename1}.') 41 | -------------------------------------------------------------------------------- /preprocess.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | 3 | import pandas as pd 4 | 5 | from weather import derive_nth_day_feature, features 6 | 7 | with open('records_pt2.pkl', 'rb') as fp: 8 | records = pickle.load(fp) 9 | 10 | df = pd.DataFrame(records, columns=features).set_index('date') 11 | 12 | for feature in features: 13 | if feature != 'date': 14 | for N in range(1, 4): 15 | derive_nth_day_feature(df, feature, N) 16 | 17 | # make list of original features without temperatureMean, temperatureMin, and temperatureMax 18 | to_remove = [ 19 | feature 20 | for feature in features 21 | if feature not in ['temperatureMean', 'temperatureMin', 'temperatureMax'] 22 | ] 23 | 24 | # make a list of columns to keep 25 | to_keep = [col for col in df.columns if col not in to_remove] 26 | 27 | # select only the columns in to_keep and assign to df 28 | df = df[to_keep] 29 | 30 | df = df.apply(pd.to_numeric, errors='coerce') 31 | 32 | # Call describe on df and transpose it due to the large number of columns 33 | spread = df.describe().T 34 | 35 | # precalculate interquartile range for ease of use in next calculation 36 | IQR = spread['75%'] - spread['25%'] 37 | 38 | # create an outliers column which is either 3 IQRs below the first quartile or 39 | # 3 IQRs above the third quartile 40 | spread['outliers'] = (spread['min'] < (spread['25%'] - (3 * IQR))) | ( 41 | spread['max'] > (spread['75%'] + 3 * IQR) 42 | ) 43 | 44 | # iterate over the precip columns 45 | for precip_col in ['precipProbability_1', 'precipProbability_2', 'precipProbability_3']: 46 | # create a boolean array of values representing nans 47 | missing_vals = pd.isnull(df[precip_col]) 48 | df[precip_col][missing_vals] = 0 49 | 50 | df = df.dropna() 51 | 52 | with open('end-part1_df.pkl', 'wb') as f: 53 | pickle.dump(df, f) 54 | -------------------------------------------------------------------------------- /train_test_dnn.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | 3 | import numpy as np 4 | import tensorflow as tf 5 | from sklearn.metrics import ( 6 | explained_variance_score, 7 | mean_absolute_error, 8 | median_absolute_error, 9 | ) 10 | from sklearn.model_selection import train_test_split 11 | 12 | with open('end-part1_df.pkl', 'rb') as fp: 13 | df = pickle.load(fp) 14 | 15 | df.index = df.index.values.astype(float) 16 | 17 | # First drop the temperatureMax and temperatureMin from the dataframe 18 | df = df.drop(['temperatureMin', 'temperatureMax'], axis=1) 19 | 20 | # X will be a pandas dataframe of all columns except temperatureMean 21 | X = df[[col for col in df.columns if col != 'temperatureMean']] 22 | 23 | # y will be a pandas series of the temperatureMean 24 | y = df['temperatureMean'] 25 | 26 | # split data into training set and a temporary set 27 | X_train, X_tmp, y_train, y_tmp = train_test_split(X, y, test_size=0.2, random_state=23) 28 | 29 | # split the remaining 20% of data evenly 30 | X_test, X_val, y_test, y_val = train_test_split( 31 | X_tmp, y_tmp, test_size=0.5, random_state=23 32 | ) 33 | 34 | X_train.shape, X_test.shape, X_val.shape 35 | print( 36 | f'Training instances {X_train.shape[0]}, Training features {X_train.shape[1]}' 37 | ) 38 | print(f'Validation instances {X_val.shape[0]}, Validation features {X_val.shape[1]}') 39 | print(f'Testing instances {X_test.shape[0]}, Testing features {X_test.shape[1]}') 40 | 41 | feature_cols = [tf.feature_column.numeric_column(col) for col in X.columns] 42 | 43 | regressor = tf.estimator.DNNRegressor( 44 | feature_columns=feature_cols, 45 | hidden_units=[50, 50], 46 | model_dir='~/Projects/machine-learning-predict-weather/tf_models/tf_wx_model', 47 | ) 48 | 49 | 50 | def wx_input_fn(X, y=None, num_epochs=None, shuffle=True, batch_size=400): 51 | return tf.estimator.inputs.pandas_input_fn( 52 | x=X, y=y, num_epochs=num_epochs, shuffle=shuffle, batch_size=batch_size 53 | ) 54 | 55 | 56 | evaluations = [] 57 | STEPS = 400 58 | for i in range(100): 59 | regressor.train(input_fn=wx_input_fn(X_train, y=y_train), steps=STEPS) 60 | evaluations.append( 61 | regressor.evaluate( 62 | input_fn=wx_input_fn(X_val, y_val, num_epochs=1, shuffle=False) 63 | ) 64 | ) 65 | 66 | pred = regressor.predict(input_fn=wx_input_fn(X_test, num_epochs=1, shuffle=False)) 67 | predictions = np.array([p['predictions'][0] for p in pred]) 68 | 69 | print(f'The Explained Variance: {explained_variance_score(y_test, predictions):.2f}') 70 | print( 71 | f'The Mean Absolute Error: {mean_absolute_error(y_test, predictions):.2f} degrees Celcius' 72 | ) 73 | print( 74 | f'The Median Absolute Error: {median_absolute_error(y_test, predictions):.2f} degrees Celcius' 75 | ) 76 | -------------------------------------------------------------------------------- /train_test.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | 3 | import pandas as pd 4 | import statsmodels.api as sm 5 | from sklearn.linear_model import LinearRegression 6 | from sklearn.metrics import mean_absolute_error, median_absolute_error 7 | from sklearn.model_selection import train_test_split 8 | 9 | with open('end-part1_df.pkl', 'rb') as fp: 10 | df = pickle.load(fp) 11 | 12 | df_corr = df.corr()[['temperatureMean']].sort_values('temperatureMean') 13 | df_corr_fil = df_corr[abs(df_corr['temperatureMean']) > 0.55] 14 | 15 | unwanted = ['temperatureMin', 'temperatureMax', 'temperatureMean'] 16 | predictors = df_corr_fil.index.tolist() 17 | predictors = [i for i in predictors if i not in unwanted] 18 | 19 | df2 = df[['temperatureMean'] + predictors] 20 | 21 | X = df2[predictors] 22 | y = df2['temperatureMean'] 23 | alpha = 0.05 24 | 25 | 26 | def stepwise_selection( 27 | X, y, initial_list=predictors, threshold_out=alpha, verbose=True 28 | ): 29 | """ Perform a forward-backward feature selection 30 | based on p-value from statsmodels.api.OLS 31 | Arguments: 32 | X - pandas.DataFrame with candidate features 33 | y - list-like with the target 34 | initial_list - list of features to start with (column names of X) 35 | threshold_in - include a feature if its p-value < threshold_in 36 | threshold_out - exclude a feature if its p-value > threshold_out 37 | verbose - whether to print the sequence of inclusions and exclusions 38 | Returns: list of selected features 39 | See https://en.wikipedia.org/wiki/Stepwise_regression for the details 40 | """ 41 | included = list(initial_list) 42 | while True: 43 | changed = False 44 | model = sm.OLS(y, sm.add_constant(pd.DataFrame(X[included]))).fit() 45 | # use all coefs except intercept 46 | pvalues = model.pvalues.iloc[1:] 47 | worst_pval = pvalues.max() # null if pvalues is empty 48 | if worst_pval > threshold_out: 49 | changed = True 50 | worst_feature = pvalues.idxmax() 51 | included.remove(worst_feature) 52 | if verbose: 53 | print('Drop {:30} with p-value {:.6}'.format(worst_feature, worst_pval)) 54 | if not changed: 55 | break 56 | return included 57 | 58 | 59 | result = stepwise_selection(X, y) 60 | 61 | print('resulting features:') 62 | print(result) 63 | 64 | X = X[result] 65 | model = sm.OLS(y, X).fit() 66 | print(model.summary()) 67 | 68 | X_train, X_test, y_train, y_test = train_test_split( 69 | X, y, test_size=0.2, random_state=12 70 | ) 71 | 72 | regressor = LinearRegression() 73 | 74 | regressor.fit(X_train, y_train) 75 | 76 | prediction = regressor.predict(X_test) 77 | 78 | print(f'The Explained Variance: {regressor.score(X_test, y_test):.2f}') 79 | print( 80 | f'The Mean Absolute Error: {mean_absolute_error(y_test, prediction):.2f} degrees celcius' 81 | ) 82 | print( 83 | f'The Median Absolute Error: {median_absolute_error(y_test, prediction):.2f} degrees celcius' 84 | ) 85 | -------------------------------------------------------------------------------- /weather.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | from collections import namedtuple 4 | from datetime import datetime, timedelta 5 | 6 | import requests 7 | from pyprind import ProgBar 8 | 9 | loc = '30.578806,-97.853065' 10 | 11 | API_KEY = os.environ.get('MY_API_KEY') 12 | BASE_URL = 'https://api.darksky.net/forecast/{}/{},{}' 13 | 14 | features = [ 15 | 'date', 16 | 'temperatureMean', 17 | 'dewPoint', 18 | 'pressure', 19 | 'humidity', 20 | 'temperatureMax', 21 | 'temperatureMin', 22 | 'precipProbability', 23 | ] 24 | DailySummary = namedtuple('DailySummary', features) 25 | 26 | 27 | def extract_weather_data(url, api_key, target_date, days): 28 | """Call Wunderground API to extract weather data.""" 29 | records = [] 30 | bar = ProgBar(days) 31 | for _ in range(days): 32 | request = BASE_URL.format( 33 | API_KEY, loc, target_date.strftime('%Y-%m-%dT%H:%M:%S') 34 | ) 35 | response = requests.get(request) 36 | if response.status_code == 200: 37 | 38 | def get_mean_temp(): 39 | """Return average temperature across a 24 hour period.""" 40 | total_temp = 0 41 | for i in range(len(hdata)): 42 | try: 43 | total_temp += hdata[i]['temperature'] 44 | except KeyError: 45 | total_temp += hdata[i-1]['temperature'] 46 | meanTemp = total_temp / 24 47 | return meanTemp 48 | 49 | data = response.json()['daily']['data'][0] 50 | hdata = response.json()['hourly']['data'] 51 | try: 52 | records.append( 53 | DailySummary( 54 | date=target_date, 55 | temperatureMean=get_mean_temp(), 56 | dewPoint=data['dewPoint'], 57 | pressure=data['pressure'], 58 | humidity=data['humidity'], 59 | temperatureMax=data['temperatureMax'], 60 | temperatureMin=data['temperatureMin'], 61 | precipProbability=data['precipProbability'], 62 | ) 63 | ) 64 | except KeyError: 65 | records.append( 66 | DailySummary( 67 | date=target_date, 68 | temperatureMean=get_mean_temp(), 69 | dewPoint=data['dewPoint'], 70 | pressure=data['pressure'], 71 | humidity=data['humidity'], 72 | temperatureMax=data['temperatureMax'], 73 | temperatureMin=data['temperatureMin'], 74 | precipProbability=0, 75 | ) 76 | ) 77 | # time.sleep(6) 78 | bar.update() 79 | target_date += timedelta(days=1) 80 | return records 81 | 82 | 83 | def get_target_date(): 84 | """Return target date 1000 days prior to current date.""" 85 | current_date = datetime.now() 86 | target_date = current_date - timedelta(days=1000) 87 | return target_date 88 | 89 | 90 | def derive_nth_day_feature(df, feature, N): 91 | nth_prior_measurements = df[feature].shift(periods=N) 92 | col_name = f'{feature}_{N}' 93 | df[col_name] = nth_prior_measurements 94 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Using Machine Learning to Predict the Weather ([Powered by Dark Sky](https://darksky.net/poweredby/)) 2 | This project is based on a three-part article written by Adam McQuistan in [stackabuse.com](http://stackabuse.com/using-machine-learning-to-predict-the-weather-part-1/). 3 | 4 | ## Update regarding the weather API 5 | My original disclaimer was Weather Underground ([wunderground.com](https://www.wunderground.com/)) was no longer providing free API accounts. At some point (I don't know exactly when), they discontinued their API service altogether. I have since signed up for a [Dark Sky API](https://darksky.net/dev). They don't have a free tier but they do have a trial account which allows 1,000 API calls per day to evaluate the service. Every API request over the free daily limit costs $0.0001. 6 | 7 | ## Summary 8 | I won't go into too much detail about the project since you can go to the original article on stackabuse.com; however, here is a little background if you wish to save time. (Although checkout the series, it's worth the read.) 9 | 10 | The project is split into three separate Jupyter Notebooks: one to collect the weather data from the Wunderground.com developer's API (again I'm using Dark Sky's API), inspect it, and clean it; a second to further refine the features and fit the data to a Linear Regression model; and a third to train and evaluate a deep neural net regressor. 11 | 12 | ## Changes 13 | For the most part I did not deviate from the author's original process. I did seek to automate and streamline the code. For example, I added a progress bar to the data collection function and created another function to automatically set a target date that is 1000 days prior to the current date. I automated the code to remove features that did not show a strong correlation and implemented a stepwise regression function to automate removing features that had p-values that were too high. (The original author did this manually.) 14 | 15 | ## Added modules 16 | Automating the code allowed me to adapt the Python code in the Jupyter Notebooks to regular .py files. Jupyter Notebooks are fantastic tools but I believe the final product should be Python scripts that run in the background. Here are the scripts I added and a quick summary: 17 | 18 | 1. weather.py- a utility file that contains reused methods and variables 19 | 2. collect_weather.py- uses the Requests library to download weather data for 1000 days. Also uses ```os.path.isfile()``` and a ```if/elif/else``` statement to determine whether the data from the first 500 days should be collected, data from the second 500 days should be collected, or no data is to be collected. (This no longer necessary since the daily limit is 1,000 calls.) 20 | 3. preprocess.py- creates a Pandas DataFrame from the weather records and cleans the data 21 | 4. train_test.py- performs some additional preprocessing and fits the data to a Linear Regression model 22 | 5. train_test_dnn- uses the same weather data to train, evaluate, and test a deep neural network regressor 23 | 24 | ## Still To Do 25 | * Update collect_weather.py to make 1,000 API calls at once instead of 500 over two days 26 | * Update the Jupyter Notebooks for the Dark Sky API 27 | * Replace/remove some deprecated methods in the train_test.py and train_test_dnn.py modules 28 | * Add better documentation in the form of markdown cells to the notebooks. 29 | * Apply the model to future forecasts and validate against actual weather data. 30 | -------------------------------------------------------------------------------- /Collect Weather Data API.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import os\n", 10 | "import pickle\n", 11 | "import time\n", 12 | "from collections import namedtuple\n", 13 | "from datetime import datetime, timedelta\n", 14 | "\n", 15 | "import pandas as pd\n", 16 | "import requests\n", 17 | "\n", 18 | "import matplotlib.pyplot as plt\n", 19 | "from pyprind import ProgBar\n", 20 | "\n", 21 | "%matplotlib inline" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 2, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "API_KEY = os.environ.get('MY_API_KEY')\n", 31 | "BASE_URL = 'http://api.wunderground.com/api/{}/history_{}/q/TX/Round_Rock.json'" 32 | ] 33 | }, 34 | { 35 | "cell_type": "code", 36 | "execution_count": 3, 37 | "metadata": {}, 38 | "outputs": [], 39 | "source": [ 40 | "features = [\n", 41 | " \"date\", \"meantempm\", \"meandewptm\", \"meanpressurem\", \"maxhumidity\",\n", 42 | " \"minhumidity\", \"maxtempm\", \"mintempm\", \"maxdewptm\", \"mindewptm\",\n", 43 | " \"maxpressurem\", \"minpressurem\", \"precipm\"\n", 44 | "]\n", 45 | "DailySummary = namedtuple('DailySummary', features)" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 4, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "def extract_weather_data(url, api_key, target_date, days):\n", 55 | " \"\"\"Call Wunderground API to extract weather data.\"\"\"\n", 56 | " records = []\n", 57 | " bar = ProgBar(days)\n", 58 | " for _ in range(days):\n", 59 | " request = BASE_URL.format(API_KEY, target_date.strftime('%Y%m%d'))\n", 60 | " response = requests.get(request)\n", 61 | " if response.status_code == 200:\n", 62 | " data = response.json()['history']['dailysummary'][0]\n", 63 | " records.append(DailySummary(\n", 64 | " date=target_date,\n", 65 | " meantempm=data['meantempm'],\n", 66 | " meandewptm=data['meandewptm'],\n", 67 | " meanpressurem=data['meanpressurem'],\n", 68 | " maxhumidity=data['maxhumidity'],\n", 69 | " minhumidity=data['minhumidity'],\n", 70 | " maxtempm=data['maxtempm'],\n", 71 | " mintempm=data['mintempm'],\n", 72 | " maxdewptm=data['maxdewptm'],\n", 73 | " mindewptm=data['mindewptm'],\n", 74 | " maxpressurem=data['maxpressurem'],\n", 75 | " minpressurem=data['minpressurem'],\n", 76 | " precipm=data['precipm']))\n", 77 | " time.sleep(6)\n", 78 | " bar.update()\n", 79 | " target_date += timedelta(days=1)\n", 80 | " return records" 81 | ] 82 | }, 83 | { 84 | "cell_type": "code", 85 | "execution_count": 5, 86 | "metadata": {}, 87 | "outputs": [], 88 | "source": [ 89 | "# Do not run this cell when collecting data on day 2\n", 90 | "def get_target_date():\n", 91 | " \"\"\"Return target date 1000 days prior to current date.\"\"\"\n", 92 | " current_date = datetime.now()\n", 93 | " target_date = current_date - timedelta(days=1000)\n", 94 | " return target_date\n", 95 | "\n", 96 | "target_date = get_target_date()" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 6, 102 | "metadata": {}, 103 | "outputs": [ 104 | { 105 | "name": "stderr", 106 | "output_type": "stream", 107 | "text": [ 108 | "0% [##############################] 100% | ETA: 00:00:00\n", 109 | "Total time elapsed: 00:53:56\n" 110 | ] 111 | } 112 | ], 113 | "source": [ 114 | "records = extract_weather_data(BASE_URL, API_KEY, target_date, 500)" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 7, 120 | "metadata": {}, 121 | "outputs": [ 122 | { 123 | "data": { 124 | "text/plain": [ 125 | "[DailySummary(date=datetime.datetime(2015, 10, 3, 22, 13, 6, 559948), meantempm='21', meandewptm='6', meanpressurem='1012', maxhumidity='63', minhumidity='20', maxtempm='29', mintempm='14', maxdewptm='8', mindewptm='4', maxpressurem='1014', minpressurem='1010', precipm='0.00'),\n", 126 | " DailySummary(date=datetime.datetime(2015, 10, 4, 22, 13, 6, 559948), meantempm='22', meandewptm='8', meanpressurem='1015', maxhumidity='63', minhumidity='25', maxtempm='29', mintempm='15', maxdewptm='10', mindewptm='7', maxpressurem='1017', minpressurem='1013', precipm='0.00'),\n", 127 | " DailySummary(date=datetime.datetime(2015, 10, 5, 22, 13, 6, 559948), meantempm='24', meandewptm='11', meanpressurem='1018', maxhumidity='64', minhumidity='35', maxtempm='29', mintempm='19', maxdewptm='13', mindewptm='8', maxpressurem='1020', minpressurem='1015', precipm='0.00'),\n", 128 | " DailySummary(date=datetime.datetime(2015, 10, 6, 22, 13, 6, 559948), meantempm='23', meandewptm='11', meanpressurem='1019', maxhumidity='73', minhumidity='25', maxtempm='30', mintempm='17', maxdewptm='14', mindewptm='8', maxpressurem='1022', minpressurem='1017', precipm='0.00'),\n", 129 | " DailySummary(date=datetime.datetime(2015, 10, 7, 22, 13, 6, 559948), meantempm='24', meandewptm='13', meanpressurem='1017', maxhumidity='72', minhumidity='31', maxtempm='32', mintempm='17', maxdewptm='16', mindewptm='10', maxpressurem='1020', minpressurem='1015', precipm='0.00')]" 130 | ] 131 | }, 132 | "execution_count": 7, 133 | "metadata": {}, 134 | "output_type": "execute_result" 135 | } 136 | ], 137 | "source": [ 138 | "# Look at first five records\n", 139 | "records[:5]" 140 | ] 141 | }, 142 | { 143 | "cell_type": "code", 144 | "execution_count": 8, 145 | "metadata": { 146 | "scrolled": true 147 | }, 148 | "outputs": [ 149 | { 150 | "data": { 151 | "text/plain": [ 152 | "500" 153 | ] 154 | }, 155 | "execution_count": 8, 156 | "metadata": {}, 157 | "output_type": "execute_result" 158 | } 159 | ], 160 | "source": [ 161 | "len(records)" 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": 9, 167 | "metadata": {}, 168 | "outputs": [], 169 | "source": [ 170 | "# save records list\n", 171 | "with open('records_pt1.pkl', 'wb') as f:\n", 172 | " pickle.dump(records, f)" 173 | ] 174 | }, 175 | { 176 | "cell_type": "code", 177 | "execution_count": 5, 178 | "metadata": {}, 179 | "outputs": [], 180 | "source": [ 181 | "# load records list - still need to run cells 1-4\n", 182 | "with open('records_pt1.pkl', 'rb') as fp:\n", 183 | " records = pickle.load(fp)" 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": 6, 189 | "metadata": {}, 190 | "outputs": [ 191 | { 192 | "data": { 193 | "text/plain": [ 194 | "DailySummary(date=datetime.datetime(2017, 2, 13, 22, 13, 6, 559948), meantempm='20', meandewptm='13', meanpressurem='1018', maxhumidity='94', minhumidity='42', maxtempm='25', mintempm='16', maxdewptm='18', mindewptm='5', maxpressurem='1022', minpressurem='1012', precipm='0.00')" 195 | ] 196 | }, 197 | "execution_count": 6, 198 | "metadata": {}, 199 | "output_type": "execute_result" 200 | } 201 | ], 202 | "source": [ 203 | "# Inspect last record to date; next target date should be plus one day\n", 204 | "records[-1]" 205 | ] 206 | }, 207 | { 208 | "cell_type": "code", 209 | "execution_count": 7, 210 | "metadata": {}, 211 | "outputs": [], 212 | "source": [ 213 | "# set new target date based on date above plus one day\n", 214 | "target_date = datetime(2017, 2, 14)" 215 | ] 216 | }, 217 | { 218 | "cell_type": "code", 219 | "execution_count": 8, 220 | "metadata": {}, 221 | "outputs": [ 222 | { 223 | "name": "stderr", 224 | "output_type": "stream", 225 | "text": [ 226 | "0% [##############################] 100% | ETA: 00:00:00\n", 227 | "Total time elapsed: 00:53:38\n" 228 | ] 229 | } 230 | ], 231 | "source": [ 232 | "records += extract_weather_data(BASE_URL, API_KEY, target_date, 500)" 233 | ] 234 | }, 235 | { 236 | "cell_type": "code", 237 | "execution_count": 9, 238 | "metadata": { 239 | "scrolled": false 240 | }, 241 | "outputs": [ 242 | { 243 | "data": { 244 | "text/plain": [ 245 | "1000" 246 | ] 247 | }, 248 | "execution_count": 9, 249 | "metadata": {}, 250 | "output_type": "execute_result" 251 | } 252 | ], 253 | "source": [ 254 | "len(records)" 255 | ] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": 10, 260 | "metadata": {}, 261 | "outputs": [], 262 | "source": [ 263 | "# with open('records_pt2.pkl', 'wb') as f:\n", 264 | "# pickle.dump(records, f)" 265 | ] 266 | }, 267 | { 268 | "cell_type": "code", 269 | "execution_count": 11, 270 | "metadata": {}, 271 | "outputs": [], 272 | "source": [ 273 | "# load records list - still need to run cells 1 and 3\n", 274 | "# with open('records_pt2.pkl', 'rb') as fp:\n", 275 | "# records = pickle.load(fp)" 276 | ] 277 | }, 278 | { 279 | "cell_type": "code", 280 | "execution_count": 12, 281 | "metadata": {}, 282 | "outputs": [], 283 | "source": [ 284 | "df = pd.DataFrame(records, columns=features).set_index('date')" 285 | ] 286 | }, 287 | { 288 | "cell_type": "code", 289 | "execution_count": 13, 290 | "metadata": {}, 291 | "outputs": [ 292 | { 293 | "data": { 294 | "text/html": [ 295 | "
\n", 296 | "\n", 309 | "\n", 310 | " \n", 311 | " \n", 312 | " \n", 313 | " \n", 314 | " \n", 315 | " \n", 316 | " \n", 317 | " \n", 318 | " \n", 319 | " \n", 320 | " \n", 321 | " \n", 322 | " \n", 323 | " \n", 324 | " \n", 325 | " \n", 326 | " \n", 327 | " \n", 328 | " \n", 329 | " \n", 330 | " \n", 331 | " \n", 332 | " \n", 333 | " \n", 334 | " \n", 335 | " \n", 336 | " \n", 337 | " \n", 338 | " \n", 339 | " \n", 340 | " \n", 341 | " \n", 342 | " \n", 343 | " \n", 344 | " \n", 345 | " \n", 346 | " \n", 347 | " \n", 348 | " \n", 349 | " \n", 350 | " \n", 351 | " \n", 352 | " \n", 353 | " \n", 354 | " \n", 355 | " \n", 356 | " \n", 357 | " \n", 358 | " \n", 359 | " \n", 360 | " \n", 361 | " \n", 362 | " \n", 363 | " \n", 364 | " \n", 365 | " \n", 366 | " \n", 367 | " \n", 368 | " \n", 369 | " \n", 370 | " \n", 371 | " \n", 372 | " \n", 373 | " \n", 374 | "
meantempmmeandewptm
date
2015-10-03 22:13:06.559948216
2015-10-04 22:13:06.559948228
2015-10-05 22:13:06.5599482411
2015-10-06 22:13:06.5599482311
2015-10-07 22:13:06.5599482413
2015-10-08 22:13:06.5599482617
2015-10-09 22:13:06.5599482617
2015-10-10 22:13:06.5599482414
2015-10-11 22:13:06.5599482616
2015-10-12 22:13:06.5599482819
\n", 375 | "
" 376 | ], 377 | "text/plain": [ 378 | " meantempm meandewptm\n", 379 | "date \n", 380 | "2015-10-03 22:13:06.559948 21 6\n", 381 | "2015-10-04 22:13:06.559948 22 8\n", 382 | "2015-10-05 22:13:06.559948 24 11\n", 383 | "2015-10-06 22:13:06.559948 23 11\n", 384 | "2015-10-07 22:13:06.559948 24 13\n", 385 | "2015-10-08 22:13:06.559948 26 17\n", 386 | "2015-10-09 22:13:06.559948 26 17\n", 387 | "2015-10-10 22:13:06.559948 24 14\n", 388 | "2015-10-11 22:13:06.559948 26 16\n", 389 | "2015-10-12 22:13:06.559948 28 19" 390 | ] 391 | }, 392 | "execution_count": 13, 393 | "metadata": {}, 394 | "output_type": "execute_result" 395 | } 396 | ], 397 | "source": [ 398 | "tmp = df[['meantempm', 'meandewptm']].head(10)\n", 399 | "tmp" 400 | ] 401 | }, 402 | { 403 | "cell_type": "code", 404 | "execution_count": 14, 405 | "metadata": {}, 406 | "outputs": [ 407 | { 408 | "data": { 409 | "text/html": [ 410 | "
\n", 411 | "\n", 424 | "\n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | " \n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | " \n", 464 | " \n", 465 | " \n", 466 | " \n", 467 | " \n", 468 | " \n", 469 | " \n", 470 | " \n", 471 | " \n", 472 | " \n", 473 | " \n", 474 | " \n", 475 | " \n", 476 | " \n", 477 | " \n", 478 | " \n", 479 | " \n", 480 | " \n", 481 | " \n", 482 | " \n", 483 | " \n", 484 | " \n", 485 | " \n", 486 | " \n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | " \n", 498 | " \n", 499 | " \n", 500 | " \n", 501 | "
meantempmmeandewptmmeantempm_1
date
2015-10-03 22:13:06.559948216NaN
2015-10-04 22:13:06.55994822821
2015-10-05 22:13:06.559948241122
2015-10-06 22:13:06.559948231124
2015-10-07 22:13:06.559948241323
2015-10-08 22:13:06.559948261724
2015-10-09 22:13:06.559948261726
2015-10-10 22:13:06.559948241426
2015-10-11 22:13:06.559948261624
2015-10-12 22:13:06.559948281926
\n", 502 | "
" 503 | ], 504 | "text/plain": [ 505 | " meantempm meandewptm meantempm_1\n", 506 | "date \n", 507 | "2015-10-03 22:13:06.559948 21 6 NaN\n", 508 | "2015-10-04 22:13:06.559948 22 8 21\n", 509 | "2015-10-05 22:13:06.559948 24 11 22\n", 510 | "2015-10-06 22:13:06.559948 23 11 24\n", 511 | "2015-10-07 22:13:06.559948 24 13 23\n", 512 | "2015-10-08 22:13:06.559948 26 17 24\n", 513 | "2015-10-09 22:13:06.559948 26 17 26\n", 514 | "2015-10-10 22:13:06.559948 24 14 26\n", 515 | "2015-10-11 22:13:06.559948 26 16 24\n", 516 | "2015-10-12 22:13:06.559948 28 19 26" 517 | ] 518 | }, 519 | "execution_count": 14, 520 | "metadata": {}, 521 | "output_type": "execute_result" 522 | } 523 | ], 524 | "source": [ 525 | "# 1 day prior\n", 526 | "N = 1\n", 527 | "\n", 528 | "# target measurement of mean temperature\n", 529 | "feature = 'meantempm'\n", 530 | "\n", 531 | "# total number of rows\n", 532 | "rows = tmp.shape[0]\n", 533 | "\n", 534 | "# a list representing Nth prior measurements of feature\n", 535 | "nth_prior_measurements = tmp[feature].shift(periods=N)\n", 536 | "\n", 537 | "# makee a new column name of feature_N and add to DataFrame\n", 538 | "col_name = f'{feature}_{N}'\n", 539 | "tmp[col_name] = nth_prior_measurements\n", 540 | "tmp" 541 | ] 542 | }, 543 | { 544 | "cell_type": "code", 545 | "execution_count": 15, 546 | "metadata": {}, 547 | "outputs": [], 548 | "source": [ 549 | "def derive_nth_day_feature(df, feature, N):\n", 550 | " nth_prior_measurements = df[feature].shift(periods=N)\n", 551 | " col_name = f'{feature}_{N}'\n", 552 | " df[col_name] = nth_prior_measurements" 553 | ] 554 | }, 555 | { 556 | "cell_type": "code", 557 | "execution_count": 16, 558 | "metadata": {}, 559 | "outputs": [], 560 | "source": [ 561 | "for feature in features:\n", 562 | " if feature != 'date':\n", 563 | " for N in range(1, 4):\n", 564 | " derive_nth_day_feature(df, feature, N)" 565 | ] 566 | }, 567 | { 568 | "cell_type": "code", 569 | "execution_count": 17, 570 | "metadata": {}, 571 | "outputs": [ 572 | { 573 | "data": { 574 | "text/plain": [ 575 | "Index(['meantempm', 'meandewptm', 'meanpressurem', 'maxhumidity',\n", 576 | " 'minhumidity', 'maxtempm', 'mintempm', 'maxdewptm', 'mindewptm',\n", 577 | " 'maxpressurem', 'minpressurem', 'precipm', 'meantempm_1', 'meantempm_2',\n", 578 | " 'meantempm_3', 'meandewptm_1', 'meandewptm_2', 'meandewptm_3',\n", 579 | " 'meanpressurem_1', 'meanpressurem_2', 'meanpressurem_3',\n", 580 | " 'maxhumidity_1', 'maxhumidity_2', 'maxhumidity_3', 'minhumidity_1',\n", 581 | " 'minhumidity_2', 'minhumidity_3', 'maxtempm_1', 'maxtempm_2',\n", 582 | " 'maxtempm_3', 'mintempm_1', 'mintempm_2', 'mintempm_3', 'maxdewptm_1',\n", 583 | " 'maxdewptm_2', 'maxdewptm_3', 'mindewptm_1', 'mindewptm_2',\n", 584 | " 'mindewptm_3', 'maxpressurem_1', 'maxpressurem_2', 'maxpressurem_3',\n", 585 | " 'minpressurem_1', 'minpressurem_2', 'minpressurem_3', 'precipm_1',\n", 586 | " 'precipm_2', 'precipm_3'],\n", 587 | " dtype='object')" 588 | ] 589 | }, 590 | "execution_count": 17, 591 | "metadata": {}, 592 | "output_type": "execute_result" 593 | } 594 | ], 595 | "source": [ 596 | "df.columns" 597 | ] 598 | }, 599 | { 600 | "cell_type": "code", 601 | "execution_count": 18, 602 | "metadata": {}, 603 | "outputs": [ 604 | { 605 | "data": { 606 | "text/plain": [ 607 | "Index(['meantempm', 'maxtempm', 'mintempm', 'meantempm_1', 'meantempm_2',\n", 608 | " 'meantempm_3', 'meandewptm_1', 'meandewptm_2', 'meandewptm_3',\n", 609 | " 'meanpressurem_1', 'meanpressurem_2', 'meanpressurem_3',\n", 610 | " 'maxhumidity_1', 'maxhumidity_2', 'maxhumidity_3', 'minhumidity_1',\n", 611 | " 'minhumidity_2', 'minhumidity_3', 'maxtempm_1', 'maxtempm_2',\n", 612 | " 'maxtempm_3', 'mintempm_1', 'mintempm_2', 'mintempm_3', 'maxdewptm_1',\n", 613 | " 'maxdewptm_2', 'maxdewptm_3', 'mindewptm_1', 'mindewptm_2',\n", 614 | " 'mindewptm_3', 'maxpressurem_1', 'maxpressurem_2', 'maxpressurem_3',\n", 615 | " 'minpressurem_1', 'minpressurem_2', 'minpressurem_3', 'precipm_1',\n", 616 | " 'precipm_2', 'precipm_3'],\n", 617 | " dtype='object')" 618 | ] 619 | }, 620 | "execution_count": 18, 621 | "metadata": {}, 622 | "output_type": "execute_result" 623 | } 624 | ], 625 | "source": [ 626 | "# make list of original features without meantempm, mintempm, and maxtempm\n", 627 | "to_remove = [feature\n", 628 | " for feature in features\n", 629 | " if feature not in ['meantempm', 'mintempm', 'maxtempm']]\n", 630 | "\n", 631 | "# make a list of columns to keep\n", 632 | "to_keep = [col for col in df.columns if col not in to_remove]\n", 633 | "\n", 634 | "# select only the columns in to_keep and assign to df\n", 635 | "df = df[to_keep]\n", 636 | "df.columns" 637 | ] 638 | }, 639 | { 640 | "cell_type": "code", 641 | "execution_count": 19, 642 | "metadata": {}, 643 | "outputs": [ 644 | { 645 | "name": "stdout", 646 | "output_type": "stream", 647 | "text": [ 648 | "\n", 649 | "DatetimeIndex: 1000 entries, 2015-10-03 22:13:06.559948 to 2018-06-28 00:00:00\n", 650 | "Data columns (total 39 columns):\n", 651 | "meantempm 1000 non-null object\n", 652 | "maxtempm 1000 non-null object\n", 653 | "mintempm 1000 non-null object\n", 654 | "meantempm_1 999 non-null object\n", 655 | "meantempm_2 998 non-null object\n", 656 | "meantempm_3 997 non-null object\n", 657 | "meandewptm_1 999 non-null object\n", 658 | "meandewptm_2 998 non-null object\n", 659 | "meandewptm_3 997 non-null object\n", 660 | "meanpressurem_1 999 non-null object\n", 661 | "meanpressurem_2 998 non-null object\n", 662 | "meanpressurem_3 997 non-null object\n", 663 | "maxhumidity_1 999 non-null object\n", 664 | "maxhumidity_2 998 non-null object\n", 665 | "maxhumidity_3 997 non-null object\n", 666 | "minhumidity_1 999 non-null object\n", 667 | "minhumidity_2 998 non-null object\n", 668 | "minhumidity_3 997 non-null object\n", 669 | "maxtempm_1 999 non-null object\n", 670 | "maxtempm_2 998 non-null object\n", 671 | "maxtempm_3 997 non-null object\n", 672 | "mintempm_1 999 non-null object\n", 673 | "mintempm_2 998 non-null object\n", 674 | "mintempm_3 997 non-null object\n", 675 | "maxdewptm_1 999 non-null object\n", 676 | "maxdewptm_2 998 non-null object\n", 677 | "maxdewptm_3 997 non-null object\n", 678 | "mindewptm_1 999 non-null object\n", 679 | "mindewptm_2 998 non-null object\n", 680 | "mindewptm_3 997 non-null object\n", 681 | "maxpressurem_1 999 non-null object\n", 682 | "maxpressurem_2 998 non-null object\n", 683 | "maxpressurem_3 997 non-null object\n", 684 | "minpressurem_1 999 non-null object\n", 685 | "minpressurem_2 998 non-null object\n", 686 | "minpressurem_3 997 non-null object\n", 687 | "precipm_1 999 non-null object\n", 688 | "precipm_2 998 non-null object\n", 689 | "precipm_3 997 non-null object\n", 690 | "dtypes: object(39)\n", 691 | "memory usage: 312.5+ KB\n" 692 | ] 693 | } 694 | ], 695 | "source": [ 696 | "df.info()" 697 | ] 698 | }, 699 | { 700 | "cell_type": "code", 701 | "execution_count": 20, 702 | "metadata": {}, 703 | "outputs": [ 704 | { 705 | "name": "stdout", 706 | "output_type": "stream", 707 | "text": [ 708 | "\n", 709 | "DatetimeIndex: 1000 entries, 2015-10-03 22:13:06.559948 to 2018-06-28 00:00:00\n", 710 | "Data columns (total 39 columns):\n", 711 | "meantempm 997 non-null float64\n", 712 | "maxtempm 997 non-null float64\n", 713 | "mintempm 997 non-null float64\n", 714 | "meantempm_1 996 non-null float64\n", 715 | "meantempm_2 995 non-null float64\n", 716 | "meantempm_3 994 non-null float64\n", 717 | "meandewptm_1 996 non-null float64\n", 718 | "meandewptm_2 995 non-null float64\n", 719 | "meandewptm_3 994 non-null float64\n", 720 | "meanpressurem_1 994 non-null float64\n", 721 | "meanpressurem_2 993 non-null float64\n", 722 | "meanpressurem_3 992 non-null float64\n", 723 | "maxhumidity_1 996 non-null float64\n", 724 | "maxhumidity_2 995 non-null float64\n", 725 | "maxhumidity_3 994 non-null float64\n", 726 | "minhumidity_1 996 non-null float64\n", 727 | "minhumidity_2 995 non-null float64\n", 728 | "minhumidity_3 994 non-null float64\n", 729 | "maxtempm_1 996 non-null float64\n", 730 | "maxtempm_2 995 non-null float64\n", 731 | "maxtempm_3 994 non-null float64\n", 732 | "mintempm_1 996 non-null float64\n", 733 | "mintempm_2 995 non-null float64\n", 734 | "mintempm_3 994 non-null float64\n", 735 | "maxdewptm_1 996 non-null float64\n", 736 | "maxdewptm_2 995 non-null float64\n", 737 | "maxdewptm_3 994 non-null float64\n", 738 | "mindewptm_1 996 non-null float64\n", 739 | "mindewptm_2 995 non-null float64\n", 740 | "mindewptm_3 994 non-null float64\n", 741 | "maxpressurem_1 994 non-null float64\n", 742 | "maxpressurem_2 993 non-null float64\n", 743 | "maxpressurem_3 992 non-null float64\n", 744 | "minpressurem_1 994 non-null float64\n", 745 | "minpressurem_2 993 non-null float64\n", 746 | "minpressurem_3 992 non-null float64\n", 747 | "precipm_1 999 non-null float64\n", 748 | "precipm_2 998 non-null float64\n", 749 | "precipm_3 997 non-null float64\n", 750 | "dtypes: float64(39)\n", 751 | "memory usage: 312.5 KB\n" 752 | ] 753 | } 754 | ], 755 | "source": [ 756 | "df = df.apply(pd.to_numeric, errors='coerce')\n", 757 | "df.info()" 758 | ] 759 | }, 760 | { 761 | "cell_type": "code", 762 | "execution_count": 21, 763 | "metadata": {}, 764 | "outputs": [ 765 | { 766 | "data": { 767 | "text/html": [ 768 | "
\n", 769 | "\n", 782 | "\n", 783 | " \n", 784 | " \n", 785 | " \n", 786 | " \n", 787 | " \n", 788 | " \n", 789 | " \n", 790 | " \n", 791 | " \n", 792 | " \n", 793 | " \n", 794 | " \n", 795 | " \n", 796 | " \n", 797 | " \n", 798 | " \n", 799 | " \n", 800 | " \n", 801 | " \n", 802 | " \n", 803 | " \n", 804 | " \n", 805 | " \n", 806 | " \n", 807 | " \n", 808 | " \n", 809 | " \n", 810 | " \n", 811 | " \n", 812 | " \n", 813 | " \n", 814 | " \n", 815 | " \n", 816 | " \n", 817 | " \n", 818 | " \n", 819 | " \n", 820 | " \n", 821 | " \n", 822 | " \n", 823 | " \n", 824 | " \n", 825 | " \n", 826 | " \n", 827 | " \n", 828 | " \n", 829 | " \n", 830 | " \n", 831 | " \n", 832 | " \n", 833 | " \n", 834 | " \n", 835 | " \n", 836 | " \n", 837 | " \n", 838 | " \n", 839 | " \n", 840 | " \n", 841 | " \n", 842 | " \n", 843 | " \n", 844 | " \n", 845 | " \n", 846 | " \n", 847 | " \n", 848 | " \n", 849 | " \n", 850 | " \n", 851 | " \n", 852 | " \n", 853 | " \n", 854 | " \n", 855 | " \n", 856 | " \n", 857 | " \n", 858 | " \n", 859 | " \n", 860 | " \n", 861 | " \n", 862 | " \n", 863 | " \n", 864 | " \n", 865 | " \n", 866 | " \n", 867 | " \n", 868 | " \n", 869 | " \n", 870 | " \n", 871 | " \n", 872 | " \n", 873 | " \n", 874 | " \n", 875 | " \n", 876 | " \n", 877 | " \n", 878 | " \n", 879 | " \n", 880 | " \n", 881 | " \n", 882 | " \n", 883 | " \n", 884 | " \n", 885 | " \n", 886 | " \n", 887 | " \n", 888 | " \n", 889 | " \n", 890 | " \n", 891 | " \n", 892 | " \n", 893 | " \n", 894 | " \n", 895 | " \n", 896 | " \n", 897 | " \n", 898 | " \n", 899 | " \n", 900 | " \n", 901 | " \n", 902 | " \n", 903 | " \n", 904 | " \n", 905 | " \n", 906 | " \n", 907 | "
countmeanstdmin25%50%75%maxoutliers
maxhumidity_1996.094.32630510.73204745.094.0100.0100.0100.00True
maxhumidity_2995.094.32060310.73593445.094.0100.0100.0100.00True
maxhumidity_3994.094.31488910.73982545.094.0100.0100.0100.00True
minpressurem_1994.01014.2303825.858541996.01011.01014.01017.01037.00True
minpressurem_2993.01014.2316215.861363996.01011.01014.01017.01037.00True
minpressurem_3992.01014.2318555.864315996.01011.01014.01017.01037.00True
precipm_1999.01.4191097.9586520.00.00.00.0131.57True
precipm_2998.01.4205317.9625150.00.00.00.0131.57True
precipm_3997.01.4219567.9663840.00.00.00.0131.57True
\n", 908 | "
" 909 | ], 910 | "text/plain": [ 911 | " count mean std min 25% 50% 75% \\\n", 912 | "maxhumidity_1 996.0 94.326305 10.732047 45.0 94.0 100.0 100.0 \n", 913 | "maxhumidity_2 995.0 94.320603 10.735934 45.0 94.0 100.0 100.0 \n", 914 | "maxhumidity_3 994.0 94.314889 10.739825 45.0 94.0 100.0 100.0 \n", 915 | "minpressurem_1 994.0 1014.230382 5.858541 996.0 1011.0 1014.0 1017.0 \n", 916 | "minpressurem_2 993.0 1014.231621 5.861363 996.0 1011.0 1014.0 1017.0 \n", 917 | "minpressurem_3 992.0 1014.231855 5.864315 996.0 1011.0 1014.0 1017.0 \n", 918 | "precipm_1 999.0 1.419109 7.958652 0.0 0.0 0.0 0.0 \n", 919 | "precipm_2 998.0 1.420531 7.962515 0.0 0.0 0.0 0.0 \n", 920 | "precipm_3 997.0 1.421956 7.966384 0.0 0.0 0.0 0.0 \n", 921 | "\n", 922 | " max outliers \n", 923 | "maxhumidity_1 100.00 True \n", 924 | "maxhumidity_2 100.00 True \n", 925 | "maxhumidity_3 100.00 True \n", 926 | "minpressurem_1 1037.00 True \n", 927 | "minpressurem_2 1037.00 True \n", 928 | "minpressurem_3 1037.00 True \n", 929 | "precipm_1 131.57 True \n", 930 | "precipm_2 131.57 True \n", 931 | "precipm_3 131.57 True " 932 | ] 933 | }, 934 | "execution_count": 21, 935 | "metadata": {}, 936 | "output_type": "execute_result" 937 | } 938 | ], 939 | "source": [ 940 | "# Call describe on df and transpose it due to the large number of columns\n", 941 | "spread = df.describe().T\n", 942 | "\n", 943 | "# precalculate interquartile range for ease of use in next calculation\n", 944 | "IQR = spread['75%'] - spread['25%']\n", 945 | "\n", 946 | "# create an outliers column which is either 3 IQRs below the first quartile or\n", 947 | "# 3 IQRs above the third quartile\n", 948 | "spread['outliers'] = (spread['min'] <\n", 949 | " (spread['25%'] -\n", 950 | " (3 * IQR))) | (spread['max'] >\n", 951 | " (spread['75%'] + 3 * IQR))\n", 952 | "\n", 953 | "# just display the features containing extreame outliers\n", 954 | "spread.loc[spread.outliers, ]" 955 | ] 956 | }, 957 | { 958 | "cell_type": "code", 959 | "execution_count": 22, 960 | "metadata": {}, 961 | "outputs": [], 962 | "source": [ 963 | "# iterate over the precip columns\n", 964 | "for precip_col in ['precipm_1', 'precipm_2', 'precipm_3']:\n", 965 | " # create a boolean array of values representing nans\n", 966 | " missing_vals = pd.isnull(df[precip_col])\n", 967 | " df[precip_col][missing_vals] = 0" 968 | ] 969 | }, 970 | { 971 | "cell_type": "code", 972 | "execution_count": 23, 973 | "metadata": {}, 974 | "outputs": [], 975 | "source": [ 976 | "df = df.dropna()" 977 | ] 978 | }, 979 | { 980 | "cell_type": "code", 981 | "execution_count": 24, 982 | "metadata": {}, 983 | "outputs": [ 984 | { 985 | "data": { 986 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzgAAAHxCAYAAACh9BnXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X+YZmV5J/jvHdCooIAYOwho69irJjIidlgSZ5lGnAk4zkD2ColJVGBxOjvrmh/iJh3HJGaSTMzMJkY3EyesGMAxIuPGgYjrhKA1xmTQiD9QxAhBhRaEqIC2+Au954/3tCmKarq6u4qqfurzua663nOe87zn3G+9D9X15XneU9XdAQAAGMF3rXYBAAAAy0XAAQAAhiHgAAAAwxBwAACAYQg4AADAMAQcAABgGAIOwDKrqv9YVb+8TOd6bFXtqKoDpv25qnrRcpx7Ot//X1VnLtf59uC6v1FVn6+qzz3Q115Qx6er6tkP0LWuraotuzi2paq2L6UvAPfvwNUuAGB/UlWfTrIhyT1JvpXk40kuSnJed387Sbr7f9+Dc72ou/98V326+6YkB+9b1d+53iuTPLG7nz/v/Kcux7n3sI6jk5yb5HHdffsDff3V0t3fvzd9F3vf9kZVnZTkV5Icl+SO7t64L+cDWKvM4ADsuX/e3Q9P8rgkr0ryi0nOX+6LVNWo/xPqcUm+sJ7CzRrxlSRvSPJ/rXYhACtJwAHYS919V3dfluTHk5xZVU9Nkqq6oKp+Y9p+VFW9varurKovVtVfVNV3VdUbkzw2yZ9OS9B+oao2VlVX1TlVdVOSd81rmx92/kFVvb+q7qqqS6vqkdO17rXMaWr7dFU9u6pOSfLyJD8+Xe8j0/HvLHmb6npFVX2mqm6vqouq6pDp2M46zqyqm6blZf96V9+bqjpkev7fTed7xXT+Zye5IsljpjouWOS5W6pq+/Q9ub2qbq2q06vqOVX1yen7+PJ5/Y+vqv8+fY9vrarfr6oHT8d+aKr16Gn/aVO/J8+75LFVdc30/XxLVT1k6ntWVb13QW1dVU+c9z7/wbTMb0dV/WVVfW9V/V5V3VFVn6iqpy98L6bth07Pv6OqPp7kB5b6vlXVGVV19YL+51bVf9nV+5Ek3f3+7n5jkhvvrx/A/k7AAdhH3f3+JNuT/C+LHD53OvY9mS1te/nsKf2CJDdlNht0cHf/u3nP+cdJnpLkh3dxyRcm+d+SPCazpXKvXUKN70zyb5O8Zbre0xbpdtb0dVKSJ2S2NO73F/T5R0melOTkJL9SVU/ZxSX/nySHTOf5x1PNZ0/L8U5NcstUx1m7eP73JnlIkiMzW1b1/yZ5fpJnZPZ9/pWqesLU91tJfj7Jo5L84FTb/zG97r9K8odJLqyqhyZ5Y5JXdPcn5l3rx5KckuTxSf7h9D1Yqh9L8orp2l9P8t+TfHDaf2uS393F8341yT+Yvn44yaKfg9rF+3ZZkscv+N4/f3ptAOuegAOwPG5J8shF2r+Z5IjMPm/yze7+i+7u3Zzrld39le7+6i6Ov7G7P9bdX0nyy0l+rKabEOyjn0ryu919Y3fvSPJLSZ63YPbo17r7q939kSQfSXKfoDTV8uNJfqm7v9zdn07yO0lesAe1fDPJb3b3N5NcnFlgeM10vmuTXJtZGEl3X93dV3X3PdO1/jCzULXTKzMLW+/P7H36Dwuu9druvqW7v5jkT5Mcuwd1vm26/teSvC3J17r7ou7+VpK3JHn6Lp73Y9Pr+2J335wlhNSduvvr07mfnyRV9f1JNiZ5+x7UDTAsAQdgeRyZ5IuLtP/7JDck+bOqurGqti3hXDfvwfHPJHlQZgFgXz1mOt/8cx+Y2czTTvPvenZ3Fr8BwqOSPHiRcx25B7V8YQoJSbIz6N027/hXd167qv6naRng56rqS5nNeHzn+zGFpAuSPDXJ7ywSMJfymnZlYU2L1riIx+S+7+OeuDDJT1ZVZRYcL5mCD8C6J+AA7KOq+oHMfnl/78Jj04zDud39hCT/PMlLq+rknYd3ccrdzfAcPW/7sZnNdnw+sw+RP2xeXQdktjRuqee9JbMbAMw/9z259y/tS/H5qaaF5/rsHp5nqV6X5BNJNnX3IzJbBlg7D1bVkZktCfujJL9TVd+9xPMu/H5+77JVnNya+76Pu3Kf9627r0ryjcyW6/1kLE8D+A4BB2AvVdUjquq5mS2h+k/d/dFF+jy3qp44/Z/2L2X2eZGdMxO3ZfYZlT31/Kr6vqp6WJJ/k+St02zHJ5M8pKr+WVU9KLPPhsz/Zf62JBuralc/+9+c5Oer6vFVdXD+/rMf9+xJcVMtlyT5zap6eFU9LslLk/ynPTnPHnh4Zt/bHdPNA/7VzgPT9/2CzO5yd05mweLXl3jejyT5/qo6drrxwCuXseZLkvxSVR1WVUclecn99N3V+3ZRZp+Ruqe77xOuF5pu8vCQzGb8qqoesvNmDAAjEXAA9tyfVtWXM1ti9K8z+yD52bvouynJnyfZkdkH0P+gu+emY7+V5BXTXb1etgfXf2Nmv7R/LrMP4v9MMrurW2Yfrn99ZrMlX8nsBgc7/efp8QtV9cFFzvuG6dzvSfKpJF/L/f/ifX9eMl3/xsxmtv54Ov9KeFlmsxhfzuxmBG+Zd+xnMlti98vT0rSzk5xdVYvdEOJeuvuTmQXIP09yfRaZodsHv5bZsrRPJfmz3P8MzK7etzdmtuxuqbM3J2a2bO4dmc0YfXW6NsBQavefdQUA1prprnC3Jzmuu69f7XoA1gozOACwf/pXSf5auAG4t1H/SjYADKuqPp3ZjRROX9B+be59c4edfrq73/QAlAaw6ixRAwAAhmGJGgAAMAwBBwAAGMaa+AzOox71qN64ceNql8HkK1/5Sg466KDVLoNVZAxgDJAYBxgDrK0xcPXVV3++u79nd/3WRMDZuHFjPvCBD6x2GUzm5uayZcuW1S6DVWQMYAyQGAcYA6ytMVBVn1lKP0vUAACAYQg4AADAMAQcAABgGAIOAAAwDAEHAAAYhoADAAAMY7cBp6qeVFUfnvf1par6uap6ZFVdUVXXT4+HTf2rql5bVTdU1TVVddzKvwwAAIAlBJzu/pvuPra7j03yjCR3J3lbkm1JruzuTUmunPaT5NQkm6avrUletxKFAwAALLSnS9ROTvK33f2ZJKcluXBqvzDJ6dP2aUku6pmrkhxaVUcsS7UAAAD3Y08DzvOSvHna3tDdtybJ9Pjoqf3IJDfPe872qQ0AAGBFVXcvrWPVg5PckuT7u/u2qrqzuw+dd/yO7j6sqi5P8lvd/d6p/cokv9DdVy8439bMlrBlw4YNz7j44ouX5xWxz3bs2JGDDz54tctgFRkDGAMkxgHGAGtrDJx00klXd/fm3fU7cA/OeWqSD3b3bdP+bVV1RHffOi1Bu31q357k6HnPOyqzYHQv3X1ekvOSZPPmzb1ly5Y9KIWVNDc3F+/H+mYMYAyQGAcYA+yfY2BPlqj9RP5+eVqSXJbkzGn7zCSXzmt/4XQ3tROS3LVzKRsAAMBKWtIMTlU9LMk/SfLT85pfleSSqjonyU1Jzpja35HkOUluyOyOa2cvW7UAAAD3Y0kBp7vvTnL4grYvZHZXtYV9O8mLl6U6AACAPbCnd1EDAABYswQcAABgGAIOAAAwDAEHAAAYxp78HRwAABjOxm2Xr3YJa9YFpxy02iXsMTM4AADAMAQcAABgGAIOAAAwDAEHAAAYhoADAAAMQ8ABAACGIeAAAADDEHAAAIBhCDgAAMAwBBwAAGAYAg4AADAMAQcAABiGgAMAAAxDwAEAAIYh4AAAAMMQcAAAgGEIOAAAwDAEHAAAYBgCDgAAMAwBBwAAGIaAAwAADEPAAQAAhiHgAAAAwxBwAACAYQg4AADAMAQcAABgGAIOAAAwDAEHAAAYhoADAAAMQ8ABAACGIeAAAADDEHAAAIBhCDgAAMAwBBwAAGAYAg4AADAMAQcAABiGgAMAAAxDwAEAAIYh4AAAAMMQcAAAgGEIOAAAwDAEHAAAYBgCDgAAMAwBBwAAGIaAAwAADEPAAQAAhrGkgFNVh1bVW6vqE1V1XVX9YFU9sqquqKrrp8fDpr5VVa+tqhuq6pqqOm5lXwIAAMDMUmdwXpPknd395CRPS3Jdkm1JruzuTUmunPaT5NQkm6avrUlet6wVAwAA7MJuA05VPSLJiUnOT5Lu/kZ335nktCQXTt0uTHL6tH1akot65qokh1bVEcteOQAAwAJLmcF5QpK/S/JHVfWhqnp9VR2UZEN335ok0+Ojp/5HJrl53vO3T20AAAArqrr7/jtUbU5yVZJndvf7quo1Sb6U5CXdfei8fnd092FVdXmS3+ru907tVyb5he6+esF5t2a2hC0bNmx4xsUXX7ycr4t9sGPHjhx88MGrXQaryBjAGCAxDlg/Y+Cjn71rtUtYsx5/yAFrZgycdNJJV3f35t31O3AJ59qeZHt3v2/af2tmn7e5raqO6O5bpyVot8/rf/S85x+V5JaFJ+3u85KclySbN2/uLVu2LKEUHghzc3PxfqxvxgDGAIlxwPoZA2dtu3y1S1izLjjloP1uDOx2iVp3fy7JzVX1pKnp5CQfT3JZkjOntjOTXDptX5bkhdPd1E5IctfOpWwAAAAraSkzOEnykiRvqqoHJ7kxydmZhaNLquqcJDclOWPq+44kz0lyQ5K7p74AAAArbkkBp7s/nGSx9W4nL9K3k7x4H+sCAADYY0v9OzgAAABrnoADAAAMQ8ABAACGIeAAAADDEHAAAIBhCDgAAMAwBBwAAGAYAg4AADAMAQcAABiGgAMAAAxDwAEAAIYh4AAAAMMQcAAAgGEIOAAAwDAEHAAAYBgCDgAAMAwBBwAAGIaAAwAADEPAAQAAhiHgAAAAwxBwAACAYQg4AADAMAQcAABgGAIOAAAwDAEHAAAYhoADAAAMQ8ABAACGIeAAAADDEHAAAIBhCDgAAMAwBBwAAGAYAg4AADAMAQcAABiGgAMAAAxDwAEAAIYh4AAAAMMQcAAAgGEIOAAAwDAEHAAAYBgCDgAAMAwBBwAAGIaAAwAADEPAAQAAhiHgAAAAwxBwAACAYQg4AADAMAQcAABgGAIOAAAwDAEHAAAYhoADAAAMQ8ABAACGIeAAAADDWFLAqapPV9VHq+rDVfWBqe2RVXVFVV0/PR42tVdVvbaqbqiqa6rquJV8AQAAADvtyQzOSd19bHdvnva3JbmyuzcluXLaT5JTk2yavrYmed1yFQsAAHB/9mWJ2mlJLpy2L0xy+rz2i3rmqiSHVtUR+3AdAACAJVlqwOkkf1ZVV1fV1qltQ3ffmiTT46On9iOT3DzvudunNgAAgBV14BL7PbO7b6mqRye5oqo+cT99a5G2vk+nWVDamiQbNmzI3NzcEkthpe3YscP7sc4ZAxgDJMYB62cMnHvMPatdwpq1P46BJQWc7r5lery9qt6W5Pgkt1XVEd1967QE7fap+/YkR897+lFJblnknOclOS9JNm/e3Fu2bNnrF8Hympubi/djfTMGMAZIjAPWzxg4a9vlq13CmnXBKQftd2Ngt0vUquqgqnr4zu0k/zTJx5JcluTMqduZSS6dti9L8sLpbmonJLlr51I2AACAlbSUGZwNSd5WVTv7/3F3v7Oq/jrJJVV1TpKbkpwx9X9HkuckuSHJ3UnOXvaqAQAAFrHbgNPdNyZ52iLtX0hy8iLtneTFy1IdAADAHtiX20QDAACsKQIOAAAwDAEHAAAYhoADAAAMQ8ABAACGIeAAAADDEHAAAIBhCDgAAMAwBBwAAGAYAg4AADAMAQcAABiGgAMAAAxDwAEAAIYh4AAAAMMQcAAAgGEIOAAAwDAEHAAAYBgCDgAAMAwBBwAAGIaAAwAADEPAAQAAhiHgAAAAwxBwAACAYQg4AADAMAQcAABgGAIOAAAwDAEHAAAYhoADAAAMQ8ABAACGIeAAAADDEHAAAIBhCDgAAMAwBBwAAGAYAg4AADAMAQcAABiGgAMAAAxDwAEAAIYh4AAAAMMQcAAAgGEIOAAAwDAEHAAAYBgCDgAAMAwBBwAAGIaAAwAADEPAAQAAhiHgAAAAwxBwAACAYQg4AADAMAQcAABgGAIOAAAwDAEHAAAYxpIDTlUdUFUfqqq3T/uPr6r3VdX1VfWWqnrw1P7d0/4N0/GNK1M6AADAve3JDM7PJrlu3v5vJ3l1d29KckeSc6b2c5Lc0d1PTPLqqR8AAMCKW1LAqaqjkvyzJK+f9ivJs5K8depyYZLTp+3Tpv1Mx0+e+gMAAKyopc7g/F6SX0jy7Wn/8CR3dvc90/72JEdO20cmuTlJpuN3Tf0BAABW1IG761BVz01ye3dfXVVbdjYv0rWXcGz+ebcm2ZokGzZsyNzc3FLq5QGwY8cO78c6ZwxgDJAYB6yfMXDuMffsvtM6tT+Ogd0GnCTPTPIvquo5SR6S5BGZzegcWlUHTrM0RyW5Zeq/PcnRSbZX1YFJDknyxYUn7e7zkpyXJJs3b+4tW7bs40thuczNzcX7sb4ZAxgDJMYB62cMnLXt8tUuYc264JSD9rsxsNslat39S919VHdvTPK8JO/q7p9K8u4kPzp1OzPJpdP2ZdN+puPv6u77zOAAAAAst335Ozi/mOSlVXVDZp+xOX9qPz/J4VP7S5Ns27cSAQAAlmYpS9S+o7vnksxN2zcmOX6RPl9LcsYy1AYAALBH9mUGBwAAYE0RcAAAgGEIOAAAwDAEHAAAYBgCDgAAMAwBBwAAGIaAAwAADEPAAQAAhiHgAAAAwxBwAACAYQg4AADAMAQcAABgGAIOAAAwDAEHAAAYhoADAAAMQ8ABAACGIeAAAADDEHAAAIBhCDgAAMAwBBwAAGAYAg4AADAMAQcAABiGgAMAAAxDwAEAAIYh4AAAAMMQcAAAgGEIOAAAwDAEHAAAYBgCDgAAMAwBBwAAGIaAAwAADEPAAQAAhiHgAAAAwxBwAACAYQg4AADAMAQcAABgGAIOAAAwDAEHAAAYhoADAAAMQ8ABAACGIeAAAADDEHAAAIBhCDgAAMAwBBwAAGAYAg4AADAMAQcAABiGgAMAAAxDwAEAAIYh4AAAAMMQcAAAgGEIOAAAwDAEHAAAYBi7DThV9ZCqen9VfaSqrq2qX5vaH19V76uq66vqLVX14Kn9u6f9G6bjG1f2JQAAAMwsZQbn60me1d1PS3JsklOq6oQkv53k1d29KckdSc6Z+p+T5I7ufmKSV0/9AAAAVtxuA07P7Jh2HzR9dZJnJXnr1H5hktOn7dOm/UzHT66qWraKAQAAdmFJn8GpqgOq6sNJbk9yRZK/TXJnd98zddme5Mhp+8gkNyfJdPyuJIcvZ9EAAACLOXApnbr7W0mOrapDk7wtyVMW6zY9LjZb0wsbqmprkq1JsmHDhszNzS2lFB4AO3bs8H6sc8YAxgCJccD6GQPnHnPP7jutU/vjGFhSwNmpu++sqrkkJyQ5tKoOnGZpjkpyy9Rte5Kjk2yvqgOTHJLki4uc67wk5yXJ5s2be8uWLXv7Glhmc3Nz8X6sb8YAxgCJccD6GQNnbbt8tUtYsy445aD9bgws5S5q3zPN3KSqHprk2UmuS/LuJD86dTszyaXT9mXTfqbj7+ru+8zgAAAALLelzOAckeTCqjogs0B0SXe/vao+nuTiqvqNJB9Kcv7U//wkb6yqGzKbuXneCtQNAABwH7sNON19TZKnL9J+Y5LjF2n/WpIzlqU6AACAPbCku6gBAADsDwQcAABgGAIOAAAwDAEHAAAYhoADAAAMQ8ABAACGIeAAAADDEHAAAIBhCDgAAMAwBBwAAGAYAg4AADAMAQcAABiGgAMAAAxDwAEAAIYh4AAAAMMQcAAAgGEIOAAAwDAEHAAAYBgCDgAAMAwBBwAAGIaAAwAADEPAAQAAhiHgAAAAwxBwAACAYQg4AADAMAQcAABgGAIOAAAwDAEHAAAYhoADAAAMQ8ABAACGIeAAAADDEHAAAIBhCDgAAMAwBBwAAGAYAg4AADAMAQcAABiGgAMAAAxDwAEAAIYh4AAAAMMQcAAAgGEIOAAAwDAEHAAAYBgCDgAAMAwBBwAAGIaAAwAADEPAAQAAhiHgAAAAwxBwAACAYQg4AADAMAQcAABgGAIOAAAwjN0GnKo6uqreXVXXVdW1VfWzU/sjq+qKqrp+ejxsaq+qem1V3VBV11TVcSv9IgAAAJKlzeDck+Tc7n5KkhOSvLiqvi/JtiRXdvemJFdO+0lyapJN09fWJK9b9qoBAAAWsduA0923dvcHp+0vJ7kuyZFJTkty4dTtwiSnT9unJbmoZ65KcmhVHbHslQMAACywR5/BqaqNSZ6e5H1JNnT3rcksBCV59NTtyCQ3z3va9qkNAABgRR241I5VdXCS/y/Jz3X3l6pql10XaetFzrc1syVs2bBhQ+bm5pZaCitsx44d3o91zhjAGCAxDlg/Y+DcY+5Z7RLWrP1xDCwp4FTVgzILN2/q7j+Zmm+rqiO6+9ZpCdrtU/v2JEfPe/pRSW5ZeM7uPi/JeUmyefPm3rJly969Apbd3NxcvB/rmzGAMUBiHLB+xsBZ2y5f7RLWrAtOOWi/GwNLuYtaJTk/yXXd/bvzDl2W5Mxp+8wkl85rf+F0N7UTkty1cykbAADASlrKDM4zk7wgyUer6sNT28uTvCrJJVV1TpKbkpwxHXtHkuckuSHJ3UnOXtaKAQAAdmG3Aae735vFP1eTJCcv0r+TvHgf6wIAANhje3QXNQAAgLVMwAEAAIYh4AAAAMMQcAAAgGEIOAAAwDAEHAAAYBgCDgAAMAwBBwAAGIaAAwAADEPAAQAAhiHgAAAAwxBwAACAYQg4AADAMAQcAABgGAIOAAAwDAEHAAAYhoADAAAMQ8ABAACGIeAAAADDEHAAAIBhCDgAAMAwBBwAAGAYAg4AADAMAQcAABiGgAMAAAxDwAEAAIYh4AAAAMMQcAAAgGEIOAAAwDAEHAAAYBgCDgAAMAwBBwAAGIaAAwAADEPAAQAAhiHgAAAAwxBwAACAYQg4AADAMAQcAABgGAIOAAAwDAEHAAAYhoADAAAMQ8ABAACGIeAAAADDEHAAAIBhCDgAAMAwBBwAAGAYAg4AADAMAQcAABiGgAMAAAxDwAEAAIYh4AAAAMMQcAAAgGHsNuBU1Ruq6vaq+ti8tkdW1RVVdf30eNjUXlX12qq6oaquqarjVrJ4AACA+ZYyg3NBklMWtG1LcmV3b0py5bSfJKcm2TR9bU3yuuUpEwAAYPd2G3C6+z1Jvrig+bQkF07bFyY5fV77RT1zVZJDq+qI5SoWAADg/lR3775T1cYkb+/up077d3b3ofOO39Hdh1XV25O8qrvfO7VfmeQXu/sDi5xza2azPNmwYcMzLr744mV4OSyHHTt25OCDD17tMlhFxgDGAIlxwPoZAx/97F2rXcKa9fhDDlgzY+Ckk066urs3767fgct83VqkbdEE1d3nJTkvSTZv3txbtmxZ5lLYW3Nzc/F+rG/GAMYAiXHA+hkDZ227fLVLWLMuOOWg/W4M7O1d1G7bufRserx9at+e5Oh5/Y5KcsvelwcAALB0extwLkty5rR9ZpJL57W/cLqb2glJ7uruW/exRgAAgCXZ7RK1qnpzki1JHlVV25P8apJXJbmkqs5JclOSM6bu70jynCQ3JLk7ydkrUDMAAMCidhtwuvsndnHo5EX6dpIX72tRAAAAe2O5bzIAAMAatHEvPkh/7jH3+AA++529/QwOAADAmiPgAAAAwxBwAACAYQg4AADAMAQcAABgGAIOAAAwDAEHAAAYhoADAAAMQ8ABAACGIeAAAADDEHAAAIBhCDgAAMAwBBwAAGAYAg4AADAMAQcAABjGgatdAADActm47fLVLgFYZWZwAACAYQg4AADAMAQcAABgGAIOAAAwDAEHAAAYhoADAAAMQ8ABAACGIeAAAADDEHAAAIBhCDgAAMAwBBwAAGAYB652AQDAntm47fIH5DrnHnNPznqArgWwXMzgAAAAwxBwAACAYQg4AADAMAQcAABgGAIOAAAwDAEHAAAYhoADAAAMQ8ABAACG4Q99ArAmPVB/zBKAsZjBAQAAhmEGB2AVrdVZinOPuSdnrdHaAOD+CDjAilurv8QDAOOxRA0AABiGgAMAAAxDwAEAAIYh4AAAAMMQcAAAgGEIOAAAwDAEHAAAYBj+Dg4sk5H+1os/8ggA7K/M4AAAAMMQcAAAgGGsSMCpqlOq6m+q6oaq2rYS1wAAAFho2T+DU1UHJPkPSf5Jku1J/rqqLuvujy/3tVbKSJ+l2Bs+fwEAwP5qJWZwjk9yQ3ff2N3fSHJxktNW4DoAAAD3shIB58gkN8/b3z61AQAArKjq7uU9YdUZSX64u1807b8gyfHd/ZIF/bYm2TrtPinJ3yxrIeyLRyX5/GoXwaoyBjAGSIwDjAHW1hh4XHd/z+46rcTfwdme5Oh5+0cluWVhp+4+L8l5K3B99lFVfaC7N692HaweYwBjgMQ4wBhg/xwDK7FE7a+TbKqqx1fVg5M8L8llK3AdAACAe1n2GZzuvqeq/s8k/zXJAUne0N3XLvd1AAAAFlqJJWrp7nckecdKnJsHhKWDGAMYAyTGAcYA++EYWPabDAAAAKyWlfgMDgAAwKoQcEhVfbqqPlpVH66qD0xtj6yqK6rq+unxsNWuk5VTVYdW1Vur6hNVdV1V/aAxsH5U1ZOm//53fn2pqn7OGFhfqurnq+raqvpYVb25qh4y3TDofdMYeMt08yAGVVU/O73/11bVz01tfg4MrKreUFW3V9XH5rUt+p7XzGur6oaquqaqjlu9yu+fgMNOJ3X3sfNuA7gtyZXdvSnJldM+43pNknd295OTPC3JdTEG1o3u/pvpv/9jkzwjyd1J3hZjYN2oqiOT/EySzd391MxuEvS8JL+d5NXTGLgjyTmrVyUrqaqemuRfJjk+s38HnltVm+LnwOguSHLKgrZdveenJtk0fW1N8roHqMY9JuCwK6cluXDavjDJ6atYCyuoqh6R5MQk5ydJd3+ju++MMbBenZzkb7v7MzEG1psDkzy0qg5M8rAktyYTVC0pAAAGDElEQVR5VpK3TseNgbE9JclV3X13d9+T5L8l+ZH4OTC07n5Pki8uaN7Ve35akot65qokh1bVEQ9MpXtGwCFJOsmfVdXVVbV1atvQ3bcmyfT46FWrjpX2hCR/l+SPqupDVfX6qjooxsB69bwkb562jYF1ors/m+T/TnJTZsHmriRXJ7lz+mU3mf0h7yNXp0IeAB9LcmJVHV5VD0vynMz+cLufA+vPrt7zI5PcPK/fmv2ZIOCQJM/s7uMym3p8cVWduNoF8YA6MMlxSV7X3U9P8pVYgrAuTZ+v+BdJ/vNq18IDa1pjf1qSxyd5TJKDMvs3YSG3Xh1Ud1+X2ZLEK5K8M8lHktxzv09ivalF2tbkzwQBh3T3LdPj7Zmtuz8+yW07px2nx9tXr0JW2PYk27v7fdP+WzMLPMbA+nNqkg92923TvjGwfjw7yae6+++6+5tJ/iTJD2W2BGXn38w7Ksktq1UgK6+7z+/u47r7xMyWLV0fPwfWo12959szm9Xbac3+TBBw1rmqOqiqHr5zO8k/zWya+rIkZ07dzkxy6epUyErr7s8lubmqnjQ1nZzk4zEG1qOfyN8vT0uMgfXkpiQnVNXDqqry9z8H3p3kR6c+xsDgqurR0+Njk/yvmf088HNg/dnVe35ZkhdOd1M7IcldO5eyrTX+0Oc6V1VPyGzWJpktVfrj7v7Nqjo8ySVJHpvZP3xndPfCD6ExiKo6Nsnrkzw4yY1Jzs7sf4AYA+vEtOb+5iRP6O67pjY/B9aRqvq1JD+e2bKkDyV5UWbr6y9O8sip7fnd/fVVK5IVVVV/keTwJN9M8tLuvtLPgbFV1ZuTbEnyqCS3JfnVJP8li7zn0//8+P3M7rp2d5Kzu/sDq1H37gg4AADAMCxRAwAAhiHgAAAAwxBwAACAYQg4AADAMAQcAABgGAIOAAAwDAEHgGVXVa+sqpet0Ln/ahftF1TVj07br6+q75u2X74P13pnVd1ZVW/f23MA8MAScADYr3T3Dy2hz4u6++PT7l4HnCT/PskL9uH5ADzABByAda6qNlbVJ6ZZj49V1Zuq6tlV9ZdVdX1VHT99/VVVfWh6fNL03JdW1Rum7WOm5z9sOvX3VdVcVd1YVT8z71ofm3ftl1XVK6ftuap6dVW9p6quq6ofqKo/mWr4jXnP2TE9VlX9flV9vKouT/LoeX3mqmpzVb0qyUOr6sPT6/r1qvrZef1+c2dti+nuK5N8eZ+/yQA8YAQcAJLkiUlek+QfJnlykp9M8o+SvCyzGZBPJDmxu5+e5FeS/Nvpeb+X5IlV9SNJ/ijJT3f33dOxJyf54STHJ/nVqnrQEur4RnefmOQ/Jrk0yYuTPDXJWVV1+IK+P5LkSUmOSfIvk9xnZqe7tyX5ancf290/leT8JGcmSVV9V5LnJXnTEuoCYD9x4GoXAMCa8Knu/miSVNW1Sa7s7q6qjybZmOSQJBdW1aYkneRBSdLd366qs5Jck+QPu/sv553z8u7+epKvV9XtSTYsoY7LpsePJrm2u2+daroxydFJvjCv74lJ3tzd30pyS1W9a3cn7+5PV9UXqurpUz0f6u4v7O55AOw/BBwAkuTr87a/PW//25n9W/HrSd7d3T9SVRuTzM3rvynJjiSPuZ9zfms6zz259+qBh+ziOfNrmF/HQr1I2+68PslZSb43yRv24vkArGGWqAGwFIck+ey0fdbOxqo6JLOlbScmOXznXczux21JHl1Vh1fVdyd57j7U9J4kz6uqA6rqiCQn7aLfNxcsj3tbklOS/ECS/7oP1wdgDTKDA8BS/LvMlqi9NMn8pWCvTvIH3f3Jqjonybur6j27Okl3f7Oq/k2S9yX5VGaf7dlbb0vyrMyWs30yyX/bRb/zklxTVR/s7p/q7m9U1buT3Dktb9ulqvqLzD5LdHBVbU9yTncLRQBrWHXvzew+AOyfppsLfDDJGd19/WrXA8DyskQNgHVj+uOfN2R2EwXhBmBAZnAAWNeq6pgkb1zQ/PXu/p9Xox4A9o2AAwAADMMSNQAAYBgCDgAAMAwBBwAAGIaAAwAADEPAAQAAhvE/AGg77iGPRjOKAAAAAElFTkSuQmCC\n", 987 | "text/plain": [ 988 | "
" 989 | ] 990 | }, 991 | "metadata": {}, 992 | "output_type": "display_data" 993 | } 994 | ], 995 | "source": [ 996 | "fig, ax = plt.subplots(figsize = (14, 8))\n", 997 | "ax.hist(df.maxhumidity_1)\n", 998 | "ax.set_title('Distribution of maxhumidity_1')\n", 999 | "ax.set_xlabel('maxhumidity_1')\n", 1000 | "ax.grid()" 1001 | ] 1002 | }, 1003 | { 1004 | "cell_type": "code", 1005 | "execution_count": 25, 1006 | "metadata": {}, 1007 | "outputs": [ 1008 | { 1009 | "data": { 1010 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzkAAAHxCAYAAABONnLpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X20ZWddJ/jvj4Q3CRIgWB1CpGhItwYjUaojtnRTEZa8qUFHmjCACcRJ9xpsmu4wWtIKKGQmOg2sZlR6otAJLxIiLxJI0IlproiIBBASQkRiUpAiTCLvqQhIhV//cXbBobhVdW/VfX3y+ax11z3n2c/e+7fPedap87372buquwMAADCKO613AQAAACtJyAEAAIYi5AAAAEMRcgAAgKEIOQAAwFCEHAAAYChCDsAKqKr/VlW/tkLb+t6q2l1VR0zPF6rqF1Zi29P23llVZ6zU9pax35dU1Wer6v9fgW09v6p+fyXqAmA85f/JATiwqtqZZEuSPUluT/KxJK9Jcn53f+MQtvUL3f2ny1hnIcnrunvZX+qr6kVJHtLdT1/uuiupqo5P8rdJHtjdt6xnLXxLVb04yZOSfH+Sl3T3i9a3IoCV4UwOwNL8VHffM8kDk5yX5JeTvGqld1JVR670NjeIByb53EYPOBvh9V/jGq5L8ktJLl3DfQKsOiEHYBm6+0vdfUmSpyQ5o6p+IEmq6oKqesn0+JiqekdVfbGqPl9Vf15Vd6qq1yb53iRvn6aj/VJVba2qrqqzqupTSf7HXNv8l90HV9X7q+pLVfW2qrrPtK/tVbVrvsaq2llVj6mqxyV5fpKnTPv7yLT8m9Pfprp+tao+WVW3VNVrqupe07K9dZxRVZ+appr95/29NlV1r2n9v5+296vT9h+T5PIk95/quGCRdbdX1a7pNbmlqj5TVU+qqidU1d9Or+Pz5/q/qKpet5Q6p75vqqo3VtWtVfWhqnrYPq/XL1fVVUluq6ojq+r+VfXm6VhuqKrnzPU/pao+UFVfrqqbq+plU/vdqup1VfW56b2/sqq2zL8nB6n/m2Ngan9EVb132tZHqmr73PoL0/S/906v6dur6r5V9fqpriurauv+3qu9uvvC7n5nklsP1hdgMxFyAA5Bd78/ya4k/2qRxedMy+6X2TS3589W6Wck+VRmZ4WO6u7fmlvnUZlNGXrsfnb580meleT+mU2be8USavzjJP9nkjdO+3vYIt3OnH5OTfJPkxyV5Lf36fPIJP88yaOTvKCqvn8/u/x/ktxr2s6jppqfOU3Ne3ySm6Y6ztzP+v8kyd2SHJfkBUl+L8nTkzw8s9f5BVX1Tw9wyAeq87Qkf5jkPkn+IMkfVdWd55Y/NckTkxyd5BtJ3p7kI1Mtj07y3Kra+9781yT/tbu/O8mDk1w8tZ8xHf/xSe6b5N8l+coB6t3XN8dAVR2X2dmVl0w1Py/Jm6vqfnP9T0/yjKnGByf5yyT/fep/bZIXLmPfAEMRcgAO3U2ZfaHc19eTHJvZ9Sdf7+4/74NfAPmi7r6tu/f3pfi13f3R7r4tya8l+Tc13ZjgMD0tycu6+/ru3p3kV5Kcvs9ZpF/v7q9090cy++L/HWFpquUpSX6lu2/t7p1JXprZl/Cl+nqSc7v760kuSnJMZmHi1u6+Jsk1SX7wAOsfqM4Pdvebpm2/LLMw9Yi55a/o7hun1/9fJLlfd/9Gd/9jd1+fWeA6fa7Oh1TVMd29u7vfN9d+38yugbq9uz/Y3V9exvHPj4GnJ7msuy/r7m909+VJPpDkCXP9/3t3/113fynJO5P8XXf/aXfvySzQ/dAy9g0wFCEH4NAdl+Tzi7T/35ld6/D/VdX1VbVjCdu6cRnLP5nkzpmFgMN1/2l789s+MrMzUHvN3w3tHzI727OvY5LcZZFtHbeMWj7X3bdPj/eGvZvnln9lP/teSp3ffP2mm0XsyuzYv2N5ZtcP3X+aJvbFqvpiZmfj9r4mZyX5Z0n+ZpoW9pNT+2uT/EmSi6rqpqr6rX3OFh3MvjU8eZ8aHplZeN5r39dmOa8VwNDW/QJLgM2oqv5FZl/g37Pvsu6+NbMpa+dU1UOTvKuqruzuK5Ls74zOwc70HD/3+HszO2vw2SS3JfmuubqOyGya3FK3e1NmX6jnt70nsy/MDzjIuvM+O9X0wMzuPrd3W59exjZW0zdfv6q6U2bHdtPc8vnX6cYkN3T3CYttqLs/keSp03Z+Nsmbquq+01m2X0/y69P1MJcl+XhmN6j4tvcps6l537HpfWp4bXf/b0s6OgC+jTM5AMtQVd89/eX+osxu63z1In1+sqoeUlWV5MuZ3XZ67xmKmzO7ZmW5nl5VJ1bVdyX5jSRvms56/G2Su1XVE6ezBr+a5K5z692cZOv0hXwxb0jyH6vqQVV1VL51Dc+e5RQ31XJxknOr6p5V9cAk/ynJ65aznVX08Kr62Wka3nOTfC3J+/bT9/1JvjzdjODuVXVEVf3AFGxTVU+vqvtNZ4S+OK1ze1WdWlUnTUHzy5mFvr3v+4czmwZ456raluTnDlLv65L8VFU9dtr/3Wp2c4blBM+Dmuq5W2bfB46c9rMS0yAB1pWQA7A0b6+qWzP7C/t/zuy6jmfup+8JSf40ye7MLgb/3e5emJb9X0l+dZqC9Lxl7P+1SS7IbErW3ZI8J5nd7S3J/57k9zM7a3JbZlOx9vrD6ffnqupDi2z31dO2353khiRfTfLvl1HXvH8/7f/6zM5w/cG0/Y3gbZldM/SFzK4T+tnp+pzvMAW2n0pycmavyWcze33vNXV5XJJrqmp3ZjchOL27v5rZ2Zk3ZRZwrk3yZ/lWyPu1zG4O8IXMzvb8wYGK7e4bM7tZwvOT/H1m4+7/yMr/u/17mU1te2pm4/orWd51VAAbkv8MFICh1Qb5D1EBWDvO5AAAAENx4wEAGFRV/avMbi/9Hbrb3deAYZmuBgAADMV0NQAAYChCDgAAMJQNcU3OMccc01u3bl3vMjiA2267Lfe4xz3WuwxYMmOWzcaYZTMybllLH/zgBz/b3fc7eM8NEnK2bt2aD3zgA+tdBgewsLCQ7du3r3cZsGTGLJuNMctmZNyylqrqk0vta7oaAAAwFCEHAAAYipADAAAMRcgBAACGIuQAAABDEXIAAIChCDkAAMBQhBwAAGAoQg4AADAUIQcAABiKkAMAAAxFyAEAAIYi5AAAAEMRcgAAgKEIOQAAwFCEHAAAYChCDgAAMBQhBwAAGIqQAwAADOXI9S4A4I5s645L17uEDWvneU9c7xIA2KScyQEAAIYi5AAAAEMRcgAAgKEIOQAAwFCEHAAAYChCDgAAMBQhBwAAGIqQAwAADEXIAQAAhiLkAAAAQxFyAACAoQg5AADAUIQcAABgKEIOAAAwFCEHAAAYipADAAAM5aAhp6ruVlXvr6qPVNU1VfXrU/uDquqvquoTVfXGqrrL1H7X6fl10/Ktq3sIAAAA37KUMzlfS/Lj3f2wJCcneVxVPSLJbyZ5eXefkOQLSc6a+p+V5Avd/ZAkL5/6AQAArImDhpye2T09vfP000l+PMmbpvYLkzxpenza9DzT8kdXVa1YxQAAAAewpGtyquqIqvpwkluSXJ7k75J8sbv3TF12JTluenxckhuTZFr+pST3XcmiAQAA9ufIpXTq7tuTnFxVRyd5a5LvX6zb9Huxsza9b0NVnZ3k7CTZsmVLFhYWllIK62T37t3eIzaVzTJmzzlpz8E73UFthvdvJW2WMQvzjFs2qiWFnL26+4tVtZDkEUmOrqojp7M1D0hy09RtV5Ljk+yqqiOT3CvJ5xfZ1vlJzk+Sbdu29fbt2w/1GFgDCwsL8R6xmWyWMXvmjkvXu4QNa+fTtq93CWtqs4xZmGfcslEt5e5q95vO4KSq7p7kMUmuTfKuJD83dTsjydumx5dMzzMt/x/d/R1ncgAAAFbDUs7kHJvkwqo6IrNQdHF3v6OqPpbkoqp6SZK/TvKqqf+rkry2qq7L7AzO6atQNwAAwKIOGnK6+6okP7RI+/VJTlmk/atJnrwi1QEAACzTku6uBgAAsFkIOQAAwFCEHAAAYChCDgAAMBQhBwAAGIqQAwAADEXIAQAAhiLkAAAAQxFyAACAoQg5AADAUIQcAABgKEIOAAAwFCEHAAAYipADAAAMRcgBAACGIuQAAABDEXIAAIChCDkAAMBQhBwAAGAoQg4AADAUIQcAABiKkAMAAAxFyAEAAIYi5AAAAEMRcgAAgKEIOQAAwFCEHAAAYChCDgAAMBQhBwAAGIqQAwAADEXIAQAAhiLkAAAAQxFyAACAoQg5AADAUIQcAABgKEIOAAAwFCEHAAAYipADAAAMRcgBAACGIuQAAABDEXIAAIChCDkAAMBQhBwAAGAoQg4AADAUIQcAABiKkAMAAAxFyAEAAIYi5AAAAEMRcgAAgKEIOQAAwFCEHAAAYChCDgAAMBQhBwAAGIqQAwAADEXIAQAAhiLkAAAAQzloyKmq46vqXVV1bVVdU1X/YWp/UVV9uqo+PP08YW6dX6mq66rq41X12NU8AAAAgHlHLqHPniTndPeHquqeST5YVZdPy17e3f9lvnNVnZjk9CQPTXL/JH9aVf+su29fycIBAAAWc9AzOd39me7+0PT41iTXJjnuAKucluSi7v5ad9+Q5Lokp6xEsQAAAAezrGtyqmprkh9K8ldT0y9W1VVV9eqquvfUdlySG+dW25UDhyIAAIAVU929tI5VRyX5syTndvdbqmpLks8m6SQvTnJsdz+rqn4nyV929+um9V6V5LLufvM+2zs7ydlJsmXLlodfdNFFK3VMrILdu3fnqKOOWu8yYMk2y5i9+tNfWu8SNqyTjrvXepewpjbLmIV5xi1r6dRTT/1gd29bSt+lXJOTqrpzkjcneX13vyVJuvvmueW/l+Qd09NdSY6fW/0BSW7ad5vdfX6S85Nk27ZtvX379qWUwjpZWFiI94jNZLOM2TN3XLreJWxYO5+2fb1LWFObZczCPOOWjWopd1erJK9Kcm13v2yu/di5bj+T5KPT40uSnF5Vd62qByU5Icn7V65kAACA/VvKmZwfS/KMJFdX1YentucneWpVnZzZdLWdSf5tknT3NVV1cZKPZXZntme7sxoAALBWDhpyuvs9SWqRRZcdYJ1zk5x7GHUBAAAckiVdkwNwOLauw3Un55y0x/UuAHAHtaxbSAMAAGx0Qg4AADAUIQcAABiKkAMAAAxFyAEAAIYi5AAAAEMRcgAAgKEIOQAAwFCEHAAAYChCDgAAMBQhBwAAGIqQAwAADEXIAQAAhiLkAAAAQxFyAACAoQg5AADAUIQcAABgKEIOAAAwFCEHAAAYipADAAAMRcgBAACGIuQAAABDEXIAAIChCDkAAMBQhBwAAGAoQg4AADAUIQcAABiKkAMAAAxFyAEAAIYi5AAAAEMRcgAAgKEIOQAAwFCEHAAAYChCDgAAMBQhBwAAGIqQAwAADEXIAQAAhiLkAAAAQxFyAACAoQg5AADAUIQcAABgKEIOAAAwFCEHAAAYipADAAAMRcgBAACGIuQAAABDEXIAAIChCDkAAMBQhBwAAGAoQg4AADAUIQcAABiKkAMAAAxFyAEAAIYi5AAAAEMRcgAAgKEIOQAAwFAOGnKq6viqeldVXVtV11TVf5ja71NVl1fVJ6bf957aq6peUVXXVdVVVfXDq30QAAAAey3lTM6eJOd09/cneUSSZ1fViUl2JLmiu09IcsX0PEken+SE6efsJK9c8aoBAAD246Ahp7s/090fmh7fmuTaJMclOS3JhVO3C5M8aXp8WpLX9Mz7khxdVceueOUAAACLWNY1OVW1NckPJfmrJFu6+zPJLAgl+Z6p23FJbpxbbdfUBgAAsOqOXGrHqjoqyZuTPLe7v1xV++26SFsvsr2zM5vOli1btmRhYWGppbAOdu/e7T3ikJ1z0p413+eWu6/Pflk5d7TPHJ+zbEbGLRvVkkJOVd05s4Dz+u5+y9R8c1Ud292fmaaj3TK170py/NzqD0hy077b7O7zk5yfJNu2bevt27cf2hGwJhYWFuI94lCduePSNd/nOSftyUuvXvLfcdiAdj5t+3qXsKZ8zrIZGbdsVEu5u1oleVWSa7v7ZXOLLklyxvT4jCRvm2v/+ekua49I8qW909oAAABW21L+zPljSZ6R5Oqq+vDU9vwk5yW5uKrOSvKpJE+ell2W5AlJrkvyD0meuaIVAwAAHMBBQ053vyeLX2eTJI9epH8nefZh1gUAAHBIlnV3NQAAgI1OyAEAAIYi5AAAAEMRcgAAgKEIOQAAwFCEHAAAYChCDgAAMBQhBwAAGIqQAwAADEXIAQAAhiLkAAAAQxFyAACAoQg5AADAUIQcAABgKEIOAAAwFCEHAAAYipADAAAMRcgBAACGIuQAAABDEXIAAIChCDkAAMBQhBwAAGAoQg4AADAUIQcAABiKkAMAAAxFyAEAAIYi5AAAAEMRcgAAgKEIOQAAwFCEHAAAYChCDgAAMBQhBwAAGIqQAwAADEXIAQAAhiLkAAAAQxFyAACAoQg5AADAUIQcAABgKEIOAAAwFCEHAAAYipADAAAMRcgBAACGIuQAAABDEXIAAIChCDkAAMBQhBwAAGAoQg4AADAUIQcAABiKkAMAAAxFyAEAAIYi5AAAAEMRcgAAgKEIOQAAwFCEHAAAYChCDgAAMBQhBwAAGMpBQ05Vvbqqbqmqj861vaiqPl1VH55+njC37Feq6rqq+nhVPXa1CgcAAFjMUs7kXJDkcYu0v7y7T55+LkuSqjoxyelJHjqt87tVdcRKFQsAAHAwBw053f3uJJ9f4vZOS3JRd3+tu29Icl2SUw6jPgAAgGU5nGtyfrGqrpqms917ajsuyY1zfXZNbQAAAGviyENc75VJXpykp98vTfKsJLVI315sA1V1dpKzk2TLli1ZWFg4xFJYC7t37/YeccjOOWnPmu9zy93XZ7+snDvaZ47PWTYj45aN6pBCTnffvPdxVf1ekndMT3clOX6u6wOS3LSfbZyf5Pwk2bZtW2/fvv1QSmGNLCwsxHvEoTpzx6Vrvs9zTtqTl159qH/HYSPY+bTt613CmvI5y2Zk3LJRHdJ0tao6du7pzyTZe+e1S5KcXlV3raoHJTkhyfsPr0QAAIClO+ifOavqDUm2JzmmqnYleWGS7VV1cmZT0XYm+bdJ0t3XVNXFST6WZE+SZ3f37atTOgAAwHc6aMjp7qcu0vyqA/Q/N8m5h1MUAADAoTqcu6sBAABsOEIOAAAwFCEHAAAYipADAAAMRcgBAACGIuQAAABDEXIAAIChCDkAAMBQhBwAAGAoQg4AADAUIQcAABjKketdAAAsZuuOS9e7hDV1zkl7cuYSj3nneU9c5WoANjdncgAAgKEIOQAAwFCEHAAAYChCDgAAMBQhBwAAGIqQAwAADEXIAQAAhiLkAAAAQxFyAACAoQg5AADAUIQcAABgKEIOAAAwFCEHAAAYipADAAAMRcgBAACGIuQAAABDEXIAAIChCDkAAMBQhBwAAGAoQg4AADAUIQcAABiKkAMAAAxFyAEAAIYi5AAAAEMRcgAAgKEIOQAAwFCEHAAAYChCDgAAMBQhBwAAGIqQAwAADEXIAQAAhiLkAAAAQxFyAACAoQg5AADAUIQcAABgKEIOAAAwFCEHAAAYipADAAAMRcgBAACGIuQAAABDEXIAAIChCDkAAMBQhBwAAGAoQg4AADAUIQcAABjKQUNOVb26qm6pqo/Otd2nqi6vqk9Mv+89tVdVvaKqrquqq6rqh1ezeAAAgH0duYQ+FyT57SSvmWvbkeSK7j6vqnZMz385yeOTnDD9/EiSV06/YXhbd1y63iUAAJAlnMnp7ncn+fw+zacluXB6fGGSJ821v6Zn3pfk6Ko6dqWKBQAAOJhDvSZnS3d/Jkmm398ztR+X5Ma5frumNgAAgDWxlOlqy1GLtPWiHavOTnJ2kmzZsiULCwsrXAoraffu3d6jgzjnpD3rXQJzttzde8Lmspwx6/OYjcL3AzaqQw05N1fVsd39mWk62i1T+64kx8/1e0CSmxbbQHefn+T8JNm2bVtv3779EEthLSwsLMR7dGBnuiZnQznnpD156dUr/XccWD3LGbM7n7Z9dYuBJfL9gI3qUKerXZLkjOnxGUneNtf+89Nd1h6R5Et7p7UBAACshYP+yaiq3pBke5JjqmpXkhcmOS/JxVV1VpJPJXny1P2yJE9Icl2Sf0jyzFWoGQAAYL8OGnK6+6n7WfToRfp2kmcfblEAwP65Zf3+7TzvietdArABHOp0NQAAgA1JyAEAAIYi5AAAAEMRcgAAgKEIOQAAwFCEHAAAYChCDgAAMBQhBwAAGIqQAwAADEXIAQAAhiLkAAAAQxFyAACAoQg5AADAUIQcAABgKEIOAAAwFCEHAAAYipADAAAMRcgBAACGIuQAAABDEXIAAIChCDkAAMBQhBwAAGAoQg4AADAUIQcAABiKkAMAAAxFyAEAAIYi5AAAAEMRcgAAgKEIOQAAwFCEHAAAYChCDgAAMBQhBwAAGIqQAwAADEXIAQAAhiLkAAAAQxFyAACAoQg5AADAUIQcAABgKEIOAAAwFCEHAAAYipADAAAMRcgBAACGIuQAAABDEXIAAIChCDkAAMBQhBwAAGAoQg4AADAUIQcAABiKkAMAAAxFyAEAAIYi5AAAAEMRcgAAgKEIOQAAwFCEHAAAYChCDgAAMBQhBwAAGIqQAwAADOXIw1m5qnYmuTXJ7Un2dPe2qrpPkjcm2ZpkZ5J/091fOLwyAQAAlmYlzuSc2t0nd/e26fmOJFd09wlJrpieAwAArInVmK52WpILp8cXJnnSKuwDAABgUdXdh75y1Q1JvpCkk/y/3X1+VX2xu4+e6/OF7r73IuueneTsJNmyZcvDL7rookOug9W3e/fuHHXUUetdxoZ29ae/tN4lMGfL3ZObv7LeVcDSGbMr46Tj7rXeJdyh+H7AWjr11FM/ODd77IAO65qcJD/W3TdV1fckubyq/mapK3b3+UnOT5Jt27b19u3bD7MUVtPCwkK8Rwd25o5L17sE5pxz0p689OrD/YiDtWPMroydT9u+3iXcofh+wEZ1WNPVuvum6fctSd6a5JQkN1fVsUky/b7lcIsEAABYqkMOOVV1j6q6597HSX4iyUeTXJLkjKnbGUnedrhFAgAALNXhnBffkuStVbV3O3/Q3X9cVVcmubiqzkryqSRPPvwyAQAAluaQQ053X5/kYYu0fy7Jow+nKAAAgEO1GreQBgAAWDdCDgAAMBQhBwAAGIqQAwAADEXIAQAAhiLkAAAAQxFyAACAoQg5AADAUIQcAABgKEIOAAAwFCEHAAAYipADAAAMRcgBAACGcuR6FwAAsFK27rh0vUvYsHae98T1LgHWjDM5AADAUIQcAABgKEIOAAAwFCEHAAAYihsPsCwu6AQAYKNzJgcAABiKkAMAAAxFyAEAAIYi5AAAAEMRcgAAgKEIOQAAwFCEHAAAYChCDgAAMBQhBwAAGIqQAwAADEXIAQAAhiLkAAAAQxFyAACAoQg5AADAUIQcAABgKEIOAAAwFCEHAAAYipADAAAMRcgBAACGIuQAAABDEXIAAIChCDkAAMBQhBwAAGAoQg4AADAUIQcAABiKkAMAAAxFyAEAAIYi5AAAAEMRcgAAgKEcud4FAACw+rbuuHTFt3nOSXty5ipsd63tPO+J610CK8yZHAAAYChCDgAAMBQhBwAAGIprchaxGnNWN7tR5twCADA+IQcAgDs0f+Dev816UwbT1QAAgKEIOQAAwFBWLeRU1eOq6uNVdV1V7Vit/QAAAMxblZBTVUck+Z0kj09yYpKnVtWJq7EvAACAeat1JueUJNd19/Xd/Y9JLkpy2irtCwAA4JtWK+Qcl+TGuee7pjYAAIBVtVq3kK5F2vrbOlSdneTs6enuqvr4KtXCCnhOckySz653HbBUxiybjTHLZmTcjq9+c70r+DYPXGrH1Qo5u5IcP/f8AUlumu/Q3ecnOX+V9s8Kq6oPdPe29a4DlsqYZbMxZtmMjFs2qtWarnZlkhOq6kFVdZckpye5ZJX2BQAA8E2rcianu/dU1S8m+ZMkRyR5dXdfsxr7AgAAmLda09XS3ZcluWy1ts+aM7WQzcaYZbMxZtmMjFs2pOrug/cCAADYJFbrmhwAAIB1IeTcgVXVq6vqlqr66Fzbfarq8qr6xPT73lN7VdUrquq6qrqqqn54bp0zpv6fqKoz1uNYuGNY5pj9vqr6y6r6WlU9b5/tPK6qPj6N5x1rfRzccSxzzD5t+ny9qqreW1UPm1vHmGVNLHPMnjaN1w9X1Qeq6pFz6/huwLoScu7YLkjyuH3adiS5ortPSHLF9DxJHp/khOnn7CSvTGYffElemORHkpyS5IV7P/xgFVyQpY/Zzyd5TpL/Mt+5qo5I8juZjekTkzy1qk5cxZq5Y7sgSx+zNyR5VHf/YJIXZ7rWwZhljV2QpY/ZK5I8rLtPTvKsJL+f+G7AxiDk3IF197sz+yI477QkF06PL0zypLn21/TM+5IcXVXHJnlsksu7+/Pd/YUkl+c7PxxhRSxnzHb3Ld19ZZKv79P/lCTXdff13f2PSS6atgErbplj9r3T52iSvC+z/2MuMWZZQ8scs7v7Wxd33yPf+o/ffTdg3Qk57GtLd38mSabf3zO1H5fkxrl+u6a2/bXDWtnfmN0fY5b1tpQxe1aSd06PjVnW237HbFX9TFX9TZJLMzubkxizbABCDktVi7T1AdphozJm2dCq6tTMQs4v721apJsxy4bQ3W/t7u/L7OzOi6dmY5Z1J+Swr5unaWiZft8yte9KcvxcvwckuekA7bBW9jdm98eYZb3td8xW1Q9mdl3Dad39uanZmGW9HfRzdprm9uCqOibGLBuAkMO+Lkmy9y4oZyR521z7z093WXtEki9Np6z/JMlPVNW9p4sKf2Jqg7WyvzG7P1cmOaGqHlRVd0ly+rQNWCuLjtmq+t4kb0nyjO7+27n+xizrbX9j9iFVVdPjH05ylySfi+8GbABHrncBrJ+qekOS7UmOqapdmd2L9IXHAAAERElEQVQJ5bwkF1fVWUk+leTJU/fLkjwhyXVJ/iHJM5Okuz9fVS/O7B/hJPmN7t73gkVYEcsZs1X1T5J8IMl3J/lGVT03yYnd/eWq+sXM/sE9Ismru/uaNT8Y7hCW+Tn7giT3TfK70/fGPd29rbv3GLOslWWO2f8lsz+Afj3JV5I8ZboRge8GrLv61k0xAAAANj/T1QAAgKEIOQAAwFCEHAAAYChCDgAAMBQhBwAAGIqQAwAADEXIAWBRVfXTVbVjvevYKKrq3Kq6sap2r3ctAByY/ycHgFVXVUd09+2beV9V9Ygkn0zyie4+aqW3D8DKcSYH4A6oqrZW1d9U1e9X1Uer6vVV9Ziq+ouq+kRVnVJVZ1bVb0/9L6iqV1TVe6vq+qr6ual9e1W9u6reWlUfq6r/VlV3mpbtrqrfqKq/SvKjVfXwqvqzqvpgVf1JVR079XvOtO5VVXXR1Paoqvrw9PPXVXXPaV/vmDuG366qM6fHO6vqBVX1niRPrqoHV9UfT/v686r6vrnjeGVVvWs6jkdV1aur6tqquuBAr1l3v6+7P7PS7wUAK+/I9S4AgHXzkCRPTnJ2kiuT/K9JHpnkp5M8P8kf7dP/2Gn59yW5JMmbpvZTkpyY2VmOP07ys9OyeyT5aHe/oKrunOTPkpzW3X9fVU9Jcm6SZyXZkeRB3f21qjp62ubzkjy7u/+iqo5K8tUlHM9Xu/uRSVJVVyT5d939iar6kSS/m+THp373nh7/dJK3J/mxJL+Q5MqqOrm7P7yEfQGwgQk5AHdcN3T31UlSVdckuaK7u6quTrJ1kf5/1N3fSPKxqtoy1/7+7r5+2s4bMgtCb0pye5I3T33+eZIfSHJ5VSXJEUn2nhW5Ksnrq+qP8q1g9RdJXlZVr0/ylu7eNa13IG+cajgqyb9M8odz69x1rt/b547z5n1eg61JhByATU7IAbjj+trc42/MPf9GFv/3Yb7/fOLY9+LOvc+/OndtTCW5prt/dJHtPjHJv87szMqvVdVDu/u8qro0yROSvK+qHpNkT759mvXd9tnObdPvOyX5YnefvMi+5o9j/pj3PvfvIsAAXJMDwOE6paoeNF2L85Qk71mkz8eT3K+qfjRJqurOVfXQaZ3ju/tdSX4pydFJjqqqB3f31d39m0k+kNkUuU8mObGq7lpV90ry6MWK6e4vJ7mhqp487auq6mEre8gAbGRCDgCH6y+TnJfko0luSPLWfTt09z8m+bkkv1lVH8lsSti/zGza2uumqWN/neTl3f3FJM+dbojwkSRfSfLO7r4xycWZprdN/ffnaUnOmta/Jslph3uQVfVbVbUryXdV1a6qetHhbhOA1eEW0gAcsqranuR53f2T610LAOzlTA4AADAUZ3IAYM70//rcdZ/mZ+y9CxsAG5+QAwAADMV0NQAAYChCDgAAMBQhBwAAGIqQAwAADEXIAQAAhvI/AaLo3tfrKesTAAAAAElFTkSuQmCC\n", 1011 | "text/plain": [ 1012 | "
" 1013 | ] 1014 | }, 1015 | "metadata": {}, 1016 | "output_type": "display_data" 1017 | } 1018 | ], 1019 | "source": [ 1020 | "fig, ax = plt.subplots(figsize = (14, 8))\n", 1021 | "ax.hist(df.minpressurem_1)\n", 1022 | "ax.set_title('Distribution of minpressurem_1')\n", 1023 | "ax.set_xlabel('minpressurem_1')\n", 1024 | "ax.grid()" 1025 | ] 1026 | }, 1027 | { 1028 | "cell_type": "code", 1029 | "execution_count": 26, 1030 | "metadata": {}, 1031 | "outputs": [], 1032 | "source": [ 1033 | "# import pickle\n", 1034 | "with open('end-part1_df.pkl', 'wb') as f:\n", 1035 | " pickle.dump(df, f)" 1036 | ] 1037 | } 1038 | ], 1039 | "metadata": { 1040 | "kernelspec": { 1041 | "display_name": "Python 3", 1042 | "language": "python", 1043 | "name": "python3" 1044 | }, 1045 | "language_info": { 1046 | "codemirror_mode": { 1047 | "name": "ipython", 1048 | "version": 3 1049 | }, 1050 | "file_extension": ".py", 1051 | "mimetype": "text/x-python", 1052 | "name": "python", 1053 | "nbconvert_exporter": "python", 1054 | "pygments_lexer": "ipython3", 1055 | "version": "3.6.5" 1056 | } 1057 | }, 1058 | "nbformat": 4, 1059 | "nbformat_minor": 2 1060 | } 1061 | --------------------------------------------------------------------------------