├── regressionmetrics ├── __init__.py ├── __pycache__ │ ├── keras.cpython-38.pyc │ ├── __init__.cpython-38.pyc │ └── metrics.cpython-38.pyc ├── keras.py └── metrics.py ├── regressionmetrics.egg-info ├── zip-safe ├── dependency_links.txt ├── top_level.txt ├── requires.txt ├── SOURCES.txt └── PKG-INFO ├── MANIFEST.in ├── metrics_list.xlsx ├── dist ├── regressionmetrics-1.4.0.tar.gz └── regressionmetrics-1.4.0-py3-none-any.whl ├── examples ├── test.py ├── test_with_adjr2.py └── testkeras.py ├── CHANGLOG.txt ├── setup.py ├── LICENSE ├── README.md └── RegressionMetricsDemo.ipynb /regressionmetrics/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /regressionmetrics.egg-info/zip-safe: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.md -------------------------------------------------------------------------------- /regressionmetrics.egg-info/dependency_links.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /regressionmetrics.egg-info/top_level.txt: -------------------------------------------------------------------------------- 1 | regressionmetrics 2 | -------------------------------------------------------------------------------- /metrics_list.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashishpatel26/regressionmetrics/HEAD/metrics_list.xlsx -------------------------------------------------------------------------------- /regressionmetrics.egg-info/requires.txt: -------------------------------------------------------------------------------- 1 | pandas 2 | numpy 3 | scikit-learn 4 | tensorflow-cpu 5 | tensorflow-gpu 6 | -------------------------------------------------------------------------------- /dist/regressionmetrics-1.4.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashishpatel26/regressionmetrics/HEAD/dist/regressionmetrics-1.4.0.tar.gz -------------------------------------------------------------------------------- /dist/regressionmetrics-1.4.0-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashishpatel26/regressionmetrics/HEAD/dist/regressionmetrics-1.4.0-py3-none-any.whl -------------------------------------------------------------------------------- /regressionmetrics/__pycache__/keras.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashishpatel26/regressionmetrics/HEAD/regressionmetrics/__pycache__/keras.cpython-38.pyc -------------------------------------------------------------------------------- /regressionmetrics/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashishpatel26/regressionmetrics/HEAD/regressionmetrics/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /regressionmetrics/__pycache__/metrics.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ashishpatel26/regressionmetrics/HEAD/regressionmetrics/__pycache__/metrics.cpython-38.pyc -------------------------------------------------------------------------------- /regressionmetrics.egg-info/SOURCES.txt: -------------------------------------------------------------------------------- 1 | LICENSE 2 | MANIFEST.in 3 | README.md 4 | setup.py 5 | regressionmetrics/__init__.py 6 | regressionmetrics/keras.py 7 | regressionmetrics/metrics.py 8 | regressionmetrics.egg-info/PKG-INFO 9 | regressionmetrics.egg-info/SOURCES.txt 10 | regressionmetrics.egg-info/dependency_links.txt 11 | regressionmetrics.egg-info/requires.txt 12 | regressionmetrics.egg-info/top_level.txt 13 | regressionmetrics.egg-info/zip-safe -------------------------------------------------------------------------------- /examples/test.py: -------------------------------------------------------------------------------- 1 | from regressionmetrics.metrics import * 2 | y_true = [3, 0.5, 2, 7] 3 | y_pred = [2.5, 0.0, 2, -8] 4 | 5 | print("R2Score: ",R2CoefScore(y_true, y_pred)) 6 | print("Adjusted_R2_Score:",AdjR2CoefScore(y_true, y_pred)) 7 | print("RMSE:", RootMeanSqrtErr(y_true, y_pred)) 8 | print("MAE:",MeanAbsoErr(y_true, y_pred)) 9 | print("RMSLE with Neg Value:", RootMeanSqrtLogErrNeg(y_true, y_pred)) 10 | print("MSE:", MeanSqrtErr(y_true, y_pred)) 11 | print("MAPE: ", MeanAbsPercErr(y_true, y_pred)) 12 | -------------------------------------------------------------------------------- /CHANGLOG.txt: -------------------------------------------------------------------------------- 1 | - Changelog for v1.4.0 (2021-11-18) 2 | -------------------------------------------------- 3 | * Name clashes resolved with keras names 4 | 5 | - Changelog for v1.3.0 (2021-11-18) 6 | -------------------------------------------------- 7 | * new regresson metrics are added with details explaination 8 | 9 | - Changelog for v1.2.0 (2021-10-31) 10 | -------------------------------------------------- 11 | * Adjusted r2 score error solved 12 | 13 | - Changelog for v1.1.0 (2021-10-31) 14 | -------------------------------------------------- 15 | * SomeError solved 16 | 17 | - Changelog for v1.0.0 (2021-10-31) 18 | -------------------------------------------------- 19 | * regressionmetrics package first release 1.0.0. 20 | -------------------------------------------------------------------------------- /examples/test_with_adjr2.py: -------------------------------------------------------------------------------- 1 | from regressionmetrics.keras import * 2 | 3 | import pandas as pd 4 | import numpy as np 5 | 6 | import tensorflow as tf 7 | from tensorflow import keras 8 | from tensorflow.keras import layers 9 | 10 | (x_train, y_train), (x_test, y_test) = tf.keras.datasets.boston_housing.load_data(path="boston_housing.npz", test_split=0.2, seed=113) 11 | 12 | print(x_train.dtype, y_train.dtype, x_test.dtype, y_test.dtype) 13 | model = keras.Sequential([ 14 | layers.Dense(64, activation='relu', input_shape=(x_train.shape[1],)), 15 | layers.Dense(64, activation='relu'), 16 | layers.Dense(1) 17 | ]) 18 | model.compile(optimizer='rmsprop', loss='mae', metrics=[R2CoefScore, AdjR2CoefScore]) 19 | model.fit(x_train, y_train, epochs=10, batch_size=32, validation_data=(x_test, y_test)) -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | import os 3 | 4 | with open("README.md", encoding="utf8") as f: 5 | long_description = f.read() 6 | 7 | # Setting up 8 | setup( 9 | name="regressionmetrics", 10 | version="1.4.0", 11 | author="ashishpatel26", 12 | author_email="ashishpatel.ce.2011@gmail.com", 13 | description="Regression Metrics Calculation Made easy.", 14 | long_description_content_type="text/markdown", 15 | long_description=long_description, 16 | packages=find_packages(), 17 | install_requires=['pandas', 'numpy', 'scikit-learn', 'tensorflow-cpu', 'tensorflow-gpu'], 18 | keywords=['regressionmertics','regression', 'metrics', 'regression metrics', 'regression metrics calculation', 'regression metrics calculation made easy'], 19 | url='http://github.com/ashishpatel26/regressionmetrics', 20 | include_package_data=True, 21 | classifiers=[ 22 | "License :: OSI Approved :: MIT License", 23 | "Programming Language :: Python :: 3", 24 | ], 25 | platforms=["any"], 26 | zip_safe=True, 27 | ) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Ashish Patel 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /examples/testkeras.py: -------------------------------------------------------------------------------- 1 | from regressionmetrics.keras import * 2 | import pandas as pd 3 | import numpy as np 4 | 5 | import tensorflow as tf 6 | from tensorflow import keras 7 | from tensorflow.keras import layers 8 | 9 | (x_train, y_train), (x_test, y_test) = tf.keras.datasets.boston_housing.load_data(path="boston_housing.npz", test_split=0.2, seed=113) 10 | 11 | model = keras.Sequential([ 12 | layers.Dense(64, activation='relu', input_shape=(x_train.shape[1],)), 13 | layers.Dense(64, activation='relu'), 14 | layers.Dense(1) 15 | ]) 16 | model.compile(optimizer='rmsprop', loss='mse', metrics=[R2CoefScore, 17 | MeanAbsoErr, 18 | MeanSqrtErr, 19 | RootMeanSqrtErr, 20 | MeanAbsPercErr, 21 | RootMeanSqrtLogErr, 22 | NormRootMeanSqrtErr]) 23 | model.fit(x_train, y_train, epochs=10, batch_size=32, validation_data=(x_test, y_test)) -------------------------------------------------------------------------------- /regressionmetrics/keras.py: -------------------------------------------------------------------------------- 1 | import tensorflow.keras.backend as K 2 | import tensorflow as tf 3 | 4 | 5 | def MeanAbsoErr(y_true, y_pred): 6 | """ 7 | Mean absolute error regression loss. 8 | 9 | Args: 10 | y_true ([np.array]): test samples 11 | y_pred ([np.array]): predicted samples 12 | 13 | Returns: 14 | [float]: mean absolute error 15 | """ 16 | return K.mean(K.abs(y_pred - y_true), axis=-1) 17 | 18 | 19 | def MeanSqrtErr(y_true, y_pred): 20 | """ 21 | Mean squared error regression loss. 22 | 23 | Args: 24 | y_true ([np.array]): test samples 25 | y_pred ([np.array]): predicted samples 26 | 27 | Returns: 28 | [float]: mean squared error 29 | """ 30 | return K.mean(K.square(y_pred - y_true), axis=-1) 31 | 32 | 33 | def MeanAbsPercErr(y_true, y_pred): 34 | """ 35 | Mean absolute percentage error regression loss. 36 | 37 | Args: 38 | y_true ([np.array]): test samples 39 | y_pred ([np.array]): predicted samples 40 | 41 | Returns: 42 | [float]: mean absolute percentage error 43 | """ 44 | diff = K.abs((y_true - y_pred) / K.clip(K.abs(y_true), K.epsilon(), None)) 45 | return 100. * K.mean(diff, axis=-1) 46 | 47 | 48 | def MeanSqrtLogErr(y_true, y_pred): 49 | """ 50 | Mean squared logarithmic error regression loss. 51 | 52 | Args: 53 | y_true ([np.array]): test samples 54 | y_pred ([np.array]): predicted samples 55 | 56 | Returns: 57 | [float]: mean squared logarithmic error 58 | """ 59 | first_log = K.log(K.clip(y_pred, K.epsilon(), None) + 1.) 60 | second_log = K.log(K.clip(y_true, K.epsilon(), None) + 1.) 61 | return K.mean(K.square(first_log - second_log), axis=-1) 62 | 63 | 64 | def R2CoefScore(y_true, y_pred): 65 | """ 66 | :math:`R^2` (coefficient of determination) regression score function. 67 | 68 | Best possible score is 1.0, lower values are worse. 69 | 70 | Args: 71 | y_true ([np.array]): test samples 72 | y_pred ([np.array]): predicted samples 73 | 74 | Returns: 75 | [float]: R2 76 | """ 77 | SS_res = tf.reduce_sum(tf.square(y_true - y_pred), axis=-1) 78 | SS_tot = tf.reduce_sum( 79 | tf.square(y_true - tf.reduce_mean(y_true, axis=-1)), axis=-1) 80 | return (1 - SS_res/(SS_tot + tf.keras.backend.epsilon())) 81 | 82 | 83 | def AdjR2CoefScore(y_true, y_pred): 84 | """ 85 | Adjusted R2 regression score function with default inputs. 86 | 87 | Best possible score is 1.0, lower values are worse. 88 | 89 | Args: 90 | y_true ([np.array]): test samples 91 | y_pred ([np.array]): predicted samples 92 | 93 | Returns: 94 | [float]: adjusted R2 95 | """ 96 | SS_res = tf.reduce_sum(tf.square(y_true - y_pred), axis=-1) 97 | SS_tot = tf.reduce_sum( 98 | tf.square(y_true - tf.reduce_mean(y_true, axis=-1)), axis=-1) 99 | return (1 - SS_res/(SS_tot + tf.keras.backend.epsilon())) * (1 - (1 - R2CoefScore(y_true, y_pred)) * (tf.cast(tf.size(y_true), tf.float32) - 1) / (tf.cast(tf.size(y_true), tf.float32) - tf.cast(tf.rank(y_true), tf.float32) - 1)) 100 | # SS_res = tf.reduce_sum(tf.square(y_true - y_pred), axis=-1) 101 | # SS_tot = tf.reduce_sum(tf.square(y_true - tf.reduce_mean(y_true, axis=-1)), axis=-1) 102 | # adj_SS_res = tf.cast(SS_res / (K.shape(y_true)[0] - 1), tf.int32) 103 | # adj_SS_tot = tf.cast(SS_tot / (K.shape(y_true)[0] - 1), tf.int32) 104 | # return (1 - adj_SS_res/(adj_SS_tot + tf.keras.backend.epsilon())) 105 | 106 | 107 | def RootMeanSqrtLogErr(y_true, y_pred): 108 | """ 109 | Root Mean Squared Logarithm Error 110 | Args: 111 | y_true ([np.array]): test samples 112 | y_pred ([np.array]): predicted samples 113 | 114 | Returns: 115 | [float]: root mean squared logarithm error 116 | """ 117 | first_log = K.log(K.clip(y_pred, K.epsilon(), None) + 1.) 118 | second_log = K.log(K.clip(y_true, K.epsilon(), None) + 1.) 119 | return K.sqrt(K.mean(K.square(first_log - second_log), axis=-1)) 120 | 121 | 122 | def RootMeanSqrtErr(y_true, y_pred): 123 | """ 124 | Root Mean Squared Error 125 | Args: 126 | y_true ([np.array]): test samples 127 | y_pred ([np.array]): predicted samples 128 | 129 | Returns: 130 | [float]: root mean squared error 131 | """ 132 | return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1)) 133 | 134 | 135 | def SymMeanAbsPercErr(y_true, y_pred): 136 | """ 137 | Symmetric mean absolute percentage error regression loss. 138 | 139 | Args: 140 | y_true ([np.array]): test samples 141 | y_pred ([np.array]): predicted samples 142 | 143 | Returns: 144 | [float]: symmetric mean absolute percentage error 145 | """ 146 | diff = K.abs((y_true - y_pred) / K.clip(K.abs(y_true), K.epsilon(), None)) 147 | return 100. * K.mean(K.mean(diff, axis=-1)) 148 | 149 | 150 | def SymMeanAbsPercLogErr(y_true, y_pred): 151 | """ 152 | Symmetric mean absolute percentage log error regression loss. 153 | 154 | Args: 155 | y_true ([np.array]): test samples 156 | y_pred ([np.array]): predicted samples 157 | 158 | Returns: 159 | [float]: symmetric mean absolute percentage error 160 | """ 161 | diff = K.abs((y_true - y_pred) / K.clip(K.abs(y_true), K.epsilon(), None)) 162 | return K.log(K.mean(K.mean(diff, axis=-1))) 163 | 164 | 165 | def NormRootMeanSqrtErr(y_true, y_pred): 166 | """ 167 | Normalized Root Mean Squared Error 168 | Args: 169 | y_true ([np.array]): test samples 170 | y_pred ([np.array]): predicted samples 171 | 172 | Returns: 173 | [float]: normalized root mean squared error 174 | """ 175 | return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1)) / K.mean(K.abs(y_true), axis=-1) 176 | -------------------------------------------------------------------------------- /regressionmetrics.egg-info/PKG-INFO: -------------------------------------------------------------------------------- 1 | Metadata-Version: 2.1 2 | Name: regressionmetrics 3 | Version: 1.4.0 4 | Summary: Regression Metrics Calculation Made easy. 5 | Home-page: http://github.com/ashishpatel26/regressionmetrics 6 | Author: ashishpatel26 7 | Author-email: ashishpatel.ce.2011@gmail.com 8 | License: UNKNOWN 9 | Description: # Regression Metrics 10 | 11 | ## Installation 12 | 13 | To install the package from the PyPi repository you can execute the following 14 | command: 15 | ```sh 16 | pip install regressionmetrics 17 | ``` 18 | If you prefer, you can clone it and run the setup.py file. Use the following commands to get a copy from GitHub and install all dependencies: 19 | ```bash 20 | git clone https://github.com/ashishpatel26/regressionmetrics.git 21 | cd regressionmetrics 22 | pip install . 23 | ``` 24 | 25 | | Metrics | Full Form | Interpretation | Sklearn | Keras | 26 | | --------------------------- | ---------------------------------------------- | ----------------------------------- | ------- | ----- | 27 | | mae | Mean Absolute Error | Smaller is better (Best value is 0) | ☑️ | ☑️ | 28 | | mse | Mean Sqaured Error | Smaller is better(Best value is 0) | ☑️ | ☑️ | 29 | | rmse | Root Mean Square Error | Smaller is better(Best value is 0) | ☑️ | ☑️ | 30 | | rmsle | Root Mean Square Log Error | Smaller is better(Best value is 0) | ☑️ | ☑️ | 31 | | rmsle_with_negval | Root Mean Square Log Error with neg. value | Smaller is better(Best value is 0) | ☑️ | | 32 | | r2 score | coefficient of determination | Best possible score is 1 | ☑️ | ☑️ | 33 | | Adjusted r2 score | Adjusted R2 score | Best possible score is 1 | ☑️ | ☑️ | 34 | | mape | Mean Absolute Percentage Error | Smaller is better(Best value is 0) | ☑️ | ☑️ | 35 | | msle | Mean Sqaured Logarithm Error | Smaller is better(Best value is 0) | ☑️ | ☑️ | 36 | | smape | Symmetric mean absolute percentage error | Smaller is better(Best value is 0) | ☑️ | | 37 | | nrmse | Normalized Root Mean Square Error. | | ☑️ | ☑️ | 38 | | nrmsle | Normalized Root Mean Squared Logarithmic Error | | ☑️ | | 39 | | medianAE | Median Absolute Error | Smaller is better(Best value is 0) | ☑️ | | 40 | | mre | Mean Relative Error | Smaller is better(Best value is 0) | ☑️ | | 41 | | maape | Mean Arctangent Absolute Percentage Error | Smaller is better(Best value is 0) | ☑️ | | 42 | | nse | Nash-Sutcliffe Efficiency Coefficient | Larger is better (Best = 1) | ☑️ | | 43 | | willmott_index_of_agreement | Willmott Index | Larger is better (Best = 1) | ☑️ | | 44 | 45 | ## Usage 46 | 47 | > Usage with scikit learn : 48 | 49 | ```python 50 | from regressionmetrics.metrics import * 51 | 52 | y_true = np.array([3, 0.5, 2, 7]) 53 | y_pred = np.array([2.5, 0.0, 2, -8]) 54 | 55 | 56 | print("R2Score: ",r2(y_true, y_pred)) 57 | print("Adjusted_R2_Score:",adj_r2(y_true, y_pred)) 58 | print("RMSE:", rmse(y_true, y_pred)) 59 | print("MAE:",mae(y_true, y_pred)) 60 | print("RMSLE with Neg Value:", rmsle_with_negval(y_true, y_pred)) 61 | print("MSE:", mse(y_true, y_pred)) 62 | print("MAPE: ", mape(y_true, y_pred)) 63 | ``` 64 | > Usage with Tensorflow keras: 65 | 66 | ```python 67 | from regressionmetrics.keras import * 68 | import pandas as pd 69 | import numpy as np 70 | 71 | import tensorflow as tf 72 | from tensorflow import keras 73 | from tensorflow.keras import layers 74 | 75 | (x_train, y_train), (x_test, y_test) = tf.keras.datasets.boston_housing.load_data(path="boston_housing.npz", test_split=0.2, seed=113) 76 | 77 | model = keras.Sequential([ 78 | layers.Dense(64, activation='relu', input_shape=(x_train.shape[1],)), 79 | layers.Dense(64, activation='relu'), 80 | layers.Dense(1) 81 | ]) 82 | model.compile(optimizer='rmsprop', loss='mse', metrics=[r2, mae, mse, rmse, mape, rmsle, nrmse]) 83 | model.fit(x_train, y_train, epochs=10, batch_size=32, validation_data=(x_test, y_test)) 84 | ``` 85 | 86 | ```bash 87 | Epoch 1/10 88 | 1/13 [=>............................] - ETA: 7s - loss: 1574.7567 - r2: 0.6597 - mae: 37.1803 - mse: 1574.7567 - rmse: 37.1802 - mape: 159.261313/13 [==============================] - 1s 15ms/step - loss: 270.0653 - r2: 0.9472 - mae: 11.5427 - mse: 270.0653 - rmse: 11.5427 - mape: 57.3519 - rmsle: 0.6445 - nrmse: 0.5735 - val_loss: 88.6351 - val_r2: 0.9727 - val_mae: 6.6028 - val_mse: 88.6351 - val_rmse: 6.6028 - val_mape: 29.6502 - val_rmsle: 0.3161 - val_nrmse: 0.2965 89 | Epoch 2/10 90 | 1/13 [=>............................] - ETA: 0s - loss: 74.6623 - r2: 0.9913 - mae: 5.5958 - mse: 74.6623 - rmse: 5.5958 - mape: 25.3655 - rmsl13/13 [==============================] - 0s 3ms/step - loss: 87.1876 - r2: 0.9856 - mae: 6.9466 - mse: 87.1876 - rmse: 6.9466 - mape: 33.4256 - rmsle: 0.3057 - nrmse: 0.3343 - val_loss: 81.7884 - val_r2: 0.9712 - val_mae: 6.6424 - val_mse: 81.7884 - val_rmse: 6.6424 - val_mape: 28.8687 - val_rmsle: 0.3334 - val_nrmse: 0.2887 91 | Epoch 3/10 92 | 1/13 [=>............................] - ETA: 0s - loss: 41.2790 - r2: 0.9722 - mae: 5.3798 - mse: 41.2790 - rmse: 5.3798 - mape: 28.7497 - rmsl13/13 [==============================] - 0s 3ms/step - loss: 103.6462 - r2: 0.9825 - mae: 7.1041 - mse: 103.6462 - rmse: 7.1041 - mape: 34.6278 - rmsle: 0.3231 - nrmse: 0.3463 - val_loss: 71.7539 - val_r2: 0.9769 - val_mae: 6.1455 - val_mse: 71.7539 - val_rmse: 6.1455 - val_mape: 27.5078 - val_rmsle: 0.2893 - val_nrmse: 0.2751 93 | Epoch 4/10 94 | 1/13 [=>............................] - ETA: 0s - loss: 113.6758 - r2: 0.9917 - mae: 6.6575 - mse: 113.6758 - rmse: 6.6575 - mape: 20.8683 - rm13/13 [==============================] - 0s 3ms/step - loss: 88.1601 - r2: 0.9823 - mae: 6.8479 - mse: 88.1601 - rmse: 6.8479 - mape: 32.5867 - rmsle: 0.3080 - nrmse: 0.3259 - val_loss: 63.3707 - val_r2: 0.9829 - val_mae: 6.0845 - val_mse: 63.3707 - val_rmse: 6.0845 - val_mape: 33.1628 - val_rmsle: 0.2747 - val_nrmse: 0.3316 95 | Epoch 5/10 96 | 1/13 [=>............................] - ETA: 0s - loss: 85.8188 - r2: 0.9893 - mae: 7.0097 - mse: 85.8188 - rmse: 7.0097 - mape: 34.8362 - rmsl13/13 [==============================] - 0s 3ms/step - loss: 82.3233 - r2: 0.9860 - mae: 6.5795 - mse: 82.3233 - rmse: 6.5795 - mape: 32.5198 - rmsle: 0.3105 - nrmse: 0.3252 - val_loss: 74.4783 - val_r2: 0.9813 - val_mae: 6.8936 - val_mse: 74.4783 - val_rmse: 6.8936 - val_mape: 41.9492 - val_rmsle: 0.3067 - val_nrmse: 0.4195 97 | Epoch 7/10 98 | 1/13 [=>............................] - ETA: 0s - loss: 105.6430 - r2: 0.9658 - mae: 9.4737 - mse: 105.6430 - rmse: 9.4737 - mape: 53.0854 - rm13/13 [==============================] - 0s 3ms/step - loss: 76.0740 - r2: 0.9856 - mae: 6.4234 - mse: 76.0740 - rmse: 6.4234 - mape: 31.8728 - rmsle: 0.2828 - nrmse: 0.3187 - val_loss: 104.1779 - val_r2: 0.9679 - val_mae: 7.5539 - val_mse: 104.1779 - val_rmse: 7.5539 - val_mape: 30.9401 - val_rmsle: 0.3692 - val_nrmse: 0.3094 99 | Epoch 8/10 100 | 1/13 [=>............................] - ETA: 0s - loss: 100.0114 - r2: 0.9833 - mae: 6.8492 - mse: 100.0114 - rmse: 6.8492 - mape: 27.9621 - rm13/13 [==============================] - 0s 4ms/step - loss: 68.4268 - r2: 0.9892 - mae: 5.9540 - mse: 68.4268 - rmse: 5.9540 - mape: 29.7586 - rmsle: 0.2623 - nrmse: 0.2976 - val_loss: 171.7968 - val_r2: 0.9412 - val_mae: 10.5855 - val_mse: 171.7968 - val_rmse: 10.5855 - val_mape: 47.9010 - val_rmsle: 0.7561 - val_nrmse: 0.4790 101 | Epoch 9/10 102 | 1/13 [=>............................] - ETA: 0s - loss: 291.8670 - r2: 0.9725 - mae: 13.9899 - mse: 291.8670 - rmse: 13.9899 - mape: 61.3658 - 13/13 [==============================] - 0s 3ms/step - loss: 92.3889 - r2: 0.9796 - mae: 6.8932 - mse: 92.3889 - rmse: 6.8932 - mape: 33.2856 - rmsle: 0.3333 - nrmse: 0.3329 - val_loss: 67.2208 - val_r2: 0.9808 - val_mae: 5.8498 - val_mse: 67.2208 - val_rmse: 5.8498 - val_mape: 26.4504 - val_rmsle: 0.2680 - val_nrmse: 0.2645 103 | Epoch 10/10 104 | 1/13 [=>............................] - ETA: 0s - loss: 97.0853 - r2: 0.9923 - mae: 5.9866 - mse: 97.0853 - rmse: 5.9866 - mape: 24.9878 - rmsl13/13 [==============================] - 0s 3ms/step - loss: 78.3823 - r2: 0.9856 - mae: 6.5958 - mse: 78.3823 - rmse: 6.5958 - mape: 32.8136 - rmsle: 0.3025 - nrmse: 0.3281 - val_loss: 69.5314 - val_r2: 0.9787 - val_mae: 6.8302 - val_mse: 69.5314 - val_rmse: 6.8302 - val_mape: 37.3933 - val_rmsle: 0.2974 - val_nrmse: 0.3739 105 | ``` 106 | 107 | :smiley: Thanks for reading and forking. 108 | --- 109 | 110 | Keywords: regressionmertics,regression,metrics,regression metrics,regression metrics calculation,regression metrics calculation made easy 111 | Platform: any 112 | Classifier: License :: OSI Approved :: MIT License 113 | Classifier: Programming Language :: Python :: 3 114 | Description-Content-Type: text/markdown 115 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Regression Metrics 2 | 3 | ## Installation 4 | 5 | To install the package from the PyPi repository you can execute the following 6 | command: 7 | ```bash 8 | pip install regressionmetrics 9 | ``` 10 | If you prefer, you can clone it and run the setup.py file. Use the following commands to get a copy from GitHub and install all dependencies: 11 | ```bash 12 | git clone https://github.com/ashishpatel26/regressionmetrics.git 13 | cd regressionmetrics 14 | pip install . 15 | ``` 16 | 17 | | Metrics | Full Form | Interpretation | Sklearn | Keras | 18 | | --------------------------- | ---------------------------------------------- | ----------------------------------- | ------- | ----- | 19 | | **MeanAbsoErr** | Mean Absolute Error | Smaller is better (Best value is 0) | ☑️ | ☑️ | 20 | | **MeanSqrtErr** | Mean Sqaured Error | Smaller is better(Best value is 0) | ☑️ | ☑️ | 21 | | **RootMeanSqrtErr** | Root Mean Square Error | Smaller is better(Best value is 0) | ☑️ | ☑️ | 22 | | **RootMeanSqrtLogErr** | Root Mean Square Log Error | Smaller is better(Best value is 0) | ☑️ | ☑️ | 23 | | **RootMeanSqrtLogErrNeg** | Root Mean Square Log Error with neg. value | Smaller is better(Best value is 0) | ☑️ | | 24 | | **R2CoefScore** | coefficient of determination | Best possible score is 1 | ☑️ | ☑️ | 25 | | **AdjR2CoefScore** | Adjusted R2 score | Best possible score is 1 | ☑️ | ☑️ | 26 | | **MeanAbsPercErr** | Mean Absolute Percentage Error | Smaller is better(Best value is 0) | ☑️ | ☑️ | 27 | | **MeanSqrtLogErr** | Mean Sqaured Logarithm Error | Smaller is better(Best value is 0) | ☑️ | ☑️ | 28 | | **SymMeanAbsPercErr** | Symmetric mean absolute percentage error | Smaller is better(Best value is 0) | ☑️ | | 29 | | **NormRootMeanSqrtErr** | Normalized Root Mean Square Error. | | ☑️ | ☑️ | 30 | | **NormRootMeanSqrtLogErr** | Normalized Root Mean Squared Logarithmic Error | | ☑️ | | 31 | | **MedianAbsErr** | Median Absolute Error | Smaller is better(Best value is 0) | ☑️ | | 32 | | **MediaRelErr** | Mean Relative Error | Smaller is better(Best value is 0) | ☑️ | | 33 | | **MeanArcAbsPercErr** | Mean Arctangent Absolute Percentage Error | Smaller is better(Best value is 0) | ☑️ | | 34 | | **NashSutCoeff** | Nash-Sutcliffe Efficiency Coefficient | Larger is better (Best = 1) | ☑️ | | 35 | | **WillMottIndexAgreeMent** | Willmott Index | Larger is better (Best = 1) | ☑️ | | 36 | 37 | * Mean Absolute Error - `sklearn, keras` 38 | 39 | * Mean Square Error - `sklearn, keras` 40 | * Root Mean Square Error - `sklearn, keras` 41 | * Root Mean Square Logarithmic Error - `sklearn, keras` 42 | * Root Mean Square Logarithmic Error with negative value handle - `sklearn` 43 | * R2 Score - `sklearn, keras` 44 | * Adjusted R2 Score - `sklearn, keras` 45 | * Mean Absolute Percentage Error - `sklearn, keras` 46 | * Mean squared logarithmic Error - `sklearn, keras` 47 | * Symmetric mean absolute percentage error - `sklearn, keras` 48 | * Normalized Root Mean Squared Error - `sklearn, keras` 49 | 50 | Colaboratory File : [![Open In Colab](https://camo.githubusercontent.com/84f0493939e0c4de4e6dbe113251b4bfb5353e57134ffd9fcab6b8714514d4d1/68747470733a2f2f636f6c61622e72657365617263682e676f6f676c652e636f6d2f6173736574732f636f6c61622d62616467652e737667)](https://colab.research.google.com/github/ashishpatel26/regressionmetrics/blob/main/RegressionMetricsDemo.ipynb) 51 | 52 | ## Usage 53 | 54 | > **Usage with scikit learn :** 55 | 56 | ```python 57 | from regressionmetrics.metrics import * 58 | y_true = [3, 0.5, 2, 7] 59 | y_pred = [2.5, 0.0, 2, -8] 60 | 61 | print("R2Score: ",R2CoefScore(y_true, y_pred)) 62 | print("Adjusted_R2_Score:",AdjR2CoefScore(y_true, y_pred)) 63 | print("RMSE:", RootMeanSqrtErr(y_true, y_pred)) 64 | print("MAE:",MeanAbsoErr(y_true, y_pred)) 65 | print("RMSLE with Neg Value:", RootMeanSqrtLogErrNeg(y_true, y_pred)) 66 | print("MSE:", MeanSqrtErr(y_true, y_pred)) 67 | print("MAPE: ", MeanAbsPercErr(y_true, y_pred)) 68 | ``` 69 | **Output:** 70 | 71 | ```bash 72 | R2Score: -8.725067385444744 73 | Adjusted_R2_Score: 20.450134770889484 74 | RMSE: 7.508328708840604 75 | MAE: 4.0 76 | RMSLE with Neg Value: 0.21344354447336292 77 | MSE: 56.375 78 | MAPE: 0.8273809523809523 79 | ``` 80 | 81 | > **Usage with TensorFlow keras:** 82 | 83 | ```python 84 | try: 85 | from regressionmetrics.keras import * 86 | except: 87 | import os 88 | os.system("pip install regressionmetrics") 89 | 90 | from regressionmetrics.keras import * 91 | import pandas as pd 92 | import numpy as np 93 | 94 | import tensorflow as tf 95 | from tensorflow import keras 96 | from tensorflow.keras import layers 97 | 98 | (x_train, y_train), (x_test, y_test) = tf.keras.datasets.boston_housing.load_data(path="boston_housing.npz", test_split=0.2, seed=113) 99 | 100 | model = keras.Sequential([ 101 | layers.Dense(64, activation='relu', input_shape=(x_train.shape[1],)), 102 | layers.Dense(64, activation='relu'), 103 | layers.Dense(1) 104 | ]) 105 | model.compile(optimizer='rmsprop', loss='mse', metrics=[R2CoefScore, 106 | MeanAbsoErr, 107 | MeanSqrtErr, 108 | RootMeanSqrtErr, 109 | MeanAbsPercErr, 110 | RootMeanSqrtLogErr, 111 | NormRootMeanSqrtErr]) 112 | model.fit(x_train, y_train, epochs=10, batch_size=32, validation_data=(x_test, y_test)) 113 | ``` 114 | **Output** 115 | 116 | ```bash 117 | Epoch 1/10 118 | 13/13 [==============================] - 2s 29ms/step - loss: 461.6622 - R2CoefScore: 0.9004 - MeanAbsoErr: 14.6653 - MeanSqrtErr: 461.6622 - RootMeanSqrtErr: 14.6653 - MeanAbsPercErr: 75.2677 - RootMeanSqrtLogErr: 0.7278 - NormRootMeanSqrtErr: 0.7527 - val_loss: 300.4463 - val_R2CoefScore: 0.8947 - val_MeanAbsoErr: 15.4050 - val_MeanSqrtErr: 300.4463 - val_RootMeanSqrtErr: 15.4050 - val_MeanAbsPercErr: 69.2703 - val_RootMeanSqrtLogErr: 1.2662 - val_NormRootMeanSqrtErr: 0.6927 119 | Epoch 2/10 120 | 13/13 [==============================] - 0s 4ms/step - loss: 184.7860 - R2CoefScore: 0.9527 - MeanAbsoErr: 10.9894 - MeanSqrtErr: 184.7860 - RootMeanSqrtErr: 10.9894 - MeanAbsPercErr: 56.5819 - RootMeanSqrtLogErr: 0.5995 - NormRootMeanSqrtErr: 0.5658 - val_loss: 305.9124 - val_R2CoefScore: 0.8910 - val_MeanAbsoErr: 15.4291 - val_MeanSqrtErr: 305.9124 - val_RootMeanSqrtErr: 15.4291 - val_MeanAbsPercErr: 71.9620 - val_RootMeanSqrtLogErr: 1.3943 - val_NormRootMeanSqrtErr: 0.7196 121 | Epoch 3/10 122 | 13/13 [==============================] - 0s 5ms/step - loss: 198.5649 - R2CoefScore: 0.9507 - MeanAbsoErr: 12.0198 - MeanSqrtErr: 198.5649 - RootMeanSqrtErr: 12.0198 - MeanAbsPercErr: 62.6733 - RootMeanSqrtLogErr: 0.6901 - NormRootMeanSqrtErr: 0.6267 - val_loss: 80.2263 - val_R2CoefScore: 0.9807 - val_MeanAbsoErr: 7.0446 - val_MeanSqrtErr: 80.2263 - val_RootMeanSqrtErr: 7.0446 - val_MeanAbsPercErr: 43.2890 - val_RootMeanSqrtLogErr: 0.3114 - val_NormRootMeanSqrtErr: 0.4329 123 | Epoch 4/10 124 | 13/13 [==============================] - 0s 6ms/step - loss: 197.9205 - R2CoefScore: 0.9613 - MeanAbsoErr: 10.8593 - MeanSqrtErr: 197.9205 - RootMeanSqrtErr: 10.8593 - MeanAbsPercErr: 56.8981 - RootMeanSqrtLogErr: 0.5830 - NormRootMeanSqrtErr: 0.5690 - val_loss: 139.6424 - val_R2CoefScore: 0.9512 - val_MeanAbsoErr: 9.2244 - val_MeanSqrtErr: 139.6424 - val_RootMeanSqrtErr: 9.2244 - val_MeanAbsPercErr: 38.9547 - val_RootMeanSqrtLogErr: 0.5582 - val_NormRootMeanSqrtErr: 0.3895 125 | Epoch 5/10 126 | 13/13 [==============================] - 0s 4ms/step - loss: 164.3372 - R2CoefScore: 0.9641 - MeanAbsoErr: 10.6009 - MeanSqrtErr: 164.3372 - RootMeanSqrtErr: 10.6009 - MeanAbsPercErr: 55.5600 - RootMeanSqrtLogErr: 0.5740 - NormRootMeanSqrtErr: 0.5556 - val_loss: 142.1380 - val_R2CoefScore: 0.9564 - val_MeanAbsoErr: 10.7172 - val_MeanSqrtErr: 142.1380 - val_RootMeanSqrtErr: 10.7172 - val_MeanAbsPercErr: 63.0724 - val_RootMeanSqrtLogErr: 0.4243 - val_NormRootMeanSqrtErr: 0.6307 127 | Epoch 6/10 128 | 13/13 [==============================] - 0s 5ms/step - loss: 176.5649 - R2CoefScore: 0.9584 - MeanAbsoErr: 11.0135 - MeanSqrtErr: 176.5649 - RootMeanSqrtErr: 11.0135 - MeanAbsPercErr: 56.6267 - RootMeanSqrtLogErr: 0.5719 - NormRootMeanSqrtErr: 0.5663 - val_loss: 217.2575 - val_R2CoefScore: 0.9235 - val_MeanAbsoErr: 12.4566 - val_MeanSqrtErr: 217.2575 - val_RootMeanSqrtErr: 12.4566 - val_MeanAbsPercErr: 55.4557 - val_RootMeanSqrtLogErr: 0.9559 - val_NormRootMeanSqrtErr: 0.5546 129 | Epoch 7/10 130 | 13/13 [==============================] - 0s 4ms/step - loss: 157.5359 - R2CoefScore: 0.9567 - MeanAbsoErr: 9.5872 - MeanSqrtErr: 157.5359 - RootMeanSqrtErr: 9.5872 - MeanAbsPercErr: 50.5483 - RootMeanSqrtLogErr: 0.5250 - NormRootMeanSqrtErr: 0.5055 - val_loss: 411.2795 - val_R2CoefScore: 0.8542 - val_MeanAbsoErr: 18.6303 - val_MeanSqrtErr: 411.2795 - val_RootMeanSqrtErr: 18.6303 - val_MeanAbsPercErr: 85.9467 - val_RootMeanSqrtLogErr: 1.6382 - val_NormRootMeanSqrtErr: 0.8595 131 | Epoch 8/10 132 | 13/13 [==============================] - 0s 4ms/step - loss: 115.8139 - R2CoefScore: 0.9795 - MeanAbsoErr: 7.9076 - MeanSqrtErr: 115.8139 - RootMeanSqrtErr: 7.9076 - MeanAbsPercErr: 39.5189 - RootMeanSqrtLogErr: 0.3936 - NormRootMeanSqrtErr: 0.3952 - val_loss: 72.1911 - val_R2CoefScore: 0.9813 - val_MeanAbsoErr: 6.7830 - val_MeanSqrtErr: 72.1911 - val_RootMeanSqrtErr: 6.7830 - val_MeanAbsPercErr: 40.6487 - val_RootMeanSqrtLogErr: 0.2993 - val_NormRootMeanSqrtErr: 0.4065 133 | Epoch 9/10 134 | 13/13 [==============================] - 0s 5ms/step - loss: 214.5103 - R2CoefScore: 0.9397 - MeanAbsoErr: 10.9144 - MeanSqrtErr: 214.5103 - RootMeanSqrtErr: 10.9144 - MeanAbsPercErr: 56.2217 - RootMeanSqrtLogErr: 0.5520 - NormRootMeanSqrtErr: 0.5622 - val_loss: 87.2555 - val_R2CoefScore: 0.9733 - val_MeanAbsoErr: 6.8626 - val_MeanSqrtErr: 87.2555 - val_RootMeanSqrtErr: 6.8626 - val_MeanAbsPercErr: 28.4989 - val_RootMeanSqrtLogErr: 0.3236 - val_NormRootMeanSqrtErr: 0.2850 135 | Epoch 10/10 136 | 13/13 [==============================] - 0s 6ms/step - loss: 159.1116 - R2CoefScore: 0.9662 - MeanAbsoErr: 9.1501 - MeanSqrtErr: 159.1116 - RootMeanSqrtErr: 9.1501 - MeanAbsPercErr: 46.7719 - RootMeanSqrtLogErr: 0.5018 - NormRootMeanSqrtErr: 0.4677 - val_loss: 69.8977 - val_R2CoefScore: 0.9841 - val_MeanAbsoErr: 6.0780 - val_MeanSqrtErr: 69.8977 - val_RootMeanSqrtErr: 6.0780 - val_MeanAbsPercErr: 32.4612 - val_RootMeanSqrtLogErr: 0.2741 - val_NormRootMeanSqrtErr: 0.3246 137 | 138 | ``` 139 | 140 | :smiley: Thanks for reading and forking. 141 | --- 142 | -------------------------------------------------------------------------------- /RegressionMetricsDemo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "name": "RegressionMetricsDemo.ipynb", 7 | "provenance": [] 8 | }, 9 | "kernelspec": { 10 | "name": "python3", 11 | "display_name": "Python 3" 12 | }, 13 | "language_info": { 14 | "name": "python" 15 | }, 16 | "accelerator": "GPU" 17 | }, 18 | "cells": [ 19 | { 20 | "cell_type": "code", 21 | "execution_count": 2, 22 | "metadata": { 23 | "colab": { 24 | "base_uri": "https://localhost:8080/" 25 | }, 26 | "id": "6wxtEPndx0g6", 27 | "outputId": "849ec6f8-33ff-4f4c-b152-a0c67ac03972" 28 | }, 29 | "outputs": [ 30 | { 31 | "output_type": "stream", 32 | "name": "stdout", 33 | "text": [ 34 | "Epoch 1/10\n", 35 | "13/13 [==============================] - 2s 29ms/step - loss: 461.6622 - R2CoefScore: 0.9004 - MeanAbsoErr: 14.6653 - MeanSqrtErr: 461.6622 - RootMeanSqrtErr: 14.6653 - MeanAbsPercErr: 75.2677 - RootMeanSqrtLogErr: 0.7278 - NormRootMeanSqrtErr: 0.7527 - val_loss: 300.4463 - val_R2CoefScore: 0.8947 - val_MeanAbsoErr: 15.4050 - val_MeanSqrtErr: 300.4463 - val_RootMeanSqrtErr: 15.4050 - val_MeanAbsPercErr: 69.2703 - val_RootMeanSqrtLogErr: 1.2662 - val_NormRootMeanSqrtErr: 0.6927\n", 36 | "Epoch 2/10\n", 37 | "13/13 [==============================] - 0s 4ms/step - loss: 184.7860 - R2CoefScore: 0.9527 - MeanAbsoErr: 10.9894 - MeanSqrtErr: 184.7860 - RootMeanSqrtErr: 10.9894 - MeanAbsPercErr: 56.5819 - RootMeanSqrtLogErr: 0.5995 - NormRootMeanSqrtErr: 0.5658 - val_loss: 305.9124 - val_R2CoefScore: 0.8910 - val_MeanAbsoErr: 15.4291 - val_MeanSqrtErr: 305.9124 - val_RootMeanSqrtErr: 15.4291 - val_MeanAbsPercErr: 71.9620 - val_RootMeanSqrtLogErr: 1.3943 - val_NormRootMeanSqrtErr: 0.7196\n", 38 | "Epoch 3/10\n", 39 | "13/13 [==============================] - 0s 5ms/step - loss: 198.5649 - R2CoefScore: 0.9507 - MeanAbsoErr: 12.0198 - MeanSqrtErr: 198.5649 - RootMeanSqrtErr: 12.0198 - MeanAbsPercErr: 62.6733 - RootMeanSqrtLogErr: 0.6901 - NormRootMeanSqrtErr: 0.6267 - val_loss: 80.2263 - val_R2CoefScore: 0.9807 - val_MeanAbsoErr: 7.0446 - val_MeanSqrtErr: 80.2263 - val_RootMeanSqrtErr: 7.0446 - val_MeanAbsPercErr: 43.2890 - val_RootMeanSqrtLogErr: 0.3114 - val_NormRootMeanSqrtErr: 0.4329\n", 40 | "Epoch 4/10\n", 41 | "13/13 [==============================] - 0s 6ms/step - loss: 197.9205 - R2CoefScore: 0.9613 - MeanAbsoErr: 10.8593 - MeanSqrtErr: 197.9205 - RootMeanSqrtErr: 10.8593 - MeanAbsPercErr: 56.8981 - RootMeanSqrtLogErr: 0.5830 - NormRootMeanSqrtErr: 0.5690 - val_loss: 139.6424 - val_R2CoefScore: 0.9512 - val_MeanAbsoErr: 9.2244 - val_MeanSqrtErr: 139.6424 - val_RootMeanSqrtErr: 9.2244 - val_MeanAbsPercErr: 38.9547 - val_RootMeanSqrtLogErr: 0.5582 - val_NormRootMeanSqrtErr: 0.3895\n", 42 | "Epoch 5/10\n", 43 | "13/13 [==============================] - 0s 4ms/step - loss: 164.3372 - R2CoefScore: 0.9641 - MeanAbsoErr: 10.6009 - MeanSqrtErr: 164.3372 - RootMeanSqrtErr: 10.6009 - MeanAbsPercErr: 55.5600 - RootMeanSqrtLogErr: 0.5740 - NormRootMeanSqrtErr: 0.5556 - val_loss: 142.1380 - val_R2CoefScore: 0.9564 - val_MeanAbsoErr: 10.7172 - val_MeanSqrtErr: 142.1380 - val_RootMeanSqrtErr: 10.7172 - val_MeanAbsPercErr: 63.0724 - val_RootMeanSqrtLogErr: 0.4243 - val_NormRootMeanSqrtErr: 0.6307\n", 44 | "Epoch 6/10\n", 45 | "13/13 [==============================] - 0s 5ms/step - loss: 176.5649 - R2CoefScore: 0.9584 - MeanAbsoErr: 11.0135 - MeanSqrtErr: 176.5649 - RootMeanSqrtErr: 11.0135 - MeanAbsPercErr: 56.6267 - RootMeanSqrtLogErr: 0.5719 - NormRootMeanSqrtErr: 0.5663 - val_loss: 217.2575 - val_R2CoefScore: 0.9235 - val_MeanAbsoErr: 12.4566 - val_MeanSqrtErr: 217.2575 - val_RootMeanSqrtErr: 12.4566 - val_MeanAbsPercErr: 55.4557 - val_RootMeanSqrtLogErr: 0.9559 - val_NormRootMeanSqrtErr: 0.5546\n", 46 | "Epoch 7/10\n", 47 | "13/13 [==============================] - 0s 4ms/step - loss: 157.5359 - R2CoefScore: 0.9567 - MeanAbsoErr: 9.5872 - MeanSqrtErr: 157.5359 - RootMeanSqrtErr: 9.5872 - MeanAbsPercErr: 50.5483 - RootMeanSqrtLogErr: 0.5250 - NormRootMeanSqrtErr: 0.5055 - val_loss: 411.2795 - val_R2CoefScore: 0.8542 - val_MeanAbsoErr: 18.6303 - val_MeanSqrtErr: 411.2795 - val_RootMeanSqrtErr: 18.6303 - val_MeanAbsPercErr: 85.9467 - val_RootMeanSqrtLogErr: 1.6382 - val_NormRootMeanSqrtErr: 0.8595\n", 48 | "Epoch 8/10\n", 49 | "13/13 [==============================] - 0s 4ms/step - loss: 115.8139 - R2CoefScore: 0.9795 - MeanAbsoErr: 7.9076 - MeanSqrtErr: 115.8139 - RootMeanSqrtErr: 7.9076 - MeanAbsPercErr: 39.5189 - RootMeanSqrtLogErr: 0.3936 - NormRootMeanSqrtErr: 0.3952 - val_loss: 72.1911 - val_R2CoefScore: 0.9813 - val_MeanAbsoErr: 6.7830 - val_MeanSqrtErr: 72.1911 - val_RootMeanSqrtErr: 6.7830 - val_MeanAbsPercErr: 40.6487 - val_RootMeanSqrtLogErr: 0.2993 - val_NormRootMeanSqrtErr: 0.4065\n", 50 | "Epoch 9/10\n", 51 | "13/13 [==============================] - 0s 5ms/step - loss: 214.5103 - R2CoefScore: 0.9397 - MeanAbsoErr: 10.9144 - MeanSqrtErr: 214.5103 - RootMeanSqrtErr: 10.9144 - MeanAbsPercErr: 56.2217 - RootMeanSqrtLogErr: 0.5520 - NormRootMeanSqrtErr: 0.5622 - val_loss: 87.2555 - val_R2CoefScore: 0.9733 - val_MeanAbsoErr: 6.8626 - val_MeanSqrtErr: 87.2555 - val_RootMeanSqrtErr: 6.8626 - val_MeanAbsPercErr: 28.4989 - val_RootMeanSqrtLogErr: 0.3236 - val_NormRootMeanSqrtErr: 0.2850\n", 52 | "Epoch 10/10\n", 53 | "13/13 [==============================] - 0s 6ms/step - loss: 159.1116 - R2CoefScore: 0.9662 - MeanAbsoErr: 9.1501 - MeanSqrtErr: 159.1116 - RootMeanSqrtErr: 9.1501 - MeanAbsPercErr: 46.7719 - RootMeanSqrtLogErr: 0.5018 - NormRootMeanSqrtErr: 0.4677 - val_loss: 69.8977 - val_R2CoefScore: 0.9841 - val_MeanAbsoErr: 6.0780 - val_MeanSqrtErr: 69.8977 - val_RootMeanSqrtErr: 6.0780 - val_MeanAbsPercErr: 32.4612 - val_RootMeanSqrtLogErr: 0.2741 - val_NormRootMeanSqrtErr: 0.3246\n" 54 | ] 55 | }, 56 | { 57 | "output_type": "execute_result", 58 | "data": { 59 | "text/plain": [ 60 | "" 61 | ] 62 | }, 63 | "metadata": {}, 64 | "execution_count": 2 65 | } 66 | ], 67 | "source": [ 68 | "try:\n", 69 | " from regressionmetrics.keras import *\n", 70 | "except:\n", 71 | " import os\n", 72 | " os.system(\"pip install regressionmetrics\")\n", 73 | "\n", 74 | "from regressionmetrics.keras import *\n", 75 | "import pandas as pd\n", 76 | "import numpy as np\n", 77 | "\n", 78 | "import tensorflow as tf\n", 79 | "from tensorflow import keras\n", 80 | "from tensorflow.keras import layers\n", 81 | "\n", 82 | "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.boston_housing.load_data(path=\"boston_housing.npz\", test_split=0.2, seed=113)\n", 83 | "\n", 84 | "model = keras.Sequential([\n", 85 | " layers.Dense(64, activation='relu', input_shape=(x_train.shape[1],)),\n", 86 | " layers.Dense(64, activation='relu'),\n", 87 | " layers.Dense(1)\n", 88 | "])\n", 89 | "model.compile(optimizer='rmsprop', loss='mse', metrics=[R2CoefScore, \n", 90 | " MeanAbsoErr, \n", 91 | " MeanSqrtErr, \n", 92 | " RootMeanSqrtErr, \n", 93 | " MeanAbsPercErr, \n", 94 | " RootMeanSqrtLogErr, \n", 95 | " NormRootMeanSqrtErr])\n", 96 | "model.fit(x_train, y_train, epochs=10, batch_size=32, validation_data=(x_test, y_test))" 97 | ] 98 | }, 99 | { 100 | "cell_type": "markdown", 101 | "source": [ 102 | "### Adjusted R2 Score Checking" 103 | ], 104 | "metadata": { 105 | "id": "sdrNkoNv76MP" 106 | } 107 | }, 108 | { 109 | "cell_type": "code", 110 | "source": [ 111 | "from regressionmetrics.keras import *\n", 112 | "\n", 113 | "import pandas as pd\n", 114 | "import numpy as np\n", 115 | "\n", 116 | "import tensorflow as tf\n", 117 | "from tensorflow import keras\n", 118 | "from tensorflow.keras import layers\n", 119 | "\n", 120 | "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.boston_housing.load_data(path=\"boston_housing.npz\", test_split=0.2, seed=113)\n", 121 | "\n", 122 | "print(x_train.dtype, y_train.dtype, x_test.dtype, y_test.dtype)\n", 123 | "model = keras.Sequential([\n", 124 | " layers.Dense(64, activation='relu', input_shape=(x_train.shape[1],)),\n", 125 | " layers.Dense(64, activation='relu'),\n", 126 | " layers.Dense(1)\n", 127 | "])\n", 128 | "model.compile(optimizer='rmsprop', loss='mae', metrics=[R2CoefScore, AdjR2CoefScore])\n", 129 | "model.fit(x_train, y_train, epochs=10, batch_size=32, validation_data=(x_test, y_test))" 130 | ], 131 | "metadata": { 132 | "colab": { 133 | "base_uri": "https://localhost:8080/" 134 | }, 135 | "id": "e3WRdtl779I_", 136 | "outputId": "c5a1c786-d033-4423-98f1-52db8e2cfb37" 137 | }, 138 | "execution_count": 3, 139 | "outputs": [ 140 | { 141 | "output_type": "stream", 142 | "name": "stdout", 143 | "text": [ 144 | "float64 float64 float64 float64\n", 145 | "Epoch 1/10\n", 146 | "13/13 [==============================] - 1s 21ms/step - loss: 38.6285 - R2CoefScore: -0.2374 - AdjR2CoefScore: 12.3941 - val_loss: 8.1920 - val_R2CoefScore: 0.9699 - val_AdjR2CoefScore: 0.9369\n", 147 | "Epoch 2/10\n", 148 | "13/13 [==============================] - 0s 5ms/step - loss: 13.6232 - R2CoefScore: 0.9301 - AdjR2CoefScore: 0.8683 - val_loss: 11.6862 - val_R2CoefScore: 0.9450 - val_AdjR2CoefScore: 0.8898\n", 149 | "Epoch 3/10\n", 150 | "13/13 [==============================] - 0s 4ms/step - loss: 10.9884 - R2CoefScore: 0.9536 - AdjR2CoefScore: 0.9100 - val_loss: 6.5358 - val_R2CoefScore: 0.9813 - val_AdjR2CoefScore: 0.9599\n", 151 | "Epoch 4/10\n", 152 | "13/13 [==============================] - 0s 5ms/step - loss: 10.7548 - R2CoefScore: 0.9638 - AdjR2CoefScore: 0.9278 - val_loss: 16.7073 - val_R2CoefScore: 0.8927 - val_AdjR2CoefScore: 0.7934\n", 153 | "Epoch 5/10\n", 154 | "13/13 [==============================] - 0s 4ms/step - loss: 11.1149 - R2CoefScore: 0.9575 - AdjR2CoefScore: 0.9161 - val_loss: 9.2592 - val_R2CoefScore: 0.9590 - val_AdjR2CoefScore: 0.9160\n", 155 | "Epoch 6/10\n", 156 | "13/13 [==============================] - 0s 4ms/step - loss: 10.9138 - R2CoefScore: 0.9559 - AdjR2CoefScore: 0.9134 - val_loss: 15.3846 - val_R2CoefScore: 0.9112 - val_AdjR2CoefScore: 0.8257\n", 157 | "Epoch 7/10\n", 158 | "13/13 [==============================] - 0s 4ms/step - loss: 11.0101 - R2CoefScore: 0.9544 - AdjR2CoefScore: 0.9107 - val_loss: 8.6935 - val_R2CoefScore: 0.9658 - val_AdjR2CoefScore: 0.9292\n", 159 | "Epoch 8/10\n", 160 | "13/13 [==============================] - 0s 4ms/step - loss: 9.2991 - R2CoefScore: 0.9637 - AdjR2CoefScore: 0.9298 - val_loss: 13.9994 - val_R2CoefScore: 0.9245 - val_AdjR2CoefScore: 0.8508\n", 161 | "Epoch 9/10\n", 162 | "13/13 [==============================] - 0s 4ms/step - loss: 10.8715 - R2CoefScore: 0.9513 - AdjR2CoefScore: 0.9067 - val_loss: 7.4039 - val_R2CoefScore: 0.9737 - val_AdjR2CoefScore: 0.9444\n", 163 | "Epoch 10/10\n", 164 | "13/13 [==============================] - 0s 4ms/step - loss: 10.5106 - R2CoefScore: 0.9632 - AdjR2CoefScore: 0.9270 - val_loss: 8.5580 - val_R2CoefScore: 0.9733 - val_AdjR2CoefScore: 0.9446\n" 165 | ] 166 | }, 167 | { 168 | "output_type": "execute_result", 169 | "data": { 170 | "text/plain": [ 171 | "" 172 | ] 173 | }, 174 | "metadata": {}, 175 | "execution_count": 3 176 | } 177 | ] 178 | }, 179 | { 180 | "cell_type": "markdown", 181 | "source": [ 182 | "### With Scikit Learn" 183 | ], 184 | "metadata": { 185 | "id": "p6vGElnq8DfS" 186 | } 187 | }, 188 | { 189 | "cell_type": "code", 190 | "source": [ 191 | "from regressionmetrics.metrics import *\n", 192 | "y_true = [3, 0.5, 2, 7]\n", 193 | "y_pred = [2.5, 0.0, 2, -8]\n", 194 | "\n", 195 | "print(\"R2Score: \",R2CoefScore(y_true, y_pred))\n", 196 | "print(\"Adjusted_R2_Score:\",AdjR2CoefScore(y_true, y_pred))\n", 197 | "print(\"RMSE:\", RootMeanSqrtErr(y_true, y_pred))\n", 198 | "print(\"MAE:\",MeanAbsoErr(y_true, y_pred))\n", 199 | "print(\"RMSLE with Neg Value:\", RootMeanSqrtLogErrNeg(y_true, y_pred))\n", 200 | "print(\"MSE:\", MeanSqrtErr(y_true, y_pred))\n", 201 | "print(\"MAPE: \", MeanAbsPercErr(y_true, y_pred))" 202 | ], 203 | "metadata": { 204 | "colab": { 205 | "base_uri": "https://localhost:8080/" 206 | }, 207 | "id": "DL0qWyV28Ftd", 208 | "outputId": "e9d53433-7193-4c79-bd88-e339fa5388ad" 209 | }, 210 | "execution_count": 4, 211 | "outputs": [ 212 | { 213 | "output_type": "stream", 214 | "name": "stdout", 215 | "text": [ 216 | "R2Score: -8.725067385444744\n", 217 | "Adjusted_R2_Score: 20.450134770889484\n", 218 | "RMSE: 7.508328708840604\n", 219 | "MAE: 4.0\n", 220 | "RMSLE with Neg Value: 0.21344354447336292\n", 221 | "MSE: 56.375\n", 222 | "MAPE: 0.8273809523809523\n" 223 | ] 224 | } 225 | ] 226 | } 227 | ] 228 | } -------------------------------------------------------------------------------- /regressionmetrics/metrics.py: -------------------------------------------------------------------------------- 1 | from sklearn.metrics import * 2 | import numpy as np 3 | 4 | EPSILON = 1e-10 5 | 6 | def MeanAbsoErr(y_true, y_pred): 7 | """ 8 | Mean absolute error regression loss. smaller is better. 9 | 10 | interpretation: smaller is better.(Best value is 0.0) 11 | - MAE value is between 0 to inf. 12 | - MAE treats larger and small errors equally. Not sensitive to outliers in the data. 13 | - MAE is Robust to outliers compared to RMSE. 14 | - 15 | 16 | Args: 17 | y_true ([np.array]): test samples 18 | y_pred ([np.array]): predicted samples 19 | 20 | Returns: 21 | [float]: mean absolute error 22 | """ 23 | return mean_absolute_error(y_true, y_pred) 24 | 25 | 26 | def MeanSqrtErr(y_true, y_pred): 27 | """ 28 | Mean squared error regression loss. 29 | 30 | interpretation: smaller is better.(Best value is 0.0) 31 | - MSE = 0.0, indicates that model is perfectly fitting the data. 32 | - MSE = inf, indicates that model is overfitting the data. 33 | - MSE = NaN, indicates that model is not fitting the data. 34 | - MSE value is between 0 to inf. 35 | - MSE treats Sensitive to outliers, punishes larger error more. 36 | 37 | Args: 38 | y_true ([np.array]): test samples 39 | y_pred ([np.array]): predicted samples 40 | 41 | Returns: 42 | [float]: mean squared error 43 | """ 44 | return mean_squared_error(y_true, y_pred) 45 | 46 | def RootMeanSqrtErr(y_true, y_pred): 47 | """ 48 | Root Mean Square Error regression loss. 49 | 50 | interpretation: smaller is better.(Best value is 0.0) 51 | - RMSE = 0.0, indicates that model is perfectly fitting the data. 52 | - RMSE = inf, indicates that model is overfitting the data. 53 | - RMSE = NaN, indicates that model is not fitting the data. 54 | - RMSE value is between 0 to inf. 55 | - RMSE treats Sensitive to outliers, punishes larger error more. 56 | 57 | Args: 58 | y_true ([np.array]): test samples 59 | y_pred ([np.array]): predicted samples 60 | 61 | Returns: 62 | [float]: mean squared error 63 | """ 64 | return np.sqrt(mean_squared_error(y_true, y_pred)) 65 | 66 | def RootMeanSqrtLogErr(y_true, y_pred): 67 | """ 68 | Root Mean Squared Logarithm Error regression loss. 69 | 70 | interpretation: smaller is better.(Best value is 0.0) 71 | - RMSLE = 0.0, indicates that model is perfectly fitting the data. 72 | - RMSLE = inf, indicates that model is overfitting the data. 73 | - RMSLE = NaN, indicates that model is not fitting the data. 74 | - RMSLE is usually used when you don't want to penalize the large differences in the predicted and the actual values when the predicted 75 | and the actual values are big numbers. 76 | - RMSLE is used when y has long tail distribution, or we are interested in the ratio of true value and predicted value. 77 | 78 | Args: 79 | y_true ([np.array]): test samples 80 | y_pred ([np.array]): predicted samples 81 | 82 | Returns: 83 | [float]: root mean squared logarithm error 84 | """ 85 | for i in range(len(y_true)): 86 | if y_true[i] < 0 or y_pred[i] < 0: 87 | continue 88 | 89 | return np.sqrt(mean_squared_log_error(y_true, y_pred)) 90 | 91 | 92 | def RootMeanSqrtLogErrNeg(y_true, y_pred): 93 | """ 94 | Root Mean Squared Logarithmic Error with negative values regression loss. 95 | 96 | interpretation: smaller is better.(Best value is 0.0) 97 | 98 | Args: 99 | y_true ([np.array]): test samples 100 | y_pred ([np.array]): predicted samples 101 | 102 | Returns: 103 | [float]: root mean squared logarithm error 104 | """ 105 | sum=0.0 106 | for x in range(len(y_pred)): 107 | if y_pred[x]<0 or y_true[x]<0: #check for negative values 108 | continue 109 | p = np.log(y_pred[x]+1) 110 | r = np.log(y_true[x]+1) 111 | sum = sum + (p - r)**2 112 | 113 | RMSLE = (sum/len(y_pred))**0.5 114 | 115 | return RMSLE 116 | 117 | def R2CoefScore(y_true, y_pred): 118 | """ 119 | :math:`R^2` (coefficient of determination) regression score function. 120 | 121 | interpretation: Best possible score is 1.0(perfect prediction), lower values are worse. 122 | - 0.0 <= r2<= 1.0 is standard value but worst cases are -1.0 and -2.0 or any negative values. 123 | - r2 > 0.9 is good. 124 | - 0.7 < r2 < 0.9 indicates high level of correlation. 125 | - 0.4 < r2 < 0.7 indicates medium level of correlation. 126 | - r2 < 0.4 indicates low level of correlation. 127 | - 128 | Args: 129 | y_true ([np.array]): test samples 130 | y_pred ([np.array]): predicted samples 131 | 132 | Returns: 133 | [float]: R2 134 | """ 135 | return r2_score(y_true, y_pred) 136 | 137 | def AdjR2CoefScore(y_true, y_pred): 138 | """ 139 | Adjusted R2 regression score function.(Best value is 1.0) 140 | 141 | interpretation: Best possible score is 1.0, lower values are worse. 142 | - adj_r2 < 1.0 indicates that at least some variability in the data is explained by the model. 143 | - adj_r2 =0.5 indicates that the variability in the outcome data cannot be explained by the model. 144 | - adj_r2 < 0.0 indicates that model fitted wrong. 145 | - Adjusted R-squared to compare the goodness-of-fit for regression models that contain differing numbers of independent variables. 146 | - Adjusted R2 is the better model when you compare models that have a different amount of variables. 147 | - 148 | 149 | Args: 150 | y_true ([np.array]): test samples 151 | y_pred ([np.array]): predicted samples 152 | 153 | Returns: 154 | [float]: adjusted R2 155 | """ 156 | return r2_score(y_true, y_pred) - ((1 - r2_score(y_true, y_pred)) * (len(y_true) - 1) / (len(y_true) - len(y_pred) - 1)) 157 | 158 | def MeanAbsPercErr(y_true, y_pred): 159 | """ 160 | Mean absolute percentage error regression loss. 161 | 162 | interpretation: smaller is better.(Best value is 0.0) 163 | - MAPE < 10 % Very Good 164 | - 10 % < MAPE < 20% Good 165 | - 20 % < MAPE < 50% Average 166 | - MAPE > 50% Not Good 167 | - MAPE widely used measure for checking forecast accuracy. 168 | 169 | Args: 170 | y_true ([np.array]): test samples 171 | y_pred ([np.array]): predicted samples 172 | 173 | Returns: 174 | [float]: mean absolute percentage error 175 | """ 176 | return mean_absolute_percentage_error(y_true, y_pred) 177 | 178 | def MeanSqrtLogErr(y_true, y_pred): 179 | """ 180 | Mean squared logarithmic error regression loss. 181 | 182 | interpretation: smaller is better.(Best value is 0.0) 183 | - MSLE = 0.0, indicates that model is perfectly fitting the data. 184 | - MSLE = inf, indicates that model is overfitting the data. 185 | - MSLE used when target, conditioned on the input, is normally distributed, and 186 | you don’t want large errors to be significantly more penalized than small ones, 187 | in those cases where the range of the target value is large. 188 | 189 | Args: 190 | y_true ([np.array]): test samples 191 | y_pred ([np.array]): predicted samples 192 | 193 | Returns: 194 | [float]: mean squared logarithmic error 195 | """ 196 | return mean_squared_log_error(y_true, y_pred) 197 | 198 | def SymMeanAbsPercErr(y_true: np.ndarray, y_pred: np.ndarray): 199 | """ 200 | :math:`SMAPE` Symmetric mean absolute percentage error regression loss. 201 | 202 | interpretation: smaller is better.(Best value is 0.0) 203 | 204 | Args: 205 | y_true ([np.array]): test samples 206 | y_pred ([np.array]): predicted samples 207 | 208 | Returns: 209 | [float]: SMAPE 210 | """ 211 | return np.mean(2.0 * np.abs(y_true - y_pred) / ((np.abs(y_true) + np.abs(y_pred)) + EPSILON)) 212 | 213 | def NormRootMeanSqrtErr(y_true, y_pred, type = "minmax"): 214 | """ 215 | Normalized Root Mean Square Error. 216 | 217 | interpretation: smaller is better. 218 | 219 | Args: 220 | y_true ([np.array]): test samples 221 | y_pred ([np.array]): predicted samples 222 | type (str): type of normalization. default is "minmax" 223 | - sd (standard deviation) : it divides the value by the standard deviation of the data. 224 | - mean (mean) : it divides the value by the mean of the data. 225 | - minmax (min-max) : it divides the value by the range of the data. 226 | - iqr (interquartile range) : it divides the value by the interquartile range of the data. 227 | 228 | Returns: 229 | [float]: normalized root mean square error 230 | """ 231 | if type=="sd": 232 | return np.sqrt(mean_squared_error(y_true, y_pred))/np.std(y_true) 233 | elif type=="mean": 234 | return np.sqrt(mean_squared_error(y_true, y_pred))/np.mean(y_true) 235 | elif type=="minmax": 236 | return np.sqrt(mean_squared_error(y_true, y_pred))/(np.max(y_true) - np.min(y_true)) 237 | elif type=="iqr": 238 | return np.sqrt(mean_squared_error(y_true, y_pred))/(np.quantile(y_true, 0.75) - np.quantile(y_true, 0.25)) 239 | elif type!="": 240 | raise ValueError("type must be either 'sd', 'mean', 'minmax', or 'iqr'") 241 | 242 | def NormRootMeanSqrtLogErr(y_true, y_pred): 243 | """ 244 | Normalized Root Mean Squared Logarithmic Error. 245 | 246 | interrpretation: smaller is better. 247 | 248 | Args: 249 | y_true ([np.array]): test samples 250 | y_pred ([np.array]): predicted samples 251 | type (str): type of normalization. default is "minmax" 252 | - sd (standard deviation) : it divides the value by the standard deviation of the data. 253 | - mean (mean) : it divides the value by the mean of the data. 254 | - minmax (min-max) : it divides the value by the range of the data. 255 | - iqr (interquartile range) : it divides the value by the interquartile range of the data. 256 | 257 | Returns: 258 | [float]: normalized root mean squared logarithm error 259 | """ 260 | if type=="sd": 261 | return np.sqrt(mean_squared_log_error(y_true, y_pred))/np.std(y_true) 262 | elif type=="mean": 263 | return np.sqrt(mean_squared_log_error(y_true, y_pred))/np.mean(y_true) 264 | elif type=="minmax": 265 | return np.sqrt(mean_squared_log_error(y_true, y_pred))/(np.max(y_true) - np.min(y_true)) 266 | elif type=="iqr": 267 | return np.sqrt(mean_squared_log_error(y_true, y_pred))/(np.quantile(y_true, 0.75) - np.quantile(y_true, 0.25)) 268 | elif type!="": 269 | raise ValueError("type must be either 'sd', 'mean', 'minmax', or 'iqr'") 270 | 271 | def MedianAbsErr(y_true, y_pred): 272 | """ 273 | Median absolute error regression loss. (Best value is 0.0) 274 | 275 | interpretation: smaller is better. 276 | 277 | Args: 278 | y_true ([np.array]): test samples 279 | y_pred ([np.array]): predicted samples 280 | 281 | Returns: 282 | [float]: median absolute error 283 | """ 284 | return median_absolute_error(y_true, y_pred) 285 | 286 | 287 | def MediaRelErr(y_true, y_pred): 288 | """ 289 | Mean relative error regression loss. 290 | 291 | interpretation: smaller is better.(Best value is 0.0) 292 | 293 | Args: 294 | y_true ([np.array]): test samples 295 | y_pred ([np.array]): predicted samples 296 | 297 | Returns: 298 | [float]: mean relative error 299 | """ 300 | return np.mean(np.abs(y_true - y_pred) / (y_true + EPSILON)) 301 | 302 | def MeanArcAbsPercErr(y_true, y_pred): 303 | """ 304 | Mean Arctangent Absolute Percentage Error regression loss. 305 | 306 | interpretation: smaller is better.(Best value is 0.0) 307 | 308 | Args: 309 | y_true ([np.array]): test samples 310 | y_pred ([np.array]): predicted samples 311 | 312 | Returns: 313 | [float]: mean arctangent absolute percentage error 314 | """ 315 | return np.mean(np.abs(np.arctan(y_true) - np.arctan(y_pred))) 316 | 317 | 318 | def MeanAbsScaErr(y_true, y_pred, y_train): 319 | """ 320 | Mean absolute scale error regression loss. 321 | 322 | Reference : R.J. Hyndman, A.B. Koehler, Another look at measures of forecast accuracy, International Journal of Forecasting, 22 (2006), pp. 679-688 323 | 324 | interpretation: smaller is better.(Best value is 0.0) 325 | - MASE = 0.5 means model has doubled the prediction accurracy. 326 | - MASE > 1.0 means model has overpredicted. 327 | - MASE < 1.0 means model has underpredicted. 328 | 329 | Args: 330 | y_true ([np.array]): test samples 331 | y_pred ([np.array]): predicted samples 332 | 333 | Returns: 334 | [float]: mean absolute scale error 335 | """ 336 | e_t = np.mean(np.abs(y_true - y_train)) 337 | scale = mean_absolute_error(y_true[1:] - y_pred[:-1]) 338 | return np.mean(np.abs(e_t/ scale)) 339 | 340 | def NashSutCoeff(y_true, y_pred): 341 | """ 342 | The Nash-Sutcliffe efficiency (NSE) is a normalized statistic that determines the relative magnitude of the residual variance compared to the measured data variance (Nash and Sutcliffe, 1970). 343 | 344 | Reference: McCuen, R.H., Knight, Z. and Cutter, A.G., 2006. Evaluation of the Nash–Sutcliffe efficiency index. Journal of hydrologic engineering, 11(6), pp.597-602. 345 | 346 | interpretation: Larger is better.(Best possible score is 1.0, lower values are worse.) 347 | - NSE = 1, corresponds to a perfect match of the model to the observed data. 348 | - NSE = 0, indicates that the model predictions are as accurate as the mean of the observed data. 349 | - Inf < NSE < 0, indicates that the observed mean is a better predictor than the model. 350 | 351 | Args: 352 | y_true ([np.array]): test samples 353 | y_pred ([np.array]): predicted samples 354 | 355 | Returns: 356 | [float]: Nash-Sutcliffe efficiency coefficient 357 | """ 358 | return 1 - ((np.sum((y_true - y_pred)**2)) / (np.sum((y_true - np.mean(y_true))**2))) 359 | 360 | 361 | def WillMottIndexAgreeMent(y_true, y_pred): 362 | """ 363 | Willmott (1981) proposed an index of agreement (d) as a standardized measure of the degree of model prediction error which varies between 0 and 1. 364 | 365 | References : Willmott, C.J., 1981. On the validation of models. Physical geography, 2(2), pp.184-194. 366 | 367 | interpretation: Larger is better (Best value is 1.0) 368 | - d = 1, indicates that the model prediction is perfect match. 369 | - d = 0, indicates that no agreement at all 370 | 371 | Args: 372 | y_true ([np.array]): test samples 373 | y_pred ([np.array]): predicted samples 374 | 375 | Returns: 376 | [float]: Willmott index of agreement 377 | """ 378 | residuals = y_true - y_pred 379 | abs_diff_pred = np.abs(y_pred - np.nanmean(y_true)) 380 | abs_diff_obs = np.abs(y_true - np.nanmean(y_true)) 381 | return 1 - np.nansum(residuals**2) / np.nansum((abs_diff_pred * abs_diff_obs)**2) 382 | --------------------------------------------------------------------------------