├── tests ├── __init__.py ├── test_average_recall.py └── test_metrics.py ├── requirements.txt ├── setup.cfg ├── .travis.yml ├── keras_metrics ├── casts.py ├── __init__.py └── metrics.py ├── LICENSE.txt ├── setup.py ├── .gitignore └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Keras>=2.1.5,<2.3.0 2 | tensorflow>=1.8.0,<2.0.0 3 | scikit-learn>=0.20.2 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | [metadata] 5 | description-file = README.md 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | - "3.4" 5 | - "3.5" 6 | - "3.6" 7 | script: 8 | - pytest 9 | notifications: 10 | email: false 11 | -------------------------------------------------------------------------------- /keras_metrics/casts.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | 3 | 4 | def binary(y_true, y_pred, dtype="int32", label=0): 5 | return categorical(y_true, y_pred, dtype, label) 6 | 7 | 8 | def categorical(y_true, y_pred, dtype="int32", label=0): 9 | column = slice(label, label+1) 10 | 11 | y_true = y_true[..., column] 12 | y_pred = y_pred[..., column] 13 | 14 | y_true = K.cast(K.round(y_true), dtype) 15 | y_pred = K.cast(K.round(y_pred), dtype) 16 | 17 | return y_true, y_pred 18 | 19 | 20 | def sparse_categorical(y_true, y_pred, dtype="int32", label=0): 21 | return categorical(y_true, y_pred) 22 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Yasha Bubnov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import keras_metrics 2 | import os 3 | import setuptools 4 | 5 | 6 | here = os.path.abspath(os.path.dirname(__file__)) 7 | 8 | # Get the long description from the README file. 9 | with open(os.path.join(here, "README.md"), encoding="utf-8") as md: 10 | long_description = md.read() 11 | 12 | 13 | setuptools.setup( 14 | name="keras-metrics", 15 | version=keras_metrics.__version__, 16 | 17 | long_description=long_description, 18 | long_description_content_type="text/markdown", 19 | description="Metrics for Keras model evaluation", 20 | 21 | url="https://github.com/netrack/keras-metrics", 22 | author="Yasha Bubnov", 23 | author_email="girokompass@gmail.com", 24 | 25 | classifiers=[ 26 | "Intended Audience :: Developers", 27 | "License :: OSI Approved :: MIT License", 28 | "Topic :: Software Development :: Libraries", 29 | 30 | 'Programming Language :: Python :: 2', 31 | 'Programming Language :: Python :: 2.7', 32 | 'Programming Language :: Python :: 3', 33 | 'Programming Language :: Python :: 3.4', 34 | 'Programming Language :: Python :: 3.5', 35 | 'Programming Language :: Python :: 3.6', 36 | ], 37 | 38 | keywords="keras metrics evaluation", 39 | 40 | packages=setuptools.find_packages(exclude=["tests"]), 41 | install_requires=["Keras>=2.1.5,<2.3.0"], 42 | ) 43 | -------------------------------------------------------------------------------- /tests/test_average_recall.py: -------------------------------------------------------------------------------- 1 | import keras 2 | import keras.utils 3 | import keras_metrics as km 4 | import numpy 5 | import unittest 6 | 7 | 8 | class TestAverageRecall(unittest.TestCase): 9 | 10 | def create_samples(self, n, labels=1): 11 | x = numpy.random.uniform(0, numpy.pi/2, (n, labels)) 12 | y = numpy.random.randint(labels, size=(n, 1)) 13 | return x, keras.utils.to_categorical(y) 14 | 15 | def test_average_recall(self): 16 | model = keras.models.Sequential() 17 | model.add(keras.layers.Activation(keras.backend.sin)) 18 | model.add(keras.layers.Activation(keras.backend.abs)) 19 | model.add(keras.layers.Softmax()) 20 | model.compile(optimizer="sgd", 21 | loss="categorical_crossentropy", 22 | metrics=[ 23 | km.categorical_recall(label=0), 24 | km.categorical_recall(label=1), 25 | km.categorical_recall(label=2), 26 | km.categorical_average_recall(labels=3), 27 | ]) 28 | 29 | x, y = self.create_samples(10000, labels=3) 30 | 31 | model.fit(x, y, epochs=10, batch_size=100) 32 | metrics = model.evaluate(x, y, batch_size=100)[1:] 33 | 34 | r0, r1, r2 = metrics[0:3] 35 | average_recall = metrics[3] 36 | 37 | expected_recall = (r0+r1+r2)/3.0 38 | self.assertAlmostEqual(expected_recall, average_recall, places=3) 39 | 40 | 41 | if __name__ == "__main__": 42 | unittest.main() 43 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # Swap 107 | [._]*.s[a-v][a-z] 108 | [._]*.sw[a-p] 109 | [._]s[a-v][a-z] 110 | [._]sw[a-p] 111 | 112 | # Session 113 | Session.vim 114 | 115 | # Temporary 116 | .netrwhist 117 | *~ 118 | # Auto-generated tag files 119 | tags 120 | # Persistent undo 121 | [._]*.un~ 122 | 123 | *.hdf5 124 | -------------------------------------------------------------------------------- /keras_metrics/__init__.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | from keras_metrics import metrics as m 3 | from keras_metrics import casts 4 | 5 | 6 | __version__ = "1.2.1" 7 | 8 | 9 | def metric_fn(cls, cast_strategy): 10 | def fn(label=0, **kwargs): 11 | metric = cls(label=label, cast_strategy=cast_strategy, **kwargs) 12 | metric.__name__ = "%s_%s" % (cast_strategy.__name__, cls.__name__) 13 | return metric 14 | return fn 15 | 16 | 17 | binary_metric = partial( 18 | metric_fn, cast_strategy=casts.binary) 19 | 20 | 21 | categorical_metric = partial( 22 | metric_fn, cast_strategy=casts.categorical) 23 | 24 | 25 | sparse_categorical_metric = partial( 26 | metric_fn, cast_strategy=casts.sparse_categorical) 27 | 28 | 29 | binary_true_positive = binary_metric(m.true_positive) 30 | binary_true_negative = binary_metric(m.true_negative) 31 | binary_false_positive = binary_metric(m.false_positive) 32 | binary_false_negative = binary_metric(m.false_negative) 33 | binary_precision = binary_metric(m.precision) 34 | binary_recall = binary_metric(m.recall) 35 | binary_f1_score = binary_metric(m.f1_score) 36 | binary_average_recall = binary_metric(m.average_recall) 37 | 38 | 39 | categorical_true_positive = categorical_metric(m.true_positive) 40 | categorical_true_negative = categorical_metric(m.true_negative) 41 | categorical_false_positive = categorical_metric(m.false_positive) 42 | categorical_false_negative = categorical_metric(m.false_negative) 43 | categorical_precision = categorical_metric(m.precision) 44 | categorical_recall = categorical_metric(m.recall) 45 | categorical_f1_score = categorical_metric(m.f1_score) 46 | categorical_average_recall = categorical_metric(m.average_recall) 47 | 48 | 49 | sparse_categorical_true_positive = sparse_categorical_metric(m.true_positive) 50 | sparse_categorical_true_negative = sparse_categorical_metric(m.true_negative) 51 | sparse_categorical_false_positive = sparse_categorical_metric(m.false_positive) 52 | sparse_categorical_false_negative = sparse_categorical_metric(m.false_negative) 53 | sparse_categorical_precision = sparse_categorical_metric(m.precision) 54 | sparse_categorical_recall = sparse_categorical_metric(m.recall) 55 | sparse_categorical_f1_score = sparse_categorical_metric(m.f1_score) 56 | sparse_categorical_average_recall = sparse_categorical_metric(m.average_recall) 57 | 58 | 59 | # For backward compatibility. 60 | true_positive = binary_true_positive 61 | true_negative = binary_true_negative 62 | false_positive = binary_false_positive 63 | false_negative = binary_false_negative 64 | precision = binary_precision 65 | recall = binary_recall 66 | f1_score = binary_f1_score 67 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Keras Metrics 2 | 3 | ## Deprecation Warning 4 | 5 | Since Keras version `2.3.0`, it provides all metrics available in this package. 6 | It's preferrable to use metrics from the original Keras package. 7 | 8 | This package will be maintained for older version of Keras (`<2.3.0`). 9 | 10 | [![Build Status][BuildStatus]](https://travis-ci.org/netrack/keras-metrics) 11 | 12 | This package provides metrics for evaluation of Keras classification models. 13 | The metrics are safe to use for batch-based model evaluation. 14 | 15 | ## Installation 16 | 17 | To install the package from the PyPi repository you can execute the following 18 | command: 19 | ```sh 20 | pip install keras-metrics 21 | ``` 22 | 23 | ## Usage 24 | 25 | The usage of the package is simple: 26 | ```py 27 | import keras 28 | import keras_metrics as km 29 | 30 | model = models.Sequential() 31 | model.add(keras.layers.Dense(1, activation="sigmoid", input_dim=2)) 32 | model.add(keras.layers.Dense(1, activation="softmax")) 33 | 34 | model.compile(optimizer="sgd", 35 | loss="binary_crossentropy", 36 | metrics=[km.binary_precision(), km.binary_recall()]) 37 | ``` 38 | 39 | Similar configuration for multi-label binary crossentropy: 40 | ```py 41 | import keras 42 | import keras_metrics as km 43 | 44 | model = models.Sequential() 45 | model.add(keras.layers.Dense(1, activation="sigmoid", input_dim=2)) 46 | model.add(keras.layers.Dense(2, activation="softmax")) 47 | 48 | # Calculate precision for the second label. 49 | precision = km.binary_precision(label=1) 50 | 51 | # Calculate recall for the first label. 52 | recall = km.binary_recall(label=0) 53 | 54 | model.compile(optimizer="sgd", 55 | loss="binary_crossentropy", 56 | metrics=[precision, recall]) 57 | ``` 58 | 59 | Keras metrics package also supports metrics for categorical crossentropy and 60 | sparse categorical crossentropy: 61 | ```py 62 | import keras_metrics as km 63 | 64 | c_precision = km.categorical_precision() 65 | sc_precision = km.sparse_categorical_precision() 66 | 67 | # ... 68 | ``` 69 | 70 | ## Tensorflow Keras 71 | 72 | Tensorflow library provides the ```keras``` package as parts of its API, in 73 | order to use ```keras_metrics``` with Tensorflow Keras, you are advised to 74 | perform model training with initialized global variables: 75 | ```py 76 | import numpy as np 77 | import keras_metrics as km 78 | import tensorflow as tf 79 | import tensorflow.keras as keras 80 | 81 | model = keras.Sequential() 82 | model.add(keras.layers.Dense(1, activation="softmax")) 83 | model.compile(optimizer="sgd", 84 | loss="binary_crossentropy", 85 | metrics=[km.binary_true_positive()]) 86 | 87 | x = np.array([[0], [1], [0], [1]]) 88 | y = np.array([1, 0, 1, 0]) 89 | 90 | # Wrap model.fit into the session with global 91 | # variables initialization. 92 | with tf.Session() as s: 93 | s.run(tf.global_variables_initializer()) 94 | model.fit(x=x, y=y) 95 | ``` 96 | 97 | [BuildStatus]: https://travis-ci.org/netrack/keras-metrics.svg?branch=master 98 | -------------------------------------------------------------------------------- /tests/test_metrics.py: -------------------------------------------------------------------------------- 1 | import keras 2 | import keras.backend 3 | import keras.utils 4 | import keras_metrics as km 5 | import itertools 6 | import numpy 7 | import tempfile 8 | import unittest 9 | 10 | from keras import backend as K 11 | from sklearn.metrics import confusion_matrix 12 | 13 | 14 | class TestMetrics(unittest.TestCase): 15 | 16 | binary_metrics = [ 17 | km.binary_true_positive, 18 | km.binary_true_negative, 19 | km.binary_false_positive, 20 | km.binary_false_negative, 21 | km.binary_precision, 22 | km.binary_recall, 23 | km.binary_f1_score, 24 | ] 25 | 26 | categorical_metrics = [ 27 | km.categorical_true_positive, 28 | km.categorical_true_negative, 29 | km.categorical_false_positive, 30 | km.categorical_false_negative, 31 | km.categorical_precision, 32 | km.categorical_recall, 33 | km.categorical_f1_score, 34 | ] 35 | 36 | sparse_categorical_metrics = [ 37 | km.sparse_categorical_true_positive, 38 | km.sparse_categorical_true_negative, 39 | km.sparse_categorical_false_positive, 40 | km.sparse_categorical_false_negative, 41 | km.sparse_categorical_precision, 42 | km.sparse_categorical_recall, 43 | km.sparse_categorical_f1_score, 44 | ] 45 | 46 | def create_binary_samples(self, n): 47 | x = numpy.random.uniform(0, numpy.pi/2, (n, 1)) 48 | y = numpy.random.randint(2, size=(n, 1)) 49 | return x, y 50 | 51 | def create_categorical_samples(self, n): 52 | x, y = self.create_binary_samples(n) 53 | return x, keras.utils.to_categorical(y) 54 | 55 | def create_sparse_categorical_samples(self, n, num_classes=2): 56 | x = numpy.random.uniform(0, numpy.pi/2, (n, 1)) 57 | y = numpy.random.randint(num_classes, size=(n, 1)) 58 | return x, y 59 | 60 | def create_metrics(self, metrics_fns): 61 | return list(map(lambda m: m(), metrics_fns)) 62 | 63 | def create_model(self, outputs, loss, metrics_fns): 64 | model = keras.models.Sequential() 65 | model.add(keras.layers.Activation(keras.backend.sin)) 66 | model.add(keras.layers.Activation(keras.backend.abs)) 67 | model.add(keras.layers.Dense(outputs, activation="softmax")) 68 | model.compile(optimizer="sgd", 69 | loss=loss, 70 | metrics=self.create_metrics(metrics_fns)) 71 | return model 72 | 73 | def assert_save_load(self, model, metrics_fns, samples_fn): 74 | metrics = [m() for m in metrics_fns] 75 | 76 | custom_objects = {m.__name__: m for m in metrics} 77 | custom_objects["sin"] = keras.backend.sin 78 | custom_objects["abs"] = keras.backend.abs 79 | 80 | x, y = samples_fn(100) 81 | model.fit(x, y, epochs=10) 82 | 83 | with tempfile.NamedTemporaryFile() as file: 84 | model.save(file.name, overwrite=True) 85 | 86 | loaded_model = keras.models.load_model( 87 | file.name, custom_objects=custom_objects) 88 | 89 | expected = model.evaluate(x, y)[1:] 90 | received = loaded_model.evaluate(x, y)[1:] 91 | 92 | self.assertEqual(expected, received) 93 | 94 | def test_save_load_binary_metrics(self): 95 | model = self.create_model(1, "binary_crossentropy", 96 | self.binary_metrics) 97 | self.assert_save_load(model, 98 | self.binary_metrics, 99 | self.create_binary_samples) 100 | 101 | def test_save_load_categorical_metrics(self): 102 | model = self.create_model(2, "categorical_crossentropy", 103 | self.categorical_metrics) 104 | self.assert_save_load(model, 105 | self.categorical_metrics, 106 | self.create_categorical_samples) 107 | 108 | def test_save_load_sparse_categorical_metrics(self): 109 | model = self.create_model(2, "sparse_categorical_crossentropy", 110 | self.sparse_categorical_metrics) 111 | self.assert_save_load(model, 112 | self.sparse_categorical_metrics, 113 | self.create_binary_samples) 114 | 115 | def assert_metrics(self, model, samples_fn): 116 | samples = 10000 117 | batch_size = 100 118 | 119 | x, y = samples_fn(samples) 120 | 121 | model.fit(x, y, epochs=10, batch_size=batch_size) 122 | y_pred = model.predict(x).round() 123 | 124 | metrics = model.evaluate(x, y, batch_size=batch_size)[1:] 125 | metrics = list(map(float, metrics)) 126 | 127 | cm = confusion_matrix(y[:,0], y_pred[:,0]) 128 | 129 | tp_val = metrics[0] 130 | tn_val = metrics[1] 131 | fp_val = metrics[2] 132 | fn_val = metrics[3] 133 | 134 | # Cross validation on SKLearn library. 135 | self.assertEqual(tp_val, cm[1][1]) 136 | self.assertEqual(tn_val, cm[0][0]) 137 | self.assertEqual(fp_val, cm[0][1]) 138 | self.assertEqual(fn_val, cm[1][0]) 139 | 140 | precision = metrics[4] 141 | recall = metrics[5] 142 | f1 = metrics[6] 143 | 144 | expected_precision = tp_val / (tp_val + fp_val + K.epsilon()) 145 | expected_recall = tp_val / (tp_val + fn_val + K.epsilon()) 146 | 147 | f1_divident = (expected_precision*expected_recall) 148 | f1_divisor = (expected_precision+expected_recall) 149 | expected_f1 = 2 * f1_divident / (f1_divisor + K.epsilon()) 150 | 151 | self.assertGreaterEqual(tp_val, 0.0) 152 | self.assertGreaterEqual(fp_val, 0.0) 153 | self.assertGreaterEqual(fn_val, 0.0) 154 | self.assertGreaterEqual(tn_val, 0.0) 155 | 156 | self.assertEqual(sum(metrics[0:4]), samples) 157 | 158 | places = 4 159 | self.assertAlmostEqual(expected_precision, precision, places=places) 160 | self.assertAlmostEqual(expected_recall, recall, places=places) 161 | self.assertAlmostEqual(expected_f1, f1, places=places) 162 | 163 | def test_binary_metrics(self): 164 | model = self.create_model(1, "binary_crossentropy", 165 | self.binary_metrics) 166 | self.assert_metrics(model, self.create_binary_samples) 167 | 168 | def test_categorical_metrics(self): 169 | model = self.create_model(2, "categorical_crossentropy", 170 | self.categorical_metrics) 171 | self.assert_metrics(model, self.create_categorical_samples) 172 | 173 | def test_sparse_categorical_metrics(self): 174 | model = self.create_model(2, "sparse_categorical_crossentropy", 175 | self.sparse_categorical_metrics) 176 | self.assert_metrics(model, self.create_sparse_categorical_samples) 177 | 178 | 179 | if __name__ == "__main__": 180 | unittest.main() 181 | -------------------------------------------------------------------------------- /keras_metrics/metrics.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras.layers import Layer 3 | from operator import truediv 4 | 5 | 6 | class layer(Layer): 7 | 8 | def __init__(self, name=None, label=0, cast_strategy=None, **kwargs): 9 | super(layer, self).__init__(name=name, **kwargs) 10 | 11 | self.stateful = True 12 | self.label = label 13 | self.cast_strategy = cast_strategy 14 | self.epsilon = None 15 | 16 | def init_tensors(self, name=None): 17 | """Method to delay initialization of tensors until the first time the 18 | metric is called. Allows the class to be pickled and single-file models 19 | to be shipped. 20 | """ 21 | if self.epsilon is None: 22 | self.epsilon = K.constant(K.epsilon(), dtype="float64") 23 | if name and not hasattr(self, name): 24 | setattr(self, name, K.variable(0, dtype="int32")) 25 | 26 | def cast(self, y_true, y_pred, dtype="int32"): 27 | """Convert the specified true and predicted output to the specified 28 | destination type (int32 by default). 29 | """ 30 | return self.cast_strategy( 31 | y_true, y_pred, dtype=dtype, label=self.label) 32 | 33 | def __getattribute__(self, name): 34 | if name == "get_config": 35 | raise AttributeError 36 | return object.__getattribute__(self, name) 37 | 38 | 39 | class true_positive(layer): 40 | """Create a metric for model's true positives amount calculation. 41 | 42 | A true positive is an outcome where the model correctly predicts the 43 | positive class. 44 | """ 45 | 46 | def __init__(self, name="true_positive", **kwargs): 47 | super(true_positive, self).__init__(name=name, **kwargs) 48 | 49 | def reset_states(self): 50 | """Reset the state of the metric.""" 51 | K.set_value(self.tp, 0) 52 | 53 | def __call__(self, y_true, y_pred): 54 | if self.epsilon is None: 55 | self.init_tensors("tp") 56 | 57 | y_true, y_pred = self.cast(y_true, y_pred) 58 | 59 | tp = K.sum(y_true * y_pred) 60 | current_tp = self.tp * 1 61 | 62 | tp_update = K.update_add(self.tp, tp) 63 | self.add_update(tp_update, inputs=[y_true, y_pred]) 64 | 65 | return tp + current_tp 66 | 67 | 68 | class true_negative(layer): 69 | """Create a metric for model's true negatives amount calculation. 70 | 71 | A true negative is an outcome where the model correctly predicts the 72 | negative class. 73 | """ 74 | 75 | def __init__(self, name="true_negative", **kwargs): 76 | super(true_negative, self).__init__(name=name, **kwargs) 77 | 78 | def reset_states(self): 79 | """Reset the state of the metric.""" 80 | K.set_value(self.tn, 0) 81 | 82 | def __call__(self, y_true, y_pred): 83 | if self.epsilon is None: 84 | self.init_tensors("tn") 85 | 86 | y_true, y_pred = self.cast(y_true, y_pred) 87 | 88 | neg_y_true = 1 - y_true 89 | neg_y_pred = 1 - y_pred 90 | 91 | tn = K.sum(neg_y_true * neg_y_pred) 92 | current_tn = self.tn * 1 93 | 94 | tn_update = K.update_add(self.tn, tn) 95 | self.add_update(tn_update, inputs=[y_true, y_pred]) 96 | 97 | return tn + current_tn 98 | 99 | 100 | class false_negative(layer): 101 | """Create a metric for model's false negatives amount calculation. 102 | 103 | A false negative is an outcome where the model incorrectly predicts the 104 | negative class. 105 | """ 106 | 107 | def __init__(self, name="false_negative", **kwargs): 108 | super(false_negative, self).__init__(name=name, **kwargs) 109 | 110 | def reset_states(self): 111 | """Reset the state of the metric.""" 112 | K.set_value(self.fn, 0) 113 | 114 | def __call__(self, y_true, y_pred): 115 | if self.epsilon is None: 116 | self.init_tensors("fn") 117 | 118 | y_true, y_pred = self.cast(y_true, y_pred) 119 | neg_y_pred = 1 - y_pred 120 | 121 | fn = K.sum(y_true * neg_y_pred) 122 | current_fn = self.fn * 1 123 | 124 | fn_update = K.update_add(self.fn, fn) 125 | self.add_update(fn_update, inputs=[y_true, y_pred]) 126 | 127 | return fn + current_fn 128 | 129 | 130 | class false_positive(layer): 131 | """Create a metric for model's false positive amount calculation. 132 | 133 | A false positive is an outcome where the model incorrectly predicts the 134 | positive class. 135 | """ 136 | 137 | def __init__(self, name="false_positive", **kwargs): 138 | super(false_positive, self).__init__(name=name, **kwargs) 139 | 140 | def reset_states(self): 141 | """Reset the state of the metric.""" 142 | K.set_value(self.fp, 0) 143 | 144 | def __call__(self, y_true, y_pred): 145 | if self.epsilon is None: 146 | self.init_tensors("fp") 147 | 148 | y_true, y_pred = self.cast(y_true, y_pred) 149 | neg_y_true = 1 - y_true 150 | 151 | fp = K.sum(neg_y_true * y_pred) 152 | current_fp = self.fp * 1 153 | 154 | fp_update = K.update_add(self.fp, fp) 155 | self.add_update(fp_update, inputs=[y_true, y_pred]) 156 | 157 | return fp + current_fp 158 | 159 | 160 | class recall(layer): 161 | """Create a metric for model's recall calculation. 162 | 163 | Recall measures proportion of actual positives that was identified 164 | correctly. 165 | """ 166 | 167 | def __init__(self, name="recall", **kwargs): 168 | super(recall, self).__init__(name=name, **kwargs) 169 | 170 | self.tp = true_positive(**kwargs) 171 | self.fn = false_negative(**kwargs) 172 | 173 | def reset_states(self): 174 | """Reset the state of the metrics.""" 175 | self.tp.reset_states() 176 | self.fn.reset_states() 177 | 178 | def __call__(self, y_true, y_pred): 179 | if self.epsilon is None: 180 | self.init_tensors() 181 | 182 | tp = self.tp(y_true, y_pred) 183 | fn = self.fn(y_true, y_pred) 184 | 185 | self.add_update(self.tp.updates) 186 | self.add_update(self.fn.updates) 187 | 188 | tp = K.cast(tp, self.epsilon.dtype) 189 | fn = K.cast(fn, self.epsilon.dtype) 190 | 191 | return truediv(tp, tp + fn + self.epsilon) 192 | 193 | 194 | class precision(layer): 195 | """Create a metric for model's precision calculation. 196 | 197 | Precision measures proportion of positives identifications that were 198 | actually correct. 199 | """ 200 | 201 | def __init__(self, name="precision", **kwargs): 202 | super(precision, self).__init__(name=name, **kwargs) 203 | 204 | self.tp = true_positive(**kwargs) 205 | self.fp = false_positive(**kwargs) 206 | 207 | def reset_states(self): 208 | """Reset the state of the metrics.""" 209 | self.tp.reset_states() 210 | self.fp.reset_states() 211 | 212 | def __call__(self, y_true, y_pred): 213 | if self.epsilon is None: 214 | self.init_tensors() 215 | 216 | tp = self.tp(y_true, y_pred) 217 | fp = self.fp(y_true, y_pred) 218 | 219 | self.add_update(self.tp.updates) 220 | self.add_update(self.fp.updates) 221 | 222 | tp = K.cast(tp, self.epsilon.dtype) 223 | fp = K.cast(fp, self.epsilon.dtype) 224 | 225 | return truediv(tp, tp + fp + self.epsilon) 226 | 227 | 228 | class f1_score(layer): 229 | """Create a metric for the model's F1 score calculation. 230 | 231 | The F1 score is the harmonic mean of precision and recall. 232 | """ 233 | 234 | def __init__(self, name="f1_score", **kwargs): 235 | super(f1_score, self).__init__(name=name, **kwargs) 236 | 237 | self.precision = precision(**kwargs) 238 | self.recall = recall(**kwargs) 239 | 240 | def reset_states(self): 241 | """Reset the state of the metrics.""" 242 | self.precision.reset_states() 243 | self.recall.reset_states() 244 | 245 | def __call__(self, y_true, y_pred): 246 | if self.epsilon is None: 247 | self.init_tensors() 248 | 249 | pr = self.precision(y_true, y_pred) 250 | rec = self.recall(y_true, y_pred) 251 | 252 | self.add_update(self.precision.updates) 253 | self.add_update(self.recall.updates) 254 | 255 | return 2 * truediv(pr * rec, pr + rec + K.epsilon()) 256 | 257 | 258 | class average_recall(layer): 259 | """Create a metric for the average recall calculation. 260 | """ 261 | 262 | def __init__(self, name="average_recall", labels=1, **kwargs): 263 | super(average_recall, self).__init__(name=name, **kwargs) 264 | 265 | self.labels = labels 266 | 267 | def init_tensors(self, name=None): 268 | self.epsilon = K.constant(K.epsilon(), dtype="float64") 269 | if name and not hasattr(self, name): 270 | setattr(self, name, K.zeros(self.labels, dtype="int32")) 271 | 272 | def reset_states(self): 273 | K.set_value(self.tp, [0]*self.labels) 274 | K.set_value(self.fn, [0]*self.labels) 275 | 276 | def __call__(self, y_true, y_pred): 277 | if self.epsilon is None: 278 | self.init_tensors("tp") 279 | self.init_tensors("fn") 280 | 281 | y_true = K.cast(K.round(y_true), "int32") 282 | y_pred = K.cast(K.round(y_pred), "int32") 283 | neg_y_pred = 1 - y_pred 284 | 285 | tp = K.sum(K.transpose(y_true * y_pred), axis=-1) 286 | fn = K.sum(K.transpose(y_true * neg_y_pred), axis=-1) 287 | 288 | current_tp = K.cast(self.tp + tp, self.epsilon.dtype) 289 | current_fn = K.cast(self.fn + fn, self.epsilon.dtype) 290 | 291 | tp_update = K.update_add(self.tp, tp) 292 | fn_update = K.update_add(self.fn, fn) 293 | 294 | self.add_update(tp_update, inputs=[y_true, y_pred]) 295 | self.add_update(fn_update, inputs=[y_true, y_pred]) 296 | 297 | return K.mean(truediv(current_tp, current_tp + current_fn + self.epsilon)) 298 | --------------------------------------------------------------------------------