├── tests ├── __init__.py ├── metric │ ├── __init__.py │ ├── test_bipartite_soft_rankloss.py │ ├── test_expected_entropy.py │ ├── test_wloss.py │ └── test_precision_at_recall_k.py └── test_probabilistic_scoring_list.py ├── docs ├── _static │ └── .gitkeep ├── _templates │ └── .gitkeep ├── api.rst ├── index.rst ├── Makefile ├── make.bat └── conf.py ├── skpsl ├── helper │ ├── __init__.py │ ├── calibrators │ │ ├── __init__.py │ │ ├── sigmoid_transform.py │ │ └── beta_transform.py │ └── binarization_optimizers.py ├── preprocessing │ ├── __init__.py │ └── binarizer.py ├── __init__.py ├── metrics │ ├── expected_entropy.py │ ├── wloss.py │ ├── __init__.py │ ├── precision_at_recall_k.py │ ├── soft_rankingloss.py │ └── ambiguity_aware_accuracy.py └── estimators │ ├── __init__.py │ ├── probabilistic_scoring_system.py │ ├── probabilistic_scoring_list.py │ └── multiclass_scoring_list.py ├── .gitignore ├── CITATION.cff ├── examples ├── fitting psl on partially continuous data.py ├── simple.py └── using binarizer pipeline.py ├── pyproject.toml ├── LICENSE ├── experiments └── performance.ipynb ├── README.md ├── CHANGELOG.md └── scratch ├── default_value_identification.py ├── default_value_idenficiation.ipynb ├── psl_describe.ipynb └── pretest data ├── data_for_otree_pretest_test.csv └── msl on pretest data.ipynb /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/_static/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/_templates/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/metric/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /skpsl/helper/__init__.py: -------------------------------------------------------------------------------- 1 | from .binarization_optimizers import create_optimizer -------------------------------------------------------------------------------- /skpsl/preprocessing/__init__.py: -------------------------------------------------------------------------------- 1 | from .binarizer import MinEntropyBinarizer 2 | -------------------------------------------------------------------------------- /skpsl/__init__.py: -------------------------------------------------------------------------------- 1 | from skpsl.estimators import ProbabilisticScoringList, MulticlassScoringList 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /dist 3 | .ipynb_checkpoints/ 4 | 5 | /docs/_build/ 6 | __pycache__/ 7 | /data/ -------------------------------------------------------------------------------- /skpsl/helper/calibrators/__init__.py: -------------------------------------------------------------------------------- 1 | from .beta_transform import BetaTransformer 2 | from .sigmoid_transform import SigmoidTransformer -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | API 4 | === 5 | 6 | .. automodule:: skpsl 7 | :members: 8 | :undoc-members: 9 | 10 | .. autoclass:: skpsl.ProbabilisticScoringList 11 | :members: 12 | :undoc-members: 13 | -------------------------------------------------------------------------------- /skpsl/metrics/expected_entropy.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from scipy.stats import entropy 3 | 4 | 5 | def expected_entropy_loss(y_prob, sample_weight=None): 6 | y_prob = np.array(y_prob) 7 | return np.average(entropy([1 - y_prob, y_prob], base=2), weights=sample_weight) 8 | -------------------------------------------------------------------------------- /skpsl/metrics/wloss.py: -------------------------------------------------------------------------------- 1 | from sklearn.metrics import confusion_matrix 2 | 3 | 4 | def weighted_loss(y_true, y_prob, m=10, *, sample_weight=None): 5 | tn, fp, fn, tp = confusion_matrix( 6 | y_true, 1 - y_prob < m * y_prob, sample_weight=sample_weight, normalize="all" 7 | ).ravel() 8 | return fp + m * fn 9 | -------------------------------------------------------------------------------- /skpsl/estimators/__init__.py: -------------------------------------------------------------------------------- 1 | from .probabilistic_scoring_list import ProbabilisticScoringList 2 | from .probabilistic_scoring_system import ProbabilisticScoringSystem 3 | from .multiclass_scoring_list import MulticlassScoringList 4 | 5 | __all__ = ["ProbabilisticScoringSystem", "ProbabilisticScoringList", "MulticlassScoringList"] 6 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "If you use this software, please cite it as below." 3 | license: "MIT" 4 | authors: 5 | - family-names: "Stefan" 6 | given-names: "Heid" 7 | orcid: "https://orcid.org/0000-0002-9461-7372" 8 | - family-names: "Jonas" 9 | given-names: "Hanselle" 10 | orcid: "https://orcid.org/0000-0002-1231-4985" 11 | title: "scikit-psl" 12 | version: 0.7.2 13 | date-released: 2024-10-11 14 | url: "https://github.com/trr318/scikit-psl" 15 | -------------------------------------------------------------------------------- /skpsl/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | from .ambiguity_aware_accuracy import ambiguity_aware_accuracy 2 | from .expected_entropy import expected_entropy_loss 3 | from .precision_at_recall_k import precision_at_recall_k_score 4 | from .soft_rankingloss import soft_ranking_loss 5 | from .wloss import weighted_loss 6 | 7 | __all__ = [ 8 | "expected_entropy_loss", 9 | "weighted_loss", 10 | "precision_at_recall_k_score", 11 | "soft_ranking_loss", 12 | "ambiguity_aware_accuracy" 13 | ] 14 | -------------------------------------------------------------------------------- /tests/metric/test_bipartite_soft_rankloss.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from skpsl.metrics.soft_rankingloss import bipartite_soft_label_ranking_loss 4 | 5 | 6 | @pytest.mark.parametrize( 7 | "y_true,y_score,expected", 8 | [ 9 | ([[0, 1]], [[0.5, 0.5]], 0.5), 10 | ([[0, 0.5]], [[0, 0.1]], 0), 11 | ([[0, 1]], [[0.2, 0]], 1), 12 | ], 13 | ) 14 | def test_expected_entropy_loss_weighted(y_true, y_score, expected): 15 | assert bipartite_soft_label_ranking_loss(y_true, y_score) == expected 16 | -------------------------------------------------------------------------------- /skpsl/helper/calibrators/sigmoid_transform.py: -------------------------------------------------------------------------------- 1 | from sklearn.base import TransformerMixin 2 | from sklearn.exceptions import NotFittedError 3 | from sklearn.linear_model import LogisticRegression 4 | 5 | 6 | class SigmoidTransformer(TransformerMixin): 7 | def __init__(self): 8 | self.clf = None 9 | 10 | def fit(self, X, y): 11 | self.clf = LogisticRegression().fit(X, y) 12 | return self 13 | 14 | def transform(self, X): 15 | if self.clf is None: 16 | raise NotFittedError() 17 | return self.clf.predict_proba(X)[:, 1] 18 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Probabilistic Scoring List classifier documentation master file, created by 2 | sphinx-quickstart on Tue Aug 8 13:43:26 2023. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to Probabilistic Scoring List classifier's documentation! 7 | ================================================================= 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | api.rst 14 | 15 | Indices and tables 16 | ================== 17 | 18 | * :ref:`genindex` 19 | * :ref:`modindex` 20 | * :ref:`search` 21 | -------------------------------------------------------------------------------- /skpsl/metrics/precision_at_recall_k.py: -------------------------------------------------------------------------------- 1 | from operator import itemgetter 2 | 3 | from sklearn.metrics import precision_recall_curve 4 | 5 | 6 | def precision_at_recall_k_score( 7 | y_true, y_prob, *, recall_level=0.9, return_threshold=False 8 | ): 9 | # maximum precision for a given recall level 10 | prec, threshold = max( 11 | ( 12 | (p, t) 13 | for p, r, t in zip(*precision_recall_curve(y_true, y_prob)) 14 | if r >= recall_level 15 | ), 16 | key=itemgetter(0), 17 | ) 18 | if return_threshold: 19 | return prec, threshold 20 | else: 21 | return prec -------------------------------------------------------------------------------- /tests/metric/test_expected_entropy.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from skpsl.metrics import expected_entropy_loss 4 | 5 | 6 | @pytest.mark.parametrize("y_prob,expected", [([0.5, 0.5], 1), ([0.5, 0], 0.5)]) 7 | def test_expected_entropy_loss(y_prob, expected): 8 | assert expected_entropy_loss(y_prob) == expected 9 | 10 | 11 | @pytest.mark.parametrize( 12 | "y_prob,w,expected", 13 | [ 14 | ([0.5, 0], [0.5, 0.5], 0.5), 15 | ([0.5, 0], [1, 0], 1), 16 | ([0.5, 0], [0.2, 0.8], 0.2), 17 | ], 18 | ) 19 | def test_expected_entropy_loss_weighted(y_prob, w, expected): 20 | assert expected_entropy_loss(y_prob, w) == expected 21 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /tests/metric/test_wloss.py: -------------------------------------------------------------------------------- 1 | from skpsl.metrics import precision_at_recall_k_score, weighted_loss 2 | import numpy as np 3 | from pytest import approx 4 | from sklearn.metrics import precision_score 5 | 6 | p, t = weighted_loss( 7 | [0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1], 8 | np.linspace(0, 1, 16), 9 | recall_level=0.5, 10 | return_threshold=True, 11 | ) 12 | assert p == approx(0.833, 0.01) 13 | p = weighted_loss( 14 | [0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1], np.linspace(0, 1, 16) >= t 15 | ) 16 | assert p == approx(0.833, 0.01) 17 | p = weighted_loss( 18 | [0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1], 19 | np.linspace(0, 1, 16), 20 | ) 21 | assert p == approx(0.615, 0.01) 22 | -------------------------------------------------------------------------------- /tests/metric/test_precision_at_recall_k.py: -------------------------------------------------------------------------------- 1 | from skpsl.metrics import precision_at_recall_k_score 2 | import numpy as np 3 | from pytest import approx 4 | from sklearn.metrics import precision_score 5 | 6 | p, t = precision_at_recall_k_score( 7 | [0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1], 8 | np.linspace(0, 1, 16), 9 | recall_level=0.5, 10 | return_threshold=True, 11 | ) 12 | assert p == approx(0.833, 0.01) 13 | p = precision_score( 14 | [0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1], np.linspace(0, 1, 16) >= t 15 | ) 16 | assert p == approx(0.833, 0.01) 17 | p = precision_at_recall_k_score( 18 | [0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1], 19 | np.linspace(0, 1, 16), 20 | ) 21 | assert p == approx(0.615, 0.01) 22 | -------------------------------------------------------------------------------- /examples/fitting psl on partially continuous data.py: -------------------------------------------------------------------------------- 1 | from sklearn.datasets import make_classification 2 | from sklearn.model_selection import train_test_split 3 | 4 | from skpsl import ProbabilisticScoringList 5 | 6 | if __name__ == '__main__': 7 | X, y = make_classification(n_informative=10, random_state=42) 8 | # make 9 | X[:, [9, 10]] = X[:, [9, 10]] > 0 10 | X[:, [2, 6, 7]] = X[:, [2, 6, 7]] > -5 11 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=42) 12 | 13 | psl = ProbabilisticScoringList({-1, 1, 2}) 14 | psl.fit(X_train, y_train) 15 | print(f"Brier score: {psl.score(X_test, y_test):.4f}") 16 | 17 | df = psl.inspect(6) 18 | print(df.to_string(index=False, na_rep="-", justify="center", float_format=lambda x: f"{x:.2f}")) 19 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /skpsl/metrics/soft_rankingloss.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def bipartite_soft_label_ranking_loss(y_true, y_score): 5 | y_true, y_score = np.array(y_true), np.array(y_score) 6 | assert y_true.shape[1] == y_score.shape[1] == 2 7 | t = np.sign(y_true[:, 0] - y_true[:, 1]) 8 | p = np.sign(y_score[:, 0] - y_score[:, 1]) 9 | 10 | # assume that labels contain no ties 11 | assert np.count_nonzero(t) == t.size 12 | 13 | tie_mask = p == 0 14 | 15 | # ties incure a loss of .5 and false orderings a loss of 1 16 | return (0.5 * p[tie_mask].size + np.count_nonzero(p[~tie_mask] != t[~tie_mask])) / y_true.shape[0] 17 | 18 | 19 | def soft_ranking_loss(y_true, y_score): 20 | # bisect ytrue into pos and neg 21 | pos_idx = np.where(y_true == 1)[0] 22 | neg_idx = np.where(y_true != 1)[0] 23 | 24 | # create all pairs of (pos,neg) (as indices) 25 | tuples = np.array(np.meshgrid(pos_idx, neg_idx)).T.reshape(-1, 2) 26 | 27 | return bipartite_soft_label_ranking_loss(y_true[tuples], y_score[tuples]) 28 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "scikit-psl" 3 | version = "0.7.2" 4 | description = "Probabilistic Scoring List classifier" 5 | license = "MIT" 6 | authors = ["Stefan Heid ", "Jonas Hanselle "] 7 | readme = ["README.md", "CHANGELOG.md"] 8 | repository = "https://github.com/trr318/scikit-psl" 9 | packages = [{ include = "skpsl" }] 10 | 11 | [tool.poetry.dependencies] 12 | python = ">=3.10,<3.13" 13 | scikit-learn = "^1.5.2" 14 | numpy = "^1.26.4" 15 | scipy = "^1.14.1" 16 | joblib = "^1.4.2" 17 | pandas = "^2.2.3" 18 | sortedcontainers = "^2.4.0" 19 | pygad = "^3.3.1" 20 | 21 | [tool.poetry.group.dev.dependencies] 22 | sphinx = "^7.2.5" 23 | sphinx_rtd_theme = "^1.3" 24 | black = "^23.11.0" 25 | swig = "^4.2.1.post0" 26 | smac = "^2.2.0" 27 | 28 | pytest = "^7.4" 29 | ipykernel = "^6.29.5" 30 | 31 | 32 | [tool.poetry.group.extra.dependencies] 33 | seaborn = "^0.13.2" 34 | tqdm = "^4.67.1" 35 | 36 | 37 | [tool.poetry.group.experiments.dependencies] 38 | networkx = "^3.4.2" 39 | 40 | [build-system] 41 | requires = ["poetry-core"] 42 | build-backend = "poetry.core.masonry.api" 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) [2021-2022] [Tanja Tornede, Alexander Tornede, Lukas Fehring, Lukas Gehring, Helena Graf, Jonas Hanselle, Felix Mohr, Marcel Wever] 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /examples/simple.py: -------------------------------------------------------------------------------- 1 | from sklearn.datasets import make_classification 2 | from sklearn.model_selection import train_test_split 3 | 4 | from skpsl import ProbabilisticScoringList 5 | 6 | if __name__ == '__main__': 7 | X, y = make_classification(n_informative=10, random_state=42) 8 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=42) 9 | 10 | psl = ProbabilisticScoringList({-1, 1, 2}) 11 | psl.fit(X_train, y_train) 12 | print(f"Brier score: {psl.score(X_test, y_test):.4f}") 13 | """ 14 | Brier score: 0.2438 (lower is better) 15 | """ 16 | 17 | df = psl.inspect(5) 18 | print(df.to_string(index=False, na_rep="-", justify="center", float_format=lambda x: f"{x:.2f}")) 19 | """ 20 | Stage Threshold Score T = -2 T = -1 T = 0 T = 1 T = 2 T = 3 T = 4 T = 5 21 | 0 - - - - 0.51 - - - - - 22 | 1 >-2.4245 2.00 - - 0.00 - 0.63 - - - 23 | 2 >-0.9625 -1.00 - 0.00 0.00 0.48 1.00 - - - 24 | 3 >0.4368 -1.00 0.00 0.00 0.12 0.79 1.00 - - - 25 | 4 >-0.9133 1.00 0.00 0.00 0.12 0.12 0.93 1.00 - - 26 | 5 >2.4648 2.00 0.00 0.00 0.07 0.07 0.92 1.00 1.00 1.00 27 | """ 28 | -------------------------------------------------------------------------------- /examples/using binarizer pipeline.py: -------------------------------------------------------------------------------- 1 | from sklearn.datasets import make_classification 2 | from sklearn.model_selection import train_test_split 3 | from sklearn.pipeline import Pipeline 4 | 5 | from skpsl import ProbabilisticScoringList 6 | from skpsl.preprocessing import MinEntropyBinarizer 7 | 8 | if __name__ == '__main__': 9 | X, y = make_classification(n_informative=10, random_state=42) 10 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=42) 11 | 12 | # The MinEntropyBinarizer calculates an optimal threshold beforehand. 13 | # It will calculate a threshold for binarizing the continuous data 14 | # The optimization criterion is an entropy based impurity measure, similar to the ones used for decision trees. 15 | # The PSL than operates on the binary data 16 | pipe = Pipeline([("binarizer", MinEntropyBinarizer()), 17 | ("psl", ProbabilisticScoringList({-1, 1, 2}))]) 18 | pipe.fit(X_train, y_train) 19 | print(f"Brier score: {pipe.score(X_test, y_test):.4f}") 20 | # > Brier score: 0.2184 (lower is better) 21 | 22 | df = pipe["binarizer"].inspect() 23 | print(df.to_string(index=False, na_rep="-", justify="center", float_format=lambda x: f"{x:.2f}")) 24 | 25 | df = pipe["psl"].inspect(5) 26 | print(df.to_string(index=False, na_rep="-", justify="center", float_format=lambda x: f"{x:.2f}")) 27 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | 6 | # -- Project information ----------------------------------------------------- 7 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 8 | 9 | project = 'Probabilistic Scoring List classifier' 10 | copyright = '2023, "Stefan Heid ", "Jonas Hanselle "' 11 | author = '"Stefan Heid ", "Jonas Hanselle "' 12 | release = '0.1.0' 13 | 14 | # -- General configuration --------------------------------------------------- 15 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 16 | 17 | extensions = ['sphinx_rtd_theme', 18 | 'sphinx.ext.todo', 19 | 'sphinx.ext.viewcode', 20 | 'sphinx.ext.autodoc'] 21 | 22 | templates_path = ['_templates'] 23 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 24 | 25 | # -- Options for HTML output ------------------------------------------------- 26 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 27 | 28 | html_theme = "sphinx_rtd_theme" 29 | html_theme_path = ["_themes", ] 30 | html_static_path = ['_static'] -------------------------------------------------------------------------------- /skpsl/metrics/ambiguity_aware_accuracy.py: -------------------------------------------------------------------------------- 1 | """ 2 | This loss is very similar to accuracy, but it is implemented using predict-proba to detect if there are ties among the most probable class. 3 | if so, its assumed that all tied classes will be predicted and a loss will only be incured it the true class is not among those argmax classes 4 | """ 5 | import numpy as np 6 | 7 | 8 | def ambiguity_aware_accuracy(y_true, y_prob, alpha=0.5): 9 | """ 10 | Computes a weighted accuracy score inversely proportional to the number 11 | of selected classes, modulated by alpha. 12 | 13 | Parameters: 14 | - y_true: np.ndarray of shape (n_samples,), true class indices. 15 | - y_prob: np.ndarray of shape (n_samples, n_classes), predicted probabilities. 16 | - alpha: float, weighting factor for ambiguous cases (0 to 1). 17 | 18 | Returns: 19 | - float, the mean of the computed metric. 20 | """ 21 | # Determine the selected predictions 22 | if len(y_prob.shape) == 1: 23 | y_prob = np.vstack((np.ones_like(y_prob) - y_prob, y_prob)).T 24 | y_pred = y_prob == np.max(y_prob, axis=1, keepdims=True) 25 | 26 | # One-hot encode y_true 27 | true_one_hot = np.zeros_like(y_prob, dtype=bool) 28 | true_one_hot[np.arange(len(y_true)), y_true] = True 29 | 30 | # Cases where the true label is selected 31 | true_selected = y_pred[np.arange(len(y_true)), y_true] 32 | 33 | # Count the number of selected classes for each row 34 | selected_count = y_pred.sum(axis=1) 35 | 36 | # Calculate the metric 37 | scores = np.where( 38 | true_selected, # If the true label is selected 39 | alpha * (1 / selected_count) + (1 - alpha) * 1, 40 | 0 # If the true label is not selected 41 | ) 42 | 43 | return scores.mean() 44 | -------------------------------------------------------------------------------- /experiments/performance.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "f101aee3-ac9c-45c2-8d67-b729f22ea8b0", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "from sklearn.datasets import load_breast_cancer\n", 11 | "from sklearn.model_selection import cross_val_score, ShuffleSplit\n", 12 | "\n", 13 | "from skpsl import ProbabilisticScoringList, MinEntropyBinarizer" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": 2, 19 | "id": "0b07fa87-3c3c-40d4-bca7-605cfe5052c9", 20 | "metadata": {}, 21 | "outputs": [], 22 | "source": [ 23 | "# Generating synthetic data with continuous features and a binary target variable\n", 24 | "\n", 25 | "data = load_breast_cancer()\n", 26 | "X = MinEntropyBinarizer().fit_transform(data.data,data.target)\n", 27 | "y = data.target" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 3, 33 | "id": "3e77f88f-8aa7-4794-9c19-aa9fc73bbb99", 34 | "metadata": {}, 35 | "outputs": [ 36 | { 37 | "name": "stdout", 38 | "output_type": "stream", 39 | "text": [ 40 | "0.0274\n", 41 | "CPU times: user 33.9 ms, sys: 87 ms, total: 121 ms\n", 42 | "Wall time: 3.03 s\n" 43 | ] 44 | } 45 | ], 46 | "source": [ 47 | "%%time\n", 48 | "clf = ProbabilisticScoringList([-1, 1, 2])\n", 49 | "print(f\"{cross_val_score(clf, X, y, fit_params=dict(l=1), cv=ShuffleSplit(5, test_size=.2, random_state=42), n_jobs=-1).mean():.4f}\")" 50 | ] 51 | }, 52 | { 53 | "cell_type": "markdown", 54 | "id": "eb1d191d-38b6-4f1c-8740-7101c9bd192c", 55 | "metadata": {}, 56 | "source": [ 57 | "non-neg 0.2349\n", 58 | "l1 0.0265\n", 59 | "l2 0.0276 20\n", 60 | "l2 0.0315 5\n", 61 | "l3 0.0320 5\n", 62 | "\n", 63 | "### 0.2.0\n", 64 | "- l=2 14min 11s, 0.9599\n", 65 | "- l=1 30s, 0.9604\n", 66 | "\n", 67 | "### 0.1.0\n", 68 | "- 30s, 0.9604" 69 | ] 70 | } 71 | ], 72 | "metadata": { 73 | "kernelspec": { 74 | "display_name": "skpsl", 75 | "language": "python", 76 | "name": "skpsl" 77 | }, 78 | "language_info": { 79 | "codemirror_mode": { 80 | "name": "ipython", 81 | "version": 3 82 | }, 83 | "file_extension": ".py", 84 | "mimetype": "text/x-python", 85 | "name": "python", 86 | "nbconvert_exporter": "python", 87 | "pygments_lexer": "ipython3", 88 | "version": "3.11.3" 89 | } 90 | }, 91 | "nbformat": 4, 92 | "nbformat_minor": 5 93 | } 94 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![License](https://img.shields.io/github/license/trr318/scikit-psl)](https://github.com/trr318/scikit-psl/blob/master/LICENSE) 2 | [![Pip](https://img.shields.io/pypi/v/scikit-psl)](https://pypi.org/project/scikit-psl) 3 | [![Paper](https://img.shields.io/badge/doi-10.1007%2F978--3--031--45275--8__13-green)](https://doi.org/10.1007/978-3-031-45275-8_13) 4 | 5 | 6 | # Probabilistic Scoring Lists 7 | 8 | Probabilistic scoring lists are incremental models that evaluate one feature of the dataset at a time. 9 | PSLs can be seen as a extension to *scoring systems* in two ways: 10 | - they can be evaluated at any stage allowing to trade of model complexity and prediction speed. 11 | - they provide probablistic predictions instead of deterministic decisions for each possible score. 12 | 13 | Scoring systems are used as decision support systems for human experts e.g. in medical or judical decision making. 14 | 15 | This implementation adheres to the [sklearn-api](https://scikit-learn.org/stable/glossary.html#glossary-estimator-types). 16 | 17 | # Install 18 | ```bash 19 | pip install scikit-psl 20 | ``` 21 | 22 | # Usage 23 | 24 | For examples have a look at the `examples` folder, but here is a simple example 25 | 26 | 27 | ```python 28 | from sklearn.datasets import make_classification 29 | from sklearn.model_selection import train_test_split 30 | 31 | from skpsl import ProbabilisticScoringList 32 | 33 | # Generating synthetic data with continuous features and a binary target variable 34 | X, y = make_classification(n_informative=10, random_state=42) 35 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=42) 36 | 37 | psl = ProbabilisticScoringList({-1, 1, 2}) 38 | psl.fit(X_train, y_train) 39 | print(f"Brier score: {psl.score(X_test, y_test, -1):.4f}") 40 | """ 41 | Brier score: 0.2438 (lower is better) 42 | """ 43 | 44 | df = psl.inspect(5) 45 | print(df.to_string(index=False, na_rep="-", justify="center", float_format=lambda x: f"{x:.2f}")) 46 | """ 47 | Stage Threshold Score T = -2 T = -1 T = 0 T = 1 T = 2 T = 3 T = 4 T = 5 48 | 0 - - - - 0.51 - - - - - 49 | 1 >-2.4245 2.00 - - 0.00 - 0.63 - - - 50 | 2 >-0.9625 -1.00 - 0.00 0.00 0.48 1.00 - - - 51 | 3 >0.4368 -1.00 0.00 0.00 0.12 0.79 1.00 - - - 52 | 4 >-0.9133 1.00 0.00 0.00 0.12 0.12 0.93 1.00 - - 53 | 5 >2.4648 2.00 0.00 0.00 0.07 0.07 0.92 1.00 1.00 1.00 54 | """ 55 | ``` 56 | -------------------------------------------------------------------------------- /skpsl/preprocessing/binarizer.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from functools import partial 3 | 4 | import numpy as np 5 | import pandas as pd 6 | from scipy.stats import entropy 7 | from sklearn.base import TransformerMixin, BaseEstimator 8 | from sklearn.exceptions import NotFittedError 9 | 10 | from skpsl.helper import create_optimizer 11 | 12 | logger = logging.getLogger() 13 | 14 | 15 | class MinEntropyBinarizer(BaseEstimator, TransformerMixin, auto_wrap_output_keys=None): 16 | def __init__(self, method="bisect"): 17 | self.method = method 18 | 19 | self.threshs = None 20 | self._optimizer = create_optimizer(method) 21 | 22 | def fit(self, X, y=None): 23 | def binarize(x): 24 | uniq_f_vals = np.unique(x) 25 | is_data_binary = len(set(uniq_f_vals)) <= 2 and set(uniq_f_vals.astype(float)) <= {0.0, 1.0} 26 | if is_data_binary: 27 | return np.nan 28 | return self._optimizer(partial(MinEntropyBinarizer._cut_entropy, y=y), x) 29 | 30 | self.threshs = np.apply_along_axis(binarize, 0, X) 31 | return self 32 | 33 | def transform(self, X): 34 | if self.threshs is None: 35 | raise NotFittedError() 36 | threshs = self.threshs 37 | threshs[np.isnan(threshs)] = .5 38 | return (np.array(X) > threshs).astype(int) 39 | 40 | def inspect(self, feature_names=None) -> pd.DataFrame: 41 | """ 42 | Returns a dataframe that visualizes the internal model 43 | """ 44 | k = len(self.threshs) 45 | df = pd.DataFrame(columns=["Threshold"], data=[f">{t:.4f}" for t in self.threshs]) 46 | if feature_names is not None: 47 | df.insert(0, "Feature", feature_names[:k] + [np.nan] * (k - len(feature_names))) 48 | return df 49 | 50 | @staticmethod 51 | def _cut_entropy(thresh: float, x: np.array, y: np.array) -> float: 52 | """ 53 | https://www.ijcai.org/Proceedings/93-2/Papers/022.pdf 54 | 55 | :param x: one-dimensional float array of the feature variable 56 | :param y: one-dimensional float array of the target variable 57 | :param thresh: scalar 58 | :return: combined entropy as float 59 | """ 60 | mask = x > np.array(thresh).squeeze() 61 | s1, s2 = y[mask], y[~mask] 62 | _, s1_freqs = np.unique(s1, return_counts=True) 63 | _, s2_freqs = np.unique(s2, return_counts=True) 64 | 65 | return (s1.size * entropy(s1_freqs, base=2) + 66 | s2.size * entropy(s2_freqs, base=2)) / x.size 67 | 68 | 69 | if __name__ == '__main__': 70 | from sklearn.datasets import make_classification 71 | 72 | logging.basicConfig() 73 | logger.setLevel(logging.DEBUG) 74 | 75 | x1 = np.linspace(0, 100, 10) 76 | y_ = (x1 > 30).astype(int) 77 | X_ = x1.reshape(-1, 1) 78 | print(np.hstack([X_, y_.reshape(-1, 1)])) 79 | print(MinEntropyBinarizer().fit(X_, y_).inspect()) 80 | 81 | X_, y_ = make_classification(n_samples=50, n_features=3, n_informative=2, n_redundant=0, random_state=42) 82 | print(np.hstack([X_, y_.reshape(-1, 1)])) 83 | print(MinEntropyBinarizer().fit(X_, y_).inspect(feature_names=["width", "height"])) 84 | -------------------------------------------------------------------------------- /skpsl/helper/binarization_optimizers.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from sortedcontainers import SortedSet 3 | 4 | 5 | def create_optimizer(method: str): 6 | match method: 7 | case "bisect": 8 | return binary_search_optimizer 9 | case "brute": 10 | return brute_search_optimizer 11 | case _: 12 | ValueError(f'No optimizer "{method}" defined. Choose "bisect" or "brute"') 13 | 14 | 15 | def binary_search_optimizer(func: callable, data: np.array, minimize=True) -> float: 16 | """ 17 | This algorithm employs a hierarchical logarithmic search to find the local minimum of a parametrized metric. 18 | 19 | Parameter data is used to calculate potential threshold values used for cutting data. 20 | It is passed to func without changes. The function is assumed to be pseudo convex in x. 21 | This allows for the hierarchical logarithmic search to find a global optimum efficiently. 22 | 23 | :param data: one-dimensional float array of the feature variable 24 | :param func: Metric must have the signature (x, data) and return a score to be minimized 25 | :return: optimal threshold 26 | """ 27 | values = np.sort(np.unique(data)) 28 | # Adding the extremal values max might not be necessary, but it is not trivial to proof. 29 | cuts = np.concatenate([[data.min() - 1], (values[:-1] + values[1:]) / 2, [data.max() + 1]]) 30 | minimal_points = set() 31 | sgn = (-1) ** int(not minimize) 32 | min_ = np.inf 33 | thresh = None 34 | evaluated = SortedSet() 35 | to_evaluate = {0, cuts.size - 1} 36 | 37 | while to_evaluate: 38 | # evaluate points 39 | while to_evaluate: 40 | k = to_evaluate.pop() 41 | evaluated.add(k) 42 | value = sgn * func(cuts[k], data) 43 | if value < min_: 44 | min_ = value 45 | thresh = cuts[k] 46 | minimal_points = {k} 47 | elif value == min_: 48 | minimal_points.add(k) 49 | 50 | # calculate new points to evaluate 51 | for k in minimal_points: 52 | k_pos = evaluated.index(k) 53 | candidates = set() 54 | for offset in [-1, 1]: 55 | try: 56 | candidates.add(k + (evaluated[k_pos + offset] - k) // 2) 57 | except IndexError: 58 | pass 59 | to_evaluate.update(candidates - evaluated) 60 | return thresh 61 | 62 | 63 | def brute_search_optimizer(func: callable, data: np.array, minimize=True) -> float: 64 | """ 65 | This algorithm employs a brute force search to find the global minimum of a parametrized metric. 66 | 67 | Parameter data is used to calculate potential threshold values used for cutting data. 68 | It is passed to func without changes. The function is assumed to be pseudo convex in x. 69 | This allows for the hierarchical logarithmic search to find a global optimum efficiently. 70 | 71 | :param data: one-dimensional float array of the feature variable 72 | :param func: Metric must have the signature (x, data) and return a score to be minimized 73 | :return: optimal threshold 74 | """ 75 | values = np.sort(np.unique(data)) 76 | # Adding the extremal values max might not be necessary, but it is not trivial to proof. 77 | cuts = np.concatenate([[data.min() - 1], (values[:-1] + values[1:]) / 2, [data.max() + 1]]) 78 | sgn = (-1) ** int(not minimize) 79 | 80 | thresh = cuts[np.argmin([sgn * func(cut, data) for cut in cuts])] 81 | 82 | return thresh 83 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | 9 | 10 | ## 0.7.2 - 2024-10-11 11 | 12 | ### Fixed 13 | 14 | - MaxEntropyBinarizer also uses stricter test to detect binary features 15 | 16 | ## 0.7.1 - 2024-10-09 17 | 18 | ### Fixed 19 | 20 | - PSL uses stricter test to detect binary features 21 | 22 | 23 | ## 0.7.0 - 2024-03-27 24 | 25 | ### Added 26 | 27 | - PSL classifier 28 | - probability predictions can return confidence intervals 29 | - probability calibration using BetaCalibration 30 | - stages can now be sliced and iterated (\__getitem\__(), \__iter\__()) 31 | - Metrics 32 | - Weighted loss metric 33 | - Rank loss metric 34 | 35 | ### Fixed 36 | 37 | - PSL more robust against non-standard class labels like "True"/"False" instead of boolean values 38 | 39 | ## 0.6.3 - 2024-03-08 40 | 41 | ### Added 42 | 43 | - PSL supports Dataframes as inputs 44 | 45 | ## 0.6.2 - 2024-02-01 46 | 47 | ### Added 48 | 49 | - PSL supports instance weights 50 | 51 | ## 0.6.1 - 2023-12-15 52 | 53 | ### Added 54 | 55 | - Extended precision at recall function 56 | 57 | ## 0.6.0 - 2023-12-07 58 | 59 | ### Added 60 | 61 | - Significantly extended the configuration capabilities with predefined features to limit the PSLs searchspace 62 | 63 | ### Changed 64 | 65 | - PSL global loss defaults to sum(cascade) 66 | - rewrote/extracted expected entropy calculation 67 | 68 | ### Fixed 69 | 70 | - PSL inspect is now more robust 71 | 72 | ## 0.5.1 - 2023-11-24 73 | 74 | ### Fixed 75 | 76 | - PSL classifier optimization regarding global loss was incorrect 77 | 78 | ## 0.5.0 - 2023-11-16 79 | 80 | ### Added 81 | 82 | - _ClassifierAtK 83 | - Sigmoid calibration additional to isotonic 84 | - PSL classifier 85 | - Make optimization loss configurable 86 | - Small `searchspace_analyisis(·)` function makes lookahead choice more informed 87 | 88 | ### Fixed 89 | 90 | - Fixed lookahead search space and considering global loss for model-sequence evaluation 91 | 92 | ### Changed 93 | 94 | - Updated dependencies and added black 95 | - Moved Binarizer to different module 96 | - Moved PSL hyperparameters to constructor 97 | 98 | ## 0.4.2 - 2023-11-09 99 | 100 | ### Fixed 101 | 102 | - _ClassifierAtK 103 | - Expected entropy for stage 0 now also calculated wrt. base 2 104 | - Data with only 0 or 1 is now also interpret as binary data 105 | 106 | ## 0.4.1 - 2023-10-19 107 | 108 | ### Fixed 109 | 110 | - Small import error 111 | 112 | ## 0.4.0 - 2023-10-17 113 | 114 | ### Added 115 | 116 | - Add brute force threshold optimization method to find the global optimum, bisect optimizer remains default method 117 | 118 | ### Changed 119 | 120 | - Restructured source files 121 | 122 | ## 0.3.1 - 2023-09-12 123 | 124 | ### Fixed 125 | 126 | - PSL is now correctly handles when all instances belong to the negative class 127 | - [#1](../../issues/1) if the first feature is assigned a negative score, it is now assigned the most negative score 128 | 129 | ## 0.3.0 - 2023-08-10 130 | 131 | ### Added 132 | 133 | - PSL classifier can now run with continuous data and optimally (wrt. expected entropy) select thresholds to binarize 134 | the data 135 | 136 | ### Changed 137 | 138 | - Significantly improved optimum calculation for MinEntropyBinarizer (the same optimization algorithm is shared with the 139 | psls internal binarization algorithm) 140 | 141 | ## 0.2.0 - 2023-08-10 142 | 143 | ### Added 144 | 145 | - PSL classifier 146 | - introduced parallelization 147 | - implemented l-step lookahead 148 | - simple inspect(·) method that creates a tabular representation of the model 149 | 150 | ## 0.1.0 - 2023-08-08 151 | 152 | ### Added 153 | 154 | - Initial implementation of the PSL algorithm 155 | -------------------------------------------------------------------------------- /tests/test_probabilistic_scoring_list.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | import pytest 4 | from sklearn.datasets import make_classification 5 | from sklearn.model_selection import train_test_split, cross_val_score 6 | from sklearn.pipeline import Pipeline 7 | 8 | from skpsl import ProbabilisticScoringList 9 | from skpsl.preprocessing import MinEntropyBinarizer 10 | 11 | 12 | @pytest.fixture 13 | def numpy_randomness(): 14 | np.random.seed(0) 15 | 16 | 17 | def test_binary_data(): 18 | X, y = make_classification(random_state=42) 19 | X = (X > 0.5).astype(int) 20 | 21 | psl = ProbabilisticScoringList({-1, 1, 2}) 22 | 23 | X_train, X_test, y_train, y_test = train_test_split( 24 | X, y, test_size=0.2, random_state=42 25 | ) 26 | psl.fit(X_train, y_train) 27 | assert psl.thresholds == [np.nan] * X.shape[1] 28 | 29 | 30 | def test_inspect(): 31 | # Generating synthetic data with continuous features and a binary target variable 32 | X, y = make_classification(n_features=5, random_state=42) 33 | X = (X > 0.5).astype(int) 34 | 35 | psl = ProbabilisticScoringList({-1, 1, 2}) 36 | 37 | X_train, X_test, y_train, y_test = train_test_split( 38 | X, y, test_size=0.2, random_state=42 39 | ) 40 | psl.fit(X_train, y_train) 41 | df = psl.inspect(3) 42 | print( 43 | df.to_string( 44 | index=False, na_rep="-", justify="center", float_format=lambda x: f"{x:.2f}" 45 | ) 46 | ) 47 | 48 | 49 | def test_improvement_for_internalized_binarization(): 50 | X, y = make_classification(n_features=6, n_informative=4, random_state=42) 51 | 52 | clf_ = Pipeline( 53 | [ 54 | ("binarizer", MinEntropyBinarizer()), 55 | ("psl", ProbabilisticScoringList({-1, 1, 2})), 56 | ] 57 | ) 58 | pipe_score = cross_val_score(clf_, X, y, cv=5).mean() 59 | 60 | clf_ = ProbabilisticScoringList({-1, 1, 2}) 61 | psl_score = cross_val_score(clf_, X, y, cv=5).mean() 62 | 63 | # lower score is better 64 | # the internalized binarization should perform better 65 | assert psl_score < pipe_score 66 | 67 | 68 | def test_gh1_maximally_negative_score_for_first_feature(): 69 | X, y = make_classification(n_features=6, n_informative=4, random_state=42) 70 | X = (X > 0.5).astype(int) 71 | y = 1 - X[:, 2] 72 | psl = ProbabilisticScoringList({-2, -1, 1, 2}) 73 | psl.fit(X, y) 74 | assert psl.scores[0] == -2 75 | 76 | 77 | def test_only_negative_classes(): 78 | X, y = make_classification(n_features=6, n_informative=4, random_state=42) 79 | y.fill(0) 80 | psl = ProbabilisticScoringList({-2, -1, 1, 2}) 81 | psl.fit(X, y) 82 | assert not np.isnan(psl.stage_clfs[0].score(X)) 83 | 84 | 85 | def test_sample_weight(): 86 | X, y = make_classification( 87 | n_samples=10, n_features=6, n_informative=4, random_state=42 88 | ) 89 | psl = ProbabilisticScoringList({-1, 1}) 90 | weighted = psl.fit(X, y).score( 91 | X, 92 | y, 93 | sample_weight=np.random.exponential( 94 | X.shape[0], 95 | ), 96 | ) 97 | unweighted = psl.fit(X, y).score(X, y) 98 | assert unweighted != weighted 99 | 100 | 101 | def test_dataframe(): 102 | X, y = make_classification( 103 | n_samples=10, n_features=6, n_informative=4, random_state=42 104 | ) 105 | X = pd.DataFrame(X) 106 | y = pd.Series(y) 107 | 108 | psl = ProbabilisticScoringList({-1, 1}) 109 | psl.fit(X, y).score(X, y) 110 | 111 | 112 | 113 | def test_predef(): 114 | from sklearn.datasets import make_classification 115 | from sklearn.model_selection import cross_val_score 116 | 117 | # Generating synthetic data with continuous features and a binary target variable 118 | X_, y_ = make_classification(random_state=42) 119 | 120 | clf_ = ProbabilisticScoringList({-1, 1, 2}, lookahead=2) 121 | clf_.fit(X_, y_, predef_features=[3, 2, 1], predef_scores=[2, 2, 1], strict=True) 122 | -------------------------------------------------------------------------------- /scratch/default_value_identification.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | # sys.path.insert(0, "/dss/dsshome1/04/ra43rid2/msl/scikit-psl") 4 | 5 | import pandas as pd 6 | from sklearn.linear_model import LogisticRegression 7 | from sklearn.model_selection import cross_validate, cross_val_score 8 | from sklearn.metrics import balanced_accuracy_score, make_scorer 9 | from sklearn.datasets import load_iris 10 | from sklearn.model_selection import train_test_split 11 | from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay 12 | from sklearn.pipeline import make_pipeline 13 | import numpy as np 14 | from skpsl.preprocessing import MinEntropyBinarizer 15 | from skpsl.estimators import MultinomialScoringList 16 | from matplotlib import pyplot as plt 17 | 18 | from sklearn.datasets import load_iris, load_wine 19 | 20 | from ConfigSpace import Configuration, ConfigurationSpace 21 | from smac import HyperparameterOptimizationFacade, Scenario 22 | from math import ceil, floor, sqrt 23 | from statistics import fmean 24 | 25 | from joblib import Parallel, delayed 26 | 27 | if __name__ == "__main__": 28 | df = pd.read_csv("../data/player_processed.csv") 29 | X_soccer = df.iloc[:, 1:] 30 | y_soccer = df.iloc[:, 0] 31 | 32 | X_iris, y_iris = load_iris(return_X_y=True) 33 | X_wine, y_wine = load_wine(return_X_y=True) 34 | 35 | def evaluate(config: Configuration, seed: int = 0) -> float: 36 | 37 | def evaluate_for_dataset(name, X, y): 38 | num_features = X.shape[1] 39 | num_classes = len(np.unique(y)) 40 | num_scores_to_assign = num_features * num_classes 41 | args = dict(config) 42 | args["crossover_type"] = str(args["crossover_type"]) 43 | parents_mating_factor = args.pop("parents_mating_factor") 44 | mutation_num_genes = ( 45 | args.pop("mutation_num_genes_factor") * num_scores_to_assign 46 | ) 47 | init_pop_factor_minus_mating = args.pop("init_pop_factor_minus_mating") 48 | 49 | ga_params = args 50 | ga_params["popsize"] = ceil(sqrt(num_scores_to_assign)) 51 | ga_params["parents_mating"] = floor( 52 | parents_mating_factor * ga_params["popsize"] 53 | ) 54 | ga_params["init_pop_factor"] = ( 55 | init_pop_factor_minus_mating + parents_mating_factor 56 | ) 57 | ga_params["mutation_num_genes"] = floor(mutation_num_genes) 58 | 59 | return ( 60 | name, 61 | -cross_val_score( 62 | MultinomialScoringList( 63 | score_set=set(range(-3, 4)), 64 | ga_params=ga_params, 65 | random_state=seed, 66 | cascade_loss=fmean, 67 | ), 68 | X, 69 | y, 70 | n_jobs=-1, 71 | ).mean(), 72 | ) 73 | 74 | names = ["soccer", "iris", "wine"] 75 | data = [(X_soccer, y_soccer), (X_iris, y_iris), (X_wine, y_wine)] 76 | results = Parallel(n_jobs=-1)( 77 | delayed(evaluate_for_dataset)(name, X, y) 78 | for name, (X, y) in zip(names, data) 79 | ) 80 | 81 | return dict(results) 82 | 83 | cs = ConfigurationSpace( 84 | { 85 | "crossover_type": ["single_point", "two_points", "uniform", "scattered"], 86 | "init_pop_factor_minus_mating": (0.1, 5.0), 87 | "parents_mating_factor": (0.3, 0.7), 88 | "init_pop_noise": (0.05, 0.5), 89 | "mutation_num_genes_factor": (0.1, 0.5), 90 | "maxiter": [100], 91 | } 92 | ) 93 | 94 | scenario = Scenario( 95 | cs, 96 | deterministic=False, 97 | objectives=["soccer", "iris", "wine"], 98 | cputime_limit=72000, 99 | n_trials=250, 100 | ) 101 | smac = HyperparameterOptimizationFacade( 102 | scenario, 103 | evaluate, 104 | overwrite=False, 105 | multi_objective_algorithm=HyperparameterOptimizationFacade.get_multi_objective_algorithm( 106 | scenario, objective_weights=[1, 1, 1] 107 | ), 108 | ) 109 | smac.optimize() 110 | -------------------------------------------------------------------------------- /skpsl/helper/calibrators/beta_transform.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | 3 | import numpy as np 4 | from scipy.optimize import minimize_scalar 5 | from sklearn.base import TransformerMixin, BaseEstimator 6 | from sklearn.exceptions import NotFittedError 7 | from sklearn.linear_model import LogisticRegression 8 | from sklearn.preprocessing import MinMaxScaler 9 | from sklearn.utils import column_or_1d, indexable 10 | 11 | 12 | class BetaTransformer(BaseEstimator, TransformerMixin): 13 | """ 14 | Adapted from: 15 | 16 | Beta regression model with three parameters introduced in 17 | Kull, M., Silva Filho, T.M. and Flach, P. Beta calibration: a well-founded 18 | and easily implemented improvement on logistic calibration for binary 19 | classifiers. AISTATS 2017. 20 | 21 | Attributes 22 | ---------- 23 | map_ : array-like, shape (3,) 24 | Array containing the coefficients of the model (a and b) and the 25 | midpoint m. Takes the form map_ = [a, b, m] 26 | 27 | lr_ : sklearn.linear_model.LogisticRegression 28 | Internal logistic regression used to train the model. 29 | """ 30 | 31 | def __init__(self, penalty=None): 32 | self.penalty = penalty 33 | 34 | self.scaler_ = None 35 | self.lr_ = None 36 | self.map_ = None 37 | 38 | def fit(self, X, y, sample_weight=None): 39 | """Fit the model using X, y as training data. 40 | 41 | Parameters 42 | ---------- 43 | X : array-like, shape (n_samples,) 44 | Training data. 45 | 46 | y : array-like, shape (n_samples,) 47 | Training target. 48 | 49 | sample_weight : array-like, shape = [n_samples] or None 50 | Sample weights. If None, then samples are equally weighted. 51 | 52 | Returns 53 | ------- 54 | self : object 55 | Returns an instance of self. 56 | """ 57 | 58 | self.scaler_ = MinMaxScaler().fit(X.astype(float), y) 59 | X = self.scaler_.transform(X) 60 | X = column_or_1d(X) 61 | y = column_or_1d(y) 62 | X, y = indexable(X, y) 63 | warnings.filterwarnings("ignore"), 64 | 65 | df = column_or_1d(X).reshape(-1, 1) 66 | eps = np.finfo(df.dtype).eps 67 | df = np.clip(df, eps, 1 - eps) 68 | y = column_or_1d(y) 69 | 70 | x = np.hstack((df, 1.0 - df)) 71 | x = np.log(x) 72 | x[:, 1] *= -1 73 | 74 | lr = LogisticRegression(penalty=self.penalty) 75 | lr.fit(x, y, sample_weight) 76 | coefs = lr.coef_[0] 77 | 78 | if coefs[0] < 0: 79 | x = x[:, 1].reshape(-1, 1) 80 | lr = LogisticRegression(penalty=self.penalty) 81 | lr.fit(x, y, sample_weight) 82 | coefs = lr.coef_[0] 83 | a = None 84 | b = coefs[0] 85 | elif coefs[1] < 0: 86 | x = x[:, 0].reshape(-1, 1) 87 | lr = LogisticRegression(penalty=self.penalty) 88 | lr.fit(x, y, sample_weight) 89 | coefs = lr.coef_[0] 90 | a = coefs[0] 91 | b = None 92 | else: 93 | a = coefs[0] 94 | b = coefs[1] 95 | inter = lr.intercept_[0] 96 | 97 | a_, b_ = a or 0, b or 0 98 | m = minimize_scalar( 99 | lambda mh: np.abs(b_ * np.log(1.0 - mh) - a_ * np.log(mh) - inter), 100 | bounds=[0, 1], 101 | method="Bounded", 102 | ).x 103 | 104 | self.map_, self.lr_ = [a, b, m], lr 105 | 106 | return self 107 | 108 | def predict(self, S): 109 | """Predict new values. 110 | 111 | Parameters 112 | ---------- 113 | S : array-like, shape (n_samples,) 114 | Data to predict from. 115 | 116 | Returns 117 | ------- 118 | S_ : array, shape (n_samples,) 119 | The predicted values. 120 | """ 121 | df = column_or_1d(S).reshape(-1, 1) 122 | eps = np.finfo(df.dtype).eps 123 | df = np.clip(df, eps, 1 - eps) 124 | 125 | x = np.hstack((df, 1.0 - df)) 126 | x = np.log(x) 127 | x[:, 1] *= -1 128 | if self.map_[0] is None: 129 | x = x[:, 1].reshape(-1, 1) 130 | elif self.map_[1] is None: 131 | x = x[:, 0].reshape(-1, 1) 132 | 133 | return self.lr_.predict_proba(x)[:, 1] 134 | 135 | def transform(self, X): 136 | if self.lr_ is None: 137 | raise NotFittedError() 138 | return self.predict(self.scaler_.transform(X.astype(float))) 139 | -------------------------------------------------------------------------------- /scratch/default_value_idenficiation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 66, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import pandas as pd\n", 10 | "from sklearn.linear_model import LogisticRegression\n", 11 | "from sklearn.model_selection import cross_validate, cross_val_score\n", 12 | "from sklearn.metrics import balanced_accuracy_score, make_scorer\n", 13 | "from sklearn.datasets import load_iris\n", 14 | "from sklearn.model_selection import train_test_split\n", 15 | "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n", 16 | "from sklearn.pipeline import make_pipeline\n", 17 | "import numpy as np\n", 18 | "from skpsl.preprocessing import MinEntropyBinarizer\n", 19 | "from skpsl.estimators import MultinomialScoringList\n", 20 | "from matplotlib import pyplot as plt\n", 21 | "\n", 22 | "\n", 23 | "from ConfigSpace import Configuration, ConfigurationSpace\n", 24 | "from smac import HyperparameterOptimizationFacade, Scenario" 25 | ] 26 | }, 27 | { 28 | "cell_type": "code", 29 | "execution_count": 67, 30 | "metadata": {}, 31 | "outputs": [], 32 | "source": [ 33 | "df = pd.read_csv(\"data/player_processed.csv\")\n", 34 | "X = df.iloc[:,1:]\n", 35 | "y = df.iloc[:,0]" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 71, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [ 44 | "def evaluate(config: Configuration, seed: int=0) -> float:\n", 45 | " args = dict(config)\n", 46 | " args[\"crossover_type\"] = str(args[\"crossover_type\"])\n", 47 | " score_set = set(args.pop(\"score_set\"))\n", 48 | " ga_params = args\n", 49 | " clf = MultinomialScoringList(score_set=score_set, ga_params=ga_params, random_state=seed).fit(X,y)\n", 50 | " return clf.score(X,y)" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 72, 56 | "metadata": {}, 57 | "outputs": [], 58 | "source": [ 59 | "cs = ConfigurationSpace({\"popsize\":(10,50),\"crossover_type\":[\"single_point\", \"two_points\", \"uniform\", \"scattered\"],\"parents_mating\":(2,10), \"maxiter\":[250],\"score_set\":[[-3,-2,-1,1,2,3]]})" 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": 73, 65 | "metadata": {}, 66 | "outputs": [ 67 | { 68 | "name": "stderr", 69 | "output_type": "stream", 70 | "text": [ 71 | "/home/jonas/Documents/Research/msl/scikit-psl/.venv/lib/python3.11/site-packages/pygad/pygad.py:1139: UserWarning: The 'delay_after_gen' parameter is deprecated starting from PyGAD 3.3.0. To delay or pause the evolution after each generation, assign a callback function/method to the 'on_generation' parameter to adds some time delay.\n", 72 | " warnings.warn(\"The 'delay_after_gen' parameter is deprecated starting from PyGAD 3.3.0. To delay or pause the evolution after each generation, assign a callback function/method to the 'on_generation' parameter to adds some time delay.\")\n" 73 | ] 74 | }, 75 | { 76 | "data": { 77 | "text/plain": [ 78 | "7.29410135696483" 79 | ] 80 | }, 81 | "execution_count": 73, 82 | "metadata": {}, 83 | "output_type": "execute_result" 84 | } 85 | ], 86 | "source": [ 87 | "evaluate(cs.sample_configuration())" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [ 96 | "scenario = Scenario(cs, deterministic=True, n_trials=200)" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": null, 102 | "metadata": {}, 103 | "outputs": [ 104 | { 105 | "name": "stdout", 106 | "output_type": "stream", 107 | "text": [ 108 | "[INFO][abstract_initial_design.py:147] Using 50 initial design configurations and 0 additional configurations.\n" 109 | ] 110 | } 111 | ], 112 | "source": [ 113 | "smac = HyperparameterOptimizationFacade(scenario, evaluate)" 114 | ] 115 | } 116 | ], 117 | "metadata": { 118 | "kernelspec": { 119 | "display_name": ".venv", 120 | "language": "python", 121 | "name": "python3" 122 | }, 123 | "language_info": { 124 | "codemirror_mode": { 125 | "name": "ipython", 126 | "version": 3 127 | }, 128 | "file_extension": ".py", 129 | "mimetype": "text/x-python", 130 | "name": "python", 131 | "nbconvert_exporter": "python", 132 | "pygments_lexer": "ipython3", 133 | "version": "3.11.5" 134 | } 135 | }, 136 | "nbformat": 4, 137 | "nbformat_minor": 2 138 | } 139 | -------------------------------------------------------------------------------- /skpsl/estimators/probabilistic_scoring_system.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from collections import defaultdict 3 | from typing import Optional, Union 4 | 5 | import numpy as np 6 | from scipy.stats import beta 7 | from sklearn.base import BaseEstimator, ClassifierMixin, TransformerMixin 8 | from sklearn.exceptions import NotFittedError 9 | from sklearn.isotonic import IsotonicRegression 10 | 11 | from skpsl.helper.calibrators import BetaTransformer, SigmoidTransformer 12 | from skpsl.metrics import expected_entropy_loss 13 | 14 | 15 | class ProbabilisticScoringSystem(ClassifierMixin, BaseEstimator): 16 | """ 17 | Internal class for the classifier at stage k of the probabilistic scoring list 18 | """ 19 | 20 | def __init__( 21 | self, 22 | features: list[int], 23 | scores: list[int], 24 | initial_feature_thresholds: Optional[list[Optional[float]]] = None, 25 | threshold_optimizer: Optional[callable] = None, 26 | calibration_method="isotonic", 27 | balance_class_weights=False, 28 | loss_function=None, 29 | ): 30 | """ 31 | Regardless of the stage-loss, thresholds are optimized with respect to expected entropy 32 | 33 | :param features: tuple of feature indices. used for selecting data from X 34 | :param scores: tuple of scores, corresponding to the feature indices 35 | :param initial_feature_thresholds: tuple of thresholds to binarize the feature values 36 | """ 37 | self.features = features 38 | self.scores = scores 39 | self.initial_feature_thresholds = initial_feature_thresholds 40 | self.threshold_optimizer = threshold_optimizer 41 | self.calibration_method = calibration_method 42 | self.balance_class_weights = balance_class_weights 43 | self.loss_function = loss_function 44 | 45 | match loss_function: 46 | case None: 47 | self.loss_function_ = ( 48 | lambda _, y_prob, sample_weight=None: expected_entropy_loss( 49 | y_prob, sample_weight=sample_weight 50 | ) 51 | ) 52 | case _: 53 | self.loss_function_ = lambda true, prob, sample_weight=None: ( 54 | loss_function(true, prob) 55 | if sample_weight is None 56 | else loss_function(true, prob, sample_weight) 57 | ) 58 | 59 | self.classes_ = None 60 | self.scores_ = np.array(scores) 61 | self.feature_thresholds = ( 62 | initial_feature_thresholds 63 | if initial_feature_thresholds is not None 64 | else [None] * len(features) 65 | ) 66 | self.logger = logging.getLogger(__name__) 67 | self.calibrator = None 68 | self.class_counts_per_score = None 69 | self.decision_threshold = 0.5 70 | 71 | def fit(self, X, y, sample_weight=None) -> "ProbabilisticScoringSystem": 72 | X, y = np.array(X), np.array(y) 73 | 74 | self.classes_ = np.unique(y) 75 | y = np.array(y == self.classes_[1], dtype=int) 76 | 77 | if self.balance_class_weights: 78 | n = y.size 79 | n_pos = np.count_nonzero(y == 1) 80 | self.decision_threshold = n_pos / n 81 | 82 | for i, (f, t) in enumerate(zip(self.features, self.feature_thresholds)): 83 | feature_values = X[:, f] 84 | uniq_f_vals = np.unique(feature_values) 85 | is_data_binary = len(set(uniq_f_vals)) <= 2 and set(uniq_f_vals.astype(float)) <= {0.0, 1.0} 86 | if (t is np.nan or t is None) and not is_data_binary: 87 | self.logger.debug( 88 | f"feature {f} is non-binary and threshold not set: calculating threshold..." 89 | ) 90 | if self.threshold_optimizer is None: 91 | raise ValueError( 92 | "threshold_optimizer mustn't be None, when non-binary features with unset " 93 | "thresholds exist." 94 | ) 95 | 96 | # fit optimal threshold 97 | # Note: The threshold is optimized with the expected entropy, regardless of the stageloss used in the PSL 98 | self.feature_thresholds[i] = self.threshold_optimizer( 99 | lambda t_, _: self.loss_function_( 100 | y, 101 | self._create_calibrator().fit_transform( 102 | self._compute_total_scores( 103 | X, 104 | self.features[: i + 1], 105 | self.scores_[: i + 1], 106 | self.feature_thresholds[:i] + [t_], 107 | ), 108 | y, 109 | ), 110 | sample_weight=sample_weight, 111 | ), 112 | feature_values, 113 | ) 114 | 115 | total_scores = ProbabilisticScoringSystem._compute_total_scores( 116 | X, self.features, self.scores_, self.feature_thresholds 117 | ) 118 | self.calibrator = self._create_calibrator().fit(total_scores, y) 119 | uniq_total_scores, idx = np.unique(total_scores, return_inverse=True) 120 | self.class_counts_per_score = defaultdict(lambda: {0: 0, 1: 0}) | { 121 | int(score): {0: 0, 1: 0} 122 | | { 123 | c_: count 124 | for c_, count in zip(*np.unique(y[idx == i], return_counts=True)) 125 | } 126 | for i, score in enumerate(uniq_total_scores) 127 | } 128 | 129 | return self 130 | 131 | def _create_calibrator(self): 132 | if ( 133 | hasattr(self.calibration_method, "fit") 134 | and hasattr(self.calibration_method, "transform") 135 | and isinstance(self.calibration_method, TransformerMixin) 136 | ): 137 | return self.calibration_method 138 | 139 | # compute probabilities 140 | match self.calibration_method: 141 | case "isotonic": 142 | return IsotonicRegression( 143 | y_min=0.0, y_max=1.0, increasing=True, out_of_bounds="clip" 144 | ) 145 | case "sigmoid": 146 | return SigmoidTransformer() 147 | case "beta": 148 | return BetaTransformer() 149 | case "beta_reg": 150 | return BetaTransformer(penalty="l2") 151 | case _: 152 | raise ValueError( 153 | f'Calibration method "{self.calibration_method}" doesn\'t exist. ' 154 | 'Did you mean "isotonic" or "sigmoid"' 155 | ) 156 | 157 | def predict(self, X): 158 | if self.calibrator is None: 159 | raise NotFittedError() 160 | return self.classes_[ 161 | np.array(self.predict_proba(X)[:, 1] > self.decision_threshold, dtype=int) 162 | ] 163 | 164 | def predict_proba(self, X, ci: Optional[Union[float, tuple]] = None): 165 | """ 166 | Predicts the probability for the given datapoint 167 | :param X: 168 | :param ci: if given, it will return triples of probabilities (lower, proba, upper) with Clopper-Pearson confidence intervals. 169 | The confidence interval must be between 0 and 1. It can also be a tuple of confidence intervals, one for the lower bound, one for the higher. 170 | """ 171 | if self.calibrator is None: 172 | raise NotFittedError() 173 | scores = self._compute_total_scores( 174 | X, self.features, self.scores_, self.feature_thresholds 175 | ) 176 | sigma, idx = np.unique(scores, return_inverse=True) 177 | ps = self.calibrator.transform(sigma.reshape(-1, 1)) 178 | 179 | if ci is None: 180 | proba_pos = ps[idx] 181 | return np.vstack([1 - proba_pos, proba_pos]).T 182 | if isinstance(ci, float): 183 | # binomial proportion ci bounds, scaled by bonferroni correction 184 | al = au = (1 - ci) / len(sigma) 185 | else: 186 | assert isinstance(ci, (tuple, list)) and len(ci) == 2 187 | al, au = (1 - ci[0]) / len(sigma), (1 - ci[1]) / len(sigma) 188 | 189 | # https://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval#Clopper%E2%80%93Pearson_interval 190 | ls, us = [], [] 191 | for p, i_ in zip(ps, sigma): 192 | c = self.class_counts_per_score[int(i_)] 193 | neg, pos = c[0], c[1] 194 | 195 | l, u = beta.ppf([al / 2, 1 - au / 2], [pos, pos + 1], [neg + 1, neg]) 196 | # make sure the bounds are sensible wrt. proba estimate 197 | ls.append(min(np.nan_to_num(l, nan=0), p)) 198 | us.append(max(np.nan_to_num(u, nan=1), p)) 199 | 200 | # make sure the bounds are monotonic in the scoreset 201 | ls = np.array([max([l] + ls[:i]) for i, l in enumerate(ls)]) 202 | us = np.array([min([u] + us[i:]) for i, u in enumerate(us)]) 203 | 204 | return np.vstack([ls[idx], ps[idx], us[idx]]).T 205 | 206 | def score(self, X, y, sample_weight=None): 207 | """ 208 | Calculates the expected entropy of the fitted model 209 | :param X: 210 | :param y: 211 | :param sample_weight: 212 | :return: 213 | """ 214 | if self.calibrator is None: 215 | raise NotFittedError() 216 | return self.loss_function_( 217 | y, self.predict_proba(X)[:, 1], sample_weight=sample_weight 218 | ) 219 | 220 | @staticmethod 221 | def _compute_total_scores(X, features, scores: np.ndarray, thresholds): 222 | X = np.array(X) 223 | if len(features) == 0: 224 | return np.zeros((X.shape[0], 1)) 225 | data = np.array(X)[:, features] 226 | thresholds = np.array(thresholds, dtype=float) 227 | thresholds[np.isnan(thresholds)] = 0.5 228 | return ((data > thresholds[None, :]) @ scores).reshape(-1, 1) 229 | -------------------------------------------------------------------------------- /scratch/psl_describe.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "adc6c663-b073-4451-9821-216c578bbd69", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "from skpsl import ProbabilisticScoringList\n", 11 | "from sklearn.datasets import make_classification\n", 12 | "from sklearn.model_selection import cross_val_score, ShuffleSplit\n", 13 | "from functools import reduce\n", 14 | "from operator import or_\n", 15 | "import numpy as np\n", 16 | "import pandas as pd" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": 6, 22 | "id": "764414e5-261c-4a77-b14b-2235472b9baf", 23 | "metadata": {}, 24 | "outputs": [ 25 | { 26 | "name": "stdout", 27 | "output_type": "stream", 28 | "text": [ 29 | "Brier score: 0.6500\n" 30 | ] 31 | } 32 | ], 33 | "source": [ 34 | "# Generating synthetic data with continuous features and a binary target variable\n", 35 | "X, y = make_classification(random_state=42)\n", 36 | "#X = (X*10).round().astype(int)\n", 37 | "X = (X > .5).astype(int)\n", 38 | "\n", 39 | "for train, test in ShuffleSplit(1, test_size=.2, random_state=42).split(X):\n", 40 | " psl = ProbabilisticScoringList([-1, 1, 2])\n", 41 | " psl.fit(X[train], y[train])\n", 42 | " print(f\"Brier score: {psl.score(X[test], y[test]):.4f}\")" 43 | ] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "execution_count": 7, 48 | "id": "26a044d2-d3a1-43eb-bbef-b8e7050ce568", 49 | "metadata": {}, 50 | "outputs": [ 51 | { 52 | "data": { 53 | "text/html": [ 54 | "
\n", 55 | "\n", 68 | "\n", 69 | " \n", 70 | " \n", 71 | " \n", 72 | " \n", 73 | " \n", 74 | " \n", 75 | " \n", 76 | " \n", 77 | " \n", 78 | " \n", 79 | " \n", 80 | " \n", 81 | " \n", 82 | " \n", 83 | " \n", 84 | " \n", 85 | " \n", 86 | " \n", 87 | " \n", 88 | " \n", 89 | " \n", 90 | " \n", 91 | " \n", 92 | " \n", 93 | " \n", 94 | " \n", 95 | " \n", 96 | " \n", 97 | " \n", 98 | " \n", 99 | " \n", 100 | " \n", 101 | " \n", 102 | " \n", 103 | " \n", 104 | " \n", 105 | " \n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | " \n", 110 | " \n", 111 | " \n", 112 | " \n", 113 | " \n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | " \n", 118 | " \n", 119 | " \n", 120 | " \n", 121 | " \n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | " \n", 126 | " \n", 127 | " \n", 128 | " \n", 129 | " \n", 130 | " \n", 131 | " \n", 132 | " \n", 133 | " \n", 134 | " \n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | "
StageScoreT = -4.0T = -3.1666666666666665T = -2.333333333333333T = -1.5T = -0.6666666666666665T = 0.16666666666666696T = 1.0
00-0.53750.53750.53750.53750.53750.53750.5375
111.00.18180.18180.18180.18180.18180.31360.9722
22-1.00.00000.00000.00000.00000.12930.48981.0000
33-1.00.26920.26920.26920.26920.35900.61541.0000
44-1.00.33330.33330.33330.33330.43730.70431.0000
55-1.00.00000.00000.16670.31730.57220.95611.0000
\n", 158 | "
" 159 | ], 160 | "text/plain": [ 161 | " Stage Score T = -4.0 T = -3.1666666666666665 T = -2.333333333333333 \\\n", 162 | "0 0 - 0.5375 0.5375 0.5375 \n", 163 | "1 1 1.0 0.1818 0.1818 0.1818 \n", 164 | "2 2 -1.0 0.0000 0.0000 0.0000 \n", 165 | "3 3 -1.0 0.2692 0.2692 0.2692 \n", 166 | "4 4 -1.0 0.3333 0.3333 0.3333 \n", 167 | "5 5 -1.0 0.0000 0.0000 0.1667 \n", 168 | "\n", 169 | " T = -1.5 T = -0.6666666666666665 T = 0.16666666666666696 T = 1.0 \n", 170 | "0 0.5375 0.5375 0.5375 0.5375 \n", 171 | "1 0.1818 0.1818 0.3136 0.9722 \n", 172 | "2 0.0000 0.1293 0.4898 1.0000 \n", 173 | "3 0.2692 0.3590 0.6154 1.0000 \n", 174 | "4 0.3333 0.4373 0.7043 1.0000 \n", 175 | "5 0.3173 0.5722 0.9561 1.0000 " 176 | ] 177 | }, 178 | "execution_count": 7, 179 | "metadata": {}, 180 | "output_type": "execute_result" 181 | } 182 | ], 183 | "source": [ 184 | "df = psl.inspect(5)\n", 185 | "\n", 186 | "pd.set_option(\"display.precision\", 4)\n", 187 | "df.fillna(\"-\")" 188 | ] 189 | }, 190 | { 191 | "cell_type": "code", 192 | "execution_count": 3, 193 | "id": "aa327ca6-0ec9-4ca2-8c9e-3d59a26701e4", 194 | "metadata": {}, 195 | "outputs": [ 196 | { 197 | "data": { 198 | "text/html": [ 199 | "
\n", 200 | "\n", 213 | "\n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | " \n", 255 | " \n", 256 | " \n", 257 | " \n", 258 | " \n", 259 | " \n", 260 | " \n", 261 | " \n", 262 | " \n", 263 | " \n", 264 | " \n", 265 | " \n", 266 | " \n", 267 | " \n", 268 | " \n", 269 | " \n", 270 | " \n", 271 | " \n", 272 | " \n", 273 | " \n", 274 | " \n", 275 | " \n", 276 | " \n", 277 | " \n", 278 | " \n", 279 | " \n", 280 | " \n", 281 | " \n", 282 | " \n", 283 | " \n", 284 | " \n", 285 | " \n", 286 | " \n", 287 | " \n", 288 | " \n", 289 | " \n", 290 | " \n", 291 | " \n", 292 | " \n", 293 | " \n", 294 | " \n", 295 | " \n", 296 | " \n", 297 | " \n", 298 | " \n", 299 | " \n", 300 | " \n", 301 | " \n", 302 | "
StageScoreT = -4.0T = -3.0T = -2.0T = -1.0T = 0.0T = 1.0T = 2.0
00-0.53750.53750.53750.53750.53750.53750.5375
11-1.00.36670.36670.36670.36670.47831.00001.0000
22-1.00.00000.00000.00000.30650.71431.00001.0000
33-1.00.00000.00000.00000.13640.61111.00001.0000
442.00.00000.00000.00000.04001.00001.00001.0000
55-1.00.00000.00000.00000.00001.00001.00001.0000
\n", 303 | "
" 304 | ], 305 | "text/plain": [ 306 | " Stage Score T = -4.0 T = -3.0 T = -2.0 T = -1.0 T = 0.0 T = 1.0 \\\n", 307 | "0 0 - 0.5375 0.5375 0.5375 0.5375 0.5375 0.5375 \n", 308 | "1 1 -1.0 0.3667 0.3667 0.3667 0.3667 0.4783 1.0000 \n", 309 | "2 2 -1.0 0.0000 0.0000 0.0000 0.3065 0.7143 1.0000 \n", 310 | "3 3 -1.0 0.0000 0.0000 0.0000 0.1364 0.6111 1.0000 \n", 311 | "4 4 2.0 0.0000 0.0000 0.0000 0.0400 1.0000 1.0000 \n", 312 | "5 5 -1.0 0.0000 0.0000 0.0000 0.0000 1.0000 1.0000 \n", 313 | "\n", 314 | " T = 2.0 \n", 315 | "0 0.5375 \n", 316 | "1 1.0000 \n", 317 | "2 1.0000 \n", 318 | "3 1.0000 \n", 319 | "4 1.0000 \n", 320 | "5 1.0000 " 321 | ] 322 | }, 323 | "execution_count": 3, 324 | "metadata": {}, 325 | "output_type": "execute_result" 326 | } 327 | ], 328 | "source": [ 329 | "df = psl.inspect(5)\n", 330 | "\n", 331 | "pd.set_option(\"display.precision\", 4)\n", 332 | "df.fillna(\"-\")" 333 | ] 334 | } 335 | ], 336 | "metadata": { 337 | "kernelspec": { 338 | "display_name": "Python 3 (ipykernel)", 339 | "language": "python", 340 | "name": "python3" 341 | }, 342 | "language_info": { 343 | "codemirror_mode": { 344 | "name": "ipython", 345 | "version": 3 346 | }, 347 | "file_extension": ".py", 348 | "mimetype": "text/x-python", 349 | "name": "python", 350 | "nbconvert_exporter": "python", 351 | "pygments_lexer": "ipython3", 352 | "version": "3.11.3" 353 | } 354 | }, 355 | "nbformat": 4, 356 | "nbformat_minor": 5 357 | } 358 | -------------------------------------------------------------------------------- /skpsl/estimators/probabilistic_scoring_list.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from collections import defaultdict 3 | from itertools import permutations, product, chain 4 | from typing import Optional 5 | 6 | import numpy as np 7 | import pandas as pd 8 | from joblib import Parallel, delayed 9 | from sklearn.base import BaseEstimator, ClassifierMixin 10 | from sklearn.exceptions import NotFittedError 11 | from sklearn.metrics import brier_score_loss 12 | 13 | from skpsl.estimators.probabilistic_scoring_system import ProbabilisticScoringSystem 14 | from skpsl.helper import create_optimizer 15 | from skpsl.metrics import soft_ranking_loss 16 | 17 | 18 | class ProbabilisticScoringList(ClassifierMixin,BaseEstimator): 19 | """ 20 | Probabilistic scoring list classifier. 21 | A probabilistic classifier that greedily creates a PSL selecting one feature at a time 22 | """ 23 | 24 | def __init__( 25 | self, 26 | score_set: set, 27 | loss_cutoff: float = None, 28 | method="bisect", 29 | lookahead=1, 30 | n_jobs=None, 31 | stage_loss=None, 32 | cascade_loss=None, 33 | stage_clf_params=None, 34 | ): 35 | """ 36 | 37 | :param score_set: Set score values to be considered. Basically feature weights. 38 | :param loss_cutoff: minimal loss at which to stop fitting further stages. None means fitting the whole stage 39 | """ 40 | self.score_set = score_set 41 | self.loss_cutoff = loss_cutoff 42 | self.method = method 43 | self.lookahead = lookahead 44 | self.n_jobs = n_jobs 45 | self.stage_loss = stage_loss 46 | self.cascade_loss = cascade_loss 47 | self.stage_clf_params = stage_clf_params 48 | 49 | self.stage_clf_params_ = (self.stage_clf_params or dict()) | dict( 50 | threshold_optimizer=create_optimizer(method), loss_function=self.stage_loss 51 | ) 52 | match cascade_loss: 53 | case None: 54 | self.cascade_loss_ = sum 55 | case _: 56 | self.cascade_loss_ = cascade_loss 57 | self.logger = logging.getLogger(__name__) 58 | self.score_set_ = np.array(sorted(self.score_set, reverse=True, key=abs)) 59 | self.classes_ = None 60 | assert self.score_set_.size > 0 61 | self.stage_clfs = None # type: Optional[list[ProbabilisticScoringSystem]] 62 | 63 | def fit( 64 | self, 65 | X, 66 | y, 67 | sample_weight=None, 68 | predef_features: Optional[np.ndarray] = None, 69 | predef_scores: Optional[np.ndarray] = None, 70 | strict=True, 71 | ) -> "ProbabilisticScoringList": 72 | """ 73 | Fits a probabilistic scoring list to the given data 74 | 75 | :param X: 76 | :param y: 77 | :param predef_features: 78 | :param predef_scores: 79 | :return: The fitted classifier 80 | """ 81 | X, y = np.array(X), np.array(y) 82 | predef_features = predef_features or [] 83 | predef_scores = predef_scores or [] 84 | 85 | self.classes_ = np.unique(y) 86 | if predef_scores and predef_features: 87 | assert len(predef_features) <= len(predef_scores) 88 | 89 | predef_scores = defaultdict(lambda: list(self.score_set_)) | { 90 | predef_features[i]: [s] for i, s in enumerate(predef_scores) 91 | } 92 | 93 | number_features = X.shape[1] 94 | remaining_features = set(range(number_features)) 95 | self.stage_clfs = [] 96 | 97 | # Stage 0 classifier 98 | losses = [self._fit_and_store_clf_at_k(X, y, sample_weight)] 99 | stage = 0 100 | 101 | while remaining_features and ( 102 | self.loss_cutoff is None or losses[-1] > self.loss_cutoff 103 | ): 104 | len_ = min(self.lookahead, len(remaining_features)) 105 | len_pre = min(len(set(predef_features) & remaining_features), len_) 106 | len_rest = len_ - len_pre 107 | 108 | if strict and predef_features: 109 | prefixes = [ 110 | [f_ for f_ in predef_features if f_ in remaining_features][:len_pre] 111 | ] 112 | else: 113 | prefixes = permutations( 114 | set(predef_features) & remaining_features, len_pre 115 | ) 116 | 117 | f_exts = [ 118 | list(pre) + list(suf) 119 | for (pre, suf) in product( 120 | prefixes, 121 | permutations(remaining_features - set(predef_features), len_rest), 122 | ) 123 | ] 124 | 125 | new_cascade_losses, f, s, t = zip( 126 | *Parallel(n_jobs=self.n_jobs)( 127 | delayed(self._optimize)( 128 | list(f_seq), list(s_seq), losses, X, y, sample_weight 129 | ) 130 | for (f_seq, s_seq) in chain.from_iterable( 131 | product([fext], product(*[predef_scores[f] for f in fext])) 132 | for fext in f_exts 133 | ) 134 | ) 135 | ) 136 | 137 | i = np.argmin(new_cascade_losses) 138 | remaining_features.remove(f[i]) 139 | 140 | losses.append( 141 | self._fit_and_store_clf_at_k( 142 | X, 143 | y, 144 | sample_weight, 145 | self.features + [f[i]], 146 | self.scores + [s[i]], 147 | self.thresholds + [t[i]], 148 | ) 149 | ) 150 | stage += 1 151 | return self 152 | 153 | def _fit_and_store_clf_at_k(self, X, y, sample_weight=None, f=None, s=None, t=None): 154 | f, s, t = f or [], s or [], t or [] 155 | k_clf = ProbabilisticScoringSystem( 156 | features=f, 157 | scores=s, 158 | initial_feature_thresholds=t, 159 | **self.stage_clf_params_, 160 | ).fit(X, y) 161 | self.stage_clfs.append(k_clf) 162 | return k_clf.score(X, y, sample_weight) 163 | 164 | def _optimize( 165 | self, feature_extension, score_extension, cascade_losses, X, y, sample_weight 166 | ): 167 | cascade_losses = cascade_losses.copy() 168 | # build cascade extension 169 | new_thresholds = [] 170 | for i in range(1, len(feature_extension) + 1): 171 | # Regardless of the stage-loss, thresholds are optimized with respect to expected entropy 172 | clf = ProbabilisticScoringSystem( 173 | features=self.features + feature_extension[:i], 174 | scores=self.scores + score_extension[:i], 175 | initial_feature_thresholds=self.thresholds + new_thresholds + [None], 176 | **self.stage_clf_params_, 177 | ).fit(X, y) 178 | cascade_losses.append(clf.score(X, y, sample_weight)) 179 | new_thresholds.append(clf.feature_thresholds[-1]) 180 | 181 | return ( 182 | self.cascade_loss_(cascade_losses), 183 | feature_extension[0], 184 | score_extension[0], 185 | new_thresholds[0], 186 | ) 187 | 188 | def predict(self, X, k=-1): 189 | """ 190 | Predicts a probabilistic scoring list to the given data 191 | :param X: Dataset to predict the probabilities for 192 | :param k: Classifier stage to use for prediction 193 | :return: 194 | """ 195 | 196 | return self.predict_proba(X, k).argmax(axis=1) 197 | 198 | def predict_proba(self, X, k=-1, **kwargs): 199 | """ 200 | Predicts the probability using the k-th or last classifier 201 | :param X: Dataset to predict the probabilities for 202 | :param k: Classifier stage to use for prediction 203 | :return: 204 | """ 205 | if self.stage_clfs is None: 206 | raise NotFittedError( 207 | "Please fit the probabilistic scoring classifier before usage." 208 | ) 209 | 210 | return self[k].predict_proba(X, **kwargs) 211 | 212 | def score(self, X, y, k=None, sample_weight=None): 213 | """ 214 | Calculates the Brier score of the model 215 | :param X: 216 | :param y: 217 | :param k: Classifier stage to use for prediction 218 | :param sample_weight: ignored 219 | :return: 220 | """ 221 | if self.stage_clfs is None: 222 | raise NotFittedError() 223 | if k is None: 224 | return self.cascade_loss_( 225 | [ 226 | brier_score_loss(y, self.predict_proba(X, k=k)[:, 1]) 227 | for k in range(len(self)) 228 | ] 229 | ) 230 | return brier_score_loss(y, self.predict_proba(X, k=k)[:, 1]) 231 | 232 | def searchspace_analysis(self, X): 233 | """ 234 | Prints some useful information about the search space 235 | 236 | :param X: only used to derive number of features 237 | :return: None 238 | """ 239 | f, s, l = X.shape[1], len(self.score_set), self.lookahead 240 | 241 | # models = calculate number of models at each stage 242 | print(f"Searchspace size: {(s+1)**f:.2g}") 243 | 244 | # calculate lookahead induced subspace 245 | effective_searchspace = sum( 246 | ( 247 | s ** min(l, k) 248 | * np.array([k_ + 1 for k_ in range(max(k - l, 0), k)]).prod() 249 | for k in range(f, 0, -1) 250 | ) 251 | ) 252 | print(f"Models to evaluate (ignoring caching): {effective_searchspace:g}") 253 | 254 | def inspect(self, k=None, feature_names=None) -> pd.DataFrame: 255 | """ 256 | Returns a dataframe that visualizes the internal model 257 | 258 | :param k: maximum stage to include in the visualization (default: all stages) 259 | :param feature_names: names of the features. 260 | :return: 261 | """ 262 | k = k or len(self) - 1 263 | 264 | scores = self.stage_clfs[k].scores 265 | features = self.stage_clfs[k].features 266 | thresholds = self.stage_clfs[k].feature_thresholds 267 | 268 | all_total_scores = [{0}] 269 | for i, score in enumerate(scores): 270 | all_total_scores.append( 271 | all_total_scores[i] 272 | | {prev_sum + score for prev_sum in all_total_scores[i]} 273 | ) 274 | 275 | data = [] 276 | for clf, T in zip(self.stage_clfs[: k + 1], all_total_scores): 277 | a = {t: np.nan for t in all_total_scores[-1]} 278 | probas = clf.calibrator.transform(np.array(list(T))) 279 | a.update(dict(zip(T, probas))) 280 | data.append(a) 281 | 282 | df = pd.DataFrame(data) 283 | df = df[sorted(df.columns)] 284 | df.columns = [f"T = {t_}" for t_ in df.columns] 285 | df.insert(0, "Score", np.array([np.nan] + list(scores))) 286 | if feature_names is not None: 287 | if len(feature_names) != len(features): 288 | raise ValueError( 289 | f"Passed feature names are of incorrect length! Passed {len(feature_names)}, expected {len(features)}." 290 | ) 291 | feature_names = [feature_names[i] for i in features] 292 | df.insert(0, "Feature", [np.nan] + feature_names[:k]) 293 | else: 294 | df.insert( 295 | 0, 296 | "Feature Index", 297 | [np.nan] + features[:k], 298 | ) 299 | if not all(t is None or np.isnan(t) for t in thresholds): 300 | df.insert( 301 | 0, 302 | "Threshold", 303 | [np.nan] 304 | + [ 305 | (np.nan if t is None or np.isnan(t) else f">{t:.4f}") 306 | for t in thresholds 307 | ], 308 | ) 309 | return df.reset_index(names=["Stage"]) 310 | 311 | def __len__(self): 312 | if self.stage_clfs is None: 313 | return 0 314 | return len(self.stage_clfs) 315 | 316 | def __getitem__(self, item) -> ProbabilisticScoringSystem: 317 | if self.stage_clfs is None: 318 | raise AttributeError("Cant get any clf, no clfs fitted") 319 | return self.stage_clfs[item] 320 | 321 | @property 322 | def features(self) -> list[int]: 323 | return self.stage_clfs[-1].features if self.stage_clfs else [] 324 | 325 | @property 326 | def scores(self) -> list[int]: 327 | return self.stage_clfs[-1].scores if self.stage_clfs else [] 328 | 329 | @property 330 | def thresholds(self) -> list[int]: 331 | return self.stage_clfs[-1].feature_thresholds if self.stage_clfs else [] 332 | 333 | 334 | if __name__ == "__main__": 335 | from sklearn.datasets import make_classification 336 | from sklearn.model_selection import cross_val_score 337 | 338 | # Generating synthetic data with continuous features and a binary target variable 339 | X_, y_ = make_classification(random_state=42) 340 | 341 | clf_ = ProbabilisticScoringList({-1, 1, 2}, stage_loss=soft_ranking_loss, stage_clf_params=dict(calibration_method="beta")) 342 | clf_.searchspace_analysis(X_) 343 | print("Total Brier score:", cross_val_score(clf_, X_, y_, cv=5, n_jobs=5).mean()) 344 | -------------------------------------------------------------------------------- /skpsl/estimators/multiclass_scoring_list.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import logging 3 | from itertools import product, repeat 4 | from operator import itemgetter 5 | 6 | import numpy as np 7 | import pandas as pd 8 | from joblib.parallel import delayed, Parallel 9 | from pygad import pygad 10 | from scipy.special import softmax 11 | from scipy.stats import rankdata, truncnorm 12 | from sklearn.base import BaseEstimator, ClassifierMixin 13 | from sklearn.exceptions import NotFittedError 14 | from sklearn.linear_model import LogisticRegression 15 | from sklearn.metrics import log_loss, accuracy_score 16 | from sklearn.preprocessing import LabelBinarizer 17 | from tqdm import tqdm 18 | 19 | from skpsl.preprocessing import MinEntropyBinarizer 20 | 21 | LOGGER = logging.getLogger(__name__) 22 | 23 | 24 | class MulticlassScoringList(ClassifierMixin,BaseEstimator): 25 | 26 | def __init__(self, score_set, method=None, cascade_loss=None, random_state=None, ga_params=None, **kwargs): 27 | """ 28 | :param score_set: 29 | :param cascade_loss: function to aggregate the stage losses 30 | - None: will give the sum of the whole cascade 31 | - lambda x:x[-1]: will give the loss of the last stage 32 | """ 33 | self.score_set = score_set 34 | self.cascade_loss = cascade_loss 35 | self.method = method if method is not None else "greedy" 36 | self.random_state = random_state 37 | self.ga_params = ga_params 38 | 39 | self.score_set_ = np.array(sorted(score_set, reverse=True, key=abs)) 40 | self.scores = None 41 | self.f_ranks = None 42 | self.classes_ = None 43 | self.binarizer = None 44 | self.majority_class_idx = None 45 | self.n_classes = None 46 | self.n_features = None 47 | self.ga_instance = None 48 | self.stage = None 49 | self.l2 = kwargs.get("l2", 0) 50 | self.n_jobs = kwargs.get("n_jobs", 1) 51 | ga_default = dict(maxiter=50, popsize=10, init_pop_factor=1, init_pop_noise=.2, parents_mating=5) 52 | self.ga_params_ = ga_default | ga_params if ga_params is not None else None 53 | match cascade_loss: 54 | case None: 55 | self.cascade_loss_ = sum 56 | case _: 57 | self.cascade_loss_ = cascade_loss 58 | 59 | def __iter__(self): 60 | self.stage = 0 61 | while self.stage != len(self) + 1: 62 | yield self 63 | self.stage = self.stage + 1 64 | self.stage = None 65 | 66 | def __getitem__(self, item): 67 | self.stage = item 68 | return self 69 | 70 | def fit(self, X, y): 71 | self.classes_, counts = np.unique(y, return_counts=True) 72 | self.n_classes = self.classes_.size 73 | n_instances, self.n_features = X.shape 74 | 75 | self.binarizer = MinEntropyBinarizer().fit(X, y) 76 | X = self.binarizer.transform(X) 77 | 78 | # Compute cross-entropy loss 79 | y_trans = LabelBinarizer().fit_transform(y) 80 | 81 | match self.method: 82 | case "greedy": 83 | features, scores = [], [] 84 | 85 | X = np.hstack((np.ones((n_instances, 1)), X)) 86 | 87 | def opt(s, f, logits): 88 | logits_ = logits + X[:, [f + 1]] @ np.array([s]) 89 | return f, list(s), logits_, log_loss(y_trans, softmax(logits_, axis=1)) + self.l2 * ( 90 | (np.array(s) / max(abs(self.score_set_))) ** 2).mean() 91 | 92 | logits = np.zeros((n_instances, self.n_classes)) # np.repeat([s], n_instances, axis=0) 93 | 94 | # bias term 95 | res = tqdm( 96 | Parallel(n_jobs=1, return_as="generator")( 97 | delayed(opt)(s, -1, logits) 98 | for (s) in 99 | sorted(product(*repeat(self.score_set_, self.n_classes)), key=lambda x: sum(abs(np.array(x)))) 100 | ), 101 | total=len(self.score_set_) ** self.n_classes) 102 | _, s, logits, _ = min(res, key=itemgetter(3)) 103 | LOGGER.info(f"bias terms: {s}") 104 | scores.append(s) 105 | 106 | # n features 107 | remaining_features = set(range(self.n_features)) 108 | while remaining_features: 109 | res = tqdm( 110 | Parallel(n_jobs=self.n_jobs, return_as="generator")( 111 | delayed(opt)(s, f, logits) 112 | for (f, s) in 113 | product(remaining_features, product(*repeat(self.score_set_, self.n_classes))) 114 | ), 115 | total=len(remaining_features) * len(self.score_set_) ** self.n_classes) 116 | 117 | f, s, logits, _ = min(res, key=itemgetter(3)) 118 | 119 | LOGGER.info(f"scores for stage {len(scores)}: {s}") 120 | features.append(f) 121 | scores.append(s) 122 | remaining_features.remove(f) 123 | 124 | # convert orderings to ranks 125 | self.f_ranks = np.argsort(features) 126 | # index ordering scores by ranks to get scores in feature permutation (congrent to the f_ranks) 127 | self.scores = np.array(scores)[[0] + list(self.f_ranks + 1)].T 128 | 129 | case "ga": 130 | # fit lr as a seed for the genetic search 131 | lr = LogisticRegression().fit(X, y) 132 | 133 | # FEATURE RANKINGS 134 | self.f_ranks = np.argsort(lr.coef_.mean(axis=0)) 135 | 136 | # CALCULATE SCORES 137 | # extract logits from LR and rescale and round to the score_set 138 | # (this destroys the proper scaling for the softmax and hence probability estimates) 139 | self.scores = self._rescale_round_closest(np.hstack([lr.intercept_.reshape(-1, 1), lr.coef_])) 140 | 141 | # FIT SCALE PARAMETERS 142 | if self.ga_params_ is not None: 143 | # do GA optimization 144 | X_trans = np.array(self.binarizer.transform(X)) 145 | y_trans = LabelBinarizer().fit_transform(y) 146 | 147 | def mutation(offspring, ga_instance): 148 | new_gen = [] 149 | for i in range(len(offspring)): 150 | rng = np.random.default_rng(ga_instance.random_seed) 151 | f_ranks, scores = self._unpack(offspring[0], self.n_features) 152 | score_set = np.array(sorted(self.score_set_)) 153 | 154 | # replace the rank of one feature 155 | f_ranks[rng.integers(0, len(f_ranks))] = rng.random() 156 | 157 | # select scores indices to replace and convert into score indices 158 | flat_indices = rng.choice(scores.size, ga_instance.mutation_num_genes, replace=False) 159 | current_indices = np.searchsorted(score_set, scores.flat[flat_indices]) 160 | 161 | # the clip is still necessary, because the documentation does not say if the bounds 162 | # are inclusive, and it still seems to crash from time to time 163 | a, b = (0 - current_indices), (len(score_set) - 1 - current_indices) 164 | new_indices = np.clip(np.round( 165 | truncnorm.rvs(a, b, loc=current_indices, scale=1, 166 | size=ga_instance.mutation_num_genes, random_state=rng)).astype(int), 0, 167 | len(score_set) - 1) 168 | 169 | # replace values 170 | scores.flat[flat_indices] = score_set[new_indices] 171 | 172 | new_gen.append(self._pack(f_ranks, scores)) 173 | 174 | return np.array(new_gen) 175 | 176 | def objective(instance, solution_, solution_idx): 177 | f_ranks, scores = self._unpack(solution_, self.n_features) 178 | 179 | quality = [] 180 | for k in range(1, len(self) + 1): 181 | # we can omit stage 0 from the optimization as it is not influenced by the selection of scores 182 | mask = f_ranks < k 183 | pred = self.classes_[ 184 | np.argmax(scores[:, 0] + X_trans[:, mask] @ scores[:, 1:][:, mask].T, axis=1)] 185 | quality.append(accuracy_score(y, pred)) 186 | 187 | return sum(quality) 188 | 189 | seed = self._pack(self.f_ranks, self.scores) 190 | rng = np.random.default_rng(self.random_state) 191 | initial_pop = [seed] 192 | for _ in range(int(self.ga_params_.get("init_pop_factor") * self.ga_params_.get("popsize")) - 1): 193 | f_ranks, scores = self._unpack(seed, self.n_features) 194 | n = int(scores.size * self.ga_params_.get("init_pop_noise")) 195 | scores.flat[rng.choice(scores.size, n, replace=False)] = \ 196 | self.score_set_[rng.integers(0, self.score_set_.size, n)] 197 | initial_pop.append(self._pack(rng.permutation(f_ranks), scores)) 198 | 199 | admissible_args = inspect.getfullargspec(pygad.GA)[0] 200 | kwargs = {k: v for k, v in self.ga_params_.items() if k in admissible_args} 201 | params = dict(fitness_func=objective, 202 | initial_population=initial_pop, 203 | mutation_type=mutation, # adaptive 204 | num_genes=(self.n_classes + 1) * self.n_features, 205 | gene_space=[dict(low=0, high=1)] * self.n_features + 206 | [sorted(self.score_set)] * (self.n_features + 1) * self.n_classes, 207 | num_generations=self.ga_params_.get("maxiter"), 208 | num_parents_mating=self.ga_params_.get("parents_mating"), 209 | sol_per_pop=self.ga_params_.get("popsize"), 210 | random_seed=self.random_state, 211 | ) 212 | self.ga_instance = pygad.GA(**(params | kwargs)) 213 | self.ga_instance.run() 214 | solution, fitness, _ = self.ga_instance.best_solution() 215 | self.f_ranks, self.scores = self._unpack(solution, self.n_features) 216 | 217 | return self 218 | 219 | @staticmethod 220 | def _unpack(solution, m): 221 | solution = np.array(solution) 222 | return (rankdata(solution[:m], method="ordinal") - 1).astype(int), solution[m:].reshape(-1, m + 1).astype(int) 223 | 224 | @staticmethod 225 | def _pack(f_ranks, scores): 226 | return list(f_ranks / len(f_ranks)) + list(scores.flatten()) 227 | 228 | def _rescale_round_closest(self, values): 229 | # self.score_set is sorted differently (by absolute value) so we need to resort it. 230 | sorted_scoreset = np.array(sorted(self.score_set_)) 231 | 232 | scale = max(values.min() / sorted_scoreset.min(), values.max() / sorted_scoreset.max()) 233 | values = values / scale 234 | 235 | # Find the insertion points for each value in values 236 | pos = np.searchsorted(sorted_scoreset, values) 237 | 238 | # Clip positions to handle edge cases where pos is out of bounds 239 | pos = np.clip(pos, 1, len(sorted_scoreset) - 1) 240 | 241 | # Get the closest values before and after each position 242 | before = sorted_scoreset[pos - 1] 243 | after = sorted_scoreset[pos] 244 | 245 | # Choose the closer value by comparing absolute differences 246 | return np.where(np.abs(values - before) <= np.abs(values - after), before, after) 247 | 248 | def __len__(self): 249 | return len(self.f_ranks) 250 | 251 | def predict(self, X): 252 | # get random argmax 253 | arr = self._logit(X, self.stage) 254 | is_max = arr == arr.max(axis=1, keepdims=True) # Boolean mask of max locations 255 | rand_vals = np.random.rand(*arr.shape) * is_max # Assign random values only to max locations 256 | return self.classes_[rand_vals.argmax(axis=1)] 257 | 258 | def _logit(self, X, k=None): 259 | if self.scores is None: 260 | raise NotFittedError() 261 | X_bin = np.array(self.binarizer.transform(X)) 262 | if k is None or k == -1: 263 | k = len(self) 264 | mask = self.f_ranks < k 265 | return self.scores[:, 0] + X_bin[:, mask] @ self.scores[:, 1:][:, mask].T 266 | 267 | def predict_proba(self, X): 268 | k = self.stage if self.stage is not None else -1 269 | return softmax(self._logit(X, k), axis=1) 270 | 271 | def score(self, X, y, sample_weight=None): 272 | k = self.stage 273 | if k is not None: 274 | return accuracy_score(y, self.predict(X)) 275 | loss = self.cascade_loss_([stage.score(X, y) for stage in self]) 276 | self.stage = None 277 | return loss 278 | 279 | def inspect(self, feature_names=None, class_names=None): 280 | # when using the f_ranks for indexing they have to be converted into orderings 281 | f_ordering = np.argsort(self.f_ranks) 282 | 283 | sections = [] 284 | if feature_names is not None: 285 | feature_names = np.array(feature_names) 286 | sections.append(pd.DataFrame([""] + feature_names[f_ordering].tolist(), columns=["Feature"])) 287 | else: 288 | sections.append( 289 | pd.DataFrame([-1] + np.array(list(range(self.n_features)))[f_ordering].tolist(), columns=["Feature"])) 290 | sections.append( 291 | pd.DataFrame([np.nan] + self.binarizer.threshs[f_ordering].T.tolist(), columns=["Thresholds (>)"])) 292 | sections.append(pd.DataFrame(self.scores[:, [0] + (f_ordering + 1).tolist()].T, 293 | columns=(class_names if class_names is not None else self.classes_))) 294 | return pd.concat(sections, axis=1) 295 | 296 | @property 297 | def features(self): 298 | return np.where(self.f_ranks < self.stage)[0] 299 | 300 | if __name__ == '__main__': 301 | from sklearn.datasets import load_iris 302 | 303 | data = load_iris() 304 | X, y = data.data, data.target 305 | clf = MulticlassScoringList(score_set=set(range(-3, 4)), # cascade_loss=lambda x: x[-1] 306 | method="greedy").fit(X, y) 307 | print(clf.inspect(data.feature_names, data.target_names)) 308 | clf.predict(X) -------------------------------------------------------------------------------- /scratch/pretest data/data_for_otree_pretest_test.csv: -------------------------------------------------------------------------------- 1 | ,player_id,position,Player Age,Player Height (cm),Matches Played,Average Minutes per Match,Average Goals per Match,Average Assists per Match,Average Shots per Match,Average Pass Success Rate (%),Average Aerial Duels Won per Match,Average Yellow Cards per Match,Average Red Cards per Match,Average Man of the Match per Match 2 | 2458,3550,Midfielder,25,168,135,48.46,0.12,0.13,1.5,81.7,0.1,0.1,0.01,0.04 3 | 3669,5184,Midfielder,22,187,92,61.46,0.27,0.2,1.5,81.1,0.8,0.1,0.0,0.1 4 | 3179,4534,Defender,23,183,97,74.56,0.03,0.02,0.4,86.7,1.1,0.13,0.01,0.01 5 | 3421,4846,Defender,28,185,209,80.54,0.04,0.01,0.4,84.7,2.9,0.15,0.02,0.02 6 | 713,1046,Defender,32,186,296,82.35,0.08,0.05,1.0,78.0,1.9,0.28,0.01,0.06 7 | 3052,4372,Goalkeeper,29,193,171,90.1,0.01,0.0,0.0,67.6,0.4,0.04,0.01,0.04 8 | 1921,2810,Midfielder,30,177,178,72.75,0.01,0.02,0.4,84.7,0.7,0.18,0.01,0.02 9 | 2620,3775,Midfielder,20,183,75,36.55,0.09,0.05,0.8,73.4,0.2,0.07,0.01,0.04 10 | 2254,3257,Defender,23,190,114,78.63,0.06,0.04,0.6,71.0,4.3,0.11,0.0,0.07 11 | 592,859,Midfielder,33,181,149,69.17,0.03,0.05,0.8,80.8,0.6,0.32,0.02,0.01 12 | 2680,3857,Forward,26,195,145,65.51,0.32,0.04,2.3,57.4,2.9,0.23,0.03,0.04 13 | 927,1362,Midfielder,21,184,178,57.74,0.27,0.19,1.7,84.6,0.3,0.06,0.0,0.1 14 | 1775,2595,Midfielder,27,178,126,67.42,0.19,0.15,1.3,80.5,0.6,0.1,0.0,0.09 15 | 1464,2158,Defender,30,194,81,83.85,0.02,0.0,0.3,84.4,1.7,0.11,0.0,0.01 16 | 1000,1479,Midfielder,25,181,103,79.61,0.22,0.05,1.7,83.7,1.5,0.2,0.01,0.08 17 | 3402,4820,Midfielder,24,181,124,70.38,0.02,0.06,0.7,78.7,0.7,0.22,0.0,0.02 18 | 118,178,Defender,21,182,53,57.53,0.02,0.02,0.2,78.8,0.6,0.06,0.0,0.0 19 | 722,1059,Midfielder,27,182,78,64.68,0.01,0.01,0.3,86.5,0.4,0.14,0.0,0.01 20 | 1749,2555,Midfielder,27,182,171,69.07,0.06,0.04,0.9,70.5,0.9,0.19,0.0,0.01 21 | 274,395,Defender,20,188,56,78.91,0.0,0.0,0.3,88.3,1.3,0.11,0.0,0.02 22 | 1641,2389,Goalkeeper,34,192,583,90.01,0.0,0.0,0.0,62.2,0.1,0.02,0.0,0.03 23 | 204,284,Defender,37,190,227,87.7,0.06,0.01,0.6,83.9,2.5,0.11,0.01,0.04 24 | 1780,2605,Defender,32,192,386,88.05,0.07,0.03,0.8,84.7,3.0,0.18,0.02,0.06 25 | 1567,2286,Midfielder,24,182,145,53.39,0.15,0.12,1.1,87.2,0.2,0.02,0.0,0.01 26 | 2295,3317,Midfielder,30,174,231,71.86,0.13,0.13,1.3,75.7,0.7,0.15,0.0,0.04 27 | 1372,2039,Forward,27,192,82,46.12,0.22,0.06,1.4,72.4,1.0,0.12,0.0,0.05 28 | 480,700,Forward,26,179,77,76.95,0.43,0.04,2.2,78.3,1.0,0.12,0.0,0.06 29 | 820,1178,Forward,23,183,155,52.68,0.32,0.08,1.3,77.1,0.7,0.03,0.01,0.04 30 | 517,759,Midfielder,29,185,88,62.95,0.08,0.05,0.8,79.2,0.9,0.19,0.0,0.05 31 | 98,147,Forward,24,177,183,71.78,0.46,0.12,2.7,69.4,0.6,0.1,0.01,0.11 32 | 3515,4971,Midfielder,25,174,122,67.45,0.09,0.03,0.8,84.8,0.4,0.12,0.0,0.03 33 | 3407,4827,Midfielder,25,178,101,73.03,0.23,0.15,2.2,77.8,0.7,0.05,0.0,0.09 34 | 2241,3240,Midfielder,23,184,66,55.24,0.08,0.09,0.8,83.7,0.3,0.14,0.0,0.0 35 | 373,525,Midfielder,29,176,258,67.33,0.07,0.05,0.7,83.4,0.6,0.24,0.0,0.01 36 | 1615,2348,Defender,30,194,207,89.34,0.07,0.03,0.7,82.7,4.0,0.08,0.0,0.09 37 | 2123,3088,Midfielder,21,175,75,35.37,0.05,0.07,1.0,77.4,0.2,0.07,0.0,0.0 38 | 242,339,Goalkeeper,26,195,205,88.68,0.0,0.0,0.0,80.4,0.2,0.0,0.0,0.04 39 | 448,648,Midfielder,26,180,128,57.26,0.03,0.02,0.7,85.6,0.3,0.14,0.0,0.0 40 | 1722,2510,Midfielder,29,188,416,71.27,0.1,0.08,0.9,88.4,1.3,0.16,0.01,0.05 41 | 3486,4933,Defender,29,190,66,81.29,0.08,0.0,0.8,81.5,2.5,0.3,0.03,0.08 42 | 2874,4118,Defender,31,184,311,82.63,0.01,0.02,0.3,80.2,3.2,0.05,0.0,0.01 43 | 1720,2504,Midfielder,22,181,128,43.48,0.07,0.1,0.6,80.4,0.1,0.01,0.0,0.03 44 | 121,183,Goalkeeper,33,186,318,88.64,0.0,0.0,0.0,61.0,0.4,0.03,0.01,0.06 45 | 1872,2745,Defender,33,188,206,85.81,0.08,0.03,0.7,80.4,2.7,0.16,0.01,0.04 46 | 2533,3646,Defender,28,187,67,83.49,0.09,0.03,0.6,85.0,1.9,0.16,0.01,0.04 47 | 1231,1818,Defender,29,190,116,61.61,0.02,0.01,0.4,80.0,1.8,0.21,0.01,0.03 48 | 2055,2992,Midfielder,32,183,231,60.6,0.11,0.1,1.3,81.3,0.3,0.11,0.0,0.03 49 | 1649,2401,Defender,24,178,84,70.44,0.04,0.02,0.6,80.5,0.8,0.23,0.01,0.02 50 | 3359,4756,Goalkeeper,26,187,73,87.22,0.0,0.0,0.0,58.7,0.5,0.04,0.0,0.01 51 | 3747,5292,Forward,26,181,56,55.34,0.23,0.07,1.5,74.6,0.6,0.09,0.0,0.07 52 | 3356,4753,Midfielder,26,180,54,56.33,0.09,0.09,1.2,79.2,0.2,0.11,0.0,0.0 53 | 3330,4720,Midfielder,27,180,70,40.86,0.06,0.07,0.8,79.7,0.3,0.14,0.0,0.03 54 | 2831,4060,Defender,32,188,87,69.4,0.01,0.02,0.2,83.7,2.0,0.07,0.01,0.0 55 | 3540,5010,Forward,28,178,119,47.82,0.28,0.1,1.4,76.7,0.6,0.08,0.0,0.05 56 | 3561,5033,Forward,21,185,64,36.05,0.17,0.06,1.2,76.3,0.7,0.14,0.0,0.03 57 | 2099,3051,Defender,25,188,171,82.39,0.08,0.04,1.1,84.4,2.5,0.2,0.0,0.05 58 | 1330,1971,Defender,33,195,389,89.04,0.08,0.03,0.9,88.2,4.0,0.09,0.01,0.07 59 | 925,1359,Midfielder,30,173,418,71.3,0.14,0.12,1.1,84.8,0.8,0.11,0.01,0.05 60 | 981,1455,Goalkeeper,37,190,179,88.29,0.0,0.0,0.0,49.9,0.3,0.09,0.01,0.02 61 | 1674,2431,Midfielder,32,184,90,71.1,0.01,0.02,0.3,87.7,0.6,0.19,0.02,0.01 62 | 1826,2678,Midfielder,39,172,648,74.25,0.09,0.14,1.1,89.2,0.4,0.13,0.0,0.06 63 | 2722,3916,Midfielder,24,175,113,62.31,0.16,0.19,0.9,82.4,0.3,0.05,0.0,0.05 64 | 2444,3535,Defender,26,188,115,81.1,0.05,0.02,0.7,79.3,1.3,0.15,0.03,0.02 65 | 2348,3387,Defender,23,175,71,66.18,0.01,0.03,0.5,78.6,0.5,0.15,0.01,0.03 66 | 1364,2030,Defender,22,184,65,61.8,0.02,0.05,0.4,77.5,0.6,0.11,0.02,0.02 67 | 2745,3940,Defender,26,191,99,83.71,0.0,0.03,0.3,87.2,1.2,0.11,0.01,0.01 68 | 1291,1907,Defender,28,192,202,82.9,0.04,0.02,0.4,87.3,2.4,0.22,0.02,0.0 69 | 3136,4476,Defender,33,185,382,86.79,0.05,0.01,0.6,79.0,3.6,0.15,0.01,0.05 70 | 483,704,Forward,25,183,145,61.01,0.34,0.1,1.7,77.6,1.0,0.12,0.01,0.06 71 | 3423,4850,Goalkeeper,29,188,130,90.94,0.0,0.0,0.0,62.9,0.2,0.03,0.01,0.04 72 | 3435,4865,Goalkeeper,31,188,203,89.73,0.0,0.0,0.0,76.3,0.3,0.03,0.0,0.08 73 | 405,577,Defender,28,192,278,84.67,0.04,0.04,0.5,82.9,2.3,0.18,0.01,0.03 74 | 2062,3002,Midfielder,22,172,62,59.58,0.1,0.03,0.7,84.7,0.2,0.11,0.0,0.0 75 | 254,358,Goalkeeper,31,188,375,88.99,0.0,0.02,0.0,82.4,0.2,0.09,0.01,0.01 76 | 1709,2484,Midfielder,28,184,314,72.23,0.03,0.06,0.9,87.2,1.4,0.11,0.01,0.05 77 | 3269,4647,Goalkeeper,27,191,59,90.0,0.0,0.0,0.0,66.4,0.3,0.03,0.0,0.07 78 | 404,575,Midfielder,29,185,329,67.06,0.05,0.08,0.7,83.1,0.5,0.21,0.0,0.02 79 | 1129,1672,Midfielder,30,178,294,75.64,0.2,0.18,1.6,80.4,0.4,0.14,0.0,0.07 80 | 2561,3691,Goalkeeper,32,185,63,88.17,0.0,0.0,0.0,57.0,0.2,0.0,0.0,0.03 81 | 459,669,Midfielder,28,178,249,66.1,0.02,0.03,0.5,90.1,0.4,0.16,0.0,0.0 82 | 1338,1984,Midfielder,24,186,169,66.85,0.19,0.15,1.7,83.5,0.5,0.08,0.01,0.08 83 | 3161,4511,Goalkeeper,25,190,84,87.58,0.0,0.0,0.0,64.3,0.3,0.12,0.0,0.01 84 | 1863,2735,Forward,22,180,136,48.29,0.35,0.15,1.7,76.8,1.2,0.09,0.0,0.07 85 | 1302,1923,Midfielder,26,178,209,67.67,0.03,0.05,0.8,85.5,0.4,0.14,0.01,0.01 86 | 1747,2553,Defender,22,193,61,53.1,0.02,0.03,0.3,86.1,1.4,0.13,0.0,0.03 87 | 2436,3526,Midfielder,27,175,182,58.83,0.03,0.06,0.4,67.1,1.2,0.21,0.01,0.04 88 | 669,980,Defender,26,185,253,82.42,0.04,0.02,0.4,85.8,1.8,0.17,0.01,0.02 89 | 670,982,Midfielder,25,172,243,53.72,0.13,0.09,1.4,76.7,0.4,0.05,0.0,0.04 90 | 3588,5073,Midfielder,22,184,70,73.93,0.04,0.04,0.3,82.5,0.8,0.16,0.01,0.03 91 | 1610,2341,Midfielder,22,183,97,47.59,0.06,0.02,0.6,81.4,0.6,0.14,0.01,0.0 92 | 1428,2106,Goalkeeper,42,188,440,88.6,0.0,0.01,0.0,71.4,0.2,0.05,0.01,0.02 93 | 1690,2455,Midfielder,31,188,285,62.12,0.06,0.05,0.8,84.9,1.0,0.22,0.0,0.01 94 | 1797,2633,Defender,23,191,152,70.95,0.01,0.03,0.3,85.2,1.2,0.13,0.01,0.0 95 | 2532,3645,Midfielder,28,176,184,54.3,0.02,0.04,0.6,84.2,0.4,0.1,0.0,0.01 96 | 111,168,Midfielder,20,172,51,38.92,0.06,0.04,0.5,84.2,0.3,0.06,0.02,0.0 97 | 386,545,Defender,23,194,153,81.5,0.05,0.02,0.5,88.0,2.5,0.16,0.0,0.03 98 | 3380,4787,Defender,24,188,82,74.84,0.01,0.0,0.4,88.8,1.8,0.18,0.01,0.0 99 | 2352,3392,Defender,22,193,90,79.5,0.01,0.01,0.3,84.4,1.5,0.14,0.01,0.03 100 | 1339,1986,Defender,27,188,233,67.91,0.0,0.04,0.3,84.5,1.9,0.14,0.0,0.0 101 | 47,67,Midfielder,23,181,69,25.93,0.03,0.04,0.2,78.0,0.2,0.01,0.0,0.0 102 | 2787,3999,Midfielder,23,180,162,58.1,0.17,0.07,1.7,72.7,0.4,0.08,0.0,0.04 103 | 1118,1654,Defender,27,194,151,86.9,0.02,0.01,0.4,85.3,2.2,0.15,0.0,0.03 104 | 1830,2683,Defender,25,167,113,75.34,0.04,0.08,0.6,84.2,0.4,0.06,0.0,0.0 105 | 3093,4421,Forward,31,190,91,35.46,0.12,0.01,0.7,74.2,1.7,0.09,0.0,0.01 106 | 1442,2122,Defender,25,188,159,84.43,0.08,0.04,0.6,75.6,3.8,0.14,0.01,0.04 107 | 1308,1930,Midfielder,25,178,202,70.33,0.12,0.12,1.7,76.5,0.9,0.11,0.0,0.05 108 | 586,853,Forward,32,186,101,80.97,0.26,0.15,2.3,59.4,4.2,0.22,0.02,0.07 109 | 3114,4448,Defender,31,194,173,75.08,0.02,0.03,0.3,82.1,2.0,0.18,0.0,0.0 110 | 263,376,Midfielder,27,187,295,65.67,0.04,0.1,0.7,86.9,1.0,0.23,0.01,0.03 111 | 671,983,Midfielder,21,178,150,52.85,0.01,0.03,0.5,82.4,0.6,0.17,0.01,0.0 112 | 3138,4479,Defender,27,188,159,79.33,0.02,0.02,0.4,79.6,3.3,0.16,0.01,0.02 113 | 3469,4912,Forward,23,171,85,50.45,0.14,0.04,1.4,66.7,2.6,0.13,0.0,0.02 114 | 1266,1865,Forward,23,177,85,57.04,0.14,0.07,1.6,75.3,0.6,0.07,0.0,0.02 115 | 2168,3152,Midfielder,25,169,79,47.86,0.0,0.0,0.2,85.7,0.5,0.27,0.03,0.0 116 | 2919,4185,Defender,24,184,58,66.34,0.02,0.05,0.1,84.7,1.0,0.19,0.02,0.0 117 | 1523,2229,Midfielder,29,179,79,57.78,0.06,0.06,0.7,79.3,0.9,0.09,0.0,0.01 118 | 1096,1616,Forward,30,201,156,62.81,0.56,0.07,2.5,71.2,3.3,0.11,0.0,0.13 119 | 2353,3394,Forward,24,185,73,64.71,0.21,0.1,1.5,72.6,1.5,0.1,0.01,0.01 120 | 837,1213,Defender,29,186,279,77.83,0.04,0.02,0.4,87.8,1.7,0.22,0.02,0.02 121 | 1038,1534,Midfielder,28,178,192,73.49,0.07,0.11,1.1,83.7,0.4,0.15,0.01,0.02 122 | 947,1396,Defender,39,190,511,83.26,0.02,0.02,0.3,86.0,1.9,0.21,0.0,0.02 123 | 2279,3298,Midfielder,23,172,56,28.14,0.09,0.09,0.7,72.5,0.1,0.18,0.0,0.0 124 | 3571,5045,Defender,33,182,211,83.93,0.04,0.0,0.4,75.5,2.3,0.16,0.0,0.04 125 | 1001,1480,Forward,24,180,95,62.97,0.32,0.05,1.8,67.8,1.5,0.11,0.0,0.05 126 | 624,899,Forward,29,183,93,31.59,0.11,0.06,1.0,73.5,0.6,0.08,0.0,0.0 127 | 3044,4361,Defender,28,185,159,81.63,0.03,0.02,0.7,80.0,4.0,0.08,0.0,0.04 128 | 975,1441,Midfielder,32,187,302,69.53,0.04,0.06,0.6,87.8,1.1,0.19,0.01,0.03 129 | 2952,4228,Goalkeeper,29,188,194,88.37,0.0,0.01,0.0,65.2,0.3,0.06,0.01,0.03 130 | 1654,2409,Midfielder,27,183,224,69.96,0.06,0.06,1.2,83.3,1.1,0.19,0.01,0.02 131 | 1139,1684,Forward,22,178,145,61.12,0.28,0.05,1.6,83.6,0.3,0.06,0.01,0.02 132 | 3737,5275,Defender,36,185,79,81.76,0.03,0.0,0.3,79.3,2.2,0.16,0.0,0.01 133 | 1840,2699,Defender,24,189,120,81.08,0.02,0.03,0.5,78.6,3.0,0.18,0.02,0.05 134 | 1839,2698,Defender,31,186,159,85.58,0.05,0.04,0.9,67.3,4.6,0.23,0.0,0.03 135 | 240,332,Goalkeeper,33,195,118,88.16,0.0,0.01,0.0,71.1,0.5,0.03,0.0,0.04 136 | 1099,1624,Midfielder,20,190,77,42.22,0.01,0.01,0.3,87.1,0.5,0.16,0.01,0.0 137 | 2636,3797,Midfielder,33,173,85,53.42,0.07,0.04,0.8,79.7,0.4,0.15,0.01,0.0 138 | 1332,1973,Goalkeeper,32,193,380,89.76,0.0,0.01,0.0,80.2,0.3,0.03,0.0,0.03 139 | 2205,3196,Defender,22,185,52,73.31,0.02,0.1,0.4,81.1,1.0,0.12,0.04,0.0 140 | 3154,4503,Midfielder,26,178,65,62.22,0.06,0.14,1.6,71.4,0.3,0.11,0.0,0.03 141 | 721,1057,Midfielder,24,184,174,56.7,0.14,0.06,1.4,80.6,0.3,0.11,0.0,0.03 142 | 3190,4548,Defender,30,189,125,87.41,0.07,0.03,0.7,82.4,2.4,0.14,0.0,0.06 143 | 1657,2413,Defender,35,193,194,85.96,0.04,0.0,0.6,85.7,2.8,0.15,0.02,0.07 144 | 3744,5287,Defender,24,183,58,64.45,0.0,0.02,0.3,84.7,1.2,0.12,0.0,0.02 145 | 2329,3361,Defender,30,190,65,84.94,0.05,0.0,0.4,87.2,2.0,0.12,0.0,0.03 146 | 2120,3083,Midfielder,25,176,97,45.21,0.11,0.08,1.2,80.1,0.2,0.05,0.0,0.03 147 | 1121,1659,Goalkeeper,31,195,270,88.51,0.0,0.0,0.0,68.6,0.3,0.03,0.0,0.06 148 | 2564,3695,Forward,25,181,98,41.76,0.22,0.08,1.2,77.4,0.7,0.12,0.0,0.03 149 | 1310,1935,Goalkeeper,30,189,222,89.09,0.0,0.0,0.0,76.4,0.3,0.07,0.0,0.02 150 | 1396,2066,Midfielder,22,180,119,62.86,0.01,0.04,0.3,87.5,0.6,0.13,0.02,0.02 151 | 821,1187,Goalkeeper,31,194,112,89.92,0.0,0.0,0.0,65.0,0.3,0.05,0.0,0.02 152 | 1044,1543,Defender,34,188,243,71.71,0.04,0.02,0.3,82.2,2.1,0.16,0.0,0.02 153 | 509,744,Midfielder,25,178,139,62.63,0.17,0.06,1.5,81.4,0.5,0.1,0.0,0.03 154 | 1125,1665,Forward,31,189,273,64.14,0.37,0.12,2.2,63.8,3.9,0.07,0.0,0.08 155 | 3081,4407,Midfielder,21,182,87,61.9,0.07,0.15,1.3,75.7,0.7,0.17,0.0,0.05 156 | 3848,5424,Midfielder,25,170,111,64.57,0.08,0.06,0.7,79.0,0.1,0.07,0.0,0.05 157 | 2243,3243,Defender,31,188,227,80.2,0.02,0.01,0.2,83.4,1.6,0.12,0.0,0.02 158 | 1658,2414,Midfielder,26,183,75,54.76,0.09,0.11,1.2,72.9,0.8,0.04,0.0,0.04 159 | 1329,1970,Midfielder,27,178,215,65.63,0.31,0.12,2.0,83.9,0.8,0.06,0.0,0.11 160 | 3817,5379,Defender,25,189,73,62.37,0.07,0.01,0.5,85.6,1.5,0.12,0.0,0.01 161 | 457,665,Defender,23,191,61,78.36,0.0,0.02,0.4,81.7,2.3,0.25,0.0,0.02 162 | 3422,4847,Forward,27,188,304,61.75,0.32,0.03,2.0,64.6,2.6,0.09,0.01,0.06 163 | 423,613,Forward,22,191,87,34.82,0.11,0.03,0.8,60.8,1.6,0.14,0.01,0.01 164 | 1312,1939,Midfielder,21,178,119,65.71,0.05,0.07,0.7,78.7,0.8,0.24,0.0,0.03 165 | 2433,3519,Midfielder,25,186,60,60.87,0.1,0.13,0.9,72.2,0.7,0.13,0.0,0.02 166 | 1870,2743,Midfielder,30,181,99,68.72,0.15,0.16,1.4,72.9,0.9,0.12,0.01,0.06 167 | 2803,4024,Midfielder,21,182,63,55.79,0.02,0.08,0.2,78.6,0.4,0.13,0.0,0.0 168 | 1141,1689,Midfielder,25,175,116,51.85,0.21,0.19,1.7,79.7,0.3,0.06,0.0,0.1 169 | 2566,3698,Midfielder,32,178,129,71.49,0.05,0.05,0.8,85.5,0.4,0.21,0.0,0.05 170 | 485,707,Defender,25,183,106,83.09,0.05,0.01,0.5,84.0,1.2,0.19,0.01,0.02 171 | 668,979,Midfielder,28,191,248,56.87,0.1,0.07,1.0,86.7,1.1,0.08,0.0,0.04 172 | 3283,4663,Goalkeeper,26,185,105,89.33,0.0,0.01,0.0,62.3,0.2,0.08,0.0,0.06 173 | 471,690,Goalkeeper,25,186,161,90.32,0.0,0.01,0.0,76.5,0.2,0.02,0.01,0.03 174 | 37,52,Goalkeeper,22,190,53,90.57,0.0,0.0,0.0,69.0,0.2,0.0,0.0,0.04 175 | 3207,4573,Midfielder,24,180,51,42.55,0.12,0.04,1.0,75.4,0.9,0.02,0.02,0.02 176 | 105,157,Forward,29,185,302,63.27,0.36,0.08,2.2,74.9,1.3,0.07,0.0,0.07 177 | 2289,3309,Midfielder,28,180,203,56.78,0.14,0.07,1.5,84.3,0.4,0.12,0.0,0.03 178 | 2169,3154,Defender,33,183,113,81.62,0.02,0.0,0.4,85.6,1.2,0.12,0.01,0.02 179 | 3452,4892,Midfielder,25,176,110,66.4,0.23,0.19,2.4,75.4,0.3,0.07,0.0,0.12 180 | 1917,2805,Midfielder,26,180,97,38.93,0.03,0.05,0.4,84.1,0.4,0.08,0.02,0.03 181 | 2930,4202,Goalkeeper,35,196,219,89.29,0.0,0.0,0.0,64.1,0.3,0.06,0.0,0.02 182 | 994,1472,Goalkeeper,33,187,301,89.85,0.0,0.0,0.0,69.9,0.3,0.07,0.0,0.06 183 | 2617,3771,Midfielder,22,175,60,35.05,0.05,0.03,0.5,82.2,0.1,0.13,0.0,0.03 184 | 3243,4614,Midfielder,31,180,262,71.54,0.02,0.03,0.5,84.4,1.0,0.08,0.0,0.0 185 | 1012,1496,Midfielder,23,176,121,77.79,0.02,0.02,0.5,93.2,0.3,0.28,0.02,0.02 186 | 1691,2456,Forward,26,188,99,54.58,0.28,0.07,1.6,67.4,1.1,0.07,0.0,0.05 187 | 3454,4894,Defender,27,182,89,73.28,0.01,0.0,0.3,84.2,1.2,0.22,0.02,0.0 188 | 1606,2333,Midfielder,25,185,213,56.18,0.08,0.12,1.3,77.6,0.6,0.16,0.0,0.03 189 | 211,292,Midfielder,26,188,54,67.04,0.02,0.02,0.7,76.8,1.4,0.28,0.0,0.02 190 | 2888,4138,Midfielder,25,175,99,68.25,0.07,0.06,1.2,79.9,1.2,0.21,0.0,0.03 191 | 3743,5285,Midfielder,25,170,100,78.01,0.09,0.13,1.1,81.3,0.4,0.26,0.02,0.02 192 | 555,815,Midfielder,30,183,76,58.91,0.13,0.07,1.3,79.9,0.2,0.12,0.03,0.03 193 | 3828,5400,Goalkeeper,27,186,91,90.0,0.0,0.01,0.0,84.3,0.3,0.02,0.0,0.07 194 | 1859,2729,Goalkeeper,41,188,206,90.0,0.0,0.01,0.0,65.7,0.1,0.02,0.0,0.02 195 | 2576,3711,Defender,29,193,367,85.36,0.08,0.04,1.0,67.1,5.0,0.2,0.0,0.05 196 | 642,938,Defender,29,188,336,84.82,0.04,0.01,0.5,91.9,1.4,0.12,0.01,0.02 197 | 3610,5111,Forward,24,180,72,29.82,0.1,0.06,0.6,73.9,0.3,0.07,0.0,0.0 198 | 3287,4668,Midfielder,29,183,159,68.0,0.01,0.04,0.3,78.7,1.3,0.16,0.01,0.0 199 | 2000,2922,Forward,36,196,356,75.47,0.43,0.21,1.9,66.0,3.6,0.08,0.0,0.12 200 | 3767,5314,Defender,35,186,201,82.88,0.01,0.03,0.5,78.6,1.7,0.11,0.01,0.0 201 | 1048,1547,Forward,35,188,219,58.01,0.29,0.05,1.9,67.4,4.3,0.21,0.0,0.08 202 | 1248,1844,Midfielder,24,174,177,71.64,0.07,0.07,1.0,73.5,1.2,0.19,0.0,0.03 203 | 249,347,Midfielder,20,179,85,55.51,0.13,0.19,1.2,81.5,0.1,0.12,0.0,0.09 204 | 1091,1610,Midfielder,25,173,170,67.32,0.02,0.04,0.3,87.7,0.8,0.24,0.01,0.02 205 | 2527,3638,Defender,30,194,101,87.27,0.06,0.02,0.5,86.6,2.6,0.18,0.0,0.05 206 | 406,578,Midfielder,24,180,120,79.72,0.02,0.07,0.9,86.1,0.7,0.22,0.02,0.02 207 | 1350,2007,Goalkeeper,24,193,55,88.15,0.0,0.0,0.0,52.0,0.3,0.05,0.0,0.04 208 | 3550,5021,Defender,26,186,88,87.57,0.07,0.06,0.6,73.3,1.8,0.09,0.01,0.08 209 | 3818,5383,Forward,26,184,114,30.59,0.11,0.04,0.9,60.9,0.8,0.16,0.01,0.02 210 | 1725,2517,Midfielder,22,188,102,60.75,0.08,0.08,0.9,86.1,0.3,0.15,0.01,0.02 211 | 1572,2292,Midfielder,24,175,137,35.61,0.13,0.09,1.0,88.1,0.1,0.04,0.0,0.03 212 | 2399,3461,Midfielder,28,178,128,79.19,0.06,0.03,1.2,68.7,2.7,0.28,0.04,0.04 213 | 1055,1557,Defender,26,195,162,76.35,0.03,0.03,0.4,81.0,3.2,0.23,0.03,0.04 214 | 634,924,Midfielder,20,174,84,62.69,0.06,0.1,0.9,89.2,1.1,0.05,0.0,0.08 215 | 349,494,Defender,29,173,53,68.77,0.0,0.02,0.1,76.3,0.7,0.09,0.0,0.04 216 | 1475,2174,Forward,29,185,71,45.56,0.21,0.06,1.0,64.6,2.7,0.31,0.0,0.11 217 | 804,1154,Goalkeeper,30,187,260,89.95,0.0,0.0,0.0,64.5,0.3,0.1,0.0,0.05 218 | 2178,3167,Goalkeeper,28,186,70,89.61,0.0,0.0,0.0,55.7,0.2,0.0,0.0,0.07 219 | 2165,3149,Forward,32,185,216,51.19,0.2,0.04,1.4,75.6,0.9,0.11,0.0,0.01 220 | 3456,4896,Midfielder,23,181,69,82.68,0.01,0.04,0.6,88.6,0.5,0.26,0.01,0.03 221 | 1297,1916,Midfielder,24,174,160,74.91,0.03,0.08,0.6,85.9,0.4,0.09,0.01,0.02 222 | 134,198,Midfielder,23,173,70,52.54,0.13,0.11,1.2,80.6,0.2,0.09,0.0,0.04 223 | 3559,5031,Midfielder,23,174,77,54.47,0.04,0.1,1.0,79.7,0.4,0.12,0.01,0.03 224 | 2262,3273,Defender,36,191,283,82.78,0.03,0.0,0.3,83.2,2.3,0.12,0.01,0.01 225 | 53,76,Midfielder,29,184,61,36.44,0.07,0.11,0.6,77.5,0.3,0.08,0.02,0.03 226 | 1628,2370,Goalkeeper,29,195,100,89.55,0.0,0.0,0.0,55.9,0.2,0.06,0.0,0.09 227 | 452,654,Midfielder,34,184,294,62.61,0.06,0.04,0.6,86.8,1.0,0.19,0.0,0.02 228 | 2155,3134,Midfielder,22,174,52,45.23,0.1,0.08,0.9,77.3,0.3,0.12,0.0,0.02 229 | 1824,2674,Defender,26,186,240,80.76,0.07,0.02,0.7,85.7,2.5,0.16,0.01,0.09 230 | 259,367,Defender,26,185,227,84.93,0.06,0.03,0.5,86.3,2.5,0.3,0.03,0.04 231 | 1284,1895,Forward,23,193,163,58.58,0.31,0.15,1.8,74.1,0.5,0.09,0.0,0.07 232 | 661,969,Defender,23,194,76,79.16,0.04,0.04,0.6,78.9,2.6,0.3,0.03,0.05 233 | 328,471,Defender,30,184,125,86.29,0.04,0.02,0.3,82.2,2.9,0.29,0.02,0.09 234 | 394,558,Midfielder,26,178,168,76.77,0.14,0.15,1.3,80.1,0.7,0.11,0.01,0.09 235 | 3448,4885,Midfielder,28,184,222,51.26,0.06,0.11,0.9,68.5,0.6,0.07,0.0,0.01 236 | 1232,1820,Defender,23,188,100,64.51,0.01,0.02,0.3,87.0,1.1,0.18,0.0,0.0 237 | 2290,3311,Midfielder,25,177,51,79.55,0.2,0.08,1.8,78.0,0.9,0.08,0.0,0.02 238 | 1144,1692,Midfielder,20,182,106,49.61,0.08,0.02,0.8,76.5,0.4,0.19,0.0,0.0 239 | 88,132,Midfielder,26,176,172,59.95,0.09,0.12,0.8,86.6,0.4,0.15,0.01,0.03 240 | 857,1247,Midfielder,25,175,135,62.59,0.08,0.07,0.8,84.4,0.5,0.23,0.01,0.01 241 | 658,963,Forward,31,181,116,23.6,0.09,0.02,0.7,63.1,0.9,0.09,0.0,0.0 242 | 699,1017,Goalkeeper,30,194,176,89.75,0.0,0.0,0.0,58.9,0.4,0.06,0.0,0.06 243 | 3794,5347,Defender,30,180,96,85.8,0.06,0.01,0.7,82.0,2.5,0.1,0.02,0.02 244 | 3050,4370,Midfielder,27,184,169,71.73,0.09,0.14,0.6,77.8,0.9,0.09,0.0,0.07 245 | 1426,2103,Midfielder,28,184,90,57.89,0.08,0.04,1.1,77.1,1.2,0.23,0.01,0.02 246 | 1903,2783,Defender,26,185,139,83.05,0.02,0.0,0.4,84.8,3.3,0.13,0.01,0.01 247 | 3302,4685,Midfielder,24,185,85,66.93,0.08,0.06,0.5,80.5,0.8,0.11,0.0,0.01 248 | 2752,3947,Midfielder,28,180,212,83.89,0.21,0.12,2.1,83.5,0.2,0.22,0.0,0.08 249 | 1733,2528,Defender,22,188,64,66.02,0.06,0.11,0.5,84.7,1.5,0.25,0.0,0.03 250 | 667,977,Midfielder,25,185,213,74.81,0.05,0.08,1.1,82.3,0.5,0.11,0.02,0.02 251 | 2404,3473,Defender,33,196,159,76.6,0.06,0.01,0.5,76.3,3.2,0.1,0.01,0.06 252 | 421,609,Defender,27,188,158,81.68,0.03,0.01,0.7,67.2,2.3,0.32,0.01,0.04 253 | 2464,3557,Forward,24,182,60,46.47,0.12,0.08,1.3,64.4,1.5,0.05,0.0,0.0 254 | 3732,5269,Midfielder,28,182,109,52.17,0.02,0.02,0.3,77.3,0.7,0.08,0.03,0.0 255 | 1422,2098,Defender,29,186,243,82.72,0.09,0.03,0.7,81.2,3.6,0.14,0.01,0.05 256 | 831,1204,Defender,28,187,209,83.77,0.02,0.01,0.5,88.2,3.1,0.23,0.01,0.01 257 | 1065,1574,Forward,30,186,151,50.75,0.32,0.02,1.5,65.4,1.9,0.05,0.0,0.07 258 | 2172,3160,Defender,27,174,115,67.1,0.03,0.06,0.4,78.1,0.9,0.17,0.02,0.01 259 | 1619,2358,Midfielder,26,178,166,73.99,0.05,0.05,0.6,86.3,0.4,0.21,0.01,0.05 260 | 1223,1810,Midfielder,34,181,266,71.9,0.06,0.15,1.0,73.4,0.4,0.06,0.01,0.02 261 | 101,150,Midfielder,34,178,325,69.65,0.06,0.11,0.8,86.9,0.4,0.17,0.0,0.02 262 | 893,1314,Midfielder,20,185,91,51.77,0.09,0.1,1.3,77.2,0.3,0.13,0.0,0.02 263 | 3677,5202,Defender,25,185,97,63.98,0.03,0.03,0.5,86.9,2.1,0.06,0.02,0.0 264 | 127,190,Defender,27,192,105,84.64,0.02,0.02,0.7,82.0,3.2,0.29,0.02,0.1 265 | 327,467,Defender,33,187,177,75.11,0.05,0.02,0.4,77.6,2.7,0.1,0.02,0.04 266 | 578,844,Midfielder,28,170,180,52.78,0.08,0.09,1.1,78.8,0.2,0.08,0.02,0.01 267 | 976,1443,Defender,23,188,111,78.32,0.03,0.02,0.4,87.5,1.9,0.15,0.02,0.01 268 | 2845,4075,Midfielder,25,178,105,68.64,0.01,0.03,0.3,83.0,0.7,0.13,0.01,0.0 269 | 2849,4079,Midfielder,24,175,70,32.41,0.04,0.01,0.4,73.1,0.4,0.03,0.0,0.0 270 | 3289,4670,Midfielder,27,188,54,38.74,0.07,0.04,0.7,74.9,0.7,0.02,0.0,0.0 271 | 542,797,Forward,21,194,102,44.67,0.19,0.09,1.6,66.1,1.9,0.07,0.01,0.07 272 | 84,119,Midfielder,20,173,123,66.69,0.07,0.08,0.7,88.8,0.9,0.28,0.02,0.02 273 | 1608,2336,Midfielder,22,179,67,56.72,0.06,0.07,1.0,77.8,0.3,0.03,0.0,0.01 274 | 2247,3248,Forward,26,193,82,32.88,0.15,0.01,0.9,58.3,1.6,0.01,0.0,0.02 275 | 966,1427,Defender,28,185,199,79.86,0.02,0.01,0.3,86.9,1.9,0.28,0.04,0.04 276 | 132,196,Midfielder,25,178,104,38.11,0.09,0.08,0.8,91.1,0.4,0.12,0.01,0.01 277 | 3672,5193,Forward,21,185,136,46.83,0.29,0.07,1.5,78.1,0.8,0.05,0.0,0.04 278 | 3348,4742,Defender,22,185,63,53.0,0.02,0.02,0.7,78.7,1.1,0.13,0.02,0.0 279 | 905,1329,Midfielder,27,184,239,58.24,0.17,0.13,1.1,82.2,0.5,0.08,0.0,0.04 280 | 1311,1938,Defender,22,189,99,88.54,0.01,0.0,0.2,85.1,1.6,0.13,0.01,0.0 281 | 3339,4731,Midfielder,28,169,180,53.6,0.08,0.08,1.5,81.0,0.1,0.08,0.01,0.04 282 | 1323,1960,Defender,37,190,295,83.44,0.03,0.02,0.4,79.6,1.7,0.21,0.03,0.04 283 | 1363,2029,Midfielder,22,175,150,41.22,0.1,0.04,0.9,75.6,0.3,0.1,0.0,0.01 284 | 49,71,Goalkeeper,26,190,124,89.64,0.0,0.0,0.0,69.6,0.2,0.0,0.0,0.04 285 | 3379,4786,Midfielder,26,171,125,35.58,0.06,0.03,0.8,85.0,0.2,0.1,0.0,0.02 286 | 3197,4561,Defender,24,175,144,83.62,0.04,0.01,0.5,69.9,3.0,0.18,0.0,0.01 287 | 1432,2110,Defender,31,193,159,77.9,0.04,0.02,0.4,76.0,1.6,0.29,0.02,0.02 288 | 3531,4993,Goalkeeper,28,191,229,88.9,0.0,0.01,0.0,59.9,0.4,0.11,0.0,0.04 289 | 917,1345,Midfielder,26,191,217,79.34,0.06,0.06,1.0,81.8,1.5,0.24,0.01,0.06 290 | 1184,1748,Defender,30,189,261,86.61,0.04,0.03,0.3,86.3,1.7,0.11,0.0,0.02 291 | 2837,4066,Goalkeeper,27,183,64,89.89,0.0,0.0,0.0,62.5,0.2,0.06,0.0,0.03 292 | 3156,4505,Midfielder,24,172,83,67.35,0.22,0.07,1.3,79.2,0.3,0.12,0.0,0.06 293 | 955,1410,Defender,22,187,76,55.14,0.07,0.01,0.3,89.4,1.5,0.17,0.04,0.04 294 | 2891,4143,Midfielder,25,181,81,68.2,0.1,0.16,1.5,79.8,0.4,0.12,0.0,0.06 295 | 78,109,Goalkeeper,32,187,506,89.58,0.0,0.0,0.0,80.4,0.3,0.03,0.0,0.02 296 | 802,1152,Midfielder,30,175,330,69.4,0.14,0.12,1.3,78.7,0.6,0.13,0.02,0.03 297 | 334,478,Midfielder,26,180,101,66.13,0.1,0.13,1.5,82.4,0.2,0.08,0.01,0.03 298 | 554,814,Goalkeeper,30,191,183,89.52,0.0,0.01,0.0,61.9,0.4,0.1,0.0,0.03 299 | 1774,2594,Midfielder,20,179,66,41.67,0.03,0.03,0.8,88.8,0.7,0.18,0.02,0.02 300 | 3702,5234,Defender,30,189,63,87.3,0.0,0.02,0.6,75.2,2.4,0.21,0.0,0.0 301 | 2868,4107,Midfielder,26,183,182,58.24,0.06,0.07,1.0,83.1,0.5,0.13,0.01,0.01 302 | 3366,4765,Forward,33,188,56,38.23,0.2,0.05,1.2,60.7,2.0,0.11,0.02,0.02 303 | 2225,3221,Midfielder,26,178,82,40.98,0.06,0.02,0.6,79.4,0.6,0.24,0.02,0.01 304 | 1574,2295,Defender,24,185,133,80.01,0.02,0.06,0.3,85.4,1.3,0.13,0.01,0.01 305 | 1011,1495,Midfielder,26,170,162,62.22,0.06,0.09,0.8,83.9,0.5,0.12,0.01,0.06 306 | 1103,1630,Midfielder,26,185,300,70.92,0.05,0.07,0.9,86.0,1.0,0.24,0.0,0.04 307 | 591,858,Defender,29,185,63,79.89,0.03,0.0,0.6,81.9,2.0,0.21,0.02,0.06 308 | 701,1024,Defender,26,188,111,80.5,0.03,0.01,0.3,83.0,1.9,0.12,0.0,0.0 309 | 1821,2667,Midfielder,25,183,60,56.38,0.17,0.05,1.8,62.8,2.0,0.08,0.0,0.07 310 | 370,521,Defender,30,190,190,78.93,0.07,0.02,0.5,85.6,2.2,0.17,0.02,0.03 311 | 1110,1640,Midfielder,26,175,239,73.08,0.09,0.08,1.0,85.5,0.8,0.25,0.01,0.03 312 | 2041,2973,Defender,30,172,82,73.07,0.01,0.0,0.3,78.5,1.6,0.13,0.01,0.02 313 | 1627,2368,Defender,24,183,78,76.01,0.0,0.06,0.4,73.0,0.7,0.08,0.03,0.03 314 | 913,1340,Defender,22,180,56,88.38,0.02,0.04,0.6,79.1,1.4,0.16,0.0,0.0 315 | 3738,5276,Midfielder,20,177,70,42.9,0.04,0.06,0.7,82.5,0.6,0.06,0.0,0.01 316 | 2611,3765,Midfielder,25,179,108,47.63,0.07,0.09,0.9,76.6,0.4,0.09,0.0,0.01 317 | 2933,4207,Midfielder,24,189,98,59.63,0.06,0.03,0.8,81.8,1.4,0.14,0.02,0.04 318 | 1603,2327,Goalkeeper,25,195,53,89.87,0.0,0.0,0.0,58.3,0.4,0.11,0.0,0.06 319 | 3734,5271,Midfielder,25,180,62,46.13,0.03,0.0,0.4,84.8,0.2,0.18,0.02,0.0 320 | 2377,3435,Forward,26,175,110,75.36,0.37,0.21,2.1,64.0,0.6,0.09,0.02,0.03 321 | 2558,3688,Defender,21,180,72,61.03,0.01,0.06,0.4,82.4,0.9,0.19,0.03,0.01 322 | 914,1341,Midfielder,22,179,84,48.88,0.08,0.14,1.5,79.9,1.2,0.11,0.0,0.07 323 | 838,1220,Midfielder,28,171,306,56.95,0.09,0.15,1.0,83.9,0.3,0.07,0.0,0.06 324 | 3449,4887,Midfielder,24,177,68,80.63,0.35,0.16,2.0,84.2,0.5,0.04,0.0,0.18 325 | 258,366,Midfielder,27,175,271,75.32,0.26,0.21,2.3,83.5,0.2,0.15,0.0,0.11 326 | 3842,5417,Defender,30,188,73,79.62,0.01,0.0,0.3,85.9,1.9,0.19,0.01,0.0 327 | 689,1006,Goalkeeper,23,189,77,89.62,0.0,0.0,0.0,72.6,0.3,0.04,0.0,0.08 328 | 3719,5253,Goalkeeper,33,192,131,89.79,0.0,0.01,0.0,47.3,0.4,0.11,0.0,0.07 329 | 2443,3533,Forward,21,182,53,35.77,0.15,0.06,0.9,79.1,0.5,0.0,0.02,0.02 330 | 2968,4251,Midfielder,34,175,340,86.66,0.09,0.09,1.0,90.6,0.7,0.05,0.0,0.03 331 | 1814,2657,Midfielder,25,170,206,65.08,0.03,0.01,0.6,90.0,0.3,0.09,0.0,0.01 332 | 1752,2560,Defender,24,189,118,65.69,0.03,0.01,0.2,80.4,2.1,0.22,0.01,0.01 333 | 384,542,Midfielder,24,180,244,65.58,0.23,0.27,1.1,84.9,0.1,0.03,0.0,0.1 334 | 916,1344,Defender,27,195,264,85.16,0.06,0.02,0.6,85.8,2.8,0.19,0.02,0.02 335 | 1265,1864,Midfielder,32,167,152,71.62,0.15,0.08,1.3,69.8,0.3,0.15,0.01,0.04 336 | 3477,4924,Defender,32,187,170,86.31,0.05,0.0,0.6,78.0,3.0,0.24,0.01,0.06 337 | 631,915,Midfielder,27,181,163,49.55,0.15,0.2,1.4,78.3,0.5,0.13,0.01,0.04 338 | 2029,2955,Midfielder,27,181,55,60.67,0.09,0.04,0.8,84.0,0.9,0.16,0.0,0.05 339 | 505,734,Forward,27,174,336,67.04,0.46,0.1,3.0,71.1,1.0,0.16,0.01,0.13 340 | 882,1293,Defender,23,191,68,56.74,0.04,0.01,0.2,79.3,1.2,0.21,0.0,0.0 341 | 114,171,Defender,25,187,106,73.4,0.06,0.01,0.4,84.0,1.5,0.22,0.01,0.0 342 | 2785,3997,Midfielder,27,185,215,72.18,0.02,0.07,0.7,79.6,1.0,0.28,0.0,0.03 343 | 3024,4336,Forward,25,189,90,35.23,0.14,0.07,1.3,60.9,2.1,0.14,0.0,0.02 344 | 2915,4179,Midfielder,28,175,131,62.15,0.08,0.16,1.8,76.5,0.3,0.05,0.01,0.05 345 | 1960,2870,Midfielder,20,175,53,73.13,0.08,0.06,0.9,81.7,0.8,0.17,0.02,0.04 346 | 830,1201,Goalkeeper,30,191,220,89.45,0.0,0.0,0.0,66.5,0.2,0.06,0.0,0.05 347 | 2528,3639,Forward,27,182,113,61.26,0.3,0.04,2.3,74.7,2.0,0.27,0.0,0.09 348 | 794,1143,Defender,23,187,73,85.36,0.03,0.0,0.3,87.4,1.7,0.16,0.03,0.0 349 | 2721,3915,Midfielder,23,188,52,27.25,0.12,0.08,0.6,64.2,0.4,0.08,0.0,0.0 350 | 1477,2178,Defender,38,186,52,83.96,0.1,0.0,0.5,70.3,2.3,0.25,0.02,0.0 351 | 2210,3203,Defender,32,185,258,83.36,0.03,0.02,0.3,84.1,1.7,0.13,0.01,0.01 352 | 3008,4314,Defender,31,185,325,85.19,0.02,0.02,0.3,80.5,2.1,0.15,0.01,0.01 353 | 1286,1897,Defender,30,187,309,82.93,0.02,0.02,0.2,89.0,1.9,0.08,0.0,0.01 354 | 748,1092,Defender,29,177,144,79.51,0.03,0.1,0.6,75.7,0.8,0.13,0.0,0.04 355 | 910,1337,Goalkeeper,36,186,65,85.2,0.0,0.0,0.0,55.8,0.4,0.09,0.02,0.03 356 | 2775,3982,Defender,34,179,254,78.47,0.02,0.11,0.4,79.9,0.7,0.09,0.0,0.01 357 | 2904,4161,Midfielder,26,180,98,68.68,0.07,0.09,0.5,86.1,0.7,0.1,0.02,0.03 358 | 3079,4402,Midfielder,34,174,54,40.94,0.07,0.09,0.9,76.3,0.1,0.15,0.04,0.0 359 | 3175,4529,Midfielder,26,179,131,62.63,0.05,0.15,0.6,78.2,0.5,0.11,0.0,0.02 360 | 911,1338,Forward,32,191,360,67.26,0.34,0.06,1.8,62.3,3.3,0.03,0.0,0.05 361 | 1230,1817,Midfielder,22,173,150,63.43,0.06,0.12,1.4,75.6,0.3,0.18,0.01,0.05 362 | 2676,3846,Midfielder,29,174,201,77.57,0.05,0.02,0.7,82.9,0.9,0.16,0.0,0.0 363 | 1127,1668,Midfielder,27,180,282,67.23,0.16,0.11,1.1,83.7,0.4,0.14,0.0,0.04 364 | 2551,3672,Defender,28,183,176,84.02,0.01,0.01,0.4,90.2,2.1,0.12,0.01,0.02 365 | 494,721,Defender,24,186,62,74.02,0.05,0.0,0.5,86.9,1.7,0.19,0.03,0.02 366 | 1793,2625,Midfielder,22,184,122,47.38,0.12,0.08,1.2,83.7,0.3,0.04,0.0,0.06 367 | 460,670,Midfielder,24,175,144,62.26,0.01,0.01,0.2,88.6,0.7,0.19,0.0,0.01 368 | 1591,2314,Forward,29,180,113,47.81,0.19,0.03,1.5,70.1,1.1,0.16,0.02,0.04 369 | 1375,2042,Midfielder,26,183,55,64.07,0.18,0.15,1.5,81.7,0.5,0.09,0.0,0.07 370 | 2158,3142,Midfielder,24,174,74,67.23,0.03,0.05,0.6,84.0,0.3,0.15,0.01,0.0 371 | 1833,2688,Forward,18,173,63,45.98,0.29,0.02,1.5,72.6,0.4,0.17,0.0,0.03 372 | 1431,2109,Defender,31,185,112,80.01,0.03,0.04,0.3,81.8,1.5,0.21,0.02,0.03 373 | 698,1016,Defender,33,192,277,84.59,0.05,0.03,0.6,77.1,3.7,0.24,0.0,0.05 374 | 2198,3188,Midfielder,24,186,76,74.5,0.0,0.13,0.5,74.8,1.7,0.12,0.01,0.03 375 | 2632,3791,Midfielder,34,175,341,82.56,0.04,0.1,0.6,80.6,0.4,0.18,0.0,0.01 376 | 1798,2635,Defender,34,185,336,79.42,0.04,0.05,0.6,82.0,1.4,0.24,0.01,0.03 377 | 1260,1859,Defender,33,189,232,86.76,0.05,0.03,1.0,77.9,2.6,0.17,0.03,0.03 378 | 862,1256,Defender,41,188,492,86.55,0.04,0.03,0.5,89.3,2.0,0.2,0.02,0.02 379 | 1009,1492,Goalkeeper,31,193,299,90.13,0.0,0.01,0.0,52.0,0.3,0.06,0.0,0.05 380 | 122,184,Midfielder,32,183,100,77.47,0.02,0.02,0.6,78.8,1.4,0.14,0.0,0.0 381 | 1005,1488,Defender,23,187,54,83.04,0.11,0.04,0.6,82.5,2.5,0.28,0.02,0.06 382 | 488,712,Forward,28,188,83,55.59,0.28,0.1,1.7,72.6,1.5,0.05,0.0,0.08 383 | 90,134,Goalkeeper,34,192,379,89.74,0.0,0.0,0.0,70.3,0.2,0.03,0.01,0.02 384 | 985,1461,Midfielder,26,183,132,62.45,0.05,0.08,0.9,80.5,1.0,0.08,0.02,0.02 385 | 2227,3223,Midfielder,27,180,101,51.32,0.03,0.0,0.5,84.8,0.4,0.08,0.01,0.01 386 | 1124,1664,Midfielder,23,174,117,65.04,0.23,0.1,1.7,78.1,0.2,0.14,0.0,0.15 387 | 643,940,Defender,20,186,54,80.98,0.04,0.0,0.4,89.6,1.4,0.3,0.04,0.0 388 | 3088,4415,Defender,32,187,55,68.87,0.05,0.04,0.6,76.2,1.3,0.22,0.02,0.02 389 | 2856,4087,Midfielder,22,183,70,59.0,0.1,0.13,1.7,78.6,0.2,0.01,0.0,0.04 390 | 3819,5386,Midfielder,25,181,63,44.95,0.0,0.0,0.3,79.7,0.8,0.22,0.02,0.0 391 | 823,1192,Midfielder,27,167,149,53.48,0.1,0.09,1.4,72.3,0.2,0.08,0.0,0.03 392 | 1602,2326,Defender,28,189,71,67.17,0.0,0.0,0.5,84.1,1.7,0.1,0.01,0.03 393 | 3266,4642,Midfielder,30,182,79,58.42,0.1,0.13,0.9,80.5,0.4,0.09,0.0,0.04 394 | 1462,2155,Defender,28,180,71,77.96,0.01,0.1,0.5,82.6,1.1,0.1,0.03,0.03 395 | 2379,3437,Midfielder,24,163,132,71.62,0.06,0.05,1.0,81.1,0.4,0.16,0.01,0.02 396 | 2791,4009,Goalkeeper,28,185,124,89.51,0.0,0.0,0.0,56.2,0.5,0.02,0.0,0.05 397 | 3153,4502,Midfielder,22,170,74,43.34,0.12,0.07,0.9,73.5,0.3,0.09,0.0,0.01 398 | 2949,4225,Defender,31,183,138,61.24,0.0,0.04,0.2,77.3,0.6,0.1,0.0,0.0 399 | 1642,2391,Midfielder,30,180,280,55.5,0.04,0.06,0.7,86.4,0.2,0.24,0.01,0.01 400 | 3816,5378,Midfielder,32,175,127,59.4,0.19,0.17,1.6,82.8,0.4,0.08,0.0,0.06 401 | 811,1165,Midfielder,31,180,88,53.32,0.09,0.05,0.5,84.1,0.5,0.14,0.02,0.03 402 | 1618,2357,Goalkeeper,33,190,280,89.81,0.0,0.0,0.0,70.5,0.3,0.02,0.0,0.01 403 | 762,1109,Midfielder,24,178,122,59.11,0.02,0.01,0.3,85.3,0.8,0.39,0.01,0.0 404 | 3051,4371,Midfielder,25,181,135,70.74,0.1,0.13,0.9,79.2,0.3,0.23,0.0,0.04 405 | 382,539,Midfielder,23,178,132,80.35,0.05,0.07,0.7,86.9,0.9,0.26,0.0,0.02 406 | 1067,1576,Midfielder,22,172,82,61.35,0.11,0.09,1.2,85.1,0.9,0.13,0.0,0.04 407 | 880,1288,Midfielder,26,183,100,54.17,0.01,0.05,0.7,80.5,0.6,0.19,0.0,0.01 408 | 1090,1609,Defender,28,189,242,84.86,0.04,0.01,0.3,84.1,2.4,0.23,0.0,0.04 409 | 1342,1991,Midfielder,21,170,147,54.18,0.09,0.16,1.3,83.1,0.2,0.06,0.0,0.02 410 | 1117,1652,Midfielder,29,192,239,83.23,0.17,0.05,1.4,76.4,4.0,0.15,0.0,0.09 411 | 3261,4637,Defender,26,180,51,82.18,0.04,0.04,0.5,80.9,2.6,0.14,0.0,0.04 412 | 287,413,Midfielder,25,191,75,40.72,0.05,0.01,0.3,78.8,0.8,0.17,0.0,0.0 413 | 2,6,Midfielder,21,182,68,62.74,0.19,0.04,1.7,83.8,0.3,0.07,0.0,0.04 414 | 3839,5414,Midfielder,25,169,53,51.26,0.11,0.26,1.3,74.0,0.2,0.15,0.0,0.04 415 | 3557,5029,Midfielder,24,189,64,57.75,0.2,0.02,1.1,75.6,0.8,0.05,0.0,0.03 416 | 1946,2849,Goalkeeper,28,196,183,89.53,0.0,0.01,0.0,65.1,0.3,0.03,0.0,0.04 417 | 653,952,Midfielder,25,178,116,48.01,0.03,0.05,0.4,86.0,0.6,0.14,0.02,0.01 418 | 1522,2228,Goalkeeper,28,184,63,87.95,0.0,0.0,0.0,54.7,0.2,0.14,0.0,0.02 419 | 3374,4776,Defender,31,186,83,84.16,0.0,0.04,0.4,83.0,1.7,0.17,0.01,0.0 420 | 1443,2123,Defender,29,194,150,85.56,0.05,0.04,0.5,71.7,4.7,0.13,0.01,0.07 421 | 1348,2004,Midfielder,22,183,61,51.43,0.15,0.13,1.1,83.8,0.1,0.15,0.0,0.02 422 | 2726,3920,Defender,29,189,196,76.28,0.04,0.04,0.7,77.2,3.5,0.22,0.01,0.05 423 | 2157,3141,Goalkeeper,25,185,79,89.91,0.0,0.0,0.0,67.9,0.2,0.14,0.01,0.09 424 | 2901,4156,Midfielder,28,175,224,47.95,0.09,0.08,1.0,73.4,0.9,0.2,0.0,0.02 425 | 510,745,Midfielder,30,180,466,63.36,0.14,0.11,1.4,85.9,0.3,0.09,0.0,0.02 426 | 368,516,Goalkeeper,27,202,131,89.21,0.0,0.0,0.0,57.8,0.3,0.08,0.02,0.03 427 | 1578,2299,Midfielder,33,185,403,70.88,0.13,0.11,1.3,89.1,0.6,0.09,0.0,0.05 428 | 467,684,Midfielder,22,188,106,57.2,0.08,0.06,0.9,88.3,1.1,0.21,0.0,0.0 429 | 963,1423,Midfielder,27,183,219,71.29,0.04,0.06,0.6,84.7,0.7,0.17,0.0,0.01 430 | 2323,3355,Midfielder,26,186,96,76.65,0.1,0.21,1.5,67.9,2.0,0.21,0.0,0.19 431 | 3777,5326,Defender,27,193,74,62.47,0.03,0.01,0.5,81.6,2.4,0.2,0.0,0.01 432 | 2734,3929,Midfielder,26,183,95,79.38,0.16,0.05,0.9,84.9,1.0,0.17,0.0,0.04 433 | 2371,3424,Midfielder,29,185,220,54.85,0.07,0.09,0.7,84.8,0.1,0.11,0.0,0.04 434 | 2731,3926,Defender,28,194,106,83.97,0.07,0.03,0.5,84.8,3.2,0.1,0.01,0.04 435 | 2552,3674,Midfielder,31,185,84,60.64,0.04,0.0,0.5,85.1,0.7,0.24,0.0,0.0 436 | 803,1153,Defender,26,190,198,82.85,0.05,0.04,0.6,83.3,2.4,0.17,0.03,0.05 437 | 512,747,Midfielder,22,175,94,42.35,0.03,0.03,0.8,89.1,0.1,0.1,0.0,0.01 438 | 2596,3740,Defender,32,185,79,85.38,0.0,0.03,0.5,76.8,1.9,0.23,0.0,0.03 439 | 2685,3863,Goalkeeper,28,193,146,90.0,0.0,0.0,0.0,54.3,0.4,0.03,0.0,0.05 440 | 2613,3767,Defender,29,185,91,80.82,0.02,0.0,0.3,88.0,1.9,0.09,0.01,0.01 441 | 477,697,Midfielder,27,177,195,68.91,0.06,0.05,0.8,84.9,0.8,0.19,0.01,0.03 442 | 801,1151,Goalkeeper,26,189,69,89.65,0.0,0.0,0.0,75.6,0.1,0.1,0.01,0.01 443 | 1077,1589,Midfielder,24,170,54,38.3,0.06,0.04,0.8,73.0,0.1,0.04,0.0,0.02 444 | 3600,5094,Defender,34,190,289,85.2,0.03,0.03,0.6,79.2,4.1,0.17,0.02,0.05 445 | 1024,1516,Midfielder,34,175,424,69.47,0.1,0.15,1.2,79.0,0.8,0.12,0.0,0.05 446 | 607,878,Goalkeeper,35,185,237,89.18,0.0,0.01,0.0,64.8,0.1,0.02,0.0,0.03 447 | 720,1056,Midfielder,28,169,51,41.25,0.16,0.1,0.9,77.4,0.3,0.04,0.0,0.06 448 | 1688,2452,Defender,33,184,371,84.84,0.06,0.03,0.4,85.7,1.5,0.17,0.02,0.03 449 | 819,1176,Forward,31,195,223,67.82,0.3,0.08,2.0,65.8,5.0,0.13,0.0,0.1 450 | 3695,5223,Defender,26,190,146,81.18,0.03,0.03,0.3,84.6,1.9,0.1,0.01,0.01 451 | 2349,3388,Midfielder,28,180,322,81.01,0.11,0.09,1.2,78.6,1.5,0.15,0.01,0.05 452 | 1294,1910,Defender,28,190,212,85.66,0.05,0.02,0.3,80.2,1.9,0.15,0.01,0.0 453 | 2364,3410,Midfielder,21,172,87,56.72,0.15,0.15,1.4,81.8,0.3,0.2,0.0,0.07 454 | 788,1137,Midfielder,23,184,84,65.81,0.05,0.02,1.1,86.0,0.9,0.27,0.0,0.01 455 | 2536,3651,Defender,31,190,103,65.32,0.01,0.0,0.2,81.3,1.2,0.08,0.01,0.0 456 | 1605,2331,Defender,25,191,141,79.46,0.09,0.01,0.5,83.9,2.3,0.18,0.01,0.03 457 | 1575,2296,Forward,23,178,104,42.17,0.21,0.03,1.9,66.3,2.1,0.06,0.0,0.04 458 | 3670,5187,Midfielder,21,179,90,65.67,0.27,0.18,2.5,85.2,0.4,0.08,0.0,0.07 459 | 988,1464,Defender,25,188,99,74.4,0.06,0.03,0.5,88.2,1.8,0.07,0.01,0.02 460 | 1120,1656,Defender,26,194,156,80.26,0.04,0.01,0.9,82.5,2.6,0.15,0.01,0.07 461 | 365,513,Forward,33,189,364,66.2,0.38,0.14,2.4,72.3,1.9,0.08,0.0,0.07 462 | 3276,4655,Midfielder,23,183,169,60.55,0.25,0.1,2.4,76.1,0.2,0.08,0.01,0.09 463 | 338,482,Midfielder,24,188,134,54.01,0.13,0.07,1.2,80.3,0.6,0.08,0.01,0.02 464 | 548,805,Midfielder,23,180,67,73.9,0.06,0.03,0.7,87.6,0.8,0.18,0.01,0.01 465 | 1852,2721,Midfielder,22,182,117,68.32,0.18,0.13,1.3,85.7,0.6,0.12,0.01,0.03 466 | 2647,3809,Midfielder,35,180,202,77.19,0.12,0.03,1.4,87.1,1.0,0.19,0.0,0.04 467 | 565,825,Midfielder,30,188,204,69.84,0.07,0.03,1.1,82.1,1.4,0.1,0.0,0.04 468 | 3573,5049,Midfielder,33,174,115,66.3,0.17,0.17,1.5,78.6,0.3,0.2,0.0,0.05 469 | 982,1456,Forward,20,180,63,37.44,0.11,0.08,1.2,69.5,1.0,0.13,0.0,0.02 470 | 3049,4368,Defender,20,190,57,74.82,0.02,0.0,0.2,87.6,1.9,0.07,0.0,0.02 471 | 1423,2099,Forward,26,183,177,46.41,0.2,0.03,1.4,72.8,0.5,0.05,0.0,0.02 472 | 3733,5270,Midfielder,27,181,109,39.05,0.04,0.04,0.6,84.0,0.5,0.05,0.0,0.0 473 | 511,746,Defender,32,190,393,81.59,0.06,0.03,0.5,90.0,1.9,0.08,0.01,0.02 474 | 1930,2825,Defender,23,184,58,79.69,0.0,0.05,0.5,83.7,1.6,0.21,0.02,0.0 475 | 2924,4194,Defender,33,185,324,83.33,0.03,0.02,0.4,85.8,2.3,0.17,0.02,0.02 476 | 1514,2219,Forward,34,186,330,55.92,0.19,0.04,1.8,70.3,2.4,0.1,0.01,0.03 477 | 813,1170,Midfielder,32,168,285,72.74,0.01,0.02,0.2,88.8,0.3,0.15,0.0,0.0 478 | 710,1039,Goalkeeper,32,191,254,89.56,0.0,0.0,0.0,47.6,0.4,0.06,0.0,0.09 479 | 1734,2529,Goalkeeper,29,183,271,90.31,0.0,0.0,0.0,64.1,0.4,0.03,0.0,0.04 480 | 3045,4363,Midfielder,26,178,124,64.44,0.15,0.1,1.5,82.5,0.5,0.12,0.0,0.06 481 | 3789,5341,Forward,31,175,57,59.18,0.35,0.09,1.9,79.1,1.1,0.04,0.0,0.05 482 | 2089,3039,Midfielder,25,186,51,56.25,0.08,0.06,0.5,86.0,0.7,0.08,0.0,0.02 483 | 3267,4645,Forward,24,188,62,57.95,0.35,0.06,1.9,77.3,1.4,0.16,0.0,0.11 484 | 2296,3318,Forward,27,185,115,45.7,0.14,0.05,1.1,75.7,1.1,0.08,0.02,0.02 485 | 352,497,Midfielder,27,184,114,50.04,0.04,0.02,0.9,80.4,0.8,0.21,0.0,0.0 486 | 3593,5081,Midfielder,30,187,170,71.66,0.11,0.04,1.1,75.8,1.3,0.21,0.01,0.05 487 | 3746,5290,Defender,34,183,186,79.23,0.06,0.04,0.4,85.0,1.9,0.12,0.01,0.02 488 | 1495,2199,Defender,24,189,120,76.35,0.05,0.05,0.6,65.2,2.9,0.11,0.02,0.01 489 | 1789,2619,Forward,25,186,148,65.95,0.39,0.07,2.5,70.6,2.0,0.15,0.01,0.05 490 | 647,944,Forward,30,194,272,67.81,0.29,0.1,2.0,63.4,4.3,0.12,0.01,0.11 491 | 1556,2269,Midfielder,29,183,197,61.68,0.03,0.02,0.8,82.2,1.1,0.16,0.02,0.02 492 | 2380,3439,Defender,27,193,113,84.92,0.03,0.01,0.6,74.5,4.1,0.19,0.02,0.04 493 | 2183,3172,Defender,27,190,58,73.91,0.02,0.05,0.5,84.1,1.6,0.16,0.0,0.02 494 | 1784,2613,Forward,20,183,69,44.97,0.2,0.04,1.3,78.8,0.5,0.06,0.0,0.01 495 | 1163,1715,Goalkeeper,33,188,83,86.39,0.0,0.0,0.0,56.6,0.1,0.01,0.0,0.02 496 | 1743,2547,Midfielder,26,174,113,49.22,0.12,0.11,1.1,85.3,0.3,0.06,0.0,0.04 497 | 1327,1965,Midfielder,32,170,237,64.82,0.14,0.09,1.7,87.2,0.3,0.16,0.01,0.03 498 | 1054,1556,Midfielder,22,174,76,56.12,0.12,0.12,1.2,85.2,0.3,0.14,0.0,0.04 499 | 388,547,Goalkeeper,26,197,118,89.32,0.0,0.0,0.0,72.3,0.4,0.1,0.01,0.03 500 | 18,26,Goalkeeper,34,188,135,89.24,0.0,0.01,0.0,56.9,0.3,0.01,0.01,0.02 501 | 504,733,Midfielder,27,175,348,79.02,0.09,0.13,1.1,83.5,0.7,0.24,0.01,0.05 502 | 1656,2412,Defender,27,190,112,74.41,0.02,0.01,0.2,84.8,1.7,0.21,0.04,0.01 503 | 1405,2078,Goalkeeper,32,190,274,90.19,0.0,0.0,0.0,71.0,0.4,0.08,0.0,0.04 504 | 2272,3289,Forward,29,185,130,58.95,0.52,0.05,2.4,70.2,1.9,0.2,0.0,0.08 505 | 3696,5224,Midfielder,32,177,259,64.17,0.1,0.08,1.2,73.4,0.7,0.22,0.0,0.02 506 | 3151,4499,Defender,24,192,74,56.0,0.08,0.0,0.6,76.5,1.5,0.23,0.0,0.03 507 | 1528,2235,Forward,27,184,111,32.26,0.11,0.05,0.9,64.6,1.3,0.01,0.0,0.01 508 | 1115,1645,Midfielder,23,178,59,55.12,0.05,0.12,0.7,88.0,0.1,0.15,0.0,0.0 509 | 1485,2187,Defender,27,191,61,68.95,0.03,0.0,0.5,77.6,2.9,0.21,0.03,0.05 510 | 1933,2830,Midfielder,26,165,186,74.46,0.19,0.14,2.0,80.6,0.4,0.12,0.01,0.08 511 | 2763,3966,Forward,20,180,69,37.35,0.1,0.03,1.1,65.9,0.6,0.09,0.0,0.01 512 | 883,1296,Midfielder,25,185,110,70.25,0.03,0.13,0.7,81.4,0.5,0.16,0.01,0.01 513 | 990,1466,Defender,27,190,212,79.72,0.05,0.02,0.6,80.8,1.6,0.23,0.01,0.04 514 | 390,551,Midfielder,23,178,127,77.32,0.08,0.14,1.4,86.4,0.7,0.19,0.0,0.03 515 | 2048,2980,Midfielder,26,172,63,62.86,0.06,0.03,0.5,72.9,1.0,0.21,0.0,0.02 516 | 2293,3315,Goalkeeper,29,187,124,88.97,0.0,0.0,0.0,60.9,0.3,0.04,0.0,0.04 517 | 1910,2793,Midfielder,31,183,140,71.36,0.07,0.06,1.1,71.7,0.9,0.14,0.0,0.04 518 | 2259,3267,Goalkeeper,30,191,129,89.64,0.0,0.0,0.0,65.1,0.3,0.09,0.02,0.05 519 | 1425,2102,Midfielder,21,177,53,62.13,0.04,0.02,0.4,89.7,0.3,0.3,0.0,0.0 520 | 1249,1845,Midfielder,31,179,297,69.44,0.15,0.13,1.2,86.7,0.5,0.13,0.0,0.06 521 | 2473,3573,Midfielder,31,170,204,63.36,0.14,0.13,0.9,78.9,0.3,0.11,0.0,0.03 522 | 2530,3641,Midfielder,33,173,185,61.2,0.15,0.14,1.6,80.5,0.2,0.14,0.01,0.03 523 | 3691,5219,Midfielder,24,175,104,66.38,0.03,0.02,0.5,82.4,0.6,0.02,0.01,0.02 524 | 3429,4858,Midfielder,24,183,85,52.31,0.24,0.11,1.3,78.3,0.2,0.08,0.01,0.09 525 | 74,103,Defender,25,180,296,85.18,0.04,0.05,0.6,87.3,2.3,0.1,0.01,0.04 526 | 515,754,Defender,23,190,59,84.76,0.03,0.02,0.4,83.9,1.8,0.12,0.02,0.03 527 | 3496,4944,Defender,24,186,93,59.24,0.01,0.03,0.3,80.1,0.7,0.06,0.0,0.02 528 | 223,305,Midfielder,28,171,67,43.69,0.03,0.04,0.7,71.6,0.2,0.12,0.01,0.0 529 | 1448,2130,Midfielder,21,174,60,55.88,0.18,0.1,1.5,78.2,0.3,0.13,0.0,0.07 530 | 1250,1846,Goalkeeper,23,199,84,90.74,0.0,0.0,0.0,78.1,0.3,0.05,0.0,0.04 531 | 2573,3706,Forward,27,191,76,46.91,0.14,0.01,0.8,72.5,1.0,0.01,0.0,0.0 532 | 238,329,Defender,29,195,347,76.93,0.05,0.03,0.6,89.6,1.8,0.08,0.0,0.01 533 | 1192,1761,Forward,33,190,222,63.03,0.31,0.04,1.7,60.8,3.9,0.18,0.0,0.09 534 | 929,1366,Defender,26,186,286,80.59,0.02,0.03,0.4,88.2,2.3,0.18,0.02,0.02 535 | 3824,5392,Midfielder,24,177,76,61.07,0.04,0.08,0.8,78.7,0.2,0.12,0.0,0.03 536 | 1862,2734,Defender,30,190,199,75.03,0.07,0.02,0.4,88.4,2.0,0.11,0.0,0.01 537 | 1285,1896,Midfielder,23,182,140,68.34,0.01,0.04,0.5,89.8,0.5,0.25,0.01,0.03 538 | 1087,1601,Goalkeeper,26,190,176,90.35,0.0,0.01,0.0,57.1,0.2,0.02,0.0,0.04 539 | 3026,4338,Defender,21,180,73,54.51,0.05,0.03,0.4,75.3,0.9,0.19,0.01,0.01 540 | 2256,3264,Defender,25,188,124,84.07,0.08,0.04,0.7,80.5,2.6,0.29,0.01,0.04 541 | 1072,1581,Midfielder,31,178,314,61.68,0.04,0.09,0.9,78.8,0.5,0.15,0.0,0.01 542 | 3796,5349,Defender,37,189,271,80.02,0.05,0.01,0.5,82.9,1.7,0.19,0.03,0.01 543 | 126,188,Midfielder,23,179,156,68.94,0.01,0.07,0.4,80.8,0.6,0.14,0.01,0.01 544 | 1825,2677,Defender,32,173,433,80.26,0.03,0.13,0.4,86.5,0.9,0.28,0.01,0.03 545 | 2064,3004,Midfielder,23,175,73,46.14,0.03,0.03,0.5,82.9,0.4,0.21,0.0,0.01 546 | 2649,3811,Defender,27,187,127,80.85,0.02,0.02,0.4,86.1,2.2,0.11,0.01,0.02 547 | 1100,1625,Defender,30,185,173,79.18,0.02,0.02,0.3,80.7,1.5,0.18,0.02,0.03 548 | 912,1339,Midfielder,24,182,163,53.46,0.13,0.11,1.1,86.2,0.2,0.02,0.0,0.03 549 | 3850,5427,Midfielder,21,170,117,36.26,0.1,0.12,1.3,83.6,0.1,0.2,0.02,0.04 550 | 944,1393,Midfielder,35,182,576,79.94,0.14,0.15,1.0,87.1,0.6,0.22,0.01,0.09 551 | 1215,1796,Goalkeeper,30,194,171,89.79,0.01,0.0,0.0,72.9,0.3,0.02,0.0,0.04 552 | 2986,4277,Midfielder,22,185,150,79.27,0.06,0.13,0.8,84.7,0.8,0.09,0.02,0.0 553 | 3847,5423,Midfielder,25,188,60,66.82,0.0,0.0,0.3,80.9,1.0,0.25,0.03,0.0 554 | 1264,1863,Midfielder,30,177,115,75.53,0.01,0.0,0.4,82.4,1.0,0.21,0.01,0.01 555 | 1263,1862,Defender,26,188,98,79.35,0.02,0.01,0.3,81.7,1.5,0.22,0.05,0.0 556 | 2529,3640,Goalkeeper,34,187,142,88.78,0.0,0.01,0.0,60.5,0.5,0.09,0.0,0.07 557 | 608,879,Midfielder,31,179,189,70.87,0.06,0.16,0.8,84.1,0.9,0.2,0.02,0.07 558 | 1384,2053,Midfielder,30,188,209,63.47,0.11,0.05,1.2,79.1,1.3,0.22,0.01,0.01 559 | 784,1132,Defender,24,186,60,87.8,0.03,0.03,0.4,85.7,2.1,0.32,0.02,0.0 560 | 1274,1880,Defender,26,183,198,69.55,0.05,0.08,0.8,85.5,1.1,0.09,0.01,0.03 561 | 1925,2814,Defender,24,189,134,75.67,0.01,0.02,0.4,78.5,2.6,0.16,0.01,0.01 562 | 604,873,Midfielder,35,177,139,51.42,0.09,0.06,0.9,76.2,0.4,0.13,0.01,0.02 563 | 441,639,Midfielder,31,185,262,80.18,0.13,0.09,1.5,84.0,0.6,0.2,0.02,0.06 564 | 3383,4792,Forward,33,190,258,60.23,0.36,0.08,2.1,68.7,2.2,0.1,0.0,0.09 565 | 1991,2913,Goalkeeper,28,188,183,88.32,0.0,0.0,0.0,63.7,0.3,0.1,0.0,0.04 566 | 3020,4332,Forward,29,190,136,59.38,0.27,0.08,1.5,63.3,2.5,0.15,0.01,0.06 567 | 1648,2400,Midfielder,25,184,77,58.4,0.18,0.08,1.5,83.6,0.3,0.04,0.0,0.05 568 | 1702,2473,Midfielder,21,183,53,57.57,0.04,0.02,0.9,81.0,0.8,0.21,0.0,0.02 569 | 304,436,Forward,25,188,189,69.76,0.23,0.04,2.0,68.1,1.9,0.09,0.01,0.03 570 | 1093,1612,Goalkeeper,34,193,205,90.14,0.0,0.0,0.0,51.2,0.4,0.03,0.0,0.03 571 | 1233,1821,Midfielder,26,176,128,53.0,0.06,0.06,1.0,76.5,0.8,0.15,0.02,0.02 572 | 2661,3826,Defender,24,188,86,66.84,0.01,0.01,0.1,82.7,1.0,0.1,0.02,0.0 573 | 3011,4318,Defender,29,189,122,75.97,0.03,0.02,0.4,80.9,2.3,0.08,0.01,0.01 574 | 3408,4828,Defender,25,191,139,83.84,0.05,0.04,0.6,69.3,2.7,0.21,0.03,0.06 575 | 1014,1500,Midfielder,23,172,98,52.7,0.07,0.08,1.0,79.8,0.2,0.12,0.0,0.02 576 | 1695,2461,Defender,31,185,350,85.37,0.03,0.03,0.6,74.0,4.0,0.22,0.0,0.09 577 | 2677,3849,Defender,34,191,332,83.76,0.06,0.01,0.6,72.7,4.3,0.14,0.0,0.06 578 | 2351,3391,Midfielder,24,183,134,50.04,0.1,0.09,1.1,80.5,0.3,0.07,0.0,0.04 579 | 3025,4337,Midfielder,25,183,149,62.22,0.13,0.09,1.7,76.9,0.8,0.11,0.0,0.07 580 | 2087,3037,Defender,30,188,241,84.02,0.04,0.02,0.6,78.9,1.8,0.24,0.01,0.03 581 | 538,791,Defender,24,185,162,85.06,0.07,0.01,0.7,83.6,1.9,0.15,0.02,0.02 582 | 2943,4219,Midfielder,28,178,127,67.96,0.14,0.09,1.5,77.2,1.0,0.16,0.02,0.04 583 | 3251,4623,Midfielder,29,180,117,62.68,0.21,0.11,2.1,77.7,0.5,0.15,0.0,0.08 584 | 3856,5436,Goalkeeper,27,192,56,90.0,0.0,0.0,0.0,71.0,0.3,0.07,0.0,0.07 585 | 3016,4328,Midfielder,32,174,327,71.81,0.14,0.11,0.9,79.0,0.5,0.16,0.0,0.04 586 | 2914,4177,Midfielder,24,174,100,55.55,0.03,0.09,0.4,83.1,0.2,0.14,0.0,0.0 587 | 1760,2571,Midfielder,31,175,221,68.23,0.11,0.16,1.4,75.3,0.3,0.2,0.0,0.06 588 | 2108,3066,Midfielder,30,185,246,62.47,0.05,0.03,0.6,81.5,1.0,0.19,0.0,0.02 589 | 2546,3665,Forward,29,183,219,64.76,0.33,0.04,2.2,73.3,2.0,0.11,0.0,0.05 590 | -------------------------------------------------------------------------------- /scratch/pretest data/msl on pretest data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import pandas as pd\n", 10 | "from skpsl.estimators import MulticlassScoringList\n", 11 | "from sklearn.metrics import ConfusionMatrixDisplay, balanced_accuracy_score\n", 12 | "import matplotlib.pyplot as plt\n", 13 | "import logging\n", 14 | "import numpy as np\n", 15 | "import seaborn as sns\n", 16 | "logging.basicConfig(level=logging.INFO)\n", 17 | "\n", 18 | "sns.set(font_scale=1,rc={'text.usetex' : True})\n", 19 | "sns.set_style(\"white\")\n", 20 | "plt.rc('font', **{'family': 'serif'})\n", 21 | "plt.rcParams[\"figure.figsize\"] = (7.45, 3.5)" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 2, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "df_train = pd.read_csv(\"scratch/pretest data/data_for_otree_pretest_train.csv\", index_col=0)#.sample(200)\n", 31 | "X_train= df_train.iloc[:,2:]\n", 32 | "y_train = df_train.iloc[:,1]\n", 33 | "df_test = pd.read_csv(\"scratch/pretest data/data_for_otree_pretest_test.csv\", index_col=0)\n", 34 | "X_test= df_test.iloc[:,2:]\n", 35 | "y_test = df_test.iloc[:,1]" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 3, 41 | "metadata": {}, 42 | "outputs": [ 43 | { 44 | "name": "stderr", 45 | "output_type": "stream", 46 | "text": [ 47 | "100%|██████████| 2401/2401 [00:04<00:00, 510.26it/s]\n", 48 | "INFO:skpsl.estimators.multiclass_scoring_list:bias terms: [1, 0, 0, 1]\n", 49 | "100%|██████████| 28812/28812 [00:56<00:00, 512.79it/s]\n", 50 | "INFO:skpsl.estimators.multiclass_scoring_list:scores for stage 1: [1, 3, -3, 3]\n", 51 | "100%|██████████| 26411/26411 [00:52<00:00, 502.50it/s]\n", 52 | "INFO:skpsl.estimators.multiclass_scoring_list:scores for stage 2: [2, -3, 3, -1]\n", 53 | "100%|██████████| 24010/24010 [00:49<00:00, 485.90it/s]\n", 54 | "INFO:skpsl.estimators.multiclass_scoring_list:scores for stage 3: [1, -1, -1, 1]\n", 55 | "100%|██████████| 21609/21609 [00:39<00:00, 546.57it/s]\n", 56 | "INFO:skpsl.estimators.multiclass_scoring_list:scores for stage 4: [2, 3, -3, 1]\n", 57 | "100%|██████████| 19208/19208 [00:35<00:00, 538.21it/s]\n", 58 | "INFO:skpsl.estimators.multiclass_scoring_list:scores for stage 5: [-1, -1, 2, -1]\n", 59 | "100%|██████████| 16807/16807 [00:30<00:00, 557.79it/s]\n", 60 | "INFO:skpsl.estimators.multiclass_scoring_list:scores for stage 6: [1, 3, -3, 3]\n", 61 | "100%|██████████| 14406/14406 [00:26<00:00, 538.71it/s]\n", 62 | "INFO:skpsl.estimators.multiclass_scoring_list:scores for stage 7: [1, 0, 0, 1]\n", 63 | "100%|██████████| 12005/12005 [00:22<00:00, 533.78it/s]\n", 64 | "INFO:skpsl.estimators.multiclass_scoring_list:scores for stage 8: [1, 3, -3, 2]\n", 65 | "100%|██████████| 9604/9604 [00:18<00:00, 515.52it/s]\n", 66 | "INFO:skpsl.estimators.multiclass_scoring_list:scores for stage 9: [0, -1, 2, -1]\n", 67 | "100%|██████████| 7203/7203 [00:12<00:00, 573.06it/s]\n", 68 | "INFO:skpsl.estimators.multiclass_scoring_list:scores for stage 10: [-1, 0, 0, 0]\n", 69 | "100%|██████████| 4802/4802 [00:08<00:00, 554.62it/s]\n", 70 | "INFO:skpsl.estimators.multiclass_scoring_list:scores for stage 11: [1, 0, 0, 1]\n", 71 | "100%|██████████| 2401/2401 [00:04<00:00, 567.76it/s]\n", 72 | "INFO:skpsl.estimators.multiclass_scoring_list:scores for stage 12: [0, 0, 1, 0]\n" 73 | ] 74 | } 75 | ], 76 | "source": [ 77 | "clf = MulticlassScoringList(score_set=range(-3,4), l2=1e-6).fit(X_train,y_train)" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 4, 83 | "metadata": {}, 84 | "outputs": [ 85 | { 86 | "name": "stdout", 87 | "output_type": "stream", 88 | "text": [ 89 | "9.681972789115646 12\n", 90 | "acc: 0.3690 info: 0.0015 0 0\n", 91 | "acc: 0.5289 info: 0.0906 1 1\n", 92 | "acc: 0.6361 info: 0.3375 2 2\n", 93 | "acc: 0.6667 info: 0.3393 3 3\n", 94 | "acc: 0.7585 info: 0.5546 4 4\n", 95 | "acc: 0.8282 info: 0.7549 5 5\n", 96 | "acc: 0.8367 info: 0.7574 6 6\n", 97 | "acc: 0.8350 info: 0.6866 7 7\n", 98 | "acc: 0.8384 info: 0.7893 8 8\n", 99 | "acc: 0.8401 info: 0.7990 9 9\n", 100 | "acc: 0.8520 info: 0.8081 10 10\n", 101 | "acc: 0.8418 info: 0.7542 11 11\n", 102 | "acc: 0.8435 info: 0.7596 12 12\n" 103 | ] 104 | } 105 | ], 106 | "source": [ 107 | "print(clf.score(X_test,y_test), len(clf))\n", 108 | "for stage in clf:\n", 109 | " print(\"acc:\", f\"{stage.score(X_test,y_test):.4f}\",\"info:\", f\"{balanced_accuracy_score(y_test,clf.predict(X_test), adjusted=True):.4f}\", stage.stage, len(stage.features))" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 5, 115 | "metadata": {}, 116 | "outputs": [ 117 | { 118 | "data": { 119 | "text/html": [ 120 | "
\n", 121 | "\n", 134 | "\n", 135 | " \n", 136 | " \n", 137 | " \n", 138 | " \n", 139 | " \n", 140 | " \n", 141 | " \n", 142 | " \n", 143 | " \n", 144 | " \n", 145 | " \n", 146 | " \n", 147 | " \n", 148 | " \n", 149 | " \n", 150 | " \n", 151 | " \n", 152 | " \n", 153 | " \n", 154 | " \n", 155 | " \n", 156 | " \n", 157 | " \n", 158 | " \n", 159 | " \n", 160 | " \n", 161 | " \n", 162 | " \n", 163 | " \n", 164 | " \n", 165 | " \n", 166 | " \n", 167 | " \n", 168 | " \n", 169 | " \n", 170 | " \n", 171 | " \n", 172 | " \n", 173 | " \n", 174 | " \n", 175 | " \n", 176 | " \n", 177 | " \n", 178 | " \n", 179 | " \n", 180 | " \n", 181 | " \n", 182 | " \n", 183 | " \n", 184 | " \n", 185 | " \n", 186 | " \n", 187 | " \n", 188 | " \n", 189 | " \n", 190 | " \n", 191 | " \n", 192 | " \n", 193 | " \n", 194 | " \n", 195 | " \n", 196 | " \n", 197 | " \n", 198 | " \n", 199 | " \n", 200 | " \n", 201 | " \n", 202 | " \n", 203 | " \n", 204 | " \n", 205 | " \n", 206 | " \n", 207 | " \n", 208 | " \n", 209 | " \n", 210 | " \n", 211 | " \n", 212 | " \n", 213 | " \n", 214 | " \n", 215 | " \n", 216 | " \n", 217 | " \n", 218 | " \n", 219 | " \n", 220 | " \n", 221 | " \n", 222 | " \n", 223 | " \n", 224 | " \n", 225 | " \n", 226 | " \n", 227 | " \n", 228 | " \n", 229 | " \n", 230 | " \n", 231 | " \n", 232 | " \n", 233 | " \n", 234 | " \n", 235 | " \n", 236 | " \n", 237 | " \n", 238 | " \n", 239 | " \n", 240 | " \n", 241 | " \n", 242 | " \n", 243 | " \n", 244 | " \n", 245 | " \n", 246 | " \n", 247 | " \n", 248 | " \n", 249 | " \n", 250 | " \n", 251 | " \n", 252 | " \n", 253 | " \n", 254 | " \n", 255 | " \n", 256 | " \n", 257 | " \n", 258 | " \n", 259 | " \n", 260 | " \n", 261 | " \n", 262 | " \n", 263 | " \n", 264 | " \n", 265 | "
FeatureThresholds (>)ForwardMidfielderDefenderGoalkeeper
0NaN0110
1Average Shots per Match0.550331-3
2Average Minutes per Match78.795-3-123
3Average Pass Success Rate (%)74.550-111-1
4Average Aerial Duels Won per Match0.650312-3
5Player Height (cm)183.500-1-1-12
6Average Assists per Match0.035331-3
7Average Yellow Cards per Match0.0950110
8Average Goals per Match0.085321-3
9Average Man of the Match per Match0.025-1-102
10Matches Played249.50000-10
11Average Red Cards per Match0.0050110
12Player Age29.5000001
\n", 266 | "
" 267 | ], 268 | "text/plain": [ 269 | " Feature Thresholds (>) Forward Midfielder \\\n", 270 | "0 NaN 0 1 \n", 271 | "1 Average Shots per Match 0.550 3 3 \n", 272 | "2 Average Minutes per Match 78.795 -3 -1 \n", 273 | "3 Average Pass Success Rate (%) 74.550 -1 1 \n", 274 | "4 Average Aerial Duels Won per Match 0.650 3 1 \n", 275 | "5 Player Height (cm) 183.500 -1 -1 \n", 276 | "6 Average Assists per Match 0.035 3 3 \n", 277 | "7 Average Yellow Cards per Match 0.095 0 1 \n", 278 | "8 Average Goals per Match 0.085 3 2 \n", 279 | "9 Average Man of the Match per Match 0.025 -1 -1 \n", 280 | "10 Matches Played 249.500 0 0 \n", 281 | "11 Average Red Cards per Match 0.005 0 1 \n", 282 | "12 Player Age 29.500 0 0 \n", 283 | "\n", 284 | " Defender Goalkeeper \n", 285 | "0 1 0 \n", 286 | "1 1 -3 \n", 287 | "2 2 3 \n", 288 | "3 1 -1 \n", 289 | "4 2 -3 \n", 290 | "5 -1 2 \n", 291 | "6 1 -3 \n", 292 | "7 1 0 \n", 293 | "8 1 -3 \n", 294 | "9 0 2 \n", 295 | "10 -1 0 \n", 296 | "11 1 0 \n", 297 | "12 0 1 " 298 | ] 299 | }, 300 | "execution_count": 5, 301 | "metadata": {}, 302 | "output_type": "execute_result" 303 | } 304 | ], 305 | "source": [ 306 | "#print(clf.inspect(X_train.columns)[[\"Feature\",\"Thresholds (>)\",\"Forward\",\"Midfielder\",\"Defender\",\"Goalkeeper\"]].to_latex())\n", 307 | "clf.inspect(X_train.columns)[[\"Feature\",\"Thresholds (>)\",\"Forward\",\"Midfielder\",\"Defender\",\"Goalkeeper\"]]" 308 | ] 309 | }, 310 | { 311 | "cell_type": "code", 312 | "execution_count": 9, 313 | "metadata": {}, 314 | "outputs": [ 315 | { 316 | "data": { 317 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvcAAAFpCAYAAAAY+wcVAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAdHNJREFUeJzt3Xt8FNd9N/6PEEggaWflC7KENTKJDQSvSMDBTViB7VaOWclOGrBhSZw0YCNoUhu1jfRL0wQ9LU6aFCltoG7yoAXj52lia8HhaZNYWtkmF2OtktgxdtDKF3wBjUAyF6OdlcC6zu+P9Yw02vtqpb193n2p8e7MOXN22D3znTPfOZOhKIoCIiIiIiJKerPi3QAiIiIiIooNBvdERERERCmCwT0RERERUYpgcE9ERERElCIY3BMRERERpQgG90REREREKYLBPRERERFRimBwT0RERESUImbHuwFERESpRpIkNDU1Yf/+/RBFEVarVXu/r68PlZWVsFgscW5lcquvrwcA5OfnQxAEGI1GWCwW1NfXo7a2NqbbcjqdcDgcAACz2cx/O0poGXxCLaWbUAcEl8uFhoYGSJKE5557bsbbJ8sy9u3bh5KSEhiNRrjdbi0wmC48SBJNjy1btkAURezatUv3/p133gmr1Yqqqqop1W+32336h/r6ekiShL1794ZVR6Trx5q/zxCMy+XCzp07UVNTA7PZrL0vSRLq6+vR2dkZ8757yZIlePHFF9HR0QEAuu1GKt77m9KAQpQmOjo6lHXr1iltbW2697u6upSHH35YKS8v195ra2vTvZ4Ou3fvVh5++GGf98vLy5Wuri7t9cqVK3WvYymSfRIrixcvVtxut9LW1uaz3UgF2odEiWLz5s3Kzp07fd5vbGxUFi9ePOX6/X3/29ralJaWlrDriHT9WIv0N1xeXh6w75iOvrujoyOmdcZ7f1PqY1oOpY3q6mrs2rXLZ8RFFEVs2rQJdXV12ntGo3Ha21NWVgZZlnXvuVwuCIIAURS19/bs2aN7HUuR7JNYcLlcEEURgiBMaeRL5W8fEiUDQRAAeK/Uqf8dKbvdju7ubp/3I/1txeK3GK1AnyEQ9SpjoDabzeZp6S+j/TfyJ577m9IDg3tKC/E6IATjry1ut9vnxGK6DgQ8SBLFj8vlgslkgiAIkGUZdrsdoijC6XTCarXCZDIB8KaxNTQ0oLi4GJs2bUJbWxsA74ltW1sbJEmCzWYDAFRVVQVNK7Tb7brXVqvV7/rqNtX0PMB7EtLX16dL0wun3QDwyCOPQJIk7X4DtQ6n0+n3MwTT2toa8nc/uQ6bzab1ZZIkacvDaaPL5YLdbtfaKIoiRFH02Wf19fU4dOgQampqtBQjdd/IsgxJkiAIAkpLSwP++0ylnUQ68b50QDQTysvL/V4an2jiZV5/l2HdbrfS2NiotLS0KDt37lQ6Ojp0y5uamrTLrY2NjUpTU1PA9zs6OpTNmzf7pAKpqTCNjY1KY2OjsnPnTmXlypU+l6DVdqj/q5Zft26d8vDDDyttbW3K7t27ld27d8dsn0zcrrrtieutW7dOWbdundLR0aEtn7j9jo4O7fOo9fjbD7t371ZWrlyp7b9I9mEs2kkUS5PTctR+ZN26dYrb7VYUxfudn5h6V15eri1TFEVpaWnR0uc6Ojq076v6fZ7MX/81+Xuu/j4Crd/S0qIsXrxY167du3frPkuodqspMhP7kfLycl3fGegzBLJ48WLdbzoUtT9UdXV1KZs3b46ojf72j7/0n82bN2v9VktLi64P6+rq0l77qy8W7SRScSpMSguSJIUchQ41GrRv3z5YLBZYLBbs2rUL1dXVWkrI5BtE1dGuQO+bTCbU1NT4bH/Tpk0QBAFVVVWoqqrCrl27fNq9Y8cOiKIIi8WCqqoq2O12uFwumM1mbNu2TbvEXVlZGdN9smPHDphMJu1zWCwWbNmyRVuvpqYGsizD7XZrbWttbYXL5dI+s9VqhdFoRFVVFSwWC0wmk88oW21tLUpLS7XXkezDWLSTKNY6Ojpgt9tht9vR0tICs9mMI0eOaFexJEmC0+nU1ldHwlWCIGi/cZPJFPForSzLaGhowPbt27X3mpubIUlSwDKCIMBkMun6iO3bt2uj2OG022g0QpIkXT8iimLQ7caSy+VCe3u7z/bdbrfWzmjb6C9102Aw6F47HA7tGCGKoq5fm6l2UnpicE8UplAHskAdebgdfLhtaG1t1c0wY7FYtMvtUw0CAuFBkih6paWlsFqt2p+auqLau3cvrFYrZFmGy+WC2+2G2+3WrTOVFLmOjg4IgqBLidu7d2/EM/WodagnwtG022Aw+KwTiXB+q+ryjo4Ov/utuLhYS2+ajjYC0ProW2+9FevXr4fNZvP5d1fFs52UmhjcU1qI5IAQSLADWaCOPJIOPhxOpxOCIMDpdGp/XV1duraHGwTwIBnbdhJFy+VyYceOHWhpaYEoiiguLvZZJ5x7VQL9nqfrpvNw2h2pUH3S2rVrdYMq/qjLZ/pme4/Ho3t98OBBHDlyBBUVFbDb7dp9BZNxUgCKNQb3lBYiOSAEEupAFqgjD7eDD4csyxBFEWazWfurra3FwYMHtXXCvWGVB0mi+JNlGZs3b8b27dthtVohCIL2/Y/0alKg1DKTyeT3txHp70WWZciyrNUXq3ZPFCo9buLNuIHaqF4NNJvNftvS3d2NZcuWRd3GQCYOEKhXU9XUwyNHjqClpcVvuZluJ6U+BveUFiI5IARaHuxAFqgjj6SDD4fJZPJ7EIgmqOVBkij+JEnSAmaV+v0PFehOvPomSVLAK1qiKGLt2rW6k2JZlkP2RS6XS9e37Nu3D1arVdtuNO2efOIe7meYaM+ePWhoaPDpu9TZeybel7Nq1Srdemrbgj08b3Ib/Zl85VOdEUfdX2pbJgp0ZWM620npicE9pY1wDwj+hDqQBerII+ngw2E2m1FaWqrdZKqK9oSBB8nw20kUCXXqxI6ODjidTthsNr8n4SaTCVu3bkV9fb2Wardnzx4tjUwtq9Y3OQVv48aNWllRFOFyubBv3z7d9JKAN62wr68PNpsNDocDLS0t2lSY/tZX26a2yWazIT8/X3vSbqh2+6tX3R92u13rw/x9hlBMJhOOHDmCtrY21NfXw2azaTcrT76PYO/evWhra9NuaG5ubsaRI0fCbuPEderr67X+RBAE1NTUaPvT6XRi1apVWjn1CqrD4YDD4YDdbsd3vvOdoP8+U2kn0UQZiqIo8W4E0UxS53fPz8/XOuCJjz5XO9LW1lZs3bpVG+FWy5WVlQHwHpTq6+tRWVmpBfrqSLckSbBarVrQPfl9SZK0bdTU1KCqqko7gHZ0dGDbtm2oqqqCzWZDY2MjSktLUVVVpd0EWl9fj5KSEq1ei8XiU95isYSdfx9qn0xcr6SkBADQ1dWlmwt68udR2y6KIrZt2wZRFHX7tbKyUjtZUg9Yanubm5vR2dmJmpqagPvW3z6MRTuDnQQQpQt1bnU1wCSi5MHgnoiIiHQY3BMlL6blEBERERGlCAb3REREpFFT/Fwu15Rm9yIiL/Wejcn3jwWi3suh/m+kZkdcgoiIiFKWOs0uEU2d0+mELMvo7OxEfn5+yPXr6upgsVi036D6VPpInpHDkXsiIiIiomlgNpthsVh8nrweiN1u151cl5WVhT3ir2JwT0REREQUZ+pT6P29Hwmm5aSglStXYmhoCPPnz493U4goSZ0/fx5ZWVl46aWXplwX+yQiioVY9Etf+tKX0NPTE1XZc+fOoaCgIODyo0ePRtssAP4fSGk0GnUPdgwHg/sUNDg4iMGRYUj9ffFuSkLJGM6IdxMSSmbfYLybQAlsdPYQBq8MxaSuwcFBDI0Mo/eDizGpL1UolzLj3YSEkjEyFu8mJBxlZCTeTUgoo7OHMNUZ3Ht6etDTI6EocIzuv9w5YLrDZrfbrT3TRSUIQsRPoWdwn4IKCgog9fdhuPreeDclocx1zYt3ExLK9d9vj3cTEg8f+6E5s/D1mNVVUFCA3g8uQvyX22NWZyoY+ufCeDchoWSf5snfZCOnTse7CQnlzMLXUSBGGJX7UVQAtDZFdjK5dtMsILNoyqPzwUwO7AHvaL6/VJ1gGNwTERERUdpQAIwqkQX3CmZhuq//C4Lgk4LjbzQ/FAb3RERERJRWxpB4V2rNZrNPCo4syxFPTcvZcoiIiIgojSgYi/D/ME0nAy6XS/egqrVr1+pmx3E6nbBarRHVyZF7IiIiIqJp4HQ64XK50N7eju7ubgCAxWKBKIoAgObmZnR2dsJisQAA9u7di/r6ekiSBFmWYbVaI3qAFcDgnoiIiIjSiDfnPrKReAWIKudefeJzVVWV3+W1tbVhvRcJBvdERERElFYSMec+VhjcExEREVFaGWVwT0RERESU/BREPnKfTKcCDO6JiIiIKK1EmnOfTBjcExEREVFaiewRVsmFwT0RERERpQ0FkefcJ9M4P4N7IiIiIkoro8kUrUeIwT0RERERpRWm5RARERERpQBvWk5kj6RKpoF+BvdERERElD4UYCzSaD2JonsG90RERESUViIduU8mDO6JiIiIKG2kelrOrHg3gIiIiIiIYoMj90RERESUVsYUpuUQEREREaUE5twTEREREaUABRkYjTAzXUmikwEG90RERESUVpiWQ0RERESUIpiWQ0RERESUAhQAo0qkaTnJg8E9EREREaWVsRSeDZ7BPRERERGlDe8NtZGWSR4pE9y7XC7Y7XbY7XaYTCZUVFQAAPr6+tDe3g5ZlvHcc8/FuZXhkSQJdXV1cLvdOHLkSLybo5P/y/cwlpMJAJh1eRR991wXdP15HTKMv7qIy6UGDBdkIafDgw8+moOBP7tKV6cqs38EFzddPz2NnwYP3HIcnsFsAIAhexCPvbwiovK2v/wFqv7ns7r3DFmDsCx6G3fd9LbPskSw4avvoV/2fgfyhFEc/nHw70A4ZYItX7HGg8ovXcDLzxvQ25WNW9Z48MarOXjh6XxtnYr7LyBPGEW/nIkFNwyh6dECDMjJ171t+No59Ls/3A/GURz+UUGcW5T4xp70AHkfjsD1j2HWFwxB1x/9p/cx68/nAUWZ4+U+lLHA+51R+seg/PoKlN9eQWbDtdPS7um08Z4TGLicBQDIzRnCoV8uC1kmN2cQd3z6FG771Cl843tr/dapEvI+wP6mW2PX4Bi794snMdA/BwCQmzeMnz2xaMpl7v3iSe2/BeMwDv74Zu31Nx95Ec8fvR69Z3O0OlS9Z3Oj/hyJIlX7pUjTcoCxaWnHdEi+o18AJpMJu3btgt1uh9VqhdVq1S3fsmVLnFoWOVEUUVNTg507d8a7KTpqEC7/hfdgN69DxvwDXTj/YEnAMrMuj2Jehwd5f+jDcEEWLt1znS6wL9z7Li6XGrQ6hV9dwDVNZ5IiwH/gluMAgMMubyf/abEb/+vPf4t//vXtYZW/68a3YS7p1r23dP55lBachyF7EMa5H8S2wTGw4ave70DLT73/XivWeLDjXyXs/YYYdZlQy3OFUaxY3Y/b7nHj7KksHPpRgS6w3/DV99D802u0YD5XGMHf1kv47vaPxOpjz4gNXzsHAGj56TUAwtu36W7sSQ8AYNZnvQGU8sdBjP2gD7O+nh+40MlhjP3Wz2/r9rnI/Kerobw5BOWNYaB/DJCTaazOSw3Cn/7VEgDALaVn8bcPtuGHB8oClrlp4QUs+ehF5OYMwZA36LN8Z/Wv8fKJBVqdd//FG9i66cWEDPDVINzx84UAgOUrz+Oh2lfxaP0noi7zzUdexPEX52vLLZ87hS1f7dQC/BuXuLH6L3p86n3hV0X43s7E20eRSOV+aSyFb6hNuYQjQRD8vm+1WiHL8gy3JnqBPkc8XfXz9yD/+TXa6yulAoy/vhiy3Ol/vxlv/WQFTv+bSQviAWD2uUHk/aEP/Z/K197r/1Q+rvrlOcwaGIlp26fD1k++jMMd46M3v5OKsbG0M6yyhqxBGOf6HkRfOz8fh103o9udeP/+AGB96Byafzr+HTh+zIC7vxT8OxCqTDh1fmXVUqy9fjm2lN2snQSobrnNoxulH5BnI0+I9IJr/FkfOofmn0zaD19+P44tSnzKE/3I+Oz4yGjGJ7Oh/PJy0DIZ9+Qg89cLdH8Zf29E5j9d7V2+OAuzPpurjeInmy/85Z/w9K8Wa69f7liAe8rfDFrmrVPX4ulfLUHPOd+rHkUFHtz+qVP4ze8Wau/95ncLselzHcjN8e3D4m3jl9/SgnAAeOWl+aj4/OmoyxQuGMDqv+jBsV+NDzgd+9X1uO9LbyE3bxgA0PLfN+Duss/p/v7jXz+e9IE9kLr9kgJgFLMi+kumU/2UC+4nkyQJgHdk3+12x7k1yWv2uUFkXh7FWK7vAW9eR3QnTXPOeQ8ME+tU/zv73eAH6HgrFmQY5w7BM5Tts+zTYrefEnqWRW/DcfLG6WjatCksGYQhf9RvusuKNZ6oykRT52T9cia+9+RbyBVGtG32dPn+uySy8f2Q6bMs3P2QbpSzI0C/gow838OY8sfAQWfG7fN81s1YMifA2smlqMADQ+4QBi77fv9vKT0bdZ0AdHWq/73ko6EHd2ZS4YIB5AnDPqkxgHc0PpoyhQu8x6KJy9X/XvSxPgBA228W+JQ7+Xp+NB8hoaR6vzSqzIroL5kk59BEBBwOB6qqqiCKImRZRn19PZYtW6YF+mr6jtPpRENDA1atWoWSkhKtnCzLaGhogCAI2LNnD5qamrB//35s3boVtbW1qK+vx6FDh1BTUwOr1QqXywW32w1JkuByubRtB9uG2WyGLMvYt28fSkpKYDQatZOSRKEG4pON5mRi1uXgo6R5v+/DaF4mMvtHMefcoJZyM1zgPUDMGhjxOWmYc24IV2LQ7ulSbPR/QuP+IAtCVvDRrE+L3WiXiqejWdOq6IYhv+97+jKRG2CkPFSZcOu87bN98PTNhiF/BAtuGMKBfxk/mP6wVsSjLW/iyGsdOPSfBTh7OivpLhlHs2/TXk+A/ZKX4U2pCWDiiLxydgTK2REtrSfZqYH4ZJ6BLOTm+P+OhaKO5ufmDPqcNATaXryogfhk/fIcbZQ90jJvv2kE4M3Dn3wCULhgAMB8XV594YIBFC4YwCsvLYz8AySY1O6XMqKYLSd50nhSMrhva2uDLMuQJAmSJKGqqgoAsHnzZjz++ONayovNZoPNZtMCbKvVCpvNpt14azQaYTabtXpEUURtbS1aW1uxadMmAEBtbS3y8/O1k4Tq6mpdoF9dXa3dFBtoG2rbHnnkEZhMJgDek5JkMPZh0B7I4MIcAMDIh4G88KsLKNz7Lnp3fAQjBdm4XGrAPJdHy8OP9ipAonAPzvWbbjORkDWIblmAIcRJQLLw9GXCkB9ZGpVaxt+I0OQ63zrhHWnt/XA0vuL+C/jWvne1nPoBeTYO/agAt9zmwca/OYeXn8/D87/IT8obaifz7odkP4jOMGEWFDm8bFrlyf7g+fkpwtOfDcFPLn04es4Z8McTC3BLaQ+O/WEhgOivAsSLxzMHBiGykxu1TO/ZXBx/8VosX3leG6EPdBUAAO67/62g+f2pIBX6Je8895EF60zLibOysjJUVVXpRs0dDgeMRqMul91isaCxsVFXVl1utVq1QNtqtaKlpQUAIMsyBEFAU1OTVu/Em3f37NkDs9kMwJsK5HK5fNo3eRtOpxOyLGvbA6C1O9HNChLYA96gXg3sAW9Ofd4f+rSc+rP/cBPmvnMZwq8uIPcPl7QR/OGCrOlr9DQyZge/CXaDqRPPvJ1c6TihRNPJhyozcXlvV7YW2APA87/Ix233uLU0nAf/8Sx6u7Lx3e0fwVdWLYUhfxSPtgTPMU4WyX4AjQs5vBktlDejG8lORv5uko3EN763Fks+eh53/8UbWPNnpyD3e/tnfzn6ichg8D9qH26Zb/+tGYuX9sHyuVMou+MsPLJ3BH/yTDg3Lu6bUjuTBfulxJf8Q1tBiKKoBdonTpyAwaDviIxGI2RZ1gJ2ACgtLfWpRxAEiKKoBeF79uzBli1bUFtbC7fbrTthEEVRS68JZPI2JElKyBtoJxou8J/DnHl5NGggnvuHS7rZcdTgfc65IQx+xPvfE2fGUYP+wY/kTLnN0ynQDa/GuUOQZP/Lls4/j45z86ezWdOq57T/f2dD/qgu+I6kTDh1rr67Tzc7jjoiX1QyhH55FLnGURw/5v1t93Zl46GKJXi05Q2fcoks+H5IzhPdaVfk/6oP+pWwboZVfnEZWBCgjiQVKNg25A5NORCfODOOeiPtG+9cE2j1uOg96/+4kScMB5ySMtwyE6e+VFN8JufVV/zlafScSexjVyRSvV8aTc3xbQApOnI/kcViAQCUlJSgu1t/o6Oa1x5OYG21WtHU1AS32w1RFCGKIlwul5ZWA3hH9cvLy1FZWekzHWewmXrU+wES2UhBNkZzMjHbT+79lVL/+2/WwAiK9p7SlVGDd/WEYPKNs/NcHvT/Wb7fG3cTSbcswP1BFooF33+33wXIpzfOHcQqsRsP3HIcD9xyHH9f9jsA3ik177rx7Wltbyz0dmXD05eJwhLf74AaXEdaJtTyXGEEOxtP6ZarI/Y9XVkoumEIA27fAO3pnyRW0BFKNPs23WUsmA3kZXhvrJ287JOhb6hWXh70mec+2fWcM8AzkOU3F/7ljgV+SoTnpoUXdK9vKe3Bb3+/0O+Nu/HUezYX/fKcD3Ph9V55yf/ASjhlJo/IL195Hi/8qsgnB3/5ref93pibrFK6X1KAMWVWRH/JlJeTWj0bAgfR6lSYE5c3NzejpqYmrHorKirQ3t6uBfMWiwXV1dXalQEA6OjoAAAtvWZiSo7T6QxYt1rHxJtoOzo6Ei7gv/S565DTMX7QyP3DJbgnTI05+9yg7oFUY7mzcemeAl1ajvDri7rgvfA/3tXl2Rt/dREXNkV/EJpJ+/94C1ZNmBnnrhvfxqEJU2MWC7I2Fz7gDfofe3mF9qeu+9jLK3xSdULl7ceL/dEC3SwJq+/u0wXShSWD2rz14ZYJtnxAno1D/1mguzJQef9FPP9LIwbk2Th+zICbll3WAn7Voo9fSZpRe5V3P/Rrr1ff3Yen/+vqOLYo8WV8MU83M47y2yvIuGd85FQ5O6LNhe/j7GjQEX4lzPSeRPPk/3xclxO/5s9O4ZdHx6fGLCrw6B5INVGgvPy66t/o6ryn/A3sf3JljFocW4f+6yZdTnzZHWfR8t83aK8LFwzoHkgVTplvfucl3fKKz5/SjeSriq6/nBIPrZooVfsl7xNqI50KkzfUzjiXy4Xm5mYAgN1uhyzLsFgsutz1I0eOaCkzsiwjPz9fu9nW6XRq5dSbbCcSBAGrVq3SrgRUVFTA5XLpRv3NZjMqKipgt9u17W7duhX19fXYtGlT0G2obVu2zPskQfWG4Pr6etTW1k7DHotc3z3XIf+X7yH3D5cAAHPfuax7gFX2qcsw/uqC7qm1lz57nc8TaHt3jD9c6NwDIrJPXcGcc0OYc24Q5x4QdScDieyxl1foRt1Lrzune4DVzfPPY0Npp9+n1t5149uwLHoLAPD35nY4JRG/k4pRLMi46ybvMlPBBfy9uR0d7xUkTJ7+4R9fhw1ffQ+r7+4DACz5xGXdzDQ3LbuCyi9d1D1hNlSZUMubHi3QnTAIV43qHlD1ne0Lsemhc5AvZaJfzkSeMIoD3y2K5ceeEYd/VIANXzs3vh+WX066WX9m2qwvGDD2pAfKb71zaymvD+tvkD057J333t9TaxdkAgbfg7VydgTKb69A+fUH3gde7ZOR8bE5PlNoJqpDv1yGjfecwJo/OwUAWPLR87oHWN208CLuKX9D99TaogIP1vzZKdyx6l0s/shFbN30It54Z752A+0PD5hx08KLKCrwoKhAxg8PmBM23/5nTyzCvV88ibI7vCcji5f26W5wvXGxGxWfP617Am2oMo/u/gRuXNyHwgUDKLr+Mh7d/Qm/QXzPmRwtHz9VpHK/FOkNtckkQ1GUJLrQQOEoLy+H1N+H4ep7492UhDLXlRwH55lS/P32eDch8bA71JxZ+DoAoPMd/6O8kSgvL0fvBxch/kt4T29OF0P/XBjvJiSU7NOJNW9+Ihh5N/gDuNLNmYWvo+gjBTh69GjUdZSXl6N/+Cy+duCq0CtP8KMHLyFvzoIpbXumpMzIPRERERFRKN6pMCPLTE+moR8G90RERESUVsJ7GkZyYnBPRERERGkkI+KRez6hloiIiIgoASmIfJ57puUQERERESWosRSeLYfBPRERERGllVR+Qi2DeyIiIiJKGwoyvE+djbBMsmBwT0RERERpZTSJgvVIMbgnIiIiorQS6ch9MkndT0ZERERElGY4ck9EREREacM7FWZkaTlTnQrTZrNBFEVIkgRRFGGxWEKuLwgCZFkOa/2JGNwTERERUVqZybScuro6WCwWmM1mAMCOHTsgiiJMJpPf9W02G6qqqnSvXS5XwPUnY1oOEREREaUPxfuE2kj+MIV58e12uxbYA0BZWRnsdnvA9U+cOKF7bTab4XQ6w94eg3siIiIiShsKgDFkRPQXbVqO0+mEIAh+3w+ks7NTF/x3dHSEPWoPMC2HiIiIiNLMaBRpOT09PSgvLw+4/OjRoz7vybLs857RaITb7Q5YT1VVFerq6tDW1obt27dDlmXdyH8oDO6JiIiIKK2MTSHNJhJutxtGo1H3nnqjbCBWqxUulwt2ux3t7e14/PHHI9omg3siIiIiShve2XIifUItUFRU5Hd0PpjJgT3gHc33l6qjUnP01RH89evX4+DBg2GP3jPnnoiIiIjSSAbGlMj+EOUTbQVB8EnB8Tear5IkCW1tbbBYLBBFEQcPHkRNTQ0aGhrC3iaDeyIiIiJKK2OYFdFftMxms08KTrAcepfLhWXLlunemzgtZjgY3BMRERFR2lAAjCoZEf1N5SFWa9eu1c2O43Q6YbVatdculwsOhwOA/2kvJUlCRUVF2Ntjzj0RERERpZWZuqEWAPbu3Yv6+npIkgRZlmG1WnVTWzY3N6OzsxMWiwWCIKCmpgb19fUoKSnR1olk9J7BPRERERGllZl8Qi0A1NbWhr3MZDJFNK/9ZAzuiYiIiChtKMjAaIQ3yCpR3lAbD8y5JyIiIiJKERy5JyIiIqL0oUSRcz+VO2pnGIN7IiIiIkorM51zP5MY3BMRERFRWhlLohz6SDG4JyIiIqK0oc5zH2mZZMHgnoiIiIjSCtNyKOnMuTiCjz70TrybkVA2/c4V7yYkFLt9dbybkHBG3jkV7yakLOVSJoZ3FsS7GQnl7b/KjHcTEor49HXxbkLCye15L95NSCgZGbFKpcmI4iFWyZPGw+CeiIiIiNKGgshz7pmWQ0RERESUoCIfuU8eDO6JiIiIKK0w556IiIiIKEVw5J6IiIiIKAUw556IiIiIKFUoUcyWk0Qj/QzuiYiIiCitMC2HiIiIiChFpHJwn7q3ChMRERERpRmO3BMRERFR2lAQ+cg9b6glIiIiIkpQkc6Wk0wY3BMRERFRWknlnHsG90RERESUNpiWQ0RERESUQjhyT0RERESUCvgQKyIiIiKi1KEkUbAeKQb3RERERJRWOFsOEREREVEK4A21REREREQphGk5REREREQpgrPlEBERERGlCI7cExERERGlgFTPuZ8V7wYQEREREVFscOSeiIiIiNKKkkxD8RFicE9EREREaSQjinnukydHn8E9EREREaUPJYobapNopJ/BPRERERGlFU6FSWnpvgck9Hu8X5E8wwieekyMaZnv7v8TvrX141EvTwSu/XnIMnhP54c8GTBt7Q9Z5qQ9F0OeDGQZFPRLs2HaJiNL0A8JHG8wIk8cAQBkG8dQYrkS+8bPkHu/eBID/XMAALl5w/jZE4tClsnNG8aavziD1X9+Ft/+O/N0NzFhbPjaOfS7MwEAecZRHP5RQZxblPg2fq4D/QNZAIC83CEc+nlpyDK5OUO4fdUp3LbqFP7hO3f5LK8sfxN5uUPoH8jCgkIPnvx/yzBwOSvmbZ8uVzl6MJbj/R7NujyKS5aioOvndLphfP48Lt8sYPjabOS8JuODhbno/+TVYS1PBl/4zKvov/Lh92TeEJ589hNhlQGA6+fLAICGJ9Zoyz75sTP43OrX8NLr1+PsBQG3fqwbr52ej98e/+g0tH7q7tt2FgOy99icK4zgqcYFUy5z37azAICikkEAwH98+yM+dTzwjS70nJ4LAPC4M/FCyzXRf4gYS+Wc+4ScLcflcqG+vh5LlizBli1bAq63ZcsWLFmyBPX19ZAkCQCwY8cO1NfX+11fkiRs2bIF69ev97s9h8MBh8MBl8sVso1OpxPr16/Hjh07ItpWsrjvAe/+dBwuguNwEd56LQ8P/dObMStTdtd5rDD3Bawr1PJE4NqfBwBYZB3AIusArr55GL+vyw9Z5oaKyzBt7cci6wBM22T8fudV2vIhOQMt6wtg2iZjkXUA15QO4djfJk5nGKl7v3gSAOD4+UI4fr4Qb79pxEO1rwQtc+PiPqz5izPIzRuGQRiagVYmhg1fOwcAaPnpNWj56TV468Q87PhXKc6tSmwbP9cBAGg+uhjNRxfj5LtXo7qqPWiZmz5yEbevOoW83CEY8gb91vnb9oU49PNSNB9djCf/3zL8/XbntLR/Olzl6AEAuG8rgPu2AgyW5KDgv04FLTPryihyXpdx3U9Oo+AnpzE8P1sXuIdanujUIP0XLyzFL15Yijela1HzxWNBy/z153+PJ5/9BJ589hNaUP+DHc3a8rx5g1i59Axq738BNV88hjMXhIQO7AGgpakALU0FeMuVi4e/8+6UyjzwjS481bgATzUu0IL67/7f17TluYYR7P2fE7D/aAFamgrw5olcfOs/34r1R4uaAm9aTkR/8W50BBIyuDeZTKitrcXWrVvhdDohy7LPOpIkwWAwaOuKoneEuLKyEmVlZX7rFUURNTU1uvdkWcbOnTtRW1sLi8UCSZLQ0NAQso1msxnbtm0LuNzftpLJhioJLYfHR3teab8KlRt7Y1Im1zACg3EkYD2hlieKzkYBizYOaK+LzIN461Be0DK9zrm6UfosQcGQZ/xneLzBiJLKy9o6V5uG8RePnY9xy2fOxi+fhOPnC7XXr7xUgIq/PB20zNtv5sPx84XoPZs7za1LLNaHzqH5J+MncsePGXD3l9+PY4sS36bPn8DTzy3WXh8/sQD3fCb4IMRb716D5qOL0fOe/9/qLR8/qxulH7ichdzc5DnJvNrRA/dt87XXl282Iv9Y6D7k3X/5ON5svBWn/uXjcN/me8Uo1PJE9iXLK/jFCx/TXv/x9evxuTWvB1w/b94gFpVcRN688ZO/nx9biluXnkHRtePxiPXbm3DbV6vwhbpN+MULS6en8TGw8atn0fLk+L/ZK21GVH7xXNRlcg0juNE0gFzD+HG6+ckC3LJaRqH4AQDggW9IeP7pazDw4ZX8t125+OaXP4ZEEmlwn0wSMrhXlZSUwGQywW63+yxzuVxYtmyZz/sWiwVmc+DL+IIg6F47nU6Ulo5fxrVardi1a9cUWh14W8misPgKDMYR7Uc50fJVl6ZcZo3lPI455vusF+7yROCRMjEkz/JJpwGAHmd2wHJZhjEcfeBaDMkZWj15xeMd5FuH8lCy9go8UqZWT5HZd3QxGRQuGECeYVhLyZlo+crgB5Z0U1gyCEP+KAbkTJ9lK9Z44tCixFdY4IEhb8hvusyKZWejrndgIAvf//YzyM0Z0rbT854h6vpm0pzzHyDz8ijGcnz74ZxOdxxaFH9F18ow5Ayh/4pvv/zJj50JWO5jN5zHgmvHf3tnL3i/A4Z5yXOiBwCF4gcwGEf9H5vL/H8nwimzaNkACkvGj029Xd79myuMAgAqv3gOL7RcjULxA63MK23GqX2YGBtTMiL6SyYJHdwD3mC7paVl2up3u90wGMY7bkEQtKsA6Uo9857M456tO1OPpszyVZfwSnt+wG2HWp4o+iX/t6tkCWMYlgP/rD71yCX0S7Nx+M+ux/EGI3qdc/GpXX0AvIE+AFxyZWFYnoU8cQS/r8sPerKQyAoXDPh9v98zB7l5iX9lZiYV3eA/YPD0ZWoHS9Irus7/SY+nPwt5OdEHYP+2z4yiAg/++/EnsfX+P+KWZT3YY1sVdX0zac55/wMBozmZmHUl+PfI8NL7yPvj+zA+fw7X/sw3HSzU8kQ1MUCfyHM5SzcyP1H/lWzc/fWv4E3pWu099URADfIB4M8/+Q5uX/EOPrv6Nfz1538fw1bHzsQAfCKPOxN5QoDjeYgyA57Z2LhiJd52jV9dXbHaG8D3dmVr8cBNpQPIFUbR25WNh7/zbsCTiXhRlMj+kknC31BrtVpRV1cHSZK0oNvlcsFsNvuM6LtcLuzcuRPFxcXYu3cvAG/azb59+1BSUgKj0ajl5gPeUfu2tjZ0d3drefqdnZ1wu904cuQIAG/6T1NTE5YtW4YTJ06gsrISJpPJb1uDbStYXU6nEw0NDVi1ahVKSkrgcDhQVVUV9ApEPPS7Z0ecLjO5TK5hBL3d8wKeJIRanuiyjGMYdAcO7rMEBTdv9aDXmY3O/QYUmj/ADRXeNBz1hGGOMIarTcMAgBU1bvzPnUXY8IfoRyITjUeek1a59FPh6cuEIZ/BfSQ8/Vkw5EX//Rq4nAX7/yzDJz9+Fta/7MAf/1SE37YvTKobaicbzZmNzIHAfepgSQ4AYHi+98ZH4/PnUPS/30LPX98U1vJkJA9kQ8gN/6rolyyvov6nq7UrAGrg33PBe4X+s6tfwz9vfQ7/a/+dsW/sNOjvmw2DMbK+JViZjX99Fnv/8SMY8MzGoo97B3b65dnaCcBj/yri4POvYOOKlVNreAwlW6pNJKZl5P4HP/hBTOtbu3YtmpqatNeSJPlNeTGZTD558Js3b0ZlZSWsVissFotuVN5sNqOsrAylpaWora1FbW2tT578li1bsH37dlgsFtTW1qK6ujpgO4NtK1hdZrMZVqsVra2tWlmjMbEuXwFAXhR58BPLWDb0oO2ZwOk2oZYng6EggT0wPgvOmj3v43PP9mCobxZa7r1Ot841peOBSZagYEielbSj9/4YhOF4NyFpMLCP3FQCewDYev8f0XMuD4/8+x348kPrYcgbxI++/4sYtS4+Mi8H77uH58/VAncA8Ky8GoaXL2HWh+VCLU9GkQT2f/353+NXL31Ul1ffc0HQAnsA+PUfP4o//+S7Aa8GJJq8/CiO5wHKPPCNLjz/9DVoadLfi3HyT+Mj+wMe74lBwozeR3ozrZIBJNHJQNgj93fd5TtdmD+KoqC7uxtf//rXo27UZJs2bUJ1dTVqa2sBhJ/Lrt6MO3GkPVTKzcS67XY7BEHQvScIAlwul8/ofahthVOXusxqtYb1+aZLrzTX7/sG4wh6u/0vC1XmxqUenOwIfLNpqOWJRp2mcrKhD9Np/PHm6WdoOfQGcRQVR86hZX0BuhzzcJXJf1CSJYx9OKqfHAcNVaAbYvMMw+g9mzPDrUlsPaf9jwob8kfR25W8I8bTKVAevCFvCD3noutLCgs8yM0ZwvET3in/es8Z8Df/8Fn85/d/gTWfOoVjv18YbXNnxPB8/4MAmZdHMXxt4AGCvD++r5v9Rs3Zn3N+EIM3zA65PJFNTKOZyJAzhLMXQscSt694B2cuCD43zN6+4h3d7DjqiP6Caz14U0qcwRg1F34yg3EUPQHaGUmZ1RUX0XN6ri6wD1Te485EkTiIV8Jo90xIskybiIT9qywuLkZVVVXIEWVFUdDY2Djlhk2kpqc4nU7d61ACjfCHq6urCwDgcDi097Zt2+b3BCHUtsKpa+KNvfHU2z0PHvdsFBZfQW/3PN2yV9qviqrM8lWXcNPSfqxY1QcAKCzxztt+3wMSerrnYsAzO+jyRBvRN4ijyBLG4JEyYRD1o6uBboDtl2b7vQH3JuuAVmeeOIJ+abaWlgN4TxgmjuYni96zuej3zEHhggGfQP+Vl5Jrto3p1tuVDU9fJgpLBn0OjMePJcfNnDOt95wBnv4sFBZ40HtOv4/U4DxSRdd5/KbfPP3skqjqm2nD8+diNCcTc85/oBtpB7yz5vgz6/IIFux7G+9+N0crMz5inx1yeaLruSDAczkLRdfKupF2wDtrTjBqnr0a2OfNG4QhdxCegWw8su0orDuv1epUR+wDnUzES680Fx53JgrFD3wG4QLd4BpuGXUUXg3scw0jMOSPoFeai57T2SgsGcTbrvEw02AcxZsnEmMWNAWY8ZF4m80GURS1NHOLxRJ0/Ymp3IA39g03pg07uK+pqcHNN98c1rrbt28Pt9qwVVRUwOFwRJSHLoqi32k0w1VSUoL29vaQ/wDhbCuSuhLBYZuI5av64DjsDdTL7jqP5kOF2vLC4itYfdcF3UOqgpV5pf0q3YnBjUs9qNzYqysfanmiuXmbjF7nXBg+DM67HPNw08bxh1h5pEx0tc7THmxVZB5E534DhuQMXZD/vmuOdlPtiq+7cbolB1eb3FqdheYPdMF+Mjn0X4uwfOV5OH7u7dDL7jiLlv+5QVteuGAAZXec9ftgq3TLy7c/WoAVa/rR8lNvwLT67j48/V/JM5d4PDT99zLcsqwHzUe9AdWaT53CL58dnxqzsMCD2z592u+Drfyl7xw/sQDWv+xAbo5+Fp5FH72YNDfVvm8pQs5rMtwfBuJ5f3wffWvGB0fmnP8AeX+8pD3YaixnNt5fW6g7GTA+fx6eW67SRuhDLU90P3Esx8qPncEvXvAGRreveAc/PzY+LWPRtTLuWPGu7sFWi8ULWCxewG+Of0Sb/vKOFe/iFy98DP1XsvFE68d1JwufXf06fv3Hj/idlSfeDv14AVaUyWhp8v4brq64iOYnxgdYCsUPsLrifd1DqkKVudE0gJtMA9qMON513temz3xst4jb77mo5dyvrriIl18QdDfhxt0MDt3X1dXpZnPcsWMHRFEMeA+nJEmorq7W7v/csWMHJElCVVVVWNsL+5fpL7B/7bXX8OSTT+LMmTM4cOAAPB4PHA4HNmzYEG61Qamj3YA3VWX9+vVhpax4PN6749WdOPFm3I6OjqBBuCzL2nKr1QqbzaYr73A4YDKZtNfhbiucuhLJU4+JuO8BCWV3eedHXlzqwaP/NH7QvPHmflRs7NEF36HKqMruOo/bKrzrbPn7d3B8UuAfanmiMG3th2t/Hroc3pOZix1ZWpAOeGe9ecuep3tq7ZofXoSrUUBW/iiyDAqGPBlYUTOeg1hiuYJB9yztAVlDfZkof+zCzHygafCzJxbh3i+eRNkd3huCFy+9hEfrl2vLb1zsRsVfntIF92rAf1v5Gdy0xI0tX3XhzdeuQttvohuNTRaHf1SADV87h9V39wEAliy/jL3fSLy+IZEc+nkpNn6uA2s+dQoAsOQmfRC+6CMXcfedb+iCezXgv938LhZ/9H1svf+PeOOta7SUm0f+7Q58Yd0JyJ5s9A9kIS93CPt/+smZ/FhTcslShKscPcj7o/cZCXNPDeDclxdqy7O7LsP4/HndU2vfryjSHn4FAJkDI7qbZUMtT3RPPvsJfOEzr+L2Fe8AAJbecF73tNnF4gV8bs1rWnCfN28Q//63T8OQM4Svrv+DT10A8JPW5drDsQDAmPtBwt5M+1TjAty37SxWV1wEACz++IDuabI3lQ6g8gvndMF9sDK5hhF87yevwWAcxYP/oJ84RK3jhZZrYDCOag/DEq4awbf+KnGfBTDd7Ha7bpr1srIyn/cmqq+v18W727dvjyhWzFCU6Cb4aW1thd1uh8ViQVdXl+5G1GeeeSbsHH1/XC4X9u3bh/b2dmzcuFHLtd+xY4c2C47NZkNLSwskScK2bdtgsVggyzIaGhrQ0dGBRx55RHtv37592mUN9SFVW7duRVlZGRoaGiDLsjY7TX19PVpbW1FTU4OqqiqtTpPJBKPRCEEQYDab4XK5tG3V1NTAarUG3VZtbW3AutTZcmRZhtVqDfvMLJDy8nL0nr4A8f1bp1RPqtn0u9BPHk4n9srV8W5Cwhl551S8m5Awziz0PuSn850TU66rvLwcPT19KLo2OZ/YPV3e+ivfZ0CkM/Hp5Llhcabktv4p3k1IKN1Fr6Jw4XwcPXo06jrKy8shDfRB+fq6iMpl/OD/QczNj3jbTqcT1dXVePHFF7X37HY7bDYbnnvuOb9llixZgueeey7qwd+og/v9+/dj69atAID29nasWjU+WtLa2oq1a9dG1SCaOgb3/jG412Nw74vB/TgG99OPwb0eg3tfDO71Yhncj/19ZMH9rH/7f5gtX0FRUVHAdfy1y+FwYOfOnbrg3t97KpfLhfXr1+PgwYOQZRlut/fqfiSTrUSdMFdSUhJw2VTy3ImIiIiIptNMzXPvdrt9JqMRBCFgrKw+I0mWZe0+zS1btsBoNIZ932bUwX1XV5ff9JtnnnlGlytPRERERJRQogjui4qKIr5q4G+WSVmWQ858M/FmW7PZjMbGxukP7rdu3Yrq6mpUV1dDEAQUFxeju7sbpaWlOHDgQLTVEhERERFNHwWIOCk9ytl1BEHQUmtU/kbzVWqe/eR8e3VEPxxTmsdqz549kCQJ7e3t2gOcJubeExERERElnBmaCtNsNvuk4MiyHHBqd3XEfuLsin19fRHdXDsryrZqMjIykJ+fj5KSkoSc0pGIiIiIaCJFyYjobyrWrl2rPYgV8M6gM/EGWZfLpXvI6datW3Xrd3Z2Ytu2bWFvb0oj93V1dTh06BBEUYTb7YbH48HGjRvxz//8z1OploiIiIho+szgQ6z27t2L+vp6SJKkTXs+Mae+ubkZnZ2dWk59bW0t6uvrUV9fj/z8fFgslogeghp1cL9//36Ulpb6TMB/6NAhHDhwAA8++GC0VRMRERERTZuZmi1HpT6zKdxlwdYPJeq0HEEQsHHjRp/3N27cCIPBEHWDiIiIiIimlRLhXxKJeuQ+0F2+oZYREREREcVX6j40LeqR+4yMDPT39/u839/fj4yM1N1hRERERJTkOHIPPPjgg+jr69NeK4qC6upq3Hzzzbr1uru78cMf/jBW7SMiIiIiiq0kC9gjEXZwf+nSJWzfvj3kE7UApuUQERERUYJSMiJ/Qu0M34A7FWEH97W1tXxAFRERERFRAgs7uA8nsO/u7kZnZycA4K677oq+VURERERE00ABoESYlpNMWTxTeohVf38/nE4n3G637v22tjYG90RERESUmJIpWo9Q1MF9Z2cnqqurIYoi+vr6IIoiPB4P3G439uzZE8s2EhERERHFThLl0Ecq6uDebrfj2WefBeAN9EVR1B5e1d7ejuLi4ti0kIiIiIgoRjIAZEQ4cp9MpwJRz3NvNpu1/xZFEQ6HIyYNIiIiIiKaVik8z33Uwb0kSeju7sbhw4dhMBjwwgsv4PXXXwcAOJ3OmDWQiIiIiCim1Okww/1LIlEH91arFU1NTWhrawMAbNu2DX/1V3+FpUuXxqxxREREREQxFemofZKN3kedc28wGFBTU6O9NplMOHr0KCRJ8nlqLRERERFRwkiiYD1SU5oKczKDwcDAnoiIiIgSWwoH91Gn5QTzgx/8YDqqJSIiIiKauhTOuQ975D7ch1IpioLu7m58/etfj7pRRERERETTJdKpMJNJ2MF9cXExqqqqYDQag66nKAoaGxun3DAiIiIiomnB4B6oqakJO59++/btUTeIiIiIiIiiE3bOfSQ3yvKmWiIiIiKimRfT2XIocShjYxh1y/FuRkJ58r47492EhPKVltZ4NyHhHFxyQ7ybkLIyhkYw57WueDcjoSz82Ufj3YSEMvx3F+PdhIST8UJOvJuQWDJid2Mrc+6JiIiIiFJFks2AEwkG90RERESUPqJ54mwSjfQzuCciIiKi9JJEwXqkGNwTERERUVpJ5Zz7KT2h9rXXXkNdXR0efPBBAIDH48Hhw4dj0jAiIiIiommhRPiXRKIO7ltbW1FfX4/S0lIsXboUAGAwGLBhwwY888wzMWsgEREREVFMpXBwH3VajiRJeOyxxwAA7e3tumWKkmR7gYiIiIjSRiqn5UQd3JeUlARcJsucX52IiIiIElQKT4UZdVpOV1eX3/SbZ555Bl1dfFAJERERESWgSFNykiw1J+qR+61bt6K6uhrV1dUQBAHFxcXo7u5GaWkpDhw4EMs2EhERERHFDNNyAtizZw8kSUJ7eztkWYbJZMKqVati1TYiIiIiothjcB+YKIoQRVH33jPPPIO77rprqlUTEREREcUcR+79CDSfvSzLsNvtDO6JiIiIKDExuPelznFvMBgAeB9g1dfXh+7ubpjN5pg1kIiIiIiIwhN1cL9x40bU1NT4vO/xeOB0OqfUKCIiIiKiacORe1/+AnvA+5TajIzUnTuUiIiIiJJXBpDSwX3U89wHI0nSdFRLRERERERBRD1y/5nPfMbvCL0kSQFH9YmIiIiI4i6FR+6jDu5FUURVVRWMRqPP++pNtkRERERECUWJfCpMJYlOBqaUc3/zzTfHsi1ERERERNMviYL1SEWdc9/c3Iwf/OAHsWwLEREREdH0UyL8SyJTuqG2srLS7/vd3d1TqZaIiIiIaNpkKJH9JZOog/uysjL09fX5Xdba2hpttURERERE02uGR+5tNhscDof2v+GSJAn19fURbSvqnPumpiZIkoTXXntNdxOtoig4c+YMHnzwwWirJiIiIiKaNjM5Gl9XVweLxQKz2QwA2LFjB0RRhMlkCqusKIoRbS+q4N7j8UCSJGzfvh2CIOiWKYqC/fv3R1MtEREREdH0m8Hg3m63Y9euXdrrsrIyn/f8cTqdEQf2QATB/b333ov8/HxYLBYUFxejtrYWq1at8rsun1BLRERERAkpmlSbKE8GnE6nz0C4+n4okiRBFMWIHw4bdnCvKAoOHDgQ1rqBgn4iIiIioniLJi2np6cH5eXlAZcfPXrU5z1Zln3eMxqNcLvdQbdlt9thtVphs9kibmfYN9SqeULheOaZZyJuCBERERHRjJihG2rdbrfPA18FQfAb9KtkWY4qHUcV9sj9VVddFXalkV4+SBQOhwOCIER0IpNKNnz1PfTLmQCAPGEUh3983ZTLRFLn9558C9/8wk3a6xVrPKj80gW8/LwBvV3ZuGWNB2+8moMXns6P9KNNm/s2vIaBgSwAQG7uEJ46vDRkmdzcIdx2u4TVt0n41j/c4XcZABQV9aOwqB97/v1WbRvJ4IRNQJYwBgAYkmdhWVXgDkz1hj0PQ/IsZAljkLtm4+Pb3cgW9L3pi/X5EEpGAADZxjEstFyOfePjbMPXzqHf/eHvxTiKwz8qiHOLEs99W06j3zMHAJBnGMZTB2+YUpnln34fFRvO4nj71ejtnosVqy7hzQ4D2p71v++/2/gKvrVt+dQ/yDTaVPEn9F/29hl5OUNoavl4WGUAYEGB9/f6b/9ndUTLE92cw31Qcr3jmRkDYxjekB9euQMXoRR5vzuKYRZG1+Rpy2Y3y8gYGIOSOwsZPcMYtuYDeZmxbnpM8HcTG0VFRX5H54OZHNgD3uDdX6qOqqWlBVarNeL2qcIO7p9++umAU19OJMsy2tvbp3W2HJfLhebmZuzfvx8mkwkVFRUAxk8qqqqqIj7j2bJlC3bt2oW6ujoAkV2pCEaSJNTV1cHtduPIkSMxqXM6bPjqewCAlp9eC8AbWO/4Vwl7vxF4P4YqE0mdq+/uwy239eveyxVGsWJ1P267x42zp7Jw6EcFCRfYA0BL840AgOUrevFw9Yv4jz23Bixz403vY/HiS8jNHYLBMOSz/IGtr+KwfSl6e70HkIerX8Q/7nT6nAQkqhM2b2e1xOr9tzzrnIu2uqtRtuv9oGUWWz1aMD8oZ8D57Wvw53svaK9bN1+HtY+/h2xBwQVXFn6xvghb3jg9zZ9mZm342jkAQMtPrwEQ3m8w3dy3xftv7nhqAQBvgPFQ3Rt4dNeSqMvkGkaw/NOXsOau8+iR5uLwYzcEDFDKPnMOK1ZditnnmQ5qEP7L334MAHDLzWfw9195IWgwvm3DH9B4+M+013//lRdQX9OC2oaKsJYnujmH+wAAI5Xe/mnWy5eRtec8hqrnBy7UP4q5/9CDD75fBORlYtbJQcx7+AwGHHlancMVhvFgvn8U2T+8gMFvhx4Um2n83QQwQzfUCoLgk4LjbzRf5XQ6pxyDhp2W4/F44HA40NbWFvTP6XRO+8i9yWRCbW0tBEGA1WpFVVUVqqqqsGvXLlRVVWHLli1wuVxh16euK4oidu3aFdORe1EUUVNTE7P6pov1oXNo/jCoAIDjxwy4+0sXp1Qm3DpzhREY8kf8buMrq5Zi7fXLsaXsZu0kIVFs3PSaFtgDwCvHC1F59ztBy7z91tVoab5RC94nKywawOo147+fnrN5WLQocGCcaP7U6A3UVQvMH+BNuyFombPOubpR+mxBwZBnvGt6qeEqfKTisrbOtaYhrD34XoxbHn/Wh86h+SeTfi9fTp5/+5mwYWsXWj4MNgDgld9djcoNZ6dc5gHLp1G57M/xYOUqLZiZLNcwDIPRfz+VSL5496v4xW/Gg7aXO6/HZ+94I+D6ufMGseiGi8idN6i994vffAwrTWdRNF8OuTwZzLF/GIh/aOyWHMxp8QQpAWQdeB+jt+dpwfvYomxc+ZdCbXnmy1f0o/R5mcgYGIttw2OEvxv/ZuohVmaz2ScFR5bloLGmOh++zWZDS0sLOjo6YLPZgqbyTBR2cL927Vo8++yzOHLkSNC/Z599Nq5z3IuiiKqqKlRXV4ddxu12a/P0TyXHKZBgl14SQWHJIAz5oxiQfS/krFjjvwMMVSaSOm/7bB+e/0V+dI2Pk8LCfhgMw37TZZav6I263m/9wx261J7FS97HK8cTbyTIH480G0Nypk86DeAN4APJMoyhdUsBBuUMrZ48cfxg8KbdgIWWy/BIs7V6Fpg/iHHr42v89+J7ST/QbzDdFBZfgUEYwcCHaQITLf+0/5OgaMoEsmbteRxrDTLSmwCK5ssw5A5h4Eq2z7Jbbj4TsNyShRewoGD8e3b2nPd4mJczFNbyRJbRM4yM/jG/6TKzXg6c2jenxYOR1bnI6BnW1hu7JUdbruTOwtxv9gD9o9p2xgqjfnTQtOHvJogZfIjV2rVrdbPjOJ1OXdqNy+XSHmxlNpu1QeuqqiqsWrUKxcXFqKqqCjueDDu4j2R6y/z8/LDXnQ5WqxWSJEX0BLB0VnSD/w7a05eJXGE0qjLh1rlijQfHjwUe2b3ts31YfXcfKu6/gAf/MfhIw0wqLOr3+77HMwd5ecMx2cbqNRJy84ax598Dp/kkEo/k/8CWJYxiSA7c1Zi/cxEeaTaeuLUEL9bn46xzrpbGo9Z50ZWFQXkWDOII2uquDnqykIyi+Q2mm8LiK37f98izkWvwPzIYbpk1a8+j7DPnYLnvLLb83ds+6y//9Pt45Xfh33cWL0Xz/Z8IegayAgbiA1ey8bmHvoyTp8evjH7S5O1rz54zhFye6DJ6/PfHSt6sgCPtaplZbw0io38MStEcZO05rzsZGPzba5HRO4zc+05jzoGLyDx+JXiaT5zwdxPYTI3cA8DevXvR1tYGu90Om80Gq9Wqe4BVc3Mz7Ha7TzmbzYbW1lZ0dnZGNGtO2KeZkaTabN26Nex1p4soijhx4gQsFgsAb/ubmpqwbNkynDhxApWVlTCZTHC5XGhqatJ2nNlshslkCri+0+lEQ0MDSktLtbodDgfMZrP2WpZl7Nu3DyUlJTAajT77LlTdq1atQklJCRwOB6qqquJ2g6+nLzNgukyoMv5GIP3VmSuMorcrG7mC73beOjEPANDb5R2Fqrj/Ar617118d/tHImrTTOr3ZPnNpY+EelNtbu4QXnheTKqbaf3JNo5h0B04uM8WFCyrknG2bS469huxwHwFCysGkC0oWnCfJYzhWpN3v66suYSnyq/H/S92z0j748n7e2FwH0y/e3bEl/0nlnn7NW+A2tvt7W8s953FN3/Qge99vVRbP9cwgt7uecg1xObEfabJA9kQcgdDr/ih++9+FT94vMzvFYBwlicDxTALGZ7gwb2SOwtji7yfcejBq5GzWcLlpxZ6V8rLxPCGfGS+fAVZh90YXTEPI7flJuwNtZPxd4MZfYgVANTW1ka8TB29j1TYI/dOpxMHDhxAf7//EctEIwgCOjs7tddbtmzB9u3bYbFYUFtbq6XtmEwmbNq0SUvnUc+kAq1vNpthtVrhdDpRWlqqvW5oaNC2tXnzZlRWVsJqtcJisfik+oSqu7W1VSsb6IaLmRBNUBGqzMTlFfdfCHqDbG9XthbYA8Dzv8jHbfe4/Z4IJIq8KQb2ADAwkIWW5hu19JxDPzuC3NzEv/wdSLDAHvDOgmMQR/Dney/gvufOYNA9C79YX6Rb55rS8cAkW1AwJGem3Oi9PwzsQ8uLIp93Ypne7nlagAIAx1rnY81d57WAxHLf2YA3CiaLSAL7bRv+gF//4aPaDbmRLk8WgQL7icYWTzh5yctERv+YNno/58BFjBXOxuC3r8PlgyLgGcW8hwOnPiWatP/dRJqSE4PUnJkUdnD/s5/9DKtWrcKJEyeSYh77iXOE2u12CIKgy1USBCHgTbeh1jcajbrloihqd0I7nU7Isqy73DIxuA+nLeqyyZdtpkvPaf8jw4b8UV1wHUmZUMtvKr2Mk6/m+F1HtfruPt1rNX+/qCT+gW5vj/8bYg2GYfT05EZVZ27uEB548FVdIH/85etgMAxjxS2JfwOpQfR/sBiSMwMu80izMeSZpeXQG8QRfO5IL7KEMZxy5AQslyWMBkwDSkbBfy/JfeUmViYGEhMZhBH0dvs/0QunTNlnzumWqXnGhcUf4MalHpx0JX7qiarnvP+2GnKHcDbAsoluW/kuzp4TAk6dGWp5IlKnsZwso38sYI58oDJK3izM6h3R8vjVHHylaA4+eLQYSu4sZB5LrAFQ/m4Cy4jwL5mEfXScjhtNp5MkSVpg3NXVBQC6HPxt27YF/EzhrB9oRF2SpKA3PIRTd2lpqU+56dTblQ1PXyYKSwZ9gvlA+fDhlAm2fMUaD24qvazdLFi00BvQbvjqe+jpysbxY3nY2XgKX1m1VCuvjtj3JECw09ubB49nDgoL+31mvnnleGGAUsEVFvVjg/V1ND994/jc+Xne/dLf7/9gk0gM4ogWdE8OygPdAOuRZiPL4DuCpk6laRBHYBCH4ZHmINs0ftIzJGfimtL4n+TFSjS/wXTT2z0PHnk2Couv+AQfr/zu6qjK5BqG8a1/c+GBCoO2XB157O2ei0UmD25a6sGKD28iLBS93+P7tpxGT/e8hBuZ7DkvwDOQhaL5MnrO649DL3deH7SsesOtOiKfO28QQt6gVk+o5YlKKZrjza/vGfYJ2ifeIDu5zFjRbMzqGcHYovE0m4z+MYwtyvbWlec7NqpOtZlI+LsJIolG4iMV9sh9MlEfRqXeiVxSUgIAsFgsur9AQXik608kimLQqYqmUvd0sj9aoJuVY/XdfXh6wrR8hSWD2rz14ZYJtvz4MQMO//g67a/5v7zvH/7xdXjh6XwMyLNx6D8LdIFO5f0X8fwvjX5n4ImHQ01LdSPqq9dIaH76o9rrwsJ+bS78yfzl5b/91tU4bP+Y7mTh9tslnDx5VdQnDDPt49tkXbrMKUeObmpMjzRbmwsf8Ab973dmaTPlqC64srSHVK2s6cO7zeMH4VOOHCwwX9Fy8FOF9/cyPuq3+u4+PP1f/g++6erw/hIs//T4fNllnzmH5sPjU/AVFl/R5ucOp8yAZw4OP1aiC2Aq7juLY8/Mx4BnDl753dV46uAN2l/Lh+WeOhh4Tu94e+LpT+CTN49PPnDbynd1U2MWzZe1ufBVi264gMU3XMTJ09egaL6MovkyPnvHG5D7s8NanuiGrfnIPD5+k2jmsX7d1JgZPcPaXPiqoQeuRubz/boyoyvmYWxRNsZuyUHmW0PaTDmqWScHdQ+5ShT83fjKQOQ31CbT6H1iREkxJMsyGhoasGfPHu09q9UKm80GSZK0EXKHwwGTyeQ3GA+1fjDqza8Ty3Z0dGjbmErd0+nwj6/Dhq++p6XCLPnEZd3Dc25adgWVX7qoe8JsqDKhlqtW392H2z/n7UQe/MezePmYAcePGdD0aIHuhEK4ajShbqZ96vBS3LfhNW1e+sWL39c9wOqmRZdQeffbuqktCwv7sXqNhNvukLBo0SU88OCrePPNq/HCsQ9TyJqW6k4IcvOG8M3/746Z+UAxsKxKxgmbgFMObzB+/kSW7gFWF11ZeMOep3tq7R17zuNP+4yYmz+GLGEMQ/IsrKwZP6gstFzGoHuWdlLwQd8srD2ovyScCg7/qAAbvnZu/Pey3P/vJZ09dfAG3LfltJYSsLjUo3sQz41LPajYcFb3JM1QZQ7tL9EFNob8Ed1Ngaqyz5zDbRZvHVv+7m0cb78q4MhnPDW1fBybKv6E21a+CwD42EfO6x5gteiGi7jnjte11JrceYP4QW0LDLlD2L7xRZ+6Qi1PBsMb8jHncJ+WMjPrzUHdzDaz3hrE7GZZ99Ta0TV5yPCMjQf98ig++N74vUAffKsAc+x9gJDpfULtwBiGHky87wPA301AKTxyn6EoStJ9vEBPqO3r64PH4/H7hFo16DeZTFrOvNlshsvlQkNDAzo6OrBt2zZYrVYIghDW+jU1NTCbzbDZbLDb7di6dStqa2u12XKWLVsGwBvoNzQ06Jb7q1udLUeWZe3hXNEoLy9Hz7vncP3ppaFXTiOzTIGfxpeOvvKz1ng3IeEcXBL6kezp4szC1wEAne+cmHJd5eXl6D19AaL701OuK5VcWfnR0CulkeG/40PbJhPu74t3ExKKZPwdCm+4FkePHo26jvLycpy56Eb2bfdHVG7w+Z/i+muMU9r2TEnK4J6CY3DvH4N7PQb3vhjcj2NwP/0Y3OsxuPfF4F4vlsH93DWRBfcfHEue4D7l0nKIiIiIiIJK4aHtlLyhloiIiIgoHXHknoiIiIjSSkYKj9wzuCciIiKi9BHNE2eT6GSAwT0RERERpRWO3BMRERERpQoG90REREREKYLBPRERERFRamBaDhERERFRqmBwT0RERESUGjKU1I3uGdwTERERUfrgVJhERERERKmDOfdERERERKmCwT0RERERUWrgyD0RERERUapgcE9ERERElPwyEPnIfca0tGR6zIp3A4iIiIiIKDY4ck9ERERE6YVpOUREREREqYE31BIRERERpQIFQKRPqE2ikwEG90RERESUVjhyT0RERESUKhjcExERERGlhoyxeLdg+jC4JyIiIqL0wpF7IiIiIqLUwJx7IiIiIqJUwNlyiIiIiIhSB0fuKTlFelaa4sY6Xo93ExLK46Yb492EhPPzM+3xbkLCuPsLWTGtTxkbw+jF92NaZ7Kb9woPwRPlfHVevJuQcCqPnYx3ExLK/902HLvKUjhEYs9CRERERGmFI/dERERERKkihbMbGNwTERERUdrIQOQj9xnT0pLpMSveDSAiIiIiotjgyD0RERERpZfUzcphcE9ERERE6YU31BIRERERpQIFwBgfYkVERERElBqSKFiPFIN7IiIiIkorM52WY7PZIIoiJEmCKIqwWCwB15VlGXa7HQDgdDphtVqDrj8Zg3siIiIiSiNKFPPcR382UFdXB4vFArPZDADYsWMHRFGEyWTyu35DQwN27doFALBarbj11ltx5MiRgOtPxqkwiYiIiCitZCiR/U2F3W7XAnsAKCsr00bmJ5MkCZIkQZZlAIAgCDCbzdi3b1/Y22NwT0RERETpRYnwL0pOpxOCIPh9P5COjg643W7ttSiK6O7uDnubTMshIiIiovShABmRpuUoQE9PD8rLywOucvToUZ/31BH4iYxGoy54n0gURbz44ou695xOp27kPxQG90RERESUXsZmZjNutxtGo1H3niAIfoN+f1wuF9xuN2pqasLeJoN7IiIiIkorEY/cAygqKvI7Oh/M5MAe8I7m+0vV8Wfnzp14/PHHw14fYHBPREREROlmhqbCFATBJwXH32i+P/X19XjkkUfCniVHxRtqiYiIiCi9KEpkf1Eym80+KTiyLIfMobfb7aisrNQC+2A34E7G4J6IiIiIaJqsXbtWF5yrD6ZSuVwuOBwO3XLAO+ovSRJcLhdcLlfY22NaDhERERGllZl8Qu3evXtRX1+vzV9vtVp1qTbNzc3o7OyExWKBLMvYsmWLTx28oZaIiIiIKJAppNpEo7a2NqxlgiDgjTfemNK2GNwTERERUfpQgIxIp8Kc2XOBKWFwT0RERETpZYZH7mcSg3siIiIiSi+pG9szuCciIiKi9JGByB9ilTE9TZkWDO6JiIiIKL0wLYeIiIiIKAUoAHhDLRERERFRKlAiTstJpuiewT1Niw1fO4d+dyYAIM84isM/Kohzi+IvVffJfX/diwHZ+7lyhVE89b8Lp1wmVxjBbfdcwpq7L+Ef71+sW7ZitYzK+8/j5WMCeruysWKNjDdfzcULzVfF6BPF3s9+VIhcYRQAMCBn4t6v9YZVRuW5NBubv9Ud0XLylbK/wa+cQr/HezjPM4zgqf+zMCZltuw4iZ7ueQAAjzwHbc9dBwBY/qmLqLi3G8d/dw16z8zDik+9jzc7BW15vN37xZMY6J8DAMjNG8bPnlg05TL3fvGk9t+CcRgHf3yz9vqbj7yI549ej96zOVodqt6zuVF/jul08sBczDF4g9VhTwYWPfhByDKnDmVj2JOBOQYFl6VMLKq6gjmC/4DX+aAB5gOemLY55lI4LWdWvBtAqWfD184BAFp+eg1afnoN3joxDzv+VYpzq+IrVffJfX/tDVJbnpiPlifm4+2OHOz43ukplbmp9DJuu+cScoVRGPJHfcrnCqNYvtqD6u934eHvnUbP6eyED+wBwPKl87B86TxuLL2M//z/bgha5vvbbkSuMIp7v9aLe7/Wi8IbBvH4d4vDXk6+UvY3+JVTAADHkWI4jhTjrdcNeOhbnVMqk5s3jD0//R0OPbYQjiPFONkp4Fv1fxpfbhjB8k+9jx07X8PD33oNPd3zEiqwBwDHzxfC8fOFePvNfDxU++qUynzzkRcx0D8HP3tiEX72xCL0nMnBlq+O768bl7jxj999CXsPPo8Dh49qfxPXSSQnD8wFACzcOIiFGweRv3QUr/6v4CchJw/MxfWWISx68AMs3DiIRVVX8Eqd/zJnW7NwoT0r5u2OOUWJ7C+JJERwL0kS6urqYLPZYLPZYLfbIUkSbDYbZFmOyTacTifWr1+PHTt26F7X1dXFpH4aZ33oHJp/co32+vgxA+7+8vtxbFH8peo+sf5NL1qeuFZ7ffwFAZX3X5hSmbc6ctDyxHz0dmUHrGNzWSksJZ/EA2uWoeWJ+VP4BNPvqUeLsPb+89rr5bfJaP1p4BHj3tPZcD59NVZ/dvz7sfqz7+PIj4rQ784MuZz8S9Xf4IYH3kXLz67XXr/y+2tQed+ZKZXZUn0Sx54p1Eah335dwD/+9S26Oh64ezUqV3wGD35uNRxHEufEcuOX34Lj5wu116+8NB8Vnw8+4BCsTOGCAaz+ix4c+9X4/jr2q+tx35feQm7eMACg5b9vwN1ln9P9/ce/fhzf23lr7D5YDL1lm4eFGwa11/PNwzh9eG7QMuedc3Sj9HMEBcMe3/ljhuUMDLmTZF6ZsQj/kkjcg3u73Y66ujrU1NSgqqoKVVVVsFqtcDgcaGhoiNl2zGYztm3bpntttVpjVj95FZYMwpA/qqVcTLRiTYJfopsmqbpPCksGYTCOYkD2ze5bsdr/SXk0ZZJZ7+lsDLhnI8/oewXileeFgGUA6Mqo//3Wq7khl5OvlP0NXn8ZBmHEJxUE8KbORFum8r4zeOG5AhRef1l775XfX+OzfqIpXDCAPGHY/2dbed5PidBlChdcBgDdcvW/F32sDwDQ9psFPuVOvp4fzUeYdgPSLAzLs/ym05x3+u4D1RyDAueDBgzLGVo9ucW+Ee8ZRxautwzFrsHTKENRIvpLJnEN7l0uF+rq6rBnzx4Igv5AV1VVBVEU49QyilbRDf5/1J6+TC3nON2k6j4pKhn0+77HHfhzRVPGn9vuuYTVlZdQ8cXzeOCbiZtrrgbik+UaR/wGmgBQeIN3H/kbhe/tyg65nHyl6m+wsPiK3/c98mzkGkaiKlN4vTeYvXGpB7mGEfR2z8ND3+r0OVlYc9d7KLvzPVjWd2PLjpP+qpxxaiA+Wb88Rxtlj7RM79kcAPBbvnDBAAB9Xn3hggEULhjA22/mR9L0GXNZ8t/vzBHG/I7Eq5bvGsDl7ky0fPpqdP4gB+fb5+AT/zygW+e8cw7mr/K/nxMS03KmR0NDA9auXesT2KvWrl07wy2i6eLpy/SbP53OUnWfeD+X/8AiFmXe6sjB8RcEvNB8FVqemI+e09n41o/fjqapcWPIH4Gnz/98BoU3DOITa9x49dh4vzhxlD/Ucgpfqv4G+91zYBAiC7LUMmrwP+CZjbdfF9B7JgcH9yzCN3eP59y//ZoBr/z+arQ9dx0cR4rR0z0P39wdPK89njyeOTAIkY0mq2V6z+bi+IvX6kb+A10FAID77ten+CSLOUYlaDrNHEHBTQ9eQdFdg3jrwDycbc3SRvFVw54M5IpJlr+SouI6W47T6URNTU3A5du3b9cF/rIsY9++fVi2bBncbjcA6FJrXC4X3G43JEmCy+WKaPTf5XJh/fr1MJvNqKmpgSAIaGpqwrJly3DixAlUVlbCZDIB8N4j4G+Z0+lEQ0MDiouLUVZWptVrNpthsVjCKrtq1SqUlJTA4XCgqqoKZrM5sp2aoFLxADpVqbpPovlckZSZPDr9/C+vQvX3u5ArjPhN90lEgQJ71SNNb+Lx7xbD0zcbhvwRbbS+8MMrH6GWU3hS9TeYZ4x89HRymZOu8WPvQP8cGIQRLP/URbzy+2vQeyZHt+6xZ67Djp2vYW+e//SWeDMYIt8fE8t8+2/N2PLVThiEIXjkLG00f/JMODcu7ptSO+NpOESefOcPcjB/1TAW/rAfA9JlvPR3Bvx2gxF3tvYB8M6ks3BjEvU/CiIfjU+iwfu4HQnVG2UDjdr7W7Z582Y8/vjj2vvqDbhVVVUAgOrqatTU1MBqtcLlcqG6uhpHjhwJqz1utxu7du3SThbuvPNOHDlyBIIgwGKx4M4778Rzzz0HANiyZYvfZWoev91u15103HnnnRAEAWazOWRZm82mbcdoNIbV9kTSc9r/HfKG/FH0diXB3fPTIFX3SU+AFBCDcTRgekg0ZSZbXXlJNzuOGtAXlQzhrY7ECu7VoHuyAffskIH4xKkt1RScmz4xEPZyGpeqv8HeD6epnMwgjKD3jP9locoEWu6RZ6Pweu+oftmd7+lmx1ED+sLiK3j79fgF92rQPVmeMBxwSspwy0yc+lJN0ZmcV1/xl6fRc8Z/fYkiR/R/Qjss+8+hB9Q8/QzMN3s/d644htufcuO39xlxtjULOeIo8ksju1qbEJIs1SYScUvLUQP0ybPhyLKszZhjt9u1GXMcDgeMRqMu4LdYLGhsbNRe79mzRxvpNplMcLlcYbXF4XBAkiQtILfb7RAEQbctQRDgcrmCLlMVF+tnDlCD9nDKqsusVqt2pSCZ9HZlw9OX6TdwOX7MEIcWxV+q7pPermx43AE+1wsBbhaNosxEucIIvv2/39GVzxW8B5WeBAzSCm8YRK5xxG/u/fLbAt9A/PYJfYDw6jEB5rvf126cDbWc9FL2N3gm58Og2zdvPNANsKHK9J7JQY80zyc33yCM4GSngNy8YXyr/k+68mqwG+jEYKb0ns1FvzxHy4Wf6JWX/M+qFU6ZySPyy1eexwu/KvK5SrH81vMJeeViolxxDHOEMQxIvuGfGrxPdlnK9HsD7g0bvXPjD/fNwvn2OTh5YC5OHpiLzh94+6eTB+bibGvi9csazpYzPcxmM06cOKF7TxAEbSS+rq4OZrMZgiDgxIkTMBj0nbDRaIQsy9oJgiiK2Ldvn3ZiEA6n06lNu6nq6uoC4A361b9t27ZBFMWgywIRBAGSJIVVtrS0NKx2JzL7owVYsaZfe7367j48/V9Xx7FF8Zeq+8T+n4W6WW5WV15C80/Hp7ksLBnU5rUPt4zKXw7+gDwbh358nW6Uv+KLF3Ds6fyETcm576EevDIhP77tl1dh7f3ntNe9p7N1D6QCgH/dfqMuj97xk/m6kfpQy8lXqv4GDz/2ESz/1PiUnmV3vofmp8anbSy8/rI2r324ZR7buwi33fWebvnx312Nt18XMNA/B4cPLtSl5lTcewbHni1IiMD20H/dpMuJL7vjLFr+e/y5EoULBnQPpAqnzDe/85JuecXnT+lG8lVF119O2IdWTXRT1RWcbx//tzrbmoUbNow/xGpAmqXNhQ94g/6+zkyfHHu3azYWrB3CfPMwFj34gfZ3g9Vb16IHP8CCtYk6c05kM+V4Z8tJnpH+uB4Na2pqsH79esiy7JOCowa5auBbUlKC9vZ23TqS5H0AiSAIkGUZ5eXlePzxx7URb3UOe3/1q8xmM6qqqiBJEurr61FbW6ttS82TnyjYskBkWYYoilGVTUaHf1SADV87h9V39wEAliy/jL3fSO+Zj1J1nzz1vwtx31/3YnXlJQDA4k8MYO83xw+KN5VeRuX953VPoA1VprBkEKsrL+H2z17ComWX8cA3u3VPoLX/Z6HuhEG4agTf/eqN0/o5p+Ler/XiZz8qRNsvve0/+Wou/mb3+Lzbb5/IQetP5uueWvu175/G2x056O3KRu/pbPzNv57WpfiEWk6+UvY3+H8W4r6vnELZnd5gfPHNMh797njgeeNSDyru7dY9gTZUmbbnroNBGNZOCgzGYXzrq5/Ulh96bKHuhMFgHMb3/r9PTMOni9zPnliEe794EmV3nAUALF7ah0frx9t242I3Kj5/WvcE2lBlHt39Cdy4uA+FCwZQdP1lPLr7E36D+J4zOfDI8T/BCWXRgx/oRtX7OmbrZr5xd87G6UNzdU+tvfXf+3HSNg9z8scwx+Cd4/7mr/te/TnbmoUzLd561Tz9QFcE4i6F03IyFCW+n04dZZ+YSw948+kbGhrw4osvau9PzIMHgPr6euTn56OqqgpOpxPV1dV48cUXAYzfIPvGG2/A4XDAYrHA4XDAbrfj4MGD2rZdLhd27doFALj11lu1k4M777wTBw8e1E4uHA4HTCYTRFEMukz9PBNz/e+8807s2bMnZL2T2xOt8vJy9Lx7Dtef+tiU6qHUljE7MUe64+l/TreHXilN3P0F7wH6V795c8p1sU/yL/O6wA8zS0cZOfFN60lElU+/HO8mJJT/u60XwpwiHD16NOo6ysvL0dv9PkoQ2YyMXWhFYfHVU9r2TIn70d1qtcJsNqOhoUELeE+cOIGysjKfh0wdOXIE+/btQ0lJCWRZ1gJ7wDsCX1FRAbvdrtWzdetW1NfXY9OmTVq+fEdHh7aO3W7X8vnNZjOMRiOqq6tRVVWFI0eOoKGhASaTScv1V+sNtgzwpgs5HA7ts+zatUu7mhCorNPp1Noz8SZhIiIiIoohRft/KSnuI/epxm63o62tDXv37o1bGzhKRuHgyL0vjtyP48j99OPIvR5H7n1x5F4vZiP30vsoUT4TUbmujGdRKHLknoiIiIgo8UQ6th38UQAJJa6z5aQaNbWmvb097Nl6iIiIiGiGjSmR/SURjtzHkNlsDvuhWUREREQUDwqgRDp5ffIE+AzuiYiIiCi9pPAtpwzuiYiIiCh9KIg81SaJzgUY3BMRERFReknhkXveUEtERERElCI4ck9ERERE6SWFR+4Z3BMRERFRemFwT0RERESUIsYinQozeTC4JyIiIqI0okQxcp88I/0M7omIiIgofSiIPLhPntiewT0RERERpZlI57lPIgzuiYiIiCitKApz7omIiIiIUgNH7omIiIiIUgSnwiQiIiIiSgGKEvlUmEl0MsDgnoiIiIjSSxIF65FicE9EREREaUXhQ6yIiIiIiFJECo/cz4p3A4iIiIiIKDY4ck9ERERE6YVTYRIRERERpQBFASJ9iNUU03hsNhtEUYQkSRBFERaLJabrT8TgnoiIiIjSijKDI/d1dXWwWCwwm80AgB07dkAURZhMppisPxlz7omIiIgovShjkf1Ngd1u1wJ1ACgrK4Pdbo/Z+pNx5J6IiIiI0oaCyEfuox3ndzqdEATB7/uxWN8fBvcp6Ny5cxidPYQzC1+Pd1MogWXEuwEJ6O4vZMW7CQmj9xyQmRmbutgn+ZeReTLeTUgsGeyVJvu/2wbj3YSE4rkwig8yz025ntHZQzhzQ2fEZXp6elBeXh5wnaNHj/q8J8uyz3tGoxFut9tvHZGu7w+D+xSUnZ2NjIwMzJ8/P95NIaIkNXv2eWRlxeZkh30SEcXCYAz6paKioqjLnjsX+YmF2+2G0WjUvScIgt8gPpr1/WFwn4JeeumleDeBiEjDPomIEsVPfvKTGd3e5EAd8I7O+0u9iWZ9f3hDLRERERHRNBAEwSelxt/ofLTr+8PgnoiIiIhoGpjNZp+UGlmWdbPhTGV9fxjcExERERFNk7Vr1+pmu3E6nbBardprl8sFh8MR9vqhZCjKFB+5RUREREREAdXX16OkpASyLPs8cba+vh6dnZ04ePBgWOuHwuCeiIiIiChFMC2HiIiIiChFMLgnIiIiIkoRDO6JiIiIiFIEg3siIiIiohTB4J6IiIiIKEXMjncDaOa5XC7Y7XbY7XaYTCZUVFQAAPr6+tDe3g5ZlvHcc8/FuZXhkSQJdXV1cLvdOHLkSLybQwE4HA4IghDRQzgofbBPopnGPolSmkJpa/HixUpTU5PP+5s3b45Da6LX0dGhrFu3Lqz1du/erSxevDjoZ9y8ebOyePFiZffu3UpXV5eiKIry8MMPK7t37/a7fldXl7J582afNqjba2lpUVpaWpSOjo6QbWxra1PWrVunPPzwwxFtK1Ym7qN169YpjY2NSmNjo7Jz505l586d2v6IxObNm7V2t7W1xayt07Evurq6lJ07d2qfu6mpSenq6lIaGxsVt9sdk21M/jdWX+/cuTMm9SezdOuT1HXZLwWW7n2SWi/7JYoEg/s0tnLlSr8H0paWlph1GDOhq6sros5UPVD4+4xdXV3Kww8/7FNfS0tL0IPA5IO52+3WvW5sbAw7QGlpaQl4EPW3reng77vR1dWllJeXhxUMqDo6OrTPHc1BOJz6Y7UvmpqalM2bN/t8LxobGwN+X6I1+d+4qamJB1ElffskRWG/FEo69kmKwn6JosOce9JIkgQAMJlMcLvdcW7N9CkpKYHJZILdbvdZ5nK5sGzZMp/3LRZL0Mu3giDoXjudTpSWlmqvrVYrdu3aNYVWB97WTBFFEVVVVaiurg67jNvthsFg0MrHWqz2hcvlQl1dHfbs2eNTZ1VV1bS0nUJLlz4JYL8UjVTukwD2SxQ9BvekcTgcALwdntFoRH19PRwOh5YLq3I6nVi/fj3q6+tht9uxZcsWOJ1OOBwO3HnnnVi/fj0kSUJ9fT2WLFmC+vp6AN5HKd96661aXS6XC06nE3a7HXV1ddqBPNg2AECWZe19h8OhtTsSVqsVLS0tUe+rUCYeQABvh58KHbHVaoUkSVHt80TW0NCAtWvXBjwwr127doZbREB69UkA+6VopGqfBLBfoujxhto019bWBlmWIUkSJElCVVUVAGDz5s14/PHHtU7FZrPBZrOhqqoKZrMZVqsVNptNu8nNaDTCbDZr9YiiiNraWrS2tmLTpk0AgNraWuTn58NqtQIAqqurUVNTA6vVCpfLherqau0GtEDbUNv2yCOPwGQyAUDUwb168FYPbi6XC2az2WfkzOVyYefOnSguLsbevXsBeA/m+/btQ0lJCYxGo08Q0NbWhu7ubi2I6Ozs1N1gJ0kSmpqasGzZMpw4cQKVlZXa55ks2LaC1eV0OtHQ0IBVq1ahpKQEDodD+/ebClEUceLECVgslqDbd7lcaGpqQmdnJ2w2G8xmM0wmU8j2lpaWanU7HA6YzWbt9XTtC6fTiZqamoCfefv27boDrNqOZcuWaSPK6vca8H5n3G43JEmCy+WKaJTN5XJh/fr1MJvNqKmpgSAIAb8roT5vcXExysrKtHon7suZ/t6EK137JID9UrRSsU9S/83YLyVGv5R04p0XRPEzMYdRvWFHUbx5d5PzMLu6upSVK1dqr5uamvzmFbrdbm09Nb9TveFrct5sR0eH7vXixYt1dfnbRltbm1JeXq57L9IcR/UzT74ZraWlRVEUby6jv9zWibmI69at0+V5trS06MpMzlWc3Mby8nLdZ5/4mSLdVrC6mpqatNdNTU1h56YGyn1W2zPx+xFs+21tbT7fpXDaqy7v6OjQLZ+OfeF2uwPeyBnIunXrdNtRb3SbuF31++Tv+xkst7WtrU3XlmCfKdTnnbzd8vJyLUd7Or43U5WufZJat6KwXwoknfokRWG/FKhsPPqlZMS0HALgHflQz4BPnDihu3QLeEeoZFmGLMvaexNzN1XqZV6n0wmn04k9e/agtbUVgPeS8MRRBlEUsW/fPp9L7BNN3oYkSTHLady0aRMOHTqka3s4nE4nZFnWjWiFGv2YWLfdbocgCLr3BEGAy+WKeFvh1KUus1qtAUfhIiHLstaGSD5LOOsbjUbdclEUtRGo6doX6nsTv9vqa5vNpn0/bTYbZFmGw+HQ2qmyWCxobGzUXu/Zs0f7PakjhuFwOByQJEkbbQv2mcL5vMXFxbr61ZHneHxvIpWOfRLAfikaqdYnTXyf/VJi9UvJgmk5pFEvi5WUlKC9vV23TL3UGM6Bxmq1oqmpCWVlZRBFEaIowuVyaZewAW8HVV5ejscff1z7gdbV1WnLAm1HFEWfzi5aE1MyJr4OZaoH866uLgD6S/fbtm3zeyAOta1w6vIX8EyFJEnav1kknyXc9Sd+TyZvd7r2hdlsxokTJ3TvCYKAqqoqLf/6yJEjEAQhZKCpBpPqpfpwOZ1OiKIIu92uHUSDfabm5uaQn3cyQRAgSVJcvjfRSLc+CWC/FI1U7JMA9kv+yiZCv5QMGNynsUAHJPUseuIBrbm5OWju30QVFRVoaGhAZWUlAO8BemLuKgB0dHQAgNYhTzwzdzqd2kF9MvVANzEntaOjI+qDa0VFhZZDGa6pHszVQCXQZ4xkW5HUFQvqg1/UTj7S7U+lvdO5L2pqarB+/Xq/QZx6MFG/b6ECzWiDRLPZjKqqKu3Gz9ra2qCfKZrPq45wzvT3Jlzsk8bby34pPKnaJwHslyh6TMtJQy6XS7uhSr2sN/kmoCNHjmiXp202G/Lz87Ub29TZJJxOJ2w2m0/9giBg1apV2g+0oqICZrNZ13mYzWZUVFRo9bjdbmzduhX19fXazTOBtnHkyBE0NTVps1KoN9+pnykUdXQA8AYNdrs9rJuKPB6P1nYAun0W6mA+MX3AarVqbVaplz0j3VY4dcWKLMtoaGjAnj17tPdCbX/yPplKe6dzX5hMJuzatQubN2/2abM6gjr5M0xcb2KgGSpIVKn/xpPt2rULhw4dgsvlCvqZwvm83d3durrtdrt2w+hMfW/Cke59EsB+KRqp3CcB7Jf8laXwZCiKosS7EUQzweVyYd++fWhvb8fGjRtRW1sLANixY4c224TNZkNLSwskScK2bdtgsVi0A0hHRwceeeQR7T11VgLA27k3NDRg69atKCsrQ0NDA2RZ1u7mr6+vR2trK2pqalBVVaXVaTKZtDxJs9kMl8ulbWtiZxdoW7W1tQHrUmcXkGUZVqtVC4RC7aPm5mbs378fJpMJFRUVAIC+vj54PB6/syuE81m2bdsGq9WqjSCF89nNZrOWhznxs07nvpAkCTabTfuMJ06cQFlZGVwulzZDhPqZ1cvb6sF0Yp11dXUwmUxaPW1tbQC8+dQTv081NTUQRVFrm/q5169fr9WpjjpP/kzB9j0AbVpGdURT/SyhykbzvaHosV8K/v1K9z5JrZP9EvulSDC4JyJKQXa7HW1tbVqASEQUb+yXZgbTcoiIiIiIUgSDeyKiFKPmh7e3twec0pGIaCaxX5o5TMshIiIiIkoRHLknIiIiIkoRDO6JiIiIiFIEg3siIiIiohTB4J6IiIiIKEUwuCcKQX3S5JIlS3DnnXfCZrPBZrOhrq4OO3bsgMPhiPk26+rqcOutt/o8hXCmhdsOl8uFLVu24M477wy7bqfTifXr12PLli1RtS2abRKlCvZL7JeIAlKIKCybN29Wdu7c6fN+eXm50tjYGPPtrVu3Tmlra9Ne7969W3n44YcjqqOpqSnm7Qikra1NKS8vj6julpYWZd26ddE2LaptEqUS9kvBsV+idMSRe6IpslqtaGhomPbtlJWVobKyMqIy6uPFZ4LRaIy4jPrY9JncJlE6YL/kxX6J0tHseDeAKNmpBwJZlqd8UAjGbDZHtL7dbkd3d/c0tYaIEhn7JaL0xeCeaIpcLhdMJhMEQYDT6URDQwOKi4uxadMmbYSqtrYWAGCz2SCKIiRJgiiKsFgsWj02mw2CIPgd9XG5XGhoaIAkSXjuued0yyY/6c9qtcLpdKKtrQ2SJMFmswEAqqqqdNuKph2RkmUZdrsdoijC6XTCarXCZDL5rKfmB7vdbsiyrGtrqPYSkS/2S4GxX6KUF++8IKJkMTm31e12K42Njcq6desUt9utva/ma7a1tSkdHR3K7t27FUVRlIcfflhpaWnR1dfR0aEoijdvdWIeqtvtVhYvXqzLKe3o6PDJ42xsbNTqV7etbqOtrc1v3uhU2xGIv/bt3r1b6erq0l6Xl5fr9lVbW5uyePFi3XtNTU26/Rysvf62SZRO2C8Fx36J0hFz7oki0NHRAbvdDrvdjpaWFpjNZhw5ckR32VsQBLhcLpjNZphMJtTW1kKSJLS2tupGdiwWC+x2O2RZxv79+2G1WnV1+BtJmkiWZTQ0NGD79u3ae83NzZAkKWCZ6WhHMJIk6Wa0UEfKJlJHF1VWqxV2ux2SJAVtLxF5sV+KDPslSnVMyyGKQGlpqe4gE4goirrXTqdTuzyu6urq0g4y0eTEdnR0QBAEXdm9e/cGLTMd7QhGbY8sy5AkCW63G263O2Q5URThcrngdrsDtpeIvNgvRYb9EqU6BvdE02DywUiWZYiiqLv5TP1vh8MRVR6pLMsRrS9J0rS0IxiXy4V9+/ahrKwMFRUVKC4uDquceqAN1l4iigz7JS/2S5TqmJZDNANMJpPfUR1ZlgMuC6dOfwfSQAdX9Qa7WLcjEFmWsXnzZmzfvh1WqxWCIMDj8QBAyO2o7QnWXiKaGvZL7JcoNTG4J5oBZrMZpaWlPk+NbGlpgSiKWj6nSpZluFyuoAcLURSxdu1abdYJtVxLS4u2XD0ASZIEk8k0Le0IRB2Rm5gbq458uVwun/VUdrsdVqtVGxkL1F4imhr2S17slyjVZCiKosS7EUSJTJIkOBwONDY2wmg0wmq1aiM+kzmdTthsNnR0dGDbtm2wWCy6PNf6+nqUlJRol5knT/UmiqJWr/oAmpqaGhiNRuzbtw+tra2oqanRTclWX1+P/Px8iKIIt9uty72tr68HAJSUlPi8H007Al16Vi9zt7a2YuvWrdoUe+r2y8rKAHgP7PX19aisrITFYtHyV9WDaKAp5/y1N9A2idIB+yX2S0SBMLgnIiIiIkoRTMshIiIiIkoRDO6JiIiIiFIEg3siIiIiohTB4J6IiIiIKEUwuCciIiIiShEM7omIiIiIUgSDeyIiIiKiFMHgnoiIiIgoRTC4JyIiIiJKEQzuiYiIiIhSBIN7IiIiIqIUweCeiIiIiChF/P+xqXYbb+nCVQAAAABJRU5ErkJggg==", 318 | "text/plain": [ 319 | "
" 320 | ] 321 | }, 322 | "metadata": {}, 323 | "output_type": "display_data" 324 | } 325 | ], 326 | "source": [ 327 | "fig, (a1,a2) = plt.subplots(1,2, layout='constrained')\n", 328 | "\n", 329 | "ConfusionMatrixDisplay.from_predictions(np.repeat(y_test,10), clf.predict(np.repeat(X_test,10,axis=0)), labels=[\"Forward\", \"Midfielder\", \"Defender\", \"Goalkeeper\"], normalize=\"true\", ax=a1, colorbar=False, im_kw=dict(vmin=0, vmax=1))\n", 330 | "a1.set_title(\"Classifier Confusion\")\n", 331 | "\n", 332 | "cm =np.array([[22,6,3,0],[28,85,38,4],[5,20,63,5],[0,2,3,26]])\n", 333 | "cm = cm/cm.sum(axis=1)[:,None]\n", 334 | "ConfusionMatrixDisplay(cm, display_labels=[\"Forward\", \"Midfielder\", \"Defender\", \"Goalkeeper\"]).plot(ax=a2, im_kw=dict(vmin=0, vmax=1))\n", 335 | "a2.yaxis.set_visible(False)\n", 336 | "\n", 337 | "a2.set_title(\"Participant Confusion\")\n", 338 | "fig.savefig(\"Confustion Matrix.pdf\", bbox_inches=\"tight\")\n" 339 | ] 340 | } 341 | ], 342 | "metadata": { 343 | "kernelspec": { 344 | "display_name": "scikit-psl-VP3OoPiz-py3.12", 345 | "language": "python", 346 | "name": "python3" 347 | }, 348 | "language_info": { 349 | "codemirror_mode": { 350 | "name": "ipython", 351 | "version": 3 352 | }, 353 | "file_extension": ".py", 354 | "mimetype": "text/x-python", 355 | "name": "python", 356 | "nbconvert_exporter": "python", 357 | "pygments_lexer": "ipython3", 358 | "version": "3.12.9" 359 | } 360 | }, 361 | "nbformat": 4, 362 | "nbformat_minor": 2 363 | } 364 | --------------------------------------------------------------------------------