├── MANIFEST.in ├── .github ├── assets │ ├── img_2.png │ └── img_3.png └── workflows │ └── publish-package.yml ├── requirements.txt ├── docs ├── source │ ├── pages │ │ ├── metaperceptron.rst │ │ ├── metaperceptron.helpers.rst │ │ ├── metaperceptron.core.rst │ │ ├── tutorial │ │ │ ├── data.rst │ │ │ ├── data_transformer.rst │ │ │ ├── model_functions.rst │ │ │ ├── provided_classes.rst │ │ │ ├── mha_mlp_tuner.rst │ │ │ ├── mha_mlp_comparator.rst │ │ │ └── all_model_classes.rst │ │ ├── quick_start.rst │ │ └── support.rst │ ├── conf.py │ └── index.rst ├── requirements.txt ├── Makefile └── make.bat ├── .flake8 ├── examples ├── __init__.py ├── core │ ├── __init__.py │ ├── exam_base_mlp.py │ ├── exam_mha_mlp.py │ └── exam_gradient_mlp.py ├── tuner │ ├── __init__.py │ ├── exam_mha_mlp_regressor_tuner.py │ ├── exam_mha_mlp_multiclass_classifier_tuner.py │ └── exam_mha_mlp_binary_classifier_tuner.py ├── comparator │ ├── __init__.py │ ├── exam_mha_mlp_multiclass_classifier_comparator.py │ ├── exam_mha_mlp_binary_classifier_comparator.py │ └── exam_mha_mlp_regressor_comparator.py ├── compare_sklearn │ └── exam_feature_selection.py ├── exam_mha_mlp_multi_class_classification.py ├── exam_gradient_mlp_regression.py ├── exam_mha_mlp_regression.py ├── exam_mha_mlp_binary_classification.py ├── exam_gradient_mlp_multi_class_classification.py ├── exam_gradient_mlp_binary_classification.py ├── exam_gradient_mlp_regression_sklearn.py └── exam_gradient_mlp_classification_sklearn.py ├── tests ├── __init__.py ├── test_MhaMlpRegressor.py ├── test_MhaMlpComparator.py ├── test_MlpRegressor.py ├── test_MhaMlpClassifier.py ├── test_MlpClassifier.py └── test_MhaMlpTuner.py ├── metaperceptron ├── helpers │ ├── __init__.py │ ├── metric_util.py │ ├── validator.py │ ├── scaler_util.py │ └── preprocessor.py ├── core │ ├── __init__.py │ ├── tuner.py │ ├── comparator.py │ ├── metaheuristic_mlp.py │ └── gradient_mlp.py └── __init__.py ├── CITATION.cff ├── .readthedocs.yaml ├── .gitignore ├── ChangeLog.md ├── CODE_OF_CONDUCT.md ├── setup.py └── README.md /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md LICENSE CODE_OF_CONDUCT.md ChangeLog.md examples.md CITATION.cff 2 | -------------------------------------------------------------------------------- /.github/assets/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thieu1995/MetaPerceptron/HEAD/.github/assets/img_2.png -------------------------------------------------------------------------------- /.github/assets/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thieu1995/MetaPerceptron/HEAD/.github/assets/img_3.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy>=1.17.1 2 | scipy>=1.7.1 3 | scikit-learn>=1.2.1 4 | pandas>=1.3.5 5 | mealpy>=3.0.2 6 | permetrics>=2.0.0 7 | torch>=2.0.0 8 | -------------------------------------------------------------------------------- /docs/source/pages/metaperceptron.rst: -------------------------------------------------------------------------------- 1 | MetaPerceptron Package 2 | ====================== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | metaperceptron.core 8 | metaperceptron.helpers 9 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = __pycache__, built, build, mytest, htmlcov, drafts 3 | ignore = E203, E266, W291, W503 4 | max-line-length = 180 5 | max-complexity = 18 6 | select = B,C,E,F,W,T4,B9 7 | -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 22:28, 14/08/2023 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # Defining the exact version will make sure things don't break 2 | sphinx>=4.4.0 3 | sphinx_rtd_theme>=1.0.1 4 | readthedocs-sphinx-search>=0.1.1 5 | numpy>=1.17.1 6 | scipy>=1.7.1 7 | scikit-learn>=1.2.1 8 | pandas>=1.3.5 9 | mealpy>=3.0.2 10 | permetrics>=2.0.0 11 | torch>=2.0.0 12 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 22:47, 02/11/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | -------------------------------------------------------------------------------- /examples/core/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 14:17, 26/10/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | -------------------------------------------------------------------------------- /metaperceptron/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 23:32, 10/08/2023 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | -------------------------------------------------------------------------------- /examples/tuner/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 19:24, 16/08/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | -------------------------------------------------------------------------------- /examples/comparator/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 23:30, 17/08/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | -------------------------------------------------------------------------------- /metaperceptron/core/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 11:22, 17/08/2023 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.1.0 2 | message: "If you use this software, please cite it as below." 3 | authors: 4 | - family-names: "Van Thieu" 5 | given-names: "Nguyen" 6 | orcid: "https://orcid.org/0000-0001-9994-8747" 7 | title: "MetaPerceptron: A Standardized Framework for Metaheuristic-Trained Multi-Layer Perceptron" 8 | version: v2.2.0 9 | doi: 10.5281/zenodo.10251021 10 | date-released: 2023-12-02 11 | url: "https://github.com/thieu1995/MetaPerceptron" 12 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # Build all formats 8 | formats: all 9 | 10 | build: 11 | os: "ubuntu-20.04" 12 | tools: 13 | python: "3.8" 14 | 15 | # Build documentation in the docs/ directory with Sphinx 16 | sphinx: 17 | configuration: docs/source/conf.py 18 | 19 | python: 20 | install: 21 | - requirements: docs/requirements.txt 22 | 23 | submodules: 24 | include: all 25 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /metaperceptron/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 15:23, 10/08/2023 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | __version__ = "2.2.0" 8 | 9 | from metaperceptron.helpers.preprocessor import Data, DataTransformer 10 | from metaperceptron.core.gradient_mlp import MlpClassifier, MlpRegressor 11 | from metaperceptron.core.metaheuristic_mlp import MhaMlpClassifier, MhaMlpRegressor 12 | from metaperceptron.core.comparator import MhaMlpComparator 13 | from metaperceptron.core.tuner import MhaMlpTuner 14 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.https://www.sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /examples/core/exam_base_mlp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 14:24, 26/10/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from metaperceptron.core.base_mlp import CustomMLP 8 | 9 | 10 | def check_CustomMLP_class(): 11 | # Example usage 12 | input_size = 10 13 | output_size = 2 14 | hidden_layers = [64, 32, 16] # Three hidden layers with specified nodes 15 | activations = ["ReLU", "Tanh", "ReLU"] # Activation functions for each layer 16 | dropouts = [0.2, 0.3, 0.0] # Dropout rates for each hidden layer 17 | 18 | model = CustomMLP(input_size, output_size, hidden_layers, activations, dropouts) 19 | print(model) 20 | 21 | 22 | check_CustomMLP_class() 23 | -------------------------------------------------------------------------------- /docs/source/pages/metaperceptron.helpers.rst: -------------------------------------------------------------------------------- 1 | metaperceptron.helpers package 2 | ============================== 3 | 4 | 5 | metaperceptron.helpers.metric\_util module 6 | ------------------------------------------ 7 | 8 | .. automodule:: metaperceptron.helpers.metric_util 9 | :members: 10 | :undoc-members: 11 | :show-inheritance: 12 | 13 | metaperceptron.helpers.preprocessor module 14 | ------------------------------------------ 15 | 16 | .. automodule:: metaperceptron.helpers.preprocessor 17 | :members: 18 | :undoc-members: 19 | :show-inheritance: 20 | 21 | metaperceptron.helpers.scaler\_util module 22 | ------------------------------------------ 23 | 24 | .. automodule:: metaperceptron.helpers.scaler_util 25 | :members: 26 | :undoc-members: 27 | :show-inheritance: 28 | 29 | metaperceptron.helpers.validator module 30 | --------------------------------------- 31 | 32 | .. automodule:: metaperceptron.helpers.validator 33 | :members: 34 | :undoc-members: 35 | :show-inheritance: 36 | -------------------------------------------------------------------------------- /docs/source/pages/metaperceptron.core.rst: -------------------------------------------------------------------------------- 1 | metaperceptron.core package 2 | =========================== 3 | 4 | 5 | metaperceptron.core.base\_mlp module 6 | ------------------------------------ 7 | 8 | .. automodule:: metaperceptron.core.base_mlp 9 | :members: 10 | :undoc-members: 11 | :show-inheritance: 12 | 13 | metaperceptron.core.gradient\_mlp module 14 | ---------------------------------------- 15 | 16 | .. automodule:: metaperceptron.core.gradient_mlp 17 | :members: 18 | :undoc-members: 19 | :show-inheritance: 20 | 21 | metaperceptron.core.metaheuristic\_mlp module 22 | --------------------------------------------- 23 | 24 | .. automodule:: metaperceptron.core.metaheuristic_mlp 25 | :members: 26 | :undoc-members: 27 | :show-inheritance: 28 | 29 | metaperceptron.core.comparator module 30 | ------------------------------------- 31 | 32 | .. automodule:: metaperceptron.core.comparator 33 | :members: 34 | :undoc-members: 35 | :show-inheritance: 36 | 37 | metaperceptron.core.tuner module 38 | -------------------------------- 39 | 40 | .. automodule:: metaperceptron.core.tuner 41 | :members: 42 | :undoc-members: 43 | :show-inheritance: 44 | -------------------------------------------------------------------------------- /docs/source/pages/tutorial/data.rst: -------------------------------------------------------------------------------- 1 | Data class 2 | ========== 3 | 4 | + You can load your dataset into `Data` class 5 | + You can split dataset to train and test set 6 | + You can scale dataset without using `DataTransformer` class 7 | + You can scale labels using `LabelEncoder` 8 | 9 | For example. 10 | 11 | .. code-block:: python 12 | 13 | from metaperceptron import Data 14 | import pandas as pd 15 | 16 | dataset = pd.read_csv('Position_Salaries.csv') 17 | X = dataset.iloc[:, 1:5].values 18 | y = dataset.iloc[:, 5].values 19 | 20 | ## Create data object 21 | data = Data(X, y, name="position_salaries") 22 | 23 | ## Split dataset into train and test set 24 | data.split_train_test(test_size=0.2, shuffle=True, random_state=100, inplace=True) 25 | 26 | ## Feature Scaling 27 | data.X_train, scaler_X = data.scale(data.X_train, scaling_methods=("standard", "sqrt", "minmax")) 28 | data.X_test = scaler_X.transform(data.X_test) 29 | 30 | data.y_train, scaler_y = data.encode_label(data.y_train) # This is for classification problem only 31 | data.y_test = scaler_y.transform(data.y_test) 32 | 33 | .. toctree:: 34 | :maxdepth: 4 35 | 36 | .. toctree:: 37 | :maxdepth: 4 38 | 39 | .. toctree:: 40 | :maxdepth: 4 41 | -------------------------------------------------------------------------------- /docs/source/pages/quick_start.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Installation 3 | ============ 4 | 5 | Library name: `MetaPerceptron`, but distributed as: `metaperceptron`. Therefore, you can 6 | 7 | 8 | * Install the `current PyPI release `_:: 9 | 10 | $ pip install metaperceptron==2.2.0 11 | 12 | 13 | * Install directly from source code:: 14 | 15 | $ git clone https://github.com/thieu1995/MetaPerceptron.git 16 | $ cd MetaPerceptron 17 | $ python setup.py install 18 | 19 | * In case, you want to install the development version from Github:: 20 | 21 | $ pip install git+https://github.com/thieu1995/MetaPerceptron 22 | 23 | 24 | After installation, you can import MetaPerceptron as any other Python module:: 25 | 26 | $ python 27 | >>> import metaperceptron 28 | >>> metaperceptron.__version__ 29 | 30 | ======== 31 | Tutorial 32 | ======== 33 | 34 | .. include:: tutorial/provided_classes.rst 35 | .. include:: tutorial/data_transformer.rst 36 | .. include:: tutorial/data.rst 37 | .. include:: tutorial/all_model_classes.rst 38 | .. include:: tutorial/model_functions.rst 39 | .. include:: tutorial/mha_mlp_tuner.rst 40 | .. include:: tutorial/mha_mlp_comparator.rst 41 | 42 | 43 | .. toctree:: 44 | :maxdepth: 4 45 | 46 | .. toctree:: 47 | :maxdepth: 4 48 | 49 | .. toctree:: 50 | :maxdepth: 4 51 | -------------------------------------------------------------------------------- /examples/compare_sklearn/exam_feature_selection.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 11:52, 20/12/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.feature_selection import SequentialFeatureSelector 8 | from sklearn.neighbors import KNeighborsClassifier 9 | from sklearn.datasets import load_iris 10 | from metaperceptron import MhaMlpClassifier 11 | 12 | X, y = load_iris(return_X_y=True) 13 | 14 | ## Create model 15 | knn = KNeighborsClassifier(n_neighbors=3) 16 | ## Run KNN-feature selector 17 | sfs1 = SequentialFeatureSelector(knn, n_features_to_select=3) 18 | sfs1.fit(X, y) 19 | print(sfs1.get_support()) 20 | 21 | 22 | ## Create model 23 | woa_mlp = MhaMlpClassifier(hidden_layers=(10,), act_names="ELU", dropout_rates=None, act_output=None, 24 | optim="OriginalWOA", optim_params={"epoch": 50, "pop_size": 20}, 25 | obj_name="F1S", seed=42, verbose=False, 26 | lb=None, ub=None, mode='single', n_workers=None, termination=None) 27 | ## Run WOA-MLP-feature selector 28 | sfs2 = SequentialFeatureSelector(woa_mlp, n_features_to_select=3) 29 | sfs2.fit(X, y) 30 | print(sfs2.get_support()) 31 | -------------------------------------------------------------------------------- /docs/source/pages/tutorial/data_transformer.rst: -------------------------------------------------------------------------------- 1 | DataTransformer class 2 | ===================== 3 | 4 | We provide many scaler classes that you can select and make a combination of transforming your data via DataTransformer class. For example: 5 | 6 | * You want to scale data by `Loge` and then `Sqrt` and then `MinMax`. 7 | 8 | .. code-block:: python 9 | 10 | from metaperceptron import DataTransformer 11 | import pandas as pd 12 | from sklearn.model_selection import train_test_split 13 | 14 | dataset = pd.read_csv('Position_Salaries.csv') 15 | X = dataset.iloc[:, 1:5].values 16 | y = dataset.iloc[:, 5].values 17 | X_train, y_train, X_test, y_test = train_test_split(X, y, test_size=0.2) 18 | 19 | dt = DataTransformer(scaling_methods=("loge", "sqrt", "minmax")) 20 | X_train_scaled = dt.fit_transform(X_train) 21 | X_test_scaled = dt.transform(X_test) 22 | 23 | 24 | * I want to scale data by `YeoJohnson` and then `Standard`. 25 | 26 | .. code-block:: python 27 | 28 | from metaperceptron import DataTransformer 29 | import pandas as pd 30 | from sklearn.model_selection import train_test_split 31 | 32 | dataset = pd.read_csv('Position_Salaries.csv') 33 | X = dataset.iloc[:, 1:5].values 34 | y = dataset.iloc[:, 5].values 35 | X_train, y_train, X_test, y_test = train_test_split(X, y, test_size=0.2) 36 | 37 | dt = DataTransformer(scaling_methods=("yeo-johnson", "standard")) 38 | X_train_scaled = dt.fit_transform(X_train) 39 | X_test_scaled = dt.transform(X_test) 40 | 41 | .. toctree:: 42 | :maxdepth: 4 43 | 44 | .. toctree:: 45 | :maxdepth: 4 46 | 47 | .. toctree:: 48 | :maxdepth: 4 49 | -------------------------------------------------------------------------------- /docs/source/pages/tutorial/model_functions.rst: -------------------------------------------------------------------------------- 1 | Function in model object 2 | ======================== 3 | 4 | After you define model, here are several functions you can call in model object. 5 | 6 | .. code-block:: python 7 | 8 | from metaperceptron import MhaMlpRegressor, Data 9 | 10 | data = Data(X, y) # Assumption that you have provide this object like above 11 | 12 | model = MhaMlpRegressor(...) 13 | 14 | ## Train the model 15 | model.fit(data.X_train, data.y_train) 16 | 17 | ## Predicting a new result 18 | y_pred = model.predict(data.X_test) 19 | 20 | ## Calculate metrics using score or scores functions. 21 | print(model.score(data.X_test, data.y_test)) 22 | 23 | ## Calculate metrics using evaluate function 24 | print(model.evaluate(data.y_test, y_pred, list_metrics=("MAPE", "NNSE", "KGE", "MASE", "R2", "R", "R2S"))) 25 | 26 | ## Save performance metrics to csv file 27 | model.save_evaluation_metrics(data.y_test, y_pred, list_metrics=("RMSE", "MAE"), save_path="history", filename="metrics.csv") 28 | 29 | ## Save training loss to csv file 30 | model.save_training_loss(save_path="history", filename="loss.csv") 31 | 32 | ## Save predicted label 33 | model.save_y_predicted(X=data.X_test, y_true=data.y_test, save_path="history", filename="y_predicted.csv") 34 | 35 | ## Save model 36 | model.save_model(save_path="history", filename="traditional_mlp.pkl") 37 | 38 | ## Load model 39 | trained_model = MlpRegressor.load_model(load_path="history", filename="traditional_mlp.pkl") 40 | 41 | .. toctree:: 42 | :maxdepth: 4 43 | 44 | .. toctree:: 45 | :maxdepth: 4 46 | 47 | .. toctree:: 48 | :maxdepth: 4 49 | -------------------------------------------------------------------------------- /examples/exam_mha_mlp_multi_class_classification.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 21:35, 02/11/2023 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.datasets import load_iris 8 | from metaperceptron import Data, MhaMlpClassifier 9 | 10 | 11 | ## Load data object 12 | X, y = load_iris(return_X_y=True) 13 | data = Data(X, y) 14 | 15 | ## Split train and test 16 | data.split_train_test(test_size=0.2, random_state=2, inplace=True, shuffle=True) 17 | print(data.X_train.shape, data.X_test.shape) 18 | 19 | ## Scaling dataset 20 | data.X_train, scaler_X = data.scale(data.X_train, scaling_methods=("standard", "minmax")) 21 | data.X_test = scaler_X.transform(data.X_test) 22 | 23 | data.y_train, scaler_y = data.encode_label(data.y_train) 24 | data.y_test = scaler_y.transform(data.y_test) 25 | 26 | ## Create model 27 | model = MhaMlpClassifier(hidden_layers=(50,), act_names="Tanh", 28 | dropout_rates=None, act_output=None, 29 | optim="BaseGA", optim_params={"name": "WOA", "epoch": 100, "pop_size": 20}, 30 | obj_name="F1S", seed=42, verbose=True, 31 | lb=-2., ub=2., mode='single', n_workers=None, termination=None) 32 | ## Train the model 33 | model.fit(X=data.X_train, y=data.y_train) 34 | 35 | ## Test the model 36 | y_pred = model.predict(data.X_test) 37 | print(y_pred) 38 | 39 | ## Calculate some metrics 40 | print(model.evaluate(y_true=data.y_test, y_pred=y_pred, list_metrics=["F2S", "CKS", "FBS"])) 41 | -------------------------------------------------------------------------------- /examples/exam_gradient_mlp_regression.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 10:12, 17/08/2023 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.datasets import load_diabetes 8 | from metaperceptron import Data, MlpRegressor 9 | 10 | 11 | ## Load data object 12 | X, y = load_diabetes(return_X_y=True) 13 | data = Data(X, y) 14 | 15 | ## Split train and test 16 | data.split_train_test(test_size=0.2, random_state=2, inplace=True) 17 | print(data.X_train.shape, data.X_test.shape) 18 | 19 | ## Scaling dataset 20 | data.X_train, scaler_X = data.scale(data.X_train, scaling_methods=("standard", "minmax")) 21 | data.X_test = scaler_X.transform(data.X_test) 22 | 23 | data.y_train, scaler_y = data.scale(data.y_train, scaling_methods=("standard", "minmax")) 24 | data.y_test = scaler_y.transform(data.y_test) 25 | 26 | print(type(data.X_train), type(data.y_train)) 27 | 28 | ## Create model 29 | model = MlpRegressor(hidden_layers=(30,), act_names="Tanh", dropout_rates=None, act_output=None, 30 | epochs=10, batch_size=16, optim="Adam", optim_params=None, 31 | early_stopping=True, n_patience=10, epsilon=0.001, valid_rate=0.1, 32 | seed=42, verbose=True, device="cpu") 33 | ## Train the model 34 | model.fit(data.X_train, data.y_train) 35 | 36 | ## Test the model 37 | y_pred = model.predict(data.X_test) 38 | print(y_pred) 39 | 40 | ## Calculate some metrics 41 | print(model.evaluate(y_true=data.y_test, y_pred=y_pred, list_metrics=["R2", "NSE", "MAPE", "NNSE"])) 42 | -------------------------------------------------------------------------------- /examples/exam_mha_mlp_regression.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 21:35, 02/11/2023 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | import numpy as np 8 | from sklearn.datasets import load_diabetes 9 | from metaperceptron import Data, MhaMlpRegressor 10 | 11 | 12 | ## Load data object 13 | X, y = load_diabetes(return_X_y=True) 14 | data = Data(X, y) 15 | 16 | ## Split train and test 17 | data.split_train_test(test_size=0.2, random_state=2) 18 | print(data.X_train.shape, data.X_test.shape) 19 | 20 | ## Scaling dataset 21 | data.X_train, scaler_X = data.scale(data.X_train, scaling_methods=("standard")) 22 | data.X_test = scaler_X.transform(data.X_test) 23 | 24 | data.y_train, scaler_y = data.scale(data.y_train, scaling_methods=("minmax", )) 25 | data.y_test = scaler_y.transform(np.reshape(data.y_test, (-1, 1))) 26 | 27 | ## Create model 28 | model = MhaMlpRegressor(hidden_layers=(30, 15,), act_names="ELU", dropout_rates=0.2, act_output=None, 29 | optim="BaseGA", optim_params={"name": "WOA", "epoch": 10, "pop_size": 30}, 30 | obj_name="MSE", seed=42, verbose=True, 31 | lb=None, ub=None, mode='single', n_workers=None, termination=None) 32 | ## Train the model 33 | model.fit(data.X_train, data.y_train) 34 | 35 | ## Test the model 36 | y_pred = model.predict(data.X_test) 37 | print(y_pred) 38 | 39 | ## Calculate some metrics 40 | print(model.evaluate(y_true=data.y_test, y_pred=y_pred, list_metrics=["R2", "NSE", "MAPE", "NNSE"])) 41 | 42 | print(model.optimizer.name) 43 | -------------------------------------------------------------------------------- /examples/exam_mha_mlp_binary_classification.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 21:35, 02/11/2023 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.datasets import load_breast_cancer 8 | from metaperceptron import Data, MhaMlpClassifier 9 | 10 | 11 | ## Load data object 12 | X, y = load_breast_cancer(return_X_y=True) 13 | data = Data(X, y) 14 | 15 | ## Split train and test 16 | data.split_train_test(test_size=0.2, random_state=2, inplace=True, shuffle=True) 17 | print(data.X_train.shape, data.X_test.shape) 18 | 19 | ## Scaling dataset 20 | data.X_train, scaler_X = data.scale(data.X_train, scaling_methods=("standard", "minmax")) 21 | data.X_test = scaler_X.transform(data.X_test) 22 | 23 | data.y_train, scaler_y = data.encode_label(data.y_train) 24 | data.y_test = scaler_y.transform(data.y_test) 25 | 26 | ## Create model 27 | print(MhaMlpClassifier.SUPPORTED_CLS_OBJECTIVES) 28 | model = MhaMlpClassifier(hidden_layers=(100,), act_names="ReLU", dropout_rates=None, act_output=None, 29 | optim="BaseGA", optim_params={"name": "WOA", "epoch": 100, "pop_size": 30}, 30 | obj_name="F1S", seed=42, verbose=True, 31 | lb=None, ub=None, mode='single', n_workers=None, termination=None) 32 | ## Train the model 33 | model.fit(X=data.X_train, y=data.y_train) 34 | 35 | ## Test the model 36 | y_pred = model.predict(data.X_test) 37 | print(y_pred) 38 | 39 | ## Calculate some metrics 40 | print(model.evaluate(y_true=data.y_test, y_pred=y_pred, list_metrics=["AS", "F1S", "PS", "FBS"])) 41 | -------------------------------------------------------------------------------- /examples/exam_gradient_mlp_multi_class_classification.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 16:38, 25/10/2023 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.datasets import load_iris 8 | from metaperceptron import Data, MlpClassifier 9 | 10 | 11 | ## Load data object 12 | X, y = load_iris(return_X_y=True) 13 | data = Data(X, y) 14 | 15 | ## Split train and test 16 | data.split_train_test(test_size=0.2, random_state=2, inplace=True, shuffle=True) 17 | print(data.X_train.shape, data.X_test.shape) 18 | 19 | ## Scaling dataset 20 | data.X_train, scaler_X = data.scale(data.X_train, scaling_methods=("standard", "minmax")) 21 | data.X_test = scaler_X.transform(data.X_test) 22 | 23 | data.y_train, scaler_y = data.encode_label(data.y_train) 24 | data.y_test = scaler_y.transform(data.y_test) 25 | 26 | print(type(data.X_train), type(data.y_train)) 27 | 28 | ## Create model 29 | model = MlpClassifier(hidden_layers=(30,), act_names="Tanh", dropout_rates=None, act_output=None, 30 | epochs=10, batch_size=16, optim="Adam", optim_params=None, 31 | early_stopping=True, n_patience=10, epsilon=0.001, valid_rate=0.1, 32 | seed=42, verbose=True, device="cpu") 33 | ## Train the model 34 | model.fit(X=data.X_train, y=data.y_train) 35 | 36 | ## Test the model 37 | y_pred = model.predict(data.X_test) 38 | print(y_pred) 39 | print(model.predict_proba(data.X_test)) 40 | 41 | ## Calculate some metrics 42 | print(model.evaluate(y_true=data.y_test, y_pred=y_pred, list_metrics=["F2S", "CKS", "FBS", "PS", "RS", "NPV", "F1S"])) 43 | -------------------------------------------------------------------------------- /examples/exam_gradient_mlp_binary_classification.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 08:24, 25/10/2023 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.datasets import load_breast_cancer 8 | from metaperceptron import Data, MlpClassifier 9 | 10 | 11 | ## Load data object 12 | X, y = load_breast_cancer(return_X_y=True) 13 | data = Data(X, y) 14 | 15 | ## Split train and test 16 | data.split_train_test(test_size=0.2, random_state=2, inplace=True, shuffle=True) 17 | print(data.X_train.shape, data.X_test.shape) 18 | 19 | ## Scaling dataset 20 | data.X_train, scaler_X = data.scale(data.X_train, scaling_methods=("standard", "minmax")) 21 | data.X_test = scaler_X.transform(data.X_test) 22 | 23 | data.y_train, scaler_y = data.encode_label(data.y_train) 24 | data.y_test = scaler_y.transform(data.y_test) 25 | 26 | print(type(data.X_train), type(data.y_train)) 27 | 28 | ## Create model 29 | model = MlpClassifier(hidden_layers=(30,), act_names="ReLU", dropout_rates=None, act_output=None, 30 | epochs=10, batch_size=16, optim="Adam", optim_params=None, 31 | early_stopping=True, n_patience=10, epsilon=0.001, valid_rate=0.1, 32 | seed=42, verbose=True, device="cpu") 33 | ## Train the model 34 | model.fit(X=data.X_train, y=data.y_train) 35 | 36 | ## Test the model 37 | y_pred = model.predict(data.X_test) 38 | print(y_pred) 39 | print(model.predict_proba(data.X_test)) 40 | 41 | ## Calculate some metrics 42 | print(model.evaluate(y_true=data.y_test, y_pred=y_pred, list_metrics=["F2S", "CKS", "FBS", "PS", "RS", "NPV", "F1S"])) 43 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | examples/comparator/history/ 2 | examples/core/history/ 3 | examples/helpers/history/ 4 | metaperceptron/data/ 5 | # Pycharm 6 | .idea/ 7 | tut_upcode.md 8 | drafts/ 9 | docs/refs/ 10 | # Byte-compiled / optimized / DLL files 11 | __pycache__/ 12 | *.py[cod] 13 | *$py.class 14 | 15 | # C extensions 16 | *.so 17 | 18 | # Distribution / packaging 19 | .Python 20 | build/ 21 | develop-eggs/ 22 | dist/ 23 | downloads/ 24 | eggs/ 25 | .eggs/ 26 | lib/ 27 | lib64/ 28 | parts/ 29 | sdist/ 30 | var/ 31 | wheels/ 32 | *.egg-info/ 33 | .installed.cfg 34 | *.egg 35 | MANIFEST 36 | 37 | # PyInstaller 38 | # Usually these files are written by a python script from a template 39 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 40 | *.manifest 41 | *.spec 42 | 43 | # Installer logs 44 | pip-log.txt 45 | pip-delete-this-directory.txt 46 | 47 | # Unit test / coverage reports 48 | htmlcov/ 49 | .tox/ 50 | .coverage 51 | .coverage.* 52 | .cache 53 | nosetests.xml 54 | coverage.xml 55 | *.cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | 59 | # Translations 60 | *.mo 61 | *.pot 62 | 63 | # Django stuff: 64 | *.log 65 | local_settings.py 66 | db.sqlite3 67 | 68 | # Flask stuff: 69 | instance/ 70 | .webassets-cache 71 | 72 | # Scrapy stuff: 73 | .scrapy 74 | 75 | # Sphinx documentation 76 | docs/_build/ 77 | 78 | # PyBuilder 79 | target/ 80 | 81 | # Jupyter Notebook 82 | .ipynb_checkpoints 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # celery beat schedule file 88 | celerybeat-schedule 89 | 90 | # SageMath parsed files 91 | *.sage.py 92 | 93 | # Environments 94 | .env 95 | .venv 96 | env/ 97 | venv/ 98 | ENV/ 99 | env.bak/ 100 | venv.bak/ 101 | 102 | # Spyder project settings 103 | .spyderproject 104 | .spyproject 105 | 106 | # Rope project settings 107 | .ropeproject 108 | 109 | # mkdocs documentation 110 | /site 111 | 112 | # mypy 113 | .mypy_cache/ 114 | -------------------------------------------------------------------------------- /ChangeLog.md: -------------------------------------------------------------------------------- 1 | 2 | # Version 2.2.0 3 | 4 | + Update latest version of dependencies `mealpy` to 3.0.2 and `permetrics` to 2.0.0 5 | + Update init() in `MhaMlpRegressor`, `MhaMlpClassifier`, and `MhaMlpComparator` classes. 6 | + Update workflows, examples, tests, documents, citations, and readme. 7 | 8 | ---------------------------------------------------------------------------------------- 9 | 10 | # Version 2.1.0 11 | 12 | + Fix bug name of optimizer in `MhaMlp` models 13 | + Fix bug encoding labels data in `Data` class 14 | + Add GPU support for Gradient Descent-trained MLP models 15 | + Rename `optim_paras` to `optim_params` in all models. 16 | + Fix bug in `validator` module 17 | + Fix bug missing loss_train in Gradient Descent-trained MLP models 18 | + Update examples, tests, documents, citations, and readme. 19 | 20 | ---------------------------------------------------------------------------------------- 21 | 22 | # Version 2.0.0 23 | 24 | + Re-structure and re-implement the MLP, Metaheuristic-trained MLP, and Gradient Descent-trained MLP. 25 | + Remove the dependence of Skorch library. 26 | + Add `MhaMlpTuner` class: It can tune the hyper-parameters of Metaheuristic-based MLP models. 27 | + Add `MhaMlpComparator` class: It can compare several Metaheuristic-based MLP models. 28 | + Update examples, tests, and documents 29 | + Update requirements, citations, readme. 30 | 31 | ---------------------------------------------------------------------------------------- 32 | 33 | # Version 1.1.0 34 | 35 | + Add `MhaMlpRegressor` and `MhaMlpClassifier` classes 36 | + Add docs folder and document website 37 | + Update examples and tests folders 38 | + Add zenodo DOI, CITATION.cff 39 | 40 | ---------------------------------------------------------------------------------------- 41 | 42 | # Version 1.0.1 43 | 44 | + Add infors (CODE_OF_CONDUCT.md, MANIFEST.in, LICENSE, requirements.txt) 45 | + Add helpers modules (act_util, metric_util, scaler_util, validator, preprocessor) 46 | + Add MlpRegressor and MlpClassifier classes (Based on Pytorch and Skorch) 47 | + Add publish workflow 48 | + Add examples and tests folders 49 | -------------------------------------------------------------------------------- /examples/tuner/exam_mha_mlp_regressor_tuner.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 19:27, 16/08/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.model_selection import train_test_split 8 | from sklearn.preprocessing import StandardScaler 9 | from sklearn.datasets import load_diabetes 10 | from metaperceptron import MhaMlpTuner 11 | 12 | 13 | ## Load data object 14 | X, y = load_diabetes(return_X_y=True) 15 | 16 | # Split data into train and test sets 17 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 18 | 19 | # Standardize the features 20 | scaler = StandardScaler() 21 | X_train = scaler.fit_transform(X_train) 22 | X_test = scaler.transform(X_test) 23 | print(X_train.shape, X_test.shape) 24 | 25 | # Example parameter grid of tuning hyper-parameter for Genetic Algorithm-based MLP 26 | param_dict = { 27 | 'hidden_layers': [30, 40], 28 | 'act_names': ['Tanh', 'ELU'], 29 | 'dropout_rates': [0.2, None], 30 | 'optim': ['BaseGA'], 31 | 'optim_params': [ 32 | {"epoch": 10, "pop_size": 20}, 33 | {"epoch": 20, "pop_size": 20}, 34 | ], 35 | 'obj_name': ["MSE"], 36 | 'seed': [42], 37 | "verbose": [False], 38 | } 39 | 40 | # Initialize the tuner 41 | tuner = MhaMlpTuner( 42 | task="regression", 43 | param_dict=param_dict, 44 | search_method="gridsearch", # or "randomsearch" 45 | scoring='MSE', 46 | cv=3, 47 | verbose=2, # Example additional argument 48 | n_jobs=4 # Parallelization 49 | ) 50 | 51 | # Perform tuning 52 | tuner.fit(X_train, y_train) 53 | print("Best Parameters: ", tuner.best_params_) 54 | print("Best Estimator: ", tuner.best_estimator_) 55 | 56 | y_pred = tuner.predict(X_test) 57 | print(tuner.best_estimator_.evaluate(y_test, y_pred, list_metrics=["MSE", "MAPE", "R2", "KGE", "NSE"])) 58 | -------------------------------------------------------------------------------- /examples/tuner/exam_mha_mlp_multiclass_classifier_tuner.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 06:50, 17/08/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.model_selection import train_test_split 8 | from sklearn.preprocessing import StandardScaler 9 | from sklearn.datasets import load_iris 10 | from metaperceptron import MhaMlpTuner 11 | 12 | 13 | ## Load data object 14 | X, y = load_iris(return_X_y=True) 15 | 16 | # Split data into train and test sets 17 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 18 | 19 | # Standardize the features 20 | scaler = StandardScaler() 21 | X_train = scaler.fit_transform(X_train) 22 | X_test = scaler.transform(X_test) 23 | print(X_train.shape, X_test.shape) 24 | 25 | # Example parameter grid of tuning hyper-parameter for Genetic Algorithm-based MLP 26 | param_dict = { 27 | 'hidden_layers': [(10,), (20, 10)], 28 | 'act_names': ['Tanh', 'ELU'], 29 | 'dropout_rates': [0.2, None], 30 | 'optim': ['BaseGA'], 31 | 'optim_params': [ 32 | {"epoch": 10, "pop_size": 20}, 33 | {"epoch": 20, "pop_size": 20}, 34 | ], 35 | 'obj_name': ["F1S"], 36 | 'seed': [42], 37 | "verbose": [False], 38 | } 39 | 40 | # Initialize the tuner 41 | tuner = MhaMlpTuner( 42 | task="classification", 43 | param_dict=param_dict, 44 | search_method="randomsearch", # or "randomsearch" 45 | scoring='accuracy', 46 | cv=3, 47 | verbose=2, # Example additional argument 48 | random_state=42, # Additional parameter for RandomizedSearchCV 49 | n_jobs=4 # Parallelization 50 | ) 51 | 52 | # Perform tuning 53 | tuner.fit(X_train, y_train) 54 | print("Best Parameters: ", tuner.best_params_) 55 | print("Best Estimator: ", tuner.best_estimator_) 56 | 57 | y_pred = tuner.predict(X_test) 58 | print(tuner.best_estimator_.evaluate(y_test, y_pred, list_metrics=["AS", "PS", "RS", "F1S", "NPV"])) 59 | -------------------------------------------------------------------------------- /examples/comparator/exam_mha_mlp_multiclass_classifier_comparator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 19:55, 19/08/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.model_selection import train_test_split 8 | from sklearn.preprocessing import StandardScaler 9 | from sklearn.datasets import load_iris 10 | from metaperceptron import MhaMlpComparator 11 | 12 | 13 | ## Load data object 14 | X, y = load_iris(return_X_y=True) 15 | 16 | # Split data into train and test sets 17 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 18 | 19 | # Standardize the features 20 | scaler = StandardScaler() 21 | X_train = scaler.fit_transform(X_train) 22 | X_test = scaler.transform(X_test) 23 | print(X_train.shape, X_test.shape) 24 | 25 | # Here is the list of optimizers you want to compare 26 | optim_dict = { 27 | 'BaseGA': {"epoch": 10, "pop_size": 20}, 28 | "OriginalPSO": {"epoch": 10, "pop_size": 20}, 29 | } 30 | 31 | # Initialize the comparator 32 | compartor = MhaMlpComparator( 33 | optim_dict=optim_dict, 34 | task="classification", 35 | hidden_layers=(10, ), 36 | act_names="ReLU", 37 | dropout_rates=None, 38 | act_output=None, 39 | obj_name="F1S", 40 | verbose=True, 41 | seed=42, 42 | lb=None, ub=None, mode='single', n_workers=None, termination=None 43 | ) 44 | 45 | # Perform comparison 46 | results = compartor.compare_cross_val_score(X_train, y_train, metric="AS", cv=4, n_trials=2, to_csv=True) 47 | print(results) 48 | 49 | results = compartor.compare_cross_validate(X_train, y_train, metrics=["AS", "PS", "F1S", "NPV"], 50 | cv=4, return_train_score=True, n_trials=2, to_csv=True) 51 | print(results) 52 | 53 | results = compartor.compare_train_test(X_train, y_train, X_test, y_test, 54 | metrics=["AS", "PS", "F1S", "NPV"], n_trials=2, to_csv=True) 55 | print(results) 56 | -------------------------------------------------------------------------------- /examples/tuner/exam_mha_mlp_binary_classifier_tuner.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 06:36, 17/08/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.model_selection import train_test_split 8 | from sklearn.preprocessing import StandardScaler 9 | from sklearn.datasets import load_breast_cancer 10 | from metaperceptron import MhaMlpTuner 11 | 12 | 13 | ## Load data object 14 | X, y = load_breast_cancer(return_X_y=True) 15 | 16 | # Split data into train and test sets 17 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 18 | 19 | # Standardize the features 20 | scaler = StandardScaler() 21 | X_train = scaler.fit_transform(X_train) 22 | X_test = scaler.transform(X_test) 23 | print(X_train.shape, X_test.shape) 24 | 25 | # Example parameter grid of tuning hyper-parameter for Genetic Algorithm-based MLP 26 | param_dict = { 27 | 'hidden_layers': [(10,), (20, 10)], 28 | 'act_names': ['Tanh', 'ELU'], 29 | 'dropout_rates': [0.2, None], 30 | 'optim': ['BaseGA'], 31 | 'optim_params': [ 32 | {"epoch": 10, "pop_size": 20}, 33 | {"epoch": 20, "pop_size": 20}, 34 | ], 35 | 'obj_name': ["F1S"], 36 | 'seed': [42], 37 | "verbose": [False], 38 | } 39 | 40 | # Initialize the tuner 41 | tuner = MhaMlpTuner( 42 | task="classification", 43 | param_dict=param_dict, 44 | search_method="randomsearch", # or "randomsearch" 45 | scoring='accuracy', 46 | cv=3, 47 | verbose=2, # Example additional argument 48 | random_state=42, # Additional parameter for RandomizedSearchCV 49 | n_jobs=4 # Parallelization 50 | ) 51 | 52 | # Perform tuning 53 | tuner.fit(X_train, y_train) 54 | print("Best Parameters: ", tuner.best_params_) 55 | print("Best Estimator: ", tuner.best_estimator_) 56 | 57 | y_pred = tuner.predict(X_test) 58 | print(tuner.best_estimator_.evaluate(y_test, y_pred, list_metrics=["AS", "PS", "RS", "F1S", "NPV"])) 59 | -------------------------------------------------------------------------------- /examples/comparator/exam_mha_mlp_binary_classifier_comparator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 23:30, 17/08/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.model_selection import train_test_split 8 | from sklearn.preprocessing import StandardScaler 9 | from sklearn.datasets import load_breast_cancer 10 | from metaperceptron import MhaMlpComparator 11 | 12 | 13 | ## Load data object 14 | X, y = load_breast_cancer(return_X_y=True) 15 | 16 | # Split data into train and test sets 17 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 18 | 19 | # Standardize the features 20 | scaler = StandardScaler() 21 | X_train = scaler.fit_transform(X_train) 22 | X_test = scaler.transform(X_test) 23 | print(X_train.shape, X_test.shape) 24 | 25 | # Here is the list of optimizers you want to compare 26 | optim_dict = { 27 | 'BaseGA': {"epoch": 10, "pop_size": 20}, 28 | "OriginalPSO": {"epoch": 10, "pop_size": 20}, 29 | } 30 | 31 | # Initialize the comparator 32 | compartor = MhaMlpComparator( 33 | optim_dict=optim_dict, 34 | task="classification", 35 | hidden_layers=(10, ), 36 | act_names="ReLU", 37 | dropout_rates=None, 38 | act_output=None, 39 | obj_name="F1S", 40 | verbose=True, 41 | seed=42, 42 | lb=None, ub=None, mode='single', n_workers=None, termination=None 43 | ) 44 | 45 | ## Perform comparison 46 | results = compartor.compare_cross_val_score(X_train, y_train, metric="AS", cv=4, n_trials=2, to_csv=True) 47 | print(results) 48 | 49 | results = compartor.compare_cross_validate(X_train, y_train, metrics=["AS", "PS", "F1S", "NPV"], 50 | cv=4, return_train_score=True, n_trials=2, to_csv=True) 51 | print(results) 52 | 53 | results = compartor.compare_train_test(X_train, y_train, X_test, y_test, 54 | metrics=["AS", "PS", "F1S", "NPV"], n_trials=2, to_csv=True) 55 | print(results) 56 | -------------------------------------------------------------------------------- /docs/source/pages/tutorial/provided_classes.rst: -------------------------------------------------------------------------------- 1 | Provided Classes 2 | ================ 3 | 4 | * `Data`: A class for managing and structuring data. It includes methods for loading, splitting, and scaling datasets for neural network models. 5 | 6 | * `DataTransformer`: A transformer class that applies preprocessing operations, such as scaling, normalization, or encoding, to prepare data for neural network models. 7 | 8 | * `MlpRegressor`: A standard multi-layer perceptron (MLP) model for regression tasks. It includes configurable parameters for the number and size of hidden layers, activation functions, learning rate, and optimizer. 9 | 10 | * `MlpClassifier`: A standard multi-layer perceptron (MLP) model for classification tasks. This model supports flexible architectures with customizable hyperparameters for improved classification performance. 11 | 12 | * `MhaMlpRegressor`: An MLP regressor model enhanced with Meta-Heuristic Algorithms (MHA), designed to optimize training by applying metaheuristic techniques (such as Genetic Algorithms or Particle Swarm Optimization) to find better network weights and hyperparameters for complex regression tasks. 13 | 14 | * `MhaMlpClassifier`: A classification model that combines MLP with Meta-Heuristic Algorithms (MHA) to improve training efficiency. This approach allows for robust exploration of the optimization landscape, which is beneficial for complex, high-dimensional classification problems. 15 | 16 | * `MhaMlpTuner`: A tuner class designed to optimize MLP model hyperparameters using Meta-Heuristic Algorithms (MHA). This class leverages algorithms like Genetic Algorithms, Particle Swarm Optimization, and others to automate the tuning of parameters such as learning rate, number of hidden layers, and neuron configuration, aiming to achieve optimal model performance. 17 | 18 | * `MhaMlpComparator`: A comparator class for evaluating and comparing the performance of different MLP models or configurations, particularly useful for assessing the impact of various Meta-Heuristic Algorithms (MHAs) on model training. The comparator allows side-by-side performance evaluation of models with distinct hyperparameter settings or MHA enhancements. 19 | 20 | .. toctree:: 21 | :maxdepth: 4 22 | 23 | .. toctree:: 24 | :maxdepth: 4 25 | 26 | .. toctree:: 27 | :maxdepth: 4 28 | -------------------------------------------------------------------------------- /docs/source/pages/tutorial/mha_mlp_tuner.rst: -------------------------------------------------------------------------------- 1 | MhaMlpTuner class 2 | ================= 3 | 4 | In this example, we use Genetic Algorithm-trained MLP network for Breast Cancer classification dataset. 5 | We tune several hyper-paramaters of both network structure and optimizer's parameters. 6 | 7 | .. code-block:: python 8 | 9 | from sklearn.model_selection import train_test_split 10 | from sklearn.preprocessing import StandardScaler 11 | from sklearn.datasets import load_breast_cancer 12 | from metaperceptron import MhaMlpTuner 13 | 14 | ## Load data object 15 | X, y = load_breast_cancer(return_X_y=True) 16 | 17 | # Split data into train and test sets 18 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 19 | 20 | # Standardize the features 21 | scaler = StandardScaler() 22 | X_train = scaler.fit_transform(X_train) 23 | X_test = scaler.transform(X_test) 24 | print(X_train.shape, X_test.shape) 25 | 26 | # Example parameter grid of tuning hyper-parameter for Genetic Algorithm-based MLP 27 | param_dict = { 28 | 'hidden_layers': [(10,), (20, 10)], 29 | 'act_names': ['Tanh', 'ELU'], 30 | 'dropout_rates': [0.2, None], 31 | 'optim': ['BaseGA'], 32 | 'optim_params': [ 33 | {"epoch": 10, "pop_size": 20}, 34 | {"epoch": 20, "pop_size": 20}, 35 | ], 36 | 'obj_name': ["F1S"], 37 | 'seed': [42], 38 | "verbose": [False], 39 | } 40 | 41 | # Initialize the tuner 42 | tuner = MhaMlpTuner( 43 | task="classification", 44 | param_dict=param_dict, 45 | search_method="randomsearch", # or "gridsearch" 46 | scoring='accuracy', 47 | cv=3, 48 | verbose=2, # Example additional argument 49 | random_state=42, # Additional parameter for RandomizedSearchCV 50 | n_jobs=4 # Parallelization 51 | ) 52 | 53 | # Perform tuning 54 | tuner.fit(X_train, y_train) 55 | print("Best Parameters: ", tuner.best_params_) 56 | print("Best Estimator: ", tuner.best_estimator_) 57 | 58 | y_pred = tuner.predict(X_test) 59 | print(tuner.best_estimator_.evaluate(y_test, y_pred, list_metrics=["AS", "PS", "RS", "F1S", "NPV"])) 60 | 61 | .. toctree:: 62 | :maxdepth: 4 63 | 64 | .. toctree:: 65 | :maxdepth: 4 66 | 67 | .. toctree:: 68 | :maxdepth: 4 69 | -------------------------------------------------------------------------------- /examples/comparator/exam_mha_mlp_regressor_comparator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 19:59, 19/08/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.model_selection import train_test_split 8 | from sklearn.preprocessing import StandardScaler, MinMaxScaler 9 | from sklearn.datasets import load_diabetes 10 | from metaperceptron import MhaMlpComparator 11 | 12 | 13 | ## Load data object 14 | X, y = load_diabetes(return_X_y=True) 15 | 16 | # Split data into train and test sets 17 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 18 | 19 | # Standardize the features 20 | scaler = StandardScaler() 21 | X_train = scaler.fit_transform(X_train) 22 | X_test = scaler.transform(X_test) 23 | print(X_train.shape, X_test.shape) 24 | 25 | # Min-max the output 26 | y_scaler = MinMaxScaler(feature_range=(0, 1)) 27 | y_train = y_scaler.fit_transform(y_train.reshape(-1, 1)) 28 | y_test = y_scaler.transform(y_test.reshape(-1, 1)) 29 | 30 | 31 | # Here is the list of optimizers you want to compare 32 | optim_dict = { 33 | 'BaseGA': {"epoch": 100, "pop_size": 20}, 34 | "OriginalPSO": {"epoch": 100, "pop_size": 20}, 35 | } 36 | 37 | # Initialize the comparator 38 | compartor = MhaMlpComparator( 39 | optim_dict=optim_dict, 40 | task="regression", 41 | hidden_layers=(10, ), 42 | act_names="ELU", 43 | dropout_rates=None, 44 | act_output=None, 45 | obj_name="NSE", 46 | verbose=True, 47 | seed=42, 48 | lb=None, ub=None, mode='single', n_workers=None, termination=None 49 | ) 50 | 51 | ## Perform comparison 52 | results = compartor.compare_cross_val_score(X_train, y_train, metric="RMSE", cv=4, n_trials=2, to_csv=True) 53 | print(results) 54 | 55 | results = compartor.compare_cross_validate(X_train, y_train, metrics=["MSE", "MAPE", "R2", "KGE", "NSE"], 56 | cv=4, return_train_score=True, n_trials=2, to_csv=True) 57 | print(results) 58 | 59 | results = compartor.compare_train_test(X_train, y_train, X_test, y_test, 60 | metrics=["MSE", "MAPE", "R2", "KGE", "NSE"], n_trials=2, to_csv=True) 61 | print(results) 62 | -------------------------------------------------------------------------------- /docs/source/pages/tutorial/mha_mlp_comparator.rst: -------------------------------------------------------------------------------- 1 | MhaMlpComparator 2 | ================ 3 | 4 | In this example, we will use Iris classification dataset. We compare 3 models includes `GA-MLP`, `PSO-MLP`, and `WOA-MLP`. 5 | 6 | .. code-block:: python 7 | 8 | from sklearn.model_selection import train_test_split 9 | from sklearn.preprocessing import StandardScaler 10 | from sklearn.datasets import load_iris 11 | from metaperceptron import MhaMlpComparator 12 | 13 | ## Load data object 14 | X, y = load_iris(return_X_y=True) 15 | 16 | ## Split data into train and test sets 17 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 18 | 19 | ## Standardize the features 20 | scaler = StandardScaler() 21 | X_train = scaler.fit_transform(X_train) 22 | X_test = scaler.transform(X_test) 23 | print(X_train.shape, X_test.shape) 24 | 25 | ## Here is the list of optimizers you want to compare 26 | optim_dict = { 27 | "BaseGA": {"epoch": 100, "pop_size": 20}, 28 | "OriginalPSO": {"epoch": 100, "pop_size": 20}, 29 | "OriginalWOA": {"epoch": 100, "pop_size": 20}, 30 | } 31 | 32 | ## Initialize the comparator 33 | compartor = MhaMlpComparator( 34 | optim_dict=optim_dict, 35 | task="classification", 36 | hidden_layers=(30, ), 37 | act_names="ReLU", 38 | dropout_rates=None, 39 | act_output=None, 40 | obj_name="F1S", 41 | verbose=True, 42 | seed=42, 43 | lb=None, ub=None, mode='single', n_workers=None, termination=None 44 | ) 45 | 46 | ## Perform comparison 47 | 48 | ## You can perform cross validation score method 49 | results = compartor.compare_cross_val_score(X_train, y_train, metric="AS", cv=4, n_trials=2, to_csv=True) 50 | print(results) 51 | 52 | ## Or you can perform cross validation method 53 | results = compartor.compare_cross_validate(X_train, y_train, metrics=["AS", "PS", "F1S", "NPV"], 54 | cv=4, return_train_score=True, n_trials=2, to_csv=True) 55 | print(results) 56 | 57 | ## Or you can perform train and test method 58 | results = compartor.compare_train_test(X_train, y_train, X_test, y_test, 59 | metrics=["AS", "PS", "F1S", "NPV"], n_trials=2, to_csv=True) 60 | print(results) 61 | 62 | .. toctree:: 63 | :maxdepth: 4 64 | 65 | .. toctree:: 66 | :maxdepth: 4 67 | 68 | .. toctree:: 69 | :maxdepth: 4 70 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | 17 | import sphinx_rtd_theme 18 | import os 19 | import sys 20 | 21 | sys.path.insert(0, os.path.abspath('.')) 22 | sys.path.insert(0, os.path.abspath('../../')) 23 | sys.path.insert(1, os.path.abspath('../../metaperceptron')) 24 | 25 | 26 | # -- Project information ----------------------------------------------------- 27 | 28 | project = 'metaperceptron' 29 | copyright = '2023, Thieu' 30 | author = 'Thieu' 31 | 32 | # The full version, including alpha/beta/rc tags 33 | release = '2.2.0' 34 | 35 | 36 | # -- General configuration --------------------------------------------------- 37 | 38 | # Add any Sphinx extension module names here, as strings. They can be 39 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 40 | # ones. 41 | extensions = [ 42 | "sphinx.ext.autodoc", 43 | "sphinx.ext.napoleon", 44 | "sphinx.ext.intersphinx", 45 | "sphinx.ext.viewcode", 46 | ] 47 | 48 | # Add any paths that contain templates here, relative to this directory. 49 | templates_path = ['_templates'] 50 | 51 | # List of patterns, relative to source directory, that match files and 52 | # directories to ignore when looking for source files. 53 | # This pattern also affects html_static_path and html_extra_path. 54 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 55 | 56 | 57 | # -- Options for HTML output ------------------------------------------------- 58 | 59 | # The theme to use for HTML and HTML Help pages. See the documentation for 60 | # a list of builtin themes. 61 | # 62 | html_theme = 'sphinx_rtd_theme' 63 | 64 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 65 | 66 | # Add any paths that contain custom static files (such as style sheets) here, 67 | # relative to this directory. They are copied after the builtin static files, 68 | # so a file named "default.css" will overwrite the builtin "default.css". 69 | html_static_path = ['_static'] 70 | 71 | highlight_language = 'python' 72 | -------------------------------------------------------------------------------- /docs/source/pages/tutorial/all_model_classes.rst: -------------------------------------------------------------------------------- 1 | Define all model classes 2 | ======================== 3 | 4 | Here is how you define all of provided classes. 5 | 6 | .. code-block:: python 7 | 8 | from metaperceptron import MhaMlpRegressor, MhaMlpClassifier, MlpRegressor, MlpClassifier 9 | 10 | ## Use Metaheuristic Algorithm-trained MLP model for regression problem 11 | print(MhaMlpRegressor.SUPPORTED_OPTIMIZERS) 12 | print(MhaMlpRegressor.SUPPORTED_REG_OBJECTIVES) 13 | 14 | opt_paras = {"epoch": 250, "pop_size": 30, "name": "GA"} 15 | model = MhaMlpRegressor(hidden_layers=(30, 15,), act_names="ELU", dropout_rates=0.2, act_output=None, 16 | optim="BaseGA", optim_params=opt_paras, obj_name="MSE", seed=42, verbose=True, 17 | lb=None, ub=None, mode='single', n_workers=None, termination=None) 18 | 19 | 20 | ## Use Metaheuristic Algorithm-trained MLP model for classification problem 21 | print(MhaMlpClassifier.SUPPORTED_OPTIMIZERS) 22 | print(MhaMlpClassifier.SUPPORTED_CLS_OBJECTIVES) 23 | 24 | opt_paras = {"epoch": 250, "pop_size": 30, "name": "WOA"} 25 | model = MhaMlpClassifier(hidden_layers=(100, 20), act_names="ReLU", dropout_rates=None, act_output=None, 26 | optim="OriginalWOA", optim_params=opt_paras, obj_name="F1S", seed=42, verbose=True, 27 | lb=None, ub=None, mode='single', n_workers=None, termination=None) 28 | 29 | 30 | ## Use Gradient Descent-trained (Adam Optimizer) to train MLP model for regression problem 31 | print(MhaMlpClassifier.SUPPORTED_OPTIMIZERS) 32 | 33 | model = MlpRegressor(hidden_layers=(30, 10), act_names="Tanh", dropout_rates=None, act_output=None, 34 | epochs=100, batch_size=16, optim="Adagrad", optim_params=None, 35 | early_stopping=True, n_patience=10, epsilon=0.001, valid_rate=0.1, 36 | seed=42, verbose=True) 37 | 38 | 39 | ## Use Gradient Descent-trained (Adam Optimizer) to train MLP model for classification problem 40 | print(MhaMlpClassifier.SUPPORTED_OPTIMIZERS) 41 | 42 | model = MlpClassifier(hidden_layers=(30, 20), act_names="ReLU", dropout_rates=None, act_output=None, 43 | epochs=100, batch_size=16, optim="Adam", optim_params=None, 44 | early_stopping=True, n_patience=10, epsilon=0.001, valid_rate=0.1, 45 | seed=42, verbose=True) 46 | 47 | .. toctree:: 48 | :maxdepth: 4 49 | 50 | .. toctree:: 51 | :maxdepth: 4 52 | 53 | .. toctree:: 54 | :maxdepth: 4 55 | -------------------------------------------------------------------------------- /tests/test_MhaMlpRegressor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 22:45, 02/11/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | import pytest 8 | from sklearn.datasets import make_regression 9 | from sklearn.metrics import r2_score 10 | from sklearn.model_selection import train_test_split 11 | from metaperceptron import MhaMlpRegressor 12 | 13 | 14 | @pytest.fixture 15 | def regression_data(): 16 | # Create a synthetic dataset for regression 17 | X, y = make_regression(n_samples=100, n_features=10, noise=0.1, random_state=42) 18 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 19 | return X_train, X_test, y_train, y_test 20 | 21 | 22 | @pytest.fixture 23 | def regressor(): 24 | # Initialize the MhaMlpRegressor with default parameters 25 | return MhaMlpRegressor( 26 | hidden_layers=[50, 25], 27 | act_names="ReLU", 28 | dropout_rates=0.2, 29 | optim="BaseGA", 30 | optim_params={"epoch": 10, "pop_size": 20}, 31 | seed=42, 32 | verbose=False 33 | ) 34 | 35 | 36 | def test_initialization(regressor): 37 | # Test if the regressor initializes correctly 38 | assert isinstance(regressor, MhaMlpRegressor) 39 | assert regressor.seed == 42 40 | 41 | 42 | def test_fit(regressor, regression_data): 43 | X_train, X_test, y_train, y_test = regression_data 44 | # Test if the regressor can fit the model 45 | regressor.fit(X_train, y_train) 46 | assert regressor.task in ["regression", "multi_regression"] # Task should be set 47 | assert hasattr(regressor, "network") # Model should be built after fitting 48 | 49 | 50 | def test_predict(regressor, regression_data): 51 | X_train, X_test, y_train, y_test = regression_data 52 | # Train the model before predicting 53 | regressor.fit(X_train, y_train) 54 | predictions = regressor.predict(X_test) 55 | 56 | # Check if predictions have the same length as test samples 57 | assert len(predictions) == len(y_test) 58 | 59 | 60 | def test_score(regressor, regression_data): 61 | X_train, X_test, y_train, y_test = regression_data 62 | # Train the model and calculate R^2 score 63 | regressor.fit(X_train, y_train) 64 | r2 = regressor.score(X_test, y_test) 65 | 66 | # Compare with sklearn's R^2 score 67 | predictions = regressor.predict(X_test) 68 | expected_r2 = r2_score(y_test, predictions) 69 | assert r2 == pytest.approx(expected_r2, 0.01) 70 | 71 | 72 | def test_evaluate(regressor, regression_data): 73 | X_train, X_test, y_train, y_test = regression_data 74 | # Train the model and get predictions 75 | regressor.fit(X_train, y_train) 76 | predictions = regressor.predict(X_test) 77 | 78 | # Evaluate with custom metrics 79 | metrics = regressor.evaluate(y_test, predictions, list_metrics=("MSE", "MAE")) 80 | 81 | # Check if metrics dictionary contains requested metrics 82 | assert "MSE" in metrics 83 | assert "MAE" in metrics 84 | assert isinstance(metrics["MSE"], float) 85 | assert isinstance(metrics["MAE"], float) 86 | -------------------------------------------------------------------------------- /examples/exam_gradient_mlp_regression_sklearn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 09:30, 24/05/2025 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.datasets import load_diabetes 8 | from sklearn.model_selection import cross_val_score, train_test_split, GridSearchCV 9 | from sklearn.pipeline import Pipeline 10 | from metaperceptron import DataTransformer, MlpRegressor 11 | 12 | 13 | def get_cross_val_score(X, y, cv=3): 14 | ## Train and test 15 | model = MlpRegressor(hidden_layers=(30,), act_names="Tanh", dropout_rates=None, act_output=None, 16 | epochs=10, batch_size=16, optim="Adam", optim_params=None, 17 | early_stopping=True, n_patience=10, epsilon=0.001, valid_rate=0.1, 18 | seed=42, verbose=True, device="cpu") 19 | return cross_val_score(model, X, y, cv=cv) 20 | 21 | 22 | def get_pipe_line(X, y): 23 | ## Split train and test 24 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=2) 25 | 26 | ## Train and test 27 | model = MlpRegressor(hidden_layers=(30,), act_names="Tanh", dropout_rates=None, act_output=None, 28 | epochs=10, batch_size=16, optim="Adam", optim_params=None, 29 | early_stopping=True, n_patience=10, epsilon=0.001, valid_rate=0.1, 30 | seed=42, verbose=True, device="cpu") 31 | 32 | pipe = Pipeline([ 33 | ("dt", DataTransformer(scaling_methods=("standard", "minmax"))), 34 | ("grnn", model) 35 | ]) 36 | 37 | pipe.fit(X_train, y_train) 38 | y_pred = pipe.predict(X_test) 39 | 40 | return model.evaluate(y_true=y_test, y_pred=y_pred, list_metrics=["MAE", "RMSE", "R", "NNSE", "KGE", "R2"]) 41 | 42 | 43 | def get_grid_search(X, y): 44 | ## Split train and test 45 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=2) 46 | 47 | para_grid = { 48 | 'act_names': ("ReLU", "Tanh", "Sigmoid"), 49 | 'hidden_layers': [(10,), (20,), (30,) ] 50 | } 51 | 52 | ## Create a gridsearch 53 | model = MlpRegressor(dropout_rates=None, act_output=None, 54 | epochs=10, batch_size=16, optim="Adam", optim_params=None, 55 | early_stopping=True, n_patience=10, epsilon=0.001, valid_rate=0.1, 56 | seed=42, verbose=True, device="cpu") 57 | clf = GridSearchCV(model, para_grid, cv=5, scoring='neg_mean_squared_error', verbose=2) 58 | clf.fit(X_train, y_train) 59 | print("Best parameters found: ", clf.best_params_) 60 | print("Best model: ", clf.best_estimator_) 61 | print("Best training score: ", clf.best_score_) 62 | print(clf) 63 | 64 | ## Predict 65 | y_pred = clf.predict(X_test) 66 | return model.evaluate(y_true=y_test, y_pred=y_pred, list_metrics=["MAE", "RMSE", "R", "NNSE", "KGE", "R2"]) 67 | 68 | 69 | ## Load data object 70 | X, y = load_diabetes(return_X_y=True) 71 | 72 | print(get_cross_val_score(X, y, cv=3)) 73 | print(get_pipe_line(X, y)) 74 | print(get_grid_search(X, y)) 75 | -------------------------------------------------------------------------------- /examples/exam_gradient_mlp_classification_sklearn.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 09:15, 24/05/2025 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.datasets import load_breast_cancer 8 | from sklearn.model_selection import cross_val_score, train_test_split, GridSearchCV 9 | from sklearn.pipeline import Pipeline 10 | from metaperceptron import DataTransformer, MlpClassifier 11 | 12 | 13 | def get_cross_val_score(X, y, cv=3): 14 | ## Train and test 15 | model = MlpClassifier(hidden_layers=(30,), act_names="ReLU", dropout_rates=None, act_output=None, 16 | epochs=10, batch_size=16, optim="Adam", optim_params=None, 17 | early_stopping=True, n_patience=10, epsilon=0.001, valid_rate=0.1, 18 | seed=42, verbose=True, device="cpu") 19 | return cross_val_score(model, X, y, cv=cv) 20 | 21 | 22 | def get_pipe_line(X, y): 23 | ## Split train and test 24 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=2) 25 | 26 | ## Train and test 27 | model = MlpClassifier(hidden_layers=(30,), act_names="ReLU", dropout_rates=None, act_output=None, 28 | epochs=10, batch_size=16, optim="Adam", optim_params=None, 29 | early_stopping=True, n_patience=10, epsilon=0.001, valid_rate=0.1, 30 | seed=42, verbose=True, device="cpu") 31 | 32 | pipe = Pipeline([ 33 | ("dt", DataTransformer(scaling_methods=("standard", "minmax"))), 34 | ("pnn", model) 35 | ]) 36 | 37 | pipe.fit(X_train, y_train) 38 | y_pred = pipe.predict(X_test) 39 | 40 | return model.evaluate(y_true=y_test, y_pred=y_pred, list_metrics=["F2S", "CKS", "FBS", "AS", "RS", "PS"]) 41 | 42 | 43 | def get_grid_search(X, y): 44 | ## Split train and test 45 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=2) 46 | 47 | para_grid = { 48 | 'act_names': ("ReLU", "Tanh", "Sigmoid"), 49 | 'hidden_layers': [(10,), (20,), (30,) ] 50 | } 51 | 52 | ## Create a gridsearch 53 | model = MlpClassifier(dropout_rates=None, act_output=None, 54 | epochs=10, batch_size=16, optim="Adam", optim_params=None, 55 | early_stopping=True, n_patience=10, epsilon=0.001, valid_rate=0.1, 56 | seed=42, verbose=True, device="cpu") 57 | clf = GridSearchCV(model, para_grid, cv=3, scoring='accuracy', verbose=2) 58 | clf.fit(X_train, y_train) 59 | print("Best parameters found: ", clf.best_params_) 60 | print("Best model: ", clf.best_estimator_) 61 | print("Best training score: ", clf.best_score_) 62 | print(clf) 63 | 64 | ## Predict 65 | y_pred = clf.predict(X_test) 66 | return model.evaluate(y_true=y_test, y_pred=y_pred, list_metrics=["F2S", "CKS", "FBS", "AS", "RS", "PS"]) 67 | 68 | 69 | ## Load data object 70 | X, y = load_breast_cancer(return_X_y=True) 71 | 72 | print(get_cross_val_score(X, y, cv=3)) 73 | print(get_pipe_line(X, y)) 74 | print(get_grid_search(X, y)) 75 | -------------------------------------------------------------------------------- /.github/workflows/publish-package.yml: -------------------------------------------------------------------------------- 1 | name: Tests & Publishes to PyPI 2 | 3 | on: 4 | release: 5 | types: [published] 6 | push: 7 | branches: 8 | - main 9 | pull_request: 10 | branches: 11 | - "*" 12 | 13 | env: 14 | PROJECT_NAME: metaperceptron 15 | 16 | jobs: 17 | build: 18 | runs-on: ubuntu-latest 19 | strategy: 20 | fail-fast: false 21 | matrix: 22 | python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] 23 | 24 | steps: 25 | - uses: actions/checkout@v3 26 | with: 27 | fetch-depth: 9 28 | 29 | - name: Set up Python ${{ matrix.python-version }} 30 | uses: actions/setup-python@v4 31 | with: 32 | python-version: ${{ matrix.python-version }} 33 | 34 | - uses: actions/cache@v3 35 | id: depcache 36 | with: 37 | path: deps 38 | key: requirements-pip-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }} 39 | 40 | - name: Download dependencies 41 | if: steps.depcache.outputs.cache-hit != 'true' 42 | run: | 43 | pip download --dest=deps -r requirements.txt 44 | 45 | - name: Install dependencies 46 | run: | 47 | pip install -U --no-index --find-links=deps deps/* 48 | pip install pytest pytest-cov flake8 49 | 50 | - name: Run tests 51 | run: | 52 | pytest --doctest-modules --junitxml=junit/pytest-results-${{ matrix.python-version }}.xml --cov=$PROJECT_NAME --cov-report=xml tests/ 53 | flake8 tests/ 54 | 55 | - name: Upload test results 56 | uses: actions/upload-artifact@v4 57 | with: 58 | name: pytest-results-${{ matrix.python-version }} 59 | path: junit/pytest-results-${{ matrix.python-version }}.xml 60 | if: always() 61 | 62 | - name: Install build dependencies (only for Python 3.11) 63 | if: matrix.python-version == '3.11' 64 | run: | 65 | pip install -r requirements.txt 66 | pip install --upgrade setuptools wheel twine 67 | 68 | - name: Build package (only for Python 3.11) 69 | if: matrix.python-version == '3.11' 70 | run: | 71 | python setup.py sdist bdist_wheel 72 | 73 | - name: Upload dist as artifact (only for Python 3.11) 74 | if: matrix.python-version == '3.11' 75 | uses: actions/upload-artifact@v4 76 | with: 77 | name: dist-package 78 | path: dist 79 | 80 | publish: 81 | runs-on: ubuntu-latest 82 | needs: build 83 | if: github.event_name == 'release' 84 | permissions: 85 | id-token: write 86 | contents: read 87 | steps: 88 | - name: Download dist artifact 89 | uses: actions/download-artifact@v4 90 | with: 91 | name: dist-package 92 | path: dist 93 | 94 | - name: Publish to Test PyPI 95 | uses: pypa/gh-action-pypi-publish@release/v1 96 | with: 97 | repository-url: https://test.pypi.org/legacy/ 98 | skip-existing: true 99 | attestations: false 100 | 101 | - name: Publish to PyPI 102 | uses: pypa/gh-action-pypi-publish@release/v1 103 | with: 104 | skip-existing: true 105 | attestations: true 106 | -------------------------------------------------------------------------------- /tests/test_MhaMlpComparator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 23:03, 02/11/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | import pytest 8 | import numpy as np 9 | from metaperceptron import MhaMlpComparator 10 | 11 | 12 | # Sample optimizer dictionary for testing 13 | optim_dict = { 14 | 'BaseGA': {"epoch": 10, "pop_size": 20}, 15 | "OriginalPSO": {"epoch": 10, "pop_size": 20}, 16 | } 17 | 18 | # Sample data 19 | X = np.random.rand(100, 5) # 100 samples, 5 features 20 | y_classification = np.random.randint(0, 2, 100) # Binary classification target 21 | y_regression = np.random.rand(100) # Regression target 22 | 23 | 24 | @pytest.fixture 25 | def classifier_comparator(): 26 | return MhaMlpComparator( 27 | optim_dict=optim_dict, 28 | task="classification", 29 | hidden_layers=(10,), 30 | act_names="ReLU", 31 | dropout_rates=None, 32 | act_output=None, 33 | obj_name="F1S", 34 | verbose=True, 35 | seed=42 36 | ) 37 | 38 | 39 | @pytest.fixture 40 | def regressor_comparator(): 41 | return MhaMlpComparator( 42 | optim_dict=optim_dict, 43 | task="regression", 44 | hidden_layers=(10,), 45 | act_names="ReLU", 46 | dropout_rates=None, 47 | act_output=None, 48 | obj_name="R2", 49 | verbose=True, 50 | seed=42 51 | ) 52 | 53 | 54 | def test_initialization(classifier_comparator, regressor_comparator): 55 | assert classifier_comparator.task == "classification" 56 | assert regressor_comparator.task == "regression" 57 | assert len(classifier_comparator.models) == len(optim_dict) 58 | assert len(regressor_comparator.models) == len(optim_dict) 59 | 60 | 61 | def test_compare_cross_validate_classification(classifier_comparator): 62 | results = classifier_comparator.compare_cross_validate( 63 | X, y_classification, metrics=["AS", "PS", "F1S", "NPV"], cv=3, n_trials=2, to_csv=False 64 | ) 65 | assert not results.empty 66 | assert "mean_test_AS" in results.columns 67 | assert "std_test_AS" in results.columns 68 | assert "mean_test_F1S" in results.columns 69 | assert "mean_train_F1S" in results.columns 70 | 71 | 72 | def test_compare_cross_validate_regression(regressor_comparator): 73 | results = regressor_comparator.compare_cross_validate( 74 | X, y_regression, metrics=["MSE", "MAPE", "R2", "KGE", "NSE"], cv=3, n_trials=2, to_csv=False 75 | ) 76 | assert not results.empty 77 | assert "mean_test_MSE" in results.columns 78 | assert "std_test_MSE" in results.columns 79 | assert "mean_train_MSE" in results.columns 80 | assert "std_train_MSE" in results.columns 81 | 82 | 83 | def test_compare_train_test_split(classifier_comparator): 84 | X_train, y_train = X[:80], y_classification[:80] 85 | X_test, y_test = X[80:], y_classification[80:] 86 | results = classifier_comparator.compare_train_test( 87 | X_train, y_train, X_test, y_test, metrics=["AS", "PS", "F1S", "NPV"], n_trials=2, to_csv=False 88 | ) 89 | assert not results.empty 90 | assert "PS_train" in results.columns 91 | assert "PS_test" in results.columns 92 | assert "AS_train" in results.columns 93 | assert "AS_test" in results.columns 94 | assert "F1S_test" in results.columns 95 | assert "F1S_train" in results.columns 96 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at nguyenthieu2102@gmail.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /tests/test_MlpRegressor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 22:60, 02/11/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | import pytest 8 | import numpy as np 9 | import torch 10 | from metaperceptron import MlpRegressor 11 | 12 | # Test data 13 | X = np.random.rand(100, 5) # 100 samples, 5 features each 14 | y = np.random.rand(100, 1) # 100 target values 15 | 16 | 17 | @pytest.fixture 18 | def model(): 19 | """Fixture to initialize the MlpRegressor model with default parameters.""" 20 | return MlpRegressor(hidden_layers=(50, 50), epochs=10, batch_size=8, seed=42, verbose=False) 21 | 22 | 23 | def test_initialization(model): 24 | """Test that the MlpRegressor initializes with correct default parameters.""" 25 | assert model.hidden_layers == (50, 50) 26 | assert model.epochs == 10 27 | assert model.batch_size == 8 28 | assert model.seed == 42 29 | assert model.verbose is False 30 | 31 | 32 | def test__process_data(model): 33 | """Test the data processing and tensor conversion.""" 34 | train_loader, X_valid_tensor, y_valid_tensor = model._process_data(X, y) 35 | 36 | # Check the training loader data format 37 | for batch_X, batch_y in train_loader: 38 | assert isinstance(batch_X, torch.Tensor) 39 | assert isinstance(batch_y, torch.Tensor) 40 | break # Check only the first batch 41 | 42 | # Check if validation tensors are None when valid_rate is not set 43 | if model.valid_rate == 0: 44 | assert X_valid_tensor is None 45 | assert y_valid_tensor is None 46 | else: 47 | assert X_valid_tensor is not None 48 | assert y_valid_tensor is not None 49 | 50 | 51 | def test_fit(model): 52 | """Test the fitting process of the model.""" 53 | # Fit the model 54 | model.fit(X, y) 55 | 56 | # Check if the model is trained (weights initialized) 57 | assert hasattr(model, 'network') 58 | for param in model.network.parameters(): 59 | assert param.requires_grad 60 | 61 | 62 | def test_predict_shape(model): 63 | """Test that the predict method outputs the correct shape.""" 64 | model.fit(X, y) # Fit before predicting 65 | predictions = model.predict(X) 66 | assert predictions.shape == y.shape # Shape of predictions should match shape of y 67 | 68 | 69 | def test_predict_values(model): 70 | """Test that predict outputs reasonable values for a trained model.""" 71 | model.fit(X, y) 72 | predictions = model.predict(X) 73 | # Assert predictions are finite numbers 74 | assert np.all(np.isfinite(predictions)) 75 | 76 | 77 | def test_score_r2(model): 78 | """Test the scoring method to verify R2 score calculation.""" 79 | model.fit(X, y) 80 | r2 = model.score(X, y) 81 | assert isinstance(r2, float) 82 | assert -1 <= r2 <= 1 # R2 score should be within [-1, 1] 83 | 84 | 85 | def test_evaluate(model): 86 | """Test the evaluate method to verify performance metrics.""" 87 | model.fit(X, y) 88 | predictions = model.predict(X) 89 | 90 | # Evaluate with Mean Squared Error (MSE) and Mean Absolute Error (MAE) 91 | results = model.evaluate(y, predictions, list_metrics=["MSE", "MAE"]) 92 | 93 | # Check if results contain the expected keys 94 | assert "MSE" in results 95 | assert "MAE" in results 96 | assert results["MSE"] >= 0 # MSE should be non-negative 97 | assert results["MAE"] >= 0 # MAE should be non-negative 98 | -------------------------------------------------------------------------------- /tests/test_MhaMlpClassifier.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 22:40, 02/11/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | import pytest 8 | from sklearn.datasets import make_classification 9 | from sklearn.model_selection import train_test_split 10 | from sklearn.metrics import accuracy_score 11 | from metaperceptron import MhaMlpClassifier 12 | 13 | 14 | @pytest.fixture 15 | def data(): 16 | # Create a synthetic dataset for classification 17 | X, y = make_classification(n_samples=100, n_features=20, n_classes=2, random_state=42) 18 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 19 | return X_train, X_test, y_train, y_test 20 | 21 | 22 | @pytest.fixture 23 | def classifier(): 24 | # Initialize the MhaMlpClassifier with default parameters 25 | return MhaMlpClassifier( 26 | hidden_layers=[50, 25], 27 | act_names="ReLU", 28 | dropout_rates=0.2, 29 | optim="BaseGA", 30 | optim_params={"epoch": 10, "pop_size": 20}, 31 | seed=42, 32 | verbose=False 33 | ) 34 | 35 | 36 | def test_initialization(classifier): 37 | # Test if the classifier initializes correctly 38 | assert isinstance(classifier, MhaMlpClassifier) 39 | assert classifier.seed == 42 40 | assert classifier.task == "classification" # Default should be classification 41 | 42 | 43 | def test_fit(classifier, data): 44 | X_train, X_test, y_train, y_test = data 45 | # Test if the classifier can fit the model 46 | classifier.fit(X_train, y_train) 47 | assert classifier.classes_ is not None 48 | assert len(classifier.classes_) == 2 49 | 50 | 51 | def test_predict(classifier, data): 52 | X_train, X_test, y_train, y_test = data 53 | # Train the model before predicting 54 | classifier.fit(X_train, y_train) 55 | predictions = classifier.predict(X_test) 56 | 57 | # Check if predictions have the same length as test samples 58 | assert len(predictions) == len(y_test) 59 | 60 | 61 | def test_predict_proba(classifier, data): 62 | X_train, X_test, y_train, y_test = data 63 | # Train the model before predicting probabilities 64 | classifier.fit(X_train, y_train) 65 | 66 | # Check if predict_proba returns probabilities 67 | probs = classifier.predict_proba(X_test) 68 | assert probs.shape[0] == len(X_test) 69 | assert probs.shape[1] == (1 if classifier.task == "binary_classification" else len(classifier.classes_)) 70 | 71 | 72 | def test_score(classifier, data): 73 | X_train, X_test, y_train, y_test = data 74 | # Train the model and calculate accuracy 75 | classifier.fit(X_train, y_train) 76 | accuracy = classifier.score(X_test, y_test) 77 | 78 | # Compare with sklearn's accuracy_score 79 | predictions = classifier.predict(X_test) 80 | expected_accuracy = accuracy_score(y_test, predictions) 81 | assert accuracy == pytest.approx(expected_accuracy, 0.01) 82 | 83 | 84 | def test_evaluate(classifier, data): 85 | X_train, X_test, y_train, y_test = data 86 | # Train the model and get predictions 87 | classifier.fit(X_train, y_train) 88 | predictions = classifier.predict(X_test) 89 | 90 | # Evaluate with custom metrics 91 | metrics = classifier.evaluate(y_test, predictions, list_metrics=("AS", "RS")) 92 | 93 | # Check if metrics dictionary contains requested metrics 94 | assert "AS" in metrics 95 | assert "RS" in metrics 96 | assert isinstance(metrics["AS"], float) 97 | assert isinstance(metrics["RS"], float) 98 | -------------------------------------------------------------------------------- /docs/source/pages/support.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | Citation Request 3 | ================ 4 | 5 | If you want to understand how Metaheuristic is applied to Multi-Layer Perceptron, you need to read the paper `link `_ 6 | 7 | 8 | Please include these citations if you plan to use this library:: 9 | 10 | @article{van2025metaperceptron, 11 | title={MetaPerceptron: A Standardized Framework for Metaheuristic-Driven Multi-Layer Perceptron Optimization}, 12 | author={Van Thieu, Nguyen and Mirjalili, Seyedali and Garg, Harish and Hoang, Nguyen Thanh}, 13 | journal={Computer Standards \& Interfaces}, 14 | pages={103977}, 15 | year={2025}, 16 | publisher={Elsevier}, 17 | doi={10.1016/j.csi.2025.103977}, 18 | url={https://doi.org/10.1016/j.csi.2025.103977} 19 | } 20 | 21 | @software{nguyen_van_thieu_2023_10251022, 22 | author = {Nguyen Van Thieu}, 23 | title = {MetaPerceptron: A Standardized Framework for Metaheuristic-Trained Multi-Layer Perceptron}, 24 | month = dec, 25 | year = 2023, 26 | publisher = {Zenodo}, 27 | doi = {10.5281/zenodo.10251021}, 28 | url = {https://github.com/thieu1995/MetaPerceptron} 29 | } 30 | 31 | @article{van2023mealpy, 32 | title={MEALPY: An open-source library for latest meta-heuristic algorithms in Python}, 33 | author={Van Thieu, Nguyen and Mirjalili, Seyedali}, 34 | journal={Journal of Systems Architecture}, 35 | year={2023}, 36 | publisher={Elsevier}, 37 | doi={10.1016/j.sysarc.2023.102871} 38 | } 39 | 40 | @article{van2023groundwater, 41 | title={Groundwater level modeling using Augmented Artificial Ecosystem Optimization}, 42 | author={Van Thieu, Nguyen and Barma, Surajit Deb and Van Lam, To and Kisi, Ozgur and Mahesha, Amai}, 43 | journal={Journal of Hydrology}, 44 | volume={617}, 45 | pages={129034}, 46 | year={2023}, 47 | publisher={Elsevier} 48 | } 49 | 50 | @article{thieu2019efficient, 51 | title={Efficient time-series forecasting using neural network and opposition-based coral reefs optimization}, 52 | author={Thieu Nguyen, Tu Nguyen and Nguyen, Binh Minh and Nguyen, Giang}, 53 | journal={International Journal of Computational Intelligence Systems}, 54 | volume={12}, 55 | number={2}, 56 | pages={1144--1161}, 57 | year={2019} 58 | } 59 | 60 | 61 | If you have an open-ended or a research question, you can contact me via `nguyenthieu2102@gmail.com` 62 | 63 | 64 | =============== 65 | Important links 66 | =============== 67 | 68 | * Official source code repo: https://github.com/thieu1995/MetaPerceptron 69 | * Official document: https://metaperceptron.readthedocs.io/ 70 | * Download releases: https://pypi.org/project/metaperceptron/ 71 | * Issue tracker: https://github.com/thieu1995/MetaPerceptron/issues 72 | * Notable changes log: https://github.com/thieu1995/MetaPerceptron/blob/master/ChangeLog.md 73 | 74 | * This project also related to our another projects which are "optimization" and "machine learning", check it here: 75 | * https://github.com/thieu1995/mealpy 76 | * https://github.com/thieu1995/metaheuristics 77 | * https://github.com/thieu1995/opfunu 78 | * https://github.com/thieu1995/enoppy 79 | * https://github.com/thieu1995/permetrics 80 | * https://github.com/thieu1995/MetaCluster 81 | * https://github.com/thieu1995/pfevaluator 82 | * https://github.com/thieu1995/intelelm 83 | * https://github.com/thieu1995/reflame 84 | * https://github.com/aiir-team 85 | 86 | 87 | ======= 88 | License 89 | ======= 90 | 91 | The project is licensed under GNU General Public License (GPL) V3 license. 92 | 93 | 94 | .. toctree:: 95 | :maxdepth: 4 96 | 97 | .. toctree:: 98 | :maxdepth: 4 99 | 100 | .. toctree:: 101 | :maxdepth: 4 102 | -------------------------------------------------------------------------------- /tests/test_MlpClassifier.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 22:55, 02/11/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | import pytest 8 | import numpy as np 9 | from sklearn.metrics import accuracy_score 10 | from metaperceptron import MlpClassifier 11 | 12 | 13 | @pytest.fixture 14 | def create_mlp_classifier(): 15 | """Fixture to create an instance of MlpClassifier with default parameters.""" 16 | return MlpClassifier(hidden_layers=(50, 50), act_names="ReLU", dropout_rates=0.2, 17 | epochs=10, batch_size=4, seed=42, verbose=False) 18 | 19 | 20 | @pytest.fixture 21 | def generate_data(): 22 | """Fixture to generate a small synthetic dataset for testing.""" 23 | X = np.random.rand(50, 4) 24 | y = np.random.randint(0, 2, size=50) 25 | return X, y 26 | 27 | 28 | def test_initialization(create_mlp_classifier): 29 | """Test that MlpClassifier initializes with the correct parameters.""" 30 | clf = create_mlp_classifier 31 | assert clf.hidden_layers == (50, 50) 32 | assert clf.act_names == "ReLU" 33 | assert clf.dropout_rates == 0.2 34 | assert clf.epochs == 10 35 | assert clf.batch_size == 4 36 | assert clf.seed == 42 37 | 38 | 39 | def test__process_data(create_mlp_classifier, generate_data): 40 | """Test data processing method with validation split.""" 41 | clf = create_mlp_classifier 42 | X, y = generate_data 43 | 44 | train_loader, X_valid_tensor, y_valid_tensor = clf._process_data(X, y) 45 | assert train_loader is not None 46 | assert X_valid_tensor is not None 47 | assert y_valid_tensor is not None 48 | 49 | 50 | def test_fit(create_mlp_classifier, generate_data): 51 | """Test fitting the model on synthetic data.""" 52 | clf = create_mlp_classifier 53 | X, y = generate_data 54 | 55 | clf.fit(X, y) 56 | assert clf.size_input == X.shape[1] 57 | assert clf.size_output == 1 # Assuming binary classification 58 | assert clf.classes_ is not None 59 | 60 | 61 | def test_predict(create_mlp_classifier, generate_data): 62 | """Test predictions on synthetic data.""" 63 | clf = create_mlp_classifier 64 | X, y = generate_data 65 | clf.fit(X, y) 66 | 67 | y_pred = clf.predict(X) 68 | assert y_pred.shape == y.shape 69 | assert set(np.unique(y_pred)).issubset(clf.classes_) 70 | 71 | 72 | def test_score(create_mlp_classifier, generate_data): 73 | """Test scoring method by checking accuracy against sklearn's accuracy_score.""" 74 | clf = create_mlp_classifier 75 | X, y = generate_data 76 | clf.fit(X, y) 77 | 78 | score = clf.score(X, y) 79 | y_pred = clf.predict(X) 80 | expected_score = accuracy_score(y, y_pred) 81 | assert np.isclose(score, expected_score) 82 | 83 | 84 | def test_predict_proba(create_mlp_classifier, generate_data): 85 | """Test probability prediction for classification tasks.""" 86 | clf = create_mlp_classifier 87 | X, y = generate_data 88 | clf.fit(X, y) 89 | 90 | probas = clf.predict_proba(X) 91 | assert probas.shape == (len(X), clf.size_output) 92 | assert np.all((probas >= 0) & (probas <= 1)) 93 | 94 | 95 | def test_evaluate(create_mlp_classifier, generate_data): 96 | """Test the evaluation method with default metrics.""" 97 | clf = create_mlp_classifier 98 | X, y = generate_data 99 | clf.fit(X, y) 100 | 101 | y_pred = clf.predict(X) 102 | results = clf.evaluate(y, y_pred) 103 | assert isinstance(results, dict) 104 | assert "AS" in results # Assuming "AS" is accuracy score as an example 105 | assert "RS" in results # Assuming "RS" is another metric as per your setup 106 | 107 | 108 | if __name__ == "__main__": 109 | pytest.main() 110 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. MetaPerceptron documentation master file, created by 2 | sphinx-quickstart on Sat May 20 16:59:33 2023. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to MetaPerceptron's documentation! 7 | ========================================== 8 | 9 | .. image:: https://img.shields.io/badge/release-2.2.0-yellow.svg 10 | :target: https://github.com/thieu1995/MetaPerceptron/releases 11 | 12 | .. image:: https://img.shields.io/pypi/wheel/gensim.svg 13 | :target: https://pypi.python.org/pypi/metaperceptron 14 | 15 | .. image:: https://badge.fury.io/py/metaperceptron.svg 16 | :target: https://badge.fury.io/py/metaperceptron 17 | 18 | .. image:: https://img.shields.io/pypi/pyversions/metaperceptron.svg 19 | :target: https://www.python.org/ 20 | 21 | .. image:: https://img.shields.io/pypi/status/metaperceptron.svg 22 | :target: https://img.shields.io/pypi/status/metaperceptron.svg 23 | 24 | .. image:: https://img.shields.io/pypi/dm/metaperceptron.svg 25 | :target: https://img.shields.io/pypi/dm/metaperceptron.svg 26 | 27 | .. image:: https://github.com/thieu1995/metaperceptron/actions/workflows/publish-package.yml/badge.svg 28 | :target: https://github.com/thieu1995/metaperceptron/actions/workflows/publish-package.yml 29 | 30 | .. image:: https://pepy.tech/badge/metaperceptron 31 | :target: https://pepy.tech/project/metaperceptron 32 | 33 | .. image:: https://img.shields.io/github/release-date/thieu1995/metaperceptron.svg 34 | :target: https://img.shields.io/github/release-date/thieu1995/metaperceptron.svg 35 | 36 | .. image:: https://readthedocs.org/projects/metaperceptron/badge/?version=latest 37 | :target: https://metaperceptron.readthedocs.io/en/latest/?badge=latest 38 | 39 | .. image:: https://img.shields.io/badge/Chat-on%20Telegram-blue 40 | :target: https://t.me/+fRVCJGuGJg1mNDg1 41 | 42 | .. image:: https://img.shields.io/github/contributors/thieu1995/metaperceptron.svg 43 | :target: https://img.shields.io/github/contributors/thieu1995/metaperceptron.svg 44 | 45 | .. image:: https://img.shields.io/badge/PR-Welcome-%23FF8300.svg? 46 | :target: https://git-scm.com/book/en/v2/GitHub-Contributing-to-a-Project 47 | 48 | .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.10251021.svg 49 | :target: https://zenodo.org/doi/10.5281/zenodo.10251021 50 | 51 | .. image:: https://img.shields.io/badge/License-GPLv3-blue.svg 52 | :target: https://www.gnu.org/licenses/gpl-3.0 53 | 54 | 55 | MetaPerceptron (Metaheuristic-optimized Multi-Layer Perceptron) is a Python library that implements variants and the 56 | traditional version of Multi-Layer Perceptron models. These include Metaheuristic-optimized MLP models (GA, PSO, WOA, TLO, DE, ...) 57 | and Gradient Descent-optimized MLP models (SGD, Adam, Adelta, Adagrad, ...). It provides a comprehensive list of 58 | optimizers for training MLP models and is also compatible with the Scikit-Learn library. With MetaPerceptron, 59 | you can perform searches and hyperparameter tuning using the features provided by the Scikit-Learn library. 60 | 61 | * **Free software:** GNU General Public License (GPL) V3 license 62 | * **Provided Estimator**: `MlpRegressor`, `MlpClassifier`, `MhaMlpRegressor`, `MhaMlpClassifier` 63 | * **Provided Utility**: `MhaMlpTuner` and `MhaMlpComparator` 64 | * **Total Metaheuristic-based MLP Regressor**: > 200 Models 65 | * **Total Metaheuristic-based MLP Classifier**: > 200 Models 66 | * **Total Gradient Descent-based MLP Regressor**: 12 Models 67 | * **Total Gradient Descent-based MLP Classifier**: 12 Models 68 | * **Supported performance metrics**: >= 67 (47 regressions and 20 classifications) 69 | * **Supported utility functions**: GPU for Gradient-based models, Scikit-learn compatibility, and more 70 | * **Documentation:** https://metaperceptron.readthedocs.io 71 | * **Python versions:** >= 3.8.x 72 | * **Dependencies:** numpy, scipy, scikit-learn, pytorch, mealpy, pandas, permetrics. 73 | 74 | .. toctree:: 75 | :maxdepth: 4 76 | :caption: Quick Start: 77 | 78 | pages/quick_start.rst 79 | 80 | .. toctree:: 81 | :maxdepth: 4 82 | :caption: Models API: 83 | 84 | pages/metaperceptron.rst 85 | 86 | .. toctree:: 87 | :maxdepth: 4 88 | :caption: Support: 89 | 90 | pages/support.rst 91 | 92 | 93 | 94 | Indices and tables 95 | ================== 96 | 97 | * :ref:`genindex` 98 | * :ref:`modindex` 99 | * :ref:`search` 100 | -------------------------------------------------------------------------------- /tests/test_MhaMlpTuner.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 23:23, 02/11/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | import pytest 8 | import numpy as np 9 | from sklearn.model_selection import GridSearchCV, RandomizedSearchCV 10 | from sklearn.exceptions import NotFittedError 11 | from metaperceptron import MhaMlpTuner 12 | 13 | 14 | @pytest.fixture 15 | def sample_data(): 16 | """Creates sample data for testing""" 17 | X = np.random.rand(20, 5) 18 | y_classification = np.random.randint(0, 2, size=20) 19 | y_regression = np.random.rand(20) 20 | return X, y_classification, y_regression 21 | 22 | 23 | @pytest.fixture 24 | def param_dict(): 25 | """Creates a sample parameter grid""" 26 | return { 27 | 'hidden_layers': [(10,), ], 28 | 'act_names': ['Tanh', 'ELU'], 29 | 'dropout_rates': [None], 30 | 'optim': ['BaseGA'], 31 | 'optim_params': [ 32 | {"epoch": 10, "pop_size": 20}, 33 | {"epoch": 20, "pop_size": 20}, 34 | ], 35 | 'obj_name': ["F1S"], 36 | 'seed': [42], 37 | "verbose": [False], 38 | } 39 | 40 | 41 | def test_init_invalid_task(): 42 | """Test initialization with an invalid task""" 43 | with pytest.raises(ValueError, match="Invalid task type"): 44 | MhaMlpTuner(task="invalid_task") 45 | 46 | 47 | def test_get_search_object_invalid_method(param_dict): 48 | """Test _get_search_object with invalid search method""" 49 | tuner = MhaMlpTuner(task="classification", param_dict=param_dict, search_method="invalid_method", 50 | scoring="accuracy") 51 | with pytest.raises(ValueError, match="Unsupported searching method"): 52 | tuner._get_search_object() 53 | 54 | 55 | def test_get_search_object_no_param_dict(): 56 | """Test _get_search_object without param_dict""" 57 | tuner = MhaMlpTuner(task="classification", scoring="accuracy") 58 | with pytest.raises(ValueError, match="requires a param_dict"): 59 | tuner._get_search_object() 60 | 61 | 62 | def test_get_search_object_no_scoring(param_dict): 63 | """Test _get_search_object without scoring""" 64 | tuner = MhaMlpTuner(task="classification", param_dict=param_dict) 65 | with pytest.raises(ValueError, match="requires a scoring method"): 66 | tuner._get_search_object() 67 | 68 | 69 | def test_fit_classification_grid_search(sample_data, param_dict): 70 | """Test fitting with classification task and GridSearchCV""" 71 | X, y_classification, _ = sample_data 72 | tuner = MhaMlpTuner(task="classification", param_dict=param_dict, search_method="gridsearch", scoring="accuracy") 73 | tuner.fit(X, y_classification) 74 | assert isinstance(tuner.searcher, GridSearchCV) 75 | assert tuner.best_estimator_ is not None 76 | assert tuner.best_params_ is not None 77 | 78 | 79 | def test_fit_regression_random_search(sample_data, param_dict): 80 | """Test fitting with regression task and RandomizedSearchCV""" 81 | X, _, y_regression = sample_data 82 | param_dict["obj_name"] = ["MSE"] 83 | tuner = MhaMlpTuner(task="regression", param_dict=param_dict, search_method="randomsearch", 84 | scoring="neg_mean_squared_error", n_iter=2) 85 | tuner.fit(X, y_regression) 86 | assert isinstance(tuner.searcher, RandomizedSearchCV) 87 | assert tuner.best_estimator_ is not None 88 | assert tuner.best_params_ is not None 89 | 90 | 91 | def test_predict_before_fit(sample_data, param_dict): 92 | """Test predict method before calling fit""" 93 | X, _, _ = sample_data 94 | tuner = MhaMlpTuner(task="classification", param_dict=param_dict, scoring="accuracy") 95 | with pytest.raises(NotFittedError, match="not fitted yet"): 96 | tuner.predict(X) 97 | 98 | 99 | def test_predict_after_fit(sample_data, param_dict): 100 | """Test predict method after fitting""" 101 | X, y_classification, _ = sample_data 102 | tuner = MhaMlpTuner(task="classification", param_dict=param_dict, search_method="gridsearch", scoring="accuracy") 103 | tuner.fit(X, y_classification) 104 | predictions = tuner.predict(X) 105 | assert predictions.shape == y_classification.shape 106 | 107 | 108 | def test_best_params_after_fit(sample_data, param_dict): 109 | """Check if best_params_ is set after fitting""" 110 | X, y_classification, _ = sample_data 111 | tuner = MhaMlpTuner(task="classification", param_dict=param_dict, search_method="gridsearch", scoring="accuracy") 112 | tuner.fit(X, y_classification) 113 | assert tuner.best_params_ is not None 114 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 13:24, 25/05/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | import setuptools 8 | import os 9 | import re 10 | 11 | 12 | with open("requirements.txt") as f: 13 | REQUIREMENTS = f.read().splitlines() 14 | 15 | 16 | def get_version(): 17 | init_path = os.path.join(os.path.dirname(__file__), 'metaperceptron', '__init__.py') 18 | with open(init_path, 'r', encoding='utf-8') as f: 19 | init_content = f.read() 20 | version_match = re.search(r"^__version__ = ['\"]([^'\"]+)['\"]", init_content, re.M) 21 | if version_match: 22 | return version_match.group(1) 23 | raise RuntimeError("Unable to find version string.") 24 | 25 | 26 | def readme(): 27 | with open('README.md', encoding='utf-8') as f: 28 | res = f.read() 29 | return res 30 | 31 | 32 | setuptools.setup( 33 | name="metaperceptron", 34 | version=get_version(), 35 | author="Thieu", 36 | author_email="nguyenthieu2102@gmail.com", 37 | description="MetaPerceptron: A Standardized Framework for Metaheuristic-Trained Multi-Layer Perceptron", 38 | long_description=readme(), 39 | long_description_content_type="text/markdown", 40 | keywords=["multi-layer perceptron", "machine learning", "artificial intelligence", 41 | "deep learning", "neural networks", "single hidden layer network", 42 | "random projection", "FLANN", "feed-forward neural network", "artificial neural network", 43 | "classification", "regression", "supervised learning", "online learning", "generalization", 44 | "optimization algorithms", "Kernel MLP", "Cross-validation" 45 | "Genetic algorithm (GA)", "Particle swarm optimization (PSO)", "Ant colony optimization (ACO)", 46 | "Differential evolution (DE)", "Simulated annealing", "Grey wolf optimizer (GWO)", 47 | "Whale Optimization Algorithm (WOA)", "confusion matrix", "recall", "precision", "accuracy", 48 | "pearson correlation coefficient (PCC)", "spearman correlation coefficient (SCC)", 49 | "Global optimization", "Convergence analysis", "Search space exploration", "Local search", 50 | "Computational intelligence", "Robust optimization", "metaheuristic", "metaheuristic algorithms", 51 | "nature-inspired computing", "nature-inspired algorithms", "swarm-based computation", 52 | "metaheuristic-based multi-layer perceptron", "metaheuristic-optimized MLP", 53 | "Performance analysis", "Intelligent optimization", "Simulations"], 54 | url="https://github.com/thieu1995/MetaPerceptron", 55 | project_urls={ 56 | 'Documentation': 'https://metaperceptron.readthedocs.io/', 57 | 'Source Code': 'https://github.com/thieu1995/MetaPerceptron', 58 | 'Bug Tracker': 'https://github.com/thieu1995/MetaPerceptron/issues', 59 | 'Change Log': 'https://github.com/thieu1995/MetaPerceptron/blob/main/ChangeLog.md', 60 | 'Forum': 'https://t.me/+fRVCJGuGJg1mNDg1', 61 | }, 62 | packages=setuptools.find_packages(exclude=['tests*', 'examples*']), 63 | include_package_data=True, 64 | license="GPLv3", 65 | classifiers=[ 66 | "Development Status :: 5 - Production/Stable", 67 | "Intended Audience :: Developers", 68 | "Intended Audience :: Education", 69 | "Intended Audience :: Information Technology", 70 | "Intended Audience :: Science/Research", 71 | "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", 72 | "Natural Language :: English", 73 | "Programming Language :: Python :: 3.8", 74 | "Programming Language :: Python :: 3.9", 75 | "Programming Language :: Python :: 3.10", 76 | "Programming Language :: Python :: 3.11", 77 | "Programming Language :: Python :: 3.12", 78 | "Programming Language :: Python :: 3.13", 79 | "Topic :: System :: Benchmark", 80 | "Topic :: Scientific/Engineering", 81 | "Topic :: Scientific/Engineering :: Mathematics", 82 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 83 | "Topic :: Scientific/Engineering :: Information Analysis", 84 | "Topic :: Scientific/Engineering :: Visualization", 85 | "Topic :: Scientific/Engineering :: Bio-Informatics", 86 | "Topic :: Software Development :: Build Tools", 87 | "Topic :: Software Development :: Libraries", 88 | "Topic :: Software Development :: Libraries :: Python Modules", 89 | "Topic :: Utilities", 90 | ], 91 | install_requires=REQUIREMENTS, 92 | extras_require={ 93 | "dev": ["pytest==7.1.2", "pytest-cov==4.0.0", "flake8>=4.0.1"], 94 | }, 95 | python_requires='>=3.8', 96 | ) 97 | -------------------------------------------------------------------------------- /metaperceptron/core/tuner.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 18:32, 16/08/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.exceptions import NotFittedError 8 | from sklearn.model_selection import GridSearchCV, RandomizedSearchCV 9 | from metaperceptron.helpers.metric_util import get_metric_sklearn 10 | from metaperceptron.core.metaheuristic_mlp import MhaMlpRegressor, MhaMlpClassifier 11 | 12 | 13 | class MhaMlpTuner: 14 | """ 15 | Automated hyperparameter tuner for MhaMlp models. 16 | 17 | Performs hyperparameter tuning for MhaMlp models using either GridSearchCV or RandomizedSearchCV. 18 | Provides an interface for fitting and predicting using the best found model. 19 | 20 | Attributes: 21 | model_class (class): The MhaMlp model class (MhaMlpRegressor or MhaMlpClassifier). 22 | param_grid (dict): The parameter grid for hyperparameter tuning. 23 | search_method (str): The optimization method ('gridsearch' or 'randomsearch'). 24 | kwargs (dict): Additional keyword arguments for the search method. 25 | searcher (GridSearchCV or RandomizedSearchCV): The searcher 26 | best_estimator_ (sklearn.base.BaseEstimator): The best estimator found during tuning. 27 | best_params_ (dict): The best hyperparameters found during tuning. 28 | 29 | Methods: 30 | fit(X, y): Fits the tuner to the data and tunes hyperparameters. 31 | predict(X): Predicts using the best estimator. 32 | """ 33 | 34 | def __init__(self, task="classification", param_dict=None, search_method="gridsearch", scoring=None, cv=3, **kwargs): 35 | """ 36 | Initializes the tuner 37 | 38 | Args: 39 | task (str): The task to be tuned (e.g., classification or regression). 40 | param_dict (dict): The parameter grid or distributions for hyperparameter tuning. 41 | search_method (str): The method for tuning (e.g., 'gridsearch', 'randomsearch'). 42 | **kwargs: Additional arguments for tuning methods like cv, n_iter, etc. 43 | """ 44 | self.task = task 45 | if task not in ("classification", "regression"): 46 | raise ValueError(f"Invalid task type: {task}. Supported tasks are 'classification' and 'regression'.") 47 | self.model_class = MhaMlpClassifier if task == "classification" else MhaMlpRegressor 48 | self.param_dict = param_dict 49 | self.search_method = search_method.lower() 50 | self.scoring = scoring 51 | self.cv = cv 52 | self.kwargs = kwargs 53 | self.searcher = None 54 | self.best_estimator_ = None 55 | self.best_params_ = None 56 | 57 | def _get_search_object(self): 58 | """ 59 | Returns an instance of GridSearchCV or RandomizedSearchCV based on the chosen search method. 60 | 61 | Raises: 62 | ValueError: If an unsupported search method is specified or if a parameter grid is missing for RandomizedSearchCV. 63 | """ 64 | if not self.param_dict: 65 | raise ValueError("Searching hyper-parameter requires a param_dict as a dictionary.") 66 | if not self.scoring: 67 | raise ValueError("Searching hyper-parameter requires a scoring method.") 68 | metrics = get_metric_sklearn(task=self.task, metric_names=[self.scoring]) 69 | if len(metrics) == 1: 70 | self.scoring = metrics[self.scoring] 71 | if self.search_method == "gridsearch": 72 | return GridSearchCV(estimator=self.model_class(), param_grid=self.param_dict, 73 | scoring=self.scoring, cv=self.cv, **self.kwargs) 74 | elif self.search_method == "randomsearch": 75 | return RandomizedSearchCV(estimator=self.model_class(), param_distributions=self.param_dict, 76 | scoring=self.scoring, cv=self.cv, **self.kwargs) 77 | else: 78 | raise ValueError(f"Unsupported searching method: {self.search_method}") 79 | 80 | def fit(self, X, y): 81 | """ 82 | Fits the tuner to the data and tunes the hyperparameters. 83 | 84 | Args: 85 | X (array-like): Training features. 86 | y (array-like): Training target values. 87 | 88 | Returns: 89 | self: Fitted tuner object. 90 | """ 91 | self.searcher = self._get_search_object() 92 | self.searcher.fit(X, y) 93 | self.best_estimator_ = self.searcher.best_estimator_ 94 | self.best_params_ = self.searcher.best_params_ 95 | return self 96 | 97 | def predict(self, X): 98 | """ 99 | Predicts using the best estimator found during tuning. 100 | 101 | Args: 102 | X (array-like): Data to predict. 103 | 104 | Returns: 105 | array-like: Predicted values. 106 | """ 107 | if self.best_estimator_ is None: 108 | raise NotFittedError("This MhaMlpTuner instance is not fitted yet. Call 'fit' before using this estimator.") 109 | 110 | return self.best_estimator_.predict(X) 111 | -------------------------------------------------------------------------------- /metaperceptron/helpers/metric_util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 06:52, 10/08/2023 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.metrics import make_scorer 8 | from sklearn.model_selection import cross_val_score 9 | from sklearn.model_selection import cross_val_predict 10 | from sklearn.model_selection import cross_validate 11 | from sklearn.model_selection import learning_curve 12 | from sklearn.model_selection import permutation_test_score 13 | from sklearn.model_selection import validation_curve 14 | from permetrics.regression import RegressionMetric 15 | from permetrics.classification import ClassificationMetric 16 | 17 | 18 | def get_metrics(problem, y_true, y_pred, metrics=None, testcase="test"): 19 | """ 20 | Calculates metrics for regression or classification tasks. 21 | 22 | This function takes the true labels (y_true), predicted labels (y_pred), problem type 23 | (regression or classification), a dictionary or list of metrics to calculate, and an 24 | optional test case name. It returns a dictionary containing the calculated metrics with 25 | descriptive names. 26 | 27 | Args: 28 | problem (str): The type of problem, either "regression" or "classification". 29 | y_true (array-like): The true labels. 30 | y_pred (array-like): The predicted labels. 31 | metrics (dict or list, optional): A dictionary or list of metrics to calculate. Defaults to None. 32 | testcase (str, optional): An optional test case name to prepend to the metric names. Defaults to "test". 33 | 34 | Returns: 35 | dict: A dictionary containing the calculated metrics with descriptive names. 36 | 37 | Raises: 38 | ValueError: If the `metrics` parameter is not a list or dictionary. 39 | """ 40 | if problem == "regression": 41 | evaluator = RegressionMetric(y_true, y_pred) 42 | paras = [{}, ] * len(metrics) # Create empty parameter lists for metrics 43 | else: 44 | evaluator = ClassificationMetric(y_true, y_pred) 45 | paras = [{"average": "weighted"}, ] * len(metrics) # Set default parameters for classification metrics 46 | if type(metrics) is dict: 47 | result = evaluator.get_metrics_by_dict(metrics) # Calculate metrics using a dictionary 48 | elif type(metrics) in (tuple, list): 49 | result = evaluator.get_metrics_by_list_names(metrics, paras) # Calculate metrics using a list of names and parameters 50 | else: 51 | raise ValueError("metrics parameter should be a list or dict") 52 | final_result = {} 53 | for key, value in result.items(): 54 | if testcase is None or testcase == "": 55 | final_result[f"{key}"] = value # Add metric name without test case prefix if not provided 56 | else: 57 | final_result[f"{key}_{testcase}"] = value # Add metric name with test case prefix 58 | return final_result 59 | 60 | 61 | def get_all_regression_metrics(): 62 | """ 63 | Gets a dictionary of all supported regression metrics. 64 | 65 | This function returns a dictionary where keys are metric names and values are their optimization types ("min" or "max"). 66 | 67 | Returns: 68 | dict: A dictionary containing all supported regression metrics. 69 | """ 70 | UNUSED_METRICS = ("RE", "RB", "AE", "SE", "SLE") # List of unused metrics 71 | dict_results = {} 72 | for key, value in RegressionMetric.SUPPORT.items(): 73 | if (key not in UNUSED_METRICS) and (value["type"] in ("min", "max")): 74 | dict_results[key] = value["type"] 75 | return dict_results 76 | 77 | 78 | def get_all_classification_metrics(): 79 | """ 80 | Gets a dictionary of all supported classification metrics. 81 | 82 | This function returns a dictionary where keys are metric names and values are their optimization types ("min" or "max"). 83 | 84 | Returns: 85 | dict: A dictionary containing all supported classification metrics. 86 | """ 87 | dict_results = {} 88 | for key, value in ClassificationMetric.SUPPORT.items(): 89 | if value["type"] in ("min", "max"): 90 | dict_results[key] = value["type"] # Add supported metrics and their optimization types 91 | return dict_results 92 | 93 | 94 | def get_metric_sklearn(task="classification", metric_names=None): 95 | """ 96 | Creates a dictionary of scorers for scikit-learn cross-validation. 97 | 98 | This function takes the task type (classification or regression) and a list of metric names. 99 | It creates an appropriate metrics instance (ClassificationMetric or RegressionMetric) and iterates 100 | through the provided metric names. For each metric name, it checks if it exists in the metrics 101 | instance and retrieves the corresponding method. Finally, it uses `make_scorer` to convert the 102 | method to a scorer and adds it to a dictionary. 103 | 104 | Args: 105 | task (str, optional): The task type, either "classification" or "regression". Defaults to "classification". 106 | metric_names (list, optional): A list of metric names. Defaults to None. 107 | 108 | Returns: 109 | dict: A dictionary of scorers for scikit-learn cross-validation. 110 | """ 111 | if task == "classification": 112 | met = ClassificationMetric() 113 | else: 114 | met = RegressionMetric() 115 | # Initialize an empty dictionary to hold scorers 116 | scorers = {} 117 | # Loop through metric names, dynamically create scorers, and add them to the dictionary 118 | for metric_name in metric_names: 119 | # Get the method from the metrics instance 120 | if hasattr(met, metric_name): 121 | metric_method = getattr(met, metric_name.upper()) 122 | else: 123 | continue 124 | # Convert the method to a scorer using make_scorer 125 | if met.SUPPORT[metric_name]["type"] == "min": 126 | greater_is_better = False 127 | else: 128 | greater_is_better = True 129 | scorers[metric_name] = make_scorer(metric_method, greater_is_better=greater_is_better) 130 | # Now, you can use this scorers dictionary 131 | return scorers 132 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 |

3 | MetaPerceptron 4 |

5 | 6 | 7 | --- 8 | 9 | [![GitHub release](https://img.shields.io/badge/release-2.2.0-yellow.svg)](https://github.com/thieu1995/MetaPerceptron/releases) 10 | [![Wheel](https://img.shields.io/pypi/wheel/gensim.svg)](https://pypi.python.org/pypi/metaperceptron) 11 | [![PyPI version](https://badge.fury.io/py/metaperceptron.svg)](https://badge.fury.io/py/metaperceptron) 12 | ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/metaperceptron.svg) 13 | ![PyPI - Downloads](https://img.shields.io/pypi/dm/metaperceptron.svg) 14 | [![Downloads](https://pepy.tech/badge/metaperceptron)](https://pepy.tech/project/metaperceptron) 15 | [![Tests & Publishes to PyPI](https://github.com/thieu1995/metaperceptron/actions/workflows/publish-package.yml/badge.svg)](https://github.com/thieu1995/metaperceptron/actions/workflows/publish-package.yml) 16 | [![Documentation Status](https://readthedocs.org/projects/metaperceptron/badge/?version=latest)](https://metaperceptron.readthedocs.io/en/latest/?badge=latest) 17 | [![Chat](https://img.shields.io/badge/Chat-on%20Telegram-blue)](https://t.me/+fRVCJGuGJg1mNDg1) 18 | [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10251021.svg)](https://zenodo.org/doi/10.5281/zenodo.10251021) 19 | [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) 20 | 21 | 22 | `MetaPerceptron` (Metaheuristic-optimized Multi-Layer Perceptron) is a powerful and extensible Python library that 23 | brings the best of both worlds: metaheuristic optimization and deep learning via Multi-Layer Perceptron (MLP). 24 | Whether you're working with classic Gradient Descent techniques or state-of-the-art metaheuristic algorithms 25 | like GA, PSO, WOA, DE, etc., `MetaPerceptron` has you covered. With `MetaPerceptron`, you can perform searches, 26 | feature selection, and hyperparameter tuning using the features provided by the Scikit-Learn library. 27 | 28 | ## 🚀 Features at a Glance 29 | 30 | - 🔧 **Estimators**: `MlpRegressor`, `MlpClassifier`, `MhaMlpRegressor`, `MhaMlpClassifier` 31 | - 📊 **Utilities**: `MhaMlpTuner`, `MhaMlpComparator` 32 | - 🧠 **Model Zoo**: 33 | - 200+ Metaheuristic-trained MLP Regressors 34 | - 200+ Metaheuristic-trained MLP Classifiers 35 | - 12 Gradient Descent-trained MLP Regressors 36 | - 12 Gradient Descent-trained MLP Classifiers 37 | - 📏 **67+ Performance Metrics** (47 for regression, 20 for classification) 38 | - ⚙️ **Support**: GPU support (for GD-based models), Scikit-learn compatible API 39 | - 📚 **Documentation**: https://metaperceptron.readthedocs.io 40 | - 🐍 **Python**: 3.8+ 41 | - 📦 **Dependencies**: numpy, scipy, scikit-learn, pytorch, mealpy, pandas, permetrics 42 | 43 | 44 | ## 📖 Citation 45 | 46 | If MetaPerceptron supports your work, please consider citing the following: 47 | 48 | ```bibtex 49 | @article{van2025metaperceptron, 50 | title={MetaPerceptron: A Standardized Framework for Metaheuristic-Driven Multi-Layer Perceptron Optimization}, 51 | author={Van Thieu, Nguyen and Mirjalili, Seyedali and Garg, Harish and Hoang, Nguyen Thanh}, 52 | journal={Computer Standards \& Interfaces}, 53 | pages={103977}, 54 | year={2025}, 55 | publisher={Elsevier}, 56 | doi={10.1016/j.csi.2025.103977}, 57 | url={https://doi.org/10.1016/j.csi.2025.103977} 58 | } 59 | 60 | @article{van2023mealpy, 61 | title={MEALPY: An open-source library for latest meta-heuristic algorithms in Python}, 62 | author={Van Thieu, Nguyen and Mirjalili, Seyedali}, 63 | journal={Journal of Systems Architecture}, 64 | year={2023}, 65 | publisher={Elsevier}, 66 | doi={10.1016/j.sysarc.2023.102871} 67 | } 68 | 69 | @article{van2023groundwater, 70 | title={Groundwater level modeling using Augmented Artificial Ecosystem Optimization}, 71 | author={Van Thieu, Nguyen and Barma, Surajit Deb and Van Lam, To and Kisi, Ozgur and Mahesha, Amai}, 72 | journal={Journal of Hydrology}, 73 | volume={617}, 74 | pages={129034}, 75 | year={2023}, 76 | publisher={Elsevier}, 77 | doi={10.1016/j.jhydrol.2022.129034} 78 | } 79 | ``` 80 | 81 | 82 | ## 🧪 Quick Start 83 | 84 | Install the latest version using pip: 85 | 86 | ```bash 87 | pip install metaperceptron 88 | ``` 89 | 90 | After that, check the version to ensure successful installation: 91 | 92 | ```python 93 | import metaperceptron 94 | print(metaperceptron.__version__) 95 | ``` 96 | 97 | 98 | ### ✅ Import core components 99 | 100 | Here is how you can import all provided classes from `MetaPerceptron` 101 | 102 | ```python 103 | from metaperceptron import DataTransformer, Data 104 | from metaperceptron import MhaMlpRegressor, MhaMlpClassifier, MlpRegressor, MlpClassifier 105 | from metaperceptron import MhaMlpTuner, MhaMlpComparator 106 | ``` 107 | 108 | ### 🔍 Example: Training an MLP Classifier with Genetic Algorithm 109 | 110 | In this tutorial, we will use Genetic Algorithm to train Multi-Layer Perceptron network for classification task. 111 | For more complex examples and use cases, please check the folder [examples](examples). 112 | 113 | ```python 114 | from sklearn.datasets import load_iris 115 | from sklearn.model_selection import train_test_split 116 | from metaperceptron import DataTransformer, MhaMlpClassifier 117 | 118 | ## Load the dataset 119 | X, y = load_iris(return_X_y=True) 120 | 121 | ## Split train and test 122 | X_train, y_train, X_test, y_test = train_test_split(X, y, test_size=0.2) 123 | 124 | ## Scale dataset with two methods: standard and minmax 125 | dt = DataTransformer(scaling_methods=("standard", "minmax")) 126 | X_train_scaled = dt.fit_transform(X_train) 127 | X_test_scaled = dt.transform(X_test) 128 | 129 | ## Define Genetic Algorithm-trained Multi-Layer Perceptron 130 | model = MhaMlpClassifier(hidden_layers=(50, 15), act_names="Tanh", 131 | dropout_rates=None, act_output=None, 132 | optim="BaseGA", optim_params={"epoch": 100, "pop_size": 20, "name": "GA"}, 133 | obj_name="F1S", seed=42, verbose=True) 134 | ## Train the model 135 | model.fit(X=X_train_scaled, y=y_train) 136 | 137 | ## Test the model 138 | y_pred = model.predict(X_test) 139 | print(y_pred) 140 | 141 | ## Print the score 142 | print(model.score(X_test_scaled, y_test)) 143 | 144 | ## Calculate some metrics 145 | print(model.evaluate(y_true=y_test, y_pred=y_pred, list_metrics=["AS", "PS", "RS", "F2S", "CKS", "FBS"])) 146 | ``` 147 | 148 | ## 💬 Support 149 | 150 | - 📦 [Source Code](https://github.com/thieu1995/MetaPerceptron) 151 | - 📖 [Documentation](https://metaperceptron.readthedocs.io/) 152 | - ⬇️ [PyPI Releases](https://pypi.org/project/metaperceptron/) 153 | - ❗ [Report Issues](https://github.com/thieu1995/MetaPerceptron/issues) 154 | - 📝 [Changelog](https://github.com/thieu1995/MetaPerceptron/blob/master/ChangeLog.md) 155 | - 💬 [Chat Group](https://t.me/+fRVCJGuGJg1mNDg1) 156 | 157 | --- 158 | 159 | Developed by: [Thieu](mailto:nguyenthieu2102@gmail.com?Subject=MetaPerceptron_QUESTIONS) @ 2025 160 | -------------------------------------------------------------------------------- /metaperceptron/helpers/validator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 21:39, 10/08/2023 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | import operator 8 | import numpy as np 9 | from numbers import Number 10 | 11 | 12 | SEQUENCE = (list, tuple, np.ndarray) 13 | DIGIT = (int, np.integer) 14 | REAL = (float, np.floating) 15 | 16 | 17 | def is_in_bound(value, bound): 18 | """ 19 | Checks if a value falls within a specified numerical bound. 20 | 21 | Args: 22 | value (float): The value to check. 23 | bound (tuple): A tuple representing the lower and upper bound (inclusive for lists). 24 | 25 | Returns: 26 | bool: True if the value is within the bound, False otherwise. 27 | 28 | Raises: 29 | ValueError: If the bound is not a tuple or list. 30 | """ 31 | ops = None 32 | if type(bound) is tuple: 33 | ops = operator.lt 34 | elif type(bound) is list: 35 | ops = operator.le 36 | if bound[0] == float("-inf") and bound[1] == float("inf"): 37 | return True 38 | elif bound[0] == float("-inf") and ops(value, bound[1]): 39 | return True 40 | elif ops(bound[0], value) and bound[1] == float("inf"): 41 | return True 42 | elif ops(bound[0], value) and ops(value, bound[1]): 43 | return True 44 | return False 45 | 46 | 47 | def is_str_in_list(value: str, my_list: list): 48 | """ 49 | Checks if a string value exists within a provided list. 50 | 51 | Args: 52 | value (str): The string value to check. 53 | my_list (list, optional): The list of possible values. 54 | 55 | Returns: 56 | bool: True if the value is in the list, False otherwise. 57 | """ 58 | if type(value) == str and my_list is not None: 59 | return True if value in my_list else False 60 | return False 61 | 62 | 63 | def check_int(name: str, value: None, bound=None): 64 | """ 65 | Checks if a value is an integer and optionally verifies it falls within a specified bound. 66 | 67 | Args: 68 | name (str): The name of the variable being checked. 69 | value (int or float): The value to check. 70 | bound (tuple, optional): A tuple representing the lower and upper bound (inclusive). 71 | 72 | Returns: 73 | int: The validated integer value. 74 | 75 | Raises: 76 | ValueError: If the value is not an integer or falls outside the bound (if provided). 77 | """ 78 | if isinstance(value, Number): 79 | if bound is None: 80 | return int(value) 81 | elif is_in_bound(value, bound): 82 | return int(value) 83 | bound = "" if bound is None else f"and value should be in range: {bound}" 84 | raise ValueError(f"'{name}' is an integer {bound}.") 85 | 86 | 87 | def check_float(name: str, value: None, bound=None): 88 | """ 89 | Checks if a value is a float and optionally verifies it falls within a specified bound. 90 | 91 | Args: 92 | name (str): The name of the variable being checked. 93 | value (int or float): The value to check. 94 | bound (tuple, optional): A tuple representing the lower and upper bound (inclusive). 95 | 96 | Returns: 97 | float: The validated float value. 98 | 99 | Raises: 100 | ValueError: If the value is not a float or falls outside the bound (if provided). 101 | """ 102 | if isinstance(value, Number): 103 | if bound is None: 104 | return float(value) 105 | elif is_in_bound(value, bound): 106 | return float(value) 107 | bound = "" if bound is None else f"and value should be in range: {bound}" 108 | raise ValueError(f"'{name}' is a float {bound}.") 109 | 110 | 111 | def check_str(name: str, value: str, bound=None): 112 | """ 113 | Checks if a value is a string and optionally verifies it exists within a provided list. 114 | 115 | Args: 116 | name (str): The name of the variable being checked. 117 | value (str): The value to check. 118 | bound (list, optional): A list of allowed string values. 119 | 120 | Returns: 121 | str: The validated string value. 122 | 123 | Raises: 124 | ValueError: If the value is not a string or not found in the bound list (if provided). 125 | """ 126 | if type(value) is str: 127 | if bound is None or is_str_in_list(value, bound): 128 | return value 129 | bound = "" if bound is None else f"and value should be one of this: {bound}" 130 | raise ValueError(f"'{name}' is a string {bound}.") 131 | 132 | 133 | def check_bool(name: str, value: bool, bound=(True, False)): 134 | """ 135 | Checks if a value is a boolean and optionally verifies it matches a specified bound. 136 | 137 | Args: 138 | name (str): The name of the variable being checked. 139 | value (bool): The value to check. 140 | bound (tuple, optional): A tuple of allowed boolean values. 141 | 142 | Returns: 143 | bool: The validated boolean value. 144 | 145 | Raises: 146 | ValueError: If the value is not a boolean or not in the bound (if provided). 147 | """ 148 | if type(value) is bool: 149 | if value in bound: 150 | return value 151 | bound = "" if bound is None else f"and value should be one of this: {bound}" 152 | raise ValueError(f"'{name}' is a boolean {bound}.") 153 | 154 | 155 | def check_tuple_int(name: str, values: None, bounds=None): 156 | """ 157 | Checks if a tuple contains only integers and optionally verifies they fall within specified bounds. 158 | 159 | Args: 160 | name (str): The name of the variable being checked. 161 | values (tuple): The tuple of values to check. 162 | bounds (list of tuples, optional): A list of tuples representing lower and upper bounds for each value. 163 | 164 | Returns: 165 | tuple: The validated tuple of integers. 166 | 167 | Raises: 168 | ValueError: If the values are not all integers or do not fall within the specified bounds. 169 | """ 170 | if isinstance(values, SEQUENCE) and len(values) > 1: 171 | value_flag = [isinstance(item, DIGIT) for item in values] 172 | if np.all(value_flag): 173 | if bounds is not None and len(bounds) == len(values): 174 | value_flag = [is_in_bound(item, bound) for item, bound in zip(values, bounds)] 175 | if np.all(value_flag): 176 | return values 177 | else: 178 | return values 179 | bounds = "" if bounds is None else f"and values should be in range: {bounds}" 180 | raise ValueError(f"'{name}' are integer {bounds}.") 181 | 182 | 183 | def check_tuple_float(name: str, values: tuple, bounds=None): 184 | """ 185 | Checks if a tuple contains only floats or integers and optionally verifies they fall within specified bounds. 186 | 187 | Args: 188 | name (str): The name of the variable being checked. 189 | values (tuple): The tuple of values to check. 190 | bounds (list of tuples, optional): A list of tuples representing lower and upper bounds for each value. 191 | 192 | Returns: 193 | tuple: The validated tuple of floats. 194 | 195 | Raises: 196 | ValueError: If the values are not all floats or integers or do not fall within the specified bounds. 197 | """ 198 | if isinstance(values, SEQUENCE) and len(values) > 1: 199 | value_flag = [isinstance(item, Number) for item in values] 200 | if np.all(value_flag): 201 | if bounds is not None and len(bounds) == len(values): 202 | value_flag = [is_in_bound(item, bound) for item, bound in zip(values, bounds)] 203 | if np.all(value_flag): 204 | return values 205 | else: 206 | return values 207 | bounds = "" if bounds is None else f"and values should be in range: {bounds}" 208 | raise ValueError(f"'{name}' are float {bounds}.") 209 | -------------------------------------------------------------------------------- /metaperceptron/helpers/scaler_util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 12:36, 17/09/2023 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | import numpy as np 8 | from scipy.stats import boxcox, yeojohnson 9 | from scipy.special import inv_boxcox 10 | from sklearn.base import BaseEstimator, TransformerMixin 11 | from sklearn.preprocessing import StandardScaler, MinMaxScaler, MaxAbsScaler, RobustScaler 12 | 13 | 14 | class ObjectiveScaler: 15 | """ 16 | For label scaler in classification (binary and multiple classification) 17 | """ 18 | def __init__(self, obj_name="sigmoid", ohe_scaler=None): 19 | """ 20 | ohe_scaler: Need to be an instance of One-Hot-Encoder for softmax scaler (multiple classification problem) 21 | """ 22 | self.obj_name = obj_name 23 | self.ohe_scaler = ohe_scaler 24 | 25 | def transform(self, data): 26 | if self.obj_name == "sigmoid" or self.obj_name == "self": 27 | return data 28 | elif self.obj_name == "hinge": 29 | data = np.squeeze(np.array(data)) 30 | data[np.where(data == 0)] = -1 31 | return data 32 | elif self.obj_name == "softmax": 33 | data = self.ohe_scaler.transform(np.reshape(data, (-1, 1))) 34 | return data 35 | 36 | def inverse_transform(self, data): 37 | if self.obj_name == "sigmoid": 38 | data = np.squeeze(np.array(data)) 39 | data = np.rint(data).astype(int) 40 | elif self.obj_name == "hinge": 41 | data = np.squeeze(np.array(data)) 42 | data = np.ceil(data).astype(int) 43 | data[np.where(data == -1)] = 0 44 | elif self.obj_name == "softmax": 45 | data = np.squeeze(np.array(data)) 46 | data = np.argmax(data, axis=1) 47 | return data 48 | 49 | 50 | class Log1pScaler(BaseEstimator, TransformerMixin): 51 | 52 | def fit(self, X, y=None): 53 | # LogETransformer doesn't require fitting, so we simply return self. 54 | return self 55 | 56 | def transform(self, X): 57 | # Apply the natural logarithm to each element of the input data 58 | return np.log1p(X) 59 | 60 | def inverse_transform(self, X): 61 | # Apply the exponential function to reverse the logarithmic transformation 62 | return np.expm1(X) 63 | 64 | 65 | class LogeScaler(BaseEstimator, TransformerMixin): 66 | 67 | def fit(self, X, y=None): 68 | # LogETransformer doesn't require fitting, so we simply return self. 69 | return self 70 | 71 | def transform(self, X): 72 | # Apply the natural logarithm (base e) to each element of the input data 73 | return np.log(X) 74 | 75 | def inverse_transform(self, X): 76 | # Apply the exponential function to reverse the logarithmic transformation 77 | return np.exp(X) 78 | 79 | 80 | class SqrtScaler(BaseEstimator, TransformerMixin): 81 | 82 | def fit(self, X, y=None): 83 | # SqrtScaler doesn't require fitting, so we simply return self. 84 | return self 85 | 86 | def transform(self, X): 87 | # Apply the square root transformation to each element of the input data 88 | return np.sqrt(X) 89 | 90 | def inverse_transform(self, X): 91 | # Apply the square of each element to reverse the square root transformation 92 | return X ** 2 93 | 94 | 95 | class BoxCoxScaler(BaseEstimator, TransformerMixin): 96 | 97 | def __init__(self, lmbda=None): 98 | self.lmbda = lmbda 99 | 100 | def fit(self, X, y=None): 101 | # Estimate the lambda parameter from the data if not provided 102 | if self.lmbda is None: 103 | _, self.lmbda = boxcox(X.flatten()) 104 | return self 105 | 106 | def transform(self, X): 107 | # Apply the Box-Cox transformation to the data 108 | X_new = boxcox(X.flatten(), lmbda=self.lmbda) 109 | return X_new.reshape(X.shape) 110 | 111 | def inverse_transform(self, X): 112 | # Inverse transform using the original lambda parameter 113 | return inv_boxcox(X, self.lmbda) 114 | 115 | 116 | class YeoJohnsonScaler(BaseEstimator, TransformerMixin): 117 | 118 | def __init__(self, lmbda=None): 119 | self.lmbda = lmbda 120 | 121 | def fit(self, X, y=None): 122 | # Estimate the lambda parameter from the data if not provided 123 | if self.lmbda is None: 124 | _, self.lmbda = yeojohnson(X.flatten()) 125 | return self 126 | 127 | def transform(self, X): 128 | # Apply the Yeo-Johnson transformation to the data 129 | X_new = boxcox(X.flatten(), lmbda=self.lmbda) 130 | return X_new.reshape(X.shape) 131 | 132 | def inverse_transform(self, X): 133 | # Inverse transform using the original lambda parameter 134 | return inv_boxcox(X, self.lmbda) 135 | 136 | 137 | class SinhArcSinhScaler(BaseEstimator, TransformerMixin): 138 | # https://stats.stackexchange.com/questions/43482/transformation-to-increase-kurtosis-and-skewness-of-normal-r-v 139 | def __init__(self, epsilon=0.1, delta=1.0): 140 | self.epsilon = epsilon 141 | self.delta = delta 142 | 143 | def fit(self, X, y=None): 144 | return self 145 | 146 | def transform(self, X): 147 | return np.sinh(self.delta * np.arcsinh(X) - self.epsilon) 148 | 149 | def inverse_transform(self, X): 150 | return np.sinh((np.arcsinh(X) + self.epsilon) / self.delta) 151 | 152 | 153 | class DataTransformer(BaseEstimator, TransformerMixin): 154 | 155 | SUPPORTED_SCALERS = {"standard": StandardScaler, "minmax": MinMaxScaler, "max-abs": MaxAbsScaler, 156 | "log1p": Log1pScaler, "loge": LogeScaler, "sqrt": SqrtScaler, 157 | "sinh-arc-sinh": SinhArcSinhScaler, "robust": RobustScaler, 158 | "box-cox": BoxCoxScaler, "yeo-johnson": YeoJohnsonScaler} 159 | 160 | def __init__(self, scaling_methods=('standard', ), list_dict_paras=None): 161 | if type(scaling_methods) is str: 162 | if list_dict_paras is None: 163 | self.list_dict_paras = [{}] 164 | elif type(list_dict_paras) is dict: 165 | self.list_dict_paras = [list_dict_paras] 166 | else: 167 | raise TypeError(f"You use only 1 scaling method, the list_dict_paras should be dict of parameter for that scaler.") 168 | self.scaling_methods = [scaling_methods] 169 | elif type(scaling_methods) in (tuple, list, np.ndarray): 170 | if list_dict_paras is None: 171 | self.list_dict_paras = [{}, ]*len(scaling_methods) 172 | elif type(list_dict_paras) in (tuple, list, np.ndarray): 173 | self.list_dict_paras = list(list_dict_paras) 174 | else: 175 | raise TypeError(f"Invalid type of list_dict_paras. Supported type are: tuple, list, or np.ndarray of parameter dict") 176 | self.scaling_methods = list(scaling_methods) 177 | else: 178 | raise TypeError(f"Invalid type of scaling_methods. Supported type are: str, tuple, list, or np.ndarray") 179 | 180 | self.scalers = [self._get_scaler(technique, paras) for (technique, paras) in zip(self.scaling_methods, self.list_dict_paras)] 181 | 182 | def _get_scaler(self, technique, paras): 183 | if technique in self.SUPPORTED_SCALERS.keys(): 184 | if type(paras) is not dict: 185 | paras = {} 186 | return self.SUPPORTED_SCALERS[technique](**paras) 187 | else: 188 | raise ValueError(f"Invalid scaling technique. Supported techniques are {self.SUPPORTED_SCALERS.keys()}") 189 | 190 | def fit(self, X, y=None): 191 | for idx, _ in enumerate(self.scalers): 192 | X = self.scalers[idx].fit_transform(X) 193 | return self 194 | 195 | def transform(self, X): 196 | for scaler in self.scalers: 197 | X = scaler.transform(X) 198 | return X 199 | 200 | def inverse_transform(self, X): 201 | for scaler in reversed(self.scalers): 202 | X = scaler.inverse_transform(X) 203 | return X 204 | -------------------------------------------------------------------------------- /metaperceptron/helpers/preprocessor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 23:33, 10/08/2023 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | import numpy as np 8 | from metaperceptron.helpers.scaler_util import DataTransformer 9 | from sklearn.model_selection import train_test_split 10 | 11 | 12 | class OneHotEncoder: 13 | """ 14 | Encode categorical features as a one-hot numeric array. 15 | This is useful for converting categorical variables into a format that can be provided to ML algorithms. 16 | """ 17 | def __init__(self): 18 | self.categories_ = None 19 | 20 | def fit(self, X): 21 | """Fit the encoder to unique categories in X.""" 22 | self.categories_ = np.unique(X) 23 | return self 24 | 25 | def transform(self, X): 26 | """Transform X into one-hot encoded format.""" 27 | if self.categories_ is None: 28 | raise ValueError("The encoder has not been fitted yet.") 29 | one_hot = np.zeros((X.shape[0], len(self.categories_)), dtype=int) 30 | for i, val in enumerate(X): 31 | index = np.where(self.categories_ == val)[0][0] 32 | one_hot[i, index] = 1 33 | return one_hot 34 | 35 | def fit_transform(self, X): 36 | """Fit the encoder to X and transform X.""" 37 | self.fit(X) 38 | return self.transform(X) 39 | 40 | def inverse_transform(self, one_hot): 41 | """Convert one-hot encoded format back to original categories.""" 42 | if self.categories_ is None: 43 | raise ValueError("The encoder has not been fitted yet.") 44 | if one_hot.shape[1] != len(self.categories_): 45 | raise ValueError("The shape of the input does not match the number of categories.") 46 | original = np.array([self.categories_[np.argmax(row)] for row in one_hot]) 47 | return original 48 | 49 | 50 | class LabelEncoder: 51 | """ 52 | Encode categorical features as integer labels. 53 | """ 54 | 55 | def __init__(self): 56 | self.unique_labels = None 57 | self.label_to_index = {} 58 | 59 | def fit(self, y): 60 | """ 61 | Fit label encoder to a given set of labels. 62 | 63 | Parameters 64 | ---------- 65 | y : array-like 66 | Labels to encode. 67 | """ 68 | self.unique_labels = np.unique(y) 69 | self.label_to_index = {label: i for i, label in enumerate(self.unique_labels)} 70 | 71 | def transform(self, y): 72 | """ 73 | Transform labels to encoded integer labels. 74 | 75 | Parameters 76 | ---------- 77 | y : array-like 78 | Labels to encode. 79 | 80 | Returns: 81 | -------- 82 | encoded_labels : array-like 83 | Encoded integer labels. 84 | """ 85 | y = np.ravel(y) 86 | if self.unique_labels is None: 87 | raise ValueError("Label encoder has not been fit yet.") 88 | return np.array([self.label_to_index[label] for label in y]) 89 | 90 | def fit_transform(self, y): 91 | """Fit label encoder and return encoded labels. 92 | 93 | Parameters 94 | ---------- 95 | y : array-like of shape (n_samples,) 96 | Target values. 97 | 98 | Returns 99 | ------- 100 | y : array-like of shape (n_samples,) 101 | Encoded labels. 102 | """ 103 | self.fit(y) 104 | return self.transform(y) 105 | 106 | def inverse_transform(self, y): 107 | """ 108 | Transform integer labels to original labels. 109 | 110 | Parameters 111 | ---------- 112 | y : array-like 113 | Encoded integer labels. 114 | 115 | Returns 116 | ------- 117 | original_labels : array-like 118 | Original labels. 119 | """ 120 | if self.unique_labels is None: 121 | raise ValueError("Label encoder has not been fit yet.") 122 | return np.array([self.unique_labels[i] if i in self.label_to_index.values() else "unknown" for i in y]) 123 | 124 | 125 | class TimeSeriesDifferencer: 126 | 127 | def __init__(self, interval=1): 128 | if interval < 1: 129 | raise ValueError("Interval for differencing must be at least 1.") 130 | self.interval = interval 131 | 132 | def difference(self, X): 133 | self.original_data = X.copy() 134 | return np.array([X[i] - X[i - self.interval] for i in range(self.interval, len(X))]) 135 | 136 | def inverse_difference(self, diff_data): 137 | if self.original_data is None: 138 | raise ValueError("Original data is required for inversion.") 139 | return np.array([diff_data[i - self.interval] + self.original_data[i - self.interval] for i in 140 | range(self.interval, len(self.original_data))]) 141 | 142 | 143 | class FeatureEngineering: 144 | def __init__(self): 145 | """ 146 | Initialize the FeatureEngineering class 147 | """ 148 | # Check if the threshold is a valid number 149 | pass 150 | 151 | def create_threshold_binary_features(self, X, threshold): 152 | """ 153 | Perform feature engineering to add binary indicator columns for values below the threshold. 154 | Add each new column right after the corresponding original column. 155 | 156 | Args: 157 | X (numpy.ndarray): The input 2D matrix of shape (n_samples, n_features). 158 | threshold (float): The threshold value for identifying low values. 159 | 160 | Returns: 161 | numpy.ndarray: The updated 2D matrix with binary indicator columns. 162 | """ 163 | # Check if X is a NumPy array 164 | if not isinstance(X, np.ndarray): 165 | raise ValueError("Input X should be a NumPy array.") 166 | # Check if the threshold is a valid number 167 | if not (isinstance(threshold, int) or isinstance(threshold, float)): 168 | raise ValueError("Threshold should be a numeric value.") 169 | 170 | # Create a new matrix to hold the original and new columns 171 | X_new = np.zeros((X.shape[0], X.shape[1] * 2)) 172 | # Iterate over each column in X 173 | for idx in range(X.shape[1]): 174 | feature_values = X[:, idx] 175 | # Create a binary indicator column for values below the threshold 176 | indicator_column = (feature_values < threshold).astype(int) 177 | # Add the original column and indicator column to the new matrix 178 | X_new[:, idx * 2] = feature_values 179 | X_new[:, idx * 2 + 1] = indicator_column 180 | return X_new 181 | 182 | 183 | class Data: 184 | """ 185 | The structure of our supported Data class 186 | 187 | Parameters 188 | ---------- 189 | X : np.ndarray 190 | The features of your data 191 | 192 | y : np.ndarray 193 | The labels of your data 194 | """ 195 | 196 | SUPPORT = { 197 | "scaler": list(DataTransformer.SUPPORTED_SCALERS.keys()) 198 | } 199 | 200 | def __init__(self, X=None, y=None, name="Unknown"): 201 | self.X = X 202 | self.y = self.check_y(y) 203 | self.name = name 204 | self.X_train, self.y_train, self.X_test, self.y_test = None, None, None, None 205 | 206 | @staticmethod 207 | def check_y(y): 208 | if y is None: 209 | return y 210 | y = np.squeeze(np.asarray(y)) 211 | if y.ndim == 1: 212 | y = np.reshape(y, (-1, 1)) 213 | return y 214 | 215 | @staticmethod 216 | def scale(X, scaling_methods=('standard',), list_dict_paras=None): 217 | X = np.squeeze(np.asarray(X)) 218 | if X.ndim == 1: 219 | X = np.reshape(X, (-1, 1)) 220 | if X.ndim >= 3: 221 | raise TypeError(f"Invalid X data type. It should be array-like with shape (n samples, m features)") 222 | scaler = DataTransformer(scaling_methods=scaling_methods, list_dict_paras=list_dict_paras) 223 | data = scaler.fit_transform(X) 224 | return data, scaler 225 | 226 | @staticmethod 227 | def encode_label(y): 228 | y = np.squeeze(np.asarray(y)) 229 | if y.ndim != 1: 230 | raise TypeError(f"Invalid y data type. It should be a vector / array-like with shape (n samples,)") 231 | scaler = LabelEncoder() 232 | data = scaler.fit_transform(y) 233 | return data, scaler 234 | 235 | def split_train_test(self, test_size=0.2, train_size=None, 236 | random_state=41, shuffle=True, stratify=None, inplace=True): 237 | """ 238 | The wrapper of the split_train_test function in scikit-learn library. 239 | """ 240 | self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(self.X, self.y, test_size=test_size, 241 | train_size=train_size, 242 | random_state=random_state, 243 | shuffle=shuffle, stratify=stratify) 244 | if not inplace: 245 | return self.X_train, self.X_test, self.y_train, self.y_test 246 | 247 | def set_train_test(self, X_train=None, y_train=None, X_test=None, y_test=None): 248 | """ 249 | Function use to set your own X_train, y_train, X_test, y_test in case you don't want to use our split function 250 | 251 | Parameters 252 | ---------- 253 | X_train : np.ndarray 254 | y_train : np.ndarray 255 | X_test : np.ndarray 256 | y_test : np.ndarray 257 | """ 258 | self.X_train = X_train 259 | self.y_train = y_train 260 | self.X_test = X_test 261 | self.y_test = y_test 262 | return self 263 | -------------------------------------------------------------------------------- /examples/core/exam_mha_mlp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 09:10, 01/11/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.datasets import load_iris, load_breast_cancer, load_diabetes, make_regression 8 | from sklearn.model_selection import train_test_split, GridSearchCV 9 | from sklearn.preprocessing import StandardScaler 10 | from sklearn.metrics import accuracy_score 11 | from metaperceptron import MhaMlpClassifier, MhaMlpRegressor 12 | 13 | 14 | def check_MhaMlpClassifier_multi_class(): 15 | # Load and prepare the dataset 16 | data = load_iris() 17 | X = data.data 18 | y = data.target 19 | 20 | # Split data into train and test sets 21 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 22 | 23 | # Standardize the features 24 | scaler = StandardScaler() 25 | X_train = scaler.fit_transform(X_train) 26 | X_test = scaler.transform(X_test) 27 | 28 | model = MhaMlpClassifier(hidden_layers=(100, ), act_names="ELU", dropout_rates=None, act_output=None, 29 | optim="BaseGA", optim_params=None, obj_name="F1S", seed=42, verbose=True) 30 | # print(model.SUPPORTED_CLS_OBJECTIVES) 31 | 32 | model.fit(X_train, y_train, mode="swarm", n_workers=6) 33 | res = model.score(X_test, y_test) 34 | print(res) 35 | print(model.network) 36 | print(model.network.get_weights()) 37 | 38 | 39 | def check_MhaMlpClassifier_multi_class_gridsearch(): 40 | # Load and prepare the dataset 41 | data = load_iris() 42 | X = data.data 43 | y = data.target 44 | 45 | # Split data into train and test sets 46 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 47 | 48 | # Standardize the features 49 | scaler = StandardScaler() 50 | X_train = scaler.fit_transform(X_train) 51 | X_test = scaler.transform(X_test) 52 | 53 | # Define parameters for grid search 54 | param_grid = { 55 | 'hidden_layers': [(30,), [20, 10], (50, 20)], 56 | 'act_names': ["ReLU", "Tanh", "Sigmoid"], 57 | 'dropout_rates': [0.2, 0.3, None], 58 | 'optim': ['BaseGA', 'OriginalWOA'], 59 | "optim_params": [ 60 | {"epoch": 10, "pop_size": 30 }, 61 | {"epoch": 20, "pop_size": 30 }, 62 | ], 63 | "obj_name": ["F1S"] 64 | } 65 | 66 | # Hyperparameter tuning with GridSearchCV 67 | searcher = GridSearchCV(estimator=MhaMlpClassifier(), param_grid=param_grid, cv=3, scoring='accuracy', verbose=2) 68 | searcher.fit(X_train, y_train) 69 | 70 | # Get best parameters and accuracy 71 | print("Best Parameters:", searcher.best_params_) 72 | print("Best Cross-Validation Accuracy:", searcher.best_score_) 73 | 74 | # Evaluate on test set 75 | best_model = searcher.best_estimator_ 76 | y_pred = best_model.predict(X_test) 77 | test_accuracy = accuracy_score(y_test, y_pred) 78 | print("Test Accuracy:", test_accuracy) 79 | 80 | print(best_model.network) 81 | 82 | 83 | def check_MhaMlpClassifier_binary_class(): 84 | # Load and prepare the dataset 85 | data = load_breast_cancer() 86 | X = data.data 87 | y = data.target 88 | 89 | # Split data into train and test sets 90 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 91 | 92 | # Standardize the features 93 | scaler = StandardScaler() 94 | X_train = scaler.fit_transform(X_train) 95 | X_test = scaler.transform(X_test) 96 | 97 | model = MhaMlpClassifier(hidden_layers=(100,), act_names="ELU", dropout_rates=0.2, act_output=None, 98 | optim="BaseGA", optim_params=None, obj_name="F1S", seed=42, verbose=True) 99 | model.fit(X_train, y_train) 100 | res = model.score(X_test, y_test) 101 | print(res) 102 | print(model.network) 103 | 104 | 105 | def check_MhaMlpClassifier_binary_class_gridsearch(): 106 | # Load and prepare the dataset 107 | data = load_breast_cancer() 108 | X = data.data 109 | y = data.target 110 | 111 | # Split data into train and test sets 112 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 113 | 114 | # Standardize the features 115 | scaler = StandardScaler() 116 | X_train = scaler.fit_transform(X_train) 117 | X_test = scaler.transform(X_test) 118 | 119 | # Define parameters for grid search 120 | param_grid = { 121 | 'hidden_layers': [(30,), (50, 20)], 122 | 'act_names': ["ReLU", "Tanh", "Sigmoid"], 123 | 'dropout_rates': [0.2, None], 124 | 'optim': ['BaseGA', 'OriginalWOA'], 125 | "optim_params": [ 126 | {"epoch": 10, "pop_size": 30 }, 127 | {"epoch": 20, "pop_size": 30 }, 128 | ], 129 | "obj_name": ["F1S"] 130 | } 131 | 132 | # Hyperparameter tuning with GridSearchCV 133 | searcher = GridSearchCV(estimator=MhaMlpClassifier(), param_grid=param_grid, cv=3, verbose=2) 134 | searcher.fit(X_train, y_train) 135 | 136 | # Get best parameters and accuracy 137 | print("Best Parameters:", searcher.best_params_) 138 | print("Best Cross-Validation Accuracy:", searcher.best_score_) 139 | 140 | # Evaluate on test set 141 | best_model = searcher.best_estimator_ 142 | y_pred = best_model.predict(X_test) 143 | test_accuracy = accuracy_score(y_test, y_pred) 144 | print("Test Accuracy:", test_accuracy) 145 | 146 | print(best_model.network) 147 | 148 | 149 | def check_MhaMlpRegressor_single_output(): 150 | # Load and prepare the dataset 151 | data = load_diabetes() 152 | X = data.data 153 | y = data.target 154 | 155 | # Split data into train and test sets 156 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 157 | 158 | # Standardize the features 159 | X_scaler = StandardScaler() 160 | X_train = X_scaler.fit_transform(X_train) 161 | X_test = X_scaler.transform(X_test) 162 | 163 | y_scaler = StandardScaler() 164 | y_train = y_scaler.fit_transform(y_train.reshape(-1, 1)) 165 | y_test = y_scaler.transform(y_test.reshape(-1, 1)) 166 | 167 | model = MhaMlpRegressor(hidden_layers=(30, 15,), act_names="ELU", dropout_rates=0.2, act_output=None, 168 | optim="BaseGA", optim_params=None, obj_name="MSE", seed=42, verbose=True) 169 | model.fit(X_train, y_train) 170 | res = model.score(X_test, y_test) 171 | print(res) 172 | print(model.network) 173 | 174 | 175 | def check_MhaMlpRegressor_single_output_gridsearch(): 176 | # Load and prepare the dataset 177 | data = load_diabetes() 178 | X = data.data 179 | y = data.target 180 | 181 | # Split data into train and test sets 182 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 183 | 184 | # Standardize the features 185 | X_scaler = StandardScaler() 186 | X_train = X_scaler.fit_transform(X_train) 187 | X_test = X_scaler.transform(X_test) 188 | 189 | y_scaler = StandardScaler() 190 | y_train = y_scaler.fit_transform(y_train.reshape(-1, 1)) 191 | y_test = y_scaler.transform(y_test.reshape(-1, 1)) 192 | 193 | # Define parameters for grid search 194 | param_grid = { 195 | 'hidden_layers': [(60,), (100,)], 196 | 'act_names': ["ELU", "Tanh", "Sigmoid"], 197 | 'dropout_rates': [0.2, None], 198 | 'optim': ['BaseGA', 'OriginalWOA'], 199 | "optim_params": [ 200 | {"epoch": 10, "pop_size": 30}, 201 | {"epoch": 20, "pop_size": 30}, 202 | ], 203 | "obj_name": ["MSE"] 204 | } 205 | 206 | # Hyperparameter tuning with GridSearchCV 207 | searcher = GridSearchCV(estimator=MhaMlpRegressor(), param_grid=param_grid, cv=3, 208 | scoring='neg_mean_squared_error', verbose=2) 209 | searcher.fit(X_train, y_train) 210 | 211 | # Get best parameters and accuracy 212 | print("Best Parameters:", searcher.best_params_) 213 | print("Best Cross-Validation Accuracy:", searcher.best_score_) 214 | 215 | # Evaluate on test set 216 | best_model = searcher.best_estimator_ 217 | # y_pred = best_model.predict(X_test) 218 | print(best_model.score(X_test, y_test)) 219 | print(best_model.network) 220 | 221 | 222 | def check_MhaMlpRegressor_multi_output(): 223 | # Load and prepare the dataset 224 | # 1. Generate synthetic multi-output regression dataset 225 | # Setting n_targets=2 for multi-output (2 target variables) 226 | X, y = make_regression(n_samples=1000, n_features=10, n_targets=2, noise=0.1, random_state=42) 227 | 228 | # Split into train and test sets 229 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 230 | 231 | # Standardize the features 232 | X_scaler = StandardScaler() 233 | X_train = X_scaler.fit_transform(X_train) 234 | X_test = X_scaler.transform(X_test) 235 | 236 | y_scaler = StandardScaler() 237 | y_train = y_scaler.fit_transform(y_train) 238 | y_test = y_scaler.transform(y_test) 239 | 240 | model = MhaMlpRegressor(hidden_layers=(30, 15,), act_names="ELU", dropout_rates=0.2, act_output=None, 241 | optim="BaseGA", optim_params=None, obj_name="MSE", seed=42, verbose=True) 242 | model.fit(X_train, y_train) 243 | res = model.score(X_test, y_test) 244 | print(res) 245 | print(model.network) 246 | 247 | 248 | def check_MhaMlpRegressor_multi_output_gridsearch(): 249 | # Load and prepare the dataset 250 | X, y = make_regression(n_samples=1000, n_features=10, n_targets=2, noise=0.1, random_state=42) 251 | 252 | # Split data into train and test sets 253 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 254 | 255 | # Standardize the features 256 | X_scaler = StandardScaler() 257 | X_train = X_scaler.fit_transform(X_train) 258 | X_test = X_scaler.transform(X_test) 259 | 260 | y_scaler = StandardScaler() 261 | y_train = y_scaler.fit_transform(y_train) 262 | y_test = y_scaler.transform(y_test) 263 | 264 | # Define parameters for grid search 265 | param_grid = { 266 | 'hidden_layers': [(60,), (100,)], 267 | 'act_names': ["ELU", "Tanh", "Sigmoid"], 268 | 'dropout_rates': [0.2, None], 269 | 'optim': ['BaseGA', 'OriginalWOA'], 270 | "optim_params": [ 271 | {"epoch": 10, "pop_size": 30}, 272 | {"epoch": 20, "pop_size": 30}, 273 | ], 274 | "obj_name": ["MSE"] 275 | } 276 | 277 | # Hyperparameter tuning with GridSearchCV 278 | searcher = GridSearchCV(estimator=MhaMlpRegressor(), param_grid=param_grid, cv=3, 279 | scoring='neg_mean_squared_error', verbose=2) 280 | searcher.fit(X_train, y_train) 281 | 282 | # Get best parameters and accuracy 283 | print("Best Parameters:", searcher.best_params_) 284 | print("Best Cross-Validation Accuracy:", searcher.best_score_) 285 | 286 | # Evaluate on test set 287 | best_model = searcher.best_estimator_ 288 | # y_pred = best_model.predict(X_test) 289 | print(best_model.score(X_test, y_test)) 290 | print(best_model.network) 291 | 292 | 293 | check_MhaMlpClassifier_multi_class() 294 | check_MhaMlpClassifier_multi_class_gridsearch() 295 | check_MhaMlpClassifier_binary_class() 296 | check_MhaMlpClassifier_binary_class_gridsearch() 297 | check_MhaMlpRegressor_single_output() 298 | check_MhaMlpRegressor_single_output_gridsearch() 299 | check_MhaMlpRegressor_multi_output() 300 | check_MhaMlpRegressor_multi_output_gridsearch() 301 | -------------------------------------------------------------------------------- /examples/core/exam_gradient_mlp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 17:25, 01/11/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from sklearn.datasets import load_iris, load_breast_cancer, load_diabetes, make_regression 8 | from sklearn.model_selection import train_test_split, GridSearchCV 9 | from sklearn.preprocessing import StandardScaler 10 | from sklearn.metrics import accuracy_score 11 | from metaperceptron import MlpClassifier, MlpRegressor 12 | 13 | 14 | def check_MlpClassifier_multi_class(): 15 | # Load and prepare the dataset 16 | data = load_iris() 17 | X = data.data 18 | y = data.target 19 | 20 | # Split data into train and test sets 21 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 22 | 23 | # Standardize the features 24 | scaler = StandardScaler() 25 | X_train = scaler.fit_transform(X_train) 26 | X_test = scaler.transform(X_test) 27 | 28 | model = MlpClassifier(hidden_layers=(100, ), act_names="ReLU", dropout_rates=None, act_output=None, 29 | epochs=10, batch_size=16, optim="Adam", optim_params=None, 30 | early_stopping=True, n_patience=10, epsilon=0.001, valid_rate=0.1, 31 | seed=42, verbose=True, device="cpu") 32 | model.fit(X_train, y_train) 33 | res = model.score(X_test, y_test) 34 | print(res) 35 | print(model.network) 36 | print(model.network.get_weights()) 37 | 38 | 39 | def check_MlpClassifier_multi_class_gridsearch(): 40 | # Load and prepare the dataset 41 | data = load_iris() 42 | X = data.data 43 | y = data.target 44 | 45 | # Split data into train and test sets 46 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 47 | 48 | # Standardize the features 49 | scaler = StandardScaler() 50 | X_train = scaler.fit_transform(X_train) 51 | X_test = scaler.transform(X_test) 52 | 53 | # Define parameters for grid search 54 | param_grid = { 55 | 'hidden_layers': [(30,), (50,), [20, 10], (50, 20)], 56 | 'act_names': ["ReLU", "Tanh", "Sigmoid"], 57 | 'dropout_rates': [0.2, 0.3, None], 58 | 'epochs': [10, 20], 59 | 'batch_size': [16, 24], 60 | 'optim': ['Adam', 'SGD'], 61 | "early_stopping": [True], 62 | "valid_rate": [0.1, 0.2], 63 | } 64 | 65 | # Hyperparameter tuning with GridSearchCV 66 | searcher = GridSearchCV(estimator=MlpClassifier(), param_grid=param_grid, cv=3, scoring='accuracy', verbose=2) 67 | searcher.fit(X_train, y_train) 68 | 69 | # Get best parameters and accuracy 70 | print("Best Parameters:", searcher.best_params_) 71 | print("Best Cross-Validation Accuracy:", searcher.best_score_) 72 | 73 | # Evaluate on test set 74 | best_net = searcher.best_estimator_ 75 | y_pred = best_net.predict(X_test) 76 | test_accuracy = accuracy_score(y_test, y_pred) 77 | print("Test Accuracy:", test_accuracy) 78 | 79 | print(best_net.network) 80 | 81 | 82 | def check_MlpClassifier_binary_class(): 83 | # Load and prepare the dataset 84 | data = load_breast_cancer() 85 | X = data.data 86 | y = data.target 87 | 88 | # Split data into train and test sets 89 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 90 | 91 | # Standardize the features 92 | scaler = StandardScaler() 93 | X_train = scaler.fit_transform(X_train) 94 | X_test = scaler.transform(X_test) 95 | 96 | model = MlpClassifier(hidden_layers=(100, 50), act_names="ReLU", dropout_rates=0.2, act_output=None, 97 | epochs=50, batch_size=8, optim="Adam", optim_params=None, 98 | early_stopping=True, n_patience=5, epsilon=0.0, valid_rate=0.1, 99 | seed=42, verbose=True, device="cpu") 100 | model.fit(X_train, y_train) 101 | res = model.score(X_test, y_test) 102 | print(res) 103 | print(model.network) 104 | 105 | 106 | def check_MlpClassifier_binary_class_gridsearch(): 107 | # Load and prepare the dataset 108 | data = load_breast_cancer() 109 | X = data.data 110 | y = data.target 111 | 112 | # Split data into train and test sets 113 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 114 | 115 | # Standardize the features 116 | scaler = StandardScaler() 117 | X_train = scaler.fit_transform(X_train) 118 | X_test = scaler.transform(X_test) 119 | 120 | # Define parameters for grid search 121 | param_grid = { 122 | 'hidden_layers': [(60,), (100,)], 123 | 'act_names': ["ReLU", "Tanh", "Sigmoid"], 124 | 'dropout_rates': [0.2, None], 125 | 'epochs': [10, 20], 126 | 'batch_size': [16, 24], 127 | 'optim': ['Adam', 'SGD'], 128 | "early_stopping": [True], 129 | "valid_rate": [0.1, 0.2], 130 | } 131 | 132 | # Hyperparameter tuning with GridSearchCV 133 | searcher = GridSearchCV(estimator=MlpClassifier(), param_grid=param_grid, cv=3, scoring='accuracy', verbose=2) 134 | searcher.fit(X_train, y_train) 135 | 136 | # Get best parameters and accuracy 137 | print("Best Parameters:", searcher.best_params_) 138 | print("Best Cross-Validation Accuracy:", searcher.best_score_) 139 | 140 | # Evaluate on test set 141 | best_net = searcher.best_estimator_ 142 | y_pred = best_net.predict(X_test) 143 | test_accuracy = accuracy_score(y_test, y_pred) 144 | print("Test Accuracy:", test_accuracy) 145 | 146 | print(best_net.network) 147 | 148 | 149 | def check_MlpRegressor_single_output(): 150 | # Load and prepare the dataset 151 | data = load_diabetes() 152 | X = data.data 153 | y = data.target 154 | 155 | # Split data into train and test sets 156 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 157 | 158 | # Standardize the features 159 | X_scaler = StandardScaler() 160 | X_train = X_scaler.fit_transform(X_train) 161 | X_test = X_scaler.transform(X_test) 162 | 163 | y_scaler = StandardScaler() 164 | y_train = y_scaler.fit_transform(y_train.reshape(-1, 1)) 165 | y_test = y_scaler.transform(y_test.reshape(-1, 1)) 166 | 167 | model = MlpRegressor(hidden_layers=(30, 15), act_names="ELU", dropout_rates=0.2, act_output=None, 168 | epochs=50, batch_size=8, optim="Adam", optim_params=None, 169 | early_stopping=True, n_patience=5, epsilon=0.0, valid_rate=0.1, 170 | seed=42, verbose=True, device="cpu") 171 | model.fit(X_train, y_train) 172 | res = model.score(X_test, y_test) 173 | print(res) 174 | print(model.network) 175 | 176 | 177 | def check_MlpRegressor_single_output_gridsearch(): 178 | # Load and prepare the dataset 179 | data = load_diabetes() 180 | X = data.data 181 | y = data.target 182 | 183 | # Split data into train and test sets 184 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 185 | 186 | # Standardize the features 187 | X_scaler = StandardScaler() 188 | X_train = X_scaler.fit_transform(X_train) 189 | X_test = X_scaler.transform(X_test) 190 | 191 | y_scaler = StandardScaler() 192 | y_train = y_scaler.fit_transform(y_train.reshape(-1, 1)) 193 | y_test = y_scaler.transform(y_test.reshape(-1, 1)) 194 | 195 | # Define parameters for grid search 196 | param_grid = { 197 | 'hidden_layers': [(60,), (100,)], 198 | 'act_names': ["ELU", "Tanh", "Sigmoid"], 199 | 'dropout_rates': [0.2, None], 200 | 'epochs': [10, 20], 201 | 'batch_size': [16, 24], 202 | 'optim': ['Adam', 'SGD'], 203 | "early_stopping": [True], 204 | "valid_rate": [0.1, 0.2], 205 | } 206 | 207 | # Hyperparameter tuning with GridSearchCV 208 | searcher = GridSearchCV(estimator=MlpRegressor(), param_grid=param_grid, cv=3, 209 | scoring='neg_mean_squared_error', verbose=2) 210 | searcher.fit(X_train, y_train) 211 | 212 | # Get best parameters and accuracy 213 | print("Best Parameters:", searcher.best_params_) 214 | print("Best Cross-Validation Accuracy:", searcher.best_score_) 215 | 216 | # Evaluate on test set 217 | best_net = searcher.best_estimator_ 218 | # y_pred = best_net.predict(X_test) 219 | print(best_net.score(X_test, y_test)) 220 | print(best_net.network) 221 | 222 | 223 | def check_MlpRegressor_multi_output(): 224 | # Load and prepare the dataset 225 | # 1. Generate synthetic multi-output regression dataset 226 | # Setting n_targets=2 for multi-output (2 target variables) 227 | X, y = make_regression(n_samples=1000, n_features=10, n_targets=2, noise=0.1, random_state=42) 228 | 229 | # Split into train and test sets 230 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 231 | 232 | # Standardize the features 233 | X_scaler = StandardScaler() 234 | X_train = X_scaler.fit_transform(X_train) 235 | X_test = X_scaler.transform(X_test) 236 | 237 | y_scaler = StandardScaler() 238 | y_train = y_scaler.fit_transform(y_train) 239 | y_test = y_scaler.transform(y_test) 240 | 241 | model = MlpRegressor(hidden_layers=(30, 15), act_names="ELU", dropout_rates=0.2, act_output=None, 242 | epochs=50, batch_size=8, optim="Adam", optim_params=None, 243 | early_stopping=True, n_patience=5, epsilon=0.0, valid_rate=0.1, 244 | seed=42, verbose=True, device="cpu") 245 | model.fit(X_train, y_train) 246 | res = model.score(X_test, y_test) 247 | print(res) 248 | print(model.network) 249 | 250 | 251 | def check_MlpRegressor_multi_output_gridsearch(): 252 | # Load and prepare the dataset 253 | X, y = make_regression(n_samples=1000, n_features=10, n_targets=2, noise=0.1, random_state=42) 254 | 255 | # Split data into train and test sets 256 | X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) 257 | 258 | # Standardize the features 259 | X_scaler = StandardScaler() 260 | X_train = X_scaler.fit_transform(X_train) 261 | X_test = X_scaler.transform(X_test) 262 | 263 | y_scaler = StandardScaler() 264 | y_train = y_scaler.fit_transform(y_train) 265 | y_test = y_scaler.transform(y_test) 266 | 267 | # Define parameters for grid search 268 | param_grid = { 269 | 'hidden_layers': [(60,), (100,)], 270 | 'act_names': ["ELU", "Tanh", "Sigmoid"], 271 | 'dropout_rates': [0.2, None], 272 | 'epochs': [10, 20], 273 | 'batch_size': [16, 24], 274 | 'optim': ['Adam', 'SGD'], 275 | "early_stopping": [True], 276 | "valid_rate": [0.1, 0.2], 277 | } 278 | 279 | # Hyperparameter tuning with GridSearchCV 280 | searcher = GridSearchCV(estimator=MlpRegressor(), param_grid=param_grid, cv=3, 281 | scoring='neg_mean_squared_error', verbose=2) 282 | searcher.fit(X_train, y_train) 283 | 284 | # Get best parameters and accuracy 285 | print("Best Parameters:", searcher.best_params_) 286 | print("Best Cross-Validation Accuracy:", searcher.best_score_) 287 | 288 | # Evaluate on test set 289 | best_net = searcher.best_estimator_ 290 | # y_pred = best_net.predict(X_test) 291 | print(best_net.score(X_test, y_test)) 292 | print(best_net.network) 293 | 294 | 295 | check_MlpClassifier_multi_class() 296 | check_MlpClassifier_multi_class_gridsearch() 297 | check_MlpClassifier_binary_class() 298 | check_MlpClassifier_binary_class_gridsearch() 299 | check_MlpRegressor_single_output() 300 | check_MlpRegressor_single_output_gridsearch() 301 | check_MlpRegressor_multi_output() 302 | check_MlpRegressor_multi_output_gridsearch() 303 | -------------------------------------------------------------------------------- /metaperceptron/core/comparator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 10:42, 17/08/2024 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | from pathlib import Path 8 | import numpy as np 9 | import pandas as pd 10 | from sklearn.model_selection import cross_val_score, cross_validate 11 | from metaperceptron import MhaMlpRegressor, MhaMlpClassifier 12 | from metaperceptron.helpers.metric_util import get_metric_sklearn 13 | 14 | 15 | class MhaMlpComparator: 16 | """ 17 | Automated compare different MhaMlp models based on provided optimizer configurations. 18 | 19 | This class facilitates the comparison of multiple MhaMlp models with varying optimizer 20 | configurations. It provides methods for cross-validation and train-test split evaluation. 21 | 22 | Args: 23 | optim_dict (dict, optional): A dictionary of optimizer names and parameters. 24 | task (str, optional): The task type (classification or regression). Defaults to 'classification'. 25 | hidden_layers (int, list, tuple, optional): The number of nodes in each hidden layer. Defaults is (10, ). 26 | act_names (str, optional): The activation function name. Defaults to 'ELU'. 27 | obj_name (str, optional): The objective function name. Defaults to None. 28 | verbose (bool, optional): Whether to print verbose output. Defaults to False. 29 | seed (int, optional): Random seed for reproducibility. Defaults to None. 30 | obj_weights (array-like, optional): Weights for the objective function. Defaults to None. 31 | **kwargs: Additional keyword arguments for model initialization. 32 | """ 33 | 34 | def __init__(self, optim_dict=None, task="classification", hidden_layers=(10, ), act_names="ELU", 35 | dropout_rates=None, act_output=None, obj_name=None, verbose=False, seed=None, 36 | lb=None, ub=None, mode='single', n_workers=None, termination=None): 37 | self.optim_dict = self._set_optimizer_dict(optim_dict) 38 | self.hidden_layers = hidden_layers 39 | self.act_names = act_names 40 | self.dropout_rates = dropout_rates 41 | self.act_output = act_output 42 | self.obj_name = obj_name 43 | self.verbose = verbose 44 | self.lb = lb 45 | self.ub = ub 46 | self.mode = mode 47 | self.n_workers = n_workers 48 | self.termination = termination 49 | self.generator = np.random.default_rng(seed) 50 | self.task = task 51 | if self.task == "classification": 52 | self.models = [MhaMlpClassifier(hidden_layers=hidden_layers, act_names=act_names, 53 | dropout_rates=dropout_rates, act_output=act_output, obj_name=obj_name, 54 | verbose=verbose, seed=seed, lb=lb, ub=ub, mode=mode, n_workers=n_workers, 55 | termination=termination) for _ in range(len(self.optim_dict))] 56 | else: 57 | self.models = [MhaMlpRegressor(hidden_layers=hidden_layers, act_names=act_names, 58 | dropout_rates=dropout_rates, act_output=act_output, obj_name=obj_name, 59 | verbose=verbose, seed=seed, lb=lb, ub=ub, mode=mode, n_workers=n_workers, 60 | termination=termination) for _ in range(len(self.optim_dict))] 61 | self.best_estimator_ = None 62 | self.best_params_ = None 63 | self.result_cross_val_scores_ = None 64 | self.result_train_test_ = None 65 | 66 | def _set_optimizer_dict(self, opt_dict): 67 | """Validates the optimizer dictionary.""" 68 | if type(opt_dict) is not dict: 69 | raise TypeError(f"Support optim_dict hyper-parameter as dict only: {type(opt_dict)}") 70 | return opt_dict 71 | 72 | def _filter_metric_results(self, results, metric_names=None, return_train_score=False): 73 | """Filters and formats metric results.""" 74 | list_metrics = [] 75 | for idx, metric in enumerate(metric_names): 76 | list_metrics.append(f"test_{metric}") 77 | if return_train_score: 78 | list_metrics.append(f"train_{metric}") 79 | final_results = {} 80 | for idx, metric in enumerate(list_metrics): 81 | final_results[f"mean_{metric}"] = np.mean(results[metric]) 82 | final_results[f"std_{metric}"] = np.std(results[metric]) 83 | return final_results 84 | 85 | def _rename_metrics(self, results: dict, suffix="train"): 86 | """Renames metric keys with a specified suffix.""" 87 | res = {} 88 | for metric_name, metric_value in results.items(): 89 | res[f"{metric_name}_{suffix}"] = metric_value 90 | return res 91 | 92 | def _results_to_csv(self, to_csv=False, results=None, saved_file_path="history/results.csv"): 93 | """Saves results to a CSV file.""" 94 | Path(f"{Path.cwd()}/{Path(saved_file_path).parent}").mkdir(parents=True, exist_ok=True) 95 | df = pd.DataFrame(results) 96 | if to_csv: 97 | if not saved_file_path.lower().endswith(".csv"): 98 | saved_file_path += ".csv" 99 | df.to_csv(saved_file_path, index=False) 100 | return df 101 | 102 | def compare_cross_validate(self, X, y, metrics=None, cv=5, return_train_score=True, n_trials=10, 103 | to_csv=True, saved_file_path="history/results_cross_validate.csv", **kwargs): 104 | """Performs cross-validation for model comparison. 105 | 106 | Compares different MhaMlp models using cross-validation. 107 | 108 | Args: 109 | X (array-like): The feature matrix. 110 | y (array-like): The target vector. 111 | metrics (list, optional): A list of metric names. Defaults to None. 112 | cv (int, optional): The number of cross-validation folds. Defaults to 5. 113 | return_train_score (bool, optional): Whether to return train scores. Defaults to True. 114 | n_trials (int, optional): The number of trials. Defaults to 10. 115 | to_csv (bool, optional): Whether to save results to a CSV file. Defaults to True. 116 | saved_file_path (str, optional): The path to save the CSV file. Defaults to 'history/results_cross_validate.csv'. 117 | **kwargs: Additional keyword arguments for cross_validate. 118 | 119 | Returns: 120 | pandas.DataFrame: The comparison results. 121 | """ 122 | list_seeds = self.generator.choice(list(range(0, 1000)), n_trials, replace=False) 123 | scoring = get_metric_sklearn(task=self.task, metric_names=metrics) 124 | results = [] 125 | for idx, (opt_name, opt_paras) in enumerate(self.optim_dict.items()): 126 | self.models[idx].set_optim_and_paras(opt_name, opt_paras) 127 | for trial, seed in enumerate(list_seeds): 128 | self.models[idx].set_seed(seed) 129 | res = cross_validate(self.models[idx], X, y, scoring=scoring, cv=cv, return_train_score=return_train_score, **kwargs) 130 | final_res = self._filter_metric_results(res, list(scoring.keys()), return_train_score) 131 | temp = {"model_name": f"{opt_name}-{opt_paras}", "trial": trial, **final_res} 132 | results.append(temp) 133 | return self._results_to_csv(to_csv, results=results, saved_file_path=saved_file_path) 134 | 135 | def compare_cross_val_score(self, X, y, metric=None, cv=5, n_trials=10, to_csv=True, 136 | saved_file_path="history/results_cross_val_score.csv", **kwargs): 137 | """Performs cross-validation with a single metric. 138 | 139 | Compares different MhaMlp models using cross-validation with a single metric. 140 | 141 | Args: 142 | X (array-like): The feature matrix. 143 | y (array-like): The target vector. 144 | metric (str, optional): The metric to evaluate. Defaults to None. 145 | cv (int, optional): The number of cross-validation folds. Defaults to 5. 146 | n_trials (int, optional): The number of trials. Defaults to 10. 147 | to_csv (bool, optional): Whether to save results to a CSV file. Defaults to True. 148 | saved_file_path (str, optional): The path to save the CSV file. Defaults to 'history/results_cross_val_score.csv'. 149 | **kwargs: Additional keyword arguments for cross_val_score. 150 | 151 | Returns: 152 | pandas.DataFrame: The comparison results. 153 | """ 154 | list_seeds = self.generator.choice(list(range(0, 1000)), n_trials, replace=False) 155 | scoring = get_metric_sklearn(task=self.task, metric_names=[metric])[metric] 156 | results = [] 157 | for idx, (opt_name, opt_paras) in enumerate(self.optim_dict.items()): 158 | self.models[idx].set_optim_and_paras(opt_name, opt_paras) 159 | for trial, seed in enumerate(list_seeds): 160 | self.models[idx].set_seed(seed) 161 | res = cross_val_score(self.models[idx], X, y, scoring=scoring, cv=cv, **kwargs) 162 | final_res = {"model_name": f"{opt_name}-{opt_paras}", "trial": trial, f"mean_{metric}": np.mean(res), f"std_{metric}": np.std(res)} 163 | results.append(final_res) 164 | return self._results_to_csv(to_csv, results=results, saved_file_path=saved_file_path) 165 | 166 | def compare_train_test(self, X_train, y_train, X_test, y_test, metrics=None, n_trials=10, 167 | to_csv=True, saved_file_path="history/results_train_test.csv"): 168 | """Compares models using train-test split. 169 | 170 | Compares different MhaMlp models using train-test split evaluation. 171 | 172 | Args: 173 | X_train (array-like): The training feature matrix. 174 | y_train (array-like): The training target vector. 175 | X_test (array-like): The testing feature matrix. 176 | y_test (array-like): The testing target vector. 177 | metrics (list, optional): A list of metric names. Defaults to None. 178 | n_trials (int, optional): The number of trials. Defaults to 10. 179 | to_csv (bool, optional): Whether to save results to a CSV file. Defaults to True. 180 | saved_file_path (str, optional): The path to save the CSV file. Defaults to 'history/results_train_test.csv'. 181 | 182 | Returns: 183 | pandas.DataFrame: The comparison results. 184 | """ 185 | list_seeds = self.generator.choice(list(range(0, 1000)), n_trials, replace=False) 186 | for idx, (opt_name, opt_paras) in enumerate(self.optim_dict.items()): 187 | self.models[idx].set_optim_and_paras(opt_name, opt_paras) 188 | results = [] 189 | for idx, _ in enumerate(self.models): 190 | for trial, seed in enumerate(list_seeds): 191 | self.models[idx].set_seed(seed) 192 | self.models[idx].fit(X_train, y_train) 193 | model_name = self.models[idx].get_name() 194 | y_train_pred = self.models[idx].predict(X_train) 195 | res1 = self.models[idx].evaluate(y_train, y_train_pred, metrics) 196 | y_test_pred = self.models[idx].predict(X_test) 197 | res2 = self.models[idx].evaluate(y_test, y_test_pred, metrics) 198 | res1 = self._rename_metrics(res1, suffix="train") 199 | res2 = self._rename_metrics(res2, suffix="test") 200 | if self.verbose: 201 | temp = {**res1, **res2} 202 | print(f"{model_name} model is trained and evaluated with score: {temp}") 203 | results.append({"model": model_name, "trial": trial, **res1, **res2}) 204 | return self._results_to_csv(to_csv, results=results, saved_file_path=saved_file_path) 205 | -------------------------------------------------------------------------------- /metaperceptron/core/metaheuristic_mlp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 17:43, 13/09/2023 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | import numpy as np 8 | import torch 9 | from sklearn.metrics import accuracy_score, r2_score 10 | from sklearn.base import ClassifierMixin, RegressorMixin 11 | from permetrics import ClassificationMetric, RegressionMetric 12 | from metaperceptron.core.base_mlp import BaseMhaMlp 13 | 14 | 15 | class MhaMlpClassifier(BaseMhaMlp, ClassifierMixin): 16 | """ 17 | A Metaheuristic-based MLP Classifier that extends the BaseMhaMlp class and implements 18 | the ClassifierMixin interface from Scikit-Learn for classification tasks. 19 | 20 | Attributes 21 | ---------- 22 | classes_ : np.ndarray 23 | Unique classes found in the target variable. 24 | 25 | metric_class : type 26 | The metric class used for evaluating classification performance. 27 | 28 | Parameters 29 | ---------- 30 | hidden_layers : list of int, tuple of int, or np.ndarray of int 31 | The structure of the hidden layers (default is (100,)). 32 | 33 | act_names : str 34 | Activation function name to use in hidden layers (default is "ReLU"). 35 | 36 | dropout_rates : float, optional 37 | Dropout rate for regularization (default is 0.2). 38 | 39 | act_output : any, optional 40 | Activation function for the output layer (default is None). 41 | 42 | optim : str 43 | The optimization algorithm to use (default is "BaseGA"). 44 | 45 | optim_params : dict 46 | Parameters for the optimizer (default is None). 47 | 48 | obj_name : str, optional 49 | The objective name for the optimization (default is "F1S"). 50 | 51 | seed : int, optional 52 | Random seed for reproducibility (default is 42). 53 | 54 | verbose : bool, optional 55 | Whether to print detailed logs during fitting (default is True). 56 | 57 | lb : int, float, tuple, list, np.ndarray, optional 58 | Lower bounds for weights and biases in network. 59 | 60 | ub : int, float, tuple, list, np.ndarray, optional 61 | Upper bounds for weights and biases in network. 62 | 63 | mode : str, optional 64 | Mode for optimization (default is 'single'). 65 | 66 | n_workers : int, optional 67 | Number of workers for parallel processing (default is None). 68 | 69 | termination : any, optional 70 | Termination criteria for optimization (default is None). 71 | """ 72 | 73 | def __init__(self, hidden_layers=(100,), act_names="ReLU", dropout_rates=0.2, act_output=None, 74 | optim="BaseGA", optim_params=None, obj_name="F1S", seed=42, verbose=True, 75 | lb=None, ub=None, mode='single', n_workers=None, termination=None): 76 | """ 77 | Initializes the MhaMlpClassifier with specified parameters. 78 | """ 79 | super().__init__(hidden_layers, act_names, dropout_rates, act_output, 80 | optim, optim_params, obj_name, seed, verbose, 81 | lb, ub, mode, n_workers, termination) 82 | self.classes_ = None # Initialize classes to None 83 | self.metric_class = ClassificationMetric # Set the metric class for evaluation 84 | 85 | def fit(self, X, y): 86 | """ 87 | Fits the model to the training data. 88 | 89 | Parameters 90 | ---------- 91 | X : array-like, shape (n_samples, n_features) 92 | Training data. 93 | y : array-like, shape (n_samples,) 94 | Target values. 95 | 96 | Returns 97 | ------- 98 | self : MhaMlpClassifier 99 | Returns the instance of the fitted model. 100 | """ 101 | ## Check the parameters 102 | self.size_input = X.shape[1] # Number of features 103 | y = np.squeeze(np.array(y)) # Convert y to a numpy array and squeeze dimensions 104 | if y.ndim != 1: 105 | y = np.argmax(y, axis=1) # Convert to 1D if it’s not already 106 | self.classes_ = np.unique(y) # Get unique classes from y 107 | if len(self.classes_) == 2: 108 | self.task = "binary_classification" # Set task for binary classification 109 | self.size_output = 1 # Output size for binary classification 110 | else: 111 | self.task = "classification" # Set task for multi-class classification 112 | self.size_output = len(self.classes_) # Output size for multi-class 113 | 114 | ## Process data 115 | X_tensor = torch.tensor(X, dtype=torch.float32) # Convert input data to tensor 116 | 117 | ## Build model 118 | self.build_model() # Build the model architecture 119 | 120 | ## Fit the data 121 | self._fit(X_tensor, y) # Fit the model 122 | 123 | return self # Return the fitted model 124 | 125 | def predict(self, X): 126 | """ 127 | Predicts the class labels for the provided input data. 128 | 129 | Parameters 130 | ---------- 131 | X : array-like, shape (n_samples, n_features) 132 | Input data for prediction. 133 | 134 | Returns 135 | ------- 136 | np.ndarray 137 | Predicted class labels for each sample. 138 | """ 139 | X_tensor = torch.tensor(X, dtype=torch.float32) # Convert input data to tensor 140 | self.network.eval() # Set model to evaluation mode 141 | with torch.no_grad(): 142 | output = self.network(X_tensor) # Get model predictions 143 | if self.task =="classification": # Multi-class classification 144 | _, predicted = torch.max(output, 1) 145 | else: # Binary classification 146 | predicted = (output > 0.5).int().squeeze() 147 | return predicted.numpy() # Return as a numpy array 148 | 149 | def score(self, X, y): 150 | """ 151 | Computes the accuracy score of the model based on predictions. 152 | 153 | Parameters 154 | ---------- 155 | X : array-like, shape (n_samples, n_features) 156 | Input data for scoring. 157 | y : array-like, shape (n_samples,) 158 | True labels for comparison. 159 | 160 | Returns 161 | ------- 162 | float 163 | Accuracy score of the model. 164 | """ 165 | y_pred = self.predict(X) # Get predictions 166 | return accuracy_score(y, y_pred) # Calculate and return accuracy score 167 | 168 | def predict_proba(self, X): 169 | """ 170 | Computes the probability estimates for each class (for classification tasks only). 171 | 172 | Parameters 173 | ---------- 174 | X : array-like, shape (n_samples, n_features) 175 | Input data for which to predict probabilities. 176 | 177 | Returns 178 | ------- 179 | np.ndarray 180 | Probability predictions for each class. 181 | 182 | Raises 183 | ------ 184 | ValueError 185 | If the task is not a classification task. 186 | """ 187 | X_tensor = torch.tensor(X, dtype=torch.float32) # Convert input data to tensor 188 | if self.task not in ["classification", "binary_classification"]: 189 | raise ValueError( 190 | "predict_proba is only available for classification tasks.") # Raise error if task is invalid 191 | 192 | self.network.eval() # Ensure model is in evaluation mode 193 | with torch.no_grad(): 194 | probs = self.network.forward(X_tensor) # Get the output from forward pass 195 | return probs.numpy() # Return probabilities as a numpy array 196 | 197 | def evaluate(self, y_true, y_pred, list_metrics=("AS", "RS")): 198 | """ 199 | Return the list of performance metrics on the given test data and labels. 200 | 201 | Parameters 202 | ---------- 203 | y_true : array-like of shape (n_samples,) or (n_samples, n_outputs) 204 | True values for `X`. 205 | 206 | y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs) 207 | Predicted values for `X`. 208 | 209 | list_metrics : list, default=("AS", "RS") 210 | List of metrics to compute using Permetrics library: 211 | https://github.com/thieu1995/permetrics 212 | 213 | Returns 214 | ------- 215 | results : dict 216 | A dictionary containing the results of the requested metrics. 217 | """ 218 | return self._evaluate_cls(y_true, y_pred, list_metrics) # Call evaluation method 219 | 220 | 221 | class MhaMlpRegressor(BaseMhaMlp, RegressorMixin): 222 | """ 223 | A Metaheuristic-based MLP Regressor that extends the BaseMhaMlp class and implements 224 | the RegressorMixin interface from Scikit-Learn for regression tasks. 225 | 226 | Attributes 227 | ---------- 228 | metric_class : type 229 | The metric class used for evaluating regression performance. 230 | 231 | Parameters 232 | ---------- 233 | hidden_layers : list of int, tuple of int, or np.ndarray of int 234 | The structure of the hidden layers (default is (100,)). 235 | act_names : str 236 | Activation function name to use in hidden layers (default is "ELU"). 237 | dropout_rates : float, optional 238 | Dropout rate for regularization (default is 0.2). 239 | act_output : any, optional 240 | Activation function for the output layer (default is None). 241 | optim : str 242 | The optimization algorithm to use (default is "BaseGA"). 243 | optim_params : dict 244 | Parameters for the optimizer (default is None). 245 | obj_name : str, optional 246 | The objective name for the optimization (default is "MSE"). 247 | seed : int, optional 248 | Random seed for reproducibility (default is 42). 249 | verbose : bool, optional 250 | Whether to print detailed logs during fitting (default is True). 251 | lb : int, float, tuple, list, np.ndarray, optional 252 | Lower bounds for weights and biases in network. 253 | ub : int, float, tuple, list, np.ndarray, optional 254 | Upper bounds for weights and biases in network. 255 | mode : str, optional 256 | Mode for optimization (default is 'single'). 257 | n_workers : int, optional 258 | Number of workers for parallel processing (default is None). 259 | termination : any, optional 260 | Termination criteria for optimization (default is None). 261 | """ 262 | 263 | def __init__(self, hidden_layers=(100,), act_names="ELU", dropout_rates=0.2, act_output=None, 264 | optim="BaseGA", optim_params=None, obj_name="MSE", seed=42, verbose=True, 265 | lb=None, ub=None, mode='single', n_workers=None, termination=None): 266 | """ 267 | Initializes the MhaMlpRegressor with specified parameters. 268 | """ 269 | super().__init__(hidden_layers, act_names, dropout_rates, act_output, 270 | optim, optim_params, obj_name, seed, verbose, 271 | lb, ub, mode, n_workers, termination) 272 | self.metric_class = RegressionMetric # Set the metric class for evaluation 273 | 274 | def fit(self, X, y): 275 | """ 276 | Fits the model to the training data. 277 | 278 | Parameters 279 | ---------- 280 | X : array-like, shape (n_samples, n_features) 281 | Training data. 282 | y : array-like, shape (n_samples,) or (n_samples, n_outputs) 283 | Target values. 284 | 285 | Returns 286 | ------- 287 | self : MhaMlpRegressor 288 | Returns the instance of the fitted model. 289 | """ 290 | ## Check the parameters 291 | self.size_input = X.shape[1] # Number of input features 292 | y = np.squeeze(np.array(y)) # Convert y to a numpy array and squeeze dimensions 293 | self.size_output = 1 # Default output size for single-output regression 294 | self.task = "regression" # Default task is regression 295 | 296 | if y.ndim == 2: 297 | self.task = "multi_regression" # Set task for multi-output regression 298 | self.size_output = y.shape[1] # Update output size for multi-output 299 | 300 | ## Process data 301 | X_tensor = torch.tensor(X, dtype=torch.float32) # Convert input data to tensor 302 | 303 | ## Build model 304 | self.build_model() # Build the model architecture 305 | 306 | ## Fit the data 307 | self._fit(X_tensor, y) 308 | 309 | return self # Return the fitted model 310 | 311 | def predict(self, X): 312 | """ 313 | Predicts the output values for the provided input data. 314 | 315 | Parameters 316 | ---------- 317 | X : array-like, shape (n_samples, n_features) 318 | Input data for prediction. 319 | 320 | Returns 321 | ------- 322 | np.ndarray 323 | Predicted output values for each sample. 324 | """ 325 | X_tensor = torch.tensor(X, dtype=torch.float32) # Convert input data to tensor 326 | self.network.eval() # Set model to evaluation mode 327 | with torch.no_grad(): 328 | predicted = self.network(X_tensor) # Get model predictions 329 | return predicted.numpy() # Return predictions as a numpy array 330 | 331 | def score(self, X, y): 332 | """ 333 | Computes the R2 score of the model based on predictions. 334 | 335 | Parameters 336 | ---------- 337 | X : array-like, shape (n_samples, n_features) 338 | Input data for scoring. 339 | y : array-like, shape (n_samples,) 340 | True labels for comparison. 341 | 342 | Returns 343 | ------- 344 | float 345 | R2 score of the model. 346 | """ 347 | y_pred = self.predict(X) # Get predictions 348 | return r2_score(y, y_pred) # Calculate and return R^2 score 349 | 350 | def evaluate(self, y_true, y_pred, list_metrics=("AS", "RS")): 351 | """ 352 | Return the list of performance metrics on the given test data and labels. 353 | 354 | Parameters 355 | ---------- 356 | y_true : array-like of shape (n_samples,) or (n_samples, n_outputs) 357 | True values for `X`. 358 | 359 | y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs) 360 | Predicted values for `X`. 361 | 362 | list_metrics : list, default=("AS", "RS") 363 | List of metrics to compute using Permetrics library: 364 | https://github.com/thieu1995/permetrics 365 | 366 | Returns 367 | ------- 368 | results : dict 369 | A dictionary containing the results of the requested metrics. 370 | """ 371 | return self._evaluate_reg(y_true, y_pred, list_metrics) # Call evaluation method 372 | -------------------------------------------------------------------------------- /metaperceptron/core/gradient_mlp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Created by "Thieu" at 09:52, 17/08/2023 ----------% 3 | # Email: nguyenthieu2102@gmail.com % 4 | # Github: https://github.com/thieu1995 % 5 | # --------------------------------------------------% 6 | 7 | import numpy as np 8 | import torch 9 | from torch.utils.data import TensorDataset, DataLoader 10 | from sklearn.model_selection import train_test_split 11 | from sklearn.metrics import accuracy_score, r2_score 12 | from sklearn.base import ClassifierMixin, RegressorMixin 13 | from metaperceptron.core.base_mlp import BaseStandardMlp 14 | 15 | 16 | class MlpClassifier(BaseStandardMlp, ClassifierMixin): 17 | """ 18 | Multi-layer Perceptron (MLP) Classifier that inherits from BaseStandardMlp and ClassifierMixin. 19 | 20 | Parameters 21 | ---------- 22 | hidden_layers : tuple, default=(100,) 23 | Defines the number of hidden layers and the units per layer in the network. 24 | 25 | act_names : str or list of str, default="ReLU" 26 | Activation function(s) for each layer. Can be a single activation name for all layers or a list of names. 27 | 28 | dropout_rates : float or list of float, default=0.2 29 | Dropout rates for each hidden layer to prevent overfitting. If a single float, the same rate is applied to all layers. 30 | 31 | act_output : str, default=None 32 | Activation function for the output layer. 33 | 34 | epochs : int, default=1000 35 | Number of training epochs. 36 | 37 | batch_size : int, default=16 38 | Batch size used in training. 39 | 40 | optim : str, default="Adam" 41 | Optimizer to use, selected from the supported optimizers. 42 | 43 | optim_params : dict, default=None 44 | Parameters for the optimizer, such as learning rate, beta values, etc. 45 | 46 | early_stopping : bool, default=True 47 | If True, training will stop early if validation loss does not improve. 48 | 49 | n_patience : int, default=10 50 | Number of epochs to wait for an improvement in validation loss before stopping. 51 | 52 | epsilon : float, default=0.001 53 | Minimum improvement in validation loss to continue training. 54 | 55 | valid_rate : float, default=0.1 56 | Fraction of data to use for validation. 57 | 58 | seed : int, default=42 59 | Seed for random number generation. 60 | 61 | verbose : bool, default=True 62 | If True, prints training progress and validation loss during training. 63 | 64 | device : str, default=None 65 | Device to use for training (e.g., "cpu" or "gpu"). If None, uses the default device. 66 | """ 67 | 68 | def __init__(self, hidden_layers=(100,), act_names="ReLU", dropout_rates=0.2, act_output=None, 69 | epochs=1000, batch_size=16, optim="Adam", optim_params=None, 70 | early_stopping=True, n_patience=10, epsilon=0.001, valid_rate=0.1, 71 | seed=42, verbose=True, device=None): 72 | # Call superclass initializer with the specified parameters. 73 | super().__init__(hidden_layers, act_names, dropout_rates, act_output, 74 | epochs, batch_size, optim, optim_params, 75 | early_stopping, n_patience, epsilon, valid_rate, 76 | seed, verbose, device) 77 | self.classes_ = None 78 | 79 | def _process_data(self, X, y, **kwargs): 80 | """ 81 | Prepares and processes data for training, including optional splitting into validation data. 82 | 83 | Parameters 84 | ---------- 85 | X : array-like, shape (n_samples, n_features) 86 | Training data. 87 | 88 | y : array-like, shape (n_samples,) 89 | Target values. 90 | 91 | Returns 92 | ------- 93 | tuple : (train_loader, X_valid_tensor, y_valid_tensor) 94 | Data loader for training data, and tensors for validation data (if specified). 95 | """ 96 | X_valid_tensor, y_valid_tensor, X_valid, y_valid = None, None, None, None 97 | 98 | # Split data into training and validation sets based on valid_rate 99 | if self.valid_rate is not None: 100 | if 0 < self.valid_rate < 1: 101 | # Activate validation mode if valid_rate is set between 0 and 1 102 | self.valid_mode = True 103 | X, X_valid, y, y_valid = train_test_split(X, y, test_size=self.valid_rate, 104 | random_state=self.seed, shuffle=True, stratify=y) 105 | else: 106 | raise ValueError("Validation rate must be between 0 and 1.") 107 | 108 | # Convert data to tensors and set up DataLoader 109 | X_tensor = torch.tensor(X, dtype=torch.float32).to(self.device) 110 | y_tensor = torch.tensor(y, dtype=torch.long).to(self.device) 111 | if self.task == "binary_classification": 112 | y_tensor = torch.tensor(y, dtype=torch.float32) 113 | y_tensor = torch.unsqueeze(y_tensor, 1).to(self.device) 114 | 115 | train_loader = DataLoader(TensorDataset(X_tensor, y_tensor), batch_size=self.batch_size, shuffle=True) 116 | 117 | if self.valid_mode: 118 | X_valid_tensor = torch.tensor(X_valid, dtype=torch.float32).to(self.device) 119 | y_valid_tensor = torch.tensor(y_valid, dtype=torch.long).to(self.device) 120 | if self.task == "binary_classification": 121 | y_valid_tensor = torch.tensor(y_valid, dtype=torch.float32) 122 | y_valid_tensor = torch.unsqueeze(y_valid_tensor, 1).to(self.device) 123 | 124 | return train_loader, X_valid_tensor, y_valid_tensor 125 | 126 | def fit(self, X, y, **kwargs): 127 | """ 128 | Trains the MLP model on the provided data. 129 | 130 | Parameters 131 | ---------- 132 | X : array-like, shape (n_samples, n_features) 133 | Training data. 134 | 135 | y : array-like, shape (n_samples,) 136 | Target values. 137 | 138 | Returns 139 | ------- 140 | self : object 141 | Fitted classifier. 142 | """ 143 | # Set input and output sizes based on data and initialize task 144 | self.size_input = X.shape[1] 145 | y = np.squeeze(np.array(y)) 146 | if y.ndim != 1: 147 | y = np.argmax(y, axis=1) 148 | 149 | self.classes_ = np.unique(y) 150 | if len(self.classes_) == 2: 151 | self.task = "binary_classification" 152 | self.size_output = 1 153 | else: 154 | self.task = "classification" 155 | self.size_output = len(self.classes_) 156 | 157 | # Process data for training and validation 158 | data = self._process_data(X, y, **kwargs) 159 | 160 | # Build the model architecture 161 | self.build_model() 162 | 163 | # Train the model using processed data 164 | self._fit(data, **kwargs) 165 | 166 | return self 167 | 168 | def predict(self, X): 169 | """ 170 | Predicts the class labels for the given input data. 171 | 172 | Parameters 173 | ---------- 174 | X : array-like, shape (n_samples, n_features) 175 | Input data. 176 | 177 | Returns 178 | ------- 179 | numpy.ndarray 180 | Predicted class labels for each sample. 181 | """ 182 | X_tensor = torch.tensor(X, dtype=torch.float32).to(self.device) 183 | self.network.eval() 184 | with torch.no_grad(): 185 | output = self.network(X_tensor) # Get model predictions 186 | if self.task == "classification": # Multi-class classification 187 | _, predicted = torch.max(output, 1) 188 | else: # Binary classification 189 | predicted = (output > 0.5).int().squeeze() 190 | return predicted.cpu().numpy() 191 | 192 | def score(self, X, y): 193 | """ 194 | Computes the accuracy score for the classifier. 195 | 196 | Parameters 197 | ---------- 198 | X : array-like, shape (n_samples, n_features) 199 | Input data. 200 | 201 | y : array-like, shape (n_samples,) 202 | True class labels. 203 | 204 | Returns 205 | ------- 206 | float 207 | Accuracy score of the classifier. 208 | """ 209 | y_pred = self.predict(X) 210 | return accuracy_score(y, y_pred) 211 | 212 | def predict_proba(self, X): 213 | """ 214 | Computes the probability estimates for each class (for classification tasks only). 215 | 216 | Parameters 217 | ---------- 218 | X : array-like, shape (n_samples, n_features) 219 | Input data. 220 | 221 | Returns 222 | ------- 223 | numpy.ndarray 224 | Probability predictions for each class. 225 | """ 226 | X_tensor = torch.tensor(X, dtype=torch.float32).to(self.device) 227 | if self.task not in ["classification", "binary_classification"]: 228 | raise ValueError("predict_proba is only available for classification tasks.") 229 | 230 | self.network.eval() # Set model to evaluation mode 231 | with torch.no_grad(): 232 | probs = self.network.forward(X_tensor) # Forward pass to get probability estimates 233 | 234 | return probs.cpu().numpy() # Return as numpy array 235 | 236 | def evaluate(self, y_true, y_pred, list_metrics=("AS", "RS")): 237 | """ 238 | Returns performance metrics for the model on the provided test data. 239 | 240 | Parameters 241 | ---------- 242 | y_true : array-like of shape (n_samples,) 243 | True class labels. 244 | 245 | y_pred : array-like of shape (n_samples,) 246 | Predicted class labels. 247 | 248 | list_metrics : list, default=("AS", "RS") 249 | List of performance metrics to calculate. Refer to Permetrics (https://github.com/thieu1995/permetrics) library for available metrics. 250 | 251 | Returns 252 | ------- 253 | dict 254 | Dictionary with results for the specified metrics. 255 | """ 256 | return self._evaluate_cls(y_true=y_true, y_pred=y_pred, list_metrics=list_metrics) 257 | 258 | 259 | class MlpRegressor(BaseStandardMlp, RegressorMixin): 260 | """ 261 | Multi-Layer Perceptron (MLP) Regressor for predicting continuous values. 262 | 263 | Parameters 264 | ---------- 265 | hidden_layers : tuple, optional 266 | A tuple indicating the number of neurons in each hidden layer (default is (100,)). 267 | act_names : str or list of str, optional 268 | Activation function(s) for hidden layers (default is "ELU"). 269 | dropout_rates : float or list of float, optional 270 | Dropout rate(s) for regularization (default is 0.2). 271 | act_output : str, optional 272 | Activation function for the output layer (default is None). 273 | epochs : int, optional 274 | Number of epochs for training (default is 1000). 275 | batch_size : int, optional 276 | Size of the mini-batch during training (default is 16). 277 | optim : str, optional 278 | Optimization algorithm (default is "Adam"). 279 | optim_params : dict, optional 280 | Additional parameters for the optimizer (default is None). 281 | early_stopping : bool, optional 282 | Flag to enable early stopping during training (default is True). 283 | n_patience : int, optional 284 | Number of epochs to wait for improvement before stopping (default is 10). 285 | epsilon : float, optional 286 | Tolerance for improvement (default is 0.001). 287 | valid_rate : float, optional 288 | Proportion of data to use for validation (default is 0.1). 289 | seed : int, optional 290 | Random seed for reproducibility (default is 42). 291 | verbose : bool, optional 292 | Flag to enable verbose output during training (default is True). 293 | device : str, optional 294 | Device to use for training (default is None, which uses the default device). 295 | """ 296 | 297 | def __init__(self, hidden_layers=(100,), act_names="ELU", dropout_rates=0.2, act_output=None, 298 | epochs=1000, batch_size=16, optim="Adam", optim_params=None, 299 | early_stopping=True, n_patience=10, epsilon=0.001, valid_rate=0.1, 300 | seed=42, verbose=True, device=None): 301 | super().__init__(hidden_layers, act_names, dropout_rates, act_output, 302 | epochs, batch_size, optim, optim_params, 303 | early_stopping, n_patience, epsilon, valid_rate, 304 | seed, verbose, device) 305 | 306 | def _process_data(self, X, y, **kwargs): 307 | """ 308 | Prepares the input data for training and validation by converting it to tensors 309 | and creating DataLoaders for batch processing. 310 | 311 | Parameters 312 | ---------- 313 | X : array-like 314 | Input features for the regression task. 315 | y : array-like 316 | Target values for the regression task. 317 | 318 | Returns 319 | ------- 320 | tuple 321 | A tuple containing the training DataLoader and optional validation tensors. 322 | """ 323 | X_valid_tensor, y_valid_tensor, X_valid, y_valid = None, None, None, None 324 | 325 | # Check for validation rate and split the data if applicable 326 | if self.valid_rate is not None: 327 | if 0 < self.valid_rate < 1: 328 | # Split data into training and validation sets 329 | self.valid_mode = True 330 | X, X_valid, y, y_valid = train_test_split(X, y, test_size=self.valid_rate, 331 | random_state=self.seed, shuffle=True) 332 | else: 333 | raise ValueError("Validation rate must be between 0 and 1.") 334 | 335 | # Convert training data to tensors 336 | X_tensor = torch.tensor(X, dtype=torch.float32).to(self.device) 337 | y_tensor = torch.tensor(y, dtype=torch.float32).to(self.device) 338 | 339 | # Ensure the target tensor has correct dimensions 340 | if y_tensor.ndim == 1: 341 | y_tensor = y_tensor.unsqueeze(1) 342 | 343 | # Create DataLoader for training data 344 | train_loader = DataLoader(TensorDataset(X_tensor, y_tensor), batch_size=self.batch_size, shuffle=True) 345 | 346 | # Process validation data if in validation mode 347 | if self.valid_mode: 348 | X_valid_tensor = torch.tensor(X_valid, dtype=torch.float32).to(self.device) 349 | y_valid_tensor = torch.tensor(y_valid, dtype=torch.float32).to(self.device) 350 | if y_valid_tensor.ndim == 1: 351 | y_valid_tensor = y_valid_tensor.unsqueeze(1) 352 | 353 | return train_loader, X_valid_tensor, y_valid_tensor 354 | 355 | def fit(self, X, y, **kwargs): 356 | """ 357 | Fits the MLP model to the provided training data. 358 | 359 | Parameters 360 | ---------- 361 | X : array-like 362 | Input features for training. 363 | y : array-like 364 | Target values for training. 365 | 366 | Returns 367 | ------- 368 | self : MlpRegressor 369 | Returns the instance of the fitted model. 370 | """ 371 | # Check and prepare the input parameters 372 | self.size_input = X.shape[1] 373 | y = np.squeeze(np.array(y)) 374 | self.size_output = 1 # Default output size for regression 375 | self.task = "regression" # Default task is regression 376 | 377 | if y.ndim == 2: 378 | # Adjust task if multi-dimensional target is provided 379 | self.task = "multi_regression" 380 | self.size_output = y.shape[1] 381 | 382 | # Process data to prepare DataLoader and tensors 383 | data = self._process_data(X, y, **kwargs) 384 | 385 | # Build the MLP model 386 | self.build_model() 387 | 388 | # Fit the model to the training data 389 | self._fit(data, **kwargs) 390 | 391 | return self 392 | 393 | def predict(self, X): 394 | """ 395 | Predicts the target values for the given input features. 396 | 397 | Parameters 398 | ---------- 399 | X : array-like 400 | Input features for prediction. 401 | 402 | Returns 403 | ------- 404 | numpy.ndarray 405 | Predicted values for the input features. 406 | """ 407 | # Convert input features to tensor 408 | X_tensor = torch.tensor(X, dtype=torch.float32).to(self.device) 409 | self.network.eval() # Set model to evaluation mode 410 | with torch.no_grad(): 411 | predicted = self.network(X_tensor) # Forward pass to get predictions 412 | return predicted.cpu().numpy() # Convert predictions to numpy array 413 | 414 | def score(self, X, y): 415 | """ 416 | Computes the R2 score of the predictions. 417 | 418 | Parameters 419 | ---------- 420 | X : array-like 421 | Input features for scoring. 422 | y : array-like 423 | True target values for the input features. 424 | 425 | Returns 426 | ------- 427 | float 428 | R2 score indicating the model's performance. 429 | """ 430 | y_pred = self.predict(X) # Obtain predictions 431 | return r2_score(y, y_pred) # Calculate and return R^2 score 432 | 433 | def evaluate(self, y_true, y_pred, list_metrics=("MSE", "MAE")): 434 | """ 435 | Returns a list of performance metrics for the predictions. 436 | 437 | Parameters 438 | ---------- 439 | y_true : array-like of shape (n_samples,) or (n_samples, n_outputs) 440 | True values for the input features. 441 | y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs) 442 | Predicted values for the input features. 443 | list_metrics : list, default=("MSE", "MAE") 444 | List of metrics to evaluate (can be from Permetrics library: https://github.com/thieu1995/permetrics). 445 | 446 | Returns 447 | ------- 448 | results : dict 449 | A dictionary containing the results of the specified metrics. 450 | """ 451 | return self._evaluate_reg(y_true, y_pred, list_metrics) # Call the evaluation method 452 | --------------------------------------------------------------------------------