├── .gitignore ├── LICENSE ├── README.md ├── docs ├── README.md ├── appendix │ ├── tips.md │ └── tips_md │ │ └── example.jpg ├── build_docs.sh ├── ds │ ├── brn.md │ ├── overview.md │ ├── rn.md │ └── usecases.md ├── formatting │ └── extra.css ├── index.md ├── install │ └── install.md ├── reference │ ├── models.md │ ├── modules.md │ ├── nn.md │ └── utils.md ├── static │ ├── torchlogic_logo.png │ └── torchlogic_logo_black.png └── tutorials │ └── brn.md ├── experiments ├── NRN_Paper_Plots_AAAI.ipynb ├── README.md ├── explain.py ├── explain_global.py ├── explain_nam.py ├── incremental_deletion_RF.py ├── incremental_deletion_RF_SHAP.py ├── incremental_deletion_RNRN.py ├── incremental_deletion_nam.py ├── main.py ├── nam_shap_size.py ├── poetry.lock ├── pyproject.toml ├── query_openml.py └── src │ ├── BRCG.py │ ├── LEURN │ ├── Causality_Example.png │ ├── DATA.py │ ├── DEMO.py │ ├── LEURN.py │ ├── LICENSE │ ├── Presentation_Product.pdf │ ├── Presentation_Technical.pdf │ ├── README.md │ ├── TRAINER.py │ ├── UI.py │ ├── credit_scoring.csv │ └── requirements.txt │ ├── __init__.py │ ├── cofrnet.py │ ├── danet │ ├── DAN_Task.py │ ├── Figures │ │ └── DAN.jpg │ ├── LICENSE │ ├── README.md │ ├── __init__.py │ ├── abstract_model.py │ ├── config │ │ ├── MSLR.yaml │ │ ├── cardio.yaml │ │ ├── click.yaml │ │ ├── default.py │ │ ├── epsilon.yaml │ │ ├── forest_cover_type.yaml │ │ ├── yahoo.yaml │ │ └── year.yaml │ ├── data │ │ ├── data_util.py │ │ └── dataset.py │ ├── main.py │ ├── model │ │ ├── AcceleratedModule.py │ │ ├── DANet.py │ │ ├── __init__.py │ │ └── sparsemax │ │ │ ├── __init__.py │ │ │ └── sparsemax.py │ ├── predict.py │ └── requirements.txt │ ├── danet_model.py │ ├── datasets.py │ ├── difflogic │ ├── INSTALLATION_SUPPORT.md │ ├── LICENSE │ ├── README.md │ ├── difflogic │ │ ├── __init__.py │ │ ├── compiled_model.py │ │ ├── cuda │ │ │ ├── difflogic.cpp │ │ │ └── difflogic_kernel.cu │ │ ├── difflogic.py │ │ ├── functional.py │ │ └── packbitstensor.py │ ├── difflogic_logo.png │ ├── experiments │ │ ├── apply_compiled_net.py │ │ ├── main.py │ │ ├── main_baseline.py │ │ ├── mnist_dataset.py │ │ ├── requirements.txt │ │ ├── results_json.py │ │ └── uci_datasets.py │ └── setup.py │ ├── difflogic_model.py │ ├── encoders.py │ ├── fttransformer.py │ ├── iris_datasets.py │ ├── mlp.py │ ├── nam.py │ ├── neural_additive_models │ ├── README.md │ ├── __init__.py │ ├── data_utils.py │ ├── graph_builder.py │ ├── models.py │ ├── nam_train.py │ ├── nam_train_test.py │ ├── requirements.txt │ ├── run.sh │ ├── setup.py │ └── tests │ │ ├── __init__.py │ │ ├── data_utils_test.py │ │ ├── graph_builder_test.py │ │ └── models_test.py │ └── tuners.py ├── mkdocs.yml ├── notebooks └── tutorials │ ├── Bandit-NRN Tutorial - Binary-Class.ipynb │ ├── Bandit-NRN Tutorial - Domain Knowledge.ipynb │ ├── Bandit-NRN Tutorial - Multi-Class.ipynb │ ├── Bandit-NRN Tutorial - Regression.ipynb │ └── configs │ └── logging.yaml ├── poetry.lock ├── project.yaml ├── pyproject.toml ├── setup.py ├── tests ├── README.md ├── models │ ├── test__boosted_brrn.py │ ├── test__brrn.py │ ├── test_mixin_classifier.py │ └── test_mixin_regressor.py ├── modules │ ├── test_attn.py │ ├── test_brnn.py │ └── test_var.py ├── nn │ ├── test_base_blocks.py │ ├── test_base_predicates.py │ ├── test_base_utils.py │ ├── test_forward_blocks.py │ └── test_forward_utils.py ├── sklogic │ ├── test__base__estimator.py │ ├── test__rnrn_classifier.py │ └── test__rnrn_regressor.py └── utils │ ├── test_base_trainer.py │ ├── test_boosted_brrn_trainer.py │ ├── test_brrn_trainer.py │ ├── test_explanations.py │ └── test_simplification.py └── torchlogic ├── __init__.py ├── models ├── __init__.py ├── attn_classifier.py ├── attn_regressor.py ├── base │ ├── __init__.py │ ├── attn.py │ ├── boosted_brn.py │ ├── brn.py │ ├── pruningrn.py │ ├── rn.py │ └── var.py ├── brn_classifier.py ├── brn_regressor.py ├── mixins │ ├── __init__.py │ ├── _base_mixin.py │ ├── classifier.py │ └── regressor.py ├── var_classifier.py └── var_regressor.py ├── modules ├── __init__.py ├── attn.py ├── brn.py └── var.py ├── nn ├── __init__.py ├── base │ ├── __init__.py │ ├── _core.py │ ├── blocks.py │ ├── constants.py │ ├── predicates.py │ └── utils.py ├── blocks.py ├── predicates.py └── utils.py ├── sklogic ├── __init__.py ├── base │ ├── __init__.py │ └── base_estimator.py ├── classifiers │ ├── RNRNClassifier.py │ └── __init__.py ├── datasets │ ├── __init__.py │ └── simple_dataset.py └── regressors │ ├── RNRNRegressor.py │ └── __init__.py └── utils ├── __init__.py ├── distributed.py ├── explanations ├── __init__.py ├── explanations.py └── simplification.py ├── operations.py └── trainers ├── __init__.py ├── attnnrntrainer.py ├── banditnrntrainer.py ├── base ├── __init__.py └── basetrainer.py └── boostedbanditnrntrainer.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Mac 2 | .DS_Store 3 | 4 | # Editors 5 | .vscode/ 6 | .vs/ 7 | 8 | # MLflow 9 | mlruns/ 10 | 11 | # data and credentials directories have .gitignore in their directory 12 | 13 | # ==================== GitHub Defaults ===================== # 14 | # Byte-compiled / optimized / DLL files 15 | __pycache__/ 16 | *.py[cod] 17 | *$py.class 18 | 19 | # C extensions 20 | *.so 21 | 22 | # Distribution / packaging 23 | .Python 24 | build/ 25 | develop-eggs/ 26 | dist/ 27 | downloads/ 28 | eggs/ 29 | .eggs/ 30 | lib/ 31 | lib64/ 32 | parts/ 33 | sdist/ 34 | var/ 35 | wheels/ 36 | share/python-wheels/ 37 | *.egg-info/ 38 | .installed.cfg 39 | *.egg 40 | MANIFEST 41 | 42 | # PyInstaller 43 | # Usually these files are written by a python script from a template 44 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 45 | *.manifest 46 | *.spec 47 | 48 | # Installer logs 49 | pip-log.txt 50 | pip-delete-this-directory.txt 51 | 52 | # Unit test / coverage reports 53 | htmlcov/ 54 | .tox/ 55 | .nox/ 56 | .coverage 57 | .coverage.* 58 | .cache 59 | nosetests.xml 60 | coverage.xml 61 | *.cover 62 | *.py,cover 63 | .hypothesis/ 64 | .pytest_cache/ 65 | cover/ 66 | 67 | # Translations 68 | *.mo 69 | *.pot 70 | 71 | # Django stuff: 72 | *.log 73 | local_settings.py 74 | db.sqlite3 75 | db.sqlite3-journal 76 | 77 | # Flask stuff: 78 | instance/ 79 | .webassets-cache 80 | 81 | # Scrapy stuff: 82 | .scrapy 83 | 84 | # Sphinx documentation 85 | docs/_build/ 86 | 87 | # PyBuilder 88 | .pybuilder/ 89 | target/ 90 | 91 | # Jupyter Notebook 92 | .ipynb_checkpoints 93 | 94 | # IPython 95 | profile_default/ 96 | ipython_config.py 97 | 98 | # pyenv 99 | # For a library or package, you might want to ignore these files since the code is 100 | # intended to run in multiple environments; otherwise, check them in: 101 | # .python-version 102 | 103 | # pipenv 104 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 105 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 106 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 107 | # install all needed dependencies. 108 | #Pipfile.lock 109 | 110 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 111 | __pypackages__/ 112 | 113 | # Celery stuff 114 | celerybeat-schedule 115 | celerybeat.pid 116 | 117 | # SageMath parsed files 118 | *.sage.py 119 | 120 | # Environments 121 | .env 122 | .venv 123 | env/ 124 | venv/ 125 | ENV/ 126 | env.bak/ 127 | venv.bak/ 128 | 129 | # Spyder project settings 130 | .spyderproject 131 | .spyproject 132 | 133 | # Rope project settings 134 | .ropeproject 135 | 136 | # mkdocs documentation 137 | /site 138 | 139 | # mypy 140 | .mypy_cache/ 141 | .dmypy.json 142 | dmypy.json 143 | 144 | # Pyre type checker 145 | .pyre/ 146 | 147 | # pytype static type analyzer 148 | .pytype/ 149 | 150 | # Cython debug symbols 151 | cython_debug/ 152 | 153 | .idea/* 154 | .pytest_cache/* 155 | 156 | # poetry 157 | dist/* -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [//]: # (![Coverage Report](./assets/coverage.svg)) 2 | 3 | ![](./static/torchlogic_logo.png) 4 | 5 | # TORCHLOGIC DOCUMENTATION 6 | 7 | _torchlogic_ is a pytorch framework for developing Neuro-Symbolic AI systems 8 | based on [Weighted Lukasiewicz Logic](https://arxiv.org/abs/2006.13155) that we denote as _Neural Reasoning Networks_. 9 | The design principles of the _torchlogic_ provide computational efficiency for Neuro-Symbolic AI through 10 | GPU scaling. 11 | 12 | ### Design Principles 13 | 14 | - _Neural == Symbolic_: Symbolic operations should not deviate computationally from Neural operations and leverage PyTorch directly 15 | - _Masked Tensors_: Reasoning Networks use tensors and masking to represent any logical structure _and_ leverage GPU optimized computations 16 | - _Neural -> Symbolic Extension_: Symbolic operations in _torchlogic_ are PyTorch Modules and can therefore integrate with existing Deep Neural Networks seamlessly 17 | 18 | With these principles, _torchlogic_ and Neural Reasoning Networks are able 19 | to extend and integrate with our current state-of-the-art technologies that leverage advances in 20 | Deep Learning. Neural Reasoning Networks developed with _torchlogic_ can scale with 21 | multi-GPU support. Finally, those familiar with PyTorch development principles will have only a small step 22 | in skill building to develop with _torchlogic_. 23 | 24 | ### Documentation 25 | 26 | The API reference and additional documentation for torchlogic are available 27 | on through the site. 28 | The current code is in an Alpha state so there may be bugs and the functionality 29 | is expanding quickly. We'll do our best to keep the documentation up to date 30 | with the latest changes to our API reflected there. 31 | 32 | ### Tutorial 33 | 34 | There are several tutorials demonstrating how to use the R-NRN algorithm 35 | in multiple use cases. 36 | 37 | [Tutorial Source](./tutorials/brrn.md) 38 | 39 | ### Data Science 40 | 41 | To understand the basic of the torchlogic framework and Neural Reasoning Networks 42 | check out the [Data Science](./ds/rn.md) section, which gives an introduction to some of the 43 | models developed so far using torchlogic. 44 | 45 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # `docs/` 2 | A directory for storing all documentation. 3 | 4 | ## Quick Start 5 | To start, install the required and recommended libraries. 6 | 7 | ```bash 8 | pip install mkdocs 9 | pip install mkdocs-git-revision-date-plugin 10 | pip install pymdown-extensions 11 | pip install mkdocs-material 12 | pip install mkdocs-autorefs 13 | ``` 14 | 15 | In your root folder, type the following command to preview the documentation locally. 16 | 17 | ``` 18 | mkdocs serve 19 | ``` 20 | Ensure that your main branch has atleast one commit in order for the above command to work. 21 | 22 | Open up http://127.0.0.1:8000/ in your browser, and you'll see the default home page being displayed. -------------------------------------------------------------------------------- /docs/appendix/tips.md: -------------------------------------------------------------------------------- 1 | # Tips 2 | Are you here for some tips on writing? Here are some resources and formatting tricks you can play around with markdown writing! 3 | 4 | ## Markdown Tricks 5 | ### Make use of the built-in navigation 6 | Look at the right side of the page. When you have multiple sections in a page, you can see the section break downs on the right. It comes for free! 7 | 8 | ### Reference 9 | 10 | Anything within the `docs/` folder are cross-reference-able. 11 | 12 | For example, [click me](../arch/index.md) (`[click me](../arch/index.md)`)will take you to the Architecture page. 13 | 14 | 15 | ### Attachment 16 | 17 | Similarly, this is also how you include a picture. 18 | 19 | !!! example Include a Cat Picture 20 | ``` 21 | ![cat](tips_md/example.jpg) 22 | ``` 23 | 24 | ![cat](tips_md/example.jpg) 25 | 26 | 27 | ## Admonition 28 | !!! tip "" 29 | Admonition is powerful to make your documentation vivid. 30 | 31 | ??? success 32 | And it's cool! 33 | 34 | Check [here](https://squidfunk.github.io/mkdocs-material/reference/admonitions/) for a full list of supported admonition. 35 | -------------------------------------------------------------------------------- /docs/appendix/tips_md/example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/torchlogic/20d10b0462b6866f3853336ef3cfad19218dcb18/docs/appendix/tips_md/example.jpg -------------------------------------------------------------------------------- /docs/build_docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # -------ATTENTION: Remove this block after thourough testing ------------ # 4 | # Please DO NOT run this template directly. 5 | # Double check each line and make sure it's consistent with your project 6 | # setting before running the script. 7 | # When you are ready to use the script, change the permission of this file 8 | # to executable so that it can be picked up by Travis. 9 | # e.g. git update-index --chmod=+x docs/build_docs.sh 10 | # ----------------------------------------------------------------------- # 11 | 12 | pip install mkdocs 13 | pip install mkdocs-git-revision-date-plugin 14 | pip install pymdown-extensions 15 | pip install mkdocs-material 16 | pip install mkdocs-autorefs 17 | 18 | git config user.name Travis; 19 | git config user.email your@email.com; 20 | git remote add gh-token https://${GITHUB_TOKEN}@github..com/.git; 21 | git fetch gh-token && git fetch gh-token gh-pages:gh-pages; 22 | mkdocs gh-deploy -v --clean --force --remote-name gh-token; 23 | -------------------------------------------------------------------------------- /docs/ds/overview.md: -------------------------------------------------------------------------------- 1 | # torchlogic Overview 2 | 3 | The algorithms implemented in torchlogic aims to learn a set of 4 | logical rules from data that will classify, or rank, depending on the loss function. 5 | The core technology to this algorithm is 6 | [Weighted Lukasiewicz Logic](https://arxiv.org/abs/2006.13155), which enables a Data Scientist 7 | to define logic that can be used for a prediction task, and leverage data to learn weights 8 | for that logic that minimize loss on that prediction task. 9 | 10 | Leveraging this technology, we develop learning 11 | algorithms that mines useful rules from raw data. The algorithm 12 | implemented thus far is Bandit-Reinforced Neural Reasoning Network 13 | (R-NRN). 14 | 15 | NRNs are a fully explainable AI approach that enables the seamless integration between 16 | expert knowledge (e.g. sellers, sales leaders etc.) and patterns learned 17 | from data so that those who use its predictions can understand the predictive 18 | logic, learn from patterns detected in data, and influence future predictions by 19 | adding their own knowledge. Some key properties of this algorithm that make it especially suited to 20 | enterprise application are as follows: 21 | 22 | - Transparency: The logic learned by the algorithm is fully transparent. It can be inspected, and for complex logic one can control the level of detail to surface during inspection -- limiting the exposed logic by order of importance. 23 | - Domain Knowledge: The algorithm can be easily extended to make use of expert knowledge. This knowledge can be encoded in the RN and used for predictions. If the knowledge is not useful, it will be forgotten. 24 | - Control: Enterprise application of AI often involve use of business rules. Similar to included expert knowledge, one can introduce business rules to the RN in the form of encoded logic, and by fixing the level of importance for those rules, ensure that the model will obey them. 25 | - Scalability: The algorithm performs well at all scales of data. If you have 100 samples or 1M samples, 10 features or 10K features, the algorithm can effectively learn to classify. 26 | - Performance: In many experiments, the algorithms performance can exceed that of traditional ML methods such as Random Forest by 5%-10%. -------------------------------------------------------------------------------- /docs/ds/rn.md: -------------------------------------------------------------------------------- 1 | # Neural Reasoning Networks (NRN) 2 | 3 | Reasoning Networks (NRN) provides the building 4 | blocks from which logical rules can be learned. The current version of RN 5 | consists of 4 logical operators: 6 | 1. `And` 7 | 2. `Or` 8 | 3. `Not` 9 | 4. `Predicate` 10 | 11 | !!! info "Truth value table for AND and OR" 12 | 13 | | P | Q | AND | OR | 14 | |-----|-----|-----|-----| 15 | | T | T | T | T | 16 | | T | F | F | T | 17 | | F | T | F | T | 18 | | F | F | F | F | 19 | 20 | These logical operations are familiar to most, except for Predicate. Within the RN 21 | framework, Predicates can be understood as being analogous to a column of data. They 22 | represent some information about the known state of the world. For example, if we 23 | are classifying cats and dogs the predicate `BARKS` and `MEOWS`, where the truth value 24 | for any particular sample is `True` or `False`, would be quite useful. 25 | 26 | ### Toy Example: 27 | 28 | For example, if we would like to classify samples as cats or dogs, one could create an RN as follows 29 | supposing we have the data below. 30 | 31 | | MEOWS | BARKS | HAS FUR | IS CAT | 32 | |--------|-------|---------|--------| 33 | | T | F | T | T | 34 | | T | F | F | T | 35 | | F | T | T | F | 36 | | T | T | T | F | 37 | 38 | !!! example "RN Logic for Cat/Dog Classification" 39 | - `CLASS CAT: AND(MEOWS, NOT(BARKS))` 40 | - `CLASS DOG: AND(OR(MEOWS, BARKS), HAS FUR)` 41 | 42 | The Data Scientist can construct this model directly and proceed to use it to 43 | make predictions. Passing our data through the logic will result in the logic 44 | for `CLASS CAT` evaluating to `True` for cats and `False` for dogs. `CLASS DOG` 45 | logic will evaluate to `True` for dogs and `False` for cats. Since this problem 46 | is a binary classification we could use either `CLASS CAT` or `CLASS DOG` logic, 47 | but the example shows how RN can be used for multi-class and multi-label 48 | problems by created logic for each class. 49 | 50 | Each logical operation in an RN contains weights, which can be changed 51 | using backpropogation and optimized via gradient descent to identify a 52 | weighted logic that will minimize the loss for the classification problem. 53 | For example, the predicate `BARKS` in our data above is perfectly correlated 54 | to our target `IS CAT`. Thus, optimizing the weights for the `CLASS CAT` 55 | logic `AND(MEOWS, NOT(BARKS))` might result in the following weights for 56 | `MEOWS` AND `NOT(BARKS)` in the `AND` node: 57 | 58 | !!! example "Learned RN Weights" 59 | - `AND(MEOWS, NOT(BARKS))` 60 | - `WEIGHTS: [0.5, 5.0]` 61 | - `BIAS: 1.0` 62 | 63 | The relatively high weight on the `NOT(BARKS)` predicate indicates that it is 64 | more important to the truth of the `IS CAT` logic then the `MEOWS` 65 | predicate. This makes sense, given that we have one instance where a 66 | dog both barks and meows, but a cat never barks. -------------------------------------------------------------------------------- /docs/ds/usecases.md: -------------------------------------------------------------------------------- 1 | # Use Cases 2 | 3 | As previously mentioned, Neural Reasoning Networks are especially suited 4 | to achieve the following benefits: 5 | 6 | - Transparency: The logic learned by the algorithm is fully transparent. It can be inspected, and for complex logic one can control the level of detail to surface during inspection -- limiting the exposed logic by order of importance. 7 | - Domain Knowledge: The algorithm can be easily extended to make use of expert knowledge. This knowledge can be encoded in the RN and used for predictions. If the knowledge is not useful, it will be forgotten. 8 | - Control: Enterprise application of AI often involve use of business rules. Similar to included expert knowledge, one can introduce business rules to the RN in the form of encoded logic, and by fixing the level of importance for those rules, ensure that the model will obey them. 9 | - Scalability: The algorithm performs well at all scales of data. If you have 100 samples or 1M samples, 10 features or 10K features, the algorithm can effectively learn to classify. 10 | - Performance: In many experiments, the algorithms performance can exceed that of traditional ML methods such as Random Forest by 5%-10%. 11 | 12 | These benefits lend themselves to certain use cases within the AI workflow. 13 | Below are examples of when and how Reasoning Networks might be used. 14 | This list is certainly not exhaustive. 15 | 16 | ## Use Case 1: Data Understanding 17 | 18 | NRNs should likely not serve as an initial baseline model due to their 19 | relatively higher difficulty in training compared to easily trained models such 20 | as Random Forest. However, they may be quite useful during model development 21 | as a tool to understand ones data and debug a system. 22 | 23 | NRNs unique model explanation capabilities can enable a Data Scientist 24 | to train a model and understand directly how the data is used to generate 25 | predictions. For example, one can gain some basic understanding of 26 | how to classify tumors from data by inspecting a trained NRN. 27 | 28 | !!! Example "Benign Class" 29 | - Predicted Value: 0.7 30 | 31 | - The patient is in the benign class because: 32 | - ANY of the following are TRUE: 33 | - ALL of the following are TRUE: 34 | - the concave points error is greater than 0.037, 35 | - AND the worst perimeter is NOT greater than 61.855, 36 | - OR ALL of the following are TRUE: 37 | - the mean radius is NOT greater than 12.073, 38 | - AND the mean concavity is NOT greater than 0.195 39 | 40 | !!! Example "Malignant Class" 41 | - Predicted Value: 0.3 42 | 43 | - The patient is in the negative of benign class because: 44 | - ANY of the following are TRUE: 45 | - ALL of the following are TRUE: 46 | - the concave points error is NOT greater than 0.033, 47 | - AND the worst perimeter is greater than 64.666, 48 | - AND the worst symmetry is NOT greater than 0.374, 49 | - OR ALL of the following are TRUE: 50 | - the mean radius is greater than 13.214, 51 | - AND the mean concavity is greater than 0.239 52 | 53 | Without any prior knowledge of the data, one can begin to understand that 54 | tumors with larger perimeters, a larger radius, lower symmetry and more 55 | concavity are likely to be tumors. Furthermore, we can see some of the 56 | critical boundaries in the data for predictions that are fairly confident 57 | of either classification. 58 | 59 | ## Use Case 2: End User Transparency 60 | 61 | In many industries, such as Business, Healthcare, Finance, or Law, the 62 | ability for end users to understand a model's behavior is at the very least 63 | a motivator for adopting model predictions, and in many cases a requirement 64 | for using a model all together. 65 | 66 | Reinforced Reasoning Networks enable Data Scientists to produce sample 67 | level explanations of the model's predictions, which can be used to 68 | directly understand the context of the prediction, increasing trust. 69 | 70 | !!! Example "Malignant Class" 71 | - Sample 0: The patient was in the **negative** of benign class because: 72 | - ANY of the following are TRUE: 73 | - ALL of the following are TRUE: 74 | - the concave points error is NOT greater than 0.05279, 75 | - AND the worst perimeter is greater than 56.23291, 76 | - AND the worst symmetry is NOT greater than 0.6638, 77 | - OR ALL of the following are TRUE: 78 | - the mean radius is greater than 14.270505, 79 | - AND the perimeter error is greater than 3.346206, 80 | - AND the mean concavity is greater than 0.3000404 81 | 82 | In the case above, a Doctor using this model might review the 83 | proposed classification and its reasoning and decide if the 84 | logic soundly classifies this specific patient or identify specific 85 | aspects of the case to review more deeply. 86 | 87 | ## Use Case 3: Domain Knowledge and Control 88 | 89 | In some applications, one might have Domain Knowledge, such as 90 | Predictive Rules, an existing Knowledge Base, or required constraints. 91 | NRNs can leverage this pre-existing logic, along with training data 92 | to either improve a model's performance, expand its capabilities beyond 93 | those possibly from supervised training only, or control its behavior 94 | to meet pre-defined criteria. 95 | 96 | !!! Example "Classifying Flowers" 97 | - A flower is in the setosa class because: 98 | - ALL of the following are TRUE: 99 | - ANY of the following are TRUE: 100 | - NOT the following: 101 | - petal length (cm) greater than 0.95, 102 | - OR NOT the following: 103 | - sepal length (cm) greater than 0.969, 104 | - OR NOT the following: 105 | - sepal width (cm) greater than 0.078, 106 | - OR sepal length (cm) NOT greater than 0.146, 107 | - AND ANY of the following are TRUE: 108 | - petal length (cm) NOT greater than 0.164, 109 | - OR sepal width (cm) greater than 0.885 110 | 111 | In the example above, the logic below was encoded before training 112 | the NRN, and is a part of the logic used to classify the Setotsa 113 | flowers. 114 | 115 | !!! Example "Our Domain Knowledge" 116 | - ANY of the following are TRUE: 117 | - petal length (cm) NOT greater than 0.164, 118 | - OR sepal width (cm) greater than 0.885 119 | 120 | This principle can be extended to many more complex, and useful 121 | applications. -------------------------------------------------------------------------------- /docs/formatting/extra.css: -------------------------------------------------------------------------------- 1 | /** Below is the CIO CSS Theme **/ 2 | 3 | /* Indentation for mkdocstrings items. */ 4 | div.doc-contents:not(.first) { 5 | padding-left: 25px; 6 | border-left: 4px solid rgba(230, 230, 230); 7 | margin-bottom: 80px; 8 | } 9 | 10 | nav.md-nav.md-nav--secondary { 11 | font-size: 14px; 12 | color: #77757a; 13 | background-color: #f5f7fa; 14 | } 15 | 16 | body > div > main > div > div.md-sidebar.md-sidebar--primary > div > div > nav > label { 17 | background-color: #252525 18 | } 19 | 20 | .md-typeset ul { 21 | margin-top: 2px; 22 | margin-bottom: 3px; 23 | } 24 | 25 | .md-typeset ul li { margin-bottom: 1px ; } 26 | 27 | body > div > main > div > div.md-sidebar.md-sidebar--primary > div > div > nav > label > a > i { 28 | display: none; 29 | } 30 | 31 | body > div > main > div > div.md-sidebar.md-sidebar--primary > div > div > nav > label { 32 | background-color: #fff; 33 | } 34 | 35 | a.md-header-nav__button.md-logo { 36 | display: none; 37 | } 38 | 39 | nav.md-nav.md-nav--primary{ 40 | font-size: 16.5px; 41 | } 42 | 43 | body { 44 | background-color: #fff; 45 | font-family: "PlexSans", Source Sans Pro, Helvetica Neue, Arial, sans-serif; 46 | font-size: 15px; 47 | } 48 | 49 | .md-header { 50 | background-color: #252525 51 | } 52 | 53 | body > div > main > div > div.md-sidebar.md-sidebar--secondary > div > div > nav > label { 54 | visibility: hidden; 55 | } 56 | 57 | body > div > main > div > div.md-sidebar.md-sidebar--secondary > div > div > nav > label:after { 58 | content:'Topic contents'; 59 | visibility: visible; 60 | display: block; 61 | position: absolute; 62 | top: 0.001px; 63 | padding-top: 28px; 64 | } 65 | 66 | body > div > footer > div.md-footer-meta.md-typeset > div > div { 67 | visibility: hidden; 68 | } 69 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | [//]: # (![Coverage Report](./assets/coverage.svg)) 2 | 3 | ![](./static/torchlogic_logo.png) 4 | 5 | # TORCHLOGIC DOCUMENTATION 6 | 7 | _torchlogic_ is a pytorch framework for developing Neuro-Symbolic AI systems 8 | based on [Weighted Lukasiewicz Logic](https://arxiv.org/abs/2006.13155) that we denote as _Reasoning Networks_. 9 | However, the design principles of the _torchlogic_ framework depart from previous related works to provide a 10 | critical breakthrough in computational efficiency for Neuro-Symbolic AI, full GPU scaling. 11 | 12 | ### Design Principles 13 | 14 | - _Neural == Symbolic_: Symbolic operations should not deviate computationally from Neural operations and leverage PyTorch directly 15 | - _Masked Tensors_: Reasoning Networks use tensors and masking to represent any logical structure _and_ leverage GPU optimized computations 16 | - _Neural -> Symbolic Extension_: Symbolic operations in _torchlogic_ are PyTorch Modules and can therefore integrate with existing Deep Neural Networks seamlessly 17 | 18 | With these principles, _torchlogic_ and Reasoning Networks are able 19 | to extend and integrate with our current state-of-the-art technologies that leverage advances in 20 | Deep Learning. Reasoning Networks developed with _torchlogic_ can scale with 21 | multi-GPU support enabling reasoning at speeds not previously possible. Finally, 22 | those familiar with PyTorch development principles will have only a small step 23 | in skill building to develop with _torchlogic_. Happy reasoning! 24 | 25 | ### Documentation 26 | 27 | The current code is in an Beta state so there may be some bugs. We've established a 28 | stable set of functionality and want to hear from you on how to improve further! We'll do our best to keep the 29 | documentation up to date with the latest changes to our API reflected there. 30 | 31 | ### Tutorial 32 | 33 | The best way to get started using torchlogic is by exploring the various tutorials. 34 | There are several tutorials demonstrating how to use the Bandit-NRN algorithm 35 | in multiple use cases. The tutorials demonstrate the key concepts in torchlogic and 36 | when using Reasoning Networks for explainable AI solutions. These concepts are 37 | also discussed in more theoretical terms in the Data Science section of this 38 | documentation. 39 | 40 | [Tutorial Source](./tutorials/brn.md) 41 | 42 | ### Data Science 43 | 44 | To understand the basic of the torchlogic framework and Reasoning Networks 45 | check out the [Data Science](./ds/rn.md) section, which gives an introduction to some of the 46 | models developed so far using torchlogic. 47 | 48 | -------------------------------------------------------------------------------- /docs/install/install.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | 3 | #### Install the latest version of torchlogic 4 | 5 | The `torchlogic` package can be installed by cloning the repo, using poetry and pip. 6 | Installing via pip will enable you to 7 | use the R-NRN to solve classification and regression problems, or 8 | to develop new Neural Reasoning Network based models using the `torchlogic` 9 | framework. 10 | 11 | ```commandline 12 | poetry install 13 | poetry build 14 | pip install 15 | ``` -------------------------------------------------------------------------------- /docs/reference/models.md: -------------------------------------------------------------------------------- 1 | ::: torchlogic.models.BanditNRNClassifier 2 | 3 | ::: torchlogic.models.BanditNRNRegressor -------------------------------------------------------------------------------- /docs/reference/modules.md: -------------------------------------------------------------------------------- 1 | ::: torchlogic.modules.BanditNRNModule -------------------------------------------------------------------------------- /docs/reference/nn.md: -------------------------------------------------------------------------------- 1 | ::: torchlogic.nn.Predicates 2 | 3 | ::: torchlogic.nn.LukasiewiczChannelAndBlock 4 | 5 | ::: torchlogic.nn.LukasiewiczChannelOrBlock 6 | 7 | ::: torchlogic.nn.LukasiewiczChannelXOrBlock 8 | 9 | ::: torchlogic.nn.LukasiewiczChannelXOrBlock 10 | 11 | ::: torchlogic.nn.ConcatenateBlocksLogic -------------------------------------------------------------------------------- /docs/reference/utils.md: -------------------------------------------------------------------------------- 1 | ::: torchlogic.utils.trainers.BanditNRNTrainer -------------------------------------------------------------------------------- /docs/static/torchlogic_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/torchlogic/20d10b0462b6866f3853336ef3cfad19218dcb18/docs/static/torchlogic_logo.png -------------------------------------------------------------------------------- /docs/static/torchlogic_logo_black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/torchlogic/20d10b0462b6866f3853336ef3cfad19218dcb18/docs/static/torchlogic_logo_black.png -------------------------------------------------------------------------------- /docs/tutorials/brn.md: -------------------------------------------------------------------------------- 1 | # Bandit-NRN Tutorials 2 | 3 | ### A tutorial demonstrating use of Bandit-NRN for multi-class classification: 4 | 5 | [Bandit-NRN Tutorial - Multi-Class]("link/to/notebook") 6 | 7 | In this tutorial you will learn how to use a BanditNRNClassifier and the BanditNRNTrainer 8 | together to solve a multi-class classification problem. Some key concepts covered include: 9 | 10 | - Data preprocessing for Reasoning Networks 11 | - Feature naming conventions that improve interpretation of the model's natural language explanations 12 | - Using optuna to run a hyper-parameter search for BanditNRN 13 | - Evaluating BanditNRN predictive performance 14 | - Producing Global and Sample level explanations for predictions from BanditNRN 15 | - Controlling the outputs of Global and Sample level explanations. 16 | - Examining a detailed view of the induced weighted logic 17 | 18 | ### A tutorial demonstrating use of Bandit-NRN for binary-class classification: 19 | 20 | [Bandit-NRN Tutorial - Binary-Class](link/to/notebook) 21 | 22 | In this tutorial you will learn how to use a BanditNRNClassifier and the BanditNRNTrainer 23 | together to solve a binary-class classification problem. Some key concepts covered include: 24 | 25 | - Data preprocessing for Reasoning Networks 26 | - Feature naming conventions that improve interpretation of the model's natural language explanations 27 | - Using optuna to run a hyper-parameter search for BanditNRN 28 | - Evaluating BanditNRN predictive performance 29 | - Calibrating BanditNRN predictions to probability scores 30 | - Producing Global and Sample level explanations for predictions from BanditNRN 31 | - Examining a detailed view of the induced weighted logic 32 | 33 | ### A tutorial demonstrating use of Bandit-NRN for regression: 34 | 35 | [Bandit-NRN Tutorial - Regression]("link/to/notebook") 36 | 37 | In this tutorial you will learn how to use a BanditNRNRegressor and the BanditNRNTrainer 38 | together to solve a regression problem. Some key concepts covered include: 39 | 40 | - Data preprocessing for Reasoning Networks 41 | - Feature naming conventions that improve interpretation of the model's natural language explanations 42 | - Binarizing numeric data using a Feature Binarization From Trees -- a method often used with BanditNRN to solve complex problems that have numeric inputs 43 | - Using optuna to run a hyper-parameter search for BanditNRN 44 | - Evaluating BanditNRN predictive performance 45 | - Producing Global and Sample level explanations for predictions from BanditNRN 46 | - Examining a detailed view of the induced weighted logic 47 | 48 | ### A tutorial demonstrating use of Bandit-NRN for multi-class classification with custom domain knowledge: 49 | 50 | [Bandit-NRN Tutorial - Multi-Class Domain Knowledge](link/to/notebook) 51 | 52 | In this tutorial you will learn how to use a BanditNRNClassifier, with added domain knowledge, 53 | and the BanditNRNTrainer together to solve a multi-class classification problem. Some key concepts covered include: 54 | 55 | - Data preprocessing for Reasoning Networks 56 | - Encoding domain knowledge into a Reasoning Network 57 | - Extending BanditNRN to include domain knowledge in addition to supervised learning 58 | - Using optuna to run a hyper-parameter search for BanditNRN 59 | - Evaluating BanditNRN predictive performance 60 | - Producing Global and Sample level explanations for predictions from BanditNRN 61 | - Examining a detailed view of the induced weighted logic 62 | 63 | ### A tutorial demonstrating use of Boosted-Bandit-NRN for binary-class classification: 64 | 65 | [Bandit-NRN Tutorial - Boosted-Binary-Class](link/to/notebook) 66 | 67 | In this tutorial you will learn how to use a BanditNRNClassifier and the BoostedBanditNRNTrainer 68 | together to solve a binary-class classification problem with the BoostedBanditNRN ensemble model. 69 | Some key concepts covered include: 70 | 71 | - Data preprocessing for Reasoning Networks 72 | - Feature naming conventions that improve interpretation of the model's natural language explanations 73 | - Using optuna to run a hyper-parameter search for BoostedBanditNRN 74 | - Evaluating BanditNRN predictive performance 75 | - Calibrating BanditNRN predictions to probability scores 76 | - Producing Global and Sample level explanations for predictions from BanditNRN 77 | - Examining a detailed view of the induced weighted logic -------------------------------------------------------------------------------- /experiments/README.md: -------------------------------------------------------------------------------- 1 | # cao-exps -------------------------------------------------------------------------------- /experiments/nam_shap_size.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | import pandas as pd 4 | 5 | from src.datasets import OpemlMLDataset 6 | 7 | BENCHMARK_DATASETS = [44120, 44121, 44122, 44123, 44124, 44125, 44126, 44127, 44128, 44129, 44130, 44131, 8 | 44089, 44090, 44091, 44156, 44157, 44158, 44159, 44160, 44161, 44162] 9 | 10 | 11 | if __name__ == "__main__": 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument("-t", "--train_size", type=float, default=0.6) 14 | parser.add_argument("-e", "--test_size", type=float, default=0.5) 15 | parser.add_argument("-s", "--random_state", type=int, default=42) 16 | args = parser.parse_args() 17 | 18 | openml_ids = [] 19 | explanation_sizes = [] 20 | for openml_id in BENCHMARK_DATASETS: 21 | dataset = OpemlMLDataset( 22 | openml_id, args.train_size, args.test_size, args.random_state 23 | ) 24 | # explanation_size = dataset.X_test.nunique().sum() + dataset.X_test.shape[1] 25 | explanation_size = dataset.X_test.shape[1] 26 | openml_ids += [openml_id] 27 | explanation_sizes += [explanation_size] 28 | 29 | result = pd.DataFrame({'openml_id': openml_ids, 'explanation_size': explanation_sizes}) 30 | result.to_csv('./plots/nam_shap_explanation_sizes.csv', index=False) 31 | 32 | # import pandas as pd 33 | # import numpy as np 34 | # import os 35 | # import glob 36 | # def calculate_rnrn_explanation_sizes(data_path): 37 | # files = glob.glob(os.path.join(data_path, '*auc_fix_explanations_with_percentiles_rnrn_size.csv')) 38 | # sizes = [] 39 | # for file in files: 40 | # df = pd.read_csv(file) 41 | # df['explanation'] = df['simple_sample_explain_pos'].combine_first(df['simple_sample_explain_neg']) 42 | # df['explanation_size'] = df['explanation'].apply(lambda x: x.count('\n\t') if x != 'FAILED' else np.nan) 43 | # failed_instances = df['explanation'].str.contains('FAILED').sum() 44 | # if failed_instances > 0: 45 | # print(f"{failed_instances} FAILED INSTANCES for {file}!!") 46 | # sizes += [df['explanation_size'].mean()] 47 | # print(np.mean(sizes)) 48 | # print(pd.DataFrame({'file': files, 'size': sizes})) 49 | # 50 | # 51 | # calculate_rnrn_explanation_sizes('./plots') 52 | -------------------------------------------------------------------------------- /experiments/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "cao-exps" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Your Name "] 6 | readme = "README.md" 7 | 8 | [tool.poetry.dependencies] 9 | python = "^3.9" 10 | openml = "^0.14.1" 11 | torch = "^2.1.0" 12 | xgboost = "^2.0.2" 13 | cvxpy-base = "1.4.1" 14 | cvxopt = "1.3.2" 15 | aix360 = "^0.3.0" 16 | optuna = "^3.3.0" 17 | qhoptim = "^1.1.0" 18 | tensorflow = "2.15.0" 19 | yacs = "0.1.8" 20 | 21 | 22 | [build-system] 23 | requires = ["poetry-core"] 24 | build-backend = "poetry.core.masonry.api" 25 | -------------------------------------------------------------------------------- /experiments/query_openml.py: -------------------------------------------------------------------------------- 1 | import openml 2 | 3 | 4 | def get_openml_df(max_features, max_classes, min_instances, max_instances): 5 | openml_df = openml.datasets.list_datasets(output_format="dataframe") 6 | openml_df = openml_df.query( 7 | "NumberOfInstancesWithMissingValues == 0 & " 8 | "NumberOfMissingValues == 0 & " 9 | "NumberOfClasses > 1 & " 10 | #'NumberOfClasses <= 30 & ' 11 | "NumberOfSymbolicFeatures == 1 & " 12 | #'NumberOfInstances > 999 & ' 13 | "NumberOfFeatures >= 2 & " 14 | "NumberOfNumericFeatures == NumberOfFeatures -1 &" 15 | "NumberOfClasses <= " + str(max_classes) + " & " 16 | "NumberOfFeatures <= " + str(max_features + 1) + " & " 17 | "NumberOfInstances >= " + str(min_instances) + " & " 18 | "NumberOfInstances <= " + str(max_instances) 19 | ) 20 | 21 | openml_df = openml_df[ 22 | ["name", "did", "NumberOfClasses", "NumberOfInstances", "NumberOfFeatures"] 23 | ] 24 | 25 | return openml_df 26 | 27 | 28 | get_openml_df(100, 20, 160, 1200).to_csv("openml.csv", index=False) 29 | -------------------------------------------------------------------------------- /experiments/src/BRCG.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from aix360.algorithms.rbm import FeatureBinarizerFromTrees 3 | from aix360.algorithms.rbm.boolean_rule_cg import BooleanRuleCG 4 | 5 | 6 | class BRCG(object): 7 | 8 | def __init__( 9 | self, 10 | lambda0, 11 | lambda1, 12 | CNF, 13 | K, 14 | D, 15 | B, 16 | iterMax, 17 | timeMax, 18 | eps, 19 | solver, 20 | fbt=None 21 | ): 22 | self.fbt = fbt 23 | self.model = BooleanRuleCG( 24 | lambda0=lambda0, 25 | lambda1=lambda1, 26 | CNF=CNF, 27 | K=K, 28 | D=D, 29 | B=B, 30 | iterMax=iterMax, 31 | timeMax=timeMax, 32 | eps=eps, 33 | solver=solver 34 | ) 35 | 36 | def fit(self, X_train, y_train): 37 | if not isinstance(X_train, pd.DataFrame): 38 | X_train = pd.DataFrame(X_train, columns=[f'feature_{i}' for i in range(X_train.shape[1])]) 39 | self.model.fit(X_train, y_train.ravel()) 40 | 41 | def predict(self, X): 42 | return self.model.predict(X) 43 | -------------------------------------------------------------------------------- /experiments/src/LEURN/Causality_Example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/torchlogic/20d10b0462b6866f3853336ef3cfad19218dcb18/experiments/src/LEURN/Causality_Example.png -------------------------------------------------------------------------------- /experiments/src/LEURN/DEMO.py: -------------------------------------------------------------------------------- 1 | """ 2 | @author: Caglar Aytekin 3 | contact: caglar@deepcause.ai 4 | """ 5 | # %% IMPORT 6 | from LEURN import LEURN 7 | import torch 8 | from DATA import split_and_processing 9 | from TRAINER import Trainer 10 | import numpy as np 11 | import openml 12 | 13 | 14 | 15 | #DEMO FOR CREDIT SCORING DATASET: OPENML ID : 31 16 | #MORE INFO: https://www.openml.org/search?type=data&sort=runs&id=31&status=active 17 | #%% Set Neural Network Hyperparameters 18 | depth=2 19 | batch_size=1024 20 | lr=1e-3 21 | epochs=500 22 | droprate=0.0 23 | output_type=1 #0: regression, 1: binary classification, 2: multi-class classification 24 | 25 | #%% Check if CUDA is available and set the device accordingly 26 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 27 | print("Using device:", device) 28 | 29 | 30 | #%% Load the dataset 31 | #Read dataset from openml 32 | open_ml_dataset_id=31 33 | dataset = openml.datasets.get_dataset(open_ml_dataset_id) 34 | X, y, categoricals, attribute_names = dataset.get_data(target=dataset.default_target_attribute) 35 | #Alternatively load your own dataset from another source (excel,csv etc) 36 | #Be mindful that X and y should be dataframes, categoricals is a boolean list indicating categorical features, attribute_names is a list of feature names 37 | 38 | # %% Process data, save useful statistics 39 | X_train,X_val,X_test,y_train,y_val,y_test,preprocessor=split_and_processing(X,y,categoricals,output_type,attribute_names) 40 | 41 | 42 | 43 | #%% Initialize model, loss function, optimizer, and learning rate scheduler 44 | model = LEURN(preprocessor, depth=depth,droprate=droprate).to(device) 45 | 46 | 47 | #%%Train model 48 | model_trainer=Trainer(model, X_train, X_val, y_train, y_val,lr=lr,batch_size=batch_size,epochs=epochs,problem_type=output_type) 49 | model_trainer.train() 50 | #Load best weights 51 | model.load_state_dict(torch.load('best_model_weights.pth')) 52 | 53 | #%%Evaluate performance 54 | perf=model_trainer.evaluate(X_train, y_train) 55 | perf=model_trainer.evaluate(X_test, y_test) 56 | perf=model_trainer.evaluate(X_val, y_val) 57 | 58 | #%%TESTS 59 | model.eval() 60 | 61 | #%%Check sample in original format: 62 | print(preprocessor.inverse_transform_X(X_test[0:1])) 63 | #%% Explain single example 64 | Exp_df_test_sample,result,result_original_format=model.explain(X_test[0:1]) 65 | #%% Check results 66 | print(result,result_original_format) 67 | #%% Check explanation 68 | print(Exp_df_test_sample) 69 | #%% tests 70 | #model output and sum of contributions should be the same 71 | print(result,model.output,model(X_test[0:1]),Exp_df_test_sample['Contribution'].values.sum()) 72 | 73 | 74 | #%% GENERATION FROM SAME CATEGORY 75 | generated_sample_nn_friendly, generated_sample_original_input_format,output=model.generate_from_same_category(X_test[0:1]) 76 | #%%Check sample in original format: 77 | print(preprocessor.inverse_transform_X(X_test[0:1])) 78 | print(generated_sample_original_input_format) 79 | #%% Explain single example 80 | Exp_df_generated_sample,result,result_original_format=model.explain(generated_sample_nn_friendly) 81 | print(Exp_df_generated_sample) 82 | print(Exp_df_test_sample.equals(Exp_df_generated_sample)) #this should be true 83 | 84 | 85 | #%% GENERATE FROM SCRATCH 86 | generated_sample_nn_friendly, generated_sample_original_input_format,output=model.generate() 87 | Exp_df_generated_sample,result,result_original_format=model.explain(generated_sample_nn_friendly) 88 | print(Exp_df_generated_sample) 89 | print(result,result_original_format) 90 | 91 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /experiments/src/LEURN/Presentation_Product.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/torchlogic/20d10b0462b6866f3853336ef3cfad19218dcb18/experiments/src/LEURN/Presentation_Product.pdf -------------------------------------------------------------------------------- /experiments/src/LEURN/Presentation_Technical.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/torchlogic/20d10b0462b6866f3853336ef3cfad19218dcb18/experiments/src/LEURN/Presentation_Technical.pdf -------------------------------------------------------------------------------- /experiments/src/LEURN/README.md: -------------------------------------------------------------------------------- 1 | # LEURN 2 | Official Repository for LEURN: Learning Explainable Univariate Rules with Neural Networks 3 | https://arxiv.org/abs/2303.14937 4 | 5 | Detailed information about LEURN is given in the presentations. 6 | A demo is provided for training, making local explanations and data generation in DEMO.py 7 | 8 | NEW! Streamlit demo is now available 9 | Just activate the environment and run the following in your command line. 10 | streamlit run UI.py 11 | Make sure you check the explanation video at: 12 | https://www.linkedin.com/posts/caglaraytekin_ai-machinelearning-dataanalysis-activity-7172866316691869697-5-nB?utm_source=share&utm_medium=member_desktop 13 | 14 | NEW! LEURN now includes Causal Effects 15 | Thanks to its unique design, LEURN can make controlled experiments at lightning speed, discovering average causal effects. 16 | ![plot](./Causality_Example.png) 17 | 18 | Main difference of this implementation from the paper: 19 | - LEURN is now much simpler and uses binarized tanh (k=1 always) with no degradation in performance. 20 | 21 | Notes: 22 | - For top performance, a thorough hyperparameter search as described in paper is needed. 23 | - Human-in-the-loop continuous training is not implemented in this repository. 24 | - Deepcause provides consultancy services to make the most out of LEURN 25 | 26 | Contact: 27 | caglar@deepcause.ai 28 | -------------------------------------------------------------------------------- /experiments/src/LEURN/requirements.txt: -------------------------------------------------------------------------------- 1 | torch --index-url https://download.pytorch.org/whl/cu118 2 | pandas 3 | openml 4 | numpy 5 | scikit-learn 6 | streamlit==1.29.0 -------------------------------------------------------------------------------- /experiments/src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/torchlogic/20d10b0462b6866f3853336ef3cfad19218dcb18/experiments/src/__init__.py -------------------------------------------------------------------------------- /experiments/src/cofrnet.py: -------------------------------------------------------------------------------- 1 | # Imports and Seeds 2 | import copy 3 | 4 | from aix360.algorithms.cofrnet.CustomizedLinearClasses import CustomizedLinearFunction 5 | from aix360.algorithms.cofrnet.CustomizedLinearClasses import CustomizedLinear 6 | from aix360.algorithms.cofrnet.utils import generate_connections 7 | from aix360.algorithms.cofrnet.utils import process_data 8 | from aix360.algorithms.cofrnet.utils import train 9 | from aix360.algorithms.cofrnet.utils import OnlyTabularDataset 10 | from aix360.algorithms.cofrnet.CoFrNet import CoFrNet_Model 11 | from aix360.algorithms.cofrnet.CoFrNet import generate_connections 12 | from aix360.algorithms.cofrnet.CoFrNet import CoFrNet_Explainer 13 | from torch.utils.data import DataLoader 14 | from tqdm import tqdm 15 | import numpy as np 16 | from torch.utils.data import Dataset 17 | import torch # import main library 18 | import torch.nn as nn # import modules 19 | from torch.autograd import Function # import Function to create custom activations 20 | from torch.nn.parameter import Parameter # import Parameter to create custom activations with learnable parameters 21 | import torch.nn.functional as F # import torch functions 22 | from sklearn.preprocessing import MinMaxScaler 23 | from sklearn.model_selection import train_test_split 24 | import torch.optim as optim 25 | import random 26 | from sklearn.datasets import load_breast_cancer 27 | 28 | from sklearn.metrics import roc_auc_score 29 | 30 | 31 | def onehot_encoding(label, n_classes): 32 | """Conduct one-hot encoding on a label vector.""" 33 | label = label.view(-1) 34 | onehot = torch.zeros(label.size(0), n_classes).float().to(label.device) 35 | onehot.scatter_(1, label.view(-1, 1), 1) 36 | 37 | return onehot 38 | 39 | class Cofrnet(object): 40 | 41 | def __init__(self, network_depth, variant, input_size, output_size, lr, momentum, epochs, weight_decay, early_stopping_plateau_count): 42 | self.network_depth = network_depth 43 | self.variant = variant 44 | self.input_size = input_size 45 | self.output_size = output_size 46 | self.lr = lr 47 | self.momentum = momentum 48 | self.weight_decay = weight_decay 49 | self.epochs = epochs 50 | self.early_stopping_plateau_count = early_stopping_plateau_count 51 | self.model = CoFrNet_Model( 52 | generate_connections(network_depth, input_size, output_size, variant)) 53 | 54 | self.best_model = None 55 | 56 | if torch.cuda.is_available(): 57 | self.model.cuda() 58 | 59 | def train(self, X_train, y_train, X_holdout, y_holdout): 60 | # CONVERTING TO TENSOR 61 | tensor_x_train = torch.Tensor(X_train) 62 | tensor_y_train = torch.Tensor(y_train).long() 63 | 64 | tensor_x_holdout = torch.Tensor(X_holdout) 65 | tensor_y_holdout = torch.Tensor(y_holdout).long() 66 | 67 | train_dataset = OnlyTabularDataset(tensor_x_train, 68 | tensor_y_train) 69 | 70 | batch_size = 128 71 | dataloader = DataLoader(train_dataset, batch_size) 72 | self._train( 73 | self.model, 74 | dataloader, 75 | tensor_x_holdout, 76 | tensor_y_holdout, 77 | self.output_size, 78 | epochs=self.epochs, 79 | lr=self.lr, 80 | momentum=self.momentum, 81 | weight_decay=self.weight_decay, 82 | 83 | ) 84 | 85 | def predict(self, X): 86 | tensor_x = torch.Tensor(X) 87 | if torch.cuda.is_available(): 88 | tensor_x = tensor_x.cuda() 89 | return self.model(tensor_x).detach().cpu() 90 | 91 | def eval(self, X, y): 92 | if y.shape[1] == 1: 93 | predictions = self.predict(X)[:, -1] 94 | else: 95 | predictions = self.predict(X) 96 | predictions = predictions.cpu() 97 | return roc_auc_score(y, predictions, multi_class='ovo', average='micro') 98 | 99 | def _train(self, 100 | model, 101 | dataloader, 102 | X_holdout, 103 | y_holdout, 104 | num_classes, 105 | lr=0.001, 106 | momentum=0.9, 107 | epochs=20, 108 | weight_decay=0.00001 109 | ): 110 | criterion = nn.CrossEntropyLoss() 111 | # criterion = nn.MSELoss(reduction="sum") 112 | # optimizer = optim.SGD(model.parameters(), lr=lr, momentum=momentum) 113 | optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay, betas=(momentum, 0.999)) 114 | 115 | best_holdout_score = 0.0 116 | plateau_count = 0 117 | 118 | EPOCHS = epochs 119 | for epoch in range(EPOCHS): # loop over the dataset multiple times 120 | print("Epoch: ", epoch) 121 | if epoch > 0: 122 | holdout_score = self.eval(X_holdout, y_holdout) 123 | if holdout_score > best_holdout_score: 124 | print(f"Best holdout score updated from {best_holdout_score} to {holdout_score}") 125 | best_holdout_score = holdout_score 126 | self.best_model = copy.deepcopy(model) 127 | plateau_count = 0 128 | else: 129 | plateau_count += 1 130 | if plateau_count >= self.early_stopping_plateau_count: 131 | break 132 | 133 | running_loss = 0.0 134 | # for i, data in enumerate(trainloader, 0): 135 | for i, batch in tqdm(enumerate(dataloader)): 136 | # get the inputs; data is a list of [inputs, labels] 137 | # forward + backward + optimize 138 | 139 | tabular = batch['tabular'].cuda() 140 | target = batch['target'].cuda() 141 | 142 | tabular.requires_grad = True 143 | if torch.cuda.is_available(): 144 | tabular = tabular.cuda() 145 | target = target.cuda() 146 | 147 | outputs = model(tabular) 148 | 149 | one_hot_encoded_target = onehot_encoding(target, num_classes) 150 | 151 | # loss = criterion(outputs, batch['target']) 152 | loss = criterion(outputs, one_hot_encoded_target) 153 | 154 | # zero the parameter gradients 155 | optimizer.zero_grad() 156 | 157 | loss.backward() 158 | optimizer.step() 159 | 160 | # print statistics 161 | running_loss += loss.item() 162 | # print(f'[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}') 163 | print("Loss: ", running_loss) 164 | 165 | self.model = self.best_model 166 | 167 | print('Finished Training') -------------------------------------------------------------------------------- /experiments/src/danet/DAN_Task.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from scipy.special import softmax 4 | from .lib.utils import PredictDataset 5 | from .abstract_model import DANsModel 6 | from .lib.multiclass_utils import infer_output_dim, check_output_dim 7 | from torch.utils.data import DataLoader 8 | from torch.nn.functional import cross_entropy, mse_loss 9 | 10 | class DANetClassifier(DANsModel): 11 | def __post_init__(self): 12 | super(DANetClassifier, self).__post_init__() 13 | self._task = 'classification' 14 | self._default_loss = cross_entropy 15 | self._default_metric = 'accuracy' 16 | 17 | def weight_updater(self, weights): 18 | """ 19 | Updates weights dictionary according to target_mapper. 20 | 21 | Parameters 22 | ---------- 23 | weights : bool or dict 24 | Given weights for balancing training. 25 | 26 | Returns 27 | ------- 28 | bool or dict 29 | Same bool if weights are bool, updated dict otherwise. 30 | 31 | """ 32 | if isinstance(weights, int): 33 | return weights 34 | elif isinstance(weights, dict): 35 | return {self.target_mapper[key]: value for key, value in weights.items()} 36 | else: 37 | return weights 38 | 39 | def prepare_target(self, y): 40 | return np.vectorize(self.target_mapper.get)(y) 41 | 42 | def compute_loss(self, y_pred, y_true): 43 | return self.loss_fn(y_pred, y_true.long()) 44 | 45 | def update_fit_params( 46 | self, 47 | X_train, 48 | y_train, 49 | eval_set 50 | ): 51 | output_dim, train_labels = infer_output_dim(y_train) 52 | for X, y in eval_set: 53 | check_output_dim(train_labels, y) 54 | self.output_dim = output_dim 55 | self._default_metric = 'accuracy' 56 | self.classes_ = train_labels 57 | self.target_mapper = {class_label: index for index, class_label in enumerate(self.classes_)} 58 | self.preds_mapper = {str(index): class_label for index, class_label in enumerate(self.classes_)} 59 | 60 | def stack_batches(self, list_y_true, list_y_score): 61 | y_true = np.hstack(list_y_true) 62 | y_score = np.vstack(list_y_score) 63 | y_score = softmax(y_score, axis=1) 64 | return y_true, y_score 65 | 66 | def predict_func(self, outputs): 67 | outputs = np.argmax(outputs, axis=1) 68 | return outputs 69 | 70 | def predict_proba(self, X): 71 | """ 72 | Make predictions for classification on a batch (valid) 73 | 74 | Parameters 75 | ---------- 76 | X : a :tensor: `torch.Tensor` 77 | Input data 78 | 79 | Returns 80 | ------- 81 | res : np.ndarray 82 | 83 | """ 84 | self.network.eval() 85 | 86 | dataloader = DataLoader( 87 | PredictDataset(X), 88 | batch_size=1024, 89 | shuffle=False, 90 | ) 91 | 92 | results = [] 93 | for batch_nb, data in enumerate(dataloader): 94 | data = data.to(self.device).float() 95 | output = self.network(data) 96 | predictions = torch.nn.Softmax(dim=1)(output).cpu().detach().numpy() 97 | results.append(predictions) 98 | res = np.vstack(results) 99 | return res 100 | 101 | 102 | class DANetRegressor(DANsModel): 103 | def __post_init__(self): 104 | super(DANetRegressor, self).__post_init__() 105 | self._task = 'regression' 106 | self._default_loss = mse_loss 107 | self._default_metric = 'mse' 108 | 109 | def prepare_target(self, y): 110 | return y 111 | 112 | def compute_loss(self, y_pred, y_true): 113 | return self.loss_fn(y_pred, y_true) 114 | 115 | def update_fit_params( 116 | self, 117 | X_train, 118 | y_train, 119 | eval_set 120 | ): 121 | if len(y_train.shape) != 2: 122 | msg = "Targets should be 2D : (n_samples, n_regression) " + \ 123 | f"but y_train.shape={y_train.shape} given.\n" + \ 124 | "Use reshape(-1, 1) for single regression." 125 | raise ValueError(msg) 126 | self.output_dim = y_train.shape[1] 127 | self.preds_mapper = None 128 | 129 | 130 | def predict_func(self, outputs): 131 | return outputs 132 | 133 | def stack_batches(self, list_y_true, list_y_score): 134 | y_true = np.vstack(list_y_true) 135 | y_score = np.vstack(list_y_score) 136 | return y_true, y_score 137 | -------------------------------------------------------------------------------- /experiments/src/danet/Figures/DAN.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/torchlogic/20d10b0462b6866f3853336ef3cfad19218dcb18/experiments/src/danet/Figures/DAN.jpg -------------------------------------------------------------------------------- /experiments/src/danet/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Ronnie Rocket 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /experiments/src/danet/README.md: -------------------------------------------------------------------------------- 1 | # Deep Abstract Networks 2 | A PyTorch implementation of AAAI-2022 paper **[DANets: Deep Abstract Networks for Tabular Data Classification and Regression](https://arxiv.org/abs/2112.02962)** for reference. 3 | 4 | ## Brief Introduction 5 | Tabular data are ubiquitous in real world applications. Although many commonly-used neural components (e.g., convolution) and extensible neural networks (e.g., ResNet) have been developed by the machine learning community, few of them were effective for tabular data and few designs were adequately tailored for tabular data structures. In this paper, we propose a novel and flexible neural component for tabular data, called Abstract Layer (AbstLay), which learns to explicitly group correlative input features and generate higher-level features for semantics abstraction. Also, we design a structure re-parameterization method to compress AbstLay, thus reducing the computational complexity by a clear margin in the reference phase. A special basic block is built using AbstLays, and we construct a family of Deep Abstract Networks (DANets) for tabular data classification and regression by stacking such blocks. In DANets, a special shortcut path is introduced to fetch information from raw tabular features, assisting feature interactions across different levels. Comprehensive experiments on real-world tabular datasets show that our AbstLay and DANets are effective for tabular data classification and regression, and the computational complexity is superior to competitive methods. 6 | 7 | ## DANets illustration 8 | ![DANets](./Figures/DAN.jpg) 9 | 10 | ## Downloads 11 | ### Dataset 12 | Download the datasets from the following links: 13 | - [Cardiovascular Disease](https://www.kaggle.com/sulianova/cardiovascular-disease-dataset) 14 | - [Click](https://www.kaggle.com/c/kddcup2012-track2/) 15 | - [Epsilon](https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary.html) 16 | - [Forest Cover Type](https://archive.ics.uci.edu/ml/datasets/covertype) 17 | - [Microsoft WEB-10K](https://www.microsoft.com/en-us/research/project/mslr/) 18 | - [Yahoo! Learn to Rank Challenge version 2.0](https://webscope.sandbox.yahoo.com/catalog.php?datatype=c) 19 | - [YearPrediction](https://archive.ics.uci.edu/ml/datasets/yearpredictionmsd) 20 | 21 | (Optional) Before starting the program, you may change the file format to `.pkl` by using `svm2pkl()` or `csv2pkl()` functions in `./data/data_util.py`. 22 | 23 | ## How to use 24 | 25 | ### Setting 26 | 1. Clone or download this repository, and `cd` the path. 27 | 2. Build a working python environment. Python 3.7 is fine for this repository. 28 | 3. Install packages following the `requirements.txt`, e.g., by using `pip install -r requirements.txt`. 29 | 30 | ### Training 31 | 1. Set the hyperparameters in config files (`./config/default.py ` or `./config/*.yaml`). 32 | Notably, the hyperparameters in `.yaml` file will cover those in `default.py`. 33 | 34 | 2. Run by `python main.py --c [config_path] --g [gpu_id]`. 35 | - `-c`: The config file path 36 | - `-g`: GPU device ID 37 | 3. The checkpoint models and best models will be saved at the `./logs` file. 38 | 39 | ### Inference 40 | 1. Replace the `resume_dir` path with the file path containing your trained model/weight. 41 | 2. Run codes by using `python predict.py -d [dataset_name] -m [model_file_path] -g [gpu_id]`. 42 | - `-d`: Dataset name 43 | - `-m`: Model path for loading 44 | - `-g`: GPU device ID 45 | 46 | ### Config Hyperparameters 47 | #### Normal parameters 48 | - `dataset`: str 49 | The dataset name given must match those in `./data/dataset.py`. 50 | 51 | - `task`: str 52 | Choose one of the pre-given tasks 'classification' and 'regression'. 53 | 54 | - `resume_dir`: str 55 | The log path containing the checkpoint models. 56 | 57 | - `logname`: str 58 | The directory names of the models save at `./logs`. 59 | 60 | - `seed`: int 61 | The random seed. 62 | 63 | #### Model parameters 64 | - `layer`: int (default=20) 65 | Number of abstract layers to stack 66 | 67 | - `k`: int (default=5) 68 | Number of masks 69 | 70 | - `base_outdim`: int (default=64) 71 | The output feature dimension in abstract layer. 72 | 73 | - `drop_rate`: float (default=0.1) 74 | Dropout rate in shortcut module 75 | 76 | #### Fit parameters 77 | - `lr`: float (default=0.008) 78 | Learning rate 79 | 80 | - `max_epochs`: int (default=5000) 81 | Maximum number of epochs in training. 82 | 83 | - `patience`: int (default=1500) 84 | Number of consecutive epochs without improvement before performing early stopping. If patience is set to 0, then no early stopping will be performed. 85 | 86 | - `batch_size`: int (default=8192) 87 | Number of examples per batch. 88 | 89 | - `virtual_batch_size`: int (default=256) 90 | Size of the mini batches used for "Ghost Batch Normalization". `virtual_batch_size` must divide `batch_size`. 91 | 92 | ### Citations 93 | ``` 94 | @inproceedings{danets, 95 | title={DANets: Deep Abstract Networks for Tabular Data Classification and Regression}, 96 | author={Chen, Jintai and Liao, Kuanlun and Wan, Yao and Chen, Danny Z and Wu, Jian}, 97 | booktitle={AAAI}, 98 | year={2022} 99 | } 100 | ``` 101 | 102 | 103 | -------------------------------------------------------------------------------- /experiments/src/danet/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/torchlogic/20d10b0462b6866f3853336ef3cfad19218dcb18/experiments/src/danet/__init__.py -------------------------------------------------------------------------------- /experiments/src/danet/config/MSLR.yaml: -------------------------------------------------------------------------------- 1 | dataset: 'MSLR' 2 | task: 'regression' 3 | resume_dir: '' 4 | logname: 'layer20' 5 | 6 | fit: 7 | max_epochs: 2000 8 | patience: 500 9 | lr: 0.008 10 | 11 | model: 12 | layer: 20 13 | base_outdim: 64 14 | k: 5 15 | drop_rate: 0.1 -------------------------------------------------------------------------------- /experiments/src/danet/config/cardio.yaml: -------------------------------------------------------------------------------- 1 | dataset: 'cardio' 2 | task: 'classification' 3 | resume_dir: '' 4 | logname: 'layer8' 5 | 6 | fit: 7 | max_epochs: 500 8 | patience: 200 9 | lr: 0.008 10 | 11 | model: 12 | layer: 8 13 | base_outdim: 64 14 | k: 5 15 | drop_rate: 0.1 -------------------------------------------------------------------------------- /experiments/src/danet/config/click.yaml: -------------------------------------------------------------------------------- 1 | dataset: 'click' 2 | task: 'classification' 3 | resume_dir: '' 4 | logname: '' 5 | 6 | fit: 7 | max_epochs: 1700 8 | patience: 1000 9 | lr: 0.008 10 | 11 | model: 12 | layer: 8 13 | base_outdim: 64 14 | k: 5 15 | drop_rate: 0.1 -------------------------------------------------------------------------------- /experiments/src/danet/config/default.py: -------------------------------------------------------------------------------- 1 | from yacs.config import CfgNode as Node 2 | cfg = Node() 3 | cfg.seed = 324 4 | cfg.dataset = 'forest_cover_type' 5 | cfg.task = 'classification' 6 | cfg.resume_dir = '' 7 | cfg.logname = '' 8 | 9 | cfg.model = Node() 10 | cfg.model.base_outdim = 64 11 | cfg.model.k = 5 12 | cfg.model.drop_rate = 0.1 13 | cfg.model.layer = 20 14 | 15 | cfg.fit = Node() 16 | cfg.fit.lr = 0.008 17 | cfg.fit.max_epochs = 4000 18 | cfg.fit.patience = 1500 19 | cfg.fit.batch_size = 8192 20 | cfg.fit.virtual_batch_size = 256 21 | -------------------------------------------------------------------------------- /experiments/src/danet/config/epsilon.yaml: -------------------------------------------------------------------------------- 1 | dataset: 'epsilon' 2 | task: 'classification' 3 | train_ratio: 1.0 4 | resume_dir: '' 5 | logname: 'layer32' 6 | 7 | fit: 8 | max_epochs: 1500 9 | patience: 500 10 | lr: 0.02 11 | 12 | model: 13 | layer: 32 14 | base_outdim: 96 15 | k: 8 16 | drop_rate: 0.1 -------------------------------------------------------------------------------- /experiments/src/danet/config/forest_cover_type.yaml: -------------------------------------------------------------------------------- 1 | dataset: 'forest' 2 | task: 'classification' 3 | resume_dir: '' 4 | logname: 'layer20' 5 | 6 | fit: 7 | max_epochs: 5000 8 | patience: 1500 9 | lr: 0.008 10 | 11 | model: 12 | layer: 20 13 | base_outdim: 64 14 | k: 5 15 | -------------------------------------------------------------------------------- /experiments/src/danet/config/yahoo.yaml: -------------------------------------------------------------------------------- 1 | dataset: 'yahoo' 2 | task: 'regression' 3 | resume_dir: '' 4 | logname: 'layer32' 5 | 6 | fit: 7 | max_epochs: 2000 8 | patience: 500 9 | lr: 0.02 10 | 11 | model: 12 | layer: 32 13 | base_outdim: 96 14 | k: 8 15 | drop_rate: 0.1 -------------------------------------------------------------------------------- /experiments/src/danet/config/year.yaml: -------------------------------------------------------------------------------- 1 | dataset: 'year' 2 | task: 'regression' 3 | resume_dir: '' 4 | logname: '' 5 | 6 | fit: 7 | max_epochs: 150 8 | patience: 80 9 | lr: 0.008 10 | 11 | model: 12 | layer: 20 13 | base_outdim: 64 14 | k: 5 15 | drop_rate: 0.1 -------------------------------------------------------------------------------- /experiments/src/danet/data/data_util.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import os 3 | from sklearn.datasets import load_svmlight_file 4 | 5 | def svm2pkl(source, save_path): 6 | X_train, y_train = load_svmlight_file(os.path.join(source, 'train')) 7 | X_valid, y_valid = load_svmlight_file(os.path.join(source, 'vali')) 8 | X_test, y_test = load_svmlight_file(os.path.join(source, 'test')) 9 | 10 | X_train = pd.DataFrame(X_train.todense()) 11 | y_train = pd.Series(y_train) 12 | pd.concat([y_train, X_train], axis=1).T.reset_index(drop=True).T.to_pickle(os.path.join(save_path, 'train.pkl')) 13 | 14 | X_valid = pd.DataFrame(X_valid.todense()) 15 | y_valid = pd.Series(y_valid) 16 | pd.concat([y_valid, X_valid], axis=1).T.reset_index(drop=True).T.to_pickle(os.path.join(save_path, 'valid.pkl')) 17 | 18 | X_test = pd.DataFrame(X_test.todense()) 19 | y_test = pd.Series(y_test) 20 | pd.concat([y_test, X_test], axis=1).T.reset_index(drop=True).T.to_pickle(os.path.join(save_path, 'test.pkl')) 21 | 22 | def csv2pkl(source, save_path): 23 | data = pd.read_csv(source) 24 | data.to_pickle(save_path) 25 | 26 | if __name__ == '__main__': 27 | source = '/data/dataset/MSLR-WEB10K/Fold1' 28 | save_path = './data/MSLR-WEB10K' 29 | svm2pkl(source, save_path) 30 | -------------------------------------------------------------------------------- /experiments/src/danet/main.py: -------------------------------------------------------------------------------- 1 | from DAN_Task import DANetClassifier, DANetRegressor 2 | import argparse 3 | import os 4 | import torch.distributed 5 | import torch.backends.cudnn 6 | from sklearn.metrics import accuracy_score, mean_squared_error 7 | from data.dataset import get_data 8 | from lib.utils import normalize_reg_label 9 | from qhoptim.pyt import QHAdam 10 | from config.default import cfg 11 | os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" 12 | 13 | def get_args(): 14 | parser = argparse.ArgumentParser(description='PyTorch v1.4, DANet Task Training') 15 | parser.add_argument('-c', '--config', type=str, required=False, default='config/forest_cover_type.yaml', metavar="FILE", help='Path to config file') 16 | parser.add_argument('-g', '--gpu_id', type=str, default='1', help='GPU ID') 17 | 18 | args = parser.parse_args() 19 | os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id 20 | torch.backends.cudnn.benchmark = True if len(args.gpu_id) < 2 else False 21 | if args.config: 22 | cfg.merge_from_file(args.config) 23 | cfg.freeze() 24 | task = cfg.task 25 | seed = cfg.seed 26 | train_config = {'dataset': cfg.dataset, 'resume_dir': cfg.resume_dir, 'logname': cfg.logname} 27 | fit_config = dict(cfg.fit) 28 | model_config = dict(cfg.model) 29 | print('Using config: ', cfg) 30 | 31 | return train_config, fit_config, model_config, task, seed, len(args.gpu_id) 32 | 33 | def set_task_model(task, std=None, seed=1): 34 | if task == 'classification': 35 | clf = DANetClassifier( 36 | optimizer_fn=QHAdam, 37 | optimizer_params=dict(lr=fit_config['lr'], weight_decay=1e-5, nus=(0.8, 1.0)), 38 | scheduler_params=dict(gamma=0.95, step_size=20), 39 | scheduler_fn=torch.optim.lr_scheduler.StepLR, 40 | layer=model_config['layer'], 41 | base_outdim=model_config['base_outdim'], 42 | k=model_config['k'], 43 | drop_rate=model_config['drop_rate'], 44 | seed=seed 45 | ) 46 | eval_metric = ['accuracy'] 47 | 48 | elif task == 'regression': 49 | clf = DANetRegressor( 50 | std=std, 51 | optimizer_fn=QHAdam, 52 | optimizer_params=dict(lr=fit_config['lr'], weight_decay=fit_config['weight_decay'], nus=(0.8, 1.0)), 53 | scheduler_params=dict(gamma=0.95, step_size=fit_config['schedule_step']), 54 | scheduler_fn=torch.optim.lr_scheduler.StepLR, 55 | layer=model_config['layer'], 56 | base_outdim=model_config['base_outdim'], 57 | k=model_config['k'], 58 | seed=seed 59 | ) 60 | eval_metric = ['mse'] 61 | return clf, eval_metric 62 | 63 | if __name__ == '__main__': 64 | 65 | print('===> Setting configuration ...') 66 | train_config, fit_config, model_config, task, seed, n_gpu = get_args() 67 | logname = None if train_config['logname'] == '' else train_config['dataset'] + '/' + train_config['logname'] 68 | print('===> Getting data ...') 69 | X_train, y_train, X_valid, y_valid, X_test, y_test = get_data(train_config['dataset']) 70 | mu, std = None, None 71 | if task == 'regression': 72 | mu, std = y_train.mean(), y_train.std() 73 | print("mean = %.5f, std = %.5f" % (mu, std)) 74 | y_train = normalize_reg_label(y_train, std, mu) 75 | y_valid = normalize_reg_label(y_valid, std, mu) 76 | y_test = normalize_reg_label(y_test, std, mu) 77 | 78 | clf, eval_metric = set_task_model(task, std, seed) 79 | 80 | clf.fit( 81 | X_train=X_train, y_train=y_train, 82 | eval_set=[(X_valid, y_valid)], 83 | eval_name=['valid'], 84 | eval_metric=eval_metric, 85 | max_epochs=fit_config['max_epochs'], patience=fit_config['patience'], 86 | batch_size=fit_config['batch_size'], virtual_batch_size=fit_config['virtual_batch_size'], 87 | logname=logname, 88 | resume_dir=train_config['resume_dir'], 89 | n_gpu=n_gpu 90 | ) 91 | 92 | preds_test = clf.predict(X_test) 93 | 94 | if task == 'classification': 95 | test_acc = accuracy_score(y_pred=preds_test, y_true=y_test) 96 | print(f"FINAL TEST ACCURACY FOR {train_config['dataset']} : {test_acc}") 97 | 98 | elif task == 'regression': 99 | test_mse = mean_squared_error(y_pred=preds_test, y_true=y_test) 100 | print(f"FINAL TEST MSE FOR {train_config['dataset']} : {test_mse}") 101 | -------------------------------------------------------------------------------- /experiments/src/danet/model/AcceleratedModule.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | class AcceleratedCreator(object): 6 | def __init__(self, input_dim, base_out_dim, k): 7 | super(AcceleratedCreator, self).__init__() 8 | self.input_dim = input_dim 9 | self.base_out_dim = base_out_dim 10 | self.computer = Extractor(k) 11 | 12 | def __call__(self, network): 13 | network.init_layer = self.extract_module(network.init_layer, self.input_dim, self.input_dim) 14 | for i in range(len(network.layer)): 15 | network.layer[i] = self.extract_module(network.layer[i], self.base_out_dim, self.input_dim) 16 | return network 17 | 18 | def extract_module(self, basicblock, base_input_dim, fix_input_dim): 19 | basicblock.conv1 = self.computer(basicblock.conv1, base_input_dim, self.base_out_dim // 2) 20 | basicblock.conv2 = self.computer(basicblock.conv2, self.base_out_dim // 2, self.base_out_dim) 21 | basicblock.downsample = self.computer(basicblock.downsample._modules['1'], fix_input_dim, self.base_out_dim) 22 | return basicblock 23 | 24 | 25 | class Extractor(object): 26 | def __init__(self, k): 27 | super(Extractor, self).__init__() 28 | self.k = k 29 | 30 | @staticmethod 31 | def get_parameter(abs_layer): 32 | bn = abs_layer.bn.bn 33 | alpha, beta, eps = bn.weight.data, bn.bias.data, bn.eps # [240] 34 | mu, var = bn.running_mean.data, bn.running_var.data 35 | locality = abs_layer.masker 36 | sparse_weight = locality.smax(locality.weight.data) # 6, 10 37 | 38 | feat_pro = abs_layer.fc 39 | process_weight = feat_pro.weight.data # ([240, 10, 1]) [240] 40 | process_bias = feat_pro.bias.data if feat_pro.bias is not None else None 41 | return alpha, beta, eps, mu, var, sparse_weight, process_weight, process_bias 42 | 43 | @staticmethod 44 | def compute_weights(a, b, eps, mu, var, sw, pw, pb, base_input_dim, base_output_dim, k): 45 | """ 46 | standard shape: [path, output_shape, input_shape, branch] 47 | """ 48 | sw_ = sw[:, None, :, None] 49 | pw_ = pw.view(k, 2, base_output_dim, base_input_dim).permute(0, 2, 3, 1) 50 | if pb is not None: 51 | pb_ = pb.view(k, 2, base_output_dim).permute(0, 2, 1)[:, :, None, :] 52 | a_ = a.view(k, 2, base_output_dim).permute(0, 2, 1)[:, :, None, :] 53 | b_ = b.view(k, 2, base_output_dim).permute(0, 2, 1)[:, :, None, :] 54 | mu_ = mu.view(k, 2, base_output_dim).permute(0, 2, 1)[:, :, None, :] 55 | var_ = var.view(k, 2, base_output_dim).permute(0, 2, 1)[:, :, None, :] 56 | 57 | W = sw_ * pw_ 58 | if pb is not None: 59 | mu_ = mu_ - pb_ 60 | W = a_ / (var_ + eps).sqrt() * W 61 | B = b_ - a_ / (var_ + eps).sqrt() * mu_ 62 | 63 | W_att = W[..., 0] 64 | B_att = B[..., 0] 65 | 66 | W_fc = W[..., 1] 67 | B_fc = B[..., 1] 68 | 69 | return W_att, W_fc, B_att.squeeze(), B_fc.squeeze() 70 | 71 | def __call__(self, abslayer, input_dim, base_out_dim): 72 | (a, b, e, m, v, s, pw, pb) = self.get_parameter(abslayer) 73 | wa, wf, ba, bf = self.compute_weights(a, b, e, m, v, s, pw, pb, input_dim, base_out_dim, self.k) 74 | return CompressAbstractLayer(wa, wf, ba, bf) 75 | 76 | 77 | class CompressAbstractLayer(nn.Module): 78 | def __init__(self, att_w, f_w, att_b, f_b): 79 | super(CompressAbstractLayer, self).__init__() 80 | self.att_w = nn.Parameter(att_w) 81 | self.f_w = nn.Parameter(f_w) 82 | self.att_bias = nn.Parameter(att_b[None, :, :]) 83 | self.f_bias = nn.Parameter(f_b[None, :, :]) 84 | 85 | def forward(self, x): 86 | att = torch.sigmoid(torch.einsum('poi,bi->bpo', self.att_w, x) + self.att_bias) # (2 * i + 2) * p * o 87 | y = torch.einsum('poi,bi->bpo', self.f_w, x) + self.f_bias # (2 * i + 1) * p * o 88 | return torch.sum(F.relu(att * y), dim=-2, keepdim=False) # 3 * p * o 89 | 90 | 91 | if __name__ == '__main__': 92 | import torch.optim as optim 93 | from DANet import AbstractLayer 94 | 95 | input_feat = torch.rand((8, 10), requires_grad=False) 96 | loss_function = nn.L1Loss() 97 | target = torch.rand((8, 20), requires_grad=False) 98 | abs_layer = AbstractLayer(base_input_dim=10, base_output_dim=20, k=6, virtual_batch_size=4, bias=False) 99 | y_ = abs_layer(input_feat) 100 | optimizer = optim.SGD(abs_layer.parameters(), lr=0.3) 101 | abs_layer.zero_grad() 102 | loss_function(y_, target).backward() 103 | optimizer.step() 104 | 105 | abs_layer = abs_layer.eval() 106 | y = abs_layer(input_feat) 107 | computer = Extractor(k=6) 108 | (a, b, e, m, v, s, pw, pb) = computer.get_parameter(abs_layer) 109 | wa, wf, ba, bf = computer.compute_weights(a, b, e, m, v, s, pw, pb, 10, 20, 6) 110 | acc_abs = CompressAbstractLayer(wa, wf, ba, bf) 111 | y2 = acc_abs(input_feat) 112 | -------------------------------------------------------------------------------- /experiments/src/danet/model/DANet.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import numpy as np 4 | import torch.nn.functional as F 5 | from .sparsemax import sparsemax 6 | 7 | def initialize_glu(module, input_dim, output_dim): 8 | gain_value = np.sqrt((input_dim + output_dim) / np.sqrt(input_dim)) 9 | torch.nn.init.xavier_normal_(module.weight, gain=gain_value) 10 | return 11 | 12 | class GBN(torch.nn.Module): 13 | """ 14 | Ghost Batch Normalization 15 | https://arxiv.org/abs/1705.08741 16 | """ 17 | def __init__(self, input_dim, virtual_batch_size=512): 18 | super(GBN, self).__init__() 19 | self.input_dim = input_dim 20 | self.virtual_batch_size = virtual_batch_size 21 | self.bn = nn.BatchNorm1d(self.input_dim) 22 | 23 | def forward(self, x): 24 | if self.training == True: 25 | chunks = x.chunk(int(np.ceil(x.shape[0] / self.virtual_batch_size)), 0) 26 | res = [self.bn(x_) for x_ in chunks] 27 | return torch.cat(res, dim=0) 28 | else: 29 | return self.bn(x) 30 | 31 | class LearnableLocality(nn.Module): 32 | 33 | def __init__(self, input_dim, k): 34 | super(LearnableLocality, self).__init__() 35 | self.register_parameter('weight', nn.Parameter(torch.rand(k, input_dim))) 36 | self.smax = sparsemax.Entmax15(dim=-1) 37 | 38 | def forward(self, x): 39 | mask = self.smax(self.weight) 40 | masked_x = torch.einsum('nd,bd->bnd', mask, x) # [B, k, D] 41 | return masked_x 42 | 43 | class AbstractLayer(nn.Module): 44 | def __init__(self, base_input_dim, base_output_dim, k, virtual_batch_size, bias=True): 45 | super(AbstractLayer, self).__init__() 46 | self.masker = LearnableLocality(input_dim=base_input_dim, k=k) 47 | self.fc = nn.Conv1d(base_input_dim * k, 2 * k * base_output_dim, kernel_size=1, groups=k, bias=bias) 48 | initialize_glu(self.fc, input_dim=base_input_dim * k, output_dim=2 * k * base_output_dim) 49 | self.bn = GBN(2 * base_output_dim * k, virtual_batch_size) 50 | self.k = k 51 | self.base_output_dim = base_output_dim 52 | 53 | def forward(self, x): 54 | b = x.size(0) 55 | x = self.masker(x) # [B, D] -> [B, k, D] 56 | x = self.fc(x.reshape(b, -1, 1)) # [B, k, D] -> [B, k * D, 1] -> [B, k * (2 * D'), 1] 57 | x = self.bn(x) 58 | chunks = x.chunk(self.k, 1) # k * [B, 2 * D', 1] 59 | x = sum([F.relu(torch.sigmoid(x_[:, :self.base_output_dim, :]) * x_[:, self.base_output_dim:, :]) for x_ in chunks]) # k * [B, D', 1] -> [B, D', 1] 60 | return x.squeeze(-1) 61 | 62 | 63 | class BasicBlock(nn.Module): 64 | def __init__(self, input_dim, base_outdim, k, virtual_batch_size, fix_input_dim, drop_rate): 65 | super(BasicBlock, self).__init__() 66 | self.conv1 = AbstractLayer(input_dim, base_outdim // 2, k, virtual_batch_size) 67 | self.conv2 = AbstractLayer(base_outdim // 2, base_outdim, k, virtual_batch_size) 68 | 69 | self.downsample = nn.Sequential( 70 | nn.Dropout(drop_rate), 71 | AbstractLayer(fix_input_dim, base_outdim, k, virtual_batch_size) 72 | ) 73 | 74 | def forward(self, x, pre_out=None): 75 | if pre_out == None: 76 | pre_out = x 77 | out = self.conv1(pre_out) 78 | out = self.conv2(out) 79 | identity = self.downsample(x) 80 | out += identity 81 | return F.leaky_relu(out, 0.01) 82 | 83 | 84 | class DANet(nn.Module): 85 | def __init__(self, input_dim, num_classes, layer_num, base_outdim, k, virtual_batch_size, drop_rate=0.1): 86 | super(DANet, self).__init__() 87 | params = {'base_outdim': base_outdim, 'k': k, 'virtual_batch_size': virtual_batch_size, 88 | 'fix_input_dim': input_dim, 'drop_rate': drop_rate} 89 | self.init_layer = BasicBlock(input_dim, **params) 90 | self.lay_num = layer_num 91 | self.layer = nn.ModuleList() 92 | for i in range((layer_num // 2) - 1): 93 | self.layer.append(BasicBlock(base_outdim, **params)) 94 | self.drop = nn.Dropout(0.1) 95 | 96 | self.fc = nn.Sequential(nn.Linear(base_outdim, 256), 97 | nn.ReLU(inplace=True), 98 | nn.Linear(256, 512), 99 | nn.ReLU(inplace=True), 100 | nn.Linear(512, num_classes)) 101 | 102 | def forward(self, x): 103 | out = self.init_layer(x) 104 | for i in range(len(self.layer)): 105 | out = self.layer[i](x, out) 106 | out = self.drop(out) 107 | out = self.fc(out) 108 | return out 109 | -------------------------------------------------------------------------------- /experiments/src/danet/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/torchlogic/20d10b0462b6866f3853336ef3cfad19218dcb18/experiments/src/danet/model/__init__.py -------------------------------------------------------------------------------- /experiments/src/danet/model/sparsemax/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/torchlogic/20d10b0462b6866f3853336ef3cfad19218dcb18/experiments/src/danet/model/sparsemax/__init__.py -------------------------------------------------------------------------------- /experiments/src/danet/predict.py: -------------------------------------------------------------------------------- 1 | from DAN_Task import DANetClassifier, DANetRegressor 2 | from sklearn.metrics import accuracy_score, mean_squared_error 3 | from lib.multiclass_utils import infer_output_dim 4 | from lib.utils import normalize_reg_label 5 | import numpy as np 6 | import argparse 7 | from data.dataset import get_data 8 | import os 9 | os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" 10 | 11 | def get_args(): 12 | parser = argparse.ArgumentParser(description='PyTorch v1.4, DANet Testing') 13 | parser.add_argument('-d', '--dataset', type=str, default='forest', help='Dataset Name for extracting data') 14 | parser.add_argument('-m', '--model_file', type=str, default='./weights/forest_layer32.pth', metavar="FILE", help='Inference model path') 15 | parser.add_argument('-g', '--gpu_id', type=str, default='1', help='GPU ID') 16 | args = parser.parse_args() 17 | os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id 18 | dataset = args.dataset 19 | model_file = args.model_file 20 | task = 'regression' if dataset in ['year', 'yahoo', 'MSLR'] else 'classification' 21 | 22 | return dataset, model_file, task, len(args.gpu_id) 23 | 24 | def set_task_model(task): 25 | if task == 'classification': 26 | clf = DANetClassifier() 27 | metric = accuracy_score 28 | elif task == 'regression': 29 | clf = DANetRegressor() 30 | metric = mean_squared_error 31 | return clf, metric 32 | 33 | def prepare_data(task, y_train, y_valid, y_test): 34 | output_dim = 1 35 | mu, std = None, None 36 | if task == 'classification': 37 | output_dim, train_labels = infer_output_dim(y_train) 38 | target_mapper = {class_label: index for index, class_label in enumerate(train_labels)} 39 | y_train = np.vectorize(target_mapper.get)(y_train) 40 | y_valid = np.vectorize(target_mapper.get)(y_valid) 41 | y_test = np.vectorize(target_mapper.get)(y_test) 42 | 43 | elif task == 'regression': 44 | mu, std = y_train.mean(), y_train.std() 45 | print("mean = %.5f, std = %.5f" % (mu, std)) 46 | y_train = normalize_reg_label(y_train, mu, std) 47 | y_valid = normalize_reg_label(y_valid, mu, std) 48 | y_test = normalize_reg_label(y_test, mu, std) 49 | 50 | return output_dim, std, y_train, y_valid, y_test 51 | 52 | if __name__ == '__main__': 53 | dataset, model_file, task, n_gpu = get_args() 54 | print('===> Getting data ...') 55 | X_train, y_train, X_valid, y_valid, X_test, y_test = get_data(dataset) 56 | output_dim, std, y_train, y_valid, y_test = prepare_data(task, y_train, y_valid, y_test) 57 | clf, metric = set_task_model(task) 58 | 59 | filepath = model_file 60 | clf.load_model(filepath, input_dim=X_test.shape[1], output_dim=output_dim, n_gpu=n_gpu) 61 | 62 | preds_test = clf.predict(X_test) 63 | test_value = metric(y_pred=preds_test, y_true=y_test) 64 | 65 | if task == 'classification': 66 | print(f"FINAL TEST ACCURACY FOR {dataset} : {test_value}") 67 | 68 | elif task == 'regression': 69 | print(f"FINAL TEST MSE FOR {dataset} : {test_value}") 70 | -------------------------------------------------------------------------------- /experiments/src/danet/requirements.txt: -------------------------------------------------------------------------------- 1 | torch>=1.4.0 2 | category_encoders 3 | yacs 4 | tensorboard>=2.2.2 5 | qhoptim -------------------------------------------------------------------------------- /experiments/src/danet_model.py: -------------------------------------------------------------------------------- 1 | from .danet.DAN_Task import DANetClassifier, DANetRegressor 2 | import argparse 3 | import os 4 | import torch.distributed 5 | import torch.backends.cudnn 6 | from sklearn.metrics import accuracy_score, mean_squared_error 7 | from .danet.lib.utils import normalize_reg_label 8 | from qhoptim.pyt import QHAdam 9 | from .danet.config.default import cfg 10 | os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" 11 | from uuid import uuid4 12 | 13 | 14 | class DANet(object): 15 | 16 | def __init__(self, max_epochs, patience, lr, layer, base_outdim, k, drop_rate, seed): 17 | self.max_epochs = max_epochs 18 | self.patience = patience 19 | self.lr = lr 20 | self.layer = layer 21 | self.base_outdim = base_outdim 22 | self.k = k 23 | self.drop_rate = drop_rate 24 | 25 | self.clf = DANetClassifier( 26 | optimizer_fn=QHAdam, 27 | optimizer_params=dict(lr=self.lr, weight_decay=1e-5, nus=(0.8, 1.0)), 28 | scheduler_params=dict(gamma=0.95, step_size=20), 29 | scheduler_fn=torch.optim.lr_scheduler.StepLR, 30 | layer=self.layer, 31 | base_outdim=self.base_outdim, 32 | k=self.k, 33 | drop_rate=self.drop_rate, 34 | seed=seed 35 | ) 36 | 37 | def fit(self, X_train, y_train, X_valid, y_valid): 38 | self.clf.fit( 39 | X_train=X_train, y_train=y_train.ravel(), 40 | eval_set=[(X_valid, y_valid.ravel())], 41 | eval_name=['valid'], 42 | eval_metric=['accuracy'], 43 | max_epochs=self.max_epochs, patience=self.patience, 44 | batch_size=8192, 45 | virtual_batch_size=256, 46 | logname=f'DANet-{uuid4()}', 47 | # resume_dir=train_config['resume_dir'], 48 | n_gpu=1 49 | ) 50 | 51 | def predict(self, X): 52 | return self.clf.predict(X) 53 | -------------------------------------------------------------------------------- /experiments/src/difflogic/INSTALLATION_SUPPORT.md: -------------------------------------------------------------------------------- 1 | # Installation Support 2 | 3 | `difflogic` is a Python 3.6+ and PyTorch 1.9.0+ based library for training and inference with logic gate networks. 4 | The library can be installed with: 5 | ```shell 6 | pip install difflogic 7 | ``` 8 | > ⚠️ Note that `difflogic` requires CUDA, the CUDA Toolkit (for compilation), and `torch>=1.9.0` (matching the CUDA version). 9 | 10 | **It is very important that the installed version of PyTorch was compiled with a CUDA version that is compatible with the CUDA version of the locally installed CUDA Toolkit.** 11 | 12 | You can check your CUDA version by running `nvidia-smi`. 13 | 14 | You can install PyTorch and torchvision of a specific version, e.g., via 15 | 16 | ```shell 17 | pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 -f https://download.pytorch.org/whl/torch_stable.html # CUDA version 11.1 18 | pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 -f https://download.pytorch.org/whl/torch_stable.html # CUDA version 11.3 19 | pip install torch==1.13.0+cu117 torchvision==0.14.0+cu117 -f https://download.pytorch.org/whl/torch_stable.html # CUDA version 11.7 20 | ``` 21 | 22 | If you get the following error message: 23 | 24 | ``` 25 | Failed to build difflogic 26 | 27 | ... 28 | 29 | RuntimeError: 30 | The detected CUDA version (11.2) mismatches the version that was used to compile 31 | PyTorch (11.7). Please make sure to use the same CUDA versions. 32 | ``` 33 | 34 | You need to make sure that the versions match, typically by installing a different PyTorch version. 35 | Note that there are some versions of PyTorch that have been compiled with CUDA versions different from the advertised 36 | versions, so in case it should match but doesn't, a quick fix can be to try some other (e.g., older) PyTorch versions. 37 | 38 | --- 39 | 40 | `difflogic` has been tested with PyTorch versions between 1.9 and 1.13. 41 | 42 | --- 43 | 44 | For the experiments, please make sure all dependencies in `experiments/requirements.txt` are installed in the Python environment. 45 | 46 | -------------------------------------------------------------------------------- /experiments/src/difflogic/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021-2023 Dr. Felix Petersen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /experiments/src/difflogic/difflogic/__init__.py: -------------------------------------------------------------------------------- 1 | from .difflogic import LogicLayer, GroupSum 2 | from .packbitstensor import PackBitsTensor 3 | from .compiled_model import CompiledLogicNet 4 | 5 | -------------------------------------------------------------------------------- /experiments/src/difflogic/difflogic/cuda/difflogic.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | 6 | namespace py = pybind11; 7 | 8 | torch::Tensor logic_layer_cuda_forward( 9 | torch::Tensor x, 10 | torch::Tensor a, 11 | torch::Tensor b, 12 | torch::Tensor w 13 | ); 14 | torch::Tensor logic_layer_cuda_backward_w( 15 | torch::Tensor x, 16 | torch::Tensor a, 17 | torch::Tensor b, 18 | torch::Tensor grad_y 19 | ); 20 | torch::Tensor logic_layer_cuda_backward_x( 21 | torch::Tensor x, 22 | torch::Tensor a, 23 | torch::Tensor b, 24 | torch::Tensor w, 25 | torch::Tensor grad_y, 26 | torch::Tensor given_x_indices_of_y_start, 27 | torch::Tensor given_x_indices_of_y 28 | ); 29 | torch::Tensor logic_layer_cuda_eval( 30 | torch::Tensor x, 31 | torch::Tensor a, 32 | torch::Tensor b, 33 | torch::Tensor w 34 | ); 35 | std::tuple tensor_packbits_cuda( 36 | torch::Tensor t, 37 | const int bit_count 38 | ); 39 | torch::Tensor groupbitsum( 40 | torch::Tensor b, 41 | const int pad_len, 42 | const int k 43 | ); 44 | 45 | PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 46 | m.def( 47 | "forward", 48 | [](torch::Tensor x, torch::Tensor a, torch::Tensor b, torch::Tensor w) { 49 | return logic_layer_cuda_forward(x, a, b, w); 50 | }, 51 | "logic layer forward (CUDA)"); 52 | m.def( 53 | "backward_w", [](torch::Tensor x, torch::Tensor a, torch::Tensor b, torch::Tensor grad_y) { 54 | return logic_layer_cuda_backward_w(x, a, b, grad_y); 55 | }, 56 | "logic layer backward w (CUDA)"); 57 | m.def( 58 | "backward_x", 59 | [](torch::Tensor x, torch::Tensor a, torch::Tensor b, torch::Tensor w, torch::Tensor grad_y, torch::Tensor given_x_indices_of_y_start, torch::Tensor given_x_indices_of_y) { 60 | return logic_layer_cuda_backward_x(x, a, b, w, grad_y, given_x_indices_of_y_start, given_x_indices_of_y); 61 | }, 62 | "logic layer backward x (CUDA)"); 63 | m.def( 64 | "eval", 65 | [](torch::Tensor x, torch::Tensor a, torch::Tensor b, torch::Tensor w) { 66 | return logic_layer_cuda_eval(x, a, b, w); 67 | }, 68 | "logic layer eval (CUDA)"); 69 | m.def( 70 | "tensor_packbits_cuda", 71 | [](torch::Tensor t, const int bit_count) { 72 | return tensor_packbits_cuda(t, bit_count); 73 | }, 74 | "ltensor_packbits_cuda (CUDA)"); 75 | m.def( 76 | "groupbitsum", 77 | [](torch::Tensor b, const int pad_len, const unsigned int k) { 78 | if (b.size(0) % k != 0) { 79 | throw py::value_error("in_dim (" + std::to_string(b.size(0)) + ") has to be divisible by k (" + std::to_string(k) + ") but it is not"); 80 | } 81 | return groupbitsum(b, pad_len, k); 82 | }, 83 | "groupbitsum (CUDA)"); 84 | } 85 | -------------------------------------------------------------------------------- /experiments/src/difflogic/difflogic/functional.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | 4 | BITS_TO_NP_DTYPE = {8: np.int8, 16: np.int16, 32: np.int32, 64: np.int64} 5 | 6 | 7 | # | id | Operator | AB=00 | AB=01 | AB=10 | AB=11 | 8 | # |----|----------------------|-------|-------|-------|-------| 9 | # | 0 | 0 | 0 | 0 | 0 | 0 | 10 | # | 1 | A and B | 0 | 0 | 0 | 1 | 11 | # | 2 | not(A implies B) | 0 | 0 | 1 | 0 | 12 | # | 3 | A | 0 | 0 | 1 | 1 | 13 | # | 4 | not(B implies A) | 0 | 1 | 0 | 0 | 14 | # | 5 | B | 0 | 1 | 0 | 1 | 15 | # | 6 | A xor B | 0 | 1 | 1 | 0 | 16 | # | 7 | A or B | 0 | 1 | 1 | 1 | 17 | # | 8 | not(A or B) | 1 | 0 | 0 | 0 | 18 | # | 9 | not(A xor B) | 1 | 0 | 0 | 1 | 19 | # | 10 | not(B) | 1 | 0 | 1 | 0 | 20 | # | 11 | B implies A | 1 | 0 | 1 | 1 | 21 | # | 12 | not(A) | 1 | 1 | 0 | 0 | 22 | # | 13 | A implies B | 1 | 1 | 0 | 1 | 23 | # | 14 | not(A and B) | 1 | 1 | 1 | 0 | 24 | # | 15 | 1 | 1 | 1 | 1 | 1 | 25 | 26 | def bin_op(a, b, i): 27 | assert a[0].shape == b[0].shape, (a[0].shape, b[0].shape) 28 | if a.shape[0] > 1: 29 | assert a[1].shape == b[1].shape, (a[1].shape, b[1].shape) 30 | 31 | if i == 0: 32 | return torch.zeros_like(a) 33 | elif i == 1: 34 | return a * b 35 | elif i == 2: 36 | return a - a * b 37 | elif i == 3: 38 | return a 39 | elif i == 4: 40 | return b - a * b 41 | elif i == 5: 42 | return b 43 | elif i == 6: 44 | return a + b - 2 * a * b 45 | elif i == 7: 46 | return a + b - a * b 47 | elif i == 8: 48 | return 1 - (a + b - a * b) 49 | elif i == 9: 50 | return 1 - (a + b - 2 * a * b) 51 | elif i == 10: 52 | return 1 - b 53 | elif i == 11: 54 | return 1 - b + a * b 55 | elif i == 12: 56 | return 1 - a 57 | elif i == 13: 58 | return 1 - a + a * b 59 | elif i == 14: 60 | return 1 - a * b 61 | elif i == 15: 62 | return torch.ones_like(a) 63 | 64 | 65 | def bin_op_s(a, b, i_s): 66 | r = torch.zeros_like(a) 67 | for i in range(16): 68 | u = bin_op(a, b, i) 69 | r = r + i_s[..., i] * u 70 | return r 71 | 72 | 73 | ######################################################################################################################## 74 | 75 | 76 | def get_unique_connections(in_dim, out_dim, device='cuda'): 77 | assert out_dim * 2 >= in_dim, 'The number of neurons ({}) must not be smaller than half of the number of inputs ' \ 78 | '({}) because otherwise not all inputs could be used or considered.'.format( 79 | out_dim, in_dim 80 | ) 81 | 82 | x = torch.arange(in_dim).long().unsqueeze(0) 83 | 84 | # Take pairs (0, 1), (2, 3), (4, 5), ... 85 | a, b = x[..., ::2], x[..., 1::2] 86 | if a.shape[-1] != b.shape[-1]: 87 | m = min(a.shape[-1], b.shape[-1]) 88 | a = a[..., :m] 89 | b = b[..., :m] 90 | 91 | # If this was not enough, take pairs (1, 2), (3, 4), (5, 6), ... 92 | if a.shape[-1] < out_dim: 93 | a_, b_ = x[..., 1::2], x[..., 2::2] 94 | a = torch.cat([a, a_], dim=-1) 95 | b = torch.cat([b, b_], dim=-1) 96 | if a.shape[-1] != b.shape[-1]: 97 | m = min(a.shape[-1], b.shape[-1]) 98 | a = a[..., :m] 99 | b = b[..., :m] 100 | 101 | # If this was not enough, take pairs with offsets >= 2: 102 | offset = 2 103 | while out_dim > a.shape[-1] > offset: 104 | a_, b_ = x[..., :-offset], x[..., offset:] 105 | a = torch.cat([a, a_], dim=-1) 106 | b = torch.cat([b, b_], dim=-1) 107 | offset += 1 108 | assert a.shape[-1] == b.shape[-1], (a.shape[-1], b.shape[-1]) 109 | 110 | if a.shape[-1] >= out_dim: 111 | a = a[..., :out_dim] 112 | b = b[..., :out_dim] 113 | else: 114 | assert False, (a.shape[-1], offset, out_dim) 115 | 116 | perm = torch.randperm(out_dim) 117 | 118 | a = a[:, perm].squeeze(0) 119 | b = b[:, perm].squeeze(0) 120 | 121 | a, b = a.to(torch.int64), b.to(torch.int64) 122 | a, b = a.to(device), b.to(device) 123 | a, b = a.contiguous(), b.contiguous() 124 | return a, b 125 | 126 | 127 | ######################################################################################################################## 128 | 129 | 130 | class GradFactor(torch.autograd.Function): 131 | @staticmethod 132 | def forward(ctx, x, f): 133 | ctx.f = f 134 | return x 135 | 136 | @staticmethod 137 | def backward(ctx, grad_y): 138 | return grad_y * ctx.f, None 139 | 140 | 141 | ######################################################################################################################## 142 | 143 | 144 | 145 | -------------------------------------------------------------------------------- /experiments/src/difflogic/difflogic/packbitstensor.py: -------------------------------------------------------------------------------- 1 | import difflogic_cuda 2 | import torch 3 | import numpy as np 4 | 5 | 6 | class PackBitsTensor: 7 | def __init__(self, t: torch.BoolTensor, bit_count=32, device='cuda'): 8 | 9 | assert len(t.shape) == 2, t.shape 10 | 11 | self.bit_count = bit_count 12 | self.device = device 13 | 14 | if device == 'cuda': 15 | t = t.to(device).T.contiguous() 16 | self.t, self.pad_len = difflogic_cuda.tensor_packbits_cuda(t, self.bit_count) 17 | else: 18 | raise NotImplementedError(device) 19 | 20 | def group_sum(self, k): 21 | assert self.device == 'cuda', self.device 22 | return difflogic_cuda.groupbitsum(self.t, self.pad_len, k) 23 | 24 | def flatten(self, start_dim=0, end_dim=-1, **kwargs): 25 | """ 26 | Returns the PackBitsTensor object itself. 27 | Arguments are ignored. 28 | """ 29 | return self 30 | 31 | def _get_member_repr(self, member): 32 | if len(member) <= 4: 33 | result = [(np.binary_repr(integer, width=self.bit_count))[::-1] for integer in member] 34 | return ' '.join(result) 35 | first_three = [(np.binary_repr(integer, width=self.bit_count))[::-1] for integer in member[:3]] 36 | sep = "..." 37 | final = np.binary_repr(member[-1], width=self.bit_count)[::-1] 38 | return f"{' '.join(first_three)} {sep} {final}" 39 | 40 | def __repr__(self): 41 | return '\n'.join([self._get_member_repr(item) for item in self.t]) -------------------------------------------------------------------------------- /experiments/src/difflogic/difflogic_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/torchlogic/20d10b0462b6866f3853336ef3cfad19218dcb18/experiments/src/difflogic/difflogic_logo.png -------------------------------------------------------------------------------- /experiments/src/difflogic/experiments/apply_compiled_net.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torchvision 3 | 4 | import mnist_dataset 5 | from difflogic import CompiledLogicNet 6 | 7 | torch.set_num_threads(1) 8 | 9 | dataset = 'mnist20x20' 10 | batch_size = 1_000 11 | 12 | transforms = torchvision.transforms.Compose([ 13 | torchvision.transforms.ToTensor(), 14 | torchvision.transforms.Lambda(lambda x: x.round()), 15 | ]) 16 | test_set = mnist_dataset.MNIST('./data-mnist', train=False, transform=transforms, remove_border=True) 17 | test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, shuffle=False, pin_memory=True, drop_last=True) 18 | 19 | for num_bits in [ 20 | # 8, 21 | # 16, 22 | # 32, 23 | 64 24 | ]: 25 | save_lib_path = 'lib/{:08d}_{}.so'.format(0, num_bits) 26 | compiled_model = CompiledLogicNet.load(save_lib_path, 10, num_bits) 27 | 28 | correct, total = 0, 0 29 | for (data, labels) in test_loader: 30 | data = torch.nn.Flatten()(data).bool().numpy() 31 | 32 | output = compiled_model.forward(data) 33 | 34 | correct += (output.argmax(-1) == labels).float().sum() 35 | total += output.shape[0] 36 | 37 | acc3 = correct / total 38 | print('COMPILED MODEL', num_bits, acc3) 39 | -------------------------------------------------------------------------------- /experiments/src/difflogic/experiments/requirements.txt: -------------------------------------------------------------------------------- 1 | tqdm 2 | scikit-learn 3 | torch 4 | torchvision 5 | difflogic 6 | -------------------------------------------------------------------------------- /experiments/src/difflogic/experiments/results_json.py: -------------------------------------------------------------------------------- 1 | import json 2 | import time 3 | import socket 4 | import os 5 | 6 | 7 | class ResultsJSON(object): 8 | 9 | def __init__(self, eid: int, path: str): 10 | self.eid = eid 11 | self.path = path 12 | 13 | self.init_time = time.time() 14 | self.save_time = None 15 | self.total_time = None 16 | 17 | self.args = None 18 | 19 | self.server_name = socket.gethostname().split('.')[0] 20 | 21 | def store_args(self, args): 22 | 23 | self.args = vars(args) 24 | 25 | def store_results(self, results: dict): 26 | 27 | for key, val in results.items(): 28 | if not hasattr(self, key): 29 | setattr(self, key, list()) 30 | 31 | getattr(self, key).append(val) 32 | 33 | def store_final_results(self, results: dict): 34 | 35 | for key, val in results.items(): 36 | key = key + '_' 37 | 38 | setattr(self, key, val) 39 | 40 | def save(self): 41 | self.save_time = time.time() 42 | self.total_time = self.save_time - self.init_time 43 | 44 | json_str = json.dumps(self.__dict__) 45 | 46 | with open(os.path.join(self.path, '{:08d}.json'.format(self.eid)), mode='w') as f: 47 | f.write(json_str) 48 | 49 | @staticmethod 50 | def load(eid: int, path: str, get_dict=False): 51 | with open(os.path.join(path, '{:08d}.json'.format(eid)), mode='r') as f: 52 | data = json.loads(f.read()) 53 | 54 | if get_dict: 55 | return data 56 | 57 | self = ResultsJSON(-1, '') 58 | self.__dict__.update(data) 59 | 60 | assert eid == self.eid 61 | 62 | return self 63 | 64 | 65 | if __name__ == '__main__': 66 | 67 | r = ResultsJSON(101, './') 68 | 69 | print(r.__dict__) 70 | 71 | r.save() 72 | 73 | r2 = ResultsJSON.load(101, './') 74 | 75 | print(r2.__dict__) -------------------------------------------------------------------------------- /experiments/src/difflogic/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | from torch.utils.cpp_extension import BuildExtension, CUDAExtension 3 | 4 | with open('README.md', 'r', encoding='utf-8') as fh: 5 | long_description = fh.read() 6 | 7 | setup( 8 | name='difflogic', 9 | version='0.1.0', 10 | author='Felix Petersen', 11 | author_email='ads0600@felix-petersen.de', 12 | long_description=long_description, 13 | long_description_content_type='text/markdown', 14 | url='https://github.com/Felix-Petersen/difflogic', 15 | classifiers=[ 16 | 'Programming Language :: Python :: 3', 17 | 'License :: OSI Approved :: MIT License', 18 | 'Operating System :: OS Independent', 19 | 'Topic :: Scientific/Engineering', 20 | 'Topic :: Scientific/Engineering :: Mathematics', 21 | 'Topic :: Scientific/Engineering :: Artificial Intelligence', 22 | 'Topic :: Software Development', 23 | 'Topic :: Software Development :: Libraries', 24 | 'Topic :: Software Development :: Libraries :: Python Modules', 25 | ], 26 | package_dir={'difflogic': 'difflogic'}, 27 | packages=['difflogic'], 28 | ext_modules=[CUDAExtension('difflogic_cuda', [ 29 | 'difflogic/cuda/difflogic.cpp', 30 | 'difflogic/cuda/difflogic_kernel.cu', 31 | ], extra_compile_args={'nvcc': ['-lineinfo']})], 32 | cmdclass={'build_ext': BuildExtension}, 33 | python_requires='>=3.6', 34 | install_requires=[ 35 | 'torch>=1.6.0', 36 | 'numpy', 37 | ], 38 | ) 39 | -------------------------------------------------------------------------------- /experiments/src/encoders.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from sklearn.pipeline import make_pipeline 3 | from sklearn.compose import ColumnTransformer 4 | from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder, LabelEncoder 5 | 6 | 7 | class FeatureEncoder: 8 | 9 | def __init__( 10 | self, 11 | numerical_features: list, 12 | categorical_features: list, 13 | ordinal_features: list, 14 | label_encode_categorical: bool 15 | ): 16 | self.numerical_features = numerical_features 17 | self.categorical_features = categorical_features 18 | self.ordinal_features = ordinal_features 19 | self.label_encode_categorical = label_encode_categorical 20 | self.mixed_pipe = None 21 | 22 | def _create_pipeline(self, features: pd.DataFrame): 23 | n_unique_categories = features[self.categorical_features].nunique().sort_values(ascending=False) 24 | high_cardinality_features = n_unique_categories[n_unique_categories > 255].index 25 | low_cardinality_features = n_unique_categories[n_unique_categories <= 255].index 26 | 27 | if not self.label_encode_categorical: 28 | mixed_encoded_preprocessor = ColumnTransformer( 29 | [ 30 | ("numerical", "passthrough", self.numerical_features), 31 | ( 32 | "high_cardinality", 33 | OneHotEncoder(handle_unknown="ignore", sparse_output=False), 34 | high_cardinality_features, 35 | ), 36 | ( 37 | "low_cardinality", 38 | OneHotEncoder(handle_unknown="ignore", sparse_output=False), 39 | low_cardinality_features, 40 | ), 41 | # ( 42 | # "ordinal", 43 | # OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-1), 44 | # self.ordinal_features, 45 | # ), 46 | ( 47 | "ordinal_onehot", 48 | OneHotEncoder(handle_unknown="ignore", sparse_output=False), 49 | self.ordinal_features, 50 | ), 51 | ], 52 | verbose_feature_names_out=False, 53 | ) 54 | 55 | mixed_encoded_preprocessor.set_output(transform='pandas') 56 | 57 | else: 58 | mixed_encoded_preprocessor = ColumnTransformer( 59 | [ 60 | ("numerical", "passthrough", self.numerical_features), 61 | ( 62 | "high_cardinality", 63 | OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-1), 64 | high_cardinality_features, 65 | ), 66 | ( 67 | "low_cardinality", 68 | OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-1), 69 | low_cardinality_features, 70 | ), 71 | # ( 72 | # "ordinal", 73 | # OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-1), 74 | # self.ordinal_features, 75 | # ), 76 | ( 77 | "ordinal_onehot", 78 | OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-1), 79 | self.ordinal_features, 80 | ), 81 | ], 82 | verbose_feature_names_out=False, 83 | ) 84 | 85 | self.mixed_pipe = make_pipeline( 86 | mixed_encoded_preprocessor, 87 | ) 88 | 89 | def fit_transform(self, features: pd.DataFrame): 90 | self._create_pipeline(features=features) 91 | out = self.mixed_pipe.fit_transform(features) 92 | if not isinstance(out, pd.DataFrame): 93 | out = pd.DataFrame(out) 94 | return out 95 | 96 | def fit(self, features: pd.DataFrame): 97 | self._create_pipeline(features=features) 98 | out = self.mixed_pipe.fit(features) 99 | if not isinstance(out, pd.DataFrame): 100 | out = pd.DataFrame(out) 101 | return out 102 | 103 | def transform(self, features: pd.DataFrame): 104 | out = self.mixed_pipe.transform(features) 105 | if not isinstance(out, pd.DataFrame): 106 | out = pd.DataFrame(out) 107 | return out 108 | -------------------------------------------------------------------------------- /experiments/src/iris_datasets.py: -------------------------------------------------------------------------------- 1 | import openml 2 | import pandas as pd 3 | from sklearn.model_selection import train_test_split 4 | from sklearn.preprocessing import LabelEncoder, MinMaxScaler 5 | 6 | 7 | class IRISDataset: 8 | def __init__(self, openml_id, test_size, val_size, random_state): 9 | mms = MinMaxScaler() 10 | encoder = LabelEncoder() 11 | 12 | uc = pd.HDFStore("iris.transformed.10class.h5") 13 | df = uc.select_as_multiple( 14 | ['df{}'.format(i) for i in range(len(uc.keys()))], 15 | selector='df0') 16 | uc.close() 17 | 18 | df = df.drop(["SNAPSHOT_DATE"], axis=1) 19 | df = df.drop(["CUST_ID"], axis=1) 20 | df = df.drop(["COUNTRY_CODE"], axis=1) 21 | 22 | target_features = [x for x in list(df.columns) if x.endswith("_label")] 23 | target_values = target_features 24 | X = df.drop(target_features, axis=1) 25 | feature_names=X.columns 26 | y = df[target_features] 27 | 28 | train_data, test_data = train_test_split( 29 | df, 30 | test_size=test_size, 31 | random_state=random_state, 32 | ) 33 | 34 | train_data, val_data = train_test_split( 35 | df, 36 | test_size=val_size, 37 | random_state=random_state, 38 | ) 39 | 40 | X_train = train_data.drop(target_features, axis=1) 41 | X_val = val_data.drop(target_features, axis=1) 42 | X_test = test_data.drop(target_features, axis=1) 43 | 44 | y_train = train_data[target_features].values 45 | y_val = val_data[target_features].values 46 | y_test = test_data[target_features].values 47 | 48 | self.numerical_features = X.columns 49 | self.categorical_features = [] 50 | self.ordinal_features = [] 51 | 52 | self.X = X 53 | self.y = y 54 | self.feature_names = feature_names 55 | self.target_values = target_values 56 | self.df = df 57 | self.train_data = train_data 58 | self.val_data = val_data 59 | self.test_data = test_data 60 | self.X_train = X_train 61 | self.X_val = X_val 62 | self.X_test = X_test 63 | self.y_train = y_train 64 | self.y_val = y_val 65 | self.y_test = y_test 66 | 67 | 68 | __all__ = ["IRISDataset"] 69 | -------------------------------------------------------------------------------- /experiments/src/neural_additive_models/README.md: -------------------------------------------------------------------------------- 1 | ## Neural Additive Models: Interpretable Machine Learning with Neural Nets 2 | 3 | # [![Website](https://img.shields.io/badge/www-Website-green)](https://neural-additive-models.github.io) [![Visualization Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1E3_t7Inhol-qVPmFNq1Otj9sWt1vU_DQ?usp=sharing) 4 | 5 | 6 | This repository contains open-source code 7 | for the paper 8 | [Neural Additive Models: Interpretable Machine Learning with Neural Nets](https://arxiv.org/abs/2004.13912). 9 | 10 | Neural Additive Model 11 | 12 | Currently, 13 | we release the `tf.keras.Model` for NAM which can be simply plugged into any neural network training procedure. We also provide helpers for 14 | building a computation graph using NAM for classification/regression problems with `tf.compat.v1`. 15 | The `nam_train.py` file provides the example of a training script on a single 16 | dataset split. 17 | 18 | Use `./run.sh` test script to ensure that the setup is correct. 19 | 20 | ## Multi-task NAMs 21 | The code for multi task NAMs can be found at [https://github.com/lemeln/nam](https://github.com/lemeln/nam). 22 | 23 | ## Dependencies 24 | 25 | The code was tested under Ubuntu 16 and uses these packages: 26 | 27 | - tensorflow>=1.15 28 | - numpy>=1.15.2 29 | - sklearn>=0.23 30 | - pandas>=0.24 31 | - absl-py 32 | 33 | ## Datasets 34 | 35 | The datasets used in the paper (except MIMIC-II) can be found in the public GCP bucket `gs://nam_datasets/data`, which can be downloaded using [gsutil][gsutil]. To install gsutil, follow the instructions [here][gsutil_install]. The preprocessed version of MIMIC-II dataset, used in the NAM paper, can be 36 | shared only if you provide us with the signed data use agreement to the MIMIC-III Clinical 37 | Database on the PhysioNet website. 38 | 39 | Citing 40 | ------ 41 | If you use this code in your research, please cite the following paper: 42 | 43 | > Agarwal, R., Melnick, L., Frosst, N., Zhang, X., Lengerich, B., Caruana, 44 | > R., & Hinton, G. E. (2021). Neural additive models: Interpretable machine > learning with neural nets. Advances in Neural Information Processing 45 | > Systems, 34. 46 | 47 | @article{agarwal2021neural, 48 | title={Neural additive models: Interpretable machine learning with neural nets}, 49 | author={Agarwal, Rishabh and Melnick, Levi and Frosst, Nicholas and Zhang, Xuezhou and Lengerich, Ben and Caruana, Rich and Hinton, Geoffrey E}, 50 | journal={Advances in Neural Information Processing Systems}, 51 | volume={34}, 52 | year={2021} 53 | } 54 | 55 | --- 56 | 57 | *Disclaimer about COMPAS dataset: It is important to note that 58 | developing a machine learning model to predict pre-trial detention has a 59 | number of important ethical considerations. You can learn more about these 60 | issues in the Partnership on AI 61 | [Report on Algorithmic Risk Assessment Tools in the U.S. Criminal Justice System](https://www.partnershiponai.org/report-on-machine-learning-in-risk-assessment-tools-in-the-u-s-criminal-justice-system/). 62 | The Partnership on AI is a multi-stakeholder organization -- of which Google 63 | is a member -- that creates guidelines around AI.* 64 | 65 | *We’re using the COMPAS dataset only as an example of how to identify and 66 | remediate fairness concerns in data. This dataset is canonical in the 67 | algorithmic fairness literature.* 68 | 69 | *Disclaimer: This is not an official Google product.* 70 | 71 | [gsutil_install]: https://cloud.google.com/storage/docs/gsutil_install#install 72 | [gsutil]: https://cloud.google.com/storage/docs/gsutil 73 | -------------------------------------------------------------------------------- /experiments/src/neural_additive_models/__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2024 The Google Research Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | -------------------------------------------------------------------------------- /experiments/src/neural_additive_models/nam_train_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2024 The Google Research Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Tests functionality of training NAM models.""" 17 | 18 | import os 19 | 20 | from absl import flags 21 | from absl.testing import absltest 22 | from absl.testing import flagsaver 23 | from absl.testing import parameterized 24 | import tensorflow.compat.v1 as tf 25 | 26 | from neural_additive_models import nam_train 27 | 28 | FLAGS = flags.FLAGS 29 | 30 | 31 | class NAMTrainingTest(parameterized.TestCase): 32 | """Tests whether NAMs can be run without error.""" 33 | 34 | @parameterized.named_parameters( 35 | ('classification', 'BreastCancer', False), 36 | ('regression', 'Housing', True), 37 | ) 38 | @flagsaver.flagsaver 39 | def test_nam(self, dataset_name, regression): 40 | """Test whether the NAM training pipeline runs successfully or not.""" 41 | FLAGS.training_epochs = 4 42 | FLAGS.save_checkpoint_every_n_epochs = 2 43 | FLAGS.early_stopping_epochs = 2 44 | FLAGS.dataset_name = dataset_name 45 | FLAGS.regression = regression 46 | FLAGS.num_basis_functions = 16 47 | logdir = os.path.join(self.create_tempdir().full_path, dataset_name) 48 | tf.gfile.MakeDirs(logdir) 49 | data_gen, _ = nam_train.create_test_train_fold(fold_num=1) 50 | nam_train.single_split_training(data_gen, logdir) 51 | 52 | 53 | if __name__ == '__main__': 54 | absltest.main() 55 | -------------------------------------------------------------------------------- /experiments/src/neural_additive_models/requirements.txt: -------------------------------------------------------------------------------- 1 | tensorflow>=1.15, <=2.13 2 | numpy>=1.15.2 3 | sklearn 4 | pandas>=0.24 5 | absl-py 6 | -------------------------------------------------------------------------------- /experiments/src/neural_additive_models/run.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2024 The Google Research Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | #!/bin/bash 16 | set -e 17 | set -x 18 | 19 | virtualenv -p python3 . 20 | source ./bin/activate 21 | 22 | pip install -r neural_additive_models/requirements.txt 23 | python -m neural_additive_models.nam_train_test.py 24 | -------------------------------------------------------------------------------- /experiments/src/neural_additive_models/setup.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2024 The Google Research Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Setup script for NAMs.""" 17 | 18 | import pathlib 19 | from setuptools import find_packages 20 | from setuptools import setup 21 | 22 | here = pathlib.Path(__file__).parent.resolve() 23 | 24 | long_description = (here / 'README.md').read_text(encoding='utf-8') 25 | 26 | install_requires = [ 27 | 'tensorflow>=1.15', 28 | 'numpy>=1.15.2', 29 | 'sklearn', 30 | 'pandas>=0.24', 31 | 'absl-py', 32 | ] 33 | 34 | nam_description = ('Neural Additive Models: Intepretable ML with Neural Nets') 35 | 36 | setup( 37 | name='neural_additive_models', 38 | version=0.1, 39 | description=nam_description, 40 | long_description=long_description, 41 | long_description_content_type='text/markdown', 42 | url='https://github.com/agarwl/google-research/tree/master/neural_additive_models', 43 | author='Rishabh Agarwal', 44 | classifiers=[ 45 | 'Development Status :: 4 - Beta', 46 | 47 | 'Intended Audience :: Developers', 48 | 'Intended Audience :: Education', 49 | 'Intended Audience :: Science/Research', 50 | 51 | 'License :: OSI Approved :: Apache Software License', 52 | 53 | 'Programming Language :: Python :: 3', 54 | 'Programming Language :: Python :: 3.5', 55 | 'Programming Language :: Python :: 3.6', 56 | 'Programming Language :: Python :: 3.7', 57 | 'Programming Language :: Python :: 3.8', 58 | 'Programming Language :: Python :: 3 :: Only', 59 | 60 | 'Topic :: Scientific/Engineering', 61 | 'Topic :: Scientific/Engineering :: Mathematics', 62 | 'Topic :: Scientific/Engineering :: Artificial Intelligence', 63 | 'Topic :: Software Development', 64 | 'Topic :: Software Development :: Libraries', 65 | 'Topic :: Software Development :: Libraries :: Python Modules', 66 | 67 | ], 68 | keywords='nam, interpretability, machine, learning, research', 69 | include_package_data=True, 70 | packages=find_packages(exclude=['docs']), 71 | install_requires=install_requires, 72 | license='Apache 2.0', 73 | ) 74 | -------------------------------------------------------------------------------- /experiments/src/neural_additive_models/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2024 The Google Research Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | -------------------------------------------------------------------------------- /experiments/src/neural_additive_models/tests/data_utils_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2024 The Google Research Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Tests functionality of loading the different datasets.""" 17 | 18 | from absl.testing import absltest 19 | from absl.testing import parameterized 20 | 21 | import numpy as np 22 | from neural_additive_models import data_utils 23 | 24 | 25 | class LoadDataTest(parameterized.TestCase): 26 | """Data Loading Tests.""" 27 | 28 | @parameterized.named_parameters( 29 | ('breast_cancer', 'BreastCancer', 569), 30 | ('fico', 'Fico', 9861), 31 | ('housing', 'Housing', 20640), 32 | ('recidivism', 'Recidivism', 6172), 33 | ('credit', 'Credit', 284807), 34 | ('adult', 'Adult', 32561), 35 | ('telco', 'Telco', 7043)) 36 | def test_data(self, dataset_name, dataset_size): 37 | """Test whether a dataset is loaded properly with specified size.""" 38 | x, y, _ = data_utils.load_dataset(dataset_name) 39 | self.assertIsInstance(x, np.ndarray) 40 | self.assertIsInstance(y, np.ndarray) 41 | self.assertLen(x, dataset_size) 42 | 43 | if __name__ == '__main__': 44 | absltest.main() 45 | -------------------------------------------------------------------------------- /experiments/src/neural_additive_models/tests/graph_builder_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2024 The Google Research Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Tests functionality of building tensorflow graph.""" 17 | 18 | from absl.testing import absltest 19 | from absl.testing import parameterized 20 | 21 | import tensorflow.compat.v1 as tf 22 | from neural_additive_models import data_utils 23 | from neural_additive_models import graph_builder 24 | 25 | 26 | class GraphBuilderTest(parameterized.TestCase): 27 | """Tests whether neural net models can be run without error.""" 28 | 29 | @parameterized.named_parameters(('classification', 'BreastCancer', False), 30 | ('regression', 'Housing', True)) 31 | def test_build_graph(self, dataset_name, regression): 32 | """Test whether build_graph works as expected.""" 33 | data_x, data_y, _ = data_utils.load_dataset(dataset_name) 34 | data_gen = data_utils.split_training_dataset( 35 | data_x, data_y, n_splits=5, stratified=not regression) 36 | (x_train, y_train), (x_validation, y_validation) = next(data_gen) 37 | sess = tf.InteractiveSession() 38 | graph_tensors_and_ops, metric_scores = graph_builder.build_graph( 39 | x_train=x_train, 40 | y_train=y_train, 41 | x_test=x_validation, 42 | y_test=y_validation, 43 | activation='exu', 44 | learning_rate=1e-3, 45 | batch_size=256, 46 | shallow=True, 47 | regression=regression, 48 | output_regularization=0.1, 49 | dropout=0.1, 50 | decay_rate=0.999, 51 | name_scope='model', 52 | l2_regularization=0.1) 53 | # Run initializer ops 54 | sess.run(tf.global_variables_initializer()) 55 | sess.run([ 56 | graph_tensors_and_ops['iterator_initializer'], 57 | graph_tensors_and_ops['running_vars_initializer'] 58 | ]) 59 | for _ in range(2): 60 | sess.run(graph_tensors_and_ops['train_op']) 61 | self.assertIsInstance(metric_scores['train'](sess), float) 62 | sess.close() 63 | 64 | 65 | if __name__ == '__main__': 66 | absltest.main() 67 | -------------------------------------------------------------------------------- /experiments/src/neural_additive_models/tests/models_test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2024 The Google Research Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Tests functionality of loading different models.""" 17 | 18 | from absl.testing import absltest 19 | from absl.testing import parameterized 20 | 21 | import numpy as np 22 | import tensorflow.compat.v1 as tf 23 | from neural_additive_models import models 24 | 25 | 26 | class LoadModelsTest(parameterized.TestCase): 27 | """Tests whether neural net models can be run without error.""" 28 | 29 | @parameterized.named_parameters(('exu_nam', 'exu_nam'), 30 | ('relu_nam', 'relu_nam'), ('dnn', 'dnn')) 31 | def test_model(self, architecture): 32 | """Test whether a model with specified architecture can be run.""" 33 | x = np.random.rand(5, 10).astype('float32') 34 | sess = tf.InteractiveSession() 35 | if architecture == 'exu_nam': 36 | model = models.NAM( 37 | num_inputs=x.shape[1], num_units=1024, shallow=True, activation='exu') 38 | elif architecture == 'relu_nam': 39 | model = models.NAM( 40 | num_inputs=x.shape[1], num_units=64, shallow=False, activation='relu') 41 | elif architecture == 'dnn': 42 | model = models.DNN() 43 | else: 44 | raise ValueError('Architecture {} not found'.format(architecture)) 45 | out_op = model(x) 46 | sess.run(tf.global_variables_initializer()) 47 | self.assertIsInstance(sess.run(out_op), np.ndarray) 48 | sess.close() 49 | 50 | 51 | if __name__ == '__main__': 52 | absltest.main() 53 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: "torchlogic" 2 | repo_url: "" 3 | docs_dir: 'docs' 4 | 5 | nav: 6 | - Welcome: 7 | - Overview: 'index.md' 8 | - Installation: 'install/install.md' 9 | - Tutorials: 10 | - Bandit-RRN: 'tutorials/brn.md' 11 | - Data Science: 12 | - torchlogic Overview: 'ds/overview.md' 13 | - Neural Reasoning Networks: 'ds/rn.md' 14 | - Bandit-NRN Algorithm: 'ds/brn.md' 15 | - Use Cases in AI Workflows: 'ds/usecases.md' 16 | - API Reference: 17 | - nn: 'reference/nn.md' 18 | - modules: 'reference/modules.md' 19 | - models: 'reference/models.md' 20 | - utils: 'reference/utils.md' 21 | 22 | theme: 23 | name: material 24 | language: en 25 | logo: static/torchlogic_logo_black.png 26 | 27 | plugins: 28 | - search 29 | - git-revision-date 30 | - autorefs 31 | - mkdocstrings: 32 | default_handler: python 33 | handlers: 34 | python: 35 | setup_commands: 36 | - import sys 37 | - sys.path.append("./torchlogic") 38 | 39 | markdown_extensions: 40 | - admonition 41 | - attr_list 42 | - def_list 43 | - pymdownx.tasklist: 44 | custom_checkbox: true 45 | - pymdownx.emoji 46 | - pymdownx.magiclink 47 | - pymdownx.snippets: 48 | check_paths: true 49 | - pymdownx.superfences: 50 | - pymdownx.tabbed 51 | - pymdownx.tasklist 52 | - mkdocs-click 53 | - pymdownx.details 54 | - footnotes 55 | 56 | extra_css: 57 | - 'formatting/extra.css' 58 | -------------------------------------------------------------------------------- /notebooks/tutorials/configs/logging.yaml: -------------------------------------------------------------------------------- 1 | version: 1 2 | disable_existing_loggers: False 3 | formatters: 4 | simple: 5 | format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s" 6 | 7 | handlers: 8 | console: 9 | # for simple logging use logging.StreamHandler 10 | class: carrot.logger.ColorLogHandler 11 | level: DEBUG 12 | formatter: simple 13 | stream: ext://sys.stdout 14 | 15 | info_file_handler: 16 | class: carrot.logger.IndefiniteRotatingFileHandler 17 | level: INFO 18 | formatter: simple 19 | filename: info.log 20 | maxBytes: 10485760 # 10MB 21 | backupCount: 20 22 | encoding: utf8 23 | delay: True 24 | 25 | error_file_handler: 26 | class: logging.handlers.RotatingFileHandler 27 | level: ERROR 28 | formatter: simple 29 | filename: errors.log 30 | maxBytes: 10485760 # 10MB 31 | backupCount: 20 32 | encoding: utf8 33 | delay: True 34 | 35 | loggers: 36 | anyconfig: 37 | level: WARNING 38 | handlers: [console] 39 | propagate: no 40 | 41 | root: 42 | level: INFO 43 | handlers: [console, info_file_handler, error_file_handler] -------------------------------------------------------------------------------- /project.yaml: -------------------------------------------------------------------------------- 1 | CarrotProjectDetails: 2 | carrot_version: 1.4.1 3 | project_repo_name: torchlogic 4 | tracking_enabled: false 5 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "torchlogic" 3 | version = "0.0.3-beta" 4 | authors = ["Anonymous"] 5 | description = "A PyTorch framework for rapidly developing Neural Reasoning Networks." 6 | readme = "README.md" 7 | 8 | [tool.poetry.dependencies] 9 | python = ">=3.9.5, <3.11" 10 | torch = "^2.0.0" 11 | numpy = "^1.25.0" 12 | pandas = "^2.0.2" 13 | scipy = "^1.10.1" 14 | scikit-learn = "^1.2.2" 15 | pytest = "^7.4.0" 16 | pytest-cov = "^4.1.0" 17 | pytorch_optimizer = "^2.12.0" 18 | torchvision = "^0.16" 19 | xgboost = "^2.0.2" 20 | setuptools = "^69.0.2" 21 | aix360 = "^0.3.0" 22 | minepy = "^1.2.6" 23 | cvxpy = "^1.5.3" 24 | 25 | [build-system] 26 | requires = ["poetry-core"] 27 | build-backend = "poetry.core.masonry.api" 28 | 29 | [tool.isort] 30 | multi_line_output = 3 31 | include_trailing_comma = true 32 | force_grid_wrap = 0 33 | line_length = 120 34 | default_section = "THIRDPARTY" 35 | #NOTE: Do not use use_paranthesis setting as it is not compatible with black 36 | 37 | [tool.black] 38 | exclude = "^tests/" 39 | line-length = 120 40 | skip-string-normalization = true 41 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | if __name__ == "__main__": 4 | with open("requirements.txt") as f: 5 | requirements = f.read().splitlines() 6 | 7 | setup( 8 | name="torchlogic", 9 | packages=find_packages('torchlogic'), 10 | package_dir={'': 'torchlogic'}, 11 | version="0.0.3-beta", 12 | description="A PyTorch framework for rapidly developing Neural Reasoning Networks.", 13 | classifiers=["Programming Language :: Python :: 3", "Operating System :: OS Independent"], 14 | authors="Anonymous", 15 | python_requires=">=3.6", 16 | install_requires=requirements, 17 | ) -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | # tests 2 | For storing test codes, using libraries such as pytest, unittest. 3 | -------------------------------------------------------------------------------- /tests/models/test__boosted_brrn.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import numpy as np 4 | import pandas as pd 5 | 6 | import torch 7 | from torch.utils.data import Dataset, DataLoader 8 | 9 | from torchlogic.models.base import BoostedBanditNRNModel 10 | from torchlogic.utils.trainers import BoostedBanditNRNTrainer 11 | 12 | from pytest import fixture 13 | 14 | ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) 15 | PARENT_DIR = os.path.abspath(ROOT_DIR) 16 | 17 | 18 | class TestBoostedBanditRRN: 19 | 20 | @fixture 21 | def data_loaders(self): 22 | class StaticDataset(Dataset): 23 | def __init__(self): 24 | super(StaticDataset, self).__init__() 25 | self.X = pd.DataFrame( 26 | {'feature1': [1, 0, 1, 0] * 1000, 27 | 'feature2': [0, 1, 0, 1] * 1000}).values 28 | self.y = pd.DataFrame( 29 | {'class1': [1, 0, 1, 0] * 1000, 30 | 'class2': [0, 1, 0, 1] * 1000}).values 31 | self.sample_idx = np.arange(self.X.shape[0]) # index of samples 32 | 33 | def __len__(self): 34 | return self.X.shape[0] 35 | 36 | def __getitem__(self, idx): 37 | features = torch.from_numpy(self.X[idx, :]).float() 38 | target = torch.from_numpy(self.y[idx, :]) 39 | return {'features': features, 'target': target, 'sample_idx': idx} 40 | 41 | return (DataLoader(StaticDataset(), batch_size=1000, shuffle=False), 42 | DataLoader(StaticDataset(), batch_size=1000, shuffle=False)) 43 | 44 | @fixture 45 | def model(self): 46 | model = BoostedBanditNRNModel( 47 | target_names=['class1', 'class2'], 48 | feature_names=['feat1', 'feat1'], 49 | input_size=2, 50 | output_size=2, 51 | layer_sizes=[5, ], 52 | n_selected_features_input=2, 53 | n_selected_features_internal=2, 54 | n_selected_features_output=2, 55 | perform_prune_quantile=0.5, 56 | ucb_scale=2.5 57 | ) 58 | 59 | for item in model.rn.state_dict(): 60 | if item.find('weights') > -1: 61 | weights = model.rn.state_dict()[item] 62 | weights.data.copy_(torch.zeros_like(weights.data)) 63 | 64 | return model 65 | 66 | @fixture 67 | def trainer(self, model): 68 | return BoostedBanditNRNTrainer( 69 | model, 70 | torch.nn.BCELoss(), 71 | torch.optim.Adam(model.rn.parameters()), 72 | perform_prune_plateau_count=0, 73 | augment=None 74 | ) 75 | 76 | @staticmethod 77 | def test__predict(trainer, data_loaders): 78 | train_dl, val_dl = data_loaders 79 | trainer.boost(train_dl) 80 | predictions, all_targets = trainer.model.predict(val_dl) 81 | class_predictions = np.array(np.vstack([[True, False], [False, True], [True, False], [False, True]] * 1000)) 82 | assert np.equal(predictions.values > 0.5, class_predictions).all(), "did not make correct boosted predictions" 83 | -------------------------------------------------------------------------------- /tests/modules/test_attn.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torchlogic.modules import AttentionNRNModule 3 | from torchlogic.nn import (ConcatenateBlocksLogic, AttentionLukasiewiczChannelAndBlock, 4 | AttentionLukasiewiczChannelOrBlock) 5 | 6 | 7 | class TestAttnNRNModule: 8 | 9 | @staticmethod 10 | def test__init(): 11 | 12 | layer_sizes = [5, 5] 13 | input_size=4 14 | output_size = 2 15 | 16 | module = AttentionNRNModule( 17 | input_size, 18 | output_size, 19 | layer_sizes, 20 | feature_names=['feat1', 'feat1', 'feat3', 'feat4'], 21 | add_negations=False, 22 | weight_init=0.2, 23 | attn_emb_dim=8, 24 | attn_n_layers=2, 25 | normal_form='dnf' 26 | ) 27 | 28 | for i in range(len(module.model)): 29 | if i % 2 == 0: 30 | assert isinstance(module.model[i], AttentionLukasiewiczChannelAndBlock), "incorrect block type" 31 | else: 32 | assert isinstance(module.model[i], AttentionLukasiewiczChannelOrBlock), "incorrect block type" 33 | assert module.model[i].channels == output_size, "channels is incorrect" 34 | assert isinstance(module.output_layer, AttentionLukasiewiczChannelAndBlock), \ 35 | "output_layer should be of type 'AttentionLukasiewiczChannelAndBlock'" 36 | assert len(module.model) == len(layer_sizes), "number of layers is incorrect" 37 | assert module.model[0].in_features == input_size, \ 38 | "disjuction input layer's input size is incorrect" 39 | assert module.output_layer.channels == output_size 40 | 41 | module = AttentionNRNModule( 42 | input_size, 43 | output_size, 44 | layer_sizes, 45 | feature_names=['feat1', 'feat1', 'feat3', 'feat4'], 46 | add_negations=True, 47 | weight_init=0.2, 48 | attn_emb_dim=8, 49 | attn_n_layers=2, 50 | normal_form='cnf' 51 | ) 52 | 53 | for i in range(len(module.model)): 54 | if i % 2 == 0: 55 | assert isinstance(module.model[i], AttentionLukasiewiczChannelOrBlock), "incorrect block type" 56 | else: 57 | assert isinstance(module.model[i], AttentionLukasiewiczChannelAndBlock), "incorrect block type" 58 | assert module.model[i].channels == output_size, "channels is incorrect" 59 | assert isinstance(module.output_layer, AttentionLukasiewiczChannelOrBlock), \ 60 | "output_layer should be of type 'AttentionLukasiewiczChannelOrBlock'" 61 | assert module.output_layer.channels == output_size 62 | 63 | @staticmethod 64 | def test__forward(): 65 | layer_sizes = [5, 5] 66 | input_size = 4 67 | output_size = 2 68 | 69 | module = AttentionNRNModule( 70 | input_size, 71 | output_size, 72 | layer_sizes, 73 | feature_names=['feat1', 'feat1', 'feat3', 'feat4'], 74 | ) 75 | 76 | x = torch.ones(10, 4) 77 | output = module(x) 78 | 79 | assert output.size() == (x.size(0), output_size) 80 | assert output.min() >= 0 81 | assert output.max() <= 1 82 | 83 | 84 | -------------------------------------------------------------------------------- /tests/modules/test_brnn.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torchlogic.modules import BanditNRNModule 3 | 4 | 5 | class TestBaseClassifier: 6 | 7 | @staticmethod 8 | def test__init(): 9 | 10 | layer_sizes = [5, 5] 11 | input_size=4 12 | output_size = 2 13 | 14 | module = BanditNRNModule( 15 | input_size, 16 | output_size, 17 | layer_sizes, 18 | feature_names=['feat1', 'feat1', 'feat3', 'feat4'], 19 | n_selected_features_input=2, 20 | n_selected_features_internal=1, 21 | n_selected_features_output=1, 22 | perform_prune_quantile=0.5, 23 | ucb_scale=2.5 24 | ) 25 | 26 | assert len(module.model) == len(layer_sizes) 27 | assert module.model[0].in_features == input_size 28 | assert all([m.channels == output_size for m in module.model]) 29 | assert module.output_layer.channels == output_size 30 | 31 | @staticmethod 32 | def test__forward(): 33 | layer_sizes = [5, 5] 34 | input_size = 4 35 | output_size = 2 36 | 37 | module = BanditNRNModule( 38 | input_size, 39 | output_size, 40 | layer_sizes, 41 | feature_names=['feat1', 'feat1', 'feat3', 'feat4'], 42 | n_selected_features_input=2, 43 | n_selected_features_internal=1, 44 | n_selected_features_output=1, 45 | perform_prune_quantile=0.5, 46 | ucb_scale=2.5 47 | ) 48 | 49 | x = torch.ones(10, 4) 50 | output = module(x) 51 | 52 | assert output.size() == (x.size(0), output_size) 53 | assert output.min() >= 0 54 | assert output.max() <= 1 55 | 56 | 57 | -------------------------------------------------------------------------------- /tests/modules/test_var.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torchlogic.modules import VarNRNModule 3 | from torchlogic.nn import (ConcatenateBlocksLogic, VariationalLukasiewiczChannelAndBlock, 4 | VariationalLukasiewiczChannelOrBlock) 5 | 6 | 7 | class TestAttnNRNModule: 8 | 9 | @staticmethod 10 | def test__init(): 11 | 12 | layer_sizes = [5, 5] 13 | input_size=4 14 | output_size = 2 15 | 16 | module = VarNRNModule( 17 | input_size, 18 | output_size, 19 | layer_sizes, 20 | feature_names=['feat1', 'feat1', 'feat3', 'feat4'], 21 | add_negations=False, 22 | weight_init=0.2, 23 | var_emb_dim=8, 24 | var_n_layers=2, 25 | normal_form='dnf' 26 | ) 27 | 28 | for i in range(len(module.model)): 29 | if i % 2 == 0: 30 | assert isinstance(module.model[i], VariationalLukasiewiczChannelAndBlock), "incorrect block type" 31 | else: 32 | assert isinstance(module.model[i], VariationalLukasiewiczChannelOrBlock), "incorrect block type" 33 | assert module.model[i].channels == output_size, "channels is incorrect" 34 | assert isinstance(module.output_layer, VariationalLukasiewiczChannelAndBlock), \ 35 | "output_layer should be of type 'AttentionLukasiewiczChannelAndBlock'" 36 | assert len(module.model) == len(layer_sizes), "number of layers is incorrect" 37 | assert module.model[0].in_features == input_size, \ 38 | "disjuction input layer's input size is incorrect" 39 | assert module.output_layer.channels == output_size 40 | 41 | module = VarNRNModule( 42 | input_size, 43 | output_size, 44 | layer_sizes, 45 | feature_names=['feat1', 'feat1', 'feat3', 'feat4'], 46 | add_negations=True, 47 | weight_init=0.2, 48 | var_emb_dim=8, 49 | var_n_layers=2, 50 | normal_form='cnf' 51 | ) 52 | 53 | for i in range(len(module.model)): 54 | if i % 2 == 0: 55 | assert isinstance(module.model[i], VariationalLukasiewiczChannelOrBlock), "incorrect block type" 56 | else: 57 | assert isinstance(module.model[i], VariationalLukasiewiczChannelAndBlock), "incorrect block type" 58 | assert module.model[i].channels == output_size, "channels is incorrect" 59 | assert isinstance(module.output_layer, VariationalLukasiewiczChannelOrBlock), \ 60 | "output_layer should be of type 'AttentionLukasiewiczChannelOrBlock'" 61 | assert module.output_layer.channels == output_size 62 | 63 | @staticmethod 64 | def test__forward(): 65 | layer_sizes = [5, 5] 66 | input_size = 4 67 | output_size = 2 68 | 69 | module = VarNRNModule( 70 | input_size, 71 | output_size, 72 | layer_sizes, 73 | feature_names=['feat1', 'feat1', 'feat3', 'feat4'], 74 | ) 75 | 76 | x = torch.ones(10, 4) 77 | output = module(x) 78 | 79 | assert output.size() == (x.size(0), output_size) 80 | assert output.min() >= 0 81 | assert output.max() <= 1 82 | 83 | 84 | -------------------------------------------------------------------------------- /tests/nn/test_forward_blocks.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | 4 | from torchlogic.nn import LukasiewiczChannelAndBlock, LukasiewiczChannelOrBlock, LukasiewiczChannelXOrBlock, Predicates 5 | 6 | from pytest import fixture 7 | 8 | ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) 9 | PARENT_DIR = os.path.abspath(ROOT_DIR) 10 | 11 | 12 | class TestLukasiewiczChannelBlock: 13 | 14 | @fixture 15 | def predicates(self): 16 | return Predicates(['feat1', 'feat2']) 17 | 18 | @staticmethod 19 | def test__forward_lukasiewicz_channel_block(predicates): 20 | block = LukasiewiczChannelAndBlock( 21 | channels=2, 22 | in_features=10, 23 | out_features=4, 24 | n_selected_features=5, 25 | parent_weights_dimension='channels', 26 | operands=predicates, 27 | outputs_key='0' 28 | ) 29 | 30 | # INPUT: [BATCH_SIZE, 1, IN_FEATURES] OR [BATCH_SIZE, CHANNELS, 1, IN_FEATURES] 31 | x = torch.randn(32, 1, 10) 32 | out = block(x) 33 | # OUTPUT: [BATCH_SIZE, OUT_CHANNELS, 1, OUT_FEATURES] 34 | assert out.size() == (32, 2, 1, 4) 35 | assert torch.all(out <= 1.0) 36 | assert torch.all(out >= 0.0) 37 | 38 | block = LukasiewiczChannelOrBlock( 39 | channels=2, 40 | in_features=10, 41 | out_features=4, 42 | n_selected_features=5, 43 | parent_weights_dimension='channels', 44 | operands=predicates, 45 | outputs_key='0' 46 | ) 47 | 48 | # INPUT: [BATCH_SIZE, 1, IN_FEATURES] OR [BATCH_SIZE, CHANNELS, 1, IN_FEATURES] 49 | x = torch.randn(32, 1, 10) 50 | out = block(x) 51 | # OUTPUT: [BATCH_SIZE, OUT_CHANNELS, 1, OUT_FEATURES] 52 | assert out.size() == (32, 2, 1, 4) 53 | assert torch.all(out <= 1.0) 54 | assert torch.all(out >= 0.0) 55 | 56 | block = LukasiewiczChannelXOrBlock( 57 | channels=2, 58 | in_features=10, 59 | out_features=4, 60 | n_selected_features=5, 61 | parent_weights_dimension='channels', 62 | operands=predicates, 63 | outputs_key='0' 64 | ) 65 | 66 | # INPUT: [BATCH_SIZE, 1, IN_FEATURES] OR [BATCH_SIZE, CHANNELS, 1, IN_FEATURES] 67 | x = torch.randn(32, 1, 10) 68 | out = block(x) 69 | # OUTPUT: [BATCH_SIZE, OUT_CHANNELS, 1, OUT_FEATURES] 70 | assert out.size() == (32, 2, 1, 4) 71 | assert torch.all(out <= 1.0) 72 | assert torch.all(out >= 0.0) -------------------------------------------------------------------------------- /tests/nn/test_forward_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | 4 | from torchlogic.nn import LukasiewiczChannelAndBlock, LukasiewiczChannelOrBlock, Predicates, ConcatenateBlocksLogic 5 | 6 | from pytest import fixture 7 | 8 | ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) 9 | PARENT_DIR = os.path.abspath(ROOT_DIR) 10 | 11 | 12 | class TestLukasiewiczChannelBlock: 13 | 14 | @fixture 15 | def predicates(self): 16 | return Predicates([f'feat{i}' for i in range(10)]) 17 | 18 | @fixture 19 | def concatenated_blocks(self, predicates): 20 | block1 = LukasiewiczChannelAndBlock( 21 | channels=2, 22 | in_features=10, 23 | out_features=1, 24 | n_selected_features=5, 25 | parent_weights_dimension='out_features', 26 | operands=predicates, 27 | outputs_key='0' 28 | ) 29 | 30 | block2 = LukasiewiczChannelOrBlock( 31 | channels=2, 32 | in_features=10, 33 | out_features=1, 34 | n_selected_features=5, 35 | parent_weights_dimension='out_features', 36 | operands=predicates, 37 | outputs_key='1' 38 | ) 39 | 40 | cat_blocks = ConcatenateBlocksLogic([block1, block2], '2') 41 | 42 | return cat_blocks 43 | 44 | @staticmethod 45 | def test__forward_lukasiewicz_channel_block(concatenated_blocks): 46 | # INPUT: [BATCH_SIZE, CHANNELS, 1, IN_FEATURES] 47 | x1 = torch.rand(32, 2, 1, 1) 48 | x2 = torch.rand(32, 2, 1, 1) 49 | out = concatenated_blocks(x1, x2) 50 | # OUTPUT: [BATCH_SIZE, OUT_CHANNELS, 1, OUT_FEATURES] 51 | assert out.size() == (32, 2, 1, 2) 52 | assert torch.all(out <= 1.0) 53 | assert torch.all(out >= 0.0) 54 | -------------------------------------------------------------------------------- /tests/sklogic/test__rnrn_regressor.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | import numpy as np 4 | import pandas as pd 5 | from sklearn.datasets import make_regression 6 | 7 | import torch 8 | from torch.utils.data import Dataset 9 | 10 | from pytest import fixture 11 | 12 | from torchlogic.sklogic.regressors import RNRNRegressor 13 | 14 | 15 | class TestRNRNRegressor: 16 | 17 | @fixture 18 | def data_frames(self): 19 | X, y = make_regression( 20 | n_samples=10, n_features=4, n_informative=4, n_targets=1, tail_strength=0.1, random_state=0) 21 | X = pd.DataFrame(X, columns=[f"feature{i}" for i in range(X.shape[1])]) 22 | y = pd.DataFrame(y, columns=["Target1"]) 23 | return X, y 24 | 25 | @fixture 26 | def data_sets(self, data_frames): 27 | class StaticDataset(Dataset): 28 | def __init__(self): 29 | X, y = data_frames 30 | super(StaticDataset, self).__init__() 31 | self.X = X.values 32 | self.y = y.values 33 | self.sample_idx = np.arange(self.X.shape[0]) # index of samples 34 | 35 | def __len__(self): 36 | return self.X.shape[0] 37 | 38 | def __getitem__(self, idx): 39 | features = torch.from_numpy(self.X[idx, :]).float() 40 | target = torch.from_numpy(self.y[idx, :]) 41 | return {'features': features, 'target': target, 'sample_idx': idx} 42 | 43 | return (StaticDataset(), StaticDataset()) 44 | 45 | @fixture 46 | def model(self): 47 | torch.manual_seed(0) 48 | np.random.seed(0) 49 | 50 | model = RNRNRegressor(holdout_pct=0.5) 51 | return model 52 | 53 | @staticmethod 54 | def test_fit(model, data_frames): 55 | model1 = copy.copy(model) 56 | X, y = data_frames 57 | model1.fit(X, y) 58 | assert model1._fbt_is_fitted 59 | assert model1.fbt is not None 60 | assert model1.model is not None 61 | 62 | # should still fit if arrays are passed 63 | model2 = copy.copy(model) 64 | model2.fit(X.to_numpy(), y.to_numpy()) 65 | assert model2._fbt_is_fitted 66 | assert model2.fbt is not None 67 | assert model2.model is not None 68 | 69 | @staticmethod 70 | def test_predict(model, data_frames): 71 | torch.manual_seed(0) 72 | np.random.seed(0) 73 | model1 = copy.copy(model) 74 | X, y = data_frames 75 | model1.fit(X, y) 76 | predictions = model1.predict(X) 77 | assert predictions.max().values <= y.max().values, "range was not correct" 78 | assert predictions.min().values >= y.min().values, "range was not correct" 79 | assert list(predictions.columns) == ["Target"] 80 | 81 | model2 = copy.copy(model) 82 | model2.target_name = 'The Target' 83 | torch.manual_seed(0) 84 | np.random.seed(0) 85 | model2.fit(X, y) 86 | predictions = model2.predict(X) 87 | assert predictions.max().values <= y.max().values, "range was not correct" 88 | assert predictions.min().values >= y.min().values, "range was not correct" 89 | assert list(predictions.columns) == ['The Target'] 90 | 91 | @staticmethod 92 | def test_score(model, data_frames): 93 | torch.manual_seed(0) 94 | np.random.seed(0) 95 | 96 | model1 = copy.copy(model) 97 | 98 | X, y = data_frames 99 | 100 | model1.fit(X, y) 101 | score = model1.score(X, y) 102 | # TODO: what other tests are reasonable? 103 | assert score > 0 104 | assert isinstance(score, float) 105 | 106 | @staticmethod 107 | def test_explain_sample(model, data_frames): 108 | torch.manual_seed(0) 109 | np.random.seed(0) 110 | 111 | model1 = copy.copy(model) 112 | 113 | X, y = data_frames 114 | model1.fit(X, y) 115 | explanation = model1.explain_sample(X, sample_index=0) 116 | assert explanation == ('0: The sample has a predicted Target of -87.051 because: ' 117 | '\n\n\nAND \n\tThe feature0 was greater than -0.204\n\tThe ' 118 | 'feature1 was greater than -1.479\n\tThe feature2 was greater ' 119 | 'than -1.328\n\tThe feature3 was greater than 0.234') 120 | -------------------------------------------------------------------------------- /tests/utils/test_base_trainer.py: -------------------------------------------------------------------------------- 1 | import os 2 | from copy import deepcopy 3 | 4 | import numpy as np 5 | import pandas as pd 6 | 7 | import torch 8 | from torch.utils.data import Dataset, DataLoader 9 | 10 | from torchlogic.modules import BanditNRNModule 11 | from torchlogic.models.base import BaseBanditNRNModel 12 | from torchlogic.utils.trainers.base import BaseReasoningNetworkDistributedTrainer 13 | 14 | import pytest 15 | from pytest import fixture 16 | 17 | ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) 18 | PARENT_DIR = os.path.abspath(ROOT_DIR) 19 | 20 | 21 | class TestBaseTrainer: 22 | 23 | @fixture 24 | def data_loaders(self): 25 | 26 | class StaticDataset(Dataset): 27 | def __init__(self): 28 | 29 | super(StaticDataset, self).__init__() 30 | self.X = pd.DataFrame( 31 | {'feature1': [1, 1, 1, 1], 32 | 'feature2': [1, 0, 1, 0], 33 | 'feature3': [1, 0, 1, 0], 34 | 'feature4': [1, 0, 1, 0]}).values 35 | self.y = pd.DataFrame( 36 | {'class1': [1, 0, 1, 0], 37 | 'class2': [0, 1, 0, 1]}).values 38 | self.sample_idx = np.arange(self.X.shape[0]) # index of samples 39 | 40 | def __len__(self): 41 | return self.X.shape[0] 42 | 43 | def __getitem__(self, idx): 44 | features = torch.from_numpy(self.X[idx, :]).float() 45 | target = torch.from_numpy(self.y[idx, :]) 46 | return {'features': features, 'target': target, 'sample_idx': idx} 47 | 48 | return (DataLoader(StaticDataset(), batch_size=2, shuffle=False), 49 | DataLoader(StaticDataset(), batch_size=2, shuffle=False)) 50 | 51 | @fixture 52 | def model(self): 53 | model = BaseBanditNRNModel( 54 | target_names=['class1', 'class2'], 55 | feature_names=['feat1', 'feat1', 'feat3', 'feat4'], 56 | input_size=4, 57 | output_size=2, 58 | layer_sizes=[5, ], 59 | n_selected_features_input=2, 60 | n_selected_features_internal=2, 61 | n_selected_features_output=2, 62 | perform_prune_quantile=0.5, 63 | ucb_scale=2.5 64 | ) 65 | 66 | for item in model.rn.state_dict(): 67 | if item.find('weights') > -1: 68 | weights = model.rn.state_dict()[item] 69 | weights.data.copy_(torch.ones_like(weights.data)) 70 | 71 | return model 72 | 73 | @fixture 74 | def trainer(self, model): 75 | return BaseReasoningNetworkDistributedTrainer( 76 | model, 77 | torch.nn.BCELoss(), 78 | torch.optim.Adam(model.rn.parameters()) 79 | ) 80 | 81 | @staticmethod 82 | def test__validate_state_dicts(model, trainer): 83 | state_dict1 = deepcopy(model.rn.state_dict()) 84 | 85 | for item in model.rn.state_dict(): 86 | if item.find('weights') > -1: 87 | weights = model.rn.state_dict()[item] 88 | weights.data.copy_(torch.ones_like(weights.data) * 2) 89 | 90 | state_dict2 = deepcopy(model.rn.state_dict()) 91 | 92 | assert not trainer._validate_state_dicts(state_dict1, state_dict2) 93 | assert trainer._validate_state_dicts(state_dict1, state_dict1) 94 | 95 | @staticmethod 96 | def test__save_best_state(model, trainer): 97 | trainer.save_best_state() 98 | 99 | assert trainer._validate_state_dicts(model.best_state['rn'].state_dict(), model.rn.state_dict()) 100 | assert model.best_state['epoch'] == 0 101 | assert not model.best_state['was_pruned'] 102 | 103 | @staticmethod 104 | def test__set_best_state(model, trainer): 105 | trainer.initialized_optimizer = torch.optim.Adam(model.rn.parameters()) 106 | trainer.save_best_state() 107 | 108 | # change states 109 | for item in model.rn.state_dict(): 110 | if item.find('weights') > -1: 111 | weights = model.rn.state_dict()[item] 112 | weights.data.copy_(torch.ones_like(weights.data) * 2) 113 | trainer.epoch = 10 114 | trainer.was_pruned = True 115 | 116 | assert model.best_state['epoch'] == 0 117 | assert not model.best_state['was_pruned'] 118 | assert trainer.set_best_state() 119 | 120 | @staticmethod 121 | def test__evaluate_step(data_loaders, trainer): 122 | train_dl, val_dl = data_loaders 123 | 124 | out = trainer.evaluate_step(dl=val_dl, epoch=0, plateau_counter=0) 125 | 126 | assert out == 0 127 | assert trainer.best_val_performance == 0.5 128 | assert isinstance(trainer.model.best_state['rn'], BanditNRNModule) 129 | assert not trainer.model.best_state['was_pruned'] 130 | assert trainer.model.best_state['epoch'] == 0 131 | 132 | @staticmethod 133 | def test___validation_step(data_loaders, trainer): 134 | train_dl, val_dl = data_loaders 135 | 136 | out = trainer._validation_step(val_dl=val_dl, epoch=0, plateau_counter=0) 137 | 138 | assert out == {'plateau_counter': 0} 139 | assert trainer.best_val_performance == 0.5 140 | assert isinstance(trainer.model.best_state['rn'], BanditNRNModule) 141 | assert not trainer.model.best_state['was_pruned'] 142 | assert trainer.model.best_state['epoch'] == 0 143 | 144 | @staticmethod 145 | def test__train(data_loaders, model, trainer): 146 | train_dl, val_dl = data_loaders 147 | 148 | # basic trainer 149 | trainer.epochs = 2 150 | trainer.train(train_dl, val_dl) 151 | 152 | assert trainer.epoch == 1 153 | assert trainer.best_val_performance == 0.5 154 | assert isinstance(trainer.model.best_state['rn'], BanditNRNModule) 155 | assert not trainer.model.best_state['was_pruned'] 156 | assert trainer.model.best_state['epoch'] == 1 157 | 158 | # basic trainer without validation set provided 159 | trainer.epochs = 2 160 | trainer.train(train_dl) 161 | 162 | assert trainer.epoch == 1 163 | assert trainer.best_val_performance == 0.5 164 | assert isinstance(trainer.model.best_state['rn'], BanditNRNModule) 165 | assert not trainer.model.best_state['was_pruned'] 166 | assert trainer.model.best_state['epoch'] == 1 167 | -------------------------------------------------------------------------------- /tests/utils/test_boosted_brrn_trainer.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import numpy as np 4 | import pandas as pd 5 | 6 | import torch 7 | from torch.utils.data import Dataset, DataLoader 8 | 9 | from torchlogic.models.base import BoostedBanditNRNModel 10 | from torchlogic.utils.trainers import BoostedBanditNRNTrainer 11 | 12 | from pytest import fixture 13 | 14 | ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) 15 | PARENT_DIR = os.path.abspath(ROOT_DIR) 16 | 17 | 18 | class TestBoostedBanditRRNTrainer: 19 | 20 | @fixture 21 | def data_loaders(self): 22 | class StaticDataset(Dataset): 23 | def __init__(self): 24 | super(StaticDataset, self).__init__() 25 | self.X = pd.DataFrame( 26 | {'feature1': [1, 1, 1, 1], 27 | 'feature2': [1, 1, 1, 1], 28 | 'feature3': [1, 1, 1, 1], 29 | 'feature4': [1, 1, 1, 1]}).values 30 | self.y = pd.DataFrame( 31 | {'class1': [1, 0, 1, 0], 32 | 'class2': [0, 1, 0, 1]}).values 33 | self.sample_idx = np.arange(self.X.shape[0]) # index of samples 34 | 35 | def __len__(self): 36 | return self.X.shape[0] 37 | 38 | def __getitem__(self, idx): 39 | features = torch.from_numpy(self.X[idx, :]).float() 40 | target = torch.from_numpy(self.y[idx, :]) 41 | return {'features': features, 'target': target, 'sample_idx': idx} 42 | 43 | return (DataLoader(StaticDataset(), batch_size=2, shuffle=False), 44 | DataLoader(StaticDataset(), batch_size=2, shuffle=False)) 45 | 46 | @fixture 47 | def model(self): 48 | model = BoostedBanditNRNModel( 49 | target_names=['class1', 'class2'], 50 | feature_names=['feat1', 'feat1', 'feat3', 'feat4'], 51 | input_size=4, 52 | output_size=2, 53 | layer_sizes=[5, ], 54 | n_selected_features_input=2, 55 | n_selected_features_internal=2, 56 | n_selected_features_output=2, 57 | perform_prune_quantile=0.5, 58 | ucb_scale=2.5 59 | ) 60 | 61 | for item in model.rn.state_dict(): 62 | if item.find('weights') > -1: 63 | weights = model.rn.state_dict()[item] 64 | weights.data.copy_(torch.ones_like(weights.data)) 65 | 66 | return model 67 | 68 | @fixture 69 | def trainer(self, model): 70 | return BoostedBanditNRNTrainer( 71 | model, 72 | torch.nn.BCELoss(), 73 | torch.optim.Adam(model.rn.parameters()), 74 | perform_prune_plateau_count=0, 75 | augment=None 76 | ) 77 | 78 | @staticmethod 79 | def test__boost(trainer, data_loaders): 80 | train_dl, val_dl = data_loaders 81 | trainer.boost(train_dl) 82 | assert trainer.model.xgb_is_fitted, "boosting did not complete correctly" 83 | -------------------------------------------------------------------------------- /torchlogic/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/torchlogic/20d10b0462b6866f3853336ef3cfad19218dcb18/torchlogic/__init__.py -------------------------------------------------------------------------------- /torchlogic/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .brn_regressor import BanditNRNRegressor 2 | from .brn_classifier import BanditNRNClassifier 3 | from .var_classifier import VarNRNClassifier 4 | from .var_regressor import VarNRNRegressor 5 | from .attn_classifier import AttnNRNClassifier 6 | from .attn_regressor import AttnNRNRegressor 7 | 8 | __all__ = [ 9 | BanditNRNRegressor, 10 | BanditNRNClassifier, 11 | VarNRNClassifier, 12 | VarNRNRegressor, 13 | AttnNRNClassifier, 14 | AttnNRNRegressor 15 | ] 16 | -------------------------------------------------------------------------------- /torchlogic/models/attn_classifier.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import List, Union 3 | 4 | import numpy as np 5 | 6 | from ..models.base import BaseAttnNRNModel 7 | from .mixins import ReasoningNetworkClassifierMixin 8 | 9 | 10 | class AttnNRNClassifier(BaseAttnNRNModel, ReasoningNetworkClassifierMixin): 11 | 12 | def __init__( 13 | self, 14 | target_names: List[str], 15 | feature_names: List[str], 16 | input_size: int, 17 | output_size: int, 18 | layer_sizes: List[int], 19 | swa: bool = True, 20 | add_negations: bool = False, 21 | weight_init: float = 0.2, 22 | attn_emb_dim: int = 32, 23 | attn_n_layers: int = 2, 24 | normal_form: str = 'dnf', 25 | logits: bool = True 26 | ): 27 | """ 28 | Initialize a AttnNRNClassifier model. 29 | 30 | Example: 31 | model = AttnNRNClassifier( 32 | target_names=['class1', 'class2'], 33 | feature_names=['feature1', 'feature2', 'feature3'], 34 | input_size=3, 35 | output_size=2, 36 | layer_sizes=[3, 3] 37 | out_type='And', 38 | swa=False, 39 | add_negations=True, 40 | weight_init=0.2, 41 | attn_emb_dim=32, 42 | attn_n_layers=2 43 | ) 44 | 45 | Args: 46 | target_names (list): A list of the target names. 47 | feature_names (list): A list of feature names. 48 | input_size (int): number of features from input. 49 | output_size (int): number of outputs. 50 | layer_sizes (list): A list containing the number of output logics for each layer. 51 | out_type (str): 'And' or 'Or'. The logical type of the output layer. 52 | swa (bool): Use stochastic weight averaging 53 | add_negations (bool): add negations of logic. 54 | weight_init (float): Upper bound of uniform weight initialization. Lower bound is negated value. 55 | attn_emb_dim (int): Hidden layer size for attention model. 56 | attn_n_layers (int): Number of layers for attention model. 57 | """ 58 | ReasoningNetworkClassifierMixin.__init__(self, output_size=output_size, logits=logits) 59 | BaseAttnNRNModel.__init__( 60 | self, 61 | target_names=target_names, 62 | input_size=input_size, 63 | output_size=output_size, 64 | layer_sizes=layer_sizes, 65 | feature_names=feature_names, 66 | swa=swa, 67 | weight_init=weight_init, 68 | add_negations=add_negations, 69 | attn_emb_dim=attn_emb_dim, 70 | attn_n_layers=attn_n_layers, 71 | normal_form=normal_form, 72 | logits=logits 73 | ) 74 | self.set_modules(self.rn) 75 | self.logger = logging.getLogger(self.__class__.__name__) 76 | 77 | def explain( 78 | self, 79 | quantile: float = 0.5, 80 | required_output_thresholds: Union[float, np.float64] = 0.9, 81 | threshold: float = None, 82 | explain_type: str = 'both', 83 | print_type: str = 'logical', 84 | target_names: list = ['positive'], 85 | explanation_prefix: str = "A sample is in the", 86 | ignore_uninformative: bool = False, 87 | rounding_precision: int = 3, 88 | inverse_transform=None, 89 | decision_boundary: float = 0.5, 90 | show_bounds: bool = True, 91 | simplify: bool = False 92 | ) -> str: 93 | raise UserWarning("Global explanations are not available for AttentionNRNClassifier.") 94 | 95 | def print( 96 | self, 97 | quantile=0.5, 98 | required_output_thresholds=None, 99 | threshold=None, 100 | explain_type='both', 101 | print_type='logical', 102 | target_names=['positive'], 103 | ignore_uninformative: bool = False, 104 | rounding_precision: int = 3, 105 | inverse_transform=None, 106 | decision_boundary: float = 0.5, 107 | show_bounds: bool = True 108 | ): 109 | raise UserWarning("Global explanations are not available for AttentionNRNClassifier.") 110 | 111 | 112 | __all__ = [AttnNRNClassifier] 113 | -------------------------------------------------------------------------------- /torchlogic/models/attn_regressor.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import List 3 | 4 | from ..models.base import BaseAttnNRNModel 5 | from .mixins import ReasoningNetworkRegressorMixin 6 | 7 | 8 | class AttnNRNRegressor(BaseAttnNRNModel, ReasoningNetworkRegressorMixin): 9 | 10 | def __init__( 11 | self, 12 | target_names: List[str], 13 | feature_names: List[str], 14 | input_size: int, 15 | layer_sizes: List[int], 16 | swa: bool = True, 17 | add_negations: bool = False, 18 | weight_init: float = 0.2, 19 | attn_emb_dim: int = 32, 20 | attn_n_layers: int = 2, 21 | normal_form: str = 'dnf' 22 | ): 23 | """ 24 | Initialize a AttnNRNRegressor model. 25 | 26 | Example: 27 | model = AttnNRNRegressor( 28 | target_names=['class1', 'class2'], 29 | feature_names=['feature1', 'feature2', 'feature3'], 30 | input_size=3, 31 | output_size=2, 32 | layer_sizes=[3, 3] 33 | out_type='And', 34 | swa=False, 35 | add_negations=True, 36 | weight_init=0.2, 37 | attn_emb_dim=32, 38 | attn_n_layers=2 39 | ) 40 | 41 | Args: 42 | target_names (list): A list of the target names. 43 | feature_names (list): A list of feature names. 44 | input_size (int): number of features from input. 45 | output_size (int): number of outputs. 46 | layer_sizes (list): A list containing the number of output logics for each layer. 47 | out_type (str): 'And' or 'Or'. The logical type of the output layer. 48 | swa (bool): Use stochastic weight averaging 49 | add_negations (bool): add negations of logic. 50 | weight_init (float): Upper bound of uniform weight initialization. Lower bound is negated value. 51 | attn_emb_dim (int): Hidden layer size for attention model. 52 | attn_n_layers (int): Number of layers for attention model. 53 | """ 54 | ReasoningNetworkRegressorMixin.__init__(self, output_size=1) 55 | BaseAttnNRNModel.__init__( 56 | self, 57 | target_names=target_names, 58 | input_size=input_size, 59 | output_size=1, 60 | layer_sizes=layer_sizes, 61 | feature_names=feature_names, 62 | swa=swa, 63 | weight_init=weight_init, 64 | add_negations=add_negations, 65 | attn_emb_dim=attn_emb_dim, 66 | attn_n_layers=attn_n_layers, 67 | normal_form=normal_form, 68 | logits=False 69 | ) 70 | self.set_modules(self.rn) 71 | self.logger = logging.getLogger(self.__class__.__name__) 72 | 73 | 74 | __all__ = [AttnNRNRegressor] 75 | -------------------------------------------------------------------------------- /torchlogic/models/base/__init__.py: -------------------------------------------------------------------------------- 1 | from .rn import ReasoningNetworkModel 2 | from .pruningrn import PruningReasoningNetworkModel 3 | from .brn import BaseBanditNRNModel 4 | from .boosted_brn import BoostedBanditNRNModel 5 | from .var import BaseVarNRNModel 6 | from .attn import BaseAttnNRNModel 7 | 8 | __all__ = [ReasoningNetworkModel, PruningReasoningNetworkModel, BaseBanditNRNModel, BoostedBanditNRNModel, 9 | BaseVarNRNModel, BaseAttnNRNModel] 10 | -------------------------------------------------------------------------------- /torchlogic/models/base/attn.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | import logging 3 | 4 | import torch 5 | from torch import nn 6 | from torch.optim.swa_utils import AveragedModel 7 | 8 | from torchlogic.modules import AttentionNRNModule 9 | from .rn import ReasoningNetworkModel 10 | 11 | 12 | class BaseAttnNRNModel(ReasoningNetworkModel): 13 | 14 | def __init__( 15 | self, 16 | target_names: List[str], 17 | feature_names: List[str], 18 | input_size: int, 19 | output_size: int, 20 | layer_sizes: List[int], 21 | swa: bool = False, 22 | add_negations: bool = False, 23 | weight_init: float = 0.2, 24 | attn_emb_dim: int = 32, 25 | attn_n_layers: int = 2, 26 | normal_form: str = 'dnf', 27 | logits: bool = False 28 | ): 29 | """ 30 | Initialize a Attention Neural Reasoning Network model. 31 | 32 | Example: 33 | model = BaseAttnNRNModel( 34 | target_names=['class1', 'class2'], 35 | feature_names=['feature1', 'feature2', 'feature3'], 36 | input_size=3, 37 | output_size=2, 38 | layer_sizes=[3, 3] 39 | out_type='And', 40 | swa=False, 41 | add_negations=True, 42 | weight_init=0.2, 43 | attn_emb_dim=32, 44 | attn_n_layer=2 45 | ) 46 | 47 | Args: 48 | target_names (list): A list of the target names. 49 | feature_names (list): A list of feature names. 50 | input_size (int): number of features from input. 51 | output_size (int): number of outputs. 52 | layer_sizes (list): A list containing the number of output logics for each layer. 53 | out_type (str): 'And' or 'Or'. The logical type of the output layer. 54 | swa (bool): Use stochastic weight averaging 55 | add_negations (bool): add negations of logic. 56 | weight_init (float): Upper bound of uniform weight initialization. Lower bound is negated value. 57 | attn_emb_dim (int): Hidden layer size for attention model. 58 | attn_n_layers (int): Number of layers for attention model. 59 | """ 60 | ReasoningNetworkModel.__init__(self) 61 | 62 | self.target_names = target_names 63 | self.feature_names = feature_names 64 | self.input_size = input_size 65 | self.output_size = output_size 66 | self.swa = swa 67 | self.add_negations = add_negations 68 | self.weight_init = weight_init 69 | self.attn_emb_dim = attn_emb_dim 70 | self.attn_n_layers = attn_n_layers 71 | self.normal_form = normal_form 72 | self.logits = logits 73 | 74 | self.rn = AttentionNRNModule( 75 | input_size=input_size, 76 | output_size=output_size, 77 | layer_sizes=layer_sizes, 78 | feature_names=feature_names, 79 | add_negations=add_negations, 80 | weight_init=weight_init, 81 | attn_emb_dim=attn_emb_dim, 82 | attn_n_layers=attn_n_layers, 83 | normal_form=normal_form, 84 | logits=logits 85 | ) 86 | 87 | self.averaged_rn = None 88 | if self.swa: 89 | self.averaged_rn = AveragedModel(self.rn) 90 | 91 | self.target_names = target_names 92 | 93 | if torch.cuda.device_count() > 1: 94 | self.logger.info(f"Using {torch.cuda.device_count()} GPUs!") 95 | self.rn = nn.DataParallel(self.rn) 96 | if self.USE_CUDA: 97 | self.logger.info(f"Using GPU") 98 | self.rn = self.rn.cuda() 99 | elif self.USE_MPS: 100 | self.logger.info(f"Using MPS") 101 | self.rn = self.rn.to('mps') 102 | 103 | self.USE_DATA_PARALLEL = isinstance(self.rn, torch.nn.DataParallel) 104 | 105 | self.logger = logging.getLogger(self.__class__.__name__) 106 | 107 | 108 | __all__ = [BaseAttnNRNModel] 109 | -------------------------------------------------------------------------------- /torchlogic/models/base/pruningrn.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from torchlogic.models.base import ReasoningNetworkModel 3 | 4 | 5 | class PruningReasoningNetworkModel(ReasoningNetworkModel): 6 | 7 | def __init__(self): 8 | """ 9 | Initialize a PruningReasoningNetworkModel 10 | """ 11 | super(PruningReasoningNetworkModel, self).__init__() 12 | self.best_state = {} 13 | self.target_names = None 14 | self.rn = None 15 | self.logger = logging.getLogger(self.__class__.__name__) 16 | 17 | def perform_prune(self) -> None: 18 | """ 19 | Update the nodes in the PruningLogicalNetwork by pruning and growing. 20 | """ 21 | raise Warning("Abstract method, must be implemented!") 22 | 23 | 24 | __all__ = [PruningReasoningNetworkModel] 25 | -------------------------------------------------------------------------------- /torchlogic/models/base/var.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | import logging 3 | 4 | import torch 5 | from torch import nn 6 | from torch.optim.swa_utils import AveragedModel 7 | 8 | from torchlogic.modules import VarNRNModule 9 | from .rn import ReasoningNetworkModel 10 | 11 | 12 | class BaseVarNRNModel(ReasoningNetworkModel): 13 | 14 | def __init__( 15 | self, 16 | target_names: List[str], 17 | feature_names: List[str], 18 | input_size: int, 19 | output_size: int, 20 | layer_sizes: List[int], 21 | swa: bool = False, 22 | add_negations: bool = False, 23 | weight_init: float = 0.2, 24 | var_emb_dim: int = 50, 25 | var_n_layers: int = 2, 26 | normal_form: str = 'dnf', 27 | logits: bool = False 28 | ): 29 | """ 30 | Initialize a Attention Neural Reasoning Network model. 31 | 32 | Example: 33 | model = BaseAttnNRNModel( 34 | target_names=['class1', 'class2'], 35 | feature_names=['feature1', 'feature2', 'feature3'], 36 | input_size=3, 37 | output_size=2, 38 | layer_sizes=[3, 3] 39 | out_type='And', 40 | swa=False, 41 | add_negations=True, 42 | weight_init=0.2, 43 | tau_min=0.2, 44 | tau_warmup=50, 45 | attn_emb_dim=50 46 | ) 47 | 48 | Args: 49 | target_names (list): A list of the target names. 50 | feature_names (list): A list of feature names. 51 | input_size (int): number of features from input. 52 | output_size (int): number of outputs. 53 | layer_sizes (list): A list containing the number of output logics for each layer. 54 | out_type (str): 'And' or 'Or'. The logical type of the output layer. 55 | swa (bool): Use stochastic weight averaging 56 | add_negations (bool): add negations of logic. 57 | weight_init (float): Upper bound of uniform weight initialization. Lower bound is negated value. 58 | tau_min (float): Minimum tau value for gumbel softmax. 59 | tau_warmup (int): Number of epochs used to linearly warmup to tau_min. 60 | var_emb_dim (int): Embedding dimension for latent variational space. 61 | """ 62 | ReasoningNetworkModel.__init__(self) 63 | 64 | self.target_names = target_names 65 | self.feature_names = feature_names 66 | self.input_size = input_size 67 | self.output_size = output_size 68 | self.swa = swa 69 | self.add_negations = add_negations 70 | self.weight_init = weight_init 71 | self.var_emb_dim = var_emb_dim 72 | self.var_n_layers = var_n_layers 73 | self.normal_form = normal_form 74 | self.logits = logits 75 | 76 | self.rn = VarNRNModule( 77 | input_size=input_size, 78 | output_size=output_size, 79 | layer_sizes=layer_sizes, 80 | feature_names=feature_names, 81 | add_negations=add_negations, 82 | weight_init=weight_init, 83 | var_emb_dim=var_emb_dim, 84 | var_n_layers=var_n_layers, 85 | normal_form=normal_form, 86 | logits=logits 87 | ) 88 | 89 | self.averaged_rn = None 90 | if self.swa: 91 | self.averaged_rn = AveragedModel(self.rn) 92 | 93 | self.target_names = target_names 94 | 95 | if torch.cuda.device_count() > 1: 96 | self.logger.info(f"Using {torch.cuda.device_count()} GPUs!") 97 | self.rn = nn.DataParallel(self.rn) 98 | if self.USE_CUDA: 99 | self.logger.info(f"Using GPU") 100 | self.rn = self.rn.cuda() 101 | elif self.USE_MPS: 102 | self.logger.info(f"Using MPS") 103 | self.rn = self.rn.to('mps') 104 | 105 | self.USE_DATA_PARALLEL = isinstance(self.rn, torch.nn.DataParallel) 106 | 107 | self.logger = logging.getLogger(self.__class__.__name__) 108 | 109 | 110 | __all__ = [BaseVarNRNModel] 111 | -------------------------------------------------------------------------------- /torchlogic/models/brn_classifier.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import List 3 | 4 | import torch 5 | 6 | from ..models.base import BoostedBanditNRNModel 7 | from .mixins import ReasoningNetworkClassifierMixin 8 | 9 | 10 | class BanditNRNClassifier(BoostedBanditNRNModel, ReasoningNetworkClassifierMixin): 11 | 12 | def __init__( 13 | self, 14 | target_names: List[str], 15 | feature_names: List[str], 16 | input_size: int, 17 | output_size: int, 18 | layer_sizes: List[int], 19 | n_selected_features_input: int, 20 | n_selected_features_internal: int, 21 | n_selected_features_output: int, 22 | perform_prune_quantile: float, 23 | ucb_scale: float, 24 | normal_form: str = 'dnf', 25 | delta: float = 2.0, 26 | prune_strategy: str = 'class', 27 | bootstrap: bool = True, 28 | swa: bool = True, 29 | add_negations: bool = False, 30 | weight_init: float = 0.2, 31 | policy_init: torch.Tensor = None, 32 | xgb_max_depth: int = 4, 33 | xgb_n_estimators: int = 200, 34 | xgb_min_child_weight: float = 1.0, 35 | xgb_subsample: float = 0.7, 36 | xgb_learning_rate: float = 0.001, 37 | xgb_colsample_bylevel: float = 0.7, 38 | xgb_colsample_bytree: float = 0.7, 39 | xgb_gamma: float = 1, 40 | xgb_reg_lambda: float = 2.0, 41 | xgb_reg_alpha: float = 2.0, 42 | logits: bool = False 43 | ): 44 | """ 45 | Initialize a BERrnClassifier model. 46 | 47 | Example: 48 | model = BERrnClassifier( 49 | target_names=['class1', 'class2'], 50 | feature_names=['feature1', 'feature2', 'feature3'], 51 | input_size=3, 52 | output_size=2, 53 | layer_sizes=[3, 3] 54 | n_selected_features_input=2, 55 | n_selected_features_internal=2, 56 | n_selected_features_output=1, 57 | ucb_scale=1.96, 58 | perform_prune_quantile=0.7, 59 | normal_form='dnf', 60 | prune_strategy='class', 61 | bootstrap=True, 62 | distributed=False 63 | ) 64 | 65 | Args: 66 | target_names (list): A list of the target names. 67 | feature_names (list): A list of feature names. 68 | input_size (int): number of features from input. 69 | output_size (int): number of outputs. 70 | layer_sizes (list): A list containing the number of output logics for each layer. 71 | n_selected_features_input (int): The number of features to include in each logic in the input layer. 72 | n_selected_features_internal (int): The number of logics to include in each logic in the internal layers. 73 | n_selected_features_output (int): The number of logics to include in each logic in the output layer. 74 | perform_prune_quantile (float): The quantile to use for pruning randomized rn. 75 | ucb_scale (float): The scale of the confidence interval in the multi-armed bandit policy. 76 | c = 1.96 is a 95% confidence interval. 77 | normal_form (str): 'dnf' for disjunctive normal form network; 'cnf' for conjunctive normal form network. 78 | delta (float): higher values increase diversity of logic generation away from existing logics. 79 | prune_strategy(str): Either 'class' or 'logic'. Determines which pruning strategy to use. 80 | bootstrap (bool): Use boostrap samples to evaluate each atomic logic in logic prune strategy. 81 | swa (bool): Use stochastic weight averaging 82 | add_negations (bool): add negations of logic. 83 | weight_init (float): Upper bound of uniform weight initialization. Lower bound is negated value. 84 | xgb_max_depth (int): Max depth for XGBoost boosting model. 85 | xgb_n_estimators (int): Number of estimators for XGBoost boosting model. 86 | xgb_min_child_weight (float): Minimum child weight for XGBoost boosting model. 87 | xgb_subsample (float): Subsample percentage for XGBoost boosting model. 88 | xgb_learning_rate (float): Learning rate for XGBoost boosting model. 89 | xgb_colsample_bylevel (float): Column subsample percent for XGBoost boosting model. 90 | xgb_colsample_bytree (float): Tree subsample percent for XGBoost boosting model. 91 | xgb_gamma (float): Gamma parameter for XGBoost boosting model. 92 | xgb_reg_lambda (float): Lambda regularization parameter for XGBoost boosting model. 93 | xgb_reg_alpha (float): Alpha regularization parameter for XGBoost boosting model. 94 | """ 95 | ReasoningNetworkClassifierMixin.__init__(self, output_size=output_size) 96 | BoostedBanditNRNModel.__init__( 97 | self, 98 | target_names=target_names, 99 | input_size=input_size, 100 | output_size=output_size, 101 | layer_sizes=layer_sizes, 102 | feature_names=feature_names, 103 | n_selected_features_input=n_selected_features_input, 104 | n_selected_features_internal=n_selected_features_internal, 105 | n_selected_features_output=n_selected_features_output, 106 | perform_prune_quantile=perform_prune_quantile, 107 | ucb_scale=ucb_scale, 108 | normal_form=normal_form, 109 | delta=delta, 110 | prune_strategy=prune_strategy, 111 | bootstrap=bootstrap, 112 | swa=swa, 113 | weight_init=weight_init, 114 | policy_init=policy_init, 115 | add_negations=add_negations, 116 | xgb_max_depth=xgb_max_depth, 117 | xgb_n_estimators=xgb_n_estimators, 118 | xgb_min_child_weight=xgb_min_child_weight, 119 | xgb_subsample=xgb_subsample, 120 | xgb_learning_rate=xgb_learning_rate, 121 | xgb_colsample_bylevel=xgb_colsample_bylevel, 122 | xgb_colsample_bytree=xgb_colsample_bytree, 123 | xgb_gamma=xgb_gamma, 124 | xgb_reg_lambda=xgb_reg_lambda, 125 | xgb_reg_alpha=xgb_reg_alpha, 126 | logits=logits 127 | ) 128 | self.set_modules(self.rn) 129 | self.logger = logging.getLogger(self.__class__.__name__) 130 | 131 | 132 | __all__ = [BanditNRNClassifier] 133 | -------------------------------------------------------------------------------- /torchlogic/models/brn_regressor.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import List 3 | 4 | import torch 5 | 6 | from ..models.base import BoostedBanditNRNModel 7 | from .mixins import ReasoningNetworkRegressorMixin 8 | 9 | 10 | class BanditNRNRegressor(BoostedBanditNRNModel, ReasoningNetworkRegressorMixin): 11 | 12 | def __init__( 13 | self, 14 | target_names: str, 15 | feature_names: List[str], 16 | input_size: int, 17 | layer_sizes: List[int], 18 | n_selected_features_input: int, 19 | n_selected_features_internal: int, 20 | n_selected_features_output: int, 21 | perform_prune_quantile: float, 22 | ucb_scale: float, 23 | normal_form: str = 'dnf', 24 | delta: float = 2.0, 25 | prune_strategy: str = 'class', 26 | bootstrap: bool = True, 27 | swa: bool = False, 28 | add_negations: bool = False, 29 | weight_init: float = 0.2, 30 | policy_init: torch.Tensor = None, 31 | xgb_max_depth: int = 4, 32 | xgb_n_estimators: int = 200, 33 | xgb_min_child_weight: float = 1.0, 34 | xgb_subsample: float = 0.7, 35 | xgb_learning_rate: float = 0.001, 36 | xgb_colsample_bylevel: float = 0.7, 37 | xgb_colsample_bytree: float = 0.7, 38 | xgb_gamma: float = 1, 39 | xgb_reg_lambda: float = 2.0, 40 | xgb_reg_alpha: float = 2.0 41 | ): 42 | """ 43 | Initialize a BERrnRegressor model. 44 | 45 | Example: 46 | model = BERrnRegressor( 47 | target_names='metric1', 48 | feature_names=['feature1', 'feature2', 'feature3'], 49 | input_size=3, 50 | layer_sizes=[3, 3] 51 | n_selected_features_input=2, 52 | n_selected_features_internal=2, 53 | n_selected_features_output=1, 54 | ucb_scale=1.96, 55 | perform_prune_quantile=0.7, 56 | normal_form='dnf', 57 | prune_strategy='class', 58 | distributed=False 59 | ) 60 | 61 | Args: 62 | target_names (list): A list of the target names. 63 | feature_names (list): A list of feature names. 64 | input_size (int): number of features from input. 65 | layer_sizes (list): A list containing the number of output logics for each layer. 66 | n_selected_features_input (int): The number of features to include in each logic in the input layer. 67 | n_selected_features_internal (int): The number of logics to include in each logic in the internal layers. 68 | n_selected_features_output (int): The number of logics to include in each logic in the output layer. 69 | perform_prune_quantile (float): The quantile to use for pruning randomized rn. 70 | ucb_scale (float): The scale of the confidence interval in the multi-armed bandit policy. 71 | c = 1.96 is a 95% confidence interval. 72 | normal_form (str): 'dnf' for disjunctive normal form network; 'cnf' for conjunctive normal form network. 73 | delta (float): higher values increase diversity of logic generation away from existing logics. 74 | prune_strategy(str): Either 'class' or 'logic'. Determines which pruning strategy to use. 75 | bootstrap (bool): Use boostrap samples to evaluate each atomic logic in logic prune strategy. 76 | swa (bool): Use stochastic weight averaging 77 | add_negations (bool): add negations of logic. 78 | weight_init (float): Upper bound of uniform weight initialization. Lower bound is negated value. 79 | xgb_max_depth (int): Max depth for XGBoost boosting model. 80 | xgb_n_estimators (int): Number of estimators for XGBoost boosting model. 81 | xgb_min_child_weight (float): Minimum child weight for XGBoost boosting model. 82 | xgb_subsample (float): Subsample percentage for XGBoost boosting model. 83 | xgb_learning_rate (float): Learning rate for XGBoost boosting model. 84 | xgb_colsample_bylevel (float): Column subsample percent for XGBoost boosting model. 85 | xgb_colsample_bytree (float): Tree subsample percent for XGBoost boosting model. 86 | xgb_gamma (float): Gamma parameter for XGBoost boosting model. 87 | xgb_reg_lambda (float): Lambda regularization parameter for XGBoost boosting model. 88 | xgb_reg_alpha (float): Alpha regularization parameter for XGBoost boosting model. 89 | """ 90 | ReasoningNetworkRegressorMixin.__init__(self, output_size=1) 91 | BoostedBanditNRNModel.__init__( 92 | self, 93 | target_names=[target_names], 94 | input_size=input_size, 95 | output_size=1, 96 | layer_sizes=layer_sizes, 97 | feature_names=feature_names, 98 | n_selected_features_input=n_selected_features_input, 99 | n_selected_features_internal=n_selected_features_internal, 100 | n_selected_features_output=n_selected_features_output, 101 | perform_prune_quantile=perform_prune_quantile, 102 | ucb_scale=ucb_scale, 103 | normal_form=normal_form, 104 | delta=delta, 105 | prune_strategy=prune_strategy, 106 | bootstrap=bootstrap, 107 | swa=swa, 108 | add_negations=add_negations, 109 | weight_init=weight_init, 110 | policy_init=policy_init, 111 | xgb_max_depth=xgb_max_depth, 112 | xgb_n_estimators=xgb_n_estimators, 113 | xgb_min_child_weight=xgb_min_child_weight, 114 | xgb_subsample=xgb_subsample, 115 | xgb_learning_rate=xgb_learning_rate, 116 | xgb_colsample_bylevel=xgb_colsample_bylevel, 117 | xgb_colsample_bytree=xgb_colsample_bytree, 118 | xgb_gamma=xgb_gamma, 119 | xgb_reg_lambda=xgb_reg_lambda, 120 | xgb_reg_alpha=xgb_reg_alpha, 121 | logits=False 122 | ) 123 | self.set_modules(self.rn) 124 | self.logger = logging.getLogger(self.__class__.__name__) 125 | 126 | 127 | __all__ = [BanditNRNRegressor] 128 | -------------------------------------------------------------------------------- /torchlogic/models/mixins/__init__.py: -------------------------------------------------------------------------------- 1 | from .classifier import ReasoningNetworkClassifierMixin 2 | from .regressor import ReasoningNetworkRegressorMixin 3 | 4 | __all__ = [ReasoningNetworkClassifierMixin, ReasoningNetworkRegressorMixin] 5 | -------------------------------------------------------------------------------- /torchlogic/models/var_classifier.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import List 3 | 4 | from ..models.base import BaseVarNRNModel 5 | from .mixins import ReasoningNetworkClassifierMixin 6 | 7 | 8 | class VarNRNClassifier(BaseVarNRNModel, ReasoningNetworkClassifierMixin): 9 | 10 | def __init__( 11 | self, 12 | target_names: List[str], 13 | feature_names: List[str], 14 | input_size: int, 15 | output_size: int, 16 | layer_sizes: List[int], 17 | swa: bool = True, 18 | add_negations: bool = False, 19 | weight_init: float = 0.2, 20 | var_emb_dim: int = 50, 21 | var_n_layers: int = 2, 22 | normal_form: str = 'dnf', 23 | logits: bool = False 24 | ): 25 | """ 26 | Initialize a AttnNRNClassifier model. 27 | 28 | Example: 29 | model = AttnNRNClassifier( 30 | target_names=['class1', 'class2'], 31 | feature_names=['feature1', 'feature2', 'feature3'], 32 | input_size=3, 33 | output_size=2, 34 | layer_sizes=[3, 3] 35 | out_type='And', 36 | swa=False, 37 | add_negations=True, 38 | weight_init=0.2, 39 | tau_min=0.2, 40 | tau_warmup=50, 41 | attn_emb_dim=50 42 | ) 43 | 44 | Args: 45 | target_names (list): A list of the target names. 46 | feature_names (list): A list of feature names. 47 | input_size (int): number of features from input. 48 | output_size (int): number of outputs. 49 | layer_sizes (list): A list containing the number of output logics for each layer. 50 | out_type (str): 'And' or 'Or'. The logical type of the output layer. 51 | swa (bool): Use stochastic weight averaging 52 | add_negations (bool): add negations of logic. 53 | weight_init (float): Upper bound of uniform weight initialization. Lower bound is negated value. 54 | var_emb_dim (int): Embedding dimension for latent variational space. 55 | """ 56 | ReasoningNetworkClassifierMixin.__init__(self, output_size=output_size) 57 | BaseVarNRNModel.__init__( 58 | self, 59 | target_names=target_names, 60 | input_size=input_size, 61 | output_size=output_size, 62 | layer_sizes=layer_sizes, 63 | feature_names=feature_names, 64 | swa=swa, 65 | weight_init=weight_init, 66 | add_negations=add_negations, 67 | var_emb_dim=var_emb_dim, 68 | var_n_layers=var_n_layers, 69 | normal_form=normal_form, 70 | logits=logits 71 | ) 72 | self.set_modules(self.rn) 73 | self.logger = logging.getLogger(self.__class__.__name__) 74 | 75 | 76 | __all__ = [VarNRNClassifier] 77 | -------------------------------------------------------------------------------- /torchlogic/models/var_regressor.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import List 3 | 4 | from ..models.base import BaseVarNRNModel 5 | from .mixins import ReasoningNetworkRegressorMixin 6 | 7 | 8 | class VarNRNRegressor(BaseVarNRNModel, ReasoningNetworkRegressorMixin): 9 | 10 | def __init__( 11 | self, 12 | target_names: List[str], 13 | feature_names: List[str], 14 | input_size: int, 15 | output_size: int, 16 | layer_sizes: List[int], 17 | swa: bool = True, 18 | add_negations: bool = False, 19 | weight_init: float = 0.2, 20 | var_emb_dim: int = 50, 21 | var_n_layers: int = 2, 22 | normal_form: str = 'dnf' 23 | ): 24 | """ 25 | Initialize a AttnNRNRegressor model. 26 | 27 | Example: 28 | model = AttnNRNRegressor( 29 | target_names=['class1', 'class2'], 30 | feature_names=['feature1', 'feature2', 'feature3'], 31 | input_size=3, 32 | output_size=2, 33 | layer_sizes=[3, 3] 34 | out_type='And', 35 | swa=False, 36 | add_negations=True, 37 | weight_init=0.2, 38 | tau_min=0.2, 39 | tau_warmup=50, 40 | attn_emb_dim=50 41 | ) 42 | 43 | Args: 44 | target_names (list): A list of the target names. 45 | feature_names (list): A list of feature names. 46 | input_size (int): number of features from input. 47 | output_size (int): number of outputs. 48 | layer_sizes (list): A list containing the number of output logics for each layer. 49 | out_type (str): 'And' or 'Or'. The logical type of the output layer. 50 | swa (bool): Use stochastic weight averaging 51 | add_negations (bool): add negations of logic. 52 | weight_init (float): Upper bound of uniform weight initialization. Lower bound is negated value. 53 | var_emb_dim (int): Embedding dimension for latent variational space. 54 | """ 55 | ReasoningNetworkRegressorMixin.__init__(self, output_size=output_size) 56 | BaseVarNRNModel.__init__( 57 | self, 58 | target_names=target_names, 59 | input_size=input_size, 60 | output_size=output_size, 61 | layer_sizes=layer_sizes, 62 | feature_names=feature_names, 63 | swa=swa, 64 | weight_init=weight_init, 65 | add_negations=add_negations, 66 | var_emb_dim=var_emb_dim, 67 | var_n_layers=var_n_layers, 68 | normal_form=normal_form, 69 | logits=False 70 | ) 71 | self.set_modules(self.rn) 72 | self.logger = logging.getLogger(self.__class__.__name__) 73 | 74 | 75 | __all__ = [VarNRNRegressor] 76 | -------------------------------------------------------------------------------- /torchlogic/modules/__init__.py: -------------------------------------------------------------------------------- 1 | from .brn import BanditNRNModule 2 | from .var import VarNRNModule 3 | from .attn import AttentionNRNModule 4 | 5 | __all__ = [BanditNRNModule, VarNRNModule, AttentionNRNModule] 6 | 7 | -------------------------------------------------------------------------------- /torchlogic/nn/__init__.py: -------------------------------------------------------------------------------- 1 | from .predicates import Predicates 2 | from .blocks import (LukasiewiczChannelAndBlock, LukasiewiczChannelOrBlock, LukasiewiczChannelXOrBlock, 3 | VariationalLukasiewiczChannelAndBlock, VariationalLukasiewiczChannelOrBlock, 4 | VariationalLukasiewiczChannelXOrBlock, AttentionLukasiewiczChannelAndBlock, 5 | AttentionLukasiewiczChannelOrBlock) 6 | from .utils import ConcatenateBlocksLogic 7 | 8 | __all__ = [Predicates, 9 | LukasiewiczChannelAndBlock, 10 | LukasiewiczChannelOrBlock, 11 | LukasiewiczChannelXOrBlock, 12 | VariationalLukasiewiczChannelAndBlock, 13 | VariationalLukasiewiczChannelOrBlock, 14 | VariationalLukasiewiczChannelXOrBlock, 15 | AttentionLukasiewiczChannelAndBlock, 16 | AttentionLukasiewiczChannelOrBlock, 17 | ConcatenateBlocksLogic] 18 | -------------------------------------------------------------------------------- /torchlogic/nn/base/__init__.py: -------------------------------------------------------------------------------- 1 | from .blocks import LukasiewiczChannelBlock, VariationalLukasiewiczChannelBlock, AttentionLukasiewiczChannelBlock 2 | from .predicates import BasePredicates 3 | from .utils import BaseConcatenateBlocksLogic 4 | 5 | __all__ = [LukasiewiczChannelBlock, BasePredicates, BaseConcatenateBlocksLogic, 6 | VariationalLukasiewiczChannelBlock, AttentionLukasiewiczChannelBlock] 7 | -------------------------------------------------------------------------------- /torchlogic/nn/base/constants.py: -------------------------------------------------------------------------------- 1 | keyword_constraints = ['scenarios', 'requirements', 'conditions', 'situations', 'circumstances', 'context'] 2 | 3 | or_options = ["There are several scenarios that could be met for this prediction to hold true. " 4 | "The first scenario that could be met is as follows. ", 5 | "At least one of the following requirements are met. The first requirement is as follows. ", 6 | "There are several more conditions that could be met. The conditions are described next. ", 7 | "In fact, there are additional situations that could be met. ", 8 | "It must be true that at least one of the following circumstances are met. " 9 | "The first circumstance is the following. ", 10 | "There is additional context that could be met. The contexts are described next. "] 11 | and_options = ["There are several scenarios that must be met for this prediction to hold true. " 12 | "The first scenario is as follows. ", 13 | "All of the following requirements are met. The first requirement is as follows. ", 14 | "There are several more conditions that must be met. The conditions are described next. ", 15 | "In fact, there are additional situations that must be met. ", 16 | "It must be true that the following circumstances are met. The first circumstance is the following. ", 17 | "There is additional context that must be met. The contexts are described next. "] 18 | 19 | or_options_negated = ["There are several scenarios that must NOT be met for this prediction to hold true. " 20 | "The first scenario that must NOT be met is as follows. ", 21 | "All of the following requirements are NOT met. The first requirement is as follows. ", 22 | "There are several more conditions that must NOT be met. The conditions are described next. ", 23 | "In fact, there are additional situations that must NOT be met. " 24 | "The first situation is as follows. ", 25 | "It must be true that all of the following circumstances are NOT met. " 26 | "The first circumstance is the following. ", 27 | "There is additional context that must NOT be met. The contexts are described next. "] 28 | and_options_negated = ["There are several scenarios, at least one of which must NOT hold," 29 | " for this prediction to hold true. " 30 | "The first scenario is as follows. ", 31 | "It must be true that at least one of the " 32 | "following requirements are NOT met. The first requirement is as follows. ", 33 | "There are several more conditions, at least one of which must NOT be met. " 34 | "The conditions are described next. ", 35 | "In fact, there are additional situations, at least one of which must NOT be met. ", 36 | "It must be true that at least one of the following circumstances are NOT met. " 37 | "The first circumstance is the following. ", 38 | "There is additional context, at least one of which must NOT be met. " 39 | "The contexts are described next. "] 40 | 41 | and_joining_options = [".\n\nThe next scenario that must be met is as follows. ", 42 | ". An additional requirement that must be met is the following. ", 43 | ". As well as the following conditions. ", 44 | ".\n\nThe next situation that must be met is as follows. ", 45 | ". An additional circumstance that must be met is the following. ", 46 | ". As well as the following context. "] 47 | or_joining_options = [".\n\nThe next scenario that could be met is as follows. ", 48 | ". An additional requirement that could be met is the following. ", 49 | ". Or the following conditions. ", 50 | ".\n\nThe next situation that could be met is as follows. ", 51 | ". An additional circumstance that could be met is the following. ", 52 | ". Or the following context. "] 53 | 54 | 55 | and_joining_options_negated = [".\n\nThe next scenario that could NOT be met is as follows. ", 56 | ". An additional requirement that could NOT be met is the following. ", 57 | ". Or the following conditions are NOT met. ", 58 | ".\n\nThe next situation that coule NOT be met is as follows. ", 59 | ". An additional circumstance that could NOT be be met is the following. ", 60 | ". Or the following context is NOT met. "] 61 | or_joining_options_negated = [".\n\nThe next scenario that must NOT be met is as follows. ", 62 | ". An additional requirement that must NOT be met is the following. ", 63 | ". The following conditions are also NOT met. ", 64 | ".\n\nThe next situation that must NOT be met is as follows. ", 65 | ". An additional circumstance that must NOT be met is the following. ", 66 | ". And the following context is NOT met. "] -------------------------------------------------------------------------------- /torchlogic/nn/predicates.py: -------------------------------------------------------------------------------- 1 | from .base import BasePredicates 2 | import logging 3 | 4 | 5 | class Predicates(BasePredicates): 6 | 7 | def __init__(self, feature_names: list): 8 | """ 9 | Initialize a Predicates object. The Predicates object is passed to the operands parameter of a 10 | LukasiewiczLayer or LukasiewiczChannelBlock and enables the explanation functionality of the torchlogic 11 | blocks, layers and modules. 12 | 13 | Example: 14 | feature_names = ['feature1', 'feature2', 'feature3', 'feature4'] 15 | 16 | input_layer_and = LukasiewiczChannelAndBlock( 17 | channels=output_size, 18 | in_features=input_size, 19 | out_features=layer_sizes, 20 | n_selected_features=n_selected_features_input, 21 | parent_weights_dimension='out_features', 22 | operands=Predicates(feature_names=feature_names) 23 | ) 24 | 25 | Args: 26 | feature_names (list): A list of feature names. 27 | """ 28 | super(Predicates, self).__init__(feature_names=feature_names) 29 | self.logger = logging.getLogger(self.__class__.__name__) 30 | -------------------------------------------------------------------------------- /torchlogic/nn/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from .base import BaseConcatenateBlocksLogic 3 | 4 | 5 | class ConcatenateBlocksLogic(BaseConcatenateBlocksLogic): 6 | 7 | def __init__(self, modules, outputs_key): 8 | super(ConcatenateBlocksLogic, self).__init__(modules, outputs_key) 9 | 10 | def forward(self, *inputs): 11 | """ 12 | A channel logical xor. 13 | 14 | Args: 15 | *inputs: comma separated tensors of [BATCH_SIZE, CHANNELS, 1, 1] 16 | 17 | Returns: 18 | out: output tensor [BATCH_SIZE, OUT_CHANNELS, 1, # of input tensors] 19 | """ 20 | return torch.cat(inputs, dim=-1) 21 | -------------------------------------------------------------------------------- /torchlogic/sklogic/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/torchlogic/20d10b0462b6866f3853336ef3cfad19218dcb18/torchlogic/sklogic/__init__.py -------------------------------------------------------------------------------- /torchlogic/sklogic/base/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/torchlogic/20d10b0462b6866f3853336ef3cfad19218dcb18/torchlogic/sklogic/base/__init__.py -------------------------------------------------------------------------------- /torchlogic/sklogic/classifiers/__init__.py: -------------------------------------------------------------------------------- 1 | from .RNRNClassifier import RNRNClassifier 2 | 3 | __all__ = [RNRNClassifier] -------------------------------------------------------------------------------- /torchlogic/sklogic/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/torchlogic/20d10b0462b6866f3853336ef3cfad19218dcb18/torchlogic/sklogic/datasets/__init__.py -------------------------------------------------------------------------------- /torchlogic/sklogic/datasets/simple_dataset.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from torch.utils.data import Dataset 4 | 5 | 6 | class SimpleDataset(Dataset): 7 | 8 | """ 9 | Class for a simple dataset used in sklogic. 10 | """ 11 | 12 | def __init__( 13 | self, 14 | X: np.array, 15 | y: np.array 16 | ): 17 | """ 18 | Dataset suitable for SKLogic models model from torchlogic 19 | 20 | Args: 21 | X (np.array): features data scaled to [0, 1] 22 | y (np.array): target data of classes 0, 1 23 | """ 24 | super(SimpleDataset, self).__init__() 25 | self.X = X.astype('float') 26 | self.y = y 27 | self.sample_idx = np.arange(X.shape[0]) # index of samples 28 | 29 | def __len__(self): 30 | return self.X.shape[0] 31 | 32 | def __getitem__(self, idx): 33 | features = torch.from_numpy(self.X[idx, :]).float() 34 | target = torch.from_numpy(self.y[idx, :]) 35 | return {'features': features, 'target': target, 'sample_idx': idx} -------------------------------------------------------------------------------- /torchlogic/sklogic/regressors/__init__.py: -------------------------------------------------------------------------------- 1 | from .RNRNRegressor import RNRNRegressor 2 | 3 | __all__ = [RNRNRegressor] -------------------------------------------------------------------------------- /torchlogic/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .distributed import grad_agg, float_agg, tensor_agg 2 | from .explanations import simplification, remove_duplicate_words 3 | 4 | __all__ = [grad_agg, float_agg, tensor_agg, simplification, remove_duplicate_words] 5 | -------------------------------------------------------------------------------- /torchlogic/utils/distributed.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | import torch 4 | import torch.distributed as dist 5 | 6 | 7 | def grad_agg(params): 8 | r"""Aggregate gradients across multiple processes doing training.""" 9 | for p in params: 10 | if p.requires_grad: 11 | world_size = dist.get_world_size() 12 | val = torch.tensor([float(p.grad is not None)]) 13 | votes = float_agg(val) 14 | assert (votes == world_size) or (votes == 0) 15 | if p.grad is not None: 16 | dist.all_reduce(p.grad.data, op=dist.ReduceOp.SUM) 17 | 18 | 19 | def float_agg(val): 20 | r"""Aggregate any value val across multiple processes.""" 21 | val = torch.tensor([val]) 22 | dist.all_reduce(val, op=dist.ReduceOp.SUM) 23 | return val.item() 24 | 25 | 26 | def tensor_agg(tensor_in: Union[torch.FloatTensor, torch.LongTensor, torch.DoubleTensor, torch.Tensor]): 27 | r"""Aggregate any tensor by concatenation across multiple processes""" 28 | world_size = dist.get_world_size() 29 | tensor_out = [torch.zeros_like(tensor_in.reshape(-1)) for _ in range(world_size)] 30 | dist.all_gather(tensor_out, tensor_in.reshape(-1)) 31 | tensor_out = torch.cat(tensor_out) 32 | return tensor_out 33 | 34 | 35 | __all__ = [grad_agg, float_agg, tensor_agg] -------------------------------------------------------------------------------- /torchlogic/utils/explanations/__init__.py: -------------------------------------------------------------------------------- 1 | from .explanations import remove_duplicate_words, remove_character_combo, register_hooks, simplification 2 | 3 | __all__ = [remove_duplicate_words, remove_character_combo, register_hooks, simplification] 4 | -------------------------------------------------------------------------------- /torchlogic/utils/explanations/explanations.py: -------------------------------------------------------------------------------- 1 | import re 2 | from torchlogic.utils.explanations.simplification import Explanation 3 | 4 | 5 | def remove_duplicate_words(input1: str): 6 | # Regex to matching repeated words 7 | regex = r'\b(\w+)(?:\W+\1\b)+' 8 | 9 | return re.sub(regex, r'\1', input1, flags=re.IGNORECASE) 10 | 11 | 12 | def remove_character_combo(input1: str, char1: str, char2: str, replacement_char: str): 13 | # retplace the string ";," with ";" 14 | 15 | return input1.replace(f"{char1}{char2}", f"{replacement_char}") 16 | 17 | 18 | def get_outputs(name, outputs): 19 | def hook(model, input, output): 20 | if isinstance(output, tuple): 21 | outputs[name] = output[0].detach() 22 | else: 23 | outputs[name] = output.detach() 24 | 25 | return hook 26 | 27 | 28 | def register_hooks(model, outputs, mode='explanation'): 29 | for x in model.named_children(): 30 | x[1].register_forward_hook(get_outputs(x[0], outputs)) 31 | register_hooks(x[1], outputs) 32 | 33 | 34 | def simplification( 35 | explanation_str, 36 | print_type, 37 | simplify, 38 | sample_level=False, 39 | verbose=False, 40 | ndigits: int = 3, 41 | exclusions: list[str] = None, 42 | min_max_feature_dict: dict = None, 43 | feature_importances: bool = False 44 | ): 45 | if verbose: 46 | print("\n_____\nInput string:\n\n", explanation_str) 47 | 48 | if simplify: 49 | assert print_type in ['logical', 'logical-natural'], \ 50 | "print_type must be 'logical' or 'logical-natural' if simplify is True" 51 | 52 | if feature_importances: 53 | exp = Explanation(explanation_str, print_type) 54 | exp.root.sort_operands() 55 | if verbose: 56 | print("\n_____\nExplanation tree for feature importances (not simplified):\n", exp.root.tree_to_string()) 57 | return exp.root 58 | 59 | if print_type == 'natural': 60 | return explanation_str 61 | 62 | exp = Explanation(explanation_str, print_type) 63 | if verbose: 64 | exp.root.sort_operands() 65 | print("\n_____\nExplanation tree:\n", exp.root.tree_to_string()) 66 | 67 | if not simplify: 68 | exp.root.sort_operands() 69 | # return exp.root.tree_to_string() 70 | return exp.root 71 | 72 | exp.root = Explanation.push_negations_down(exp.root) 73 | if verbose: 74 | exp.root.sort_operands() 75 | print("\n_____\nNegations pushed down:\n\n", exp.root.tree_to_string()) 76 | 77 | exp.root = Explanation.recursively_collapse_consecutive_repeated_operands(exp.root) 78 | if verbose: 79 | exp.root.sort_operands() 80 | print("\n_____\nConsecutive operands collapsed:\n", exp.root.tree_to_string()) 81 | 82 | Explanation.recursive_explanation_parsing(exp.root) 83 | exp.root = Explanation.remove_redundant(exp.root) 84 | if verbose: 85 | exp.root.sort_operands() 86 | print("\n_____\nRedundant features removed:\n", exp.root.tree_to_string()) 87 | 88 | exp.root = Explanation.recursively_collapse_single_operands(exp.root) 89 | if verbose: 90 | exp.root.sort_operands() 91 | print("\n_____\nSingle operands collapsed:\n", exp.root.tree_to_string()) 92 | 93 | Explanation.recursive_explanation_parsing(exp.root) 94 | exp.root = Explanation.remove_redundant(exp.root) 95 | if verbose: 96 | exp.root.sort_operands() 97 | print("\n_____\nRedundant features removed:\n", exp.root.tree_to_string()) 98 | 99 | if not sample_level: 100 | Explanation.recursive_explanation_parsing(exp.root) # Adding feature, sign, value to each node 101 | exp.simple_root = Explanation.create_between_explanations(exp.root) 102 | exp.simple_root.sort_operands() 103 | if verbose: 104 | print("\n_____\nBetween explanations created:\n", exp.simple_root.tree_to_string()) 105 | # return exp.simple_root.tree_to_string() 106 | return exp.simple_root 107 | 108 | # Now we know that simplify is True, print_type is 'logical' and sample_level is True 109 | # Sample level simplification -- for sample level explanations only! 110 | exp.sample_root = Explanation.collapse_sample_explanation(exp.root) 111 | if verbose: 112 | exp.sample_root.sort_operands() 113 | print("\n_____\nThe tree collapsed to sample level:\n", exp.sample_root.tree_to_string()) 114 | 115 | Explanation.recursive_explanation_parsing(exp.sample_root) 116 | exp.sample_root = Explanation.remove_redundant(exp.sample_root, verbose=verbose) 117 | if verbose: 118 | exp.sample_root.sort_operands() 119 | print("\n_____\nSample level redundant features removed:\n", exp.sample_root.tree_to_string()) 120 | 121 | Explanation.recursive_explanation_parsing(exp.sample_root) # To add feature, sign, value to each node 122 | exp.sample_root = Explanation.create_between_explanations__sample_level(exp.sample_root) 123 | exp.sample_root.sort_operands() 124 | if verbose: 125 | print("\n_____\nSample level between explanations created:\n", exp.sample_root.tree_to_string()) 126 | 127 | Explanation.format_explanation(exp, exp.sample_root, ndigits, exclusions, min_max_feature_dict) 128 | if verbose: 129 | print("\n_____\nSample level formatted explanations created:\n", exp.sample_root.tree_to_string()) 130 | 131 | exp.sample_root.sort_operands() 132 | # return exp.sample_root.tree_to_string() 133 | return exp.sample_root 134 | 135 | -------------------------------------------------------------------------------- /torchlogic/utils/trainers/__init__.py: -------------------------------------------------------------------------------- 1 | from .banditnrntrainer import BanditNRNTrainer 2 | from .boostedbanditnrntrainer import BoostedBanditNRNTrainer 3 | from .attnnrntrainer import AttnNRNTrainer 4 | 5 | __all__ = [BanditNRNTrainer, BoostedBanditNRNTrainer, AttnNRNTrainer] 6 | -------------------------------------------------------------------------------- /torchlogic/utils/trainers/base/__init__.py: -------------------------------------------------------------------------------- 1 | from .basetrainer import BaseReasoningNetworkDistributedTrainer 2 | 3 | __all__ = [BaseReasoningNetworkDistributedTrainer] 4 | --------------------------------------------------------------------------------