├── .gitignore ├── BUILD.md ├── CHANGELOG.md ├── COMMITS.md ├── LICENSE ├── README.md ├── TODO.md ├── change-log.sh ├── docs ├── example.tar.gz ├── index.md ├── markdown.md └── options.md ├── mkdocs-api.sh ├── mkdocs-build.sh ├── mkdocs-deploy.sh ├── mkdocs-serve.sh ├── mkdocs.yml ├── pyproject.toml ├── requirements.txt ├── scripts ├── git-user-hooks │ ├── gitautoversion.py │ └── post-commit ├── solverpy-autotune ├── solverpy-compress ├── solverpy-decompress └── solverpy-deconflict ├── src └── solverpy │ ├── __init__.py │ ├── benchmark │ ├── __init__.py │ ├── db │ │ ├── __init__.py │ │ ├── cachedprovider.py │ │ ├── db.py │ │ ├── provider.py │ │ └── providers │ │ │ ├── __init__.py │ │ │ ├── jsons.py │ │ │ ├── loader.py │ │ │ ├── solved.py │ │ │ └── status.py │ ├── launcher.py │ ├── path │ │ ├── __init__.py │ │ ├── bids.py │ │ └── sids.py │ ├── reports │ │ ├── data.py │ │ ├── markdown.py │ │ └── progress.py │ └── setups │ │ ├── __init__.py │ │ ├── common.py │ │ ├── loop.py │ │ ├── setup.py │ │ ├── solver.py │ │ └── tuner.py │ ├── builder │ ├── __init__.py │ ├── autotune │ │ ├── __init__.py │ │ ├── autotune.py │ │ ├── build.py │ │ ├── check.py │ │ ├── listener.py │ │ └── tune.py │ ├── autotuner.py │ ├── builder.py │ ├── cvc5ml.py │ ├── enigma.py │ ├── plugins │ │ ├── __init__.py │ │ ├── cvc5.py │ │ ├── enigma.py │ │ ├── multi.py │ │ ├── svm.py │ │ └── trains.py │ └── svm.py │ ├── solver │ ├── __init__.py │ ├── atp │ │ ├── __init__.py │ │ ├── cvc5.py │ │ ├── eprover.py │ │ ├── lash.py │ │ ├── prover9.py │ │ └── vampire.py │ ├── object.py │ ├── plugins │ │ ├── __init__.py │ │ ├── db │ │ │ ├── __init__.py │ │ │ ├── bid.py │ │ │ ├── errors.py │ │ │ ├── outputs.py │ │ │ └── sid.py │ │ ├── decorator.py │ │ ├── plugin.py │ │ ├── shell │ │ │ ├── __init__.py │ │ │ ├── limits.py │ │ │ ├── memory.py │ │ │ ├── time.py │ │ │ └── timeout.py │ │ ├── status │ │ │ ├── __init__.py │ │ │ ├── limiter.py │ │ │ ├── smt.py │ │ │ └── tptp.py │ │ └── translator.py │ ├── pluginsolver.py │ ├── reloader.py │ ├── shellsolver.py │ ├── smt │ │ ├── __init__.py │ │ ├── bitwuzla.py │ │ ├── cvc5.py │ │ └── z3.py │ ├── solver.py │ ├── solverpy.py │ └── stdinsolver.py │ ├── task │ ├── __init__.py │ ├── bar.py │ ├── launcher.py │ ├── shelltask.py │ ├── solvertask.py │ └── task.py │ └── tools │ ├── __init__.py │ ├── human.py │ ├── log.py │ ├── patterns.py │ ├── redirect.py │ ├── timeme.py │ └── typing.py └── strats └── cvc5 ├── cvc5-smtcomp01 ├── cvc5-smtcomp02 ├── cvc5-smtcomp03 ├── cvc5-smtcomp04 ├── cvc5-smtcomp05 ├── cvc5-smtcomp06 ├── cvc5-smtcomp07 ├── cvc5-smtcomp08 ├── cvc5-smtcomp09 ├── cvc5-smtcomp10 ├── cvc5-smtcomp11 ├── cvc5-smtcomp12 ├── cvc5-smtcomp13 ├── cvc5-smtcomp14 ├── cvc5-smtcomp15 ├── cvc5-smtcomp16 ├── cvc5-smtcomp17 ├── cvc5-smtcomp18 ├── cvc5-smtcomp19 ├── cvc5-smtcomp20 ├── cvc5-smtcomp21 ├── cvc5-smtcomp22 └── cvc5-smtcomp23 /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | Pipfile 163 | Pipfile.lock 164 | _dust/ 165 | tags 166 | 167 | -------------------------------------------------------------------------------- /BUILD.md: -------------------------------------------------------------------------------- 1 | * to build: 2 | 3 | $ python3 -m build 4 | 5 | * to upload: 6 | 7 | $ twine upload dist/* 8 | 9 | * to generate requirements.txt of the package 10 | 11 | $ pipreqs 12 | 13 | * to see the current version string 14 | 15 | $ setuptools-git-versioning 16 | 17 | * to update the version tags 18 | 19 | $ ./version-tags.py 20 | 21 | and run the commands outputed; 22 | and then push the tags: 23 | 24 | $ git push --tags 25 | 26 | * to update the CHANGELOG.md run: 27 | 28 | $ ./change-log.sh 29 | 30 | + stuff about git amend: https://www.atlassian.com/git/tutorials/rewriting-history 31 | 32 | ```bash 33 | # Edit hello.py and main.py 34 | git add hello.py 35 | git commit 36 | # Realize you forgot to add the changes from main.py 37 | git add main.py 38 | git commit --amend --no-edit 39 | ``` 40 | 41 | + git hooks: https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks 42 | 43 | 44 | + automatically push tags for this repo 45 | 46 | $ git config push.followTags true 47 | 48 | or for all the repos globally: 49 | 50 | $ git config --global push.followTags true 51 | 52 | 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `solverpy`: Python Interface for Automated Solvers 2 | 3 | `solverpy` is a Python package providing a uniform interface to launch automated problem solvers from Python and process their outputs. Currently supported solvers are: 4 | 5 | * E Prover (solver.atp.eprover.E) 6 | * Vampire (solver.atp.vampire.Vampire) 7 | * Prover9 (solver.atp.prover9.Prover9) 8 | * Lash (solver.atp.lash.Lash) 9 | * cvc5 (solver.smt.cvc5.Cvc5) 10 | * Z3 (solver.smt.z3.Z3) 11 | * Bitwuzla (solver.smt.bitwuzla.Bitwuzla) 12 | 13 | Note that the solver binaries are not part of this Python package and must be installed separately by the user. The respective binaries must be (by default) in `PATH` if you wish to use them from `solverpy`. 14 | 15 | ## Installation 16 | 17 | ```bash 18 | $ pip install solverpy 19 | ``` 20 | 21 | ## Single problem solving 22 | 23 | To call the solver on one problem instance, start by creating the solver object. 24 | 25 | ```python 26 | from solverpy.solver.smt.cvc5 import Cvc5 27 | 28 | cvc5 = Cvc5("T5") # time limit of 5 seconds 29 | ``` 30 | 31 | The constructor argument is a resource limit string, in this case, a time limit `T` in seconds. All solvers support `T` and additional resource limits might be available depending on the solver. Multiple resource limits can be used (separated by `-`, like `T10-R50000`). The limit string must, however, always start with `T`. 32 | 33 | Then call the `solve` method: 34 | 35 | ```python 36 | result = cvc5.solve("myproblem.smt2", "--enum-inst") 37 | ``` 38 | 39 | The first argument is the problem filename, the second is the solver-dependent strategy description (typically command line options as a string). 40 | 41 | The result is a Python `dict` with results and statistics. The keys and values are solver-specific. Nevertheless, the result always contains keys `status` (with the value of type `str`) and `runtime` (type `float`). 42 | 43 | Hint: Call `cvc5.run(p,s)` instead of `cvc5.solve(p,s)` to get the raw solver output without any processing. Call `cvc5.command(p,s)` to output the shell command that is going to be executed to launch the solver. 44 | 45 | ## Parallel benchmark evaluation 46 | 47 | To evaluate a set of strategies on a set of benchmark problems, you just need to provide your experiment description as a Python `dict` and launch the experiments. 48 | 49 | ```python 50 | from solverpy.benchmark import setups 51 | 52 | mysetup = ... 53 | 54 | setups.launch(mysetup) 55 | ``` 56 | 57 | The experiment setup (`mysetup`) must have specific keys. The module `solverpy.benchmark.setups` contains methods to fill in the required keys and values. 58 | 59 | You must specify at least the following: 60 | 61 | | key | type | description | 62 | |--------------|-----------|---------------| 63 | | `cores` | `int` | number of CPU cores to use for parallel evaluation | 64 | | `sidlist` | `[str]` | list of strategies to evaluate | 65 | | `bidlist` | `[str]` | list of problems to evaluate on | 66 | | `limit` | `str` | the resource limit for a single solver run | 67 | 68 | ### Strategies and _strategy id's_ 69 | 70 | Strategies are stored in files in the directory `solverpy_db/strats` which must exist in the current working directory (the directory is adjustable by the `SOLVERPY_DB` environment variable). 71 | The filename of each strategy is used to reference the strategy in `sidlist` and it is called the _strategy id_ (`sid`). 72 | 73 | Hence, for every `sid` in `sidlist` in `mysetup`, there must be the file `solverpy_db/strats/sid` in the current working directory. 74 | This file contains the strategy definition (typically command line options) to pass to the `solver.solve` method. 75 | 76 | ### Problems and _benchmark id's_ 77 | 78 | Benchmark problem sets are represented by _benchark id's_ (`bid`). The _benchmark id_ is a file path relative to the current working directory (adjustable by the `SOLVERPY_BENCHMARKS` environment variable) 79 | pointing either to a file or to a directory. 80 | 81 | If the path leads to a: 82 | 83 | * `directory`: Then every regular file _directly_ in this directory is considered a benchmark problem. 84 | Directories and hidden files are ignored and no recursive search is performed. 85 | This variant is useful when you have a set of problem files, all in one directory. 86 | * `file`: Then the file consists of lines containing paths to corresponding problem files. 87 | The paths are relative to the directory of the `bid` file. For example, if the `bid` is `myproblems/subset1` and this file contains (among others) the line `category1/problem23.smt2` then the problem must be placed in `myproblems/category1/problem23.smt2` (because the directory of the `bid` file is `myproblems`). 88 | This variant is useful when your benchmarks are structured in subdirectories and you don't want to merge them into one directory. 89 | 90 | ### Experiments example 91 | 92 | Suppose you have some SMT problems in the directory `myproblems` and that you want to evaluate your cvc5 strategies `buzzard`, `sparrow`, and `chickadee`, which you have placed in the directory `solverpy_db/strats`. You can download the archive with files for this example [here](https://github.com/cbboyan/solverpy/raw/main/docs/example.tar.gz). 93 | 94 | You proceed as follows. First, you create the description of your experiments in `mysetup`. 95 | 96 | ```python 97 | from solverpy.benchmark import setups 98 | 99 | mysetup = { 100 | "cores": 4, 101 | "limit": "T10", 102 | "bidlist": ["myproblems"], 103 | "sidlist": ["buzzard", "sparrow", "chickadee"], 104 | } 105 | ``` 106 | 107 | Hint: Add `mysetup["options"] = ["outputs"]` if you want to keep raw solver output files from all solver runs. 108 | 109 | Hint: Options are slightly more described [here](docs/options.md). 110 | 111 | Then you specify that you want to use cvc5 and that you wish to launch an evaluation. These methods update `mysetup` and fill in some keys required by `setups.launch()`. 112 | 113 | ```python 114 | setups.cvc5(mysetup) 115 | setups.evaluation(mysetup) 116 | ``` 117 | 118 | Finally, you launch the experiments. 119 | 120 | ```python 121 | setups.launch(mysetup) 122 | ``` 123 | 124 | You will see the progress of the experiments on the screen. Once finished, you will find the following subdirectories inside `solverpy_db`: 125 | 126 | | directory | content | 127 | |--------------|-----------| 128 | | `results` | Results by each strategy (`sid`) for each `bid`. The result for each `sid` and `bid` is a JSON file (gzip-ed) with a Python dictionary `{problem: result}`. | 129 | | `solved` | List of solved problem names by each strategy for each `bid`. One per line, easy to `grep` and `cat`. | 130 | | `status` | Statuses of all problems by each strategy for each `bid`. Problem name and status at one line, TAB separated. Easy to `cut`. | 131 | | `log` | Console log for each `solverpy` experiment run. | 132 | | `outputs` | Raw solver output files for each solver run (only if selected). | 133 | 134 | Now run the script again and notice that it finished much faster. It is because the cached results were reused and no solvers were actually launched. So be careful and always clean the database if you want to force recompute. Simply delete all the directories in `solverpy_db` except `strats` (see the script `clean_db.sh` in the [example archive](https://github.com/cbboyan/solverpy/raw/main/docs/example.tar.gz)). 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | [ ] automated restriction to solvable problems (force implementation) 2 | [x] re-using trains 3 | [ ] trains regeneration 4 | [ ] show best iteration in the training report 5 | [ ] show the initial model in the report 6 | [ ] reporting (!!!) 7 | [ ] improve total bar eta by not including skipped problems 8 | [ ] improve eta by considering timeout (?) 9 | -------------------------------------------------------------------------------- /change-log.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | previous_tag=0 3 | #for current_tag in $(git tag --sort=-creatordate) 4 | for current_tag in $(git tag --sort=-v:refname) 5 | do 6 | 7 | if [ "$previous_tag" != 0 ];then 8 | tag_date=$(git log -1 --pretty=format:'%ad' --date=short ${previous_tag}) 9 | printf "## ${previous_tag} (${tag_date})\n\n" 10 | git log ${current_tag}...${previous_tag} --pretty=format:'* %s [View](https://github.com/cbboyan/solverpy/commit/%H)' --reverse | grep -v Merge 11 | printf "\n\n" 12 | fi 13 | previous_tag=${current_tag} 14 | done | grep -vi changelog | tee CHANGELOG.md 15 | -------------------------------------------------------------------------------- /docs/example.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbboyan/solverpy/caa8da555ff766a50e1b0b0a01f2747df65d8bdb/docs/example.tar.gz -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Python Interface for Automated Solvers 2 | 3 | `solverpy` is a Python package providing a uniform interface to launch automated problem solvers from Python and process their outputs. Currently supported solvers are: 4 | 5 | * E Prover [`E`][solverpy.solver.solver] 6 | * Vampire (solver.atp.vampire.Vampire) 7 | * Prover9 (solver.atp.prover9.Prover9) 8 | * Lash (solver.atp.lash.Lash) 9 | * cvc5 (solver.smt.cvc5.Cvc5) 10 | * Z3 (solver.smt.z3.Z3) 11 | * Bitwuzla (solver.smt.bitwuzla.Bitwuzla) 12 | 13 | > 🗒️ **Note**: The solver binaries/libraries are not part of this Python 14 | > package and must be installed separately. The binaries must be (by default) 15 | > in `PATH` or specified using `bin` parameter, if you wish to use them from 16 | > `solverpy`. 17 | 18 | ## Installation 19 | 20 | ```sh 21 | $ pip install solverpy 22 | ``` 23 | 24 | ## Single problem solving 25 | 26 | To call the solver on one problem instance, start by creating the solver object. 27 | 28 | ```python 29 | from solverpy.solver.smt.cvc5 import Cvc5 30 | 31 | cvc5 = Cvc5("T5") # time limit of 5 seconds 32 | ``` 33 | 34 | The constructor argument is a resource limit string, in this case, a time limit `T` in seconds. All solvers support `T` and additional resource limits might be available depending on the solver. Multiple resource limits can be used (separated by `-`, like `T10-R50000`). The limit string must, however, always start with `T`. 35 | 36 | Then call the `solve` method: 37 | 38 | ```python 39 | result = cvc5.solve("myproblem.smt2", "--enum-inst") 40 | ``` 41 | 42 | The first argument is the problem filename, the second is the solver-dependent strategy description (typically command line options as a string). 43 | 44 | The result is a Python `dict` with results and statistics. The keys and values are solver-specific. Nevertheless, the result always contains keys `status` (with the value of type `str`) and `runtime` (type `float`). 45 | 46 | 💡 Call `cvc5.run(p,s)` instead of `cvc5.solve(p,s)` to get the raw solver output without any processing. 47 | 48 | 💡 Call `cvc5.command(p,s)` to output the shell command that is going to be executed to launch the solver. 49 | 50 | 51 | ## Parallel benchmark evaluation 52 | 53 | To evaluate a set of strategies on a set of benchmark problems, you just need to provide your experiment description as a Python `dict` and launch the experiments. 54 | 55 | ```python 56 | from solverpy.benchmark import setups 57 | 58 | mysetup = ... 59 | 60 | setups.launch(mysetup) 61 | ``` 62 | 63 | The experiment setup (`mysetup`) must have specific keys. The module `solverpy.benchmark.setups` contains methods to fill in the required keys and values. 64 | 65 | You must specify at least the following: 66 | 67 | | key | type | description | 68 | |--------------|-----------|---------------| 69 | | `cores` | `int` | number of CPU cores to use for parallel evaluation | 70 | | `sidlist` | `[str]` | list of strategies to evaluate | 71 | | `bidlist` | `[str]` | list of problems to evaluate on | 72 | | `limit` | `str` | the resource limit for a single solver run | 73 | 74 | ### Strategies and _strategy id's_ 75 | 76 | Strategies are stored in files in the directory `solverpy_db/strats` which must exist in the current working directory (the directory is adjustable by the `SOLVERPY_DB` environment variable). 77 | The filename of each strategy is used to reference the strategy in `sidlist` and it is called the _strategy id_ (`sid`). 78 | 79 | Hence, for every `sid` in `sidlist` in `mysetup`, there must be the file `solverpy_db/strats/sid` in the current working directory. 80 | This file contains the strategy definition (typically command line options) to pass to the `solver.solve` method. 81 | 82 | ### Problems and _benchmark id's_ 83 | 84 | Benchmark problem sets are represented by _benchark id's_ (`bid`). The _benchmark id_ is a file path relative to the current working directory (adjustable by the `SOLVERPY_BENCHMARKS` environment variable) 85 | pointing either to a file or to a directory. 86 | 87 | If the path leads to a: 88 | 89 | * `directory`: Then every regular file _directly_ in this directory is considered a benchmark problem. 90 | Directories and hidden files are ignored and no recursive search is performed. 91 | This variant is useful when you have a set of problem files, all in one directory. 92 | * `file`: Then the file consists of lines containing paths to corresponding problem files. 93 | The paths are relative to the directory of the `bid` file. For example, if the `bid` is `myproblems/subset1` and this file contains (among others) the line `category1/problem23.smt2` then the problem must be placed in `myproblems/category1/problem23.smt2` (because the directory of the `bid` file is `myproblems`). 94 | This variant is useful when your benchmarks are structured in subdirectories and you don't want to merge them into one directory. 95 | 96 | ### Experiments example 97 | 98 | Suppose you have some SMT problems in the directory `myproblems` and that you want to evaluate your cvc5 strategies `buzzard`, `sparrow`, and `chickadee`, which you have placed in the directory `solverpy_db/strats`. You can download the archive with files for this example [here](https://github.com/cbboyan/solverpy/raw/main/docs/example.tar.gz). 99 | 100 | You proceed as follows. First, you create the description of your experiments in `mysetup`. 101 | 102 | ```python 103 | from solverpy.benchmark import setups 104 | 105 | mysetup = { 106 | "cores": 4, 107 | "limit": "T10", 108 | "bidlist": ["myproblems"], 109 | "sidlist": ["buzzard", "sparrow", "chickadee"], 110 | } 111 | ``` 112 | 113 | Hint: Add `mysetup["options"] = ["outputs"]` if you want to keep raw solver output files from all solver runs. 114 | 115 | Hint: Options are slightly more described [here](options.md). 116 | 117 | Then you specify that you want to use cvc5 and that you wish to launch an evaluation. These methods update `mysetup` and fill in some keys required by `setups.launch()`. 118 | 119 | ```python 120 | setups.cvc5(mysetup) 121 | setups.evaluation(mysetup) 122 | ``` 123 | 124 | Finally, you launch the experiments. 125 | 126 | ```python 127 | setups.launch(mysetup) 128 | ``` 129 | 130 | You will see the progress of the experiments on the screen. Once finished, you will find the following subdirectories inside `solverpy_db`: 131 | 132 | | directory | content | 133 | |--------------|-----------| 134 | | `results` | Results by each strategy (`sid`) for each `bid`. The result for each `sid` and `bid` is a JSON file (gzip-ed) with a Python dictionary `{problem: result}`. | 135 | | `solved` | List of solved problem names by each strategy for each `bid`. One per line, easy to `grep` and `cat`. | 136 | | `status` | Statuses of all problems by each strategy for each `bid`. Problem name and status at one line, TAB separated. Easy to `cut`. | 137 | | `log` | Console log for each `solverpy` experiment run. | 138 | | `outputs` | Raw solver output files for each solver run (only if selected). | 139 | 140 | Now run the script again and notice that it finished much faster. It is because the cached results were reused and no solvers were actually launched. So be careful and always clean the database if you want to force recompute. Simply delete all the directories in `solverpy_db` except `strats` (see the script `clean_db.sh` in the [example archive](https://github.com/cbboyan/solverpy/raw/main/docs/example.tar.gz)). 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | -------------------------------------------------------------------------------- /docs/markdown.md: -------------------------------------------------------------------------------- 1 | 🛠️ Unicode Symbols for Markdown Documentation 2 | 3 | ℹ️ Info Symbol Unicode Meaning Usage Example 4 | 5 | ℹ️ U+2139 Information ℹ️ Use with caution in debug mode. 6 | 🛈 U+1F6C8 Alternative info symbol 🛈 API is subject to change. 7 | 📘 U+1F4D8 Reference/Blue book 📘 See section 4.2 of the guide. 8 | 🧾 U+1F9FE Documentation/Receipt 🧾 Generated logs are saved to /var/log. 9 | 🧠 U+1F9E0 Insight/Brain 🧠 This technique boosts performance. 10 | 11 | 📝 Note Symbol Unicode Meaning Usage Example 12 | 13 | 📝 U+1F4DD Memo/Note 📝 This method is deprecated. 14 | 🗒️ U+1F5D2 Notepad 🗒️ Make sure to configure both fields. 15 | 📌 U+1F4CC Pinned Note 📌 This setting is environment-specific. 16 | 📎 U+1F4CE Paperclip/Attached 📎 Attached are the config templates. 17 | 🧐 U+1F9D0 Observational Note 🧐 Rare edge case: input may be null. 18 | 19 | 💡 Hint / Tip Symbol Unicode Meaning Usage Example 20 | 21 | 💡 U+1F4A1 Lightbulb 💡 Try caching the results for speed. 22 | 🔍 U+1F50D Insight/Search 🔍 You can filter logs by severity. 23 | 🧠 U+1F9E0 Brain/Insight 🧠 Consider using memoization here. 24 | 🔧 U+1F527 Tool/Fix tip 🔧 Use the --fix flag to auto-correct. 25 | 26 | ⚠️ Warning Symbol Unicode Meaning Usage Example 27 | ⚠️ U+26A0 Warning ⚠️ Do not delete system files. 28 | ❗ U+2757 Important/Alert ❗ Backup your data before proceeding. 29 | ❕ U+2755 Light Warning ❕ This may cause a minor delay. 30 | 🚧 U+1F6A7 Construction/Partial 🚧 Feature under development. 31 | 32 | ❌ Error / Danger Symbol Unicode Meaning Usage Example 33 | 34 | ❌ U+274C Error ❌ Invalid configuration file. 35 | 🔥 U+1F525 Hot Issue 🔥 Major performance hit detected. 36 | 🛑 U+1F6D1 Stop/Critical 🛑 Service will shut down immediately. 37 | 38 | ✅ Success / Done Symbol Unicode Meaning Usage Example 39 | 40 | ✅ U+2705 Success/Passed ✅ All unit tests passed. 41 | ☑️ U+2611 Checkbox ☑️ Enabled via config. 42 | 🎉 U+1F389 Celebration 🎉 Initial setup complete! 43 | 44 | -------------------------------------------------------------------------------- /docs/options.md: -------------------------------------------------------------------------------- 1 | # SolverPy Options 2 | 3 | + An *option* is identified by its string name, and represents a boolean yes/no value. 4 | + Put the option name into the string list under the key`"options"` in the setup. 5 | + Use `no-`option, like `no-compress`, to set the option to `no` 6 | 7 | |option|description|default| 8 | |------|-----------|-------| 9 | |`outputs`|Keep raw solver output files from all runs|no| 10 | |`compress`|Compress output files (outputs, trains, results).|yes| 11 | |`flatten`|Put all output files in a single directory (replace `/` with `_._`).|yes| 12 | |`compress-trains`|Compress trains.|yes| 13 | |`debug-trains`|Dump training data for each file separately.|no| 14 | 15 | -------------------------------------------------------------------------------- /mkdocs-api.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for i in `find src -name "*.py" | grep -v __init__ | cut -d/ -f2-`; do 4 | j="${i:0:-3}"; 5 | k=`echo $j | tr "/" "."`; 6 | mkdir -p `dirname docs/api/$j`; 7 | echo "::: $k" > docs/api/$j.md; 8 | echo "done: $k" 9 | done 10 | -------------------------------------------------------------------------------- /mkdocs-build.sh: -------------------------------------------------------------------------------- 1 | mkdocs build 2 | -------------------------------------------------------------------------------- /mkdocs-deploy.sh: -------------------------------------------------------------------------------- 1 | mkdocs gh-deploy 2 | -------------------------------------------------------------------------------- /mkdocs-serve.sh: -------------------------------------------------------------------------------- 1 | mkdocs serve 2 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: SolverPy 2 | site_url: https://cbboyan.github.io/solverpy/ 3 | 4 | theme: 5 | name: material 6 | 7 | markdown_extensions: 8 | - pymdownx.highlight 9 | - pymdownx.superfences 10 | - pymdownx.inlinehilite 11 | 12 | plugins: 13 | - search 14 | - mkdocstrings: 15 | handlers: 16 | python: 17 | paths: [src] 18 | - autorefs: {} 19 | 20 | nav: 21 | - Home: index.md 22 | - Options: options.md 23 | - Documentation: api/reference.md 24 | 25 | 26 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools","setuptools-git-versioning<2"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "solverpy" 7 | description = "Python inteface for Automated Reasoning (AR) solvers, provers, and checkers." 8 | readme = "README.md" 9 | requires-python = ">=3.12" 10 | classifiers = [ 11 | "Programming Language :: Python :: 3", 12 | "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", 13 | "Operating System :: OS Independent", 14 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 15 | ] 16 | dependencies = [ 17 | "PyYAML", 18 | "numpy", 19 | "Requests", 20 | "scikit_learn", 21 | "scipy", 22 | "tqdm", 23 | ] 24 | dynamic = ["version"] 25 | 26 | [project.urls] 27 | "Homepage" = "https://github.com/cbboyan/solverpy" 28 | "Bug Tracker" = "https://github.com/cbboyan/solverpy/issues" 29 | 30 | [tool.setuptools-git-versioning] 31 | enabled = true 32 | 33 | [tool.yapf] 34 | based_on_style = "pep8" 35 | indent_width = 3 36 | continuation_indent_width = 3 37 | 38 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | lightgbm 2 | numpy 3 | optuna 4 | Requests 5 | scikit_learn 6 | scipy 7 | tqdm 8 | antlr4-python3-runtime==4.10 9 | -------------------------------------------------------------------------------- /scripts/git-user-hooks/gitautoversion.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import subprocess 4 | 5 | # change this to you GitHub names (or adjust the link format in the code) 6 | GIT_USER = "cbboyan" 7 | 8 | GIT_PROJECT = "solverpy" 9 | 10 | # commits with the following types are skiped in the change log 11 | SKIP_TYPE = ["chore", "merge"] 12 | 13 | # skip all commits with one of these keywords in the change log 14 | SKIP_MSG = ["README", "ChangeLog", "CHANGELOG", "changelog", "aaa", "Merge branch"] 15 | 16 | def gitlog(last_hash=True): 17 | commits = [] 18 | logs = subprocess.check_output("git log --oneline --decorate --decorate-refs=tags", shell=True) 19 | logs = logs.decode().strip().split("\n") 20 | for commit in logs: 21 | parts = commit.split(" ") 22 | hsh = parts[0] 23 | i = 1 24 | tag = None 25 | if parts[1] == "(tag:": 26 | tag = parts[2].rstrip(")") 27 | i = 3 28 | ver = None 29 | if tag and tag.startswith("v"): 30 | ver = [int(x) for x in tag[1:].split(".")] 31 | typ = None 32 | if parts[i].endswith(":"): 33 | typ = parts[i].rstrip(":") 34 | i += 1 35 | msg = " ".join(parts[i:]) 36 | commits.append([hsh, ver, typ, msg]) 37 | if not last_hash: 38 | commits[0][0] = "" 39 | commits.reverse() 40 | return commits 41 | 42 | def gitversion(ver): 43 | tag = f"v{'.'.join(map(str,ver))}" 44 | return tag 45 | 46 | def gittags(commits, update_commits=False): 47 | cur = [0, 0, 0] # [MAJOR, MINOR, PATCH] 48 | tags = [] 49 | for commit in commits: 50 | (hsh, ver, typ, msg) = commit 51 | if ver: 52 | cur = ver 53 | continue # this commit already has a version tag 54 | if typ: # but not ver 55 | if typ.endswith("!!!"): 56 | cur = [cur[0]+1, 0, 0] # increase MAJOR 57 | elif typ.endswith("!!"): 58 | cur = [cur[0], cur[1]+1, 0] # increase MINOR 59 | elif typ.endswith("!"): 60 | cur = [cur[0], cur[1], cur[2]+1] # increase PATCH 61 | else: 62 | continue # this commit needs no version tag 63 | tags.append((hsh, cur)) 64 | if update_commits: 65 | commit[1] = cur # update the version 66 | return tags 67 | 68 | def gitchanges(commits): 69 | changes = [] 70 | cur = [] 71 | for (hsh, ver, typ, msg) in commits: 72 | if (typ in SKIP_TYPE) or any((skip in msg) for skip in SKIP_MSG): 73 | continue # skip this commit in the change log 74 | cur.append((hsh, typ, msg)) 75 | if ver: 76 | cur.reverse() 77 | changes.append((gitversion(ver), cur)) 78 | cur = [] 79 | if cur: 80 | cur.reverse() 81 | changes.append(("Unreleased changes", cur)) 82 | changes.reverse() 83 | return changes 84 | 85 | def gitdate(hsh): 86 | cmd = f"git log -1 --pretty=format:'%ad' --date=short {hsh}" 87 | return subprocess.check_output(cmd, shell=True).decode() 88 | 89 | def changelog(commits): 90 | changes = gitchanges(commits) 91 | lines = [] 92 | lines.append("# Change Log") 93 | lines.append("") 94 | for (ver, coms) in changes: 95 | date = gitdate(coms[0][0]) 96 | lines.append(f"## {ver} ({date})") 97 | lines.append("") 98 | for (hsh, typ, msg) in coms: 99 | typ = f"{typ}: " if typ else "" 100 | url1 = f"https://github.com/{GIT_USER}/{GIT_PROJECT}/commit/{hsh}" 101 | url2 = f"https://github.com/{GIT_USER}/{GIT_PROJECT}/tree/{hsh}" 102 | lines.append(f"* {typ}{msg} [[details]({url1}) | [browse]({url2})]") 103 | lines.append("") 104 | if lines: 105 | lines.append("") 106 | return "\n".join(lines) 107 | 108 | -------------------------------------------------------------------------------- /scripts/git-user-hooks/post-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | import os 5 | 6 | import gitautoversion as gita 7 | 8 | LOCKFILE = ".skip-post-commit-hook.lock" 9 | 10 | if os.path.isfile(LOCKFILE): 11 | sys.exit(0) 12 | 13 | print("git-auto-version[post-commit]: updating version tags and CHANGELOG.md") 14 | 15 | # update change log 16 | commits = gita.gitlog(last_hash=False) 17 | gita.gittags(commits, update_commits=True) 18 | changelog = gita.changelog(commits) 19 | with open("CHANGELOG.md", "w") as f: 20 | f.write(changelog) 21 | 22 | # update (ammend) the commit with the update change log 23 | os.system("git add CHANGELOG.md") 24 | open(LOCKFILE, "w").close() 25 | os.system("git commit --amend --no-edit") 26 | os.remove(LOCKFILE) 27 | 28 | # set version tags 29 | commits = gita.gitlog() 30 | tags = gita.gittags(commits) 31 | for (hsh, ver) in tags: 32 | tag = gita.gitversion(ver) 33 | cmd = f"git tag -a {tag} {hsh} -m 'Version {tag}'" 34 | print(f"git-auto-version[post-commit]: settting version tag {tag} for commit {hsh}") 35 | os.system(cmd) 36 | 37 | -------------------------------------------------------------------------------- /scripts/solverpy-autotune: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import logging 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | def arguments(): 8 | import argparse 9 | parser = argparse.ArgumentParser() 10 | parser.add_argument("train", 11 | help="Trainig examples file in the SVM format (plain or compressed).") 12 | parser.add_argument("test", nargs="?", 13 | help="Testing examples file in the SVM format (plain or compressed).") 14 | parser.add_argument("--phases", metavar="p", default="l:b:m:r", 15 | help="LightGBM arguments to tune. (default: 'l:b:m:r')") 16 | parser.add_argument("--iters", type=int, metavar="it", default=16, 17 | help="Maximal number of trial models to build (default: 16).") 18 | parser.add_argument("--timeout", type=int, metavar="t", default=None, 19 | help="Overall timeout for tuning in seconds (default: unlimited).") 20 | parser.add_argument("--min_leaves", type=int, metavar="min", default=16, 21 | help="Minimal number of leaves for the leaves trial phase `l` (default: 16)."), 22 | parser.add_argument("--max_leaves", type=int, metavar="max", default=2048, 23 | help="Maximal number of leaves for the leaves trial phase `l` (default: 2048)."), 24 | args = parser.parse_args() 25 | args.test = args.test or args.train 26 | return args 27 | 28 | def main(args): 29 | import json 30 | from solverpy.builder import autotune 31 | from solverpy.tools import human 32 | 33 | best = autotune.prettytuner( 34 | f_train=args.train, 35 | f_test=args.test, 36 | phases=args.phases, 37 | timeout=args.timeout, 38 | iters=args.iters, 39 | min_leaves=args.min_leaves, 40 | max_leaves=args.max_leaves, 41 | ) 42 | 43 | print( 44 | f"Best model score: {best[0]}\n" 45 | f"Best model accuracy: {human.humanacc(best[1])}\n" 46 | f"Best model file: {best[2]}\n" 47 | f"Best model training time: {human.humantime(best[3])}\n" 48 | f"Positive training samples: {best[5]:.0f}\n" 49 | f"Negative training samples: {best[6]:.0f}\n" 50 | f"Best model training parameters: {json.dumps(best[4],indent=2,sort_keys=True)}\n" 51 | ) 52 | 53 | if __name__ == "__main__": 54 | logging.basicConfig(level=logging.INFO, format="%(message)s") 55 | args = arguments() 56 | main(args) 57 | 58 | -------------------------------------------------------------------------------- /scripts/solverpy-compress: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | from solverpy.builder import svm 5 | 6 | f_train = sys.argv[1] if len(sys.argv) > 1 else "train.in" 7 | 8 | svm.compress(f_train) 9 | 10 | -------------------------------------------------------------------------------- /scripts/solverpy-decompress: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | from solverpy.builder import svm 5 | 6 | f_train = sys.argv[1] if len(sys.argv) > 1 else "train.in" 7 | 8 | svm.decompress(f_train, keep=True) 9 | 10 | -------------------------------------------------------------------------------- /scripts/solverpy-deconflict: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | def arguments(): 4 | import argparse 5 | parser = argparse.ArgumentParser(description= 6 | "Process input and optional output file.") 7 | parser.add_argument("input", metavar="input.in", help="Path to input file") 8 | parser.add_argument("output", metavar="output.in", nargs='?', 9 | default="deconflicted.in", help="Path to output file (default: deconflicted.in)") 10 | parser.add_argument("-v", "--verbose", action="store_true", 11 | help="Show more details.") 12 | args = parser.parse_args() 13 | return args 14 | 15 | def main(f_in, f_out): 16 | from solverpy.builder import svm 17 | (xs, ys) = svm.load(f_in) 18 | (xs0, ys0) = svm.deconflict(xs, ys) 19 | svm.save(xs0, ys0, f_out) 20 | 21 | if __name__ == "__main__": 22 | import logging 23 | #logger = logging.getLogger(__name__) 24 | args = arguments() 25 | level = logging.DEBUG if args.verbose else logging.INFO 26 | logging.basicConfig(level=level, format="%(message)s") 27 | main(args.input, args.output) 28 | 29 | -------------------------------------------------------------------------------- /src/solverpy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbboyan/solverpy/caa8da555ff766a50e1b0b0a01f2747df65d8bdb/src/solverpy/__init__.py -------------------------------------------------------------------------------- /src/solverpy/benchmark/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbboyan/solverpy/caa8da555ff766a50e1b0b0a01f2747df65d8bdb/src/solverpy/benchmark/__init__.py -------------------------------------------------------------------------------- /src/solverpy/benchmark/db/__init__.py: -------------------------------------------------------------------------------- 1 | from .db import DB 2 | from .providers.jsons import Jsons, JsonsStore 3 | from .providers.solved import Solved 4 | from .providers.status import Status 5 | from .providers.loader import Loader, SlashLoader 6 | 7 | def default(delfix=None): 8 | return DB([Jsons.Maker(), Solved.Maker(delfix=delfix), Status.Maker(delfix=delfix)]) 9 | 10 | __all__ = ["DB", "Jsons", "JsonsStore", "Solved", "Status", "Loader", 11 | "SlashLoader", "default"] 12 | 13 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/db/cachedprovider.py: -------------------------------------------------------------------------------- 1 | from typing import TextIO 2 | import os 3 | import gzip 4 | import logging 5 | 6 | from .provider import Provider 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | class CachedProvider(Provider): 12 | 13 | def __init__( 14 | self, 15 | bid: str, 16 | sid: str, 17 | limit: (str | None) = None, 18 | store_cache: bool = False, 19 | compress: bool = False, 20 | ): 21 | Provider.__init__( 22 | self, 23 | bid, 24 | sid, 25 | limit, 26 | store_cache, 27 | ) 28 | self.compress = compress 29 | logger.debug(f"creating provider {self} for {sid} @ {bid}") 30 | self.load() 31 | 32 | def __repr__(self) -> str: 33 | if self.limit: 34 | return f"{type(self).__name__}({self.bid},{self.sid},{self.limit})" 35 | else: 36 | return f"{type(self).__name__}({self.bid},{self.sid})" 37 | 38 | def commit(self) -> None: 39 | if self.cache is None: 40 | logger.warning("empty cache commit") 41 | return 42 | if self._uptodate: 43 | logger.debug(f"commit skipped; cache {self} is up-to-date") 44 | return 45 | ext = ".gz" if self.compress else "" 46 | f = f"{self.cachepath()}{ext}" 47 | logger.debug(f"cache {self} writing {f}") 48 | os.makedirs(os.path.dirname(f), exist_ok=True) 49 | ouropen = gzip.open if self.compress else open 50 | with ouropen(f, "wt") as fw: 51 | self.cachedump(fw) 52 | logger.debug(f"cache {self} saved {len(self.cache)} entries") 53 | self._uptodate = True 54 | 55 | def load(self) -> None: 56 | if self._uptodate: 57 | logger.debug(f"loading skipped; cache {self} is up-to-date") 58 | return 59 | ext = ".gz" if self.compress else "" 60 | f = f"{self.cachepath()}{ext}" 61 | logger.debug(f"cache {self} loading {f}") 62 | if os.path.isfile(f): 63 | ouropen = gzip.open if self.compress else open 64 | with ouropen(f, "rt") as fr: 65 | self.cacheload(fr) 66 | logger.debug(f"cache {self} loaded {len(self.cache)} entries") 67 | else: 68 | self.cache = {} 69 | logger.debug(f"cache {self} not found {f}") 70 | self._uptodate = True 71 | 72 | def cachepath(self) -> str: 73 | raise NotImplementedError() 74 | 75 | def cachedump(self, fw: TextIO) -> None: 76 | del fw 77 | raise NotImplementedError() 78 | 79 | def cacheload(self, fr: TextIO) -> None: 80 | del fr 81 | raise NotImplementedError() 82 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/db/db.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | import logging 3 | 4 | from ...solver.object import SolverPyObj 5 | from ...benchmark.path import bids 6 | 7 | if TYPE_CHECKING: 8 | from .provider import ProviderMaker 9 | from .provider import Provider 10 | from ...task.solvertask import SolverTask 11 | from ...tools.typing import Result 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | class DB(SolverPyObj): 17 | 18 | def __init__(self, providers: list["ProviderMaker"]): 19 | SolverPyObj.__init__(self) 20 | self._providers = providers 21 | self.loaded: dict[tuple[str, str], list["Provider"]] = {} 22 | 23 | def represent(self) -> dict[str, Any]: 24 | return dict( 25 | path=bids.dbpath(), 26 | providers=self._providers, 27 | ) 28 | 29 | def connect(self, bid: str, sid: str, limit: str) -> None: 30 | if (bid, sid) not in self.loaded: 31 | logger.debug(f"connecting providers for {sid} @ {bid}") 32 | insts = [maker(bid, sid, limit) for maker in self._providers] 33 | self.loaded[(bid, sid)] = insts 34 | logger.debug(f"connected to {len(insts)} providers") 35 | 36 | def providers(self, task: "SolverTask") -> list["Provider"]: 37 | return self.loaded[(task.bid, task.sid)] 38 | 39 | def commit(self) -> None: 40 | for key in self.loaded: 41 | logger.debug(f"db commit: {key}") 42 | for provider in self.loaded[key]: 43 | provider.commit() 44 | 45 | def querytask( 46 | self, 47 | task: "SolverTask", 48 | ) -> "Result | None": 49 | self.connect(task.bid, task.sid, task.solver._limits.limit) 50 | for provider in self.providers(task): 51 | provider.check(task) 52 | result = provider.query(task) 53 | if result: 54 | return result 55 | return None 56 | 57 | def cachedtask( 58 | self, 59 | task: "SolverTask", 60 | result: "Result", 61 | ) -> None: 62 | self.connect(task.bid, task.sid, task.solver._limits.limit) 63 | for provider in self.providers(task): 64 | provider.check(task) 65 | provider.cached(task, result) 66 | 67 | def storetask( 68 | self, 69 | task: "SolverTask", 70 | result: "Result", 71 | ) -> None: 72 | self.connect(task.bid, task.sid, task.solver._limits.limit) 73 | for provider in self.providers(task): 74 | provider.check(task) 75 | provider.store(task, result) 76 | 77 | def query( 78 | self, 79 | tasks: list["SolverTask"], 80 | ) -> dict["SolverTask", dict[str, Any]]: 81 | logger.debug(f"db query on {len(tasks)} tasks") 82 | results = {} 83 | for task in tasks: 84 | result = self.querytask(task) 85 | if result: 86 | results[task] = result 87 | logger.debug(f"announcing {len(results)} cached results") 88 | for (task, result) in results.items(): 89 | self.cachedtask(task, result) 90 | logger.debug(f"db query done: {len(results)} tasks already done") 91 | return results 92 | 93 | def store( 94 | self, 95 | tasks: list["SolverTask"], 96 | results: list["Result"], 97 | ) -> None: 98 | logger.debug(f"db store on {len(tasks)} tasks") 99 | count = 0 100 | for (task, result) in zip(tasks, results): 101 | self.storetask(task, result) 102 | count += 1 103 | self.commit() 104 | logger.debug(f"db store done: {count} commits") 105 | 106 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/db/provider.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, TYPE_CHECKING 2 | from ...solver.object import SolverPyObj 3 | 4 | if TYPE_CHECKING: 5 | from ...task.solvertask import SolverTask 6 | from ...tools.typing import ProviderMaker 7 | 8 | class Provider(SolverPyObj): 9 | """A data provider that stores and/or queries results of tasks.""" 10 | 11 | def __init__( 12 | self, 13 | bid: str, 14 | sid: str, 15 | limit: (str | None) = None, 16 | caching: bool = False, 17 | ): 18 | self.bid = bid 19 | self.sid = sid 20 | self.limit = limit 21 | self.caching = caching 22 | self._uptodate = False 23 | "call store for cached results." 24 | 25 | @classmethod 26 | def Maker(cls, **kwargs) -> "ProviderMaker": 27 | 28 | class MakerMaker(SolverPyObj): 29 | 30 | def __init__(self): 31 | SolverPyObj.__init__( 32 | self, 33 | cls_name = cls.__name__, 34 | **kwargs, 35 | ) 36 | 37 | def __call__( 38 | self, 39 | bid: str, 40 | sid: str, 41 | limit: (str | None) = None, 42 | ): 43 | return cls(bid, sid, limit, **kwargs) 44 | 45 | return MakerMaker() 46 | 47 | def query( 48 | self, 49 | task: "SolverTask", 50 | ) -> (dict[str, Any] | None): 51 | """Return the cached result for `task` or None if not available.""" 52 | del task # unused argument 53 | return None 54 | 55 | def store( 56 | self, 57 | task: "SolverTask", 58 | result: dict[str, Any], 59 | ): 60 | """New result for `task` was found. Update the cache.""" 61 | del task, result # unused arguments 62 | pass 63 | 64 | def cached( 65 | self, 66 | task: "SolverTask", 67 | result: dict[str, Any], 68 | ): 69 | """Announcement that the cached `result` for `task` was found.""" 70 | if self.caching: 71 | self.store(task, result) 72 | 73 | def commit(self) -> None: 74 | """Save/flush the data.""" 75 | pass 76 | 77 | def check(self, task: "SolverTask") -> None: 78 | if (self.bid != task.bid) or (self.sid != task.sid): 79 | raise Exception("Error: Operation on invalid bid/sid in a provider.") 80 | 81 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/db/providers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbboyan/solverpy/caa8da555ff766a50e1b0b0a01f2747df65d8bdb/src/solverpy/benchmark/db/providers/__init__.py -------------------------------------------------------------------------------- /src/solverpy/benchmark/db/providers/jsons.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TextIO, TYPE_CHECKING 2 | import os, json 3 | import logging 4 | 5 | from ..cachedprovider import CachedProvider 6 | from ...path import bids, sids 7 | 8 | if TYPE_CHECKING: 9 | from ....task.solvertask import SolverTask 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | NAME = "results" 14 | 15 | 16 | class Jsons(CachedProvider): 17 | 18 | def __init__( 19 | self, 20 | bid: str, 21 | sid: str, 22 | limit: (str | None) = None, # TODO: make just `str` here? 23 | caching: bool = False, 24 | ): 25 | CachedProvider.__init__( 26 | self, 27 | bid, 28 | sid, 29 | limit, 30 | caching, 31 | compress=True, 32 | ) 33 | 34 | def query( 35 | self, 36 | task: "SolverTask", 37 | ) -> (dict[str, Any] | None): 38 | if task.problem in self.cache: 39 | result = self.cache[task.problem] 40 | return task.solver.determine(result) 41 | return None 42 | 43 | def store( 44 | self, 45 | task: "SolverTask", 46 | result: dict[str, Any], 47 | ) -> None: 48 | if task.solver.valid(result): 49 | self.cache[task.problem] = result 50 | self._uptodate = False 51 | 52 | def cachepath(self): 53 | return os.path.join( 54 | bids.dbpath(NAME), 55 | bids.name(self.bid, limit=self.limittype()), 56 | sids.name(self.sid), 57 | ).rstrip("/") + ".json" 58 | 59 | def cacheload(self, fr: TextIO) -> None: 60 | self.cache = json.load(fr) 61 | 62 | def cachedump(self, fw: TextIO) -> None: 63 | json.dump(self.cache, fw, indent=3, sort_keys=True) 64 | 65 | def limittype(self) -> str: 66 | if self.limit is None: 67 | return "unlimited" 68 | return "".join(sorted(x[0] for x in self.limit.split("-"))) 69 | 70 | 71 | # TODO: refactor and remove, use Maker instead 72 | class JsonsStore(Jsons): 73 | 74 | def __init__( 75 | self, 76 | bid: str, 77 | sid: str, 78 | limit: (str | None) = None, 79 | ): 80 | Jsons.__init__( 81 | self, 82 | bid, 83 | sid, 84 | limit, 85 | caching=True, 86 | ) 87 | 88 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/db/providers/loader.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | import os 3 | import gzip 4 | import logging 5 | 6 | from ..provider import Provider 7 | #from ...path import bids, sids 8 | from ....solver.plugins.db.outputs import Outputs 9 | 10 | if TYPE_CHECKING: 11 | from ....task.solvertask import SolverTask 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | class Loader(Provider): 17 | 18 | def __init__( 19 | self, 20 | bid: str, 21 | sid: str, 22 | limit: (str | None) = None, 23 | flatten: bool = True, 24 | ): 25 | Provider.__init__(self, bid, sid, limit, False) 26 | self.outputs = Outputs(flatten=flatten) 27 | 28 | def query(self, task: "SolverTask") -> (dict[str, Any] | None): 29 | self.outputs.solver = task.solver 30 | f = self.outputs.path(task.instance, task.strategy) 31 | #logger.debug(f"loading output for task {task} from {f}") 32 | if os.path.isfile(f + ".gz"): 33 | fr = gzip.open(f + ".gz", "rb") 34 | elif os.path.isfile(f): 35 | fr = open(f, "rb") 36 | else: 37 | return None 38 | output = fr.read().decode() 39 | fr.close() 40 | result = task.solver.process(output) 41 | task.solver.update(task.instance, task.strategy, output, result) 42 | #logger.debug(f"result = {result}") 43 | return result 44 | 45 | 46 | class SlashLoader(Loader): 47 | 48 | def __init__( 49 | self, 50 | bid: str, 51 | sid: str, 52 | limit: (str | None) = None, 53 | ): 54 | Loader.__init__( 55 | self, 56 | bid, 57 | sid, 58 | limit, 59 | flatten=False, 60 | ) 61 | 62 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/db/providers/solved.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TextIO, TYPE_CHECKING 2 | import os 3 | import logging 4 | 5 | from ..cachedprovider import CachedProvider 6 | from ...path import bids, sids 7 | 8 | if TYPE_CHECKING: 9 | from ....task.solvertask import SolverTask 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | NAME = "solved" 14 | 15 | 16 | class Solved(CachedProvider): 17 | 18 | def __init__( 19 | self, 20 | bid: str, 21 | sid: str, 22 | limit: str, 23 | caching: bool = True, 24 | delfix: (str | int | None) = None, 25 | ): 26 | CachedProvider.__init__( 27 | self, 28 | bid, 29 | sid, 30 | limit, 31 | caching, 32 | ) 33 | self._delfix = delfix 34 | 35 | def store( 36 | self, 37 | task: "SolverTask", 38 | result: dict[str, Any], 39 | ) -> None: 40 | if task.solver.solved(result): 41 | problem = delfix(task.problem, self._delfix) 42 | if problem not in self.cache: 43 | self.cache.add(problem) 44 | self._uptodate = False 45 | 46 | def load(self) -> None: 47 | super().load() 48 | if not self.cache: 49 | self.cache = set() 50 | 51 | def cachepath(self) -> str: 52 | return os.path.join( 53 | bids.dbpath(NAME), 54 | bids.name(self.bid, limit=self.limit), 55 | sids.name(self.sid), 56 | ).rstrip("/") 57 | 58 | def cacheload(self, fr: TextIO) -> None: 59 | self.cache = set(x for x in fr.read().strip().split("\n") if x) 60 | 61 | def cachedump(self, fw: TextIO) -> None: 62 | if self.cache: 63 | fw.write("\n".join(sorted(self.cache)) + "\n") 64 | 65 | 66 | def delfix( 67 | problem: str, 68 | fix: (str | int | None), 69 | ) -> str: 70 | """Delete a prefix of a problem name.""" 71 | if not fix: # covers None, 0, "" (also False) 72 | return problem 73 | if (type(fix) is str) and problem.startswith(fix): 74 | return problem[len(fix):] 75 | if (type(fix) is int) and problem.count("/") >= fix: 76 | parts = problem.split("/") 77 | return "/".join(parts[fix:]) 78 | logger.warning(f"Uknown delfix value type {type(fix)} of '{fix}'") 79 | return problem 80 | 81 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/db/providers/status.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TextIO, TYPE_CHECKING 2 | import os 3 | import logging 4 | 5 | from ..cachedprovider import CachedProvider 6 | from ...path import bids, sids 7 | from .solved import delfix 8 | 9 | if TYPE_CHECKING: 10 | from ....task.solvertask import SolverTask 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | NAME = "status" 15 | 16 | DELIM = "\t" 17 | 18 | 19 | class Status(CachedProvider): 20 | 21 | def __init__( 22 | self, 23 | bid: str, 24 | sid: str, 25 | limit: (str | None) = None, 26 | caching: bool = True, 27 | delfix: (str | int | None) = None, 28 | ): 29 | CachedProvider.__init__( 30 | self, 31 | bid, 32 | sid, 33 | limit, 34 | caching, 35 | ) 36 | self._delfix = delfix 37 | 38 | def store( 39 | self, 40 | task: "SolverTask", 41 | result: dict[str, Any], 42 | ) -> None: 43 | if task.solver.valid(result): 44 | problem = delfix(task.problem, self._delfix) 45 | val = (result["status"], f'{result["runtime"]:0.3f}') 46 | if (problem not in self.cache) or (self.cache[problem] != val): 47 | self.cache[problem] = val 48 | self._uptodate = False 49 | 50 | def cachepath(self) -> str: 51 | return os.path.join( 52 | bids.dbpath(NAME), 53 | bids.name(self.bid, limit=self.limit), 54 | sids.name(self.sid), 55 | ).rstrip("/") 56 | 57 | def cacheload(self, fr: TextIO) -> None: 58 | 59 | def entry(line): 60 | line = line.split(DELIM) 61 | return (line[0], DELIM.join(line[1:])) 62 | 63 | lines = fr.read().strip().split("\n") 64 | self.cache = dict(entry(l) for l in lines if l) 65 | 66 | def cachedump(self, fw: TextIO) -> None: 67 | 68 | def entry(problem): 69 | return DELIM.join(self.cache[problem]) 70 | 71 | if self.cache: 72 | fw.write("\n".join(f"{p}{DELIM}{entry(p)}" 73 | for p in sorted(self.cache)) + "\n") 74 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/path/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbboyan/solverpy/caa8da555ff766a50e1b0b0a01f2747df65d8bdb/src/solverpy/benchmark/path/__init__.py -------------------------------------------------------------------------------- /src/solverpy/benchmark/path/bids.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | DEFAULT_NAME = "." 4 | DEFAULT_DIR = os.getenv("SOLVERPY_BENCHMARKS", DEFAULT_NAME) 5 | 6 | DB_NAME = "solverpy_db" 7 | DB_DIR = os.getenv("SOLVERPY_DB", DB_NAME) 8 | 9 | 10 | def bidpath(bid: str) -> str: 11 | return os.path.join(DEFAULT_DIR, bid) 12 | 13 | 14 | def dbpath(subdir: str | None = None) -> str: 15 | # TODO: move this elsewhere 16 | return os.path.join(DB_DIR, subdir) if subdir else DB_DIR 17 | 18 | 19 | def path( 20 | bid: str, 21 | problem: str, 22 | flatten: bool | str = False, 23 | ) -> str: 24 | p_bid = bidpath(bid) 25 | if os.path.isfile(p_bid): 26 | p_bid = os.path.dirname(p_bid) 27 | if flatten and problem: 28 | problem = problem.replace("/", "_._" if flatten is True else flatten) 29 | return os.path.join(p_bid, problem).rstrip("/") 30 | 31 | 32 | def name(bid: str, limit: str | None = None) -> str: 33 | bid = bid.replace("/", "--") 34 | if limit: 35 | bid = f"{bid}--{limit}" 36 | return bid 37 | 38 | 39 | def problems( 40 | bid: str, 41 | cache: dict[str, list[str]] = {}, 42 | ) -> list[str]: 43 | if bid in cache: 44 | return cache[bid] 45 | p_bid = bidpath(bid) 46 | if os.path.isfile(p_bid): 47 | probs = open(p_bid).read().strip().split("\n") 48 | else: # now os.path.isdir(p_bid) holds 49 | probs = [x for x in os.listdir(p_bid) \ 50 | if os.path.isfile(os.path.join(p_bid,x))] 51 | cache[bid] = probs 52 | return probs 53 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/path/sids.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | 4 | from . import bids 5 | 6 | NAME = "strats" 7 | 8 | ARGUMENT = re.compile(r"@@@\s*([^@: ]*)\s*:\s*([^@: ]*)\s*@@@") 9 | 10 | def path(sid: str) -> str: 11 | f_sid = sid.split("@")[0] if ("@" in sid) else sid 12 | return os.path.join(bids.dbpath(NAME), f_sid) 13 | 14 | def load(sid: str) -> str: 15 | return open(path(sid)).read().strip() 16 | 17 | def save(sid: str, strategy: str) -> None: 18 | f_sid = path(sid) 19 | os.makedirs(os.path.dirname(f_sid), exist_ok=True) 20 | open(f_sid, "w").write(strategy.strip()) 21 | 22 | def name(sid: str) -> str: 23 | return sid.replace("/", "--") 24 | 25 | def unspace(strategy: str) -> str: 26 | return " ".join(x for x in strategy.split() if x) 27 | 28 | def split(sid: str) -> tuple[str, dict[str, str]]: 29 | args = {} 30 | if "@" in sid: 31 | (sid, args) = sid.split("@") 32 | args = args.split(":") 33 | args = [x.split("=") for x in args] 34 | args = {x.strip():y.strip() for (x,y) in args} 35 | return (sid, args) 36 | 37 | def instatiate(strategy: str, args: dict[str, str]) -> str: 38 | args0 = defaults(strategy) 39 | args0.update(args) 40 | ret = ARGUMENT.sub(lambda mo: args0[mo.group(1)], strategy) 41 | return ret 42 | 43 | def defaults(strategy: str) -> dict[str, str]: 44 | ret = ARGUMENT.findall(strategy) 45 | ret = {x.strip():y.strip() for (x,y) in ret} 46 | return ret 47 | 48 | def fmt(base: str, args: dict[str, str]) -> str: 49 | args0 = ":".join(f"{x}={args[x]}" for x in sorted(args)) 50 | return f"{base}@{args0}" 51 | 52 | def normalize(sid: str) -> str: 53 | strategy = load(sid) 54 | defs = defaults(strategy) 55 | (sid, args) = split(sid) 56 | args = {x:y for (x,y) in args.items() if y != defs[x]} 57 | return fmt(sid, args) 58 | 59 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/reports/data.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | if TYPE_CHECKING: 4 | from ...tools.typing import Result 5 | from ...solver.solverpy import SolverPy 6 | 7 | 8 | def par2score( 9 | solver: "SolverPy", 10 | result: "Result", 11 | ) -> float: 12 | if result and ("runtime" in result) and solver.solved(result): 13 | # NOTE: timeouts after solving when dumping proof happen 14 | # (then "runtime" might not be filled) 15 | return result["runtime"] 16 | else: 17 | return 2 * solver._limits.timeout 18 | 19 | 20 | def summary( 21 | solver: "SolverPy", 22 | bid: str, 23 | sid: str, 24 | results: dict[str, "Result"], 25 | refsolved: frozenset[str] | None = None, 26 | refpar2: float | None = None, 27 | ) -> tuple[int, str, int, int, int] | tuple[int, int, int, str, str, int, int, 28 | int]: 29 | del bid, sid 30 | solved = set() 31 | errors = 0 32 | unsolved = 0 33 | timeouts = 0 34 | par2 = 0 35 | for (problem, res) in results.items(): 36 | par2 += par2score(solver, res) 37 | if solver.solved(res): 38 | solved.add(problem) 39 | elif not solver.valid(res): 40 | errors += 1 41 | elif res["status"] in solver.timeouts: 42 | timeouts += 1 43 | else: 44 | unsolved += 1 45 | 46 | #errors = [p for (p,r) in results.items() if solver. 47 | if refsolved is None: 48 | par2 = f"{par2:0.2f}" 49 | return ( 50 | len(solved), 51 | par2, 52 | unsolved, 53 | timeouts, 54 | errors, 55 | ) 56 | else: 57 | assert refpar2 != None 58 | par2plus = (refpar2 - par2) / (refpar2 / 100.0) 59 | par2plus = f"{par2plus:+0.2f}%" 60 | par2 = f"{par2:0.2f}" 61 | plus = len(solved - refsolved) 62 | minus = len(refsolved - solved) 63 | return ( 64 | len(solved), 65 | plus, 66 | minus, 67 | par2, 68 | par2plus, 69 | unsolved, 70 | timeouts, 71 | errors, 72 | ) 73 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/reports/markdown.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, TYPE_CHECKING 2 | import yaml as pyyaml 3 | import logging 4 | 5 | from . import data 6 | 7 | if TYPE_CHECKING: 8 | from ...tools.typing import Report, SolverJob, Result 9 | from ...solver.solverpy import SolverPy 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | __all__ = ["newline", "text", "heading", "table"] 14 | 15 | 16 | def widths( 17 | rows: list[list[str]], 18 | header: list[str] | None = None, 19 | ) -> list[int]: 20 | rows = (rows + [header]) if header else rows 21 | ncols = len(rows[0]) 22 | width = [0] * ncols 23 | for i in range(ncols): 24 | width[i] = max(len(str(row[i])) for row in rows) 25 | return width 26 | 27 | 28 | def join( 29 | row: list[str], 30 | width: list[int], 31 | sep: str = "|", 32 | padding: str = " ", 33 | ) -> str: 34 | psep = f"{padding}{sep}{padding}" 35 | line = psep.join(f"{val:{width[i]}}" for (i, val) in enumerate(row)) 36 | return f"{sep}{padding}{line}{padding}{sep}" 37 | 38 | 39 | def newline() -> "Report": 40 | return [""] 41 | 42 | 43 | def heading(title: str, level: int = 1) -> "Report": 44 | level0 = "#" * level 45 | return [f"{level0} {title}", ""] 46 | 47 | 48 | def text(txt: str) -> "Report": 49 | return [txt] 50 | 51 | 52 | def yaml(obj: Any) -> "Report": 53 | if type(obj) is str: 54 | txt = obj 55 | else: 56 | txt = pyyaml.dump(obj, default_flow_style=False) 57 | lines = [] 58 | lines.append("```yaml") 59 | lines.extend(txt.strip().split("\n")) 60 | lines.append("```") 61 | return lines 62 | 63 | 64 | def table( 65 | header: list[str], 66 | rows: list[list[Any]], 67 | key: Callable[[list[Any]], Any] | None = None, 68 | ) -> "Report": 69 | logger.debug( 70 | f"making table with {len(rows)} rows and {len(rows[0])} columns") 71 | width = widths(rows, header=header) 72 | lines = [] 73 | if header: 74 | lines.append(join(header, width)) 75 | delims = ["-" * (w + 2) for w in width] 76 | lines.append(join(delims, width, padding="")) 77 | if key is not None: 78 | rows = sorted(rows, reverse=True, key=key) 79 | for row in rows: 80 | lines.append(join(row, width)) 81 | return lines 82 | 83 | 84 | def summary( 85 | results: dict["SolverJob", "Result"], 86 | nicks: dict["SolverJob", str], 87 | ref: "SolverJob | None" = None, 88 | ) -> "Report": 89 | logger.debug(f"creating summary for {len(results)} results") 90 | if ref is None: 91 | header = ["name", "solved", "PAR2", "unsolved", "timeouts", "errors"] 92 | refsolved = None 93 | refpar2 = None 94 | else: 95 | header = [ 96 | "name", "solved", "ref+", "ref-", "PAR2", "PAR2+", "unsolved", 97 | "timeouts", "errors" 98 | ] 99 | refsolved = frozenset(p for (p, r) in results[ref].items() 100 | if ref[0].solved(r)) 101 | refpar2 = sum(data.par2score(ref[0], r) for r in results[ref].values()) 102 | 103 | rows: list[list[Any]] = [] 104 | for ((solver, bid, sid), res) in results.items(): 105 | row: list[Any] = [nicks[(solver, bid, sid)]] 106 | row.extend(data.summary(solver, bid, sid, res, refsolved, refpar2)) 107 | rows.append(row) 108 | lines = table(header, rows, key=lambda x: x[1:]) 109 | logger.debug(f"summary created") 110 | return lines 111 | 112 | 113 | def statuses( 114 | results: dict["SolverJob", "Result"], 115 | nicks: dict["SolverJob", str], 116 | ) -> "Report": 117 | logger.debug(f"creating statuses for {len(results)} results") 118 | 119 | def safestat(r: "Result | None") -> str: 120 | if r is None: 121 | return "NONE" 122 | elif "status" in r: 123 | return r["status"] 124 | else: 125 | return "MISSING" 126 | 127 | def count(status: str, res: dict[str, "Result | None"]) -> int: 128 | return sum(1 for r in res.values() if safestat(r) == status) 129 | 130 | some: "SolverPy" = list(results.keys())[0][0] 131 | 132 | def rank(status: str) -> tuple[int, str]: 133 | if status in some.success: 134 | return (0, status) 135 | elif status in some.timeouts: 136 | return (2, status) 137 | else: 138 | return (1, status) 139 | 140 | allstats = frozenset( 141 | safestat(r) for res in results.values() for r in res.values()) 142 | allstats = sorted(allstats, key=rank) 143 | header = ["name"] + allstats 144 | rows = [] 145 | for ((solver, bid, sid), res) in results.items(): 146 | row = [nicks[(solver, bid, sid)]] 147 | row += [count(status, res) for status in allstats] 148 | rows.append(row) 149 | lines = table(header, rows, key=lambda x: x[1:]) 150 | logger.debug(f"statuses created") 151 | return lines 152 | 153 | 154 | def collect( 155 | report: "Report", 156 | acc: list[str] | None = None, 157 | ) -> list[str]: 158 | if acc is None: acc = [] 159 | for line in report: 160 | if isinstance(line, str): 161 | acc.append(line) 162 | else: 163 | assert (type(line) is list) or (type(line) is tuple) 164 | collect(line, acc) 165 | return acc 166 | 167 | 168 | def dump(report: "Report", prefix: str = "") -> str: 169 | if not report: 170 | return "" 171 | return f"{prefix}" + f"\n{prefix}".join(collect(report)) 172 | 173 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/reports/progress.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, TYPE_CHECKING 2 | import logging 3 | 4 | from . import markdown 5 | from .markdown import * 6 | from ...tools import human 7 | 8 | if TYPE_CHECKING: 9 | from ...tools.typing import Report 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | #def reporter(fun: Callable[..., "Report"]) -> Callable[..., "Report"]: 14 | # def wrapper(*args, dump=True, prefix="> ", **kwargs) -> "Report": 15 | # rep = fun(*args, **kwargs) 16 | # return markdown.dump(rep, prefix=prefix) if dump else rep 17 | 18 | def reporter(fun: Callable[..., "Report"]) -> Callable[..., str]: 19 | def wrapper(*args, prefix="> ", **kwargs) -> str: 20 | rep = fun(*args, **kwargs) 21 | return markdown.dump(rep, prefix=prefix) 22 | 23 | return wrapper 24 | 25 | 26 | def typename(obj: object, quote: str = ""): 27 | t = type(obj) 28 | return f"{quote}{t.__module__}.{t.__name__}{quote}" 29 | 30 | 31 | @reporter 32 | def compress( 33 | f_in: str, 34 | size_in: int, 35 | size_z: int, 36 | data: object, 37 | label: object, 38 | ) -> "Report": 39 | return [ 40 | newline(), 41 | heading("Data compression report", level=4), 42 | text(f"* data file: `{f_in}`"), 43 | newline(), 44 | table(["", ""], 45 | [["uncompressed", human.humanbytes(size_in)], 46 | ["compressed", human.humanbytes(size_z)], 47 | ["ratio", f"{size_in/size_z:.2f} x"], 48 | ["data-type", typename(data, quote="`")], 49 | ["label-type ", typename(label, quote="`")]]), 50 | newline(), 51 | ] 52 | 53 | 54 | @reporter 55 | def build( 56 | dataname: str, 57 | score: float, 58 | acc: float, 59 | trainacc: float, 60 | f_best: str, 61 | dur: float, 62 | params: dict[str, int | float | str], 63 | pos: int, 64 | neg: int, 65 | ) -> "Report": 66 | 67 | def valfmt(y: int | float | str) -> str: 68 | return f"{y:g}" if type(y) is float else str(y) 69 | 70 | return [ 71 | newline(), 72 | heading(f"Model `{dataname}`", level=3), 73 | heading("Best model statistics", level=4), 74 | text(f"* best model: `{f_best}`"), 75 | newline(), 76 | table(["", ""], [ 77 | ["trains", f"{pos+neg} ({pos} / {neg})"], 78 | ["acc / train", human.humanacc(trainacc)], 79 | ["acc / devel", human.humanacc(acc)], 80 | ["duration", human.humantime(dur)], 81 | ["score", f"{score:0.3f}"], 82 | ]), 83 | newline(), 84 | heading("Best model training parameters", level=4), 85 | table(["param", "value"], [[x, valfmt(y)] for (x, y) in params.items()]), 86 | newline(), 87 | ] 88 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/setups/__init__.py: -------------------------------------------------------------------------------- 1 | from .setup import Setup 2 | from .solver import eprover, vampire, prover9, cvc5, bitwuzla, z3 3 | from .loop import evaluation, launch 4 | from .tuner import cvc5ml, enigma 5 | 6 | __all__ = [ 7 | "Setup", 8 | "eprover", 9 | "vampire", 10 | "cvc5", 11 | "z3", 12 | "prover9", 13 | "bitwuzla", 14 | "evaluation", 15 | "launch", 16 | "cvc5ml", 17 | "enigma", 18 | ] 19 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/setups/common.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | import logging 3 | 4 | from ...solver import plugins 5 | from .setup import Setup 6 | 7 | if TYPE_CHECKING: 8 | from ...tools.typing import SolverMaker 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | # generic arguments accepted by solver constructors 13 | GENERICS = frozenset(["binary", "plugins", "static", "complete"]) 14 | 15 | 16 | def default(setup: Setup, key: str, val: Any) -> None: 17 | if key not in setup: 18 | setup[key] = val 19 | 20 | 21 | def ensure(options: list[str], option: str) -> None: 22 | baseopt = option if not option.startswith("no-") else option[3:] 23 | if (baseopt not in options) and (f"no-{baseopt}" not in options): 24 | options.append(option) 25 | 26 | 27 | def init(setup: Setup) -> Setup: 28 | default(setup, "options", ["flatten", "compress"]) 29 | assert "options" in setup 30 | options = setup["options"] 31 | ensure(options, "flatten") 32 | ensure(options, "compress") 33 | default(setup, "limit", "T1") 34 | default(setup, "dataname", "noname") 35 | 36 | if "plugins" not in setup: 37 | if "outputs" not in options: 38 | plugs = plugins.db() 39 | else: 40 | plugs = plugins.outputs( 41 | flatten="flatten" in options, 42 | compress="compress" in options, 43 | ) 44 | default(setup, "plugins", plugs) 45 | return setup 46 | 47 | 48 | def solver(setup: Setup, mk_solver: "SolverMaker") -> Setup: 49 | assert "static" in setup 50 | assert "limit" in setup 51 | kwargs = {x: setup[x] for x in setup if x in GENERICS} 52 | kwargs["static"] = " ".join(setup["static"]) 53 | _solver = mk_solver(setup["limit"], **kwargs) 54 | setup["solver"] = _solver 55 | return setup 56 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/setups/loop.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from .. import launcher, db 4 | from ...tools import log 5 | from .common import default 6 | from ...builder.builder import Builder 7 | from .setup import Setup 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | def loopinit(setup: Setup) -> Setup: 13 | assert "basedataname" in setup 14 | assert "trains" in setup 15 | base = setup["basedataname"] 16 | if "it" not in setup: 17 | setup["it"] = 0 18 | filename = "train.in" 19 | else: 20 | setup["it"] += 1 21 | setup["previous_trains"] = setup["trains"].path() 22 | filename = "addon.in" 23 | it = setup["it"] 24 | setup["dataname"] = f"{base}/loop{it:02d}" 25 | setup["trains"].reset(setup["dataname"], filename) 26 | if "builder" in setup: 27 | builder: Builder = setup["builder"] 28 | builder.reset(setup["dataname"]) 29 | return setup 30 | 31 | 32 | def looping(setup: Setup) -> Setup: 33 | assert "dataname" in setup 34 | setup["basedataname"] = setup["dataname"] 35 | loopinit(setup) 36 | return setup 37 | 38 | 39 | def evaluation(setup: Setup) -> Setup: 40 | default(setup, "cores", 4) 41 | default(setup, "ref", True) 42 | default(setup, "bidfile", "bids") 43 | assert "bidfile" in setup 44 | default(setup, "sidfile", "sids") 45 | assert "sidfile" in setup 46 | default(setup, "delfix", None) 47 | assert "delfix" in setup 48 | default(setup, "db", db.default(delfix=setup["delfix"])) 49 | default(setup, "ntfy", None) 50 | if "sidlist" not in setup: 51 | with open(setup["sidfile"]) as f: 52 | setup["sidlist"] = f.read().strip().split("\n") 53 | if "bidlist" not in setup: 54 | with open(setup["bidfile"]) as f: 55 | setup["bidlist"] = f.read().strip().split("\n") 56 | if "loops" in setup: 57 | looping(setup) 58 | return setup 59 | 60 | 61 | def oneloop(setup: Setup) -> Setup: 62 | 63 | def is_last(setup): 64 | return ("loops" in setup) and (setup["it"] == setup["loops"]) 65 | 66 | def trains_compress(setup: Setup): 67 | assert "options" in setup 68 | options = setup["options"] 69 | if ("trains" in setup) and ("compress" in options) and \ 70 | ("no-compress-trains" not in options): 71 | setup["trains"].compress() 72 | 73 | def trains_merge(setup): 74 | if ("previous_trains" in setup) and not is_last(setup): 75 | setup["trains"].merge(setup["previous_trains"], "train.in") 76 | #f_out = setup["trains"].path(filename="train.in") 77 | #svm.merge(setup["previous_trains"], setup["trains"].path(), f_out=f_out) 78 | setup["trains"].reset(filename="train.in") 79 | 80 | def model_build(setup): 81 | if "builder" not in setup: 82 | return 83 | builder = setup["builder"] 84 | if builder and not is_last(setup): 85 | builder.build() 86 | setup["news"] = builder.strategies 87 | logger.info("New ML strategies:\n" + "\n".join(setup["news"])) 88 | 89 | assert "dataname" in setup 90 | it = setup['it'] if 'it' in setup else 0 91 | logger.info( 92 | f"Running evaluation loop {it} on data {setup['dataname']}.\n> \n> ## Evaluation `{setup['dataname']}` ##\n> " 93 | ) 94 | if (it > 0) or ("start_dataname" not in setup): 95 | launcher.launch(**setup) 96 | trains_compress(setup) 97 | trains_merge(setup) 98 | elif "trains" in setup: 99 | logger.info( 100 | f"Evaluation skipped. Starting with data {setup['start_dataname']}") 101 | setup["trains"].reset(setup["start_dataname"]) 102 | model_build(setup) 103 | return setup 104 | 105 | 106 | def launch(setup: Setup, devels: Setup | None = None) -> Setup: 107 | 108 | def do_loop(col): 109 | if not col: return 110 | oneloop(col) 111 | 112 | def do_iter(col): 113 | if not col: return 114 | #col["sidlist"] = list(setup["refs"]) if "refs" in setup else [] 115 | col["sidlist"].extend(setup["news"] if "news" in setup else []) 116 | loopinit(col) 117 | oneloop(col) 118 | 119 | log.ntfy(setup, "solverpy: init") 120 | launcher.init(setup) 121 | do_loop(devels) 122 | do_loop(setup) 123 | if "loops" in setup: 124 | assert "it" in setup 125 | while setup["it"] < setup["loops"]: 126 | log.ntfy(setup, f"solverpy: iter #{setup['it']}") 127 | do_iter(devels) 128 | if devels and (setup['it'] + 1 == setup["loops"]): 129 | # skip evaluation on trains in the last loop if devel is used 130 | break 131 | do_iter(setup) 132 | log.ntfy(setup, "solverpy: done") 133 | return setup 134 | 135 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/setups/setup.py: -------------------------------------------------------------------------------- 1 | from typing import TypedDict, TYPE_CHECKING 2 | 3 | if TYPE_CHECKING: 4 | from ..db import DB 5 | from ...builder.builder import Builder 6 | from ...solver.solverpy import SolverPy 7 | from ...solver.plugins.plugin import Plugin 8 | from ...builder.plugins.svm import SvmTrains 9 | 10 | 11 | class Setup(TypedDict, total=False): 12 | 13 | limit: str 14 | cores: int 15 | ref: (bool | int | str | None) 16 | bidfile: str 17 | sidfile: str 18 | bidlist: list[str] 19 | sidlist: list[str] 20 | binary: str 21 | static: list[str] 22 | ntfy: str 23 | it: int 24 | loops: int 25 | news: list[str] 26 | refs: list[str] 27 | options: list[str] 28 | delfix: (int | str | None) 29 | 30 | 31 | force: bool 32 | shuffle: bool 33 | 34 | dataname: str 35 | start_dataname: str 36 | basedataname: str 37 | db: "DB" 38 | builder: "Builder" 39 | solver: "SolverPy" 40 | trains: "SvmTrains" 41 | previous_trains: str 42 | plugins: list["Plugin"] 43 | 44 | e_training_examples: str 45 | gen_features: str 46 | sel_features: str 47 | 48 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/setups/solver.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | import logging 3 | 4 | from ...solver.atp.eprover import E_STATIC, E 5 | from ...solver.atp.prover9 import Prover9 6 | from ...solver.atp.vampire import Vampire 7 | from ...builder.plugins import * 8 | from ...solver.smt import Cvc5, Z3, Bitwuzla 9 | from ...solver.smt.cvc5 import CVC5_STATIC 10 | from ...solver.smt.z3 import Z3_STATIC 11 | from .common import default, init, solver 12 | from .setup import Setup 13 | 14 | if TYPE_CHECKING: 15 | from ...builder.plugins.svm import SvmTrains 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | def eprover(setup: Setup, training: bool = False) -> Setup: 21 | init(setup) 22 | assert "plugins" in setup 23 | assert "options" in setup 24 | static = E_STATIC.split() 25 | if training: 26 | default(setup, "dataname", "data/model") 27 | assert "dataname" in setup 28 | default(setup, "e_training_examples", "11") 29 | assert "e_training_examples" in setup 30 | default(setup, "sel_features", None) 31 | assert "sel_features" in setup 32 | default(setup, "gen_features", None) 33 | assert "gen_features" in setup 34 | sel = setup["sel_features"] 35 | gen = setup["gen_features"] 36 | dataname = setup["dataname"] 37 | static.append(f"--training-examples={setup['e_training_examples']}") 38 | trains: "SvmTrains" 39 | if sel: 40 | static.append(f'--enigmatic-sel-features="{sel}"') 41 | if gen: 42 | static.append(f'--enigmatic-gen-features="{gen}"') 43 | if sel and gen: 44 | trains = EnigmaMultiTrains(dataname, sel, gen) 45 | elif sel: 46 | trains = EnigmaTrains(dataname, sel, "sel") 47 | elif gen: 48 | trains = EnigmaTrains(dataname, gen, "gen") 49 | else: 50 | raise ValueError( 51 | "`sel_features` or `gen_features` must be provided in setup to generate trains." 52 | ) 53 | setup["trains"] = trains 54 | plugs = setup["plugins"] 55 | plugs.append(trains) 56 | flatten = "flatten" in setup["options"] 57 | if "debug-trains" in setup["options"]: 58 | if sel: 59 | plugs.append(EnigmaTrainsDebug(sel, "sel", flatten)) 60 | if gen: 61 | plugs.append(EnigmaTrainsDebug(gen, "gen", flatten)) 62 | default(setup, "static", static) 63 | return solver(setup, E) 64 | 65 | ####default(setup, "sel_features", "C(l,v,h,s,c,d,a)") 66 | ###setup["static"] += " ".join(["", # make a space 67 | ### f"--training-examples={setup['e_training_examples']}", 68 | ### f"--enigmatic-sel-features=\"{setup['sel_features']}\"", 69 | ###]) 70 | ###default(setup, "dataname", "data/model") 71 | ###trains = EnigmaTrains(setup["dataname"], setup["sel_features"]) 72 | ###plugs = setup["plugins"] 73 | ###plugs.append(trains) 74 | ###if "debug-trains" in setup["options"]: 75 | ### plugs.append(EnigmaTrainsDebug( 76 | ### setup["sel_features"], "flatten" in setup["options"])) 77 | ###setup["trains"] = trains 78 | 79 | 80 | def vampire(setup: Setup) -> Setup: 81 | init(setup) 82 | return solver(setup, Vampire) 83 | 84 | 85 | def prover9(setup: Setup) -> Setup: 86 | init(setup) 87 | return solver(setup, Prover9) 88 | 89 | 90 | def bitwuzla(setup: Setup) -> Setup: 91 | init(setup) 92 | return solver(setup, Bitwuzla) 93 | 94 | 95 | def z3(setup: Setup) -> Setup: 96 | default(setup, "static", "") 97 | init(setup) 98 | return solver(setup, Z3) 99 | 100 | 101 | def cvc5(setup: Setup, training: bool = False) -> Setup: 102 | init(setup) 103 | assert "plugins" in setup 104 | assert "options" in setup 105 | static = CVC5_STATIC.split() 106 | if training: 107 | static.extend([ 108 | "--produce-proofs", 109 | "--produce-models", 110 | "--dump-instantiations", 111 | "--print-inst-full", 112 | "--ml-engine", 113 | ]) 114 | default(setup, "dataname", "data/model") 115 | assert "dataname" in setup 116 | trains = Cvc5Trains(setup["dataname"]) 117 | plugs = setup["plugins"] 118 | plugs.append(trains) 119 | options = setup["options"] 120 | if "debug-trains" in options: 121 | plugs.append(Cvc5TrainsDebug("flatten" in options)) 122 | setup["trains"] = trains 123 | default(setup, "static", static) 124 | return solver(setup, Cvc5) 125 | -------------------------------------------------------------------------------- /src/solverpy/benchmark/setups/tuner.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, TYPE_CHECKING 2 | import logging 3 | 4 | from ...builder.cvc5ml import Cvc5ML 5 | from ...builder.enigma import Enigma 6 | from .setup import Setup 7 | 8 | if TYPE_CHECKING: 9 | from ...builder.builder import Builder 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | BuilderMaker = Callable[ 14 | [Setup, Setup | None, dict[str, Any] | None], 15 | "Builder", 16 | ] 17 | 18 | 19 | def autotuner( 20 | trains: Setup, 21 | devels: (Setup | None), 22 | tuneargs: (dict[str, Any] | None), 23 | mk_builder: BuilderMaker, 24 | ) -> Setup: 25 | if "refs" not in trains: 26 | assert "ref" in trains 27 | assert "sidlist" in trains 28 | ref = trains["ref"] 29 | idx = ref if (type(ref) is int) else 0 30 | trains["refs"] = [trains["sidlist"][idx]] 31 | trains["builder"] = mk_builder(trains, devels, tuneargs) 32 | return trains 33 | 34 | 35 | def cvc5ml( 36 | trains: Setup, 37 | devels: (Setup | None) = None, 38 | tuneargs: (dict[str, Any] | None) = None, 39 | ) -> Setup: 40 | return autotuner(trains, devels, tuneargs, Cvc5ML) 41 | 42 | 43 | def enigma( 44 | trains: Setup, 45 | devels: (Setup | None) = None, 46 | tuneargs: (dict[str, Any] | None) = None, 47 | ) -> Setup: 48 | return autotuner(trains, devels, tuneargs, Enigma) 49 | -------------------------------------------------------------------------------- /src/solverpy/builder/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbboyan/solverpy/caa8da555ff766a50e1b0b0a01f2747df65d8bdb/src/solverpy/builder/__init__.py -------------------------------------------------------------------------------- /src/solverpy/builder/autotune/__init__.py: -------------------------------------------------------------------------------- 1 | from .autotune import tuner, prettytuner 2 | 3 | __all__ = ["tuner", "prettytuner"] 4 | 5 | -------------------------------------------------------------------------------- /src/solverpy/builder/autotune/autotune.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, TYPE_CHECKING 2 | import os 3 | import time 4 | import logging 5 | import lightgbm as lgb 6 | import multiprocessing 7 | 8 | from ...tools import human, redirect 9 | from ...builder import svm 10 | from . import tune, build 11 | from .listener import AutotuneListener 12 | 13 | if TYPE_CHECKING: 14 | from queue import Queue 15 | from .tune import TuneResult 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | PHASES: dict[str, Callable[..., "TuneResult"]] = { 20 | "l": tune.leaves_grid, 21 | "b": tune.bagging, 22 | "r": tune.regular, 23 | "m": tune.min_data, 24 | "d": tune.depth, 25 | } 26 | 27 | DEFAULTS: dict[str, Any] = { 28 | 'learning_rate': 0.15, 29 | 'objective': 'binary', 30 | 'num_round': 200, 31 | 'max_depth': 0, 32 | 'num_leaves': 300, 33 | # default values from the docs: 34 | 'min_data': 20, 35 | 'max_bin': 255, 36 | 'feature_fraction': 1.0, 37 | 'bagging_fraction': 1.0, 38 | 'bagging_freq': 0, 39 | 'lambda_l1': 0.0, 40 | 'lambda_l2': 0.0, 41 | } 42 | 43 | 44 | def tuner( 45 | f_train: str, 46 | f_test: str, 47 | d_tmp: str = "optuna-tmp", 48 | phases: str = "l:b:m:r", 49 | iters: int = 100, 50 | timeout: (int | None) = None, 51 | init_params: (dict[str, Any] | None) = None, 52 | min_leaves: int = 16, 53 | max_leaves: int = 2048, 54 | queue: "Queue[Any] | None" = None, 55 | ) -> tuple[Any, ...] | None: 56 | if queue: queue.put(("tuning", time.time())) 57 | (xs, ys) = svm.load(f_train) 58 | dtrain = lgb.Dataset(xs, label=ys, free_raw_data=False) 59 | dtrain.construct() 60 | (xs0, ys0) = svm.load(f_test) if f_test != f_train else (xs, ys) 61 | dtest = lgb.Dataset(xs0, label=ys0, free_raw_data=False) 62 | dtest.construct() 63 | 64 | os.makedirs(d_tmp, exist_ok=True) 65 | 66 | params = dict(DEFAULTS) 67 | if init_params: params.update(init_params) 68 | pos = sum(ys) 69 | neg = len(ys) - pos 70 | #params["scale_pos_weight"] = neg / pos 71 | params["is_unbalance"] = "true" if neg != pos else "false" 72 | phases0 = phases.split(":") 73 | if "m" in phases: 74 | params["feature_pre_filter"] = "false" 75 | timeout0 = timeout / len(phases0) if timeout else None 76 | iters0 = iters // len(phases0) if iters else None 77 | args = dict( 78 | dtrain=dtrain, 79 | dtest=dtest, 80 | d_tmp=d_tmp, 81 | iters=iters0, 82 | timeout=timeout0, 83 | queue=queue, 84 | min_leaves=min_leaves, 85 | max_leaves=max_leaves, 86 | ) 87 | 88 | if init_params is not None: 89 | f_mod = os.path.join(d_tmp, "init", "model.lgb") 90 | #(score, acc, trainacc, dur) = build.model(params, dtrain, dtest, f_mod, queue) 91 | (_, stats) = build.model(params, dtrain, dtest, f_mod, queue) 92 | acc = stats["valid_acc"] 93 | best = (stats["score"], acc, stats["train_acc"], f_mod, 94 | stats["duration"]) 95 | logger.debug("- initial model: %s" % human.humanacc(acc)) 96 | else: 97 | best = (-1, None, None, None, None) 98 | 99 | for phase in phases0: 100 | (best0, params0) = PHASES[phase](params=params, **args) 101 | if best0[0] > best[0]: 102 | best = best0 103 | params.update(params0) 104 | 105 | if queue: queue.put(("tuned", time.time())) 106 | ret = best + (params, pos, neg) 107 | if queue: 108 | queue.put(("result", (ret, ))) 109 | else: 110 | return ret 111 | 112 | 113 | def prettytuner(*args, **kwargs) -> Any: 114 | 115 | listener = AutotuneListener() 116 | 117 | d_tmp = kwargs["d_tmp"] 118 | os.makedirs(d_tmp, exist_ok=True) 119 | queue = multiprocessing.Queue() 120 | kwargs["queue"] = queue 121 | kwargs["f_log"] = os.path.join(d_tmp, "autotune.log") 122 | kwargs["target"] = tuner 123 | p = multiprocessing.Process(target=redirect.call, args=args, kwargs=kwargs) 124 | 125 | try: 126 | p.start() 127 | while True: 128 | msg = queue.get() 129 | result = listener.listen(msg) 130 | if result: 131 | break 132 | except (Exception, KeyboardInterrupt) as e: 133 | p.terminate() 134 | raise e 135 | finally: 136 | p.join() 137 | 138 | return result 139 | -------------------------------------------------------------------------------- /src/solverpy/builder/autotune/build.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | import os 3 | import time 4 | import lightgbm as lgb 5 | from numpy import ndarray 6 | from scipy.sparse import csr_matrix 7 | 8 | if TYPE_CHECKING: 9 | from queue import Queue 10 | from lightgbm import Booster, Dataset 11 | Talker = Queue[tuple[str, tuple[Any, ...]]] 12 | 13 | POS_ACC_WEIGHT = 2.0 14 | 15 | 16 | def accuracy( 17 | bst: "Booster", 18 | xs: "csr_matrix", 19 | ys: "ndarray", 20 | ) -> tuple[float, float, float]: 21 | 22 | def getacc(pairs: list[tuple[float, float]]) -> float: 23 | if not pairs: return 0 24 | return sum([1 for (x, y) in pairs if int(x > 0.5) == y]) / len(pairs) 25 | 26 | if hasattr(bst, "best_iteration"): 27 | preds = bst.predict(xs, num_iteration=bst.best_iteration) 28 | else: 29 | preds = bst.predict(xs) 30 | 31 | assert type(preds) is ndarray 32 | preds0 = list(zip(preds, ys)) 33 | acc = getacc(preds0) 34 | posacc = getacc([(x, y) for (x, y) in preds0 if y == 1]) 35 | negacc = getacc([(x, y) for (x, y) in preds0 if y == 0]) 36 | return (acc, posacc, negacc) 37 | 38 | 39 | def model( 40 | params: dict[str, Any], 41 | dtrain: "Dataset", 42 | dtest: "Dataset", 43 | f_mod: str, 44 | queue: "Talker | None" = None, 45 | ) -> tuple["Booster", dict[str, Any]]: 46 | callbacks = bst = begin = end = score = acc = trainacc = None 47 | 48 | def report(key: str, *content: Any) -> None: 49 | if queue: 50 | queue.put((key, content)) 51 | 52 | def queue_callback(env: Any) -> None: 53 | results = env.evaluation_result_list 54 | report("debug", str(results)) 55 | loss = [r[2] for r in results] 56 | report("iteration", env.iteration, env.end_iteration, loss) 57 | 58 | def setup_dirs() -> None: 59 | d_mod = os.path.dirname(f_mod) 60 | os.makedirs(d_mod, exist_ok=True) 61 | # f_log = f_mod + ".log" 62 | 63 | def setup_callbacks() -> None: 64 | nonlocal callbacks, params 65 | callbacks = [] 66 | callbacks.append(lgb.log_evaluation(1)) 67 | if "early_stopping" in params: 68 | # rounds = params["early_stopping"] 69 | params = dict(params) 70 | rounds = params.pop("early_stopping") # this also removes it 71 | # rounds can be `bool` or `int` (or int-convertable) 72 | rounds = 10 if (rounds is True) else int( 73 | rounds) # True => 10; False => 0 74 | if rounds: 75 | report("debug", 76 | f"activating early stopping: stopping_rounds={rounds}") 77 | callbacks.append( 78 | lgb.early_stopping(rounds, first_metric_only=True, 79 | verbose=True)) 80 | if queue: 81 | callbacks.append(queue_callback) 82 | 83 | def build_model() -> "Booster": 84 | nonlocal bst, begin, end, params, callbacks 85 | # build the model 86 | report("building", f_mod, params["num_round"]) 87 | begin = time.time() 88 | bst = lgb.train( 89 | params, 90 | dtrain, 91 | valid_sets=[dtrain, dtest], 92 | valid_names=["train", "valid"], 93 | # valid_sets=[dtest], 94 | callbacks=callbacks) 95 | end = time.time() 96 | if hasattr(bst, "best_iteration"): 97 | report("debug", 98 | f"early stopping: best_iteration={bst.best_iteration}") 99 | bst.save_model(f_mod) 100 | return bst 101 | 102 | def check_model() -> None: 103 | nonlocal score, acc, trainacc 104 | assert bst 105 | # compute the accuracy on the testing data 106 | (axs, ays) = (dtest.get_data(), dtest.get_label()) 107 | assert type(axs) is csr_matrix 108 | assert type(ays) is ndarray 109 | acc = accuracy(bst, axs, ays) 110 | (taxs, tays) = (dtrain.get_data(), dtrain.get_label()) 111 | assert type(taxs) is csr_matrix 112 | assert type(tays) is ndarray 113 | trainacc = accuracy(bst, taxs, tays) 114 | bst.free_dataset() 115 | bst.free_network() 116 | # compute the score of this model 117 | score = POS_ACC_WEIGHT * acc[1] + acc[2] 118 | report("built", score) 119 | 120 | setup_dirs() 121 | setup_callbacks() 122 | bst = build_model() # make typing happy 123 | check_model() 124 | 125 | assert begin and end 126 | stats = dict( 127 | score=score, 128 | valid_acc=acc, 129 | train_acc=trainacc, 130 | duration=end - begin, 131 | ) 132 | 133 | #return (score, acc, trainacc, end-begin) 134 | return (bst, stats) 135 | -------------------------------------------------------------------------------- /src/solverpy/builder/autotune/check.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | import os 3 | 4 | from . import build 5 | 6 | if TYPE_CHECKING: 7 | from optuna import Trial 8 | from lightgbm import Dataset 9 | from .build import Talker 10 | 11 | 12 | def check( 13 | trial: "Trial", 14 | params: dict[str, Any], 15 | dtrain: "Dataset", 16 | dtest: "Dataset", 17 | d_tmp: str, 18 | queue: "Talker | None" = None, 19 | **args: Any, 20 | ) -> float: 21 | del args # unused argument 22 | f_mod = os.path.join(d_tmp, "model%04d" % trial.number, "model.lgb") 23 | #(score, acc, trainacc, dur) = build.model(params, dtrain, dtest, f_mod, queue) 24 | (_, stats) = build.model(params, dtrain, dtest, f_mod, queue) 25 | 26 | trial.set_user_attr(key="model", value=f_mod) 27 | trial.set_user_attr(key="score", value=stats["score"]) 28 | trial.set_user_attr(key="acc", value=stats["valid_acc"]) 29 | trial.set_user_attr(key="trainacc", value=stats["train_acc"]) 30 | trial.set_user_attr(key="time", value=stats["duration"]) 31 | #if queue: queue.put(("tried", (score, acc, trainacc, dur))) 32 | if queue: queue.put(("tried", (stats, ))) 33 | return stats["score"] 34 | 35 | 36 | def leaves( 37 | trial: "Trial", 38 | params: dict[str, Any], 39 | min_leaves: int, 40 | max_leaves: int, 41 | queue: "Talker | None", 42 | **args: Any, 43 | ) -> float: 44 | args = dict(args, queue=queue) 45 | #num_leaves_base = trial.suggest_int('num_leaves_base', 16, 31) 46 | #num_leaves = round(2**(num_leaves_base/2)) 47 | num_leaves = trial.suggest_int('num_leaves', min_leaves, max_leaves) 48 | if queue: queue.put(("trying", ("leaves", trial.number, (num_leaves, )))) 49 | params = dict(params, num_leaves=num_leaves) 50 | score = check(trial, params, **args) 51 | #acc = human.humanacc(trial.user_attrs["acc"]) 52 | return score 53 | 54 | 55 | def bagging( 56 | trial: "Trial", 57 | params: dict[str, Any], 58 | queue: "Talker | None", 59 | **args: Any, 60 | ) -> float: 61 | bagging_freq = trial.suggest_int("bagging_freq", 1, 7) 62 | bagging_fraction = min( 63 | trial.suggest_float("bagging_fraction", 0.4, 1.0 + 1e-12), 1.0) 64 | if queue: 65 | queue.put(("trying", ("bagging", trial.number, (bagging_freq, 66 | bagging_fraction)))) 67 | params = dict(params, 68 | bagging_freq=bagging_freq, 69 | bagging_fraction=bagging_fraction) 70 | score = check(trial, params, queue=queue, **args) 71 | return score 72 | 73 | 74 | def min_data( 75 | trial: "Trial", 76 | params: dict[str, Any], 77 | queue: "Talker | None", 78 | **args: Any, 79 | ) -> float: 80 | min_data = trial.suggest_int("min_data", 5, 10000) 81 | if queue: queue.put(("trying", ("min_data", trial.number, (min_data, )))) 82 | params = dict(params, min_data=min_data) 83 | score = check(trial, params, queue=queue, **args) 84 | return score 85 | 86 | 87 | def regular( 88 | trial: "Trial", 89 | params: dict[str, Any], 90 | queue: "Talker | None", 91 | **args: Any, 92 | ) -> float: 93 | lambda_l1 = trial.suggest_float("lambda_l1", 1e-8, 10.0) 94 | lambda_l2 = trial.suggest_float("lambda_l2", 1e-8, 10.0) 95 | if queue: 96 | queue.put(("trying", ("regular", trial.number, (lambda_l1, lambda_l2)))) 97 | params = dict(params, lambda_l1=lambda_l1, lambda_l2=lambda_l2) 98 | score = check(trial, params, queue=queue, **args) 99 | return score 100 | 101 | 102 | def depth( 103 | trial: "Trial", 104 | params: dict[str, Any], 105 | queue: "Talker | None", 106 | **args: Any, 107 | ) -> float: 108 | max_depth = trial.suggest_int("max_depth", 3, 50) 109 | if queue: queue.put(("trying", ("depth", trial.number, (max_depth, )))) 110 | params = dict(params, max_depth=max_depth) 111 | score = check(trial, params, queue=queue, **args) 112 | return score 113 | -------------------------------------------------------------------------------- /src/solverpy/builder/autotune/listener.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TypeVar, TYPE_CHECKING 2 | import logging 3 | 4 | from ...tools import human 5 | from ...task.bar import BuilderBar 6 | from ...benchmark.reports import markdown 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | if TYPE_CHECKING: 11 | T = TypeVar("T") 12 | 13 | 14 | class Listener: 15 | 16 | def listen(self, message) -> Any: 17 | logger.debug(f"listening: {message}") 18 | try: 19 | (key, content) = message 20 | except (TypeError, ValueError): 21 | logger.exception(f"Incorect message: '{message}'") 22 | return None 23 | if not (isinstance(content, tuple) or isinstance(content, list)): 24 | content = (content, ) 25 | try: 26 | handler = getattr(self, key) 27 | except AttributeError: 28 | logger.warning( 29 | f"Unknown listener message: {key} (content: {content})") 30 | return None 31 | return handler(*content) 32 | 33 | 34 | class AutotuneListener(Listener): 35 | 36 | def __init__(self): 37 | self.bar = None 38 | self.desc = "trial" 39 | self.iters = "" 40 | self.t_start = 0 41 | self.t_end = 0 42 | self.f_mod = None 43 | self.nick = None 44 | self.it = None 45 | self.values = None 46 | self.table = None 47 | self.header = None 48 | 49 | def result(self, val: "T") -> "T": 50 | return val 51 | 52 | def building(self, f_mod: str, total: int) -> None: 53 | logger.debug(f"building model: {f_mod}") 54 | self.bar = BuilderBar(total, self.desc) 55 | 56 | def built(self, score: float) -> None: 57 | assert self.bar 58 | self.bar.close() 59 | logger.debug(f"model {self.f_mod} built: score={score:.4f}") 60 | 61 | def iteration(self, n: int, total: int, loss: list[float]) -> None: 62 | del n, total # unused argument 63 | assert self.bar 64 | self.bar.done(loss) 65 | 66 | def trials(self, nick: str, iters: int, timeout: int) -> None: 67 | del timeout 68 | logger.info( 69 | f"Running tuning phase: {nick}\n> \n> ### Tuning `{nick}` ###\n> ") 70 | self.iters = f"/{iters}" if iters else "" 71 | self.header = ["it", nick, "score", "test.acc", "train.acc", "time"] 72 | self.table = [] 73 | 74 | def trying(self, nick: str, it: int, values: list[float]) -> None: 75 | self.it = it + 1 76 | self.desc = f"{nick}[{it+1}{self.iters}]" 77 | self.values = ", ".join("%.4f" % v if type(v) is float else str(v) 78 | for v in values) 79 | self.desc = f"[{it+1}{self.iters}] {self.values:8s}" 80 | 81 | #def tried(self, score, acc, trainacc, duration): 82 | def tried(self, stats: dict[str, Any]) -> None: 83 | assert self.table 84 | self.table.append(( 85 | self.it, 86 | self.values, 87 | f"{stats['score']:.4f}", 88 | human.humanacc(stats["valid_acc"]), 89 | human.humanacc(stats["train_acc"]), 90 | human.humantime(stats["duration"]), 91 | )) 92 | 93 | def trialed(self, nick: str) -> None: 94 | del nick 95 | assert self.table and self.header 96 | lines = [] 97 | lines.append("") 98 | lines.extend( 99 | markdown.table( 100 | self.header, 101 | self.table, 102 | key=lambda x: float(x[2]), 103 | )) 104 | lines.append("") 105 | logger.info("\n" + markdown.dump(lines, prefix="> ")) 106 | 107 | def tuning(self, t_start: int) -> None: 108 | self.t_start = t_start 109 | 110 | def tuned(self, t_end: int) -> None: 111 | self.t_end = t_end 112 | 113 | def info(self, msg: str) -> None: 114 | logger.info(msg) 115 | 116 | def debug(self, msg: str) -> None: 117 | logger.debug(msg) 118 | -------------------------------------------------------------------------------- /src/solverpy/builder/autotune/tune.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, TYPE_CHECKING 2 | import os 3 | import math 4 | import optuna 5 | 6 | from . import check 7 | 8 | if TYPE_CHECKING: 9 | from .build import Talker 10 | from optuna.samplers import BaseSampler 11 | 12 | #UserAttrs = tuple[Any, ...] 13 | UserAttrs = tuple[float, float, float, str, float] 14 | TuneResult = tuple[UserAttrs, dict[str, Any]] 15 | 16 | 17 | def tune( 18 | check_fun: Callable[..., float], 19 | nick: str, 20 | iters: int, 21 | timeout: (int | None), 22 | d_tmp: str, 23 | queue: "Talker | None" = None, 24 | sampler: "BaseSampler | None" = None, 25 | **args: Any, 26 | ) -> TuneResult: 27 | d_tmp = os.path.join(d_tmp, nick) 28 | if queue: queue.put(("trials", (nick, iters, timeout))) 29 | study = optuna.create_study(direction='maximize', sampler=sampler) 30 | objective = lambda trial: check_fun(trial, d_tmp=d_tmp, queue=queue, **args) 31 | study.optimize(objective, n_trials=iters, timeout=timeout) 32 | best = tuple(study.best_trial.user_attrs[x] for x in [ 33 | "score", 34 | "acc", 35 | "trainacc", 36 | "model", 37 | "time", 38 | ]) 39 | if queue: queue.put(("trialed", (nick,))) 40 | return (best, study.best_trial.params) 41 | 42 | 43 | def leaves_grid(min_leaves: int, max_leaves: int, **args: Any) -> TuneResult: 44 | args = dict(args, min_leaves=min_leaves, max_leaves=max_leaves) 45 | min_base = round(2 * math.log2(min_leaves)) 46 | max_base = round(2 * math.log2(max_leaves)) 47 | values = list([round(2**(n / 2)) for n in range(min_base, max_base + 1)]) 48 | sampler = optuna.samplers.GridSampler({"num_leaves": values}) 49 | return tune(check.leaves, "leaves", sampler=sampler, **args) 50 | 51 | 52 | def bagging(**args: Any) -> TuneResult: 53 | return tune(check.bagging, "bagging", **args) 54 | 55 | 56 | def min_data(**args: Any) -> TuneResult: 57 | values = [5, 10, 20, 50, 100, 500, 1000, 2000, 5000, 10000] 58 | sampler = optuna.samplers.GridSampler({"min_data": values}) 59 | return tune(check.min_data, "min_data", sampler=sampler, **args) 60 | 61 | 62 | def regular(**args: Any) -> TuneResult: 63 | return tune(check.regular, "regular", **args) 64 | 65 | 66 | def depth(**args: Any) -> TuneResult: 67 | return tune(check.depth, "depth", **args) 68 | -------------------------------------------------------------------------------- /src/solverpy/builder/autotuner.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | import os 3 | import shutil 4 | import logging 5 | 6 | from .builder import Builder 7 | from .autotune import autotune 8 | from ..benchmark.reports import progress 9 | from ..benchmark.setups.setup import Setup 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | TUNEARGS = dict( 14 | phases="l:b:m:r", 15 | timeout=None, 16 | iters=8, 17 | min_leaves=2, 18 | max_leaves=16, 19 | #init_params=dict(num_round=1000), 20 | ) 21 | 22 | 23 | class AutoTuner(Builder): 24 | 25 | def __init__( 26 | self, 27 | trains: Setup, 28 | devels: (Setup | None) = None, 29 | tuneargs: (dict[str, Any] | None) = None, 30 | ): 31 | assert "dataname" in trains 32 | Builder.__init__(self, trains["dataname"]) 33 | self._trains = trains 34 | self._devels = devels or trains 35 | self._tuneargs: dict[str, Any] = TUNEARGS | (tuneargs or {}) 36 | 37 | def path(self, modelfile: str = "model.lgb") -> str: 38 | if modelfile: 39 | return os.path.join(super().path(), modelfile) 40 | else: 41 | return super().path() 42 | 43 | def build(self) -> None: 44 | assert "trains" in self._trains 45 | assert "trains" in self._devels 46 | assert "refs" in self._trains 47 | logger.info( 48 | f"Building model: {self._dataname}\n> \n> ## Building model `{self._dataname}` ##\n> " 49 | ) 50 | logger.debug(f'using trains: {self._trains["trains"].path()}') 51 | 52 | f_model = self.path() 53 | if os.path.exists(f_model): 54 | logger.info(f"Skipped model building; model {self._dataname} exists.") 55 | self._strats = self.applies(self._trains["refs"], self._dataname) 56 | return 57 | 58 | f_train = self._trains["trains"].path() 59 | f_test = self._devels["trains"].path() 60 | 61 | logger.info(f"Tunning learning params: train={f_train} test={f_test}") 62 | ret = autotune.prettytuner( 63 | f_train=f_train, 64 | f_test=f_test, 65 | d_tmp=self.path("opt"), 66 | **self._tuneargs, 67 | ) 68 | 69 | #f_best = ret[3] 70 | (score, acc, trainacc, f_best, dur, params, pos, neg) = ret 71 | (pos, neg) = (int(pos), int(neg)) 72 | shutil.copyfile(f_best, f_model) 73 | #self._models = [f_model] 74 | self._strats = self.applies(self._trains["refs"], self._dataname) 75 | 76 | report = progress.build(self._dataname, *ret) 77 | logger.info(f"Model {self._dataname} built.\n{report}") 78 | -------------------------------------------------------------------------------- /src/solverpy/builder/builder.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | 4 | from ..benchmark.path import bids 5 | from ..solver.object import SolverPyObj 6 | 7 | NAME = "models" 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | class Builder(SolverPyObj): 13 | """Build the model from the training samples.""" 14 | 15 | def __init__(self, dataname: str): 16 | """Construct the builder and store the dataname.""" 17 | SolverPyObj.__init__(self, dataname=dataname) 18 | self._strats = [] 19 | self._dataname = dataname 20 | 21 | def path(self) -> str: 22 | """Return the model filename.""" 23 | return os.path.join(bids.dbpath(NAME), self._dataname) 24 | 25 | def reset(self, dataname: str) -> None: 26 | """Reset the dataname, for example, when a new loop is initiated.""" 27 | self._dataname = dataname 28 | 29 | def build(self) -> None: 30 | """Build the model(s). Save the list of new strategies `self._strats`.""" 31 | raise NotImplementedError() 32 | 33 | def apply(self, sid: str, model: str) -> list[str]: 34 | """Combine the `model` with strategy `sid`.""" 35 | del sid, model 36 | raise NotImplementedError() 37 | 38 | @property 39 | def strategies(self): 40 | """Return all created strategies.""" 41 | return self._strats 42 | 43 | def applies(self, sidlist: list[str], model: str) -> list[str]: 44 | """Combine the `model` with several strategies `sidlist`.""" 45 | new = [] 46 | for ref in sidlist: 47 | new.extend(self.apply(ref, model)) 48 | return new 49 | 50 | -------------------------------------------------------------------------------- /src/solverpy/builder/cvc5ml.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | import os 3 | import logging 4 | 5 | from .builder import NAME 6 | from .autotuner import AutoTuner 7 | from ..benchmark.path import sids, bids 8 | from ..benchmark.setups.setup import Setup 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class Cvc5ML(AutoTuner): 14 | 15 | def __init__( 16 | self, 17 | trains: Setup, 18 | devels: (Setup | None) = None, 19 | tuneargs: (dict[str, Any] | None) = None, 20 | ): 21 | AutoTuner.__init__( 22 | self, 23 | trains, 24 | devels, 25 | tuneargs, 26 | ) 27 | 28 | def template(self, sid : str) -> str: 29 | "`sid` must be base strategy without parameters" 30 | if sid.endswith("-ml"): 31 | logger.debug(f"strategy {sid} already ml-enhanced") 32 | return sid 33 | sidml = f"{sid}-ml" 34 | if os.path.exists(sids.path(sidml)): 35 | logger.debug(f"ml strategy {sidml} already exists") 36 | return sidml 37 | dbpath = bids.dbpath(NAME) 38 | mod = f"{dbpath}/@@@model:default@@@/model.lgb" 39 | strat = sids.load(sid).rstrip() 40 | strat = self.mlstrat(strat, mod) 41 | sids.save(sidml, strat) 42 | logger.debug( 43 | f"created parametric ml strategy {sidml} inherited from {sid}:\n{strat}" 44 | ) 45 | return sidml 46 | 47 | def mlstrat(self, strat: str, model: str) -> str: 48 | adds = "\n".join([ 49 | f"--ml-engine", 50 | f"--ml-model={model}", 51 | f"--ml-usage=@@@usage:1.0@@@", 52 | f"--ml-fallback=@@@fallback:0@@@", 53 | f"--ml-selector=@@@sel:orig@@@", 54 | f"--ml-selector-value=@@@val:0.5@@@", 55 | ]) 56 | return f"{strat}\n{adds}" 57 | 58 | def apply(self, sid: str, model: str) -> list[str]: 59 | (base, args) = sids.split(sid) 60 | tpl = self.template(base) 61 | sidml = sids.fmt(tpl, dict(args, model=model)) 62 | logger.debug(f"new strategy: {sidml}") 63 | return [sidml] 64 | -------------------------------------------------------------------------------- /src/solverpy/builder/enigma.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, TYPE_CHECKING 2 | import os 3 | import re 4 | import logging 5 | 6 | from .builder import Builder, NAME 7 | from .autotuner import AutoTuner 8 | from ..benchmark.path import sids, bids 9 | from .plugins import enigma 10 | from ..benchmark.setups.setup import Setup 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | class EnigmaModel(AutoTuner): 16 | 17 | def __init__( 18 | self, 19 | trains: Setup, 20 | devels: (Setup | None), 21 | tuneargs: (dict[str, Any] | None), 22 | variant: str, 23 | ): 24 | AutoTuner.__init__(self, trains, devels, tuneargs) 25 | self._variant = variant 26 | self._fkey = f"{variant}_features" 27 | self.reset(self._dataname) 28 | 29 | def featurepath(self) -> str: 30 | fpath = enigma.featurepath(self._trains[self._fkey]) 31 | return f"{self._variant}_{fpath}" 32 | 33 | def reset(self, dataname: str) -> None: 34 | dataname = os.path.join(dataname, self.featurepath()) 35 | super().reset(dataname) # does: self._dataname = dataname 36 | 37 | def template( 38 | self, 39 | sid: str, 40 | name: str, 41 | mk_strat: Callable[[str], str], 42 | ) -> str: 43 | sidml = f"{sid}-{name}" 44 | if os.path.exists(sids.path(sidml)): 45 | logger.debug(f"ml strategy {sidml} already exists") 46 | return sidml 47 | strat = mk_strat(sid) 48 | sids.save(sidml, strat) 49 | logger.debug( 50 | f"created parametric ml strategy {sidml} inherited from {sid}:\n{strat}" 51 | ) 52 | return sidml 53 | 54 | def build(self) -> None: 55 | super().build() 56 | f_map = self.path("enigma.map") 57 | features = self._trains[self._fkey] 58 | open(f_map, "w").write(f'features("{features}").\n') 59 | 60 | 61 | class EnigmaSel(EnigmaModel): 62 | 63 | def __init__( 64 | self, 65 | trains: Setup, 66 | devels: (Setup | None) = None, 67 | tuneargs: (dict[str, Any] | None) = None, 68 | ): 69 | EnigmaModel.__init__( 70 | self, 71 | trains, 72 | devels, 73 | tuneargs, 74 | "sel", 75 | ) 76 | 77 | def apply(self, sid: str, model: str) -> list[str]: 78 | (base, args) = sids.split(sid) 79 | sidsolo = self.template(base, "solo", solo) 80 | sidsolo = sids.fmt(sidsolo, dict(args, sel=model)) 81 | sidcoop = self.template(base, "coop", coop) 82 | sidcoop = sids.fmt(sidcoop, dict(args, sel=model)) 83 | news = [sidsolo, sidcoop] 84 | logger.debug(f"new strategies: {news}") 85 | return news 86 | 87 | 88 | class EnigmaGen(EnigmaModel): 89 | 90 | def __init__( 91 | self, 92 | trains: Setup, 93 | devels: (Setup | None) = None, 94 | tuneargs: (dict[str, Any] | None) = None, 95 | ): 96 | EnigmaModel.__init__( 97 | self, 98 | trains, 99 | devels, 100 | tuneargs, 101 | "gen", 102 | ) 103 | 104 | def apply(self, sid: str, model: str) -> list[str]: 105 | (base, args) = sids.split(sid) 106 | sidgen = self.template(base, "gen", gen) 107 | sidgen = sids.fmt(sidgen, dict(args, gen=model)) 108 | news = [sidgen] 109 | logger.debug(f"new strategies: {news}") 110 | return news 111 | 112 | 113 | class Enigma(EnigmaModel): 114 | 115 | def __init__( 116 | self, 117 | trains: Setup, 118 | devels: (Setup | None) = None, 119 | tuneargs: (dict[str, Any] | None) = None, 120 | ): 121 | AutoTuner.__init__( 122 | self, 123 | trains, 124 | devels, 125 | tuneargs, 126 | ) 127 | assert "trains" in trains 128 | assert "sel_features" in trains 129 | assert "gen_features" in trains 130 | sel = trains["sel_features"] 131 | gen = trains["gen_features"] 132 | trains0 = trains 133 | if sel and gen: 134 | # split the multi train data if it is the case 135 | assert isinstance(trains["trains"], enigma.EnigmaMultiTrains) 136 | trains0 = Setup(trains, trains=trains["trains"]._sel) 137 | self._sel = EnigmaSel(trains0, devels, tuneargs) if sel else None 138 | trains0 = trains 139 | if sel and gen: 140 | assert isinstance(trains["trains"], enigma.EnigmaMultiTrains) 141 | trains0 = Setup(trains, trains=trains["trains"]._gen) 142 | self._gen = EnigmaGen(trains0, devels, tuneargs) if gen else None 143 | 144 | def reset(self, dataname: str) -> None: 145 | if self._sel: 146 | self._sel.reset(dataname) 147 | if self._gen: 148 | self._gen.reset(dataname) 149 | Builder.reset(self, dataname) 150 | 151 | def build(self) -> None: 152 | self._strats = [] 153 | if self._sel: 154 | self._sel.build() 155 | self._strats.extend(self._sel.strategies) 156 | if self._gen: 157 | self._gen.build() 158 | self._strats.extend(self._gen.strategies) 159 | if self._sel and self._gen: 160 | assert "refs" in self._trains 161 | refs = self._trains["refs"] 162 | self._strats.extend(self.applies(refs, self._dataname)) 163 | 164 | def apply(self, sid: str, model: str) -> list[str]: 165 | del model # unused argument 166 | assert self._sel and self._gen 167 | (base, args) = sids.split(sid) 168 | sidsolo = f"{base}-solo" 169 | sidcoop = f"{base}-coop" 170 | sidsologen = self.template(sidsolo, "gen", gen) 171 | sidcoopgen = self.template(sidcoop, "gen", gen) 172 | args = dict(args, sel=self._sel._dataname, gen=self._gen._dataname) 173 | sidsologen = sids.fmt(sidsologen, args) 174 | sidcoopgen = sids.fmt(sidcoopgen, args) 175 | news = [sidsologen, sidcoopgen] 176 | logger.debug(f"new strategies: {news}") 177 | return news 178 | 179 | 180 | def cef( 181 | freq: int, 182 | model: str, 183 | efun: str = "EnigmaticLgb", 184 | prio: str = "ConstPrio", 185 | weights: int = 1, 186 | threshold: float = 0.5, 187 | ): 188 | dbpath = bids.dbpath(NAME) 189 | freq0 = f"@@@freq:{freq}@@@" 190 | prio0 = f"@@@prio:{prio}@@@" 191 | model0 = f"{dbpath}/@@@sel:{model}@@@" 192 | weights0 = str(f"@@@weights:{weights}@@@") 193 | threshold0 = f"@@@thrsel:{threshold}@@@" 194 | return f'{freq0}*{efun}({prio0},"{model0}",{weights0},{threshold0})' 195 | 196 | 197 | def solo( 198 | sid: str, 199 | *, 200 | model: str = "default", 201 | noinit: bool = False, 202 | efun: str = "EnigmaticLgb", 203 | prio: str = "ConstPrio", 204 | weights: int = 1, 205 | threshold: float = 0.5, 206 | ) -> str: 207 | strat = sids.load(sid) 208 | assert strat.find("-H'") >= 0 209 | if noinit: 210 | strat = strat.replace("--prefer-initial-clauses", "") 211 | base = strat[:strat.index("-H'")] 212 | eni = cef(1, model, efun, prio, weights, threshold) 213 | return f"{base}-H'({eni})'" 214 | 215 | 216 | def coop( 217 | sid : str, 218 | *, 219 | model: str = "default", 220 | noinit: bool = False, 221 | efun: str = "EnigmaticLgb", 222 | prio: str = "ConstPrio", 223 | weights: int = 1, 224 | threshold: float = 0.5, 225 | ) -> str: 226 | strat = sids.load(sid) 227 | assert strat.find("-H'") >= 0 228 | if noinit: 229 | strat = strat.replace("--prefer-initial-clauses", "") 230 | freq = sum(map(int, re.findall(r"(\d*)\*", strat))) 231 | eni = cef(freq, model, efun, prio, weights, threshold) 232 | strat = strat.replace("-H'(", f"-H'({eni},") 233 | return strat 234 | 235 | 236 | def solo0(sid: str, **kwargs: Any) -> str: 237 | return solo(sid, **dict(kwargs, noinit=True)) 238 | 239 | 240 | def coop0(sid: str, **kwargs: Any) -> str: 241 | return coop(sid, **dict(kwargs, noinit=True)) 242 | 243 | 244 | def gen(sid: str, *, model: str = "default", threshold: float = 0.1) -> str: 245 | strat = sids.load(sid) 246 | assert strat.count("-H'") == 1 247 | dbpath = bids.dbpath(NAME) 248 | model0 = f"{dbpath}/@@@gen:{model}@@@" 249 | threshold0 = f"@@@thrgen:{threshold}@@@" 250 | args = f'--enigmatic-gen-model="{model0}" --enigmatic-gen-threshold={threshold0}' 251 | return strat.replace("-H'", f"{args} -H'") 252 | 253 | -------------------------------------------------------------------------------- /src/solverpy/builder/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | from .cvc5 import Cvc5Trains, Cvc5TrainsDebug 2 | from .enigma import EnigmaTrains, EnigmaMultiTrains, EnigmaTrainsDebug 3 | 4 | __all__ = ["Cvc5Trains", "Cvc5TrainsDebug", "EnigmaTrains", 5 | "EnigmaMultiTrains", "EnigmaTrainsDebug"] 6 | 7 | -------------------------------------------------------------------------------- /src/solverpy/builder/plugins/cvc5.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | import re 3 | 4 | from .svm import SvmTrains 5 | from ...solver.plugins.db.outputs import Outputs 6 | from ...benchmark.path import bids 7 | 8 | NAME = "trains" 9 | 10 | SAMPLES = re.compile(r"^; QUANTIFIER SAMPLES\n(.*)^; END QUANTIFIER SAMPLES", 11 | flags=re.MULTILINE | re.DOTALL) 12 | 13 | 14 | class Cvc5Trains(SvmTrains): 15 | 16 | def __init__(self, dataname: str): 17 | SvmTrains.__init__(self, dataname) 18 | 19 | def extract( 20 | self, 21 | instance: tuple[str, str], 22 | strategy: str, 23 | output: str, 24 | result: dict[str, Any], 25 | ) -> str: 26 | del instance, strategy, result # unused arguments 27 | return cvc5samples(output) 28 | 29 | 30 | class Cvc5TrainsDebug(Outputs): 31 | 32 | def __init__(self, flatten: bool = True): 33 | Outputs.__init__(self, flatten) 34 | self._path = bids.dbpath(NAME) 35 | 36 | def path(self, instance: tuple[str, str], strategy: str) -> str: 37 | return super().path(instance, strategy) + ".in" 38 | 39 | def finished( 40 | self, 41 | instance: tuple[str, str], 42 | strategy: str, 43 | output: str, 44 | result: dict[str, Any], 45 | ) -> None: 46 | if not (output and self.solver.solved(result)): 47 | return 48 | samples = cvc5samples(output) 49 | if samples: 50 | self.write(instance, strategy, samples) 51 | 52 | 53 | def cvc5samples(output: str) -> str: 54 | mo = SAMPLES.search(output) 55 | if not mo: 56 | return "" 57 | samples = mo.group(1).strip().split("\n") 58 | samples = [x for x in samples if x and not x.startswith(";")] 59 | samples = "\n".join(samples) + "\n" if samples else "" 60 | return samples 61 | 62 | -------------------------------------------------------------------------------- /src/solverpy/builder/plugins/enigma.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | import os 3 | import re 4 | 5 | from .svm import SvmTrains 6 | from .multi import MultiTrains 7 | from ...solver.plugins.db.outputs import Outputs 8 | from ...benchmark.path import bids 9 | 10 | NAME = "trains" 11 | 12 | SEL = re.compile(r"^#SEL# .*$", flags=re.MULTILINE) 13 | GEN = re.compile(r"^#GEN# .*$", flags=re.MULTILINE) 14 | 15 | TRANS = str.maketrans("", "", "[(:,)]=") 16 | 17 | 18 | class EnigmaTrains(SvmTrains): 19 | 20 | def __init__(self, dataname: str, features: str, variant: str): 21 | self._features = features 22 | self._variant = variant 23 | SvmTrains.__init__(self, dataname) 24 | 25 | def featurepath(self) -> str: 26 | return f"{self._variant}_{featurepath(self._features)}" 27 | 28 | def reset( 29 | self, 30 | dataname: (str | None) = None, 31 | filename: str = "train.in", 32 | ) -> None: 33 | if dataname: 34 | dataname = os.path.join(dataname, self.featurepath()) 35 | super().reset(dataname, filename) 36 | 37 | def extract( 38 | self, 39 | instance: tuple[str, str], 40 | strategy: str, 41 | output: str, 42 | result: dict[str, Any], 43 | ) -> str: 44 | del instance, strategy, result # unused arguments 45 | return samples(output, self._variant) 46 | 47 | 48 | class EnigmaMultiTrains(MultiTrains): 49 | 50 | def __init__(self, dataname: str, sel: str, gen: str): 51 | MultiTrains.__init__(self, dataname) 52 | self._sel = EnigmaTrains(dataname, sel, "sel") 53 | self._gen = EnigmaTrains(dataname, gen, "gen") 54 | self.dispatch(self._sel) 55 | self.dispatch(self._gen) 56 | 57 | 58 | class EnigmaTrainsDebug(Outputs): 59 | 60 | def __init__(self, features: str, variant: str, flatten: bool = True): 61 | Outputs.__init__(self, flatten) 62 | self._path = os.path.join( 63 | bids.dbpath(NAME), 64 | "debug", 65 | f"{variant}_{featurepath(features)}", 66 | ) 67 | self._variant = variant 68 | 69 | def path( 70 | self, 71 | instance: tuple[str, str], 72 | strategy: str, 73 | ext: str = ".in", 74 | ) -> str: 75 | return super().path(instance, strategy) + ext 76 | 77 | def decorate( 78 | self, 79 | cmd: str, 80 | instance: tuple[str, str], 81 | strategy: str, 82 | ) -> str: 83 | if self._variant != "sel": 84 | return cmd 85 | enimap = self.path(instance, strategy, ".map") 86 | buckets = self.path(instance, strategy, ".buckets") 87 | os.makedirs(os.path.dirname(buckets), exist_ok=True) 88 | return f"{cmd} --enigmatic-output-map={enimap} --enigmatic-output-buckets={buckets}" 89 | 90 | def finished( 91 | self, 92 | instance: tuple[str, str], 93 | strategy: str, 94 | output: str, 95 | result: dict[str, Any], 96 | ) -> None: 97 | if not (output and self.solver.solved(result)): 98 | return 99 | vectors = samples(output, self._variant) 100 | if samples: 101 | self.write(instance, strategy, vectors) 102 | 103 | 104 | def samples(output: str, variant: str) -> str: 105 | assert variant in ["sel", "gen"] 106 | pattern = SEL if variant == "sel" else GEN 107 | vectors = pattern.findall(output) 108 | vectors = [x[7:] for x in vectors] # NOTE: this also removes the sign [+-] 109 | if vectors: vectors.append("") # new line at the end 110 | return "\n".join(vectors) if vectors else "" 111 | 112 | 113 | def featurepath(features: str) -> str: 114 | return features.translate(TRANS) 115 | 116 | -------------------------------------------------------------------------------- /src/solverpy/builder/plugins/multi.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, TYPE_CHECKING 2 | import logging 3 | 4 | from .svm import SvmTrains 5 | 6 | if TYPE_CHECKING: 7 | from ...solver.solverpy import SolverPy 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | class MultiTrains(SvmTrains): 13 | 14 | def __init__(self, dataname: str): 15 | self._trains: list["SvmTrains"] = [] 16 | self._dataname = dataname 17 | 18 | def dispatch(self, t: "SvmTrains"): 19 | self._trains.append(t) 20 | 21 | def apply(self, function: Callable[["SvmTrains"], None]) -> None: 22 | for t in self._trains: 23 | function(t) 24 | 25 | def register(self, solver: "SolverPy") -> None: 26 | super().register(solver) 27 | self._solver = solver 28 | for t in self._trains: 29 | t._solver = solver 30 | 31 | def reset( 32 | self, 33 | dataname: (str | None) = None, 34 | filename: str = "train.in", 35 | ) -> None: 36 | if dataname: 37 | self._dataname = dataname 38 | self.apply(lambda x: x.reset(dataname=dataname, filename=filename)) 39 | 40 | def path( 41 | self, 42 | dataname: (str | None) = None, 43 | filename: (str | None) = None, 44 | ) -> tuple[str, ...]: 45 | return tuple(t.path(dataname, filename) for t in self._trains) 46 | 47 | def finished(self, *args: Any, **kwargs: Any) -> None: 48 | self.apply(lambda x: x.finished(*args, **kwargs)) 49 | 50 | def extract(self, *args: Any, **kwargs: Any) -> None: 51 | self.apply(lambda x: x.extract(*args, **kwargs)) 52 | 53 | def save(self, *args: Any, **kwargs: Any) -> None: 54 | self.apply(lambda x: x.save(*args, **kwargs)) 55 | 56 | def stats(self, *args: Any, **kwargs: Any) -> None: 57 | self.apply(lambda x: x.stats(*args, **kwargs)) 58 | 59 | def compress(self, *args: Any, **kwargs: Any) -> None: 60 | self.apply(lambda x: x.compress(*args, **kwargs)) 61 | 62 | def merge( 63 | self, 64 | previous: str | tuple[str, ...], 65 | outfilename: str, 66 | ) -> None: 67 | assert len(previous) == len(self._trains) 68 | assert type(previous) is tuple 69 | for (t0, p0) in zip(self._trains, previous): 70 | t0.merge(p0, outfilename) 71 | -------------------------------------------------------------------------------- /src/solverpy/builder/plugins/svm.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | import multiprocessing 4 | 5 | from .. import svm 6 | from .trains import Trains 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | class SvmTrains(Trains): 12 | 13 | def __init__(self, dataname: str, filename: str = "train.in"): 14 | Trains.__init__(self, dataname, filename=filename) 15 | self.info = multiprocessing.Manager().Namespace() 16 | self.info.total = 0 17 | self.info.pos = 0 18 | self.info.neg = 0 19 | 20 | def stats(self, instance: tuple[str, str], strategy: str, 21 | samples: str) -> None: 22 | count = samples.count("\n") 23 | s0 = samples[0] 24 | pos = samples.count("\n1 ") + (1 if s0 == "1" else 0) 25 | neg = samples.count("\n0 ") + (1 if s0 == "0" else 0) 26 | self.info.total += count 27 | self.info.pos += pos 28 | self.info.neg += neg 29 | with open(self.path() + "-stats.txt", "a") as infa: 30 | infa.write(f"{instance} {strategy}: {count} ({pos} / {neg})\n") 31 | 32 | def compress(self) -> None: 33 | logger.info( 34 | f"Training vectors count: {self.info.total} ({self.info.pos} / {self.info.neg}) " 35 | ) 36 | if os.path.isfile(self.path()): 37 | svm.compress(self.path()) 38 | 39 | def merge( 40 | self, 41 | previous: str | tuple[str, ...], 42 | outfilename: str, 43 | ) -> None: 44 | assert self._filename != outfilename 45 | assert type(previous) is str 46 | f_out = self.path(filename=outfilename) 47 | svm.merge(previous, self.path(), f_out=f_out) 48 | #self.reset(filename=outfilename) 49 | -------------------------------------------------------------------------------- /src/solverpy/builder/plugins/trains.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | import os 3 | import logging 4 | import multiprocessing 5 | 6 | from ...solver.plugins.decorator import Decorator 7 | from ...benchmark.path import bids 8 | 9 | if TYPE_CHECKING: 10 | from ...solver.solverpy import SolverPy 11 | 12 | NAME = "trains" 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | class Trains(Decorator): 18 | 19 | def __init__(self, dataname: str, filename: str = "train.in"): 20 | self._lock = multiprocessing.Manager().Lock() 21 | self.reset(dataname, filename) 22 | 23 | def reset( 24 | self, 25 | dataname: (str | None) = None, 26 | filename: str = "train.in", 27 | ) -> None: 28 | if dataname: 29 | self._dataname = dataname 30 | self._filename = filename 31 | 32 | def path( 33 | self, 34 | dataname: (str | None) = None, 35 | filename: (str | None) = None, 36 | ) -> Any: 37 | dataname = dataname or self._dataname 38 | filename = filename or self._filename 39 | return os.path.join(bids.dbpath(NAME), dataname, filename) 40 | 41 | def register(self, solver: "SolverPy") -> None: 42 | super().register(solver) 43 | self._solver = solver 44 | 45 | def finished( 46 | self, 47 | instance: tuple[str, str], 48 | strategy: str, 49 | output: str, 50 | result: dict[str, Any], 51 | ): 52 | if not (output and self._solver.solved(result)): 53 | return 54 | samples = self.extract(instance, strategy, output, result) 55 | self.save(instance, strategy, samples) 56 | 57 | def extract( 58 | self, 59 | instance: tuple[str, str], 60 | strategy: str, 61 | output: str, 62 | result: dict[str, Any], 63 | ) -> Any: 64 | del instance, strategy, output, result # unused arguments 65 | "Extract training samples from `output`." 66 | raise NotImplementedError() 67 | 68 | def save( 69 | self, 70 | instance: tuple[str, str], 71 | strategy: str, 72 | samples: str, 73 | ) -> None: 74 | if not samples: 75 | return 76 | self._lock.acquire() 77 | try: 78 | os.makedirs(os.path.dirname(self.path()), exist_ok=True) 79 | with open(self.path(), "a") as fa: 80 | fa.write(samples) 81 | self.stats(instance, strategy, samples) 82 | finally: 83 | self._lock.release() 84 | 85 | def stats( 86 | self, 87 | instance: tuple[str, str], 88 | strategy: str, 89 | samples: str, 90 | ): 91 | "Save optional statistics." 92 | del instance, strategy, samples # unused arguments 93 | pass 94 | 95 | -------------------------------------------------------------------------------- /src/solverpy/builder/svm.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | import os 3 | import logging 4 | from collections import defaultdict 5 | 6 | import numpy 7 | import scipy 8 | from sklearn.datasets import load_svmlight_file, dump_svmlight_file 9 | 10 | from ..tools import human 11 | from ..benchmark.reports import progress 12 | 13 | if TYPE_CHECKING: 14 | from scipy.sparse import spmatrix 15 | from numpy import ndarray 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | def datafiles(f_in: str) -> tuple[str, str]: 21 | z_data = f_in + "-data.npz" 22 | z_label = f_in + "-label.npz" 23 | return (z_data, z_label) 24 | 25 | 26 | def iscompressed(f_in: str) -> bool: 27 | return all(map(os.path.isfile, datafiles(f_in))) 28 | 29 | 30 | def exists(f_in: str) -> bool: 31 | return iscompressed(f_in) or os.path.isfile(f_in) 32 | 33 | 34 | def size(f_in: str) -> int: 35 | if iscompressed(f_in): 36 | return sum(os.path.getsize(f) for f in datafiles(f_in)) 37 | else: 38 | return os.path.getsize(f_in) 39 | 40 | 41 | def format(f_in: str) -> str: 42 | if iscompressed(f_in): 43 | return "binary/npz" 44 | if os.path.isfile(f_in): 45 | return "text/svm" 46 | return "unknown" 47 | 48 | 49 | def load(f_in: str) -> tuple["spmatrix", "ndarray"]: 50 | logger.info( 51 | f"Loading trains of size {human.humanbytes(size(f_in))} from `{f_in}`.") 52 | if iscompressed(f_in): 53 | logger.debug(f"loading compressed data") 54 | (z_data, z_label) = datafiles(f_in) 55 | data = scipy.sparse.load_npz(z_data) 56 | label = numpy.load(z_label, allow_pickle=True)["label"] 57 | logger.debug(f"compressed data loaded") 58 | else: 59 | logger.debug(f"loading uncompressed data") 60 | (data, label) = load_svmlight_file(f_in, zero_based=True) # type: ignore 61 | logger.debug(f"uncompressed data loaded") 62 | logger.info("Trains loaded.") 63 | return (data, label) 64 | 65 | 66 | def save(data: "spmatrix", label: "ndarray", f_in: str) -> None: 67 | (z_data, z_label) = datafiles(f_in) 68 | logger.debug(f"saving compressed data to {z_data}") 69 | scipy.sparse.save_npz(z_data, data, compressed=True) 70 | logger.debug(f"saving compressed labels to {z_label}") 71 | numpy.savez_compressed(z_label, label=label) 72 | logger.info(f"Saved trains: {f_in}") 73 | 74 | 75 | def compress(f_in: str, keep: bool = False) -> None: 76 | logger.info( 77 | f"Compressing trains of size {human.humanbytes(size(f_in))} from `{f_in}`." 78 | ) 79 | if iscompressed(f_in): 80 | logger.warning(f"Trains {f_in} are already compressed. Skipped.") 81 | return 82 | size_in = size(f_in) # size before compression 83 | (data, label) = load_svmlight_file(f_in, zero_based=True) # type: ignore 84 | save(data, label, f_in) 85 | report = progress.compress(f_in, size_in, size(f_in), data, label) 86 | if iscompressed(f_in) and not keep: 87 | logger.debug(f"deleting the uncompressed file") 88 | os.remove(f_in) 89 | logger.info( 90 | f"Trains compressed to {human.humanbytes(size(f_in))}.\n{report}") 91 | 92 | 93 | def decompress(f_in: str, keep: bool = True) -> None: 94 | logger.info( 95 | f"Decompressing trains of size {human.humanbytes(size(f_in))} from `{f_in}`." 96 | ) 97 | if not iscompressed(f_in): 98 | logger.warning(f"Trains `{f_in}` are not compressed. Skipped.") 99 | return 100 | (data, label) = load(f_in) 101 | logger.debug(f"dumping trains to {f_in}") 102 | dump_svmlight_file(data, label, f_in) 103 | logger.debug( 104 | f"trains dumped; uncompressed size: {human.humanbytes(os.path.getsize(f_in))}" 105 | ) 106 | if os.path.isfile(f_in) and not keep: 107 | logger.debug(f"deleting the compressed files") 108 | for f in datafiles(f_in): 109 | os.remove(f) 110 | logger.info( 111 | f"Trains decompressed to {human.humanbytes(os.path.getsize(f_in))}.") 112 | 113 | 114 | def merge( 115 | f_in1: (str | None) = None, 116 | f_in2: (str | None) = None, 117 | data1: (tuple["spmatrix", "ndarray"] | None) = None, 118 | data2: (tuple["spmatrix", "ndarray"] | None) = None, 119 | f_out: (str | None) = None, 120 | ) -> tuple["spmatrix", "ndarray"]: 121 | assert data1 or f_in1 122 | assert data2 or f_in2 123 | if f_in1 and f_in2: 124 | logger.info(f"Merging trains: {f_in1} and {f_in2}") 125 | (d1, l1) = data1 if data1 else load(f_in1) # type: ignore 126 | (d2, l2) = data2 if data2 else load(f_in2) # type: ignore 127 | logger.info( 128 | f"Merging data of shapes: {d1.shape[0]}x{d1.shape[1]} and {d2.shape[0]}x{d2.shape[1]}" 129 | ) 130 | d = scipy.sparse.vstack((d1, d2)) 131 | logger.info(f"Merging labels of shapes: {l1.shape[0]} and {l2.shape[0]}") 132 | l = numpy.concatenate((l1, l2)) 133 | if f_out: 134 | save(d, l, f_out) 135 | return (d, l) 136 | 137 | 138 | def deconflict(xs: "spmatrix", ys: "ndarray") -> tuple["spmatrix", "ndarray"]: 139 | "Find conflicting positive and negative samples and remove the negative ones." 140 | assert xs.shape[0] == ys.shape[0] 141 | logger.info("Looking up conflicting samples.") 142 | logger.debug("building samples map") 143 | dups = defaultdict(list) 144 | for i in range(xs.shape[0]): 145 | row = xs.getrow(i) 146 | key = (tuple(row.indices), tuple(row.data)) 147 | dups[key].append(i) 148 | logger.debug("marking conflicting negative samples") 149 | todel = set() 150 | for ids in dups.values(): 151 | if len(ids) < 2: 152 | continue 153 | td = [] 154 | onepos = False 155 | for i in ids: 156 | if ys[i] == 0: 157 | td.append(i) # mark negative indicies to be removed 158 | else: 159 | onepos = True # there is at least one positive 160 | if onepos: 161 | todel.update(td) 162 | logger.debug("deleting marked rows") 163 | keep = [i for i in range(xs.shape[0]) if i not in todel] 164 | xs0 = xs[keep] # type: ignore 165 | ys0 = ys[keep] 166 | logger.info("\n".join([ 167 | "Data shape difference:", 168 | f"\t{xs.shape} --> {xs0.shape}", 169 | f"\t{ys.shape} --> {ys0.shape}", 170 | ])) 171 | return (xs0, ys0) 172 | -------------------------------------------------------------------------------- /src/solverpy/solver/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbboyan/solverpy/caa8da555ff766a50e1b0b0a01f2747df65d8bdb/src/solverpy/solver/__init__.py -------------------------------------------------------------------------------- /src/solverpy/solver/atp/__init__.py: -------------------------------------------------------------------------------- 1 | from .eprover import E 2 | from .lash import Lash 3 | from .vampire import Vampire 4 | from .cvc5 import Cvc5 5 | 6 | __all__ = ["E", "Lash", "Vampire", "Cvc5"] 7 | 8 | -------------------------------------------------------------------------------- /src/solverpy/solver/atp/cvc5.py: -------------------------------------------------------------------------------- 1 | from typing import Pattern, TYPE_CHECKING 2 | import re 3 | 4 | from ..shellsolver import ShellSolver 5 | from ..smt.cvc5 import CVC5_BINARY, CVC5_STATIC, CVC5_KEYS, CVC5_BUILDER 6 | from ..smt.cvc5 import Cvc5 as SmtCvc5 7 | from ..plugins.status.tptp import Tptp 8 | from ..plugins.shell.time import Time 9 | 10 | if TYPE_CHECKING: 11 | from ..plugins.plugin import Plugin 12 | from ...tools.typing import Result 13 | 14 | # depricated 15 | class Cvc5(ShellSolver): 16 | 17 | def __init__( 18 | self, 19 | limit: str, 20 | binary: str = CVC5_BINARY, 21 | static: str = CVC5_STATIC, 22 | plugins: list["Plugin"] = [], 23 | keys: list[str] = CVC5_KEYS, 24 | complete: bool = True, 25 | ): 26 | cmd = f"{binary} --lang=tptp {static}" 27 | plugins = plugins + [ 28 | Time(), 29 | Tptp(complete=complete), 30 | ] 31 | ShellSolver.__init__( 32 | self, 33 | cmd, 34 | limit, 35 | CVC5_BUILDER, 36 | plugins, 37 | wait=1, 38 | ) 39 | self.pattern: Pattern = re.compile( 40 | r"^(%s) = (.*)$" % "|".join(keys), 41 | flags=re.MULTILINE, 42 | ) 43 | 44 | def process(self, output: str) -> "Result": 45 | result = SmtCvc5.process(self, output) # type: ignore 46 | if ("status" in result) and (result["status"] == "timeout"): 47 | result["status"] = "Timeout" 48 | return result 49 | -------------------------------------------------------------------------------- /src/solverpy/solver/atp/eprover.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | import re 3 | 4 | from ..shellsolver import ShellSolver 5 | from ...tools import patterns, human 6 | from ..plugins.status.tptp import Tptp 7 | from ..plugins.shell.time import Time 8 | 9 | if TYPE_CHECKING: 10 | from ..plugins.plugin import Plugin 11 | from ...tools.typing import LimitBuilder, Result 12 | 13 | E_BINARY = "eprover" 14 | 15 | E_STATIC: str = "-s -p -R --print-statistics --tstp-format --memory-limit=2048" 16 | 17 | E_BUILDER: "LimitBuilder" = { 18 | "T": lambda x: "--soft-cpu-limit=%s --cpu-limit=%s" % (x, int(x) + 10), 19 | "P": "--processed-set-limit=%s", 20 | "C": "--processed-clauses-limit=%s", 21 | "G": "--generated-limit=%s" 22 | } 23 | 24 | E_PAT = re.compile(r"^#\s*(\S.*\S)\s*: (\S*)$", re.MULTILINE) 25 | 26 | E_TABLE = { 27 | "Processed clauses": "Processed", 28 | "Generated clauses": "Generated", 29 | "Proof object total steps": "ProofLen", 30 | "Removed by relevancy pruning/SinE": "Pruned", 31 | "Backward-subsumed": "BackSub", 32 | "Backward-rewritten": "BackRew", 33 | "Paramodulations": "Paramod", 34 | "Factorizations": "Fact", 35 | "Equation resolutions": "EqRes", 36 | "Clause-clause subsumption calls (NU)": "Subsumes", 37 | "Termbank termtop insertions": "TermBank", 38 | } 39 | 40 | 41 | class E(ShellSolver): 42 | 43 | def __init__( 44 | self, 45 | limit: str, 46 | binary: str = E_BINARY, 47 | static: str = E_STATIC, 48 | complete: bool = True, 49 | plugins: list["Plugin"] = [], 50 | ): 51 | cmd = f"{binary} {static}" 52 | plugins = plugins + [ 53 | Time(), 54 | Tptp(complete=complete), 55 | ] 56 | ShellSolver.__init__( 57 | self, 58 | cmd, 59 | limit, 60 | E_BUILDER, 61 | plugins, 62 | 15, 63 | complete, 64 | ) 65 | 66 | def process(self, output: str) -> "Result": 67 | result = patterns.keyval(E_PAT, output, E_TABLE) 68 | result = patterns.mapval(result, human.numeric) 69 | return result 70 | 71 | -------------------------------------------------------------------------------- /src/solverpy/solver/atp/lash.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | import re 3 | 4 | from ..shellsolver import ShellSolver 5 | from ...tools import patterns, human 6 | from ..plugins.status.tptp import Tptp 7 | from ..plugins.shell.time import Time 8 | 9 | if TYPE_CHECKING: 10 | from ..plugins.plugin import Plugin 11 | from ...tools.typing import LimitBuilder 12 | 13 | L_BINARY = "lash" 14 | 15 | #L_STATIC = "-p tstp -m mode0 -M %s" % getenv("LASH_MODE_DIR", "./modes") 16 | L_STATIC = "-p tstp" 17 | 18 | L_PAT = re.compile(r"^% (Steps|Mode): (\S*)$", flags=re.MULTILINE) 19 | 20 | L_BUILDER: "LimitBuilder" = { 21 | "T": "", 22 | } 23 | 24 | L_TABLE = { 25 | "Steps": "Steps", 26 | "Mode": "Mode", 27 | } 28 | 29 | 30 | class Lash(ShellSolver): 31 | 32 | def __init__( 33 | self, 34 | limit: str, 35 | binary: str = L_BINARY, 36 | static: str = L_STATIC, 37 | complete: bool = True, 38 | plugins: list["Plugin"] = [], 39 | ): 40 | cmd = f"{binary} {static}" 41 | plugins = plugins + [ 42 | Time(), 43 | Tptp(complete=complete), 44 | ] 45 | ShellSolver.__init__( 46 | self, 47 | cmd, 48 | limit, 49 | L_BUILDER, 50 | plugins, 51 | 0, 52 | complete, 53 | ) 54 | 55 | def process(self, output): 56 | result = patterns.keyval(L_PAT, output, L_TABLE) 57 | result = patterns.mapval(result, human.numeric) 58 | return result 59 | 60 | -------------------------------------------------------------------------------- /src/solverpy/solver/atp/prover9.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | import re 3 | 4 | from ..stdinsolver import StdinSolver 5 | from ..plugins.status.tptp import TPTP_OK, TPTP_INC_OK, TPTP_ALL, TPTP_TIMEOUT 6 | from ...tools import patterns, human 7 | from ..plugins.shell.time import Time 8 | 9 | if TYPE_CHECKING: 10 | from ..plugins.plugin import Plugin 11 | from ...tools.typing import LimitBuilder, Result 12 | 13 | P9_BINARY = "prover9" 14 | 15 | P9_STATIC = """ 16 | clear(print_given). 17 | clear(bell). 18 | """ 19 | 20 | P9_BUILDER: "LimitBuilder" = { 21 | "T": "assign(max_seconds, %s).\n", 22 | "M": lambda x: f"assign(max_megs, {int(x)*1000}).\n", 23 | } 24 | 25 | # termination reason regex pattern 26 | P9_REASON = re.compile( 27 | r"^-+ process[^(]*\((\S*)\) -+$", 28 | flags=re.MULTILINE, 29 | ) 30 | 31 | # pattern to extract all statistics 32 | P9_STATS = re.compile( 33 | r"^=+ STATISTICS =+$(.*)^=+ end of statistics =+$", 34 | flags=re.MULTILINE | re.DOTALL, 35 | ) 36 | 37 | # pattern to extract a single statistic (key/val pair) 38 | P9_SINGLE = re.compile(r"(\w*)=([0-9.]*[0-9])") 39 | 40 | # TPTP-compatible statuses for Prover9 termination reasons 41 | P9_STATUS = { 42 | "max_seconds": "Timeout", 43 | "max_megs": "ResourceOut", 44 | "max_proofs": "Theorem", 45 | "sos_empty": "Satisfiable", 46 | } 47 | 48 | 49 | class Prover9(StdinSolver): 50 | 51 | def __init__( 52 | self, 53 | limit: str, 54 | binary: str = P9_BINARY, 55 | static: str = P9_STATIC, 56 | complete: bool = False, 57 | plugins: list["Plugin"] = [], 58 | ): 59 | plugins = plugins + [Time()] 60 | StdinSolver.__init__( 61 | self, 62 | binary, 63 | limit, 64 | P9_BUILDER, 65 | plugins, 66 | 1, 67 | static, 68 | ) 69 | self._complete = complete 70 | 71 | def process(self, output: str) -> "Result": 72 | reason = P9_REASON.search(output) 73 | if not reason: 74 | return {} 75 | reason = reason.group(1) 76 | 77 | stats = P9_STATS.search(output) 78 | if stats: 79 | result = patterns.keyval(P9_SINGLE, stats.group(1)) 80 | else: 81 | result = {} 82 | 83 | result["reason"] = reason 84 | if reason in P9_STATUS: 85 | result["status"] = P9_STATUS[reason] 86 | elif reason.startswith("max_"): # some limit reached 87 | result["status"] = "ResourceOut" 88 | 89 | result = patterns.mapval(result, human.numeric) 90 | 91 | if "User_CPU" in result: 92 | result["runtime"] = result["User_CPU"] 93 | 94 | return result 95 | 96 | def valid(self, result: "Result") -> bool: 97 | return super().valid(result) and result["status"] in TPTP_ALL 98 | 99 | @property 100 | def success(self) -> frozenset[str]: 101 | return TPTP_OK if self._complete else TPTP_INC_OK 102 | 103 | @property 104 | def timeouts(self) -> frozenset[str]: 105 | return TPTP_TIMEOUT 106 | 107 | -------------------------------------------------------------------------------- /src/solverpy/solver/atp/vampire.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | import re 3 | 4 | from ..shellsolver import ShellSolver 5 | from ...tools import patterns, human 6 | from ..plugins.status.tptp import Tptp 7 | from ..plugins.shell.time import Time 8 | 9 | if TYPE_CHECKING: 10 | from ..plugins.plugin import Plugin 11 | from ...tools.typing import LimitBuilder, Result 12 | 13 | V_BINARY = "vampire" 14 | 15 | V_STATIC = "--proof tptp -stat full --input_syntax tptp --memory_limit 2048" 16 | 17 | V_BUILDER: "LimitBuilder" = { 18 | "T": "--time_limit %ss", 19 | "M": "--memory_limit %s", 20 | } 21 | 22 | V_PAT = re.compile(r"^% (.*): ([0-9.]*).*$", re.MULTILINE) 23 | 24 | V_TABLE = { 25 | "Active clauses": "Active", 26 | "Passive clauses": "Passive", 27 | "Generated clauses": "Generated", 28 | "Initial clauses ": "Initial", 29 | "Time elapsed": "Runtime", 30 | "Memory used [KB]": "Memory", 31 | "Split clauses": "Splits", 32 | } 33 | 34 | 35 | class Vampire(ShellSolver): 36 | 37 | def __init__( 38 | self, 39 | limit: str, 40 | binary: str = V_BINARY, 41 | static: str = V_STATIC, 42 | complete: bool = True, 43 | plugins: list["Plugin"] = [], 44 | ): 45 | cmd = f"{binary} {static}" 46 | plugins = plugins + [ 47 | Time(), 48 | Tptp(complete=complete), 49 | ] 50 | ShellSolver.__init__( 51 | self, 52 | cmd, 53 | limit, 54 | V_BUILDER, 55 | plugins, 56 | 1, 57 | complete, 58 | ) 59 | 60 | def process(self, output: str) -> "Result": 61 | result = patterns.keyval(V_PAT, output, V_TABLE) 62 | result = patterns.mapval(result, human.numeric) 63 | return result 64 | 65 | -------------------------------------------------------------------------------- /src/solverpy/solver/object.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | import logging 3 | 4 | logger = logging.getLogger(__name__) 5 | 6 | 7 | class SolverPyObj: 8 | 9 | def __init__( 10 | self, 11 | cls_name: (str | None) = None, 12 | **kwargs: Any, 13 | ): 14 | self._repr_args: dict[str, Any] = dict(kwargs) 15 | self._cls_name = cls_name or self.__class__.__name__ 16 | 17 | def __repr__(self) -> str: 18 | if hasattr(self, "_repr_args") and self._repr_args is not None: 19 | ias = [f"{x}={y}" for (x, y) in self._repr_args.items()] 20 | ias = ",".join(sorted(ias)) 21 | return f"{self._cls_name}({ias})" 22 | return object.__repr__(self) 23 | 24 | def represent(self) -> (str | list[Any] | dict[str, Any]): 25 | """Default yaml represeneter""" 26 | return repr(self) 27 | 28 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | from .db.bid import Bid 2 | from .db.sid import Sid 3 | from .db.outputs import Outputs 4 | from .db.errors import Errors 5 | 6 | def db(): 7 | return [ 8 | Bid(), 9 | Sid(), 10 | ] 11 | 12 | def outputs(flatten=True, compress=True): 13 | return [ 14 | Bid(), 15 | Sid(), 16 | Outputs(flatten, compress), 17 | Errors(flatten, compress), 18 | ] 19 | 20 | __all__ = ["db", "outputs"] 21 | 22 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/db/__init__.py: -------------------------------------------------------------------------------- 1 | from .bid import Bid 2 | from .sid import Sid 3 | 4 | __all__ = ["Bid", "Sid"] 5 | 6 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/db/bid.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from ..translator import Translator 4 | from ....benchmark.path import bids 5 | 6 | class Bid(Translator): 7 | "Benchmark ids translator." 8 | 9 | def __init__(self, **kwargs): 10 | Translator.__init__(self, **kwargs) 11 | 12 | def translate( 13 | self, 14 | instance: tuple[str,str], 15 | strategy: Any 16 | ) -> tuple[str,Any]: 17 | (bid, problem) = instance 18 | instance0 = bids.path(bid, problem) 19 | return (instance0, strategy) 20 | 21 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/db/errors.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | 3 | from .outputs import Outputs 4 | from ....benchmark.path import bids 5 | 6 | if TYPE_CHECKING: 7 | from ....tools.typing import Result 8 | 9 | NAME = "errors" 10 | 11 | 12 | class Errors(Outputs): 13 | 14 | def __init__( 15 | self, 16 | flatten: bool = True, 17 | compress: bool = True, 18 | ): 19 | Outputs.__init__(self, flatten, compress) 20 | self._path: str = bids.dbpath(NAME) 21 | 22 | def finished( 23 | self, 24 | instance: tuple[str, str], 25 | strategy: str, 26 | output: str, 27 | result: "Result", 28 | ) -> None: 29 | if output and not self.solver.valid(result): 30 | self.write(instance, strategy, output) 31 | 32 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/db/outputs.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | import os 3 | import gzip 4 | 5 | from ..decorator import Decorator 6 | from ....benchmark.path import bids, sids 7 | 8 | if TYPE_CHECKING: 9 | from ...solverpy import SolverPy 10 | 11 | NAME = "outputs" 12 | 13 | 14 | class Outputs(Decorator): 15 | 16 | def __init__( 17 | self, 18 | flatten: bool = True, 19 | compress: bool = True, 20 | ): 21 | Decorator.__init__(self, flatten=flatten, compress=compress) 22 | self._path = bids.dbpath(NAME) 23 | self._flatten = flatten 24 | self._compress = compress 25 | 26 | def register(self, solver: "SolverPy") -> None: 27 | solver.decorators.append(self) 28 | self.solver = solver 29 | 30 | def path( 31 | self, 32 | instance: tuple[str, str], 33 | strategy: str, 34 | ) -> str: 35 | (bid, problem) = instance 36 | bs = bids.name(bid, limit=self.solver._limits.limit) 37 | if self._flatten: 38 | slash = "_._" if (self._flatten is True) else self._flatten 39 | problem = problem.replace("/", slash) 40 | p = os.path.join(self._path, bs, sids.name(strategy), problem) 41 | return p 42 | 43 | def finished( 44 | self, 45 | instance: tuple[str, str], 46 | strategy: str, 47 | output: str, 48 | result: dict, 49 | ) -> None: 50 | if output and self.solver.valid(result): 51 | self.write(instance, strategy, output) 52 | 53 | def write( 54 | self, 55 | instance: tuple[str, str], 56 | strategy: str, 57 | content: str, 58 | ) -> None: 59 | f = self.path(instance, strategy) 60 | os.makedirs(os.path.dirname(f), exist_ok=True) 61 | if self._compress: 62 | fw = gzip.open(f + ".gz", "wb") 63 | else: 64 | fw = open(f, "wb") 65 | fw.write(content.encode()) 66 | fw.close() 67 | 68 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/db/sid.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from ..translator import Translator 4 | from ....benchmark.path import sids 5 | 6 | 7 | class Sid(Translator): 8 | "Strategy ids translator." 9 | 10 | def translate( 11 | self, 12 | instance: Any, 13 | strategy: str, 14 | ) -> tuple[Any, str]: 15 | sid = strategy 16 | strategy0 = sids.load(sid) 17 | if "@@@" in strategy0: 18 | (sid, args) = sids.split(sid) 19 | strategy0 = sids.instatiate(strategy0, args) 20 | return (instance, strategy0) 21 | 22 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/decorator.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | from .plugin import Plugin 3 | 4 | if TYPE_CHECKING: 5 | from ..solverpy import SolverPy 6 | from ...tools.typing import Result 7 | 8 | 9 | class Decorator(Plugin): 10 | 11 | def __init__(self, **kwargs): 12 | Plugin.__init__(self, **kwargs) 13 | 14 | def register(self, solver: "SolverPy") -> None: 15 | solver.decorators.append(self) 16 | 17 | def decorate( 18 | self, 19 | cmd: str, 20 | instance: Any, 21 | strategy: Any, 22 | ) -> str: 23 | del instance, strategy # unused arguments 24 | return cmd 25 | 26 | def update( 27 | self, 28 | instance: Any, 29 | strategy: Any, 30 | output: str, 31 | result: "Result", 32 | ) -> None: 33 | del instance, strategy, output, result # unused arguments 34 | return 35 | 36 | def finished( 37 | self, 38 | instance: Any, 39 | strategy: Any, 40 | output: str, 41 | result: "Result", 42 | ) -> None: 43 | del instance, strategy, output, result # unused arguments 44 | return 45 | 46 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/plugin.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from ..object import SolverPyObj 4 | 5 | class Plugin(SolverPyObj): 6 | 7 | def __init__(self, **kwargs: Any): 8 | SolverPyObj.__init__(self, **kwargs) 9 | 10 | def register(self, solver: Any) -> None: 11 | del solver # unused argument 12 | raise NotImplementedError() 13 | 14 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/shell/__init__.py: -------------------------------------------------------------------------------- 1 | from .limits import Limits 2 | from .timeout import Timeout 3 | from .time import Time 4 | 5 | __all__ = ["Limits", "Timeout", "Time"] 6 | 7 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/shell/limits.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | 3 | from ..plugin import Plugin 4 | from ..decorator import Decorator 5 | from ..translator import Translator 6 | 7 | if TYPE_CHECKING: 8 | from ...solverpy import SolverPy 9 | from ....tools.typing import StrMaker, LimitBuilder 10 | 11 | 12 | def build(fun: "StrMaker", arg: Any) -> str: 13 | return fun % arg if isinstance(fun, str) else fun(arg) 14 | 15 | 16 | class Limits(Decorator, Translator): 17 | """ 18 | This is either a decorator or a translator based on the value 19 | of `cmdline`. 20 | """ 21 | 22 | def __init__( 23 | self, 24 | limit: str, 25 | builder: "LimitBuilder", 26 | cmdline: bool = True, 27 | inputfile: bool = False, 28 | ): 29 | Plugin.__init__( 30 | self, 31 | limit=limit, 32 | cmdline=cmdline, 33 | ) 34 | lims = {x[0]: x[1:] for x in limit.split("-") if x} 35 | assert "T" in lims 36 | self.timeout = int(lims["T"]) 37 | self.memory = float(lims["M"]) if "M" in lims else None 38 | try: 39 | lims = [ 40 | build(builder[x], y) for (x, y) in lims.items() if x in builder 41 | ] 42 | except Exception as e: 43 | print(e) 44 | raise Exception(f"solverpy: Invalid limit string: {limit}") 45 | 46 | delim = " " if cmdline else "" 47 | self.strat = delim.join(lims) 48 | self.cmdline = cmdline 49 | 50 | #self.args = " ".join(lims) 51 | self.limit = limit 52 | self._inputfile = inputfile 53 | 54 | def register(self, solver: "SolverPy") -> None: 55 | if self.cmdline: 56 | solver.decorators.append(self) 57 | else: 58 | solver.translators.append(self) 59 | 60 | def __str__(self) -> str: 61 | return self.limit 62 | 63 | def __lt__(self, other): 64 | if self.limit and not other.limit: 65 | return None 66 | if self.memory and not other.memory: 67 | return None 68 | if not self.memory: 69 | return (self.timeout < other.timeout) 70 | else: 71 | return (self.timeout < other.timeout) or (self.memory < other.memory) 72 | 73 | #def __le__(self, other): 74 | # return (self.key == other.key) or (self < other) 75 | 76 | def decorate( 77 | self, 78 | cmd: str, 79 | instance: Any, 80 | strategy: Any, 81 | ) -> str: 82 | del instance, strategy # unused arguments 83 | if self.cmdline: 84 | input = " /dev/stdin" if self._inputfile else "" 85 | return f"{cmd} {self.strategy}{input}" if self.strategy else cmd 86 | else: 87 | return cmd 88 | 89 | def translate( 90 | self, 91 | instance: Any, 92 | strategy: str, 93 | ) -> tuple[Any, str]: 94 | if not self.cmdline: 95 | return (instance, self.strategy + strategy) 96 | else: 97 | return (instance, strategy) 98 | 99 | @property 100 | def strategy(self) -> str: 101 | return self.strat 102 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/shell/memory.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from ..decorator import Decorator 4 | 5 | ULIMIT_CMD = "ulimit -Sv %d" 6 | 7 | 8 | class Memory(Decorator): 9 | 10 | def __init__(self, giga: float = 4): 11 | Decorator.__init__(self, giga=giga) 12 | self.prefix = ULIMIT_CMD % (int(giga * 1024 * 1024) + 1024) 13 | 14 | def decorate( 15 | self, 16 | cmd: str, 17 | instance: Any, 18 | strategy: Any, 19 | ) -> str: 20 | del instance, strategy 21 | return f"{self.prefix} && {cmd}" 22 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/shell/time.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | import re 3 | 4 | from ..decorator import Decorator 5 | from ....tools import patterns 6 | 7 | if TYPE_CHECKING: 8 | from ....tools.typing import Result 9 | 10 | # real 0.01 11 | # user 0.01 12 | # sys 0.00 13 | 14 | TIME_CMD = "/usr/bin/env time -p" 15 | 16 | TIME_PAT = re.compile(r"^(real|user|sys) ([0-9.]*)$", re.MULTILINE) 17 | 18 | TIME_TABLE = { 19 | "real": "realtime", 20 | "user": "usertime", 21 | "sys": "systime", 22 | } 23 | 24 | 25 | class Time(Decorator): 26 | 27 | def __init__(self, **kwargs): 28 | Decorator.__init__(self, **kwargs) 29 | self.prefix = TIME_CMD 30 | 31 | def decorate( 32 | self, 33 | cmd: str, 34 | instance: Any, 35 | strategy: Any, 36 | ) -> str: 37 | del instance, strategy # unused arguments 38 | return f"{self.prefix} {cmd}" 39 | 40 | def update( 41 | self, 42 | instance: Any, 43 | strategy: Any, 44 | output: str, 45 | result: "Result", 46 | ) -> None: 47 | del instance, strategy # unused arguments 48 | res = patterns.keyval(TIME_PAT, output, TIME_TABLE) 49 | res = patterns.mapval(res, float) 50 | result.update(res) 51 | if ("realtime" in res) and ("systime" in res): 52 | result["runtime"] = res["realtime"] - res["systime"] 53 | #result["realtime"] = res["realtime"] 54 | #result["runtime"] = res["usertime"] 55 | 56 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/shell/timeout.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | from ..decorator import Decorator 3 | 4 | if TYPE_CHECKING: 5 | from ...solverpy import SolverPy 6 | from ....tools.typing import Result 7 | 8 | TIMEOUT_CMD = "timeout --kill-after=15 --foreground %s" 9 | 10 | 11 | class Timeout(Decorator): 12 | 13 | def __init__(self, timeout: int): 14 | Decorator.__init__(self, timeout=timeout) 15 | self.timeout = timeout 16 | self.prefix = TIMEOUT_CMD % timeout 17 | 18 | def register(self, solver: "SolverPy") -> None: 19 | self.solver = solver 20 | solver.decorators.insert(0, self) 21 | 22 | def decorate( 23 | self, 24 | cmd: str, 25 | instance: Any, 26 | strategy: Any, 27 | ) -> str: 28 | del instance, strategy # unused arguments 29 | return f"{self.prefix} {cmd}" 30 | 31 | def update( 32 | self, 33 | instance: Any, 34 | strategy: Any, 35 | output: str, 36 | result: "Result", 37 | ) -> None: 38 | del instance, strategy, output # unused arguments 39 | # see man(timeout) for timeout exit codes 40 | if self.solver._exitcode in [124, 137]: 41 | result["status"] = "TIMEOUT" 42 | 43 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/status/__init__.py: -------------------------------------------------------------------------------- 1 | from .smt import Smt 2 | from .tptp import Tptp 3 | from .limiter import Limiter 4 | 5 | __all__ = ["Smt", "Tptp", "Limiter"] 6 | 7 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/status/limiter.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING, Any 2 | 3 | from ..decorator import Decorator 4 | 5 | if TYPE_CHECKING: 6 | from ...solverpy import SolverPy 7 | from ....tools.typing import Result 8 | 9 | 10 | class Limiter(Decorator): 11 | 12 | def __init__(self, **kwargs): 13 | Decorator.__init__(self, **kwargs) 14 | 15 | def register(self, solver: "SolverPy") -> None: 16 | super().register(solver) 17 | self.timeouts = solver.timeouts 18 | self.timeout = solver._limits.timeout 19 | self.limit = solver._limits.limit 20 | 21 | def update( 22 | self, 23 | instance: Any, 24 | strategy: Any, 25 | output: Any, 26 | result: "Result", 27 | ) -> None: 28 | del instance, strategy, output # unused arguments 29 | if (not result) or ("status" not in result) or ("runtime" not in result): 30 | return 31 | if (result["status"] in self.timeouts) or \ 32 | (result["runtime"] > self.timeout): 33 | result["runtime"] = self.timeout 34 | result["limit"] = self.limit 35 | 36 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/status/smt.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | import re 3 | 4 | from ..decorator import Decorator 5 | from ....tools import patterns 6 | 7 | if TYPE_CHECKING: 8 | from ....tools.typing import Result 9 | from ...solverpy import SolverPy 10 | 11 | SMT_STATUS = re.compile(r"^(sat|unsat|unknown|timeout)$", re.MULTILINE) 12 | 13 | SMT_OK = frozenset([ 14 | 'sat', 15 | 'unsat', 16 | ]) 17 | 18 | SMT_FAILED = frozenset([ 19 | 'unknown', 20 | ]) 21 | 22 | SMT_TIMEOUT = frozenset([ 23 | 'timeout', 24 | 'memout', 25 | 'TIMEOUT', # simulated timeout 26 | ]) 27 | 28 | SMT_INCOMPLETE = frozenset([ 29 | "sat", 30 | ]) 31 | 32 | SMT_ALL = SMT_OK | SMT_FAILED | SMT_TIMEOUT 33 | 34 | SMT_INC_OK = SMT_OK - SMT_INCOMPLETE 35 | 36 | class Smt(Decorator): 37 | 38 | def __init__(self, complete=True, **kwargs): 39 | Decorator.__init__(self, complete=complete, **kwargs) 40 | self._complete = complete 41 | 42 | def register(self, solver: "SolverPy"): 43 | super().register(solver) 44 | solver._success |= SMT_OK if self._complete else SMT_INC_OK 45 | solver._timeouts |= SMT_TIMEOUT 46 | solver._statuses |= SMT_ALL 47 | 48 | def decorate( 49 | self, 50 | cmd: str, 51 | instance: Any, 52 | strategy: Any, 53 | ) -> str: 54 | del instance, strategy # unused arguments 55 | return cmd 56 | 57 | def update( 58 | self, 59 | instance: Any, 60 | strategy: Any, 61 | output: str, 62 | result: "Result", 63 | ) -> None: 64 | del instance, strategy # unused arguments 65 | status = patterns.single(SMT_STATUS, output, "") 66 | if status: 67 | result["status"] = status 68 | elif "status" not in result: 69 | result["status"] = "error" 70 | 71 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/status/tptp.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | import re 3 | 4 | from ..decorator import Decorator 5 | from ....tools import patterns 6 | 7 | if TYPE_CHECKING: 8 | from ....tools.typing import Result 9 | from ...solverpy import SolverPy 10 | 11 | TPTP_STATUS = re.compile(r"^[#%] SZS status (\S*)", re.MULTILINE) 12 | 13 | TPTP_OK = frozenset([ 14 | 'Satisfiable', 15 | 'Unsatisfiable', 16 | 'Theorem', 17 | 'CounterSatisfiable', 18 | 'ContradictoryAxioms', 19 | ]) 20 | 21 | TPTP_FAILED = frozenset([ 22 | 'GaveUp', 23 | ]) 24 | 25 | TPTP_TIMEOUT = frozenset([ 26 | 'ResourceOut', 27 | 'Timeout', 28 | "TIMEOUT", # simulated timeout 29 | ]) 30 | 31 | TPTP_INCOMPLETE = frozenset([ 32 | 'Satisfiable', 33 | 'CounterSatisfiable', 34 | ]) 35 | 36 | TPTP_ALL = TPTP_OK | TPTP_FAILED | TPTP_TIMEOUT 37 | 38 | TPTP_INC_OK = TPTP_OK - TPTP_INCOMPLETE 39 | 40 | class Tptp(Decorator): 41 | 42 | def __init__(self, complete=True, **kwargs): 43 | Decorator.__init__(self, complete=complete, **kwargs) 44 | self._complete = complete 45 | 46 | def register(self, solver: "SolverPy"): 47 | super().register(solver) 48 | solver._success |= TPTP_OK if self._complete else TPTP_INC_OK 49 | solver._timeouts |= TPTP_TIMEOUT 50 | solver._statuses |= TPTP_ALL 51 | 52 | def decorate( 53 | self, 54 | cmd: str, 55 | instance: Any, 56 | strategy: Any, 57 | ) -> str: 58 | del instance, strategy 59 | return cmd 60 | 61 | def update( 62 | self, 63 | instance: Any, 64 | strategy: Any, 65 | output: str, 66 | result: "Result", 67 | ) -> None: 68 | del instance, strategy 69 | status = patterns.single(TPTP_STATUS, output, "") 70 | if status: 71 | result["status"] = status 72 | elif "status" not in result: 73 | result["status"] = "ERROR" 74 | 75 | 76 | 77 | -------------------------------------------------------------------------------- /src/solverpy/solver/plugins/translator.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | from .plugin import Plugin 3 | 4 | if TYPE_CHECKING: 5 | from ..solverpy import SolverPy 6 | 7 | class Translator(Plugin): 8 | 9 | def __init__(self, **kwargs): 10 | Plugin.__init__(self, **kwargs) 11 | 12 | def register(self, solver: "SolverPy") -> None: 13 | solver.translators.append(self) 14 | 15 | def translate( 16 | self, 17 | instance: Any, 18 | strategy: Any 19 | ) -> tuple[Any, Any]: 20 | return (instance, strategy) 21 | 22 | -------------------------------------------------------------------------------- /src/solverpy/solver/pluginsolver.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | import logging 3 | 4 | from .solver import Solver 5 | 6 | if TYPE_CHECKING: 7 | from .plugins.plugin import Plugin 8 | from .plugins.decorator import Decorator 9 | from .plugins.translator import Translator 10 | from ..tools.typing import Result 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | class PluginSolver(Solver): 16 | 17 | def __init__(self, plugins: list["Plugin"] = [], **kwargs: Any): 18 | Solver.__init__(self, **kwargs) 19 | self.decorators: list["Decorator"] = [] 20 | self.translators: list["Translator"] = [] 21 | self.init(plugins) 22 | 23 | def represent(self): 24 | return dict( 25 | cls=self.name, 26 | decorators=[repr(x) for x in self.decorators], 27 | translators=[repr(x) for x in self.translators], 28 | ) 29 | 30 | def solve(self, instance: Any, strategy: Any) -> "Result": 31 | """ 32 | Run the solver with the strategy on the instatance. Update the 33 | solver result and announce the final version. Return the updated 34 | result. 35 | """ 36 | result = Solver.solve(self, instance, strategy) 37 | output = self._output 38 | self.update(instance, strategy, output, result) 39 | if not self.valid(result): 40 | lines = output.split("\n") 41 | if len(lines) > 3: 42 | msg = f"{lines[2]}\n{lines[3]}" # command and first output line 43 | else: 44 | msg = output 45 | logger.debug( 46 | f"failed solver run: {self}:{strategy} @ {instance}\nresult: {result}\n{msg}" 47 | ) 48 | return result 49 | 50 | def init(self, plugins: list["Plugin"]) -> None: 51 | """Plugins initialization.""" 52 | for plugin in plugins: 53 | plugin.register(self) 54 | 55 | def decorate( 56 | self, 57 | cmd: str, 58 | instance: Any, 59 | strategy: Any, 60 | ) -> str: 61 | """Decorate the command for the solver.""" 62 | for plugin in self.decorators: 63 | cmd = plugin.decorate(cmd, instance, strategy) 64 | return cmd 65 | 66 | def update( 67 | self, 68 | instance: Any, 69 | strategy: Any, 70 | output: str, 71 | result: "Result", 72 | ) -> None: 73 | """Update the solver result and announce the final version.""" 74 | for plugin in self.decorators: 75 | plugin.update(instance, strategy, output, result) 76 | for plugin in self.decorators: 77 | plugin.finished(instance, strategy, output, result) 78 | 79 | def translate( 80 | self, 81 | instance: Any, 82 | strategy: Any, 83 | ) -> tuple[Any, Any]: 84 | """Translate the instance and strategy for the solver.""" 85 | for plugin in self.translators: 86 | (instance, strategy) = plugin.translate(instance, strategy) 87 | return (instance, strategy) 88 | -------------------------------------------------------------------------------- /src/solverpy/solver/reloader.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | 3 | from .solverpy import SolverPy 4 | from .plugins.db.outputs import Outputs 5 | 6 | if TYPE_CHECKING: 7 | from .plugins.plugin import Plugin 8 | from ..tools.typing import Result 9 | 10 | 11 | class Reloader(SolverPy): 12 | 13 | def __init__( 14 | self, 15 | solver: SolverPy, 16 | plugins: list["Plugin"] = [], 17 | ): 18 | self.solver = solver 19 | SolverPy.__init__( 20 | self, 21 | solver._limits, 22 | plugins, 23 | ) 24 | self.outputs = Outputs() 25 | self.outputs.register(self.solver) 26 | 27 | def run(self, instance: Any, strategy: Any) -> str: 28 | f = self.outputs.path(instance, strategy) 29 | with open(f) as fr: 30 | return fr.read() 31 | 32 | def process(self, output: str) -> "Result": 33 | return self.solver.process(output) 34 | 35 | def valid(self, result: "Result") -> bool: 36 | return self.solver.valid(result) 37 | 38 | def decorate( 39 | self, 40 | cmd: str, 41 | instance: Any, 42 | strategy: Any, 43 | ) -> str: 44 | return self.solver.decorate(cmd, instance, strategy) 45 | 46 | def update( 47 | self, 48 | instance: Any, 49 | strategy: Any, 50 | output: str, 51 | result: "Result", 52 | ) -> None: 53 | self.solver.update(instance, strategy, output, result) 54 | 55 | def translate( 56 | self, 57 | instance: Any, 58 | strategy: Any, 59 | ) -> tuple[Any, Any]: 60 | return self.solver.translate(instance, strategy) 61 | 62 | @property 63 | def name(self) -> str: 64 | return f"{super().name}({self.solver.name})" 65 | 66 | @property 67 | def success(self) -> frozenset[str]: 68 | return self.solver.success 69 | 70 | @property 71 | def timeouts(self) -> frozenset[str]: 72 | return self.solver.timeouts 73 | 74 | -------------------------------------------------------------------------------- /src/solverpy/solver/shellsolver.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | import os 3 | import subprocess 4 | 5 | from .solverpy import SolverPy 6 | from .plugins.shell.limits import Limits 7 | from .plugins.shell.timeout import Timeout 8 | from .plugins.shell.memory import Memory 9 | from ..benchmark.path import sids 10 | 11 | if TYPE_CHECKING: 12 | from .plugins.plugin import Plugin 13 | from ..tools.typing import LimitBuilder 14 | 15 | 16 | class ShellSolver(SolverPy): 17 | 18 | def __init__( 19 | self, 20 | cmd: str, 21 | limit: str, 22 | builder: "LimitBuilder" = {}, 23 | plugins: list["Plugin"] = [], 24 | wait: (int | None) = None, 25 | unspace: bool = True, 26 | ): 27 | self._unspace = unspace 28 | limits = Limits(limit, builder) 29 | new: list["Plugin"] = [limits] 30 | if wait is not None: 31 | new.append(Timeout(limits.timeout + wait)) 32 | if limits.memory: 33 | new.append(Memory(limits.memory)) 34 | SolverPy.__init__( 35 | self, 36 | limits=limits, 37 | plugins=plugins + new, 38 | ) 39 | self._cmd = cmd 40 | 41 | @property 42 | def name(self) -> str: 43 | return f"{super().name}:{self._limits.limit}" 44 | 45 | def run(self, instance: Any, strategy: Any) -> str: 46 | cmd = self.command(instance, strategy) 47 | env0 = dict(os.environ) 48 | env0["OMP_NUM_THREADS"] = "1" 49 | #env0["CUDA_VISIBLE_DEVICES"] = "-1" 50 | if self._unspace: 51 | cmd = sids.unspace(cmd) 52 | self._exitcode = 0 53 | try: 54 | output = subprocess.check_output( 55 | cmd, 56 | shell=True, 57 | stderr=subprocess.STDOUT, 58 | env=env0, 59 | ) 60 | except subprocess.CalledProcessError as e: 61 | output = e.output 62 | self._exitcode = e.returncode 63 | return f"### INSTANCE {instance}\n### STRATEGY {strategy}\n### COMMAND: {cmd}\n" + output.decode() 64 | 65 | def command(self, instance: Any, strategy: Any) -> str: 66 | cmd = self.decorate(self._cmd, instance, strategy) 67 | (instance, strategy) = self.translate(instance, strategy) 68 | return f"{cmd} {strategy} {instance}" 69 | 70 | -------------------------------------------------------------------------------- /src/solverpy/solver/smt/__init__.py: -------------------------------------------------------------------------------- 1 | from .bitwuzla import Bitwuzla 2 | from .cvc5 import Cvc5 3 | from .z3 import Z3 4 | 5 | __all__ = ["Bitwuzla", "Cvc5", "Z3"] 6 | 7 | -------------------------------------------------------------------------------- /src/solverpy/solver/smt/bitwuzla.py: -------------------------------------------------------------------------------- 1 | from typing import Pattern, TYPE_CHECKING 2 | import re 3 | 4 | from ..shellsolver import ShellSolver 5 | from ...tools import patterns, human 6 | from ..plugins.status.smt import Smt 7 | from ..plugins.shell.time import Time 8 | 9 | if TYPE_CHECKING: 10 | from ..plugins.plugin import Plugin 11 | from ...tools.typing import LimitBuilder, Result 12 | 13 | BWZ_BINARY = "bitwuzla" 14 | 15 | BWZ_STATIC = "-v" 16 | 17 | BWZ_BUILDER: "LimitBuilder" = { 18 | "T": lambda n: f"-t={int(n)*1000}", 19 | } 20 | 21 | BWZ_TABLE = { 22 | "variable substitutions": "SubstVar", 23 | "uninterpreted function substitutions": "SubstUf", 24 | "embedded constraint substitutions": "SubstEc", 25 | "AIG vectors": "AigVec", 26 | "AIG ANDs": "AigAnd", 27 | "AIG variables": "AigVar", 28 | "CNF variables": "CnfVar", 29 | "CNF clauses": "CnfCls", 30 | "CNF literals": "CnfLit", 31 | "cached (add)": "RwcAdd", 32 | "cached (get)": "RwcGet", 33 | "udpated": "RwcUpd", 34 | } 35 | 36 | BWZ_KEYS = "|".join(BWZ_TABLE.keys()) \ 37 | .replace("(", "[(]").replace(")", "[)]") 38 | 39 | BWZ_PAT: Pattern = re.compile( 40 | r"^\[bitwuzla>core\]\s*(\d+) (%s)" % BWZ_KEYS, 41 | flags=re.MULTILINE, 42 | ) 43 | 44 | 45 | class Bitwuzla(ShellSolver): 46 | 47 | def __init__( 48 | self, 49 | limit: str, 50 | binary: str = BWZ_BINARY, 51 | static: str = BWZ_STATIC, 52 | plugins: list["Plugin"] = [], 53 | complete: bool = True, 54 | ): 55 | cmd = f"{binary} {static}" 56 | plugins = plugins + [ 57 | Time(), 58 | Smt(complete=complete), 59 | ] 60 | ShellSolver.__init__( 61 | self, 62 | cmd, 63 | limit, 64 | BWZ_BUILDER, 65 | plugins, 66 | wait=1, 67 | ) 68 | 69 | def process(self, output: str) -> "Result": 70 | result = patterns.valkey(BWZ_PAT, output, BWZ_TABLE) 71 | result = patterns.mapval(result, human.numeric) 72 | return result 73 | 74 | -------------------------------------------------------------------------------- /src/solverpy/solver/smt/cvc5.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | import re 3 | 4 | from ..shellsolver import ShellSolver 5 | from ..plugins.status.smt import Smt 6 | from ..plugins.shell.time import Time 7 | from ...tools import patterns, human 8 | 9 | if TYPE_CHECKING: 10 | from typing import Pattern 11 | from ..plugins.plugin import Plugin 12 | from ...tools.typing import LimitBuilder, Result 13 | 14 | CVC5_BINARY = "cvc5" 15 | 16 | CVC5_STATIC: str = "-Lsmt2 --stats --stats-internal" 17 | 18 | CVC5_BUILDER: "LimitBuilder" = { 19 | "T": lambda x: "--tlimit=%s" % (1000 * int(x)), 20 | "R": lambda x: "--rlimit=%s" % x 21 | } 22 | 23 | CVC5_KEYS = [ 24 | "driver::totalTime", 25 | "global::totalTime", 26 | "resource::resourceUnitsUsed", 27 | "resource::steps::resource", 28 | "Instantiate::[^ ]*", 29 | "QuantifiersEngine::[^ ]*", 30 | "SharedTermsDatabase::termsCount", 31 | "sat::conflicts", 32 | "sat::decisions", 33 | "sat::clauses_literals", 34 | "sat::propagations", 35 | ] 36 | 37 | # TODO: extend with: 38 | # GNU MP: Cannot allocate memory 39 | # (error "std::bad_alloc") 40 | # terminate called after throwing an instance of 'std::bad_alloc' 41 | # [LightGBM] [Warning] std::bad_alloc 42 | 43 | CVC5_TIMEOUT = re.compile(r"cvc5 interrupted by (timeout)") 44 | 45 | 46 | class Cvc5(ShellSolver): 47 | 48 | def __init__( 49 | self, 50 | limit: str, 51 | binary: str = CVC5_BINARY, 52 | static: str = CVC5_STATIC, 53 | plugins: list["Plugin"] = [], 54 | keys: list[str] = CVC5_KEYS, 55 | complete: bool = True, 56 | ): 57 | cmd = f"{binary} {static}" 58 | plugins = plugins + [ 59 | Time(), 60 | Smt(complete=complete), 61 | ] 62 | ShellSolver.__init__( 63 | self, 64 | cmd, 65 | limit, 66 | CVC5_BUILDER, 67 | plugins, 68 | wait=1, 69 | ) 70 | self.pattern: "Pattern" = re.compile( 71 | r"^(%s) = (.*)$" % "|".join(keys), 72 | flags=re.MULTILINE, 73 | ) 74 | 75 | def process(self, output: str) -> "Result": 76 | 77 | def parseval(val: str) -> Any: # value or dict of values 78 | if val.startswith("{") and val.endswith("}"): 79 | val0 = val.strip(" {}") 80 | val0 = val0.split(",") 81 | val0 = [x.split(":") for x in val0] 82 | return {x.strip(): human.numeric(y.strip()) for (x, y) in val0} 83 | return human.numeric(val) 84 | 85 | result = patterns.keyval(self.pattern, output) 86 | result = patterns.mapval(result, parseval) 87 | timeouted = patterns.single(CVC5_TIMEOUT, output, "") 88 | if timeouted: 89 | result["status"] = timeouted # timeouted == "timeout" 90 | return result 91 | -------------------------------------------------------------------------------- /src/solverpy/solver/smt/z3.py: -------------------------------------------------------------------------------- 1 | from typing import Pattern, TYPE_CHECKING 2 | import re 3 | 4 | from ..stdinsolver import StdinSolver 5 | from ..plugins.status.smt import Smt 6 | from ..plugins.shell.time import Time 7 | from ...tools import patterns, human 8 | 9 | if TYPE_CHECKING: 10 | from ..plugins.plugin import Plugin 11 | from ...tools.typing import LimitBuilder, Result 12 | 13 | Z3_BINARY = "z3" 14 | 15 | Z3_STATIC: str = "-smt2 -st" 16 | 17 | Z3_BUILDER: "LimitBuilder" = { 18 | "T": "-T:%s", 19 | "M": lambda giga: f"-memory:{int(1024*float(giga))}", 20 | } 21 | 22 | Z3_PAT: Pattern = re.compile( 23 | r"^.:([a-z-]*)\s*([0-9.]*)", 24 | flags=re.MULTILINE, 25 | ) 26 | 27 | Z3_MEMOUT: Pattern = re.compile( 28 | r'error "out of memory"', 29 | flags=re.MULTILINE, 30 | ) 31 | 32 | 33 | class Z3(StdinSolver): 34 | 35 | def __init__( 36 | self, 37 | limit: str, 38 | binary: str = Z3_BINARY, 39 | static: str = "", 40 | plugins: list["Plugin"] = [], 41 | complete: bool = True, 42 | ): 43 | cmd = f"{binary} {Z3_STATIC}" 44 | plugins = plugins + [ 45 | Time(), 46 | Smt(complete=complete), 47 | ] 48 | StdinSolver.__init__( 49 | self, 50 | cmd, 51 | limit, 52 | Z3_BUILDER, 53 | plugins, 54 | 1, 55 | static, 56 | True, 57 | True, 58 | ) 59 | 60 | def process(self, output: str) -> "Result": 61 | result = patterns.keyval(Z3_PAT, output) 62 | result = patterns.mapval(result, human.numeric) 63 | if re.search(Z3_MEMOUT, output): 64 | result["status"] = "memout" 65 | return result 66 | 67 | -------------------------------------------------------------------------------- /src/solverpy/solver/solver.py: -------------------------------------------------------------------------------- 1 | """ 2 | # Abstract solver interface 3 | 4 | Defines a basic interface for all solvers. 5 | 6 | ## Implemented methods 7 | 8 | + The main method [solve][solverpy.solver.solver.Solver.solve] solves the 9 | problem and returns a processed result. It calls `run` and `process` in 10 | sequence which need to implemented by subclasses. 11 | 12 | + Result queries 13 | [valid][solverpy.solver.solver.Solver.valid], 14 | [solved][solverpy.solver.solver.Solver.solved] to recognize a valid and 15 | solved results. 16 | 17 | ## Abstract methods 18 | 19 | + running the solver: [run][solverpy.solver.solver.Solver.run] 20 | + processing the output: [process][solverpy.solver.solver.Solver.process] 21 | + status sets: 22 | [success][solverpy.solver.solver.Solver.success], 23 | [timeouts][solverpy.solver.solver.Solver.timeouts], 24 | [statuses][solverpy.solver.solver.Solver.statuses] 25 | """ 26 | 27 | from typing import Any, TYPE_CHECKING 28 | from .object import SolverPyObj 29 | 30 | if TYPE_CHECKING: 31 | from ..tools.typing import Result 32 | 33 | 34 | class Solver(SolverPyObj): 35 | """ 36 | Abstract class for solvers. 37 | """ 38 | 39 | def __init__(self, **kwargs: Any): 40 | SolverPyObj.__init__(self, **kwargs) 41 | 42 | def __str__(self) -> str: 43 | return self.name 44 | 45 | def solve(self, instance: Any, strategy: Any) -> Any: 46 | """ 47 | Run the solver with the strategy on the instatance. Process the output 48 | and create the result. 49 | 50 | Args: 51 | instance: solver problem instance (filename, or a bid-problem 52 | pair, or custom). 53 | strategy: solver strategy (filename, sid, or custom). 54 | 55 | Returns: the result 56 | """ 57 | output = self.run(instance, strategy) 58 | self._output = output 59 | result = self.process(output) 60 | return result 61 | 62 | def valid(self, result: "Result") -> bool: 63 | """ 64 | A valid status contains at least keys `status` and `runtime`. The status 65 | must be a valid status. 66 | 67 | Args: 68 | result: the result 69 | """ 70 | return bool(result) and \ 71 | ("status" in result) and \ 72 | ("runtime" in result) and \ 73 | (result["status"] in self.statuses) 74 | 75 | def solved(self, result: "Result") -> bool: 76 | """ 77 | The result is solved if the status is in the success set. 78 | 79 | Args: 80 | result: the result 81 | """ 82 | return bool(result) and \ 83 | ("status" in result) and \ 84 | (result["status"] in self.success) 85 | 86 | def run(self, instance: Any, strategy: Any) -> str: 87 | """ 88 | Run the solver with the strategy on the instatnce. 89 | 90 | Args: 91 | instance: solver problem instance 92 | strategy: solver strategy 93 | 94 | Returns: raw solver output 95 | 96 | Raises: 97 | NotImplementedError: abstract method 98 | """ 99 | del instance, strategy # unused arguments 100 | raise NotImplementedError() 101 | 102 | def process(self, output: str) -> "Result": 103 | """ 104 | Process the solver output and create the result. 105 | 106 | Args: 107 | output: raw solver output 108 | 109 | Returns: processed result dictionary 110 | 111 | 112 | Raises: 113 | NotImplementedError: abstract method 114 | """ 115 | del output # unused argument 116 | raise NotImplementedError() 117 | 118 | @property 119 | def name(self) -> str: 120 | """ 121 | Solver name. The default name is the class name. 122 | """ 123 | return self.__class__.__name__ 124 | 125 | @property 126 | def success(self) -> frozenset[str]: 127 | """ 128 | The set of successful statuses. 129 | 130 | Raises: 131 | NotImplementedError: abstract property 132 | """ 133 | raise NotImplementedError() 134 | 135 | @property 136 | def timeouts(self) -> frozenset[str]: 137 | """ 138 | The set of timeout statuses. 139 | 140 | Raises: 141 | NotImplementedError: abstract property 142 | """ 143 | raise NotImplementedError() 144 | 145 | @property 146 | def statuses(self) -> frozenset[str]: 147 | """ 148 | The set of all valid statuses. 149 | 150 | Raises: 151 | NotImplementedError: abstract property 152 | """ 153 | raise NotImplementedError() 154 | 155 | -------------------------------------------------------------------------------- /src/solverpy/solver/solverpy.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | 3 | from .pluginsolver import PluginSolver 4 | from .plugins.shell.limits import Limits 5 | from .plugins.status.limiter import Limiter 6 | 7 | if TYPE_CHECKING: 8 | from .plugins.plugin import Plugin 9 | from ..tools.typing import Result 10 | 11 | 12 | class SolverPy(PluginSolver): 13 | 14 | def __init__( 15 | self, 16 | limits: Limits, 17 | plugins: list["Plugin"] = [], 18 | **kwargs: Any, 19 | ): 20 | assert limits.limit.startswith("T") 21 | self._limits: Limits = limits 22 | self._exitcode: int = -1 23 | self._timeouts = frozenset() 24 | self._success = frozenset() 25 | self._statuses = frozenset() 26 | plugins = plugins + [ 27 | Limiter(), 28 | ] 29 | PluginSolver.__init__(self, plugins=plugins, **kwargs) 30 | 31 | def determine(self, result: "Result") -> "Result | None": 32 | "Simulate run from the past result." 33 | if result["status"] in self.timeouts: 34 | #if result["status"] not in self.success: # we might want this? 35 | oldlimits = Limits(result["limit"], {}) 36 | # the cached result is timeout 37 | if oldlimits < self._limits: 38 | #if result["runtime"] < self.timeout: 39 | # recompute since we have more time or/and space 40 | return None 41 | elif result["status"] in self.success: 42 | # the cached result is solved 43 | if result["runtime"] > self._limits.timeout: 44 | #if result["runtime"] > self.timeout: 45 | # simulated timeout 46 | return dict( 47 | result, 48 | status="TIMEOUT", 49 | runtime=self._limits.timeout, 50 | ) 51 | else: 52 | # recompute unknown results (GaveUp, unknown) 53 | # TODO: do we want to always recompute? 54 | return None 55 | # the result is applicable without changes 56 | return result 57 | 58 | @property 59 | def timeouts(self) -> frozenset[str]: 60 | return self._timeouts 61 | 62 | @property 63 | def success(self) -> frozenset[str]: 64 | return self._success 65 | 66 | @property 67 | def statuses(self) -> frozenset[str]: 68 | return self._statuses 69 | -------------------------------------------------------------------------------- /src/solverpy/solver/stdinsolver.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | import os 3 | import subprocess 4 | 5 | from .solverpy import SolverPy 6 | from .plugins.shell.limits import Limits 7 | from .plugins.shell.timeout import Timeout 8 | from .plugins.shell.memory import Memory 9 | 10 | if TYPE_CHECKING: 11 | from .plugins.plugin import Plugin 12 | from ..tools.typing import LimitBuilder 13 | 14 | 15 | class StdinSolver(SolverPy): 16 | 17 | def __init__( 18 | self, 19 | cmd: str, 20 | limit: str, 21 | builder: "LimitBuilder" = {}, 22 | plugins: list["Plugin"] = [], 23 | wait: (int | None) = None, 24 | static: str = "", # TODO: rename to `prefix` 25 | cmdline: bool = False, # set limits using command line arguments? 26 | inputfile: bool = False, 27 | ): 28 | limits = Limits(limit, builder, cmdline=cmdline, inputfile=inputfile) 29 | new: list["Plugin"] = [limits] 30 | if wait is not None: 31 | new.append(Timeout(limits.timeout + wait)) 32 | if limits.memory: 33 | new.append(Memory(limits.memory)) 34 | SolverPy.__init__( 35 | self, 36 | limits=limits, 37 | plugins=plugins + new, 38 | ) 39 | self._static = static # TODO: rename to `prefix` 40 | self._cmd = cmd 41 | 42 | @property 43 | def name(self) -> str: 44 | return f"{super().name}:{self._limits.limit}" 45 | 46 | def run(self, instance: Any, strategy: Any) -> str: 47 | inputstr = self.input(instance, strategy) 48 | env0 = dict(os.environ) 49 | env0["OMP_NUM_THREADS"] = "1" 50 | #env0["CUDA_VISIBLE_DEVICES"] = "-1" 51 | cmd = self.decorate(self._cmd, instance, strategy) 52 | try: 53 | output = subprocess.check_output( 54 | cmd, 55 | input=inputstr, 56 | shell=True, 57 | stderr=subprocess.STDOUT, 58 | env=env0, 59 | ) 60 | self._exitcode = 0 61 | except subprocess.CalledProcessError as e: 62 | output = e.output 63 | self._exitcode = e.returncode 64 | return f"### INSTANCE {instance}\n### STRATEGY {strategy}\n### COMMAND: {cmd}\n" + output.decode() 65 | 66 | def input(self, instance: Any, strategy: Any) -> bytes: 67 | (instance, strategy) = self.translate(instance, strategy) 68 | inputstr = self._static.encode() 69 | inputstr += strategy.encode() 70 | inputstr += b"\n" 71 | inputstr += open(instance, "rb").read() 72 | #open(f"/home/yan/tmp.smt2", "wb").write(inputstr) 73 | return inputstr 74 | 75 | -------------------------------------------------------------------------------- /src/solverpy/task/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbboyan/solverpy/caa8da555ff766a50e1b0b0a01f2747df65d8bdb/src/solverpy/task/__init__.py -------------------------------------------------------------------------------- /src/solverpy/task/bar.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | import datetime 4 | from tqdm import tqdm 5 | 6 | RED = '\033[91m' 7 | BLUE = '\033[94m' 8 | PURPLE = '\033[95m' 9 | END = '\033[0m' 10 | 11 | #BAR_DEFAULT = "{desc}: {percentage:6.2f}%|{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}]{postfix}" 12 | #BAR_RUNNING = "{desc}: {percentage:6.2f}%|{bar}| {n_fmt}/{total_fmt} {errors} [{elapsed}<{remaining}]{postfix}" 13 | #BAR_SOLVING = "{desc}: {percentage:6.2f}%|{bar}| {n_fmt}/{total_fmt} {solved}/{unsolved}/{errors}/{solved_eta} [{elapsed}<{remaining}]{postfix}" 14 | BAR_DEFAULT = "{desc}: {percentage:6.2f}%|{bar}| {n_fmt}/{total_fmt} [{elapsed}<{speta}]{postfix}" 15 | BAR_BUILDER = "{desc}: {percentage:6.2f}%|{bar}| {n_fmt}/{total_fmt} {loss} [{elapsed}<{remaining}]{postfix}" 16 | BAR_RUNNING = "{desc}: {percentage:6.2f}%|{bar}| {n_fmt}/{total_fmt} {errors} [{elapsed}<{speta}]{postfix}" 17 | BAR_SOLVING = "{desc}: {percentage:6.2f}%|{bar}| {n_fmt}/{total_fmt} {solved}/{unsolved}/{errors}/{solved_eta} [{elapsed}<{speta}]{postfix}" 18 | 19 | 20 | class DefaultBar(tqdm): 21 | 22 | def __init__( 23 | self, 24 | total: int, 25 | desc: str, 26 | ascii: str = "┈─═━", 27 | colour: str = "green", 28 | bar_format: str = BAR_DEFAULT, 29 | *args: Any, 30 | **kwargs: Any, 31 | ): 32 | tqdm.__init__( 33 | self, 34 | total=total, 35 | desc=desc, 36 | bar_format=bar_format, 37 | ascii=ascii, 38 | colour=colour, 39 | *args, 40 | **kwargs, 41 | ) 42 | 43 | @property 44 | def format_dict(self) -> Any: 45 | d = super().format_dict 46 | speta = d["elapsed"] * (d["total"] or 0) / max(d["n"], 1) 47 | speta -= d["elapsed"] 48 | speta = str(datetime.timedelta(seconds=int(speta))) 49 | if speta.startswith("0:"): 50 | speta = speta[2:] 51 | d.update(speta=speta) 52 | return d 53 | 54 | def status(self, status, n=1) -> None: 55 | del status # unused argument 56 | self.update(n) 57 | 58 | 59 | class BuilderBar(tqdm): 60 | 61 | def __init__( 62 | self, 63 | total: int, 64 | desc: str, 65 | ascii: str = "┈─═━", 66 | colour: str = "green", 67 | bar_format: str = BAR_BUILDER, 68 | *args: Any, 69 | **kwargs: Any, 70 | ): 71 | self._loss = [] 72 | tqdm.__init__( 73 | self, 74 | total=total, 75 | desc=desc, 76 | bar_format=bar_format, 77 | ascii=ascii, 78 | colour=colour, 79 | *args, 80 | **kwargs, 81 | ) 82 | 83 | @property 84 | def format_dict(self) -> Any: 85 | d = super().format_dict 86 | d.update(loss=self._loss) 87 | return d 88 | 89 | def done( 90 | self, 91 | loss: list[float], 92 | n: int = 1, 93 | ) -> None: 94 | self._loss = "/".join(f"{x:.4f}" for x in loss) 95 | self.update(n) 96 | 97 | 98 | class RunningBar(DefaultBar): 99 | 100 | def __init__( 101 | self, 102 | total: int, 103 | desc: str, 104 | bar_format: str = BAR_RUNNING, 105 | *args: Any, 106 | **kwargs: Any, 107 | ): 108 | self._errors = 0 109 | DefaultBar.__init__( 110 | self, 111 | total, 112 | desc, 113 | bar_format=bar_format, 114 | *args, 115 | **kwargs, 116 | ) 117 | 118 | @property 119 | def format_dict(self) -> Any: 120 | d = super().format_dict 121 | asc = f"!{self._errors}" 122 | if self._errors: 123 | asc = f"{RED}{asc}{END}" 124 | d.update(errors=asc) 125 | return d 126 | 127 | def status(self, status, n=1) -> None: 128 | if status is None: 129 | self._errors += n 130 | self.update(n) 131 | 132 | 133 | class SolvingBar(RunningBar): 134 | 135 | def __init__( 136 | self, 137 | total: int, 138 | desc: str, 139 | bar_format: str = BAR_SOLVING, 140 | ascii: str = "┈─═━", 141 | colour: str = "blue", 142 | *args: Any, 143 | **kwargs: Any, 144 | ): 145 | self._solved = 0 146 | self._unsolved = 0 147 | RunningBar.__init__( 148 | self, 149 | total, 150 | desc, 151 | bar_format=bar_format, 152 | ascii=ascii, 153 | colour=colour, 154 | *args, 155 | **kwargs, 156 | ) 157 | 158 | @property 159 | def format_dict(self) -> Any: 160 | d = super().format_dict 161 | #total_time = d["elapsed"] * (d["total"] or 0) / max(d["n"], 1) 162 | solved_eta = int(self._solved * (d["total"] / max(d["n"], 1))) 163 | d.update( 164 | solved=f"{PURPLE}+{self._solved}{END}", 165 | unsolved=f"{BLUE}{self._unsolved}{END}", 166 | solved_eta=f"{PURPLE}?{solved_eta}{END}", 167 | ) 168 | return d 169 | 170 | def status( 171 | self, 172 | status: bool | None, 173 | n: int = 1, 174 | ) -> None: 175 | if status is True: 176 | self._solved += n 177 | elif status is False: 178 | self._unsolved += n 179 | else: # status is None: 180 | self._errors += n 181 | self.update(n) 182 | 183 | -------------------------------------------------------------------------------- /src/solverpy/task/launcher.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TYPE_CHECKING 2 | import logging 3 | 4 | if TYPE_CHECKING: 5 | from .task import Task 6 | from .solvertask import SolverTask 7 | from .bar import DefaultBar 8 | 9 | from multiprocessing import Pool, Manager 10 | from .task import runtask, setqueue 11 | from .bar import DefaultBar 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | WAIT = 365 * 24 * 60 * 60 # a year 16 | 17 | 18 | def run( 19 | tasks: list["Task"], 20 | cores: int = 4, 21 | chunksize: int = 1, 22 | ) -> Any: 23 | """Launch `tasks` in parallel on multiple cores and return results. 24 | 25 | :param tasks: list of task to be executed (instances of Task) 26 | :param cores: number of worker threads (Default value = 4) 27 | :param chunksize: chunksize for Pool.map_async (Default value = 1) 28 | """ 29 | pool = Pool(cores) 30 | try: 31 | runner = pool.map_async(runtask, tasks, chunksize=chunksize) 32 | results = runner.get(WAIT) 33 | pool.close() 34 | pool.join() 35 | return results 36 | except (Exception, KeyboardInterrupt) as e: 37 | pool.terminate() 38 | raise e 39 | 40 | 41 | def launch( 42 | tasks: list["SolverTask"], 43 | cores: int = 4, 44 | chunksize: int = 1, 45 | taskdone: Any = None, 46 | bar: "DefaultBar | None" = None, 47 | desc: str = "running", 48 | **others: Any, 49 | ) -> Any: 50 | """Launch `tasks` in parallel on multiple cores, communicate status over the 51 | queue, and show progress bar. 52 | 53 | :param tasks: list of task to be executed (instances of Task) 54 | :param cores: number of worker threads (Default value = 4) 55 | :param chunksize: chunksize for Pool.map_async (Default value = 1) 56 | :param taskdone: (Default value = None) 57 | :param bar: (Default value = None) 58 | :param desc: (Default value = "running") 59 | :param **others: 60 | """ 61 | del others # unused argument 62 | todo = len(tasks) 63 | pool = Pool(cores) 64 | m = Manager() 65 | queue = m.Queue() 66 | setqueue(queue, tasks) 67 | bar = bar or DefaultBar(len(tasks), desc, miniters=1) 68 | logger.debug(f"launching pool with {cores} workers for {todo} tasks") 69 | try: 70 | runner = pool.map_async(runtask, tasks, chunksize=chunksize) 71 | while todo: 72 | status = queue.get(WAIT) # type: ignore 73 | bar.status(status) 74 | if taskdone: 75 | taskdone(status) 76 | todo -= 1 77 | bar.close() 78 | logger.debug(f"all tasks done") 79 | pool.close() 80 | pool.join() 81 | logger.debug(f"pool closed") 82 | return runner.get(WAIT) 83 | except KeyboardInterrupt as e: 84 | bar.close() 85 | logger.debug("pool terminated (keyboard interupt)") 86 | pool.terminate() 87 | raise e 88 | -------------------------------------------------------------------------------- /src/solverpy/task/shelltask.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import subprocess 4 | from .task import Task 5 | 6 | class ShellTask(Task): 7 | 8 | def __init__(self, cmd): 9 | self.cmd = cmd 10 | 11 | def run(self) -> bytes: 12 | try: 13 | output = subprocess.check_output(self.cmd, shell=True, 14 | stderr=subprocess.STDOUT) 15 | except subprocess.CalledProcessError as e: 16 | output = e.output 17 | except Exception as e: 18 | print("ERROR: Shell task failed: %s" % self.cmd) 19 | raise e 20 | return output 21 | 22 | -------------------------------------------------------------------------------- /src/solverpy/task/solvertask.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | from .task import Task 3 | 4 | if TYPE_CHECKING: 5 | from ..solver.solverpy import SolverPy 6 | from ..tools.typing import Result 7 | 8 | 9 | class SolverTask(Task): 10 | 11 | def __init__( 12 | self, 13 | solver: "SolverPy", 14 | bid: str, 15 | sid: str, 16 | problem: str, 17 | ): 18 | Task.__init__(self) 19 | self.solver = solver 20 | self.bid = bid 21 | self.sid = sid 22 | self.problem = problem 23 | 24 | def __str__(self) -> str: 25 | return f"{self.solver}:{self.sid} @ {self.bid} / {self.problem}" 26 | 27 | def run(self) -> "Result": 28 | return self.solver.solve(self.instance, self.strategy) 29 | 30 | def status(self, result: "Result") -> bool | None: 31 | if not self.solver.valid(result): 32 | return None 33 | return self.solver.solved(result) 34 | 35 | @property 36 | def instance(self) -> tuple[str, str]: 37 | return (self.bid, self.problem) 38 | 39 | @property 40 | def strategy(self) -> str: 41 | return self.sid 42 | 43 | -------------------------------------------------------------------------------- /src/solverpy/task/task.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Sequence, TYPE_CHECKING 2 | import logging 3 | 4 | if TYPE_CHECKING: 5 | from queue import Queue 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | class Task: 11 | """A generic executable task/job. 12 | 13 | Represents a task that can be executed. The result can be returned or 14 | communicated over a queue. The abstract method `run` should run the task 15 | and return the result. 16 | 17 | Tasks are typically executed in different threads. Hence modification to 18 | the Task instance might not be visible to the main thread. Also the 19 | returned result should be as small as possible to limit inter-process 20 | communication. 21 | 22 | Alternatively, the communication queue can be used to send the result to the 23 | main thread. If the queue is set, the result is transformed to the `status` 24 | and this status is pushed onto the queue (see `Task.runtask`). 25 | 26 | Status is also propagated to the progress bar using the `status` bar handler 27 | (see `RunningBar.status`). `DefaultBar` ignores the status. `RunningBar` 28 | expects the status for failed tasks to be None, and non-None otherwise. 29 | `SolvingBar` expects `True` for solved tasks, `False` for unsolved, and 30 | `None` for failed tasks. 31 | """ 32 | 33 | def __init__( 34 | self, 35 | queue: "Queue[Any] | None" = None, 36 | ): 37 | """Init the task. 38 | 39 | :param queue: communication queue (optional) 40 | 41 | """ 42 | self._queue = queue 43 | 44 | def run(self) -> Any: 45 | """Run the task and return the result.""" 46 | raise NotImplementedError("Task.run: abstract method not implemented.") 47 | 48 | @property 49 | def queue(self): 50 | """Get the queue.""" 51 | return self._queue 52 | 53 | @queue.setter 54 | def queue(self, q: "Queue[Any]"): 55 | """Set the queue. 56 | 57 | :param q: the queue 58 | 59 | """ 60 | self._queue = q 61 | 62 | def status( 63 | self, 64 | result: Any, 65 | ) -> Any: 66 | """Translate the result to (typically smaller) status to send it over the 67 | queue." 68 | 69 | :param result: the result 70 | 71 | """ 72 | return (result is not None) 73 | 74 | @staticmethod 75 | def runtask(task: "Task"): 76 | """Run the task andd announce the result over the queue. 77 | 78 | :param task: the task to be ran 79 | 80 | """ 81 | try: 82 | res = task.run() 83 | status = task.status(res) 84 | except Exception as e: 85 | import traceback 86 | logger.warning(f"Exception:: {e}") 87 | logger.warning(f"Task:: {task}") 88 | logger.warning(f"Error:: {traceback.format_exc()}") 89 | status = None 90 | res = None 91 | except KeyboardInterrupt: 92 | return None 93 | if status is None: 94 | logger.debug(f"failed task: {task}") 95 | if task.queue is not None: 96 | task.queue.put(status) 97 | return res 98 | 99 | 100 | def runtask(task: Task) -> Any: 101 | """Run task and return the result. 102 | 103 | :param task: 104 | 105 | """ 106 | return task.runtask(task) 107 | 108 | 109 | def setqueue(queue: "Queue[Any]", tasks: Sequence[Task]) -> None: 110 | """Set the queue for a list of tasks. 111 | 112 | :param queue: the queue to set 113 | :param tasks: the tasks to update 114 | 115 | """ 116 | for task in tasks: 117 | task.queue = queue 118 | 119 | -------------------------------------------------------------------------------- /src/solverpy/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cbboyan/solverpy/caa8da555ff766a50e1b0b0a01f2747df65d8bdb/src/solverpy/tools/__init__.py -------------------------------------------------------------------------------- /src/solverpy/tools/human.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | 4 | def indent(string: str, size: int, left: bool = True) -> str: 5 | if left: 6 | return (" " * (size - len(string))) + string 7 | else: 8 | return string + (" " * (size - len(string))) 9 | 10 | 11 | def lindent(string: str, size: int) -> str: 12 | return indent(string, size, left=True) 13 | 14 | 15 | def rindent(string: str, size: int) -> str: 16 | return indent(string, size, left=False) 17 | 18 | 19 | def numeric(strval: str) -> int | float | str: 20 | if strval.isdigit(): 21 | return int(strval) 22 | elif strval.replace('.', '', 1).isdigit(): 23 | return float(strval) 24 | return strval 25 | 26 | 27 | def format(key: str, val: Any) -> str: 28 | unit = key[key.rfind(".") + 1:] 29 | return UNITS[unit](val) if unit in UNITS else str(val) 30 | 31 | 32 | def humanbytes(b: float) -> str: 33 | units = {0: 'Bytes', 1: 'KB', 2: 'MB', 3: 'GB', 4: 'TB', 5: 'PB'} 34 | power = 1024 35 | n = 0 36 | while b > power: 37 | b /= power 38 | n += 1 39 | return "%.2f %s" % (b, units[n]) 40 | 41 | 42 | def humanint(n: int) -> str: 43 | s = str(int(abs(n))) 44 | r = s[-3:] 45 | s = s[:-3] 46 | while s: 47 | r = s[-3:] + "," + r 48 | s = s[:-3] 49 | return r if n >= 0 else "-%s" % r 50 | 51 | 52 | def humantime(s: float) -> str: 53 | h = s // 3600 54 | s -= 3600 * h 55 | m = s // 60 56 | s -= 60 * m 57 | return "%02d:%02d:%04.1f" % (h, m, s) 58 | 59 | 60 | exps_2 = {2**n: n for n in range(256)} 61 | 62 | 63 | def humanexp(n: int) -> str: 64 | if n in exps_2: 65 | return "2e%s" % exps_2[n] 66 | return str(n) 67 | 68 | 69 | def humanloss(xy: tuple[float, str]) -> str: 70 | (x, y) = xy 71 | return "%.2f [iter %s]" % (x, y) 72 | 73 | 74 | def humanacc(xyz: Any) -> str: 75 | if len(xyz) != 3: return str(xyz) 76 | (acc, pos, neg) = xyz 77 | return "%.2f%% (%.2f%% / %.2f%%)" % (100 * acc, 100 * pos, 100 * neg) 78 | 79 | 80 | UNITS = { 81 | "acc": humanacc, 82 | "count": humanint, 83 | "loss": humanloss, 84 | "time": humantime, 85 | "seconds": lambda x: "%.2f" % x, 86 | "size": humanbytes 87 | } 88 | -------------------------------------------------------------------------------- /src/solverpy/tools/log.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING 2 | import os, sys, io 3 | import atexit, traceback 4 | from datetime import datetime 5 | import requests 6 | import socket 7 | import logging 8 | import yaml 9 | 10 | from ..benchmark.path import bids 11 | from ..solver.object import SolverPyObj 12 | 13 | if TYPE_CHECKING: 14 | from ..benchmark.setups.setup import Setup 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | NAME = "logs" 19 | 20 | 21 | def ntfy(setup: "Setup", msg: str) -> None: 22 | if (not setup) or ("ntfy" not in setup): 23 | return 24 | channel = setup["ntfy"] 25 | try: 26 | hostname = socket.gethostname() 27 | requests.post(f"https://ntfy.sh/{channel}", data=f"{hostname}: {msg}") 28 | except IOError as e: 29 | logger.warning(f"> Warning: ntfy I/O error ({e})") 30 | 31 | 32 | def filename() -> str: 33 | d_logs = bids.dbpath(NAME) 34 | os.makedirs(d_logs, exist_ok=True) 35 | script = sys.argv[0] 36 | script = script.lstrip("./").replace("/", "--") 37 | now = datetime.now() 38 | now = now.strftime("%y-%m-%d__%H:%M:%S") 39 | f_log = f"{now}__{script}.log" 40 | return os.path.join(d_logs, f_log) 41 | 42 | 43 | def init_yaml() -> None: 44 | 45 | def representer(dumper, obj: SolverPyObj): 46 | r = obj.represent() 47 | if type(r) is str: 48 | return dumper.represent_str(r) 49 | elif type(r) is list: 50 | return dumper.represent_list(r) 51 | elif type(r) is dict: 52 | return dumper.represent_dict(r) 53 | else: 54 | return dumper.represent_str(str(r)) 55 | 56 | yaml.add_multi_representer(SolverPyObj, representer) 57 | 58 | 59 | def init() -> None: 60 | # set up logging to file 61 | logging.basicConfig( 62 | level=logging.DEBUG, 63 | format="%(asctime)s %(name)-12s %(levelname)-8s %(message)s", 64 | filename=filename(), 65 | filemode="w") 66 | # define a Handler which writes INFO messages or higher to the sys.stderr 67 | #console = logging.StreamHandler() 68 | console = logging.StreamHandler( 69 | io.TextIOWrapper(os.fdopen(sys.stderr.fileno(), "wb"))) 70 | 71 | console.setLevel(logging.INFO) 72 | # set a format which is simpler for console use 73 | #formatter = logging.Formatter("%(name)-12s: %(levelname)-8s %(message)s") 74 | formatter = logging.Formatter("%(asctime)-12s: %(levelname)-8s %(message)s") 75 | #formatter = logging.Formatter("%(asctime)-12s: %(message)s") 76 | # tell the handler to use this format 77 | console.setFormatter(formatter) 78 | # add the handler to the root logger 79 | logging.getLogger("").addHandler(console) 80 | logger.info("Logger running.") 81 | atexit.register(terminating) 82 | init_yaml() 83 | 84 | 85 | def terminating() -> None: 86 | logger.info("Logger terminated.") 87 | if "last_traceback" in dir(sys): 88 | msg = traceback.format_exception(sys.last_type, sys.last_value, 89 | sys.last_traceback) 90 | msg = "".join(msg) 91 | logger.error(f"Last exception:\n{msg}") 92 | -------------------------------------------------------------------------------- /src/solverpy/tools/patterns.py: -------------------------------------------------------------------------------- 1 | from typing import Pattern, Callable, TypeVar 2 | import re 3 | 4 | 5 | def single(pattern: Pattern, output: str, default: str) -> str: 6 | mo = re.search(pattern, output) 7 | return mo.group(1) if mo else default 8 | 9 | 10 | def keyval( 11 | pattern: Pattern, 12 | output: str, 13 | table: (dict[str, str] | None) = None, 14 | ) -> dict[str, str]: 15 | res = dict(re.findall(pattern, output)) 16 | if table: 17 | res = {table[x]: res[x] for x in table if x in res} 18 | return res 19 | 20 | 21 | def valkey( 22 | pattern: Pattern, 23 | output: str, 24 | table: (dict[str, str] | None) = None, 25 | ) -> dict[str, str]: 26 | res = {k: v for (v, k) in re.findall(pattern, output)} 27 | if table: 28 | res = {table[x]: res[x] for x in table if x in res} 29 | return res 30 | 31 | 32 | K = TypeVar("K") 33 | U = TypeVar("U") 34 | V = TypeVar("V") 35 | 36 | def mapval( 37 | data: dict[K, U], 38 | apply: Callable[[U], V], 39 | ) -> dict[K, V]: 40 | return {x: apply(y) for (x, y) in data.items()} 41 | -------------------------------------------------------------------------------- /src/solverpy/tools/redirect.py: -------------------------------------------------------------------------------- 1 | from typing import Any, TextIO, BinaryIO, Callable, TypeVar 2 | import sys 3 | import os 4 | import ctypes 5 | from sys import platform 6 | 7 | libc = ctypes.CDLL(None) 8 | 9 | if platform == "linux" or platform == "linux2": 10 | c_stdout = ctypes.c_void_p.in_dll(libc, 'stdout') 11 | c_stderr = ctypes.c_void_p.in_dll(libc, 'stderr') 12 | elif platform == "darwin": 13 | c_stdout = ctypes.c_void_p.in_dll(libc, '__stdoutp') 14 | c_stderr = ctypes.c_void_p.in_dll(libc, '__stderrp') 15 | elif platform == "win32": 16 | c_stdout = ctypes.c_void_p.in_dll(libc, 'stdout') 17 | c_stderr = ctypes.c_void_p.in_dll(libc, 'stderr') 18 | 19 | 20 | class Redirector(object): 21 | 22 | def __init__(self, f_log: str) -> None: 23 | self._f_log = f_log 24 | 25 | def __enter__(self) -> None: 26 | self._redir = start(self._f_log) 27 | 28 | def __exit__(self, *args: Any) -> None: 29 | del args 30 | finish(*self._redir) 31 | 32 | 33 | def redirect(std: TextIO, fd: int) -> None: 34 | libc.fflush(c_stdout) 35 | libc.fflush(c_stderr) 36 | std_fd = std.fileno() # note that std stays open 37 | os.dup2(fd, std_fd) 38 | 39 | 40 | def start(f_log: str) -> tuple[BinaryIO, int, int]: 41 | s_log = open(f_log, mode="wb") 42 | dup_out = os.dup(sys.stdout.fileno()) 43 | dup_err = os.dup(sys.stderr.fileno()) 44 | redirect(sys.stdout, s_log.fileno()) 45 | redirect(sys.stderr, s_log.fileno()) 46 | return (s_log, dup_out, dup_err) 47 | 48 | 49 | def finish(s_log: BinaryIO, dup_out: int, dup_err: int) -> None: 50 | redirect(sys.stdout, dup_out) 51 | redirect(sys.stderr, dup_err) 52 | os.close(dup_out) 53 | os.close(dup_err) 54 | s_log.close() 55 | 56 | 57 | R = TypeVar("R") 58 | 59 | def call( 60 | target: Callable[..., R], 61 | f_log: str, 62 | *args: Any, 63 | **kwargs: Any, 64 | ) -> R: 65 | try: 66 | with Redirector(f_log): 67 | return target(*args, **kwargs) 68 | except (Exception, KeyboardInterrupt) as e: 69 | raise (e) # propagate exception to the parent 70 | -------------------------------------------------------------------------------- /src/solverpy/tools/timeme.py: -------------------------------------------------------------------------------- 1 | 2 | import time 3 | 4 | def timeme(method): 5 | def inner(*args, **kw): 6 | ts = time.perf_counter() 7 | result = method(*args, **kw) 8 | te = time.perf_counter() 9 | print('# performance: %r: %2.2f ms' % (method.__name__, (te - ts) * 1000)) 10 | return result 11 | return inner 12 | 13 | -------------------------------------------------------------------------------- /src/solverpy/tools/typing.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, TYPE_CHECKING 2 | 3 | if TYPE_CHECKING: 4 | from ..benchmark.db.provider import Provider 5 | from ..solver.solverpy import SolverPy 6 | 7 | ProviderMaker = Callable[[str, str, str], "Provider"] 8 | 9 | StrMaker = str | Callable[[Any], str] 10 | 11 | SolverMaker = Callable[..., "SolverPy"] 12 | 13 | LimitBuilder = dict[str, StrMaker] 14 | 15 | Result = dict[str, Any] 16 | 17 | Report = list["str | Report"] 18 | 19 | SolverJob = tuple["SolverPy", str, str] 20 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp01: -------------------------------------------------------------------------------- 1 | --simplification=none --enum-inst 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp02: -------------------------------------------------------------------------------- 1 | --no-e-matching --enum-inst 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp03: -------------------------------------------------------------------------------- 1 | --no-e-matching --enum-inst --enum-inst-sum 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp04: -------------------------------------------------------------------------------- 1 | --relevant-triggers --enum-inst 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp05: -------------------------------------------------------------------------------- 1 | --trigger-sel=max --enum-inst 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp06: -------------------------------------------------------------------------------- 1 | --multi-trigger-when-single --enum-inst 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp07: -------------------------------------------------------------------------------- 1 | --multi-trigger-when-single --multi-trigger-priority --enum-inst 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp08: -------------------------------------------------------------------------------- 1 | --multi-trigger-cache --enum-inst 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp09: -------------------------------------------------------------------------------- 1 | --no-multi-trigger-linear --enum-inst 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp10: -------------------------------------------------------------------------------- 1 | --pre-skolem-quant=on --enum-inst 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp11: -------------------------------------------------------------------------------- 1 | --inst-when=full --enum-inst 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp12: -------------------------------------------------------------------------------- 1 | --no-e-matching --no-cbqi --enum-inst 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp13: -------------------------------------------------------------------------------- 1 | --enum-inst --quant-ind 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp14: -------------------------------------------------------------------------------- 1 | --decision=internal --simplification=none --no-inst-no-entail --no-cbqi --enum-inst 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp15: -------------------------------------------------------------------------------- 1 | --decision=internal --enum-inst --enum-inst-sum 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp16: -------------------------------------------------------------------------------- 1 | --term-db-mode=relevant --enum-inst 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp17: -------------------------------------------------------------------------------- 1 | --enum-inst-interleave --enum-inst 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp18: -------------------------------------------------------------------------------- 1 | --finite-model-find --fmf-mbqi=none 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp19: -------------------------------------------------------------------------------- 1 | --finite-model-find --decision=internal 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp20: -------------------------------------------------------------------------------- 1 | --finite-model-find --macros-quant --macros-quant-mode=all 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp21: -------------------------------------------------------------------------------- 1 | --finite-model-find --e-matching 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp22: -------------------------------------------------------------------------------- 1 | --finite-model-find --decision=internal 2 | -------------------------------------------------------------------------------- /strats/cvc5/cvc5-smtcomp23: -------------------------------------------------------------------------------- 1 | --enum-inst 2 | --------------------------------------------------------------------------------