├── .dockerignore ├── .gitignore ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── examples ├── admissible_set │ ├── admissible_set.ipynb │ ├── admissible_set_n12_w7_size792.txt │ ├── admissible_set_n15_w10_size3003.txt │ ├── admissible_set_n21_w15_size43596.txt │ ├── admissible_set_n24_w17_size237984.txt │ ├── pre_admissible_set_for_n15_w10_size3003.txt │ ├── pre_admissible_set_for_n21_w15_size43596.txt │ └── pre_admissible_set_for_n24_w17_size237984.txt ├── bin_packing │ └── bin_packing.ipynb ├── cap_set │ ├── cap_set.ipynb │ └── n8_size512.txt ├── cap_set_spec.py ├── corner_free_set │ ├── f2_n4_size137.txt │ ├── f3_n2_size53.txt │ └── f3_n3_size370.txt ├── cyclic_graphs │ ├── cyclic_graphs.ipynb │ └── nodes11_n4_size754.txt └── specification_nonsymmetric_admissible_set.py ├── funsearch ├── __init__.py ├── __main__.py ├── code_manipulation.py ├── config.py ├── container │ ├── Dockerfile │ └── container_main.py ├── core.py ├── evaluator.py ├── programs_database.py ├── sampler.py └── sandbox.py ├── pdm.lock ├── pyproject.toml └── tests ├── code_manipulation_test.py ├── evaluator_test.py ├── funsearch_test.py ├── main_test.py ├── programs_database_test.py └── sandbox_test.py /.dockerignore: -------------------------------------------------------------------------------- 1 | data 2 | 3 | # template from: https://github.com/themattrix/python-pypi-template/blob/master/.dockerignore 4 | # Git 5 | .git 6 | .gitignore 7 | 8 | # CI 9 | .codeclimate.yml 10 | .travis.yml 11 | .taskcluster.yml 12 | 13 | # Docker 14 | docker-compose.yml 15 | .docker 16 | 17 | # Byte-compiled / optimized / DLL files 18 | __pycache__/ 19 | */__pycache__/ 20 | */*/__pycache__/ 21 | */*/*/__pycache__/ 22 | *.py[cod] 23 | */*.py[cod] 24 | */*/*.py[cod] 25 | */*/*/*.py[cod] 26 | 27 | # C extensions 28 | *.so 29 | 30 | # Distribution / packaging 31 | .Python 32 | env/ 33 | build/ 34 | develop-eggs/ 35 | dist/ 36 | downloads/ 37 | eggs/ 38 | lib/ 39 | lib64/ 40 | parts/ 41 | sdist/ 42 | var/ 43 | *.egg-info/ 44 | .installed.cfg 45 | *.egg 46 | 47 | # PyInstaller 48 | # Usually these files are written by a python script from a template 49 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 50 | *.manifest 51 | *.spec 52 | 53 | # Installer logs 54 | pip-log.txt 55 | pip-delete-this-directory.txt 56 | 57 | # Unit test / coverage reports 58 | htmlcov/ 59 | .tox/ 60 | .coverage 61 | .cache 62 | nosetests.xml 63 | coverage.xml 64 | 65 | # Translations 66 | *.mo 67 | *.pot 68 | 69 | # Django stuff: 70 | *.log 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | target/ 77 | 78 | # Virtual environment 79 | .env/ 80 | .venv/ 81 | venv/ 82 | 83 | # PyCharm 84 | .idea 85 | 86 | # Python mode for VIM 87 | .ropeproject 88 | */.ropeproject 89 | */*/.ropeproject 90 | */*/*/.ropeproject 91 | 92 | # Vim swap files 93 | *.swp 94 | */*.swp 95 | */*/*.swp 96 | */*/*/*.swp -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # Distribution / packaging 7 | .Python 8 | build/ 9 | develop-eggs/ 10 | dist/ 11 | downloads/ 12 | eggs/ 13 | .eggs/ 14 | lib/ 15 | lib64/ 16 | parts/ 17 | sdist/ 18 | var/ 19 | wheels/ 20 | share/python-wheels/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | MANIFEST 25 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | ## Contributor License Agreement 4 | 5 | Contributions to this project must be accompanied by a Contributor License 6 | Agreement. You (or your employer) retain the copyright to your contribution, 7 | this simply gives us permission to use and redistribute your contributions as 8 | part of the project. Head over to to see 9 | your current agreements on file or to sign a new one. 10 | 11 | You generally only need to submit a CLA once, so if you've already submitted one 12 | (even if it was for a different project), you probably don't need to do it 13 | again. 14 | 15 | ## Code reviews 16 | 17 | All submissions, including submissions by project members, require review. We 18 | use GitHub pull requests for this purpose. Consult 19 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 20 | information on using pull requests. 21 | 22 | ## Community Guidelines 23 | 24 | This project follows [Google's Open Source Community 25 | Guidelines](https://opensource.google/conduct/). 26 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.io/python:3.11.6 2 | 3 | WORKDIR /workspace 4 | 5 | # Use PDM to keep track of exact versions of dependencies 6 | RUN pip install pdm 7 | COPY pyproject.toml README.md pdm.lock ./ 8 | # install dependencies first. PDM also creates a /workspace/.venv here. 9 | ENV PATH="/workspace/.venv/bin:$PATH" 10 | RUN pdm install --no-self 11 | COPY examples ./examples 12 | COPY funsearch ./funsearch 13 | 14 | RUN pip install --no-deps . && rm -r ./funsearch ./build 15 | 16 | CMD /bin/bash 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # FunSearch 2 | 3 | 4 | Usage: 5 | 6 | 7 | You can run FunSearch in container using Podman or Docker 8 | 9 | ``` 10 | podman build . -t funsearch 11 | 12 | 13 | # Create a folder to share with the container 14 | mkdir data 15 | podman run -it -v ./data:/workspace/data funsearch 16 | 17 | # Set the environment variable OPENAI_API_KEY=sk-xxxx or create .env file. 18 | # "gpt-3.5-turbo-instruct" model is used by default. 19 | # Refer to 'llm' package docs to use other models. 20 | 21 | funsearch run examples/cap_set_spec.py 11 --sandbox_type ExternalProcessSandbox 22 | ``` 23 | In here we are searching for the algorithm to find maximum cap sets for dimension 11. 24 | You should see output something like 25 | ``` 26 | root@11c22cd7aeac:/workspace# funsearch run examples/cap_set_spec.py 11 --sandbox_type ExternalProcessSandbox 27 | INFO:root:Writing logs to data/1704956206 28 | INFO:absl:Best score of island 0 increased to 2048 29 | INFO:absl:Best score of island 1 increased to 2048 30 | INFO:absl:Best score of island 2 increased to 2048 31 | INFO:absl:Best score of island 3 increased to 2048 32 | INFO:absl:Best score of island 4 increased to 2048 33 | INFO:absl:Best score of island 5 increased to 2048 34 | INFO:absl:Best score of island 6 increased to 2048 35 | INFO:absl:Best score of island 7 increased to 2048 36 | INFO:absl:Best score of island 8 increased to 2048 37 | INFO:absl:Best score of island 9 increased to 2048 38 | INFO:absl:Best score of island 5 increased to 2053 39 | INFO:absl:Best score of island 1 increased to 2049 40 | INFO:absl:Best score of island 8 increased to 2684 41 | ^C^CINFO:root:Keyboard interrupt. Stopping. 42 | INFO:absl:Saving backup to data/backups/program_db_priority_1704956206_0.pickle. 43 | ``` 44 | 45 | Note that in the last command we use the ExternalProcessSandbox that is not fully safe 46 | but makes it a bit less likely that invalid code from LLM would break the search. 47 | 48 | 49 | Alternatively, you can run the main Python process on a host computer outside of any container and let 50 | the process build and run separate sandbox containers (still requires Podman/Docker). 51 | This variant could be also used, e.g., in Colab quite safely since the environment is some kind of container itself. 52 | 53 | ``` 54 | pip install . 55 | 56 | funsearch run examples/cap_set_spec.py 11 57 | ``` 58 | 59 | For more complex input data, you can provide the input also as a .json or .pickle file. 60 | 61 | Currently, the search is only using single thread with no asyncio and is somewhat slow 62 | for challenging tasks. 63 | 64 | ## Alternative LLMs 65 | 66 | The search uses gpt-3.5-turbo-instruct by default, but other models can be used with the --model_name argument 67 | and possibly installing extensions to the llm package. 68 | As an example of performance, with gpt-3.5-turbo-instruct on dimension 8 it usually around 20 tries to find a few 69 | improvements to the naive algorithm. 70 | 71 | On the other hand, using orca-mini-3b-gguf2-q4_0 doesn't seem to work quite well. 72 | The latest version has a bit improved parsing to find the last priority_vX method from the LLM response 73 | even if it contains other content like Markdown formatting. Anyway, the model seems to often 74 | use strange indentation of 1 space and thus might require some customization to be useful at all. 75 | Lastly, even with correct Python syntax, orca-mini-3b does not seem to find improvements (in 60 runs) and mostly 76 | generates code that throws "IndexError: tuple index out of range". The situation changes a bit 77 | if the search is started using a database generated by gpt-3.5-turbo prompts. 78 | 79 | Overall, all models would probably require some prompt engineering, temperatures tuning, and such for the tool 80 | to be useful at all except for very simple problems. 81 | Also, the implementation is currently lacking good tools to analyze large amount of responses properly which 82 | makes any prompt engineering more difficult. 83 | 84 | --- 85 | 86 | The repository contains a number of sample problems that can be solved with FunSearch. 87 | Currently, only the cap set problem (examples/cap_set_spec.py) has been written in the form that can be directly 88 | used with the 'funsearch' executable. 89 | 90 | This repository accompanies the publication 91 | 92 | > Romera-Paredes, B. et al. [Mathematical discoveries from program search with large language models](https://www.nature.com/articles/s41586-023-06924-6). *Nature* (2023) 93 | 94 | There are 6 independent directories: 95 | 96 | - `cap_set` contains functions discovered by FunSearch that construct large cap 97 | sets, and we also provide those cap sets in a numerical format for convenience. 98 | 99 | - `admissible_set` contains functions discovered by FunSearch that construct 100 | large admissible sets, and we also provide those admissible sets in a numerical 101 | format for convenience. 102 | 103 | - `bin_packing` contains heuristics discovered by FunSearch for online 1D bin 104 | packing problems, and an evaluation suite to reproduce the results reported in 105 | the paper. 106 | 107 | - `cyclic_graphs` contains functions discovered by FunSearch that construct 108 | large independent sets in strong products of cyclic graphs, and we also provide 109 | those sets in a numerical format for convenience. 110 | 111 | - `corner_free_sets` contains the discovered sets of indices, in numerical 112 | format, satisfying the combinatorial degeneration constraints described for the 113 | corners-free problem in the Supplementary Information. 114 | 115 | - `implementation` contains an implementation of the evolutionary algorithm, 116 | code manipulation routines, and a single-threaded implementation of the 117 | FunSearch pipeline. It does not contain language models for generating new 118 | programs, the sandbox for executing untrusted code, nor the infrastructure for 119 | running FunSearch on our distributed system. This directory is intended to be 120 | useful for understanding the details of our method, and for adapting it for use 121 | with any available language models, sandboxes, and distributed systems. 122 | 123 | ## Installation 124 | 125 | No installation is required. All notebooks can be opened and run in Google 126 | Colab. 127 | 128 | ## Usage 129 | 130 | - `cap_set`: The notebook `cap_set.ipynb` can be opened via 131 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/google-deepmind/funsearch/blob/master/cap_set/cap_set.ipynb). 132 | 133 | - `admissible_set`: The notebook `admissible_set.ipynb` can be opened 134 | via 135 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/google-deepmind/funsearch/blob/master/admissible_set/admissible_set.ipynb). 136 | 137 | - `bin_packing`: The notebook `bin_packing.ipynb` can be opened via 138 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/google-deepmind/funsearch/blob/master/bin_packing/bin_packing.ipynb). 139 | 140 | - `cyclic_graphs`: The notebook `cyclic_graphs.ipynb` can be opened via 141 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/google-deepmind/funsearch/blob/master/cyclic_graphs/cyclic_graphs.ipynb). 142 | 143 | ## Citing this work 144 | 145 | If you use the code or data in this package, please cite: 146 | 147 | ```bibtex 148 | @Article{FunSearch2023, 149 | author = {Romera-Paredes, Bernardino and Barekatain, Mohammadamin and Novikov, Alexander and Balog, Matej and Kumar, M. Pawan and Dupont, Emilien and Ruiz, Francisco J. R. and Ellenberg, Jordan and Wang, Pengming and Fawzi, Omar and Kohli, Pushmeet and Fawzi, Alhussein}, 150 | journal = {Nature}, 151 | title = {Mathematical discoveries from program search with large language models}, 152 | year = {2023}, 153 | doi = {10.1038/s41586-023-06924-6} 154 | } 155 | ``` 156 | 157 | ## License and disclaimer 158 | 159 | Copyright 2023 DeepMind Technologies Limited 160 | 161 | All software is licensed under the Apache License, Version 2.0 (Apache 2.0); 162 | you may not use this file except in compliance with the Apache 2.0 license. 163 | You may obtain a copy of the Apache 2.0 license at: 164 | https://www.apache.org/licenses/LICENSE-2.0 165 | 166 | All other materials are licensed under the Creative Commons Attribution 4.0 167 | International License (CC-BY). You may obtain a copy of the CC-BY license at: 168 | https://creativecommons.org/licenses/by/4.0/legalcode 169 | 170 | Unless required by applicable law or agreed to in writing, all software and 171 | materials distributed here under the Apache 2.0 or CC-BY licenses are 172 | distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 173 | either express or implied. See the licenses for the specific language governing 174 | permissions and limitations under those licenses. 175 | 176 | This is not an official Google product. 177 | -------------------------------------------------------------------------------- /examples/admissible_set/pre_admissible_set_for_n15_w10_size3003.txt: -------------------------------------------------------------------------------- 1 | [4, 4, 4, 4, 4] 2 | [2, 5, 4, 4, 4] 3 | [4, 2, 5, 4, 4] 4 | [4, 4, 2, 5, 4] 5 | [4, 4, 4, 2, 5] 6 | [5, 4, 4, 4, 2] 7 | [2, 3, 5, 4, 4] 8 | [3, 5, 4, 4, 2] 9 | [4, 2, 3, 5, 4] 10 | [4, 4, 2, 3, 5] 11 | [5, 4, 4, 2, 3] 12 | [1, 4, 4, 6, 4] 13 | [4, 1, 4, 4, 6] 14 | [4, 4, 6, 4, 1] 15 | [4, 6, 4, 1, 4] 16 | [6, 4, 1, 4, 4] 17 | [2, 3, 4, 4, 6] 18 | [3, 4, 4, 6, 2] 19 | [4, 4, 6, 2, 3] 20 | [4, 6, 2, 3, 4] 21 | [6, 2, 3, 4, 4] 22 | [0, 5, 4, 6, 4] 23 | [0, 5, 6, 4, 4] 24 | [4, 0, 5, 4, 6] 25 | [4, 0, 5, 6, 4] 26 | [4, 4, 0, 5, 6] 27 | [4, 6, 4, 0, 5] 28 | [5, 4, 6, 4, 0] 29 | [5, 6, 4, 4, 0] 30 | [6, 4, 0, 5, 4] 31 | [6, 4, 4, 0, 5] 32 | [0, 5, 3, 3, 5] 33 | [3, 3, 5, 0, 5] 34 | [3, 5, 0, 5, 3] 35 | [5, 0, 5, 3, 3] 36 | [5, 3, 3, 5, 0] 37 | [2, 5, 2, 5, 4] 38 | [2, 5, 4, 2, 5] 39 | [4, 2, 5, 2, 5] 40 | [5, 2, 5, 4, 2] 41 | [5, 4, 2, 5, 2] 42 | [3, 5, 2, 5, 2] 43 | [5, 2, 3, 5, 2] 44 | [5, 2, 5, 2, 3] 45 | [2, 3, 5, 2, 5] 46 | [2, 5, 2, 3, 5] 47 | [1, 2, 5, 3, 5] 48 | [2, 5, 3, 5, 1] 49 | [3, 5, 1, 2, 5] 50 | [5, 1, 2, 5, 3] 51 | [5, 3, 5, 1, 2] 52 | [0, 3, 5, 6, 4] 53 | [3, 5, 6, 4, 0] 54 | [4, 0, 3, 5, 6] 55 | [5, 6, 4, 0, 3] 56 | [6, 4, 0, 3, 5] 57 | [0, 3, 5, 3, 5] 58 | [3, 5, 0, 3, 5] 59 | [3, 5, 3, 5, 0] 60 | [5, 0, 3, 5, 3] 61 | [5, 3, 5, 0, 3] 62 | [4, 1, 1, 5, 6] 63 | [1, 1, 5, 6, 4] 64 | [1, 5, 6, 4, 1] 65 | [5, 6, 4, 1, 1] 66 | [6, 4, 1, 1, 5] 67 | [1, 1, 4, 6, 6] 68 | [1, 4, 6, 6, 1] 69 | [4, 6, 6, 1, 1] 70 | [6, 1, 1, 4, 6] 71 | [6, 6, 1, 1, 4] 72 | [1, 5, 6, 2, 3] 73 | [2, 3, 1, 5, 6] 74 | [3, 1, 5, 6, 2] 75 | [5, 6, 2, 3, 1] 76 | [6, 2, 3, 1, 5] 77 | [0, 3, 4, 6, 6] 78 | [3, 4, 6, 6, 0] 79 | [4, 6, 6, 0, 3] 80 | [6, 0, 3, 4, 6] 81 | [6, 6, 0, 3, 4] 82 | [0, 5, 6, 2, 5] 83 | [2, 5, 0, 5, 6] 84 | [5, 0, 5, 6, 2] 85 | [5, 6, 2, 5, 0] 86 | [6, 2, 5, 0, 5] 87 | [0, 5, 6, 5, 1] 88 | [1, 0, 5, 6, 5] 89 | [5, 1, 0, 5, 6] 90 | [5, 6, 5, 1, 0] 91 | [6, 5, 1, 0, 5] 92 | [0, 6, 2, 6, 5] 93 | [2, 6, 5, 0, 6] 94 | [5, 0, 6, 2, 6] 95 | [6, 2, 6, 5, 0] 96 | [6, 5, 0, 6, 2] 97 | [0, 1, 6, 6, 6] 98 | [1, 6, 6, 6, 0] 99 | [6, 0, 1, 6, 6] 100 | [6, 6, 0, 1, 6] 101 | [6, 6, 6, 0, 1] 102 | -------------------------------------------------------------------------------- /examples/admissible_set/pre_admissible_set_for_n21_w15_size43596.txt: -------------------------------------------------------------------------------- 1 | [4, 4, 4, 4, 4, 4, 6] 2 | [4, 4, 4, 4, 4, 6, 4] 3 | [4, 4, 4, 4, 6, 4, 4] 4 | [4, 4, 4, 6, 4, 4, 4] 5 | [4, 4, 6, 4, 4, 4, 4] 6 | [4, 6, 4, 4, 4, 4, 4] 7 | [6, 4, 4, 4, 4, 4, 4] 8 | [2, 6, 5, 4, 4, 4, 4] 9 | [4, 2, 6, 5, 4, 4, 4] 10 | [4, 4, 2, 6, 5, 4, 4] 11 | [4, 4, 4, 2, 6, 5, 4] 12 | [4, 4, 4, 4, 2, 6, 5] 13 | [5, 4, 4, 4, 4, 2, 6] 14 | [6, 5, 4, 4, 4, 4, 2] 15 | [2, 5, 4, 4, 6, 4, 4] 16 | [4, 2, 5, 4, 4, 6, 4] 17 | [4, 4, 2, 5, 4, 4, 6] 18 | [4, 4, 6, 4, 4, 2, 5] 19 | [4, 6, 4, 4, 2, 5, 4] 20 | [5, 4, 4, 6, 4, 4, 2] 21 | [6, 4, 4, 2, 5, 4, 4] 22 | [2, 5, 4, 4, 4, 6, 4] 23 | [2, 5, 4, 6, 4, 4, 4] 24 | [4, 2, 5, 4, 4, 4, 6] 25 | [4, 2, 5, 4, 6, 4, 4] 26 | [4, 4, 2, 5, 4, 6, 4] 27 | [4, 4, 4, 2, 5, 4, 6] 28 | [4, 4, 4, 6, 4, 2, 5] 29 | [4, 4, 6, 4, 2, 5, 4] 30 | [4, 6, 4, 2, 5, 4, 4] 31 | [4, 6, 4, 4, 4, 2, 5] 32 | [5, 4, 4, 4, 6, 4, 2] 33 | [5, 4, 6, 4, 4, 4, 2] 34 | [6, 4, 2, 5, 4, 4, 4] 35 | [6, 4, 4, 4, 2, 5, 4] 36 | [2, 5, 4, 4, 4, 4, 6] 37 | [4, 4, 4, 4, 6, 2, 5] 38 | [4, 4, 4, 6, 2, 5, 4] 39 | [4, 4, 6, 2, 5, 4, 4] 40 | [4, 6, 2, 5, 4, 4, 4] 41 | [5, 4, 4, 4, 4, 6, 2] 42 | [6, 2, 5, 4, 4, 4, 4] 43 | [2, 4, 5, 6, 4, 4, 4] 44 | [4, 2, 4, 5, 6, 4, 4] 45 | [4, 4, 2, 4, 5, 6, 4] 46 | [4, 4, 4, 2, 4, 5, 6] 47 | [4, 5, 6, 4, 4, 4, 2] 48 | [5, 6, 4, 4, 4, 2, 4] 49 | [6, 4, 4, 4, 2, 4, 5] 50 | [2, 5, 4, 4, 2, 6, 5] 51 | [2, 6, 5, 2, 5, 4, 4] 52 | [4, 2, 6, 5, 2, 5, 4] 53 | [4, 4, 2, 6, 5, 2, 5] 54 | [5, 2, 5, 4, 4, 2, 6] 55 | [5, 4, 4, 2, 6, 5, 2] 56 | [6, 5, 2, 5, 4, 4, 2] 57 | [2, 5, 2, 6, 5, 4, 4] 58 | [2, 6, 5, 4, 4, 2, 5] 59 | [4, 2, 5, 2, 6, 5, 4] 60 | [4, 4, 2, 5, 2, 6, 5] 61 | [5, 2, 6, 5, 4, 4, 2] 62 | [5, 4, 4, 2, 5, 2, 6] 63 | [6, 5, 4, 4, 2, 5, 2] 64 | [2, 3, 5, 4, 4, 6, 4] 65 | [2, 3, 5, 4, 6, 4, 4] 66 | [3, 5, 4, 4, 6, 4, 2] 67 | [3, 5, 4, 6, 4, 4, 2] 68 | [4, 2, 3, 5, 4, 4, 6] 69 | [4, 2, 3, 5, 4, 6, 4] 70 | [4, 4, 2, 3, 5, 4, 6] 71 | [4, 4, 6, 4, 2, 3, 5] 72 | [4, 6, 4, 2, 3, 5, 4] 73 | [4, 6, 4, 4, 2, 3, 5] 74 | [5, 4, 4, 6, 4, 2, 3] 75 | [5, 4, 6, 4, 4, 2, 3] 76 | [6, 4, 2, 3, 5, 4, 4] 77 | [6, 4, 4, 2, 3, 5, 4] 78 | [2, 5, 4, 2, 6, 5, 4] 79 | [2, 6, 5, 4, 2, 5, 4] 80 | [4, 2, 5, 4, 2, 6, 5] 81 | [4, 2, 6, 5, 4, 2, 5] 82 | [5, 4, 2, 5, 4, 2, 6] 83 | [5, 4, 2, 6, 5, 4, 2] 84 | [6, 5, 4, 2, 5, 4, 2] 85 | [2, 5, 2, 5, 4, 6, 4] 86 | [2, 5, 4, 6, 4, 2, 5] 87 | [4, 2, 5, 2, 5, 4, 6] 88 | [4, 6, 4, 2, 5, 2, 5] 89 | [5, 2, 5, 4, 6, 4, 2] 90 | [5, 4, 6, 4, 2, 5, 2] 91 | [6, 4, 2, 5, 2, 5, 4] 92 | [2, 3, 5, 4, 4, 4, 6] 93 | [3, 5, 4, 4, 4, 6, 2] 94 | [4, 4, 4, 6, 2, 3, 5] 95 | [4, 4, 6, 2, 3, 5, 4] 96 | [4, 6, 2, 3, 5, 4, 4] 97 | [5, 4, 4, 4, 6, 2, 3] 98 | [6, 2, 3, 5, 4, 4, 4] 99 | [2, 3, 3, 5, 4, 6, 4] 100 | [3, 3, 5, 4, 6, 4, 2] 101 | [3, 5, 4, 6, 4, 2, 3] 102 | [4, 2, 3, 3, 5, 4, 6] 103 | [4, 6, 4, 2, 3, 3, 5] 104 | [5, 4, 6, 4, 2, 3, 3] 105 | [6, 4, 2, 3, 3, 5, 4] 106 | [2, 5, 2, 5, 4, 4, 6] 107 | [2, 5, 4, 4, 6, 2, 5] 108 | [4, 4, 6, 2, 5, 2, 5] 109 | [4, 6, 2, 5, 2, 5, 4] 110 | [5, 2, 5, 4, 4, 6, 2] 111 | [5, 4, 4, 6, 2, 5, 2] 112 | [6, 2, 5, 2, 5, 4, 4] 113 | [2, 2, 5, 6, 4, 6, 4] 114 | [2, 5, 6, 4, 6, 4, 2] 115 | [4, 2, 2, 5, 6, 4, 6] 116 | [4, 6, 4, 2, 2, 5, 6] 117 | [5, 6, 4, 6, 4, 2, 2] 118 | [6, 4, 2, 2, 5, 6, 4] 119 | [6, 4, 6, 4, 2, 2, 5] 120 | [2, 5, 4, 2, 5, 4, 6] 121 | [2, 5, 4, 6, 2, 5, 4] 122 | [4, 2, 5, 4, 6, 2, 5] 123 | [4, 6, 2, 5, 4, 2, 5] 124 | [5, 4, 2, 5, 4, 6, 2] 125 | [5, 4, 6, 2, 5, 4, 2] 126 | [6, 2, 5, 4, 2, 5, 4] 127 | [2, 3, 3, 5, 4, 4, 6] 128 | [3, 3, 5, 4, 4, 6, 2] 129 | [3, 5, 4, 4, 6, 2, 3] 130 | [4, 4, 6, 2, 3, 3, 5] 131 | [4, 6, 2, 3, 3, 5, 4] 132 | [5, 4, 4, 6, 2, 3, 3] 133 | [6, 2, 3, 3, 5, 4, 4] 134 | [2, 3, 3, 5, 6, 4, 4] 135 | [3, 3, 5, 6, 4, 4, 2] 136 | [3, 5, 6, 4, 4, 2, 3] 137 | [4, 2, 3, 3, 5, 6, 4] 138 | [4, 4, 2, 3, 3, 5, 6] 139 | [5, 6, 4, 4, 2, 3, 3] 140 | [6, 4, 4, 2, 3, 3, 5] 141 | [2, 2, 5, 6, 4, 4, 6] 142 | [2, 5, 6, 4, 4, 6, 2] 143 | [4, 4, 6, 2, 2, 5, 6] 144 | [4, 6, 2, 2, 5, 6, 4] 145 | [5, 6, 4, 4, 6, 2, 2] 146 | [6, 2, 2, 5, 6, 4, 4] 147 | [6, 4, 4, 6, 2, 2, 5] 148 | [2, 3, 5, 4, 2, 6, 5] 149 | [2, 6, 5, 2, 3, 5, 4] 150 | [3, 5, 4, 2, 6, 5, 2] 151 | [4, 2, 6, 5, 2, 3, 5] 152 | [5, 2, 3, 5, 4, 2, 6] 153 | [5, 4, 2, 6, 5, 2, 3] 154 | [6, 5, 2, 3, 5, 4, 2] 155 | [2, 3, 5, 2, 6, 5, 4] 156 | [2, 6, 5, 4, 2, 3, 5] 157 | [3, 5, 2, 6, 5, 4, 2] 158 | [4, 2, 3, 5, 2, 6, 5] 159 | [5, 2, 6, 5, 4, 2, 3] 160 | [5, 4, 2, 3, 5, 2, 6] 161 | [6, 5, 4, 2, 3, 5, 2] 162 | [2, 3, 3, 3, 5, 4, 6] 163 | [3, 3, 3, 5, 4, 6, 2] 164 | [3, 3, 5, 4, 6, 2, 3] 165 | [3, 5, 4, 6, 2, 3, 3] 166 | [4, 6, 2, 3, 3, 3, 5] 167 | [5, 4, 6, 2, 3, 3, 3] 168 | [6, 2, 3, 3, 3, 5, 4] 169 | [2, 3, 3, 3, 5, 6, 4] 170 | [3, 3, 3, 5, 6, 4, 2] 171 | [3, 3, 5, 6, 4, 2, 3] 172 | [3, 5, 6, 4, 2, 3, 3] 173 | [4, 2, 3, 3, 3, 5, 6] 174 | [5, 6, 4, 2, 3, 3, 3] 175 | [6, 4, 2, 3, 3, 3, 5] 176 | [2, 5, 2, 5, 2, 6, 5] 177 | [2, 5, 2, 6, 5, 2, 5] 178 | [2, 6, 5, 2, 5, 2, 5] 179 | [5, 2, 5, 2, 5, 2, 6] 180 | [5, 2, 5, 2, 6, 5, 2] 181 | [5, 2, 6, 5, 2, 5, 2] 182 | [6, 5, 2, 5, 2, 5, 2] 183 | [2, 2, 3, 6, 5, 6, 4] 184 | [2, 3, 6, 5, 6, 4, 2] 185 | [3, 6, 5, 6, 4, 2, 2] 186 | [4, 2, 2, 3, 6, 5, 6] 187 | [5, 6, 4, 2, 2, 3, 6] 188 | [6, 4, 2, 2, 3, 6, 5] 189 | [6, 5, 6, 4, 2, 2, 3] 190 | [2, 3, 3, 5, 2, 6, 5] 191 | [2, 6, 5, 2, 3, 3, 5] 192 | [3, 3, 5, 2, 6, 5, 2] 193 | [3, 5, 2, 6, 5, 2, 3] 194 | [5, 2, 3, 3, 5, 2, 6] 195 | [5, 2, 6, 5, 2, 3, 3] 196 | [6, 5, 2, 3, 3, 5, 2] 197 | [2, 4, 5, 6, 4, 2, 5] 198 | [2, 5, 2, 4, 5, 6, 4] 199 | [4, 2, 5, 2, 4, 5, 6] 200 | [4, 5, 6, 4, 2, 5, 2] 201 | [5, 2, 4, 5, 6, 4, 2] 202 | [5, 6, 4, 2, 5, 2, 4] 203 | [6, 4, 2, 5, 2, 4, 5] 204 | [2, 3, 5, 4, 6, 2, 5] 205 | [2, 5, 2, 3, 5, 4, 6] 206 | [3, 5, 4, 6, 2, 5, 2] 207 | [4, 6, 2, 5, 2, 3, 5] 208 | [5, 2, 3, 5, 4, 6, 2] 209 | [5, 4, 6, 2, 5, 2, 3] 210 | [6, 2, 5, 2, 3, 5, 4] 211 | [2, 3, 5, 2, 5, 4, 6] 212 | [2, 5, 4, 6, 2, 3, 5] 213 | [3, 5, 2, 5, 4, 6, 2] 214 | [4, 6, 2, 3, 5, 2, 5] 215 | [5, 2, 5, 4, 6, 2, 3] 216 | [5, 4, 6, 2, 3, 5, 2] 217 | [6, 2, 3, 5, 2, 5, 4] 218 | [2, 4, 5, 6, 2, 5, 4] 219 | [2, 5, 4, 2, 4, 5, 6] 220 | [4, 2, 4, 5, 6, 2, 5] 221 | [4, 5, 6, 2, 5, 4, 2] 222 | [5, 4, 2, 4, 5, 6, 2] 223 | [5, 6, 2, 5, 4, 2, 4] 224 | [6, 2, 5, 4, 2, 4, 5] 225 | [2, 4, 2, 6, 5, 5, 4] 226 | [2, 6, 5, 5, 4, 2, 4] 227 | [4, 2, 4, 2, 6, 5, 5] 228 | [4, 2, 6, 5, 5, 4, 2] 229 | [5, 4, 2, 4, 2, 6, 5] 230 | [5, 5, 4, 2, 4, 2, 6] 231 | [6, 5, 5, 4, 2, 4, 2] 232 | [2, 2, 5, 6, 6, 4, 4] 233 | [2, 5, 6, 6, 4, 4, 2] 234 | [4, 2, 2, 5, 6, 6, 4] 235 | [4, 4, 2, 2, 5, 6, 6] 236 | [5, 6, 6, 4, 4, 2, 2] 237 | [6, 4, 4, 2, 2, 5, 6] 238 | [6, 6, 4, 4, 2, 2, 5] 239 | [2, 2, 5, 6, 2, 6, 5] 240 | [2, 5, 6, 2, 6, 5, 2] 241 | [2, 6, 5, 2, 2, 5, 6] 242 | [5, 2, 2, 5, 6, 2, 6] 243 | [5, 6, 2, 6, 5, 2, 2] 244 | [6, 2, 6, 5, 2, 2, 5] 245 | [6, 5, 2, 2, 5, 6, 2] 246 | [2, 3, 3, 3, 3, 5, 6] 247 | [3, 3, 3, 3, 5, 6, 2] 248 | [3, 3, 3, 5, 6, 2, 3] 249 | [3, 3, 5, 6, 2, 3, 3] 250 | [3, 5, 6, 2, 3, 3, 3] 251 | [5, 6, 2, 3, 3, 3, 3] 252 | [6, 2, 3, 3, 3, 3, 5] 253 | [2, 3, 2, 3, 6, 5, 6] 254 | [2, 3, 6, 5, 6, 2, 3] 255 | [3, 2, 3, 6, 5, 6, 2] 256 | [3, 6, 5, 6, 2, 3, 2] 257 | [5, 6, 2, 3, 2, 3, 6] 258 | [6, 2, 3, 2, 3, 6, 5] 259 | [6, 5, 6, 2, 3, 2, 3] 260 | [2, 3, 3, 5, 6, 2, 5] 261 | [2, 5, 2, 3, 3, 5, 6] 262 | [3, 3, 5, 6, 2, 5, 2] 263 | [3, 5, 6, 2, 5, 2, 3] 264 | [5, 2, 3, 3, 5, 6, 2] 265 | [5, 6, 2, 5, 2, 3, 3] 266 | [6, 2, 5, 2, 3, 3, 5] 267 | [2, 3, 5, 2, 4, 5, 6] 268 | [2, 4, 5, 6, 2, 3, 5] 269 | [3, 5, 2, 4, 5, 6, 2] 270 | [4, 5, 6, 2, 3, 5, 2] 271 | [5, 2, 4, 5, 6, 2, 3] 272 | [5, 6, 2, 3, 5, 2, 4] 273 | [6, 2, 3, 5, 2, 4, 5] 274 | [2, 2, 2, 6, 6, 5, 6] 275 | [2, 2, 6, 6, 5, 6, 2] 276 | [2, 6, 6, 5, 6, 2, 2] 277 | [5, 6, 2, 2, 2, 6, 6] 278 | [6, 2, 2, 2, 6, 6, 5] 279 | [6, 5, 6, 2, 2, 2, 6] 280 | [6, 6, 5, 6, 2, 2, 2] 281 | [0, 3, 3, 3, 5, 6, 5] 282 | [3, 3, 3, 5, 6, 5, 0] 283 | [3, 3, 5, 6, 5, 0, 3] 284 | [3, 5, 6, 5, 0, 3, 3] 285 | [5, 0, 3, 3, 3, 5, 6] 286 | [5, 6, 5, 0, 3, 3, 3] 287 | [6, 5, 0, 3, 3, 3, 5] 288 | [2, 2, 5, 6, 6, 2, 5] 289 | [2, 5, 2, 2, 5, 6, 6] 290 | [2, 5, 6, 6, 2, 5, 2] 291 | [5, 2, 2, 5, 6, 6, 2] 292 | [5, 6, 6, 2, 5, 2, 2] 293 | [6, 2, 5, 2, 2, 5, 6] 294 | [6, 6, 2, 5, 2, 2, 5] 295 | [0, 6, 5, 5, 2, 3, 5] 296 | [2, 3, 5, 0, 6, 5, 5] 297 | [3, 5, 0, 6, 5, 5, 2] 298 | [5, 0, 6, 5, 5, 2, 3] 299 | [5, 2, 3, 5, 0, 6, 5] 300 | [5, 5, 2, 3, 5, 0, 6] 301 | [6, 5, 5, 2, 3, 5, 0] 302 | [0, 3, 5, 2, 5, 6, 6] 303 | [2, 5, 6, 6, 0, 3, 5] 304 | [3, 5, 2, 5, 6, 6, 0] 305 | [5, 2, 5, 6, 6, 0, 3] 306 | [5, 6, 6, 0, 3, 5, 2] 307 | [6, 0, 3, 5, 2, 5, 6] 308 | [6, 6, 0, 3, 5, 2, 5] 309 | -------------------------------------------------------------------------------- /examples/admissible_set/pre_admissible_set_for_n24_w17_size237984.txt: -------------------------------------------------------------------------------- 1 | [4, 4, 4, 4, 4, 6, 4, 4] 2 | [4, 4, 4, 4, 4, 4, 6, 4] 3 | [6, 4, 4, 4, 4, 4, 4, 4] 4 | [4, 6, 4, 4, 4, 4, 4, 4] 5 | [4, 4, 6, 4, 4, 4, 4, 4] 6 | [4, 4, 4, 4, 6, 4, 4, 4] 7 | [4, 4, 4, 4, 4, 4, 4, 6] 8 | [4, 4, 4, 6, 4, 4, 4, 4] 9 | [4, 2, 6, 5, 4, 4, 4, 4] 10 | [4, 4, 4, 2, 6, 5, 4, 4] 11 | [4, 4, 2, 6, 5, 4, 4, 4] 12 | [2, 6, 5, 4, 4, 4, 4, 4] 13 | [4, 4, 4, 4, 4, 2, 6, 5] 14 | [4, 4, 4, 4, 2, 6, 5, 4] 15 | [6, 5, 4, 4, 4, 4, 4, 2] 16 | [5, 4, 4, 4, 4, 4, 2, 6] 17 | [4, 4, 2, 5, 4, 6, 4, 4] 18 | [4, 4, 4, 2, 5, 4, 6, 4] 19 | [6, 4, 4, 4, 4, 2, 5, 4] 20 | [5, 4, 6, 4, 4, 4, 4, 2] 21 | [4, 6, 4, 4, 4, 4, 2, 5] 22 | [4, 4, 4, 4, 2, 5, 4, 6] 23 | [2, 5, 4, 6, 4, 4, 4, 4] 24 | [4, 2, 5, 4, 6, 4, 4, 4] 25 | [4, 4, 2, 5, 4, 4, 6, 4] 26 | [4, 2, 5, 4, 4, 4, 6, 4] 27 | [6, 4, 4, 4, 2, 5, 4, 4] 28 | [6, 4, 4, 2, 5, 4, 4, 4] 29 | [4, 6, 4, 4, 2, 5, 4, 4] 30 | [4, 4, 2, 5, 4, 4, 4, 6] 31 | [4, 2, 5, 4, 4, 6, 4, 4] 32 | [2, 5, 4, 4, 4, 6, 4, 4] 33 | [5, 4, 4, 6, 4, 4, 4, 2] 34 | [4, 6, 4, 4, 4, 2, 5, 4] 35 | [4, 4, 6, 4, 4, 4, 2, 5] 36 | [4, 4, 6, 4, 4, 2, 5, 4] 37 | [4, 4, 4, 2, 5, 4, 4, 6] 38 | [2, 5, 4, 4, 6, 4, 4, 4] 39 | [5, 4, 4, 4, 6, 4, 4, 2] 40 | [4, 4, 4, 6, 4, 4, 2, 5] 41 | [5, 4, 4, 4, 4, 4, 6, 2] 42 | [4, 4, 4, 4, 4, 6, 2, 5] 43 | [6, 2, 5, 4, 4, 4, 4, 4] 44 | [4, 6, 2, 5, 4, 4, 4, 4] 45 | [4, 4, 6, 2, 5, 4, 4, 4] 46 | [4, 4, 4, 4, 6, 2, 5, 4] 47 | [4, 4, 4, 6, 2, 5, 4, 4] 48 | [2, 5, 4, 4, 4, 4, 4, 6] 49 | [5, 4, 4, 4, 4, 6, 4, 2] 50 | [2, 5, 4, 4, 4, 4, 6, 4] 51 | [6, 4, 2, 5, 4, 4, 4, 4] 52 | [4, 6, 4, 2, 5, 4, 4, 4] 53 | [4, 4, 6, 4, 2, 5, 4, 4] 54 | [4, 4, 4, 6, 4, 2, 5, 4] 55 | [4, 4, 4, 4, 6, 4, 2, 5] 56 | [4, 2, 5, 4, 4, 4, 4, 6] 57 | [5, 6, 4, 4, 4, 4, 2, 4] 58 | [4, 4, 4, 2, 4, 5, 6, 4] 59 | [6, 4, 4, 4, 4, 2, 4, 5] 60 | [4, 5, 6, 4, 4, 4, 4, 2] 61 | [4, 4, 4, 4, 2, 4, 5, 6] 62 | [4, 4, 2, 4, 5, 6, 4, 4] 63 | [2, 4, 5, 6, 4, 4, 4, 4] 64 | [4, 2, 4, 5, 6, 4, 4, 4] 65 | [6, 4, 4, 2, 3, 5, 4, 4] 66 | [4, 2, 3, 5, 4, 4, 6, 4] 67 | [5, 4, 4, 6, 4, 4, 2, 3] 68 | [4, 6, 4, 4, 2, 3, 5, 4] 69 | [4, 4, 6, 4, 4, 2, 3, 5] 70 | [4, 4, 2, 3, 5, 4, 4, 6] 71 | [2, 3, 5, 4, 4, 6, 4, 4] 72 | [3, 5, 4, 4, 6, 4, 4, 2] 73 | [4, 4, 2, 3, 5, 4, 6, 4] 74 | [5, 4, 6, 4, 4, 4, 2, 3] 75 | [4, 6, 4, 4, 4, 2, 3, 5] 76 | [4, 4, 4, 2, 3, 5, 4, 6] 77 | [6, 4, 4, 4, 2, 3, 5, 4] 78 | [4, 2, 3, 5, 4, 6, 4, 4] 79 | [3, 5, 4, 6, 4, 4, 4, 2] 80 | [2, 3, 5, 4, 6, 4, 4, 4] 81 | [3, 5, 4, 4, 4, 4, 6, 2] 82 | [6, 2, 3, 5, 4, 4, 4, 4] 83 | [5, 4, 4, 4, 4, 6, 2, 3] 84 | [4, 6, 2, 3, 5, 4, 4, 4] 85 | [4, 4, 6, 2, 3, 5, 4, 4] 86 | [4, 4, 4, 6, 2, 3, 5, 4] 87 | [4, 4, 4, 4, 6, 2, 3, 5] 88 | [2, 3, 5, 4, 4, 4, 4, 6] 89 | [2, 3, 5, 4, 4, 4, 6, 4] 90 | [6, 4, 2, 3, 5, 4, 4, 4] 91 | [4, 6, 4, 2, 3, 5, 4, 4] 92 | [4, 2, 3, 5, 4, 4, 4, 6] 93 | [3, 5, 4, 4, 4, 6, 4, 2] 94 | [5, 4, 4, 4, 6, 4, 2, 3] 95 | [4, 4, 6, 4, 2, 3, 5, 4] 96 | [4, 4, 4, 6, 4, 2, 3, 5] 97 | [4, 2, 3, 3, 5, 4, 6, 4] 98 | [4, 4, 2, 3, 3, 5, 4, 6] 99 | [6, 4, 4, 2, 3, 3, 5, 4] 100 | [5, 4, 6, 4, 4, 2, 3, 3] 101 | [4, 6, 4, 4, 2, 3, 3, 5] 102 | [3, 5, 4, 6, 4, 4, 2, 3] 103 | [2, 3, 3, 5, 4, 6, 4, 4] 104 | [3, 3, 5, 4, 6, 4, 4, 2] 105 | [6, 4, 2, 3, 3, 5, 4, 4] 106 | [4, 2, 3, 3, 5, 4, 4, 6] 107 | [5, 4, 4, 6, 4, 2, 3, 3] 108 | [4, 6, 4, 2, 3, 3, 5, 4] 109 | [4, 4, 6, 4, 2, 3, 3, 5] 110 | [2, 3, 3, 5, 4, 4, 6, 4] 111 | [3, 5, 4, 4, 6, 4, 2, 3] 112 | [3, 3, 5, 4, 4, 6, 4, 2] 113 | [5, 6, 4, 4, 4, 2, 3, 3] 114 | [4, 4, 4, 2, 3, 3, 5, 6] 115 | [4, 4, 2, 3, 3, 5, 6, 4] 116 | [4, 2, 3, 3, 5, 6, 4, 4] 117 | [3, 5, 6, 4, 4, 4, 2, 3] 118 | [2, 3, 3, 5, 6, 4, 4, 4] 119 | [6, 4, 4, 4, 2, 3, 3, 5] 120 | [3, 3, 5, 6, 4, 4, 4, 2] 121 | [3, 3, 5, 4, 4, 4, 6, 2] 122 | [6, 2, 3, 3, 5, 4, 4, 4] 123 | [4, 6, 2, 3, 3, 5, 4, 4] 124 | [3, 5, 4, 4, 4, 6, 2, 3] 125 | [2, 3, 3, 5, 4, 4, 4, 6] 126 | [5, 4, 4, 4, 6, 2, 3, 3] 127 | [4, 4, 6, 2, 3, 3, 5, 4] 128 | [4, 4, 4, 6, 2, 3, 3, 5] 129 | [4, 4, 2, 5, 2, 6, 5, 4] 130 | [5, 2, 6, 5, 4, 4, 4, 2] 131 | [4, 4, 4, 2, 5, 2, 6, 5] 132 | [4, 2, 5, 2, 6, 5, 4, 4] 133 | [2, 6, 5, 4, 4, 4, 2, 5] 134 | [2, 5, 2, 6, 5, 4, 4, 4] 135 | [6, 5, 4, 4, 4, 2, 5, 2] 136 | [5, 4, 4, 4, 2, 5, 2, 6] 137 | [6, 5, 4, 4, 2, 5, 4, 2] 138 | [5, 4, 4, 2, 5, 4, 2, 6] 139 | [5, 4, 2, 6, 5, 4, 4, 2] 140 | [4, 2, 6, 5, 4, 4, 2, 5] 141 | [2, 6, 5, 4, 4, 2, 5, 4] 142 | [2, 5, 4, 2, 6, 5, 4, 4] 143 | [4, 4, 2, 5, 4, 2, 6, 5] 144 | [4, 2, 5, 4, 2, 6, 5, 4] 145 | [4, 2, 6, 5, 4, 2, 5, 4] 146 | [5, 4, 4, 2, 6, 5, 4, 2] 147 | [4, 4, 2, 6, 5, 4, 2, 5] 148 | [2, 6, 5, 4, 2, 5, 4, 4] 149 | [2, 5, 4, 4, 2, 6, 5, 4] 150 | [4, 2, 5, 4, 4, 2, 6, 5] 151 | [6, 5, 4, 2, 5, 4, 4, 2] 152 | [5, 4, 2, 5, 4, 4, 2, 6] 153 | [6, 4, 4, 2, 5, 2, 5, 4] 154 | [4, 6, 4, 4, 2, 5, 2, 5] 155 | [4, 4, 2, 5, 2, 5, 4, 6] 156 | [4, 2, 5, 2, 5, 4, 6, 4] 157 | [5, 4, 6, 4, 4, 2, 5, 2] 158 | [2, 5, 4, 6, 4, 4, 2, 5] 159 | [2, 5, 2, 5, 4, 6, 4, 4] 160 | [5, 2, 5, 4, 6, 4, 4, 2] 161 | [4, 2, 6, 5, 2, 5, 4, 4] 162 | [5, 4, 4, 4, 2, 6, 5, 2] 163 | [4, 4, 4, 2, 6, 5, 2, 5] 164 | [4, 4, 2, 6, 5, 2, 5, 4] 165 | [2, 6, 5, 2, 5, 4, 4, 4] 166 | [2, 5, 4, 4, 4, 2, 6, 5] 167 | [6, 5, 2, 5, 4, 4, 4, 2] 168 | [5, 2, 5, 4, 4, 4, 2, 6] 169 | [4, 6, 4, 2, 5, 4, 2, 5] 170 | [6, 4, 2, 5, 4, 2, 5, 4] 171 | [2, 5, 4, 6, 4, 2, 5, 4] 172 | [2, 5, 4, 2, 5, 4, 6, 4] 173 | [5, 4, 6, 4, 2, 5, 4, 2] 174 | [5, 4, 2, 5, 4, 6, 4, 2] 175 | [4, 2, 5, 4, 6, 4, 2, 5] 176 | [4, 2, 5, 4, 2, 5, 4, 6] 177 | [4, 6, 2, 5, 2, 5, 4, 4] 178 | [5, 2, 5, 4, 4, 4, 6, 2] 179 | [4, 4, 6, 2, 5, 2, 5, 4] 180 | [6, 2, 5, 2, 5, 4, 4, 4] 181 | [2, 5, 2, 5, 4, 4, 4, 6] 182 | [2, 5, 4, 4, 4, 6, 2, 5] 183 | [4, 4, 4, 6, 2, 5, 2, 5] 184 | [5, 4, 4, 4, 6, 2, 5, 2] 185 | [5, 4, 4, 6, 2, 5, 4, 2] 186 | [4, 6, 2, 5, 4, 2, 5, 4] 187 | [4, 4, 6, 2, 5, 4, 2, 5] 188 | [5, 4, 2, 5, 4, 4, 6, 2] 189 | [4, 2, 5, 4, 4, 6, 2, 5] 190 | [2, 5, 4, 4, 6, 2, 5, 4] 191 | [6, 2, 5, 4, 2, 5, 4, 4] 192 | [2, 5, 4, 2, 5, 4, 4, 6] 193 | [6, 4, 2, 5, 2, 5, 4, 4] 194 | [2, 5, 2, 5, 4, 4, 6, 4] 195 | [5, 2, 5, 4, 4, 6, 4, 2] 196 | [4, 6, 4, 2, 5, 2, 5, 4] 197 | [4, 2, 5, 2, 5, 4, 4, 6] 198 | [2, 5, 4, 4, 6, 4, 2, 5] 199 | [5, 4, 4, 6, 4, 2, 5, 2] 200 | [4, 4, 6, 4, 2, 5, 2, 5] 201 | [5, 4, 4, 2, 5, 4, 6, 2] 202 | [5, 4, 6, 2, 5, 4, 4, 2] 203 | [4, 6, 2, 5, 4, 4, 2, 5] 204 | [2, 5, 4, 6, 2, 5, 4, 4] 205 | [6, 2, 5, 4, 4, 2, 5, 4] 206 | [4, 4, 2, 5, 4, 6, 2, 5] 207 | [4, 2, 5, 4, 6, 2, 5, 4] 208 | [2, 5, 4, 4, 2, 5, 4, 6] 209 | [2, 3, 3, 3, 5, 6, 4, 4] 210 | [5, 6, 4, 4, 2, 3, 3, 3] 211 | [4, 4, 2, 3, 3, 3, 5, 6] 212 | [4, 2, 3, 3, 3, 5, 6, 4] 213 | [3, 5, 6, 4, 4, 2, 3, 3] 214 | [3, 3, 5, 6, 4, 4, 2, 3] 215 | [3, 3, 3, 5, 6, 4, 4, 2] 216 | [6, 4, 4, 2, 3, 3, 3, 5] 217 | [6, 2, 3, 3, 3, 5, 4, 4] 218 | [2, 3, 3, 3, 5, 4, 4, 6] 219 | [4, 6, 2, 3, 3, 3, 5, 4] 220 | [3, 5, 4, 4, 6, 2, 3, 3] 221 | [3, 3, 3, 5, 4, 4, 6, 2] 222 | [3, 3, 5, 4, 4, 6, 2, 3] 223 | [5, 4, 4, 6, 2, 3, 3, 3] 224 | [4, 4, 6, 2, 3, 3, 3, 5] 225 | [2, 3, 3, 3, 5, 4, 6, 4] 226 | [6, 4, 2, 3, 3, 3, 5, 4] 227 | [5, 4, 6, 4, 2, 3, 3, 3] 228 | [4, 6, 4, 2, 3, 3, 3, 5] 229 | [4, 2, 3, 3, 3, 5, 4, 6] 230 | [3, 5, 4, 6, 4, 2, 3, 3] 231 | [3, 3, 5, 4, 6, 4, 2, 3] 232 | [3, 3, 3, 5, 4, 6, 4, 2] 233 | [4, 4, 2, 2, 5, 6, 4, 6] 234 | [6, 4, 6, 4, 4, 2, 2, 5] 235 | [2, 2, 5, 6, 4, 6, 4, 4] 236 | [6, 4, 4, 2, 2, 5, 6, 4] 237 | [5, 6, 4, 6, 4, 4, 2, 2] 238 | [4, 6, 4, 4, 2, 2, 5, 6] 239 | [4, 2, 2, 5, 6, 4, 6, 4] 240 | [2, 5, 6, 4, 6, 4, 4, 2] 241 | [2, 4, 2, 6, 5, 5, 4, 4] 242 | [4, 2, 4, 2, 6, 5, 5, 4] 243 | [2, 6, 5, 5, 4, 4, 2, 4] 244 | [6, 5, 5, 4, 4, 2, 4, 2] 245 | [4, 2, 6, 5, 5, 4, 4, 2] 246 | [5, 4, 4, 2, 4, 2, 6, 5] 247 | [4, 4, 2, 4, 2, 6, 5, 5] 248 | [5, 5, 4, 4, 2, 4, 2, 6] 249 | [4, 6, 4, 2, 2, 5, 6, 4] 250 | [2, 2, 5, 6, 4, 4, 6, 4] 251 | [6, 4, 4, 6, 4, 2, 2, 5] 252 | [5, 6, 4, 4, 6, 4, 2, 2] 253 | [4, 4, 6, 4, 2, 2, 5, 6] 254 | [6, 4, 2, 2, 5, 6, 4, 4] 255 | [4, 2, 2, 5, 6, 4, 4, 6] 256 | [2, 5, 6, 4, 4, 6, 4, 2] 257 | [5, 6, 4, 4, 4, 6, 2, 2] 258 | [4, 4, 6, 2, 2, 5, 6, 4] 259 | [4, 4, 4, 6, 2, 2, 5, 6] 260 | [2, 5, 6, 4, 4, 4, 6, 2] 261 | [6, 2, 2, 5, 6, 4, 4, 4] 262 | [4, 6, 2, 2, 5, 6, 4, 4] 263 | [2, 2, 5, 6, 4, 4, 4, 6] 264 | [6, 4, 4, 4, 6, 2, 2, 5] 265 | [5, 6, 4, 4, 2, 5, 2, 4] 266 | [4, 2, 5, 2, 4, 5, 6, 4] 267 | [6, 4, 4, 2, 5, 2, 4, 5] 268 | [4, 4, 2, 5, 2, 4, 5, 6] 269 | [2, 5, 2, 4, 5, 6, 4, 4] 270 | [2, 4, 5, 6, 4, 4, 2, 5] 271 | [4, 5, 6, 4, 4, 2, 5, 2] 272 | [5, 2, 4, 5, 6, 4, 4, 2] 273 | [2, 5, 6, 6, 4, 4, 4, 2] 274 | [2, 2, 5, 6, 6, 4, 4, 4] 275 | [6, 6, 4, 4, 4, 2, 2, 5] 276 | [6, 4, 4, 4, 2, 2, 5, 6] 277 | [5, 6, 6, 4, 4, 4, 2, 2] 278 | [4, 4, 4, 2, 2, 5, 6, 6] 279 | [4, 4, 2, 2, 5, 6, 6, 4] 280 | [4, 2, 2, 5, 6, 6, 4, 4] 281 | [5, 4, 4, 2, 4, 5, 6, 2] 282 | [6, 2, 5, 4, 4, 2, 4, 5] 283 | [4, 4, 2, 4, 5, 6, 2, 5] 284 | [4, 2, 4, 5, 6, 2, 5, 4] 285 | [2, 5, 4, 4, 2, 4, 5, 6] 286 | [2, 4, 5, 6, 2, 5, 4, 4] 287 | [5, 6, 2, 5, 4, 4, 2, 4] 288 | [4, 5, 6, 2, 5, 4, 4, 2] 289 | [4, 2, 5, 4, 2, 4, 5, 6] 290 | [5, 6, 4, 2, 5, 4, 2, 4] 291 | [2, 5, 4, 2, 4, 5, 6, 4] 292 | [6, 4, 2, 5, 4, 2, 4, 5] 293 | [5, 4, 2, 4, 5, 6, 4, 2] 294 | [4, 5, 6, 4, 2, 5, 4, 2] 295 | [4, 2, 4, 5, 6, 4, 2, 5] 296 | [2, 4, 5, 6, 4, 2, 5, 4] 297 | [6, 5, 4, 2, 3, 5, 4, 2] 298 | [5, 4, 2, 6, 5, 4, 2, 3] 299 | [5, 2, 6, 5, 4, 4, 2, 3] 300 | [4, 4, 2, 3, 5, 2, 6, 5] 301 | [4, 2, 6, 5, 4, 2, 3, 5] 302 | [4, 2, 3, 5, 2, 6, 5, 4] 303 | [2, 6, 5, 4, 2, 3, 5, 4] 304 | [4, 2, 3, 5, 4, 2, 6, 5] 305 | [2, 3, 5, 4, 2, 6, 5, 4] 306 | [6, 5, 4, 4, 2, 3, 5, 2] 307 | [5, 4, 4, 2, 3, 5, 2, 6] 308 | [5, 4, 2, 3, 5, 4, 2, 6] 309 | [3, 5, 4, 2, 6, 5, 4, 2] 310 | [3, 5, 2, 6, 5, 4, 4, 2] 311 | [2, 6, 5, 4, 4, 2, 3, 5] 312 | [2, 3, 5, 2, 6, 5, 4, 4] 313 | [4, 2, 6, 5, 2, 3, 5, 4] 314 | [6, 5, 2, 3, 5, 4, 4, 2] 315 | [5, 4, 4, 2, 6, 5, 2, 3] 316 | [5, 2, 3, 5, 4, 4, 2, 6] 317 | [4, 4, 2, 6, 5, 2, 3, 5] 318 | [3, 5, 4, 4, 2, 6, 5, 2] 319 | [2, 6, 5, 2, 3, 5, 4, 4] 320 | [2, 3, 5, 4, 4, 2, 6, 5] 321 | [6, 4, 2, 3, 5, 2, 5, 4] 322 | [4, 6, 4, 2, 3, 5, 2, 5] 323 | [4, 2, 3, 5, 2, 5, 4, 6] 324 | [5, 4, 6, 4, 2, 3, 5, 2] 325 | [2, 3, 5, 2, 5, 4, 6, 4] 326 | [5, 2, 5, 4, 6, 4, 2, 3] 327 | [3, 5, 2, 5, 4, 6, 4, 2] 328 | [2, 5, 4, 6, 4, 2, 3, 5] 329 | [5, 4, 6, 2, 3, 5, 4, 2] 330 | [4, 6, 2, 3, 5, 4, 2, 5] 331 | [2, 3, 5, 4, 2, 5, 4, 6] 332 | [6, 2, 3, 5, 4, 2, 5, 4] 333 | [3, 5, 4, 2, 5, 4, 6, 2] 334 | [2, 5, 4, 6, 2, 3, 5, 4] 335 | [5, 4, 2, 5, 4, 6, 2, 3] 336 | [4, 2, 5, 4, 6, 2, 3, 5] 337 | [5, 4, 4, 6, 2, 5, 2, 3] 338 | [4, 6, 2, 5, 2, 3, 5, 4] 339 | [4, 4, 6, 2, 5, 2, 3, 5] 340 | [2, 3, 5, 2, 5, 4, 4, 6] 341 | [5, 2, 3, 5, 4, 4, 6, 2] 342 | [3, 5, 2, 5, 4, 4, 6, 2] 343 | [6, 2, 5, 2, 3, 5, 4, 4] 344 | [6, 2, 3, 5, 2, 5, 4, 4] 345 | [5, 4, 4, 6, 2, 3, 5, 2] 346 | [5, 2, 5, 4, 4, 6, 2, 3] 347 | [4, 6, 2, 3, 5, 2, 5, 4] 348 | [4, 4, 6, 2, 3, 5, 2, 5] 349 | [2, 5, 4, 4, 6, 2, 3, 5] 350 | [2, 5, 2, 3, 5, 4, 4, 6] 351 | [2, 3, 5, 4, 4, 6, 2, 5] 352 | [3, 5, 4, 4, 6, 2, 5, 2] 353 | [6, 2, 5, 4, 2, 3, 5, 4] 354 | [5, 4, 2, 3, 5, 4, 6, 2] 355 | [5, 4, 6, 2, 5, 4, 2, 3] 356 | [4, 6, 2, 5, 4, 2, 3, 5] 357 | [3, 5, 4, 6, 2, 5, 4, 2] 358 | [2, 5, 4, 2, 3, 5, 4, 6] 359 | [2, 3, 5, 4, 6, 2, 5, 4] 360 | [4, 2, 3, 5, 4, 6, 2, 5] 361 | [6, 4, 2, 5, 2, 3, 5, 4] 362 | [5, 4, 6, 4, 2, 5, 2, 3] 363 | [5, 2, 3, 5, 4, 6, 4, 2] 364 | [4, 6, 4, 2, 5, 2, 3, 5] 365 | [4, 2, 5, 2, 3, 5, 4, 6] 366 | [3, 5, 4, 6, 4, 2, 5, 2] 367 | [2, 5, 2, 3, 5, 4, 6, 4] 368 | [2, 3, 5, 4, 6, 4, 2, 5] 369 | [6, 2, 3, 3, 3, 3, 5, 4] 370 | [3, 3, 3, 3, 5, 4, 6, 2] 371 | [2, 3, 3, 3, 3, 5, 4, 6] 372 | [5, 4, 6, 2, 3, 3, 3, 3] 373 | [4, 6, 2, 3, 3, 3, 3, 5] 374 | [3, 3, 3, 5, 4, 6, 2, 3] 375 | [3, 5, 4, 6, 2, 3, 3, 3] 376 | [3, 3, 5, 4, 6, 2, 3, 3] 377 | [3, 3, 3, 3, 5, 6, 4, 2] 378 | [6, 4, 2, 3, 3, 3, 3, 5] 379 | [2, 3, 3, 3, 3, 5, 6, 4] 380 | [3, 3, 3, 5, 6, 4, 2, 3] 381 | [5, 6, 4, 2, 3, 3, 3, 3] 382 | [4, 2, 3, 3, 3, 3, 5, 6] 383 | [3, 5, 6, 4, 2, 3, 3, 3] 384 | [3, 3, 5, 6, 4, 2, 3, 3] 385 | [5, 2, 6, 5, 4, 2, 3, 3] 386 | [2, 6, 5, 4, 2, 3, 3, 5] 387 | [6, 5, 4, 2, 3, 3, 5, 2] 388 | [5, 4, 2, 3, 3, 5, 2, 6] 389 | [4, 2, 3, 3, 5, 2, 6, 5] 390 | [3, 5, 2, 6, 5, 4, 2, 3] 391 | [3, 3, 5, 2, 6, 5, 4, 2] 392 | [2, 3, 3, 5, 2, 6, 5, 4] 393 | [5, 4, 2, 6, 5, 2, 3, 3] 394 | [4, 2, 6, 5, 2, 3, 3, 5] 395 | [2, 6, 5, 2, 3, 3, 5, 4] 396 | [6, 5, 2, 3, 3, 5, 4, 2] 397 | [5, 2, 3, 3, 5, 4, 2, 6] 398 | [3, 5, 4, 2, 6, 5, 2, 3] 399 | [3, 3, 5, 4, 2, 6, 5, 2] 400 | [2, 3, 3, 5, 4, 2, 6, 5] 401 | [4, 4, 2, 2, 3, 6, 5, 6] 402 | [4, 2, 2, 3, 6, 5, 6, 4] 403 | [6, 5, 6, 4, 4, 2, 2, 3] 404 | [5, 6, 4, 4, 2, 2, 3, 6] 405 | [3, 6, 5, 6, 4, 4, 2, 2] 406 | [2, 3, 6, 5, 6, 4, 4, 2] 407 | [2, 2, 3, 6, 5, 6, 4, 4] 408 | [6, 4, 4, 2, 2, 3, 6, 5] 409 | [6, 2, 3, 3, 5, 2, 5, 4] 410 | [2, 3, 3, 5, 2, 5, 4, 6] 411 | [5, 4, 6, 2, 3, 3, 5, 2] 412 | [4, 6, 2, 3, 3, 5, 2, 5] 413 | [3, 5, 2, 5, 4, 6, 2, 3] 414 | [5, 2, 5, 4, 6, 2, 3, 3] 415 | [3, 3, 5, 2, 5, 4, 6, 2] 416 | [2, 5, 4, 6, 2, 3, 3, 5] 417 | [5, 4, 2, 3, 3, 5, 6, 2] 418 | [6, 2, 5, 4, 2, 3, 3, 5] 419 | [4, 2, 3, 3, 5, 6, 2, 5] 420 | [3, 5, 6, 2, 5, 4, 2, 3] 421 | [3, 3, 5, 6, 2, 5, 4, 2] 422 | [2, 3, 3, 5, 6, 2, 5, 4] 423 | [5, 6, 2, 5, 4, 2, 3, 3] 424 | [2, 5, 4, 2, 3, 3, 5, 6] 425 | [2, 5, 2, 5, 2, 6, 5, 4] 426 | [5, 2, 5, 2, 6, 5, 4, 2] 427 | [2, 5, 2, 6, 5, 4, 2, 5] 428 | [5, 2, 6, 5, 4, 2, 5, 2] 429 | [4, 2, 5, 2, 5, 2, 6, 5] 430 | [2, 6, 5, 4, 2, 5, 2, 5] 431 | [6, 5, 4, 2, 5, 2, 5, 2] 432 | [5, 4, 2, 5, 2, 5, 2, 6] 433 | [6, 4, 2, 3, 5, 2, 4, 5] 434 | [4, 2, 3, 5, 2, 4, 5, 6] 435 | [5, 6, 4, 2, 3, 5, 2, 4] 436 | [4, 5, 6, 4, 2, 3, 5, 2] 437 | [2, 3, 5, 2, 4, 5, 6, 4] 438 | [5, 2, 4, 5, 6, 4, 2, 3] 439 | [3, 5, 2, 4, 5, 6, 4, 2] 440 | [2, 4, 5, 6, 4, 2, 3, 5] 441 | [6, 2, 5, 2, 3, 3, 5, 4] 442 | [5, 4, 6, 2, 5, 2, 3, 3] 443 | [4, 6, 2, 5, 2, 3, 3, 5] 444 | [2, 5, 2, 3, 3, 5, 4, 6] 445 | [3, 5, 4, 6, 2, 5, 2, 3] 446 | [5, 2, 3, 3, 5, 4, 6, 2] 447 | [3, 3, 5, 4, 6, 2, 5, 2] 448 | [2, 3, 3, 5, 4, 6, 2, 5] 449 | [2, 5, 2, 6, 5, 2, 5, 4] 450 | [5, 2, 6, 5, 2, 5, 4, 2] 451 | [2, 6, 5, 2, 5, 4, 2, 5] 452 | [5, 4, 2, 5, 2, 6, 5, 2] 453 | [4, 2, 5, 2, 6, 5, 2, 5] 454 | [6, 5, 2, 5, 4, 2, 5, 2] 455 | [2, 5, 4, 2, 5, 2, 6, 5] 456 | [5, 2, 5, 4, 2, 5, 2, 6] 457 | [5, 6, 2, 3, 5, 4, 2, 4] 458 | [3, 5, 4, 2, 4, 5, 6, 2] 459 | [2, 4, 5, 6, 2, 3, 5, 4] 460 | [6, 2, 3, 5, 4, 2, 4, 5] 461 | [5, 4, 2, 4, 5, 6, 2, 3] 462 | [4, 5, 6, 2, 3, 5, 4, 2] 463 | [4, 2, 4, 5, 6, 2, 3, 5] 464 | [2, 3, 5, 4, 2, 4, 5, 6] 465 | [5, 2, 3, 3, 5, 6, 4, 2] 466 | [6, 4, 2, 5, 2, 3, 3, 5] 467 | [5, 6, 4, 2, 5, 2, 3, 3] 468 | [4, 2, 5, 2, 3, 3, 5, 6] 469 | [3, 3, 5, 6, 4, 2, 5, 2] 470 | [2, 5, 2, 3, 3, 5, 6, 4] 471 | [3, 5, 6, 4, 2, 5, 2, 3] 472 | [2, 3, 3, 5, 6, 4, 2, 5] 473 | [2, 6, 5, 2, 5, 2, 5, 4] 474 | [4, 2, 6, 5, 2, 5, 2, 5] 475 | [6, 5, 2, 5, 2, 5, 4, 2] 476 | [5, 4, 2, 6, 5, 2, 5, 2] 477 | [5, 2, 5, 4, 2, 6, 5, 2] 478 | [2, 5, 4, 2, 6, 5, 2, 5] 479 | [2, 5, 2, 5, 4, 2, 6, 5] 480 | [5, 2, 5, 2, 5, 4, 2, 6] 481 | [6, 2, 3, 3, 3, 3, 3, 5] 482 | [3, 3, 3, 3, 3, 5, 6, 2] 483 | [3, 3, 3, 3, 5, 6, 2, 3] 484 | [5, 6, 2, 3, 3, 3, 3, 3] 485 | [3, 5, 6, 2, 3, 3, 3, 3] 486 | [3, 3, 3, 5, 6, 2, 3, 3] 487 | [2, 3, 3, 3, 3, 3, 5, 6] 488 | [3, 3, 5, 6, 2, 3, 3, 3] 489 | [6, 2, 5, 2, 5, 2, 5, 4] 490 | [5, 4, 6, 2, 5, 2, 5, 2] 491 | [4, 6, 2, 5, 2, 5, 2, 5] 492 | [2, 5, 4, 6, 2, 5, 2, 5] 493 | [2, 5, 2, 5, 2, 5, 4, 6] 494 | [5, 2, 5, 4, 6, 2, 5, 2] 495 | [5, 2, 5, 2, 5, 4, 6, 2] 496 | [2, 5, 2, 5, 4, 6, 2, 5] 497 | [2, 3, 5, 2, 3, 5, 4, 6] 498 | [6, 2, 3, 5, 2, 3, 5, 4] 499 | [5, 4, 6, 2, 3, 5, 2, 3] 500 | [5, 2, 3, 5, 4, 6, 2, 3] 501 | [4, 6, 2, 3, 5, 2, 3, 5] 502 | [3, 5, 4, 6, 2, 3, 5, 2] 503 | [3, 5, 2, 3, 5, 4, 6, 2] 504 | [2, 3, 5, 4, 6, 2, 3, 5] 505 | [6, 5, 2, 3, 3, 3, 5, 2] 506 | [2, 3, 3, 3, 5, 2, 6, 5] 507 | [5, 2, 6, 5, 2, 3, 3, 3] 508 | [5, 2, 3, 3, 3, 5, 2, 6] 509 | [3, 5, 2, 6, 5, 2, 3, 3] 510 | [3, 3, 3, 5, 2, 6, 5, 2] 511 | [3, 3, 5, 2, 6, 5, 2, 3] 512 | [2, 6, 5, 2, 3, 3, 3, 5] 513 | [6, 5, 4, 2, 2, 5, 6, 2] 514 | [5, 6, 2, 6, 5, 4, 2, 2] 515 | [2, 5, 6, 2, 6, 5, 4, 2] 516 | [2, 2, 5, 6, 2, 6, 5, 4] 517 | [6, 2, 6, 5, 4, 2, 2, 5] 518 | [5, 4, 2, 2, 5, 6, 2, 6] 519 | [2, 6, 5, 4, 2, 2, 5, 6] 520 | [4, 2, 2, 5, 6, 2, 6, 5] 521 | [5, 6, 4, 2, 6, 5, 2, 2] 522 | [4, 2, 6, 5, 2, 2, 5, 6] 523 | [6, 4, 2, 6, 5, 2, 2, 5] 524 | [2, 6, 5, 2, 2, 5, 6, 4] 525 | [2, 5, 6, 4, 2, 6, 5, 2] 526 | [2, 2, 5, 6, 4, 2, 6, 5] 527 | [6, 5, 2, 2, 5, 6, 4, 2] 528 | [5, 2, 2, 5, 6, 4, 2, 6] 529 | [2, 3, 3, 5, 2, 4, 5, 6] 530 | [6, 2, 3, 3, 5, 2, 4, 5] 531 | [5, 6, 2, 3, 3, 5, 2, 4] 532 | [4, 5, 6, 2, 3, 3, 5, 2] 533 | [3, 3, 5, 2, 4, 5, 6, 2] 534 | [3, 5, 2, 4, 5, 6, 2, 3] 535 | [2, 4, 5, 6, 2, 3, 3, 5] 536 | [5, 2, 4, 5, 6, 2, 3, 3] 537 | [4, 3, 2, 3, 6, 5, 6, 2] 538 | [2, 4, 3, 2, 3, 6, 5, 6] 539 | [6, 5, 6, 2, 4, 3, 2, 3] 540 | [5, 6, 2, 4, 3, 2, 3, 6] 541 | [3, 6, 5, 6, 2, 4, 3, 2] 542 | [6, 2, 4, 3, 2, 3, 6, 5] 543 | [3, 2, 3, 6, 5, 6, 2, 4] 544 | [2, 3, 6, 5, 6, 2, 4, 3] 545 | [5, 2, 3, 3, 3, 5, 6, 2] 546 | [3, 5, 6, 2, 5, 2, 3, 3] 547 | [2, 3, 3, 3, 5, 6, 2, 5] 548 | [6, 2, 5, 2, 3, 3, 3, 5] 549 | [3, 3, 5, 6, 2, 5, 2, 3] 550 | [3, 3, 3, 5, 6, 2, 5, 2] 551 | [2, 5, 2, 3, 3, 3, 5, 6] 552 | [5, 6, 2, 5, 2, 3, 3, 3] 553 | [6, 4, 6, 2, 5, 2, 2, 5] 554 | [5, 6, 4, 6, 2, 5, 2, 2] 555 | [4, 6, 2, 5, 2, 2, 5, 6] 556 | [2, 5, 2, 2, 5, 6, 4, 6] 557 | [6, 2, 5, 2, 2, 5, 6, 4] 558 | [2, 5, 6, 4, 6, 2, 5, 2] 559 | [5, 2, 2, 5, 6, 4, 6, 2] 560 | [2, 2, 5, 6, 4, 6, 2, 5] 561 | [6, 2, 2, 5, 6, 2, 5, 4] 562 | [5, 4, 6, 2, 2, 5, 6, 2] 563 | [5, 6, 2, 5, 4, 6, 2, 2] 564 | [2, 2, 5, 6, 2, 5, 4, 6] 565 | [4, 6, 2, 2, 5, 6, 2, 5] 566 | [2, 5, 6, 2, 5, 4, 6, 2] 567 | [6, 2, 5, 4, 6, 2, 2, 5] 568 | [2, 5, 4, 6, 2, 2, 5, 6] 569 | [5, 2, 5, 2, 4, 2, 6, 5] 570 | [5, 2, 4, 2, 6, 5, 5, 2] 571 | [4, 2, 6, 5, 5, 2, 5, 2] 572 | [2, 6, 5, 5, 2, 5, 2, 4] 573 | [2, 4, 2, 6, 5, 5, 2, 5] 574 | [6, 5, 5, 2, 5, 2, 4, 2] 575 | [2, 5, 2, 4, 2, 6, 5, 5] 576 | [5, 5, 2, 5, 2, 4, 2, 6] 577 | [5, 4, 2, 6, 5, 5, 2, 2] 578 | [5, 2, 2, 5, 4, 2, 6, 5] 579 | [2, 5, 4, 2, 6, 5, 5, 2] 580 | [6, 5, 5, 2, 2, 5, 4, 2] 581 | [4, 2, 6, 5, 5, 2, 2, 5] 582 | [2, 6, 5, 5, 2, 2, 5, 4] 583 | [2, 2, 5, 4, 2, 6, 5, 5] 584 | [5, 5, 2, 2, 5, 4, 2, 6] 585 | [5, 6, 2, 5, 2, 5, 2, 4] 586 | [5, 2, 5, 2, 4, 5, 6, 2] 587 | [6, 2, 5, 2, 5, 2, 4, 5] 588 | [5, 2, 4, 5, 6, 2, 5, 2] 589 | [4, 5, 6, 2, 5, 2, 5, 2] 590 | [2, 5, 2, 5, 2, 4, 5, 6] 591 | [2, 4, 5, 6, 2, 5, 2, 5] 592 | [2, 5, 2, 4, 5, 6, 2, 5] 593 | [6, 4, 2, 5, 2, 2, 5, 6] 594 | [5, 6, 6, 4, 2, 5, 2, 2] 595 | [4, 2, 5, 2, 2, 5, 6, 6] 596 | [2, 2, 5, 6, 6, 4, 2, 5] 597 | [6, 6, 4, 2, 5, 2, 2, 5] 598 | [5, 2, 2, 5, 6, 6, 4, 2] 599 | [2, 5, 6, 6, 4, 2, 5, 2] 600 | [2, 5, 2, 2, 5, 6, 6, 4] 601 | [2, 2, 5, 6, 6, 2, 5, 4] 602 | [2, 5, 6, 6, 2, 5, 4, 2] 603 | [6, 2, 5, 4, 2, 2, 5, 6] 604 | [5, 6, 6, 2, 5, 4, 2, 2] 605 | [5, 4, 2, 2, 5, 6, 6, 2] 606 | [2, 5, 4, 2, 2, 5, 6, 6] 607 | [6, 6, 2, 5, 4, 2, 2, 5] 608 | [4, 2, 2, 5, 6, 6, 2, 5] 609 | [5, 2, 3, 3, 5, 6, 2, 3] 610 | [5, 6, 2, 3, 5, 2, 3, 3] 611 | [3, 5, 2, 3, 3, 5, 6, 2] 612 | [3, 3, 5, 6, 2, 3, 5, 2] 613 | [2, 3, 5, 2, 3, 3, 5, 6] 614 | [2, 3, 3, 5, 6, 2, 3, 5] 615 | [6, 2, 3, 5, 2, 3, 3, 5] 616 | [3, 5, 6, 2, 3, 5, 2, 3] 617 | [2, 3, 5, 2, 5, 2, 6, 5] 618 | [5, 2, 5, 2, 6, 5, 2, 3] 619 | [3, 5, 2, 5, 2, 6, 5, 2] 620 | [6, 5, 2, 3, 5, 2, 5, 2] 621 | [5, 2, 6, 5, 2, 3, 5, 2] 622 | [2, 6, 5, 2, 3, 5, 2, 5] 623 | [2, 5, 2, 6, 5, 2, 3, 5] 624 | [5, 2, 3, 5, 2, 5, 2, 6] 625 | [5, 2, 6, 5, 2, 5, 2, 3] 626 | [3, 5, 2, 6, 5, 2, 5, 2] 627 | [2, 6, 5, 2, 5, 2, 3, 5] 628 | [5, 2, 3, 5, 2, 6, 5, 2] 629 | [2, 5, 2, 3, 5, 2, 6, 5] 630 | [2, 3, 5, 2, 6, 5, 2, 5] 631 | [6, 5, 2, 5, 2, 3, 5, 2] 632 | [5, 2, 5, 2, 3, 5, 2, 6] 633 | [6, 2, 2, 3, 3, 3, 6, 5] 634 | [2, 3, 3, 3, 6, 5, 6, 2] 635 | [2, 2, 3, 3, 3, 6, 5, 6] 636 | [6, 5, 6, 2, 2, 3, 3, 3] 637 | [5, 6, 2, 2, 3, 3, 3, 6] 638 | [3, 3, 3, 6, 5, 6, 2, 2] 639 | [3, 6, 5, 6, 2, 2, 3, 3] 640 | [3, 3, 6, 5, 6, 2, 2, 3] 641 | [6, 2, 2, 5, 6, 2, 4, 5] 642 | [5, 6, 2, 4, 5, 6, 2, 2] 643 | [4, 5, 6, 2, 2, 5, 6, 2] 644 | [2, 5, 6, 2, 4, 5, 6, 2] 645 | [6, 2, 4, 5, 6, 2, 2, 5] 646 | [5, 6, 2, 2, 5, 6, 2, 4] 647 | [2, 4, 5, 6, 2, 2, 5, 6] 648 | [2, 2, 5, 6, 2, 4, 5, 6] 649 | [5, 2, 2, 3, 5, 2, 6, 5] 650 | [3, 5, 2, 6, 5, 5, 2, 2] 651 | [2, 6, 5, 5, 2, 2, 3, 5] 652 | [2, 3, 5, 2, 6, 5, 5, 2] 653 | [2, 2, 3, 5, 2, 6, 5, 5] 654 | [6, 5, 5, 2, 2, 3, 5, 2] 655 | [5, 5, 2, 2, 3, 5, 2, 6] 656 | [5, 2, 6, 5, 5, 2, 2, 3] 657 | [6, 5, 6, 2, 5, 2, 2, 3] 658 | [5, 2, 2, 3, 6, 5, 6, 2] 659 | [3, 6, 5, 6, 2, 5, 2, 2] 660 | [2, 2, 3, 6, 5, 6, 2, 5] 661 | [6, 2, 5, 2, 2, 3, 6, 5] 662 | [2, 5, 2, 2, 3, 6, 5, 6] 663 | [2, 3, 6, 5, 6, 2, 5, 2] 664 | [5, 6, 2, 5, 2, 2, 3, 6] 665 | [2, 2, 5, 6, 6, 2, 3, 5] 666 | [6, 2, 3, 5, 2, 2, 5, 6] 667 | [5, 6, 6, 2, 3, 5, 2, 2] 668 | [5, 2, 2, 5, 6, 6, 2, 3] 669 | [3, 5, 2, 2, 5, 6, 6, 2] 670 | [2, 5, 6, 6, 2, 3, 5, 2] 671 | [2, 3, 5, 2, 2, 5, 6, 6] 672 | [6, 6, 2, 3, 5, 2, 2, 5] 673 | [2, 2, 5, 2, 3, 6, 5, 6] 674 | [6, 5, 6, 2, 2, 5, 2, 3] 675 | [6, 2, 2, 5, 2, 3, 6, 5] 676 | [5, 6, 2, 2, 5, 2, 3, 6] 677 | [5, 2, 3, 6, 5, 6, 2, 2] 678 | [2, 5, 2, 3, 6, 5, 6, 2] 679 | [2, 3, 6, 5, 6, 2, 2, 5] 680 | [3, 6, 5, 6, 2, 2, 5, 2] 681 | [4, 6, 4, 0, 5, 2, 5, 6] 682 | [6, 4, 6, 4, 0, 5, 2, 5] 683 | [2, 5, 6, 4, 6, 4, 0, 5] 684 | [5, 6, 4, 6, 4, 0, 5, 2] 685 | [5, 2, 5, 6, 4, 6, 4, 0] 686 | [0, 5, 2, 5, 6, 4, 6, 4] 687 | [6, 4, 0, 5, 2, 5, 6, 4] 688 | [4, 0, 5, 2, 5, 6, 4, 6] 689 | [6, 6, 4, 2, 3, 5, 0, 5] 690 | [6, 4, 2, 3, 5, 0, 5, 6] 691 | [5, 6, 6, 4, 2, 3, 5, 0] 692 | [4, 2, 3, 5, 0, 5, 6, 6] 693 | [0, 5, 6, 6, 4, 2, 3, 5] 694 | [3, 5, 0, 5, 6, 6, 4, 2] 695 | [5, 0, 5, 6, 6, 4, 2, 3] 696 | [2, 3, 5, 0, 5, 6, 6, 4] 697 | [5, 6, 0, 6, 5, 0, 6, 3] 698 | [0, 6, 5, 0, 6, 3, 5, 6] 699 | [6, 5, 0, 6, 3, 5, 6, 0] 700 | [6, 0, 6, 5, 0, 6, 3, 5] 701 | [5, 0, 6, 3, 5, 6, 0, 6] 702 | [0, 6, 3, 5, 6, 0, 6, 5] 703 | [6, 3, 5, 6, 0, 6, 5, 0] 704 | [3, 5, 6, 0, 6, 5, 0, 6] 705 | [6, 5, 0, 3, 3, 3, 3, 5] 706 | [3, 3, 3, 5, 6, 5, 0, 3] 707 | [5, 6, 5, 0, 3, 3, 3, 3] 708 | [0, 3, 3, 3, 3, 5, 6, 5] 709 | [5, 0, 3, 3, 3, 3, 5, 6] 710 | [3, 5, 6, 5, 0, 3, 3, 3] 711 | [3, 3, 5, 6, 5, 0, 3, 3] 712 | [3, 3, 3, 3, 5, 6, 5, 0] 713 | [2, 5, 6, 2, 6, 5, 0, 5] 714 | [5, 6, 2, 6, 5, 0, 5, 2] 715 | [6, 2, 6, 5, 0, 5, 2, 5] 716 | [5, 2, 5, 6, 2, 6, 5, 0] 717 | [0, 5, 2, 5, 6, 2, 6, 5] 718 | [2, 6, 5, 0, 5, 2, 5, 6] 719 | [6, 5, 0, 5, 2, 5, 6, 2] 720 | [5, 0, 5, 2, 5, 6, 2, 6] 721 | [0, 5, 6, 6, 2, 3, 3, 5] 722 | [6, 2, 3, 3, 5, 0, 5, 6] 723 | [5, 6, 6, 2, 3, 3, 5, 0] 724 | [2, 3, 3, 5, 0, 5, 6, 6] 725 | [6, 6, 2, 3, 3, 5, 0, 5] 726 | [5, 0, 5, 6, 6, 2, 3, 3] 727 | [3, 5, 0, 5, 6, 6, 2, 3] 728 | [3, 3, 5, 0, 5, 6, 6, 2] 729 | [6, 6, 2, 2, 5, 6, 0, 5] 730 | [6, 0, 5, 6, 6, 2, 2, 5] 731 | [5, 6, 6, 2, 2, 5, 6, 0] 732 | [0, 5, 6, 6, 2, 2, 5, 6] 733 | [6, 2, 2, 5, 6, 0, 5, 6] 734 | [2, 5, 6, 0, 5, 6, 6, 2] 735 | [2, 2, 5, 6, 0, 5, 6, 6] 736 | [5, 6, 0, 5, 6, 6, 2, 2] 737 | -------------------------------------------------------------------------------- /examples/cap_set/cap_set.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "RizPOl_DRB-_" 7 | }, 8 | "source": [ 9 | "```\n", 10 | "- Copyright 2023 DeepMind Technologies Limited\n", 11 | "- All software is licensed under the Apache License, Version 2.0 (Apache 2.0); you may not use this file except in compliance with the Apache 2.0 license. You may obtain a copy of the Apache 2.0 license at: https://www.apache.org/licenses/LICENSE-2.0\n", 12 | "- All other materials are licensed under the Creative Commons Attribution 4.0 International License (CC-BY). You may obtain a copy of the CC-BY license at: https://creativecommons.org/licenses/by/4.0/legalcode\n", 13 | "- Unless required by applicable law or agreed to in writing, all software and materials distributed here under the Apache 2.0 or CC-BY licenses are distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the licenses for the specific language governing permissions and limitations under those licenses.\n", 14 | "- This is not an official Google product\n", 15 | "```" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": { 21 | "id": "LVKsiNuyfoNZ" 22 | }, 23 | "source": [ 24 | "# Cap set\n", 25 | "\n", 26 | "This notebook contains\n", 27 | "\n", 28 | "1. the *skeleton* we used for FunSearch to discover large cap sets,\n", 29 | "2. the *functions* discovered by FunSearch that construct large cap sets.\n", 30 | "\n", 31 | "## Skeleton\n", 32 | "\n", 33 | "The commented-out decorators are just a way to indicate the main entry point of the program (`@funsearch.run`) and the function that *FunSearch* should evolve (`@funsearch.evolve`)." 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": null, 39 | "metadata": { 40 | "id": "3zZ0fAe6flO_" 41 | }, 42 | "outputs": [], 43 | "source": [ 44 | "\"\"\"Finds large cap sets.\"\"\"\n", 45 | "import itertools\n", 46 | "import numpy as np\n", 47 | "\n", 48 | "\n", 49 | "# @funsearch.run\n", 50 | "def evaluate(n: int) -\u003e int:\n", 51 | " \"\"\"Returns the size of an `n`-dimensional cap set.\"\"\"\n", 52 | " capset = solve(n)\n", 53 | " return len(capset)\n", 54 | "\n", 55 | "\n", 56 | "def solve(n: int) -\u003e np.ndarray:\n", 57 | " \"\"\"Returns a large cap set in `n` dimensions.\"\"\"\n", 58 | " all_vectors = np.array(list(itertools.product((0, 1, 2), repeat=n)), dtype=np.int32)\n", 59 | "\n", 60 | " # Powers in decreasing order for compatibility with `itertools.product`, so\n", 61 | " # that the relationship `i = all_vectors[i] @ powers` holds for all `i`.\n", 62 | " powers = 3 ** np.arange(n - 1, -1, -1)\n", 63 | "\n", 64 | " # Precompute all priorities.\n", 65 | " priorities = np.array([priority(tuple(vector), n) for vector in all_vectors])\n", 66 | "\n", 67 | " # Build `capset` greedily, using priorities for prioritization.\n", 68 | " capset = np.empty(shape=(0, n), dtype=np.int32)\n", 69 | " while np.any(priorities != -np.inf):\n", 70 | " # Add a vector with maximum priority to `capset`, and set priorities of\n", 71 | " # invalidated vectors to `-inf`, so that they never get selected.\n", 72 | " max_index = np.argmax(priorities)\n", 73 | " vector = all_vectors[None, max_index] # [1, n]\n", 74 | " blocking = np.einsum('cn,n-\u003ec', (- capset - vector) % 3, powers) # [C]\n", 75 | " priorities[blocking] = -np.inf\n", 76 | " priorities[max_index] = -np.inf\n", 77 | " capset = np.concatenate([capset, vector], axis=0)\n", 78 | "\n", 79 | " return capset\n", 80 | "\n", 81 | "\n", 82 | "# @funsearch.evolve\n", 83 | "def priority(el: tuple[int, ...], n: int) -\u003e float:\n", 84 | " \"\"\"Returns the priority with which we want to add `element` to the cap set.\"\"\"\n", 85 | " return 0.0" 86 | ] 87 | }, 88 | { 89 | "cell_type": "markdown", 90 | "metadata": { 91 | "id": "QY5jPdo-g1fT" 92 | }, 93 | "source": [ 94 | "By executing the skeleton with the trivial `priority` function in place we can check that the resulting cap sets are far from optimal (e.g. recall that largest known cap set for `n = 9` has size `1082`):" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": null, 100 | "metadata": { 101 | "executionInfo": { 102 | "elapsed": 134, 103 | "status": "ok", 104 | "timestamp": 1697038278379, 105 | "user": { 106 | "displayName": "", 107 | "userId": "" 108 | }, 109 | "user_tz": -60 110 | }, 111 | "id": "1cLP6xvzfn1k", 112 | "outputId": "7b371fd6-ad19-4459-d68d-0ccfa9e2927a" 113 | }, 114 | "outputs": [ 115 | { 116 | "name": "stdout", 117 | "output_type": "stream", 118 | "text": [ 119 | "3 8\n", 120 | "4 16\n", 121 | "5 32\n", 122 | "6 64\n", 123 | "7 128\n", 124 | "8 256\n", 125 | "9 512\n" 126 | ] 127 | } 128 | ], 129 | "source": [ 130 | "for n in range(3, 9+1):\n", 131 | " print(n, evaluate(n))" 132 | ] 133 | }, 134 | { 135 | "cell_type": "markdown", 136 | "metadata": { 137 | "id": "I9-mf0aThXQl" 138 | }, 139 | "source": [ 140 | "## Discovered function that builds a $512$-cap in $n = 8$ dimensions\n", 141 | "\n", 142 | "This function discovered by FunSearch results in a cap set of size $512$ in $n = 8$ dimensions, thus improving upon the previously known best construction (which had size $496$)." 143 | ] 144 | }, 145 | { 146 | "cell_type": "code", 147 | "execution_count": null, 148 | "metadata": { 149 | "id": "k-k8WyrohG8I" 150 | }, 151 | "outputs": [], 152 | "source": [ 153 | "def priority(el: tuple[int, ...], n: int) -\u003e float:\n", 154 | " score = n\n", 155 | " in_el = 0\n", 156 | " el_count = el.count(0)\n", 157 | "\n", 158 | " if el_count == 0:\n", 159 | " score += n ** 2\n", 160 | " if el[1] == el[-1]:\n", 161 | " score *= 1.5\n", 162 | " if el[2] == el[-2]:\n", 163 | " score *= 1.5\n", 164 | " if el[3] == el[-3]:\n", 165 | " score *= 1.5\n", 166 | " else:\n", 167 | " if el[1] == el[-1]:\n", 168 | " score *= 0.5\n", 169 | " if el[2] == el[-2]:\n", 170 | " score *= 0.5\n", 171 | "\n", 172 | " for e in el:\n", 173 | " if e == 0:\n", 174 | " if in_el == 0:\n", 175 | " score *= n * 0.5\n", 176 | " elif in_el == el_count - 1:\n", 177 | " score *= 0.5\n", 178 | " else:\n", 179 | " score *= n * 0.5 ** in_el\n", 180 | " in_el += 1\n", 181 | " else:\n", 182 | " score += 1\n", 183 | "\n", 184 | " if el[1] == el[-1]:\n", 185 | " score *= 1.5\n", 186 | " if el[2] == el[-2]:\n", 187 | " score *= 1.5\n", 188 | "\n", 189 | " return score\n", 190 | "\n", 191 | "\n", 192 | "# We call the `solve` function instead of `evaluate` so that we get access to\n", 193 | "# the cap set itself (rather than just its size), for verification and\n", 194 | "# inspection purposes.\n", 195 | "cap_set_n8 = solve(8)\n", 196 | "assert cap_set_n8.shape == (512, 8)" 197 | ] 198 | }, 199 | { 200 | "cell_type": "markdown", 201 | "metadata": { 202 | "id": "uTFYifHWiEO3" 203 | }, 204 | "source": [ 205 | "We make use of a helper function to verify that the cap set is indeed valid." 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": null, 211 | "metadata": { 212 | "id": "ZvQqqXS_iDhY" 213 | }, 214 | "outputs": [], 215 | "source": [ 216 | "def is_cap_set(vectors: np.ndarray) -\u003e bool:\n", 217 | " \"\"\"Returns whether `vectors` form a valid cap set.\n", 218 | "\n", 219 | " Checking the cap set property naively takes O(c^3 n) time, where c is the size\n", 220 | " of the cap set. This function implements a faster check that runs in O(c^2 n).\n", 221 | "\n", 222 | " Args:\n", 223 | " vectors: [c, n] array containing c n-dimensional vectors over {0, 1, 2}.\n", 224 | " \"\"\"\n", 225 | " _, n = vectors.shape\n", 226 | "\n", 227 | " # Convert `vectors` elements into raveled indices (numbers in [0, 3^n) ).\n", 228 | " powers = np.array([3 ** j for j in range(n - 1, -1, -1)], dtype=int) # [n]\n", 229 | " raveled = np.einsum('in,n-\u003ei', vectors, powers) # [c]\n", 230 | "\n", 231 | " # Starting from the empty set, we iterate through `vectors` one by one and at\n", 232 | " # each step check that the vector can be inserted into the set without\n", 233 | " # violating the defining property of cap set. To make this check fast we\n", 234 | " # maintain a vector `is_blocked` indicating for each element of Z_3^n whether\n", 235 | " # that element can be inserted into the growing set without violating the cap\n", 236 | " # set property.\n", 237 | " is_blocked = np.full(shape=3 ** n, fill_value=False, dtype=bool)\n", 238 | " for i, (new_vector, new_index) in enumerate(zip(vectors, raveled)):\n", 239 | " if is_blocked[new_index]:\n", 240 | " return False # Inserting the i-th element violated the cap set property.\n", 241 | " if i \u003e= 1:\n", 242 | " # Update which elements are blocked after the insertion of `new_vector`.\n", 243 | " blocking = np.einsum(\n", 244 | " 'nk,k-\u003en',\n", 245 | " (- vectors[:i, :] - new_vector[None, :]) % 3, powers)\n", 246 | " is_blocked[blocking] = True\n", 247 | " is_blocked[new_index] = True # In case `vectors` contains duplicates.\n", 248 | " return True # All elements inserted without violating the cap set property.\n", 249 | "\n", 250 | "\n", 251 | "assert is_cap_set(cap_set_n8)" 252 | ] 253 | }, 254 | { 255 | "cell_type": "markdown", 256 | "metadata": { 257 | "id": "W74wDTgB0KOn" 258 | }, 259 | "source": [ 260 | "We can start noticing some regularities in the discovered cap set if we inspect the number of nonzero entries (weights) of each of the 512 vectors:" 261 | ] 262 | }, 263 | { 264 | "cell_type": "code", 265 | "execution_count": null, 266 | "metadata": { 267 | "executionInfo": { 268 | "elapsed": 53, 269 | "status": "ok", 270 | "timestamp": 1697038278944, 271 | "user": { 272 | "displayName": "", 273 | "userId": "" 274 | }, 275 | "user_tz": -60 276 | }, 277 | "id": "_tRWqFAVzb6R", 278 | "outputId": "f36a956e-3b53-42da-a7e7-0965fad27770" 279 | }, 280 | "outputs": [ 281 | { 282 | "name": "stdout", 283 | "output_type": "stream", 284 | "text": [ 285 | "[8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8\n", 286 | " 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8\n", 287 | " 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8\n", 288 | " 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n", 289 | " 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n", 290 | " 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n", 291 | " 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n", 292 | " 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n", 293 | " 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n", 294 | " 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n", 295 | " 4 4 4 4 4 4 4 4 4 4 4 4 4 4 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5\n", 296 | " 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5\n", 297 | " 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5\n", 298 | " 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5]\n" 299 | ] 300 | } 301 | ], 302 | "source": [ 303 | "print(np.count_nonzero(cap_set_n8, axis=1))" 304 | ] 305 | }, 306 | { 307 | "cell_type": "markdown", 308 | "metadata": { 309 | "id": "iOSNsJDQ0a_e" 310 | }, 311 | "source": [ 312 | "## Explicit construction of a $512$-cap in $n = 8$ dimensions\n", 313 | "\n", 314 | "Thanks to discovering this cap set by searching in function space and noticing some regularities like the one above, we were able to manually find the following explicit construction of this new $512$-cap. See the paper's Supplementary Information for more details.\n", 315 | "\n" 316 | ] 317 | }, 318 | { 319 | "cell_type": "code", 320 | "execution_count": null, 321 | "metadata": { 322 | "id": "9D3B5-nz1k9x" 323 | }, 324 | "outputs": [], 325 | "source": [ 326 | "def build_512_cap() -\u003e list[tuple[int, ...]]:\n", 327 | " \"\"\"Returns a cap set of size 512 in `n=8` dimensions.\"\"\"\n", 328 | " n = 8\n", 329 | " V = list(itertools.product(range(3), repeat=n))\n", 330 | " support = lambda v: tuple(i for i in range(n) if v[i] != 0)\n", 331 | " reflections = lambda v: sum(1 for i in range(1, n // 2) if v[i] == v[-i])\n", 332 | "\n", 333 | " # Add all 128 weight-8 vectors that have \u003e= 2 reflections.\n", 334 | " weight8_vectors = [v for v in V\n", 335 | " if len(support(v)) == 8 # Weight is 8.\n", 336 | " and reflections(v) \u003e= 2] # At least 2 reflections.\n", 337 | "\n", 338 | " # Add all 128 weight-4 vectors that have specific support.\n", 339 | " supports_16 = [(0, 1, 2, 3), (0, 1, 2, 5), (0, 3, 6, 7), (0, 5, 6, 7),\n", 340 | " (1, 3, 4, 6), (1, 4, 5, 6), (2, 3, 4, 7), (2, 4, 5, 7)]\n", 341 | " weight4_vectors = [v for v in V\n", 342 | " if support(v) in supports_16]\n", 343 | "\n", 344 | " # Add all 128 weight-4 vectors with specific support and 1 reflection.\n", 345 | " supports_8 = [(0, 1, 2, 7), (0, 1, 2, 6), (0, 1, 3, 7), (0, 1, 6, 7),\n", 346 | " (0, 1, 5, 7), (0, 2, 3, 6), (0, 2, 6, 7), (0, 2, 5, 6),\n", 347 | " (1, 2, 4, 7), (1, 2, 4, 6), (1, 3, 4, 7), (1, 4, 6, 7),\n", 348 | " (1, 4, 5, 7), (2, 3, 4, 6), (2, 4, 6, 7), (2, 4, 5, 6)]\n", 349 | " weight4_vectors_2 = [v for v in V\n", 350 | " if support(v) in supports_8\n", 351 | " and reflections(v) == 1] # Exactly 1 reflection.\n", 352 | "\n", 353 | " # Add 128 weight-5 vectors with \u003c= 1 reflections and one more condition.\n", 354 | " allowed_zeros = [(0, 4, 7), (0, 2, 4), (0, 1, 4), (0, 4, 6),\n", 355 | " (1, 2, 6), (2, 6, 7), (1, 2, 7), (1, 6, 7)]\n", 356 | " weight5_vectors = [\n", 357 | " v for v in V\n", 358 | " if tuple(i for i in range(n) if v[i] == 0) in allowed_zeros\n", 359 | " and reflections(v) \u003c= 1 # At most 1 reflection.\n", 360 | " and (v[1] * v[7]) % 3 != 1 and (v[2] * v[6]) % 3 != 1]\n", 361 | "\n", 362 | " return weight8_vectors + weight4_vectors + weight4_vectors_2 + weight5_vectors\n", 363 | "\n", 364 | "\n", 365 | "explicit = np.array(build_512_cap(), dtype=np.int32)\n", 366 | "assert explicit.shape == (512, 8)\n", 367 | "assert is_cap_set(explicit)\n", 368 | "# The explicit construction builds the same cap set as a set (i.e. up to\n", 369 | "# permutation of rows).\n", 370 | "assert set(map(tuple, explicit)) == set(map(tuple, cap_set_n8))" 371 | ] 372 | }, 373 | { 374 | "cell_type": "markdown", 375 | "metadata": { 376 | "id": "jKTDz_ChifB8" 377 | }, 378 | "source": [ 379 | "## Discovered function that builds a $1082$-cap in $n = 9$ dimensions\n", 380 | "\n", 381 | "This matches the previously known best construction, which involves a mathematical argument utilising a special kind of product construction. Comments in the code were added by us." 382 | ] 383 | }, 384 | { 385 | "cell_type": "code", 386 | "execution_count": null, 387 | "metadata": { 388 | "id": "uOz3hX0YiWKd" 389 | }, 390 | "outputs": [], 391 | "source": [ 392 | "def priority(el: tuple[int, ...], n: int) -\u003e float:\n", 393 | " el = np.array(el, dtype=np.float32)\n", 394 | " weight = (el @ el) % 3 # Weight (mod 3) of the full vector.\n", 395 | " a = n // 3\n", 396 | " b = n - n // 3\n", 397 | " s_1 = (el[:b] @ el[:b]) % 3 # Weight (mod 3) of first two thirds.\n", 398 | " s_3 = (2 * (el[:a] @ el[:a])) % 3 # Double norm of first third.\n", 399 | " s_4 = (el[:a] @ el[a:b]) % 3 # Cross correlation.\n", 400 | " s_5 = np.sum(el[:a] == el[-1]) % 3\n", 401 | " return - 3 ** 3 * s_1 + 3 ** 2 * weight + 3 ** 3 * s_3 + 3 ** 2 * s_4 + s_5\n", 402 | "\n", 403 | "\n", 404 | "n = 9\n", 405 | "cap_set_n9 = solve(n)\n", 406 | "assert cap_set_n9.shape == (1082, 9)\n", 407 | "assert is_cap_set(cap_set_n9)" 408 | ] 409 | } 410 | ], 411 | "metadata": { 412 | "colab": { 413 | "provenance": [] 414 | }, 415 | "kernelspec": { 416 | "display_name": "Python 3", 417 | "name": "python3" 418 | }, 419 | "language_info": { 420 | "name": "python" 421 | } 422 | }, 423 | "nbformat": 4, 424 | "nbformat_minor": 0 425 | } 426 | -------------------------------------------------------------------------------- /examples/cap_set/n8_size512.txt: -------------------------------------------------------------------------------- 1 | [1, 1, 1, 1, 1, 1, 1, 1] 2 | [1, 1, 1, 1, 2, 1, 1, 1] 3 | [1, 1, 1, 2, 1, 2, 1, 1] 4 | [1, 1, 1, 2, 2, 2, 1, 1] 5 | [1, 1, 2, 1, 1, 1, 2, 1] 6 | [1, 1, 2, 1, 2, 1, 2, 1] 7 | [1, 1, 2, 2, 1, 2, 2, 1] 8 | [1, 1, 2, 2, 2, 2, 2, 1] 9 | [1, 2, 1, 1, 1, 1, 1, 2] 10 | [1, 2, 1, 1, 2, 1, 1, 2] 11 | [1, 2, 1, 2, 1, 2, 1, 2] 12 | [1, 2, 1, 2, 2, 2, 1, 2] 13 | [1, 2, 2, 1, 1, 1, 2, 2] 14 | [1, 2, 2, 1, 2, 1, 2, 2] 15 | [1, 2, 2, 2, 1, 2, 2, 2] 16 | [1, 2, 2, 2, 2, 2, 2, 2] 17 | [2, 1, 1, 1, 1, 1, 1, 1] 18 | [2, 1, 1, 1, 2, 1, 1, 1] 19 | [2, 1, 1, 2, 1, 2, 1, 1] 20 | [2, 1, 1, 2, 2, 2, 1, 1] 21 | [2, 1, 2, 1, 1, 1, 2, 1] 22 | [2, 1, 2, 1, 2, 1, 2, 1] 23 | [2, 1, 2, 2, 1, 2, 2, 1] 24 | [2, 1, 2, 2, 2, 2, 2, 1] 25 | [2, 2, 1, 1, 1, 1, 1, 2] 26 | [2, 2, 1, 1, 2, 1, 1, 2] 27 | [2, 2, 1, 2, 1, 2, 1, 2] 28 | [2, 2, 1, 2, 2, 2, 1, 2] 29 | [2, 2, 2, 1, 1, 1, 2, 2] 30 | [2, 2, 2, 1, 2, 1, 2, 2] 31 | [2, 2, 2, 2, 1, 2, 2, 2] 32 | [2, 2, 2, 2, 2, 2, 2, 2] 33 | [1, 1, 1, 1, 1, 2, 1, 1] 34 | [1, 1, 1, 1, 2, 2, 1, 1] 35 | [1, 1, 1, 2, 1, 1, 1, 1] 36 | [1, 1, 1, 2, 2, 1, 1, 1] 37 | [1, 1, 2, 1, 1, 2, 2, 1] 38 | [1, 1, 2, 1, 2, 2, 2, 1] 39 | [1, 1, 2, 2, 1, 1, 2, 1] 40 | [1, 1, 2, 2, 2, 1, 2, 1] 41 | [1, 2, 1, 1, 1, 2, 1, 2] 42 | [1, 2, 1, 1, 2, 2, 1, 2] 43 | [1, 2, 1, 2, 1, 1, 1, 2] 44 | [1, 2, 1, 2, 2, 1, 1, 2] 45 | [1, 2, 2, 1, 1, 2, 2, 2] 46 | [1, 2, 2, 1, 2, 2, 2, 2] 47 | [1, 2, 2, 2, 1, 1, 2, 2] 48 | [1, 2, 2, 2, 2, 1, 2, 2] 49 | [2, 1, 1, 1, 1, 2, 1, 1] 50 | [2, 1, 1, 1, 2, 2, 1, 1] 51 | [2, 1, 1, 2, 1, 1, 1, 1] 52 | [2, 1, 1, 2, 2, 1, 1, 1] 53 | [2, 1, 2, 1, 1, 2, 2, 1] 54 | [2, 1, 2, 1, 2, 2, 2, 1] 55 | [2, 1, 2, 2, 1, 1, 2, 1] 56 | [2, 1, 2, 2, 2, 1, 2, 1] 57 | [2, 2, 1, 1, 1, 2, 1, 2] 58 | [2, 2, 1, 1, 2, 2, 1, 2] 59 | [2, 2, 1, 2, 1, 1, 1, 2] 60 | [2, 2, 1, 2, 2, 1, 1, 2] 61 | [2, 2, 2, 1, 1, 2, 2, 2] 62 | [2, 2, 2, 1, 2, 2, 2, 2] 63 | [2, 2, 2, 2, 1, 1, 2, 2] 64 | [2, 2, 2, 2, 2, 1, 2, 2] 65 | [1, 1, 1, 1, 1, 1, 1, 2] 66 | [1, 1, 1, 1, 1, 1, 2, 1] 67 | [1, 1, 1, 1, 2, 1, 1, 2] 68 | [1, 1, 1, 1, 2, 1, 2, 1] 69 | [1, 1, 1, 2, 1, 2, 1, 2] 70 | [1, 1, 1, 2, 1, 2, 2, 1] 71 | [1, 1, 1, 2, 2, 2, 1, 2] 72 | [1, 1, 1, 2, 2, 2, 2, 1] 73 | [1, 1, 2, 1, 1, 1, 1, 1] 74 | [1, 1, 2, 1, 1, 1, 2, 2] 75 | [1, 1, 2, 1, 2, 1, 1, 1] 76 | [1, 1, 2, 1, 2, 1, 2, 2] 77 | [1, 1, 2, 2, 1, 2, 1, 1] 78 | [1, 1, 2, 2, 1, 2, 2, 2] 79 | [1, 1, 2, 2, 2, 2, 1, 1] 80 | [1, 1, 2, 2, 2, 2, 2, 2] 81 | [1, 2, 1, 1, 1, 1, 1, 1] 82 | [1, 2, 1, 1, 1, 1, 2, 2] 83 | [1, 2, 1, 1, 2, 1, 1, 1] 84 | [1, 2, 1, 1, 2, 1, 2, 2] 85 | [1, 2, 1, 2, 1, 2, 1, 1] 86 | [1, 2, 1, 2, 1, 2, 2, 2] 87 | [1, 2, 1, 2, 2, 2, 1, 1] 88 | [1, 2, 1, 2, 2, 2, 2, 2] 89 | [1, 2, 2, 1, 1, 1, 1, 2] 90 | [1, 2, 2, 1, 1, 1, 2, 1] 91 | [1, 2, 2, 1, 2, 1, 1, 2] 92 | [1, 2, 2, 1, 2, 1, 2, 1] 93 | [1, 2, 2, 2, 1, 2, 1, 2] 94 | [1, 2, 2, 2, 1, 2, 2, 1] 95 | [1, 2, 2, 2, 2, 2, 1, 2] 96 | [1, 2, 2, 2, 2, 2, 2, 1] 97 | [2, 1, 1, 1, 1, 1, 1, 2] 98 | [2, 1, 1, 1, 1, 1, 2, 1] 99 | [2, 1, 1, 1, 2, 1, 1, 2] 100 | [2, 1, 1, 1, 2, 1, 2, 1] 101 | [2, 1, 1, 2, 1, 2, 1, 2] 102 | [2, 1, 1, 2, 1, 2, 2, 1] 103 | [2, 1, 1, 2, 2, 2, 1, 2] 104 | [2, 1, 1, 2, 2, 2, 2, 1] 105 | [2, 1, 2, 1, 1, 1, 1, 1] 106 | [2, 1, 2, 1, 1, 1, 2, 2] 107 | [2, 1, 2, 1, 2, 1, 1, 1] 108 | [2, 1, 2, 1, 2, 1, 2, 2] 109 | [2, 1, 2, 2, 1, 2, 1, 1] 110 | [2, 1, 2, 2, 1, 2, 2, 2] 111 | [2, 1, 2, 2, 2, 2, 1, 1] 112 | [2, 1, 2, 2, 2, 2, 2, 2] 113 | [2, 2, 1, 1, 1, 1, 1, 1] 114 | [2, 2, 1, 1, 1, 1, 2, 2] 115 | [2, 2, 1, 1, 2, 1, 1, 1] 116 | [2, 2, 1, 1, 2, 1, 2, 2] 117 | [2, 2, 1, 2, 1, 2, 1, 1] 118 | [2, 2, 1, 2, 1, 2, 2, 2] 119 | [2, 2, 1, 2, 2, 2, 1, 1] 120 | [2, 2, 1, 2, 2, 2, 2, 2] 121 | [2, 2, 2, 1, 1, 1, 1, 2] 122 | [2, 2, 2, 1, 1, 1, 2, 1] 123 | [2, 2, 2, 1, 2, 1, 1, 2] 124 | [2, 2, 2, 1, 2, 1, 2, 1] 125 | [2, 2, 2, 2, 1, 2, 1, 2] 126 | [2, 2, 2, 2, 1, 2, 2, 1] 127 | [2, 2, 2, 2, 2, 2, 1, 2] 128 | [2, 2, 2, 2, 2, 2, 2, 1] 129 | [1, 1, 1, 1, 0, 0, 0, 0] 130 | [1, 1, 1, 2, 0, 0, 0, 0] 131 | [1, 1, 2, 1, 0, 0, 0, 0] 132 | [1, 1, 2, 2, 0, 0, 0, 0] 133 | [1, 2, 1, 1, 0, 0, 0, 0] 134 | [1, 2, 1, 2, 0, 0, 0, 0] 135 | [1, 2, 2, 1, 0, 0, 0, 0] 136 | [1, 2, 2, 2, 0, 0, 0, 0] 137 | [2, 1, 1, 1, 0, 0, 0, 0] 138 | [2, 1, 1, 2, 0, 0, 0, 0] 139 | [2, 1, 2, 1, 0, 0, 0, 0] 140 | [2, 1, 2, 2, 0, 0, 0, 0] 141 | [2, 2, 1, 1, 0, 0, 0, 0] 142 | [2, 2, 1, 2, 0, 0, 0, 0] 143 | [2, 2, 2, 1, 0, 0, 0, 0] 144 | [2, 2, 2, 2, 0, 0, 0, 0] 145 | [1, 1, 1, 0, 0, 0, 0, 2] 146 | [1, 1, 1, 0, 0, 1, 0, 0] 147 | [1, 1, 1, 0, 0, 2, 0, 0] 148 | [1, 1, 2, 0, 0, 0, 0, 2] 149 | [1, 1, 2, 0, 0, 1, 0, 0] 150 | [1, 1, 2, 0, 0, 2, 0, 0] 151 | [1, 2, 1, 0, 0, 0, 0, 1] 152 | [1, 2, 1, 0, 0, 1, 0, 0] 153 | [1, 2, 1, 0, 0, 2, 0, 0] 154 | [1, 2, 2, 0, 0, 0, 0, 1] 155 | [1, 2, 2, 0, 0, 1, 0, 0] 156 | [1, 2, 2, 0, 0, 2, 0, 0] 157 | [2, 1, 1, 0, 0, 0, 0, 2] 158 | [2, 1, 1, 0, 0, 1, 0, 0] 159 | [2, 1, 1, 0, 0, 2, 0, 0] 160 | [2, 1, 2, 0, 0, 0, 0, 2] 161 | [2, 1, 2, 0, 0, 1, 0, 0] 162 | [2, 1, 2, 0, 0, 2, 0, 0] 163 | [2, 2, 1, 0, 0, 0, 0, 1] 164 | [2, 2, 1, 0, 0, 1, 0, 0] 165 | [2, 2, 1, 0, 0, 2, 0, 0] 166 | [2, 2, 2, 0, 0, 0, 0, 1] 167 | [2, 2, 2, 0, 0, 1, 0, 0] 168 | [2, 2, 2, 0, 0, 2, 0, 0] 169 | [1, 1, 1, 0, 0, 0, 2, 0] 170 | [1, 1, 2, 0, 0, 0, 1, 0] 171 | [1, 2, 1, 0, 0, 0, 2, 0] 172 | [1, 2, 2, 0, 0, 0, 1, 0] 173 | [2, 1, 1, 0, 0, 0, 2, 0] 174 | [2, 1, 2, 0, 0, 0, 1, 0] 175 | [2, 2, 1, 0, 0, 0, 2, 0] 176 | [2, 2, 2, 0, 0, 0, 1, 0] 177 | [1, 1, 0, 0, 0, 0, 1, 2] 178 | [1, 1, 0, 0, 0, 0, 2, 2] 179 | [1, 2, 0, 0, 0, 0, 1, 1] 180 | [1, 2, 0, 0, 0, 0, 2, 1] 181 | [2, 1, 0, 0, 0, 0, 1, 2] 182 | [2, 1, 0, 0, 0, 0, 2, 2] 183 | [2, 2, 0, 0, 0, 0, 1, 1] 184 | [2, 2, 0, 0, 0, 0, 2, 1] 185 | [1, 1, 0, 1, 0, 0, 0, 2] 186 | [1, 1, 0, 2, 0, 0, 0, 2] 187 | [1, 2, 0, 1, 0, 0, 0, 1] 188 | [1, 2, 0, 2, 0, 0, 0, 1] 189 | [2, 1, 0, 1, 0, 0, 0, 2] 190 | [2, 1, 0, 2, 0, 0, 0, 2] 191 | [2, 2, 0, 1, 0, 0, 0, 1] 192 | [2, 2, 0, 2, 0, 0, 0, 1] 193 | [1, 0, 1, 0, 0, 0, 2, 1] 194 | [1, 0, 1, 0, 0, 0, 2, 2] 195 | [1, 0, 2, 0, 0, 0, 1, 1] 196 | [1, 0, 2, 0, 0, 0, 1, 2] 197 | [2, 0, 1, 0, 0, 0, 2, 1] 198 | [2, 0, 1, 0, 0, 0, 2, 2] 199 | [2, 0, 2, 0, 0, 0, 1, 1] 200 | [2, 0, 2, 0, 0, 0, 1, 2] 201 | [1, 0, 0, 0, 0, 1, 1, 1] 202 | [1, 0, 0, 0, 0, 1, 1, 2] 203 | [1, 0, 0, 0, 0, 1, 2, 1] 204 | [1, 0, 0, 0, 0, 1, 2, 2] 205 | [1, 0, 0, 0, 0, 2, 1, 1] 206 | [1, 0, 0, 0, 0, 2, 1, 2] 207 | [1, 0, 0, 0, 0, 2, 2, 1] 208 | [1, 0, 0, 0, 0, 2, 2, 2] 209 | [1, 0, 0, 1, 0, 0, 1, 1] 210 | [1, 0, 0, 1, 0, 0, 1, 2] 211 | [1, 0, 0, 1, 0, 0, 2, 1] 212 | [1, 0, 0, 1, 0, 0, 2, 2] 213 | [1, 0, 0, 2, 0, 0, 1, 1] 214 | [1, 0, 0, 2, 0, 0, 1, 2] 215 | [1, 0, 0, 2, 0, 0, 2, 1] 216 | [1, 0, 0, 2, 0, 0, 2, 2] 217 | [2, 0, 0, 0, 0, 1, 1, 1] 218 | [2, 0, 0, 0, 0, 1, 1, 2] 219 | [2, 0, 0, 0, 0, 1, 2, 1] 220 | [2, 0, 0, 0, 0, 1, 2, 2] 221 | [2, 0, 0, 0, 0, 2, 1, 1] 222 | [2, 0, 0, 0, 0, 2, 1, 2] 223 | [2, 0, 0, 0, 0, 2, 2, 1] 224 | [2, 0, 0, 0, 0, 2, 2, 2] 225 | [2, 0, 0, 1, 0, 0, 1, 1] 226 | [2, 0, 0, 1, 0, 0, 1, 2] 227 | [2, 0, 0, 1, 0, 0, 2, 1] 228 | [2, 0, 0, 1, 0, 0, 2, 2] 229 | [2, 0, 0, 2, 0, 0, 1, 1] 230 | [2, 0, 0, 2, 0, 0, 1, 2] 231 | [2, 0, 0, 2, 0, 0, 2, 1] 232 | [2, 0, 0, 2, 0, 0, 2, 2] 233 | [1, 1, 0, 0, 0, 1, 0, 2] 234 | [1, 1, 0, 0, 0, 2, 0, 2] 235 | [1, 2, 0, 0, 0, 1, 0, 1] 236 | [1, 2, 0, 0, 0, 2, 0, 1] 237 | [2, 1, 0, 0, 0, 1, 0, 2] 238 | [2, 1, 0, 0, 0, 2, 0, 2] 239 | [2, 2, 0, 0, 0, 1, 0, 1] 240 | [2, 2, 0, 0, 0, 2, 0, 1] 241 | [0, 1, 1, 0, 1, 0, 0, 2] 242 | [0, 1, 1, 0, 2, 0, 0, 2] 243 | [0, 1, 2, 0, 1, 0, 0, 2] 244 | [0, 1, 2, 0, 2, 0, 0, 2] 245 | [0, 2, 1, 0, 1, 0, 0, 1] 246 | [0, 2, 1, 0, 2, 0, 0, 1] 247 | [0, 2, 2, 0, 1, 0, 0, 1] 248 | [0, 2, 2, 0, 2, 0, 0, 1] 249 | [0, 1, 1, 0, 1, 0, 2, 0] 250 | [0, 1, 1, 0, 2, 0, 2, 0] 251 | [0, 1, 2, 0, 1, 0, 1, 0] 252 | [0, 1, 2, 0, 2, 0, 1, 0] 253 | [0, 2, 1, 0, 1, 0, 2, 0] 254 | [0, 2, 1, 0, 2, 0, 2, 0] 255 | [0, 2, 2, 0, 1, 0, 1, 0] 256 | [0, 2, 2, 0, 2, 0, 1, 0] 257 | [0, 1, 0, 0, 1, 0, 1, 2] 258 | [0, 1, 0, 0, 1, 0, 2, 2] 259 | [0, 1, 0, 0, 2, 0, 1, 2] 260 | [0, 1, 0, 0, 2, 0, 2, 2] 261 | [0, 1, 0, 1, 1, 0, 1, 0] 262 | [0, 1, 0, 1, 1, 0, 2, 0] 263 | [0, 1, 0, 1, 2, 0, 1, 0] 264 | [0, 1, 0, 1, 2, 0, 2, 0] 265 | [0, 1, 0, 2, 1, 0, 1, 0] 266 | [0, 1, 0, 2, 1, 0, 2, 0] 267 | [0, 1, 0, 2, 2, 0, 1, 0] 268 | [0, 1, 0, 2, 2, 0, 2, 0] 269 | [0, 2, 0, 0, 1, 0, 1, 1] 270 | [0, 2, 0, 0, 1, 0, 2, 1] 271 | [0, 2, 0, 0, 2, 0, 1, 1] 272 | [0, 2, 0, 0, 2, 0, 2, 1] 273 | [0, 2, 0, 1, 1, 0, 1, 0] 274 | [0, 2, 0, 1, 1, 0, 2, 0] 275 | [0, 2, 0, 1, 2, 0, 1, 0] 276 | [0, 2, 0, 1, 2, 0, 2, 0] 277 | [0, 2, 0, 2, 1, 0, 1, 0] 278 | [0, 2, 0, 2, 1, 0, 2, 0] 279 | [0, 2, 0, 2, 2, 0, 1, 0] 280 | [0, 2, 0, 2, 2, 0, 2, 0] 281 | [0, 1, 0, 0, 1, 1, 1, 0] 282 | [0, 1, 0, 0, 1, 1, 2, 0] 283 | [0, 1, 0, 0, 1, 2, 1, 0] 284 | [0, 1, 0, 0, 1, 2, 2, 0] 285 | [0, 1, 0, 0, 2, 1, 1, 0] 286 | [0, 1, 0, 0, 2, 1, 2, 0] 287 | [0, 1, 0, 0, 2, 2, 1, 0] 288 | [0, 1, 0, 0, 2, 2, 2, 0] 289 | [0, 2, 0, 0, 1, 1, 1, 0] 290 | [0, 2, 0, 0, 1, 1, 2, 0] 291 | [0, 2, 0, 0, 1, 2, 1, 0] 292 | [0, 2, 0, 0, 1, 2, 2, 0] 293 | [0, 2, 0, 0, 2, 1, 1, 0] 294 | [0, 2, 0, 0, 2, 1, 2, 0] 295 | [0, 2, 0, 0, 2, 2, 1, 0] 296 | [0, 2, 0, 0, 2, 2, 2, 0] 297 | [1, 0, 1, 1, 0, 0, 2, 0] 298 | [1, 0, 1, 2, 0, 0, 2, 0] 299 | [1, 0, 2, 1, 0, 0, 1, 0] 300 | [1, 0, 2, 2, 0, 0, 1, 0] 301 | [2, 0, 1, 1, 0, 0, 2, 0] 302 | [2, 0, 1, 2, 0, 0, 2, 0] 303 | [2, 0, 2, 1, 0, 0, 1, 0] 304 | [2, 0, 2, 2, 0, 0, 1, 0] 305 | [0, 0, 1, 1, 1, 0, 0, 1] 306 | [0, 0, 1, 1, 1, 0, 0, 2] 307 | [0, 0, 1, 1, 2, 0, 0, 1] 308 | [0, 0, 1, 1, 2, 0, 0, 2] 309 | [0, 0, 1, 2, 1, 0, 0, 1] 310 | [0, 0, 1, 2, 1, 0, 0, 2] 311 | [0, 0, 1, 2, 2, 0, 0, 1] 312 | [0, 0, 1, 2, 2, 0, 0, 2] 313 | [0, 0, 2, 1, 1, 0, 0, 1] 314 | [0, 0, 2, 1, 1, 0, 0, 2] 315 | [0, 0, 2, 1, 2, 0, 0, 1] 316 | [0, 0, 2, 1, 2, 0, 0, 2] 317 | [0, 0, 2, 2, 1, 0, 0, 1] 318 | [0, 0, 2, 2, 1, 0, 0, 2] 319 | [0, 0, 2, 2, 2, 0, 0, 1] 320 | [0, 0, 2, 2, 2, 0, 0, 2] 321 | [0, 0, 1, 0, 1, 0, 2, 1] 322 | [0, 0, 1, 0, 1, 0, 2, 2] 323 | [0, 0, 1, 0, 2, 0, 2, 1] 324 | [0, 0, 1, 0, 2, 0, 2, 2] 325 | [0, 0, 2, 0, 1, 0, 1, 1] 326 | [0, 0, 2, 0, 1, 0, 1, 2] 327 | [0, 0, 2, 0, 2, 0, 1, 1] 328 | [0, 0, 2, 0, 2, 0, 1, 2] 329 | [0, 0, 1, 0, 1, 1, 0, 1] 330 | [0, 0, 1, 0, 1, 1, 0, 2] 331 | [0, 0, 1, 0, 1, 2, 0, 1] 332 | [0, 0, 1, 0, 1, 2, 0, 2] 333 | [0, 0, 1, 0, 2, 1, 0, 1] 334 | [0, 0, 1, 0, 2, 1, 0, 2] 335 | [0, 0, 1, 0, 2, 2, 0, 1] 336 | [0, 0, 1, 0, 2, 2, 0, 2] 337 | [0, 0, 2, 0, 1, 1, 0, 1] 338 | [0, 0, 2, 0, 1, 1, 0, 2] 339 | [0, 0, 2, 0, 1, 2, 0, 1] 340 | [0, 0, 2, 0, 1, 2, 0, 2] 341 | [0, 0, 2, 0, 2, 1, 0, 1] 342 | [0, 0, 2, 0, 2, 1, 0, 2] 343 | [0, 0, 2, 0, 2, 2, 0, 1] 344 | [0, 0, 2, 0, 2, 2, 0, 2] 345 | [1, 0, 1, 0, 0, 1, 2, 0] 346 | [1, 0, 1, 0, 0, 2, 2, 0] 347 | [1, 0, 2, 0, 0, 1, 1, 0] 348 | [1, 0, 2, 0, 0, 2, 1, 0] 349 | [2, 0, 1, 0, 0, 1, 2, 0] 350 | [2, 0, 1, 0, 0, 2, 2, 0] 351 | [2, 0, 2, 0, 0, 1, 1, 0] 352 | [2, 0, 2, 0, 0, 2, 1, 0] 353 | [0, 1, 0, 1, 1, 0, 0, 2] 354 | [0, 1, 0, 1, 2, 0, 0, 2] 355 | [0, 1, 0, 2, 1, 0, 0, 2] 356 | [0, 1, 0, 2, 2, 0, 0, 2] 357 | [0, 2, 0, 1, 1, 0, 0, 1] 358 | [0, 2, 0, 1, 2, 0, 0, 1] 359 | [0, 2, 0, 2, 1, 0, 0, 1] 360 | [0, 2, 0, 2, 2, 0, 0, 1] 361 | [0, 1, 0, 0, 1, 1, 0, 2] 362 | [0, 1, 0, 0, 1, 2, 0, 2] 363 | [0, 1, 0, 0, 2, 1, 0, 2] 364 | [0, 1, 0, 0, 2, 2, 0, 2] 365 | [0, 2, 0, 0, 1, 1, 0, 1] 366 | [0, 2, 0, 0, 1, 2, 0, 1] 367 | [0, 2, 0, 0, 2, 1, 0, 1] 368 | [0, 2, 0, 0, 2, 2, 0, 1] 369 | [0, 0, 1, 1, 1, 0, 2, 0] 370 | [0, 0, 1, 1, 2, 0, 2, 0] 371 | [0, 0, 1, 2, 1, 0, 2, 0] 372 | [0, 0, 1, 2, 2, 0, 2, 0] 373 | [0, 0, 2, 1, 1, 0, 1, 0] 374 | [0, 0, 2, 1, 2, 0, 1, 0] 375 | [0, 0, 2, 2, 1, 0, 1, 0] 376 | [0, 0, 2, 2, 2, 0, 1, 0] 377 | [0, 0, 1, 0, 1, 1, 2, 0] 378 | [0, 0, 1, 0, 1, 2, 2, 0] 379 | [0, 0, 1, 0, 2, 1, 2, 0] 380 | [0, 0, 1, 0, 2, 2, 2, 0] 381 | [0, 0, 2, 0, 1, 1, 1, 0] 382 | [0, 0, 2, 0, 1, 2, 1, 0] 383 | [0, 0, 2, 0, 2, 1, 1, 0] 384 | [0, 0, 2, 0, 2, 2, 1, 0] 385 | [1, 1, 0, 1, 1, 2, 0, 0] 386 | [1, 1, 0, 1, 2, 2, 0, 0] 387 | [1, 1, 0, 2, 1, 1, 0, 0] 388 | [1, 1, 0, 2, 2, 1, 0, 0] 389 | [1, 2, 0, 1, 1, 2, 0, 0] 390 | [1, 2, 0, 1, 2, 2, 0, 0] 391 | [1, 2, 0, 2, 1, 1, 0, 0] 392 | [1, 2, 0, 2, 2, 1, 0, 0] 393 | [2, 1, 0, 1, 1, 2, 0, 0] 394 | [2, 1, 0, 1, 2, 2, 0, 0] 395 | [2, 1, 0, 2, 1, 1, 0, 0] 396 | [2, 1, 0, 2, 2, 1, 0, 0] 397 | [2, 2, 0, 1, 1, 2, 0, 0] 398 | [2, 2, 0, 1, 2, 2, 0, 0] 399 | [2, 2, 0, 2, 1, 1, 0, 0] 400 | [2, 2, 0, 2, 2, 1, 0, 0] 401 | [1, 0, 1, 1, 1, 2, 0, 0] 402 | [1, 0, 1, 1, 2, 2, 0, 0] 403 | [1, 0, 1, 2, 1, 1, 0, 0] 404 | [1, 0, 1, 2, 2, 1, 0, 0] 405 | [1, 0, 2, 1, 1, 2, 0, 0] 406 | [1, 0, 2, 1, 2, 2, 0, 0] 407 | [1, 0, 2, 2, 1, 1, 0, 0] 408 | [1, 0, 2, 2, 2, 1, 0, 0] 409 | [2, 0, 1, 1, 1, 2, 0, 0] 410 | [2, 0, 1, 1, 2, 2, 0, 0] 411 | [2, 0, 1, 2, 1, 1, 0, 0] 412 | [2, 0, 1, 2, 2, 1, 0, 0] 413 | [2, 0, 2, 1, 1, 2, 0, 0] 414 | [2, 0, 2, 1, 2, 2, 0, 0] 415 | [2, 0, 2, 2, 1, 1, 0, 0] 416 | [2, 0, 2, 2, 2, 1, 0, 0] 417 | [0, 1, 1, 1, 0, 1, 0, 2] 418 | [0, 1, 1, 1, 0, 2, 0, 2] 419 | [0, 1, 1, 2, 0, 1, 0, 2] 420 | [0, 1, 1, 2, 0, 2, 0, 2] 421 | [0, 1, 2, 1, 0, 1, 0, 2] 422 | [0, 1, 2, 1, 0, 2, 0, 2] 423 | [0, 1, 2, 2, 0, 1, 0, 2] 424 | [0, 1, 2, 2, 0, 2, 0, 2] 425 | [0, 2, 1, 1, 0, 1, 0, 1] 426 | [0, 2, 1, 1, 0, 2, 0, 1] 427 | [0, 2, 1, 2, 0, 1, 0, 1] 428 | [0, 2, 1, 2, 0, 2, 0, 1] 429 | [0, 2, 2, 1, 0, 1, 0, 1] 430 | [0, 2, 2, 1, 0, 2, 0, 1] 431 | [0, 2, 2, 2, 0, 1, 0, 1] 432 | [0, 2, 2, 2, 0, 2, 0, 1] 433 | [0, 1, 1, 1, 0, 1, 2, 0] 434 | [0, 1, 1, 1, 0, 2, 2, 0] 435 | [0, 1, 1, 2, 0, 1, 2, 0] 436 | [0, 1, 1, 2, 0, 2, 2, 0] 437 | [0, 1, 2, 1, 0, 1, 1, 0] 438 | [0, 1, 2, 1, 0, 2, 1, 0] 439 | [0, 1, 2, 2, 0, 1, 1, 0] 440 | [0, 1, 2, 2, 0, 2, 1, 0] 441 | [0, 2, 1, 1, 0, 1, 2, 0] 442 | [0, 2, 1, 1, 0, 2, 2, 0] 443 | [0, 2, 1, 2, 0, 1, 2, 0] 444 | [0, 2, 1, 2, 0, 2, 2, 0] 445 | [0, 2, 2, 1, 0, 1, 1, 0] 446 | [0, 2, 2, 1, 0, 2, 1, 0] 447 | [0, 2, 2, 2, 0, 1, 1, 0] 448 | [0, 2, 2, 2, 0, 2, 1, 0] 449 | [0, 1, 0, 1, 0, 1, 1, 2] 450 | [0, 1, 0, 1, 0, 1, 2, 2] 451 | [0, 1, 0, 1, 0, 2, 1, 2] 452 | [0, 1, 0, 1, 0, 2, 2, 2] 453 | [0, 1, 0, 2, 0, 1, 1, 2] 454 | [0, 1, 0, 2, 0, 1, 2, 2] 455 | [0, 1, 0, 2, 0, 2, 1, 2] 456 | [0, 1, 0, 2, 0, 2, 2, 2] 457 | [0, 2, 0, 1, 0, 1, 1, 1] 458 | [0, 2, 0, 1, 0, 1, 2, 1] 459 | [0, 2, 0, 1, 0, 2, 1, 1] 460 | [0, 2, 0, 1, 0, 2, 2, 1] 461 | [0, 2, 0, 2, 0, 1, 1, 1] 462 | [0, 2, 0, 2, 0, 1, 2, 1] 463 | [0, 2, 0, 2, 0, 2, 1, 1] 464 | [0, 2, 0, 2, 0, 2, 2, 1] 465 | [0, 0, 1, 1, 0, 1, 2, 1] 466 | [0, 0, 1, 1, 0, 1, 2, 2] 467 | [0, 0, 1, 1, 0, 2, 2, 1] 468 | [0, 0, 1, 1, 0, 2, 2, 2] 469 | [0, 0, 1, 2, 0, 1, 2, 1] 470 | [0, 0, 1, 2, 0, 1, 2, 2] 471 | [0, 0, 1, 2, 0, 2, 2, 1] 472 | [0, 0, 1, 2, 0, 2, 2, 2] 473 | [0, 0, 2, 1, 0, 1, 1, 1] 474 | [0, 0, 2, 1, 0, 1, 1, 2] 475 | [0, 0, 2, 1, 0, 2, 1, 1] 476 | [0, 0, 2, 1, 0, 2, 1, 2] 477 | [0, 0, 2, 2, 0, 1, 1, 1] 478 | [0, 0, 2, 2, 0, 1, 1, 2] 479 | [0, 0, 2, 2, 0, 2, 1, 1] 480 | [0, 0, 2, 2, 0, 2, 1, 2] 481 | [1, 0, 0, 1, 1, 2, 0, 1] 482 | [1, 0, 0, 1, 1, 2, 0, 2] 483 | [1, 0, 0, 1, 2, 2, 0, 1] 484 | [1, 0, 0, 1, 2, 2, 0, 2] 485 | [1, 0, 0, 2, 1, 1, 0, 1] 486 | [1, 0, 0, 2, 1, 1, 0, 2] 487 | [1, 0, 0, 2, 2, 1, 0, 1] 488 | [1, 0, 0, 2, 2, 1, 0, 2] 489 | [2, 0, 0, 1, 1, 2, 0, 1] 490 | [2, 0, 0, 1, 1, 2, 0, 2] 491 | [2, 0, 0, 1, 2, 2, 0, 1] 492 | [2, 0, 0, 1, 2, 2, 0, 2] 493 | [2, 0, 0, 2, 1, 1, 0, 1] 494 | [2, 0, 0, 2, 1, 1, 0, 2] 495 | [2, 0, 0, 2, 2, 1, 0, 1] 496 | [2, 0, 0, 2, 2, 1, 0, 2] 497 | [1, 0, 0, 1, 1, 2, 1, 0] 498 | [1, 0, 0, 1, 1, 2, 2, 0] 499 | [1, 0, 0, 1, 2, 2, 1, 0] 500 | [1, 0, 0, 1, 2, 2, 2, 0] 501 | [1, 0, 0, 2, 1, 1, 1, 0] 502 | [1, 0, 0, 2, 1, 1, 2, 0] 503 | [1, 0, 0, 2, 2, 1, 1, 0] 504 | [1, 0, 0, 2, 2, 1, 2, 0] 505 | [2, 0, 0, 1, 1, 2, 1, 0] 506 | [2, 0, 0, 1, 1, 2, 2, 0] 507 | [2, 0, 0, 1, 2, 2, 1, 0] 508 | [2, 0, 0, 1, 2, 2, 2, 0] 509 | [2, 0, 0, 2, 1, 1, 1, 0] 510 | [2, 0, 0, 2, 1, 1, 2, 0] 511 | [2, 0, 0, 2, 2, 1, 1, 0] 512 | [2, 0, 0, 2, 2, 1, 2, 0] 513 | -------------------------------------------------------------------------------- /examples/cap_set_spec.py: -------------------------------------------------------------------------------- 1 | """Finds large cap sets. 2 | 3 | On every iteration, improve priority_v1 over the priority_vX methods from previous iterations. 4 | Make only small changes. 5 | Try to make the code short. 6 | """ 7 | import itertools 8 | 9 | import numpy as np 10 | 11 | import funsearch 12 | 13 | 14 | @funsearch.run 15 | def evaluate(n: int) -> int: 16 | """Returns the size of an `n`-dimensional cap set.""" 17 | capset = solve(n) 18 | return len(capset) 19 | 20 | 21 | def solve(n: int) -> np.ndarray: 22 | """Returns a large cap set in `n` dimensions.""" 23 | all_vectors = np.array(list(itertools.product((0, 1, 2), repeat=n)), dtype=np.int32) 24 | 25 | # Powers in decreasing order for compatibility with `itertools.product`, so 26 | # that the relationship `i = all_vectors[i] @ powers` holds for all `i`. 27 | powers = 3 ** np.arange(n - 1, -1, -1) 28 | 29 | # Precompute all priorities. 30 | priorities = np.array([priority(tuple(vector), n) for vector in all_vectors]) 31 | 32 | # Build `capset` greedily, using priorities for prioritization. 33 | capset = np.empty(shape=(0, n), dtype=np.int32) 34 | while np.any(priorities != -np.inf): 35 | # Add a vector with maximum priority to `capset`, and set priorities of 36 | # invalidated vectors to `-inf`, so that they never get selected. 37 | max_index = np.argmax(priorities) 38 | vector = all_vectors[None, max_index] # [1, n] 39 | blocking = np.einsum('cn,n->c', (- capset - vector) % 3, powers) # [C] 40 | priorities[blocking] = -np.inf 41 | priorities[max_index] = -np.inf 42 | capset = np.concatenate([capset, vector], axis=0) 43 | 44 | return capset 45 | 46 | 47 | @funsearch.evolve 48 | def priority(el: tuple[int, ...], n: int) -> float: 49 | """Returns the priority with which we want to add `element` to the cap set. 50 | el is a tuple of length n with values 0-2. 51 | """ 52 | return 0.0 -------------------------------------------------------------------------------- /examples/corner_free_set/f2_n4_size137.txt: -------------------------------------------------------------------------------- 1 | [2, 1, 0, 1] 2 | [2, 1, 0, 3] 3 | [3, 3, 1, 1] 4 | [2, 1, 1, 3] 5 | [3, 3, 0, 1] 6 | [3, 3, 1, 3] 7 | [3, 3, 0, 3] 8 | [2, 1, 1, 1] 9 | [2, 2, 3, 1] 10 | [2, 0, 2, 3] 11 | [3, 2, 3, 1] 12 | [3, 2, 2, 3] 13 | [2, 2, 2, 3] 14 | [2, 0, 3, 1] 15 | [3, 0, 3, 1] 16 | [3, 0, 2, 3] 17 | [3, 3, 3, 3] 18 | [3, 1, 2, 3] 19 | [3, 1, 3, 1] 20 | [3, 3, 2, 1] 21 | [2, 1, 2, 1] 22 | [2, 3, 2, 3] 23 | [2, 3, 3, 1] 24 | [2, 1, 3, 3] 25 | [2, 2, 0, 3] 26 | [3, 0, 1, 1] 27 | [2, 2, 1, 1] 28 | [2, 2, 0, 1] 29 | [3, 0, 1, 3] 30 | [3, 0, 0, 3] 31 | [3, 2, 1, 1] 32 | [2, 0, 1, 3] 33 | [2, 0, 0, 3] 34 | [2, 0, 1, 1] 35 | [3, 2, 0, 3] 36 | [2, 0, 0, 1] 37 | [3, 0, 0, 1] 38 | [3, 2, 0, 1] 39 | [2, 2, 1, 3] 40 | [3, 2, 1, 3] 41 | [1, 0, 2, 0] 42 | [0, 2, 3, 2] 43 | [1, 0, 3, 2] 44 | [1, 0, 3, 0] 45 | [0, 2, 3, 0] 46 | [1, 0, 2, 2] 47 | [0, 2, 2, 2] 48 | [0, 2, 2, 0] 49 | [0, 3, 1, 2] 50 | [1, 1, 0, 0] 51 | [0, 3, 0, 0] 52 | [1, 1, 1, 2] 53 | [0, 1, 0, 0] 54 | [1, 3, 0, 0] 55 | [0, 1, 1, 2] 56 | [1, 3, 1, 2] 57 | [1, 1, 3, 1] 58 | [1, 1, 2, 3] 59 | [0, 1, 2, 3] 60 | [0, 3, 2, 3] 61 | [0, 1, 3, 1] 62 | [1, 3, 2, 3] 63 | [1, 3, 3, 1] 64 | [0, 3, 3, 1] 65 | [0, 3, 2, 0] 66 | [1, 3, 3, 2] 67 | [1, 3, 3, 0] 68 | [1, 1, 2, 0] 69 | [1, 1, 3, 0] 70 | [1, 1, 3, 2] 71 | [1, 1, 2, 2] 72 | [1, 3, 2, 2] 73 | [1, 3, 2, 0] 74 | [0, 1, 2, 0] 75 | [0, 1, 2, 2] 76 | [0, 1, 3, 0] 77 | [0, 1, 3, 2] 78 | [0, 3, 2, 2] 79 | [0, 3, 3, 0] 80 | [0, 3, 3, 2] 81 | [3, 3, 3, 2] 82 | [3, 3, 3, 0] 83 | [3, 3, 2, 2] 84 | [3, 3, 2, 0] 85 | [0, 0, 0, 0] 86 | [2, 1, 3, 0] 87 | [2, 1, 2, 0] 88 | [2, 1, 2, 2] 89 | [0, 2, 1, 1] 90 | [0, 2, 1, 0] 91 | [0, 2, 1, 3] 92 | [2, 1, 3, 2] 93 | [1, 0, 1, 0] 94 | [2, 2, 0, 0] 95 | [1, 0, 0, 3] 96 | [0, 2, 0, 3] 97 | [1, 0, 1, 1] 98 | [2, 0, 0, 0] 99 | [0, 2, 0, 2] 100 | [0, 2, 0, 1] 101 | [2, 0, 1, 2] 102 | [1, 0, 0, 2] 103 | [1, 0, 0, 1] 104 | [0, 0, 1, 2] 105 | [2, 2, 1, 2] 106 | [1, 0, 1, 3] 107 | [1, 2, 1, 2] 108 | [3, 0, 1, 2] 109 | [3, 2, 1, 2] 110 | [1, 2, 0, 0] 111 | [3, 0, 0, 0] 112 | [1, 3, 1, 3] 113 | [2, 0, 2, 2] 114 | [2, 0, 3, 0] 115 | [1, 3, 1, 1] 116 | [2, 0, 3, 2] 117 | [1, 3, 0, 3] 118 | [1, 2, 3, 1] 119 | [2, 2, 2, 0] 120 | [0, 0, 2, 3] 121 | [2, 2, 2, 2] 122 | [0, 1, 0, 1] 123 | [0, 1, 0, 3] 124 | [2, 2, 3, 2] 125 | [0, 1, 1, 1] 126 | [0, 2, 3, 3] 127 | [0, 3, 0, 1] 128 | [3, 2, 3, 0] 129 | [3, 2, 2, 2] 130 | [3, 2, 2, 0] 131 | [3, 0, 3, 2] 132 | [3, 0, 2, 0] 133 | [1, 1, 1, 3] 134 | [1, 1, 0, 1] 135 | [1, 0, 2, 1] 136 | [0, 3, 1, 3] 137 | [3, 1, 1, 2] 138 | -------------------------------------------------------------------------------- /examples/corner_free_set/f3_n2_size53.txt: -------------------------------------------------------------------------------- 1 | [8, 4] 2 | [4, 8] 3 | [4, 4] 4 | [4, 1] 5 | [4, 3] 6 | [1, 4] 7 | [3, 4] 8 | [0, 0] 9 | [8, 6] 10 | [8, 2] 11 | [0, 8] 12 | [4, 6] 13 | [0, 4] 14 | [8, 0] 15 | [4, 0] 16 | [0, 7] 17 | [0, 5] 18 | [0, 1] 19 | [8, 8] 20 | [7, 0] 21 | [5, 0] 22 | [3, 0] 23 | [2, 8] 24 | [6, 8] 25 | [8, 7] 26 | [6, 4] 27 | [8, 3] 28 | [1, 2] 29 | [1, 6] 30 | [3, 2] 31 | [3, 6] 32 | [4, 5] 33 | [7, 8] 34 | [5, 4] 35 | [3, 8] 36 | [0, 6] 37 | [5, 7] 38 | [7, 1] 39 | [7, 3] 40 | [3, 5] 41 | [1, 7] 42 | [1, 5] 43 | [6, 3] 44 | [3, 1] 45 | [1, 3] 46 | [1, 1] 47 | [2, 0] 48 | [7, 2] 49 | [5, 6] 50 | [6, 6] 51 | [2, 2] 52 | [2, 7] 53 | [2, 5] 54 | -------------------------------------------------------------------------------- /examples/corner_free_set/f3_n3_size370.txt: -------------------------------------------------------------------------------- 1 | [8, 8, 8] 2 | [4, 8, 8] 3 | [8, 8, 4] 4 | [0, 8, 8] 5 | [8, 8, 0] 6 | [8, 8, 6] 7 | [8, 8, 7] 8 | [8, 6, 8] 9 | [6, 8, 8] 10 | [7, 8, 8] 11 | [8, 7, 8] 12 | [8, 4, 8] 13 | [5, 8, 8] 14 | [4, 4, 8] 15 | [4, 8, 4] 16 | [8, 4, 4] 17 | [4, 4, 4] 18 | [8, 0, 8] 19 | [4, 0, 8] 20 | [0, 4, 8] 21 | [8, 0, 4] 22 | [0, 8, 4] 23 | [8, 4, 0] 24 | [4, 8, 0] 25 | [0, 4, 4] 26 | [8, 6, 4] 27 | [4, 0, 4] 28 | [4, 8, 6] 29 | [4, 4, 0] 30 | [4, 8, 7] 31 | [8, 7, 4] 32 | [8, 8, 3] 33 | [0, 0, 8] 34 | [0, 8, 0] 35 | [8, 0, 0] 36 | [0, 0, 4] 37 | [8, 6, 0] 38 | [0, 8, 6] 39 | [0, 4, 0] 40 | [4, 0, 0] 41 | [0, 8, 7] 42 | [8, 7, 0] 43 | [1, 8, 8] 44 | [8, 8, 2] 45 | [4, 6, 8] 46 | [6, 8, 4] 47 | [0, 0, 0] 48 | [4, 7, 8] 49 | [7, 8, 4] 50 | [8, 5, 8] 51 | [0, 6, 8] 52 | [4, 5, 8] 53 | [5, 8, 4] 54 | [6, 8, 0] 55 | [0, 7, 8] 56 | [6, 8, 6] 57 | [8, 6, 6] 58 | [7, 8, 0] 59 | [6, 8, 7] 60 | [8, 6, 7] 61 | [7, 8, 6] 62 | [8, 7, 6] 63 | [0, 5, 8] 64 | [6, 4, 8] 65 | [8, 7, 7] 66 | [7, 8, 7] 67 | [5, 8, 0] 68 | [8, 4, 6] 69 | [7, 4, 8] 70 | [8, 4, 7] 71 | [6, 4, 4] 72 | [4, 6, 4] 73 | [4, 4, 6] 74 | [5, 8, 6] 75 | [3, 4, 8] 76 | [4, 4, 7] 77 | [4, 7, 4] 78 | [8, 3, 4] 79 | [7, 4, 4] 80 | [5, 8, 7] 81 | [4, 8, 3] 82 | [8, 4, 3] 83 | [6, 0, 8] 84 | [4, 3, 4] 85 | [3, 4, 4] 86 | [8, 0, 6] 87 | [4, 4, 3] 88 | [6, 6, 8] 89 | [7, 0, 8] 90 | [8, 0, 7] 91 | [6, 0, 4] 92 | [0, 6, 4] 93 | [4, 6, 0] 94 | [0, 4, 6] 95 | [4, 0, 6] 96 | [6, 4, 0] 97 | [8, 1, 8] 98 | [7, 6, 8] 99 | [6, 7, 8] 100 | [3, 0, 8] 101 | [7, 0, 4] 102 | [0, 7, 4] 103 | [4, 0, 7] 104 | [0, 4, 7] 105 | [8, 0, 3] 106 | [7, 4, 0] 107 | [0, 8, 3] 108 | [8, 3, 0] 109 | [4, 7, 0] 110 | [7, 7, 8] 111 | [4, 1, 8] 112 | [1, 8, 4] 113 | [3, 0, 4] 114 | [0, 3, 4] 115 | [0, 4, 3] 116 | [3, 4, 0] 117 | [4, 0, 3] 118 | [8, 6, 3] 119 | [4, 3, 0] 120 | [5, 6, 8] 121 | [4, 8, 2] 122 | [8, 4, 2] 123 | [2, 4, 8] 124 | [8, 2, 4] 125 | [0, 6, 0] 126 | [6, 0, 0] 127 | [0, 0, 6] 128 | [8, 7, 3] 129 | [4, 4, 2] 130 | [5, 7, 8] 131 | [0, 0, 7] 132 | [4, 2, 4] 133 | [2, 4, 4] 134 | [7, 0, 0] 135 | [0, 7, 0] 136 | [0, 1, 8] 137 | [1, 8, 0] 138 | [0, 0, 3] 139 | [3, 0, 0] 140 | [0, 3, 0] 141 | [8, 0, 2] 142 | [0, 8, 2] 143 | [2, 0, 8] 144 | [8, 2, 0] 145 | [1, 8, 6] 146 | [8, 6, 2] 147 | [0, 4, 2] 148 | [4, 0, 2] 149 | [0, 2, 4] 150 | [2, 0, 4] 151 | [1, 8, 7] 152 | [6, 6, 4] 153 | [2, 4, 0] 154 | [4, 2, 0] 155 | [4, 6, 6] 156 | [8, 7, 2] 157 | [4, 6, 7] 158 | [6, 7, 4] 159 | [7, 6, 4] 160 | [4, 7, 6] 161 | [6, 8, 3] 162 | [6, 5, 8] 163 | [4, 7, 7] 164 | [7, 7, 4] 165 | [8, 5, 6] 166 | [7, 8, 3] 167 | [0, 0, 2] 168 | [7, 5, 8] 169 | [8, 5, 7] 170 | [5, 6, 4] 171 | [6, 6, 0] 172 | [0, 6, 6] 173 | [2, 0, 0] 174 | [0, 2, 0] 175 | [4, 5, 6] 176 | [3, 5, 8] 177 | [0, 6, 7] 178 | [4, 5, 7] 179 | [5, 7, 4] 180 | [7, 6, 0] 181 | [0, 7, 6] 182 | [6, 7, 0] 183 | [5, 8, 3] 184 | [1, 6, 8] 185 | [0, 7, 7] 186 | [7, 7, 0] 187 | [6, 8, 2] 188 | [1, 7, 8] 189 | [0, 5, 6] 190 | [5, 6, 0] 191 | [6, 4, 6] 192 | [7, 8, 2] 193 | [0, 5, 7] 194 | [6, 4, 7] 195 | [5, 7, 0] 196 | [7, 4, 6] 197 | [3, 2, 8] 198 | [7, 4, 7] 199 | [6, 3, 4] 200 | [4, 6, 3] 201 | [3, 4, 6] 202 | [6, 4, 3] 203 | [2, 8, 5] 204 | [2, 5, 8] 205 | [3, 4, 7] 206 | [7, 3, 4] 207 | [6, 0, 6] 208 | [4, 7, 3] 209 | [8, 3, 3] 210 | [7, 4, 3] 211 | [8, 3, 5] 212 | [6, 0, 7] 213 | [3, 3, 4] 214 | [7, 0, 6] 215 | [3, 4, 3] 216 | [4, 3, 3] 217 | [6, 6, 6] 218 | [4, 3, 5] 219 | [6, 1, 8] 220 | [6, 6, 7] 221 | [3, 5, 4] 222 | [7, 0, 7] 223 | [6, 0, 3] 224 | [0, 6, 3] 225 | [6, 7, 6] 226 | [6, 3, 0] 227 | [3, 0, 6] 228 | [7, 6, 6] 229 | [8, 1, 6] 230 | [7, 1, 8] 231 | [8, 1, 7] 232 | [1, 6, 4] 233 | [7, 6, 7] 234 | [6, 7, 7] 235 | [3, 0, 7] 236 | [7, 3, 0] 237 | [7, 7, 6] 238 | [7, 0, 3] 239 | [4, 1, 6] 240 | [0, 7, 3] 241 | [6, 4, 2] 242 | [3, 1, 8] 243 | [4, 6, 2] 244 | [2, 8, 2] 245 | [1, 7, 4] 246 | [6, 2, 4] 247 | [4, 1, 7] 248 | [7, 7, 7] 249 | [3, 8, 1] 250 | [3, 3, 0] 251 | [0, 3, 3] 252 | [5, 6, 6] 253 | [3, 0, 3] 254 | [2, 4, 6] 255 | [0, 3, 5] 256 | [8, 3, 2] 257 | [7, 4, 2] 258 | [1, 8, 5] 259 | [4, 7, 2] 260 | [2, 3, 8] 261 | [5, 6, 7] 262 | [2, 4, 7] 263 | [7, 2, 4] 264 | [3, 5, 0] 265 | [5, 7, 6] 266 | [8, 2, 3] 267 | [4, 3, 2] 268 | [3, 4, 2] 269 | [2, 3, 4] 270 | [3, 2, 4] 271 | [5, 7, 7] 272 | [4, 2, 3] 273 | [2, 4, 3] 274 | [0, 1, 6] 275 | [1, 6, 0] 276 | [6, 0, 2] 277 | [0, 6, 2] 278 | [0, 1, 7] 279 | [6, 2, 0] 280 | [2, 0, 6] 281 | [1, 7, 0] 282 | [7, 0, 2] 283 | [0, 7, 2] 284 | [2, 0, 7] 285 | [7, 2, 0] 286 | [0, 3, 2] 287 | [3, 0, 2] 288 | [2, 1, 8] 289 | [8, 2, 1] 290 | [2, 3, 0] 291 | [3, 2, 0] 292 | [0, 2, 3] 293 | [6, 6, 3] 294 | [2, 0, 3] 295 | [8, 2, 2] 296 | [4, 2, 1] 297 | [2, 1, 4] 298 | [6, 7, 3] 299 | [7, 6, 3] 300 | [6, 5, 6] 301 | [2, 4, 2] 302 | [4, 2, 2] 303 | [6, 5, 7] 304 | [2, 2, 4] 305 | [7, 7, 3] 306 | [7, 5, 6] 307 | [7, 5, 7] 308 | [5, 6, 3] 309 | [3, 5, 6] 310 | [0, 2, 1] 311 | [3, 5, 7] 312 | [1, 6, 6] 313 | [2, 1, 0] 314 | [5, 7, 3] 315 | [0, 2, 2] 316 | [2, 0, 2] 317 | [6, 6, 2] 318 | [1, 6, 7] 319 | [1, 7, 6] 320 | [2, 2, 0] 321 | [6, 7, 2] 322 | [7, 6, 2] 323 | [1, 7, 7] 324 | [7, 7, 2] 325 | [3, 2, 6] 326 | [5, 6, 2] 327 | [3, 2, 7] 328 | [6, 3, 3] 329 | [2, 5, 6] 330 | [6, 3, 5] 331 | [5, 7, 2] 332 | [2, 5, 7] 333 | [7, 3, 3] 334 | [7, 3, 5] 335 | [6, 1, 6] 336 | [3, 3, 3] 337 | [3, 3, 5] 338 | [6, 1, 7] 339 | [3, 5, 3] 340 | [7, 1, 6] 341 | [7, 1, 7] 342 | [1, 6, 3] 343 | [3, 1, 6] 344 | [6, 3, 2] 345 | [1, 7, 3] 346 | [2, 3, 6] 347 | [6, 2, 3] 348 | [7, 3, 2] 349 | [2, 3, 7] 350 | [3, 1, 3] 351 | [7, 2, 3] 352 | [3, 3, 2] 353 | [2, 3, 3] 354 | [3, 5, 2] 355 | [3, 2, 5] 356 | [1, 6, 2] 357 | [6, 2, 1] 358 | [2, 1, 6] 359 | [6, 2, 2] 360 | [1, 7, 2] 361 | [7, 2, 1] 362 | [2, 1, 7] 363 | [7, 2, 2] 364 | [3, 2, 1] 365 | [2, 1, 3] 366 | [2, 3, 2] 367 | [2, 2, 3] 368 | [5, 2, 2] 369 | [2, 1, 2] 370 | [1, 2, 2] 371 | -------------------------------------------------------------------------------- /examples/cyclic_graphs/cyclic_graphs.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "PlhRSC3iWzA1" 7 | }, 8 | "source": [ 9 | "```\n", 10 | "- Copyright 2023 DeepMind Technologies Limited\n", 11 | "- All software is licensed under the Apache License, Version 2.0 (Apache 2.0); you may not use this file except in compliance with the Apache 2.0 license. You may obtain a copy of the Apache 2.0 license at: https://www.apache.org/licenses/LICENSE-2.0\n", 12 | "- All other materials are licensed under the Creative Commons Attribution 4.0 International License (CC-BY). You may obtain a copy of the CC-BY license at: https://creativecommons.org/licenses/by/4.0/legalcode\n", 13 | "- Unless required by applicable law or agreed to in writing, all software and materials distributed here under the Apache 2.0 or CC-BY licenses are distributed on an \\\"AS IS\\\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the licenses for the specific language governing permissions and limitations under those licenses.\n", 14 | "- This is not an official Google product\n", 15 | "```" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": { 21 | "id": "3LCquqjVWy0e" 22 | }, 23 | "source": [ 24 | "# Cyclic graphs\n", 25 | "\n", 26 | "This notebook contains:\n", 27 | "1. the *skeleton* we used for FunSearch to discover large independent sets in the $n$-th strong product of cyclic graphs,\n", 28 | "2. the *functions* discovered by FunSearch that construct those independent sets.\n", 29 | "\n", 30 | "## Skeleton\n", 31 | "\n", 32 | "The commented-out decorators are just a way to indicate the main entry point of the program (`@funsearch.run`) and the function that *FunSearch* should evolve (`@funsearch.evolve`)." 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": null, 38 | "metadata": { 39 | "id": "aIkfPyeLXB4n" 40 | }, 41 | "outputs": [], 42 | "source": [ 43 | "\"\"\"Obtains maximal independent sets.\"\"\"\n", 44 | "import itertools\n", 45 | "import numpy as np\n", 46 | "\n", 47 | "\n", 48 | "# @funsearch.run\n", 49 | "def evaluate(num_nodes: int, n: int) -\u003e int:\n", 50 | " \"\"\"Returns the size of an independent set.\"\"\"\n", 51 | " independent_set = solve(num_nodes, n)\n", 52 | " return len(independent_set)\n", 53 | "\n", 54 | "\n", 55 | "def solve(num_nodes: int, n: int) -\u003e list[tuple[int, ...]]:\n", 56 | " \"\"\"Gets independent set with maximal size.\n", 57 | "\n", 58 | " Args:\n", 59 | " num_nodes: The number of nodes of the base cyclic graph.\n", 60 | " n: The power we raise the graph to.\n", 61 | "\n", 62 | " Returns:\n", 63 | " A list of `n`-tuples in `{0, 1, 2, ..., num_nodes - 1}`.\n", 64 | " \"\"\"\n", 65 | " to_block = np.array(list(itertools.product([-1, 0, 1], repeat=n)))\n", 66 | "\n", 67 | " # Powers in decreasing order for compatibility with `itertools.product`, so\n", 68 | " # that the relationship `i = children[i] @ powers` holds for all `i`.\n", 69 | " powers = num_nodes ** np.arange(n - 1, -1, -1)\n", 70 | "\n", 71 | " # Precompute the priority scores.\n", 72 | " children = np.array(\n", 73 | " list(itertools.product(range(num_nodes), repeat=n)), dtype=np.int32)\n", 74 | " scores = np.array([priority(tuple(child), num_nodes, n)\n", 75 | " for child in children])\n", 76 | "\n", 77 | " # Build `max_set` greedily, using scores for prioritization.\n", 78 | " max_set = np.empty(shape=(0, n), dtype=np.int32)\n", 79 | " while np.any(scores != -np.inf):\n", 80 | " # Add a child with a maximum score to `max_set`, and set scores of\n", 81 | " # invalidated children to -inf, so that they never get selected.\n", 82 | " max_index = np.argmax(scores)\n", 83 | " child = children[None, max_index] # [1, n]\n", 84 | "\n", 85 | " blocking = np.einsum(\n", 86 | " 'cn,n-\u003ec', (to_block + child) % num_nodes, powers) # [C]\n", 87 | " scores[blocking] = -np.inf\n", 88 | " max_set = np.concatenate([max_set, child], axis=0)\n", 89 | "\n", 90 | " return [tuple(map(int, el)) for el in max_set]\n", 91 | "\n", 92 | "\n", 93 | "# @funsearch.evolve\n", 94 | "def priority(el: tuple[int, ...], num_nodes: int, n: int) -\u003e float:\n", 95 | " \"\"\"Returns the priority with which we want to add `el` to the set.\n", 96 | "\n", 97 | " Args:\n", 98 | " el: an n-tuple representing the element to consider whether to add.\n", 99 | " num_nodes: the number of nodes of the base graph.\n", 100 | " n: an integer, power of the graph.\n", 101 | "\n", 102 | " Returns:\n", 103 | " A number reflecting the priority with which we want to add `el` to the\n", 104 | " independent set.\n", 105 | " \"\"\"\n", 106 | " return 0." 107 | ] 108 | }, 109 | { 110 | "cell_type": "markdown", 111 | "metadata": { 112 | "id": "aLkn0zVUYSbk" 113 | }, 114 | "source": [ 115 | "By executing the skeleton with the trivial `priority` function in place we can check that the resulting independent sets are far from optimal (e.g., the best known construction for the 5th strong product of the 7-node graph has size 367):\n" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "metadata": { 122 | "executionInfo": { 123 | "elapsed": 75, 124 | "status": "ok", 125 | "timestamp": 1697104375004, 126 | "user": { 127 | "displayName": "", 128 | "userId": "" 129 | }, 130 | "user_tz": -60 131 | }, 132 | "id": "9t9out8lYBSr", 133 | "outputId": "5ac9c2b2-e066-44af-947e-c94b4b826421" 134 | }, 135 | "outputs": [ 136 | { 137 | "name": "stdout", 138 | "output_type": "stream", 139 | "text": [ 140 | "1 3\n", 141 | "2 9\n", 142 | "3 27\n", 143 | "4 81\n", 144 | "5 243\n" 145 | ] 146 | } 147 | ], 148 | "source": [ 149 | "for n in range(1, 6):\n", 150 | " print(n, evaluate(num_nodes=7, n=n))" 151 | ] 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "metadata": { 156 | "id": "FBbvHTaQY3X8" 157 | }, 158 | "source": [ 159 | "## Discovered function that builds an independent set of size $367$ in $C_7^5$\n", 160 | "\n", 161 | "This matches the size of the best known construction by [Polak \u0026 Schrijver (2019)](https://ir.cwi.nl/pub/30364/30364.pdf)." 162 | ] 163 | }, 164 | { 165 | "cell_type": "code", 166 | "execution_count": null, 167 | "metadata": { 168 | "id": "w4yMoDAoYjYq" 169 | }, 170 | "outputs": [], 171 | "source": [ 172 | "def priority(el: tuple[int, ...], num_nodes: int, n: int) -\u003e float:\n", 173 | " \"\"\"Returns the priority with which we want to add `el` to the set.\"\"\"\n", 174 | " score = 0.\n", 175 | " for i in range(n):\n", 176 | " if el[i] == el[(i + 2) % n]:\n", 177 | " score += 1\n", 178 | " else:\n", 179 | " score -= 1\n", 180 | " x = ((n - 2) * el[i] - el[(i + 1) % n]\n", 181 | " - el[(i + 2) % n] - (n + 1) * el[(i + 3) % n]) % num_nodes\n", 182 | " score -= 0.5 * (x - el[(i + 1) % n]) ** 2\n", 183 | " score += 0.1 * (num_nodes - 1 - (x - 1) % num_nodes) ** 2\n", 184 | " score += 0.2 * (num_nodes - 1 - (x - 2) % num_nodes) ** 2\n", 185 | " return score" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": null, 191 | "metadata": { 192 | "id": "A_eiYDIsZLYi" 193 | }, 194 | "outputs": [], 195 | "source": [ 196 | "assert evaluate(num_nodes=7, n=5) == 367" 197 | ] 198 | }, 199 | { 200 | "cell_type": "markdown", 201 | "metadata": { 202 | "id": "zfY5hIZnZYQg" 203 | }, 204 | "source": [ 205 | "## Discovered function that builds the best known independent sets in $C_9^n$ for $n=3,...,7$\n", 206 | "\n", 207 | "These independent sets match the best known construction reported by [Matthew \u0026 Östergård (2016)](https://link.springer.com/article/10.1007/s10623-016-0194-7)." 208 | ] 209 | }, 210 | { 211 | "cell_type": "code", 212 | "execution_count": null, 213 | "metadata": { 214 | "id": "al23ssPbZUpk" 215 | }, 216 | "outputs": [], 217 | "source": [ 218 | "def priority(el: tuple[int, ...], num_nodes: int, n: int) -\u003e float:\n", 219 | " \"\"\"Returns the priority with which we want to add `el` to the set.\"\"\"\n", 220 | " s = 0.\n", 221 | " for i in range(n):\n", 222 | " s += el[i] \u003c\u003c i\n", 223 | " s %= num_nodes\n", 224 | " return (2 * el[2] - 4 * el[0] + el[1]) % num_nodes + s" 225 | ] 226 | }, 227 | { 228 | "cell_type": "markdown", 229 | "metadata": { 230 | "id": "0li-u2bKzcqs" 231 | }, 232 | "source": [ 233 | "Below, we only run the code up until $n=5$. Uncomment the line below to run also the code with $n=6$ and $n=7$, which would take about 10 minutes to execute." 234 | ] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "execution_count": null, 239 | "metadata": { 240 | "id": "4tiBK2mzadTA" 241 | }, 242 | "outputs": [], 243 | "source": [ 244 | "expected_sizes = {\n", 245 | " 3: 81,\n", 246 | " 4: 324,\n", 247 | " 5: 1458,\n", 248 | " 6: 6561,\n", 249 | " 7: 26244\n", 250 | "}\n", 251 | "range_n = range(3, 6)\n", 252 | "# range_n = range(3, 8) # Uncomment to run up until n=7.\n", 253 | "for n in range_n:\n", 254 | " assert evaluate(num_nodes=9, n=n) == expected_sizes[n]" 255 | ] 256 | }, 257 | { 258 | "cell_type": "markdown", 259 | "metadata": { 260 | "id": "m1pW8OYnarxM" 261 | }, 262 | "source": [ 263 | "## Discovered function that finds an independent set of size 754 in $C_{11}^4$\n", 264 | "\n", 265 | "This is larger than the best known independent set reported by [Matthew \u0026 Östergård (2016)](https://link.springer.com/article/10.1007/s10623-016-0194-7), which has size $748$." 266 | ] 267 | }, 268 | { 269 | "cell_type": "code", 270 | "execution_count": null, 271 | "metadata": { 272 | "id": "mDrmnumFah_t" 273 | }, 274 | "outputs": [], 275 | "source": [ 276 | "def priority(el: tuple[int, ...], num_nodes: int, n: int) -\u003e float:\n", 277 | " \"\"\"Returns the priority with which we want to add `el` to the set.\"\"\"\n", 278 | " el_clipped = np.clip(el, a_min=None, a_max=num_nodes - 3)\n", 279 | " values = 2 * np.array(list(itertools.product(range(1, n), repeat=n)))\n", 280 | " multipliers = np.array(\n", 281 | " [num_nodes ** i for i in range(n - 1, -1, -1)], dtype=np.int32)\n", 282 | " x = np.sum((1 + values + el_clipped) * multipliers, axis=-1)\n", 283 | " return np.sum(x % (num_nodes - 2), dtype=float)" 284 | ] 285 | }, 286 | { 287 | "cell_type": "code", 288 | "execution_count": null, 289 | "metadata": { 290 | "id": "dvRR69L-bD95" 291 | }, 292 | "outputs": [], 293 | "source": [ 294 | "assert evaluate(num_nodes=11, n=4) == 754" 295 | ] 296 | } 297 | ], 298 | "metadata": { 299 | "colab": { 300 | "provenance": [] 301 | }, 302 | "kernelspec": { 303 | "display_name": "Python 3", 304 | "name": "python3" 305 | }, 306 | "language_info": { 307 | "name": "python" 308 | } 309 | }, 310 | "nbformat": 4, 311 | "nbformat_minor": 0 312 | } 313 | -------------------------------------------------------------------------------- /examples/cyclic_graphs/nodes11_n4_size754.txt: -------------------------------------------------------------------------------- 1 | [0, 0, 0, 8] 2 | [0, 0, 0, 10] 3 | [0, 0, 1, 6] 4 | [0, 0, 2, 4] 5 | [0, 0, 3, 2] 6 | [0, 0, 4, 0] 7 | [0, 0, 5, 7] 8 | [0, 0, 6, 5] 9 | [0, 0, 7, 3] 10 | [0, 0, 8, 1] 11 | [0, 0, 9, 8] 12 | [0, 0, 9, 10] 13 | [0, 0, 10, 1] 14 | [0, 1, 0, 4] 15 | [0, 1, 1, 2] 16 | [0, 1, 2, 0] 17 | [0, 1, 3, 7] 18 | [0, 1, 4, 5] 19 | [0, 1, 5, 3] 20 | [0, 1, 6, 1] 21 | [0, 1, 7, 8] 22 | [0, 1, 7, 10] 23 | [0, 1, 8, 6] 24 | [0, 1, 9, 4] 25 | [0, 1, 10, 6] 26 | [0, 2, 0, 0] 27 | [0, 2, 1, 7] 28 | [0, 2, 2, 5] 29 | [0, 2, 3, 3] 30 | [0, 2, 4, 1] 31 | [0, 2, 5, 8] 32 | [0, 2, 5, 10] 33 | [0, 2, 6, 6] 34 | [0, 2, 7, 4] 35 | [0, 2, 8, 2] 36 | [0, 2, 9, 0] 37 | [0, 2, 10, 2] 38 | [0, 3, 0, 5] 39 | [0, 3, 1, 3] 40 | [0, 3, 2, 1] 41 | [0, 3, 3, 8] 42 | [0, 3, 3, 10] 43 | [0, 3, 4, 6] 44 | [0, 3, 5, 4] 45 | [0, 3, 6, 2] 46 | [0, 3, 7, 0] 47 | [0, 3, 8, 7] 48 | [0, 3, 9, 5] 49 | [0, 3, 10, 7] 50 | [0, 4, 0, 1] 51 | [0, 4, 1, 8] 52 | [0, 4, 1, 10] 53 | [0, 4, 2, 6] 54 | [0, 4, 3, 4] 55 | [0, 4, 4, 2] 56 | [0, 4, 5, 0] 57 | [0, 4, 6, 7] 58 | [0, 4, 7, 5] 59 | [0, 4, 8, 3] 60 | [0, 4, 9, 1] 61 | [0, 4, 10, 3] 62 | [0, 5, 0, 6] 63 | [0, 5, 1, 4] 64 | [0, 5, 2, 2] 65 | [0, 5, 3, 0] 66 | [0, 5, 4, 7] 67 | [0, 5, 5, 5] 68 | [0, 5, 6, 3] 69 | [0, 5, 7, 1] 70 | [0, 5, 8, 8] 71 | [0, 5, 8, 10] 72 | [0, 5, 9, 6] 73 | [0, 5, 10, 8] 74 | [0, 5, 10, 10] 75 | [0, 6, 0, 2] 76 | [0, 6, 1, 0] 77 | [0, 6, 2, 7] 78 | [0, 6, 3, 5] 79 | [0, 6, 4, 3] 80 | [0, 6, 5, 1] 81 | [0, 6, 6, 8] 82 | [0, 6, 6, 10] 83 | [0, 6, 7, 6] 84 | [0, 6, 8, 4] 85 | [0, 6, 9, 2] 86 | [0, 6, 10, 4] 87 | [0, 7, 0, 7] 88 | [0, 7, 1, 5] 89 | [0, 7, 2, 3] 90 | [0, 7, 3, 1] 91 | [0, 7, 4, 8] 92 | [0, 7, 4, 10] 93 | [0, 7, 5, 6] 94 | [0, 7, 6, 4] 95 | [0, 7, 7, 2] 96 | [0, 7, 8, 0] 97 | [0, 7, 9, 7] 98 | [0, 7, 10, 0] 99 | [0, 8, 0, 3] 100 | [0, 8, 1, 1] 101 | [0, 8, 2, 8] 102 | [0, 8, 2, 10] 103 | [0, 8, 3, 6] 104 | [0, 8, 4, 4] 105 | [0, 8, 5, 2] 106 | [0, 8, 6, 0] 107 | [0, 8, 7, 7] 108 | [0, 8, 8, 5] 109 | [0, 8, 9, 3] 110 | [0, 8, 10, 5] 111 | [0, 9, 0, 8] 112 | [0, 9, 0, 10] 113 | [0, 9, 1, 6] 114 | [0, 9, 2, 4] 115 | [0, 9, 3, 2] 116 | [0, 9, 4, 0] 117 | [0, 9, 5, 7] 118 | [0, 9, 6, 5] 119 | [0, 9, 7, 3] 120 | [0, 9, 8, 1] 121 | [0, 9, 9, 7] 122 | [0, 9, 10, 1] 123 | [0, 10, 0, 3] 124 | [0, 10, 1, 1] 125 | [0, 10, 2, 8] 126 | [0, 10, 2, 10] 127 | [0, 10, 3, 6] 128 | [0, 10, 4, 4] 129 | [0, 10, 5, 2] 130 | [0, 10, 6, 0] 131 | [0, 10, 7, 7] 132 | [0, 10, 8, 5] 133 | [0, 10, 9, 3] 134 | [0, 10, 10, 5] 135 | [1, 0, 5, 9] 136 | [1, 1, 3, 9] 137 | [1, 2, 1, 9] 138 | [1, 3, 8, 9] 139 | [1, 3, 10, 9] 140 | [1, 4, 6, 9] 141 | [1, 5, 4, 9] 142 | [1, 6, 2, 9] 143 | [1, 7, 0, 9] 144 | [1, 7, 9, 9] 145 | [1, 8, 7, 9] 146 | [1, 9, 5, 9] 147 | [1, 9, 9, 9] 148 | [1, 10, 7, 9] 149 | [2, 0, 0, 1] 150 | [2, 0, 1, 8] 151 | [2, 0, 1, 10] 152 | [2, 0, 2, 6] 153 | [2, 0, 3, 4] 154 | [2, 0, 4, 2] 155 | [2, 0, 5, 0] 156 | [2, 0, 6, 7] 157 | [2, 0, 7, 5] 158 | [2, 0, 8, 3] 159 | [2, 0, 9, 1] 160 | [2, 0, 10, 3] 161 | [2, 1, 0, 6] 162 | [2, 1, 1, 4] 163 | [2, 1, 2, 2] 164 | [2, 1, 3, 0] 165 | [2, 1, 4, 7] 166 | [2, 1, 5, 5] 167 | [2, 1, 6, 3] 168 | [2, 1, 7, 1] 169 | [2, 1, 8, 8] 170 | [2, 1, 8, 10] 171 | [2, 1, 9, 6] 172 | [2, 1, 10, 8] 173 | [2, 1, 10, 10] 174 | [2, 2, 0, 2] 175 | [2, 2, 1, 0] 176 | [2, 2, 2, 7] 177 | [2, 2, 3, 5] 178 | [2, 2, 4, 3] 179 | [2, 2, 5, 1] 180 | [2, 2, 6, 8] 181 | [2, 2, 6, 10] 182 | [2, 2, 7, 6] 183 | [2, 2, 8, 4] 184 | [2, 2, 9, 2] 185 | [2, 2, 10, 4] 186 | [2, 3, 0, 7] 187 | [2, 3, 1, 5] 188 | [2, 3, 2, 3] 189 | [2, 3, 3, 1] 190 | [2, 3, 4, 8] 191 | [2, 3, 4, 10] 192 | [2, 3, 5, 6] 193 | [2, 3, 6, 4] 194 | [2, 3, 7, 2] 195 | [2, 3, 8, 0] 196 | [2, 3, 9, 7] 197 | [2, 3, 10, 0] 198 | [2, 4, 0, 3] 199 | [2, 4, 1, 1] 200 | [2, 4, 2, 8] 201 | [2, 4, 2, 10] 202 | [2, 4, 3, 6] 203 | [2, 4, 4, 4] 204 | [2, 4, 5, 2] 205 | [2, 4, 6, 0] 206 | [2, 4, 7, 7] 207 | [2, 4, 8, 5] 208 | [2, 4, 9, 3] 209 | [2, 4, 10, 5] 210 | [2, 5, 0, 8] 211 | [2, 5, 0, 10] 212 | [2, 5, 1, 6] 213 | [2, 5, 2, 4] 214 | [2, 5, 3, 2] 215 | [2, 5, 4, 0] 216 | [2, 5, 5, 7] 217 | [2, 5, 6, 5] 218 | [2, 5, 7, 3] 219 | [2, 5, 8, 1] 220 | [2, 5, 9, 8] 221 | [2, 5, 9, 10] 222 | [2, 5, 10, 1] 223 | [2, 6, 0, 4] 224 | [2, 6, 1, 2] 225 | [2, 6, 2, 0] 226 | [2, 6, 3, 7] 227 | [2, 6, 4, 5] 228 | [2, 6, 5, 3] 229 | [2, 6, 6, 1] 230 | [2, 6, 7, 8] 231 | [2, 6, 7, 10] 232 | [2, 6, 8, 6] 233 | [2, 6, 9, 4] 234 | [2, 6, 10, 6] 235 | [2, 7, 0, 0] 236 | [2, 7, 1, 7] 237 | [2, 7, 2, 5] 238 | [2, 7, 3, 3] 239 | [2, 7, 4, 1] 240 | [2, 7, 5, 8] 241 | [2, 7, 5, 10] 242 | [2, 7, 6, 6] 243 | [2, 7, 7, 4] 244 | [2, 7, 8, 2] 245 | [2, 7, 9, 0] 246 | [2, 7, 10, 2] 247 | [2, 8, 0, 5] 248 | [2, 8, 1, 3] 249 | [2, 8, 2, 1] 250 | [2, 8, 3, 8] 251 | [2, 8, 3, 10] 252 | [2, 8, 4, 6] 253 | [2, 8, 5, 4] 254 | [2, 8, 6, 2] 255 | [2, 8, 7, 0] 256 | [2, 8, 8, 7] 257 | [2, 8, 9, 5] 258 | [2, 8, 10, 7] 259 | [2, 9, 0, 1] 260 | [2, 9, 1, 8] 261 | [2, 9, 1, 10] 262 | [2, 9, 2, 6] 263 | [2, 9, 3, 4] 264 | [2, 9, 4, 2] 265 | [2, 9, 5, 0] 266 | [2, 9, 6, 7] 267 | [2, 9, 7, 5] 268 | [2, 9, 8, 3] 269 | [2, 9, 9, 0] 270 | [2, 9, 10, 3] 271 | [2, 10, 0, 5] 272 | [2, 10, 1, 3] 273 | [2, 10, 2, 1] 274 | [2, 10, 3, 8] 275 | [2, 10, 3, 10] 276 | [2, 10, 4, 6] 277 | [2, 10, 5, 4] 278 | [2, 10, 6, 2] 279 | [2, 10, 7, 0] 280 | [2, 10, 8, 7] 281 | [2, 10, 9, 5] 282 | [2, 10, 10, 7] 283 | [3, 0, 6, 9] 284 | [3, 1, 4, 9] 285 | [3, 2, 2, 9] 286 | [3, 3, 0, 9] 287 | [3, 3, 9, 9] 288 | [3, 4, 7, 9] 289 | [3, 5, 5, 9] 290 | [3, 6, 3, 9] 291 | [3, 7, 1, 9] 292 | [3, 8, 8, 9] 293 | [3, 8, 10, 9] 294 | [3, 9, 6, 9] 295 | [3, 10, 8, 9] 296 | [3, 10, 10, 9] 297 | [4, 0, 0, 3] 298 | [4, 0, 1, 1] 299 | [4, 0, 2, 8] 300 | [4, 0, 2, 10] 301 | [4, 0, 3, 6] 302 | [4, 0, 4, 4] 303 | [4, 0, 5, 2] 304 | [4, 0, 6, 0] 305 | [4, 0, 7, 7] 306 | [4, 0, 8, 5] 307 | [4, 0, 9, 3] 308 | [4, 0, 10, 5] 309 | [4, 1, 0, 8] 310 | [4, 1, 0, 10] 311 | [4, 1, 1, 6] 312 | [4, 1, 2, 4] 313 | [4, 1, 3, 2] 314 | [4, 1, 4, 0] 315 | [4, 1, 5, 7] 316 | [4, 1, 6, 5] 317 | [4, 1, 7, 3] 318 | [4, 1, 8, 1] 319 | [4, 1, 9, 8] 320 | [4, 1, 9, 10] 321 | [4, 1, 10, 1] 322 | [4, 2, 0, 4] 323 | [4, 2, 1, 2] 324 | [4, 2, 2, 0] 325 | [4, 2, 3, 7] 326 | [4, 2, 4, 5] 327 | [4, 2, 5, 3] 328 | [4, 2, 6, 1] 329 | [4, 2, 7, 8] 330 | [4, 2, 7, 10] 331 | [4, 2, 8, 6] 332 | [4, 2, 9, 4] 333 | [4, 2, 10, 6] 334 | [4, 3, 0, 0] 335 | [4, 3, 1, 7] 336 | [4, 3, 2, 5] 337 | [4, 3, 3, 3] 338 | [4, 3, 4, 1] 339 | [4, 3, 5, 8] 340 | [4, 3, 5, 10] 341 | [4, 3, 6, 6] 342 | [4, 3, 7, 4] 343 | [4, 3, 8, 2] 344 | [4, 3, 9, 0] 345 | [4, 3, 10, 2] 346 | [4, 4, 0, 5] 347 | [4, 4, 1, 3] 348 | [4, 4, 2, 1] 349 | [4, 4, 3, 8] 350 | [4, 4, 3, 10] 351 | [4, 4, 4, 6] 352 | [4, 4, 5, 4] 353 | [4, 4, 6, 2] 354 | [4, 4, 7, 0] 355 | [4, 4, 8, 7] 356 | [4, 4, 9, 5] 357 | [4, 4, 10, 7] 358 | [4, 5, 0, 1] 359 | [4, 5, 1, 8] 360 | [4, 5, 1, 10] 361 | [4, 5, 2, 6] 362 | [4, 5, 3, 4] 363 | [4, 5, 4, 2] 364 | [4, 5, 5, 0] 365 | [4, 5, 6, 7] 366 | [4, 5, 7, 5] 367 | [4, 5, 8, 3] 368 | [4, 5, 9, 1] 369 | [4, 5, 10, 3] 370 | [4, 6, 0, 6] 371 | [4, 6, 1, 4] 372 | [4, 6, 2, 2] 373 | [4, 6, 3, 0] 374 | [4, 6, 4, 7] 375 | [4, 6, 5, 5] 376 | [4, 6, 6, 3] 377 | [4, 6, 7, 1] 378 | [4, 6, 8, 8] 379 | [4, 6, 8, 10] 380 | [4, 6, 9, 6] 381 | [4, 6, 10, 8] 382 | [4, 6, 10, 10] 383 | [4, 7, 0, 2] 384 | [4, 7, 1, 0] 385 | [4, 7, 2, 7] 386 | [4, 7, 3, 5] 387 | [4, 7, 4, 3] 388 | [4, 7, 5, 1] 389 | [4, 7, 6, 8] 390 | [4, 7, 6, 10] 391 | [4, 7, 7, 6] 392 | [4, 7, 8, 4] 393 | [4, 7, 9, 2] 394 | [4, 7, 10, 4] 395 | [4, 8, 0, 7] 396 | [4, 8, 1, 5] 397 | [4, 8, 2, 3] 398 | [4, 8, 3, 1] 399 | [4, 8, 4, 8] 400 | [4, 8, 4, 10] 401 | [4, 8, 5, 6] 402 | [4, 8, 6, 4] 403 | [4, 8, 7, 2] 404 | [4, 8, 8, 0] 405 | [4, 8, 9, 7] 406 | [4, 8, 10, 0] 407 | [4, 9, 0, 3] 408 | [4, 9, 1, 1] 409 | [4, 9, 2, 8] 410 | [4, 9, 2, 10] 411 | [4, 9, 3, 6] 412 | [4, 9, 4, 4] 413 | [4, 9, 5, 2] 414 | [4, 9, 6, 0] 415 | [4, 9, 7, 7] 416 | [4, 9, 8, 5] 417 | [4, 9, 9, 2] 418 | [4, 9, 10, 5] 419 | [4, 10, 0, 7] 420 | [4, 10, 1, 5] 421 | [4, 10, 2, 3] 422 | [4, 10, 3, 1] 423 | [4, 10, 4, 8] 424 | [4, 10, 4, 10] 425 | [4, 10, 5, 6] 426 | [4, 10, 6, 4] 427 | [4, 10, 7, 2] 428 | [4, 10, 8, 0] 429 | [4, 10, 9, 7] 430 | [4, 10, 10, 0] 431 | [5, 0, 7, 9] 432 | [5, 1, 5, 9] 433 | [5, 2, 3, 9] 434 | [5, 3, 1, 9] 435 | [5, 4, 8, 9] 436 | [5, 4, 10, 9] 437 | [5, 5, 6, 9] 438 | [5, 6, 4, 9] 439 | [5, 7, 2, 9] 440 | [5, 8, 0, 9] 441 | [5, 8, 9, 9] 442 | [5, 9, 7, 9] 443 | [5, 10, 0, 9] 444 | [5, 10, 9, 9] 445 | [6, 0, 0, 5] 446 | [6, 0, 1, 3] 447 | [6, 0, 2, 1] 448 | [6, 0, 3, 8] 449 | [6, 0, 3, 10] 450 | [6, 0, 4, 6] 451 | [6, 0, 5, 4] 452 | [6, 0, 6, 2] 453 | [6, 0, 7, 0] 454 | [6, 0, 8, 7] 455 | [6, 0, 9, 5] 456 | [6, 0, 10, 7] 457 | [6, 1, 0, 1] 458 | [6, 1, 1, 8] 459 | [6, 1, 1, 10] 460 | [6, 1, 2, 6] 461 | [6, 1, 3, 4] 462 | [6, 1, 4, 2] 463 | [6, 1, 5, 0] 464 | [6, 1, 6, 7] 465 | [6, 1, 7, 5] 466 | [6, 1, 8, 3] 467 | [6, 1, 9, 1] 468 | [6, 1, 10, 3] 469 | [6, 2, 0, 6] 470 | [6, 2, 1, 4] 471 | [6, 2, 2, 2] 472 | [6, 2, 3, 0] 473 | [6, 2, 4, 7] 474 | [6, 2, 5, 5] 475 | [6, 2, 6, 3] 476 | [6, 2, 7, 1] 477 | [6, 2, 8, 8] 478 | [6, 2, 8, 10] 479 | [6, 2, 9, 6] 480 | [6, 2, 10, 8] 481 | [6, 2, 10, 10] 482 | [6, 3, 0, 2] 483 | [6, 3, 1, 0] 484 | [6, 3, 2, 7] 485 | [6, 3, 3, 5] 486 | [6, 3, 4, 3] 487 | [6, 3, 5, 1] 488 | [6, 3, 6, 8] 489 | [6, 3, 6, 10] 490 | [6, 3, 7, 6] 491 | [6, 3, 8, 4] 492 | [6, 3, 9, 2] 493 | [6, 3, 10, 4] 494 | [6, 4, 0, 7] 495 | [6, 4, 1, 5] 496 | [6, 4, 2, 3] 497 | [6, 4, 3, 1] 498 | [6, 4, 4, 8] 499 | [6, 4, 4, 10] 500 | [6, 4, 5, 6] 501 | [6, 4, 6, 4] 502 | [6, 4, 7, 2] 503 | [6, 4, 8, 0] 504 | [6, 4, 9, 7] 505 | [6, 4, 10, 0] 506 | [6, 5, 0, 3] 507 | [6, 5, 1, 1] 508 | [6, 5, 2, 8] 509 | [6, 5, 2, 10] 510 | [6, 5, 3, 6] 511 | [6, 5, 4, 4] 512 | [6, 5, 5, 2] 513 | [6, 5, 6, 0] 514 | [6, 5, 7, 7] 515 | [6, 5, 8, 5] 516 | [6, 5, 9, 3] 517 | [6, 5, 10, 5] 518 | [6, 6, 0, 8] 519 | [6, 6, 0, 10] 520 | [6, 6, 1, 6] 521 | [6, 6, 2, 4] 522 | [6, 6, 3, 2] 523 | [6, 6, 4, 0] 524 | [6, 6, 5, 7] 525 | [6, 6, 6, 5] 526 | [6, 6, 7, 3] 527 | [6, 6, 8, 1] 528 | [6, 6, 9, 8] 529 | [6, 6, 9, 10] 530 | [6, 6, 10, 1] 531 | [6, 7, 0, 4] 532 | [6, 7, 1, 2] 533 | [6, 7, 2, 0] 534 | [6, 7, 3, 7] 535 | [6, 7, 4, 5] 536 | [6, 7, 5, 3] 537 | [6, 7, 6, 1] 538 | [6, 7, 7, 8] 539 | [6, 7, 7, 10] 540 | [6, 7, 8, 6] 541 | [6, 7, 9, 4] 542 | [6, 7, 10, 6] 543 | [6, 8, 0, 0] 544 | [6, 8, 1, 7] 545 | [6, 8, 2, 5] 546 | [6, 8, 3, 3] 547 | [6, 8, 4, 1] 548 | [6, 8, 5, 8] 549 | [6, 8, 5, 10] 550 | [6, 8, 6, 6] 551 | [6, 8, 7, 4] 552 | [6, 8, 8, 2] 553 | [6, 8, 9, 0] 554 | [6, 8, 10, 2] 555 | [6, 9, 0, 5] 556 | [6, 9, 1, 3] 557 | [6, 9, 2, 1] 558 | [6, 9, 3, 8] 559 | [6, 9, 3, 10] 560 | [6, 9, 4, 6] 561 | [6, 9, 5, 4] 562 | [6, 9, 6, 2] 563 | [6, 9, 7, 0] 564 | [6, 9, 8, 7] 565 | [6, 9, 9, 4] 566 | [6, 9, 10, 7] 567 | [6, 10, 0, 0] 568 | [6, 10, 1, 7] 569 | [6, 10, 2, 5] 570 | [6, 10, 3, 3] 571 | [6, 10, 4, 1] 572 | [6, 10, 5, 8] 573 | [6, 10, 5, 10] 574 | [6, 10, 6, 6] 575 | [6, 10, 7, 4] 576 | [6, 10, 8, 2] 577 | [6, 10, 9, 0] 578 | [6, 10, 10, 2] 579 | [7, 0, 8, 9] 580 | [7, 0, 10, 9] 581 | [7, 1, 6, 9] 582 | [7, 2, 4, 9] 583 | [7, 3, 2, 9] 584 | [7, 4, 0, 9] 585 | [7, 4, 9, 9] 586 | [7, 5, 7, 9] 587 | [7, 6, 5, 9] 588 | [7, 7, 3, 9] 589 | [7, 8, 1, 9] 590 | [7, 9, 8, 9] 591 | [7, 9, 10, 9] 592 | [7, 10, 1, 9] 593 | [8, 0, 0, 7] 594 | [8, 0, 1, 5] 595 | [8, 0, 2, 3] 596 | [8, 0, 3, 1] 597 | [8, 0, 4, 8] 598 | [8, 0, 4, 10] 599 | [8, 0, 5, 6] 600 | [8, 0, 6, 4] 601 | [8, 0, 7, 2] 602 | [8, 0, 8, 0] 603 | [8, 0, 9, 7] 604 | [8, 0, 10, 0] 605 | [8, 1, 0, 3] 606 | [8, 1, 1, 1] 607 | [8, 1, 2, 8] 608 | [8, 1, 2, 10] 609 | [8, 1, 3, 6] 610 | [8, 1, 4, 4] 611 | [8, 1, 5, 2] 612 | [8, 1, 6, 0] 613 | [8, 1, 7, 7] 614 | [8, 1, 8, 5] 615 | [8, 1, 9, 3] 616 | [8, 1, 10, 5] 617 | [8, 2, 0, 8] 618 | [8, 2, 0, 10] 619 | [8, 2, 1, 6] 620 | [8, 2, 2, 4] 621 | [8, 2, 3, 2] 622 | [8, 2, 4, 0] 623 | [8, 2, 5, 7] 624 | [8, 2, 6, 5] 625 | [8, 2, 7, 3] 626 | [8, 2, 8, 1] 627 | [8, 2, 9, 8] 628 | [8, 2, 9, 10] 629 | [8, 2, 10, 1] 630 | [8, 3, 0, 4] 631 | [8, 3, 1, 2] 632 | [8, 3, 2, 0] 633 | [8, 3, 3, 7] 634 | [8, 3, 4, 5] 635 | [8, 3, 5, 3] 636 | [8, 3, 6, 1] 637 | [8, 3, 7, 8] 638 | [8, 3, 7, 10] 639 | [8, 3, 8, 6] 640 | [8, 3, 9, 4] 641 | [8, 3, 10, 6] 642 | [8, 4, 0, 0] 643 | [8, 4, 1, 7] 644 | [8, 4, 2, 5] 645 | [8, 4, 3, 3] 646 | [8, 4, 4, 1] 647 | [8, 4, 5, 8] 648 | [8, 4, 5, 10] 649 | [8, 4, 6, 6] 650 | [8, 4, 7, 4] 651 | [8, 4, 8, 2] 652 | [8, 4, 9, 0] 653 | [8, 4, 10, 2] 654 | [8, 5, 0, 5] 655 | [8, 5, 1, 3] 656 | [8, 5, 2, 1] 657 | [8, 5, 3, 8] 658 | [8, 5, 3, 10] 659 | [8, 5, 4, 6] 660 | [8, 5, 5, 4] 661 | [8, 5, 6, 2] 662 | [8, 5, 7, 0] 663 | [8, 5, 8, 7] 664 | [8, 5, 9, 5] 665 | [8, 5, 10, 7] 666 | [8, 6, 0, 1] 667 | [8, 6, 1, 8] 668 | [8, 6, 1, 10] 669 | [8, 6, 2, 6] 670 | [8, 6, 3, 4] 671 | [8, 6, 4, 2] 672 | [8, 6, 5, 0] 673 | [8, 6, 6, 7] 674 | [8, 6, 7, 5] 675 | [8, 6, 8, 3] 676 | [8, 6, 9, 1] 677 | [8, 6, 10, 3] 678 | [8, 7, 0, 6] 679 | [8, 7, 1, 4] 680 | [8, 7, 2, 2] 681 | [8, 7, 3, 0] 682 | [8, 7, 4, 7] 683 | [8, 7, 5, 5] 684 | [8, 7, 6, 3] 685 | [8, 7, 7, 1] 686 | [8, 7, 8, 8] 687 | [8, 7, 8, 10] 688 | [8, 7, 9, 6] 689 | [8, 7, 10, 8] 690 | [8, 7, 10, 10] 691 | [8, 8, 0, 2] 692 | [8, 8, 1, 0] 693 | [8, 8, 2, 7] 694 | [8, 8, 3, 5] 695 | [8, 8, 4, 3] 696 | [8, 8, 5, 1] 697 | [8, 8, 6, 8] 698 | [8, 8, 6, 10] 699 | [8, 8, 7, 6] 700 | [8, 8, 8, 4] 701 | [8, 8, 9, 2] 702 | [8, 8, 10, 4] 703 | [8, 9, 0, 7] 704 | [8, 9, 1, 5] 705 | [8, 9, 2, 3] 706 | [8, 9, 3, 1] 707 | [8, 9, 4, 8] 708 | [8, 9, 4, 10] 709 | [8, 9, 5, 6] 710 | [8, 9, 6, 4] 711 | [8, 9, 7, 2] 712 | [8, 9, 8, 0] 713 | [8, 9, 9, 6] 714 | [8, 9, 10, 0] 715 | [8, 10, 0, 2] 716 | [8, 10, 1, 0] 717 | [8, 10, 2, 7] 718 | [8, 10, 3, 5] 719 | [8, 10, 4, 3] 720 | [8, 10, 5, 1] 721 | [8, 10, 6, 8] 722 | [8, 10, 6, 10] 723 | [8, 10, 7, 6] 724 | [8, 10, 8, 4] 725 | [8, 10, 9, 2] 726 | [8, 10, 10, 4] 727 | [9, 0, 0, 9] 728 | [9, 0, 9, 9] 729 | [9, 1, 7, 9] 730 | [9, 2, 5, 9] 731 | [9, 3, 3, 9] 732 | [9, 4, 1, 9] 733 | [9, 5, 8, 9] 734 | [9, 5, 10, 9] 735 | [9, 6, 6, 9] 736 | [9, 7, 4, 9] 737 | [9, 8, 2, 9] 738 | [9, 9, 0, 9] 739 | [9, 9, 8, 8] 740 | [9, 10, 2, 9] 741 | [10, 0, 4, 9] 742 | [10, 1, 2, 9] 743 | [10, 2, 0, 9] 744 | [10, 2, 9, 9] 745 | [10, 3, 7, 9] 746 | [10, 4, 5, 9] 747 | [10, 5, 3, 9] 748 | [10, 6, 1, 9] 749 | [10, 7, 8, 9] 750 | [10, 7, 10, 9] 751 | [10, 8, 6, 9] 752 | [10, 9, 4, 9] 753 | [10, 9, 8, 10] 754 | [10, 10, 6, 9] 755 | -------------------------------------------------------------------------------- /examples/specification_nonsymmetric_admissible_set.py: -------------------------------------------------------------------------------- 1 | """Generating maximal admissible sets of different dimensionalities.""" 2 | import itertools 3 | import numpy as np 4 | 5 | 6 | def block_children(scores, admissible_set, new_element): 7 | """Modifies `scores` to -inf for elements blocked by `new_element`.""" 8 | n = admissible_set.shape[-1] 9 | powers = np.array([3 ** i for i in range(n - 1, -1, -1)], dtype=np.int32) 10 | 11 | invalid_vals_raw = { 12 | (0, 0): (0,), 13 | (0, 1): (1,), 14 | (0, 2): (2,), 15 | (1, 0): (1,), 16 | (1, 1): (0, 1, 2), 17 | (1, 2): (1, 2), 18 | (2, 0): (2,), 19 | (2, 1): (1, 2), 20 | (2, 2): (0, 1, 2), 21 | } 22 | invalid_vals = [[np.array(invalid_vals_raw[(i, j)], dtype=np.int32) 23 | for j in range(3)] for i in range(3)] 24 | 25 | # Block 2^w elements with the same support as `new_element`. 26 | w = np.count_nonzero(new_element) 27 | all_12s = np.array(list(itertools.product((1, 2), repeat=w)), dtype=np.int32) 28 | blocking = np.einsum('aw,w->a', all_12s, powers[new_element != 0]) 29 | scores[blocking] = -np.inf 30 | 31 | # Block elements disallowed by a pair of an extant point and `new_element`. 32 | for extant_element in admissible_set: 33 | blocking = np.zeros(shape=(1,), dtype=np.int32) 34 | for e1, e2, power in zip(extant_element, new_element, powers): 35 | blocking = (blocking[:, None] + (invalid_vals[e1][e2] * power)[None, :] 36 | ).ravel() 37 | scores[blocking] = -np.inf 38 | 39 | 40 | def solve(n: int, w: int) -> np.ndarray: 41 | """Builds a large constant-weight-w n-dimensional admissible set. 42 | 43 | A constant weight admissible set is a subset of {0, 1, 2}^n. The weight of 44 | each element is the number of non-zero entries. The admissible set consists of 45 | a maximum of n choose w elements where w is the desired weight. Any three 46 | distinct elements of the set satisfy the triplet constraints, that is, there 47 | exists an index k such that the k-th components of the elements are either 48 | {0, 0, 1}, {0, 0, 2} or {0, 1, 2}. 49 | 50 | Args: 51 | n: dimension 52 | w: weight 53 | 54 | Returns: 55 | An array of shape [admissible_set_size, n], with entries in {0, 1, 2}. 56 | """ 57 | children = np.array(list(itertools.product((0, 1, 2), repeat=n)), 58 | dtype=np.int32) 59 | 60 | scores = -np.inf * np.ones((3 ** n,), dtype=np.float32) 61 | for child_index, child in enumerate(children): 62 | if sum(child == 0) == n-w: 63 | scores[child_index] = priority(np.array(child), n, w) 64 | 65 | max_admissible_set = np.empty((0, n), dtype=np.int32) 66 | while np.any(scores != -np.inf): 67 | # Find element with largest score 68 | max_index = np.argmax(scores) 69 | child = children[max_index] 70 | block_children(scores, max_admissible_set, child) 71 | max_admissible_set = np.concatenate([max_admissible_set, child[None]], 72 | axis=0) 73 | return max_admissible_set 74 | 75 | 76 | @funsearch.run 77 | def evaluate(set_description: tuple[int, int]) -> int: 78 | """Returns the size of the constructed admissible set.""" 79 | n, w = set_description 80 | admissible_set = solve(n, w) 81 | return len(admissible_set) 82 | 83 | 84 | @funsearch.evolve 85 | def priority(el: tuple[int, ...], n: int, w: int) -> float: 86 | """Trivial scoring function. 87 | 88 | Args: 89 | el: an element to be potentially added to the current admissible set. 90 | n: dimensionality of admissible set. 91 | w: weight of admissible set. 92 | 93 | Returns: 94 | A number reflecting the priority with which we want to add `el` to the set. 95 | """ 96 | return 0.0 97 | -------------------------------------------------------------------------------- /funsearch/__init__.py: -------------------------------------------------------------------------------- 1 | def evolve(func): 2 | """@funsearch.evolve decorator is used in the problem specification to detect the method that 3 | should be evolved using LLM. 4 | """ 5 | return func 6 | 7 | 8 | def run(func): 9 | """@funsearch.run decorator is used in the problem specification to detect the method that 10 | should be used to verify and grade generated code. 11 | """ 12 | return func 13 | -------------------------------------------------------------------------------- /funsearch/__main__.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import pathlib 5 | import pickle 6 | import time 7 | 8 | import click 9 | import llm 10 | from dotenv import load_dotenv 11 | 12 | 13 | from funsearch import config, core, sandbox, sampler, programs_database, code_manipulation, evaluator 14 | 15 | LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper() 16 | logging.basicConfig(level=LOGLEVEL) 17 | 18 | 19 | def get_all_subclasses(cls): 20 | all_subclasses = [] 21 | 22 | for subclass in cls.__subclasses__(): 23 | all_subclasses.append(subclass) 24 | all_subclasses.extend(get_all_subclasses(subclass)) 25 | 26 | return all_subclasses 27 | 28 | 29 | SANDBOX_TYPES = get_all_subclasses(sandbox.DummySandbox) + [sandbox.DummySandbox] 30 | SANDBOX_NAMES = [c.__name__ for c in SANDBOX_TYPES] 31 | 32 | 33 | def parse_input(filename_or_data: str): 34 | if len(filename_or_data) == 0: 35 | raise Exception("No input data specified") 36 | p = pathlib.Path(filename_or_data) 37 | if p.exists(): 38 | if p.name.endswith(".json"): 39 | return json.load(open(filename_or_data, "r")) 40 | if p.name.endswith(".pickle"): 41 | return pickle.load(open(filename_or_data, "rb")) 42 | raise Exception("Unknown file format or filename") 43 | if "," not in filename_or_data: 44 | data = [filename_or_data] 45 | else: 46 | data = filename_or_data.split(",") 47 | if data[0].isnumeric(): 48 | f = int if data[0].isdecimal() else float 49 | data = [f(v) for v in data] 50 | return data 51 | 52 | @click.group() 53 | @click.pass_context 54 | def main(ctx): 55 | pass 56 | 57 | 58 | @main.command() 59 | @click.argument("spec_file", type=click.File("r")) 60 | @click.argument('inputs') 61 | @click.option('--model_name', default="gpt-3.5-turbo-instruct", help='LLM model') 62 | @click.option('--output_path', default="./data/", type=click.Path(file_okay=False), help='path for logs and data') 63 | @click.option('--load_backup', default=None, type=click.File("rb"), help='Use existing program database') 64 | @click.option('--iterations', default=-1, type=click.INT, help='Max iterations per sampler') 65 | @click.option('--samplers', default=15, type=click.INT, help='Samplers') 66 | @click.option('--sandbox_type', default="ContainerSandbox", type=click.Choice(SANDBOX_NAMES), help='Sandbox type') 67 | def run(spec_file, inputs, model_name, output_path, load_backup, iterations, samplers, sandbox_type): 68 | """ Execute function-search algorithm: 69 | 70 | \b 71 | SPEC_FILE is a python module that provides the basis of the LLM prompt as 72 | well as the evaluation metric. 73 | See examples/cap_set_spec.py for an example.\n 74 | \b 75 | INPUTS input filename ending in .json or .pickle, or a comma-separated 76 | input data. The files are expected contain a list with at least 77 | one element. Elements shall be passed to the solve() method 78 | one by one. Examples 79 | 8 80 | 8,9,10 81 | ./examples/cap_set_input_data.json 82 | """ 83 | 84 | # Load environment variables from .env file. 85 | # 86 | # Using OpenAI APIs with 'llm' package requires setting the variable 87 | # OPENAI_API_KEY=sk-... 88 | # See 'llm' package on how to use other providers. 89 | load_dotenv() 90 | 91 | timestamp = str(int(time.time())) 92 | log_path = pathlib.Path(output_path) / timestamp 93 | if not log_path.exists(): 94 | log_path.mkdir(parents=True) 95 | logging.info(f"Writing logs to {log_path}") 96 | 97 | model = llm.get_model(model_name) 98 | model.key = model.get_key() 99 | lm = sampler.LLM(2, model, log_path) 100 | 101 | specification = spec_file.read() 102 | function_to_evolve, function_to_run = core._extract_function_names(specification) 103 | template = code_manipulation.text_to_program(specification) 104 | 105 | conf = config.Config(num_evaluators=1) 106 | database = programs_database.ProgramsDatabase( 107 | conf.programs_database, template, function_to_evolve, identifier=timestamp) 108 | if load_backup: 109 | database.load(load_backup) 110 | 111 | inputs = parse_input(inputs) 112 | 113 | sandbox_class = next(c for c in SANDBOX_TYPES if c.__name__ == sandbox_type) 114 | evaluators = [evaluator.Evaluator( 115 | database, 116 | sandbox_class(base_path=log_path), 117 | template, 118 | function_to_evolve, 119 | function_to_run, 120 | inputs, 121 | ) for _ in range(conf.num_evaluators)] 122 | 123 | # We send the initial implementation to be analysed by one of the evaluators. 124 | initial = template.get_function(function_to_evolve).body 125 | evaluators[0].analyse(initial, island_id=None, version_generated=None) 126 | assert len(database._islands[0]._clusters) > 0, ("Initial analysis failed. Make sure that Sandbox works! " 127 | "See e.g. the error files under sandbox data.") 128 | 129 | samplers = [sampler.Sampler(database, evaluators, lm) 130 | for _ in range(samplers)] 131 | 132 | core.run(samplers, database, iterations) 133 | 134 | 135 | @main.command() 136 | @click.argument("db_file", type=click.File("rb")) 137 | def ls(db_file): 138 | """List programs from a stored database (usually in data/backups/ )""" 139 | conf = config.Config(num_evaluators=1) 140 | 141 | # A bit silly way to list programs. This probably does not work if config has changed any way 142 | database = programs_database.ProgramsDatabase( 143 | conf.programs_database, None, "", identifier="") 144 | database.load(db_file) 145 | 146 | progs = database.get_best_programs_per_island() 147 | print("Found {len(progs)} programs") 148 | for i, (prog, score) in enumerate(progs): 149 | print(f"{i}: Program with score {score}") 150 | print(prog) 151 | print("\n") 152 | 153 | 154 | if __name__ == '__main__': 155 | main() 156 | -------------------------------------------------------------------------------- /funsearch/code_manipulation.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 DeepMind Technologies Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Tools for manipulating Python code. 17 | 18 | It implements 2 classes representing unities of code: 19 | - Function, containing all the information we need about functions: name, args, 20 | body and optionally a return type and a docstring. 21 | - Program, which contains a code preface (which could be imports, global 22 | variables and classes, ...) and a list of Functions. 23 | """ 24 | import ast 25 | from collections.abc import Iterator, MutableSet, Sequence 26 | import dataclasses 27 | import io 28 | import tokenize 29 | 30 | from absl import logging 31 | 32 | 33 | @dataclasses.dataclass 34 | class Function: 35 | """A parsed Python function.""" 36 | 37 | name: str 38 | args: str 39 | body: str 40 | return_type: str | None = None 41 | docstring: str | None = None 42 | 43 | def __str__(self) -> str: 44 | return_type = f' -> {self.return_type}' if self.return_type else '' 45 | 46 | function = f'def {self.name}({self.args}){return_type}:\n' 47 | if self.docstring: 48 | # self.docstring is already indented on every line except the first one. 49 | # Here, we assume the indentation is always two spaces. 50 | new_line = '\n' if self.body else '' 51 | function += f' """{self.docstring}"""{new_line}' 52 | # self.body is already indented. 53 | function += self.body + '\n\n' 54 | return function 55 | 56 | def __setattr__(self, name: str, value: str) -> None: 57 | # Ensure there aren't leading & trailing new lines in `body`. 58 | if name == 'body': 59 | value = value.strip('\n') 60 | # Ensure there aren't leading & trailing quotes in `docstring``. 61 | if name == 'docstring' and value is not None: 62 | if '"""' in value: 63 | value = value.strip() 64 | value = value.replace('"""', '') 65 | super().__setattr__(name, value) 66 | 67 | 68 | @dataclasses.dataclass(frozen=True) 69 | class Program: 70 | """A parsed Python program.""" 71 | 72 | # `preface` is everything from the beginning of the code till the first 73 | # function is found. 74 | preface: str 75 | functions: list[Function] 76 | 77 | def __str__(self) -> str: 78 | program = f'{self.preface}\n' if self.preface else '' 79 | program += '\n'.join([str(f) for f in self.functions]) 80 | return program 81 | 82 | def find_function_index(self, function_name: str) -> int: 83 | """Returns the index of input function name.""" 84 | function_names = [f.name for f in self.functions] 85 | count = function_names.count(function_name) 86 | if count == 0: 87 | raise ValueError( 88 | f'function {function_name} does not exist in program:\n{str(self)}' 89 | ) 90 | if count > 1: 91 | raise ValueError( 92 | f'function {function_name} exists more than once in program:\n' 93 | f'{str(self)}' 94 | ) 95 | index = function_names.index(function_name) 96 | return index 97 | 98 | def get_function(self, function_name: str) -> Function: 99 | index = self.find_function_index(function_name) 100 | return self.functions[index] 101 | 102 | 103 | class ProgramVisitor(ast.NodeVisitor): 104 | """Parses code to collect all required information to produce a `Program`. 105 | 106 | Note that we do not store function decorators. 107 | """ 108 | 109 | def __init__(self, sourcecode: str): 110 | self._codelines: list[str] = sourcecode.splitlines() 111 | 112 | self._preface: str = '' 113 | self._functions: list[Function] = [] 114 | self._current_function: str | None = None 115 | 116 | def visit_FunctionDef(self, # pylint: disable=invalid-name 117 | node: ast.FunctionDef) -> None: 118 | """Collects all information about the function being parsed.""" 119 | if node.col_offset == 0: # We only care about first level functions. 120 | self._current_function = node.name 121 | if not self._functions: 122 | self._preface = '\n'.join(self._codelines[:node.lineno - 1]) 123 | function_end_line = node.end_lineno 124 | body_start_line = node.body[0].lineno - 1 125 | # Extract the docstring. 126 | docstring = None 127 | if isinstance(node.body[0], ast.Expr) and isinstance(node.body[0].value, 128 | ast.Str): 129 | docstring = f' """{ast.literal_eval(ast.unparse(node.body[0]))}"""' 130 | if len(node.body) > 1: 131 | body_start_line = node.body[1].lineno - 1 132 | else: 133 | body_start_line = function_end_line 134 | 135 | self._functions.append(Function( 136 | name=node.name, 137 | args=ast.unparse(node.args), 138 | return_type=ast.unparse(node.returns) if node.returns else None, 139 | docstring=docstring, 140 | body='\n'.join(self._codelines[body_start_line:function_end_line]), 141 | )) 142 | self.generic_visit(node) 143 | 144 | def return_program(self) -> Program: 145 | return Program(preface=self._preface, functions=self._functions) 146 | 147 | 148 | def text_to_program(text: str) -> Program: 149 | """Returns Program object by parsing input text using Python AST.""" 150 | try: 151 | # We assume that the program is composed of some preface (e.g. imports, 152 | # classes, assignments, ...) followed by a sequence of functions. 153 | tree = ast.parse(text) 154 | visitor = ProgramVisitor(text) 155 | visitor.visit(tree) 156 | return visitor.return_program() 157 | except Exception as e: 158 | logging.warning('Failed parsing %s', text) 159 | raise e 160 | 161 | 162 | def text_to_function(text: str) -> Function: 163 | """Returns Function object by parsing input text using Python AST.""" 164 | program = text_to_program(text) 165 | if len(program.functions) != 1: 166 | raise ValueError(f'Only one function expected, got {len(program.functions)}' 167 | f':\n{program.functions}') 168 | return program.functions[0] 169 | 170 | 171 | def _tokenize(code: str) -> Iterator[tokenize.TokenInfo]: 172 | """Transforms `code` into Python tokens.""" 173 | code_bytes = code.encode() 174 | code_io = io.BytesIO(code_bytes) 175 | return tokenize.tokenize(code_io.readline) 176 | 177 | 178 | def _untokenize(tokens: Sequence[tokenize.TokenInfo]) -> str: 179 | """Transforms a list of Python tokens into code.""" 180 | code_bytes = tokenize.untokenize(tokens) 181 | return code_bytes.decode() 182 | 183 | 184 | def _yield_token_and_is_call( 185 | code: str) -> Iterator[tuple[tokenize.TokenInfo, bool]]: 186 | """Yields each token with a bool indicating whether it is a function call.""" 187 | try: 188 | tokens = _tokenize(code) 189 | prev_token = None 190 | is_attribute_access = False 191 | for token in tokens: 192 | if (prev_token and # If the previous token exists and 193 | prev_token.type == tokenize.NAME and # it is a Python identifier 194 | token.type == tokenize.OP and # and the current token is a delimiter 195 | token.string == '('): # and in particular it is '('. 196 | yield prev_token, not is_attribute_access 197 | is_attribute_access = False 198 | else: 199 | if prev_token: 200 | is_attribute_access = ( 201 | prev_token.type == tokenize.OP and prev_token.string == '.' 202 | ) 203 | yield prev_token, False 204 | prev_token = token 205 | if prev_token: 206 | yield prev_token, False 207 | except Exception as e: 208 | logging.warning('Failed parsing %s', code) 209 | raise e 210 | 211 | 212 | def rename_function_calls(code: str, source_name: str, target_name: str) -> str: 213 | """Renames function calls from `source_name` to `target_name`.""" 214 | if source_name not in code: 215 | return code 216 | modified_tokens = [] 217 | for token, is_call in _yield_token_and_is_call(code): 218 | if is_call and token.string == source_name: 219 | # Replace the function name token 220 | modified_token = tokenize.TokenInfo( 221 | type=token.type, 222 | string=target_name, 223 | start=token.start, 224 | end=token.end, 225 | line=token.line, 226 | ) 227 | modified_tokens.append(modified_token) 228 | else: 229 | modified_tokens.append(token) 230 | return _untokenize(modified_tokens) 231 | 232 | 233 | def get_functions_called(code: str) -> MutableSet[str]: 234 | """Returns the set of all functions called in `code`.""" 235 | return set(token.string for token, is_call in 236 | _yield_token_and_is_call(code) if is_call) 237 | 238 | 239 | def yield_decorated(code: str, module: str, name: str) -> Iterator[str]: 240 | """Yields names of functions decorated with `@module.name` in `code`.""" 241 | tree = ast.parse(code) 242 | for node in ast.walk(tree): 243 | if isinstance(node, ast.FunctionDef): 244 | for decorator in node.decorator_list: 245 | attribute = None 246 | if isinstance(decorator, ast.Attribute): 247 | attribute = decorator 248 | elif isinstance(decorator, ast.Call): 249 | attribute = decorator.func 250 | if (attribute is not None 251 | and attribute.value.id == module 252 | and attribute.attr == name): 253 | yield node.name 254 | -------------------------------------------------------------------------------- /funsearch/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 DeepMind Technologies Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Configuration of a FunSearch experiment.""" 17 | import dataclasses 18 | 19 | 20 | @dataclasses.dataclass(frozen=True) 21 | class ProgramsDatabaseConfig: 22 | """Configuration of a ProgramsDatabase. 23 | 24 | Attributes: 25 | functions_per_prompt: Number of previous programs to include in prompts. 26 | num_islands: Number of islands to maintain as a diversity mechanism. 27 | reset_period: How often (in seconds) the weakest islands should be reset. 28 | cluster_sampling_temperature_init: Initial temperature for softmax sampling 29 | of clusters within an island. 30 | cluster_sampling_temperature_period: Period of linear decay of the cluster 31 | sampling temperature. 32 | backup_period: Number of iterations before backing up the program database on disk 33 | backup_folder: Path for automatic backups 34 | """ 35 | functions_per_prompt: int = 2 36 | num_islands: int = 10 37 | reset_period: int = 4 * 60 * 60 38 | cluster_sampling_temperature_init: float = 0.1 39 | cluster_sampling_temperature_period: int = 30_000 40 | backup_period: int = 30 41 | backup_folder: str = './data/backups' 42 | 43 | 44 | @dataclasses.dataclass(frozen=True) 45 | class Config: 46 | """Configuration of a FunSearch experiment. 47 | 48 | Attributes: 49 | programs_database: Configuration of the evolutionary algorithm. 50 | num_samplers: Number of independent Samplers in the experiment. A value 51 | larger than 1 only has an effect when the samplers are able to execute 52 | in parallel, e.g. on different matchines of a distributed system. 53 | num_evaluators: Number of independent program Evaluators in the experiment. 54 | A value larger than 1 is only expected to be useful when the Evaluators 55 | can execute in parallel as part of a distributed system. 56 | samples_per_prompt: How many independently sampled program continuations to 57 | obtain for each prompt. 58 | """ 59 | programs_database: ProgramsDatabaseConfig = dataclasses.field( 60 | default_factory=ProgramsDatabaseConfig) 61 | num_samplers: int = 15 62 | num_evaluators: int = 140 63 | samples_per_prompt: int = 4 64 | -------------------------------------------------------------------------------- /funsearch/container/Dockerfile: -------------------------------------------------------------------------------- 1 | # the python version in the sandbox should be the same as in the host so that cloudpickle works reliably 2 | ARG PYTHON_VERSION=3.11.6 3 | 4 | FROM docker.io/python:${PYTHON_VERSION} 5 | 6 | ARG INSTALL_PACKAGES 7 | RUN pip install cloudpickle ${INSTALL_PACKAGES} 8 | -------------------------------------------------------------------------------- /funsearch/container/container_main.py: -------------------------------------------------------------------------------- 1 | """This file will be used as an executable script by the ContainerSandbox and ExternalProcessSandbox. 2 | """ 3 | import logging 4 | import pickle 5 | import sys 6 | 7 | 8 | def main(prog_file: str, input_file: str, output_file: str): 9 | """The method takes executable function as a cloudpickle file, then executes it with input data, 10 | and writes the output data to another file.""" 11 | logging.debug(f"Running main(): {prog_file}, {input_file}, {output_file}") 12 | with open(prog_file, "rb") as f: 13 | func = pickle.load(f) 14 | 15 | with open(input_file, "rb") as input_f: 16 | input_data = pickle.load(input_f) 17 | 18 | ret = func(input_data) 19 | with open(output_file, "wb") as of: 20 | logging.debug(f"Writing output to {output_file}") 21 | pickle.dump(ret, of) 22 | 23 | 24 | if __name__ == '__main__': 25 | if len(sys.argv) != 4: 26 | sys.exit(-1) 27 | main(sys.argv[1], sys.argv[2], sys.argv[3]) 28 | -------------------------------------------------------------------------------- /funsearch/core.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 DeepMind Technologies Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """A single-threaded implementation of the FunSearch pipeline.""" 17 | import logging 18 | 19 | from funsearch import code_manipulation 20 | 21 | 22 | def _extract_function_names(specification: str) -> tuple[str, str]: 23 | """Returns the name of the function to evolve and of the function to run.""" 24 | run_functions = list( 25 | code_manipulation.yield_decorated(specification, 'funsearch', 'run')) 26 | if len(run_functions) != 1: 27 | raise ValueError('Expected 1 function decorated with `@funsearch.run`.') 28 | evolve_functions = list( 29 | code_manipulation.yield_decorated(specification, 'funsearch', 'evolve')) 30 | if len(evolve_functions) != 1: 31 | raise ValueError('Expected 1 function decorated with `@funsearch.evolve`.') 32 | return evolve_functions[0], run_functions[0] 33 | 34 | 35 | def run(samplers, database, iterations: int = -1): 36 | """Launches a FunSearch experiment.""" 37 | 38 | try: 39 | # This loop can be executed in parallel on remote sampler machines. As each 40 | # sampler enters an infinite loop, without parallelization only the first 41 | # sampler will do any work. 42 | while iterations != 0: 43 | for s in samplers: 44 | s.sample() 45 | if iterations > 0: 46 | iterations -= 1 47 | except KeyboardInterrupt: 48 | logging.info("Keyboard interrupt. Stopping.") 49 | database.backup() 50 | 51 | -------------------------------------------------------------------------------- /funsearch/evaluator.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 DeepMind Technologies Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Class for evaluating programs proposed by the Sampler.""" 17 | import ast 18 | import re 19 | from collections.abc import Sequence 20 | import copy 21 | from typing import Any, Tuple 22 | 23 | from funsearch import code_manipulation 24 | from funsearch import programs_database 25 | from funsearch import sandbox 26 | 27 | """ 28 | Regex to find all methods named 'priority_vX'. 29 | With each match, start from the 'def priority_vX(' and continue until there's a new line with any of 30 | - a new 'def' 31 | - ` or ' or # without indentation 32 | """ 33 | METHOD_MATCHER = re.compile(r"def priority_v\d\(.*?\) -> float:(?:\s*(?:[ \t]*(?!def|#|`|').*(?:\n|$)))+") 34 | METHOD_NAME_MATCHER = re.compile(r"priority_v\d+") 35 | 36 | 37 | class _FunctionLineVisitor(ast.NodeVisitor): 38 | """Visitor that finds the last line number of a function with a given name.""" 39 | 40 | def __init__(self, target_function_name: str) -> None: 41 | self._target_function_name: str = target_function_name 42 | self._function_end_line: int | None = None 43 | 44 | def visit_FunctionDef(self, node: Any) -> None: # pylint: disable=invalid-name 45 | """Collects the end line number of the target function.""" 46 | if node.name == self._target_function_name: 47 | self._function_end_line = node.end_lineno 48 | self.generic_visit(node) 49 | 50 | @property 51 | def function_end_line(self) -> int: 52 | """Line number of the final line of function `target_function_name`.""" 53 | assert self._function_end_line is not None # Check internal correctness. 54 | return self._function_end_line 55 | 56 | 57 | def _find_method_implementation(generated_code: str) -> Tuple[str, str]: 58 | """Find the last 'def priority_vX()' method from generated code. 59 | 60 | Return the code and the name of the method. 61 | """ 62 | matches = METHOD_MATCHER.findall(generated_code) 63 | if not matches: 64 | return "", "" 65 | last_match = matches[-1] 66 | name = METHOD_NAME_MATCHER.search(last_match).group() 67 | return last_match, name 68 | 69 | 70 | def _trim_function_body(generated_code: str) -> str: 71 | """Extracts the body of the generated function, trimming anything after it.""" 72 | if not generated_code: 73 | return '' 74 | if not type(generated_code) is str: 75 | generated_code = str(generated_code) 76 | 77 | method_name = "fake_function_header" 78 | # Check is the response only a continuation for our prompt or full method implementation with header 79 | if "def priority_v" in generated_code: 80 | code, method_name = _find_method_implementation(generated_code) 81 | else: 82 | code = f'def {method_name}():\n{generated_code}' 83 | 84 | # Finally parse the code to make sure it's valid Python 85 | tree = None 86 | # We keep trying and deleting code from the end until the parser succeeds. 87 | while tree is None: 88 | try: 89 | tree = ast.parse(code) 90 | except SyntaxError as e: 91 | code = '\n'.join(code.splitlines()[:e.lineno - 1]) 92 | if not code: 93 | # Nothing could be saved from `generated_code` 94 | return '' 95 | 96 | visitor = _FunctionLineVisitor(method_name) 97 | visitor.visit(tree) 98 | body_lines = code.splitlines()[1:visitor.function_end_line] 99 | return '\n'.join(body_lines) + '\n\n' 100 | 101 | 102 | def _sample_to_program( 103 | generated_code: str, 104 | version_generated: int | None, 105 | template: code_manipulation.Program, 106 | function_to_evolve: str, 107 | ) -> tuple[code_manipulation.Function, str]: 108 | """Returns the compiled generated function and the full runnable program.""" 109 | body = _trim_function_body(generated_code) 110 | if version_generated is not None: 111 | body = code_manipulation.rename_function_calls( 112 | body, 113 | f'{function_to_evolve}_v{version_generated}', 114 | function_to_evolve) 115 | 116 | program = copy.deepcopy(template) 117 | evolved_function = program.get_function(function_to_evolve) 118 | evolved_function.body = body 119 | return evolved_function, str(program) 120 | 121 | 122 | 123 | def _calls_ancestor(program: str, function_to_evolve: str) -> bool: 124 | """Returns whether the generated function is calling an earlier version.""" 125 | for name in code_manipulation.get_functions_called(program): 126 | # In `program` passed into this function the most recently generated 127 | # function has already been renamed to `function_to_evolve` (wihout the 128 | # suffix). Therefore any function call starting with `function_to_evolve_v` 129 | # is a call to an ancestor function. 130 | if name.startswith(f'{function_to_evolve}_v'): 131 | return True 132 | return False 133 | 134 | 135 | class Evaluator: 136 | """Class that analyses functions generated by LLMs.""" 137 | 138 | def __init__( 139 | self, 140 | database: programs_database.ProgramsDatabase, 141 | sbox: sandbox.DummySandbox, 142 | template: code_manipulation.Program, 143 | function_to_evolve: str, 144 | function_to_run: str, 145 | inputs: Sequence[Any], 146 | timeout_seconds: int = 30, 147 | ): 148 | self._database = database 149 | self._template = template 150 | self._function_to_evolve = function_to_evolve 151 | self._function_to_run = function_to_run 152 | self._inputs = inputs 153 | self._timeout_seconds = timeout_seconds 154 | self._sandbox = sbox 155 | 156 | def analyse( 157 | self, 158 | sample: str, 159 | island_id: int | None, 160 | version_generated: int | None, 161 | ) -> None: 162 | """Compiles the sample into a program and executes it on test inputs.""" 163 | new_function, program = _sample_to_program( 164 | sample, version_generated, self._template, self._function_to_evolve) 165 | 166 | scores_per_test = {} 167 | for current_input in self._inputs: 168 | test_output, runs_ok = self._sandbox.run( 169 | program, self._function_to_run, current_input, self._timeout_seconds) 170 | if (runs_ok and not _calls_ancestor(program, self._function_to_evolve) 171 | and test_output is not None): 172 | if not isinstance(test_output, (int, float)): 173 | raise ValueError('@function.run did not return an int/float score.') 174 | scores_per_test[current_input] = test_output 175 | if scores_per_test: 176 | self._database.register_program(new_function, island_id, scores_per_test) 177 | -------------------------------------------------------------------------------- /funsearch/programs_database.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 DeepMind Technologies Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """A programs database that implements the evolutionary algorithm.""" 17 | import pathlib 18 | import pickle 19 | from collections.abc import Mapping, Sequence 20 | import copy 21 | import dataclasses 22 | import time 23 | from typing import Any, Iterable, Tuple 24 | 25 | from absl import logging 26 | import numpy as np 27 | import scipy 28 | 29 | from funsearch import code_manipulation 30 | from funsearch import config as config_lib 31 | 32 | Signature = tuple[float, ...] 33 | ScoresPerTest = Mapping[Any, float] 34 | 35 | 36 | def _softmax(logits: np.ndarray, temperature: float) -> np.ndarray: 37 | """Returns the tempered softmax of 1D finite `logits`.""" 38 | if not np.all(np.isfinite(logits)): 39 | non_finites = set(logits[~np.isfinite(logits)]) 40 | raise ValueError(f'`logits` contains non-finite value(s): {non_finites}') 41 | if not np.issubdtype(logits.dtype, np.floating): 42 | logits = np.array(logits, dtype=np.float32) 43 | 44 | result = scipy.special.softmax(logits / temperature, axis=-1) 45 | # Ensure that probabilities sum to 1 to prevent error in `np.random.choice`. 46 | index = np.argmax(result) 47 | result[index] = 1 - np.sum(result[0:index]) - np.sum(result[index+1:]) 48 | return result 49 | 50 | 51 | def _reduce_score(scores_per_test: ScoresPerTest) -> float: 52 | """Reduces per-test scores into a single score.""" 53 | return scores_per_test[list(scores_per_test.keys())[-1]] 54 | 55 | 56 | def _get_signature(scores_per_test: ScoresPerTest) -> Signature: 57 | """Represents test scores as a canonical signature.""" 58 | return tuple(scores_per_test[k] for k in sorted(scores_per_test.keys())) 59 | 60 | 61 | @dataclasses.dataclass(frozen=True) 62 | class Prompt: 63 | """A prompt produced by the ProgramsDatabase, to be sent to Samplers. 64 | 65 | Attributes: 66 | code: The prompt, ending with the header of the function to be completed. 67 | version_generated: The function to be completed is `_v{version_generated}`. 68 | island_id: Identifier of the island that produced the implementations 69 | included in the prompt. Used to direct the newly generated implementation 70 | into the same island. 71 | """ 72 | code: str 73 | version_generated: int 74 | island_id: int 75 | 76 | 77 | class ProgramsDatabase: 78 | """A collection of programs, organized as islands.""" 79 | 80 | def __init__( 81 | self, 82 | config: config_lib.ProgramsDatabaseConfig, 83 | template: code_manipulation.Program, 84 | function_to_evolve: str, 85 | identifier: str = "", 86 | ) -> None: 87 | self._config: config_lib.ProgramsDatabaseConfig = config 88 | self._template: code_manipulation.Program = template 89 | self._function_to_evolve: str = function_to_evolve 90 | 91 | # Initialize empty islands. 92 | self._islands: list[Island] = [] 93 | for _ in range(config.num_islands): 94 | self._islands.append( 95 | Island(template, function_to_evolve, config.functions_per_prompt, 96 | config.cluster_sampling_temperature_init, 97 | config.cluster_sampling_temperature_period)) 98 | self._best_score_per_island: list[float] = ( 99 | [-float('inf')] * config.num_islands) 100 | self._best_program_per_island: list[code_manipulation.Function | None] = ( 101 | [None] * config.num_islands) 102 | self._best_scores_per_test_per_island: list[ScoresPerTest | None] = ( 103 | [None] * config.num_islands) 104 | 105 | self._last_reset_time: float = time.time() 106 | self._program_counter = 0 107 | self._backups_done = 0 108 | self.identifier = identifier 109 | 110 | def get_best_programs_per_island(self) -> Iterable[Tuple[code_manipulation.Function | None]]: 111 | return sorted(zip(self._best_program_per_island, self._best_score_per_island), key=lambda t: t[1], reverse=True) 112 | 113 | def save(self, file): 114 | """Save database to a file""" 115 | data = {} 116 | keys = ["_islands", "_best_score_per_island", "_best_program_per_island", "_best_scores_per_test_per_island"] 117 | for key in keys: 118 | data[key] = getattr(self, key) 119 | pickle.dump(data, file) 120 | 121 | def load(self, file): 122 | """Load previously saved database""" 123 | data = pickle.load(file) 124 | for key in data.keys(): 125 | setattr(self, key, data[key]) 126 | 127 | def backup(self): 128 | filename = f"program_db_{self._function_to_evolve}_{self.identifier}_{self._backups_done}.pickle" 129 | p = pathlib.Path(self._config.backup_folder) 130 | if not p.exists(): 131 | p.mkdir(parents=True, exist_ok=True) 132 | filepath = p / filename 133 | logging.info(f"Saving backup to {filepath}.") 134 | 135 | with open(filepath, mode="wb") as f: 136 | self.save(f) 137 | self._backups_done += 1 138 | 139 | def get_prompt(self) -> Prompt: 140 | """Returns a prompt containing implementations from one chosen island.""" 141 | island_id = np.random.randint(len(self._islands)) 142 | code, version_generated = self._islands[island_id].get_prompt() 143 | return Prompt(code, version_generated, island_id) 144 | 145 | def _register_program_in_island( 146 | self, 147 | program: code_manipulation.Function, 148 | island_id: int, 149 | scores_per_test: ScoresPerTest, 150 | ) -> None: 151 | """Registers `program` in the specified island.""" 152 | self._islands[island_id].register_program(program, scores_per_test) 153 | score = _reduce_score(scores_per_test) 154 | if score > self._best_score_per_island[island_id]: 155 | self._best_program_per_island[island_id] = program 156 | self._best_scores_per_test_per_island[island_id] = scores_per_test 157 | self._best_score_per_island[island_id] = score 158 | logging.info('Best score of island %d increased to %s', island_id, score) 159 | 160 | def register_program( 161 | self, 162 | program: code_manipulation.Function, 163 | island_id: int | None, 164 | scores_per_test: ScoresPerTest, 165 | ) -> None: 166 | """Registers `program` in the database.""" 167 | # In an asynchronous implementation we should consider the possibility of 168 | # registering a program on an island that had been reset after the prompt 169 | # was generated. Leaving that out here for simplicity. 170 | if island_id is None: 171 | # This is a program added at the beginning, so adding it to all islands. 172 | for island_id in range(len(self._islands)): 173 | self._register_program_in_island(program, island_id, scores_per_test) 174 | else: 175 | self._register_program_in_island(program, island_id, scores_per_test) 176 | 177 | # Check whether it is time to reset an island. 178 | if (time.time() - self._last_reset_time > self._config.reset_period): 179 | self._last_reset_time = time.time() 180 | self.reset_islands() 181 | 182 | # Backup every N iterations 183 | if self._program_counter > 0: 184 | self._program_counter += 1 185 | if self._program_counter > self._config.backup_period: 186 | self._program_counter = 0 187 | self.backup() 188 | 189 | def reset_islands(self) -> None: 190 | """Resets the weaker half of islands.""" 191 | # We sort best scores after adding minor noise to break ties. 192 | indices_sorted_by_score: np.ndarray = np.argsort( 193 | self._best_score_per_island + 194 | np.random.randn(len(self._best_score_per_island)) * 1e-6) 195 | num_islands_to_reset = self._config.num_islands // 2 196 | reset_islands_ids = indices_sorted_by_score[:num_islands_to_reset] 197 | keep_islands_ids = indices_sorted_by_score[num_islands_to_reset:] 198 | for island_id in reset_islands_ids: 199 | self._islands[island_id] = Island( 200 | self._template, 201 | self._function_to_evolve, 202 | self._config.functions_per_prompt, 203 | self._config.cluster_sampling_temperature_init, 204 | self._config.cluster_sampling_temperature_period) 205 | self._best_score_per_island[island_id] = -float('inf') 206 | founder_island_id = np.random.choice(keep_islands_ids) 207 | founder = self._best_program_per_island[founder_island_id] 208 | founder_scores = self._best_scores_per_test_per_island[founder_island_id] 209 | self._register_program_in_island(founder, island_id, founder_scores) 210 | 211 | 212 | class Island: 213 | """A sub-population of the programs database.""" 214 | 215 | def __init__( 216 | self, 217 | template: code_manipulation.Program, 218 | function_to_evolve: str, 219 | functions_per_prompt: int, 220 | cluster_sampling_temperature_init: float, 221 | cluster_sampling_temperature_period: int, 222 | ) -> None: 223 | self._template: code_manipulation.Program = template 224 | self._function_to_evolve: str = function_to_evolve 225 | self._functions_per_prompt: int = functions_per_prompt 226 | self._cluster_sampling_temperature_init = cluster_sampling_temperature_init 227 | self._cluster_sampling_temperature_period = ( 228 | cluster_sampling_temperature_period) 229 | 230 | self._clusters: dict[Signature, Cluster] = {} 231 | self._num_programs: int = 0 232 | 233 | def register_program( 234 | self, 235 | program: code_manipulation.Function, 236 | scores_per_test: ScoresPerTest, 237 | ) -> None: 238 | """Stores a program on this island, in its appropriate cluster.""" 239 | signature = _get_signature(scores_per_test) 240 | if signature not in self._clusters: 241 | score = _reduce_score(scores_per_test) 242 | self._clusters[signature] = Cluster(score, program) 243 | else: 244 | self._clusters[signature].register_program(program) 245 | self._num_programs += 1 246 | 247 | def get_prompt(self) -> tuple[str, int]: 248 | """Constructs a prompt containing functions from this island.""" 249 | signatures = list(self._clusters.keys()) 250 | cluster_scores = np.array( 251 | [self._clusters[signature].score for signature in signatures]) 252 | 253 | # Convert scores to probabilities using softmax with temperature schedule. 254 | period = self._cluster_sampling_temperature_period 255 | temperature = self._cluster_sampling_temperature_init * ( 256 | 1 - (self._num_programs % period) / period) 257 | probabilities = _softmax(cluster_scores, temperature) 258 | 259 | # At the beginning of an experiment when we have few clusters, place fewer 260 | # programs into the prompt. 261 | functions_per_prompt = min(len(self._clusters), self._functions_per_prompt) 262 | 263 | idx = np.random.choice( 264 | len(signatures), size=functions_per_prompt, p=probabilities) 265 | chosen_signatures = [signatures[i] for i in idx] 266 | implementations = [] 267 | scores = [] 268 | for signature in chosen_signatures: 269 | cluster = self._clusters[signature] 270 | implementations.append(cluster.sample_program()) 271 | scores.append(cluster.score) 272 | 273 | indices = np.argsort(scores) 274 | sorted_implementations = [implementations[i] for i in indices] 275 | version_generated = len(sorted_implementations) + 1 276 | return self._generate_prompt(sorted_implementations), version_generated 277 | 278 | def _generate_prompt( 279 | self, 280 | implementations: Sequence[code_manipulation.Function]) -> str: 281 | """Creates a prompt containing a sequence of function `implementations`.""" 282 | implementations = copy.deepcopy(implementations) # We will mutate these. 283 | 284 | # Format the names and docstrings of functions to be included in the prompt. 285 | versioned_functions: list[code_manipulation.Function] = [] 286 | for i, implementation in enumerate(implementations): 287 | new_function_name = f'{self._function_to_evolve}_v{i}' 288 | implementation.name = new_function_name 289 | # Update the docstring for all subsequent functions after `_v0`. 290 | if i >= 1: 291 | implementation.docstring = ( 292 | f'Improved version of `{self._function_to_evolve}_v{i - 1}`.') 293 | # If the function is recursive, replace calls to itself with its new name. 294 | implementation = code_manipulation.rename_function_calls( 295 | str(implementation), self._function_to_evolve, new_function_name) 296 | versioned_functions.append( 297 | code_manipulation.text_to_function(implementation)) 298 | 299 | # Create the header of the function to be generated by the LLM. 300 | next_version = len(implementations) 301 | new_function_name = f'{self._function_to_evolve}_v{next_version}' 302 | header = dataclasses.replace( 303 | implementations[-1], 304 | name=new_function_name, 305 | body='', 306 | docstring=('Improved version of ' 307 | f'`{self._function_to_evolve}_v{next_version - 1}`.'), 308 | ) 309 | versioned_functions.append(header) 310 | 311 | # Replace functions in the template with the list constructed here. 312 | prompt = dataclasses.replace(self._template, functions=versioned_functions) 313 | return str(prompt) 314 | 315 | 316 | class Cluster: 317 | """A cluster of programs on the same island and with the same Signature.""" 318 | 319 | def __init__(self, score: float, implementation: code_manipulation.Function): 320 | self._score = score 321 | self._programs: list[code_manipulation.Function] = [implementation] 322 | self._lengths: list[int] = [len(str(implementation))] 323 | 324 | @property 325 | def score(self) -> float: 326 | """Reduced score of the signature that this cluster represents.""" 327 | return self._score 328 | 329 | def register_program(self, program: code_manipulation.Function) -> None: 330 | """Adds `program` to the cluster.""" 331 | self._programs.append(program) 332 | self._lengths.append(len(str(program))) 333 | 334 | def sample_program(self) -> code_manipulation.Function: 335 | """Samples a program, giving higher probability to shorther programs.""" 336 | normalized_lengths = (np.array(self._lengths) - min(self._lengths)) / ( 337 | max(self._lengths) + 1e-6) 338 | probabilities = _softmax(-normalized_lengths, temperature=1.0) 339 | return np.random.choice(self._programs, p=probabilities) 340 | -------------------------------------------------------------------------------- /funsearch/sampler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 DeepMind Technologies Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | """Class for sampling new programs.""" 17 | from collections.abc import Collection, Sequence 18 | 19 | import llm 20 | import numpy as np 21 | 22 | from funsearch import evaluator 23 | from funsearch import programs_database 24 | 25 | 26 | class LLM: 27 | """Language model that predicts continuation of provided source code.""" 28 | 29 | def __init__(self, samples_per_prompt: int, model: llm.Model, log_path=None) -> None: 30 | self._samples_per_prompt = samples_per_prompt 31 | self.model = model 32 | self.prompt_count = 0 33 | self.log_path = log_path 34 | 35 | def _draw_sample(self, prompt: str) -> str: 36 | """Returns a predicted continuation of `prompt`.""" 37 | response = self.model.prompt(prompt) 38 | self._log(prompt, response, self.prompt_count) 39 | self.prompt_count += 1 40 | return response 41 | 42 | def draw_samples(self, prompt: str) -> Collection[str]: 43 | """Returns multiple predicted continuations of `prompt`.""" 44 | return [self._draw_sample(prompt) for _ in range(self._samples_per_prompt)] 45 | 46 | def _log(self, prompt: str, response: str, index: int): 47 | if self.log_path is not None: 48 | with open(self.log_path / f"prompt_{index}.log", "a") as f: 49 | f.write(prompt) 50 | with open(self.log_path / f"response_{index}.log", "a") as f: 51 | f.write(str(response)) 52 | 53 | 54 | class Sampler: 55 | """Node that samples program continuations and sends them for analysis.""" 56 | 57 | def __init__( 58 | self, 59 | database: programs_database.ProgramsDatabase, 60 | evaluators: Sequence[evaluator.Evaluator], 61 | model: LLM, 62 | ) -> None: 63 | self._database = database 64 | self._evaluators = evaluators 65 | self._llm = model 66 | 67 | def sample(self): 68 | """Continuously gets prompts, samples programs, sends them for analysis.""" 69 | prompt = self._database.get_prompt() 70 | samples = self._llm.draw_samples(prompt.code) 71 | # This loop can be executed in parallel on remote evaluator machines. 72 | for sample in samples: 73 | chosen_evaluator = np.random.choice(self._evaluators) 74 | chosen_evaluator.analyse( 75 | sample, prompt.island_id, prompt.version_generated) 76 | -------------------------------------------------------------------------------- /funsearch/sandbox.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import ast 4 | import os 5 | import pathlib 6 | import sys 7 | from typing import Any 8 | 9 | import cloudpickle 10 | 11 | CONTAINER_MAIN = (pathlib.Path(__file__).parent / "container" / "container_main.py").absolute() 12 | 13 | IMAGE_NAME = "funsearch_sandbox" 14 | 15 | 16 | class DummySandbox: 17 | """Base class for Sandboxes that execute the generated code. 18 | 19 | Note: this base class executes the code but does not offer any sandboxing!!! 20 | It should be only used in unit testing or debugging, and not with real LLM 21 | unless the host environment is in some kind of sandbox itself. 22 | Even in sandboxed host, the executed code could theoretically affect later executions. 23 | """ 24 | 25 | sandboxes = 0 26 | 27 | def __init__(self, **kwargs): 28 | self.id = DummySandbox.sandboxes 29 | 30 | DummySandbox.sandboxes += 1 31 | 32 | def run( 33 | self, 34 | program: str, 35 | function_to_run: str, 36 | test_input, 37 | timeout_seconds: int, 38 | ) -> tuple[Any, bool]: 39 | """Returns `function_to_run(test_input)` and whether execution succeeded.""" 40 | 41 | # The same "program" seems to be now repeatedly parsed using AST and then compiled. 42 | # This could probably be simplified quite a bit. 43 | namespace = DummySandbox.compile_code(program) 44 | return namespace[function_to_run](test_input) 45 | 46 | @staticmethod 47 | def compile_code(program: str): 48 | namespace = {} 49 | 50 | parsed_code = ast.parse(program) 51 | compiled_code = compile(parsed_code, filename="", mode="exec") 52 | exec(compiled_code, namespace) 53 | return namespace 54 | 55 | 56 | class ExternalProcessSandbox(DummySandbox): 57 | """Sandbox that executes the code in a separate Python process in the same host. 58 | 59 | Note: This does not provide real safety and should be only used in an environment where the host process is 60 | in some kind of safe sandbox itself (i.e., a container). 61 | This kind of sandbox merely makes it more probable that single invalid call does not break the whole 62 | funsearch algorithm. It might be easier to set up and thus nice environment to tune the prompts and other code. 63 | """ 64 | 65 | def __init__(self, base_path: pathlib.Path, timeout_secs: int = 30, python_path: str = "python"): 66 | super(ExternalProcessSandbox, self).__init__() 67 | 68 | self.output_path = pathlib.Path(base_path) / f"sandbox{self.id}" 69 | self.timeout_secs = timeout_secs 70 | self.python_path = python_path 71 | self.call_count = 0 72 | 73 | self.input_path = self.output_path / "inputs" 74 | for p in [self.output_path, self.input_path]: 75 | if not p.exists(): 76 | p.mkdir(parents=True) 77 | 78 | def _exec(self, call_data_path: pathlib.Path, input_path: pathlib.Path, error_file_path: pathlib.Path): 79 | """Use podman/docker to execute python in a container. 80 | - The main.py shall execute the LLM generated method from prog.pickle file providing 81 | input.pickle as the input for the method. 82 | - main.py writes the output of the method into output.pickle. 83 | Everything except the /workspace folder will be read-only so that the environment remains good 84 | for future runs. 85 | """ 86 | prog_path = call_data_path / "prog.pickle" 87 | output_file = call_data_path / "output.pickle" 88 | cmd = (f"{self.python_path} {CONTAINER_MAIN} {prog_path} {input_path} {output_file}" 89 | f" 2> {error_file_path}") 90 | logging.debug(f"Executing: {cmd}") 91 | return os.system(cmd) 92 | 93 | def run( 94 | self, 95 | program: str, 96 | function_to_run: str, 97 | test_input, 98 | timeout_seconds: int, 99 | ) -> tuple[Any, bool]: 100 | 101 | call_data_folder = (self.output_path / f"call{self.call_count}").absolute() 102 | if not call_data_folder.exists(): 103 | call_data_folder.mkdir() 104 | 105 | input_hash = hash(test_input) 106 | input_path = (self.input_path / f"{input_hash}.pickle").absolute() 107 | 108 | if not input_path.exists(): 109 | with open(input_path, "wb") as f: 110 | cloudpickle.dump(test_input, f) 111 | try: 112 | namespace = DummySandbox.compile_code(program) 113 | 114 | prog_file = (call_data_folder / f"prog.pickle").absolute() 115 | with open(prog_file, "wb+") as f: 116 | cloudpickle.dump(namespace[function_to_run], f) 117 | 118 | error_file = self.output_path / f"stderr_{self.call_count}.log" 119 | 120 | retcode = self._exec(call_data_folder, input_path, error_file) 121 | self.call_count += 1 122 | 123 | if retcode != 0: 124 | self._save_diagnostics(program, call_data_folder) 125 | return None, False 126 | 127 | output_file = call_data_folder / f"output.pickle" 128 | with open(output_file, "rb") as f: 129 | out = cloudpickle.load(f) 130 | return out, True 131 | except Exception as e: 132 | logging.debug(f"Could not execute code: {e}") 133 | self._save_diagnostics(program, call_data_folder) 134 | return None, False 135 | 136 | @staticmethod 137 | def _save_diagnostics(program: str, output_path: pathlib.Path): 138 | filepath = output_path / "program.py" 139 | logging.debug(f"Writing program to {filepath}") 140 | with open(filepath, "w+") as f: 141 | f.write(program) 142 | 143 | 144 | class ContainerSandbox(ExternalProcessSandbox): 145 | """Basic sandbox that runs unsafe code in Podman or Docker container. 146 | - the sandbox should be safe against inadvertent bad code by LLM but not against malicious attacks. 147 | - does not require any other dependencies on the host than Podman/Docker 148 | - does not support multithreading 149 | - might provide easier or more lightweight debugging experience than some other fancier sandbox environments 150 | """ 151 | executable = "podman" 152 | image_built = False 153 | 154 | @classmethod 155 | def build_image(cls, extra_pip_packages): 156 | version = sys.version.split(" ")[0] 157 | ret = os.system("podman --version") 158 | if ret != 0: 159 | ret = os.system("docker --version") 160 | if ret != 0: 161 | raise Exception("Could not find Podman or Docker. Can not use ContainerSandbox.") 162 | else: 163 | cls.executable = "docker" 164 | 165 | dockerfile = pathlib.Path(__file__).parent / "container" / "Dockerfile" 166 | logging.debug("Building container image") 167 | extra = "" 168 | if extra_pip_packages: 169 | extra = f"--build-arg INSTALL_PACKAGES=\"{extra_pip_packages}\"" 170 | 171 | cmd = (f"{cls.executable} build --build-arg PYTHON_VERSION={version} {extra} " 172 | f"-t {IMAGE_NAME} -f {dockerfile} {CONTAINER_MAIN.parent}") 173 | logging.debug(f"Executing: {cmd}") 174 | os.system(cmd) 175 | cls.image_built = True 176 | 177 | def __init__(self, base_path: pathlib.Path, extra_pip_packages: str = "numpy", timeout_secs=30): 178 | super(ContainerSandbox, self).__init__(base_path, timeout_secs) 179 | 180 | if not ContainerSandbox.image_built: 181 | ContainerSandbox.build_image(extra_pip_packages) 182 | 183 | def _exec(self, call_data_path: pathlib.Path, input_path: pathlib.Path, error_file_path: pathlib.Path): 184 | """Use podman/docker to execute python in a container. 185 | - The main.py shall execute the LLM generated method from prog.pickle file providing 186 | input.pickle as the input for the method. 187 | - main.py writes the output of the method into output.pickle. 188 | Everything except the /workspace folder will be read-only so that the environment remains good 189 | for future runs. 190 | """ 191 | cmd = (f"{self.executable} run " 192 | f"--stop-timeout={self.timeout_secs} " 193 | f"-v {CONTAINER_MAIN}:/main.py:ro " 194 | f"-v {call_data_path}:/workspace " 195 | f"-v {input_path}:/input.pickle:ro " 196 | f"{IMAGE_NAME}:latest /usr/local/bin/python3 " 197 | f"/main.py /workspace/prog.pickle /input.pickle /workspace/output.pickle" 198 | f" 2> {error_file_path}") 199 | logging.debug(f"Executing: {cmd}") 200 | return os.system(cmd) 201 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.setuptools] 6 | packages = ["funsearch", "funsearch.container"] 7 | 8 | [project] 9 | name = "funsearch" 10 | dynamic = [ 11 | "version", 12 | ] 13 | description = "FunSearch algorithm: Mathematical discoveries from program search with large language models" 14 | readme = "README.md" 15 | requires-python = ">=3.9" 16 | 17 | classifiers = [ 18 | "Intended Audience :: Science/Research", 19 | "Programming Language :: Python", 20 | "Programming Language :: Python :: 3", 21 | "Topic :: Scientific/Engineering :: Artificial Intelligence", 22 | ] 23 | dependencies = [ 24 | "absl-py==2.0.0", 25 | "click==8.1", 26 | "cloudpickle==3.0.0", 27 | "llm==0.12", 28 | "python-dotenv==1.0.0", 29 | "scipy==1.11.4", 30 | "numpy==1.26.2", 31 | ] 32 | 33 | [project.scripts] 34 | funsearch = "funsearch.__main__:main" 35 | -------------------------------------------------------------------------------- /tests/code_manipulation_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 DeepMind Technologies Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | import itertools 17 | import textwrap 18 | 19 | from absl.testing import absltest 20 | from absl.testing import parameterized 21 | 22 | from funsearch import code_manipulation 23 | 24 | _IMPORTS: str = '''\ 25 | import itertools 26 | import numpy 27 | 28 | 29 | ''' 30 | 31 | _CLASS: str = '''\ 32 | class Helper: 33 | def __init__(self, n: int): 34 | self.n = n 35 | self.initial_capset = get_capset_v0(n) 36 | 37 | 38 | ''' 39 | 40 | _ASSIGNMENT: str = '''\ 41 | some_global_variable = 0 42 | 43 | 44 | ''' 45 | 46 | _FUNCTIONS: str = '''\ 47 | def get_capset_v0(n: int) -> set[tuple[int, ...]]: 48 | """Computes a cap set for n number of copies. 49 | 50 | A cap set is a subset of an n-dimensional affine space over a three-element 51 | field with no three elements in a line. 52 | 53 | Args: 54 | n: an integer, number of copies. 55 | 56 | Returns: 57 | A set of tuples in {0, 1, 2}. 58 | """ 59 | capset = set() 60 | for i in range(n): 61 | capset.add((0,) * i + (1,) + (0,) * (n - i - 1)) 62 | return capset 63 | 64 | 65 | def get_capset_v2(k: int): 66 | """One line docstring.""" 67 | get_capset_v0 = get_capset_v0(k) 68 | return get_capset_v0\\ 69 | 70 | ''' 71 | 72 | 73 | _SMALL_PROGRAM: str = '''\ 74 | def test() -> np.ndarray: 75 | return np.zeros(1) 76 | ''' 77 | 78 | _FUNCTION_HEADER: str = 'def get_capset_v0(n: int)' 79 | _FUNCTION_RETURN_TYPE: str = ' -> set[tuple[int, ...]]' 80 | _FUNCTION_DOCSTRING: str = ' """One line docstring."""' 81 | _FUNCTION_BODY: str = '''\ 82 | capset = set() 83 | for i in range(n): 84 | capset.add((0,) * i + (1,) + (0,) * (n - i - 1)) 85 | return capset 86 | ''' 87 | 88 | 89 | def create_test_function( 90 | has_return_type: bool, has_docstring: bool 91 | ) -> str: 92 | code = _FUNCTION_HEADER 93 | if has_return_type: 94 | code += _FUNCTION_RETURN_TYPE 95 | code += ':\n' 96 | if has_docstring: 97 | code += _FUNCTION_DOCSTRING 98 | code += '\n' 99 | code += _FUNCTION_BODY 100 | return code 101 | 102 | 103 | def create_test_program( 104 | has_imports: bool, has_class: bool, has_assignment: bool 105 | ) -> str: 106 | code = '' 107 | if has_imports: 108 | code += _IMPORTS 109 | if has_class: 110 | code += _CLASS 111 | if has_assignment: 112 | code += _ASSIGNMENT 113 | code += _FUNCTIONS 114 | return code 115 | 116 | 117 | class PromptSamplingTest(parameterized.TestCase): 118 | 119 | @parameterized.parameters(list(itertools.product([False, True], repeat=2))) 120 | def test_text_to_function(self, has_return_type: bool, has_docstring: bool): 121 | function = code_manipulation.text_to_function(create_test_function( 122 | has_return_type, has_docstring)) 123 | self.assertEqual(function.name, 'get_capset_v0') 124 | self.assertEqual(function.args, 'n: int') 125 | if has_return_type: 126 | self.assertEqual(function.return_type, 'set[tuple[int, ...]]') 127 | else: 128 | self.assertIsNone(function.return_type) 129 | if has_docstring: 130 | self.assertEqual(function.docstring, 'One line docstring.') 131 | else: 132 | self.assertIsNone(function.docstring) 133 | self.assertEqual(function.body, _FUNCTION_BODY.rstrip()) 134 | 135 | def test_small_text_to_program(self): 136 | program = code_manipulation.text_to_program(_SMALL_PROGRAM) 137 | self.assertEmpty(program.preface) 138 | self.assertLen(program.functions, 1) 139 | 140 | expected_function = code_manipulation.Function( 141 | name='test', args='', return_type='np.ndarray', 142 | body=' return np.zeros(1)') 143 | self.assertEqual(expected_function, program.functions[0]) 144 | self.assertEqual(_SMALL_PROGRAM + '\n', str(program)) 145 | 146 | # Assert that we do not add one more '\n' each time we convert to program. 147 | program_again = code_manipulation.text_to_program(str(program)) 148 | self.assertEqual(str(program), str(program_again)) 149 | 150 | @parameterized.parameters(list(itertools.product([False, True], repeat=3))) 151 | def test_text_to_program(self, has_imports: bool, has_class: bool, 152 | has_assignment: bool): 153 | code = create_test_program(has_imports, has_class, has_assignment) 154 | program = code_manipulation.text_to_program(code) 155 | self.assertLen(program.functions, 2) 156 | 157 | doc = textwrap.dedent( 158 | """\ 159 | Computes a cap set for n number of copies. 160 | 161 | A cap set is a subset of an n-dimensional affine space over a three-element 162 | field with no three elements in a line. 163 | 164 | Args: 165 | n: an integer, number of copies. 166 | 167 | Returns: 168 | A set of tuples in {0, 1, 2}. 169 | """ 170 | ) 171 | body = textwrap.dedent( 172 | """\ 173 | capset = set() 174 | for i in range(n): 175 | capset.add((0,) * i + (1,) + (0,) * (n - i - 1)) 176 | return capset""" 177 | ) 178 | expected_function_0 = code_manipulation.Function( 179 | name='get_capset_v0', 180 | args='n: int', 181 | return_type='set[tuple[int, ...]]', 182 | docstring=doc + ' ', 183 | body=textwrap.indent(body, ' '), 184 | ) 185 | expected_function_1 = code_manipulation.Function( 186 | name='get_capset_v2', 187 | args='k: int', 188 | body=''' get_capset_v0 = get_capset_v0(k) 189 | return get_capset_v0\\\n\n''', 190 | docstring='One line docstring.', 191 | ) 192 | if not has_imports and not has_class and not has_assignment: 193 | self.assertEmpty(program.preface) 194 | if has_imports: 195 | self.assertIn(_IMPORTS.rstrip(), program.preface) 196 | if has_class: 197 | self.assertIn(_CLASS.strip(), program.preface) 198 | if has_assignment: 199 | self.assertIn(_ASSIGNMENT.strip(), program.preface) 200 | self.assertEqual(expected_function_0, program.functions[0]) 201 | self.assertEqual(expected_function_1, program.functions[1]) 202 | self.assertEqual(code, str(program)) 203 | 204 | # Make sure that one can convert Function to string and then back to a 205 | # function so that it remains the same. 206 | for i in range(2): 207 | self.assertEqual( 208 | program.functions[i], 209 | code_manipulation.text_to_function(str(program.functions[i])) 210 | ) 211 | 212 | def test_get_functions_called(self): 213 | code = textwrap.dedent('''\ 214 | def f(n: int) -> int: 215 | if n == 1: 216 | return a(n) 217 | elif n == 2: 218 | return b(n) + object.c(n - 1) 219 | a = object.property 220 | g() 221 | return f(n - 1) 222 | ''') 223 | self.assertEqual(code_manipulation.get_functions_called(code), 224 | {'a', 'b', 'f', 'g'}) 225 | 226 | if __name__ == '__main__': 227 | absltest.main() 228 | -------------------------------------------------------------------------------- /tests/evaluator_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 DeepMind Technologies Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | import pathlib 16 | import textwrap 17 | 18 | from absl.testing import absltest 19 | from absl.testing import parameterized 20 | 21 | from funsearch import evaluator 22 | 23 | TESTS_FOLDER = pathlib.Path(__file__).parent 24 | 25 | 26 | class EvaluatorTest(parameterized.TestCase): 27 | 28 | def test_trim_function_body_docstring(self): 29 | code = '''\ 30 | x = 1 31 | 32 | return 0 33 | """Docstring"""''' 34 | desired = '''\ 35 | x = 1 36 | 37 | return 0 38 | 39 | ''' 40 | actual = evaluator._trim_function_body(code) 41 | self.assertEqual(desired, actual) 42 | 43 | def test_trim_function_body_function(self): 44 | code = '''\ 45 | return 0 46 | def new_f():''' 47 | desired = '''\ 48 | return 0 49 | 50 | ''' 51 | actual = evaluator._trim_function_body(code) 52 | self.assertEqual(desired, actual) 53 | 54 | def test_trim_function_body_empty(self): 55 | code = ''' return 0\n''' 56 | desired = ''' return 0\n\n''' 57 | actual = evaluator._trim_function_body(code) 58 | self.assertEqual(desired, actual) 59 | 60 | def test_trim_function_indentation_corner_case(self): 61 | code = textwrap.dedent( 62 | '''\ 63 | return (1 + 64 | 2) 65 | def unfinished_code(''' 66 | ) 67 | desired = textwrap.dedent( 68 | '''\ 69 | return (1 + 70 | 2) 71 | 72 | ''' 73 | ) 74 | actual = evaluator._trim_function_body(code) 75 | self.assertEqual(desired, actual) 76 | 77 | def test_trim_function_backlash_corner_case(self): 78 | code = textwrap.dedent( 79 | '''\ 80 | return score + ((el[0] + 1) * (el[0] + 2) * el[1] / 6 == el[2])\\ 81 | + ((el[0] + 1) * (el[0] + 2) * (el[0] + 3) * el[1] / 24 == el[2])\\ 82 | + ((el[0] + 1) * (el[0] + 2) * el[1] * el[2] / 6 == n)\\ 83 | + ((el[0] + 1) * (el[0] + 2) * el[1] * el[2] / 3 == n + el[0])\\ 84 | 85 | ''' 86 | ) 87 | actual = evaluator._trim_function_body(code) 88 | self.assertEqual(actual, code) 89 | 90 | 91 | def test_trim_function_other_llms(self): 92 | files = list((TESTS_FOLDER / "example_responses").glob("*.txt")) 93 | assert len(files) > 1, "Could not find all test files" 94 | for f in files: 95 | name = f.name 96 | response = f.read_text() 97 | actual = evaluator._trim_function_body(response) 98 | expected = (TESTS_FOLDER / "example_response_parsed" / name).read_text() 99 | self.assertEqual(expected, actual, f"{name}: Parsed code does not match") 100 | 101 | 102 | if __name__ == '__main__': 103 | absltest.main() 104 | -------------------------------------------------------------------------------- /tests/funsearch_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 DeepMind Technologies Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | from absl.testing import absltest 17 | from absl.testing import parameterized 18 | 19 | from funsearch import core 20 | 21 | _PY_PROMPT = '''\ 22 | import itertools 23 | import jax 24 | 25 | 26 | @funsearch.run 27 | @jax.jit 28 | def run(n: int): 29 | return capset(n) 30 | 31 | 32 | @funsearch.evolve 33 | def capset(n: int): 34 | """Trivial implementation of capset. 35 | 36 | Args: ... 37 | """ 38 | return [[1,] * n] 39 | ''' 40 | 41 | _PY_PROMPT_EVOLVE_RUN = '''\ 42 | import itertools 43 | 44 | 45 | @funsearch.run 46 | @funsearch.evolve 47 | def capset(n: int): 48 | return [[1,] * n] 49 | ''' 50 | 51 | _PY_PROMPT_NO_RUN = '''\ 52 | import itertools 53 | 54 | 55 | def run(n: int): 56 | return capset(n) 57 | 58 | @funsearch.evolve 59 | def capset(n: int): 60 | """Trivial implementation of capset. 61 | 62 | Args: ... 63 | """ 64 | return [[1,] * n] 65 | ''' 66 | 67 | _PY_PROMPT_NO_EVOLVE = '''\ 68 | import itertools 69 | 70 | 71 | @funsearch.run 72 | def run(n: int): 73 | return capset(n) 74 | 75 | 76 | def capset(n: int): 77 | """Trivial implementation of capset. 78 | 79 | Args: ... 80 | """ 81 | return [[1,] * n] 82 | ''' 83 | 84 | _PY_PROMPT_DOUBLE_RUN = '''\ 85 | import itertools 86 | 87 | @funsearch.run 88 | def run(n: int): 89 | return capset(n) 90 | 91 | @funsearch.run 92 | def capset(n: int): 93 | """Trivial implementation of capset. 94 | 95 | Args: ... 96 | """ 97 | return [[1,] * n] 98 | ''' 99 | 100 | 101 | class FunsearchTest(parameterized.TestCase): 102 | 103 | def test_extract_function_names(self): 104 | to_evolve, to_run = core._extract_function_names(_PY_PROMPT) 105 | self.assertEqual(to_run, 'run') 106 | self.assertEqual(to_evolve, 'capset') 107 | 108 | def test_extract_function_names_evolve_and_run(self): 109 | to_evolve, to_run = core._extract_function_names(_PY_PROMPT_EVOLVE_RUN) 110 | self.assertEqual(to_run, 'capset') 111 | self.assertEqual(to_evolve, 'capset') 112 | 113 | def test_extract_function_names_no_run(self): 114 | with self.assertRaisesRegex( 115 | ValueError, r'Expected 1 function decorated with `@funsearch.run`.'): 116 | core._extract_function_names(_PY_PROMPT_NO_RUN) 117 | 118 | def test_extract_function_names_no_evolve(self): 119 | with self.assertRaisesRegex( 120 | ValueError, r'Expected 1 function decorated with `@funsearch.evolve`.'): 121 | core._extract_function_names(_PY_PROMPT_NO_EVOLVE) 122 | 123 | def test_extract_function_names_double_run(self): 124 | with self.assertRaisesRegex( 125 | ValueError, r'Expected 1 function decorated with `@funsearch.run`.'): 126 | core._extract_function_names(_PY_PROMPT_DOUBLE_RUN) 127 | 128 | 129 | if __name__ == '__main__': 130 | absltest.main() 131 | -------------------------------------------------------------------------------- /tests/main_test.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import tempfile 3 | import unittest 4 | from pathlib import Path 5 | from unittest.mock import patch 6 | 7 | from click.testing import CliRunner 8 | 9 | from funsearch.__main__ import main, parse_input 10 | 11 | ROOT_DIR = Path(__file__).parent.parent 12 | 13 | runner = CliRunner() 14 | 15 | 16 | class TestMain(unittest.TestCase): 17 | def setUp(self): 18 | self.temp_dir = tempfile.mkdtemp() 19 | self.default_args = ["run", "--output_path", self.temp_dir, "--samplers", "1", "--iterations", "1", 20 | str(ROOT_DIR / "examples" / "cap_set_spec.py"), "5"] 21 | 22 | def tearDown(self): 23 | shutil.rmtree(self.temp_dir) 24 | 25 | def test_main(self): 26 | result = runner.invoke(main, []) 27 | assert result.exit_code == 0 28 | assert 'Usage:' in result.output 29 | with patch('funsearch.core.run', return_value=None) as mock_run: 30 | result = runner.invoke(main, self.default_args) 31 | assert result.exit_code == 0, result.output 32 | assert mock_run.call_count == 1 33 | 34 | def test_main_sample(self): 35 | with patch('funsearch.sampler.LLM._draw_sample', return_value="return 0.5") as mock_run: 36 | result = runner.invoke(main, self.default_args) 37 | assert result.exit_code == 0, result.output 38 | assert mock_run.call_count == 2, "There should be 2 sampler per sampler by default" 39 | 40 | 41 | def test_parse_input(): 42 | assert parse_input("1") == [1] 43 | assert parse_input("1,2,3") == [1, 2, 3] 44 | assert parse_input(str(ROOT_DIR / "examples" / "cap_set_input_data.json")) == [8, 9, 10] 45 | -------------------------------------------------------------------------------- /tests/programs_database_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 DeepMind Technologies Limited 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | 16 | import copy 17 | import tempfile 18 | 19 | from absl.testing import absltest 20 | from absl.testing import parameterized 21 | import numpy as np 22 | 23 | from funsearch import code_manipulation 24 | from funsearch import config 25 | from funsearch import programs_database 26 | 27 | _SKELETON = ''' 28 | """Finds large cap sets.""" 29 | import numpy as np 30 | import utils_capset 31 | 32 | 33 | def evaluate(n: int) -> int: 34 | """Returns the size of an `n`-dimensional cap set.""" 35 | capset = solve(n) 36 | return len(capset) 37 | 38 | 39 | def priority(element, n): 40 | """Returns the priority with which we want to add `element` to the cap set.""" 41 | return 0.0 42 | ''' 43 | _EXPECTED_INITIAL_PROMPT = ''' 44 | """Finds large cap sets.""" 45 | import numpy as np 46 | import utils_capset 47 | 48 | 49 | def priority_v0(element, n): 50 | """Returns the priority with which we want to add `element` to the cap set.""" 51 | return 0.0 52 | 53 | 54 | def priority_v1(element, n): 55 | """Improved version of `priority_v0`.""" 56 | 57 | ''' 58 | 59 | _SAMPLE_A = '''\ 60 | priority = element 61 | ####### 62 | # Code from lowest-scoring sampled program. 63 | ####### 64 | return ...\ 65 | ''' 66 | _SAMPLE_B = '''\ 67 | priority = element ** 2 68 | ####### 69 | # Code from highest-scoring sampled program. 70 | ####### 71 | return ...\ 72 | ''' 73 | 74 | # A prompt like this appears in the Extended Data of the paper. 75 | _EXPECTED_PROMPT = ''' 76 | """Finds large cap sets.""" 77 | import numpy as np 78 | import utils_capset 79 | 80 | 81 | def priority_v0(element, n): 82 | """Returns the priority with which we want to add `element` to the cap set.""" 83 | priority = element 84 | ####### 85 | # Code from lowest-scoring sampled program. 86 | ####### 87 | return ... 88 | 89 | 90 | def priority_v1(element, n): 91 | """Improved version of `priority_v0`.""" 92 | priority = element ** 2 93 | ####### 94 | # Code from highest-scoring sampled program. 95 | ####### 96 | return ... 97 | 98 | 99 | def priority_v2(element, n): 100 | """Improved version of `priority_v1`.""" 101 | 102 | ''' 103 | 104 | 105 | class ProgramsDatabaseTest(parameterized.TestCase): 106 | 107 | def test_initial_prompt(self): 108 | """Verifies that the first prompt looks as expected.""" 109 | 110 | # Create a programs database. 111 | template = code_manipulation.text_to_program(_SKELETON) 112 | function_to_evolve = 'priority' 113 | database = programs_database.ProgramsDatabase( 114 | config=config.ProgramsDatabaseConfig(functions_per_prompt=5), 115 | template=template, 116 | function_to_evolve=function_to_evolve, 117 | ) 118 | # Register the initial implementation provided in the skeleton template. 119 | database.register_program( 120 | program=template.get_function(function_to_evolve), 121 | island_id=None, 122 | scores_per_test={'unused': -1}) 123 | # Verify the first prompt. 124 | self.assertEqual(database.get_prompt().code, _EXPECTED_INITIAL_PROMPT) 125 | 126 | # Test saving database 127 | with tempfile.TemporaryFile() as f: 128 | database.save(f) 129 | 130 | f.seek(0) 131 | db2 = programs_database.ProgramsDatabase( 132 | config=config.ProgramsDatabaseConfig(functions_per_prompt=5), 133 | template=template, 134 | function_to_evolve=function_to_evolve, 135 | ) 136 | # Make sure the loaded database works as the original 137 | db2.load(f) 138 | self.assertEqual(db2.get_prompt().code, _EXPECTED_INITIAL_PROMPT) 139 | 140 | def test_generate_prompt(self): 141 | """Tests that we build the prompt shown in the paper.""" 142 | template = code_manipulation.text_to_program(_SKELETON) 143 | function_to_evolve = 'priority' 144 | island = programs_database.Island( 145 | template=template, 146 | function_to_evolve=function_to_evolve, 147 | functions_per_prompt=2, 148 | cluster_sampling_temperature_init=1.0, 149 | cluster_sampling_temperature_period=30_000, 150 | ) 151 | sample_a = copy.deepcopy(template.get_function(function_to_evolve)) 152 | sample_a.body = _SAMPLE_A 153 | sample_b = copy.deepcopy(template.get_function(function_to_evolve)) 154 | sample_b.body = _SAMPLE_B 155 | prompt = island._generate_prompt([sample_a, sample_b]) 156 | self.assertEqual(prompt, _EXPECTED_PROMPT) 157 | 158 | def test_destroy_islands(self): 159 | template = code_manipulation.text_to_program(_SKELETON) 160 | function_to_evolve = 'priority' 161 | database = programs_database.ProgramsDatabase( 162 | config=config.ProgramsDatabaseConfig(num_islands=10), 163 | template=template, 164 | function_to_evolve=function_to_evolve, 165 | ) 166 | scores = [7, 3, 5, 6, 7, -2, 0, -1, 4, 3] 167 | unused_function = template.get_function(function_to_evolve) 168 | for i, score in enumerate(scores): 169 | database.register_program( 170 | program=unused_function, 171 | island_id=i, 172 | scores_per_test={'unused_input': score}, 173 | ) 174 | database.register_program( 175 | unused_function, island_id=7, scores_per_test={'unused_input': 17}) 176 | 177 | expected_scores = list(scores) 178 | expected_scores[7] = 17 179 | self.assertSequenceEqual(database._best_score_per_island, expected_scores) 180 | 181 | progs = list(database.get_best_programs_per_island()) 182 | assert progs[0][1] == 17 183 | assert progs[-1][1] == -2 184 | 185 | np.random.seed(0) 186 | database.reset_islands() 187 | expected_kept = set([0, 2, 3, 4, 7]) 188 | min_kept = min(expected_scores[i] for i in expected_kept) 189 | for i, score in enumerate(expected_scores): 190 | if i in expected_kept: 191 | self.assertEqual(database._best_score_per_island[i], score) 192 | else: 193 | self.assertGreaterEqual(database._best_score_per_island[i], min_kept) 194 | 195 | @parameterized.parameters([ 196 | dict(logits=np.array([10, 9, -1000], dtype=np.float32)), 197 | dict(logits=np.array([10, 9, -1000], dtype=np.int32)), 198 | dict(logits=np.zeros(50)), 199 | ]) 200 | def test_softmax(self, logits: np.ndarray): 201 | probs_lower_temp = programs_database._softmax(logits, temperature=1.0) 202 | self.assertAlmostEqual(np.sum(probs_lower_temp), 1.0, places=6) 203 | self.assertTrue(np.all(probs_lower_temp >= 0)) 204 | self.assertTrue(np.all(probs_lower_temp <= 1)) 205 | 206 | probs_higher_temp = programs_database._softmax(logits, temperature=5.0) 207 | self.assertAlmostEqual(np.sum(probs_higher_temp), 1.0, places=6) 208 | self.assertTrue(np.all(probs_higher_temp >= 0)) 209 | self.assertTrue(np.all(probs_higher_temp <= 1)) 210 | 211 | if not np.all(logits == logits[0]): 212 | # The lower the temperature, the more confident we are on our top choice. 213 | self.assertGreater(np.max(probs_lower_temp), np.max(probs_higher_temp)) 214 | 215 | @parameterized.parameters([ 216 | dict(temperature=1, expected=[0.6590012, 0.242433, 0.0985659]), 217 | dict(temperature=2, expected=[0.50168777, 0.304289, 0.19402324]), 218 | ]) 219 | def test_softmax_output(self, temperature, expected): 220 | logits = np.array([2.0, 1.0, 0.1]) 221 | probabilities = programs_database._softmax(logits, temperature) 222 | np.testing.assert_array_almost_equal( 223 | probabilities, np.array(expected), decimal=5) 224 | 225 | @parameterized.parameters([ 226 | dict(logits=np.array([100, 200, 300, np.nan])), 227 | dict(logits=np.array([100, np.inf, 300, 200])), 228 | dict(logits=np.array([-np.inf, 200, 300, 100])), 229 | ]) 230 | def test_softmax_non_finite_error(self, logits): 231 | with self.assertRaisesRegex( 232 | ValueError, 233 | r'`logits` contains non-finite value\(s\)', 234 | ): 235 | programs_database._softmax(logits, temperature=1.0) 236 | 237 | 238 | if __name__ == '__main__': 239 | absltest.main() 240 | -------------------------------------------------------------------------------- /tests/sandbox_test.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import tempfile 3 | from funsearch.sandbox import ExternalProcessSandbox, ContainerSandbox 4 | 5 | test_prog = """ 6 | print("running!") 7 | def x(y): 8 | print(f"Received {y}") 9 | return y + 1 10 | """ 11 | 12 | 13 | def test_external_process_sandbox(): 14 | with tempfile.TemporaryDirectory() as d: 15 | sandbox = ExternalProcessSandbox(pathlib.Path(d)) 16 | ret, success = sandbox.run(test_prog, "x", 10, 1) 17 | assert success 18 | assert ret == 11 19 | 20 | 21 | def test_container_sandbox(): 22 | with tempfile.TemporaryDirectory() as d: 23 | sandbox = ContainerSandbox(pathlib.Path(d)) 24 | ret, success = sandbox.run(test_prog, "x", 10, 1) 25 | assert success 26 | assert ret == 11 27 | --------------------------------------------------------------------------------