├── .dockerignore
├── .gitignore
├── CONTRIBUTING.md
├── Dockerfile
├── LICENSE
├── README.md
├── examples
├── admissible_set
│ ├── admissible_set.ipynb
│ ├── admissible_set_n12_w7_size792.txt
│ ├── admissible_set_n15_w10_size3003.txt
│ ├── admissible_set_n21_w15_size43596.txt
│ ├── admissible_set_n24_w17_size237984.txt
│ ├── pre_admissible_set_for_n15_w10_size3003.txt
│ ├── pre_admissible_set_for_n21_w15_size43596.txt
│ └── pre_admissible_set_for_n24_w17_size237984.txt
├── bin_packing
│ └── bin_packing.ipynb
├── cap_set
│ ├── cap_set.ipynb
│ └── n8_size512.txt
├── cap_set_spec.py
├── corner_free_set
│ ├── f2_n4_size137.txt
│ ├── f3_n2_size53.txt
│ └── f3_n3_size370.txt
├── cyclic_graphs
│ ├── cyclic_graphs.ipynb
│ └── nodes11_n4_size754.txt
└── specification_nonsymmetric_admissible_set.py
├── funsearch
├── __init__.py
├── __main__.py
├── code_manipulation.py
├── config.py
├── container
│ ├── Dockerfile
│ └── container_main.py
├── core.py
├── evaluator.py
├── programs_database.py
├── sampler.py
└── sandbox.py
├── pdm.lock
├── pyproject.toml
└── tests
├── code_manipulation_test.py
├── evaluator_test.py
├── funsearch_test.py
├── main_test.py
├── programs_database_test.py
└── sandbox_test.py
/.dockerignore:
--------------------------------------------------------------------------------
1 | data
2 |
3 | # template from: https://github.com/themattrix/python-pypi-template/blob/master/.dockerignore
4 | # Git
5 | .git
6 | .gitignore
7 |
8 | # CI
9 | .codeclimate.yml
10 | .travis.yml
11 | .taskcluster.yml
12 |
13 | # Docker
14 | docker-compose.yml
15 | .docker
16 |
17 | # Byte-compiled / optimized / DLL files
18 | __pycache__/
19 | */__pycache__/
20 | */*/__pycache__/
21 | */*/*/__pycache__/
22 | *.py[cod]
23 | */*.py[cod]
24 | */*/*.py[cod]
25 | */*/*/*.py[cod]
26 |
27 | # C extensions
28 | *.so
29 |
30 | # Distribution / packaging
31 | .Python
32 | env/
33 | build/
34 | develop-eggs/
35 | dist/
36 | downloads/
37 | eggs/
38 | lib/
39 | lib64/
40 | parts/
41 | sdist/
42 | var/
43 | *.egg-info/
44 | .installed.cfg
45 | *.egg
46 |
47 | # PyInstaller
48 | # Usually these files are written by a python script from a template
49 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
50 | *.manifest
51 | *.spec
52 |
53 | # Installer logs
54 | pip-log.txt
55 | pip-delete-this-directory.txt
56 |
57 | # Unit test / coverage reports
58 | htmlcov/
59 | .tox/
60 | .coverage
61 | .cache
62 | nosetests.xml
63 | coverage.xml
64 |
65 | # Translations
66 | *.mo
67 | *.pot
68 |
69 | # Django stuff:
70 | *.log
71 |
72 | # Sphinx documentation
73 | docs/_build/
74 |
75 | # PyBuilder
76 | target/
77 |
78 | # Virtual environment
79 | .env/
80 | .venv/
81 | venv/
82 |
83 | # PyCharm
84 | .idea
85 |
86 | # Python mode for VIM
87 | .ropeproject
88 | */.ropeproject
89 | */*/.ropeproject
90 | */*/*/.ropeproject
91 |
92 | # Vim swap files
93 | *.swp
94 | */*.swp
95 | */*/*.swp
96 | */*/*/*.swp
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # Distribution / packaging
7 | .Python
8 | build/
9 | develop-eggs/
10 | dist/
11 | downloads/
12 | eggs/
13 | .eggs/
14 | lib/
15 | lib64/
16 | parts/
17 | sdist/
18 | var/
19 | wheels/
20 | share/python-wheels/
21 | *.egg-info/
22 | .installed.cfg
23 | *.egg
24 | MANIFEST
25 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # How to Contribute
2 |
3 | ## Contributor License Agreement
4 |
5 | Contributions to this project must be accompanied by a Contributor License
6 | Agreement. You (or your employer) retain the copyright to your contribution,
7 | this simply gives us permission to use and redistribute your contributions as
8 | part of the project. Head over to to see
9 | your current agreements on file or to sign a new one.
10 |
11 | You generally only need to submit a CLA once, so if you've already submitted one
12 | (even if it was for a different project), you probably don't need to do it
13 | again.
14 |
15 | ## Code reviews
16 |
17 | All submissions, including submissions by project members, require review. We
18 | use GitHub pull requests for this purpose. Consult
19 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
20 | information on using pull requests.
21 |
22 | ## Community Guidelines
23 |
24 | This project follows [Google's Open Source Community
25 | Guidelines](https://opensource.google/conduct/).
26 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM docker.io/python:3.11.6
2 |
3 | WORKDIR /workspace
4 |
5 | # Use PDM to keep track of exact versions of dependencies
6 | RUN pip install pdm
7 | COPY pyproject.toml README.md pdm.lock ./
8 | # install dependencies first. PDM also creates a /workspace/.venv here.
9 | ENV PATH="/workspace/.venv/bin:$PATH"
10 | RUN pdm install --no-self
11 | COPY examples ./examples
12 | COPY funsearch ./funsearch
13 |
14 | RUN pip install --no-deps . && rm -r ./funsearch ./build
15 |
16 | CMD /bin/bash
17 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction,
11 | and distribution as defined by Sections 1 through 9 of this document.
12 |
13 | "Licensor" shall mean the copyright owner or entity authorized by
14 | the copyright owner that is granting the License.
15 |
16 | "Legal Entity" shall mean the union of the acting entity and all
17 | other entities that control, are controlled by, or are under common
18 | control with that entity. For the purposes of this definition,
19 | "control" means (i) the power, direct or indirect, to cause the
20 | direction or management of such entity, whether by contract or
21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 | outstanding shares, or (iii) beneficial ownership of such entity.
23 |
24 | "You" (or "Your") shall mean an individual or Legal Entity
25 | exercising permissions granted by this License.
26 |
27 | "Source" form shall mean the preferred form for making modifications,
28 | including but not limited to software source code, documentation
29 | source, and configuration files.
30 |
31 | "Object" form shall mean any form resulting from mechanical
32 | transformation or translation of a Source form, including but
33 | not limited to compiled object code, generated documentation,
34 | and conversions to other media types.
35 |
36 | "Work" shall mean the work of authorship, whether in Source or
37 | Object form, made available under the License, as indicated by a
38 | copyright notice that is included in or attached to the work
39 | (an example is provided in the Appendix below).
40 |
41 | "Derivative Works" shall mean any work, whether in Source or Object
42 | form, that is based on (or derived from) the Work and for which the
43 | editorial revisions, annotations, elaborations, or other modifications
44 | represent, as a whole, an original work of authorship. For the purposes
45 | of this License, Derivative Works shall not include works that remain
46 | separable from, or merely link (or bind by name) to the interfaces of,
47 | the Work and Derivative Works thereof.
48 |
49 | "Contribution" shall mean any work of authorship, including
50 | the original version of the Work and any modifications or additions
51 | to that Work or Derivative Works thereof, that is intentionally
52 | submitted to Licensor for inclusion in the Work by the copyright owner
53 | or by an individual or Legal Entity authorized to submit on behalf of
54 | the copyright owner. For the purposes of this definition, "submitted"
55 | means any form of electronic, verbal, or written communication sent
56 | to the Licensor or its representatives, including but not limited to
57 | communication on electronic mailing lists, source code control systems,
58 | and issue tracking systems that are managed by, or on behalf of, the
59 | Licensor for the purpose of discussing and improving the Work, but
60 | excluding communication that is conspicuously marked or otherwise
61 | designated in writing by the copyright owner as "Not a Contribution."
62 |
63 | "Contributor" shall mean Licensor and any individual or Legal Entity
64 | on behalf of whom a Contribution has been received by Licensor and
65 | subsequently incorporated within the Work.
66 |
67 | 2. Grant of Copyright License. Subject to the terms and conditions of
68 | this License, each Contributor hereby grants to You a perpetual,
69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 | copyright license to reproduce, prepare Derivative Works of,
71 | publicly display, publicly perform, sublicense, and distribute the
72 | Work and such Derivative Works in Source or Object form.
73 |
74 | 3. Grant of Patent License. Subject to the terms and conditions of
75 | this License, each Contributor hereby grants to You a perpetual,
76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 | (except as stated in this section) patent license to make, have made,
78 | use, offer to sell, sell, import, and otherwise transfer the Work,
79 | where such license applies only to those patent claims licensable
80 | by such Contributor that are necessarily infringed by their
81 | Contribution(s) alone or by combination of their Contribution(s)
82 | with the Work to which such Contribution(s) was submitted. If You
83 | institute patent litigation against any entity (including a
84 | cross-claim or counterclaim in a lawsuit) alleging that the Work
85 | or a Contribution incorporated within the Work constitutes direct
86 | or contributory patent infringement, then any patent licenses
87 | granted to You under this License for that Work shall terminate
88 | as of the date such litigation is filed.
89 |
90 | 4. Redistribution. You may reproduce and distribute copies of the
91 | Work or Derivative Works thereof in any medium, with or without
92 | modifications, and in Source or Object form, provided that You
93 | meet the following conditions:
94 |
95 | (a) You must give any other recipients of the Work or
96 | Derivative Works a copy of this License; and
97 |
98 | (b) You must cause any modified files to carry prominent notices
99 | stating that You changed the files; and
100 |
101 | (c) You must retain, in the Source form of any Derivative Works
102 | that You distribute, all copyright, patent, trademark, and
103 | attribution notices from the Source form of the Work,
104 | excluding those notices that do not pertain to any part of
105 | the Derivative Works; and
106 |
107 | (d) If the Work includes a "NOTICE" text file as part of its
108 | distribution, then any Derivative Works that You distribute must
109 | include a readable copy of the attribution notices contained
110 | within such NOTICE file, excluding those notices that do not
111 | pertain to any part of the Derivative Works, in at least one
112 | of the following places: within a NOTICE text file distributed
113 | as part of the Derivative Works; within the Source form or
114 | documentation, if provided along with the Derivative Works; or,
115 | within a display generated by the Derivative Works, if and
116 | wherever such third-party notices normally appear. The contents
117 | of the NOTICE file are for informational purposes only and
118 | do not modify the License. You may add Your own attribution
119 | notices within Derivative Works that You distribute, alongside
120 | or as an addendum to the NOTICE text from the Work, provided
121 | that such additional attribution notices cannot be construed
122 | as modifying the License.
123 |
124 | You may add Your own copyright statement to Your modifications and
125 | may provide additional or different license terms and conditions
126 | for use, reproduction, or distribution of Your modifications, or
127 | for any such Derivative Works as a whole, provided Your use,
128 | reproduction, and distribution of the Work otherwise complies with
129 | the conditions stated in this License.
130 |
131 | 5. Submission of Contributions. Unless You explicitly state otherwise,
132 | any Contribution intentionally submitted for inclusion in the Work
133 | by You to the Licensor shall be under the terms and conditions of
134 | this License, without any additional terms or conditions.
135 | Notwithstanding the above, nothing herein shall supersede or modify
136 | the terms of any separate license agreement you may have executed
137 | with Licensor regarding such Contributions.
138 |
139 | 6. Trademarks. This License does not grant permission to use the trade
140 | names, trademarks, service marks, or product names of the Licensor,
141 | except as required for reasonable and customary use in describing the
142 | origin of the Work and reproducing the content of the NOTICE file.
143 |
144 | 7. Disclaimer of Warranty. Unless required by applicable law or
145 | agreed to in writing, Licensor provides the Work (and each
146 | Contributor provides its Contributions) on an "AS IS" BASIS,
147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 | implied, including, without limitation, any warranties or conditions
149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 | PARTICULAR PURPOSE. You are solely responsible for determining the
151 | appropriateness of using or redistributing the Work and assume any
152 | risks associated with Your exercise of permissions under this License.
153 |
154 | 8. Limitation of Liability. In no event and under no legal theory,
155 | whether in tort (including negligence), contract, or otherwise,
156 | unless required by applicable law (such as deliberate and grossly
157 | negligent acts) or agreed to in writing, shall any Contributor be
158 | liable to You for damages, including any direct, indirect, special,
159 | incidental, or consequential damages of any character arising as a
160 | result of this License or out of the use or inability to use the
161 | Work (including but not limited to damages for loss of goodwill,
162 | work stoppage, computer failure or malfunction, or any and all
163 | other commercial damages or losses), even if such Contributor
164 | has been advised of the possibility of such damages.
165 |
166 | 9. Accepting Warranty or Additional Liability. While redistributing
167 | the Work or Derivative Works thereof, You may choose to offer,
168 | and charge a fee for, acceptance of support, warranty, indemnity,
169 | or other liability obligations and/or rights consistent with this
170 | License. However, in accepting such obligations, You may act only
171 | on Your own behalf and on Your sole responsibility, not on behalf
172 | of any other Contributor, and only if You agree to indemnify,
173 | defend, and hold each Contributor harmless for any liability
174 | incurred by, or claims asserted against, such Contributor by reason
175 | of your accepting any such warranty or additional liability.
176 |
177 | END OF TERMS AND CONDITIONS
178 |
179 | APPENDIX: How to apply the Apache License to your work.
180 |
181 | To apply the Apache License to your work, attach the following
182 | boilerplate notice, with the fields enclosed by brackets "[]"
183 | replaced with your own identifying information. (Don't include
184 | the brackets!) The text should be enclosed in the appropriate
185 | comment syntax for the file format. We also recommend that a
186 | file or class name and description of purpose be included on the
187 | same "printed page" as the copyright notice for easier
188 | identification within third-party archives.
189 |
190 | Copyright [yyyy] [name of copyright owner]
191 |
192 | Licensed under the Apache License, Version 2.0 (the "License");
193 | you may not use this file except in compliance with the License.
194 | You may obtain a copy of the License at
195 |
196 | http://www.apache.org/licenses/LICENSE-2.0
197 |
198 | Unless required by applicable law or agreed to in writing, software
199 | distributed under the License is distributed on an "AS IS" BASIS,
200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 | See the License for the specific language governing permissions and
202 | limitations under the License.
203 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # FunSearch
2 |
3 |
4 | Usage:
5 |
6 |
7 | You can run FunSearch in container using Podman or Docker
8 |
9 | ```
10 | podman build . -t funsearch
11 |
12 |
13 | # Create a folder to share with the container
14 | mkdir data
15 | podman run -it -v ./data:/workspace/data funsearch
16 |
17 | # Set the environment variable OPENAI_API_KEY=sk-xxxx or create .env file.
18 | # "gpt-3.5-turbo-instruct" model is used by default.
19 | # Refer to 'llm' package docs to use other models.
20 |
21 | funsearch run examples/cap_set_spec.py 11 --sandbox_type ExternalProcessSandbox
22 | ```
23 | In here we are searching for the algorithm to find maximum cap sets for dimension 11.
24 | You should see output something like
25 | ```
26 | root@11c22cd7aeac:/workspace# funsearch run examples/cap_set_spec.py 11 --sandbox_type ExternalProcessSandbox
27 | INFO:root:Writing logs to data/1704956206
28 | INFO:absl:Best score of island 0 increased to 2048
29 | INFO:absl:Best score of island 1 increased to 2048
30 | INFO:absl:Best score of island 2 increased to 2048
31 | INFO:absl:Best score of island 3 increased to 2048
32 | INFO:absl:Best score of island 4 increased to 2048
33 | INFO:absl:Best score of island 5 increased to 2048
34 | INFO:absl:Best score of island 6 increased to 2048
35 | INFO:absl:Best score of island 7 increased to 2048
36 | INFO:absl:Best score of island 8 increased to 2048
37 | INFO:absl:Best score of island 9 increased to 2048
38 | INFO:absl:Best score of island 5 increased to 2053
39 | INFO:absl:Best score of island 1 increased to 2049
40 | INFO:absl:Best score of island 8 increased to 2684
41 | ^C^CINFO:root:Keyboard interrupt. Stopping.
42 | INFO:absl:Saving backup to data/backups/program_db_priority_1704956206_0.pickle.
43 | ```
44 |
45 | Note that in the last command we use the ExternalProcessSandbox that is not fully safe
46 | but makes it a bit less likely that invalid code from LLM would break the search.
47 |
48 |
49 | Alternatively, you can run the main Python process on a host computer outside of any container and let
50 | the process build and run separate sandbox containers (still requires Podman/Docker).
51 | This variant could be also used, e.g., in Colab quite safely since the environment is some kind of container itself.
52 |
53 | ```
54 | pip install .
55 |
56 | funsearch run examples/cap_set_spec.py 11
57 | ```
58 |
59 | For more complex input data, you can provide the input also as a .json or .pickle file.
60 |
61 | Currently, the search is only using single thread with no asyncio and is somewhat slow
62 | for challenging tasks.
63 |
64 | ## Alternative LLMs
65 |
66 | The search uses gpt-3.5-turbo-instruct by default, but other models can be used with the --model_name argument
67 | and possibly installing extensions to the llm package.
68 | As an example of performance, with gpt-3.5-turbo-instruct on dimension 8 it usually around 20 tries to find a few
69 | improvements to the naive algorithm.
70 |
71 | On the other hand, using orca-mini-3b-gguf2-q4_0 doesn't seem to work quite well.
72 | The latest version has a bit improved parsing to find the last priority_vX method from the LLM response
73 | even if it contains other content like Markdown formatting. Anyway, the model seems to often
74 | use strange indentation of 1 space and thus might require some customization to be useful at all.
75 | Lastly, even with correct Python syntax, orca-mini-3b does not seem to find improvements (in 60 runs) and mostly
76 | generates code that throws "IndexError: tuple index out of range". The situation changes a bit
77 | if the search is started using a database generated by gpt-3.5-turbo prompts.
78 |
79 | Overall, all models would probably require some prompt engineering, temperatures tuning, and such for the tool
80 | to be useful at all except for very simple problems.
81 | Also, the implementation is currently lacking good tools to analyze large amount of responses properly which
82 | makes any prompt engineering more difficult.
83 |
84 | ---
85 |
86 | The repository contains a number of sample problems that can be solved with FunSearch.
87 | Currently, only the cap set problem (examples/cap_set_spec.py) has been written in the form that can be directly
88 | used with the 'funsearch' executable.
89 |
90 | This repository accompanies the publication
91 |
92 | > Romera-Paredes, B. et al. [Mathematical discoveries from program search with large language models](https://www.nature.com/articles/s41586-023-06924-6). *Nature* (2023)
93 |
94 | There are 6 independent directories:
95 |
96 | - `cap_set` contains functions discovered by FunSearch that construct large cap
97 | sets, and we also provide those cap sets in a numerical format for convenience.
98 |
99 | - `admissible_set` contains functions discovered by FunSearch that construct
100 | large admissible sets, and we also provide those admissible sets in a numerical
101 | format for convenience.
102 |
103 | - `bin_packing` contains heuristics discovered by FunSearch for online 1D bin
104 | packing problems, and an evaluation suite to reproduce the results reported in
105 | the paper.
106 |
107 | - `cyclic_graphs` contains functions discovered by FunSearch that construct
108 | large independent sets in strong products of cyclic graphs, and we also provide
109 | those sets in a numerical format for convenience.
110 |
111 | - `corner_free_sets` contains the discovered sets of indices, in numerical
112 | format, satisfying the combinatorial degeneration constraints described for the
113 | corners-free problem in the Supplementary Information.
114 |
115 | - `implementation` contains an implementation of the evolutionary algorithm,
116 | code manipulation routines, and a single-threaded implementation of the
117 | FunSearch pipeline. It does not contain language models for generating new
118 | programs, the sandbox for executing untrusted code, nor the infrastructure for
119 | running FunSearch on our distributed system. This directory is intended to be
120 | useful for understanding the details of our method, and for adapting it for use
121 | with any available language models, sandboxes, and distributed systems.
122 |
123 | ## Installation
124 |
125 | No installation is required. All notebooks can be opened and run in Google
126 | Colab.
127 |
128 | ## Usage
129 |
130 | - `cap_set`: The notebook `cap_set.ipynb` can be opened via
131 | [](https://colab.research.google.com/github/google-deepmind/funsearch/blob/master/cap_set/cap_set.ipynb).
132 |
133 | - `admissible_set`: The notebook `admissible_set.ipynb` can be opened
134 | via
135 | [](https://colab.research.google.com/github/google-deepmind/funsearch/blob/master/admissible_set/admissible_set.ipynb).
136 |
137 | - `bin_packing`: The notebook `bin_packing.ipynb` can be opened via
138 | [](https://colab.research.google.com/github/google-deepmind/funsearch/blob/master/bin_packing/bin_packing.ipynb).
139 |
140 | - `cyclic_graphs`: The notebook `cyclic_graphs.ipynb` can be opened via
141 | [](https://colab.research.google.com/github/google-deepmind/funsearch/blob/master/cyclic_graphs/cyclic_graphs.ipynb).
142 |
143 | ## Citing this work
144 |
145 | If you use the code or data in this package, please cite:
146 |
147 | ```bibtex
148 | @Article{FunSearch2023,
149 | author = {Romera-Paredes, Bernardino and Barekatain, Mohammadamin and Novikov, Alexander and Balog, Matej and Kumar, M. Pawan and Dupont, Emilien and Ruiz, Francisco J. R. and Ellenberg, Jordan and Wang, Pengming and Fawzi, Omar and Kohli, Pushmeet and Fawzi, Alhussein},
150 | journal = {Nature},
151 | title = {Mathematical discoveries from program search with large language models},
152 | year = {2023},
153 | doi = {10.1038/s41586-023-06924-6}
154 | }
155 | ```
156 |
157 | ## License and disclaimer
158 |
159 | Copyright 2023 DeepMind Technologies Limited
160 |
161 | All software is licensed under the Apache License, Version 2.0 (Apache 2.0);
162 | you may not use this file except in compliance with the Apache 2.0 license.
163 | You may obtain a copy of the Apache 2.0 license at:
164 | https://www.apache.org/licenses/LICENSE-2.0
165 |
166 | All other materials are licensed under the Creative Commons Attribution 4.0
167 | International License (CC-BY). You may obtain a copy of the CC-BY license at:
168 | https://creativecommons.org/licenses/by/4.0/legalcode
169 |
170 | Unless required by applicable law or agreed to in writing, all software and
171 | materials distributed here under the Apache 2.0 or CC-BY licenses are
172 | distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
173 | either express or implied. See the licenses for the specific language governing
174 | permissions and limitations under those licenses.
175 |
176 | This is not an official Google product.
177 |
--------------------------------------------------------------------------------
/examples/admissible_set/pre_admissible_set_for_n15_w10_size3003.txt:
--------------------------------------------------------------------------------
1 | [4, 4, 4, 4, 4]
2 | [2, 5, 4, 4, 4]
3 | [4, 2, 5, 4, 4]
4 | [4, 4, 2, 5, 4]
5 | [4, 4, 4, 2, 5]
6 | [5, 4, 4, 4, 2]
7 | [2, 3, 5, 4, 4]
8 | [3, 5, 4, 4, 2]
9 | [4, 2, 3, 5, 4]
10 | [4, 4, 2, 3, 5]
11 | [5, 4, 4, 2, 3]
12 | [1, 4, 4, 6, 4]
13 | [4, 1, 4, 4, 6]
14 | [4, 4, 6, 4, 1]
15 | [4, 6, 4, 1, 4]
16 | [6, 4, 1, 4, 4]
17 | [2, 3, 4, 4, 6]
18 | [3, 4, 4, 6, 2]
19 | [4, 4, 6, 2, 3]
20 | [4, 6, 2, 3, 4]
21 | [6, 2, 3, 4, 4]
22 | [0, 5, 4, 6, 4]
23 | [0, 5, 6, 4, 4]
24 | [4, 0, 5, 4, 6]
25 | [4, 0, 5, 6, 4]
26 | [4, 4, 0, 5, 6]
27 | [4, 6, 4, 0, 5]
28 | [5, 4, 6, 4, 0]
29 | [5, 6, 4, 4, 0]
30 | [6, 4, 0, 5, 4]
31 | [6, 4, 4, 0, 5]
32 | [0, 5, 3, 3, 5]
33 | [3, 3, 5, 0, 5]
34 | [3, 5, 0, 5, 3]
35 | [5, 0, 5, 3, 3]
36 | [5, 3, 3, 5, 0]
37 | [2, 5, 2, 5, 4]
38 | [2, 5, 4, 2, 5]
39 | [4, 2, 5, 2, 5]
40 | [5, 2, 5, 4, 2]
41 | [5, 4, 2, 5, 2]
42 | [3, 5, 2, 5, 2]
43 | [5, 2, 3, 5, 2]
44 | [5, 2, 5, 2, 3]
45 | [2, 3, 5, 2, 5]
46 | [2, 5, 2, 3, 5]
47 | [1, 2, 5, 3, 5]
48 | [2, 5, 3, 5, 1]
49 | [3, 5, 1, 2, 5]
50 | [5, 1, 2, 5, 3]
51 | [5, 3, 5, 1, 2]
52 | [0, 3, 5, 6, 4]
53 | [3, 5, 6, 4, 0]
54 | [4, 0, 3, 5, 6]
55 | [5, 6, 4, 0, 3]
56 | [6, 4, 0, 3, 5]
57 | [0, 3, 5, 3, 5]
58 | [3, 5, 0, 3, 5]
59 | [3, 5, 3, 5, 0]
60 | [5, 0, 3, 5, 3]
61 | [5, 3, 5, 0, 3]
62 | [4, 1, 1, 5, 6]
63 | [1, 1, 5, 6, 4]
64 | [1, 5, 6, 4, 1]
65 | [5, 6, 4, 1, 1]
66 | [6, 4, 1, 1, 5]
67 | [1, 1, 4, 6, 6]
68 | [1, 4, 6, 6, 1]
69 | [4, 6, 6, 1, 1]
70 | [6, 1, 1, 4, 6]
71 | [6, 6, 1, 1, 4]
72 | [1, 5, 6, 2, 3]
73 | [2, 3, 1, 5, 6]
74 | [3, 1, 5, 6, 2]
75 | [5, 6, 2, 3, 1]
76 | [6, 2, 3, 1, 5]
77 | [0, 3, 4, 6, 6]
78 | [3, 4, 6, 6, 0]
79 | [4, 6, 6, 0, 3]
80 | [6, 0, 3, 4, 6]
81 | [6, 6, 0, 3, 4]
82 | [0, 5, 6, 2, 5]
83 | [2, 5, 0, 5, 6]
84 | [5, 0, 5, 6, 2]
85 | [5, 6, 2, 5, 0]
86 | [6, 2, 5, 0, 5]
87 | [0, 5, 6, 5, 1]
88 | [1, 0, 5, 6, 5]
89 | [5, 1, 0, 5, 6]
90 | [5, 6, 5, 1, 0]
91 | [6, 5, 1, 0, 5]
92 | [0, 6, 2, 6, 5]
93 | [2, 6, 5, 0, 6]
94 | [5, 0, 6, 2, 6]
95 | [6, 2, 6, 5, 0]
96 | [6, 5, 0, 6, 2]
97 | [0, 1, 6, 6, 6]
98 | [1, 6, 6, 6, 0]
99 | [6, 0, 1, 6, 6]
100 | [6, 6, 0, 1, 6]
101 | [6, 6, 6, 0, 1]
102 |
--------------------------------------------------------------------------------
/examples/admissible_set/pre_admissible_set_for_n21_w15_size43596.txt:
--------------------------------------------------------------------------------
1 | [4, 4, 4, 4, 4, 4, 6]
2 | [4, 4, 4, 4, 4, 6, 4]
3 | [4, 4, 4, 4, 6, 4, 4]
4 | [4, 4, 4, 6, 4, 4, 4]
5 | [4, 4, 6, 4, 4, 4, 4]
6 | [4, 6, 4, 4, 4, 4, 4]
7 | [6, 4, 4, 4, 4, 4, 4]
8 | [2, 6, 5, 4, 4, 4, 4]
9 | [4, 2, 6, 5, 4, 4, 4]
10 | [4, 4, 2, 6, 5, 4, 4]
11 | [4, 4, 4, 2, 6, 5, 4]
12 | [4, 4, 4, 4, 2, 6, 5]
13 | [5, 4, 4, 4, 4, 2, 6]
14 | [6, 5, 4, 4, 4, 4, 2]
15 | [2, 5, 4, 4, 6, 4, 4]
16 | [4, 2, 5, 4, 4, 6, 4]
17 | [4, 4, 2, 5, 4, 4, 6]
18 | [4, 4, 6, 4, 4, 2, 5]
19 | [4, 6, 4, 4, 2, 5, 4]
20 | [5, 4, 4, 6, 4, 4, 2]
21 | [6, 4, 4, 2, 5, 4, 4]
22 | [2, 5, 4, 4, 4, 6, 4]
23 | [2, 5, 4, 6, 4, 4, 4]
24 | [4, 2, 5, 4, 4, 4, 6]
25 | [4, 2, 5, 4, 6, 4, 4]
26 | [4, 4, 2, 5, 4, 6, 4]
27 | [4, 4, 4, 2, 5, 4, 6]
28 | [4, 4, 4, 6, 4, 2, 5]
29 | [4, 4, 6, 4, 2, 5, 4]
30 | [4, 6, 4, 2, 5, 4, 4]
31 | [4, 6, 4, 4, 4, 2, 5]
32 | [5, 4, 4, 4, 6, 4, 2]
33 | [5, 4, 6, 4, 4, 4, 2]
34 | [6, 4, 2, 5, 4, 4, 4]
35 | [6, 4, 4, 4, 2, 5, 4]
36 | [2, 5, 4, 4, 4, 4, 6]
37 | [4, 4, 4, 4, 6, 2, 5]
38 | [4, 4, 4, 6, 2, 5, 4]
39 | [4, 4, 6, 2, 5, 4, 4]
40 | [4, 6, 2, 5, 4, 4, 4]
41 | [5, 4, 4, 4, 4, 6, 2]
42 | [6, 2, 5, 4, 4, 4, 4]
43 | [2, 4, 5, 6, 4, 4, 4]
44 | [4, 2, 4, 5, 6, 4, 4]
45 | [4, 4, 2, 4, 5, 6, 4]
46 | [4, 4, 4, 2, 4, 5, 6]
47 | [4, 5, 6, 4, 4, 4, 2]
48 | [5, 6, 4, 4, 4, 2, 4]
49 | [6, 4, 4, 4, 2, 4, 5]
50 | [2, 5, 4, 4, 2, 6, 5]
51 | [2, 6, 5, 2, 5, 4, 4]
52 | [4, 2, 6, 5, 2, 5, 4]
53 | [4, 4, 2, 6, 5, 2, 5]
54 | [5, 2, 5, 4, 4, 2, 6]
55 | [5, 4, 4, 2, 6, 5, 2]
56 | [6, 5, 2, 5, 4, 4, 2]
57 | [2, 5, 2, 6, 5, 4, 4]
58 | [2, 6, 5, 4, 4, 2, 5]
59 | [4, 2, 5, 2, 6, 5, 4]
60 | [4, 4, 2, 5, 2, 6, 5]
61 | [5, 2, 6, 5, 4, 4, 2]
62 | [5, 4, 4, 2, 5, 2, 6]
63 | [6, 5, 4, 4, 2, 5, 2]
64 | [2, 3, 5, 4, 4, 6, 4]
65 | [2, 3, 5, 4, 6, 4, 4]
66 | [3, 5, 4, 4, 6, 4, 2]
67 | [3, 5, 4, 6, 4, 4, 2]
68 | [4, 2, 3, 5, 4, 4, 6]
69 | [4, 2, 3, 5, 4, 6, 4]
70 | [4, 4, 2, 3, 5, 4, 6]
71 | [4, 4, 6, 4, 2, 3, 5]
72 | [4, 6, 4, 2, 3, 5, 4]
73 | [4, 6, 4, 4, 2, 3, 5]
74 | [5, 4, 4, 6, 4, 2, 3]
75 | [5, 4, 6, 4, 4, 2, 3]
76 | [6, 4, 2, 3, 5, 4, 4]
77 | [6, 4, 4, 2, 3, 5, 4]
78 | [2, 5, 4, 2, 6, 5, 4]
79 | [2, 6, 5, 4, 2, 5, 4]
80 | [4, 2, 5, 4, 2, 6, 5]
81 | [4, 2, 6, 5, 4, 2, 5]
82 | [5, 4, 2, 5, 4, 2, 6]
83 | [5, 4, 2, 6, 5, 4, 2]
84 | [6, 5, 4, 2, 5, 4, 2]
85 | [2, 5, 2, 5, 4, 6, 4]
86 | [2, 5, 4, 6, 4, 2, 5]
87 | [4, 2, 5, 2, 5, 4, 6]
88 | [4, 6, 4, 2, 5, 2, 5]
89 | [5, 2, 5, 4, 6, 4, 2]
90 | [5, 4, 6, 4, 2, 5, 2]
91 | [6, 4, 2, 5, 2, 5, 4]
92 | [2, 3, 5, 4, 4, 4, 6]
93 | [3, 5, 4, 4, 4, 6, 2]
94 | [4, 4, 4, 6, 2, 3, 5]
95 | [4, 4, 6, 2, 3, 5, 4]
96 | [4, 6, 2, 3, 5, 4, 4]
97 | [5, 4, 4, 4, 6, 2, 3]
98 | [6, 2, 3, 5, 4, 4, 4]
99 | [2, 3, 3, 5, 4, 6, 4]
100 | [3, 3, 5, 4, 6, 4, 2]
101 | [3, 5, 4, 6, 4, 2, 3]
102 | [4, 2, 3, 3, 5, 4, 6]
103 | [4, 6, 4, 2, 3, 3, 5]
104 | [5, 4, 6, 4, 2, 3, 3]
105 | [6, 4, 2, 3, 3, 5, 4]
106 | [2, 5, 2, 5, 4, 4, 6]
107 | [2, 5, 4, 4, 6, 2, 5]
108 | [4, 4, 6, 2, 5, 2, 5]
109 | [4, 6, 2, 5, 2, 5, 4]
110 | [5, 2, 5, 4, 4, 6, 2]
111 | [5, 4, 4, 6, 2, 5, 2]
112 | [6, 2, 5, 2, 5, 4, 4]
113 | [2, 2, 5, 6, 4, 6, 4]
114 | [2, 5, 6, 4, 6, 4, 2]
115 | [4, 2, 2, 5, 6, 4, 6]
116 | [4, 6, 4, 2, 2, 5, 6]
117 | [5, 6, 4, 6, 4, 2, 2]
118 | [6, 4, 2, 2, 5, 6, 4]
119 | [6, 4, 6, 4, 2, 2, 5]
120 | [2, 5, 4, 2, 5, 4, 6]
121 | [2, 5, 4, 6, 2, 5, 4]
122 | [4, 2, 5, 4, 6, 2, 5]
123 | [4, 6, 2, 5, 4, 2, 5]
124 | [5, 4, 2, 5, 4, 6, 2]
125 | [5, 4, 6, 2, 5, 4, 2]
126 | [6, 2, 5, 4, 2, 5, 4]
127 | [2, 3, 3, 5, 4, 4, 6]
128 | [3, 3, 5, 4, 4, 6, 2]
129 | [3, 5, 4, 4, 6, 2, 3]
130 | [4, 4, 6, 2, 3, 3, 5]
131 | [4, 6, 2, 3, 3, 5, 4]
132 | [5, 4, 4, 6, 2, 3, 3]
133 | [6, 2, 3, 3, 5, 4, 4]
134 | [2, 3, 3, 5, 6, 4, 4]
135 | [3, 3, 5, 6, 4, 4, 2]
136 | [3, 5, 6, 4, 4, 2, 3]
137 | [4, 2, 3, 3, 5, 6, 4]
138 | [4, 4, 2, 3, 3, 5, 6]
139 | [5, 6, 4, 4, 2, 3, 3]
140 | [6, 4, 4, 2, 3, 3, 5]
141 | [2, 2, 5, 6, 4, 4, 6]
142 | [2, 5, 6, 4, 4, 6, 2]
143 | [4, 4, 6, 2, 2, 5, 6]
144 | [4, 6, 2, 2, 5, 6, 4]
145 | [5, 6, 4, 4, 6, 2, 2]
146 | [6, 2, 2, 5, 6, 4, 4]
147 | [6, 4, 4, 6, 2, 2, 5]
148 | [2, 3, 5, 4, 2, 6, 5]
149 | [2, 6, 5, 2, 3, 5, 4]
150 | [3, 5, 4, 2, 6, 5, 2]
151 | [4, 2, 6, 5, 2, 3, 5]
152 | [5, 2, 3, 5, 4, 2, 6]
153 | [5, 4, 2, 6, 5, 2, 3]
154 | [6, 5, 2, 3, 5, 4, 2]
155 | [2, 3, 5, 2, 6, 5, 4]
156 | [2, 6, 5, 4, 2, 3, 5]
157 | [3, 5, 2, 6, 5, 4, 2]
158 | [4, 2, 3, 5, 2, 6, 5]
159 | [5, 2, 6, 5, 4, 2, 3]
160 | [5, 4, 2, 3, 5, 2, 6]
161 | [6, 5, 4, 2, 3, 5, 2]
162 | [2, 3, 3, 3, 5, 4, 6]
163 | [3, 3, 3, 5, 4, 6, 2]
164 | [3, 3, 5, 4, 6, 2, 3]
165 | [3, 5, 4, 6, 2, 3, 3]
166 | [4, 6, 2, 3, 3, 3, 5]
167 | [5, 4, 6, 2, 3, 3, 3]
168 | [6, 2, 3, 3, 3, 5, 4]
169 | [2, 3, 3, 3, 5, 6, 4]
170 | [3, 3, 3, 5, 6, 4, 2]
171 | [3, 3, 5, 6, 4, 2, 3]
172 | [3, 5, 6, 4, 2, 3, 3]
173 | [4, 2, 3, 3, 3, 5, 6]
174 | [5, 6, 4, 2, 3, 3, 3]
175 | [6, 4, 2, 3, 3, 3, 5]
176 | [2, 5, 2, 5, 2, 6, 5]
177 | [2, 5, 2, 6, 5, 2, 5]
178 | [2, 6, 5, 2, 5, 2, 5]
179 | [5, 2, 5, 2, 5, 2, 6]
180 | [5, 2, 5, 2, 6, 5, 2]
181 | [5, 2, 6, 5, 2, 5, 2]
182 | [6, 5, 2, 5, 2, 5, 2]
183 | [2, 2, 3, 6, 5, 6, 4]
184 | [2, 3, 6, 5, 6, 4, 2]
185 | [3, 6, 5, 6, 4, 2, 2]
186 | [4, 2, 2, 3, 6, 5, 6]
187 | [5, 6, 4, 2, 2, 3, 6]
188 | [6, 4, 2, 2, 3, 6, 5]
189 | [6, 5, 6, 4, 2, 2, 3]
190 | [2, 3, 3, 5, 2, 6, 5]
191 | [2, 6, 5, 2, 3, 3, 5]
192 | [3, 3, 5, 2, 6, 5, 2]
193 | [3, 5, 2, 6, 5, 2, 3]
194 | [5, 2, 3, 3, 5, 2, 6]
195 | [5, 2, 6, 5, 2, 3, 3]
196 | [6, 5, 2, 3, 3, 5, 2]
197 | [2, 4, 5, 6, 4, 2, 5]
198 | [2, 5, 2, 4, 5, 6, 4]
199 | [4, 2, 5, 2, 4, 5, 6]
200 | [4, 5, 6, 4, 2, 5, 2]
201 | [5, 2, 4, 5, 6, 4, 2]
202 | [5, 6, 4, 2, 5, 2, 4]
203 | [6, 4, 2, 5, 2, 4, 5]
204 | [2, 3, 5, 4, 6, 2, 5]
205 | [2, 5, 2, 3, 5, 4, 6]
206 | [3, 5, 4, 6, 2, 5, 2]
207 | [4, 6, 2, 5, 2, 3, 5]
208 | [5, 2, 3, 5, 4, 6, 2]
209 | [5, 4, 6, 2, 5, 2, 3]
210 | [6, 2, 5, 2, 3, 5, 4]
211 | [2, 3, 5, 2, 5, 4, 6]
212 | [2, 5, 4, 6, 2, 3, 5]
213 | [3, 5, 2, 5, 4, 6, 2]
214 | [4, 6, 2, 3, 5, 2, 5]
215 | [5, 2, 5, 4, 6, 2, 3]
216 | [5, 4, 6, 2, 3, 5, 2]
217 | [6, 2, 3, 5, 2, 5, 4]
218 | [2, 4, 5, 6, 2, 5, 4]
219 | [2, 5, 4, 2, 4, 5, 6]
220 | [4, 2, 4, 5, 6, 2, 5]
221 | [4, 5, 6, 2, 5, 4, 2]
222 | [5, 4, 2, 4, 5, 6, 2]
223 | [5, 6, 2, 5, 4, 2, 4]
224 | [6, 2, 5, 4, 2, 4, 5]
225 | [2, 4, 2, 6, 5, 5, 4]
226 | [2, 6, 5, 5, 4, 2, 4]
227 | [4, 2, 4, 2, 6, 5, 5]
228 | [4, 2, 6, 5, 5, 4, 2]
229 | [5, 4, 2, 4, 2, 6, 5]
230 | [5, 5, 4, 2, 4, 2, 6]
231 | [6, 5, 5, 4, 2, 4, 2]
232 | [2, 2, 5, 6, 6, 4, 4]
233 | [2, 5, 6, 6, 4, 4, 2]
234 | [4, 2, 2, 5, 6, 6, 4]
235 | [4, 4, 2, 2, 5, 6, 6]
236 | [5, 6, 6, 4, 4, 2, 2]
237 | [6, 4, 4, 2, 2, 5, 6]
238 | [6, 6, 4, 4, 2, 2, 5]
239 | [2, 2, 5, 6, 2, 6, 5]
240 | [2, 5, 6, 2, 6, 5, 2]
241 | [2, 6, 5, 2, 2, 5, 6]
242 | [5, 2, 2, 5, 6, 2, 6]
243 | [5, 6, 2, 6, 5, 2, 2]
244 | [6, 2, 6, 5, 2, 2, 5]
245 | [6, 5, 2, 2, 5, 6, 2]
246 | [2, 3, 3, 3, 3, 5, 6]
247 | [3, 3, 3, 3, 5, 6, 2]
248 | [3, 3, 3, 5, 6, 2, 3]
249 | [3, 3, 5, 6, 2, 3, 3]
250 | [3, 5, 6, 2, 3, 3, 3]
251 | [5, 6, 2, 3, 3, 3, 3]
252 | [6, 2, 3, 3, 3, 3, 5]
253 | [2, 3, 2, 3, 6, 5, 6]
254 | [2, 3, 6, 5, 6, 2, 3]
255 | [3, 2, 3, 6, 5, 6, 2]
256 | [3, 6, 5, 6, 2, 3, 2]
257 | [5, 6, 2, 3, 2, 3, 6]
258 | [6, 2, 3, 2, 3, 6, 5]
259 | [6, 5, 6, 2, 3, 2, 3]
260 | [2, 3, 3, 5, 6, 2, 5]
261 | [2, 5, 2, 3, 3, 5, 6]
262 | [3, 3, 5, 6, 2, 5, 2]
263 | [3, 5, 6, 2, 5, 2, 3]
264 | [5, 2, 3, 3, 5, 6, 2]
265 | [5, 6, 2, 5, 2, 3, 3]
266 | [6, 2, 5, 2, 3, 3, 5]
267 | [2, 3, 5, 2, 4, 5, 6]
268 | [2, 4, 5, 6, 2, 3, 5]
269 | [3, 5, 2, 4, 5, 6, 2]
270 | [4, 5, 6, 2, 3, 5, 2]
271 | [5, 2, 4, 5, 6, 2, 3]
272 | [5, 6, 2, 3, 5, 2, 4]
273 | [6, 2, 3, 5, 2, 4, 5]
274 | [2, 2, 2, 6, 6, 5, 6]
275 | [2, 2, 6, 6, 5, 6, 2]
276 | [2, 6, 6, 5, 6, 2, 2]
277 | [5, 6, 2, 2, 2, 6, 6]
278 | [6, 2, 2, 2, 6, 6, 5]
279 | [6, 5, 6, 2, 2, 2, 6]
280 | [6, 6, 5, 6, 2, 2, 2]
281 | [0, 3, 3, 3, 5, 6, 5]
282 | [3, 3, 3, 5, 6, 5, 0]
283 | [3, 3, 5, 6, 5, 0, 3]
284 | [3, 5, 6, 5, 0, 3, 3]
285 | [5, 0, 3, 3, 3, 5, 6]
286 | [5, 6, 5, 0, 3, 3, 3]
287 | [6, 5, 0, 3, 3, 3, 5]
288 | [2, 2, 5, 6, 6, 2, 5]
289 | [2, 5, 2, 2, 5, 6, 6]
290 | [2, 5, 6, 6, 2, 5, 2]
291 | [5, 2, 2, 5, 6, 6, 2]
292 | [5, 6, 6, 2, 5, 2, 2]
293 | [6, 2, 5, 2, 2, 5, 6]
294 | [6, 6, 2, 5, 2, 2, 5]
295 | [0, 6, 5, 5, 2, 3, 5]
296 | [2, 3, 5, 0, 6, 5, 5]
297 | [3, 5, 0, 6, 5, 5, 2]
298 | [5, 0, 6, 5, 5, 2, 3]
299 | [5, 2, 3, 5, 0, 6, 5]
300 | [5, 5, 2, 3, 5, 0, 6]
301 | [6, 5, 5, 2, 3, 5, 0]
302 | [0, 3, 5, 2, 5, 6, 6]
303 | [2, 5, 6, 6, 0, 3, 5]
304 | [3, 5, 2, 5, 6, 6, 0]
305 | [5, 2, 5, 6, 6, 0, 3]
306 | [5, 6, 6, 0, 3, 5, 2]
307 | [6, 0, 3, 5, 2, 5, 6]
308 | [6, 6, 0, 3, 5, 2, 5]
309 |
--------------------------------------------------------------------------------
/examples/admissible_set/pre_admissible_set_for_n24_w17_size237984.txt:
--------------------------------------------------------------------------------
1 | [4, 4, 4, 4, 4, 6, 4, 4]
2 | [4, 4, 4, 4, 4, 4, 6, 4]
3 | [6, 4, 4, 4, 4, 4, 4, 4]
4 | [4, 6, 4, 4, 4, 4, 4, 4]
5 | [4, 4, 6, 4, 4, 4, 4, 4]
6 | [4, 4, 4, 4, 6, 4, 4, 4]
7 | [4, 4, 4, 4, 4, 4, 4, 6]
8 | [4, 4, 4, 6, 4, 4, 4, 4]
9 | [4, 2, 6, 5, 4, 4, 4, 4]
10 | [4, 4, 4, 2, 6, 5, 4, 4]
11 | [4, 4, 2, 6, 5, 4, 4, 4]
12 | [2, 6, 5, 4, 4, 4, 4, 4]
13 | [4, 4, 4, 4, 4, 2, 6, 5]
14 | [4, 4, 4, 4, 2, 6, 5, 4]
15 | [6, 5, 4, 4, 4, 4, 4, 2]
16 | [5, 4, 4, 4, 4, 4, 2, 6]
17 | [4, 4, 2, 5, 4, 6, 4, 4]
18 | [4, 4, 4, 2, 5, 4, 6, 4]
19 | [6, 4, 4, 4, 4, 2, 5, 4]
20 | [5, 4, 6, 4, 4, 4, 4, 2]
21 | [4, 6, 4, 4, 4, 4, 2, 5]
22 | [4, 4, 4, 4, 2, 5, 4, 6]
23 | [2, 5, 4, 6, 4, 4, 4, 4]
24 | [4, 2, 5, 4, 6, 4, 4, 4]
25 | [4, 4, 2, 5, 4, 4, 6, 4]
26 | [4, 2, 5, 4, 4, 4, 6, 4]
27 | [6, 4, 4, 4, 2, 5, 4, 4]
28 | [6, 4, 4, 2, 5, 4, 4, 4]
29 | [4, 6, 4, 4, 2, 5, 4, 4]
30 | [4, 4, 2, 5, 4, 4, 4, 6]
31 | [4, 2, 5, 4, 4, 6, 4, 4]
32 | [2, 5, 4, 4, 4, 6, 4, 4]
33 | [5, 4, 4, 6, 4, 4, 4, 2]
34 | [4, 6, 4, 4, 4, 2, 5, 4]
35 | [4, 4, 6, 4, 4, 4, 2, 5]
36 | [4, 4, 6, 4, 4, 2, 5, 4]
37 | [4, 4, 4, 2, 5, 4, 4, 6]
38 | [2, 5, 4, 4, 6, 4, 4, 4]
39 | [5, 4, 4, 4, 6, 4, 4, 2]
40 | [4, 4, 4, 6, 4, 4, 2, 5]
41 | [5, 4, 4, 4, 4, 4, 6, 2]
42 | [4, 4, 4, 4, 4, 6, 2, 5]
43 | [6, 2, 5, 4, 4, 4, 4, 4]
44 | [4, 6, 2, 5, 4, 4, 4, 4]
45 | [4, 4, 6, 2, 5, 4, 4, 4]
46 | [4, 4, 4, 4, 6, 2, 5, 4]
47 | [4, 4, 4, 6, 2, 5, 4, 4]
48 | [2, 5, 4, 4, 4, 4, 4, 6]
49 | [5, 4, 4, 4, 4, 6, 4, 2]
50 | [2, 5, 4, 4, 4, 4, 6, 4]
51 | [6, 4, 2, 5, 4, 4, 4, 4]
52 | [4, 6, 4, 2, 5, 4, 4, 4]
53 | [4, 4, 6, 4, 2, 5, 4, 4]
54 | [4, 4, 4, 6, 4, 2, 5, 4]
55 | [4, 4, 4, 4, 6, 4, 2, 5]
56 | [4, 2, 5, 4, 4, 4, 4, 6]
57 | [5, 6, 4, 4, 4, 4, 2, 4]
58 | [4, 4, 4, 2, 4, 5, 6, 4]
59 | [6, 4, 4, 4, 4, 2, 4, 5]
60 | [4, 5, 6, 4, 4, 4, 4, 2]
61 | [4, 4, 4, 4, 2, 4, 5, 6]
62 | [4, 4, 2, 4, 5, 6, 4, 4]
63 | [2, 4, 5, 6, 4, 4, 4, 4]
64 | [4, 2, 4, 5, 6, 4, 4, 4]
65 | [6, 4, 4, 2, 3, 5, 4, 4]
66 | [4, 2, 3, 5, 4, 4, 6, 4]
67 | [5, 4, 4, 6, 4, 4, 2, 3]
68 | [4, 6, 4, 4, 2, 3, 5, 4]
69 | [4, 4, 6, 4, 4, 2, 3, 5]
70 | [4, 4, 2, 3, 5, 4, 4, 6]
71 | [2, 3, 5, 4, 4, 6, 4, 4]
72 | [3, 5, 4, 4, 6, 4, 4, 2]
73 | [4, 4, 2, 3, 5, 4, 6, 4]
74 | [5, 4, 6, 4, 4, 4, 2, 3]
75 | [4, 6, 4, 4, 4, 2, 3, 5]
76 | [4, 4, 4, 2, 3, 5, 4, 6]
77 | [6, 4, 4, 4, 2, 3, 5, 4]
78 | [4, 2, 3, 5, 4, 6, 4, 4]
79 | [3, 5, 4, 6, 4, 4, 4, 2]
80 | [2, 3, 5, 4, 6, 4, 4, 4]
81 | [3, 5, 4, 4, 4, 4, 6, 2]
82 | [6, 2, 3, 5, 4, 4, 4, 4]
83 | [5, 4, 4, 4, 4, 6, 2, 3]
84 | [4, 6, 2, 3, 5, 4, 4, 4]
85 | [4, 4, 6, 2, 3, 5, 4, 4]
86 | [4, 4, 4, 6, 2, 3, 5, 4]
87 | [4, 4, 4, 4, 6, 2, 3, 5]
88 | [2, 3, 5, 4, 4, 4, 4, 6]
89 | [2, 3, 5, 4, 4, 4, 6, 4]
90 | [6, 4, 2, 3, 5, 4, 4, 4]
91 | [4, 6, 4, 2, 3, 5, 4, 4]
92 | [4, 2, 3, 5, 4, 4, 4, 6]
93 | [3, 5, 4, 4, 4, 6, 4, 2]
94 | [5, 4, 4, 4, 6, 4, 2, 3]
95 | [4, 4, 6, 4, 2, 3, 5, 4]
96 | [4, 4, 4, 6, 4, 2, 3, 5]
97 | [4, 2, 3, 3, 5, 4, 6, 4]
98 | [4, 4, 2, 3, 3, 5, 4, 6]
99 | [6, 4, 4, 2, 3, 3, 5, 4]
100 | [5, 4, 6, 4, 4, 2, 3, 3]
101 | [4, 6, 4, 4, 2, 3, 3, 5]
102 | [3, 5, 4, 6, 4, 4, 2, 3]
103 | [2, 3, 3, 5, 4, 6, 4, 4]
104 | [3, 3, 5, 4, 6, 4, 4, 2]
105 | [6, 4, 2, 3, 3, 5, 4, 4]
106 | [4, 2, 3, 3, 5, 4, 4, 6]
107 | [5, 4, 4, 6, 4, 2, 3, 3]
108 | [4, 6, 4, 2, 3, 3, 5, 4]
109 | [4, 4, 6, 4, 2, 3, 3, 5]
110 | [2, 3, 3, 5, 4, 4, 6, 4]
111 | [3, 5, 4, 4, 6, 4, 2, 3]
112 | [3, 3, 5, 4, 4, 6, 4, 2]
113 | [5, 6, 4, 4, 4, 2, 3, 3]
114 | [4, 4, 4, 2, 3, 3, 5, 6]
115 | [4, 4, 2, 3, 3, 5, 6, 4]
116 | [4, 2, 3, 3, 5, 6, 4, 4]
117 | [3, 5, 6, 4, 4, 4, 2, 3]
118 | [2, 3, 3, 5, 6, 4, 4, 4]
119 | [6, 4, 4, 4, 2, 3, 3, 5]
120 | [3, 3, 5, 6, 4, 4, 4, 2]
121 | [3, 3, 5, 4, 4, 4, 6, 2]
122 | [6, 2, 3, 3, 5, 4, 4, 4]
123 | [4, 6, 2, 3, 3, 5, 4, 4]
124 | [3, 5, 4, 4, 4, 6, 2, 3]
125 | [2, 3, 3, 5, 4, 4, 4, 6]
126 | [5, 4, 4, 4, 6, 2, 3, 3]
127 | [4, 4, 6, 2, 3, 3, 5, 4]
128 | [4, 4, 4, 6, 2, 3, 3, 5]
129 | [4, 4, 2, 5, 2, 6, 5, 4]
130 | [5, 2, 6, 5, 4, 4, 4, 2]
131 | [4, 4, 4, 2, 5, 2, 6, 5]
132 | [4, 2, 5, 2, 6, 5, 4, 4]
133 | [2, 6, 5, 4, 4, 4, 2, 5]
134 | [2, 5, 2, 6, 5, 4, 4, 4]
135 | [6, 5, 4, 4, 4, 2, 5, 2]
136 | [5, 4, 4, 4, 2, 5, 2, 6]
137 | [6, 5, 4, 4, 2, 5, 4, 2]
138 | [5, 4, 4, 2, 5, 4, 2, 6]
139 | [5, 4, 2, 6, 5, 4, 4, 2]
140 | [4, 2, 6, 5, 4, 4, 2, 5]
141 | [2, 6, 5, 4, 4, 2, 5, 4]
142 | [2, 5, 4, 2, 6, 5, 4, 4]
143 | [4, 4, 2, 5, 4, 2, 6, 5]
144 | [4, 2, 5, 4, 2, 6, 5, 4]
145 | [4, 2, 6, 5, 4, 2, 5, 4]
146 | [5, 4, 4, 2, 6, 5, 4, 2]
147 | [4, 4, 2, 6, 5, 4, 2, 5]
148 | [2, 6, 5, 4, 2, 5, 4, 4]
149 | [2, 5, 4, 4, 2, 6, 5, 4]
150 | [4, 2, 5, 4, 4, 2, 6, 5]
151 | [6, 5, 4, 2, 5, 4, 4, 2]
152 | [5, 4, 2, 5, 4, 4, 2, 6]
153 | [6, 4, 4, 2, 5, 2, 5, 4]
154 | [4, 6, 4, 4, 2, 5, 2, 5]
155 | [4, 4, 2, 5, 2, 5, 4, 6]
156 | [4, 2, 5, 2, 5, 4, 6, 4]
157 | [5, 4, 6, 4, 4, 2, 5, 2]
158 | [2, 5, 4, 6, 4, 4, 2, 5]
159 | [2, 5, 2, 5, 4, 6, 4, 4]
160 | [5, 2, 5, 4, 6, 4, 4, 2]
161 | [4, 2, 6, 5, 2, 5, 4, 4]
162 | [5, 4, 4, 4, 2, 6, 5, 2]
163 | [4, 4, 4, 2, 6, 5, 2, 5]
164 | [4, 4, 2, 6, 5, 2, 5, 4]
165 | [2, 6, 5, 2, 5, 4, 4, 4]
166 | [2, 5, 4, 4, 4, 2, 6, 5]
167 | [6, 5, 2, 5, 4, 4, 4, 2]
168 | [5, 2, 5, 4, 4, 4, 2, 6]
169 | [4, 6, 4, 2, 5, 4, 2, 5]
170 | [6, 4, 2, 5, 4, 2, 5, 4]
171 | [2, 5, 4, 6, 4, 2, 5, 4]
172 | [2, 5, 4, 2, 5, 4, 6, 4]
173 | [5, 4, 6, 4, 2, 5, 4, 2]
174 | [5, 4, 2, 5, 4, 6, 4, 2]
175 | [4, 2, 5, 4, 6, 4, 2, 5]
176 | [4, 2, 5, 4, 2, 5, 4, 6]
177 | [4, 6, 2, 5, 2, 5, 4, 4]
178 | [5, 2, 5, 4, 4, 4, 6, 2]
179 | [4, 4, 6, 2, 5, 2, 5, 4]
180 | [6, 2, 5, 2, 5, 4, 4, 4]
181 | [2, 5, 2, 5, 4, 4, 4, 6]
182 | [2, 5, 4, 4, 4, 6, 2, 5]
183 | [4, 4, 4, 6, 2, 5, 2, 5]
184 | [5, 4, 4, 4, 6, 2, 5, 2]
185 | [5, 4, 4, 6, 2, 5, 4, 2]
186 | [4, 6, 2, 5, 4, 2, 5, 4]
187 | [4, 4, 6, 2, 5, 4, 2, 5]
188 | [5, 4, 2, 5, 4, 4, 6, 2]
189 | [4, 2, 5, 4, 4, 6, 2, 5]
190 | [2, 5, 4, 4, 6, 2, 5, 4]
191 | [6, 2, 5, 4, 2, 5, 4, 4]
192 | [2, 5, 4, 2, 5, 4, 4, 6]
193 | [6, 4, 2, 5, 2, 5, 4, 4]
194 | [2, 5, 2, 5, 4, 4, 6, 4]
195 | [5, 2, 5, 4, 4, 6, 4, 2]
196 | [4, 6, 4, 2, 5, 2, 5, 4]
197 | [4, 2, 5, 2, 5, 4, 4, 6]
198 | [2, 5, 4, 4, 6, 4, 2, 5]
199 | [5, 4, 4, 6, 4, 2, 5, 2]
200 | [4, 4, 6, 4, 2, 5, 2, 5]
201 | [5, 4, 4, 2, 5, 4, 6, 2]
202 | [5, 4, 6, 2, 5, 4, 4, 2]
203 | [4, 6, 2, 5, 4, 4, 2, 5]
204 | [2, 5, 4, 6, 2, 5, 4, 4]
205 | [6, 2, 5, 4, 4, 2, 5, 4]
206 | [4, 4, 2, 5, 4, 6, 2, 5]
207 | [4, 2, 5, 4, 6, 2, 5, 4]
208 | [2, 5, 4, 4, 2, 5, 4, 6]
209 | [2, 3, 3, 3, 5, 6, 4, 4]
210 | [5, 6, 4, 4, 2, 3, 3, 3]
211 | [4, 4, 2, 3, 3, 3, 5, 6]
212 | [4, 2, 3, 3, 3, 5, 6, 4]
213 | [3, 5, 6, 4, 4, 2, 3, 3]
214 | [3, 3, 5, 6, 4, 4, 2, 3]
215 | [3, 3, 3, 5, 6, 4, 4, 2]
216 | [6, 4, 4, 2, 3, 3, 3, 5]
217 | [6, 2, 3, 3, 3, 5, 4, 4]
218 | [2, 3, 3, 3, 5, 4, 4, 6]
219 | [4, 6, 2, 3, 3, 3, 5, 4]
220 | [3, 5, 4, 4, 6, 2, 3, 3]
221 | [3, 3, 3, 5, 4, 4, 6, 2]
222 | [3, 3, 5, 4, 4, 6, 2, 3]
223 | [5, 4, 4, 6, 2, 3, 3, 3]
224 | [4, 4, 6, 2, 3, 3, 3, 5]
225 | [2, 3, 3, 3, 5, 4, 6, 4]
226 | [6, 4, 2, 3, 3, 3, 5, 4]
227 | [5, 4, 6, 4, 2, 3, 3, 3]
228 | [4, 6, 4, 2, 3, 3, 3, 5]
229 | [4, 2, 3, 3, 3, 5, 4, 6]
230 | [3, 5, 4, 6, 4, 2, 3, 3]
231 | [3, 3, 5, 4, 6, 4, 2, 3]
232 | [3, 3, 3, 5, 4, 6, 4, 2]
233 | [4, 4, 2, 2, 5, 6, 4, 6]
234 | [6, 4, 6, 4, 4, 2, 2, 5]
235 | [2, 2, 5, 6, 4, 6, 4, 4]
236 | [6, 4, 4, 2, 2, 5, 6, 4]
237 | [5, 6, 4, 6, 4, 4, 2, 2]
238 | [4, 6, 4, 4, 2, 2, 5, 6]
239 | [4, 2, 2, 5, 6, 4, 6, 4]
240 | [2, 5, 6, 4, 6, 4, 4, 2]
241 | [2, 4, 2, 6, 5, 5, 4, 4]
242 | [4, 2, 4, 2, 6, 5, 5, 4]
243 | [2, 6, 5, 5, 4, 4, 2, 4]
244 | [6, 5, 5, 4, 4, 2, 4, 2]
245 | [4, 2, 6, 5, 5, 4, 4, 2]
246 | [5, 4, 4, 2, 4, 2, 6, 5]
247 | [4, 4, 2, 4, 2, 6, 5, 5]
248 | [5, 5, 4, 4, 2, 4, 2, 6]
249 | [4, 6, 4, 2, 2, 5, 6, 4]
250 | [2, 2, 5, 6, 4, 4, 6, 4]
251 | [6, 4, 4, 6, 4, 2, 2, 5]
252 | [5, 6, 4, 4, 6, 4, 2, 2]
253 | [4, 4, 6, 4, 2, 2, 5, 6]
254 | [6, 4, 2, 2, 5, 6, 4, 4]
255 | [4, 2, 2, 5, 6, 4, 4, 6]
256 | [2, 5, 6, 4, 4, 6, 4, 2]
257 | [5, 6, 4, 4, 4, 6, 2, 2]
258 | [4, 4, 6, 2, 2, 5, 6, 4]
259 | [4, 4, 4, 6, 2, 2, 5, 6]
260 | [2, 5, 6, 4, 4, 4, 6, 2]
261 | [6, 2, 2, 5, 6, 4, 4, 4]
262 | [4, 6, 2, 2, 5, 6, 4, 4]
263 | [2, 2, 5, 6, 4, 4, 4, 6]
264 | [6, 4, 4, 4, 6, 2, 2, 5]
265 | [5, 6, 4, 4, 2, 5, 2, 4]
266 | [4, 2, 5, 2, 4, 5, 6, 4]
267 | [6, 4, 4, 2, 5, 2, 4, 5]
268 | [4, 4, 2, 5, 2, 4, 5, 6]
269 | [2, 5, 2, 4, 5, 6, 4, 4]
270 | [2, 4, 5, 6, 4, 4, 2, 5]
271 | [4, 5, 6, 4, 4, 2, 5, 2]
272 | [5, 2, 4, 5, 6, 4, 4, 2]
273 | [2, 5, 6, 6, 4, 4, 4, 2]
274 | [2, 2, 5, 6, 6, 4, 4, 4]
275 | [6, 6, 4, 4, 4, 2, 2, 5]
276 | [6, 4, 4, 4, 2, 2, 5, 6]
277 | [5, 6, 6, 4, 4, 4, 2, 2]
278 | [4, 4, 4, 2, 2, 5, 6, 6]
279 | [4, 4, 2, 2, 5, 6, 6, 4]
280 | [4, 2, 2, 5, 6, 6, 4, 4]
281 | [5, 4, 4, 2, 4, 5, 6, 2]
282 | [6, 2, 5, 4, 4, 2, 4, 5]
283 | [4, 4, 2, 4, 5, 6, 2, 5]
284 | [4, 2, 4, 5, 6, 2, 5, 4]
285 | [2, 5, 4, 4, 2, 4, 5, 6]
286 | [2, 4, 5, 6, 2, 5, 4, 4]
287 | [5, 6, 2, 5, 4, 4, 2, 4]
288 | [4, 5, 6, 2, 5, 4, 4, 2]
289 | [4, 2, 5, 4, 2, 4, 5, 6]
290 | [5, 6, 4, 2, 5, 4, 2, 4]
291 | [2, 5, 4, 2, 4, 5, 6, 4]
292 | [6, 4, 2, 5, 4, 2, 4, 5]
293 | [5, 4, 2, 4, 5, 6, 4, 2]
294 | [4, 5, 6, 4, 2, 5, 4, 2]
295 | [4, 2, 4, 5, 6, 4, 2, 5]
296 | [2, 4, 5, 6, 4, 2, 5, 4]
297 | [6, 5, 4, 2, 3, 5, 4, 2]
298 | [5, 4, 2, 6, 5, 4, 2, 3]
299 | [5, 2, 6, 5, 4, 4, 2, 3]
300 | [4, 4, 2, 3, 5, 2, 6, 5]
301 | [4, 2, 6, 5, 4, 2, 3, 5]
302 | [4, 2, 3, 5, 2, 6, 5, 4]
303 | [2, 6, 5, 4, 2, 3, 5, 4]
304 | [4, 2, 3, 5, 4, 2, 6, 5]
305 | [2, 3, 5, 4, 2, 6, 5, 4]
306 | [6, 5, 4, 4, 2, 3, 5, 2]
307 | [5, 4, 4, 2, 3, 5, 2, 6]
308 | [5, 4, 2, 3, 5, 4, 2, 6]
309 | [3, 5, 4, 2, 6, 5, 4, 2]
310 | [3, 5, 2, 6, 5, 4, 4, 2]
311 | [2, 6, 5, 4, 4, 2, 3, 5]
312 | [2, 3, 5, 2, 6, 5, 4, 4]
313 | [4, 2, 6, 5, 2, 3, 5, 4]
314 | [6, 5, 2, 3, 5, 4, 4, 2]
315 | [5, 4, 4, 2, 6, 5, 2, 3]
316 | [5, 2, 3, 5, 4, 4, 2, 6]
317 | [4, 4, 2, 6, 5, 2, 3, 5]
318 | [3, 5, 4, 4, 2, 6, 5, 2]
319 | [2, 6, 5, 2, 3, 5, 4, 4]
320 | [2, 3, 5, 4, 4, 2, 6, 5]
321 | [6, 4, 2, 3, 5, 2, 5, 4]
322 | [4, 6, 4, 2, 3, 5, 2, 5]
323 | [4, 2, 3, 5, 2, 5, 4, 6]
324 | [5, 4, 6, 4, 2, 3, 5, 2]
325 | [2, 3, 5, 2, 5, 4, 6, 4]
326 | [5, 2, 5, 4, 6, 4, 2, 3]
327 | [3, 5, 2, 5, 4, 6, 4, 2]
328 | [2, 5, 4, 6, 4, 2, 3, 5]
329 | [5, 4, 6, 2, 3, 5, 4, 2]
330 | [4, 6, 2, 3, 5, 4, 2, 5]
331 | [2, 3, 5, 4, 2, 5, 4, 6]
332 | [6, 2, 3, 5, 4, 2, 5, 4]
333 | [3, 5, 4, 2, 5, 4, 6, 2]
334 | [2, 5, 4, 6, 2, 3, 5, 4]
335 | [5, 4, 2, 5, 4, 6, 2, 3]
336 | [4, 2, 5, 4, 6, 2, 3, 5]
337 | [5, 4, 4, 6, 2, 5, 2, 3]
338 | [4, 6, 2, 5, 2, 3, 5, 4]
339 | [4, 4, 6, 2, 5, 2, 3, 5]
340 | [2, 3, 5, 2, 5, 4, 4, 6]
341 | [5, 2, 3, 5, 4, 4, 6, 2]
342 | [3, 5, 2, 5, 4, 4, 6, 2]
343 | [6, 2, 5, 2, 3, 5, 4, 4]
344 | [6, 2, 3, 5, 2, 5, 4, 4]
345 | [5, 4, 4, 6, 2, 3, 5, 2]
346 | [5, 2, 5, 4, 4, 6, 2, 3]
347 | [4, 6, 2, 3, 5, 2, 5, 4]
348 | [4, 4, 6, 2, 3, 5, 2, 5]
349 | [2, 5, 4, 4, 6, 2, 3, 5]
350 | [2, 5, 2, 3, 5, 4, 4, 6]
351 | [2, 3, 5, 4, 4, 6, 2, 5]
352 | [3, 5, 4, 4, 6, 2, 5, 2]
353 | [6, 2, 5, 4, 2, 3, 5, 4]
354 | [5, 4, 2, 3, 5, 4, 6, 2]
355 | [5, 4, 6, 2, 5, 4, 2, 3]
356 | [4, 6, 2, 5, 4, 2, 3, 5]
357 | [3, 5, 4, 6, 2, 5, 4, 2]
358 | [2, 5, 4, 2, 3, 5, 4, 6]
359 | [2, 3, 5, 4, 6, 2, 5, 4]
360 | [4, 2, 3, 5, 4, 6, 2, 5]
361 | [6, 4, 2, 5, 2, 3, 5, 4]
362 | [5, 4, 6, 4, 2, 5, 2, 3]
363 | [5, 2, 3, 5, 4, 6, 4, 2]
364 | [4, 6, 4, 2, 5, 2, 3, 5]
365 | [4, 2, 5, 2, 3, 5, 4, 6]
366 | [3, 5, 4, 6, 4, 2, 5, 2]
367 | [2, 5, 2, 3, 5, 4, 6, 4]
368 | [2, 3, 5, 4, 6, 4, 2, 5]
369 | [6, 2, 3, 3, 3, 3, 5, 4]
370 | [3, 3, 3, 3, 5, 4, 6, 2]
371 | [2, 3, 3, 3, 3, 5, 4, 6]
372 | [5, 4, 6, 2, 3, 3, 3, 3]
373 | [4, 6, 2, 3, 3, 3, 3, 5]
374 | [3, 3, 3, 5, 4, 6, 2, 3]
375 | [3, 5, 4, 6, 2, 3, 3, 3]
376 | [3, 3, 5, 4, 6, 2, 3, 3]
377 | [3, 3, 3, 3, 5, 6, 4, 2]
378 | [6, 4, 2, 3, 3, 3, 3, 5]
379 | [2, 3, 3, 3, 3, 5, 6, 4]
380 | [3, 3, 3, 5, 6, 4, 2, 3]
381 | [5, 6, 4, 2, 3, 3, 3, 3]
382 | [4, 2, 3, 3, 3, 3, 5, 6]
383 | [3, 5, 6, 4, 2, 3, 3, 3]
384 | [3, 3, 5, 6, 4, 2, 3, 3]
385 | [5, 2, 6, 5, 4, 2, 3, 3]
386 | [2, 6, 5, 4, 2, 3, 3, 5]
387 | [6, 5, 4, 2, 3, 3, 5, 2]
388 | [5, 4, 2, 3, 3, 5, 2, 6]
389 | [4, 2, 3, 3, 5, 2, 6, 5]
390 | [3, 5, 2, 6, 5, 4, 2, 3]
391 | [3, 3, 5, 2, 6, 5, 4, 2]
392 | [2, 3, 3, 5, 2, 6, 5, 4]
393 | [5, 4, 2, 6, 5, 2, 3, 3]
394 | [4, 2, 6, 5, 2, 3, 3, 5]
395 | [2, 6, 5, 2, 3, 3, 5, 4]
396 | [6, 5, 2, 3, 3, 5, 4, 2]
397 | [5, 2, 3, 3, 5, 4, 2, 6]
398 | [3, 5, 4, 2, 6, 5, 2, 3]
399 | [3, 3, 5, 4, 2, 6, 5, 2]
400 | [2, 3, 3, 5, 4, 2, 6, 5]
401 | [4, 4, 2, 2, 3, 6, 5, 6]
402 | [4, 2, 2, 3, 6, 5, 6, 4]
403 | [6, 5, 6, 4, 4, 2, 2, 3]
404 | [5, 6, 4, 4, 2, 2, 3, 6]
405 | [3, 6, 5, 6, 4, 4, 2, 2]
406 | [2, 3, 6, 5, 6, 4, 4, 2]
407 | [2, 2, 3, 6, 5, 6, 4, 4]
408 | [6, 4, 4, 2, 2, 3, 6, 5]
409 | [6, 2, 3, 3, 5, 2, 5, 4]
410 | [2, 3, 3, 5, 2, 5, 4, 6]
411 | [5, 4, 6, 2, 3, 3, 5, 2]
412 | [4, 6, 2, 3, 3, 5, 2, 5]
413 | [3, 5, 2, 5, 4, 6, 2, 3]
414 | [5, 2, 5, 4, 6, 2, 3, 3]
415 | [3, 3, 5, 2, 5, 4, 6, 2]
416 | [2, 5, 4, 6, 2, 3, 3, 5]
417 | [5, 4, 2, 3, 3, 5, 6, 2]
418 | [6, 2, 5, 4, 2, 3, 3, 5]
419 | [4, 2, 3, 3, 5, 6, 2, 5]
420 | [3, 5, 6, 2, 5, 4, 2, 3]
421 | [3, 3, 5, 6, 2, 5, 4, 2]
422 | [2, 3, 3, 5, 6, 2, 5, 4]
423 | [5, 6, 2, 5, 4, 2, 3, 3]
424 | [2, 5, 4, 2, 3, 3, 5, 6]
425 | [2, 5, 2, 5, 2, 6, 5, 4]
426 | [5, 2, 5, 2, 6, 5, 4, 2]
427 | [2, 5, 2, 6, 5, 4, 2, 5]
428 | [5, 2, 6, 5, 4, 2, 5, 2]
429 | [4, 2, 5, 2, 5, 2, 6, 5]
430 | [2, 6, 5, 4, 2, 5, 2, 5]
431 | [6, 5, 4, 2, 5, 2, 5, 2]
432 | [5, 4, 2, 5, 2, 5, 2, 6]
433 | [6, 4, 2, 3, 5, 2, 4, 5]
434 | [4, 2, 3, 5, 2, 4, 5, 6]
435 | [5, 6, 4, 2, 3, 5, 2, 4]
436 | [4, 5, 6, 4, 2, 3, 5, 2]
437 | [2, 3, 5, 2, 4, 5, 6, 4]
438 | [5, 2, 4, 5, 6, 4, 2, 3]
439 | [3, 5, 2, 4, 5, 6, 4, 2]
440 | [2, 4, 5, 6, 4, 2, 3, 5]
441 | [6, 2, 5, 2, 3, 3, 5, 4]
442 | [5, 4, 6, 2, 5, 2, 3, 3]
443 | [4, 6, 2, 5, 2, 3, 3, 5]
444 | [2, 5, 2, 3, 3, 5, 4, 6]
445 | [3, 5, 4, 6, 2, 5, 2, 3]
446 | [5, 2, 3, 3, 5, 4, 6, 2]
447 | [3, 3, 5, 4, 6, 2, 5, 2]
448 | [2, 3, 3, 5, 4, 6, 2, 5]
449 | [2, 5, 2, 6, 5, 2, 5, 4]
450 | [5, 2, 6, 5, 2, 5, 4, 2]
451 | [2, 6, 5, 2, 5, 4, 2, 5]
452 | [5, 4, 2, 5, 2, 6, 5, 2]
453 | [4, 2, 5, 2, 6, 5, 2, 5]
454 | [6, 5, 2, 5, 4, 2, 5, 2]
455 | [2, 5, 4, 2, 5, 2, 6, 5]
456 | [5, 2, 5, 4, 2, 5, 2, 6]
457 | [5, 6, 2, 3, 5, 4, 2, 4]
458 | [3, 5, 4, 2, 4, 5, 6, 2]
459 | [2, 4, 5, 6, 2, 3, 5, 4]
460 | [6, 2, 3, 5, 4, 2, 4, 5]
461 | [5, 4, 2, 4, 5, 6, 2, 3]
462 | [4, 5, 6, 2, 3, 5, 4, 2]
463 | [4, 2, 4, 5, 6, 2, 3, 5]
464 | [2, 3, 5, 4, 2, 4, 5, 6]
465 | [5, 2, 3, 3, 5, 6, 4, 2]
466 | [6, 4, 2, 5, 2, 3, 3, 5]
467 | [5, 6, 4, 2, 5, 2, 3, 3]
468 | [4, 2, 5, 2, 3, 3, 5, 6]
469 | [3, 3, 5, 6, 4, 2, 5, 2]
470 | [2, 5, 2, 3, 3, 5, 6, 4]
471 | [3, 5, 6, 4, 2, 5, 2, 3]
472 | [2, 3, 3, 5, 6, 4, 2, 5]
473 | [2, 6, 5, 2, 5, 2, 5, 4]
474 | [4, 2, 6, 5, 2, 5, 2, 5]
475 | [6, 5, 2, 5, 2, 5, 4, 2]
476 | [5, 4, 2, 6, 5, 2, 5, 2]
477 | [5, 2, 5, 4, 2, 6, 5, 2]
478 | [2, 5, 4, 2, 6, 5, 2, 5]
479 | [2, 5, 2, 5, 4, 2, 6, 5]
480 | [5, 2, 5, 2, 5, 4, 2, 6]
481 | [6, 2, 3, 3, 3, 3, 3, 5]
482 | [3, 3, 3, 3, 3, 5, 6, 2]
483 | [3, 3, 3, 3, 5, 6, 2, 3]
484 | [5, 6, 2, 3, 3, 3, 3, 3]
485 | [3, 5, 6, 2, 3, 3, 3, 3]
486 | [3, 3, 3, 5, 6, 2, 3, 3]
487 | [2, 3, 3, 3, 3, 3, 5, 6]
488 | [3, 3, 5, 6, 2, 3, 3, 3]
489 | [6, 2, 5, 2, 5, 2, 5, 4]
490 | [5, 4, 6, 2, 5, 2, 5, 2]
491 | [4, 6, 2, 5, 2, 5, 2, 5]
492 | [2, 5, 4, 6, 2, 5, 2, 5]
493 | [2, 5, 2, 5, 2, 5, 4, 6]
494 | [5, 2, 5, 4, 6, 2, 5, 2]
495 | [5, 2, 5, 2, 5, 4, 6, 2]
496 | [2, 5, 2, 5, 4, 6, 2, 5]
497 | [2, 3, 5, 2, 3, 5, 4, 6]
498 | [6, 2, 3, 5, 2, 3, 5, 4]
499 | [5, 4, 6, 2, 3, 5, 2, 3]
500 | [5, 2, 3, 5, 4, 6, 2, 3]
501 | [4, 6, 2, 3, 5, 2, 3, 5]
502 | [3, 5, 4, 6, 2, 3, 5, 2]
503 | [3, 5, 2, 3, 5, 4, 6, 2]
504 | [2, 3, 5, 4, 6, 2, 3, 5]
505 | [6, 5, 2, 3, 3, 3, 5, 2]
506 | [2, 3, 3, 3, 5, 2, 6, 5]
507 | [5, 2, 6, 5, 2, 3, 3, 3]
508 | [5, 2, 3, 3, 3, 5, 2, 6]
509 | [3, 5, 2, 6, 5, 2, 3, 3]
510 | [3, 3, 3, 5, 2, 6, 5, 2]
511 | [3, 3, 5, 2, 6, 5, 2, 3]
512 | [2, 6, 5, 2, 3, 3, 3, 5]
513 | [6, 5, 4, 2, 2, 5, 6, 2]
514 | [5, 6, 2, 6, 5, 4, 2, 2]
515 | [2, 5, 6, 2, 6, 5, 4, 2]
516 | [2, 2, 5, 6, 2, 6, 5, 4]
517 | [6, 2, 6, 5, 4, 2, 2, 5]
518 | [5, 4, 2, 2, 5, 6, 2, 6]
519 | [2, 6, 5, 4, 2, 2, 5, 6]
520 | [4, 2, 2, 5, 6, 2, 6, 5]
521 | [5, 6, 4, 2, 6, 5, 2, 2]
522 | [4, 2, 6, 5, 2, 2, 5, 6]
523 | [6, 4, 2, 6, 5, 2, 2, 5]
524 | [2, 6, 5, 2, 2, 5, 6, 4]
525 | [2, 5, 6, 4, 2, 6, 5, 2]
526 | [2, 2, 5, 6, 4, 2, 6, 5]
527 | [6, 5, 2, 2, 5, 6, 4, 2]
528 | [5, 2, 2, 5, 6, 4, 2, 6]
529 | [2, 3, 3, 5, 2, 4, 5, 6]
530 | [6, 2, 3, 3, 5, 2, 4, 5]
531 | [5, 6, 2, 3, 3, 5, 2, 4]
532 | [4, 5, 6, 2, 3, 3, 5, 2]
533 | [3, 3, 5, 2, 4, 5, 6, 2]
534 | [3, 5, 2, 4, 5, 6, 2, 3]
535 | [2, 4, 5, 6, 2, 3, 3, 5]
536 | [5, 2, 4, 5, 6, 2, 3, 3]
537 | [4, 3, 2, 3, 6, 5, 6, 2]
538 | [2, 4, 3, 2, 3, 6, 5, 6]
539 | [6, 5, 6, 2, 4, 3, 2, 3]
540 | [5, 6, 2, 4, 3, 2, 3, 6]
541 | [3, 6, 5, 6, 2, 4, 3, 2]
542 | [6, 2, 4, 3, 2, 3, 6, 5]
543 | [3, 2, 3, 6, 5, 6, 2, 4]
544 | [2, 3, 6, 5, 6, 2, 4, 3]
545 | [5, 2, 3, 3, 3, 5, 6, 2]
546 | [3, 5, 6, 2, 5, 2, 3, 3]
547 | [2, 3, 3, 3, 5, 6, 2, 5]
548 | [6, 2, 5, 2, 3, 3, 3, 5]
549 | [3, 3, 5, 6, 2, 5, 2, 3]
550 | [3, 3, 3, 5, 6, 2, 5, 2]
551 | [2, 5, 2, 3, 3, 3, 5, 6]
552 | [5, 6, 2, 5, 2, 3, 3, 3]
553 | [6, 4, 6, 2, 5, 2, 2, 5]
554 | [5, 6, 4, 6, 2, 5, 2, 2]
555 | [4, 6, 2, 5, 2, 2, 5, 6]
556 | [2, 5, 2, 2, 5, 6, 4, 6]
557 | [6, 2, 5, 2, 2, 5, 6, 4]
558 | [2, 5, 6, 4, 6, 2, 5, 2]
559 | [5, 2, 2, 5, 6, 4, 6, 2]
560 | [2, 2, 5, 6, 4, 6, 2, 5]
561 | [6, 2, 2, 5, 6, 2, 5, 4]
562 | [5, 4, 6, 2, 2, 5, 6, 2]
563 | [5, 6, 2, 5, 4, 6, 2, 2]
564 | [2, 2, 5, 6, 2, 5, 4, 6]
565 | [4, 6, 2, 2, 5, 6, 2, 5]
566 | [2, 5, 6, 2, 5, 4, 6, 2]
567 | [6, 2, 5, 4, 6, 2, 2, 5]
568 | [2, 5, 4, 6, 2, 2, 5, 6]
569 | [5, 2, 5, 2, 4, 2, 6, 5]
570 | [5, 2, 4, 2, 6, 5, 5, 2]
571 | [4, 2, 6, 5, 5, 2, 5, 2]
572 | [2, 6, 5, 5, 2, 5, 2, 4]
573 | [2, 4, 2, 6, 5, 5, 2, 5]
574 | [6, 5, 5, 2, 5, 2, 4, 2]
575 | [2, 5, 2, 4, 2, 6, 5, 5]
576 | [5, 5, 2, 5, 2, 4, 2, 6]
577 | [5, 4, 2, 6, 5, 5, 2, 2]
578 | [5, 2, 2, 5, 4, 2, 6, 5]
579 | [2, 5, 4, 2, 6, 5, 5, 2]
580 | [6, 5, 5, 2, 2, 5, 4, 2]
581 | [4, 2, 6, 5, 5, 2, 2, 5]
582 | [2, 6, 5, 5, 2, 2, 5, 4]
583 | [2, 2, 5, 4, 2, 6, 5, 5]
584 | [5, 5, 2, 2, 5, 4, 2, 6]
585 | [5, 6, 2, 5, 2, 5, 2, 4]
586 | [5, 2, 5, 2, 4, 5, 6, 2]
587 | [6, 2, 5, 2, 5, 2, 4, 5]
588 | [5, 2, 4, 5, 6, 2, 5, 2]
589 | [4, 5, 6, 2, 5, 2, 5, 2]
590 | [2, 5, 2, 5, 2, 4, 5, 6]
591 | [2, 4, 5, 6, 2, 5, 2, 5]
592 | [2, 5, 2, 4, 5, 6, 2, 5]
593 | [6, 4, 2, 5, 2, 2, 5, 6]
594 | [5, 6, 6, 4, 2, 5, 2, 2]
595 | [4, 2, 5, 2, 2, 5, 6, 6]
596 | [2, 2, 5, 6, 6, 4, 2, 5]
597 | [6, 6, 4, 2, 5, 2, 2, 5]
598 | [5, 2, 2, 5, 6, 6, 4, 2]
599 | [2, 5, 6, 6, 4, 2, 5, 2]
600 | [2, 5, 2, 2, 5, 6, 6, 4]
601 | [2, 2, 5, 6, 6, 2, 5, 4]
602 | [2, 5, 6, 6, 2, 5, 4, 2]
603 | [6, 2, 5, 4, 2, 2, 5, 6]
604 | [5, 6, 6, 2, 5, 4, 2, 2]
605 | [5, 4, 2, 2, 5, 6, 6, 2]
606 | [2, 5, 4, 2, 2, 5, 6, 6]
607 | [6, 6, 2, 5, 4, 2, 2, 5]
608 | [4, 2, 2, 5, 6, 6, 2, 5]
609 | [5, 2, 3, 3, 5, 6, 2, 3]
610 | [5, 6, 2, 3, 5, 2, 3, 3]
611 | [3, 5, 2, 3, 3, 5, 6, 2]
612 | [3, 3, 5, 6, 2, 3, 5, 2]
613 | [2, 3, 5, 2, 3, 3, 5, 6]
614 | [2, 3, 3, 5, 6, 2, 3, 5]
615 | [6, 2, 3, 5, 2, 3, 3, 5]
616 | [3, 5, 6, 2, 3, 5, 2, 3]
617 | [2, 3, 5, 2, 5, 2, 6, 5]
618 | [5, 2, 5, 2, 6, 5, 2, 3]
619 | [3, 5, 2, 5, 2, 6, 5, 2]
620 | [6, 5, 2, 3, 5, 2, 5, 2]
621 | [5, 2, 6, 5, 2, 3, 5, 2]
622 | [2, 6, 5, 2, 3, 5, 2, 5]
623 | [2, 5, 2, 6, 5, 2, 3, 5]
624 | [5, 2, 3, 5, 2, 5, 2, 6]
625 | [5, 2, 6, 5, 2, 5, 2, 3]
626 | [3, 5, 2, 6, 5, 2, 5, 2]
627 | [2, 6, 5, 2, 5, 2, 3, 5]
628 | [5, 2, 3, 5, 2, 6, 5, 2]
629 | [2, 5, 2, 3, 5, 2, 6, 5]
630 | [2, 3, 5, 2, 6, 5, 2, 5]
631 | [6, 5, 2, 5, 2, 3, 5, 2]
632 | [5, 2, 5, 2, 3, 5, 2, 6]
633 | [6, 2, 2, 3, 3, 3, 6, 5]
634 | [2, 3, 3, 3, 6, 5, 6, 2]
635 | [2, 2, 3, 3, 3, 6, 5, 6]
636 | [6, 5, 6, 2, 2, 3, 3, 3]
637 | [5, 6, 2, 2, 3, 3, 3, 6]
638 | [3, 3, 3, 6, 5, 6, 2, 2]
639 | [3, 6, 5, 6, 2, 2, 3, 3]
640 | [3, 3, 6, 5, 6, 2, 2, 3]
641 | [6, 2, 2, 5, 6, 2, 4, 5]
642 | [5, 6, 2, 4, 5, 6, 2, 2]
643 | [4, 5, 6, 2, 2, 5, 6, 2]
644 | [2, 5, 6, 2, 4, 5, 6, 2]
645 | [6, 2, 4, 5, 6, 2, 2, 5]
646 | [5, 6, 2, 2, 5, 6, 2, 4]
647 | [2, 4, 5, 6, 2, 2, 5, 6]
648 | [2, 2, 5, 6, 2, 4, 5, 6]
649 | [5, 2, 2, 3, 5, 2, 6, 5]
650 | [3, 5, 2, 6, 5, 5, 2, 2]
651 | [2, 6, 5, 5, 2, 2, 3, 5]
652 | [2, 3, 5, 2, 6, 5, 5, 2]
653 | [2, 2, 3, 5, 2, 6, 5, 5]
654 | [6, 5, 5, 2, 2, 3, 5, 2]
655 | [5, 5, 2, 2, 3, 5, 2, 6]
656 | [5, 2, 6, 5, 5, 2, 2, 3]
657 | [6, 5, 6, 2, 5, 2, 2, 3]
658 | [5, 2, 2, 3, 6, 5, 6, 2]
659 | [3, 6, 5, 6, 2, 5, 2, 2]
660 | [2, 2, 3, 6, 5, 6, 2, 5]
661 | [6, 2, 5, 2, 2, 3, 6, 5]
662 | [2, 5, 2, 2, 3, 6, 5, 6]
663 | [2, 3, 6, 5, 6, 2, 5, 2]
664 | [5, 6, 2, 5, 2, 2, 3, 6]
665 | [2, 2, 5, 6, 6, 2, 3, 5]
666 | [6, 2, 3, 5, 2, 2, 5, 6]
667 | [5, 6, 6, 2, 3, 5, 2, 2]
668 | [5, 2, 2, 5, 6, 6, 2, 3]
669 | [3, 5, 2, 2, 5, 6, 6, 2]
670 | [2, 5, 6, 6, 2, 3, 5, 2]
671 | [2, 3, 5, 2, 2, 5, 6, 6]
672 | [6, 6, 2, 3, 5, 2, 2, 5]
673 | [2, 2, 5, 2, 3, 6, 5, 6]
674 | [6, 5, 6, 2, 2, 5, 2, 3]
675 | [6, 2, 2, 5, 2, 3, 6, 5]
676 | [5, 6, 2, 2, 5, 2, 3, 6]
677 | [5, 2, 3, 6, 5, 6, 2, 2]
678 | [2, 5, 2, 3, 6, 5, 6, 2]
679 | [2, 3, 6, 5, 6, 2, 2, 5]
680 | [3, 6, 5, 6, 2, 2, 5, 2]
681 | [4, 6, 4, 0, 5, 2, 5, 6]
682 | [6, 4, 6, 4, 0, 5, 2, 5]
683 | [2, 5, 6, 4, 6, 4, 0, 5]
684 | [5, 6, 4, 6, 4, 0, 5, 2]
685 | [5, 2, 5, 6, 4, 6, 4, 0]
686 | [0, 5, 2, 5, 6, 4, 6, 4]
687 | [6, 4, 0, 5, 2, 5, 6, 4]
688 | [4, 0, 5, 2, 5, 6, 4, 6]
689 | [6, 6, 4, 2, 3, 5, 0, 5]
690 | [6, 4, 2, 3, 5, 0, 5, 6]
691 | [5, 6, 6, 4, 2, 3, 5, 0]
692 | [4, 2, 3, 5, 0, 5, 6, 6]
693 | [0, 5, 6, 6, 4, 2, 3, 5]
694 | [3, 5, 0, 5, 6, 6, 4, 2]
695 | [5, 0, 5, 6, 6, 4, 2, 3]
696 | [2, 3, 5, 0, 5, 6, 6, 4]
697 | [5, 6, 0, 6, 5, 0, 6, 3]
698 | [0, 6, 5, 0, 6, 3, 5, 6]
699 | [6, 5, 0, 6, 3, 5, 6, 0]
700 | [6, 0, 6, 5, 0, 6, 3, 5]
701 | [5, 0, 6, 3, 5, 6, 0, 6]
702 | [0, 6, 3, 5, 6, 0, 6, 5]
703 | [6, 3, 5, 6, 0, 6, 5, 0]
704 | [3, 5, 6, 0, 6, 5, 0, 6]
705 | [6, 5, 0, 3, 3, 3, 3, 5]
706 | [3, 3, 3, 5, 6, 5, 0, 3]
707 | [5, 6, 5, 0, 3, 3, 3, 3]
708 | [0, 3, 3, 3, 3, 5, 6, 5]
709 | [5, 0, 3, 3, 3, 3, 5, 6]
710 | [3, 5, 6, 5, 0, 3, 3, 3]
711 | [3, 3, 5, 6, 5, 0, 3, 3]
712 | [3, 3, 3, 3, 5, 6, 5, 0]
713 | [2, 5, 6, 2, 6, 5, 0, 5]
714 | [5, 6, 2, 6, 5, 0, 5, 2]
715 | [6, 2, 6, 5, 0, 5, 2, 5]
716 | [5, 2, 5, 6, 2, 6, 5, 0]
717 | [0, 5, 2, 5, 6, 2, 6, 5]
718 | [2, 6, 5, 0, 5, 2, 5, 6]
719 | [6, 5, 0, 5, 2, 5, 6, 2]
720 | [5, 0, 5, 2, 5, 6, 2, 6]
721 | [0, 5, 6, 6, 2, 3, 3, 5]
722 | [6, 2, 3, 3, 5, 0, 5, 6]
723 | [5, 6, 6, 2, 3, 3, 5, 0]
724 | [2, 3, 3, 5, 0, 5, 6, 6]
725 | [6, 6, 2, 3, 3, 5, 0, 5]
726 | [5, 0, 5, 6, 6, 2, 3, 3]
727 | [3, 5, 0, 5, 6, 6, 2, 3]
728 | [3, 3, 5, 0, 5, 6, 6, 2]
729 | [6, 6, 2, 2, 5, 6, 0, 5]
730 | [6, 0, 5, 6, 6, 2, 2, 5]
731 | [5, 6, 6, 2, 2, 5, 6, 0]
732 | [0, 5, 6, 6, 2, 2, 5, 6]
733 | [6, 2, 2, 5, 6, 0, 5, 6]
734 | [2, 5, 6, 0, 5, 6, 6, 2]
735 | [2, 2, 5, 6, 0, 5, 6, 6]
736 | [5, 6, 0, 5, 6, 6, 2, 2]
737 |
--------------------------------------------------------------------------------
/examples/cap_set/cap_set.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {
6 | "id": "RizPOl_DRB-_"
7 | },
8 | "source": [
9 | "```\n",
10 | "- Copyright 2023 DeepMind Technologies Limited\n",
11 | "- All software is licensed under the Apache License, Version 2.0 (Apache 2.0); you may not use this file except in compliance with the Apache 2.0 license. You may obtain a copy of the Apache 2.0 license at: https://www.apache.org/licenses/LICENSE-2.0\n",
12 | "- All other materials are licensed under the Creative Commons Attribution 4.0 International License (CC-BY). You may obtain a copy of the CC-BY license at: https://creativecommons.org/licenses/by/4.0/legalcode\n",
13 | "- Unless required by applicable law or agreed to in writing, all software and materials distributed here under the Apache 2.0 or CC-BY licenses are distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the licenses for the specific language governing permissions and limitations under those licenses.\n",
14 | "- This is not an official Google product\n",
15 | "```"
16 | ]
17 | },
18 | {
19 | "cell_type": "markdown",
20 | "metadata": {
21 | "id": "LVKsiNuyfoNZ"
22 | },
23 | "source": [
24 | "# Cap set\n",
25 | "\n",
26 | "This notebook contains\n",
27 | "\n",
28 | "1. the *skeleton* we used for FunSearch to discover large cap sets,\n",
29 | "2. the *functions* discovered by FunSearch that construct large cap sets.\n",
30 | "\n",
31 | "## Skeleton\n",
32 | "\n",
33 | "The commented-out decorators are just a way to indicate the main entry point of the program (`@funsearch.run`) and the function that *FunSearch* should evolve (`@funsearch.evolve`)."
34 | ]
35 | },
36 | {
37 | "cell_type": "code",
38 | "execution_count": null,
39 | "metadata": {
40 | "id": "3zZ0fAe6flO_"
41 | },
42 | "outputs": [],
43 | "source": [
44 | "\"\"\"Finds large cap sets.\"\"\"\n",
45 | "import itertools\n",
46 | "import numpy as np\n",
47 | "\n",
48 | "\n",
49 | "# @funsearch.run\n",
50 | "def evaluate(n: int) -\u003e int:\n",
51 | " \"\"\"Returns the size of an `n`-dimensional cap set.\"\"\"\n",
52 | " capset = solve(n)\n",
53 | " return len(capset)\n",
54 | "\n",
55 | "\n",
56 | "def solve(n: int) -\u003e np.ndarray:\n",
57 | " \"\"\"Returns a large cap set in `n` dimensions.\"\"\"\n",
58 | " all_vectors = np.array(list(itertools.product((0, 1, 2), repeat=n)), dtype=np.int32)\n",
59 | "\n",
60 | " # Powers in decreasing order for compatibility with `itertools.product`, so\n",
61 | " # that the relationship `i = all_vectors[i] @ powers` holds for all `i`.\n",
62 | " powers = 3 ** np.arange(n - 1, -1, -1)\n",
63 | "\n",
64 | " # Precompute all priorities.\n",
65 | " priorities = np.array([priority(tuple(vector), n) for vector in all_vectors])\n",
66 | "\n",
67 | " # Build `capset` greedily, using priorities for prioritization.\n",
68 | " capset = np.empty(shape=(0, n), dtype=np.int32)\n",
69 | " while np.any(priorities != -np.inf):\n",
70 | " # Add a vector with maximum priority to `capset`, and set priorities of\n",
71 | " # invalidated vectors to `-inf`, so that they never get selected.\n",
72 | " max_index = np.argmax(priorities)\n",
73 | " vector = all_vectors[None, max_index] # [1, n]\n",
74 | " blocking = np.einsum('cn,n-\u003ec', (- capset - vector) % 3, powers) # [C]\n",
75 | " priorities[blocking] = -np.inf\n",
76 | " priorities[max_index] = -np.inf\n",
77 | " capset = np.concatenate([capset, vector], axis=0)\n",
78 | "\n",
79 | " return capset\n",
80 | "\n",
81 | "\n",
82 | "# @funsearch.evolve\n",
83 | "def priority(el: tuple[int, ...], n: int) -\u003e float:\n",
84 | " \"\"\"Returns the priority with which we want to add `element` to the cap set.\"\"\"\n",
85 | " return 0.0"
86 | ]
87 | },
88 | {
89 | "cell_type": "markdown",
90 | "metadata": {
91 | "id": "QY5jPdo-g1fT"
92 | },
93 | "source": [
94 | "By executing the skeleton with the trivial `priority` function in place we can check that the resulting cap sets are far from optimal (e.g. recall that largest known cap set for `n = 9` has size `1082`):"
95 | ]
96 | },
97 | {
98 | "cell_type": "code",
99 | "execution_count": null,
100 | "metadata": {
101 | "executionInfo": {
102 | "elapsed": 134,
103 | "status": "ok",
104 | "timestamp": 1697038278379,
105 | "user": {
106 | "displayName": "",
107 | "userId": ""
108 | },
109 | "user_tz": -60
110 | },
111 | "id": "1cLP6xvzfn1k",
112 | "outputId": "7b371fd6-ad19-4459-d68d-0ccfa9e2927a"
113 | },
114 | "outputs": [
115 | {
116 | "name": "stdout",
117 | "output_type": "stream",
118 | "text": [
119 | "3 8\n",
120 | "4 16\n",
121 | "5 32\n",
122 | "6 64\n",
123 | "7 128\n",
124 | "8 256\n",
125 | "9 512\n"
126 | ]
127 | }
128 | ],
129 | "source": [
130 | "for n in range(3, 9+1):\n",
131 | " print(n, evaluate(n))"
132 | ]
133 | },
134 | {
135 | "cell_type": "markdown",
136 | "metadata": {
137 | "id": "I9-mf0aThXQl"
138 | },
139 | "source": [
140 | "## Discovered function that builds a $512$-cap in $n = 8$ dimensions\n",
141 | "\n",
142 | "This function discovered by FunSearch results in a cap set of size $512$ in $n = 8$ dimensions, thus improving upon the previously known best construction (which had size $496$)."
143 | ]
144 | },
145 | {
146 | "cell_type": "code",
147 | "execution_count": null,
148 | "metadata": {
149 | "id": "k-k8WyrohG8I"
150 | },
151 | "outputs": [],
152 | "source": [
153 | "def priority(el: tuple[int, ...], n: int) -\u003e float:\n",
154 | " score = n\n",
155 | " in_el = 0\n",
156 | " el_count = el.count(0)\n",
157 | "\n",
158 | " if el_count == 0:\n",
159 | " score += n ** 2\n",
160 | " if el[1] == el[-1]:\n",
161 | " score *= 1.5\n",
162 | " if el[2] == el[-2]:\n",
163 | " score *= 1.5\n",
164 | " if el[3] == el[-3]:\n",
165 | " score *= 1.5\n",
166 | " else:\n",
167 | " if el[1] == el[-1]:\n",
168 | " score *= 0.5\n",
169 | " if el[2] == el[-2]:\n",
170 | " score *= 0.5\n",
171 | "\n",
172 | " for e in el:\n",
173 | " if e == 0:\n",
174 | " if in_el == 0:\n",
175 | " score *= n * 0.5\n",
176 | " elif in_el == el_count - 1:\n",
177 | " score *= 0.5\n",
178 | " else:\n",
179 | " score *= n * 0.5 ** in_el\n",
180 | " in_el += 1\n",
181 | " else:\n",
182 | " score += 1\n",
183 | "\n",
184 | " if el[1] == el[-1]:\n",
185 | " score *= 1.5\n",
186 | " if el[2] == el[-2]:\n",
187 | " score *= 1.5\n",
188 | "\n",
189 | " return score\n",
190 | "\n",
191 | "\n",
192 | "# We call the `solve` function instead of `evaluate` so that we get access to\n",
193 | "# the cap set itself (rather than just its size), for verification and\n",
194 | "# inspection purposes.\n",
195 | "cap_set_n8 = solve(8)\n",
196 | "assert cap_set_n8.shape == (512, 8)"
197 | ]
198 | },
199 | {
200 | "cell_type": "markdown",
201 | "metadata": {
202 | "id": "uTFYifHWiEO3"
203 | },
204 | "source": [
205 | "We make use of a helper function to verify that the cap set is indeed valid."
206 | ]
207 | },
208 | {
209 | "cell_type": "code",
210 | "execution_count": null,
211 | "metadata": {
212 | "id": "ZvQqqXS_iDhY"
213 | },
214 | "outputs": [],
215 | "source": [
216 | "def is_cap_set(vectors: np.ndarray) -\u003e bool:\n",
217 | " \"\"\"Returns whether `vectors` form a valid cap set.\n",
218 | "\n",
219 | " Checking the cap set property naively takes O(c^3 n) time, where c is the size\n",
220 | " of the cap set. This function implements a faster check that runs in O(c^2 n).\n",
221 | "\n",
222 | " Args:\n",
223 | " vectors: [c, n] array containing c n-dimensional vectors over {0, 1, 2}.\n",
224 | " \"\"\"\n",
225 | " _, n = vectors.shape\n",
226 | "\n",
227 | " # Convert `vectors` elements into raveled indices (numbers in [0, 3^n) ).\n",
228 | " powers = np.array([3 ** j for j in range(n - 1, -1, -1)], dtype=int) # [n]\n",
229 | " raveled = np.einsum('in,n-\u003ei', vectors, powers) # [c]\n",
230 | "\n",
231 | " # Starting from the empty set, we iterate through `vectors` one by one and at\n",
232 | " # each step check that the vector can be inserted into the set without\n",
233 | " # violating the defining property of cap set. To make this check fast we\n",
234 | " # maintain a vector `is_blocked` indicating for each element of Z_3^n whether\n",
235 | " # that element can be inserted into the growing set without violating the cap\n",
236 | " # set property.\n",
237 | " is_blocked = np.full(shape=3 ** n, fill_value=False, dtype=bool)\n",
238 | " for i, (new_vector, new_index) in enumerate(zip(vectors, raveled)):\n",
239 | " if is_blocked[new_index]:\n",
240 | " return False # Inserting the i-th element violated the cap set property.\n",
241 | " if i \u003e= 1:\n",
242 | " # Update which elements are blocked after the insertion of `new_vector`.\n",
243 | " blocking = np.einsum(\n",
244 | " 'nk,k-\u003en',\n",
245 | " (- vectors[:i, :] - new_vector[None, :]) % 3, powers)\n",
246 | " is_blocked[blocking] = True\n",
247 | " is_blocked[new_index] = True # In case `vectors` contains duplicates.\n",
248 | " return True # All elements inserted without violating the cap set property.\n",
249 | "\n",
250 | "\n",
251 | "assert is_cap_set(cap_set_n8)"
252 | ]
253 | },
254 | {
255 | "cell_type": "markdown",
256 | "metadata": {
257 | "id": "W74wDTgB0KOn"
258 | },
259 | "source": [
260 | "We can start noticing some regularities in the discovered cap set if we inspect the number of nonzero entries (weights) of each of the 512 vectors:"
261 | ]
262 | },
263 | {
264 | "cell_type": "code",
265 | "execution_count": null,
266 | "metadata": {
267 | "executionInfo": {
268 | "elapsed": 53,
269 | "status": "ok",
270 | "timestamp": 1697038278944,
271 | "user": {
272 | "displayName": "",
273 | "userId": ""
274 | },
275 | "user_tz": -60
276 | },
277 | "id": "_tRWqFAVzb6R",
278 | "outputId": "f36a956e-3b53-42da-a7e7-0965fad27770"
279 | },
280 | "outputs": [
281 | {
282 | "name": "stdout",
283 | "output_type": "stream",
284 | "text": [
285 | "[8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8\n",
286 | " 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8\n",
287 | " 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8\n",
288 | " 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n",
289 | " 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n",
290 | " 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n",
291 | " 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n",
292 | " 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n",
293 | " 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n",
294 | " 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4\n",
295 | " 4 4 4 4 4 4 4 4 4 4 4 4 4 4 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5\n",
296 | " 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5\n",
297 | " 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5\n",
298 | " 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5]\n"
299 | ]
300 | }
301 | ],
302 | "source": [
303 | "print(np.count_nonzero(cap_set_n8, axis=1))"
304 | ]
305 | },
306 | {
307 | "cell_type": "markdown",
308 | "metadata": {
309 | "id": "iOSNsJDQ0a_e"
310 | },
311 | "source": [
312 | "## Explicit construction of a $512$-cap in $n = 8$ dimensions\n",
313 | "\n",
314 | "Thanks to discovering this cap set by searching in function space and noticing some regularities like the one above, we were able to manually find the following explicit construction of this new $512$-cap. See the paper's Supplementary Information for more details.\n",
315 | "\n"
316 | ]
317 | },
318 | {
319 | "cell_type": "code",
320 | "execution_count": null,
321 | "metadata": {
322 | "id": "9D3B5-nz1k9x"
323 | },
324 | "outputs": [],
325 | "source": [
326 | "def build_512_cap() -\u003e list[tuple[int, ...]]:\n",
327 | " \"\"\"Returns a cap set of size 512 in `n=8` dimensions.\"\"\"\n",
328 | " n = 8\n",
329 | " V = list(itertools.product(range(3), repeat=n))\n",
330 | " support = lambda v: tuple(i for i in range(n) if v[i] != 0)\n",
331 | " reflections = lambda v: sum(1 for i in range(1, n // 2) if v[i] == v[-i])\n",
332 | "\n",
333 | " # Add all 128 weight-8 vectors that have \u003e= 2 reflections.\n",
334 | " weight8_vectors = [v for v in V\n",
335 | " if len(support(v)) == 8 # Weight is 8.\n",
336 | " and reflections(v) \u003e= 2] # At least 2 reflections.\n",
337 | "\n",
338 | " # Add all 128 weight-4 vectors that have specific support.\n",
339 | " supports_16 = [(0, 1, 2, 3), (0, 1, 2, 5), (0, 3, 6, 7), (0, 5, 6, 7),\n",
340 | " (1, 3, 4, 6), (1, 4, 5, 6), (2, 3, 4, 7), (2, 4, 5, 7)]\n",
341 | " weight4_vectors = [v for v in V\n",
342 | " if support(v) in supports_16]\n",
343 | "\n",
344 | " # Add all 128 weight-4 vectors with specific support and 1 reflection.\n",
345 | " supports_8 = [(0, 1, 2, 7), (0, 1, 2, 6), (0, 1, 3, 7), (0, 1, 6, 7),\n",
346 | " (0, 1, 5, 7), (0, 2, 3, 6), (0, 2, 6, 7), (0, 2, 5, 6),\n",
347 | " (1, 2, 4, 7), (1, 2, 4, 6), (1, 3, 4, 7), (1, 4, 6, 7),\n",
348 | " (1, 4, 5, 7), (2, 3, 4, 6), (2, 4, 6, 7), (2, 4, 5, 6)]\n",
349 | " weight4_vectors_2 = [v for v in V\n",
350 | " if support(v) in supports_8\n",
351 | " and reflections(v) == 1] # Exactly 1 reflection.\n",
352 | "\n",
353 | " # Add 128 weight-5 vectors with \u003c= 1 reflections and one more condition.\n",
354 | " allowed_zeros = [(0, 4, 7), (0, 2, 4), (0, 1, 4), (0, 4, 6),\n",
355 | " (1, 2, 6), (2, 6, 7), (1, 2, 7), (1, 6, 7)]\n",
356 | " weight5_vectors = [\n",
357 | " v for v in V\n",
358 | " if tuple(i for i in range(n) if v[i] == 0) in allowed_zeros\n",
359 | " and reflections(v) \u003c= 1 # At most 1 reflection.\n",
360 | " and (v[1] * v[7]) % 3 != 1 and (v[2] * v[6]) % 3 != 1]\n",
361 | "\n",
362 | " return weight8_vectors + weight4_vectors + weight4_vectors_2 + weight5_vectors\n",
363 | "\n",
364 | "\n",
365 | "explicit = np.array(build_512_cap(), dtype=np.int32)\n",
366 | "assert explicit.shape == (512, 8)\n",
367 | "assert is_cap_set(explicit)\n",
368 | "# The explicit construction builds the same cap set as a set (i.e. up to\n",
369 | "# permutation of rows).\n",
370 | "assert set(map(tuple, explicit)) == set(map(tuple, cap_set_n8))"
371 | ]
372 | },
373 | {
374 | "cell_type": "markdown",
375 | "metadata": {
376 | "id": "jKTDz_ChifB8"
377 | },
378 | "source": [
379 | "## Discovered function that builds a $1082$-cap in $n = 9$ dimensions\n",
380 | "\n",
381 | "This matches the previously known best construction, which involves a mathematical argument utilising a special kind of product construction. Comments in the code were added by us."
382 | ]
383 | },
384 | {
385 | "cell_type": "code",
386 | "execution_count": null,
387 | "metadata": {
388 | "id": "uOz3hX0YiWKd"
389 | },
390 | "outputs": [],
391 | "source": [
392 | "def priority(el: tuple[int, ...], n: int) -\u003e float:\n",
393 | " el = np.array(el, dtype=np.float32)\n",
394 | " weight = (el @ el) % 3 # Weight (mod 3) of the full vector.\n",
395 | " a = n // 3\n",
396 | " b = n - n // 3\n",
397 | " s_1 = (el[:b] @ el[:b]) % 3 # Weight (mod 3) of first two thirds.\n",
398 | " s_3 = (2 * (el[:a] @ el[:a])) % 3 # Double norm of first third.\n",
399 | " s_4 = (el[:a] @ el[a:b]) % 3 # Cross correlation.\n",
400 | " s_5 = np.sum(el[:a] == el[-1]) % 3\n",
401 | " return - 3 ** 3 * s_1 + 3 ** 2 * weight + 3 ** 3 * s_3 + 3 ** 2 * s_4 + s_5\n",
402 | "\n",
403 | "\n",
404 | "n = 9\n",
405 | "cap_set_n9 = solve(n)\n",
406 | "assert cap_set_n9.shape == (1082, 9)\n",
407 | "assert is_cap_set(cap_set_n9)"
408 | ]
409 | }
410 | ],
411 | "metadata": {
412 | "colab": {
413 | "provenance": []
414 | },
415 | "kernelspec": {
416 | "display_name": "Python 3",
417 | "name": "python3"
418 | },
419 | "language_info": {
420 | "name": "python"
421 | }
422 | },
423 | "nbformat": 4,
424 | "nbformat_minor": 0
425 | }
426 |
--------------------------------------------------------------------------------
/examples/cap_set/n8_size512.txt:
--------------------------------------------------------------------------------
1 | [1, 1, 1, 1, 1, 1, 1, 1]
2 | [1, 1, 1, 1, 2, 1, 1, 1]
3 | [1, 1, 1, 2, 1, 2, 1, 1]
4 | [1, 1, 1, 2, 2, 2, 1, 1]
5 | [1, 1, 2, 1, 1, 1, 2, 1]
6 | [1, 1, 2, 1, 2, 1, 2, 1]
7 | [1, 1, 2, 2, 1, 2, 2, 1]
8 | [1, 1, 2, 2, 2, 2, 2, 1]
9 | [1, 2, 1, 1, 1, 1, 1, 2]
10 | [1, 2, 1, 1, 2, 1, 1, 2]
11 | [1, 2, 1, 2, 1, 2, 1, 2]
12 | [1, 2, 1, 2, 2, 2, 1, 2]
13 | [1, 2, 2, 1, 1, 1, 2, 2]
14 | [1, 2, 2, 1, 2, 1, 2, 2]
15 | [1, 2, 2, 2, 1, 2, 2, 2]
16 | [1, 2, 2, 2, 2, 2, 2, 2]
17 | [2, 1, 1, 1, 1, 1, 1, 1]
18 | [2, 1, 1, 1, 2, 1, 1, 1]
19 | [2, 1, 1, 2, 1, 2, 1, 1]
20 | [2, 1, 1, 2, 2, 2, 1, 1]
21 | [2, 1, 2, 1, 1, 1, 2, 1]
22 | [2, 1, 2, 1, 2, 1, 2, 1]
23 | [2, 1, 2, 2, 1, 2, 2, 1]
24 | [2, 1, 2, 2, 2, 2, 2, 1]
25 | [2, 2, 1, 1, 1, 1, 1, 2]
26 | [2, 2, 1, 1, 2, 1, 1, 2]
27 | [2, 2, 1, 2, 1, 2, 1, 2]
28 | [2, 2, 1, 2, 2, 2, 1, 2]
29 | [2, 2, 2, 1, 1, 1, 2, 2]
30 | [2, 2, 2, 1, 2, 1, 2, 2]
31 | [2, 2, 2, 2, 1, 2, 2, 2]
32 | [2, 2, 2, 2, 2, 2, 2, 2]
33 | [1, 1, 1, 1, 1, 2, 1, 1]
34 | [1, 1, 1, 1, 2, 2, 1, 1]
35 | [1, 1, 1, 2, 1, 1, 1, 1]
36 | [1, 1, 1, 2, 2, 1, 1, 1]
37 | [1, 1, 2, 1, 1, 2, 2, 1]
38 | [1, 1, 2, 1, 2, 2, 2, 1]
39 | [1, 1, 2, 2, 1, 1, 2, 1]
40 | [1, 1, 2, 2, 2, 1, 2, 1]
41 | [1, 2, 1, 1, 1, 2, 1, 2]
42 | [1, 2, 1, 1, 2, 2, 1, 2]
43 | [1, 2, 1, 2, 1, 1, 1, 2]
44 | [1, 2, 1, 2, 2, 1, 1, 2]
45 | [1, 2, 2, 1, 1, 2, 2, 2]
46 | [1, 2, 2, 1, 2, 2, 2, 2]
47 | [1, 2, 2, 2, 1, 1, 2, 2]
48 | [1, 2, 2, 2, 2, 1, 2, 2]
49 | [2, 1, 1, 1, 1, 2, 1, 1]
50 | [2, 1, 1, 1, 2, 2, 1, 1]
51 | [2, 1, 1, 2, 1, 1, 1, 1]
52 | [2, 1, 1, 2, 2, 1, 1, 1]
53 | [2, 1, 2, 1, 1, 2, 2, 1]
54 | [2, 1, 2, 1, 2, 2, 2, 1]
55 | [2, 1, 2, 2, 1, 1, 2, 1]
56 | [2, 1, 2, 2, 2, 1, 2, 1]
57 | [2, 2, 1, 1, 1, 2, 1, 2]
58 | [2, 2, 1, 1, 2, 2, 1, 2]
59 | [2, 2, 1, 2, 1, 1, 1, 2]
60 | [2, 2, 1, 2, 2, 1, 1, 2]
61 | [2, 2, 2, 1, 1, 2, 2, 2]
62 | [2, 2, 2, 1, 2, 2, 2, 2]
63 | [2, 2, 2, 2, 1, 1, 2, 2]
64 | [2, 2, 2, 2, 2, 1, 2, 2]
65 | [1, 1, 1, 1, 1, 1, 1, 2]
66 | [1, 1, 1, 1, 1, 1, 2, 1]
67 | [1, 1, 1, 1, 2, 1, 1, 2]
68 | [1, 1, 1, 1, 2, 1, 2, 1]
69 | [1, 1, 1, 2, 1, 2, 1, 2]
70 | [1, 1, 1, 2, 1, 2, 2, 1]
71 | [1, 1, 1, 2, 2, 2, 1, 2]
72 | [1, 1, 1, 2, 2, 2, 2, 1]
73 | [1, 1, 2, 1, 1, 1, 1, 1]
74 | [1, 1, 2, 1, 1, 1, 2, 2]
75 | [1, 1, 2, 1, 2, 1, 1, 1]
76 | [1, 1, 2, 1, 2, 1, 2, 2]
77 | [1, 1, 2, 2, 1, 2, 1, 1]
78 | [1, 1, 2, 2, 1, 2, 2, 2]
79 | [1, 1, 2, 2, 2, 2, 1, 1]
80 | [1, 1, 2, 2, 2, 2, 2, 2]
81 | [1, 2, 1, 1, 1, 1, 1, 1]
82 | [1, 2, 1, 1, 1, 1, 2, 2]
83 | [1, 2, 1, 1, 2, 1, 1, 1]
84 | [1, 2, 1, 1, 2, 1, 2, 2]
85 | [1, 2, 1, 2, 1, 2, 1, 1]
86 | [1, 2, 1, 2, 1, 2, 2, 2]
87 | [1, 2, 1, 2, 2, 2, 1, 1]
88 | [1, 2, 1, 2, 2, 2, 2, 2]
89 | [1, 2, 2, 1, 1, 1, 1, 2]
90 | [1, 2, 2, 1, 1, 1, 2, 1]
91 | [1, 2, 2, 1, 2, 1, 1, 2]
92 | [1, 2, 2, 1, 2, 1, 2, 1]
93 | [1, 2, 2, 2, 1, 2, 1, 2]
94 | [1, 2, 2, 2, 1, 2, 2, 1]
95 | [1, 2, 2, 2, 2, 2, 1, 2]
96 | [1, 2, 2, 2, 2, 2, 2, 1]
97 | [2, 1, 1, 1, 1, 1, 1, 2]
98 | [2, 1, 1, 1, 1, 1, 2, 1]
99 | [2, 1, 1, 1, 2, 1, 1, 2]
100 | [2, 1, 1, 1, 2, 1, 2, 1]
101 | [2, 1, 1, 2, 1, 2, 1, 2]
102 | [2, 1, 1, 2, 1, 2, 2, 1]
103 | [2, 1, 1, 2, 2, 2, 1, 2]
104 | [2, 1, 1, 2, 2, 2, 2, 1]
105 | [2, 1, 2, 1, 1, 1, 1, 1]
106 | [2, 1, 2, 1, 1, 1, 2, 2]
107 | [2, 1, 2, 1, 2, 1, 1, 1]
108 | [2, 1, 2, 1, 2, 1, 2, 2]
109 | [2, 1, 2, 2, 1, 2, 1, 1]
110 | [2, 1, 2, 2, 1, 2, 2, 2]
111 | [2, 1, 2, 2, 2, 2, 1, 1]
112 | [2, 1, 2, 2, 2, 2, 2, 2]
113 | [2, 2, 1, 1, 1, 1, 1, 1]
114 | [2, 2, 1, 1, 1, 1, 2, 2]
115 | [2, 2, 1, 1, 2, 1, 1, 1]
116 | [2, 2, 1, 1, 2, 1, 2, 2]
117 | [2, 2, 1, 2, 1, 2, 1, 1]
118 | [2, 2, 1, 2, 1, 2, 2, 2]
119 | [2, 2, 1, 2, 2, 2, 1, 1]
120 | [2, 2, 1, 2, 2, 2, 2, 2]
121 | [2, 2, 2, 1, 1, 1, 1, 2]
122 | [2, 2, 2, 1, 1, 1, 2, 1]
123 | [2, 2, 2, 1, 2, 1, 1, 2]
124 | [2, 2, 2, 1, 2, 1, 2, 1]
125 | [2, 2, 2, 2, 1, 2, 1, 2]
126 | [2, 2, 2, 2, 1, 2, 2, 1]
127 | [2, 2, 2, 2, 2, 2, 1, 2]
128 | [2, 2, 2, 2, 2, 2, 2, 1]
129 | [1, 1, 1, 1, 0, 0, 0, 0]
130 | [1, 1, 1, 2, 0, 0, 0, 0]
131 | [1, 1, 2, 1, 0, 0, 0, 0]
132 | [1, 1, 2, 2, 0, 0, 0, 0]
133 | [1, 2, 1, 1, 0, 0, 0, 0]
134 | [1, 2, 1, 2, 0, 0, 0, 0]
135 | [1, 2, 2, 1, 0, 0, 0, 0]
136 | [1, 2, 2, 2, 0, 0, 0, 0]
137 | [2, 1, 1, 1, 0, 0, 0, 0]
138 | [2, 1, 1, 2, 0, 0, 0, 0]
139 | [2, 1, 2, 1, 0, 0, 0, 0]
140 | [2, 1, 2, 2, 0, 0, 0, 0]
141 | [2, 2, 1, 1, 0, 0, 0, 0]
142 | [2, 2, 1, 2, 0, 0, 0, 0]
143 | [2, 2, 2, 1, 0, 0, 0, 0]
144 | [2, 2, 2, 2, 0, 0, 0, 0]
145 | [1, 1, 1, 0, 0, 0, 0, 2]
146 | [1, 1, 1, 0, 0, 1, 0, 0]
147 | [1, 1, 1, 0, 0, 2, 0, 0]
148 | [1, 1, 2, 0, 0, 0, 0, 2]
149 | [1, 1, 2, 0, 0, 1, 0, 0]
150 | [1, 1, 2, 0, 0, 2, 0, 0]
151 | [1, 2, 1, 0, 0, 0, 0, 1]
152 | [1, 2, 1, 0, 0, 1, 0, 0]
153 | [1, 2, 1, 0, 0, 2, 0, 0]
154 | [1, 2, 2, 0, 0, 0, 0, 1]
155 | [1, 2, 2, 0, 0, 1, 0, 0]
156 | [1, 2, 2, 0, 0, 2, 0, 0]
157 | [2, 1, 1, 0, 0, 0, 0, 2]
158 | [2, 1, 1, 0, 0, 1, 0, 0]
159 | [2, 1, 1, 0, 0, 2, 0, 0]
160 | [2, 1, 2, 0, 0, 0, 0, 2]
161 | [2, 1, 2, 0, 0, 1, 0, 0]
162 | [2, 1, 2, 0, 0, 2, 0, 0]
163 | [2, 2, 1, 0, 0, 0, 0, 1]
164 | [2, 2, 1, 0, 0, 1, 0, 0]
165 | [2, 2, 1, 0, 0, 2, 0, 0]
166 | [2, 2, 2, 0, 0, 0, 0, 1]
167 | [2, 2, 2, 0, 0, 1, 0, 0]
168 | [2, 2, 2, 0, 0, 2, 0, 0]
169 | [1, 1, 1, 0, 0, 0, 2, 0]
170 | [1, 1, 2, 0, 0, 0, 1, 0]
171 | [1, 2, 1, 0, 0, 0, 2, 0]
172 | [1, 2, 2, 0, 0, 0, 1, 0]
173 | [2, 1, 1, 0, 0, 0, 2, 0]
174 | [2, 1, 2, 0, 0, 0, 1, 0]
175 | [2, 2, 1, 0, 0, 0, 2, 0]
176 | [2, 2, 2, 0, 0, 0, 1, 0]
177 | [1, 1, 0, 0, 0, 0, 1, 2]
178 | [1, 1, 0, 0, 0, 0, 2, 2]
179 | [1, 2, 0, 0, 0, 0, 1, 1]
180 | [1, 2, 0, 0, 0, 0, 2, 1]
181 | [2, 1, 0, 0, 0, 0, 1, 2]
182 | [2, 1, 0, 0, 0, 0, 2, 2]
183 | [2, 2, 0, 0, 0, 0, 1, 1]
184 | [2, 2, 0, 0, 0, 0, 2, 1]
185 | [1, 1, 0, 1, 0, 0, 0, 2]
186 | [1, 1, 0, 2, 0, 0, 0, 2]
187 | [1, 2, 0, 1, 0, 0, 0, 1]
188 | [1, 2, 0, 2, 0, 0, 0, 1]
189 | [2, 1, 0, 1, 0, 0, 0, 2]
190 | [2, 1, 0, 2, 0, 0, 0, 2]
191 | [2, 2, 0, 1, 0, 0, 0, 1]
192 | [2, 2, 0, 2, 0, 0, 0, 1]
193 | [1, 0, 1, 0, 0, 0, 2, 1]
194 | [1, 0, 1, 0, 0, 0, 2, 2]
195 | [1, 0, 2, 0, 0, 0, 1, 1]
196 | [1, 0, 2, 0, 0, 0, 1, 2]
197 | [2, 0, 1, 0, 0, 0, 2, 1]
198 | [2, 0, 1, 0, 0, 0, 2, 2]
199 | [2, 0, 2, 0, 0, 0, 1, 1]
200 | [2, 0, 2, 0, 0, 0, 1, 2]
201 | [1, 0, 0, 0, 0, 1, 1, 1]
202 | [1, 0, 0, 0, 0, 1, 1, 2]
203 | [1, 0, 0, 0, 0, 1, 2, 1]
204 | [1, 0, 0, 0, 0, 1, 2, 2]
205 | [1, 0, 0, 0, 0, 2, 1, 1]
206 | [1, 0, 0, 0, 0, 2, 1, 2]
207 | [1, 0, 0, 0, 0, 2, 2, 1]
208 | [1, 0, 0, 0, 0, 2, 2, 2]
209 | [1, 0, 0, 1, 0, 0, 1, 1]
210 | [1, 0, 0, 1, 0, 0, 1, 2]
211 | [1, 0, 0, 1, 0, 0, 2, 1]
212 | [1, 0, 0, 1, 0, 0, 2, 2]
213 | [1, 0, 0, 2, 0, 0, 1, 1]
214 | [1, 0, 0, 2, 0, 0, 1, 2]
215 | [1, 0, 0, 2, 0, 0, 2, 1]
216 | [1, 0, 0, 2, 0, 0, 2, 2]
217 | [2, 0, 0, 0, 0, 1, 1, 1]
218 | [2, 0, 0, 0, 0, 1, 1, 2]
219 | [2, 0, 0, 0, 0, 1, 2, 1]
220 | [2, 0, 0, 0, 0, 1, 2, 2]
221 | [2, 0, 0, 0, 0, 2, 1, 1]
222 | [2, 0, 0, 0, 0, 2, 1, 2]
223 | [2, 0, 0, 0, 0, 2, 2, 1]
224 | [2, 0, 0, 0, 0, 2, 2, 2]
225 | [2, 0, 0, 1, 0, 0, 1, 1]
226 | [2, 0, 0, 1, 0, 0, 1, 2]
227 | [2, 0, 0, 1, 0, 0, 2, 1]
228 | [2, 0, 0, 1, 0, 0, 2, 2]
229 | [2, 0, 0, 2, 0, 0, 1, 1]
230 | [2, 0, 0, 2, 0, 0, 1, 2]
231 | [2, 0, 0, 2, 0, 0, 2, 1]
232 | [2, 0, 0, 2, 0, 0, 2, 2]
233 | [1, 1, 0, 0, 0, 1, 0, 2]
234 | [1, 1, 0, 0, 0, 2, 0, 2]
235 | [1, 2, 0, 0, 0, 1, 0, 1]
236 | [1, 2, 0, 0, 0, 2, 0, 1]
237 | [2, 1, 0, 0, 0, 1, 0, 2]
238 | [2, 1, 0, 0, 0, 2, 0, 2]
239 | [2, 2, 0, 0, 0, 1, 0, 1]
240 | [2, 2, 0, 0, 0, 2, 0, 1]
241 | [0, 1, 1, 0, 1, 0, 0, 2]
242 | [0, 1, 1, 0, 2, 0, 0, 2]
243 | [0, 1, 2, 0, 1, 0, 0, 2]
244 | [0, 1, 2, 0, 2, 0, 0, 2]
245 | [0, 2, 1, 0, 1, 0, 0, 1]
246 | [0, 2, 1, 0, 2, 0, 0, 1]
247 | [0, 2, 2, 0, 1, 0, 0, 1]
248 | [0, 2, 2, 0, 2, 0, 0, 1]
249 | [0, 1, 1, 0, 1, 0, 2, 0]
250 | [0, 1, 1, 0, 2, 0, 2, 0]
251 | [0, 1, 2, 0, 1, 0, 1, 0]
252 | [0, 1, 2, 0, 2, 0, 1, 0]
253 | [0, 2, 1, 0, 1, 0, 2, 0]
254 | [0, 2, 1, 0, 2, 0, 2, 0]
255 | [0, 2, 2, 0, 1, 0, 1, 0]
256 | [0, 2, 2, 0, 2, 0, 1, 0]
257 | [0, 1, 0, 0, 1, 0, 1, 2]
258 | [0, 1, 0, 0, 1, 0, 2, 2]
259 | [0, 1, 0, 0, 2, 0, 1, 2]
260 | [0, 1, 0, 0, 2, 0, 2, 2]
261 | [0, 1, 0, 1, 1, 0, 1, 0]
262 | [0, 1, 0, 1, 1, 0, 2, 0]
263 | [0, 1, 0, 1, 2, 0, 1, 0]
264 | [0, 1, 0, 1, 2, 0, 2, 0]
265 | [0, 1, 0, 2, 1, 0, 1, 0]
266 | [0, 1, 0, 2, 1, 0, 2, 0]
267 | [0, 1, 0, 2, 2, 0, 1, 0]
268 | [0, 1, 0, 2, 2, 0, 2, 0]
269 | [0, 2, 0, 0, 1, 0, 1, 1]
270 | [0, 2, 0, 0, 1, 0, 2, 1]
271 | [0, 2, 0, 0, 2, 0, 1, 1]
272 | [0, 2, 0, 0, 2, 0, 2, 1]
273 | [0, 2, 0, 1, 1, 0, 1, 0]
274 | [0, 2, 0, 1, 1, 0, 2, 0]
275 | [0, 2, 0, 1, 2, 0, 1, 0]
276 | [0, 2, 0, 1, 2, 0, 2, 0]
277 | [0, 2, 0, 2, 1, 0, 1, 0]
278 | [0, 2, 0, 2, 1, 0, 2, 0]
279 | [0, 2, 0, 2, 2, 0, 1, 0]
280 | [0, 2, 0, 2, 2, 0, 2, 0]
281 | [0, 1, 0, 0, 1, 1, 1, 0]
282 | [0, 1, 0, 0, 1, 1, 2, 0]
283 | [0, 1, 0, 0, 1, 2, 1, 0]
284 | [0, 1, 0, 0, 1, 2, 2, 0]
285 | [0, 1, 0, 0, 2, 1, 1, 0]
286 | [0, 1, 0, 0, 2, 1, 2, 0]
287 | [0, 1, 0, 0, 2, 2, 1, 0]
288 | [0, 1, 0, 0, 2, 2, 2, 0]
289 | [0, 2, 0, 0, 1, 1, 1, 0]
290 | [0, 2, 0, 0, 1, 1, 2, 0]
291 | [0, 2, 0, 0, 1, 2, 1, 0]
292 | [0, 2, 0, 0, 1, 2, 2, 0]
293 | [0, 2, 0, 0, 2, 1, 1, 0]
294 | [0, 2, 0, 0, 2, 1, 2, 0]
295 | [0, 2, 0, 0, 2, 2, 1, 0]
296 | [0, 2, 0, 0, 2, 2, 2, 0]
297 | [1, 0, 1, 1, 0, 0, 2, 0]
298 | [1, 0, 1, 2, 0, 0, 2, 0]
299 | [1, 0, 2, 1, 0, 0, 1, 0]
300 | [1, 0, 2, 2, 0, 0, 1, 0]
301 | [2, 0, 1, 1, 0, 0, 2, 0]
302 | [2, 0, 1, 2, 0, 0, 2, 0]
303 | [2, 0, 2, 1, 0, 0, 1, 0]
304 | [2, 0, 2, 2, 0, 0, 1, 0]
305 | [0, 0, 1, 1, 1, 0, 0, 1]
306 | [0, 0, 1, 1, 1, 0, 0, 2]
307 | [0, 0, 1, 1, 2, 0, 0, 1]
308 | [0, 0, 1, 1, 2, 0, 0, 2]
309 | [0, 0, 1, 2, 1, 0, 0, 1]
310 | [0, 0, 1, 2, 1, 0, 0, 2]
311 | [0, 0, 1, 2, 2, 0, 0, 1]
312 | [0, 0, 1, 2, 2, 0, 0, 2]
313 | [0, 0, 2, 1, 1, 0, 0, 1]
314 | [0, 0, 2, 1, 1, 0, 0, 2]
315 | [0, 0, 2, 1, 2, 0, 0, 1]
316 | [0, 0, 2, 1, 2, 0, 0, 2]
317 | [0, 0, 2, 2, 1, 0, 0, 1]
318 | [0, 0, 2, 2, 1, 0, 0, 2]
319 | [0, 0, 2, 2, 2, 0, 0, 1]
320 | [0, 0, 2, 2, 2, 0, 0, 2]
321 | [0, 0, 1, 0, 1, 0, 2, 1]
322 | [0, 0, 1, 0, 1, 0, 2, 2]
323 | [0, 0, 1, 0, 2, 0, 2, 1]
324 | [0, 0, 1, 0, 2, 0, 2, 2]
325 | [0, 0, 2, 0, 1, 0, 1, 1]
326 | [0, 0, 2, 0, 1, 0, 1, 2]
327 | [0, 0, 2, 0, 2, 0, 1, 1]
328 | [0, 0, 2, 0, 2, 0, 1, 2]
329 | [0, 0, 1, 0, 1, 1, 0, 1]
330 | [0, 0, 1, 0, 1, 1, 0, 2]
331 | [0, 0, 1, 0, 1, 2, 0, 1]
332 | [0, 0, 1, 0, 1, 2, 0, 2]
333 | [0, 0, 1, 0, 2, 1, 0, 1]
334 | [0, 0, 1, 0, 2, 1, 0, 2]
335 | [0, 0, 1, 0, 2, 2, 0, 1]
336 | [0, 0, 1, 0, 2, 2, 0, 2]
337 | [0, 0, 2, 0, 1, 1, 0, 1]
338 | [0, 0, 2, 0, 1, 1, 0, 2]
339 | [0, 0, 2, 0, 1, 2, 0, 1]
340 | [0, 0, 2, 0, 1, 2, 0, 2]
341 | [0, 0, 2, 0, 2, 1, 0, 1]
342 | [0, 0, 2, 0, 2, 1, 0, 2]
343 | [0, 0, 2, 0, 2, 2, 0, 1]
344 | [0, 0, 2, 0, 2, 2, 0, 2]
345 | [1, 0, 1, 0, 0, 1, 2, 0]
346 | [1, 0, 1, 0, 0, 2, 2, 0]
347 | [1, 0, 2, 0, 0, 1, 1, 0]
348 | [1, 0, 2, 0, 0, 2, 1, 0]
349 | [2, 0, 1, 0, 0, 1, 2, 0]
350 | [2, 0, 1, 0, 0, 2, 2, 0]
351 | [2, 0, 2, 0, 0, 1, 1, 0]
352 | [2, 0, 2, 0, 0, 2, 1, 0]
353 | [0, 1, 0, 1, 1, 0, 0, 2]
354 | [0, 1, 0, 1, 2, 0, 0, 2]
355 | [0, 1, 0, 2, 1, 0, 0, 2]
356 | [0, 1, 0, 2, 2, 0, 0, 2]
357 | [0, 2, 0, 1, 1, 0, 0, 1]
358 | [0, 2, 0, 1, 2, 0, 0, 1]
359 | [0, 2, 0, 2, 1, 0, 0, 1]
360 | [0, 2, 0, 2, 2, 0, 0, 1]
361 | [0, 1, 0, 0, 1, 1, 0, 2]
362 | [0, 1, 0, 0, 1, 2, 0, 2]
363 | [0, 1, 0, 0, 2, 1, 0, 2]
364 | [0, 1, 0, 0, 2, 2, 0, 2]
365 | [0, 2, 0, 0, 1, 1, 0, 1]
366 | [0, 2, 0, 0, 1, 2, 0, 1]
367 | [0, 2, 0, 0, 2, 1, 0, 1]
368 | [0, 2, 0, 0, 2, 2, 0, 1]
369 | [0, 0, 1, 1, 1, 0, 2, 0]
370 | [0, 0, 1, 1, 2, 0, 2, 0]
371 | [0, 0, 1, 2, 1, 0, 2, 0]
372 | [0, 0, 1, 2, 2, 0, 2, 0]
373 | [0, 0, 2, 1, 1, 0, 1, 0]
374 | [0, 0, 2, 1, 2, 0, 1, 0]
375 | [0, 0, 2, 2, 1, 0, 1, 0]
376 | [0, 0, 2, 2, 2, 0, 1, 0]
377 | [0, 0, 1, 0, 1, 1, 2, 0]
378 | [0, 0, 1, 0, 1, 2, 2, 0]
379 | [0, 0, 1, 0, 2, 1, 2, 0]
380 | [0, 0, 1, 0, 2, 2, 2, 0]
381 | [0, 0, 2, 0, 1, 1, 1, 0]
382 | [0, 0, 2, 0, 1, 2, 1, 0]
383 | [0, 0, 2, 0, 2, 1, 1, 0]
384 | [0, 0, 2, 0, 2, 2, 1, 0]
385 | [1, 1, 0, 1, 1, 2, 0, 0]
386 | [1, 1, 0, 1, 2, 2, 0, 0]
387 | [1, 1, 0, 2, 1, 1, 0, 0]
388 | [1, 1, 0, 2, 2, 1, 0, 0]
389 | [1, 2, 0, 1, 1, 2, 0, 0]
390 | [1, 2, 0, 1, 2, 2, 0, 0]
391 | [1, 2, 0, 2, 1, 1, 0, 0]
392 | [1, 2, 0, 2, 2, 1, 0, 0]
393 | [2, 1, 0, 1, 1, 2, 0, 0]
394 | [2, 1, 0, 1, 2, 2, 0, 0]
395 | [2, 1, 0, 2, 1, 1, 0, 0]
396 | [2, 1, 0, 2, 2, 1, 0, 0]
397 | [2, 2, 0, 1, 1, 2, 0, 0]
398 | [2, 2, 0, 1, 2, 2, 0, 0]
399 | [2, 2, 0, 2, 1, 1, 0, 0]
400 | [2, 2, 0, 2, 2, 1, 0, 0]
401 | [1, 0, 1, 1, 1, 2, 0, 0]
402 | [1, 0, 1, 1, 2, 2, 0, 0]
403 | [1, 0, 1, 2, 1, 1, 0, 0]
404 | [1, 0, 1, 2, 2, 1, 0, 0]
405 | [1, 0, 2, 1, 1, 2, 0, 0]
406 | [1, 0, 2, 1, 2, 2, 0, 0]
407 | [1, 0, 2, 2, 1, 1, 0, 0]
408 | [1, 0, 2, 2, 2, 1, 0, 0]
409 | [2, 0, 1, 1, 1, 2, 0, 0]
410 | [2, 0, 1, 1, 2, 2, 0, 0]
411 | [2, 0, 1, 2, 1, 1, 0, 0]
412 | [2, 0, 1, 2, 2, 1, 0, 0]
413 | [2, 0, 2, 1, 1, 2, 0, 0]
414 | [2, 0, 2, 1, 2, 2, 0, 0]
415 | [2, 0, 2, 2, 1, 1, 0, 0]
416 | [2, 0, 2, 2, 2, 1, 0, 0]
417 | [0, 1, 1, 1, 0, 1, 0, 2]
418 | [0, 1, 1, 1, 0, 2, 0, 2]
419 | [0, 1, 1, 2, 0, 1, 0, 2]
420 | [0, 1, 1, 2, 0, 2, 0, 2]
421 | [0, 1, 2, 1, 0, 1, 0, 2]
422 | [0, 1, 2, 1, 0, 2, 0, 2]
423 | [0, 1, 2, 2, 0, 1, 0, 2]
424 | [0, 1, 2, 2, 0, 2, 0, 2]
425 | [0, 2, 1, 1, 0, 1, 0, 1]
426 | [0, 2, 1, 1, 0, 2, 0, 1]
427 | [0, 2, 1, 2, 0, 1, 0, 1]
428 | [0, 2, 1, 2, 0, 2, 0, 1]
429 | [0, 2, 2, 1, 0, 1, 0, 1]
430 | [0, 2, 2, 1, 0, 2, 0, 1]
431 | [0, 2, 2, 2, 0, 1, 0, 1]
432 | [0, 2, 2, 2, 0, 2, 0, 1]
433 | [0, 1, 1, 1, 0, 1, 2, 0]
434 | [0, 1, 1, 1, 0, 2, 2, 0]
435 | [0, 1, 1, 2, 0, 1, 2, 0]
436 | [0, 1, 1, 2, 0, 2, 2, 0]
437 | [0, 1, 2, 1, 0, 1, 1, 0]
438 | [0, 1, 2, 1, 0, 2, 1, 0]
439 | [0, 1, 2, 2, 0, 1, 1, 0]
440 | [0, 1, 2, 2, 0, 2, 1, 0]
441 | [0, 2, 1, 1, 0, 1, 2, 0]
442 | [0, 2, 1, 1, 0, 2, 2, 0]
443 | [0, 2, 1, 2, 0, 1, 2, 0]
444 | [0, 2, 1, 2, 0, 2, 2, 0]
445 | [0, 2, 2, 1, 0, 1, 1, 0]
446 | [0, 2, 2, 1, 0, 2, 1, 0]
447 | [0, 2, 2, 2, 0, 1, 1, 0]
448 | [0, 2, 2, 2, 0, 2, 1, 0]
449 | [0, 1, 0, 1, 0, 1, 1, 2]
450 | [0, 1, 0, 1, 0, 1, 2, 2]
451 | [0, 1, 0, 1, 0, 2, 1, 2]
452 | [0, 1, 0, 1, 0, 2, 2, 2]
453 | [0, 1, 0, 2, 0, 1, 1, 2]
454 | [0, 1, 0, 2, 0, 1, 2, 2]
455 | [0, 1, 0, 2, 0, 2, 1, 2]
456 | [0, 1, 0, 2, 0, 2, 2, 2]
457 | [0, 2, 0, 1, 0, 1, 1, 1]
458 | [0, 2, 0, 1, 0, 1, 2, 1]
459 | [0, 2, 0, 1, 0, 2, 1, 1]
460 | [0, 2, 0, 1, 0, 2, 2, 1]
461 | [0, 2, 0, 2, 0, 1, 1, 1]
462 | [0, 2, 0, 2, 0, 1, 2, 1]
463 | [0, 2, 0, 2, 0, 2, 1, 1]
464 | [0, 2, 0, 2, 0, 2, 2, 1]
465 | [0, 0, 1, 1, 0, 1, 2, 1]
466 | [0, 0, 1, 1, 0, 1, 2, 2]
467 | [0, 0, 1, 1, 0, 2, 2, 1]
468 | [0, 0, 1, 1, 0, 2, 2, 2]
469 | [0, 0, 1, 2, 0, 1, 2, 1]
470 | [0, 0, 1, 2, 0, 1, 2, 2]
471 | [0, 0, 1, 2, 0, 2, 2, 1]
472 | [0, 0, 1, 2, 0, 2, 2, 2]
473 | [0, 0, 2, 1, 0, 1, 1, 1]
474 | [0, 0, 2, 1, 0, 1, 1, 2]
475 | [0, 0, 2, 1, 0, 2, 1, 1]
476 | [0, 0, 2, 1, 0, 2, 1, 2]
477 | [0, 0, 2, 2, 0, 1, 1, 1]
478 | [0, 0, 2, 2, 0, 1, 1, 2]
479 | [0, 0, 2, 2, 0, 2, 1, 1]
480 | [0, 0, 2, 2, 0, 2, 1, 2]
481 | [1, 0, 0, 1, 1, 2, 0, 1]
482 | [1, 0, 0, 1, 1, 2, 0, 2]
483 | [1, 0, 0, 1, 2, 2, 0, 1]
484 | [1, 0, 0, 1, 2, 2, 0, 2]
485 | [1, 0, 0, 2, 1, 1, 0, 1]
486 | [1, 0, 0, 2, 1, 1, 0, 2]
487 | [1, 0, 0, 2, 2, 1, 0, 1]
488 | [1, 0, 0, 2, 2, 1, 0, 2]
489 | [2, 0, 0, 1, 1, 2, 0, 1]
490 | [2, 0, 0, 1, 1, 2, 0, 2]
491 | [2, 0, 0, 1, 2, 2, 0, 1]
492 | [2, 0, 0, 1, 2, 2, 0, 2]
493 | [2, 0, 0, 2, 1, 1, 0, 1]
494 | [2, 0, 0, 2, 1, 1, 0, 2]
495 | [2, 0, 0, 2, 2, 1, 0, 1]
496 | [2, 0, 0, 2, 2, 1, 0, 2]
497 | [1, 0, 0, 1, 1, 2, 1, 0]
498 | [1, 0, 0, 1, 1, 2, 2, 0]
499 | [1, 0, 0, 1, 2, 2, 1, 0]
500 | [1, 0, 0, 1, 2, 2, 2, 0]
501 | [1, 0, 0, 2, 1, 1, 1, 0]
502 | [1, 0, 0, 2, 1, 1, 2, 0]
503 | [1, 0, 0, 2, 2, 1, 1, 0]
504 | [1, 0, 0, 2, 2, 1, 2, 0]
505 | [2, 0, 0, 1, 1, 2, 1, 0]
506 | [2, 0, 0, 1, 1, 2, 2, 0]
507 | [2, 0, 0, 1, 2, 2, 1, 0]
508 | [2, 0, 0, 1, 2, 2, 2, 0]
509 | [2, 0, 0, 2, 1, 1, 1, 0]
510 | [2, 0, 0, 2, 1, 1, 2, 0]
511 | [2, 0, 0, 2, 2, 1, 1, 0]
512 | [2, 0, 0, 2, 2, 1, 2, 0]
513 |
--------------------------------------------------------------------------------
/examples/cap_set_spec.py:
--------------------------------------------------------------------------------
1 | """Finds large cap sets.
2 |
3 | On every iteration, improve priority_v1 over the priority_vX methods from previous iterations.
4 | Make only small changes.
5 | Try to make the code short.
6 | """
7 | import itertools
8 |
9 | import numpy as np
10 |
11 | import funsearch
12 |
13 |
14 | @funsearch.run
15 | def evaluate(n: int) -> int:
16 | """Returns the size of an `n`-dimensional cap set."""
17 | capset = solve(n)
18 | return len(capset)
19 |
20 |
21 | def solve(n: int) -> np.ndarray:
22 | """Returns a large cap set in `n` dimensions."""
23 | all_vectors = np.array(list(itertools.product((0, 1, 2), repeat=n)), dtype=np.int32)
24 |
25 | # Powers in decreasing order for compatibility with `itertools.product`, so
26 | # that the relationship `i = all_vectors[i] @ powers` holds for all `i`.
27 | powers = 3 ** np.arange(n - 1, -1, -1)
28 |
29 | # Precompute all priorities.
30 | priorities = np.array([priority(tuple(vector), n) for vector in all_vectors])
31 |
32 | # Build `capset` greedily, using priorities for prioritization.
33 | capset = np.empty(shape=(0, n), dtype=np.int32)
34 | while np.any(priorities != -np.inf):
35 | # Add a vector with maximum priority to `capset`, and set priorities of
36 | # invalidated vectors to `-inf`, so that they never get selected.
37 | max_index = np.argmax(priorities)
38 | vector = all_vectors[None, max_index] # [1, n]
39 | blocking = np.einsum('cn,n->c', (- capset - vector) % 3, powers) # [C]
40 | priorities[blocking] = -np.inf
41 | priorities[max_index] = -np.inf
42 | capset = np.concatenate([capset, vector], axis=0)
43 |
44 | return capset
45 |
46 |
47 | @funsearch.evolve
48 | def priority(el: tuple[int, ...], n: int) -> float:
49 | """Returns the priority with which we want to add `element` to the cap set.
50 | el is a tuple of length n with values 0-2.
51 | """
52 | return 0.0
--------------------------------------------------------------------------------
/examples/corner_free_set/f2_n4_size137.txt:
--------------------------------------------------------------------------------
1 | [2, 1, 0, 1]
2 | [2, 1, 0, 3]
3 | [3, 3, 1, 1]
4 | [2, 1, 1, 3]
5 | [3, 3, 0, 1]
6 | [3, 3, 1, 3]
7 | [3, 3, 0, 3]
8 | [2, 1, 1, 1]
9 | [2, 2, 3, 1]
10 | [2, 0, 2, 3]
11 | [3, 2, 3, 1]
12 | [3, 2, 2, 3]
13 | [2, 2, 2, 3]
14 | [2, 0, 3, 1]
15 | [3, 0, 3, 1]
16 | [3, 0, 2, 3]
17 | [3, 3, 3, 3]
18 | [3, 1, 2, 3]
19 | [3, 1, 3, 1]
20 | [3, 3, 2, 1]
21 | [2, 1, 2, 1]
22 | [2, 3, 2, 3]
23 | [2, 3, 3, 1]
24 | [2, 1, 3, 3]
25 | [2, 2, 0, 3]
26 | [3, 0, 1, 1]
27 | [2, 2, 1, 1]
28 | [2, 2, 0, 1]
29 | [3, 0, 1, 3]
30 | [3, 0, 0, 3]
31 | [3, 2, 1, 1]
32 | [2, 0, 1, 3]
33 | [2, 0, 0, 3]
34 | [2, 0, 1, 1]
35 | [3, 2, 0, 3]
36 | [2, 0, 0, 1]
37 | [3, 0, 0, 1]
38 | [3, 2, 0, 1]
39 | [2, 2, 1, 3]
40 | [3, 2, 1, 3]
41 | [1, 0, 2, 0]
42 | [0, 2, 3, 2]
43 | [1, 0, 3, 2]
44 | [1, 0, 3, 0]
45 | [0, 2, 3, 0]
46 | [1, 0, 2, 2]
47 | [0, 2, 2, 2]
48 | [0, 2, 2, 0]
49 | [0, 3, 1, 2]
50 | [1, 1, 0, 0]
51 | [0, 3, 0, 0]
52 | [1, 1, 1, 2]
53 | [0, 1, 0, 0]
54 | [1, 3, 0, 0]
55 | [0, 1, 1, 2]
56 | [1, 3, 1, 2]
57 | [1, 1, 3, 1]
58 | [1, 1, 2, 3]
59 | [0, 1, 2, 3]
60 | [0, 3, 2, 3]
61 | [0, 1, 3, 1]
62 | [1, 3, 2, 3]
63 | [1, 3, 3, 1]
64 | [0, 3, 3, 1]
65 | [0, 3, 2, 0]
66 | [1, 3, 3, 2]
67 | [1, 3, 3, 0]
68 | [1, 1, 2, 0]
69 | [1, 1, 3, 0]
70 | [1, 1, 3, 2]
71 | [1, 1, 2, 2]
72 | [1, 3, 2, 2]
73 | [1, 3, 2, 0]
74 | [0, 1, 2, 0]
75 | [0, 1, 2, 2]
76 | [0, 1, 3, 0]
77 | [0, 1, 3, 2]
78 | [0, 3, 2, 2]
79 | [0, 3, 3, 0]
80 | [0, 3, 3, 2]
81 | [3, 3, 3, 2]
82 | [3, 3, 3, 0]
83 | [3, 3, 2, 2]
84 | [3, 3, 2, 0]
85 | [0, 0, 0, 0]
86 | [2, 1, 3, 0]
87 | [2, 1, 2, 0]
88 | [2, 1, 2, 2]
89 | [0, 2, 1, 1]
90 | [0, 2, 1, 0]
91 | [0, 2, 1, 3]
92 | [2, 1, 3, 2]
93 | [1, 0, 1, 0]
94 | [2, 2, 0, 0]
95 | [1, 0, 0, 3]
96 | [0, 2, 0, 3]
97 | [1, 0, 1, 1]
98 | [2, 0, 0, 0]
99 | [0, 2, 0, 2]
100 | [0, 2, 0, 1]
101 | [2, 0, 1, 2]
102 | [1, 0, 0, 2]
103 | [1, 0, 0, 1]
104 | [0, 0, 1, 2]
105 | [2, 2, 1, 2]
106 | [1, 0, 1, 3]
107 | [1, 2, 1, 2]
108 | [3, 0, 1, 2]
109 | [3, 2, 1, 2]
110 | [1, 2, 0, 0]
111 | [3, 0, 0, 0]
112 | [1, 3, 1, 3]
113 | [2, 0, 2, 2]
114 | [2, 0, 3, 0]
115 | [1, 3, 1, 1]
116 | [2, 0, 3, 2]
117 | [1, 3, 0, 3]
118 | [1, 2, 3, 1]
119 | [2, 2, 2, 0]
120 | [0, 0, 2, 3]
121 | [2, 2, 2, 2]
122 | [0, 1, 0, 1]
123 | [0, 1, 0, 3]
124 | [2, 2, 3, 2]
125 | [0, 1, 1, 1]
126 | [0, 2, 3, 3]
127 | [0, 3, 0, 1]
128 | [3, 2, 3, 0]
129 | [3, 2, 2, 2]
130 | [3, 2, 2, 0]
131 | [3, 0, 3, 2]
132 | [3, 0, 2, 0]
133 | [1, 1, 1, 3]
134 | [1, 1, 0, 1]
135 | [1, 0, 2, 1]
136 | [0, 3, 1, 3]
137 | [3, 1, 1, 2]
138 |
--------------------------------------------------------------------------------
/examples/corner_free_set/f3_n2_size53.txt:
--------------------------------------------------------------------------------
1 | [8, 4]
2 | [4, 8]
3 | [4, 4]
4 | [4, 1]
5 | [4, 3]
6 | [1, 4]
7 | [3, 4]
8 | [0, 0]
9 | [8, 6]
10 | [8, 2]
11 | [0, 8]
12 | [4, 6]
13 | [0, 4]
14 | [8, 0]
15 | [4, 0]
16 | [0, 7]
17 | [0, 5]
18 | [0, 1]
19 | [8, 8]
20 | [7, 0]
21 | [5, 0]
22 | [3, 0]
23 | [2, 8]
24 | [6, 8]
25 | [8, 7]
26 | [6, 4]
27 | [8, 3]
28 | [1, 2]
29 | [1, 6]
30 | [3, 2]
31 | [3, 6]
32 | [4, 5]
33 | [7, 8]
34 | [5, 4]
35 | [3, 8]
36 | [0, 6]
37 | [5, 7]
38 | [7, 1]
39 | [7, 3]
40 | [3, 5]
41 | [1, 7]
42 | [1, 5]
43 | [6, 3]
44 | [3, 1]
45 | [1, 3]
46 | [1, 1]
47 | [2, 0]
48 | [7, 2]
49 | [5, 6]
50 | [6, 6]
51 | [2, 2]
52 | [2, 7]
53 | [2, 5]
54 |
--------------------------------------------------------------------------------
/examples/corner_free_set/f3_n3_size370.txt:
--------------------------------------------------------------------------------
1 | [8, 8, 8]
2 | [4, 8, 8]
3 | [8, 8, 4]
4 | [0, 8, 8]
5 | [8, 8, 0]
6 | [8, 8, 6]
7 | [8, 8, 7]
8 | [8, 6, 8]
9 | [6, 8, 8]
10 | [7, 8, 8]
11 | [8, 7, 8]
12 | [8, 4, 8]
13 | [5, 8, 8]
14 | [4, 4, 8]
15 | [4, 8, 4]
16 | [8, 4, 4]
17 | [4, 4, 4]
18 | [8, 0, 8]
19 | [4, 0, 8]
20 | [0, 4, 8]
21 | [8, 0, 4]
22 | [0, 8, 4]
23 | [8, 4, 0]
24 | [4, 8, 0]
25 | [0, 4, 4]
26 | [8, 6, 4]
27 | [4, 0, 4]
28 | [4, 8, 6]
29 | [4, 4, 0]
30 | [4, 8, 7]
31 | [8, 7, 4]
32 | [8, 8, 3]
33 | [0, 0, 8]
34 | [0, 8, 0]
35 | [8, 0, 0]
36 | [0, 0, 4]
37 | [8, 6, 0]
38 | [0, 8, 6]
39 | [0, 4, 0]
40 | [4, 0, 0]
41 | [0, 8, 7]
42 | [8, 7, 0]
43 | [1, 8, 8]
44 | [8, 8, 2]
45 | [4, 6, 8]
46 | [6, 8, 4]
47 | [0, 0, 0]
48 | [4, 7, 8]
49 | [7, 8, 4]
50 | [8, 5, 8]
51 | [0, 6, 8]
52 | [4, 5, 8]
53 | [5, 8, 4]
54 | [6, 8, 0]
55 | [0, 7, 8]
56 | [6, 8, 6]
57 | [8, 6, 6]
58 | [7, 8, 0]
59 | [6, 8, 7]
60 | [8, 6, 7]
61 | [7, 8, 6]
62 | [8, 7, 6]
63 | [0, 5, 8]
64 | [6, 4, 8]
65 | [8, 7, 7]
66 | [7, 8, 7]
67 | [5, 8, 0]
68 | [8, 4, 6]
69 | [7, 4, 8]
70 | [8, 4, 7]
71 | [6, 4, 4]
72 | [4, 6, 4]
73 | [4, 4, 6]
74 | [5, 8, 6]
75 | [3, 4, 8]
76 | [4, 4, 7]
77 | [4, 7, 4]
78 | [8, 3, 4]
79 | [7, 4, 4]
80 | [5, 8, 7]
81 | [4, 8, 3]
82 | [8, 4, 3]
83 | [6, 0, 8]
84 | [4, 3, 4]
85 | [3, 4, 4]
86 | [8, 0, 6]
87 | [4, 4, 3]
88 | [6, 6, 8]
89 | [7, 0, 8]
90 | [8, 0, 7]
91 | [6, 0, 4]
92 | [0, 6, 4]
93 | [4, 6, 0]
94 | [0, 4, 6]
95 | [4, 0, 6]
96 | [6, 4, 0]
97 | [8, 1, 8]
98 | [7, 6, 8]
99 | [6, 7, 8]
100 | [3, 0, 8]
101 | [7, 0, 4]
102 | [0, 7, 4]
103 | [4, 0, 7]
104 | [0, 4, 7]
105 | [8, 0, 3]
106 | [7, 4, 0]
107 | [0, 8, 3]
108 | [8, 3, 0]
109 | [4, 7, 0]
110 | [7, 7, 8]
111 | [4, 1, 8]
112 | [1, 8, 4]
113 | [3, 0, 4]
114 | [0, 3, 4]
115 | [0, 4, 3]
116 | [3, 4, 0]
117 | [4, 0, 3]
118 | [8, 6, 3]
119 | [4, 3, 0]
120 | [5, 6, 8]
121 | [4, 8, 2]
122 | [8, 4, 2]
123 | [2, 4, 8]
124 | [8, 2, 4]
125 | [0, 6, 0]
126 | [6, 0, 0]
127 | [0, 0, 6]
128 | [8, 7, 3]
129 | [4, 4, 2]
130 | [5, 7, 8]
131 | [0, 0, 7]
132 | [4, 2, 4]
133 | [2, 4, 4]
134 | [7, 0, 0]
135 | [0, 7, 0]
136 | [0, 1, 8]
137 | [1, 8, 0]
138 | [0, 0, 3]
139 | [3, 0, 0]
140 | [0, 3, 0]
141 | [8, 0, 2]
142 | [0, 8, 2]
143 | [2, 0, 8]
144 | [8, 2, 0]
145 | [1, 8, 6]
146 | [8, 6, 2]
147 | [0, 4, 2]
148 | [4, 0, 2]
149 | [0, 2, 4]
150 | [2, 0, 4]
151 | [1, 8, 7]
152 | [6, 6, 4]
153 | [2, 4, 0]
154 | [4, 2, 0]
155 | [4, 6, 6]
156 | [8, 7, 2]
157 | [4, 6, 7]
158 | [6, 7, 4]
159 | [7, 6, 4]
160 | [4, 7, 6]
161 | [6, 8, 3]
162 | [6, 5, 8]
163 | [4, 7, 7]
164 | [7, 7, 4]
165 | [8, 5, 6]
166 | [7, 8, 3]
167 | [0, 0, 2]
168 | [7, 5, 8]
169 | [8, 5, 7]
170 | [5, 6, 4]
171 | [6, 6, 0]
172 | [0, 6, 6]
173 | [2, 0, 0]
174 | [0, 2, 0]
175 | [4, 5, 6]
176 | [3, 5, 8]
177 | [0, 6, 7]
178 | [4, 5, 7]
179 | [5, 7, 4]
180 | [7, 6, 0]
181 | [0, 7, 6]
182 | [6, 7, 0]
183 | [5, 8, 3]
184 | [1, 6, 8]
185 | [0, 7, 7]
186 | [7, 7, 0]
187 | [6, 8, 2]
188 | [1, 7, 8]
189 | [0, 5, 6]
190 | [5, 6, 0]
191 | [6, 4, 6]
192 | [7, 8, 2]
193 | [0, 5, 7]
194 | [6, 4, 7]
195 | [5, 7, 0]
196 | [7, 4, 6]
197 | [3, 2, 8]
198 | [7, 4, 7]
199 | [6, 3, 4]
200 | [4, 6, 3]
201 | [3, 4, 6]
202 | [6, 4, 3]
203 | [2, 8, 5]
204 | [2, 5, 8]
205 | [3, 4, 7]
206 | [7, 3, 4]
207 | [6, 0, 6]
208 | [4, 7, 3]
209 | [8, 3, 3]
210 | [7, 4, 3]
211 | [8, 3, 5]
212 | [6, 0, 7]
213 | [3, 3, 4]
214 | [7, 0, 6]
215 | [3, 4, 3]
216 | [4, 3, 3]
217 | [6, 6, 6]
218 | [4, 3, 5]
219 | [6, 1, 8]
220 | [6, 6, 7]
221 | [3, 5, 4]
222 | [7, 0, 7]
223 | [6, 0, 3]
224 | [0, 6, 3]
225 | [6, 7, 6]
226 | [6, 3, 0]
227 | [3, 0, 6]
228 | [7, 6, 6]
229 | [8, 1, 6]
230 | [7, 1, 8]
231 | [8, 1, 7]
232 | [1, 6, 4]
233 | [7, 6, 7]
234 | [6, 7, 7]
235 | [3, 0, 7]
236 | [7, 3, 0]
237 | [7, 7, 6]
238 | [7, 0, 3]
239 | [4, 1, 6]
240 | [0, 7, 3]
241 | [6, 4, 2]
242 | [3, 1, 8]
243 | [4, 6, 2]
244 | [2, 8, 2]
245 | [1, 7, 4]
246 | [6, 2, 4]
247 | [4, 1, 7]
248 | [7, 7, 7]
249 | [3, 8, 1]
250 | [3, 3, 0]
251 | [0, 3, 3]
252 | [5, 6, 6]
253 | [3, 0, 3]
254 | [2, 4, 6]
255 | [0, 3, 5]
256 | [8, 3, 2]
257 | [7, 4, 2]
258 | [1, 8, 5]
259 | [4, 7, 2]
260 | [2, 3, 8]
261 | [5, 6, 7]
262 | [2, 4, 7]
263 | [7, 2, 4]
264 | [3, 5, 0]
265 | [5, 7, 6]
266 | [8, 2, 3]
267 | [4, 3, 2]
268 | [3, 4, 2]
269 | [2, 3, 4]
270 | [3, 2, 4]
271 | [5, 7, 7]
272 | [4, 2, 3]
273 | [2, 4, 3]
274 | [0, 1, 6]
275 | [1, 6, 0]
276 | [6, 0, 2]
277 | [0, 6, 2]
278 | [0, 1, 7]
279 | [6, 2, 0]
280 | [2, 0, 6]
281 | [1, 7, 0]
282 | [7, 0, 2]
283 | [0, 7, 2]
284 | [2, 0, 7]
285 | [7, 2, 0]
286 | [0, 3, 2]
287 | [3, 0, 2]
288 | [2, 1, 8]
289 | [8, 2, 1]
290 | [2, 3, 0]
291 | [3, 2, 0]
292 | [0, 2, 3]
293 | [6, 6, 3]
294 | [2, 0, 3]
295 | [8, 2, 2]
296 | [4, 2, 1]
297 | [2, 1, 4]
298 | [6, 7, 3]
299 | [7, 6, 3]
300 | [6, 5, 6]
301 | [2, 4, 2]
302 | [4, 2, 2]
303 | [6, 5, 7]
304 | [2, 2, 4]
305 | [7, 7, 3]
306 | [7, 5, 6]
307 | [7, 5, 7]
308 | [5, 6, 3]
309 | [3, 5, 6]
310 | [0, 2, 1]
311 | [3, 5, 7]
312 | [1, 6, 6]
313 | [2, 1, 0]
314 | [5, 7, 3]
315 | [0, 2, 2]
316 | [2, 0, 2]
317 | [6, 6, 2]
318 | [1, 6, 7]
319 | [1, 7, 6]
320 | [2, 2, 0]
321 | [6, 7, 2]
322 | [7, 6, 2]
323 | [1, 7, 7]
324 | [7, 7, 2]
325 | [3, 2, 6]
326 | [5, 6, 2]
327 | [3, 2, 7]
328 | [6, 3, 3]
329 | [2, 5, 6]
330 | [6, 3, 5]
331 | [5, 7, 2]
332 | [2, 5, 7]
333 | [7, 3, 3]
334 | [7, 3, 5]
335 | [6, 1, 6]
336 | [3, 3, 3]
337 | [3, 3, 5]
338 | [6, 1, 7]
339 | [3, 5, 3]
340 | [7, 1, 6]
341 | [7, 1, 7]
342 | [1, 6, 3]
343 | [3, 1, 6]
344 | [6, 3, 2]
345 | [1, 7, 3]
346 | [2, 3, 6]
347 | [6, 2, 3]
348 | [7, 3, 2]
349 | [2, 3, 7]
350 | [3, 1, 3]
351 | [7, 2, 3]
352 | [3, 3, 2]
353 | [2, 3, 3]
354 | [3, 5, 2]
355 | [3, 2, 5]
356 | [1, 6, 2]
357 | [6, 2, 1]
358 | [2, 1, 6]
359 | [6, 2, 2]
360 | [1, 7, 2]
361 | [7, 2, 1]
362 | [2, 1, 7]
363 | [7, 2, 2]
364 | [3, 2, 1]
365 | [2, 1, 3]
366 | [2, 3, 2]
367 | [2, 2, 3]
368 | [5, 2, 2]
369 | [2, 1, 2]
370 | [1, 2, 2]
371 |
--------------------------------------------------------------------------------
/examples/cyclic_graphs/cyclic_graphs.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {
6 | "id": "PlhRSC3iWzA1"
7 | },
8 | "source": [
9 | "```\n",
10 | "- Copyright 2023 DeepMind Technologies Limited\n",
11 | "- All software is licensed under the Apache License, Version 2.0 (Apache 2.0); you may not use this file except in compliance with the Apache 2.0 license. You may obtain a copy of the Apache 2.0 license at: https://www.apache.org/licenses/LICENSE-2.0\n",
12 | "- All other materials are licensed under the Creative Commons Attribution 4.0 International License (CC-BY). You may obtain a copy of the CC-BY license at: https://creativecommons.org/licenses/by/4.0/legalcode\n",
13 | "- Unless required by applicable law or agreed to in writing, all software and materials distributed here under the Apache 2.0 or CC-BY licenses are distributed on an \\\"AS IS\\\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the licenses for the specific language governing permissions and limitations under those licenses.\n",
14 | "- This is not an official Google product\n",
15 | "```"
16 | ]
17 | },
18 | {
19 | "cell_type": "markdown",
20 | "metadata": {
21 | "id": "3LCquqjVWy0e"
22 | },
23 | "source": [
24 | "# Cyclic graphs\n",
25 | "\n",
26 | "This notebook contains:\n",
27 | "1. the *skeleton* we used for FunSearch to discover large independent sets in the $n$-th strong product of cyclic graphs,\n",
28 | "2. the *functions* discovered by FunSearch that construct those independent sets.\n",
29 | "\n",
30 | "## Skeleton\n",
31 | "\n",
32 | "The commented-out decorators are just a way to indicate the main entry point of the program (`@funsearch.run`) and the function that *FunSearch* should evolve (`@funsearch.evolve`)."
33 | ]
34 | },
35 | {
36 | "cell_type": "code",
37 | "execution_count": null,
38 | "metadata": {
39 | "id": "aIkfPyeLXB4n"
40 | },
41 | "outputs": [],
42 | "source": [
43 | "\"\"\"Obtains maximal independent sets.\"\"\"\n",
44 | "import itertools\n",
45 | "import numpy as np\n",
46 | "\n",
47 | "\n",
48 | "# @funsearch.run\n",
49 | "def evaluate(num_nodes: int, n: int) -\u003e int:\n",
50 | " \"\"\"Returns the size of an independent set.\"\"\"\n",
51 | " independent_set = solve(num_nodes, n)\n",
52 | " return len(independent_set)\n",
53 | "\n",
54 | "\n",
55 | "def solve(num_nodes: int, n: int) -\u003e list[tuple[int, ...]]:\n",
56 | " \"\"\"Gets independent set with maximal size.\n",
57 | "\n",
58 | " Args:\n",
59 | " num_nodes: The number of nodes of the base cyclic graph.\n",
60 | " n: The power we raise the graph to.\n",
61 | "\n",
62 | " Returns:\n",
63 | " A list of `n`-tuples in `{0, 1, 2, ..., num_nodes - 1}`.\n",
64 | " \"\"\"\n",
65 | " to_block = np.array(list(itertools.product([-1, 0, 1], repeat=n)))\n",
66 | "\n",
67 | " # Powers in decreasing order for compatibility with `itertools.product`, so\n",
68 | " # that the relationship `i = children[i] @ powers` holds for all `i`.\n",
69 | " powers = num_nodes ** np.arange(n - 1, -1, -1)\n",
70 | "\n",
71 | " # Precompute the priority scores.\n",
72 | " children = np.array(\n",
73 | " list(itertools.product(range(num_nodes), repeat=n)), dtype=np.int32)\n",
74 | " scores = np.array([priority(tuple(child), num_nodes, n)\n",
75 | " for child in children])\n",
76 | "\n",
77 | " # Build `max_set` greedily, using scores for prioritization.\n",
78 | " max_set = np.empty(shape=(0, n), dtype=np.int32)\n",
79 | " while np.any(scores != -np.inf):\n",
80 | " # Add a child with a maximum score to `max_set`, and set scores of\n",
81 | " # invalidated children to -inf, so that they never get selected.\n",
82 | " max_index = np.argmax(scores)\n",
83 | " child = children[None, max_index] # [1, n]\n",
84 | "\n",
85 | " blocking = np.einsum(\n",
86 | " 'cn,n-\u003ec', (to_block + child) % num_nodes, powers) # [C]\n",
87 | " scores[blocking] = -np.inf\n",
88 | " max_set = np.concatenate([max_set, child], axis=0)\n",
89 | "\n",
90 | " return [tuple(map(int, el)) for el in max_set]\n",
91 | "\n",
92 | "\n",
93 | "# @funsearch.evolve\n",
94 | "def priority(el: tuple[int, ...], num_nodes: int, n: int) -\u003e float:\n",
95 | " \"\"\"Returns the priority with which we want to add `el` to the set.\n",
96 | "\n",
97 | " Args:\n",
98 | " el: an n-tuple representing the element to consider whether to add.\n",
99 | " num_nodes: the number of nodes of the base graph.\n",
100 | " n: an integer, power of the graph.\n",
101 | "\n",
102 | " Returns:\n",
103 | " A number reflecting the priority with which we want to add `el` to the\n",
104 | " independent set.\n",
105 | " \"\"\"\n",
106 | " return 0."
107 | ]
108 | },
109 | {
110 | "cell_type": "markdown",
111 | "metadata": {
112 | "id": "aLkn0zVUYSbk"
113 | },
114 | "source": [
115 | "By executing the skeleton with the trivial `priority` function in place we can check that the resulting independent sets are far from optimal (e.g., the best known construction for the 5th strong product of the 7-node graph has size 367):\n"
116 | ]
117 | },
118 | {
119 | "cell_type": "code",
120 | "execution_count": null,
121 | "metadata": {
122 | "executionInfo": {
123 | "elapsed": 75,
124 | "status": "ok",
125 | "timestamp": 1697104375004,
126 | "user": {
127 | "displayName": "",
128 | "userId": ""
129 | },
130 | "user_tz": -60
131 | },
132 | "id": "9t9out8lYBSr",
133 | "outputId": "5ac9c2b2-e066-44af-947e-c94b4b826421"
134 | },
135 | "outputs": [
136 | {
137 | "name": "stdout",
138 | "output_type": "stream",
139 | "text": [
140 | "1 3\n",
141 | "2 9\n",
142 | "3 27\n",
143 | "4 81\n",
144 | "5 243\n"
145 | ]
146 | }
147 | ],
148 | "source": [
149 | "for n in range(1, 6):\n",
150 | " print(n, evaluate(num_nodes=7, n=n))"
151 | ]
152 | },
153 | {
154 | "cell_type": "markdown",
155 | "metadata": {
156 | "id": "FBbvHTaQY3X8"
157 | },
158 | "source": [
159 | "## Discovered function that builds an independent set of size $367$ in $C_7^5$\n",
160 | "\n",
161 | "This matches the size of the best known construction by [Polak \u0026 Schrijver (2019)](https://ir.cwi.nl/pub/30364/30364.pdf)."
162 | ]
163 | },
164 | {
165 | "cell_type": "code",
166 | "execution_count": null,
167 | "metadata": {
168 | "id": "w4yMoDAoYjYq"
169 | },
170 | "outputs": [],
171 | "source": [
172 | "def priority(el: tuple[int, ...], num_nodes: int, n: int) -\u003e float:\n",
173 | " \"\"\"Returns the priority with which we want to add `el` to the set.\"\"\"\n",
174 | " score = 0.\n",
175 | " for i in range(n):\n",
176 | " if el[i] == el[(i + 2) % n]:\n",
177 | " score += 1\n",
178 | " else:\n",
179 | " score -= 1\n",
180 | " x = ((n - 2) * el[i] - el[(i + 1) % n]\n",
181 | " - el[(i + 2) % n] - (n + 1) * el[(i + 3) % n]) % num_nodes\n",
182 | " score -= 0.5 * (x - el[(i + 1) % n]) ** 2\n",
183 | " score += 0.1 * (num_nodes - 1 - (x - 1) % num_nodes) ** 2\n",
184 | " score += 0.2 * (num_nodes - 1 - (x - 2) % num_nodes) ** 2\n",
185 | " return score"
186 | ]
187 | },
188 | {
189 | "cell_type": "code",
190 | "execution_count": null,
191 | "metadata": {
192 | "id": "A_eiYDIsZLYi"
193 | },
194 | "outputs": [],
195 | "source": [
196 | "assert evaluate(num_nodes=7, n=5) == 367"
197 | ]
198 | },
199 | {
200 | "cell_type": "markdown",
201 | "metadata": {
202 | "id": "zfY5hIZnZYQg"
203 | },
204 | "source": [
205 | "## Discovered function that builds the best known independent sets in $C_9^n$ for $n=3,...,7$\n",
206 | "\n",
207 | "These independent sets match the best known construction reported by [Matthew \u0026 Östergård (2016)](https://link.springer.com/article/10.1007/s10623-016-0194-7)."
208 | ]
209 | },
210 | {
211 | "cell_type": "code",
212 | "execution_count": null,
213 | "metadata": {
214 | "id": "al23ssPbZUpk"
215 | },
216 | "outputs": [],
217 | "source": [
218 | "def priority(el: tuple[int, ...], num_nodes: int, n: int) -\u003e float:\n",
219 | " \"\"\"Returns the priority with which we want to add `el` to the set.\"\"\"\n",
220 | " s = 0.\n",
221 | " for i in range(n):\n",
222 | " s += el[i] \u003c\u003c i\n",
223 | " s %= num_nodes\n",
224 | " return (2 * el[2] - 4 * el[0] + el[1]) % num_nodes + s"
225 | ]
226 | },
227 | {
228 | "cell_type": "markdown",
229 | "metadata": {
230 | "id": "0li-u2bKzcqs"
231 | },
232 | "source": [
233 | "Below, we only run the code up until $n=5$. Uncomment the line below to run also the code with $n=6$ and $n=7$, which would take about 10 minutes to execute."
234 | ]
235 | },
236 | {
237 | "cell_type": "code",
238 | "execution_count": null,
239 | "metadata": {
240 | "id": "4tiBK2mzadTA"
241 | },
242 | "outputs": [],
243 | "source": [
244 | "expected_sizes = {\n",
245 | " 3: 81,\n",
246 | " 4: 324,\n",
247 | " 5: 1458,\n",
248 | " 6: 6561,\n",
249 | " 7: 26244\n",
250 | "}\n",
251 | "range_n = range(3, 6)\n",
252 | "# range_n = range(3, 8) # Uncomment to run up until n=7.\n",
253 | "for n in range_n:\n",
254 | " assert evaluate(num_nodes=9, n=n) == expected_sizes[n]"
255 | ]
256 | },
257 | {
258 | "cell_type": "markdown",
259 | "metadata": {
260 | "id": "m1pW8OYnarxM"
261 | },
262 | "source": [
263 | "## Discovered function that finds an independent set of size 754 in $C_{11}^4$\n",
264 | "\n",
265 | "This is larger than the best known independent set reported by [Matthew \u0026 Östergård (2016)](https://link.springer.com/article/10.1007/s10623-016-0194-7), which has size $748$."
266 | ]
267 | },
268 | {
269 | "cell_type": "code",
270 | "execution_count": null,
271 | "metadata": {
272 | "id": "mDrmnumFah_t"
273 | },
274 | "outputs": [],
275 | "source": [
276 | "def priority(el: tuple[int, ...], num_nodes: int, n: int) -\u003e float:\n",
277 | " \"\"\"Returns the priority with which we want to add `el` to the set.\"\"\"\n",
278 | " el_clipped = np.clip(el, a_min=None, a_max=num_nodes - 3)\n",
279 | " values = 2 * np.array(list(itertools.product(range(1, n), repeat=n)))\n",
280 | " multipliers = np.array(\n",
281 | " [num_nodes ** i for i in range(n - 1, -1, -1)], dtype=np.int32)\n",
282 | " x = np.sum((1 + values + el_clipped) * multipliers, axis=-1)\n",
283 | " return np.sum(x % (num_nodes - 2), dtype=float)"
284 | ]
285 | },
286 | {
287 | "cell_type": "code",
288 | "execution_count": null,
289 | "metadata": {
290 | "id": "dvRR69L-bD95"
291 | },
292 | "outputs": [],
293 | "source": [
294 | "assert evaluate(num_nodes=11, n=4) == 754"
295 | ]
296 | }
297 | ],
298 | "metadata": {
299 | "colab": {
300 | "provenance": []
301 | },
302 | "kernelspec": {
303 | "display_name": "Python 3",
304 | "name": "python3"
305 | },
306 | "language_info": {
307 | "name": "python"
308 | }
309 | },
310 | "nbformat": 4,
311 | "nbformat_minor": 0
312 | }
313 |
--------------------------------------------------------------------------------
/examples/cyclic_graphs/nodes11_n4_size754.txt:
--------------------------------------------------------------------------------
1 | [0, 0, 0, 8]
2 | [0, 0, 0, 10]
3 | [0, 0, 1, 6]
4 | [0, 0, 2, 4]
5 | [0, 0, 3, 2]
6 | [0, 0, 4, 0]
7 | [0, 0, 5, 7]
8 | [0, 0, 6, 5]
9 | [0, 0, 7, 3]
10 | [0, 0, 8, 1]
11 | [0, 0, 9, 8]
12 | [0, 0, 9, 10]
13 | [0, 0, 10, 1]
14 | [0, 1, 0, 4]
15 | [0, 1, 1, 2]
16 | [0, 1, 2, 0]
17 | [0, 1, 3, 7]
18 | [0, 1, 4, 5]
19 | [0, 1, 5, 3]
20 | [0, 1, 6, 1]
21 | [0, 1, 7, 8]
22 | [0, 1, 7, 10]
23 | [0, 1, 8, 6]
24 | [0, 1, 9, 4]
25 | [0, 1, 10, 6]
26 | [0, 2, 0, 0]
27 | [0, 2, 1, 7]
28 | [0, 2, 2, 5]
29 | [0, 2, 3, 3]
30 | [0, 2, 4, 1]
31 | [0, 2, 5, 8]
32 | [0, 2, 5, 10]
33 | [0, 2, 6, 6]
34 | [0, 2, 7, 4]
35 | [0, 2, 8, 2]
36 | [0, 2, 9, 0]
37 | [0, 2, 10, 2]
38 | [0, 3, 0, 5]
39 | [0, 3, 1, 3]
40 | [0, 3, 2, 1]
41 | [0, 3, 3, 8]
42 | [0, 3, 3, 10]
43 | [0, 3, 4, 6]
44 | [0, 3, 5, 4]
45 | [0, 3, 6, 2]
46 | [0, 3, 7, 0]
47 | [0, 3, 8, 7]
48 | [0, 3, 9, 5]
49 | [0, 3, 10, 7]
50 | [0, 4, 0, 1]
51 | [0, 4, 1, 8]
52 | [0, 4, 1, 10]
53 | [0, 4, 2, 6]
54 | [0, 4, 3, 4]
55 | [0, 4, 4, 2]
56 | [0, 4, 5, 0]
57 | [0, 4, 6, 7]
58 | [0, 4, 7, 5]
59 | [0, 4, 8, 3]
60 | [0, 4, 9, 1]
61 | [0, 4, 10, 3]
62 | [0, 5, 0, 6]
63 | [0, 5, 1, 4]
64 | [0, 5, 2, 2]
65 | [0, 5, 3, 0]
66 | [0, 5, 4, 7]
67 | [0, 5, 5, 5]
68 | [0, 5, 6, 3]
69 | [0, 5, 7, 1]
70 | [0, 5, 8, 8]
71 | [0, 5, 8, 10]
72 | [0, 5, 9, 6]
73 | [0, 5, 10, 8]
74 | [0, 5, 10, 10]
75 | [0, 6, 0, 2]
76 | [0, 6, 1, 0]
77 | [0, 6, 2, 7]
78 | [0, 6, 3, 5]
79 | [0, 6, 4, 3]
80 | [0, 6, 5, 1]
81 | [0, 6, 6, 8]
82 | [0, 6, 6, 10]
83 | [0, 6, 7, 6]
84 | [0, 6, 8, 4]
85 | [0, 6, 9, 2]
86 | [0, 6, 10, 4]
87 | [0, 7, 0, 7]
88 | [0, 7, 1, 5]
89 | [0, 7, 2, 3]
90 | [0, 7, 3, 1]
91 | [0, 7, 4, 8]
92 | [0, 7, 4, 10]
93 | [0, 7, 5, 6]
94 | [0, 7, 6, 4]
95 | [0, 7, 7, 2]
96 | [0, 7, 8, 0]
97 | [0, 7, 9, 7]
98 | [0, 7, 10, 0]
99 | [0, 8, 0, 3]
100 | [0, 8, 1, 1]
101 | [0, 8, 2, 8]
102 | [0, 8, 2, 10]
103 | [0, 8, 3, 6]
104 | [0, 8, 4, 4]
105 | [0, 8, 5, 2]
106 | [0, 8, 6, 0]
107 | [0, 8, 7, 7]
108 | [0, 8, 8, 5]
109 | [0, 8, 9, 3]
110 | [0, 8, 10, 5]
111 | [0, 9, 0, 8]
112 | [0, 9, 0, 10]
113 | [0, 9, 1, 6]
114 | [0, 9, 2, 4]
115 | [0, 9, 3, 2]
116 | [0, 9, 4, 0]
117 | [0, 9, 5, 7]
118 | [0, 9, 6, 5]
119 | [0, 9, 7, 3]
120 | [0, 9, 8, 1]
121 | [0, 9, 9, 7]
122 | [0, 9, 10, 1]
123 | [0, 10, 0, 3]
124 | [0, 10, 1, 1]
125 | [0, 10, 2, 8]
126 | [0, 10, 2, 10]
127 | [0, 10, 3, 6]
128 | [0, 10, 4, 4]
129 | [0, 10, 5, 2]
130 | [0, 10, 6, 0]
131 | [0, 10, 7, 7]
132 | [0, 10, 8, 5]
133 | [0, 10, 9, 3]
134 | [0, 10, 10, 5]
135 | [1, 0, 5, 9]
136 | [1, 1, 3, 9]
137 | [1, 2, 1, 9]
138 | [1, 3, 8, 9]
139 | [1, 3, 10, 9]
140 | [1, 4, 6, 9]
141 | [1, 5, 4, 9]
142 | [1, 6, 2, 9]
143 | [1, 7, 0, 9]
144 | [1, 7, 9, 9]
145 | [1, 8, 7, 9]
146 | [1, 9, 5, 9]
147 | [1, 9, 9, 9]
148 | [1, 10, 7, 9]
149 | [2, 0, 0, 1]
150 | [2, 0, 1, 8]
151 | [2, 0, 1, 10]
152 | [2, 0, 2, 6]
153 | [2, 0, 3, 4]
154 | [2, 0, 4, 2]
155 | [2, 0, 5, 0]
156 | [2, 0, 6, 7]
157 | [2, 0, 7, 5]
158 | [2, 0, 8, 3]
159 | [2, 0, 9, 1]
160 | [2, 0, 10, 3]
161 | [2, 1, 0, 6]
162 | [2, 1, 1, 4]
163 | [2, 1, 2, 2]
164 | [2, 1, 3, 0]
165 | [2, 1, 4, 7]
166 | [2, 1, 5, 5]
167 | [2, 1, 6, 3]
168 | [2, 1, 7, 1]
169 | [2, 1, 8, 8]
170 | [2, 1, 8, 10]
171 | [2, 1, 9, 6]
172 | [2, 1, 10, 8]
173 | [2, 1, 10, 10]
174 | [2, 2, 0, 2]
175 | [2, 2, 1, 0]
176 | [2, 2, 2, 7]
177 | [2, 2, 3, 5]
178 | [2, 2, 4, 3]
179 | [2, 2, 5, 1]
180 | [2, 2, 6, 8]
181 | [2, 2, 6, 10]
182 | [2, 2, 7, 6]
183 | [2, 2, 8, 4]
184 | [2, 2, 9, 2]
185 | [2, 2, 10, 4]
186 | [2, 3, 0, 7]
187 | [2, 3, 1, 5]
188 | [2, 3, 2, 3]
189 | [2, 3, 3, 1]
190 | [2, 3, 4, 8]
191 | [2, 3, 4, 10]
192 | [2, 3, 5, 6]
193 | [2, 3, 6, 4]
194 | [2, 3, 7, 2]
195 | [2, 3, 8, 0]
196 | [2, 3, 9, 7]
197 | [2, 3, 10, 0]
198 | [2, 4, 0, 3]
199 | [2, 4, 1, 1]
200 | [2, 4, 2, 8]
201 | [2, 4, 2, 10]
202 | [2, 4, 3, 6]
203 | [2, 4, 4, 4]
204 | [2, 4, 5, 2]
205 | [2, 4, 6, 0]
206 | [2, 4, 7, 7]
207 | [2, 4, 8, 5]
208 | [2, 4, 9, 3]
209 | [2, 4, 10, 5]
210 | [2, 5, 0, 8]
211 | [2, 5, 0, 10]
212 | [2, 5, 1, 6]
213 | [2, 5, 2, 4]
214 | [2, 5, 3, 2]
215 | [2, 5, 4, 0]
216 | [2, 5, 5, 7]
217 | [2, 5, 6, 5]
218 | [2, 5, 7, 3]
219 | [2, 5, 8, 1]
220 | [2, 5, 9, 8]
221 | [2, 5, 9, 10]
222 | [2, 5, 10, 1]
223 | [2, 6, 0, 4]
224 | [2, 6, 1, 2]
225 | [2, 6, 2, 0]
226 | [2, 6, 3, 7]
227 | [2, 6, 4, 5]
228 | [2, 6, 5, 3]
229 | [2, 6, 6, 1]
230 | [2, 6, 7, 8]
231 | [2, 6, 7, 10]
232 | [2, 6, 8, 6]
233 | [2, 6, 9, 4]
234 | [2, 6, 10, 6]
235 | [2, 7, 0, 0]
236 | [2, 7, 1, 7]
237 | [2, 7, 2, 5]
238 | [2, 7, 3, 3]
239 | [2, 7, 4, 1]
240 | [2, 7, 5, 8]
241 | [2, 7, 5, 10]
242 | [2, 7, 6, 6]
243 | [2, 7, 7, 4]
244 | [2, 7, 8, 2]
245 | [2, 7, 9, 0]
246 | [2, 7, 10, 2]
247 | [2, 8, 0, 5]
248 | [2, 8, 1, 3]
249 | [2, 8, 2, 1]
250 | [2, 8, 3, 8]
251 | [2, 8, 3, 10]
252 | [2, 8, 4, 6]
253 | [2, 8, 5, 4]
254 | [2, 8, 6, 2]
255 | [2, 8, 7, 0]
256 | [2, 8, 8, 7]
257 | [2, 8, 9, 5]
258 | [2, 8, 10, 7]
259 | [2, 9, 0, 1]
260 | [2, 9, 1, 8]
261 | [2, 9, 1, 10]
262 | [2, 9, 2, 6]
263 | [2, 9, 3, 4]
264 | [2, 9, 4, 2]
265 | [2, 9, 5, 0]
266 | [2, 9, 6, 7]
267 | [2, 9, 7, 5]
268 | [2, 9, 8, 3]
269 | [2, 9, 9, 0]
270 | [2, 9, 10, 3]
271 | [2, 10, 0, 5]
272 | [2, 10, 1, 3]
273 | [2, 10, 2, 1]
274 | [2, 10, 3, 8]
275 | [2, 10, 3, 10]
276 | [2, 10, 4, 6]
277 | [2, 10, 5, 4]
278 | [2, 10, 6, 2]
279 | [2, 10, 7, 0]
280 | [2, 10, 8, 7]
281 | [2, 10, 9, 5]
282 | [2, 10, 10, 7]
283 | [3, 0, 6, 9]
284 | [3, 1, 4, 9]
285 | [3, 2, 2, 9]
286 | [3, 3, 0, 9]
287 | [3, 3, 9, 9]
288 | [3, 4, 7, 9]
289 | [3, 5, 5, 9]
290 | [3, 6, 3, 9]
291 | [3, 7, 1, 9]
292 | [3, 8, 8, 9]
293 | [3, 8, 10, 9]
294 | [3, 9, 6, 9]
295 | [3, 10, 8, 9]
296 | [3, 10, 10, 9]
297 | [4, 0, 0, 3]
298 | [4, 0, 1, 1]
299 | [4, 0, 2, 8]
300 | [4, 0, 2, 10]
301 | [4, 0, 3, 6]
302 | [4, 0, 4, 4]
303 | [4, 0, 5, 2]
304 | [4, 0, 6, 0]
305 | [4, 0, 7, 7]
306 | [4, 0, 8, 5]
307 | [4, 0, 9, 3]
308 | [4, 0, 10, 5]
309 | [4, 1, 0, 8]
310 | [4, 1, 0, 10]
311 | [4, 1, 1, 6]
312 | [4, 1, 2, 4]
313 | [4, 1, 3, 2]
314 | [4, 1, 4, 0]
315 | [4, 1, 5, 7]
316 | [4, 1, 6, 5]
317 | [4, 1, 7, 3]
318 | [4, 1, 8, 1]
319 | [4, 1, 9, 8]
320 | [4, 1, 9, 10]
321 | [4, 1, 10, 1]
322 | [4, 2, 0, 4]
323 | [4, 2, 1, 2]
324 | [4, 2, 2, 0]
325 | [4, 2, 3, 7]
326 | [4, 2, 4, 5]
327 | [4, 2, 5, 3]
328 | [4, 2, 6, 1]
329 | [4, 2, 7, 8]
330 | [4, 2, 7, 10]
331 | [4, 2, 8, 6]
332 | [4, 2, 9, 4]
333 | [4, 2, 10, 6]
334 | [4, 3, 0, 0]
335 | [4, 3, 1, 7]
336 | [4, 3, 2, 5]
337 | [4, 3, 3, 3]
338 | [4, 3, 4, 1]
339 | [4, 3, 5, 8]
340 | [4, 3, 5, 10]
341 | [4, 3, 6, 6]
342 | [4, 3, 7, 4]
343 | [4, 3, 8, 2]
344 | [4, 3, 9, 0]
345 | [4, 3, 10, 2]
346 | [4, 4, 0, 5]
347 | [4, 4, 1, 3]
348 | [4, 4, 2, 1]
349 | [4, 4, 3, 8]
350 | [4, 4, 3, 10]
351 | [4, 4, 4, 6]
352 | [4, 4, 5, 4]
353 | [4, 4, 6, 2]
354 | [4, 4, 7, 0]
355 | [4, 4, 8, 7]
356 | [4, 4, 9, 5]
357 | [4, 4, 10, 7]
358 | [4, 5, 0, 1]
359 | [4, 5, 1, 8]
360 | [4, 5, 1, 10]
361 | [4, 5, 2, 6]
362 | [4, 5, 3, 4]
363 | [4, 5, 4, 2]
364 | [4, 5, 5, 0]
365 | [4, 5, 6, 7]
366 | [4, 5, 7, 5]
367 | [4, 5, 8, 3]
368 | [4, 5, 9, 1]
369 | [4, 5, 10, 3]
370 | [4, 6, 0, 6]
371 | [4, 6, 1, 4]
372 | [4, 6, 2, 2]
373 | [4, 6, 3, 0]
374 | [4, 6, 4, 7]
375 | [4, 6, 5, 5]
376 | [4, 6, 6, 3]
377 | [4, 6, 7, 1]
378 | [4, 6, 8, 8]
379 | [4, 6, 8, 10]
380 | [4, 6, 9, 6]
381 | [4, 6, 10, 8]
382 | [4, 6, 10, 10]
383 | [4, 7, 0, 2]
384 | [4, 7, 1, 0]
385 | [4, 7, 2, 7]
386 | [4, 7, 3, 5]
387 | [4, 7, 4, 3]
388 | [4, 7, 5, 1]
389 | [4, 7, 6, 8]
390 | [4, 7, 6, 10]
391 | [4, 7, 7, 6]
392 | [4, 7, 8, 4]
393 | [4, 7, 9, 2]
394 | [4, 7, 10, 4]
395 | [4, 8, 0, 7]
396 | [4, 8, 1, 5]
397 | [4, 8, 2, 3]
398 | [4, 8, 3, 1]
399 | [4, 8, 4, 8]
400 | [4, 8, 4, 10]
401 | [4, 8, 5, 6]
402 | [4, 8, 6, 4]
403 | [4, 8, 7, 2]
404 | [4, 8, 8, 0]
405 | [4, 8, 9, 7]
406 | [4, 8, 10, 0]
407 | [4, 9, 0, 3]
408 | [4, 9, 1, 1]
409 | [4, 9, 2, 8]
410 | [4, 9, 2, 10]
411 | [4, 9, 3, 6]
412 | [4, 9, 4, 4]
413 | [4, 9, 5, 2]
414 | [4, 9, 6, 0]
415 | [4, 9, 7, 7]
416 | [4, 9, 8, 5]
417 | [4, 9, 9, 2]
418 | [4, 9, 10, 5]
419 | [4, 10, 0, 7]
420 | [4, 10, 1, 5]
421 | [4, 10, 2, 3]
422 | [4, 10, 3, 1]
423 | [4, 10, 4, 8]
424 | [4, 10, 4, 10]
425 | [4, 10, 5, 6]
426 | [4, 10, 6, 4]
427 | [4, 10, 7, 2]
428 | [4, 10, 8, 0]
429 | [4, 10, 9, 7]
430 | [4, 10, 10, 0]
431 | [5, 0, 7, 9]
432 | [5, 1, 5, 9]
433 | [5, 2, 3, 9]
434 | [5, 3, 1, 9]
435 | [5, 4, 8, 9]
436 | [5, 4, 10, 9]
437 | [5, 5, 6, 9]
438 | [5, 6, 4, 9]
439 | [5, 7, 2, 9]
440 | [5, 8, 0, 9]
441 | [5, 8, 9, 9]
442 | [5, 9, 7, 9]
443 | [5, 10, 0, 9]
444 | [5, 10, 9, 9]
445 | [6, 0, 0, 5]
446 | [6, 0, 1, 3]
447 | [6, 0, 2, 1]
448 | [6, 0, 3, 8]
449 | [6, 0, 3, 10]
450 | [6, 0, 4, 6]
451 | [6, 0, 5, 4]
452 | [6, 0, 6, 2]
453 | [6, 0, 7, 0]
454 | [6, 0, 8, 7]
455 | [6, 0, 9, 5]
456 | [6, 0, 10, 7]
457 | [6, 1, 0, 1]
458 | [6, 1, 1, 8]
459 | [6, 1, 1, 10]
460 | [6, 1, 2, 6]
461 | [6, 1, 3, 4]
462 | [6, 1, 4, 2]
463 | [6, 1, 5, 0]
464 | [6, 1, 6, 7]
465 | [6, 1, 7, 5]
466 | [6, 1, 8, 3]
467 | [6, 1, 9, 1]
468 | [6, 1, 10, 3]
469 | [6, 2, 0, 6]
470 | [6, 2, 1, 4]
471 | [6, 2, 2, 2]
472 | [6, 2, 3, 0]
473 | [6, 2, 4, 7]
474 | [6, 2, 5, 5]
475 | [6, 2, 6, 3]
476 | [6, 2, 7, 1]
477 | [6, 2, 8, 8]
478 | [6, 2, 8, 10]
479 | [6, 2, 9, 6]
480 | [6, 2, 10, 8]
481 | [6, 2, 10, 10]
482 | [6, 3, 0, 2]
483 | [6, 3, 1, 0]
484 | [6, 3, 2, 7]
485 | [6, 3, 3, 5]
486 | [6, 3, 4, 3]
487 | [6, 3, 5, 1]
488 | [6, 3, 6, 8]
489 | [6, 3, 6, 10]
490 | [6, 3, 7, 6]
491 | [6, 3, 8, 4]
492 | [6, 3, 9, 2]
493 | [6, 3, 10, 4]
494 | [6, 4, 0, 7]
495 | [6, 4, 1, 5]
496 | [6, 4, 2, 3]
497 | [6, 4, 3, 1]
498 | [6, 4, 4, 8]
499 | [6, 4, 4, 10]
500 | [6, 4, 5, 6]
501 | [6, 4, 6, 4]
502 | [6, 4, 7, 2]
503 | [6, 4, 8, 0]
504 | [6, 4, 9, 7]
505 | [6, 4, 10, 0]
506 | [6, 5, 0, 3]
507 | [6, 5, 1, 1]
508 | [6, 5, 2, 8]
509 | [6, 5, 2, 10]
510 | [6, 5, 3, 6]
511 | [6, 5, 4, 4]
512 | [6, 5, 5, 2]
513 | [6, 5, 6, 0]
514 | [6, 5, 7, 7]
515 | [6, 5, 8, 5]
516 | [6, 5, 9, 3]
517 | [6, 5, 10, 5]
518 | [6, 6, 0, 8]
519 | [6, 6, 0, 10]
520 | [6, 6, 1, 6]
521 | [6, 6, 2, 4]
522 | [6, 6, 3, 2]
523 | [6, 6, 4, 0]
524 | [6, 6, 5, 7]
525 | [6, 6, 6, 5]
526 | [6, 6, 7, 3]
527 | [6, 6, 8, 1]
528 | [6, 6, 9, 8]
529 | [6, 6, 9, 10]
530 | [6, 6, 10, 1]
531 | [6, 7, 0, 4]
532 | [6, 7, 1, 2]
533 | [6, 7, 2, 0]
534 | [6, 7, 3, 7]
535 | [6, 7, 4, 5]
536 | [6, 7, 5, 3]
537 | [6, 7, 6, 1]
538 | [6, 7, 7, 8]
539 | [6, 7, 7, 10]
540 | [6, 7, 8, 6]
541 | [6, 7, 9, 4]
542 | [6, 7, 10, 6]
543 | [6, 8, 0, 0]
544 | [6, 8, 1, 7]
545 | [6, 8, 2, 5]
546 | [6, 8, 3, 3]
547 | [6, 8, 4, 1]
548 | [6, 8, 5, 8]
549 | [6, 8, 5, 10]
550 | [6, 8, 6, 6]
551 | [6, 8, 7, 4]
552 | [6, 8, 8, 2]
553 | [6, 8, 9, 0]
554 | [6, 8, 10, 2]
555 | [6, 9, 0, 5]
556 | [6, 9, 1, 3]
557 | [6, 9, 2, 1]
558 | [6, 9, 3, 8]
559 | [6, 9, 3, 10]
560 | [6, 9, 4, 6]
561 | [6, 9, 5, 4]
562 | [6, 9, 6, 2]
563 | [6, 9, 7, 0]
564 | [6, 9, 8, 7]
565 | [6, 9, 9, 4]
566 | [6, 9, 10, 7]
567 | [6, 10, 0, 0]
568 | [6, 10, 1, 7]
569 | [6, 10, 2, 5]
570 | [6, 10, 3, 3]
571 | [6, 10, 4, 1]
572 | [6, 10, 5, 8]
573 | [6, 10, 5, 10]
574 | [6, 10, 6, 6]
575 | [6, 10, 7, 4]
576 | [6, 10, 8, 2]
577 | [6, 10, 9, 0]
578 | [6, 10, 10, 2]
579 | [7, 0, 8, 9]
580 | [7, 0, 10, 9]
581 | [7, 1, 6, 9]
582 | [7, 2, 4, 9]
583 | [7, 3, 2, 9]
584 | [7, 4, 0, 9]
585 | [7, 4, 9, 9]
586 | [7, 5, 7, 9]
587 | [7, 6, 5, 9]
588 | [7, 7, 3, 9]
589 | [7, 8, 1, 9]
590 | [7, 9, 8, 9]
591 | [7, 9, 10, 9]
592 | [7, 10, 1, 9]
593 | [8, 0, 0, 7]
594 | [8, 0, 1, 5]
595 | [8, 0, 2, 3]
596 | [8, 0, 3, 1]
597 | [8, 0, 4, 8]
598 | [8, 0, 4, 10]
599 | [8, 0, 5, 6]
600 | [8, 0, 6, 4]
601 | [8, 0, 7, 2]
602 | [8, 0, 8, 0]
603 | [8, 0, 9, 7]
604 | [8, 0, 10, 0]
605 | [8, 1, 0, 3]
606 | [8, 1, 1, 1]
607 | [8, 1, 2, 8]
608 | [8, 1, 2, 10]
609 | [8, 1, 3, 6]
610 | [8, 1, 4, 4]
611 | [8, 1, 5, 2]
612 | [8, 1, 6, 0]
613 | [8, 1, 7, 7]
614 | [8, 1, 8, 5]
615 | [8, 1, 9, 3]
616 | [8, 1, 10, 5]
617 | [8, 2, 0, 8]
618 | [8, 2, 0, 10]
619 | [8, 2, 1, 6]
620 | [8, 2, 2, 4]
621 | [8, 2, 3, 2]
622 | [8, 2, 4, 0]
623 | [8, 2, 5, 7]
624 | [8, 2, 6, 5]
625 | [8, 2, 7, 3]
626 | [8, 2, 8, 1]
627 | [8, 2, 9, 8]
628 | [8, 2, 9, 10]
629 | [8, 2, 10, 1]
630 | [8, 3, 0, 4]
631 | [8, 3, 1, 2]
632 | [8, 3, 2, 0]
633 | [8, 3, 3, 7]
634 | [8, 3, 4, 5]
635 | [8, 3, 5, 3]
636 | [8, 3, 6, 1]
637 | [8, 3, 7, 8]
638 | [8, 3, 7, 10]
639 | [8, 3, 8, 6]
640 | [8, 3, 9, 4]
641 | [8, 3, 10, 6]
642 | [8, 4, 0, 0]
643 | [8, 4, 1, 7]
644 | [8, 4, 2, 5]
645 | [8, 4, 3, 3]
646 | [8, 4, 4, 1]
647 | [8, 4, 5, 8]
648 | [8, 4, 5, 10]
649 | [8, 4, 6, 6]
650 | [8, 4, 7, 4]
651 | [8, 4, 8, 2]
652 | [8, 4, 9, 0]
653 | [8, 4, 10, 2]
654 | [8, 5, 0, 5]
655 | [8, 5, 1, 3]
656 | [8, 5, 2, 1]
657 | [8, 5, 3, 8]
658 | [8, 5, 3, 10]
659 | [8, 5, 4, 6]
660 | [8, 5, 5, 4]
661 | [8, 5, 6, 2]
662 | [8, 5, 7, 0]
663 | [8, 5, 8, 7]
664 | [8, 5, 9, 5]
665 | [8, 5, 10, 7]
666 | [8, 6, 0, 1]
667 | [8, 6, 1, 8]
668 | [8, 6, 1, 10]
669 | [8, 6, 2, 6]
670 | [8, 6, 3, 4]
671 | [8, 6, 4, 2]
672 | [8, 6, 5, 0]
673 | [8, 6, 6, 7]
674 | [8, 6, 7, 5]
675 | [8, 6, 8, 3]
676 | [8, 6, 9, 1]
677 | [8, 6, 10, 3]
678 | [8, 7, 0, 6]
679 | [8, 7, 1, 4]
680 | [8, 7, 2, 2]
681 | [8, 7, 3, 0]
682 | [8, 7, 4, 7]
683 | [8, 7, 5, 5]
684 | [8, 7, 6, 3]
685 | [8, 7, 7, 1]
686 | [8, 7, 8, 8]
687 | [8, 7, 8, 10]
688 | [8, 7, 9, 6]
689 | [8, 7, 10, 8]
690 | [8, 7, 10, 10]
691 | [8, 8, 0, 2]
692 | [8, 8, 1, 0]
693 | [8, 8, 2, 7]
694 | [8, 8, 3, 5]
695 | [8, 8, 4, 3]
696 | [8, 8, 5, 1]
697 | [8, 8, 6, 8]
698 | [8, 8, 6, 10]
699 | [8, 8, 7, 6]
700 | [8, 8, 8, 4]
701 | [8, 8, 9, 2]
702 | [8, 8, 10, 4]
703 | [8, 9, 0, 7]
704 | [8, 9, 1, 5]
705 | [8, 9, 2, 3]
706 | [8, 9, 3, 1]
707 | [8, 9, 4, 8]
708 | [8, 9, 4, 10]
709 | [8, 9, 5, 6]
710 | [8, 9, 6, 4]
711 | [8, 9, 7, 2]
712 | [8, 9, 8, 0]
713 | [8, 9, 9, 6]
714 | [8, 9, 10, 0]
715 | [8, 10, 0, 2]
716 | [8, 10, 1, 0]
717 | [8, 10, 2, 7]
718 | [8, 10, 3, 5]
719 | [8, 10, 4, 3]
720 | [8, 10, 5, 1]
721 | [8, 10, 6, 8]
722 | [8, 10, 6, 10]
723 | [8, 10, 7, 6]
724 | [8, 10, 8, 4]
725 | [8, 10, 9, 2]
726 | [8, 10, 10, 4]
727 | [9, 0, 0, 9]
728 | [9, 0, 9, 9]
729 | [9, 1, 7, 9]
730 | [9, 2, 5, 9]
731 | [9, 3, 3, 9]
732 | [9, 4, 1, 9]
733 | [9, 5, 8, 9]
734 | [9, 5, 10, 9]
735 | [9, 6, 6, 9]
736 | [9, 7, 4, 9]
737 | [9, 8, 2, 9]
738 | [9, 9, 0, 9]
739 | [9, 9, 8, 8]
740 | [9, 10, 2, 9]
741 | [10, 0, 4, 9]
742 | [10, 1, 2, 9]
743 | [10, 2, 0, 9]
744 | [10, 2, 9, 9]
745 | [10, 3, 7, 9]
746 | [10, 4, 5, 9]
747 | [10, 5, 3, 9]
748 | [10, 6, 1, 9]
749 | [10, 7, 8, 9]
750 | [10, 7, 10, 9]
751 | [10, 8, 6, 9]
752 | [10, 9, 4, 9]
753 | [10, 9, 8, 10]
754 | [10, 10, 6, 9]
755 |
--------------------------------------------------------------------------------
/examples/specification_nonsymmetric_admissible_set.py:
--------------------------------------------------------------------------------
1 | """Generating maximal admissible sets of different dimensionalities."""
2 | import itertools
3 | import numpy as np
4 |
5 |
6 | def block_children(scores, admissible_set, new_element):
7 | """Modifies `scores` to -inf for elements blocked by `new_element`."""
8 | n = admissible_set.shape[-1]
9 | powers = np.array([3 ** i for i in range(n - 1, -1, -1)], dtype=np.int32)
10 |
11 | invalid_vals_raw = {
12 | (0, 0): (0,),
13 | (0, 1): (1,),
14 | (0, 2): (2,),
15 | (1, 0): (1,),
16 | (1, 1): (0, 1, 2),
17 | (1, 2): (1, 2),
18 | (2, 0): (2,),
19 | (2, 1): (1, 2),
20 | (2, 2): (0, 1, 2),
21 | }
22 | invalid_vals = [[np.array(invalid_vals_raw[(i, j)], dtype=np.int32)
23 | for j in range(3)] for i in range(3)]
24 |
25 | # Block 2^w elements with the same support as `new_element`.
26 | w = np.count_nonzero(new_element)
27 | all_12s = np.array(list(itertools.product((1, 2), repeat=w)), dtype=np.int32)
28 | blocking = np.einsum('aw,w->a', all_12s, powers[new_element != 0])
29 | scores[blocking] = -np.inf
30 |
31 | # Block elements disallowed by a pair of an extant point and `new_element`.
32 | for extant_element in admissible_set:
33 | blocking = np.zeros(shape=(1,), dtype=np.int32)
34 | for e1, e2, power in zip(extant_element, new_element, powers):
35 | blocking = (blocking[:, None] + (invalid_vals[e1][e2] * power)[None, :]
36 | ).ravel()
37 | scores[blocking] = -np.inf
38 |
39 |
40 | def solve(n: int, w: int) -> np.ndarray:
41 | """Builds a large constant-weight-w n-dimensional admissible set.
42 |
43 | A constant weight admissible set is a subset of {0, 1, 2}^n. The weight of
44 | each element is the number of non-zero entries. The admissible set consists of
45 | a maximum of n choose w elements where w is the desired weight. Any three
46 | distinct elements of the set satisfy the triplet constraints, that is, there
47 | exists an index k such that the k-th components of the elements are either
48 | {0, 0, 1}, {0, 0, 2} or {0, 1, 2}.
49 |
50 | Args:
51 | n: dimension
52 | w: weight
53 |
54 | Returns:
55 | An array of shape [admissible_set_size, n], with entries in {0, 1, 2}.
56 | """
57 | children = np.array(list(itertools.product((0, 1, 2), repeat=n)),
58 | dtype=np.int32)
59 |
60 | scores = -np.inf * np.ones((3 ** n,), dtype=np.float32)
61 | for child_index, child in enumerate(children):
62 | if sum(child == 0) == n-w:
63 | scores[child_index] = priority(np.array(child), n, w)
64 |
65 | max_admissible_set = np.empty((0, n), dtype=np.int32)
66 | while np.any(scores != -np.inf):
67 | # Find element with largest score
68 | max_index = np.argmax(scores)
69 | child = children[max_index]
70 | block_children(scores, max_admissible_set, child)
71 | max_admissible_set = np.concatenate([max_admissible_set, child[None]],
72 | axis=0)
73 | return max_admissible_set
74 |
75 |
76 | @funsearch.run
77 | def evaluate(set_description: tuple[int, int]) -> int:
78 | """Returns the size of the constructed admissible set."""
79 | n, w = set_description
80 | admissible_set = solve(n, w)
81 | return len(admissible_set)
82 |
83 |
84 | @funsearch.evolve
85 | def priority(el: tuple[int, ...], n: int, w: int) -> float:
86 | """Trivial scoring function.
87 |
88 | Args:
89 | el: an element to be potentially added to the current admissible set.
90 | n: dimensionality of admissible set.
91 | w: weight of admissible set.
92 |
93 | Returns:
94 | A number reflecting the priority with which we want to add `el` to the set.
95 | """
96 | return 0.0
97 |
--------------------------------------------------------------------------------
/funsearch/__init__.py:
--------------------------------------------------------------------------------
1 | def evolve(func):
2 | """@funsearch.evolve decorator is used in the problem specification to detect the method that
3 | should be evolved using LLM.
4 | """
5 | return func
6 |
7 |
8 | def run(func):
9 | """@funsearch.run decorator is used in the problem specification to detect the method that
10 | should be used to verify and grade generated code.
11 | """
12 | return func
13 |
--------------------------------------------------------------------------------
/funsearch/__main__.py:
--------------------------------------------------------------------------------
1 | import json
2 | import logging
3 | import os
4 | import pathlib
5 | import pickle
6 | import time
7 |
8 | import click
9 | import llm
10 | from dotenv import load_dotenv
11 |
12 |
13 | from funsearch import config, core, sandbox, sampler, programs_database, code_manipulation, evaluator
14 |
15 | LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper()
16 | logging.basicConfig(level=LOGLEVEL)
17 |
18 |
19 | def get_all_subclasses(cls):
20 | all_subclasses = []
21 |
22 | for subclass in cls.__subclasses__():
23 | all_subclasses.append(subclass)
24 | all_subclasses.extend(get_all_subclasses(subclass))
25 |
26 | return all_subclasses
27 |
28 |
29 | SANDBOX_TYPES = get_all_subclasses(sandbox.DummySandbox) + [sandbox.DummySandbox]
30 | SANDBOX_NAMES = [c.__name__ for c in SANDBOX_TYPES]
31 |
32 |
33 | def parse_input(filename_or_data: str):
34 | if len(filename_or_data) == 0:
35 | raise Exception("No input data specified")
36 | p = pathlib.Path(filename_or_data)
37 | if p.exists():
38 | if p.name.endswith(".json"):
39 | return json.load(open(filename_or_data, "r"))
40 | if p.name.endswith(".pickle"):
41 | return pickle.load(open(filename_or_data, "rb"))
42 | raise Exception("Unknown file format or filename")
43 | if "," not in filename_or_data:
44 | data = [filename_or_data]
45 | else:
46 | data = filename_or_data.split(",")
47 | if data[0].isnumeric():
48 | f = int if data[0].isdecimal() else float
49 | data = [f(v) for v in data]
50 | return data
51 |
52 | @click.group()
53 | @click.pass_context
54 | def main(ctx):
55 | pass
56 |
57 |
58 | @main.command()
59 | @click.argument("spec_file", type=click.File("r"))
60 | @click.argument('inputs')
61 | @click.option('--model_name', default="gpt-3.5-turbo-instruct", help='LLM model')
62 | @click.option('--output_path', default="./data/", type=click.Path(file_okay=False), help='path for logs and data')
63 | @click.option('--load_backup', default=None, type=click.File("rb"), help='Use existing program database')
64 | @click.option('--iterations', default=-1, type=click.INT, help='Max iterations per sampler')
65 | @click.option('--samplers', default=15, type=click.INT, help='Samplers')
66 | @click.option('--sandbox_type', default="ContainerSandbox", type=click.Choice(SANDBOX_NAMES), help='Sandbox type')
67 | def run(spec_file, inputs, model_name, output_path, load_backup, iterations, samplers, sandbox_type):
68 | """ Execute function-search algorithm:
69 |
70 | \b
71 | SPEC_FILE is a python module that provides the basis of the LLM prompt as
72 | well as the evaluation metric.
73 | See examples/cap_set_spec.py for an example.\n
74 | \b
75 | INPUTS input filename ending in .json or .pickle, or a comma-separated
76 | input data. The files are expected contain a list with at least
77 | one element. Elements shall be passed to the solve() method
78 | one by one. Examples
79 | 8
80 | 8,9,10
81 | ./examples/cap_set_input_data.json
82 | """
83 |
84 | # Load environment variables from .env file.
85 | #
86 | # Using OpenAI APIs with 'llm' package requires setting the variable
87 | # OPENAI_API_KEY=sk-...
88 | # See 'llm' package on how to use other providers.
89 | load_dotenv()
90 |
91 | timestamp = str(int(time.time()))
92 | log_path = pathlib.Path(output_path) / timestamp
93 | if not log_path.exists():
94 | log_path.mkdir(parents=True)
95 | logging.info(f"Writing logs to {log_path}")
96 |
97 | model = llm.get_model(model_name)
98 | model.key = model.get_key()
99 | lm = sampler.LLM(2, model, log_path)
100 |
101 | specification = spec_file.read()
102 | function_to_evolve, function_to_run = core._extract_function_names(specification)
103 | template = code_manipulation.text_to_program(specification)
104 |
105 | conf = config.Config(num_evaluators=1)
106 | database = programs_database.ProgramsDatabase(
107 | conf.programs_database, template, function_to_evolve, identifier=timestamp)
108 | if load_backup:
109 | database.load(load_backup)
110 |
111 | inputs = parse_input(inputs)
112 |
113 | sandbox_class = next(c for c in SANDBOX_TYPES if c.__name__ == sandbox_type)
114 | evaluators = [evaluator.Evaluator(
115 | database,
116 | sandbox_class(base_path=log_path),
117 | template,
118 | function_to_evolve,
119 | function_to_run,
120 | inputs,
121 | ) for _ in range(conf.num_evaluators)]
122 |
123 | # We send the initial implementation to be analysed by one of the evaluators.
124 | initial = template.get_function(function_to_evolve).body
125 | evaluators[0].analyse(initial, island_id=None, version_generated=None)
126 | assert len(database._islands[0]._clusters) > 0, ("Initial analysis failed. Make sure that Sandbox works! "
127 | "See e.g. the error files under sandbox data.")
128 |
129 | samplers = [sampler.Sampler(database, evaluators, lm)
130 | for _ in range(samplers)]
131 |
132 | core.run(samplers, database, iterations)
133 |
134 |
135 | @main.command()
136 | @click.argument("db_file", type=click.File("rb"))
137 | def ls(db_file):
138 | """List programs from a stored database (usually in data/backups/ )"""
139 | conf = config.Config(num_evaluators=1)
140 |
141 | # A bit silly way to list programs. This probably does not work if config has changed any way
142 | database = programs_database.ProgramsDatabase(
143 | conf.programs_database, None, "", identifier="")
144 | database.load(db_file)
145 |
146 | progs = database.get_best_programs_per_island()
147 | print("Found {len(progs)} programs")
148 | for i, (prog, score) in enumerate(progs):
149 | print(f"{i}: Program with score {score}")
150 | print(prog)
151 | print("\n")
152 |
153 |
154 | if __name__ == '__main__':
155 | main()
156 |
--------------------------------------------------------------------------------
/funsearch/code_manipulation.py:
--------------------------------------------------------------------------------
1 | # Copyright 2023 DeepMind Technologies Limited
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | # ==============================================================================
15 |
16 | """Tools for manipulating Python code.
17 |
18 | It implements 2 classes representing unities of code:
19 | - Function, containing all the information we need about functions: name, args,
20 | body and optionally a return type and a docstring.
21 | - Program, which contains a code preface (which could be imports, global
22 | variables and classes, ...) and a list of Functions.
23 | """
24 | import ast
25 | from collections.abc import Iterator, MutableSet, Sequence
26 | import dataclasses
27 | import io
28 | import tokenize
29 |
30 | from absl import logging
31 |
32 |
33 | @dataclasses.dataclass
34 | class Function:
35 | """A parsed Python function."""
36 |
37 | name: str
38 | args: str
39 | body: str
40 | return_type: str | None = None
41 | docstring: str | None = None
42 |
43 | def __str__(self) -> str:
44 | return_type = f' -> {self.return_type}' if self.return_type else ''
45 |
46 | function = f'def {self.name}({self.args}){return_type}:\n'
47 | if self.docstring:
48 | # self.docstring is already indented on every line except the first one.
49 | # Here, we assume the indentation is always two spaces.
50 | new_line = '\n' if self.body else ''
51 | function += f' """{self.docstring}"""{new_line}'
52 | # self.body is already indented.
53 | function += self.body + '\n\n'
54 | return function
55 |
56 | def __setattr__(self, name: str, value: str) -> None:
57 | # Ensure there aren't leading & trailing new lines in `body`.
58 | if name == 'body':
59 | value = value.strip('\n')
60 | # Ensure there aren't leading & trailing quotes in `docstring``.
61 | if name == 'docstring' and value is not None:
62 | if '"""' in value:
63 | value = value.strip()
64 | value = value.replace('"""', '')
65 | super().__setattr__(name, value)
66 |
67 |
68 | @dataclasses.dataclass(frozen=True)
69 | class Program:
70 | """A parsed Python program."""
71 |
72 | # `preface` is everything from the beginning of the code till the first
73 | # function is found.
74 | preface: str
75 | functions: list[Function]
76 |
77 | def __str__(self) -> str:
78 | program = f'{self.preface}\n' if self.preface else ''
79 | program += '\n'.join([str(f) for f in self.functions])
80 | return program
81 |
82 | def find_function_index(self, function_name: str) -> int:
83 | """Returns the index of input function name."""
84 | function_names = [f.name for f in self.functions]
85 | count = function_names.count(function_name)
86 | if count == 0:
87 | raise ValueError(
88 | f'function {function_name} does not exist in program:\n{str(self)}'
89 | )
90 | if count > 1:
91 | raise ValueError(
92 | f'function {function_name} exists more than once in program:\n'
93 | f'{str(self)}'
94 | )
95 | index = function_names.index(function_name)
96 | return index
97 |
98 | def get_function(self, function_name: str) -> Function:
99 | index = self.find_function_index(function_name)
100 | return self.functions[index]
101 |
102 |
103 | class ProgramVisitor(ast.NodeVisitor):
104 | """Parses code to collect all required information to produce a `Program`.
105 |
106 | Note that we do not store function decorators.
107 | """
108 |
109 | def __init__(self, sourcecode: str):
110 | self._codelines: list[str] = sourcecode.splitlines()
111 |
112 | self._preface: str = ''
113 | self._functions: list[Function] = []
114 | self._current_function: str | None = None
115 |
116 | def visit_FunctionDef(self, # pylint: disable=invalid-name
117 | node: ast.FunctionDef) -> None:
118 | """Collects all information about the function being parsed."""
119 | if node.col_offset == 0: # We only care about first level functions.
120 | self._current_function = node.name
121 | if not self._functions:
122 | self._preface = '\n'.join(self._codelines[:node.lineno - 1])
123 | function_end_line = node.end_lineno
124 | body_start_line = node.body[0].lineno - 1
125 | # Extract the docstring.
126 | docstring = None
127 | if isinstance(node.body[0], ast.Expr) and isinstance(node.body[0].value,
128 | ast.Str):
129 | docstring = f' """{ast.literal_eval(ast.unparse(node.body[0]))}"""'
130 | if len(node.body) > 1:
131 | body_start_line = node.body[1].lineno - 1
132 | else:
133 | body_start_line = function_end_line
134 |
135 | self._functions.append(Function(
136 | name=node.name,
137 | args=ast.unparse(node.args),
138 | return_type=ast.unparse(node.returns) if node.returns else None,
139 | docstring=docstring,
140 | body='\n'.join(self._codelines[body_start_line:function_end_line]),
141 | ))
142 | self.generic_visit(node)
143 |
144 | def return_program(self) -> Program:
145 | return Program(preface=self._preface, functions=self._functions)
146 |
147 |
148 | def text_to_program(text: str) -> Program:
149 | """Returns Program object by parsing input text using Python AST."""
150 | try:
151 | # We assume that the program is composed of some preface (e.g. imports,
152 | # classes, assignments, ...) followed by a sequence of functions.
153 | tree = ast.parse(text)
154 | visitor = ProgramVisitor(text)
155 | visitor.visit(tree)
156 | return visitor.return_program()
157 | except Exception as e:
158 | logging.warning('Failed parsing %s', text)
159 | raise e
160 |
161 |
162 | def text_to_function(text: str) -> Function:
163 | """Returns Function object by parsing input text using Python AST."""
164 | program = text_to_program(text)
165 | if len(program.functions) != 1:
166 | raise ValueError(f'Only one function expected, got {len(program.functions)}'
167 | f':\n{program.functions}')
168 | return program.functions[0]
169 |
170 |
171 | def _tokenize(code: str) -> Iterator[tokenize.TokenInfo]:
172 | """Transforms `code` into Python tokens."""
173 | code_bytes = code.encode()
174 | code_io = io.BytesIO(code_bytes)
175 | return tokenize.tokenize(code_io.readline)
176 |
177 |
178 | def _untokenize(tokens: Sequence[tokenize.TokenInfo]) -> str:
179 | """Transforms a list of Python tokens into code."""
180 | code_bytes = tokenize.untokenize(tokens)
181 | return code_bytes.decode()
182 |
183 |
184 | def _yield_token_and_is_call(
185 | code: str) -> Iterator[tuple[tokenize.TokenInfo, bool]]:
186 | """Yields each token with a bool indicating whether it is a function call."""
187 | try:
188 | tokens = _tokenize(code)
189 | prev_token = None
190 | is_attribute_access = False
191 | for token in tokens:
192 | if (prev_token and # If the previous token exists and
193 | prev_token.type == tokenize.NAME and # it is a Python identifier
194 | token.type == tokenize.OP and # and the current token is a delimiter
195 | token.string == '('): # and in particular it is '('.
196 | yield prev_token, not is_attribute_access
197 | is_attribute_access = False
198 | else:
199 | if prev_token:
200 | is_attribute_access = (
201 | prev_token.type == tokenize.OP and prev_token.string == '.'
202 | )
203 | yield prev_token, False
204 | prev_token = token
205 | if prev_token:
206 | yield prev_token, False
207 | except Exception as e:
208 | logging.warning('Failed parsing %s', code)
209 | raise e
210 |
211 |
212 | def rename_function_calls(code: str, source_name: str, target_name: str) -> str:
213 | """Renames function calls from `source_name` to `target_name`."""
214 | if source_name not in code:
215 | return code
216 | modified_tokens = []
217 | for token, is_call in _yield_token_and_is_call(code):
218 | if is_call and token.string == source_name:
219 | # Replace the function name token
220 | modified_token = tokenize.TokenInfo(
221 | type=token.type,
222 | string=target_name,
223 | start=token.start,
224 | end=token.end,
225 | line=token.line,
226 | )
227 | modified_tokens.append(modified_token)
228 | else:
229 | modified_tokens.append(token)
230 | return _untokenize(modified_tokens)
231 |
232 |
233 | def get_functions_called(code: str) -> MutableSet[str]:
234 | """Returns the set of all functions called in `code`."""
235 | return set(token.string for token, is_call in
236 | _yield_token_and_is_call(code) if is_call)
237 |
238 |
239 | def yield_decorated(code: str, module: str, name: str) -> Iterator[str]:
240 | """Yields names of functions decorated with `@module.name` in `code`."""
241 | tree = ast.parse(code)
242 | for node in ast.walk(tree):
243 | if isinstance(node, ast.FunctionDef):
244 | for decorator in node.decorator_list:
245 | attribute = None
246 | if isinstance(decorator, ast.Attribute):
247 | attribute = decorator
248 | elif isinstance(decorator, ast.Call):
249 | attribute = decorator.func
250 | if (attribute is not None
251 | and attribute.value.id == module
252 | and attribute.attr == name):
253 | yield node.name
254 |
--------------------------------------------------------------------------------
/funsearch/config.py:
--------------------------------------------------------------------------------
1 | # Copyright 2023 DeepMind Technologies Limited
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | # ==============================================================================
15 |
16 | """Configuration of a FunSearch experiment."""
17 | import dataclasses
18 |
19 |
20 | @dataclasses.dataclass(frozen=True)
21 | class ProgramsDatabaseConfig:
22 | """Configuration of a ProgramsDatabase.
23 |
24 | Attributes:
25 | functions_per_prompt: Number of previous programs to include in prompts.
26 | num_islands: Number of islands to maintain as a diversity mechanism.
27 | reset_period: How often (in seconds) the weakest islands should be reset.
28 | cluster_sampling_temperature_init: Initial temperature for softmax sampling
29 | of clusters within an island.
30 | cluster_sampling_temperature_period: Period of linear decay of the cluster
31 | sampling temperature.
32 | backup_period: Number of iterations before backing up the program database on disk
33 | backup_folder: Path for automatic backups
34 | """
35 | functions_per_prompt: int = 2
36 | num_islands: int = 10
37 | reset_period: int = 4 * 60 * 60
38 | cluster_sampling_temperature_init: float = 0.1
39 | cluster_sampling_temperature_period: int = 30_000
40 | backup_period: int = 30
41 | backup_folder: str = './data/backups'
42 |
43 |
44 | @dataclasses.dataclass(frozen=True)
45 | class Config:
46 | """Configuration of a FunSearch experiment.
47 |
48 | Attributes:
49 | programs_database: Configuration of the evolutionary algorithm.
50 | num_samplers: Number of independent Samplers in the experiment. A value
51 | larger than 1 only has an effect when the samplers are able to execute
52 | in parallel, e.g. on different matchines of a distributed system.
53 | num_evaluators: Number of independent program Evaluators in the experiment.
54 | A value larger than 1 is only expected to be useful when the Evaluators
55 | can execute in parallel as part of a distributed system.
56 | samples_per_prompt: How many independently sampled program continuations to
57 | obtain for each prompt.
58 | """
59 | programs_database: ProgramsDatabaseConfig = dataclasses.field(
60 | default_factory=ProgramsDatabaseConfig)
61 | num_samplers: int = 15
62 | num_evaluators: int = 140
63 | samples_per_prompt: int = 4
64 |
--------------------------------------------------------------------------------
/funsearch/container/Dockerfile:
--------------------------------------------------------------------------------
1 | # the python version in the sandbox should be the same as in the host so that cloudpickle works reliably
2 | ARG PYTHON_VERSION=3.11.6
3 |
4 | FROM docker.io/python:${PYTHON_VERSION}
5 |
6 | ARG INSTALL_PACKAGES
7 | RUN pip install cloudpickle ${INSTALL_PACKAGES}
8 |
--------------------------------------------------------------------------------
/funsearch/container/container_main.py:
--------------------------------------------------------------------------------
1 | """This file will be used as an executable script by the ContainerSandbox and ExternalProcessSandbox.
2 | """
3 | import logging
4 | import pickle
5 | import sys
6 |
7 |
8 | def main(prog_file: str, input_file: str, output_file: str):
9 | """The method takes executable function as a cloudpickle file, then executes it with input data,
10 | and writes the output data to another file."""
11 | logging.debug(f"Running main(): {prog_file}, {input_file}, {output_file}")
12 | with open(prog_file, "rb") as f:
13 | func = pickle.load(f)
14 |
15 | with open(input_file, "rb") as input_f:
16 | input_data = pickle.load(input_f)
17 |
18 | ret = func(input_data)
19 | with open(output_file, "wb") as of:
20 | logging.debug(f"Writing output to {output_file}")
21 | pickle.dump(ret, of)
22 |
23 |
24 | if __name__ == '__main__':
25 | if len(sys.argv) != 4:
26 | sys.exit(-1)
27 | main(sys.argv[1], sys.argv[2], sys.argv[3])
28 |
--------------------------------------------------------------------------------
/funsearch/core.py:
--------------------------------------------------------------------------------
1 | # Copyright 2023 DeepMind Technologies Limited
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | # ==============================================================================
15 |
16 | """A single-threaded implementation of the FunSearch pipeline."""
17 | import logging
18 |
19 | from funsearch import code_manipulation
20 |
21 |
22 | def _extract_function_names(specification: str) -> tuple[str, str]:
23 | """Returns the name of the function to evolve and of the function to run."""
24 | run_functions = list(
25 | code_manipulation.yield_decorated(specification, 'funsearch', 'run'))
26 | if len(run_functions) != 1:
27 | raise ValueError('Expected 1 function decorated with `@funsearch.run`.')
28 | evolve_functions = list(
29 | code_manipulation.yield_decorated(specification, 'funsearch', 'evolve'))
30 | if len(evolve_functions) != 1:
31 | raise ValueError('Expected 1 function decorated with `@funsearch.evolve`.')
32 | return evolve_functions[0], run_functions[0]
33 |
34 |
35 | def run(samplers, database, iterations: int = -1):
36 | """Launches a FunSearch experiment."""
37 |
38 | try:
39 | # This loop can be executed in parallel on remote sampler machines. As each
40 | # sampler enters an infinite loop, without parallelization only the first
41 | # sampler will do any work.
42 | while iterations != 0:
43 | for s in samplers:
44 | s.sample()
45 | if iterations > 0:
46 | iterations -= 1
47 | except KeyboardInterrupt:
48 | logging.info("Keyboard interrupt. Stopping.")
49 | database.backup()
50 |
51 |
--------------------------------------------------------------------------------
/funsearch/evaluator.py:
--------------------------------------------------------------------------------
1 | # Copyright 2023 DeepMind Technologies Limited
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | # ==============================================================================
15 |
16 | """Class for evaluating programs proposed by the Sampler."""
17 | import ast
18 | import re
19 | from collections.abc import Sequence
20 | import copy
21 | from typing import Any, Tuple
22 |
23 | from funsearch import code_manipulation
24 | from funsearch import programs_database
25 | from funsearch import sandbox
26 |
27 | """
28 | Regex to find all methods named 'priority_vX'.
29 | With each match, start from the 'def priority_vX(' and continue until there's a new line with any of
30 | - a new 'def'
31 | - ` or ' or # without indentation
32 | """
33 | METHOD_MATCHER = re.compile(r"def priority_v\d\(.*?\) -> float:(?:\s*(?:[ \t]*(?!def|#|`|').*(?:\n|$)))+")
34 | METHOD_NAME_MATCHER = re.compile(r"priority_v\d+")
35 |
36 |
37 | class _FunctionLineVisitor(ast.NodeVisitor):
38 | """Visitor that finds the last line number of a function with a given name."""
39 |
40 | def __init__(self, target_function_name: str) -> None:
41 | self._target_function_name: str = target_function_name
42 | self._function_end_line: int | None = None
43 |
44 | def visit_FunctionDef(self, node: Any) -> None: # pylint: disable=invalid-name
45 | """Collects the end line number of the target function."""
46 | if node.name == self._target_function_name:
47 | self._function_end_line = node.end_lineno
48 | self.generic_visit(node)
49 |
50 | @property
51 | def function_end_line(self) -> int:
52 | """Line number of the final line of function `target_function_name`."""
53 | assert self._function_end_line is not None # Check internal correctness.
54 | return self._function_end_line
55 |
56 |
57 | def _find_method_implementation(generated_code: str) -> Tuple[str, str]:
58 | """Find the last 'def priority_vX()' method from generated code.
59 |
60 | Return the code and the name of the method.
61 | """
62 | matches = METHOD_MATCHER.findall(generated_code)
63 | if not matches:
64 | return "", ""
65 | last_match = matches[-1]
66 | name = METHOD_NAME_MATCHER.search(last_match).group()
67 | return last_match, name
68 |
69 |
70 | def _trim_function_body(generated_code: str) -> str:
71 | """Extracts the body of the generated function, trimming anything after it."""
72 | if not generated_code:
73 | return ''
74 | if not type(generated_code) is str:
75 | generated_code = str(generated_code)
76 |
77 | method_name = "fake_function_header"
78 | # Check is the response only a continuation for our prompt or full method implementation with header
79 | if "def priority_v" in generated_code:
80 | code, method_name = _find_method_implementation(generated_code)
81 | else:
82 | code = f'def {method_name}():\n{generated_code}'
83 |
84 | # Finally parse the code to make sure it's valid Python
85 | tree = None
86 | # We keep trying and deleting code from the end until the parser succeeds.
87 | while tree is None:
88 | try:
89 | tree = ast.parse(code)
90 | except SyntaxError as e:
91 | code = '\n'.join(code.splitlines()[:e.lineno - 1])
92 | if not code:
93 | # Nothing could be saved from `generated_code`
94 | return ''
95 |
96 | visitor = _FunctionLineVisitor(method_name)
97 | visitor.visit(tree)
98 | body_lines = code.splitlines()[1:visitor.function_end_line]
99 | return '\n'.join(body_lines) + '\n\n'
100 |
101 |
102 | def _sample_to_program(
103 | generated_code: str,
104 | version_generated: int | None,
105 | template: code_manipulation.Program,
106 | function_to_evolve: str,
107 | ) -> tuple[code_manipulation.Function, str]:
108 | """Returns the compiled generated function and the full runnable program."""
109 | body = _trim_function_body(generated_code)
110 | if version_generated is not None:
111 | body = code_manipulation.rename_function_calls(
112 | body,
113 | f'{function_to_evolve}_v{version_generated}',
114 | function_to_evolve)
115 |
116 | program = copy.deepcopy(template)
117 | evolved_function = program.get_function(function_to_evolve)
118 | evolved_function.body = body
119 | return evolved_function, str(program)
120 |
121 |
122 |
123 | def _calls_ancestor(program: str, function_to_evolve: str) -> bool:
124 | """Returns whether the generated function is calling an earlier version."""
125 | for name in code_manipulation.get_functions_called(program):
126 | # In `program` passed into this function the most recently generated
127 | # function has already been renamed to `function_to_evolve` (wihout the
128 | # suffix). Therefore any function call starting with `function_to_evolve_v`
129 | # is a call to an ancestor function.
130 | if name.startswith(f'{function_to_evolve}_v'):
131 | return True
132 | return False
133 |
134 |
135 | class Evaluator:
136 | """Class that analyses functions generated by LLMs."""
137 |
138 | def __init__(
139 | self,
140 | database: programs_database.ProgramsDatabase,
141 | sbox: sandbox.DummySandbox,
142 | template: code_manipulation.Program,
143 | function_to_evolve: str,
144 | function_to_run: str,
145 | inputs: Sequence[Any],
146 | timeout_seconds: int = 30,
147 | ):
148 | self._database = database
149 | self._template = template
150 | self._function_to_evolve = function_to_evolve
151 | self._function_to_run = function_to_run
152 | self._inputs = inputs
153 | self._timeout_seconds = timeout_seconds
154 | self._sandbox = sbox
155 |
156 | def analyse(
157 | self,
158 | sample: str,
159 | island_id: int | None,
160 | version_generated: int | None,
161 | ) -> None:
162 | """Compiles the sample into a program and executes it on test inputs."""
163 | new_function, program = _sample_to_program(
164 | sample, version_generated, self._template, self._function_to_evolve)
165 |
166 | scores_per_test = {}
167 | for current_input in self._inputs:
168 | test_output, runs_ok = self._sandbox.run(
169 | program, self._function_to_run, current_input, self._timeout_seconds)
170 | if (runs_ok and not _calls_ancestor(program, self._function_to_evolve)
171 | and test_output is not None):
172 | if not isinstance(test_output, (int, float)):
173 | raise ValueError('@function.run did not return an int/float score.')
174 | scores_per_test[current_input] = test_output
175 | if scores_per_test:
176 | self._database.register_program(new_function, island_id, scores_per_test)
177 |
--------------------------------------------------------------------------------
/funsearch/programs_database.py:
--------------------------------------------------------------------------------
1 | # Copyright 2023 DeepMind Technologies Limited
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | # ==============================================================================
15 |
16 | """A programs database that implements the evolutionary algorithm."""
17 | import pathlib
18 | import pickle
19 | from collections.abc import Mapping, Sequence
20 | import copy
21 | import dataclasses
22 | import time
23 | from typing import Any, Iterable, Tuple
24 |
25 | from absl import logging
26 | import numpy as np
27 | import scipy
28 |
29 | from funsearch import code_manipulation
30 | from funsearch import config as config_lib
31 |
32 | Signature = tuple[float, ...]
33 | ScoresPerTest = Mapping[Any, float]
34 |
35 |
36 | def _softmax(logits: np.ndarray, temperature: float) -> np.ndarray:
37 | """Returns the tempered softmax of 1D finite `logits`."""
38 | if not np.all(np.isfinite(logits)):
39 | non_finites = set(logits[~np.isfinite(logits)])
40 | raise ValueError(f'`logits` contains non-finite value(s): {non_finites}')
41 | if not np.issubdtype(logits.dtype, np.floating):
42 | logits = np.array(logits, dtype=np.float32)
43 |
44 | result = scipy.special.softmax(logits / temperature, axis=-1)
45 | # Ensure that probabilities sum to 1 to prevent error in `np.random.choice`.
46 | index = np.argmax(result)
47 | result[index] = 1 - np.sum(result[0:index]) - np.sum(result[index+1:])
48 | return result
49 |
50 |
51 | def _reduce_score(scores_per_test: ScoresPerTest) -> float:
52 | """Reduces per-test scores into a single score."""
53 | return scores_per_test[list(scores_per_test.keys())[-1]]
54 |
55 |
56 | def _get_signature(scores_per_test: ScoresPerTest) -> Signature:
57 | """Represents test scores as a canonical signature."""
58 | return tuple(scores_per_test[k] for k in sorted(scores_per_test.keys()))
59 |
60 |
61 | @dataclasses.dataclass(frozen=True)
62 | class Prompt:
63 | """A prompt produced by the ProgramsDatabase, to be sent to Samplers.
64 |
65 | Attributes:
66 | code: The prompt, ending with the header of the function to be completed.
67 | version_generated: The function to be completed is `_v{version_generated}`.
68 | island_id: Identifier of the island that produced the implementations
69 | included in the prompt. Used to direct the newly generated implementation
70 | into the same island.
71 | """
72 | code: str
73 | version_generated: int
74 | island_id: int
75 |
76 |
77 | class ProgramsDatabase:
78 | """A collection of programs, organized as islands."""
79 |
80 | def __init__(
81 | self,
82 | config: config_lib.ProgramsDatabaseConfig,
83 | template: code_manipulation.Program,
84 | function_to_evolve: str,
85 | identifier: str = "",
86 | ) -> None:
87 | self._config: config_lib.ProgramsDatabaseConfig = config
88 | self._template: code_manipulation.Program = template
89 | self._function_to_evolve: str = function_to_evolve
90 |
91 | # Initialize empty islands.
92 | self._islands: list[Island] = []
93 | for _ in range(config.num_islands):
94 | self._islands.append(
95 | Island(template, function_to_evolve, config.functions_per_prompt,
96 | config.cluster_sampling_temperature_init,
97 | config.cluster_sampling_temperature_period))
98 | self._best_score_per_island: list[float] = (
99 | [-float('inf')] * config.num_islands)
100 | self._best_program_per_island: list[code_manipulation.Function | None] = (
101 | [None] * config.num_islands)
102 | self._best_scores_per_test_per_island: list[ScoresPerTest | None] = (
103 | [None] * config.num_islands)
104 |
105 | self._last_reset_time: float = time.time()
106 | self._program_counter = 0
107 | self._backups_done = 0
108 | self.identifier = identifier
109 |
110 | def get_best_programs_per_island(self) -> Iterable[Tuple[code_manipulation.Function | None]]:
111 | return sorted(zip(self._best_program_per_island, self._best_score_per_island), key=lambda t: t[1], reverse=True)
112 |
113 | def save(self, file):
114 | """Save database to a file"""
115 | data = {}
116 | keys = ["_islands", "_best_score_per_island", "_best_program_per_island", "_best_scores_per_test_per_island"]
117 | for key in keys:
118 | data[key] = getattr(self, key)
119 | pickle.dump(data, file)
120 |
121 | def load(self, file):
122 | """Load previously saved database"""
123 | data = pickle.load(file)
124 | for key in data.keys():
125 | setattr(self, key, data[key])
126 |
127 | def backup(self):
128 | filename = f"program_db_{self._function_to_evolve}_{self.identifier}_{self._backups_done}.pickle"
129 | p = pathlib.Path(self._config.backup_folder)
130 | if not p.exists():
131 | p.mkdir(parents=True, exist_ok=True)
132 | filepath = p / filename
133 | logging.info(f"Saving backup to {filepath}.")
134 |
135 | with open(filepath, mode="wb") as f:
136 | self.save(f)
137 | self._backups_done += 1
138 |
139 | def get_prompt(self) -> Prompt:
140 | """Returns a prompt containing implementations from one chosen island."""
141 | island_id = np.random.randint(len(self._islands))
142 | code, version_generated = self._islands[island_id].get_prompt()
143 | return Prompt(code, version_generated, island_id)
144 |
145 | def _register_program_in_island(
146 | self,
147 | program: code_manipulation.Function,
148 | island_id: int,
149 | scores_per_test: ScoresPerTest,
150 | ) -> None:
151 | """Registers `program` in the specified island."""
152 | self._islands[island_id].register_program(program, scores_per_test)
153 | score = _reduce_score(scores_per_test)
154 | if score > self._best_score_per_island[island_id]:
155 | self._best_program_per_island[island_id] = program
156 | self._best_scores_per_test_per_island[island_id] = scores_per_test
157 | self._best_score_per_island[island_id] = score
158 | logging.info('Best score of island %d increased to %s', island_id, score)
159 |
160 | def register_program(
161 | self,
162 | program: code_manipulation.Function,
163 | island_id: int | None,
164 | scores_per_test: ScoresPerTest,
165 | ) -> None:
166 | """Registers `program` in the database."""
167 | # In an asynchronous implementation we should consider the possibility of
168 | # registering a program on an island that had been reset after the prompt
169 | # was generated. Leaving that out here for simplicity.
170 | if island_id is None:
171 | # This is a program added at the beginning, so adding it to all islands.
172 | for island_id in range(len(self._islands)):
173 | self._register_program_in_island(program, island_id, scores_per_test)
174 | else:
175 | self._register_program_in_island(program, island_id, scores_per_test)
176 |
177 | # Check whether it is time to reset an island.
178 | if (time.time() - self._last_reset_time > self._config.reset_period):
179 | self._last_reset_time = time.time()
180 | self.reset_islands()
181 |
182 | # Backup every N iterations
183 | if self._program_counter > 0:
184 | self._program_counter += 1
185 | if self._program_counter > self._config.backup_period:
186 | self._program_counter = 0
187 | self.backup()
188 |
189 | def reset_islands(self) -> None:
190 | """Resets the weaker half of islands."""
191 | # We sort best scores after adding minor noise to break ties.
192 | indices_sorted_by_score: np.ndarray = np.argsort(
193 | self._best_score_per_island +
194 | np.random.randn(len(self._best_score_per_island)) * 1e-6)
195 | num_islands_to_reset = self._config.num_islands // 2
196 | reset_islands_ids = indices_sorted_by_score[:num_islands_to_reset]
197 | keep_islands_ids = indices_sorted_by_score[num_islands_to_reset:]
198 | for island_id in reset_islands_ids:
199 | self._islands[island_id] = Island(
200 | self._template,
201 | self._function_to_evolve,
202 | self._config.functions_per_prompt,
203 | self._config.cluster_sampling_temperature_init,
204 | self._config.cluster_sampling_temperature_period)
205 | self._best_score_per_island[island_id] = -float('inf')
206 | founder_island_id = np.random.choice(keep_islands_ids)
207 | founder = self._best_program_per_island[founder_island_id]
208 | founder_scores = self._best_scores_per_test_per_island[founder_island_id]
209 | self._register_program_in_island(founder, island_id, founder_scores)
210 |
211 |
212 | class Island:
213 | """A sub-population of the programs database."""
214 |
215 | def __init__(
216 | self,
217 | template: code_manipulation.Program,
218 | function_to_evolve: str,
219 | functions_per_prompt: int,
220 | cluster_sampling_temperature_init: float,
221 | cluster_sampling_temperature_period: int,
222 | ) -> None:
223 | self._template: code_manipulation.Program = template
224 | self._function_to_evolve: str = function_to_evolve
225 | self._functions_per_prompt: int = functions_per_prompt
226 | self._cluster_sampling_temperature_init = cluster_sampling_temperature_init
227 | self._cluster_sampling_temperature_period = (
228 | cluster_sampling_temperature_period)
229 |
230 | self._clusters: dict[Signature, Cluster] = {}
231 | self._num_programs: int = 0
232 |
233 | def register_program(
234 | self,
235 | program: code_manipulation.Function,
236 | scores_per_test: ScoresPerTest,
237 | ) -> None:
238 | """Stores a program on this island, in its appropriate cluster."""
239 | signature = _get_signature(scores_per_test)
240 | if signature not in self._clusters:
241 | score = _reduce_score(scores_per_test)
242 | self._clusters[signature] = Cluster(score, program)
243 | else:
244 | self._clusters[signature].register_program(program)
245 | self._num_programs += 1
246 |
247 | def get_prompt(self) -> tuple[str, int]:
248 | """Constructs a prompt containing functions from this island."""
249 | signatures = list(self._clusters.keys())
250 | cluster_scores = np.array(
251 | [self._clusters[signature].score for signature in signatures])
252 |
253 | # Convert scores to probabilities using softmax with temperature schedule.
254 | period = self._cluster_sampling_temperature_period
255 | temperature = self._cluster_sampling_temperature_init * (
256 | 1 - (self._num_programs % period) / period)
257 | probabilities = _softmax(cluster_scores, temperature)
258 |
259 | # At the beginning of an experiment when we have few clusters, place fewer
260 | # programs into the prompt.
261 | functions_per_prompt = min(len(self._clusters), self._functions_per_prompt)
262 |
263 | idx = np.random.choice(
264 | len(signatures), size=functions_per_prompt, p=probabilities)
265 | chosen_signatures = [signatures[i] for i in idx]
266 | implementations = []
267 | scores = []
268 | for signature in chosen_signatures:
269 | cluster = self._clusters[signature]
270 | implementations.append(cluster.sample_program())
271 | scores.append(cluster.score)
272 |
273 | indices = np.argsort(scores)
274 | sorted_implementations = [implementations[i] for i in indices]
275 | version_generated = len(sorted_implementations) + 1
276 | return self._generate_prompt(sorted_implementations), version_generated
277 |
278 | def _generate_prompt(
279 | self,
280 | implementations: Sequence[code_manipulation.Function]) -> str:
281 | """Creates a prompt containing a sequence of function `implementations`."""
282 | implementations = copy.deepcopy(implementations) # We will mutate these.
283 |
284 | # Format the names and docstrings of functions to be included in the prompt.
285 | versioned_functions: list[code_manipulation.Function] = []
286 | for i, implementation in enumerate(implementations):
287 | new_function_name = f'{self._function_to_evolve}_v{i}'
288 | implementation.name = new_function_name
289 | # Update the docstring for all subsequent functions after `_v0`.
290 | if i >= 1:
291 | implementation.docstring = (
292 | f'Improved version of `{self._function_to_evolve}_v{i - 1}`.')
293 | # If the function is recursive, replace calls to itself with its new name.
294 | implementation = code_manipulation.rename_function_calls(
295 | str(implementation), self._function_to_evolve, new_function_name)
296 | versioned_functions.append(
297 | code_manipulation.text_to_function(implementation))
298 |
299 | # Create the header of the function to be generated by the LLM.
300 | next_version = len(implementations)
301 | new_function_name = f'{self._function_to_evolve}_v{next_version}'
302 | header = dataclasses.replace(
303 | implementations[-1],
304 | name=new_function_name,
305 | body='',
306 | docstring=('Improved version of '
307 | f'`{self._function_to_evolve}_v{next_version - 1}`.'),
308 | )
309 | versioned_functions.append(header)
310 |
311 | # Replace functions in the template with the list constructed here.
312 | prompt = dataclasses.replace(self._template, functions=versioned_functions)
313 | return str(prompt)
314 |
315 |
316 | class Cluster:
317 | """A cluster of programs on the same island and with the same Signature."""
318 |
319 | def __init__(self, score: float, implementation: code_manipulation.Function):
320 | self._score = score
321 | self._programs: list[code_manipulation.Function] = [implementation]
322 | self._lengths: list[int] = [len(str(implementation))]
323 |
324 | @property
325 | def score(self) -> float:
326 | """Reduced score of the signature that this cluster represents."""
327 | return self._score
328 |
329 | def register_program(self, program: code_manipulation.Function) -> None:
330 | """Adds `program` to the cluster."""
331 | self._programs.append(program)
332 | self._lengths.append(len(str(program)))
333 |
334 | def sample_program(self) -> code_manipulation.Function:
335 | """Samples a program, giving higher probability to shorther programs."""
336 | normalized_lengths = (np.array(self._lengths) - min(self._lengths)) / (
337 | max(self._lengths) + 1e-6)
338 | probabilities = _softmax(-normalized_lengths, temperature=1.0)
339 | return np.random.choice(self._programs, p=probabilities)
340 |
--------------------------------------------------------------------------------
/funsearch/sampler.py:
--------------------------------------------------------------------------------
1 | # Copyright 2023 DeepMind Technologies Limited
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | # ==============================================================================
15 |
16 | """Class for sampling new programs."""
17 | from collections.abc import Collection, Sequence
18 |
19 | import llm
20 | import numpy as np
21 |
22 | from funsearch import evaluator
23 | from funsearch import programs_database
24 |
25 |
26 | class LLM:
27 | """Language model that predicts continuation of provided source code."""
28 |
29 | def __init__(self, samples_per_prompt: int, model: llm.Model, log_path=None) -> None:
30 | self._samples_per_prompt = samples_per_prompt
31 | self.model = model
32 | self.prompt_count = 0
33 | self.log_path = log_path
34 |
35 | def _draw_sample(self, prompt: str) -> str:
36 | """Returns a predicted continuation of `prompt`."""
37 | response = self.model.prompt(prompt)
38 | self._log(prompt, response, self.prompt_count)
39 | self.prompt_count += 1
40 | return response
41 |
42 | def draw_samples(self, prompt: str) -> Collection[str]:
43 | """Returns multiple predicted continuations of `prompt`."""
44 | return [self._draw_sample(prompt) for _ in range(self._samples_per_prompt)]
45 |
46 | def _log(self, prompt: str, response: str, index: int):
47 | if self.log_path is not None:
48 | with open(self.log_path / f"prompt_{index}.log", "a") as f:
49 | f.write(prompt)
50 | with open(self.log_path / f"response_{index}.log", "a") as f:
51 | f.write(str(response))
52 |
53 |
54 | class Sampler:
55 | """Node that samples program continuations and sends them for analysis."""
56 |
57 | def __init__(
58 | self,
59 | database: programs_database.ProgramsDatabase,
60 | evaluators: Sequence[evaluator.Evaluator],
61 | model: LLM,
62 | ) -> None:
63 | self._database = database
64 | self._evaluators = evaluators
65 | self._llm = model
66 |
67 | def sample(self):
68 | """Continuously gets prompts, samples programs, sends them for analysis."""
69 | prompt = self._database.get_prompt()
70 | samples = self._llm.draw_samples(prompt.code)
71 | # This loop can be executed in parallel on remote evaluator machines.
72 | for sample in samples:
73 | chosen_evaluator = np.random.choice(self._evaluators)
74 | chosen_evaluator.analyse(
75 | sample, prompt.island_id, prompt.version_generated)
76 |
--------------------------------------------------------------------------------
/funsearch/sandbox.py:
--------------------------------------------------------------------------------
1 | import logging
2 |
3 | import ast
4 | import os
5 | import pathlib
6 | import sys
7 | from typing import Any
8 |
9 | import cloudpickle
10 |
11 | CONTAINER_MAIN = (pathlib.Path(__file__).parent / "container" / "container_main.py").absolute()
12 |
13 | IMAGE_NAME = "funsearch_sandbox"
14 |
15 |
16 | class DummySandbox:
17 | """Base class for Sandboxes that execute the generated code.
18 |
19 | Note: this base class executes the code but does not offer any sandboxing!!!
20 | It should be only used in unit testing or debugging, and not with real LLM
21 | unless the host environment is in some kind of sandbox itself.
22 | Even in sandboxed host, the executed code could theoretically affect later executions.
23 | """
24 |
25 | sandboxes = 0
26 |
27 | def __init__(self, **kwargs):
28 | self.id = DummySandbox.sandboxes
29 |
30 | DummySandbox.sandboxes += 1
31 |
32 | def run(
33 | self,
34 | program: str,
35 | function_to_run: str,
36 | test_input,
37 | timeout_seconds: int,
38 | ) -> tuple[Any, bool]:
39 | """Returns `function_to_run(test_input)` and whether execution succeeded."""
40 |
41 | # The same "program" seems to be now repeatedly parsed using AST and then compiled.
42 | # This could probably be simplified quite a bit.
43 | namespace = DummySandbox.compile_code(program)
44 | return namespace[function_to_run](test_input)
45 |
46 | @staticmethod
47 | def compile_code(program: str):
48 | namespace = {}
49 |
50 | parsed_code = ast.parse(program)
51 | compiled_code = compile(parsed_code, filename="", mode="exec")
52 | exec(compiled_code, namespace)
53 | return namespace
54 |
55 |
56 | class ExternalProcessSandbox(DummySandbox):
57 | """Sandbox that executes the code in a separate Python process in the same host.
58 |
59 | Note: This does not provide real safety and should be only used in an environment where the host process is
60 | in some kind of safe sandbox itself (i.e., a container).
61 | This kind of sandbox merely makes it more probable that single invalid call does not break the whole
62 | funsearch algorithm. It might be easier to set up and thus nice environment to tune the prompts and other code.
63 | """
64 |
65 | def __init__(self, base_path: pathlib.Path, timeout_secs: int = 30, python_path: str = "python"):
66 | super(ExternalProcessSandbox, self).__init__()
67 |
68 | self.output_path = pathlib.Path(base_path) / f"sandbox{self.id}"
69 | self.timeout_secs = timeout_secs
70 | self.python_path = python_path
71 | self.call_count = 0
72 |
73 | self.input_path = self.output_path / "inputs"
74 | for p in [self.output_path, self.input_path]:
75 | if not p.exists():
76 | p.mkdir(parents=True)
77 |
78 | def _exec(self, call_data_path: pathlib.Path, input_path: pathlib.Path, error_file_path: pathlib.Path):
79 | """Use podman/docker to execute python in a container.
80 | - The main.py shall execute the LLM generated method from prog.pickle file providing
81 | input.pickle as the input for the method.
82 | - main.py writes the output of the method into output.pickle.
83 | Everything except the /workspace folder will be read-only so that the environment remains good
84 | for future runs.
85 | """
86 | prog_path = call_data_path / "prog.pickle"
87 | output_file = call_data_path / "output.pickle"
88 | cmd = (f"{self.python_path} {CONTAINER_MAIN} {prog_path} {input_path} {output_file}"
89 | f" 2> {error_file_path}")
90 | logging.debug(f"Executing: {cmd}")
91 | return os.system(cmd)
92 |
93 | def run(
94 | self,
95 | program: str,
96 | function_to_run: str,
97 | test_input,
98 | timeout_seconds: int,
99 | ) -> tuple[Any, bool]:
100 |
101 | call_data_folder = (self.output_path / f"call{self.call_count}").absolute()
102 | if not call_data_folder.exists():
103 | call_data_folder.mkdir()
104 |
105 | input_hash = hash(test_input)
106 | input_path = (self.input_path / f"{input_hash}.pickle").absolute()
107 |
108 | if not input_path.exists():
109 | with open(input_path, "wb") as f:
110 | cloudpickle.dump(test_input, f)
111 | try:
112 | namespace = DummySandbox.compile_code(program)
113 |
114 | prog_file = (call_data_folder / f"prog.pickle").absolute()
115 | with open(prog_file, "wb+") as f:
116 | cloudpickle.dump(namespace[function_to_run], f)
117 |
118 | error_file = self.output_path / f"stderr_{self.call_count}.log"
119 |
120 | retcode = self._exec(call_data_folder, input_path, error_file)
121 | self.call_count += 1
122 |
123 | if retcode != 0:
124 | self._save_diagnostics(program, call_data_folder)
125 | return None, False
126 |
127 | output_file = call_data_folder / f"output.pickle"
128 | with open(output_file, "rb") as f:
129 | out = cloudpickle.load(f)
130 | return out, True
131 | except Exception as e:
132 | logging.debug(f"Could not execute code: {e}")
133 | self._save_diagnostics(program, call_data_folder)
134 | return None, False
135 |
136 | @staticmethod
137 | def _save_diagnostics(program: str, output_path: pathlib.Path):
138 | filepath = output_path / "program.py"
139 | logging.debug(f"Writing program to {filepath}")
140 | with open(filepath, "w+") as f:
141 | f.write(program)
142 |
143 |
144 | class ContainerSandbox(ExternalProcessSandbox):
145 | """Basic sandbox that runs unsafe code in Podman or Docker container.
146 | - the sandbox should be safe against inadvertent bad code by LLM but not against malicious attacks.
147 | - does not require any other dependencies on the host than Podman/Docker
148 | - does not support multithreading
149 | - might provide easier or more lightweight debugging experience than some other fancier sandbox environments
150 | """
151 | executable = "podman"
152 | image_built = False
153 |
154 | @classmethod
155 | def build_image(cls, extra_pip_packages):
156 | version = sys.version.split(" ")[0]
157 | ret = os.system("podman --version")
158 | if ret != 0:
159 | ret = os.system("docker --version")
160 | if ret != 0:
161 | raise Exception("Could not find Podman or Docker. Can not use ContainerSandbox.")
162 | else:
163 | cls.executable = "docker"
164 |
165 | dockerfile = pathlib.Path(__file__).parent / "container" / "Dockerfile"
166 | logging.debug("Building container image")
167 | extra = ""
168 | if extra_pip_packages:
169 | extra = f"--build-arg INSTALL_PACKAGES=\"{extra_pip_packages}\""
170 |
171 | cmd = (f"{cls.executable} build --build-arg PYTHON_VERSION={version} {extra} "
172 | f"-t {IMAGE_NAME} -f {dockerfile} {CONTAINER_MAIN.parent}")
173 | logging.debug(f"Executing: {cmd}")
174 | os.system(cmd)
175 | cls.image_built = True
176 |
177 | def __init__(self, base_path: pathlib.Path, extra_pip_packages: str = "numpy", timeout_secs=30):
178 | super(ContainerSandbox, self).__init__(base_path, timeout_secs)
179 |
180 | if not ContainerSandbox.image_built:
181 | ContainerSandbox.build_image(extra_pip_packages)
182 |
183 | def _exec(self, call_data_path: pathlib.Path, input_path: pathlib.Path, error_file_path: pathlib.Path):
184 | """Use podman/docker to execute python in a container.
185 | - The main.py shall execute the LLM generated method from prog.pickle file providing
186 | input.pickle as the input for the method.
187 | - main.py writes the output of the method into output.pickle.
188 | Everything except the /workspace folder will be read-only so that the environment remains good
189 | for future runs.
190 | """
191 | cmd = (f"{self.executable} run "
192 | f"--stop-timeout={self.timeout_secs} "
193 | f"-v {CONTAINER_MAIN}:/main.py:ro "
194 | f"-v {call_data_path}:/workspace "
195 | f"-v {input_path}:/input.pickle:ro "
196 | f"{IMAGE_NAME}:latest /usr/local/bin/python3 "
197 | f"/main.py /workspace/prog.pickle /input.pickle /workspace/output.pickle"
198 | f" 2> {error_file_path}")
199 | logging.debug(f"Executing: {cmd}")
200 | return os.system(cmd)
201 |
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | [build-system]
2 | requires = ["setuptools", "wheel"]
3 | build-backend = "setuptools.build_meta"
4 |
5 | [tool.setuptools]
6 | packages = ["funsearch", "funsearch.container"]
7 |
8 | [project]
9 | name = "funsearch"
10 | dynamic = [
11 | "version",
12 | ]
13 | description = "FunSearch algorithm: Mathematical discoveries from program search with large language models"
14 | readme = "README.md"
15 | requires-python = ">=3.9"
16 |
17 | classifiers = [
18 | "Intended Audience :: Science/Research",
19 | "Programming Language :: Python",
20 | "Programming Language :: Python :: 3",
21 | "Topic :: Scientific/Engineering :: Artificial Intelligence",
22 | ]
23 | dependencies = [
24 | "absl-py==2.0.0",
25 | "click==8.1",
26 | "cloudpickle==3.0.0",
27 | "llm==0.12",
28 | "python-dotenv==1.0.0",
29 | "scipy==1.11.4",
30 | "numpy==1.26.2",
31 | ]
32 |
33 | [project.scripts]
34 | funsearch = "funsearch.__main__:main"
35 |
--------------------------------------------------------------------------------
/tests/code_manipulation_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2023 DeepMind Technologies Limited
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | # ==============================================================================
15 |
16 | import itertools
17 | import textwrap
18 |
19 | from absl.testing import absltest
20 | from absl.testing import parameterized
21 |
22 | from funsearch import code_manipulation
23 |
24 | _IMPORTS: str = '''\
25 | import itertools
26 | import numpy
27 |
28 |
29 | '''
30 |
31 | _CLASS: str = '''\
32 | class Helper:
33 | def __init__(self, n: int):
34 | self.n = n
35 | self.initial_capset = get_capset_v0(n)
36 |
37 |
38 | '''
39 |
40 | _ASSIGNMENT: str = '''\
41 | some_global_variable = 0
42 |
43 |
44 | '''
45 |
46 | _FUNCTIONS: str = '''\
47 | def get_capset_v0(n: int) -> set[tuple[int, ...]]:
48 | """Computes a cap set for n number of copies.
49 |
50 | A cap set is a subset of an n-dimensional affine space over a three-element
51 | field with no three elements in a line.
52 |
53 | Args:
54 | n: an integer, number of copies.
55 |
56 | Returns:
57 | A set of tuples in {0, 1, 2}.
58 | """
59 | capset = set()
60 | for i in range(n):
61 | capset.add((0,) * i + (1,) + (0,) * (n - i - 1))
62 | return capset
63 |
64 |
65 | def get_capset_v2(k: int):
66 | """One line docstring."""
67 | get_capset_v0 = get_capset_v0(k)
68 | return get_capset_v0\\
69 |
70 | '''
71 |
72 |
73 | _SMALL_PROGRAM: str = '''\
74 | def test() -> np.ndarray:
75 | return np.zeros(1)
76 | '''
77 |
78 | _FUNCTION_HEADER: str = 'def get_capset_v0(n: int)'
79 | _FUNCTION_RETURN_TYPE: str = ' -> set[tuple[int, ...]]'
80 | _FUNCTION_DOCSTRING: str = ' """One line docstring."""'
81 | _FUNCTION_BODY: str = '''\
82 | capset = set()
83 | for i in range(n):
84 | capset.add((0,) * i + (1,) + (0,) * (n - i - 1))
85 | return capset
86 | '''
87 |
88 |
89 | def create_test_function(
90 | has_return_type: bool, has_docstring: bool
91 | ) -> str:
92 | code = _FUNCTION_HEADER
93 | if has_return_type:
94 | code += _FUNCTION_RETURN_TYPE
95 | code += ':\n'
96 | if has_docstring:
97 | code += _FUNCTION_DOCSTRING
98 | code += '\n'
99 | code += _FUNCTION_BODY
100 | return code
101 |
102 |
103 | def create_test_program(
104 | has_imports: bool, has_class: bool, has_assignment: bool
105 | ) -> str:
106 | code = ''
107 | if has_imports:
108 | code += _IMPORTS
109 | if has_class:
110 | code += _CLASS
111 | if has_assignment:
112 | code += _ASSIGNMENT
113 | code += _FUNCTIONS
114 | return code
115 |
116 |
117 | class PromptSamplingTest(parameterized.TestCase):
118 |
119 | @parameterized.parameters(list(itertools.product([False, True], repeat=2)))
120 | def test_text_to_function(self, has_return_type: bool, has_docstring: bool):
121 | function = code_manipulation.text_to_function(create_test_function(
122 | has_return_type, has_docstring))
123 | self.assertEqual(function.name, 'get_capset_v0')
124 | self.assertEqual(function.args, 'n: int')
125 | if has_return_type:
126 | self.assertEqual(function.return_type, 'set[tuple[int, ...]]')
127 | else:
128 | self.assertIsNone(function.return_type)
129 | if has_docstring:
130 | self.assertEqual(function.docstring, 'One line docstring.')
131 | else:
132 | self.assertIsNone(function.docstring)
133 | self.assertEqual(function.body, _FUNCTION_BODY.rstrip())
134 |
135 | def test_small_text_to_program(self):
136 | program = code_manipulation.text_to_program(_SMALL_PROGRAM)
137 | self.assertEmpty(program.preface)
138 | self.assertLen(program.functions, 1)
139 |
140 | expected_function = code_manipulation.Function(
141 | name='test', args='', return_type='np.ndarray',
142 | body=' return np.zeros(1)')
143 | self.assertEqual(expected_function, program.functions[0])
144 | self.assertEqual(_SMALL_PROGRAM + '\n', str(program))
145 |
146 | # Assert that we do not add one more '\n' each time we convert to program.
147 | program_again = code_manipulation.text_to_program(str(program))
148 | self.assertEqual(str(program), str(program_again))
149 |
150 | @parameterized.parameters(list(itertools.product([False, True], repeat=3)))
151 | def test_text_to_program(self, has_imports: bool, has_class: bool,
152 | has_assignment: bool):
153 | code = create_test_program(has_imports, has_class, has_assignment)
154 | program = code_manipulation.text_to_program(code)
155 | self.assertLen(program.functions, 2)
156 |
157 | doc = textwrap.dedent(
158 | """\
159 | Computes a cap set for n number of copies.
160 |
161 | A cap set is a subset of an n-dimensional affine space over a three-element
162 | field with no three elements in a line.
163 |
164 | Args:
165 | n: an integer, number of copies.
166 |
167 | Returns:
168 | A set of tuples in {0, 1, 2}.
169 | """
170 | )
171 | body = textwrap.dedent(
172 | """\
173 | capset = set()
174 | for i in range(n):
175 | capset.add((0,) * i + (1,) + (0,) * (n - i - 1))
176 | return capset"""
177 | )
178 | expected_function_0 = code_manipulation.Function(
179 | name='get_capset_v0',
180 | args='n: int',
181 | return_type='set[tuple[int, ...]]',
182 | docstring=doc + ' ',
183 | body=textwrap.indent(body, ' '),
184 | )
185 | expected_function_1 = code_manipulation.Function(
186 | name='get_capset_v2',
187 | args='k: int',
188 | body=''' get_capset_v0 = get_capset_v0(k)
189 | return get_capset_v0\\\n\n''',
190 | docstring='One line docstring.',
191 | )
192 | if not has_imports and not has_class and not has_assignment:
193 | self.assertEmpty(program.preface)
194 | if has_imports:
195 | self.assertIn(_IMPORTS.rstrip(), program.preface)
196 | if has_class:
197 | self.assertIn(_CLASS.strip(), program.preface)
198 | if has_assignment:
199 | self.assertIn(_ASSIGNMENT.strip(), program.preface)
200 | self.assertEqual(expected_function_0, program.functions[0])
201 | self.assertEqual(expected_function_1, program.functions[1])
202 | self.assertEqual(code, str(program))
203 |
204 | # Make sure that one can convert Function to string and then back to a
205 | # function so that it remains the same.
206 | for i in range(2):
207 | self.assertEqual(
208 | program.functions[i],
209 | code_manipulation.text_to_function(str(program.functions[i]))
210 | )
211 |
212 | def test_get_functions_called(self):
213 | code = textwrap.dedent('''\
214 | def f(n: int) -> int:
215 | if n == 1:
216 | return a(n)
217 | elif n == 2:
218 | return b(n) + object.c(n - 1)
219 | a = object.property
220 | g()
221 | return f(n - 1)
222 | ''')
223 | self.assertEqual(code_manipulation.get_functions_called(code),
224 | {'a', 'b', 'f', 'g'})
225 |
226 | if __name__ == '__main__':
227 | absltest.main()
228 |
--------------------------------------------------------------------------------
/tests/evaluator_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2023 DeepMind Technologies Limited
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | # ==============================================================================
15 | import pathlib
16 | import textwrap
17 |
18 | from absl.testing import absltest
19 | from absl.testing import parameterized
20 |
21 | from funsearch import evaluator
22 |
23 | TESTS_FOLDER = pathlib.Path(__file__).parent
24 |
25 |
26 | class EvaluatorTest(parameterized.TestCase):
27 |
28 | def test_trim_function_body_docstring(self):
29 | code = '''\
30 | x = 1
31 |
32 | return 0
33 | """Docstring"""'''
34 | desired = '''\
35 | x = 1
36 |
37 | return 0
38 |
39 | '''
40 | actual = evaluator._trim_function_body(code)
41 | self.assertEqual(desired, actual)
42 |
43 | def test_trim_function_body_function(self):
44 | code = '''\
45 | return 0
46 | def new_f():'''
47 | desired = '''\
48 | return 0
49 |
50 | '''
51 | actual = evaluator._trim_function_body(code)
52 | self.assertEqual(desired, actual)
53 |
54 | def test_trim_function_body_empty(self):
55 | code = ''' return 0\n'''
56 | desired = ''' return 0\n\n'''
57 | actual = evaluator._trim_function_body(code)
58 | self.assertEqual(desired, actual)
59 |
60 | def test_trim_function_indentation_corner_case(self):
61 | code = textwrap.dedent(
62 | '''\
63 | return (1 +
64 | 2)
65 | def unfinished_code('''
66 | )
67 | desired = textwrap.dedent(
68 | '''\
69 | return (1 +
70 | 2)
71 |
72 | '''
73 | )
74 | actual = evaluator._trim_function_body(code)
75 | self.assertEqual(desired, actual)
76 |
77 | def test_trim_function_backlash_corner_case(self):
78 | code = textwrap.dedent(
79 | '''\
80 | return score + ((el[0] + 1) * (el[0] + 2) * el[1] / 6 == el[2])\\
81 | + ((el[0] + 1) * (el[0] + 2) * (el[0] + 3) * el[1] / 24 == el[2])\\
82 | + ((el[0] + 1) * (el[0] + 2) * el[1] * el[2] / 6 == n)\\
83 | + ((el[0] + 1) * (el[0] + 2) * el[1] * el[2] / 3 == n + el[0])\\
84 |
85 | '''
86 | )
87 | actual = evaluator._trim_function_body(code)
88 | self.assertEqual(actual, code)
89 |
90 |
91 | def test_trim_function_other_llms(self):
92 | files = list((TESTS_FOLDER / "example_responses").glob("*.txt"))
93 | assert len(files) > 1, "Could not find all test files"
94 | for f in files:
95 | name = f.name
96 | response = f.read_text()
97 | actual = evaluator._trim_function_body(response)
98 | expected = (TESTS_FOLDER / "example_response_parsed" / name).read_text()
99 | self.assertEqual(expected, actual, f"{name}: Parsed code does not match")
100 |
101 |
102 | if __name__ == '__main__':
103 | absltest.main()
104 |
--------------------------------------------------------------------------------
/tests/funsearch_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2023 DeepMind Technologies Limited
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | # ==============================================================================
15 |
16 | from absl.testing import absltest
17 | from absl.testing import parameterized
18 |
19 | from funsearch import core
20 |
21 | _PY_PROMPT = '''\
22 | import itertools
23 | import jax
24 |
25 |
26 | @funsearch.run
27 | @jax.jit
28 | def run(n: int):
29 | return capset(n)
30 |
31 |
32 | @funsearch.evolve
33 | def capset(n: int):
34 | """Trivial implementation of capset.
35 |
36 | Args: ...
37 | """
38 | return [[1,] * n]
39 | '''
40 |
41 | _PY_PROMPT_EVOLVE_RUN = '''\
42 | import itertools
43 |
44 |
45 | @funsearch.run
46 | @funsearch.evolve
47 | def capset(n: int):
48 | return [[1,] * n]
49 | '''
50 |
51 | _PY_PROMPT_NO_RUN = '''\
52 | import itertools
53 |
54 |
55 | def run(n: int):
56 | return capset(n)
57 |
58 | @funsearch.evolve
59 | def capset(n: int):
60 | """Trivial implementation of capset.
61 |
62 | Args: ...
63 | """
64 | return [[1,] * n]
65 | '''
66 |
67 | _PY_PROMPT_NO_EVOLVE = '''\
68 | import itertools
69 |
70 |
71 | @funsearch.run
72 | def run(n: int):
73 | return capset(n)
74 |
75 |
76 | def capset(n: int):
77 | """Trivial implementation of capset.
78 |
79 | Args: ...
80 | """
81 | return [[1,] * n]
82 | '''
83 |
84 | _PY_PROMPT_DOUBLE_RUN = '''\
85 | import itertools
86 |
87 | @funsearch.run
88 | def run(n: int):
89 | return capset(n)
90 |
91 | @funsearch.run
92 | def capset(n: int):
93 | """Trivial implementation of capset.
94 |
95 | Args: ...
96 | """
97 | return [[1,] * n]
98 | '''
99 |
100 |
101 | class FunsearchTest(parameterized.TestCase):
102 |
103 | def test_extract_function_names(self):
104 | to_evolve, to_run = core._extract_function_names(_PY_PROMPT)
105 | self.assertEqual(to_run, 'run')
106 | self.assertEqual(to_evolve, 'capset')
107 |
108 | def test_extract_function_names_evolve_and_run(self):
109 | to_evolve, to_run = core._extract_function_names(_PY_PROMPT_EVOLVE_RUN)
110 | self.assertEqual(to_run, 'capset')
111 | self.assertEqual(to_evolve, 'capset')
112 |
113 | def test_extract_function_names_no_run(self):
114 | with self.assertRaisesRegex(
115 | ValueError, r'Expected 1 function decorated with `@funsearch.run`.'):
116 | core._extract_function_names(_PY_PROMPT_NO_RUN)
117 |
118 | def test_extract_function_names_no_evolve(self):
119 | with self.assertRaisesRegex(
120 | ValueError, r'Expected 1 function decorated with `@funsearch.evolve`.'):
121 | core._extract_function_names(_PY_PROMPT_NO_EVOLVE)
122 |
123 | def test_extract_function_names_double_run(self):
124 | with self.assertRaisesRegex(
125 | ValueError, r'Expected 1 function decorated with `@funsearch.run`.'):
126 | core._extract_function_names(_PY_PROMPT_DOUBLE_RUN)
127 |
128 |
129 | if __name__ == '__main__':
130 | absltest.main()
131 |
--------------------------------------------------------------------------------
/tests/main_test.py:
--------------------------------------------------------------------------------
1 | import shutil
2 | import tempfile
3 | import unittest
4 | from pathlib import Path
5 | from unittest.mock import patch
6 |
7 | from click.testing import CliRunner
8 |
9 | from funsearch.__main__ import main, parse_input
10 |
11 | ROOT_DIR = Path(__file__).parent.parent
12 |
13 | runner = CliRunner()
14 |
15 |
16 | class TestMain(unittest.TestCase):
17 | def setUp(self):
18 | self.temp_dir = tempfile.mkdtemp()
19 | self.default_args = ["run", "--output_path", self.temp_dir, "--samplers", "1", "--iterations", "1",
20 | str(ROOT_DIR / "examples" / "cap_set_spec.py"), "5"]
21 |
22 | def tearDown(self):
23 | shutil.rmtree(self.temp_dir)
24 |
25 | def test_main(self):
26 | result = runner.invoke(main, [])
27 | assert result.exit_code == 0
28 | assert 'Usage:' in result.output
29 | with patch('funsearch.core.run', return_value=None) as mock_run:
30 | result = runner.invoke(main, self.default_args)
31 | assert result.exit_code == 0, result.output
32 | assert mock_run.call_count == 1
33 |
34 | def test_main_sample(self):
35 | with patch('funsearch.sampler.LLM._draw_sample', return_value="return 0.5") as mock_run:
36 | result = runner.invoke(main, self.default_args)
37 | assert result.exit_code == 0, result.output
38 | assert mock_run.call_count == 2, "There should be 2 sampler per sampler by default"
39 |
40 |
41 | def test_parse_input():
42 | assert parse_input("1") == [1]
43 | assert parse_input("1,2,3") == [1, 2, 3]
44 | assert parse_input(str(ROOT_DIR / "examples" / "cap_set_input_data.json")) == [8, 9, 10]
45 |
--------------------------------------------------------------------------------
/tests/programs_database_test.py:
--------------------------------------------------------------------------------
1 | # Copyright 2023 DeepMind Technologies Limited
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | # ==============================================================================
15 |
16 | import copy
17 | import tempfile
18 |
19 | from absl.testing import absltest
20 | from absl.testing import parameterized
21 | import numpy as np
22 |
23 | from funsearch import code_manipulation
24 | from funsearch import config
25 | from funsearch import programs_database
26 |
27 | _SKELETON = '''
28 | """Finds large cap sets."""
29 | import numpy as np
30 | import utils_capset
31 |
32 |
33 | def evaluate(n: int) -> int:
34 | """Returns the size of an `n`-dimensional cap set."""
35 | capset = solve(n)
36 | return len(capset)
37 |
38 |
39 | def priority(element, n):
40 | """Returns the priority with which we want to add `element` to the cap set."""
41 | return 0.0
42 | '''
43 | _EXPECTED_INITIAL_PROMPT = '''
44 | """Finds large cap sets."""
45 | import numpy as np
46 | import utils_capset
47 |
48 |
49 | def priority_v0(element, n):
50 | """Returns the priority with which we want to add `element` to the cap set."""
51 | return 0.0
52 |
53 |
54 | def priority_v1(element, n):
55 | """Improved version of `priority_v0`."""
56 |
57 | '''
58 |
59 | _SAMPLE_A = '''\
60 | priority = element
61 | #######
62 | # Code from lowest-scoring sampled program.
63 | #######
64 | return ...\
65 | '''
66 | _SAMPLE_B = '''\
67 | priority = element ** 2
68 | #######
69 | # Code from highest-scoring sampled program.
70 | #######
71 | return ...\
72 | '''
73 |
74 | # A prompt like this appears in the Extended Data of the paper.
75 | _EXPECTED_PROMPT = '''
76 | """Finds large cap sets."""
77 | import numpy as np
78 | import utils_capset
79 |
80 |
81 | def priority_v0(element, n):
82 | """Returns the priority with which we want to add `element` to the cap set."""
83 | priority = element
84 | #######
85 | # Code from lowest-scoring sampled program.
86 | #######
87 | return ...
88 |
89 |
90 | def priority_v1(element, n):
91 | """Improved version of `priority_v0`."""
92 | priority = element ** 2
93 | #######
94 | # Code from highest-scoring sampled program.
95 | #######
96 | return ...
97 |
98 |
99 | def priority_v2(element, n):
100 | """Improved version of `priority_v1`."""
101 |
102 | '''
103 |
104 |
105 | class ProgramsDatabaseTest(parameterized.TestCase):
106 |
107 | def test_initial_prompt(self):
108 | """Verifies that the first prompt looks as expected."""
109 |
110 | # Create a programs database.
111 | template = code_manipulation.text_to_program(_SKELETON)
112 | function_to_evolve = 'priority'
113 | database = programs_database.ProgramsDatabase(
114 | config=config.ProgramsDatabaseConfig(functions_per_prompt=5),
115 | template=template,
116 | function_to_evolve=function_to_evolve,
117 | )
118 | # Register the initial implementation provided in the skeleton template.
119 | database.register_program(
120 | program=template.get_function(function_to_evolve),
121 | island_id=None,
122 | scores_per_test={'unused': -1})
123 | # Verify the first prompt.
124 | self.assertEqual(database.get_prompt().code, _EXPECTED_INITIAL_PROMPT)
125 |
126 | # Test saving database
127 | with tempfile.TemporaryFile() as f:
128 | database.save(f)
129 |
130 | f.seek(0)
131 | db2 = programs_database.ProgramsDatabase(
132 | config=config.ProgramsDatabaseConfig(functions_per_prompt=5),
133 | template=template,
134 | function_to_evolve=function_to_evolve,
135 | )
136 | # Make sure the loaded database works as the original
137 | db2.load(f)
138 | self.assertEqual(db2.get_prompt().code, _EXPECTED_INITIAL_PROMPT)
139 |
140 | def test_generate_prompt(self):
141 | """Tests that we build the prompt shown in the paper."""
142 | template = code_manipulation.text_to_program(_SKELETON)
143 | function_to_evolve = 'priority'
144 | island = programs_database.Island(
145 | template=template,
146 | function_to_evolve=function_to_evolve,
147 | functions_per_prompt=2,
148 | cluster_sampling_temperature_init=1.0,
149 | cluster_sampling_temperature_period=30_000,
150 | )
151 | sample_a = copy.deepcopy(template.get_function(function_to_evolve))
152 | sample_a.body = _SAMPLE_A
153 | sample_b = copy.deepcopy(template.get_function(function_to_evolve))
154 | sample_b.body = _SAMPLE_B
155 | prompt = island._generate_prompt([sample_a, sample_b])
156 | self.assertEqual(prompt, _EXPECTED_PROMPT)
157 |
158 | def test_destroy_islands(self):
159 | template = code_manipulation.text_to_program(_SKELETON)
160 | function_to_evolve = 'priority'
161 | database = programs_database.ProgramsDatabase(
162 | config=config.ProgramsDatabaseConfig(num_islands=10),
163 | template=template,
164 | function_to_evolve=function_to_evolve,
165 | )
166 | scores = [7, 3, 5, 6, 7, -2, 0, -1, 4, 3]
167 | unused_function = template.get_function(function_to_evolve)
168 | for i, score in enumerate(scores):
169 | database.register_program(
170 | program=unused_function,
171 | island_id=i,
172 | scores_per_test={'unused_input': score},
173 | )
174 | database.register_program(
175 | unused_function, island_id=7, scores_per_test={'unused_input': 17})
176 |
177 | expected_scores = list(scores)
178 | expected_scores[7] = 17
179 | self.assertSequenceEqual(database._best_score_per_island, expected_scores)
180 |
181 | progs = list(database.get_best_programs_per_island())
182 | assert progs[0][1] == 17
183 | assert progs[-1][1] == -2
184 |
185 | np.random.seed(0)
186 | database.reset_islands()
187 | expected_kept = set([0, 2, 3, 4, 7])
188 | min_kept = min(expected_scores[i] for i in expected_kept)
189 | for i, score in enumerate(expected_scores):
190 | if i in expected_kept:
191 | self.assertEqual(database._best_score_per_island[i], score)
192 | else:
193 | self.assertGreaterEqual(database._best_score_per_island[i], min_kept)
194 |
195 | @parameterized.parameters([
196 | dict(logits=np.array([10, 9, -1000], dtype=np.float32)),
197 | dict(logits=np.array([10, 9, -1000], dtype=np.int32)),
198 | dict(logits=np.zeros(50)),
199 | ])
200 | def test_softmax(self, logits: np.ndarray):
201 | probs_lower_temp = programs_database._softmax(logits, temperature=1.0)
202 | self.assertAlmostEqual(np.sum(probs_lower_temp), 1.0, places=6)
203 | self.assertTrue(np.all(probs_lower_temp >= 0))
204 | self.assertTrue(np.all(probs_lower_temp <= 1))
205 |
206 | probs_higher_temp = programs_database._softmax(logits, temperature=5.0)
207 | self.assertAlmostEqual(np.sum(probs_higher_temp), 1.0, places=6)
208 | self.assertTrue(np.all(probs_higher_temp >= 0))
209 | self.assertTrue(np.all(probs_higher_temp <= 1))
210 |
211 | if not np.all(logits == logits[0]):
212 | # The lower the temperature, the more confident we are on our top choice.
213 | self.assertGreater(np.max(probs_lower_temp), np.max(probs_higher_temp))
214 |
215 | @parameterized.parameters([
216 | dict(temperature=1, expected=[0.6590012, 0.242433, 0.0985659]),
217 | dict(temperature=2, expected=[0.50168777, 0.304289, 0.19402324]),
218 | ])
219 | def test_softmax_output(self, temperature, expected):
220 | logits = np.array([2.0, 1.0, 0.1])
221 | probabilities = programs_database._softmax(logits, temperature)
222 | np.testing.assert_array_almost_equal(
223 | probabilities, np.array(expected), decimal=5)
224 |
225 | @parameterized.parameters([
226 | dict(logits=np.array([100, 200, 300, np.nan])),
227 | dict(logits=np.array([100, np.inf, 300, 200])),
228 | dict(logits=np.array([-np.inf, 200, 300, 100])),
229 | ])
230 | def test_softmax_non_finite_error(self, logits):
231 | with self.assertRaisesRegex(
232 | ValueError,
233 | r'`logits` contains non-finite value\(s\)',
234 | ):
235 | programs_database._softmax(logits, temperature=1.0)
236 |
237 |
238 | if __name__ == '__main__':
239 | absltest.main()
240 |
--------------------------------------------------------------------------------
/tests/sandbox_test.py:
--------------------------------------------------------------------------------
1 | import pathlib
2 | import tempfile
3 | from funsearch.sandbox import ExternalProcessSandbox, ContainerSandbox
4 |
5 | test_prog = """
6 | print("running!")
7 | def x(y):
8 | print(f"Received {y}")
9 | return y + 1
10 | """
11 |
12 |
13 | def test_external_process_sandbox():
14 | with tempfile.TemporaryDirectory() as d:
15 | sandbox = ExternalProcessSandbox(pathlib.Path(d))
16 | ret, success = sandbox.run(test_prog, "x", 10, 1)
17 | assert success
18 | assert ret == 11
19 |
20 |
21 | def test_container_sandbox():
22 | with tempfile.TemporaryDirectory() as d:
23 | sandbox = ContainerSandbox(pathlib.Path(d))
24 | ret, success = sandbox.run(test_prog, "x", 10, 1)
25 | assert success
26 | assert ret == 11
27 |
--------------------------------------------------------------------------------