├── .gitignore
├── LICENSE
├── README.md
├── examples
├── README.md
├── doc_merge
│ ├── README.md
│ ├── doc_merge.py
│ ├── documents.csv
│ ├── example_prompts_document_merging.md
│ ├── plot.py
│ └── pure_documents.json
├── keyword_counting
│ ├── README.md
│ ├── countries.csv
│ ├── dataset_gen_countries.py
│ ├── example_prompts_keyword_counting.md
│ ├── keyword_counting.py
│ └── plot.py
├── set_intersection
│ ├── README.md
│ ├── dataset_gen_intersection.py
│ ├── example_prompts_set_intersection_032.md
│ ├── plot.py
│ ├── set_intersection_032.csv
│ ├── set_intersection_032.py
│ ├── set_intersection_064.csv
│ ├── set_intersection_064.py
│ ├── set_intersection_128.csv
│ ├── set_intersection_128.py
│ └── utils.py
└── sorting
│ ├── README.md
│ ├── example_prompts_sorting_032.md
│ ├── plot.py
│ ├── sorting_032.csv
│ ├── sorting_032.py
│ ├── sorting_064.csv
│ ├── sorting_064.py
│ ├── sorting_128.csv
│ ├── sorting_128.py
│ └── utils.py
├── graph_of_thoughts
├── __init__.py
├── controller
│ ├── README.md
│ ├── __init__.py
│ └── controller.py
├── language_models
│ ├── README.md
│ ├── __init__.py
│ ├── abstract_language_model.py
│ ├── chatgpt.py
│ ├── config_template.json
│ └── llamachat_hf.py
├── operations
│ ├── README.md
│ ├── __init__.py
│ ├── graph_of_operations.py
│ ├── operations.py
│ └── thought.py
├── parser
│ ├── __init__.py
│ └── parser.py
└── prompter
│ ├── __init__.py
│ └── prompter.py
├── paper
├── README.md
├── final_results_gpt35.tar.bz2
├── pics
│ └── preview.svg
├── plots.py
├── poster.pdf
└── poster.png
└── pyproject.toml
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | dist/
11 | build/
12 |
13 | # Installer logs
14 | pip-log.txt
15 |
16 | # IDEs
17 | .idea/
18 | *.vscode/
19 | *.pycproj
20 | *.user
21 | *.pyproj.user
22 |
23 | # Data
24 | results/
25 | *.out
26 | *.err
27 |
28 | # Environments
29 | env/
30 |
31 | # Config File
32 | **/config.json
33 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Copyright (c) 2023 ETH Zurich.
2 | All rights reserved.
3 |
4 | Redistribution and use in source and binary forms, with or without
5 | modification, are permitted provided that the following conditions are
6 | met:
7 |
8 | - Redistributions of source code must retain the above copyright
9 | notice, this list of conditions and the following disclaimer.
10 |
11 | - Redistributions in binary form must reproduce the above copyright
12 | notice, this list of conditions and the following disclaimer listed
13 | in this license in the documentation and/or other materials
14 | provided with the distribution.
15 |
16 | - Neither the name of the copyright holders nor the names of its
17 | contributors may be used to endorse or promote products derived from
18 | this software without specific prior written permission.
19 |
20 | The copyright holders provide no reassurances that the source code
21 | provided does not infringe any patent, copyright, or any other
22 | intellectual property rights of third parties. The copyright holders
23 | disclaim any liability to any recipient for claims brought against
24 | recipient by any third party for infringement of that parties
25 | intellectual property rights.
26 |
27 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
29 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
30 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
31 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
32 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
33 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
34 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
35 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
36 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
37 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 |
39 |
40 | Citation
41 | ========
42 |
43 | Any published work which uses this software should include the
44 | following citation:
45 |
46 | ----------------------------------------------------------------------
47 | Maciej Besta, Nils Blach, Ales Kubicek, Robert Gerstenberger, Lukas
48 | Gianinazzi, Joanna Gajda, Tomasz Lehmann, Michał Podstawski, Hubert
49 | Niewiadomski, Piotr Nyczyk, Torsten Hoefler (2024): Graph of Thoughts:
50 | Solving Elaborate Problems with Large Language Models. In: Proceedings
51 | of the AAAI Conference on Artificial Intelligence, 38(16),
52 | 17682-17690. https://doi.org/10.1609/aaai.v38i16.29720
53 | ----------------------------------------------------------------------
54 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Graph of Thoughts (GoT)
2 |
3 |
4 |
5 |
6 |
7 | This is the official implementation of [Graph of Thoughts: Solving Elaborate Problems with Large Language Models](https://arxiv.org/pdf/2308.09687.pdf).
8 | This framework gives you the ability to solve complex problems by modeling them as a Graph of Operations (GoO), which is automatically executed with a Large Language Model (LLM) as the engine.
9 | This framework is designed to be flexible and extensible, allowing you to not only solve problems using the new GoT approach, but also to implement GoOs resembling previous approaches like CoT or ToT.
10 |
11 | ## Setup Guide
12 |
13 | In order to use this framework, you need to have a working installation of Python 3.8 or newer.
14 |
15 | ### Installing GoT
16 |
17 | Before running either of the following two installation methods, make sure to activate your Python environment (if any) beforehand.
18 | If you are a user and you just want to use `graph_of_thoughts`, you can install it directly from PyPI:
19 | ```bash
20 | pip install graph_of_thoughts
21 | ```
22 | If you are a developer and you want to modify the code, you can install it in editable mode from source:
23 | ```bash
24 | git clone https://github.com/spcl/graph-of-thoughts.git
25 | cd graph-of-thoughts
26 | pip install -e .
27 | ```
28 |
29 | ### Configuring the LLM
30 |
31 | In order to use the framework, you need to have access to an LLM.
32 | Please follow the instructions in the [Controller README](graph_of_thoughts/controller/README.md) to configure the LLM of your choice.
33 |
34 | ## Quick Start
35 |
36 | The following code snippet shows how to use the framework to solve the sorting problem for a list of 32 numbers using a CoT-like approach.
37 | Make sure you have followed the [Setup Guide](#setup-guide) before running the code.
38 |
39 | ```python
40 | from examples.sorting.sorting_032 import SortingPrompter, SortingParser, utils
41 | from graph_of_thoughts import controller, language_models, operations
42 |
43 | # Problem input
44 |
45 | to_be_sorted = "[0, 2, 6, 3, 8, 7, 1, 1, 6, 7, 7, 7, 7, 9, 3, 0, 1, 7, 9, 1, 3, 5, 1, 3, 6, 4, 5, 4, 7, 3, 5, 7]"
46 |
47 | # Create the Graph of Operations
48 | gop = operations.GraphOfOperations()
49 | gop.append_operation(operations.Generate())
50 | gop.append_operation(operations.Score(scoring_function=utils.num_errors))
51 | gop.append_operation(operations.GroundTruth(utils.test_sorting))
52 |
53 | # Configure the Language Model (Assumes config.json is in the current directory with OpenAI API key)
54 | lm = language_models.ChatGPT("config.json", model_name="chatgpt")
55 |
56 | # Create the Controller
57 | ctrl = controller.Controller(
58 | lm,
59 | gop,
60 | SortingPrompter(),
61 | SortingParser(),
62 | # The following dictionary is used to configure the initial thought state
63 | {
64 | "original": to_be_sorted,
65 | "current": "",
66 | "method": "cot"
67 | }
68 | )
69 |
70 | # Run the Controller and generate the output graph
71 | ctrl.run()
72 | ctrl.output_graph("output_cot.json")
73 | ```
74 |
75 | To run the more sophisticated GoT approach, you can use the following code snippet.
76 |
77 | ```python
78 | from examples.sorting.sorting_032 import SortingPrompter, SortingParser, got, utils
79 | from graph_of_thoughts import controller, language_models, operations
80 |
81 | # Problem input
82 |
83 | to_be_sorted = "[0, 2, 6, 3, 8, 7, 1, 1, 6, 7, 7, 7, 7, 9, 3, 0, 1, 7, 9, 1, 3, 5, 1, 3, 6, 4, 5, 4, 7, 3, 5, 7]"
84 |
85 | # Retrieve the Graph of Operations
86 | gop = got()
87 |
88 | # Configure the Language Model (Assumes config.json is in the current directory with OpenAI API key)
89 | lm = language_models.ChatGPT("config.json", model_name="chatgpt")
90 |
91 | # Create the Controller
92 | ctrl = controller.Controller(
93 | lm,
94 | gop,
95 | SortingPrompter(),
96 | SortingParser(),
97 | # The following dictionary is used to configure the initial thought state
98 | {
99 | "original": to_be_sorted,
100 | "current": "",
101 | "phase": 0,
102 | "method": "got"
103 | }
104 | )
105 |
106 | # Run the Controller and generate the output graph
107 | ctrl.run()
108 | ctrl.output_graph("output_got.json")
109 | ```
110 | You can compare the two results by inspecting the output graphs `output_cot.json` and `output_got.json`.
111 | The final thought states' scores indicate the number of errors in the sorted list.
112 |
113 | ## Documentation
114 | The paper gives a high-level overview of the framework and its components.
115 | In order to understand the framework in more detail, you can read the documentation of the individual modules.
116 | Especially the [Controller](graph_of_thoughts/controller/README.md) and [Operations](graph_of_thoughts/operations/README.md) modules are important for understanding how to make the most out of the framework.
117 | We took extra care to fully document the code, so that you can easily understand how it works and how to extend it.
118 |
119 | ## Examples
120 |
121 | The [examples](examples) directory contains several examples of problems that can be solved using the framework, including the ones presented in the paper.
122 | It is a great starting point for learning how to use the framework to solve real problems.
123 | Each example contains a `README.md` file with instructions on how to run it and play with it. The code is fully documented and should be easy to follow.
124 | You can also run the examples straight from the main directory. Note that the results will be stored in the respective examples sub-directory.
125 |
126 | Try for instance:
127 | ```bash
128 | python -m examples.sorting.sorting_032
129 | python -m examples.keyword_counting.keyword_counting
130 | ```
131 | ## Paper Results
132 |
133 | You can run the experiments from the paper by following the instructions in the [examples](examples) directory.
134 | However, if you just want to inspect and replot the results, you can use the [paper](paper) directory.
135 |
136 | ## Citations
137 |
138 | If you find this repository valuable, please give it a star!
139 | Got any questions or feedback? Feel free to reach out to [nils.blach@inf.ethz.ch](mailto:nils.blach@inf.ethz.ch) or open an issue.
140 | Using this in your work? Please reference us using the provided citation:
141 |
142 | ```bibtex
143 | @article{besta2024got,
144 | title = {{Graph of Thoughts: Solving Elaborate Problems with Large Language Models}},
145 | author = {Besta, Maciej and Blach, Nils and Kubicek, Ales and Gerstenberger, Robert and Gianinazzi, Lukas and Gajda, Joanna and Lehmann, Tomasz and Podstawski, Micha{\l} and Niewiadomski, Hubert and Nyczyk, Piotr and Hoefler, Torsten},
146 | year = 2024,
147 | month = {Mar},
148 | journal = {Proceedings of the AAAI Conference on Artificial Intelligence},
149 | volume = 38,
150 | number = 16,
151 | pages = {17682-17690},
152 | publisher = {AAAI Press},
153 | doi = {10.1609/aaai.v38i16.29720},
154 | url = {https://ojs.aaai.org/index.php/AAAI/article/view/29720}
155 | }
156 | ```
157 |
--------------------------------------------------------------------------------
/examples/README.md:
--------------------------------------------------------------------------------
1 | # Examples
2 |
3 | This directory contains scripts for running various examples using the Graph of Thoughts package. Each script is a standalone Python program that sets up and runs a particular example.
4 |
5 | We further include prompt files for each example that can be used to test prompts manually in a console.
6 |
7 | Please refer to the individual example directories for more information on the specific example.
8 |
--------------------------------------------------------------------------------
/examples/doc_merge/README.md:
--------------------------------------------------------------------------------
1 | # Document Merging
2 |
3 | The use case in this directory generates new Non-Disclosure Agreement (NDA) based on several input ones that partially overlap in terms of their contents.
4 | We provide implementations of five different approaches:
5 | - IO
6 | - Chain-of-Thought (CoT)
7 | - Tree of Thought (ToT)
8 | - Graph of Thoughts (GoT):
9 | - GoT: aggregation of fully merged NDAs
10 | - GoT2: aggregation of partially merged NDAs
11 |
12 | ## Data
13 |
14 | We provide an input file with 50 samples: `documents.csv`.
15 |
16 | ## Execution
17 |
18 | The file to execute the use case is called
19 | `doc_merge.py`. In the main body, one can
20 | select the specific samples to be run (variable samples) and the
21 | approaches (variable approaches). It is also possible to set a budget in
22 | dollars (variable budget).
23 |
24 | The Python scripts will create the directory `result`, if it is not
25 | already present. In the `result` directory, another directory is created
26 | for each run: `{name of LLM}_{list of approaches}_{day}_{start time}`.
27 | Inside each execution specific directory two files (`config.json`,
28 | `log.log`) and a separate directory for each selected approach are
29 | created. `config.json` contains the configuration of the run: input data,
30 | selected approaches, name of the LLM, and the budget. `log.log` contains
31 | the prompts and responses of the LLM as well as additional debug data.
32 | The approach directories contain a separate json file for every sample
33 | and the file contains the Graph Reasoning State (GRS) for that sample.
34 |
35 | ## Plot Data
36 |
37 | Change the results directory in line 158 of `plot.py` and run `python3
38 | plot.py` to plot your data.
39 |
--------------------------------------------------------------------------------
/examples/doc_merge/plot.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # main author: Nils Blach
8 |
9 | import json
10 | import os
11 | import matplotlib.pyplot as plt
12 |
13 |
14 | def get_complete_results(base_directory):
15 | results_complete = {}
16 | for folder_name in os.listdir(base_directory):
17 | folder_path = os.path.join(base_directory, folder_name)
18 | if os.path.isdir(folder_path):
19 | results_complete[folder_name] = []
20 | for file_name in os.listdir(folder_path):
21 | if file_name.endswith(".json"):
22 | file_path = os.path.join(folder_path, file_name)
23 | with open(file_path, "r") as f:
24 | data = json.load(f)
25 | results_complete[folder_name].append(
26 | {"key": int(file_name.split(".")[0]), "data": data}
27 | )
28 | for key in results_complete.keys():
29 | results_complete[key] = sorted(
30 | results_complete[key], key=lambda x: x["key"]
31 | )
32 | return results_complete
33 |
34 |
35 | def get_final_scores(results_complete):
36 | scores = {}
37 | for method in results_complete.keys():
38 | scores[method] = []
39 | for result in results_complete[method]:
40 | score = 0
41 | solved = False
42 | cost = 1
43 | prompt_tokens = 0
44 | completion_tokens = 0
45 | for op in reversed(result["data"]):
46 | if "cost" in op:
47 | cost = op["cost"]
48 | prompt_tokens = op["prompt_tokens"]
49 | completion_tokens = op["completion_tokens"]
50 | if "operation" in op and op["operation"] == "score":
51 | try:
52 | score = max(op["scores"])
53 | break
54 | except:
55 | continue
56 | scores[method].append(
57 | [result["key"], score, solved, prompt_tokens, completion_tokens, cost]
58 | )
59 | scores[method] = sorted(scores[method], key=lambda x: x[0])
60 | return scores
61 |
62 |
63 | def get_plotting_data(base_directory):
64 | results_complete = get_complete_results(base_directory)
65 | scores = get_final_scores(results_complete)
66 | results_plotting = {
67 | method: {
68 | "scores": [x[1] for x in scores[method]],
69 | "solved": sum([1 for x in scores[method] if x[2]]),
70 | "costs": [x[5] for x in scores[method]],
71 | }
72 | for method in scores.keys()
73 | }
74 | return results_plotting
75 |
76 |
77 | def plot_results(
78 | results,
79 | methods_order=["io", "cot", "tot", "got", "got2"],
80 | model="GPT-3.5",
81 | num_ndas=4,
82 | y_lower=0,
83 | y_upper=10,
84 | cost_upper=1.8,
85 | display_solved=True,
86 | annotation_offset=1,
87 | display_left_ylabel=False,
88 | display_right_ylabel=False,
89 | ):
90 | methods_order = [method for method in methods_order if method in results]
91 | scores_ordered = [
92 | [score for score in results[method]["scores"]] for method in methods_order
93 | ]
94 | total_costs = [sum(results[method]["costs"]) for method in methods_order]
95 |
96 | # Create figure and axis
97 | fig, ax = plt.subplots(dpi=150, figsize=(3.75, 5))
98 |
99 | # Create boxplots
100 | positions = range(1, len(methods_order) + 1)
101 | ax.boxplot(scores_ordered, positions=positions)
102 |
103 | fig_fontsize = 12
104 |
105 | # Set the ticks and labels
106 | methods_labels = ["IO", "CoT", "ToT", "GoT", "GoT2"]
107 | ax.set_xticks(range(1, len(methods_order) + 1))
108 | ax.set_xticks(range(1, len(methods_order) + 1))
109 | ax.set_xticklabels(methods_labels)
110 | # ax.set_xlabel("Approach")
111 |
112 | ax.set_ylim(y_lower, 12 if display_solved else 9.75)
113 | plt.yticks(fontsize=fig_fontsize)
114 |
115 | if display_left_ylabel:
116 | ax.set_ylabel(
117 | f"Score (out of 10); the higher the better", fontsize=fig_fontsize
118 | )
119 |
120 | # ax.set_title(f"Document Merging")
121 |
122 | ax2 = ax.twinx()
123 | ax2.bar(
124 | positions,
125 | total_costs,
126 | alpha=0.5,
127 | color="blue",
128 | label="Total Cost ($); the lower the better",
129 | )
130 | ax2.yaxis.set_tick_params(colors="#1919ff", labelsize=fig_fontsize)
131 | ax2.set_ylim(0, cost_upper)
132 | number_of_ticks = len(ax.get_yticks())
133 | tick_interval = cost_upper / (number_of_ticks)
134 | ax2_ticks = [tick_interval * i for i in range(number_of_ticks)]
135 |
136 | # Set custom tick positions for ax2
137 | ax2.set_yticks(ax2_ticks)
138 |
139 | if display_right_ylabel:
140 | ax2.set_ylabel(
141 | "Total Cost ($); the lower the better",
142 | color="#1919ff",
143 | fontsize=fig_fontsize,
144 | )
145 |
146 | if display_solved:
147 | annotation_height = y_upper + annotation_offset
148 | count = 1
149 | for method in methods_order:
150 | if method not in results:
151 | continue
152 | solved = results[method]["solved"]
153 | ax.text(
154 | count, annotation_height, f"Solved: {solved}", ha="center", va="bottom"
155 | )
156 | count += 1
157 |
158 | model = model.replace(".", "").replace("-", "").lower()
159 | fig.savefig(f"doc_merge_{model}_{num_ndas}.pdf", bbox_inches="tight")
160 |
161 |
162 | plot_results(
163 | get_plotting_data("results/"),
164 | num_ndas=4,
165 | display_solved=False,
166 | model="GPT-3.5",
167 | y_upper=10,
168 | display_left_ylabel=True,
169 | cost_upper=15,
170 | )
171 |
--------------------------------------------------------------------------------
/examples/keyword_counting/README.md:
--------------------------------------------------------------------------------
1 | # Keyword Counting
2 |
3 | The use case in this directory computes the frequencies of occurring countries
4 | in a long passage of text. We provide implementations of seven different approaches:
5 | - IO
6 | - Chain-of-Thought (CoT)
7 | - Tree of Thought (ToT):
8 | - ToT: wider tree, meaning more branches per level
9 | - ToT2: tree with more levels, but fewer branches per level
10 | - Graph of Thoughts (GoT):
11 | - GoT4: split passage into 4 sub-passages
12 | - GoT8: split passage into 8 sub-passages
13 | - GoTx: split by sentences
14 |
15 | ## Data
16 |
17 | We provide an input file with 100 samples: `countries.csv`. It is also possible to use
18 | the data generator `dataset_gen_countries.py` to generate additional or
19 | different samples (using GPT-4). The parameters can be updated on line 54 (number of samples to be generated).
20 | Note that not every generated sample will be included in the dataset, as each sample is
21 | additionally tested for validity (observe script output for details).
22 |
23 | ## Execution
24 |
25 | The file to execute the use case is called
26 | `keyword_counting.py`. In the main body, one can
27 | select the specific samples to be run (variable samples) and the
28 | approaches (variable approaches). It is also possible to set a budget in
29 | dollars (variable budget).
30 |
31 | The Python scripts will create the directory `result`, if it is not
32 | already present. In the `result` directory, another directory is created
33 | for each run: `{name of LLM}_{list of approaches}_{day}_{start time}`.
34 | Inside each execution specific directory two files (`config.json`,
35 | `log.log`) and a separate directory for each selected approach are
36 | created. `config.json` contains the configuration of the run: input data,
37 | selected approaches, name of the LLM, and the budget. `log.log` contains
38 | the prompts and responses of the LLM as well as additional debug data.
39 | The approach directories contain a separate json file for every sample
40 | and the file contains the Graph Reasoning State (GRS) for that sample.
41 |
42 | ## Plot Data
43 |
44 | Change the results directory in line 150 of `plot.py` and run `python3
45 | plot.py` to plot your data.
46 |
--------------------------------------------------------------------------------
/examples/keyword_counting/dataset_gen_countries.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # main author: Ales Kubicek
8 |
9 | import csv
10 | from typing import List, Tuple
11 | from graph_of_thoughts import controller
12 |
13 |
14 | def find_country_indices(text: str, country: str) -> List[Tuple[int, str]]:
15 | """
16 | Finds the indices of the occurences of a given country in the input text.
17 |
18 | :param text: Input text.
19 | :type text: str
20 | :param country: Country to search for.
21 | :type country: str
22 | :return: List of tuples, where each tuple consists of index and country.
23 | :rtype: List[Tuple[int, str]]
24 | """
25 |
26 | indices = []
27 | index = text.find(country)
28 | while index != -1:
29 | indices.append(index)
30 | index = text.find(country, index + 1)
31 | return [(index, country) for index in indices]
32 |
33 |
34 | primary_countries = [
35 | "Afghanistan",
36 | "Argentina",
37 | "Australia",
38 | "Brazil",
39 | "Canada",
40 | "China",
41 | "Colombia",
42 | "Cuba",
43 | "Egypt",
44 | "France",
45 | "Germany",
46 | "Greece",
47 | "India",
48 | "Indonesia",
49 | "Iran",
50 | "Iraq",
51 | "Ireland",
52 | "Israel",
53 | "Italy",
54 | "Japan",
55 | "Kenya",
56 | "Mexico",
57 | "Netherlands",
58 | "New Zealand",
59 | "Nigeria",
60 | "North Korea",
61 | "Pakistan",
62 | "Peru",
63 | "Philippines",
64 | "Poland",
65 | "Portugal",
66 | "Russia",
67 | "Saudi Arabia",
68 | "South Africa",
69 | "South Korea",
70 | "Spain",
71 | "Sweden",
72 | "Switzerland",
73 | "Thailand",
74 | "Turkey",
75 | "Ukraine",
76 | "United Arab Emirates",
77 | "United Kingdom",
78 | "United States",
79 | "Venezuela",
80 | "Vietnam",
81 | "Yemen",
82 | "Zimbabwe",
83 | "Belgium",
84 | "Norway",
85 | ]
86 | primary_adjectives = [
87 | "Afghan",
88 | "Argentine ",
89 | "Argentinean",
90 | "Australian",
91 | "Brazilian",
92 | "Canadian",
93 | "Chinese",
94 | "Colombian",
95 | "Cuban",
96 | "Egyptian",
97 | "French",
98 | "German",
99 | "Greek",
100 | "Indian",
101 | "Indonesian",
102 | "Iranian",
103 | "Iraqi",
104 | "Irish",
105 | "Israeli",
106 | "Italian",
107 | "Japanese",
108 | "Kenyan",
109 | "Mexican",
110 | "Dutch",
111 | "New Zealander ",
112 | "Kiwi",
113 | "Nigerian",
114 | "North Korean",
115 | "Pakistani",
116 | "Peruvian",
117 | "Filipino",
118 | "Philippine",
119 | "Polish",
120 | "Portuguese",
121 | "Russian",
122 | "Saudi ",
123 | "Saudi Arabian",
124 | "South African",
125 | "South Korean",
126 | "Spanish",
127 | "Swedish",
128 | "Swiss",
129 | "Thai",
130 | "Turkish",
131 | "Ukrainian",
132 | "United Arab Emirates",
133 | "Emirati",
134 | "British",
135 | "American",
136 | "Venezuelan",
137 | "Vietnamese",
138 | "Yemeni",
139 | "Zimbabwean",
140 | "Belgian",
141 | "Norwegian",
142 | ]
143 | rest_countries = [
144 | "Albania",
145 | "Algeria",
146 | "Andorra",
147 | "Angola",
148 | "Antigua and Barbuda",
149 | "Armenia",
150 | "Austria",
151 | "Azerbaijan",
152 | "The Bahamas",
153 | "Bahrain",
154 | "Bangladesh",
155 | "Barbados",
156 | "Belarus",
157 | "Belize",
158 | "Benin",
159 | "Bhutan",
160 | "Bolivia",
161 | "Bosnia and Herzegovina",
162 | "Botswana",
163 | "Brunei",
164 | "Bulgaria",
165 | "Burkina Faso",
166 | "Burundi",
167 | "Cabo Verde",
168 | "Cambodia",
169 | "Cameroon",
170 | "Central African Republic",
171 | "Chad",
172 | "Chile",
173 | "Comoros",
174 | "Congo",
175 | "Costa Rica",
176 | "Côte d’Ivoire",
177 | "Croatia",
178 | "Cyprus",
179 | "Czech Republic",
180 | "Czechia",
181 | "Denmark",
182 | "Djibouti",
183 | "Dominica",
184 | "Dominican Republic",
185 | "East Timor",
186 | "Timor-Leste",
187 | "Ecuador",
188 | "El Salvador",
189 | "Equatorial Guinea",
190 | "Eritrea",
191 | "Estonia",
192 | "Eswatini",
193 | "Ethiopia",
194 | "Fiji",
195 | "Finland",
196 | "Gabon",
197 | "The Gambia",
198 | "Georgia",
199 | "Ghana",
200 | "Grenada",
201 | "Guatemala",
202 | "Guinea",
203 | "Guinea-Bissau",
204 | "Guyana",
205 | "Haiti",
206 | "Honduras",
207 | "Hungary",
208 | "Iceland",
209 | "Jamaica",
210 | "Jordan",
211 | "Kazakhstan",
212 | "Kiribati",
213 | "Kosovo",
214 | "Kuwait",
215 | "Kyrgyzstan",
216 | "Laos",
217 | "Latvia",
218 | "Lebanon",
219 | "Lesotho",
220 | "Liberia",
221 | "Libya",
222 | "Liechtenstein",
223 | "Lithuania",
224 | "Luxembourg",
225 | "Madagascar",
226 | "Malawi",
227 | "Malaysia",
228 | "Maldives",
229 | "Mali",
230 | "Malta",
231 | "Marshall Islands",
232 | "Mauritania",
233 | "Mauritius",
234 | "Micronesia",
235 | "Moldova",
236 | "Monaco",
237 | "Mongolia",
238 | "Montenegro",
239 | "Morocco",
240 | "Mozambique",
241 | "Myanmar",
242 | "Burma",
243 | "Namibia",
244 | "Nauru",
245 | "Nepal",
246 | "Nicaragua",
247 | "Niger",
248 | "North Macedonia",
249 | "Oman",
250 | "Palau",
251 | "Panama",
252 | "Papua New Guinea",
253 | "Paraguay",
254 | "Qatar",
255 | "Romania",
256 | "Rwanda",
257 | "Saint Kitts and Nevis",
258 | "Saint Lucia",
259 | "Saint Vincent and the Grenadines",
260 | "Samoa",
261 | "San Marino",
262 | "Sao Tome and Principe",
263 | "Senegal",
264 | "Serbia",
265 | "Seychelles",
266 | "Sierra Leone",
267 | "Singapore",
268 | "Slovakia",
269 | "Slovenia",
270 | "Solomon Islands",
271 | "Somalia",
272 | "Sri Lanka",
273 | "Sudan",
274 | "Suriname",
275 | "Syria",
276 | "Taiwan",
277 | "Tajikistan",
278 | "Tanzania",
279 | "Togo",
280 | "Tonga",
281 | "Trinidad and Tobago",
282 | "Tunisia",
283 | "Turkmenistan",
284 | "Tuvalu",
285 | "Uganda",
286 | "Uruguay",
287 | "Uzbekistan",
288 | "Vanuatu",
289 | "Vatican City",
290 | "Zambia",
291 | ]
292 | rest_adjectives = [
293 | "Albanian",
294 | "Algerian",
295 | "Andorran",
296 | "Angolan",
297 | "Antiguan and Barbudan",
298 | "Armenian",
299 | "Austrian",
300 | "Azerbaijani",
301 | "Bahamian",
302 | "Bahraini",
303 | "Bangladeshi",
304 | "Barbadian",
305 | "Belarusian",
306 | "Belizean",
307 | "Beninese",
308 | "Bhutanese",
309 | "Bolivian",
310 | "Bosnian and Herzegovinian",
311 | "Botswanan",
312 | "Bruneian",
313 | "Bulgarian",
314 | "Burkinabè",
315 | "Burundian",
316 | "Cape Verdean",
317 | "Cambodian",
318 | "Cameroonian",
319 | "Central African",
320 | "Chadian",
321 | "Chilean",
322 | "Comorian",
323 | "Congolese",
324 | "Costa Rican",
325 | "Ivorian",
326 | "Croatian",
327 | "Cypriot",
328 | "Czech",
329 | "Czech",
330 | "Danish",
331 | "Djiboutian",
332 | "Dominican",
333 | "Dominican",
334 | "East Timorese",
335 | "Timorese",
336 | "Ecuadorian",
337 | "Salvadoran",
338 | "Equatorial Guinean",
339 | "Eritrean",
340 | "Estonian",
341 | "Swazi",
342 | "Ethiopian",
343 | "Fijian",
344 | "Finnish",
345 | "Gabonese",
346 | "Gambian",
347 | "Georgian",
348 | "Ghanaian",
349 | "Grenadian",
350 | "Guatemalan",
351 | "Guinean",
352 | "Bissau-Guinean",
353 | "Guyanese",
354 | "Haitian",
355 | "Honduran",
356 | "Hungarian",
357 | "Icelandic",
358 | "Jamaican",
359 | "Jordanian",
360 | "Kazakh",
361 | "I-Kiribati",
362 | "Kosovar",
363 | "Kuwaiti",
364 | "Kyrgyz",
365 | "Laotian",
366 | "Latvian",
367 | "Lebanese",
368 | "Basotho",
369 | "Liberian",
370 | "Libyan",
371 | "Liechtensteiner",
372 | "Lithuanian",
373 | "Luxembourger",
374 | "Malagasy",
375 | "Malawian",
376 | "Malaysian",
377 | "Maldivian",
378 | "Malian",
379 | "Maltese",
380 | "Marshallese",
381 | "Mauritanian",
382 | "Mauritian",
383 | "Micronesian",
384 | "Moldovan",
385 | "Monégasque",
386 | "Mongolian",
387 | "Montenegrin",
388 | "Moroccan",
389 | "Mozambican",
390 | "Myanmarese",
391 | "Burmese",
392 | "Namibian",
393 | "Nauruan",
394 | "Nepali",
395 | "Nicaraguan",
396 | "Nigerien",
397 | "Macedonian",
398 | "Omani",
399 | "Palauan",
400 | "Panamanian",
401 | "Papua New Guinean",
402 | "Paraguayan",
403 | "Qatari",
404 | "Romanian",
405 | "Rwandan",
406 | "Kittitian",
407 | "Nevisian",
408 | "Saint Lucian",
409 | "Vincentian",
410 | "Samoan",
411 | "Sammarinese",
412 | "Santomean",
413 | "Senegalese",
414 | "Serbian",
415 | "Seychellois",
416 | "Sierra Leonean",
417 | "Singaporean",
418 | "Slovak",
419 | "Slovenian",
420 | "Solomon Islander",
421 | "Somali",
422 | "Sri Lankan",
423 | "Sudanese",
424 | "Surinamese",
425 | "Syrian",
426 | "Taiwanese",
427 | "Tajik",
428 | "Tanzanian",
429 | "Togolese",
430 | "Tongan",
431 | "Trinidadian ",
432 | "Tobagonian",
433 | "Tunisian",
434 | "Turkmen",
435 | "Tuvaluan",
436 | "Ugandan",
437 | "Uruguayan",
438 | "Uzbek",
439 | "Ni-Vanuatu",
440 | "Vatican",
441 | "Zambian",
442 | ]
443 |
444 | lm = controller.ChatGPT(
445 | "../../graph_of_thoughts/controller/config.json", model_name="chatgpt4"
446 | )
447 |
448 | prompt = """ Generate a continuous passage (single paragraph) of 16 sentences following the provided restrictions precisely.
449 |
450 |
451 | The following restrictions must apply to the generated text:
452 | 1. Single continuous passage of exactly 16 sentences without any paragraphs (line breaks).
453 | 2. Countries appearing in the passage must be only from the provided list. No other countries can be mentioned.
454 | 3. When a country is mentioned in the passage, it must be mentioned multiple times consecutively in the same or following sentences.
455 | 4. Passage should be creative and coherent.
456 | 5. Using adjectives of a country is NOT allowed (e.g., "Colombian coffee" should be "coffee from Colombia" instead)
457 |
458 |
459 |
460 | List of countries: [Afghanistan, Argentina, Australia, Brazil, Canada, China, Colombia, Cuba, Egypt, France, Germany, Greece, India, Indonesia, Iran, Iraq, Ireland, Israel, Italy, Japan, Kenya, Mexico, Netherlands, New Zealand, Nigeria, North Korea, Pakistan, Peru, Philippines, Poland, Portugal, Russia, Saudi Arabia, South Africa, South Korea, Spain, Sweden, Switzerland, Thailand, Turkey, Ukraine, United Arab Emirates, United Kingdom, United States, Venezuela, Vietnam, Yemen, Zimbabwe, Belgium, Norway]
461 | Passage:
462 | While exploring the ancient ruins in Greece, Sam discovered manuscripts that hinted at the hidden treasures of Egypt. It seemed these treasures were once stolen from Egypt by rogue merchants and secretly moved to Greece, only to be buried under layers of time. Intrigued, he shared the findings with his friend Maya from India, who was an expert in decoding ancient languages. She pointed out that there was a similar legend in India about treasures from China that had somehow ended up in the southern parts of India, possibly through trade or conquest. She also recounted tales from China that spoke of incredible artifacts from Indonesia, suggesting a rich tapestry of cultural exchanges throughout history. Their conversation took an interesting turn when Sam mentioned a book he'd read about the mysterious connections between Argentina and Brazil. The book detailed how both Argentina and Brazil, despite their differences, shared tales of lost civilizations and forgotten cities deep within their jungles. Maya excitedly mentioned that she'd been to the Philippines and had heard local legends about ancient ties with Indonesia and how traders from the Philippines would journey to Indonesia in search of spices and other goods. Thinking of spices, Sam fondly recalled his trip to Spain, where he had learned about the country's historical links with Portugal. Spain and Portugal, both maritime giants of their time, had extensively explored unknown lands and established trade routes. Maya, remembering her travels, said that she had been to Belgium once and was fascinated by its connections with the Netherlands. Both Belgium and the Netherlands, she explained, had rich histories of art, trade, and diplomacy that intertwined them for centuries. They both sat back, marveling at the interconnectedness of the world and how countries from Greece to the Netherlands shared tales of adventure, discovery, and mystery.
463 |
464 |
465 | List of countries: [Afghanistan, Argentina, Australia, Brazil, Canada, China, Colombia, Cuba, Egypt, France, Germany, Greece, India, Indonesia, Iran, Iraq, Ireland, Israel, Italy, Japan, Kenya, Mexico, Netherlands, New Zealand, Nigeria, North Korea, Pakistan, Peru, Philippines, Poland, Portugal, Russia, Saudi Arabia, South Africa, South Korea, Spain, Sweden, Switzerland, Thailand, Turkey, Ukraine, United Arab Emirates, United Kingdom, United States, Venezuela, Vietnam, Yemen, Zimbabwe, Belgium, Norway]
466 | Passage:
467 | """
468 |
469 | num_samples = 100
470 | sample_id = 0
471 | result = [["ID", "Text", "Countries", "Sentences", "Characters"]]
472 |
473 | """
474 | Generate passages of text that contain country names to be used as input for the
475 | keyword counting.
476 |
477 | Input(x) : Number of samples
478 | Output(y) : Passages written to a file in the CSV format.
479 | File contains the sample ID, the passage, the countries the passage
480 | contains, the sentences of the passages, number of characters of the
481 | passage.
482 | """
483 |
484 | # For x batches of y responses
485 | for _ in range(num_samples):
486 | response = lm.query(prompt, 1)
487 | texts = lm.get_response_texts(response)
488 | for text in texts:
489 | # Clean paragraphs - single long passage
490 | text = text.strip().replace("\n", "")
491 |
492 | # Get all occurrences of all primary permissible countries
493 | occurrences = []
494 | for country in [country for country in primary_countries if country in text]:
495 | occurrences.extend(find_country_indices(text, country))
496 | # Order exactly how they appear in the text
497 | ordered_occurrences = [country[1] for country in sorted(occurrences)]
498 |
499 | # Check invalid countries and adjectives
500 | invalid_primary_adjective = [
501 | adjective for adjective in primary_adjectives if adjective in text
502 | ]
503 | invalid_rest_country = [
504 | country for country in rest_countries if country in text
505 | ]
506 | invalid_rest_adjective = [
507 | adjective for adjective in rest_adjectives if adjective in text
508 | ]
509 | invalid_count = (
510 | len(invalid_primary_adjective)
511 | + len(invalid_rest_country)
512 | + len(invalid_rest_adjective)
513 | )
514 |
515 | if invalid_count > 0:
516 | print(
517 | f"Invalid countries or adjectives present: {invalid_primary_adjective}, {invalid_rest_country}, {invalid_rest_adjective}"
518 | )
519 | continue
520 |
521 | result.append(
522 | [
523 | sample_id,
524 | text,
525 | "[{0}]".format(", ".join(map(str, ordered_occurrences))),
526 | len(text.split(".")) - 1,
527 | len(text),
528 | ]
529 | )
530 | sample_id += 1
531 |
532 | # Writing to csv file
533 | with open("countries_script.csv", "w") as csvfile:
534 | csvwriter = csv.writer(csvfile)
535 | csvwriter.writerows(result)
536 |
--------------------------------------------------------------------------------
/examples/keyword_counting/plot.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # main author: Nils Blach
8 | # contributions: Ales Kubicek
9 |
10 | import json
11 | import os
12 | import matplotlib.pyplot as plt
13 |
14 |
15 | def get_complete_results(base_directory):
16 | results_complete = {}
17 | for folder_name in os.listdir(base_directory):
18 | folder_path = os.path.join(base_directory, folder_name)
19 | if os.path.isdir(folder_path):
20 | results_complete[folder_name] = []
21 | for file_name in os.listdir(folder_path):
22 | if file_name.endswith(".json"):
23 | file_path = os.path.join(folder_path, file_name)
24 | with open(file_path, "r") as f:
25 | data = json.load(f)
26 | results_complete[folder_name].append(
27 | {"key": int(file_name.split(".")[0]), "data": data}
28 | )
29 | for key in results_complete.keys():
30 | results_complete[key] = sorted(
31 | results_complete[key], key=lambda x: x["key"]
32 | )
33 | return results_complete
34 |
35 |
36 | def get_final_scores(results_complete):
37 | scores = {}
38 | for method in results_complete.keys():
39 | scores[method] = []
40 | for result in results_complete[method]:
41 | score = 100
42 | solved = False
43 | cost = 1
44 | prompt_tokens = 0
45 | completion_tokens = 0
46 | for op in result["data"]:
47 | if "operation" in op and op["operation"] == "ground_truth_evaluator":
48 | try:
49 | score = min(op["scores"])
50 | solved = any(op["problem_solved"])
51 | except:
52 | continue
53 | if "cost" in op:
54 | cost = op["cost"]
55 | prompt_tokens = op["prompt_tokens"]
56 | completion_tokens = op["completion_tokens"]
57 | scores[method].append(
58 | [result["key"], score, solved, prompt_tokens, completion_tokens, cost]
59 | )
60 | scores[method] = sorted(scores[method], key=lambda x: x[0])
61 | return scores
62 |
63 |
64 | def get_plotting_data(base_directory):
65 | results_complete = get_complete_results(base_directory)
66 | scores = get_final_scores(results_complete)
67 | results_plotting = {
68 | method: {
69 | "scores": [x[1] for x in scores[method]],
70 | "solved": sum([1 for x in scores[method] if x[2]]),
71 | "costs": [x[5] for x in scores[method]],
72 | }
73 | for method in scores.keys()
74 | }
75 | return results_plotting
76 |
77 |
78 | def plot_results(
79 | results,
80 | methods_order=["io", "cot", "tot", "tot2", "got4", "got8", "gotx"],
81 | model="GPT-3.5",
82 | y_lower=0,
83 | y_upper=40,
84 | cost_upper=1.8,
85 | display_solved=True,
86 | annotation_offset=1,
87 | display_left_ylabel=False,
88 | display_right_ylabel=False,
89 | ):
90 | methods_order = [method for method in methods_order if method in results]
91 | # Extract scores based on the order
92 | scores_ordered = [
93 | [score for score in results[method]["scores"] if score != 100 and score != 300]
94 | for method in methods_order
95 | ]
96 | total_costs = [sum(results[method]["costs"]) for method in methods_order]
97 |
98 | # Create figure and axis
99 | fig, ax = plt.subplots(dpi=150, figsize=(3.75, 4))
100 |
101 | # Create boxplots
102 | positions = range(1, len(methods_order) + 1)
103 | ax.boxplot(scores_ordered, positions=positions)
104 |
105 | fig_fontsize = 12
106 |
107 | # Set the ticks and labels
108 | methods_labels = ["IO", "CoT", "ToT", "ToT2", "GoT4", "GoT8", "GoTx"]
109 | ax.set_xticks(range(1, len(methods_order) + 1))
110 | ax.set_xticks(range(1, len(methods_order) + 1))
111 | ax.set_xticklabels(methods_labels, fontsize=10)
112 |
113 | ax.set_ylim(y_lower, (y_upper + 2) if display_solved else y_upper + 1)
114 | plt.yticks(fontsize=fig_fontsize)
115 | if display_left_ylabel:
116 | ax.set_ylabel(f"Number of errors; the lower the better", fontsize=fig_fontsize)
117 |
118 | ax.set_title(f"Keyword Counting")
119 |
120 | ax2 = ax.twinx()
121 | ax2.bar(positions, total_costs, alpha=0.5, color="blue", label="Total Cost ($)")
122 | ax2.yaxis.set_tick_params(colors="#1919ff", labelsize=fig_fontsize)
123 | ax2.set_ylim(0, cost_upper)
124 | number_of_ticks = len(ax.get_yticks())
125 | tick_interval = cost_upper / (number_of_ticks)
126 | ax2_ticks = [tick_interval * i for i in range(number_of_ticks)]
127 |
128 | ax2.set_yticks(ax2_ticks)
129 |
130 | if display_right_ylabel:
131 | ax2.set_ylabel(
132 | "Total Cost ($); the lower the better",
133 | color="#1919ff",
134 | fontsize=fig_fontsize,
135 | )
136 |
137 | if display_solved:
138 | annotation_height = y_upper + annotation_offset
139 | count = 1
140 | for method in methods_order:
141 | if method not in results:
142 | continue
143 | solved = results[method]["solved"]
144 | ax.text(
145 | count,
146 | annotation_height,
147 | f"{solved}",
148 | ha="center",
149 | va="bottom",
150 | fontsize=fig_fontsize,
151 | )
152 | count += 1
153 |
154 | model = model.replace(".", "").replace("-", "").lower()
155 | fig.savefig(f"keyword_counting_{model}.pdf", bbox_inches="tight")
156 |
157 |
158 | plot_results(
159 | get_plotting_data("results/"),
160 | display_solved=True,
161 | annotation_offset=-0.3,
162 | model="GPT-3.5",
163 | y_upper=35,
164 | display_left_ylabel=True,
165 | display_right_ylabel=True,
166 | cost_upper=9,
167 | )
168 |
--------------------------------------------------------------------------------
/examples/set_intersection/README.md:
--------------------------------------------------------------------------------
1 | # Set Intersection
2 |
3 | The use case in this directory computes the intersection of two input
4 | sets. We provide implementations of five different approaches for 32, 64
5 | and 128 elements:
6 | - IO
7 | - Chain-of-Thought (CoT)
8 | - Tree of Thought (ToT):
9 | - ToT: wider tree, meaning more branches per level
10 | - ToT2: tree with more levels, but fewer branches per level
11 | - Graph of Thoughts (GoT)
12 |
13 | ## Data
14 |
15 | We provide input files with 100 precomputed samples for each set length:
16 | `set_intersection_.csv`. It is also possible to use
17 | the data generator `dataset_gen_intersection.py` to generate additional or
18 | different samples. The parameters can be updated in lines 24 to 28 of
19 | the main body:
20 | - set_size = 32 # size of the generated sets
21 | - int_value_ubound = 64 # (exclusive) upper limit of generated numbers
22 | - seed = 42 # seed of the random number generator
23 | - num_sample = 100 # number of samples
24 | - filename = 'set_intersection_032.csv' # output filename
25 |
26 | ## Execution
27 |
28 | The files to execute the use case are called
29 | `set_intersection_.py`. In the main body, one can
30 | select the specific samples to be run (variable sample) and the
31 | approaches (variable approaches). It is also possible to set a budget in
32 | dollars (variable budget).
33 | The input filename for the samples is currently hardcoded to
34 | `set_intersection_.csv`, but can be updated in the
35 | function `run`.
36 |
37 | The Python scripts will create the directory `result`, if it is not
38 | already present. In the `result` directory, another directory is created
39 | for each run: `{name of LLM}_{list of approaches}_{day}_{start time}`.
40 | Inside each execution specific directory two files (`config.json`,
41 | `log.log`) and a separate directory for each selected approach are
42 | created. `config.json` contains the configuration of the run: input data,
43 | selected approaches, name of the LLM, and the budget. `log.log` contains
44 | the prompts and responses of the LLM as well as additional debug data.
45 | The approach directories contain a separate json file for every sample
46 | and the file contains the Graph Reasoning State (GRS) for that sample.
47 |
48 | ## Plot Data
49 |
50 | Change the results directory in line 170 of `plot.py` and update the
51 | length parameter in the subsequent line and run `python3 plot.py` to
52 | plot your data.
53 |
--------------------------------------------------------------------------------
/examples/set_intersection/dataset_gen_intersection.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # main author: Robert Gerstenberger
8 |
9 | import csv
10 | import numpy as np
11 |
12 |
13 | def scramble(array: np.ndarray, rng: np.random.Generator) -> None:
14 | """
15 | Helper function to change the order of the elements in an array randomly.
16 |
17 | :param array: Array to be scrambled.
18 | :type: numpy.ndarray
19 | :param rng: Random number generator.
20 | :type rng: numpy.random.Generator
21 | """
22 |
23 | size = array.shape[0]
24 |
25 | index_array = rng.integers(0, size, size)
26 |
27 | for i in range(size):
28 | temp = array[i]
29 | array[i] = array[index_array[i]]
30 | array[index_array[i]] = temp
31 |
32 |
33 | if __name__ == "__main__":
34 | """
35 | Input(u) : Set size.
36 | Input(v) : Range of the integer numbers in the sets: 0..v (exclusive)
37 | Input(w) : Seed for the random number generator.
38 | Input(x) : Number of samples to be generated.
39 | Input(y) : Filename for the output CSV file.
40 | Output(z) : Input sets and intersected set written a file in the CSV format.
41 | File contains the sample ID, input set 1, input set 2,
42 | intersection set.
43 | """
44 |
45 | set_size = 32 # size of the generated sets
46 | int_value_ubound = 64 # (exclusive) upper limit of generated numbers
47 | seed = 42 # seed of the random number generator
48 | num_sample = 100 # number of samples
49 | filename = "set_intersection_032.csv" # output filename
50 |
51 | assert 2 * set_size <= int_value_ubound
52 |
53 | rng = np.random.default_rng(seed)
54 |
55 | intersection_sizes = rng.integers(set_size // 4, 3 * set_size // 4, num_sample)
56 |
57 | np.set_printoptions(
58 | linewidth=np.inf
59 | ) # no wrapping in the array fields in the output file
60 |
61 | with open(filename, "w") as f:
62 | fieldnames = ["ID", "SET1", "SET2", "INTERSECTION"]
63 | writer = csv.DictWriter(f, delimiter=",", fieldnames=fieldnames)
64 | writer.writeheader()
65 |
66 | for i in range(num_sample):
67 | intersection_size = intersection_sizes[i]
68 |
69 | full_set = np.arange(0, int_value_ubound, dtype=np.int16)
70 |
71 | scramble(full_set, rng)
72 |
73 | intersection = full_set[:intersection_size].copy()
74 |
75 | sorted_intersection = np.sort(intersection)
76 |
77 | set1 = full_set[:set_size].copy()
78 | set2 = np.concatenate(
79 | [intersection, full_set[set_size : 2 * set_size - intersection_size]]
80 | )
81 |
82 | scramble(set1, rng)
83 | scramble(set2, rng)
84 |
85 | writer.writerow(
86 | {
87 | "ID": i,
88 | "SET1": set1.tolist(),
89 | "SET2": set2.tolist(),
90 | "INTERSECTION": sorted_intersection.tolist(),
91 | }
92 | )
93 |
--------------------------------------------------------------------------------
/examples/set_intersection/example_prompts_set_intersection_032.md:
--------------------------------------------------------------------------------
1 | # Set Intersection for 32-Element Sets - Prompts and Examples
2 | ## Prompt Templates
3 | ### GENERATE: split_prompt
4 | Replace `{input}` with the input list/set of numbers to be split.
5 | ```
6 | Split the following list of 32 numbers into 2 lists of 16 numbers each, the first list should contain the first 16 numbers and the second list the second 16 numbers.
7 | Only output the 2 lists in the following format without any additional text or thoughts!:
8 | {
9 | "List 1": [13, 16, 30, 6, 21, 7, 31, ...],
10 | "List 2": [25, 24, 10, 4, 27, 0, 14, ...]
11 | }
12 |
13 |
14 | Input: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
15 | Output:
16 | {
17 | "List 1": [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22],
18 | "List 2": [33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
19 | }
20 |
21 |
22 | Input: {input}
23 | ```
24 |
25 | ### GENERATE: intersect_prompt
26 | Replace `{set1}` and `{set2}` with the sets to be intersected.
27 | ```
28 | Find the intersection of two sets of numbers. Output only the set of numbers that are present in both sets, no additional text.
29 |
30 |
31 | Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
32 | Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
33 | Output: [24, 10, 20, 8]
34 |
35 | Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
36 | Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
37 | Output: [40, 15, 5, 24, 35, 59, 16, 63]
38 |
39 | Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
40 | Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
41 | Output: [115, 35, 90, 117, 63, 40, 30, 33, 15, 89, 50, 12, 2, 84, 7, 57, 96]
42 |
43 |
44 | Input Set 1: {set1}
45 | Input Set 2: {set2}
46 | ```
47 |
48 | ### AGGREGATE: merge_prompt
49 | Replace `{input1}` and `{input2}` with the lists/sets to be merged.
50 | ```
51 | Merge the following 2 lists of length 16 each, into one list of length 32 by appending the second list to the first list.
52 | Only output the final list without any additional text or thoughts!
53 |
54 | List 1: {input1}
55 | List 2: {input2}
56 |
57 | ```
58 |
59 | ## Complete Example Prompts
60 | ### The GoO Summarised
61 |
62 | 1. Split the second input set into two sub-sets of equal size (split prompt)
63 | 2. For each sub-set: Intersect the sub-set with the first input set (intersect prompt) five times; score each sort attempt; keep the best
64 | 3. Merge the resulting intersections into one full intersection set (merge prompt) 10 times; score each merge attempt; keep the best
65 |
66 | ### Input
67 | ```
68 | [31, 11, 4, 63, 38, 58, 59, 24, 61, 14, 32, 39, 27, 46, 48, 19, 52, 57, 50, 56, 3, 2, 53, 29, 5, 37, 62, 41, 36, 12, 49, 16]
69 | ```
70 |
71 | ### Step 1 - Prompt
72 | ```
73 | Split the following list of 32 numbers into 2 lists of 16 numbers each, the first list should contain the first 16 numbers and the second list the second 16 numbers.
74 | Only output the 2 lists in the following format without any additional text or thoughts!:
75 | {
76 | "List 1": [13, 16, 30, 6, 21, 7, 31, ...],
77 | "List 2": [25, 24, 10, 4, 27, 0, 14, ...]
78 | }
79 |
80 |
81 | Input: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
82 | Output:
83 | {
84 | "List 1": [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22],
85 | "List 2": [33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
86 | }
87 |
88 |
89 | Input: [31, 11, 4, 63, 38, 58, 59, 24, 61, 14, 32, 39, 27, 46, 48, 19, 52, 57, 50, 56, 3, 2, 53, 29, 5, 37, 62, 41, 36, 12, 49, 16]
90 | ```
91 |
92 | ### Step 1 - Response
93 | ```
94 | {
95 | "List 1": [31, 11, 4, 63, 38, 58, 59, 24, 61, 14, 32, 39, 27, 46, 48, 19],
96 | "List 2": [52, 57, 50, 56, 3, 2, 53, 29, 5, 37, 62, 41, 36, 12, 49, 16]
97 | }
98 | ```
99 |
100 | ### Step 2a - Prompt
101 | ```
102 | Find the intersection of two sets of numbers. Output only the set of numbers that are present in both sets, no additional text.
103 |
104 |
105 | Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
106 | Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
107 | Output: [24, 10, 20, 8]
108 |
109 | Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
110 | Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
111 | Output: [40, 15, 5, 24, 35, 59, 16, 63]
112 |
113 | Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
114 | Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
115 | Output: [115, 35, 90, 117, 63, 40, 30, 33, 15, 89, 50, 12, 2, 84, 7, 57, 96]
116 |
117 |
118 | Input Set 1: [11, 60, 1, 49, 21, 33, 14, 56, 54, 15, 23, 40, 45, 22, 7, 28, 20, 46, 51, 6, 34, 37, 3, 50, 17, 8, 25, 0, 35, 47, 18, 19]
119 | Input Set 2: [31, 11, 4, 63, 38, 58, 59, 24, 61, 14, 32, 39, 27, 46, 48, 19]
120 | ```
121 |
122 | ### Step 2a - 5 Responses
123 | ```
124 | 1. [11, 14, 46, 14, 19]
125 | 2. Output: [11, 14, 46, 19]
126 | 3. [11, 14, 46, 14, 19]
127 | 4. [11, 14, 46, 19]
128 | 5. [11, 14, 46, 14, 19]
129 | ```
130 |
131 | ### Step 2b - Prompt
132 | ```
133 | Find the intersection of two sets of numbers. Output only the set of numbers that are present in both sets, no additional text.
134 |
135 |
136 | Input Set 1: [13, 16, 30, 6, 21, 7, 31, 15, 11, 1, 24, 10, 9, 3, 20, 8]
137 | Input Set 2: [25, 24, 10, 4, 27, 0, 14, 12, 8, 2, 29, 20, 17, 19, 26, 23]
138 | Output: [24, 10, 20, 8]
139 |
140 | Input Set 1: [26, 40, 42, 57, 15, 31, 5, 32, 11, 4, 24, 28, 51, 54, 12, 22, 33, 35, 7, 13, 2, 59, 8, 23, 43, 16, 29, 55, 25, 63, 21, 18]
141 | Input Set 2: [16, 60, 36, 48, 0, 15, 5, 19, 46, 24, 1, 6, 61, 10, 38, 53, 58, 9, 44, 14, 35, 63, 52, 20, 27, 17, 39, 47, 34, 56, 40, 59]
142 | Output: [40, 15, 5, 24, 35, 59, 16, 63]
143 |
144 | Input Set 1: [115, 61, 35, 103, 90, 117, 86, 44, 63, 45, 40, 30, 74, 33, 31, 1, 118, 48, 38, 0, 119, 51, 64, 78, 15, 121, 89, 101, 79, 69, 120, 29, 58, 50, 116, 11, 60, 12, 39, 95, 23, 2, 109, 84, 7, 43, 99, 98, 52, 70, 75, 102, 57, 19, 94, 36, 114, 88, 71, 56, 83, 6, 96, 107]
145 | Input Set 2: [13, 35, 20, 96, 34, 18, 47, 127, 126, 9, 21, 16, 77, 22, 111, 122, 85, 73, 42, 105, 123, 15, 33, 59, 67, 57, 104, 8, 30, 89, 76, 12, 65, 84, 32, 40, 7, 100, 108, 50, 14, 28, 24, 53, 90, 17, 91, 81, 124, 63, 5, 46, 125, 93, 49, 66, 117, 37, 115, 113, 2, 106, 41, 72]
146 | Output: [115, 35, 90, 117, 63, 40, 30, 33, 15, 89, 50, 12, 2, 84, 7, 57, 96]
147 |
148 |
149 | Input Set 1: [11, 60, 1, 49, 21, 33, 14, 56, 54, 15, 23, 40, 45, 22, 7, 28, 20, 46, 51, 6, 34, 37, 3, 50, 17, 8, 25, 0, 35, 47, 18, 19]
150 | Input Set 2: [52, 57, 50, 56, 3, 2, 53, 29, 5, 37, 62, 41, 36, 12, 49, 16]
151 | ```
152 |
153 | ### Step 2b - 5 Responses
154 | ```
155 | 1. [50, 56, 49]
156 | 2. [50, 56, 37, 49]
157 | 3. [56, 49, 37, 3, 50]
158 | 4. [50, 56, 37, 49]
159 | 5. [50, 56, 37, 49]
160 | ```
161 |
162 | ### Step 3 - Prompt
163 | ```
164 | Merge the following 2 lists of length 16 each, into one list of length 32 by appending the second list to the first list.
165 | Only output the final list without any additional text or thoughts!
166 |
167 | List 1: [11, 14, 46, 19]
168 | List 2: [56, 49, 37, 3, 50]
169 | ```
170 |
171 | ### Step 3 - 10 Responses
172 | ```
173 | 1. [11, 14, 46, 19, 56, 49, 37, 3, 50]
174 | 2. [11, 14, 46, 19, 56, 49, 37, 3, 50]
175 | 3. [11, 14, 46, 19, 56, 49, 37, 3, 50]
176 | 4. [11, 14, 46, 19, 56, 49, 37, 3, 50]
177 | 5. [11, 14, 46, 19, 56, 49, 37, 3, 50]
178 | 6. [11, 14, 46, 19, 56, 49, 37, 3, 50]
179 | 7. [11, 14, 46, 19, 56, 49, 37, 3, 50]
180 | 8. [11, 14, 46, 19, 56, 49, 37, 3, 50]
181 | 9. [11, 14, 46, 19, 56, 49, 37, 3, 50]
182 | 10. [11, 14, 46, 19, 56, 49, 37, 3, 50]
183 | ```
184 |
185 | ### Final Result
186 | ```
187 | [11, 14, 46, 19, 56, 49, 37, 3, 50]
188 | ```
189 |
--------------------------------------------------------------------------------
/examples/set_intersection/plot.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # main author: Nils Blach
8 | # contributions: Robert Gerstenberger
9 |
10 | import json
11 | import os
12 | import matplotlib.pyplot as plt
13 |
14 |
15 | def get_complete_results(base_directory):
16 | results_complete = {}
17 | for folder_name in os.listdir(base_directory):
18 | folder_path = os.path.join(base_directory, folder_name)
19 | if os.path.isdir(folder_path):
20 | results_complete[folder_name] = []
21 | for file_name in os.listdir(folder_path):
22 | if file_name.endswith(".json"):
23 | file_path = os.path.join(folder_path, file_name)
24 | with open(file_path, "r") as f:
25 | data = json.load(f)
26 | results_complete[folder_name].append(
27 | {"key": int(file_name.split(".")[0]), "data": data}
28 | )
29 | for key in results_complete.keys():
30 | results_complete[key] = sorted(
31 | results_complete[key], key=lambda x: x["key"]
32 | )
33 | return results_complete
34 |
35 |
36 | def get_final_scores(results_complete):
37 | scores = {}
38 | for method in results_complete.keys():
39 | scores[method] = []
40 | for result in results_complete[method]:
41 | score = 100
42 | solved = False
43 | cost = 1
44 | prompt_tokens = 0
45 | completion_tokens = 0
46 | for op in result["data"]:
47 | if "operation" in op and op["operation"] == "ground_truth_evaluator":
48 | try:
49 | score = min(op["scores"])
50 | solved = any(op["problem_solved"])
51 | except:
52 | continue
53 | if "cost" in op:
54 | cost = op["cost"]
55 | prompt_tokens = op["prompt_tokens"]
56 | completion_tokens = op["completion_tokens"]
57 | scores[method].append(
58 | [result["key"], score, solved, prompt_tokens, completion_tokens, cost]
59 | )
60 | scores[method] = sorted(scores[method], key=lambda x: x[0])
61 | return scores
62 |
63 |
64 | def get_plotting_data(base_directory):
65 | results_complete = get_complete_results(base_directory)
66 | scores = get_final_scores(results_complete)
67 | results_plotting = {
68 | method: {
69 | "scores": [x[1] for x in scores[method]],
70 | "solved": sum([1 for x in scores[method] if x[2]]),
71 | "costs": [x[5] for x in scores[method]],
72 | }
73 | for method in scores.keys()
74 | }
75 | return results_plotting
76 |
77 |
78 | def plot_results(
79 | results,
80 | methods_order=["io", "cot", "tot", "tot2", "got"],
81 | model="GPT-3.5",
82 | length=32,
83 | y_lower=0,
84 | cost_upper=0.0,
85 | display_solved=True,
86 | annotation_offset=0,
87 | display_left_ylabel=False,
88 | display_right_ylabel=False,
89 | ):
90 | methods_order = [method for method in methods_order if method in results]
91 | # Extract scores based on the order
92 | scores_ordered = [
93 | [score for score in results[method]["scores"] if score != 1000]
94 | for method in methods_order
95 | ]
96 | total_costs = [sum(results[method]["costs"]) for method in methods_order]
97 |
98 | # Create figure and axis
99 | fig, ax = plt.subplots(dpi=150, figsize=(2.5, 5))
100 |
101 | # Create boxplots
102 | positions = range(1, len(methods_order) + 1)
103 | ax.boxplot(scores_ordered, positions=positions)
104 |
105 | fig_fontsize = 12
106 |
107 | # Set the ticks and labels
108 | methods_labels = ["IO", "CoT", "ToT", "ToT2", "GoT"]
109 | plt.yticks(fontsize=fig_fontsize)
110 | ax.set_xticks(range(1, len(methods_order) + 1))
111 | ax.set_xticks(range(1, len(methods_order) + 1))
112 | ax.set_xticklabels(methods_labels, fontsize=fig_fontsize)
113 |
114 | y_upper = length
115 |
116 | range_increase = 1
117 | if display_solved:
118 | if length < 48:
119 | range_increase = 2
120 | elif length < 96:
121 | range_increase = 4
122 | else:
123 | range_increase = 8
124 |
125 | ax.set_ylim(y_lower, y_upper + range_increase)
126 | ax1_yticks = range(
127 | y_lower, y_upper + 1, 2 if length < 48 else (4 if length < 96 else 8)
128 | )
129 | ax.set_yticks(ax1_yticks)
130 | if display_left_ylabel:
131 | ax.set_ylabel(
132 | f"#incorrect elements; the lower the better", fontsize=fig_fontsize
133 | )
134 |
135 | ax.set_title(f"{length} elements")
136 |
137 | ax2 = ax.twinx()
138 | ax2.bar(positions, total_costs, alpha=0.5, color="blue", label="Total Cost ($)")
139 | ax2.yaxis.set_tick_params(colors="#1919ff", labelsize=fig_fontsize)
140 | if cost_upper > 0:
141 | ax2.set_ylim(0, cost_upper)
142 | number_of_ticks = len(ax.get_yticks())
143 | tick_interval = cost_upper / (number_of_ticks)
144 | ax2_ticks = [tick_interval * i for i in range(number_of_ticks)]
145 |
146 | # Set custom tick positions for ax2
147 | ax2.set_yticks(ax2_ticks)
148 |
149 | if display_right_ylabel:
150 | ax2.set_ylabel(
151 | "Total Cost ($); the lower the better",
152 | color="#1919ff",
153 | fontsize=fig_fontsize,
154 | )
155 |
156 | if display_solved:
157 | annotation_height = y_upper + annotation_offset
158 | count = 1
159 | for method in methods_order:
160 | if method not in results:
161 | continue
162 | solved = results[method]["solved"]
163 | ax.text(
164 | count,
165 | annotation_height,
166 | f"{solved}",
167 | ha="center",
168 | va="bottom",
169 | fontsize=fig_fontsize,
170 | )
171 | count += 1
172 |
173 | model = model.replace(".", "").replace("-", "").lower()
174 | fig.savefig(f"set_intersection_{model}_{length}.pdf", bbox_inches="tight")
175 |
176 |
177 | plot_results(
178 | get_plotting_data("results/"),
179 | length=32,
180 | display_solved=True,
181 | model="GPT-3.5",
182 | display_left_ylabel=True,
183 | display_right_ylabel=True,
184 | )
185 |
--------------------------------------------------------------------------------
/examples/set_intersection/set_intersection_032.csv:
--------------------------------------------------------------------------------
1 | ID,SET1,SET2,INTERSECTION
2 | 0,"[11, 60, 1, 49, 21, 33, 14, 56, 54, 15, 23, 40, 45, 22, 7, 28, 20, 46, 51, 6, 34, 37, 3, 50, 17, 8, 25, 0, 35, 47, 18, 19]","[31, 11, 4, 63, 38, 58, 59, 24, 61, 14, 32, 39, 27, 46, 48, 19, 52, 57, 50, 56, 3, 2, 53, 29, 5, 37, 62, 41, 36, 12, 49, 16]","[3, 11, 14, 19, 37, 46, 49, 50, 56]"
3 | 1,"[28, 58, 36, 18, 37, 31, 44, 39, 34, 51, 12, 56, 21, 27, 7, 24, 46, 1, 25, 2, 41, 6, 45, 29, 49, 42, 35, 30, 54, 55, 10, 50]","[4, 16, 28, 46, 49, 21, 58, 30, 19, 1, 37, 15, 3, 59, 51, 24, 10, 12, 34, 20, 40, 44, 35, 23, 36, 0, 43, 54, 2, 31, 57, 41]","[1, 2, 10, 12, 21, 24, 28, 30, 31, 34, 35, 36, 37, 41, 44, 46, 49, 51, 54, 58]"
4 | 2,"[32, 24, 17, 27, 51, 19, 50, 59, 8, 18, 41, 36, 43, 39, 49, 4, 5, 31, 30, 60, 48, 61, 10, 16, 47, 58, 9, 54, 40, 37, 1, 11]","[44, 27, 33, 46, 51, 17, 28, 18, 4, 23, 32, 49, 58, 1, 20, 40, 39, 52, 30, 55, 26, 19, 63, 31, 57, 13, 47, 61, 10, 6, 38, 41]","[1, 4, 10, 17, 18, 19, 27, 30, 31, 32, 39, 40, 41, 47, 49, 51, 58, 61]"
5 | 3,"[22, 38, 13, 31, 63, 6, 59, 17, 27, 20, 21, 42, 24, 4, 25, 15, 28, 29, 19, 49, 40, 30, 34, 0, 53, 23, 12, 62, 36, 51, 39, 45]","[21, 29, 30, 28, 31, 55, 2, 22, 19, 4, 46, 32, 37, 43, 23, 15, 58, 56, 11, 50, 48, 54, 41, 45, 51, 7, 42, 38, 18, 9, 6, 1]","[4, 6, 15, 19, 21, 22, 23, 28, 29, 30, 31, 38, 42, 45, 51]"
6 | 4,"[4, 26, 48, 7, 42, 60, 31, 36, 25, 20, 14, 50, 21, 30, 12, 10, 23, 8, 39, 13, 46, 32, 11, 34, 6, 2, 57, 0, 17, 33, 18, 24]","[1, 37, 48, 28, 39, 58, 53, 10, 6, 50, 17, 9, 51, 56, 38, 52, 8, 30, 46, 62, 5, 24, 61, 22, 57, 14, 44, 11, 47, 35, 63, 36]","[6, 8, 10, 11, 14, 17, 24, 30, 36, 39, 46, 48, 50, 57]"
7 | 5,"[48, 50, 19, 59, 57, 63, 49, 61, 14, 11, 29, 39, 4, 17, 21, 12, 3, 35, 23, 2, 58, 60, 38, 43, 28, 47, 5, 44, 15, 55, 8, 26]","[50, 12, 30, 23, 11, 61, 17, 58, 19, 55, 62, 43, 33, 2, 37, 21, 56, 4, 15, 24, 38, 45, 1, 28, 59, 48, 35, 6, 47, 8, 27, 22]","[2, 4, 8, 11, 12, 15, 17, 19, 21, 23, 28, 35, 38, 43, 47, 48, 50, 55, 58, 59, 61]"
8 | 6,"[34, 2, 12, 54, 13, 41, 57, 42, 21, 59, 26, 10, 6, 35, 27, 16, 1, 44, 8, 11, 14, 7, 62, 20, 30, 19, 32, 9, 43, 39, 22, 24]","[18, 41, 49, 56, 1, 3, 45, 36, 17, 34, 50, 46, 29, 30, 61, 32, 14, 52, 24, 51, 5, 25, 31, 55, 37, 0, 23, 40, 28, 20, 22, 63]","[1, 14, 20, 22, 24, 30, 32, 34, 41]"
9 | 7,"[49, 61, 9, 23, 27, 21, 47, 37, 54, 32, 18, 14, 22, 13, 62, 46, 0, 26, 4, 11, 60, 34, 25, 50, 6, 51, 59, 8, 35, 41, 29, 17]","[33, 21, 49, 54, 6, 27, 3, 47, 4, 20, 43, 41, 8, 24, 0, 57, 28, 35, 15, 17, 46, 23, 13, 9, 45, 31, 22, 12, 5, 29, 50, 7]","[0, 4, 6, 8, 9, 13, 17, 21, 22, 23, 27, 29, 35, 41, 46, 47, 49, 50, 54]"
10 | 8,"[55, 59, 60, 13, 41, 15, 38, 52, 35, 5, 9, 61, 57, 34, 47, 22, 21, 1, 27, 11, 36, 2, 6, 58, 62, 51, 23, 50, 8, 39, 28, 54]","[37, 14, 4, 55, 50, 17, 8, 19, 10, 44, 0, 31, 27, 53, 39, 35, 30, 5, 62, 24, 26, 29, 54, 45, 40, 3, 43, 11, 12, 25, 41, 42]","[5, 8, 11, 27, 35, 39, 41, 50, 54, 55, 62]"
11 | 9,"[20, 12, 52, 56, 36, 51, 59, 40, 55, 33, 41, 38, 28, 45, 13, 2, 43, 27, 35, 7, 9, 0, 10, 37, 29, 15, 14, 49, 17, 63, 25, 32]","[11, 13, 19, 6, 21, 54, 36, 14, 58, 26, 2, 39, 53, 22, 50, 51, 24, 16, 5, 17, 1, 60, 57, 23, 35, 33, 42, 18, 61, 29, 47, 8]","[2, 13, 14, 17, 29, 33, 35, 36, 51]"
12 | 10,"[24, 21, 51, 37, 54, 22, 18, 38, 14, 30, 48, 49, 28, 47, 10, 44, 58, 29, 3, 45, 26, 15, 9, 6, 33, 2, 34, 17, 20, 25, 0, 1]","[3, 0, 20, 1, 24, 45, 25, 52, 13, 43, 38, 15, 22, 53, 27, 8, 42, 40, 59, 32, 21, 60, 30, 46, 51, 47, 14, 50, 5, 62, 57, 28]","[0, 1, 3, 14, 15, 20, 21, 22, 24, 25, 28, 30, 38, 45, 47, 51]"
13 | 11,"[58, 52, 51, 15, 42, 26, 19, 30, 28, 6, 40, 11, 25, 53, 18, 47, 21, 12, 41, 45, 43, 7, 35, 48, 27, 3, 32, 2, 4, 62, 17, 29]","[41, 4, 0, 35, 25, 29, 14, 62, 45, 2, 37, 11, 15, 8, 30, 39, 58, 19, 56, 42, 17, 51, 24, 28, 44, 27, 13, 18, 26, 47, 53, 43]","[2, 4, 11, 15, 17, 18, 19, 25, 26, 27, 28, 29, 30, 35, 41, 42, 43, 45, 47, 51, 53, 58, 62]"
14 | 12,"[8, 17, 23, 38, 24, 47, 61, 3, 46, 43, 58, 39, 27, 21, 26, 14, 62, 1, 15, 19, 37, 53, 55, 6, 51, 57, 52, 25, 16, 22, 33, 54]","[2, 18, 15, 17, 23, 45, 14, 36, 53, 7, 27, 32, 31, 21, 8, 38, 16, 61, 6, 58, 3, 39, 49, 19, 22, 42, 0, 12, 30, 60, 26, 46]","[3, 6, 8, 14, 15, 16, 17, 19, 21, 22, 23, 26, 27, 38, 39, 46, 53, 58, 61]"
15 | 13,"[58, 10, 28, 52, 51, 48, 49, 13, 53, 25, 55, 6, 4, 15, 30, 56, 54, 41, 38, 42, 21, 33, 44, 18, 37, 43, 31, 20, 3, 2, 50, 0]","[54, 3, 32, 44, 26, 49, 0, 9, 35, 51, 25, 33, 62, 55, 47, 2, 29, 11, 42, 18, 4, 23, 31, 41, 20, 7, 58, 52, 13, 36, 5, 53]","[0, 2, 3, 4, 13, 18, 20, 25, 31, 33, 41, 42, 44, 49, 51, 52, 53, 54, 55, 58]"
16 | 14,"[12, 18, 22, 38, 6, 23, 54, 37, 40, 44, 19, 46, 35, 47, 7, 10, 16, 24, 4, 42, 31, 30, 0, 63, 25, 57, 39, 51, 3, 60, 49, 28]","[40, 6, 12, 18, 19, 34, 17, 47, 4, 29, 53, 3, 15, 7, 37, 1, 59, 42, 33, 35, 52, 16, 61, 13, 9, 51, 32, 60, 24, 54, 38, 46]","[3, 4, 6, 7, 12, 16, 18, 19, 24, 35, 37, 38, 40, 42, 46, 47, 51, 54, 60]"
17 | 15,"[61, 62, 10, 53, 17, 20, 28, 54, 55, 59, 4, 1, 11, 34, 5, 35, 56, 38, 60, 29, 43, 42, 63, 50, 24, 40, 47, 52, 16, 39, 31, 19]","[60, 24, 50, 45, 23, 18, 20, 31, 29, 35, 43, 44, 59, 41, 51, 46, 16, 63, 38, 3, 42, 25, 4, 17, 5, 9, 61, 7, 27, 54, 19, 39]","[4, 5, 16, 17, 19, 20, 24, 29, 31, 35, 38, 39, 42, 43, 50, 54, 59, 60, 61, 63]"
18 | 16,"[12, 2, 21, 59, 49, 30, 31, 14, 23, 39, 5, 15, 4, 19, 27, 43, 6, 34, 0, 22, 16, 60, 17, 48, 40, 9, 56, 1, 63, 8, 45, 51]","[40, 4, 38, 7, 1, 47, 44, 16, 58, 11, 25, 35, 3, 51, 26, 10, 41, 60, 46, 6, 34, 19, 2, 61, 21, 31, 8, 22, 12, 53, 15, 50]","[1, 2, 4, 6, 8, 12, 15, 16, 19, 21, 22, 31, 34, 40, 51, 60]"
19 | 17,"[26, 8, 53, 37, 45, 60, 32, 44, 19, 39, 36, 16, 25, 9, 6, 50, 38, 47, 30, 63, 42, 52, 57, 10, 7, 27, 14, 1, 41, 58, 59, 62]","[28, 12, 29, 21, 63, 34, 17, 49, 56, 4, 15, 39, 1, 46, 22, 62, 9, 43, 0, 50, 23, 54, 44, 31, 5, 51, 11, 37, 8, 61, 13, 42]","[1, 8, 9, 37, 39, 42, 44, 50, 62, 63]"
20 | 18,"[9, 58, 28, 53, 3, 18, 45, 33, 11, 5, 2, 47, 4, 59, 38, 30, 61, 15, 21, 48, 41, 25, 26, 13, 60, 23, 6, 55, 17, 36, 62, 10]","[43, 15, 16, 10, 33, 5, 13, 62, 41, 53, 0, 32, 54, 23, 26, 19, 25, 9, 37, 49, 38, 21, 28, 27, 40, 45, 4, 61, 29, 3, 55, 18]","[3, 4, 5, 9, 10, 13, 15, 18, 21, 23, 25, 26, 28, 33, 38, 41, 45, 53, 55, 61, 62]"
21 | 19,"[26, 63, 42, 19, 6, 62, 39, 14, 24, 17, 44, 31, 8, 7, 10, 16, 1, 56, 45, 43, 50, 57, 54, 51, 58, 59, 5, 37, 3, 21, 13, 30]","[60, 3, 1, 34, 17, 55, 50, 21, 0, 46, 62, 59, 47, 36, 15, 22, 19, 5, 2, 38, 58, 12, 32, 27, 28, 18, 24, 31, 26, 63, 51, 53]","[1, 3, 5, 17, 19, 21, 24, 26, 31, 50, 51, 58, 59, 62, 63]"
22 | 20,"[30, 31, 28, 56, 54, 29, 61, 18, 1, 46, 34, 5, 15, 63, 26, 45, 4, 47, 40, 21, 3, 9, 2, 27, 7, 6, 24, 19, 42, 11, 35, 41]","[57, 5, 54, 42, 41, 31, 12, 13, 9, 34, 21, 23, 7, 48, 22, 29, 19, 14, 0, 43, 3, 27, 26, 4, 16, 25, 53, 51, 10, 11, 52, 20]","[3, 4, 5, 7, 9, 11, 19, 21, 26, 27, 29, 31, 34, 41, 42, 54]"
23 | 21,"[9, 12, 51, 48, 23, 6, 50, 14, 53, 15, 58, 44, 32, 59, 24, 42, 27, 41, 29, 10, 45, 34, 0, 46, 3, 36, 25, 16, 13, 63, 49, 52]","[16, 15, 51, 37, 20, 27, 47, 36, 45, 59, 0, 41, 40, 2, 28, 31, 17, 1, 21, 22, 5, 58, 18, 57, 34, 39, 54, 9, 53, 33, 60, 19]","[0, 9, 15, 16, 27, 34, 36, 41, 45, 51, 53, 58, 59]"
24 | 22,"[57, 44, 59, 3, 16, 5, 17, 6, 52, 0, 18, 40, 47, 11, 45, 14, 1, 49, 30, 55, 27, 12, 50, 10, 7, 39, 46, 22, 23, 38, 34, 41]","[44, 48, 9, 58, 28, 40, 4, 56, 49, 19, 53, 42, 47, 35, 25, 34, 29, 43, 33, 32, 6, 55, 37, 26, 20, 11, 2, 27, 31, 62, 54, 39]","[6, 11, 27, 34, 39, 40, 44, 47, 49, 55]"
25 | 23,"[32, 25, 2, 13, 11, 23, 54, 49, 9, 57, 59, 6, 61, 47, 37, 34, 18, 29, 46, 45, 56, 31, 14, 17, 48, 55, 7, 50, 27, 51, 63, 44]","[49, 34, 46, 32, 17, 50, 51, 54, 61, 14, 52, 21, 56, 16, 44, 33, 25, 57, 2, 28, 35, 19, 55, 11, 22, 9, 48, 20, 53, 29, 45, 18]","[2, 9, 11, 14, 17, 18, 25, 29, 32, 34, 44, 45, 46, 48, 49, 50, 51, 54, 55, 56, 57, 61]"
26 | 24,"[23, 16, 41, 14, 10, 42, 9, 27, 35, 51, 24, 4, 59, 47, 55, 12, 18, 58, 7, 63, 28, 32, 22, 49, 37, 31, 13, 1, 21, 39, 36, 30]","[39, 6, 63, 37, 27, 19, 40, 16, 18, 54, 22, 20, 2, 30, 23, 59, 46, 52, 43, 10, 21, 41, 31, 4, 12, 24, 32, 38, 3, 13, 47, 53]","[4, 10, 12, 13, 16, 18, 21, 22, 23, 24, 27, 30, 31, 32, 37, 39, 41, 47, 59, 63]"
27 | 25,"[20, 5, 41, 40, 24, 60, 2, 6, 59, 50, 34, 47, 10, 0, 14, 29, 35, 28, 39, 58, 13, 16, 27, 4, 18, 52, 56, 21, 45, 15, 32, 30]","[2, 23, 40, 31, 7, 52, 55, 18, 59, 56, 60, 44, 38, 54, 35, 4, 46, 1, 12, 47, 10, 3, 13, 29, 48, 43, 6, 11, 20, 24, 45, 34]","[2, 4, 6, 10, 13, 18, 20, 24, 29, 34, 35, 40, 45, 47, 52, 56, 59, 60]"
28 | 26,"[60, 39, 4, 62, 16, 57, 49, 41, 25, 59, 55, 35, 53, 33, 17, 5, 18, 13, 46, 30, 45, 38, 1, 20, 22, 44, 11, 15, 42, 7, 21, 28]","[57, 27, 24, 15, 44, 13, 60, 35, 62, 47, 38, 59, 42, 52, 49, 43, 10, 0, 53, 6, 37, 18, 19, 36, 51, 58, 3, 34, 23, 21, 63, 40]","[13, 15, 18, 21, 35, 38, 42, 44, 49, 53, 57, 59, 60, 62]"
29 | 27,"[9, 48, 28, 3, 61, 4, 1, 22, 62, 57, 18, 21, 34, 26, 25, 8, 12, 38, 27, 49, 46, 6, 40, 56, 10, 16, 36, 5, 45, 44, 51, 42]","[10, 36, 20, 5, 45, 34, 16, 18, 21, 51, 60, 17, 57, 50, 42, 54, 46, 63, 39, 38, 58, 3, 24, 53, 37, 26, 27, 48, 22, 44, 49, 8]","[3, 5, 8, 10, 16, 18, 21, 22, 26, 27, 34, 36, 38, 42, 44, 45, 46, 48, 49, 51, 57]"
30 | 28,"[17, 42, 40, 38, 58, 19, 63, 22, 50, 18, 8, 16, 12, 15, 61, 10, 52, 9, 44, 13, 60, 24, 41, 56, 35, 46, 21, 36, 28, 0, 48, 6]","[6, 33, 10, 5, 23, 41, 53, 30, 58, 22, 45, 42, 61, 7, 12, 19, 56, 60, 2, 28, 62, 36, 39, 43, 27, 20, 16, 55, 37, 40, 9, 32]","[6, 9, 10, 12, 16, 19, 22, 28, 36, 40, 41, 42, 56, 58, 60, 61]"
31 | 29,"[1, 4, 35, 57, 2, 11, 43, 56, 16, 9, 51, 8, 26, 0, 59, 31, 22, 25, 29, 50, 13, 44, 12, 32, 30, 27, 19, 36, 42, 61, 7, 38]","[59, 35, 54, 49, 7, 45, 33, 57, 32, 41, 18, 47, 55, 56, 58, 5, 4, 46, 21, 9, 63, 52, 34, 2, 62, 8, 48, 22, 51, 11, 1, 16]","[1, 2, 4, 7, 8, 9, 11, 16, 22, 32, 35, 51, 56, 57, 59]"
32 | 30,"[54, 2, 52, 36, 21, 37, 13, 28, 39, 17, 27, 0, 42, 56, 10, 4, 60, 48, 49, 24, 20, 16, 5, 62, 14, 47, 53, 1, 7, 32, 34, 12]","[38, 41, 13, 32, 31, 63, 11, 62, 1, 18, 52, 58, 56, 51, 7, 37, 36, 22, 9, 35, 43, 0, 46, 4, 55, 44, 25, 8, 24, 14, 34, 47]","[0, 1, 4, 7, 13, 14, 24, 32, 34, 36, 37, 47, 52, 56, 62]"
33 | 31,"[6, 32, 2, 58, 25, 19, 31, 0, 30, 28, 46, 49, 41, 48, 45, 44, 59, 51, 22, 3, 38, 36, 1, 15, 11, 26, 37, 24, 55, 9, 14, 61]","[10, 56, 39, 4, 34, 11, 29, 14, 57, 15, 60, 20, 9, 43, 53, 40, 8, 44, 50, 27, 47, 19, 36, 52, 6, 62, 45, 63, 12, 21, 51, 58]","[6, 9, 11, 14, 15, 19, 36, 44, 45, 51, 58]"
34 | 32,"[24, 51, 50, 19, 63, 42, 29, 59, 41, 9, 38, 28, 25, 4, 18, 40, 12, 1, 58, 34, 15, 30, 54, 6, 46, 27, 39, 55, 23, 61, 5, 36]","[56, 17, 24, 11, 35, 20, 15, 10, 3, 45, 26, 33, 21, 60, 8, 32, 22, 48, 25, 49, 41, 14, 13, 18, 19, 61, 2, 43, 52, 16, 5, 27]","[5, 15, 18, 19, 24, 25, 27, 41, 61]"
35 | 33,"[27, 37, 6, 7, 2, 32, 28, 11, 24, 43, 5, 15, 12, 18, 36, 26, 55, 35, 57, 56, 29, 54, 30, 9, 3, 20, 38, 48, 63, 25, 16, 10]","[21, 34, 3, 20, 2, 13, 56, 32, 25, 58, 61, 51, 59, 53, 6, 27, 8, 48, 50, 43, 46, 52, 63, 11, 42, 12, 19, 45, 9, 39, 55, 54]","[2, 3, 6, 9, 11, 12, 20, 25, 27, 32, 43, 48, 54, 55, 56, 63]"
36 | 34,"[22, 52, 23, 8, 46, 5, 19, 37, 31, 51, 6, 17, 21, 13, 29, 4, 58, 0, 32, 40, 9, 33, 55, 34, 11, 30, 24, 20, 3, 39, 2, 25]","[51, 20, 56, 37, 9, 19, 38, 53, 43, 35, 39, 23, 46, 8, 11, 3, 28, 49, 13, 22, 52, 2, 42, 26, 21, 30, 58, 24, 59, 40, 5, 6]","[2, 3, 5, 6, 8, 9, 11, 13, 19, 20, 21, 22, 23, 24, 30, 37, 39, 40, 46, 51, 52, 58]"
37 | 35,"[9, 37, 45, 60, 18, 38, 55, 14, 44, 49, 27, 19, 6, 2, 51, 26, 11, 24, 54, 32, 48, 57, 34, 3, 42, 5, 39, 31, 53, 62, 43, 41]","[61, 28, 30, 25, 33, 43, 46, 56, 58, 62, 21, 17, 16, 50, 29, 41, 13, 38, 54, 63, 40, 18, 35, 10, 36, 20, 7, 9, 8, 45, 19, 52]","[9, 18, 19, 38, 41, 43, 45, 54, 62]"
38 | 36,"[45, 26, 14, 2, 25, 41, 59, 58, 42, 39, 56, 46, 1, 48, 30, 31, 63, 36, 33, 5, 60, 19, 21, 57, 62, 35, 17, 37, 6, 3, 38, 13]","[43, 32, 60, 56, 36, 22, 59, 6, 39, 63, 46, 8, 26, 38, 57, 31, 29, 25, 45, 18, 55, 52, 13, 10, 62, 35, 30, 7, 47, 14, 37, 21]","[6, 13, 14, 21, 25, 26, 30, 31, 35, 36, 37, 38, 39, 45, 46, 56, 57, 59, 60, 62, 63]"
39 | 37,"[47, 52, 1, 22, 16, 56, 5, 30, 43, 54, 3, 48, 21, 17, 33, 9, 57, 11, 4, 46, 15, 26, 12, 7, 32, 44, 6, 41, 31, 14, 49, 59]","[42, 44, 49, 59, 58, 61, 57, 6, 46, 3, 7, 31, 43, 1, 34, 22, 12, 16, 4, 54, 55, 45, 27, 5, 26, 48, 13, 17, 19, 51, 39, 47]","[1, 3, 4, 5, 6, 7, 12, 16, 17, 22, 26, 31, 43, 44, 46, 47, 48, 49, 54, 57, 59]"
40 | 38,"[28, 27, 51, 11, 2, 31, 14, 6, 33, 25, 42, 49, 59, 60, 17, 10, 56, 12, 23, 36, 37, 53, 18, 52, 26, 21, 62, 63, 38, 22, 16, 44]","[52, 53, 6, 9, 40, 47, 14, 3, 34, 29, 18, 48, 7, 58, 20, 50, 51, 22, 5, 46, 28, 37, 60, 61, 55, 27, 1, 4, 49, 45, 39, 24]","[6, 14, 18, 22, 27, 28, 37, 49, 51, 52, 53, 60]"
41 | 39,"[41, 11, 6, 43, 60, 48, 37, 55, 31, 39, 7, 13, 58, 42, 62, 44, 21, 56, 35, 19, 49, 9, 36, 34, 14, 10, 8, 54, 30, 32, 52, 2]","[0, 36, 42, 58, 13, 38, 31, 45, 11, 61, 18, 28, 32, 60, 17, 57, 10, 2, 3, 59, 51, 35, 12, 9, 7, 50, 63, 41, 48, 34, 39, 49]","[2, 7, 9, 10, 11, 13, 31, 32, 34, 35, 36, 39, 41, 42, 48, 49, 58, 60]"
42 | 40,"[27, 26, 20, 51, 10, 56, 49, 22, 31, 61, 44, 24, 32, 17, 50, 29, 30, 19, 3, 4, 12, 0, 43, 41, 35, 58, 33, 15, 37, 34, 57, 25]","[54, 29, 52, 49, 45, 30, 48, 7, 11, 19, 47, 46, 10, 38, 16, 36, 40, 5, 41, 1, 23, 8, 42, 33, 56, 14, 59, 2, 58, 0, 63, 62]","[0, 10, 19, 29, 30, 33, 41, 49, 56, 58]"
43 | 41,"[44, 7, 43, 52, 38, 18, 47, 2, 49, 41, 36, 42, 57, 1, 23, 27, 39, 50, 31, 51, 16, 29, 30, 26, 40, 24, 10, 20, 6, 11, 62, 19]","[47, 21, 60, 51, 62, 55, 28, 48, 16, 2, 52, 36, 10, 18, 63, 3, 45, 32, 50, 43, 19, 44, 27, 56, 26, 1, 8, 29, 25, 41, 11, 38]","[1, 2, 10, 11, 16, 18, 19, 26, 27, 29, 36, 38, 41, 43, 44, 47, 50, 51, 52, 62]"
44 | 42,"[11, 29, 26, 5, 13, 32, 53, 10, 25, 60, 52, 39, 35, 55, 23, 17, 49, 47, 28, 20, 4, 50, 2, 31, 19, 48, 34, 6, 0, 54, 16, 37]","[59, 30, 29, 23, 49, 57, 25, 16, 37, 27, 1, 17, 20, 42, 55, 47, 41, 7, 33, 6, 52, 48, 34, 51, 13, 3, 56, 11, 26, 50, 54, 46]","[6, 11, 13, 16, 17, 20, 23, 25, 26, 29, 34, 37, 47, 48, 49, 50, 52, 54, 55]"
45 | 43,"[2, 21, 7, 46, 49, 10, 25, 20, 32, 23, 55, 26, 4, 34, 15, 16, 28, 50, 30, 42, 37, 45, 44, 1, 39, 5, 6, 13, 24, 11, 38, 14]","[29, 45, 42, 8, 21, 9, 1, 5, 15, 61, 62, 51, 54, 41, 38, 31, 7, 56, 43, 13, 36, 0, 60, 10, 59, 48, 25, 3, 34, 19, 23, 27]","[1, 5, 7, 10, 13, 15, 21, 23, 25, 34, 38, 42, 45]"
46 | 44,"[53, 2, 10, 19, 21, 8, 35, 56, 49, 5, 45, 42, 39, 16, 38, 32, 20, 9, 25, 41, 15, 29, 61, 7, 1, 11, 6, 47, 57, 36, 40, 28]","[23, 59, 3, 54, 22, 50, 47, 18, 37, 24, 27, 26, 48, 6, 34, 45, 16, 28, 55, 60, 33, 30, 52, 19, 7, 17, 13, 58, 35, 46, 62, 2]","[2, 6, 7, 16, 19, 28, 35, 45, 47]"
47 | 45,"[46, 13, 6, 58, 35, 57, 18, 2, 53, 30, 31, 28, 60, 5, 21, 56, 52, 48, 55, 39, 59, 12, 23, 45, 36, 22, 50, 0, 20, 4, 29, 32]","[45, 2, 20, 60, 13, 35, 22, 57, 24, 55, 5, 58, 51, 32, 56, 4, 0, 8, 7, 54, 41, 36, 48, 53, 23, 34, 38, 39, 28, 29, 61, 30]","[0, 2, 4, 5, 13, 20, 22, 23, 28, 29, 30, 32, 35, 36, 39, 45, 48, 53, 55, 56, 57, 58, 60]"
48 | 46,"[49, 54, 43, 3, 33, 63, 61, 57, 53, 42, 58, 48, 11, 51, 50, 55, 39, 1, 9, 28, 30, 6, 21, 20, 7, 47, 32, 19, 15, 5, 36, 40]","[44, 18, 40, 59, 58, 54, 0, 29, 38, 30, 48, 24, 55, 17, 52, 43, 11, 60, 31, 53, 20, 45, 41, 63, 3, 36, 23, 56, 35, 19, 14, 39]","[3, 11, 19, 20, 30, 36, 39, 40, 43, 48, 53, 54, 55, 58, 63]"
49 | 47,"[20, 13, 57, 11, 8, 58, 9, 4, 43, 10, 12, 61, 48, 40, 22, 32, 50, 2, 25, 41, 17, 21, 33, 5, 1, 37, 34, 19, 31, 47, 15, 3]","[58, 10, 51, 3, 1, 39, 41, 38, 46, 8, 42, 12, 20, 32, 43, 5, 2, 50, 25, 17, 18, 55, 27, 40, 16, 19, 30, 57, 31, 47, 48, 37]","[1, 2, 3, 5, 8, 10, 12, 17, 19, 20, 25, 31, 32, 37, 40, 41, 43, 47, 48, 50, 57, 58]"
50 | 48,"[39, 3, 45, 11, 22, 60, 1, 19, 25, 55, 18, 40, 35, 4, 58, 28, 8, 36, 41, 44, 30, 61, 50, 9, 2, 5, 15, 48, 12, 32, 31, 56]","[26, 19, 24, 8, 39, 44, 22, 30, 18, 31, 58, 11, 12, 63, 57, 17, 49, 45, 55, 23, 2, 42, 59, 60, 47, 0, 5, 21, 1, 29, 41, 27]","[1, 2, 5, 8, 11, 12, 18, 19, 22, 30, 31, 39, 41, 44, 45, 55, 58, 60]"
51 | 49,"[39, 40, 9, 0, 34, 57, 30, 43, 7, 35, 6, 24, 41, 63, 44, 36, 8, 13, 45, 15, 28, 55, 61, 42, 11, 48, 12, 38, 5, 27, 50, 59]","[9, 40, 0, 15, 52, 31, 45, 7, 57, 55, 28, 44, 26, 29, 11, 56, 38, 17, 3, 63, 42, 43, 5, 60, 30, 10, 58, 47, 41, 50, 54, 48]","[0, 5, 7, 9, 11, 15, 28, 30, 38, 40, 41, 42, 43, 44, 45, 48, 50, 55, 57, 63]"
52 | 50,"[27, 33, 14, 63, 0, 59, 54, 9, 62, 31, 21, 53, 39, 19, 10, 2, 22, 16, 29, 44, 17, 24, 60, 35, 12, 56, 4, 34, 36, 57, 30, 26]","[24, 36, 50, 3, 63, 58, 17, 22, 56, 0, 60, 7, 27, 55, 4, 11, 40, 31, 62, 14, 19, 35, 39, 33, 43, 21, 10, 6, 45, 59, 8, 18]","[0, 4, 10, 14, 17, 19, 21, 22, 24, 27, 31, 33, 35, 36, 39, 56, 59, 60, 62, 63]"
53 | 51,"[2, 27, 29, 42, 14, 34, 24, 58, 31, 4, 43, 62, 7, 1, 9, 5, 10, 53, 21, 49, 26, 23, 32, 3, 35, 51, 59, 55, 30, 39, 13, 16]","[57, 4, 11, 56, 37, 33, 63, 21, 30, 24, 55, 60, 48, 19, 46, 54, 44, 47, 7, 25, 31, 17, 18, 15, 45, 13, 10, 29, 0, 6, 61, 1]","[1, 4, 7, 10, 13, 21, 24, 29, 30, 31, 55]"
54 | 52,"[26, 63, 60, 20, 62, 17, 52, 35, 56, 12, 51, 10, 31, 42, 50, 18, 4, 39, 11, 44, 22, 25, 33, 5, 32, 13, 45, 0, 28, 23, 48, 59]","[36, 4, 35, 53, 22, 42, 55, 7, 18, 19, 13, 54, 52, 29, 27, 40, 25, 46, 15, 31, 63, 43, 57, 33, 38, 37, 1, 39, 9, 41, 45, 16]","[4, 13, 18, 22, 25, 31, 33, 35, 39, 42, 45, 52, 63]"
55 | 53,"[30, 10, 36, 20, 63, 2, 43, 62, 25, 33, 9, 4, 24, 60, 21, 57, 26, 40, 53, 18, 17, 45, 23, 50, 3, 48, 41, 12, 52, 47, 59, 0]","[15, 55, 6, 43, 41, 46, 32, 30, 48, 39, 9, 28, 58, 16, 8, 7, 33, 56, 20, 2, 21, 24, 34, 49, 1, 23, 17, 11, 45, 0, 44, 60]","[0, 2, 9, 17, 20, 21, 23, 24, 30, 33, 41, 43, 45, 48, 60]"
56 | 54,"[34, 24, 6, 4, 11, 52, 45, 54, 27, 49, 63, 16, 1, 28, 44, 33, 20, 22, 2, 57, 3, 40, 23, 30, 7, 41, 8, 62, 43, 58, 17, 46]","[61, 27, 43, 59, 55, 46, 10, 24, 12, 60, 1, 22, 23, 19, 52, 29, 40, 50, 39, 37, 26, 0, 63, 49, 56, 35, 6, 17, 41, 33, 9, 25]","[1, 6, 17, 22, 23, 24, 27, 33, 40, 41, 43, 46, 49, 52, 63]"
57 | 55,"[40, 62, 51, 34, 3, 26, 52, 29, 5, 44, 18, 23, 61, 10, 50, 24, 47, 43, 37, 7, 38, 11, 15, 59, 36, 49, 53, 42, 48, 45, 56, 58]","[4, 28, 22, 9, 35, 30, 5, 2, 54, 33, 26, 11, 13, 6, 25, 31, 20, 17, 34, 27, 36, 55, 48, 8, 32, 19, 56, 42, 46, 21, 57, 41]","[5, 11, 26, 34, 36, 42, 48, 56]"
58 | 56,"[49, 63, 30, 15, 54, 9, 6, 42, 8, 27, 3, 12, 48, 2, 36, 1, 26, 11, 43, 16, 61, 0, 62, 25, 53, 24, 51, 7, 19, 20, 14, 41]","[37, 55, 44, 8, 35, 43, 10, 36, 17, 32, 51, 7, 9, 59, 56, 16, 33, 19, 14, 58, 15, 6, 28, 53, 60, 50, 30, 41, 2, 23, 61, 34]","[2, 6, 7, 8, 9, 14, 15, 16, 19, 30, 36, 41, 43, 51, 53, 61]"
59 | 57,"[26, 10, 13, 42, 6, 50, 47, 34, 27, 36, 46, 3, 14, 61, 5, 44, 15, 55, 30, 41, 52, 28, 8, 2, 21, 38, 40, 11, 0, 62, 1, 16]","[19, 26, 14, 7, 13, 1, 15, 29, 4, 63, 22, 59, 47, 60, 51, 35, 23, 50, 58, 30, 57, 2, 49, 31, 62, 25, 24, 53, 20, 45, 54, 12]","[1, 2, 13, 14, 15, 26, 30, 47, 50, 62]"
60 | 58,"[43, 24, 41, 12, 33, 52, 34, 30, 39, 51, 1, 37, 49, 14, 50, 40, 23, 42, 17, 6, 31, 7, 3, 61, 21, 38, 63, 57, 20, 45, 9, 36]","[60, 34, 57, 3, 41, 49, 50, 23, 40, 14, 42, 58, 55, 19, 54, 22, 59, 21, 15, 20, 9, 28, 63, 17, 51, 27, 37, 13, 4, 2, 33, 1]","[1, 3, 9, 14, 17, 20, 21, 23, 33, 34, 37, 40, 41, 42, 49, 50, 51, 57, 63]"
61 | 59,"[5, 19, 20, 41, 52, 7, 49, 50, 28, 18, 1, 40, 33, 55, 53, 2, 22, 47, 27, 62, 29, 17, 58, 4, 43, 39, 14, 15, 45, 26, 10, 21]","[62, 61, 27, 26, 7, 25, 40, 4, 32, 14, 42, 50, 20, 39, 24, 3, 58, 2, 57, 5, 1, 17, 11, 9, 8, 38, 18, 46, 51, 29, 23, 33]","[1, 2, 4, 5, 7, 14, 17, 18, 20, 26, 27, 29, 33, 39, 40, 50, 58, 62]"
62 | 60,"[33, 32, 31, 53, 61, 4, 40, 55, 1, 43, 26, 25, 18, 17, 45, 46, 59, 21, 23, 35, 10, 27, 12, 50, 38, 42, 30, 2, 13, 29, 5, 14]","[13, 26, 30, 15, 50, 61, 2, 42, 52, 46, 32, 23, 17, 14, 21, 44, 40, 53, 8, 33, 59, 31, 28, 20, 4, 5, 55, 60, 56, 41, 24, 27]","[2, 4, 5, 13, 14, 17, 21, 23, 26, 27, 30, 31, 32, 33, 40, 42, 46, 50, 53, 55, 59, 61]"
63 | 61,"[44, 60, 25, 59, 56, 22, 42, 5, 12, 4, 31, 35, 50, 2, 43, 19, 62, 53, 24, 15, 47, 13, 37, 6, 29, 9, 34, 49, 1, 27, 55, 0]","[48, 18, 47, 60, 25, 34, 3, 13, 36, 15, 54, 35, 2, 57, 9, 40, 42, 4, 24, 43, 51, 61, 62, 26, 56, 0, 53, 11, 5, 28, 58, 19]","[0, 2, 4, 5, 9, 13, 15, 19, 24, 25, 34, 35, 42, 43, 47, 53, 56, 60, 62]"
64 | 62,"[16, 53, 6, 40, 31, 42, 45, 61, 37, 33, 27, 17, 57, 44, 2, 28, 15, 41, 8, 35, 5, 62, 1, 0, 22, 34, 21, 4, 18, 25, 36, 3]","[19, 28, 56, 32, 47, 62, 48, 43, 39, 3, 26, 38, 52, 31, 12, 33, 24, 46, 36, 51, 61, 27, 8, 57, 2, 63, 6, 9, 29, 54, 5, 13]","[2, 3, 5, 6, 8, 27, 28, 31, 33, 36, 57, 61, 62]"
65 | 63,"[46, 10, 18, 3, 15, 4, 23, 13, 31, 6, 30, 39, 16, 35, 56, 59, 28, 25, 27, 26, 50, 54, 34, 36, 12, 37, 33, 42, 40, 32, 0, 5]","[56, 41, 25, 26, 3, 4, 63, 30, 20, 59, 23, 35, 39, 62, 53, 47, 52, 27, 28, 6, 50, 18, 46, 55, 33, 13, 34, 32, 10, 29, 16, 0]","[0, 3, 4, 6, 10, 13, 16, 18, 23, 25, 26, 27, 28, 30, 32, 33, 34, 35, 39, 46, 50, 56, 59]"
66 | 64,"[47, 11, 4, 10, 52, 54, 43, 50, 18, 60, 34, 39, 36, 30, 37, 23, 41, 32, 5, 15, 2, 58, 40, 28, 27, 38, 12, 0, 20, 42, 26, 19]","[22, 18, 31, 62, 33, 15, 38, 4, 49, 17, 26, 50, 41, 27, 46, 35, 48, 36, 29, 9, 16, 24, 23, 56, 19, 5, 1, 12, 34, 61, 8, 55]","[4, 5, 12, 15, 18, 19, 23, 26, 27, 34, 36, 38, 41, 50]"
67 | 65,"[34, 4, 44, 39, 29, 25, 16, 35, 10, 51, 62, 40, 5, 20, 6, 32, 48, 42, 23, 22, 7, 33, 3, 19, 8, 27, 52, 21, 13, 53, 36, 12]","[38, 2, 46, 35, 7, 44, 1, 49, 45, 27, 57, 0, 19, 61, 59, 11, 16, 18, 51, 8, 60, 9, 12, 15, 23, 6, 40, 22, 14, 30, 50, 31]","[6, 7, 8, 12, 16, 19, 22, 23, 27, 35, 40, 44, 51]"
68 | 66,"[28, 39, 33, 2, 21, 53, 14, 12, 18, 7, 45, 43, 37, 60, 54, 42, 11, 38, 23, 9, 50, 26, 22, 62, 15, 5, 8, 0, 20, 13, 56, 30]","[51, 18, 47, 54, 4, 60, 48, 62, 61, 56, 50, 40, 14, 28, 39, 2, 25, 9, 0, 7, 13, 32, 33, 35, 30, 11, 8, 20, 23, 49, 12, 22]","[0, 2, 7, 8, 9, 11, 12, 13, 14, 18, 20, 22, 23, 28, 30, 33, 39, 50, 54, 56, 60, 62]"
69 | 67,"[56, 61, 32, 49, 34, 25, 16, 14, 0, 28, 10, 55, 46, 53, 27, 52, 39, 37, 63, 6, 43, 21, 19, 30, 29, 50, 41, 8, 31, 18, 33, 40]","[22, 51, 50, 8, 32, 45, 6, 38, 34, 3, 10, 43, 54, 37, 1, 13, 61, 55, 47, 5, 20, 23, 9, 11, 58, 59, 48, 27, 17, 42, 31, 25]","[6, 8, 10, 25, 27, 31, 32, 34, 37, 43, 50, 55, 61]"
70 | 68,"[32, 56, 63, 19, 46, 60, 23, 2, 62, 44, 53, 51, 49, 54, 33, 4, 31, 6, 47, 34, 61, 21, 12, 26, 55, 5, 27, 29, 15, 45, 52, 38]","[49, 8, 24, 36, 0, 3, 33, 51, 16, 11, 43, 40, 2, 29, 41, 38, 7, 39, 17, 55, 18, 62, 25, 20, 48, 14, 10, 13, 30, 22, 19, 50]","[2, 19, 29, 33, 38, 49, 51, 55, 62]"
71 | 69,"[10, 62, 7, 61, 52, 43, 6, 14, 42, 13, 23, 30, 11, 22, 3, 8, 21, 63, 1, 34, 35, 4, 19, 36, 27, 26, 33, 28, 29, 59, 60, 16]","[31, 63, 8, 6, 32, 52, 48, 30, 53, 49, 5, 26, 2, 14, 47, 43, 51, 18, 44, 33, 55, 4, 20, 17, 42, 7, 15, 10, 24, 3, 59, 39]","[3, 4, 6, 7, 8, 10, 14, 26, 30, 33, 42, 43, 52, 59, 63]"
72 | 70,"[9, 22, 57, 5, 0, 10, 56, 20, 8, 23, 4, 3, 58, 50, 38, 18, 1, 14, 17, 40, 21, 11, 48, 36, 27, 31, 2, 30, 34, 37, 49, 7]","[41, 36, 45, 4, 6, 42, 30, 10, 33, 54, 44, 47, 18, 40, 27, 9, 2, 21, 49, 60, 34, 1, 5, 23, 50, 46, 57, 32, 22, 26, 7, 11]","[1, 2, 4, 5, 7, 9, 10, 11, 18, 21, 22, 23, 27, 30, 34, 36, 40, 49, 50, 57]"
73 | 71,"[57, 29, 41, 52, 62, 54, 6, 11, 36, 0, 44, 9, 35, 59, 63, 32, 47, 49, 38, 34, 8, 53, 18, 13, 27, 60, 3, 10, 19, 61, 25, 28]","[10, 62, 39, 11, 16, 28, 14, 37, 23, 8, 17, 25, 58, 45, 7, 56, 36, 61, 48, 12, 13, 4, 33, 55, 27, 42, 20, 46, 32, 31, 43, 21]","[8, 10, 11, 13, 25, 27, 28, 32, 36, 61, 62]"
74 | 72,"[63, 8, 51, 13, 28, 52, 10, 41, 46, 34, 14, 9, 20, 39, 11, 25, 7, 21, 59, 5, 56, 23, 43, 47, 27, 35, 54, 30, 16, 37, 57, 2]","[19, 18, 24, 37, 58, 17, 9, 13, 2, 50, 16, 63, 3, 39, 53, 4, 35, 49, 22, 42, 45, 43, 20, 14, 47, 32, 12, 52, 26, 56, 11, 29]","[2, 9, 11, 13, 14, 16, 20, 35, 37, 39, 43, 47, 52, 56, 63]"
75 | 73,"[56, 35, 29, 20, 24, 27, 31, 12, 2, 4, 33, 11, 60, 16, 7, 8, 58, 63, 46, 49, 36, 52, 53, 41, 30, 32, 51, 47, 62, 21, 15, 0]","[51, 26, 31, 25, 47, 60, 49, 45, 9, 34, 61, 33, 38, 44, 53, 59, 3, 14, 48, 8, 22, 4, 19, 55, 6, 43, 37, 5, 50, 27, 40, 17]","[4, 8, 27, 31, 33, 47, 49, 51, 53, 60]"
76 | 74,"[17, 33, 36, 34, 27, 54, 37, 29, 38, 57, 21, 40, 43, 20, 48, 10, 41, 28, 22, 4, 32, 30, 31, 59, 47, 23, 14, 5, 1, 42, 50, 44]","[13, 23, 10, 14, 51, 12, 21, 40, 32, 3, 29, 56, 33, 50, 37, 22, 63, 47, 24, 38, 35, 43, 5, 26, 16, 27, 31, 1, 49, 46, 8, 52]","[1, 5, 10, 14, 21, 22, 23, 27, 29, 31, 32, 33, 37, 38, 40, 43, 47, 50]"
77 | 75,"[57, 37, 60, 46, 20, 41, 33, 42, 26, 55, 39, 4, 36, 10, 32, 31, 58, 9, 3, 59, 22, 2, 34, 17, 0, 61, 16, 53, 12, 25, 50, 23]","[54, 19, 15, 49, 33, 31, 8, 28, 17, 38, 27, 55, 14, 4, 45, 6, 9, 60, 10, 24, 61, 63, 13, 47, 46, 2, 20, 36, 48, 7, 53, 23]","[2, 4, 9, 10, 17, 20, 23, 31, 33, 36, 46, 53, 55, 60, 61]"
78 | 76,"[20, 56, 53, 3, 34, 4, 2, 6, 27, 15, 63, 23, 55, 39, 14, 36, 42, 25, 35, 38, 43, 1, 37, 51, 16, 29, 31, 47, 26, 28, 10, 45]","[33, 19, 20, 42, 10, 11, 48, 4, 47, 26, 9, 37, 55, 30, 58, 8, 41, 46, 59, 50, 34, 43, 18, 5, 62, 49, 23, 21, 61, 57, 51, 15]","[4, 10, 15, 20, 23, 26, 34, 37, 42, 43, 47, 51, 55]"
79 | 77,"[23, 14, 10, 28, 7, 25, 2, 0, 29, 1, 35, 8, 54, 58, 51, 24, 21, 41, 3, 38, 44, 45, 39, 5, 53, 55, 17, 48, 6, 20, 18, 34]","[2, 53, 46, 41, 62, 43, 12, 63, 10, 32, 22, 59, 51, 13, 25, 23, 14, 36, 16, 11, 58, 57, 9, 61, 42, 47, 37, 50, 33, 3, 6, 15]","[2, 3, 6, 10, 14, 23, 25, 41, 51, 53, 58]"
80 | 78,"[41, 50, 43, 38, 1, 44, 15, 61, 23, 26, 19, 9, 6, 31, 54, 18, 56, 10, 16, 29, 42, 39, 11, 12, 60, 52, 45, 63, 0, 30, 20, 53]","[58, 44, 4, 38, 11, 39, 60, 59, 22, 46, 50, 7, 15, 13, 8, 36, 5, 26, 28, 10, 53, 1, 23, 20, 43, 61, 51, 40, 57, 9, 54, 14]","[1, 9, 10, 11, 15, 20, 23, 26, 38, 39, 43, 44, 50, 53, 54, 60, 61]"
81 | 79,"[33, 36, 54, 26, 59, 28, 0, 10, 1, 63, 45, 11, 53, 56, 6, 4, 27, 40, 39, 61, 55, 18, 3, 48, 9, 22, 2, 34, 15, 16, 60, 42]","[54, 63, 16, 12, 53, 27, 34, 8, 1, 41, 36, 52, 58, 24, 59, 23, 30, 6, 44, 32, 37, 48, 3, 42, 9, 45, 51, 14, 22, 61, 50, 4]","[1, 3, 4, 6, 9, 16, 22, 27, 34, 36, 42, 45, 48, 53, 54, 59, 61, 63]"
82 | 80,"[50, 47, 26, 38, 60, 19, 14, 61, 55, 2, 31, 9, 29, 11, 62, 58, 5, 24, 25, 17, 13, 53, 37, 1, 34, 6, 43, 54, 33, 12, 41, 7]","[16, 62, 31, 60, 50, 29, 10, 20, 39, 2, 42, 14, 22, 1, 13, 59, 0, 26, 11, 34, 19, 6, 7, 54, 8, 38, 33, 61, 17, 25, 58, 47]","[1, 2, 6, 7, 11, 13, 14, 17, 19, 25, 26, 29, 31, 33, 34, 38, 47, 50, 54, 58, 60, 61, 62]"
83 | 81,"[61, 3, 10, 49, 21, 43, 46, 18, 16, 57, 4, 2, 19, 14, 34, 62, 26, 25, 1, 5, 6, 52, 63, 13, 36, 7, 45, 39, 48, 15, 54, 33]","[47, 48, 6, 16, 31, 13, 56, 17, 21, 55, 36, 40, 35, 2, 50, 3, 53, 38, 51, 23, 12, 11, 24, 34, 15, 46, 45, 8, 1, 20, 22, 39]","[1, 2, 3, 6, 13, 15, 16, 21, 34, 36, 39, 45, 46, 48]"
84 | 82,"[30, 47, 32, 19, 28, 45, 23, 13, 37, 0, 40, 16, 14, 26, 5, 43, 12, 41, 3, 6, 53, 51, 63, 24, 7, 61, 20, 22, 39, 11, 33, 34]","[9, 38, 46, 16, 6, 21, 39, 51, 1, 54, 13, 18, 57, 58, 8, 48, 29, 10, 31, 44, 17, 28, 27, 45, 34, 47, 60, 4, 5, 42, 15, 50]","[5, 6, 13, 16, 28, 34, 39, 45, 47, 51]"
85 | 83,"[18, 15, 28, 52, 23, 43, 45, 19, 34, 37, 21, 8, 27, 14, 2, 4, 47, 42, 16, 17, 60, 29, 39, 0, 58, 5, 46, 33, 12, 32, 55, 10]","[51, 27, 18, 40, 45, 32, 60, 0, 53, 4, 23, 28, 10, 15, 29, 17, 24, 2, 5, 34, 21, 22, 52, 11, 12, 50, 44, 9, 3, 42, 39, 48]","[0, 2, 4, 5, 10, 12, 15, 17, 18, 21, 23, 27, 28, 29, 32, 34, 39, 42, 45, 52, 60]"
86 | 84,"[16, 49, 22, 56, 0, 53, 12, 26, 9, 51, 20, 14, 38, 29, 36, 13, 19, 40, 6, 44, 48, 43, 32, 15, 42, 62, 39, 52, 23, 7, 60, 50]","[45, 50, 38, 19, 63, 9, 26, 20, 31, 4, 62, 54, 29, 61, 47, 16, 18, 10, 52, 39, 35, 17, 23, 56, 51, 14, 27, 49, 40, 30, 58, 15]","[9, 14, 15, 16, 19, 20, 23, 26, 29, 38, 39, 40, 49, 50, 51, 52, 56, 62]"
87 | 85,"[41, 56, 11, 28, 36, 57, 6, 23, 49, 29, 20, 5, 63, 39, 58, 51, 42, 37, 18, 0, 19, 31, 47, 10, 32, 48, 38, 7, 13, 44, 45, 8]","[29, 5, 60, 63, 11, 12, 43, 20, 42, 30, 21, 3, 6, 49, 58, 17, 36, 15, 1, 44, 24, 41, 59, 51, 53, 32, 38, 56, 37, 57, 23, 33]","[5, 6, 11, 20, 23, 29, 32, 36, 37, 38, 41, 42, 44, 49, 51, 56, 57, 58, 63]"
88 | 86,"[19, 38, 45, 33, 6, 24, 36, 10, 29, 30, 21, 9, 59, 25, 32, 11, 4, 2, 5, 57, 20, 40, 35, 55, 15, 1, 34, 44, 3, 8, 23, 18]","[37, 42, 17, 16, 6, 55, 43, 0, 56, 53, 31, 14, 21, 44, 26, 61, 49, 25, 12, 27, 28, 18, 32, 51, 13, 50, 52, 19, 54, 58, 30, 48]","[6, 18, 19, 21, 25, 30, 32, 44, 55]"
89 | 87,"[28, 38, 31, 55, 61, 37, 62, 60, 25, 6, 41, 15, 9, 4, 18, 7, 12, 49, 24, 0, 23, 46, 44, 3, 30, 59, 5, 8, 2, 34, 13, 11]","[22, 34, 53, 40, 60, 36, 27, 43, 1, 10, 17, 15, 25, 51, 48, 57, 13, 33, 54, 7, 30, 16, 38, 29, 26, 0, 44, 14, 12, 19, 42, 31]","[0, 7, 12, 13, 15, 25, 30, 31, 34, 38, 44, 60]"
90 | 88,"[60, 43, 34, 28, 48, 25, 29, 15, 16, 46, 26, 54, 4, 38, 11, 61, 39, 20, 17, 50, 27, 2, 49, 14, 36, 24, 5, 53, 41, 55, 10, 3]","[56, 48, 63, 6, 54, 38, 21, 42, 39, 47, 20, 11, 49, 14, 28, 46, 53, 61, 23, 9, 24, 34, 10, 43, 0, 52, 5, 29, 44, 59, 27, 16]","[5, 10, 11, 14, 16, 20, 24, 27, 28, 29, 34, 38, 39, 43, 46, 48, 49, 53, 54, 61]"
91 | 89,"[24, 23, 8, 45, 17, 37, 35, 10, 58, 57, 3, 61, 18, 29, 15, 4, 13, 43, 5, 19, 26, 49, 0, 1, 27, 39, 6, 20, 16, 50, 14, 34]","[38, 63, 5, 49, 47, 18, 39, 60, 29, 26, 44, 3, 0, 10, 52, 61, 14, 4, 15, 35, 59, 8, 57, 50, 54, 20, 28, 24, 16, 1, 55, 25]","[0, 1, 3, 4, 5, 8, 10, 14, 15, 16, 18, 20, 24, 26, 29, 35, 39, 49, 50, 57, 61]"
92 | 90,"[21, 35, 6, 49, 16, 39, 20, 27, 28, 57, 13, 29, 32, 61, 33, 46, 60, 38, 54, 45, 19, 59, 43, 36, 51, 31, 24, 4, 52, 50, 9, 63]","[13, 2, 42, 49, 59, 5, 27, 53, 33, 44, 16, 25, 15, 48, 52, 11, 47, 32, 7, 56, 17, 10, 14, 43, 24, 31, 0, 37, 51, 4, 30, 19]","[4, 13, 16, 19, 24, 27, 31, 32, 33, 43, 49, 51, 52, 59]"
93 | 91,"[38, 36, 34, 17, 18, 46, 20, 3, 15, 57, 23, 42, 4, 25, 2, 7, 55, 24, 19, 51, 52, 31, 21, 28, 16, 62, 9, 12, 27, 13, 48, 1]","[52, 28, 54, 22, 51, 18, 19, 8, 26, 43, 55, 4, 21, 35, 1, 37, 46, 9, 44, 39, 0, 23, 15, 42, 57, 25, 27, 49, 34, 17, 13, 11]","[1, 4, 9, 13, 15, 17, 18, 19, 21, 23, 25, 27, 28, 34, 42, 46, 51, 52, 55, 57]"
94 | 92,"[59, 17, 41, 57, 60, 46, 5, 52, 20, 38, 28, 45, 50, 58, 22, 54, 36, 56, 47, 32, 27, 3, 1, 10, 53, 24, 31, 23, 49, 6, 35, 0]","[59, 41, 20, 24, 53, 19, 46, 5, 35, 31, 38, 37, 36, 0, 52, 16, 2, 45, 17, 33, 21, 57, 50, 61, 56, 11, 8, 29, 18, 54, 10, 47]","[0, 5, 10, 17, 20, 24, 31, 35, 36, 38, 41, 45, 46, 47, 50, 52, 53, 54, 56, 57, 59]"
95 | 93,"[11, 52, 33, 43, 9, 23, 21, 61, 10, 49, 20, 15, 58, 63, 62, 16, 45, 2, 37, 28, 35, 34, 31, 24, 3, 44, 29, 30, 19, 8, 32, 38]","[49, 29, 32, 0, 20, 53, 39, 26, 44, 23, 6, 35, 10, 15, 57, 7, 61, 56, 60, 17, 4, 42, 18, 55, 46, 28, 21, 41, 24, 33, 13, 47]","[10, 15, 20, 21, 23, 24, 28, 29, 32, 33, 35, 44, 49, 61]"
96 | 94,"[8, 3, 10, 61, 33, 57, 38, 37, 20, 0, 35, 40, 58, 19, 25, 4, 24, 22, 13, 56, 45, 16, 27, 51, 46, 29, 59, 34, 15, 31, 44, 39]","[58, 55, 44, 60, 31, 8, 40, 18, 13, 26, 7, 61, 38, 19, 37, 51, 57, 53, 29, 4, 50, 46, 35, 33, 39, 5, 45, 24, 36, 34, 3, 63]","[3, 4, 8, 13, 19, 24, 29, 31, 33, 34, 35, 37, 38, 39, 40, 44, 45, 46, 51, 57, 58, 61]"
97 | 95,"[12, 23, 31, 47, 17, 22, 27, 29, 16, 37, 0, 7, 4, 50, 39, 45, 46, 19, 20, 3, 30, 41, 62, 60, 8, 18, 35, 56, 11, 32, 14, 24]","[16, 13, 5, 49, 42, 12, 40, 34, 23, 48, 36, 51, 18, 7, 62, 31, 19, 61, 25, 58, 53, 54, 60, 17, 46, 59, 41, 52, 10, 33, 63, 26]","[7, 12, 16, 17, 18, 19, 23, 31, 41, 46, 60, 62]"
98 | 96,"[22, 41, 48, 14, 11, 34, 2, 16, 3, 6, 21, 56, 61, 29, 19, 52, 18, 38, 49, 15, 12, 43, 62, 50, 63, 46, 55, 28, 0, 30, 51, 27]","[5, 2, 11, 26, 30, 40, 49, 47, 62, 54, 4, 17, 60, 44, 61, 28, 18, 35, 39, 24, 37, 57, 8, 7, 20, 45, 10, 12, 25, 3, 38, 36]","[2, 3, 11, 12, 18, 28, 30, 38, 49, 61, 62]"
99 | 97,"[2, 62, 8, 22, 15, 60, 59, 14, 0, 10, 33, 34, 50, 4, 19, 31, 58, 1, 16, 53, 17, 24, 27, 18, 32, 11, 51, 43, 54, 42, 28, 39]","[57, 54, 13, 47, 2, 15, 7, 58, 43, 16, 22, 0, 36, 35, 23, 38, 21, 55, 33, 19, 46, 8, 29, 42, 60, 27, 32, 53, 49, 31, 9, 28]","[0, 2, 8, 15, 16, 19, 22, 27, 28, 31, 32, 33, 42, 43, 53, 54, 58, 60]"
100 | 98,"[62, 54, 43, 14, 61, 0, 42, 40, 15, 12, 50, 30, 29, 47, 37, 27, 20, 33, 7, 38, 2, 5, 53, 60, 11, 16, 35, 25, 6, 21, 55, 17]","[38, 7, 54, 31, 47, 28, 34, 8, 12, 20, 3, 63, 0, 52, 36, 55, 60, 25, 17, 15, 13, 46, 33, 56, 26, 11, 14, 16, 40, 4, 37, 45]","[0, 7, 11, 12, 14, 15, 16, 17, 20, 25, 33, 37, 38, 40, 47, 54, 55, 60]"
101 | 99,"[2, 41, 3, 17, 26, 4, 9, 58, 7, 42, 49, 21, 55, 40, 53, 8, 50, 18, 29, 0, 13, 44, 63, 5, 39, 48, 24, 36, 47, 54, 19, 38]","[10, 45, 1, 18, 8, 0, 16, 15, 55, 39, 25, 6, 61, 14, 3, 34, 49, 51, 43, 46, 36, 33, 28, 57, 52, 12, 9, 35, 23, 42, 11, 31]","[0, 3, 8, 9, 18, 36, 39, 42, 49, 55]"
102 |
--------------------------------------------------------------------------------
/examples/set_intersection/utils.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # The source code is adapted from the sorting source code written by
8 | # Nils Blach.
9 | #
10 | # main author: Robert Gerstenberger
11 |
12 | from typing import Dict, List, Set
13 |
14 |
15 | def string_to_list(string: str) -> List[int]:
16 | """
17 | Helper function to convert a list encoded inside a string into a Python
18 | list object of integer elements.
19 |
20 | :param string: Input string containing a list.
21 | :type string: str
22 | :return: List of integer elements.
23 | :rtype: List[int]
24 | :raise AssertionError: If input string does not contain a list.
25 | """
26 |
27 | assert string[0] == "[" and string[-1] == "]", "String is not a list."
28 | return [int(num) for num in string[1:-1].split(",")]
29 |
30 |
31 | def string_to_set(string: str) -> Set[int]:
32 | """
33 | Helper function to convert a list encoded inside a string into a Python
34 | set object of integer elements.
35 |
36 | :param string: Input string containing a list.
37 | :type string: str
38 | :return: Set of integer elements.
39 | :rtype: Set[int]
40 | :raise AssertionError: If input string does not contain a list.
41 | """
42 |
43 | assert string[0] == "[" and string[-1] == "]", "String is not a list."
44 | return {int(num) for num in string[1:-1].split(",")}
45 |
46 |
47 | def test_set_intersection(state: Dict) -> bool:
48 | """
49 | Function to test whether the final solution matches ground truth.
50 |
51 | :param state: Thought state that represents the final solution.
52 | :type state: Dict
53 | :return: Returns whether the solution matches the ground truth.
54 | :rtype: bool
55 | """
56 |
57 | # convert string to list
58 | try:
59 | correct_list = string_to_list(state["result"])
60 | sorted_list = sorted(string_to_list(state["current"]))
61 | return sorted_list == correct_list
62 | except:
63 | return False
64 |
65 |
66 | def num_errors(state: Dict) -> float:
67 | """
68 | Function to locally count the number of errors that serves as a score.
69 |
70 | :param state: Thought state to be scored.
71 | :type state: Dict
72 | :return: Number of errors.
73 | :rtype: float
74 | """
75 |
76 | try:
77 | set1 = string_to_set(state["set1"])
78 | set2 = string_to_set(state["set2"])
79 | if "subset" in state and state["subset"] != "" and state["subset"] is not None:
80 | set2 = string_to_set(state["subset"])
81 | common = sorted(list(set1 & set2))
82 | llm_solution = sorted(string_to_list(state["current"]))
83 | num_errors = 0
84 | common_idx = 0
85 | llm_idx = 0
86 | while common_idx < len(common) and llm_idx < len(llm_solution):
87 | if common[common_idx] == llm_solution[llm_idx]:
88 | common_idx += 1
89 | llm_idx += 1
90 | elif common[common_idx] < llm_solution[llm_idx]:
91 | common_idx += 1
92 | num_errors += 1
93 | elif common[common_idx] > llm_solution[llm_idx]:
94 | llm_idx += 1
95 | num_errors += 1
96 | num_errors += len(common) - common_idx + len(llm_solution) - llm_idx
97 | return num_errors
98 | except:
99 | return 1000
100 |
--------------------------------------------------------------------------------
/examples/sorting/README.md:
--------------------------------------------------------------------------------
1 | # Sorting
2 |
3 | The use case in this directory sorts the provided list of
4 | numbers containing numbers from 0 to 9 (duplicates allowed).
5 | We provide implementations of five different approaches for
6 | 32, 64 and 128 elements:
7 | - IO
8 | - Chain-of-Thought (CoT)
9 | - Tree of Thought (ToT):
10 | - ToT: wider tree, meaning more branches per level
11 | - ToT2: tree with more levels, but fewer branches per level
12 | - Graph of Thoughts (GoT):
13 | - GoT: split into subarrays / sort / merge
14 |
15 | ## Data
16 |
17 | We provide input files with 100 precomputed samples for each list
18 | length: `sorting_.csv`.
19 |
20 | ## Execution
21 |
22 | The files to execute the use case are called
23 | `sorting_.py`. In the main body, one can select the
24 | specific samples to be run (variable sample) and the approaches
25 | (variable approaches). It is also possible to set a budget in dollars
26 | (variable budget).
27 | The input filename for the samples is currently hardcoded to
28 | `sorting_.csv`, but can be updated in the function
29 | `run`.
30 |
31 | The Python scripts will create the directory `result`, if it is not
32 | already present. In the 'result' directory, another directory is created
33 | for each run: `{name of LLM}_{list of approaches}_{day}_{start time}`.
34 | Inside each execution specific directory two files (`config.json`,
35 | `log.log`) and a separate directory for each selected approach are
36 | created. `config.json` contains the configuration of the run: input data,
37 | selected approaches, name of the LLM, and the budget. `log.log` contains
38 | the prompts and responses of the LLM as well as additional debug data.
39 | The approach directories contain a separate json file for every sample
40 | and the file contains the Graph Reasoning State (GRS) for that sample.
41 |
42 | ## Plot Data
43 |
44 | Change the results directory in line 171 of `plot.py` and update the
45 | length parameter in the subsequent line and run `python3 plot.py` to
46 | plot your data.
47 |
--------------------------------------------------------------------------------
/examples/sorting/example_prompts_sorting_032.md:
--------------------------------------------------------------------------------
1 | # Sorting 32 Numbers - Prompts and Examples
2 | ## Prompt Templates
3 | ### GENERATE: split_prompt
4 | Replace `{input}` with the input list of numbers to be split.
5 | ```
6 | Split the following list of 32 numbers into 2 lists of 16 numbers each, the first list should contain the first 16 numbers and the second list the second 16 numbers.
7 | Only output the final 2 lists in the following format without any additional text or thoughts!:
8 | {{
9 | "List 1": [3, 4, 3, 5, 7, 8, 1, ...],
10 | "List 2": [2, 9, 2, 4, 7, 1, 5, ...]
11 | }}
12 |
13 |
14 | Input: [9, 6, 7, 7, 2, 0, 2, 2, 3, 5, 0, 9, 2, 2, 4, 4, 5, 2, 5, 1, 2, 8, 3, 8, 3, 9, 6, 0, 4, 2, 2, 3]
15 | Output:
16 | {{
17 | "List 1": [9, 6, 7, 7, 2, 0, 2, 2, 3, 5, 0, 9, 2, 2, 4, 4],
18 | "List 2": [5, 2, 5, 1, 2, 8, 3, 8, 3, 9, 6, 0, 4, 2, 2, 3]
19 | }}
20 |
21 |
22 | Input: {input}
23 | ```
24 |
25 | ### GENERATE: sort_prompt
26 | Replace `{input}` with the input list of numbers to be sorted.
27 | ```
28 | Sort the following list of numbers in ascending order. Output only the sorted list of numbers, no additional text.
29 |
30 |
31 | Input: [5, 1, 0, 1, 2, 0, 4, 8, 1, 9, 5, 1, 3, 3, 9, 7]
32 | Output: [0, 0, 1, 1, 1, 1, 2, 3, 3, 4, 5, 5, 7, 8, 9, 9]
33 |
34 | Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9, 4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1]
35 | Output: [0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 9]
36 |
37 | Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
38 | Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
39 |
40 |
41 | Input: {input}
42 | ```
43 |
44 | ### IMPROVE: improve_prompt
45 | Replace `{input}` with the input list of numbers to be sorted and `{incorrectly_sorted}` with the incorrectly sorted list of numbers. `{length}` is the length of the input list.
46 | ```
47 | The following two lists represent an unsorted list of numbers and a sorted variant of that list. The sorted variant is not correct. Fix the sorted variant so that it is correct.
48 | Make sure that the output list is sorted in ascending order, has the same number of elements as the input list ({length}), and contains the same elements as the input list.
49 |
50 |
51 | To fix the incorrectly sorted list follow these steps:
52 | 1. For each number from 0 to 9, compare the frequency of that number in the incorrectly sorted list to the frequency of that number in the input list.
53 | 2. Iterate through the incorrectly sorted list and add or remove numbers as needed to make the frequency of each number in the incorrectly sorted list match the frequency of that number in the input list.
54 |
55 |
56 |
57 | Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9]
58 | Incorrectly Sorted: [0, 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 7, 7, 8, 8, 9, 9, 9, 9]
59 | Reason: The incorrectly sorted list contains four extra 0s, two extra 4s and three extra 9s and is missing two 2s.
60 | Output: [0, 1, 2, 2, 2, 2, 3, 3, 4, 5, 5, 7, 7, 8, 8, 9]
61 |
62 | Input: [6, 4, 5, 7, 5, 6, 9, 7, 6, 9, 4, 6, 9, 8, 1, 9, 2, 4, 9, 0, 7, 6, 5, 6, 6, 2, 8, 3, 9, 5, 6, 1]
63 | Incorrectly Sorted: [0, 1, 1, 2, 2, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9]
64 | Reason: The incorrectly sorted list contains two extra 4s and is missing two 6s and one 9.
65 | Output: [0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 9]
66 |
67 | Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
68 | Incorrectly Sorted: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]
69 | Reason: The incorrectly sorted list contains one extra 8 and is missing two 2s, one 3, three 4s, two 5s, one 6, six 7s and one 9.
70 | Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
71 |
72 |
73 | Input: {input}
74 | Incorrectly Sorted: {incorrectly_sorted}
75 | ```
76 |
77 | ### AGGREGATE: aggregate_prompt
78 | Replace `{input_list1}` and `{input_list2}` with the input lists of numbers to be merged. `{length}` is the length of each input list and `{length_combined}` is the length of the combined list.
79 | ```
80 | Merge the following 2 sorted lists of length {length} each, into one sorted list of length {length_combined} using a merge sort style approach.
81 | Only output the final merged list without any additional text or thoughts!:
82 |
83 |
84 | To merge the two lists in a merge-sort style approach, foloow these steps:
85 | 1. Compare the first element of both lists.
86 | 2. Append the smaller element to the merged list and move to the next element in the list from which the smaller element came.
87 | 3. Repeat steps 1 and 2 until one of the lists is empty.
88 | 4. Append the remaining elements of the non-empty list to the merged list.
89 |
90 |
91 | Merge the following two lists into one sorted list:
92 | 1: {input_list1}
93 | 2: {input_list2}
94 |
95 | Merged list:
96 | ```
97 |
98 | ## Complete Example Prompts
99 | ### The GoO Summarised
100 | 1. Split the input list into two sub-lists of equal size (split prompt)
101 | 2. For each sub-list: Sort the sub-list (sort prompt) five times; score each sort attempt; keep the best
102 | 3. Merge the sorted sub-lists into one fully sorted list (merge prompt) 10 times; score each merge attempt; keep the best
103 | 4. Fix any potential mistakes in the sorted list (improve prompt) 10 times; score each improvement attempt; keep the best
104 |
105 | ### Input
106 | ```
107 | [8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1, 0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]
108 | ```
109 |
110 | ### Step 1 - Prompt
111 | ```
112 | Split the following list of 32 numbers into 2 lists of 16 numbers each, the first list should contain the first 16 numbers and the second list the second 16 numbers.
113 | Only output the final 2 lists in the following format without any additional text or thoughts!:
114 | {
115 | "List 1": [3, 4, 3, 5, 7, 8, 1, ...],
116 | "List 2": [2, 9, 2, 4, 7, 1, 5, ...]
117 | }
118 |
119 |
120 | Input: [9, 6, 7, 7, 2, 0, 2, 2, 3, 5, 0, 9, 2, 2, 4, 4, 5, 2, 5, 1, 2, 8, 3, 8, 3, 9, 6, 0, 4, 2, 2, 3]
121 | Output:
122 | {
123 | "List 1": [9, 6, 7, 7, 2, 0, 2, 2, 3, 5, 0, 9, 2, 2, 4, 4],
124 | "List 2": [5, 2, 5, 1, 2, 8, 3, 8, 3, 9, 6, 0, 4, 2, 2, 3]
125 | }
126 |
127 |
128 | Input: [8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1, 0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]
129 | ```
130 |
131 | ### Step 1 - Response
132 | ```
133 | {
134 | "List 1": [8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1],
135 | "List 2": [0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]
136 | }
137 | ```
138 |
139 | ### Step 2a - Prompt
140 | ```
141 | Sort the following list of numbers in ascending order. Output only the sorted list of numbers, no additional text.
142 |
143 |
144 | Input: [5, 1, 0, 1, 2, 0, 4, 8, 1, 9, 5, 1, 3, 3, 9, 7]
145 | Output: [0, 0, 1, 1, 1, 1, 2, 3, 3, 4, 5, 5, 7, 8, 9, 9]
146 |
147 | Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9, 4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1]
148 | Output: [0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 9]
149 |
150 | Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
151 | Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
152 |
153 |
154 | Input: [8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1]
155 | ```
156 |
157 | ### Step 2a - 5 Responses
158 | ```
159 | 1. [0, 0, 1, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
160 | 2. [0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
161 | 3. [0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
162 | 4. [0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
163 | 5. [0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
164 | ```
165 |
166 | ### Step 2b - Prompt
167 | ```
168 | Sort the following list of numbers in ascending order. Output only the sorted list of numbers, no additional text.
169 |
170 |
171 | Input: [5, 1, 0, 1, 2, 0, 4, 8, 1, 9, 5, 1, 3, 3, 9, 7]
172 | Output: [0, 0, 1, 1, 1, 1, 2, 3, 3, 4, 5, 5, 7, 8, 9, 9]
173 |
174 | Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9, 4, 3, 5, 6, 6, 4, 4, 5, 2, 0, 9, 3, 3, 9, 2, 1]
175 | Output: [0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 9]
176 |
177 | Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
178 | Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
179 |
180 |
181 | Input: [0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]
182 | ```
183 |
184 | ### Step 2b - 5 Responses
185 | ```
186 | 1. [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 7, 8, 9]
187 | 2. [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 7, 8, 9]
188 | 3. [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 7, 8, 9]
189 | 4. [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 8, 9]
190 | 5. [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 7, 8, 9]
191 | ```
192 |
193 | ### Step 3 - Prompt
194 | ```
195 | Merge the following 2 sorted lists of length 16 each, into one sorted list of length 32 using a merge sort style approach.
196 | Only output the final merged list without any additional text or thoughts!:
197 |
198 |
199 | To merge the two lists in a merge-sort style approach, follow these steps:
200 | 1. Compare the first element of both lists.
201 | 2. Append the smaller element to the merged list and move to the next element in the list from which the smaller element came.
202 | 3. Repeat steps 1 and 2 until one of the lists is empty.
203 | 4. Append the remaining elements of the non-empty list to the merged list.
204 |
205 |
206 | Merge the following two lists into one sorted list:
207 | 1: [0, 0, 1, 1, 1, 1, 1, 1, 2, 3, 3, 4, 5, 7, 8, 9]
208 | 2: [0, 1, 2, 2, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 8, 9]
209 |
210 | Merged list:
211 | ```
212 |
213 | ### Step 3 - 10 Responses
214 | ```
215 | 1. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
216 | 2. [0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
217 | 3. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
218 | 4. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9]
219 | 5. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
220 | 6. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
221 | 7. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
222 | 8. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
223 | 9. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
224 | 10. [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
225 | ```
226 |
227 | ### Step 4 - Prompt
228 | ```
229 | The following two lists represent an unsorted list of numbers and a sorted variant of that list. The sorted variant is not correct. Fix the sorted variant so that it is correct.
230 | Make sure that the output list is sorted in ascending order, has the same number of elements as the input list (32), and contains the same elements as the input list.
231 |
232 |
233 | To fix the incorrectly sorted list follow these steps:
234 | 1. For each number from 0 to 9, compare the frequency of that number in the incorrectly sorted list to the frequency of that number in the input list.
235 | 2. Iterate through the incorrectly sorted list and add or remove numbers as needed to make the frequency of each number in the incorrectly sorted list match the frequency of that number in the input list.
236 |
237 |
238 |
239 | Input: [3, 7, 0, 2, 8, 1, 2, 2, 2, 4, 7, 8, 5, 5, 3, 9]
240 | Incorrectly Sorted: [0, 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 7, 7, 8, 8, 9, 9, 9, 9]
241 | Reason: The incorrectly sorted list contains four extra 0s, two extra 4s and three extra 9s and is missing two 2s.
242 | Output: [0, 1, 2, 2, 2, 2, 3, 3, 4, 5, 5, 7, 7, 8, 8, 9]
243 |
244 | Input: [6, 4, 5, 7, 5, 6, 9, 7, 6, 9, 4, 6, 9, 8, 1, 9, 2, 4, 9, 0, 7, 6, 5, 6, 6, 2, 8, 3, 9, 5, 6, 1]
245 | Incorrectly Sorted: [0, 1, 1, 2, 2, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9]
246 | Reason: The incorrectly sorted list contains two extra 4s and is missing two 6s and one 9.
247 | Output: [0, 1, 1, 2, 2, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 9]
248 |
249 | Input: [4, 4, 9, 7, 9, 7, 0, 0, 4, 9, 1, 7, 9, 5, 8, 7, 5, 6, 3, 8, 6, 7, 5, 8, 5, 0, 6, 3, 7, 0, 5, 3, 7, 5, 2, 4, 4, 9, 0, 7, 8, 2, 7, 7, 7, 2, 1, 3, 9, 9, 7, 9, 6, 6, 4, 5, 4, 2, 0, 8, 9, 0, 2, 2]
250 | Incorrectly Sorted: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9]
251 | Reason: The incorrectly sorted list contains one extra 8 and is missing two 2s, one 3, three 4s, two 5s, one 6, six 7s and one 9.
252 | Output: [0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]
253 |
254 |
255 | Input: [8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1, 0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]
256 | Incorrectly Sorted: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
257 | ```
258 |
259 | ### Step 4 - 10 Responses
260 | ```
261 | 1. Reason: The incorrectly sorted list is missing two 0s, one 3, one 4, one 6 and one 9. The incorrectly sorted list also contains one extra 1, one extra 2, and one extra 5.
262 |
263 | Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
264 |
265 | Note: The output list has the same number of elements as the input list, is sorted in ascending order, and contains the same elements as the input list.
266 |
267 | 2. Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
268 |
269 | 3. Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9]
270 |
271 | 4. Reason: The incorrectly sorted list is missing two 9s and one 0.
272 | Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9, 9, 9]
273 |
274 | 5. Reason: The incorrectly sorted list is missing one 0, one 3, one 4, four 6s, and one 9.
275 |
276 | Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]
277 |
278 | 6. Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]
279 |
280 | 7. Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
281 |
282 | 8. Reason: The incorrectly sorted list is missing three 0s, two 3s, two 4s, one 5, two 6s, and one 9.
283 | Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 8, 8, 9]
284 |
285 | 9. Reason: The incorrectly sorted list is missing one 0, three 2s, two 3s, two 4s, three 5s, two 6s, one 7, and one 9.
286 | Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 8, 8, 9, 9]
287 |
288 | 10. Reason: The incorrectly sorted list is missing three 1s, one 3, one 4, two 5s, one 6, one 7, one 8, and one 9.
289 | Output: [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
290 | ```
291 |
292 | ### Final Result
293 | ```
294 | [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]
295 | ```
296 |
--------------------------------------------------------------------------------
/examples/sorting/plot.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # main author: Nils Blach
8 | # contributions: Robert Gerstenberger
9 |
10 | import json
11 | import os
12 | import matplotlib.pyplot as plt
13 |
14 |
15 | def get_complete_results(base_directory):
16 | results_complete = {}
17 | for folder_name in os.listdir(base_directory):
18 | folder_path = os.path.join(base_directory, folder_name)
19 | if os.path.isdir(folder_path):
20 | results_complete[folder_name] = []
21 | for file_name in os.listdir(folder_path):
22 | if file_name.endswith(".json"):
23 | file_path = os.path.join(folder_path, file_name)
24 | with open(file_path, "r") as f:
25 | data = json.load(f)
26 | results_complete[folder_name].append(
27 | {"key": int(file_name.split(".")[0]), "data": data}
28 | )
29 | for key in results_complete.keys():
30 | results_complete[key] = sorted(
31 | results_complete[key], key=lambda x: x["key"]
32 | )
33 | return results_complete
34 |
35 |
36 | def get_final_scores(results_complete):
37 | scores = {}
38 | for method in results_complete.keys():
39 | scores[method] = []
40 | for result in results_complete[method]:
41 | score = 100
42 | solved = False
43 | cost = 1
44 | prompt_tokens = 0
45 | completion_tokens = 0
46 | for op in result["data"]:
47 | if "operation" in op and op["operation"] == "ground_truth_evaluator":
48 | try:
49 | score = min(op["scores"])
50 | solved = any(op["problem_solved"])
51 | except:
52 | continue
53 | if "cost" in op:
54 | cost = op["cost"]
55 | prompt_tokens = op["prompt_tokens"]
56 | completion_tokens = op["completion_tokens"]
57 | scores[method].append(
58 | [result["key"], score, solved, prompt_tokens, completion_tokens, cost]
59 | )
60 | scores[method] = sorted(scores[method], key=lambda x: x[0])
61 | return scores
62 |
63 |
64 | def get_plotting_data(base_directory):
65 | results_complete = get_complete_results(base_directory)
66 | scores = get_final_scores(results_complete)
67 | results_plotting = {
68 | method: {
69 | "scores": [x[1] for x in scores[method]],
70 | "solved": sum([1 for x in scores[method] if x[2]]),
71 | "costs": [x[5] for x in scores[method]],
72 | }
73 | for method in scores.keys()
74 | }
75 | return results_plotting
76 |
77 |
78 | def plot_results(
79 | results,
80 | methods_order=["io", "cot", "tot", "tot2", "got"],
81 | model="GPT-3.5",
82 | length=32,
83 | y_lower=0,
84 | cost_upper=0.0,
85 | display_solved=True,
86 | annotation_offset=0,
87 | display_left_ylabel=False,
88 | display_right_ylabel=False,
89 | ):
90 | methods_order = [method for method in methods_order if method in results]
91 | # Extract scores based on the order
92 | scores_ordered = [
93 | [
94 | min(score, length)
95 | for score in results[method]["scores"]
96 | if score != 100 and score != 300
97 | ]
98 | for method in methods_order
99 | ]
100 | total_costs = [sum(results[method]["costs"]) for method in methods_order]
101 |
102 | # Create figure and axis
103 | fig, ax = plt.subplots(dpi=150, figsize=(2.5, 5))
104 |
105 | # Create boxplots
106 | positions = range(1, len(methods_order) + 1)
107 | ax.boxplot(scores_ordered, positions=positions)
108 |
109 | fig_fontsize = 12
110 |
111 | # Set the ticks and labels
112 | method_labels = ["IO", "CoT", "ToT", "ToT2", "GoT"]
113 | plt.yticks(fontsize=fig_fontsize)
114 | ax.set_xticks(range(1, len(methods_order) + 1))
115 | ax.set_xticks(range(1, len(methods_order) + 1))
116 | ax.set_xticklabels(method_labels, fontsize=fig_fontsize)
117 |
118 | y_upper = length
119 |
120 | range_increase = 1
121 | if display_solved:
122 | if length < 48:
123 | range_increase = 2
124 | elif length < 96:
125 | range_increase = 4
126 | else:
127 | range_increase = 8
128 |
129 | ax.set_ylim(y_lower, y_upper + range_increase)
130 | ax1_yticks = range(
131 | y_lower, y_upper + 1, 2 if length < 48 else (4 if length < 96 else 8)
132 | )
133 | ax.set_yticks(ax1_yticks)
134 | if display_left_ylabel:
135 | ax.set_ylabel(f"#incorrectly sorted elements; the lower the better")
136 |
137 | ax.set_title(f"{length} elements")
138 |
139 | ax2 = ax.twinx()
140 | ax2.bar(positions, total_costs, alpha=0.5, color="blue", label="Total Cost ($)")
141 | ax2.yaxis.set_tick_params(colors="#1919ff", labelsize=fig_fontsize)
142 | if cost_upper > 0:
143 | ax2.set_ylim(0, cost_upper)
144 | number_of_ticks = len(ax.get_yticks())
145 | tick_interval = cost_upper / (number_of_ticks)
146 | ax2_ticks = [tick_interval * i for i in range(number_of_ticks)]
147 |
148 | # Set custom tick positions for ax2
149 | ax2.set_yticks(ax2_ticks)
150 |
151 | if display_right_ylabel:
152 | ax2.set_ylabel(
153 | "Total Cost ($); the lower the better",
154 | color="#1919ff",
155 | fontsize=fig_fontsize,
156 | )
157 |
158 | if display_solved:
159 | annotation_height = y_upper + annotation_offset
160 | count = 1
161 | for method in methods_order:
162 | if method not in results:
163 | continue
164 | solved = results[method]["solved"]
165 | ax.text(
166 | count,
167 | annotation_height,
168 | f"{solved}",
169 | ha="center",
170 | va="bottom",
171 | fontsize=fig_fontsize,
172 | )
173 | count += 1
174 |
175 | model = model.replace(".", "").replace("-", "").lower()
176 | fig.savefig(f"sorting_{model}_{length}.pdf", bbox_inches="tight")
177 |
178 |
179 | plot_results(
180 | get_plotting_data("results/"),
181 | length=32,
182 | display_solved=True,
183 | model="GPT-3.5",
184 | display_left_ylabel=True,
185 | display_right_ylabel=True,
186 | )
187 |
--------------------------------------------------------------------------------
/examples/sorting/sorting_032.csv:
--------------------------------------------------------------------------------
1 | ID,Unsorted,Sorted
2 | 0,"[0, 0, 5, 9, 0, 7, 9, 9, 1, 2, 6, 1, 1, 9, 0, 1, 3, 5, 2, 3, 5, 6, 0, 2, 7, 4, 6, 2, 9, 7, 5, 9]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 9, 9, 9, 9, 9, 9]"
3 | 1,"[5, 6, 2, 3, 1, 4, 9, 0, 5, 7, 0, 7, 1, 3, 2, 4, 5, 5, 6, 6, 3, 6, 4, 4, 2, 3, 7, 1, 7, 0, 2, 5]","[0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 9]"
4 | 2,"[9, 6, 7, 0, 7, 7, 4, 7, 9, 3, 6, 5, 0, 8, 1, 8, 6, 1, 5, 3, 3, 5, 3, 4, 2, 2, 4, 5, 8, 6, 0, 3]","[0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9]"
5 | 3,"[8, 7, 1, 1, 1, 1, 3, 3, 0, 9, 4, 1, 0, 2, 5, 1, 0, 5, 6, 7, 1, 4, 5, 9, 4, 6, 2, 5, 8, 6, 2, 6]","[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9]"
6 | 4,"[5, 3, 9, 6, 3, 2, 9, 8, 2, 4, 2, 1, 8, 1, 8, 0, 0, 3, 7, 4, 4, 9, 9, 5, 4, 6, 5, 4, 3, 4, 1, 1]","[0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 6, 6, 7, 8, 8, 8, 9, 9, 9, 9]"
7 | 5,"[4, 6, 9, 5, 8, 6, 5, 4, 0, 4, 2, 6, 3, 6, 8, 5, 5, 2, 3, 5, 0, 9, 8, 6, 5, 5, 5, 8, 1, 5, 8, 7]","[0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 8, 8, 8, 8, 8, 9, 9]"
8 | 6,"[9, 6, 4, 1, 9, 6, 3, 1, 1, 7, 3, 9, 2, 6, 2, 9, 9, 4, 2, 7, 4, 0, 2, 3, 1, 7, 9, 8, 3, 0, 7, 3]","[0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 6, 6, 6, 7, 7, 7, 7, 8, 9, 9, 9, 9, 9, 9]"
9 | 7,"[9, 2, 4, 7, 4, 8, 8, 4, 7, 5, 8, 9, 0, 1, 0, 0, 0, 0, 7, 3, 4, 7, 7, 7, 9, 8, 6, 3, 2, 6, 6, 8]","[0, 0, 0, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4, 4, 5, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9]"
10 | 8,"[6, 6, 9, 1, 7, 3, 5, 9, 7, 9, 1, 4, 8, 2, 5, 7, 0, 8, 8, 8, 9, 8, 1, 0, 5, 6, 5, 9, 2, 1, 7, 7]","[0, 0, 1, 1, 1, 1, 2, 2, 3, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
11 | 9,"[0, 3, 5, 9, 3, 1, 0, 2, 9, 9, 8, 1, 7, 6, 8, 1, 7, 1, 7, 1, 3, 2, 4, 2, 8, 4, 6, 8, 3, 2, 7, 0]","[0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9]"
12 | 10,"[5, 2, 2, 3, 5, 5, 6, 2, 6, 9, 9, 9, 9, 7, 5, 1, 4, 5, 0, 4, 0, 3, 6, 6, 8, 1, 2, 2, 2, 3, 7, 2]","[0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 9, 9]"
13 | 11,"[4, 1, 7, 1, 8, 8, 7, 5, 7, 1, 6, 1, 6, 1, 4, 4, 0, 0, 3, 3, 5, 6, 9, 3, 4, 9, 9, 2, 3, 0, 5, 5]","[0, 0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9]"
14 | 12,"[6, 6, 5, 3, 1, 7, 2, 8, 7, 2, 0, 5, 4, 5, 0, 5, 8, 0, 4, 8, 5, 9, 0, 0, 2, 3, 4, 3, 2, 0, 4, 4]","[0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 8, 9]"
15 | 13,"[7, 0, 9, 2, 0, 6, 5, 9, 3, 5, 6, 2, 0, 1, 4, 7, 5, 7, 0, 9, 8, 2, 1, 3, 4, 4, 8, 4, 2, 7, 9, 7]","[0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5, 5, 6, 6, 7, 7, 7, 7, 7, 8, 8, 9, 9, 9, 9]"
16 | 14,"[3, 4, 7, 1, 9, 8, 3, 0, 6, 4, 4, 1, 9, 9, 9, 2, 8, 4, 7, 4, 7, 9, 7, 6, 9, 7, 2, 1, 4, 5, 3, 1]","[0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 6, 6, 7, 7, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 9]"
17 | 15,"[8, 7, 0, 7, 8, 6, 3, 6, 6, 9, 8, 0, 3, 9, 3, 1, 7, 7, 4, 2, 5, 8, 2, 8, 7, 2, 1, 2, 6, 5, 5, 2]","[0, 0, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9]"
18 | 16,"[9, 7, 1, 8, 7, 9, 5, 0, 9, 0, 6, 4, 5, 7, 0, 9, 2, 5, 8, 4, 3, 9, 6, 4, 6, 1, 7, 2, 2, 5, 3, 0]","[0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9]"
19 | 17,"[7, 5, 1, 8, 1, 5, 9, 6, 0, 9, 8, 2, 1, 2, 7, 4, 5, 7, 3, 7, 8, 5, 1, 0, 4, 7, 8, 1, 4, 1, 4, 2]","[0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
20 | 18,"[4, 3, 6, 3, 7, 8, 5, 6, 5, 6, 8, 3, 6, 5, 5, 6, 0, 8, 4, 9, 2, 0, 4, 3, 2, 2, 5, 0, 8, 5, 2, 6]","[0, 0, 0, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 8, 8, 8, 8, 9]"
21 | 19,"[9, 7, 8, 2, 6, 2, 9, 7, 6, 8, 1, 0, 6, 3, 5, 0, 3, 6, 2, 4, 2, 0, 6, 8, 3, 1, 9, 7, 7, 2, 6, 4]","[0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9]"
22 | 20,"[8, 7, 6, 4, 5, 6, 4, 0, 2, 1, 8, 0, 2, 0, 8, 1, 8, 4, 4, 9, 7, 6, 0, 9, 6, 6, 1, 2, 5, 4, 7, 3]","[0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 4, 4, 4, 4, 4, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
23 | 21,"[6, 6, 4, 0, 7, 9, 5, 2, 0, 4, 2, 4, 8, 4, 0, 3, 7, 0, 4, 3, 1, 3, 0, 1, 9, 6, 7, 4, 9, 1, 3, 0]","[0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 6, 6, 6, 7, 7, 7, 8, 9, 9, 9]"
24 | 22,"[6, 8, 0, 2, 7, 7, 9, 7, 9, 0, 6, 6, 8, 9, 1, 4, 2, 9, 6, 6, 2, 0, 0, 7, 5, 2, 5, 1, 3, 5, 2, 6]","[0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 2, 3, 4, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 9, 9, 9, 9]"
25 | 23,"[8, 5, 0, 3, 6, 5, 6, 5, 4, 8, 0, 0, 7, 3, 9, 2, 9, 2, 0, 4, 1, 0, 5, 5, 8, 2, 8, 4, 4, 3, 5, 1]","[0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 7, 8, 8, 8, 8, 9, 9]"
26 | 24,"[8, 0, 5, 9, 3, 9, 2, 1, 7, 2, 1, 9, 7, 4, 8, 4, 2, 2, 9, 2, 1, 7, 3, 4, 4, 3, 3, 0, 8, 1, 8, 6]","[0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 6, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9]"
27 | 25,"[9, 8, 8, 9, 8, 1, 9, 6, 5, 8, 0, 8, 5, 5, 5, 4, 3, 5, 8, 4, 0, 3, 3, 6, 6, 1, 4, 7, 0, 7, 2, 8]","[0, 0, 0, 1, 1, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9]"
28 | 26,"[3, 0, 9, 6, 0, 7, 0, 2, 7, 8, 8, 9, 7, 5, 6, 0, 2, 3, 4, 9, 6, 4, 8, 9, 3, 5, 6, 2, 0, 1, 5, 3]","[0, 0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]"
29 | 27,"[6, 4, 9, 0, 5, 9, 0, 8, 5, 5, 9, 2, 6, 4, 8, 1, 9, 1, 3, 9, 4, 6, 6, 9, 7, 8, 8, 8, 9, 1, 1, 4]","[0, 0, 1, 1, 1, 1, 2, 3, 4, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
30 | 28,"[3, 9, 6, 5, 2, 1, 2, 7, 6, 9, 6, 4, 7, 2, 9, 1, 6, 6, 1, 6, 4, 7, 1, 0, 5, 7, 6, 2, 2, 2, 4, 8]","[0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 9, 9, 9]"
31 | 29,"[3, 9, 5, 1, 3, 9, 0, 8, 0, 1, 9, 5, 8, 7, 3, 1, 6, 6, 2, 2, 0, 2, 7, 2, 5, 5, 5, 1, 6, 6, 9, 0]","[0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 9, 9, 9, 9]"
32 | 30,"[5, 1, 2, 2, 1, 9, 1, 7, 7, 1, 4, 1, 1, 3, 7, 0, 2, 0, 7, 8, 0, 1, 9, 2, 0, 4, 5, 6, 6, 1, 8, 9]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 4, 4, 5, 5, 6, 6, 7, 7, 7, 7, 8, 8, 9, 9, 9]"
33 | 31,"[4, 2, 6, 4, 7, 2, 5, 2, 8, 4, 9, 5, 7, 7, 3, 2, 5, 5, 0, 2, 2, 8, 2, 8, 2, 6, 5, 3, 3, 5, 8, 4]","[0, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9]"
34 | 32,"[6, 6, 2, 2, 9, 8, 7, 2, 5, 3, 5, 7, 1, 3, 4, 0, 3, 6, 9, 1, 6, 1, 0, 4, 1, 2, 6, 7, 7, 6, 9, 1]","[0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 9, 9, 9]"
35 | 33,"[6, 3, 9, 8, 1, 4, 2, 1, 0, 0, 1, 3, 7, 9, 8, 1, 3, 1, 5, 3, 9, 7, 8, 9, 3, 9, 4, 0, 2, 7, 3, 9]","[0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 5, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9, 9, 9]"
36 | 34,"[2, 6, 4, 5, 8, 1, 9, 3, 6, 3, 9, 7, 3, 3, 8, 4, 6, 7, 2, 7, 3, 2, 3, 3, 5, 5, 6, 4, 1, 6, 6, 2]","[1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9]"
37 | 35,"[6, 6, 0, 8, 0, 6, 8, 4, 9, 8, 3, 8, 8, 1, 7, 1, 5, 0, 9, 1, 9, 9, 8, 1, 5, 3, 2, 2, 7, 0, 5, 6]","[0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
38 | 36,"[8, 9, 1, 5, 7, 4, 2, 6, 2, 8, 8, 9, 3, 0, 6, 7, 7, 8, 6, 5, 8, 3, 3, 0, 1, 6, 4, 0, 8, 8, 1, 7]","[0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9]"
39 | 37,"[9, 5, 1, 1, 3, 1, 0, 4, 4, 6, 1, 9, 4, 8, 0, 9, 4, 0, 5, 1, 9, 5, 6, 7, 9, 1, 2, 7, 1, 3, 9, 6]","[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 3, 3, 4, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 8, 9, 9, 9, 9, 9, 9]"
40 | 38,"[4, 1, 1, 8, 1, 7, 9, 1, 5, 5, 1, 9, 0, 8, 7, 7, 9, 0, 7, 0, 4, 9, 8, 5, 9, 6, 6, 2, 6, 5, 1, 1]","[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9, 9]"
41 | 39,"[6, 1, 6, 5, 7, 1, 6, 3, 8, 7, 6, 1, 3, 8, 6, 9, 3, 1, 1, 9, 4, 8, 3, 1, 9, 4, 5, 9, 5, 4, 1, 8]","[1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9]"
42 | 40,"[2, 1, 6, 3, 9, 3, 8, 3, 2, 3, 9, 9, 5, 4, 3, 2, 1, 9, 3, 4, 3, 3, 5, 4, 4, 4, 9, 7, 8, 3, 0, 8]","[0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 6, 7, 8, 8, 8, 9, 9, 9, 9, 9]"
43 | 41,"[6, 5, 6, 4, 3, 7, 3, 0, 6, 9, 1, 0, 1, 7, 0, 1, 8, 6, 5, 9, 5, 9, 4, 2, 5, 3, 8, 8, 8, 0, 9, 7]","[0, 0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9]"
44 | 42,"[9, 2, 6, 8, 4, 0, 5, 0, 8, 6, 1, 6, 6, 6, 7, 6, 5, 2, 3, 4, 3, 9, 2, 9, 3, 2, 0, 0, 1, 4, 6, 6]","[0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 8, 8, 9, 9, 9]"
45 | 43,"[7, 7, 7, 8, 7, 9, 4, 3, 5, 5, 1, 7, 7, 4, 0, 5, 6, 2, 7, 3, 2, 2, 3, 2, 0, 8, 6, 7, 8, 4, 6, 4]","[0, 0, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 9]"
46 | 44,"[0, 1, 6, 5, 4, 6, 9, 3, 7, 0, 0, 1, 1, 6, 7, 1, 9, 9, 4, 5, 9, 1, 2, 2, 7, 5, 6, 1, 9, 1, 2, 4]","[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 9, 9, 9, 9, 9]"
47 | 45,"[1, 8, 6, 5, 0, 6, 2, 4, 2, 2, 0, 8, 7, 6, 1, 3, 1, 9, 7, 5, 9, 8, 1, 8, 2, 7, 7, 3, 7, 7, 8, 7]","[0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 5, 5, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9]"
48 | 46,"[4, 7, 0, 6, 4, 5, 0, 3, 0, 3, 4, 6, 5, 2, 6, 5, 5, 9, 7, 1, 0, 8, 0, 4, 9, 1, 3, 5, 1, 7, 2, 0]","[0, 0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 9, 9]"
49 | 47,"[9, 6, 0, 1, 8, 4, 9, 6, 3, 6, 0, 7, 0, 8, 9, 3, 6, 6, 9, 3, 2, 2, 2, 9, 4, 9, 3, 8, 1, 9, 6, 1]","[0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 6, 6, 6, 6, 6, 6, 7, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
50 | 48,"[7, 3, 4, 8, 4, 2, 8, 3, 1, 0, 7, 3, 1, 9, 1, 8, 0, 1, 9, 8, 3, 2, 4, 1, 7, 3, 1, 3, 2, 0, 4, 0]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
51 | 49,"[2, 3, 7, 6, 5, 7, 5, 2, 8, 6, 8, 3, 7, 2, 3, 8, 6, 5, 4, 5, 3, 6, 8, 3, 0, 9, 2, 1, 7, 5, 5, 8]","[0, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9]"
52 | 50,"[4, 0, 9, 1, 1, 6, 5, 0, 6, 4, 8, 1, 6, 9, 6, 1, 8, 1, 3, 1, 1, 5, 8, 3, 2, 2, 9, 8, 4, 6, 6, 9]","[0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6, 6, 6, 8, 8, 8, 8, 9, 9, 9, 9]"
53 | 51,"[1, 9, 8, 5, 8, 5, 2, 4, 1, 4, 8, 0, 7, 9, 0, 5, 0, 5, 6, 4, 6, 2, 2, 3, 4, 2, 8, 8, 6, 7, 8, 9]","[0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9]"
54 | 52,"[3, 5, 2, 8, 8, 5, 4, 6, 0, 7, 2, 0, 2, 5, 2, 6, 0, 1, 8, 0, 9, 2, 8, 2, 8, 7, 3, 5, 6, 8, 0, 8]","[0, 0, 0, 0, 0, 1, 2, 2, 2, 2, 2, 2, 3, 3, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 8, 8, 8, 8, 8, 8, 8, 9]"
55 | 53,"[2, 4, 1, 8, 9, 8, 2, 4, 0, 7, 9, 6, 1, 8, 1, 5, 6, 5, 8, 8, 7, 0, 4, 7, 2, 1, 9, 4, 4, 5, 5, 6]","[0, 0, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9]"
56 | 54,"[4, 9, 4, 3, 2, 0, 9, 3, 9, 7, 7, 5, 9, 4, 3, 1, 6, 2, 5, 1, 9, 7, 2, 1, 9, 4, 6, 0, 7, 4, 9, 4]","[0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 6, 6, 7, 7, 7, 7, 9, 9, 9, 9, 9, 9, 9]"
57 | 55,"[2, 3, 6, 7, 2, 2, 1, 6, 4, 0, 0, 9, 1, 6, 9, 1, 1, 2, 5, 1, 8, 1, 7, 1, 2, 1, 6, 0, 1, 6, 4, 1]","[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 4, 4, 5, 6, 6, 6, 6, 6, 7, 7, 8, 9, 9]"
58 | 56,"[4, 7, 7, 5, 3, 5, 9, 9, 3, 1, 4, 7, 8, 3, 4, 7, 7, 3, 3, 7, 0, 0, 2, 9, 6, 5, 3, 7, 3, 0, 1, 1]","[0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 7, 7, 7, 7, 7, 7, 7, 8, 9, 9, 9]"
59 | 57,"[0, 9, 8, 9, 1, 3, 8, 0, 5, 7, 8, 5, 3, 4, 2, 5, 7, 8, 9, 6, 5, 1, 4, 5, 1, 3, 1, 8, 9, 2, 6, 9]","[0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 5, 5, 6, 6, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
60 | 58,"[6, 1, 5, 1, 3, 5, 4, 0, 1, 0, 6, 8, 9, 8, 1, 7, 6, 8, 7, 4, 6, 2, 9, 0, 8, 0, 7, 2, 6, 6, 7, 7]","[0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
61 | 59,"[2, 4, 0, 5, 4, 0, 6, 6, 1, 7, 8, 1, 7, 3, 4, 0, 9, 2, 8, 8, 8, 1, 1, 1, 7, 3, 1, 6, 0, 9, 1, 3]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
62 | 60,"[3, 8, 0, 9, 5, 6, 6, 8, 5, 2, 9, 4, 8, 2, 3, 5, 7, 6, 1, 7, 1, 4, 3, 9, 4, 5, 9, 5, 1, 9, 9, 9]","[0, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 6, 7, 7, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
63 | 61,"[8, 8, 0, 2, 4, 3, 1, 9, 2, 9, 1, 1, 3, 7, 9, 9, 9, 1, 5, 2, 1, 3, 1, 5, 7, 7, 8, 2, 8, 8, 8, 2]","[0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 4, 5, 5, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9]"
64 | 62,"[5, 0, 7, 6, 2, 6, 6, 7, 0, 5, 6, 3, 9, 5, 4, 9, 0, 0, 4, 9, 0, 3, 1, 4, 7, 5, 9, 8, 6, 0, 7, 3]","[0, 0, 0, 0, 0, 0, 1, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 9, 9, 9, 9]"
65 | 63,"[0, 0, 4, 5, 1, 1, 1, 0, 2, 2, 5, 2, 5, 0, 0, 2, 1, 4, 6, 2, 0, 8, 8, 0, 5, 4, 3, 6, 8, 1, 0, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 8, 8, 8]"
66 | 64,"[3, 7, 4, 1, 6, 1, 9, 8, 1, 9, 7, 8, 8, 4, 6, 4, 3, 0, 2, 0, 0, 7, 0, 8, 1, 5, 3, 5, 2, 2, 0, 6]","[0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
67 | 65,"[3, 3, 7, 3, 2, 5, 7, 1, 4, 7, 0, 7, 1, 2, 5, 9, 6, 9, 5, 5, 0, 3, 2, 4, 1, 2, 0, 5, 5, 0, 1, 8]","[0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 7, 8, 9, 9]"
68 | 66,"[9, 1, 9, 1, 3, 4, 4, 5, 0, 1, 9, 8, 0, 7, 1, 8, 7, 7, 6, 0, 0, 6, 1, 4, 4, 5, 8, 5, 4, 4, 4, 8]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9]"
69 | 67,"[0, 7, 5, 5, 0, 3, 0, 7, 3, 9, 8, 3, 1, 2, 1, 5, 3, 6, 8, 7, 2, 9, 9, 6, 9, 2, 0, 1, 7, 9, 2, 0]","[0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 5, 5, 5, 6, 6, 7, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9]"
70 | 68,"[1, 1, 6, 1, 1, 6, 0, 5, 0, 0, 8, 2, 5, 9, 6, 7, 1, 3, 9, 0, 9, 7, 0, 0, 3, 2, 2, 4, 0, 3, 6, 5]","[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 9]"
71 | 69,"[3, 4, 2, 5, 5, 8, 4, 8, 5, 5, 8, 3, 4, 9, 9, 8, 4, 9, 9, 9, 9, 3, 9, 0, 5, 8, 3, 7, 8, 6, 1, 1]","[0, 1, 1, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9]"
72 | 70,"[6, 4, 5, 0, 8, 3, 9, 7, 0, 2, 0, 2, 8, 5, 8, 2, 6, 2, 1, 9, 4, 5, 3, 3, 4, 0, 5, 7, 7, 2, 4, 0]","[0, 0, 0, 0, 0, 1, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9]"
73 | 71,"[0, 4, 8, 5, 2, 4, 6, 7, 2, 6, 2, 8, 3, 6, 6, 2, 0, 9, 7, 7, 5, 1, 0, 8, 6, 0, 7, 7, 0, 3, 2, 6]","[0, 0, 0, 0, 0, 1, 2, 2, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 9]"
74 | 72,"[5, 1, 7, 2, 4, 5, 0, 2, 7, 2, 2, 9, 7, 0, 5, 8, 5, 2, 2, 1, 2, 6, 9, 7, 8, 8, 4, 5, 1, 8, 2, 8]","[0, 0, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 4, 4, 5, 5, 5, 5, 5, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9]"
75 | 73,"[8, 9, 8, 8, 3, 5, 5, 3, 6, 1, 3, 2, 6, 3, 1, 3, 3, 5, 1, 7, 6, 0, 9, 6, 8, 1, 1, 0, 9, 5, 5, 6]","[0, 0, 1, 1, 1, 1, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 8, 8, 8, 8, 9, 9, 9]"
76 | 74,"[4, 3, 3, 0, 4, 7, 9, 4, 3, 5, 3, 5, 4, 2, 0, 2, 1, 5, 8, 8, 4, 4, 3, 4, 3, 0, 1, 6, 7, 3, 2, 8]","[0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 6, 7, 7, 8, 8, 8, 9]"
77 | 75,"[7, 1, 9, 4, 7, 4, 0, 5, 0, 7, 2, 0, 8, 5, 8, 4, 2, 2, 5, 9, 2, 3, 8, 2, 7, 4, 6, 6, 8, 6, 3, 3]","[0, 0, 0, 1, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
78 | 76,"[7, 3, 1, 7, 2, 0, 4, 5, 6, 5, 8, 3, 2, 8, 8, 2, 2, 5, 3, 9, 1, 3, 4, 9, 2, 7, 7, 3, 6, 8, 3, 7]","[0, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 5, 5, 5, 6, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9]"
79 | 77,"[9, 4, 1, 3, 1, 3, 4, 8, 8, 8, 7, 2, 7, 9, 5, 0, 2, 6, 4, 8, 1, 3, 5, 4, 1, 8, 7, 4, 7, 7, 0, 8]","[0, 0, 1, 1, 1, 1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 6, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9]"
80 | 78,"[3, 5, 5, 9, 6, 6, 6, 2, 9, 0, 3, 0, 2, 1, 2, 6, 5, 8, 4, 8, 5, 9, 9, 5, 7, 0, 6, 8, 9, 3, 3, 5]","[0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 3, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 8, 8, 8, 9, 9, 9, 9, 9]"
81 | 79,"[9, 5, 9, 0, 4, 5, 6, 3, 0, 1, 4, 3, 1, 5, 9, 9, 3, 2, 9, 5, 5, 2, 9, 9, 9, 7, 8, 9, 8, 0, 2, 8]","[0, 0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 5, 5, 5, 6, 7, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 9]"
82 | 80,"[9, 3, 1, 0, 5, 7, 6, 1, 8, 9, 5, 9, 1, 9, 9, 9, 9, 2, 2, 4, 3, 0, 6, 1, 9, 2, 1, 3, 5, 2, 5, 1]","[0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 5, 5, 5, 5, 6, 6, 7, 8, 9, 9, 9, 9, 9, 9, 9, 9]"
83 | 81,"[8, 1, 1, 3, 0, 2, 8, 0, 9, 5, 3, 7, 7, 6, 8, 5, 2, 9, 3, 2, 6, 4, 2, 1, 7, 9, 4, 0, 1, 9, 3, 0]","[0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]"
84 | 82,"[0, 1, 4, 1, 5, 7, 4, 9, 3, 7, 5, 3, 6, 5, 1, 4, 1, 2, 2, 0, 0, 7, 5, 3, 5, 4, 6, 2, 1, 4, 1, 3]","[0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 6, 6, 7, 7, 7, 9]"
85 | 83,"[2, 2, 4, 0, 5, 7, 4, 5, 9, 5, 7, 5, 1, 5, 9, 5, 4, 1, 7, 9, 2, 4, 4, 1, 4, 3, 7, 9, 0, 6, 1, 1]","[0, 0, 1, 1, 1, 1, 1, 2, 2, 2, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 7, 9, 9, 9, 9]"
86 | 84,"[2, 5, 4, 9, 4, 5, 3, 8, 1, 5, 0, 4, 5, 5, 7, 4, 6, 6, 7, 7, 1, 9, 6, 5, 8, 6, 3, 6, 8, 9, 0, 7]","[0, 0, 1, 1, 2, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9]"
87 | 85,"[0, 4, 3, 0, 6, 1, 1, 9, 2, 1, 3, 6, 5, 1, 2, 3, 0, 1, 6, 7, 8, 4, 1, 0, 2, 1, 2, 3, 4, 5, 6, 1]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 6, 6, 6, 6, 7, 8, 9]"
88 | 86,"[4, 6, 0, 6, 5, 1, 7, 7, 0, 8, 0, 4, 6, 0, 8, 5, 0, 1, 8, 8, 0, 1, 0, 9, 4, 5, 5, 3, 0, 1, 9, 0]","[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 8, 8, 8, 8, 9, 9]"
89 | 87,"[6, 0, 1, 6, 6, 0, 7, 9, 2, 5, 8, 4, 5, 8, 1, 4, 8, 0, 6, 4, 0, 7, 1, 2, 3, 9, 4, 8, 7, 3, 8, 2]","[0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9]"
90 | 88,"[5, 1, 5, 3, 7, 6, 5, 0, 8, 6, 6, 4, 6, 7, 2, 0, 1, 9, 5, 2, 4, 1, 3, 0, 4, 6, 7, 8, 0, 7, 2, 8]","[0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9]"
91 | 89,"[5, 2, 2, 3, 8, 5, 0, 0, 6, 7, 3, 3, 7, 7, 6, 6, 8, 2, 0, 1, 6, 3, 3, 9, 9, 9, 1, 9, 5, 6, 4, 2]","[0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 5, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 9, 9, 9, 9]"
92 | 90,"[3, 0, 3, 3, 0, 9, 8, 7, 3, 9, 7, 0, 4, 0, 9, 4, 3, 7, 5, 1, 9, 5, 6, 6, 6, 3, 6, 3, 2, 8, 6, 8]","[0, 0, 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 5, 5, 6, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]"
93 | 91,"[2, 4, 6, 0, 9, 4, 3, 5, 7, 4, 8, 8, 8, 1, 8, 5, 8, 0, 2, 6, 2, 9, 4, 0, 0, 8, 5, 0, 5, 1, 2, 8]","[0, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 8, 8, 8, 8, 8, 8, 8, 9, 9]"
94 | 92,"[6, 5, 3, 4, 8, 2, 5, 9, 1, 3, 9, 6, 7, 4, 4, 7, 9, 7, 2, 2, 2, 8, 5, 8, 6, 3, 8, 5, 0, 8, 9, 1]","[0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
95 | 93,"[3, 8, 3, 9, 8, 4, 0, 7, 3, 9, 9, 2, 6, 0, 3, 5, 1, 3, 9, 3, 2, 0, 8, 8, 7, 3, 6, 1, 5, 4, 7, 7]","[0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9]"
96 | 94,"[2, 7, 8, 3, 3, 8, 7, 9, 2, 8, 0, 6, 9, 5, 8, 4, 2, 8, 3, 6, 3, 4, 1, 4, 8, 5, 5, 6, 0, 0, 7, 6]","[0, 0, 0, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9]"
97 | 95,"[8, 1, 9, 6, 1, 7, 1, 2, 6, 9, 0, 6, 0, 6, 8, 2, 3, 5, 8, 7, 9, 0, 9, 1, 7, 5, 4, 3, 7, 1, 8, 8]","[0, 0, 0, 1, 1, 1, 1, 1, 2, 2, 3, 3, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 9, 9, 9, 9]"
98 | 96,"[3, 7, 6, 2, 0, 4, 8, 7, 1, 1, 1, 1, 8, 2, 4, 1, 4, 7, 4, 0, 8, 4, 9, 5, 0, 0, 6, 8, 3, 2, 1, 2]","[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 4, 4, 4, 4, 5, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9]"
99 | 97,"[3, 1, 2, 8, 6, 7, 0, 2, 3, 1, 6, 7, 9, 3, 1, 9, 7, 4, 5, 5, 0, 7, 8, 3, 3, 1, 4, 4, 2, 1, 1, 5]","[0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 7, 7, 7, 7, 8, 8, 9, 9]"
100 | 98,"[5, 7, 1, 7, 2, 4, 1, 8, 4, 6, 9, 2, 0, 2, 3, 9, 3, 9, 7, 8, 3, 1, 1, 3, 9, 8, 5, 0, 2, 7, 0, 2]","[0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 5, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]"
101 | 99,"[9, 4, 4, 7, 2, 3, 9, 8, 8, 3, 9, 8, 7, 1, 7, 9, 5, 1, 6, 1, 6, 2, 0, 4, 7, 6, 4, 6, 3, 2, 6, 2]","[0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 6, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9]"
--------------------------------------------------------------------------------
/examples/sorting/utils.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # main author: Nils Blach
8 |
9 | from typing import Dict, List
10 |
11 |
12 | def string_to_list(string: str) -> List[int]:
13 | """
14 | Helper function to convert a list encoded inside a string into a Python
15 | list object of string elements.
16 |
17 | :param string: Input string containing a list.
18 | :type string: str
19 | :return: List of string elements.
20 | :rtype: List[str]
21 | :raise AssertionError: If input string does not contain a list.
22 | """
23 |
24 | assert string[0] == "[" and string[-1] == "]", "String is not a list."
25 | return [int(num) for num in string[1:-1].split(",")]
26 |
27 |
28 | def test_sorting(state: Dict) -> bool:
29 | """
30 | Function to test whether the final solution matches ground truth.
31 |
32 | :param state: Thought state that represents the final solution.
33 | :type state: Dict
34 | :return: Returns whether the solution matches the ground truth.
35 | :rtype: bool
36 | """
37 |
38 | try:
39 | correct_list = sorted(string_to_list(state["original"]))
40 | sorted_list = string_to_list(state["current"])
41 | return sorted_list == correct_list
42 | except:
43 | return False
44 |
45 |
46 | def num_errors(state: Dict) -> float:
47 | """
48 | Function to locally count the number of errors that serves as a score.
49 |
50 | :param state: Thought state to be scored.
51 | :type state: Dict
52 | :return: Number of errors.
53 | :rtype: float
54 | """
55 |
56 | try:
57 | unsorted_list = state["original"]
58 | if (
59 | "unsorted_sublist" in state
60 | and state["unsorted_sublist"] != ""
61 | and state["unsorted_sublist"] is not None
62 | and len(state["unsorted_sublist"]) < len(unsorted_list) - 5
63 | ):
64 | unsorted_list = state["unsorted_sublist"]
65 | correct_list = sorted(string_to_list(unsorted_list))
66 | current_list = string_to_list(state["current"])
67 | num_errors = 0
68 | for i in range(10):
69 | num_errors += abs(
70 | sum([1 for num in current_list if num == i])
71 | - sum([1 for num in correct_list if num == i])
72 | )
73 | num_errors += sum(
74 | [1 for num1, num2 in zip(current_list, current_list[1:]) if num1 > num2]
75 | )
76 | return num_errors
77 | except:
78 | return 300
79 |
--------------------------------------------------------------------------------
/graph_of_thoughts/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/spcl/graph-of-thoughts/363421c61c7bc11edf32845a697ae2aaccd75463/graph_of_thoughts/__init__.py
--------------------------------------------------------------------------------
/graph_of_thoughts/controller/README.md:
--------------------------------------------------------------------------------
1 | # Controller
2 |
3 | The Controller class is responsible for traversing the Graph of Operations (GoO), which is a static structure that is constructed once, before the execution starts.
4 | GoO prescribes the execution plan of thought operations and the Controller invokes their execution, generating the Graph Reasoning State (GRS).
5 |
6 | In order for a GoO to be executed, an instance of Large Language Model (LLM) must be supplied to the controller (along with other required objects).
7 | Please refer to the [Language Models](../language_models/README.md) section for more information about LLMs.
8 |
9 | The following section describes how to instantiate the Controller to run a defined GoO.
10 |
11 | ## Controller Instantiation
12 | - Requires custom `Prompter`, `Parser`, as well as instantiated `GraphOfOperations` and `AbstractLanguageModel` - creation of these is described separately.
13 | - Prepare initial state (thought) as dictionary - this can be used in the initial prompts by the operations.
14 | ```
15 | lm = ...create
16 | graph_of_operations = ...create
17 |
18 | executor = controller.Controller(
19 | lm,
20 | graph_of_operations,
21 | ,
22 | ,
23 | ,
24 | )
25 | executor.run()
26 | executor.output_graph("path/to/output.json")
27 | ```
28 | - After the run the graph is written to an output file, which contains individual operations, their thoughts, information about scores and validity and total amount of used tokens / cost.
29 |
--------------------------------------------------------------------------------
/graph_of_thoughts/controller/__init__.py:
--------------------------------------------------------------------------------
1 | from .controller import Controller
2 |
--------------------------------------------------------------------------------
/graph_of_thoughts/controller/controller.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # main author: Nils Blach
8 |
9 | import json
10 | import logging
11 | from typing import List
12 | from graph_of_thoughts.language_models import AbstractLanguageModel
13 | from graph_of_thoughts.operations import GraphOfOperations, Thought
14 | from graph_of_thoughts.prompter import Prompter
15 | from graph_of_thoughts.parser import Parser
16 |
17 |
18 | class Controller:
19 | """
20 | Controller class to manage the execution flow of the Graph of Operations,
21 | generating the Graph Reasoning State.
22 | This involves language models, graph operations, prompting, and parsing.
23 | """
24 |
25 | def __init__(
26 | self,
27 | lm: AbstractLanguageModel,
28 | graph: GraphOfOperations,
29 | prompter: Prompter,
30 | parser: Parser,
31 | problem_parameters: dict,
32 | ) -> None:
33 | """
34 | Initialize the Controller instance with the language model,
35 | operations graph, prompter, parser, and problem parameters.
36 |
37 | :param lm: An instance of the AbstractLanguageModel.
38 | :type lm: AbstractLanguageModel
39 | :param graph: The Graph of Operations to be executed.
40 | :type graph: OperationsGraph
41 | :param prompter: An instance of the Prompter class, used to generate prompts.
42 | :type prompter: Prompter
43 | :param parser: An instance of the Parser class, used to parse responses.
44 | :type parser: Parser
45 | :param problem_parameters: Initial parameters/state of the problem.
46 | :type problem_parameters: dict
47 | """
48 | self.logger = logging.getLogger(self.__class__.__module__)
49 | self.lm = lm
50 | self.graph = graph
51 | self.prompter = prompter
52 | self.parser = parser
53 | self.problem_parameters = problem_parameters
54 | self.run_executed = False
55 |
56 | def run(self) -> None:
57 | """
58 | Run the controller and execute the operations from the Graph of
59 | Operations based on their readiness.
60 | Ensures the program is in a valid state before execution.
61 | :raises AssertionError: If the Graph of Operation has no roots.
62 | :raises AssertionError: If the successor of an operation is not in the Graph of Operations.
63 | """
64 | self.logger.debug("Checking that the program is in a valid state")
65 | assert self.graph.roots is not None, "The operations graph has no root"
66 | self.logger.debug("The program is in a valid state")
67 |
68 | execution_queue = [
69 | operation
70 | for operation in self.graph.operations
71 | if operation.can_be_executed()
72 | ]
73 |
74 | while len(execution_queue) > 0:
75 | current_operation = execution_queue.pop(0)
76 | self.logger.info("Executing operation %s", current_operation.operation_type)
77 | current_operation.execute(
78 | self.lm, self.prompter, self.parser, **self.problem_parameters
79 | )
80 | self.logger.info("Operation %s executed", current_operation.operation_type)
81 | for operation in current_operation.successors:
82 | assert (
83 | operation in self.graph.operations
84 | ), "The successor of an operation is not in the operations graph"
85 | if operation.can_be_executed():
86 | execution_queue.append(operation)
87 | self.logger.info("All operations executed")
88 | self.run_executed = True
89 |
90 | def get_final_thoughts(self) -> List[List[Thought]]:
91 | """
92 | Retrieve the final thoughts after all operations have been executed.
93 |
94 | :return: List of thoughts for each operation in the graph's leaves.
95 | :rtype: List[List[Thought]]
96 | :raises AssertionError: If the `run` method hasn't been executed yet.
97 | """
98 | assert self.run_executed, "The run method has not been executed"
99 | return [operation.get_thoughts() for operation in self.graph.leaves]
100 |
101 | def output_graph(self, path: str) -> None:
102 | """
103 | Serialize the state and results of the operations graph to a JSON file.
104 |
105 | :param path: The path to the output file.
106 | :type path: str
107 | """
108 | output = []
109 | for operation in self.graph.operations:
110 | operation_serialized = {
111 | "operation": operation.operation_type.name,
112 | "thoughts": [thought.state for thought in operation.get_thoughts()],
113 | }
114 | if any([thought.scored for thought in operation.get_thoughts()]):
115 | operation_serialized["scored"] = [
116 | thought.scored for thought in operation.get_thoughts()
117 | ]
118 | operation_serialized["scores"] = [
119 | thought.score for thought in operation.get_thoughts()
120 | ]
121 | if any([thought.validated for thought in operation.get_thoughts()]):
122 | operation_serialized["validated"] = [
123 | thought.validated for thought in operation.get_thoughts()
124 | ]
125 | operation_serialized["validity"] = [
126 | thought.valid for thought in operation.get_thoughts()
127 | ]
128 | if any(
129 | [
130 | thought.compared_to_ground_truth
131 | for thought in operation.get_thoughts()
132 | ]
133 | ):
134 | operation_serialized["compared_to_ground_truth"] = [
135 | thought.compared_to_ground_truth
136 | for thought in operation.get_thoughts()
137 | ]
138 | operation_serialized["problem_solved"] = [
139 | thought.solved for thought in operation.get_thoughts()
140 | ]
141 | output.append(operation_serialized)
142 |
143 | output.append(
144 | {
145 | "prompt_tokens": self.lm.prompt_tokens,
146 | "completion_tokens": self.lm.completion_tokens,
147 | "cost": self.lm.cost,
148 | }
149 | )
150 |
151 | with open(path, "w") as file:
152 | file.write(json.dumps(output, indent=2))
153 |
--------------------------------------------------------------------------------
/graph_of_thoughts/language_models/README.md:
--------------------------------------------------------------------------------
1 | # Language Models
2 |
3 | The Language Models module is responsible for managing the large language models (LLMs) used by the Controller.
4 |
5 | Currently, the framework supports the following LLMs:
6 | - GPT-4 / GPT-3.5 (Remote - OpenAI API)
7 | - LLaMA-2 (Local - HuggingFace Transformers)
8 |
9 | The following sections describe how to instantiate individual LLMs and how to add new LLMs to the framework.
10 |
11 | ## LLM Instantiation
12 | - Create a copy of `config_template.json` named `config.json`.
13 | - Fill configuration details based on the used model (below).
14 |
15 | ### GPT-4 / GPT-3.5
16 | - Adjust the predefined `chatgpt` or `chatgpt4` configurations or create a new configuration with an unique key.
17 |
18 | | Key | Value |
19 | |---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
20 | | model_id | Model name based on [OpenAI model overview](https://platform.openai.com/docs/models/overview). |
21 | | prompt_token_cost | Price per 1000 prompt tokens based on [OpenAI pricing](https://openai.com/pricing), used for calculating cumulative price per LLM instance. |
22 | | response_token_cost | Price per 1000 response tokens based on [OpenAI pricing](https://openai.com/pricing), used for calculating cumulative price per LLM instance. |
23 | | temperature | Parameter of OpenAI models that controls the randomness and the creativity of the responses (higher temperature = more diverse and unexpected responses). Value between 0.0 and 2.0, default is 1.0. More information can be found in the [OpenAI API reference](https://platform.openai.com/docs/api-reference/completions/create#completions/create-temperature). |
24 | | max_tokens | The maximum number of tokens to generate in the chat completion. Value depends on the maximum context size of the model specified in the [OpenAI model overview](https://platform.openai.com/docs/models/overview). More information can be found in the [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat/create-max_tokens). |
25 | | stop | String or array of strings specifying sequences of characters which if detected, stops further generation of tokens. More information can be found in the [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat/create-stop). |
26 | | organization | Organization to use for the API requests (may be empty). |
27 | | api_key | Personal API key that will be used to access OpenAI API. |
28 |
29 | - Instantiate the language model based on the selected configuration key (predefined / custom).
30 | ```python
31 | lm = controller.ChatGPT(
32 | "path/to/config.json",
33 | model_name=
34 | )
35 | ```
36 |
37 | ### LLaMA-2
38 | - Requires local hardware to run inference and a HuggingFace account.
39 | - Adjust the predefined `llama7b-hf`, `llama13b-hf` or `llama70b-hf` configurations or create a new configuration with an unique key.
40 |
41 | | Key | Value |
42 | |---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
43 | | model_id | Specifies HuggingFace LLaMA-2 model identifier (`meta-llama/`). |
44 | | cache_dir | Local directory where the model will be downloaded and accessed. |
45 | | prompt_token_cost | Price per 1000 prompt tokens (currently not used - local model = no cost). |
46 | | response_token_cost | Price per 1000 response tokens (currently not used - local model = no cost). |
47 | | temperature | Parameter that controls the randomness and the creativity of the responses (higher temperature = more diverse and unexpected responses). Value between 0.0 and 1.0, default is 0.6. |
48 | | top_k | Top-K sampling method described in [Transformers tutorial](https://huggingface.co/blog/how-to-generate). Default value is set to 10. |
49 | | max_tokens | The maximum number of tokens to generate in the chat completion. More tokens require more memory. |
50 |
51 | - Instantiate the language model based on the selected configuration key (predefined / custom).
52 | ```python
53 | lm = controller.Llama2HF(
54 | "path/to/config.json",
55 | model_name=
56 | )
57 | ```
58 | - Request access to LLaMA-2 via the [Meta form](https://ai.meta.com/resources/models-and-libraries/llama-downloads/) using the same email address as for the HuggingFace account.
59 | - After the access is granted, go to [HuggingFace LLaMA-2 model card](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf), log in and accept the license (a _"You have been granted access to this model"_ message should appear).
60 | - Generate HuggingFace access token.
61 | - Log in from CLI with: `huggingface-cli login --token `.
62 |
63 | Note: 4-bit quantization is used to reduce the model size for inference. During instantiation, the model is downloaded from HuggingFace into the cache directory specified in the `config.json`. Running queries using larger models will require multiple GPUs (splitting across many GPUs is done automatically by the Transformers library).
64 |
65 | ## Adding LLMs
66 | More LLMs can be added by following these steps:
67 | - Create a new class as a subclass of `AbstractLanguageModel`.
68 | - Use the constructor for loading the configuration and instantiating the language model (if needed).
69 | ```python
70 | class CustomLanguageModel(AbstractLanguageModel):
71 | def __init__(
72 | self,
73 | config_path: str = "",
74 | model_name: str = "llama7b-hf",
75 | cache: bool = False
76 | ) -> None:
77 | super().__init__(config_path, model_name, cache)
78 | self.config: Dict = self.config[model_name]
79 |
80 | # Load data from configuration into variables if needed
81 |
82 | # Instantiate LLM if needed
83 | ```
84 | - Implement the `query` abstract method that is used to get a list of responses from the LLM (remote API call or local model inference).
85 | ```python
86 | def query(self, query: str, num_responses: int = 1) -> Any:
87 | # Support caching
88 | # Call LLM and retrieve list of responses - based on num_responses
89 | # Return LLM response structure (not only raw strings)
90 | ```
91 | - Implement the `get_response_texts` abstract method that is used to get a list of raw texts from the LLM response structure produced by `query`.
92 | ```python
93 | def get_response_texts(self, query_response: Union[List[Any], Any]) -> List[str]:
94 | # Retrieve list of raw strings from the LLM response structure
95 | ```
96 |
--------------------------------------------------------------------------------
/graph_of_thoughts/language_models/__init__.py:
--------------------------------------------------------------------------------
1 | from .abstract_language_model import AbstractLanguageModel
2 | from .chatgpt import ChatGPT
3 | from .llamachat_hf import Llama2HF
4 |
--------------------------------------------------------------------------------
/graph_of_thoughts/language_models/abstract_language_model.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # main author: Nils Blach
8 |
9 | from abc import ABC, abstractmethod
10 | from typing import List, Dict, Union, Any
11 | import json
12 | import os
13 | import logging
14 |
15 |
16 | class AbstractLanguageModel(ABC):
17 | """
18 | Abstract base class that defines the interface for all language models.
19 | """
20 |
21 | def __init__(
22 | self, config_path: str = "", model_name: str = "", cache: bool = False
23 | ) -> None:
24 | """
25 | Initialize the AbstractLanguageModel instance with configuration, model details, and caching options.
26 |
27 | :param config_path: Path to the config file. Defaults to "".
28 | :type config_path: str
29 | :param model_name: Name of the language model. Defaults to "".
30 | :type model_name: str
31 | :param cache: Flag to determine whether to cache responses. Defaults to False.
32 | :type cache: bool
33 | """
34 | self.logger = logging.getLogger(self.__class__.__name__)
35 | self.config: Dict = None
36 | self.model_name: str = model_name
37 | self.cache = cache
38 | if self.cache:
39 | self.response_cache: Dict[str, List[Any]] = {}
40 | self.load_config(config_path)
41 | self.prompt_tokens: int = 0
42 | self.completion_tokens: int = 0
43 | self.cost: float = 0.0
44 |
45 | def load_config(self, path: str) -> None:
46 | """
47 | Load configuration from a specified path.
48 |
49 | :param path: Path to the config file. If an empty path provided,
50 | default is `config.json` in the current directory.
51 | :type path: str
52 | """
53 | if path == "":
54 | current_dir = os.path.dirname(os.path.abspath(__file__))
55 | path = os.path.join(current_dir, "config.json")
56 |
57 | with open(path, "r") as f:
58 | self.config = json.load(f)
59 |
60 | self.logger.debug(f"Loaded config from {path} for {self.model_name}")
61 |
62 | def clear_cache(self) -> None:
63 | """
64 | Clear the response cache.
65 | """
66 | self.response_cache.clear()
67 |
68 | @abstractmethod
69 | def query(self, query: str, num_responses: int = 1) -> Any:
70 | """
71 | Abstract method to query the language model.
72 |
73 | :param query: The query to be posed to the language model.
74 | :type query: str
75 | :param num_responses: The number of desired responses.
76 | :type num_responses: int
77 | :return: The language model's response(s).
78 | :rtype: Any
79 | """
80 | pass
81 |
82 | @abstractmethod
83 | def get_response_texts(self, query_responses: Union[List[Any], Any]) -> List[str]:
84 | """
85 | Abstract method to extract response texts from the language model's response(s).
86 |
87 | :param query_responses: The responses returned from the language model.
88 | :type query_responses: Union[List[Any], Any]
89 | :return: List of textual responses.
90 | :rtype: List[str]
91 | """
92 | pass
93 |
--------------------------------------------------------------------------------
/graph_of_thoughts/language_models/chatgpt.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # main author: Nils Blach
8 |
9 | import backoff
10 | import os
11 | import random
12 | import time
13 | from typing import List, Dict, Union
14 | from openai import OpenAI, OpenAIError
15 | from openai.types.chat.chat_completion import ChatCompletion
16 |
17 | from .abstract_language_model import AbstractLanguageModel
18 |
19 |
20 | class ChatGPT(AbstractLanguageModel):
21 | """
22 | The ChatGPT class handles interactions with the OpenAI models using the provided configuration.
23 |
24 | Inherits from the AbstractLanguageModel and implements its abstract methods.
25 | """
26 |
27 | def __init__(
28 | self, config_path: str = "", model_name: str = "chatgpt", cache: bool = False
29 | ) -> None:
30 | """
31 | Initialize the ChatGPT instance with configuration, model details, and caching options.
32 |
33 | :param config_path: Path to the configuration file. Defaults to "".
34 | :type config_path: str
35 | :param model_name: Name of the model, default is 'chatgpt'. Used to select the correct configuration.
36 | :type model_name: str
37 | :param cache: Flag to determine whether to cache responses. Defaults to False.
38 | :type cache: bool
39 | """
40 | super().__init__(config_path, model_name, cache)
41 | self.config: Dict = self.config[model_name]
42 | # The model_id is the id of the model that is used for chatgpt, i.e. gpt-4, gpt-3.5-turbo, etc.
43 | self.model_id: str = self.config["model_id"]
44 | # The prompt_token_cost and response_token_cost are the costs for 1000 prompt tokens and 1000 response tokens respectively.
45 | self.prompt_token_cost: float = self.config["prompt_token_cost"]
46 | self.response_token_cost: float = self.config["response_token_cost"]
47 | # The temperature of a model is defined as the randomness of the model's output.
48 | self.temperature: float = self.config["temperature"]
49 | # The maximum number of tokens to generate in the chat completion.
50 | self.max_tokens: int = self.config["max_tokens"]
51 | # The stop sequence is a sequence of tokens that the model will stop generating at (it will not generate the stop sequence).
52 | self.stop: Union[str, List[str]] = self.config["stop"]
53 | # The account organization is the organization that is used for chatgpt.
54 | self.organization: str = self.config["organization"]
55 | if self.organization == "":
56 | self.logger.warning("OPENAI_ORGANIZATION is not set")
57 | self.api_key: str = os.getenv("OPENAI_API_KEY", self.config["api_key"])
58 | if self.api_key == "":
59 | raise ValueError("OPENAI_API_KEY is not set")
60 | # Initialize the OpenAI Client
61 | self.client = OpenAI(api_key=self.api_key, organization=self.organization)
62 |
63 | def query(
64 | self, query: str, num_responses: int = 1
65 | ) -> Union[List[ChatCompletion], ChatCompletion]:
66 | """
67 | Query the OpenAI model for responses.
68 |
69 | :param query: The query to be posed to the language model.
70 | :type query: str
71 | :param num_responses: Number of desired responses, default is 1.
72 | :type num_responses: int
73 | :return: Response(s) from the OpenAI model.
74 | :rtype: Dict
75 | """
76 | if self.cache and query in self.response_cache:
77 | return self.response_cache[query]
78 |
79 | if num_responses == 1:
80 | response = self.chat([{"role": "user", "content": query}], num_responses)
81 | else:
82 | response = []
83 | next_try = num_responses
84 | total_num_attempts = num_responses
85 | while num_responses > 0 and total_num_attempts > 0:
86 | try:
87 | assert next_try > 0
88 | res = self.chat([{"role": "user", "content": query}], next_try)
89 | response.append(res)
90 | num_responses -= next_try
91 | next_try = min(num_responses, next_try)
92 | except Exception as e:
93 | next_try = (next_try + 1) // 2
94 | self.logger.warning(
95 | f"Error in chatgpt: {e}, trying again with {next_try} samples"
96 | )
97 | time.sleep(random.randint(1, 3))
98 | total_num_attempts -= 1
99 |
100 | if self.cache:
101 | self.response_cache[query] = response
102 | return response
103 |
104 | @backoff.on_exception(backoff.expo, OpenAIError, max_time=10, max_tries=6)
105 | def chat(self, messages: List[Dict], num_responses: int = 1) -> ChatCompletion:
106 | """
107 | Send chat messages to the OpenAI model and retrieves the model's response.
108 | Implements backoff on OpenAI error.
109 |
110 | :param messages: A list of message dictionaries for the chat.
111 | :type messages: List[Dict]
112 | :param num_responses: Number of desired responses, default is 1.
113 | :type num_responses: int
114 | :return: The OpenAI model's response.
115 | :rtype: ChatCompletion
116 | """
117 | response = self.client.chat.completions.create(
118 | model=self.model_id,
119 | messages=messages,
120 | temperature=self.temperature,
121 | max_tokens=self.max_tokens,
122 | n=num_responses,
123 | stop=self.stop,
124 | )
125 |
126 | self.prompt_tokens += response.usage.prompt_tokens
127 | self.completion_tokens += response.usage.completion_tokens
128 | prompt_tokens_k = float(self.prompt_tokens) / 1000.0
129 | completion_tokens_k = float(self.completion_tokens) / 1000.0
130 | self.cost = (
131 | self.prompt_token_cost * prompt_tokens_k
132 | + self.response_token_cost * completion_tokens_k
133 | )
134 | self.logger.info(
135 | f"This is the response from chatgpt: {response}"
136 | f"\nThis is the cost of the response: {self.cost}"
137 | )
138 | return response
139 |
140 | def get_response_texts(
141 | self, query_response: Union[List[ChatCompletion], ChatCompletion]
142 | ) -> List[str]:
143 | """
144 | Extract the response texts from the query response.
145 |
146 | :param query_response: The response dictionary (or list of dictionaries) from the OpenAI model.
147 | :type query_response: Union[List[ChatCompletion], ChatCompletion]
148 | :return: List of response strings.
149 | :rtype: List[str]
150 | """
151 | if not isinstance(query_response, List):
152 | query_response = [query_response]
153 | return [
154 | choice.message.content
155 | for response in query_response
156 | for choice in response.choices
157 | ]
158 |
--------------------------------------------------------------------------------
/graph_of_thoughts/language_models/config_template.json:
--------------------------------------------------------------------------------
1 | {
2 | "chatgpt" : {
3 | "model_id": "gpt-3.5-turbo",
4 | "prompt_token_cost": 0.0015,
5 | "response_token_cost": 0.002,
6 | "temperature": 1.0,
7 | "max_tokens": 1536,
8 | "stop": null,
9 | "organization": "",
10 | "api_key": ""
11 | },
12 | "chatgpt4" : {
13 | "model_id": "gpt-4",
14 | "prompt_token_cost": 0.03,
15 | "response_token_cost": 0.06,
16 | "temperature": 1.0,
17 | "max_tokens": 4096,
18 | "stop": null,
19 | "organization": "",
20 | "api_key": ""
21 | },
22 | "llama7b-hf" : {
23 | "model_id": "Llama-2-7b-chat-hf",
24 | "cache_dir": "/llama",
25 | "prompt_token_cost": 0.0,
26 | "response_token_cost": 0.0,
27 | "temperature": 0.6,
28 | "top_k": 10,
29 | "max_tokens": 4096
30 | },
31 | "llama13b-hf" : {
32 | "model_id": "Llama-2-13b-chat-hf",
33 | "cache_dir": "/llama",
34 | "prompt_token_cost": 0.0,
35 | "response_token_cost": 0.0,
36 | "temperature": 0.6,
37 | "top_k": 10,
38 | "max_tokens": 4096
39 | },
40 | "llama70b-hf" : {
41 | "model_id": "Llama-2-70b-chat-hf",
42 | "cache_dir": "/llama",
43 | "prompt_token_cost": 0.0,
44 | "response_token_cost": 0.0,
45 | "temperature": 0.6,
46 | "top_k": 10,
47 | "max_tokens": 4096
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/graph_of_thoughts/language_models/llamachat_hf.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # main author: Ales Kubicek
8 |
9 | import os
10 | import torch
11 | from typing import List, Dict, Union
12 | from .abstract_language_model import AbstractLanguageModel
13 |
14 |
15 | class Llama2HF(AbstractLanguageModel):
16 | """
17 | An interface to use LLaMA 2 models through the HuggingFace library.
18 | """
19 |
20 | def __init__(
21 | self, config_path: str = "", model_name: str = "llama7b-hf", cache: bool = False
22 | ) -> None:
23 | """
24 | Initialize an instance of the Llama2HF class with configuration, model details, and caching options.
25 |
26 | :param config_path: Path to the configuration file. Defaults to an empty string.
27 | :type config_path: str
28 | :param model_name: Specifies the name of the LLaMA model variant. Defaults to "llama7b-hf".
29 | Used to select the correct configuration.
30 | :type model_name: str
31 | :param cache: Flag to determine whether to cache responses. Defaults to False.
32 | :type cache: bool
33 | """
34 | super().__init__(config_path, model_name, cache)
35 | self.config: Dict = self.config[model_name]
36 | # Detailed id of the used model.
37 | self.model_id: str = self.config["model_id"]
38 | # Costs for 1000 tokens.
39 | self.prompt_token_cost: float = self.config["prompt_token_cost"]
40 | self.response_token_cost: float = self.config["response_token_cost"]
41 | # The temperature is defined as the randomness of the model's output.
42 | self.temperature: float = self.config["temperature"]
43 | # Top K sampling.
44 | self.top_k: int = self.config["top_k"]
45 | # The maximum number of tokens to generate in the chat completion.
46 | self.max_tokens: int = self.config["max_tokens"]
47 |
48 | # Important: must be done before importing transformers
49 | os.environ["TRANSFORMERS_CACHE"] = self.config["cache_dir"]
50 | import transformers
51 |
52 | hf_model_id = f"meta-llama/{self.model_id}"
53 | model_config = transformers.AutoConfig.from_pretrained(hf_model_id)
54 | bnb_config = transformers.BitsAndBytesConfig(
55 | load_in_4bit=True,
56 | bnb_4bit_quant_type="nf4",
57 | bnb_4bit_use_double_quant=True,
58 | bnb_4bit_compute_dtype=torch.bfloat16,
59 | )
60 |
61 | self.tokenizer = transformers.AutoTokenizer.from_pretrained(hf_model_id)
62 | self.model = transformers.AutoModelForCausalLM.from_pretrained(
63 | hf_model_id,
64 | trust_remote_code=True,
65 | config=model_config,
66 | quantization_config=bnb_config,
67 | device_map="auto",
68 | )
69 | self.model.eval()
70 | torch.no_grad()
71 |
72 | self.generate_text = transformers.pipeline(
73 | model=self.model, tokenizer=self.tokenizer, task="text-generation"
74 | )
75 |
76 | def query(self, query: str, num_responses: int = 1) -> List[Dict]:
77 | """
78 | Query the LLaMA 2 model for responses.
79 |
80 | :param query: The query to be posed to the language model.
81 | :type query: str
82 | :param num_responses: Number of desired responses, default is 1.
83 | :type num_responses: int
84 | :return: Response(s) from the LLaMA 2 model.
85 | :rtype: List[Dict]
86 | """
87 | if self.cache and query in self.response_cache:
88 | return self.response_cache[query]
89 | sequences = []
90 | query = f"<>You are a helpful assistant. Always follow the intstructions precisely and output the response exactly in the requested format.<>\n\n[INST] {query} [/INST]"
91 | for _ in range(num_responses):
92 | sequences.extend(
93 | self.generate_text(
94 | query,
95 | do_sample=True,
96 | top_k=self.top_k,
97 | num_return_sequences=1,
98 | eos_token_id=self.tokenizer.eos_token_id,
99 | max_length=self.max_tokens,
100 | )
101 | )
102 | response = [
103 | {"generated_text": sequence["generated_text"][len(query) :].strip()}
104 | for sequence in sequences
105 | ]
106 | if self.cache:
107 | self.response_cache[query] = response
108 | return response
109 |
110 | def get_response_texts(self, query_responses: List[Dict]) -> List[str]:
111 | """
112 | Extract the response texts from the query response.
113 |
114 | :param query_responses: The response list of dictionaries generated from the `query` method.
115 | :type query_responses: List[Dict]
116 | :return: List of response strings.
117 | :rtype: List[str]
118 | """
119 | return [query_response["generated_text"] for query_response in query_responses]
120 |
--------------------------------------------------------------------------------
/graph_of_thoughts/operations/README.md:
--------------------------------------------------------------------------------
1 | # Operations
2 |
3 | The Operations module contains operations to manipulate and process thoughts represented by the [Thought](thought.py) class.
4 | Operations interface with a language model and use other helper classes like [Prompter](../prompter/prompter.py) and [Parser](../parser/parser.py) for effective communication and extraction of results from the language model.
5 | The [Graph of Operations](graph_of_operations.py) class is the main class of the module and is responsible for orchestrating the operations, defining their relationships and maintaining the state of the thought graph, also known as Graph Reasoning State.
6 |
7 | ## Graph of Operations
8 | The [GraphOfOperations](graph_of_operations.py) class facilitates the creation and management of a directed graph representing the sequence and interrelationships of operations on thoughts. Here’s how you can construct and work with the Graph of Operations:
9 |
10 | ### Initialization
11 | Creating a new instance of GraphOfOperations:
12 |
13 | ```python
14 | from graph_of_thoughts.operations import GraphOfOperations
15 |
16 | graph = GraphOfOperations()
17 | ```
18 |
19 | Upon initialization, the graph will be empty with no operations, roots, or leaves.
20 |
21 | ### Adding Operations
22 | **Append Operation:** You can append operations to the end of the graph using the append_operation method. This ensures that the operation becomes a successor to all current leaf operations in the graph.
23 | ```python
24 | from graph_of_thoughts.operations import Generate
25 |
26 | operationA = Generate()
27 | graph.append_operation(operationA)
28 | ```
29 | **Add Operation with Relationships:** If you want to define specific relationships for an operation, use the add_operation method.
30 | ```python
31 | operationB = Generate()
32 | operationB.predecessors.append(operationA)
33 | graph.add_operation(operationB)
34 | ```
35 | Remember to set up the predecessors (and optionally successors) for your operation before adding it to the graph.
36 |
37 | ## Available Operations
38 | The following operations are available in the module:
39 |
40 | **Score:** Collect all thoughts from preceding operations and score them either using the LLM or a custom scoring function.
41 | - num_samples (Optional): The number of samples to use for scoring, defaults to 1.
42 | - combined_scoring (Optional): Whether to score all thoughts together in a single prompt or separately, defaults to False.
43 | - scoring_function (Optional): A function that takes in a list of thought states and returns a list of scores for each thought.
44 |
45 | **ValidateAndImprove:** For each thought, validate it and if it is invalid, improve it.
46 | - num_samples (Optional): The number of samples to use for validation, defaults to 1.
47 | - improve (Optional): Whether to improve the thought if it is invalid, defaults to True.
48 | - num_tries (Optional): The number of times to try improving the thought, before giving up, defaults to 3.
49 | - validate_function (Optional): A function that takes in a thought state and returns a boolean indicating whether the thought is valid.
50 |
51 | **Generate:** Generate new thoughts from the current thoughts. If no previous thoughts are available, the thoughts are initialized with the input to the [Controller](../controller/controller.py).
52 | - num_branches_prompt (Optional): Number of responses that each prompt should generate (passed to prompter). Defaults to 1.
53 | - num_branches_response (Optional): Number of responses the LLM should generate for each prompt. Defaults to 1.
54 |
55 | **Improve:** Improve the current thoughts. This operation is similar to the ValidateAndImprove operation, but it does not validate the thoughts and always tries to improve them.
56 |
57 | **Aggregate:** Aggregate the current thoughts into a single thought. This operation is useful when you want to combine multiple thoughts into a single thought.
58 | - num_responses (Optional): Number of responses to request from the LLM (generates multiple new thoughts). Defaults to 1.
59 |
60 | **KeepBestN:** Keep the best N thoughts from the preceding thoughts. Assumes that the thoughts are already scored and throws an error if they are not.
61 | - n: The number of thoughts to keep in order of score.
62 | - higher_is_better (Optional): Whether higher scores are better (True) or lower scores are better (False). Defaults to True.
63 |
64 | **KeepValid:** Keep only the valid thoughts from the preceding thoughts. Assumes that each thought has already been validated, if not, it will be considered valid.
65 |
66 | **Selector:** Select a number of thoughts from the preceding thoughts using a selection function. This is useful if subsequent operations should only be applied to a subset of the preceding thoughts.
67 | - selector: A function that takes in a list of thoughts and returns a list of thoughts to select.
68 |
69 | **GroundTruth**: Evaluates if the preceding/current thoughts solve the problem and equal the ground truth. This operation is useful for terminating the graph and checking if the final thoughts solve the problem, but is only useful if the ground truth is known.
70 | - ground_truth_evaluator: A function that takes in a thought state and returns a boolean indicating whether the thought solves the problem.
71 |
--------------------------------------------------------------------------------
/graph_of_thoughts/operations/__init__.py:
--------------------------------------------------------------------------------
1 | from .thought import Thought
2 | from .graph_of_operations import GraphOfOperations
3 | from .operations import (
4 | Operation,
5 | Score,
6 | ValidateAndImprove,
7 | Generate,
8 | Aggregate,
9 | KeepBestN,
10 | KeepValid,
11 | Selector,
12 | GroundTruth,
13 | Improve,
14 | )
15 |
--------------------------------------------------------------------------------
/graph_of_thoughts/operations/graph_of_operations.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # main author: Nils Blach
8 |
9 | from __future__ import annotations
10 | from typing import List
11 |
12 | from graph_of_thoughts.operations.operations import Operation
13 |
14 |
15 | class GraphOfOperations:
16 | """
17 | Represents the Graph of Operations, which prescribes the execution plan of thought operations.
18 | """
19 |
20 | def __init__(self) -> None:
21 | """
22 | Initializes a new Graph of Operations instance with empty operations, roots, and leaves.
23 | The roots are the entry points in the graph with no predecessors.
24 | The leaves are the exit points in the graph with no successors.
25 | """
26 | self.operations: List[Operation] = []
27 | self.roots: List[Operation] = []
28 | self.leaves: List[Operation] = []
29 |
30 | def append_operation(self, operation: Operation) -> None:
31 | """
32 | Appends an operation to all leaves in the graph and updates the relationships.
33 |
34 | :param operation: The operation to append.
35 | :type operation: Operation
36 | """
37 | self.operations.append(operation)
38 |
39 | if len(self.roots) == 0:
40 | self.roots = [operation]
41 | else:
42 | for leave in self.leaves:
43 | leave.add_successor(operation)
44 |
45 | self.leaves = [operation]
46 |
47 | def add_operation(self, operation: Operation) -> None:
48 | """
49 | Add an operation to the graph considering its predecessors and successors.
50 | Adjust roots and leaves based on the added operation's position within the graph.
51 |
52 | :param operation: The operation to add.
53 | :type operation: Operation
54 | """
55 | self.operations.append(operation)
56 | if len(self.roots) == 0:
57 | self.roots = [operation]
58 | self.leaves = [operation]
59 | assert (
60 | len(operation.predecessors) == 0
61 | ), "First operation should have no predecessors"
62 | else:
63 | if len(operation.predecessors) == 0:
64 | self.roots.append(operation)
65 | for predecessor in operation.predecessors:
66 | if predecessor in self.leaves:
67 | self.leaves.remove(predecessor)
68 | if len(operation.successors) == 0:
69 | self.leaves.append(operation)
70 |
--------------------------------------------------------------------------------
/graph_of_thoughts/operations/thought.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # main author: Nils Blach
8 |
9 | from __future__ import annotations
10 | import logging
11 | from typing import Iterator, Dict, Optional
12 | import itertools
13 |
14 |
15 | class Thought:
16 | """
17 | Represents an LLM thought with its state, constructed by the parser, and various flags.
18 | """
19 |
20 | _ids: Iterator[int] = itertools.count(0)
21 |
22 | def __init__(self, state: Optional[Dict] = None) -> None:
23 | """
24 | Initializes a new Thought instance with a state and various default flags.
25 |
26 | :param state: The state of the thought. Defaults to None.
27 | :type state: Optional[Dict]
28 | """
29 | self.logger: logging.Logger = logging.getLogger(self.__class__.__name__)
30 | self.id: int = next(Thought._ids)
31 | self.state: Dict = state
32 | self._score: float = 0.0
33 | self._valid: bool = False
34 | self._solved: bool = False
35 | self.scored: bool = False
36 | self.validated: bool = False
37 | self.compared_to_ground_truth: bool = False
38 |
39 | @staticmethod
40 | def from_thought(thought: Thought) -> Thought:
41 | """
42 | Creates a new thought from an existing one.
43 |
44 | :param thought: An instance of a Thought to clone.
45 | :return: A new Thought instance with properties copied from the input thought.
46 | """
47 | new_thought = Thought(thought.state)
48 | new_thought.score = thought.score
49 | new_thought.valid = thought.valid
50 | new_thought.solved = thought.solved
51 | new_thought.scored = thought.scored
52 | new_thought.validated = thought.validated
53 | new_thought.compared_to_ground_truth = thought.compared_to_ground_truth
54 | return new_thought
55 |
56 | @property
57 | def valid(self) -> bool:
58 | """
59 | Returns the validity of the thought.
60 |
61 | :return: The validity of the thought.
62 | :rtype: bool
63 | """
64 | return self._valid
65 |
66 | @valid.setter
67 | def valid(self, valid: bool) -> None:
68 | """
69 | Sets the validity of the thought and the validated flag.
70 |
71 | :param valid: The validity of the thought.
72 | :type valid: bool
73 | """
74 | self.validated = True
75 | self._valid = valid
76 |
77 | @property
78 | def score(self) -> float:
79 | """
80 | Returns the score of the thought.
81 |
82 | :return: The score of the thought.
83 | :rtype: float
84 | """
85 | return self._score
86 |
87 | @score.setter
88 | def score(self, new_score: float) -> None:
89 | """
90 | Sets the score of the thought and the scored flag.
91 |
92 | :param new_score: The score of the thought.
93 | :type new_score: float
94 | """
95 | self.scored = True
96 | self._score = new_score
97 |
98 | @property
99 | def solved(self) -> bool:
100 | """
101 | Returns the solved flag of the thought.
102 |
103 | :return: The solved flag of the thought.
104 | :rtype: bool
105 | """
106 | return self._solved
107 |
108 | @solved.setter
109 | def solved(self, solved: bool) -> None:
110 | """
111 | Sets the solved flag of the thought and the compared_to_ground_truth flag.
112 |
113 | :param solved: Whether the thought contains a solution to the problem.
114 | :type solved: bool
115 | """
116 | self.compared_to_ground_truth = True
117 | self._solved = solved
118 |
--------------------------------------------------------------------------------
/graph_of_thoughts/parser/__init__.py:
--------------------------------------------------------------------------------
1 | from .parser import Parser
2 |
--------------------------------------------------------------------------------
/graph_of_thoughts/parser/parser.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # main authors: Robert Gerstenberger, Nils Blach
8 |
9 | from __future__ import annotations
10 | from abc import ABC, abstractmethod
11 | from typing import Dict, List, Union
12 |
13 |
14 | class Parser(ABC):
15 | """
16 | Abstract base class that defines the interface for all parsers.
17 | Parsers are used to parse the responses from the language models.
18 | """
19 |
20 | @abstractmethod
21 | def parse_aggregation_answer(
22 | self, states: List[Dict], texts: List[str]
23 | ) -> Union[Dict, List[Dict]]:
24 | """
25 | Parse the response from the language model for a aggregation prompt.
26 |
27 | :param states: The thought states used to generate the prompt.
28 | :type states: List[Dict]
29 | :param texts: The responses to the prompt from the language model.
30 | :type texts: List[str]
31 | :return: The new thought states after parsing the response from the language model.
32 | :rtype: Union[Dict, List[Dict]]
33 | """
34 | pass
35 |
36 | @abstractmethod
37 | def parse_improve_answer(self, state: Dict, texts: List[str]) -> Dict:
38 | """
39 | Parse the response from the language model for an improve prompt.
40 |
41 | :param state: The thought state used to generate the prompt.
42 | :type state: Dict
43 | :param texts: The responses to the prompt from the language model.
44 | :type texts: List[str]
45 | :return: The new thought state after parsing the response from the language model.
46 | :rtype: Dict
47 | """
48 | pass
49 |
50 | @abstractmethod
51 | def parse_generate_answer(self, state: Dict, texts: List[str]) -> List[Dict]:
52 | """
53 | Parse the response from the language model for a generate prompt.
54 |
55 | :param state: The thought state used to generate the prompt.
56 | :type state: Dict
57 | :param texts: The responses to the prompt from the language model.
58 | :type texts: List[str]
59 | :return: The new thought states after parsing the response from the language model.
60 | :rtype: List[Dict]
61 | """
62 | pass
63 |
64 | @abstractmethod
65 | def parse_validation_answer(self, state: Dict, texts: List[str]) -> bool:
66 | """
67 | Parse the response from the language model for a validation prompt.
68 |
69 | :param state: The thought state used to generate the prompt.
70 | :type state: Dict
71 | :param texts: The responses to the prompt from the language model.
72 | :type texts: List[str]
73 | :return: Whether the thought state is valid or not.
74 | :rtype: bool
75 | """
76 | pass
77 |
78 | @abstractmethod
79 | def parse_score_answer(self, states: List[Dict], texts: List[str]) -> List[float]:
80 | """
81 | Parse the response from the language model for a score prompt.
82 |
83 | :param states: The thought states used to generate the prompt.
84 | :type states: List[Dict]
85 | :param texts: The responses to the prompt from the language model.
86 | :type texts: List[str]
87 | :return: The scores for the thought states.
88 | :rtype: List[float]
89 | """
90 | pass
91 |
--------------------------------------------------------------------------------
/graph_of_thoughts/prompter/__init__.py:
--------------------------------------------------------------------------------
1 | from .prompter import Prompter
2 |
--------------------------------------------------------------------------------
/graph_of_thoughts/prompter/prompter.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # main authors: Robert Gerstenberger, Nils Blach
8 |
9 | from __future__ import annotations
10 | from abc import ABC, abstractmethod
11 | from typing import Dict, List
12 |
13 |
14 | class Prompter(ABC):
15 | """
16 | Abstract base class that defines the interface for all prompters.
17 | Prompters are used to generate the prompts for the language models.
18 | """
19 |
20 | @abstractmethod
21 | def aggregation_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
22 | """
23 | Generate a aggregation prompt for the language model.
24 |
25 | :param state_dicts: The thought states that should be aggregated.
26 | :type state_dicts: List[Dict]
27 | :param kwargs: Additional keyword arguments.
28 | :return: The aggregation prompt.
29 | :rtype: str
30 | """
31 | pass
32 |
33 | @abstractmethod
34 | def improve_prompt(self, **kwargs) -> str:
35 | """
36 | Generate an improve prompt for the language model.
37 | The thought state is unpacked to allow for additional keyword arguments
38 | and concrete implementations to specify required arguments explicitly.
39 |
40 | :param kwargs: Additional keyword arguments.
41 | :return: The improve prompt.
42 | :rtype: str
43 | """
44 | pass
45 |
46 | @abstractmethod
47 | def generate_prompt(self, num_branches: int, **kwargs) -> str:
48 | """
49 | Generate a generate prompt for the language model.
50 | The thought state is unpacked to allow for additional keyword arguments
51 | and concrete implementations to specify required arguments explicitly.
52 |
53 | :param num_branches: The number of responses the prompt should ask the LM to generate.
54 | :type num_branches: int
55 | :param kwargs: Additional keyword arguments.
56 | :return: The generate prompt.
57 | :rtype: str
58 | """
59 | pass
60 |
61 | @abstractmethod
62 | def validation_prompt(self, **kwargs) -> str:
63 | """
64 | Generate a validation prompt for the language model.
65 | The thought state is unpacked to allow for additional keyword arguments
66 | and concrete implementations to specify required arguments explicitly.
67 |
68 | :param kwargs: Additional keyword arguments.
69 | :return: The validation prompt.
70 | :rtype: str
71 | """
72 | pass
73 |
74 | @abstractmethod
75 | def score_prompt(self, state_dicts: List[Dict], **kwargs) -> str:
76 | """
77 | Generate a score prompt for the language model.
78 |
79 | :param state_dicts: The thought states that should be scored,
80 | if more than one, they should be scored together.
81 | :type state_dicts: List[Dict]
82 | :param kwargs: Additional keyword arguments.
83 | :return: The score prompt.
84 | :rtype: str
85 | """
86 | pass
87 |
--------------------------------------------------------------------------------
/paper/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | ## Poster
6 | The poster presented at the 2024 Association for the Advancement of Artificial Intelligence (AAAI'24) conference is available as [PDF](poster.pdf) and [PNG](poster.png).
7 |
8 | ## Plot Data
9 |
10 | The data used to create the figures of the paper can be
11 | found in the `final_results_gpt35.tar.bz2` archive. Unpack the archive
12 | and run the file `plots.py`.
13 |
--------------------------------------------------------------------------------
/paper/final_results_gpt35.tar.bz2:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/spcl/graph-of-thoughts/363421c61c7bc11edf32845a697ae2aaccd75463/paper/final_results_gpt35.tar.bz2
--------------------------------------------------------------------------------
/paper/plots.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2023 ETH Zurich.
2 | # All rights reserved.
3 | #
4 | # Use of this source code is governed by a BSD-style license that can be
5 | # found in the LICENSE file.
6 | #
7 | # main author: Nils Blach
8 | # contributions: Robert Gerstenberger
9 |
10 | import json
11 | import os
12 | import matplotlib.pyplot as plt
13 |
14 |
15 | def get_complete_results(base_directory):
16 | results_complete = {}
17 | for folder_name in os.listdir(base_directory):
18 | folder_path = os.path.join(base_directory, folder_name)
19 | if os.path.isdir(folder_path):
20 | results_complete[folder_name] = []
21 | for file_name in os.listdir(folder_path):
22 | if file_name.endswith(".json"):
23 | file_path = os.path.join(folder_path, file_name)
24 | with open(file_path, "r") as f:
25 | data = json.load(f)
26 | results_complete[folder_name].append(
27 | {"key": int(file_name.split(".")[0]), "data": data}
28 | )
29 | for key in results_complete.keys():
30 | results_complete[key] = sorted(
31 | results_complete[key], key=lambda x: x["key"]
32 | )
33 | return results_complete
34 |
35 |
36 | def get_final_scores(results_complete):
37 | scores = {}
38 | for method in results_complete.keys():
39 | scores[method] = []
40 | for result in results_complete[method]:
41 | score = 100
42 | solved = False
43 | cost = 1
44 | prompt_tokens = 0
45 | completion_tokens = 0
46 | for op in result["data"]:
47 | if "operation" in op and op["operation"] == "ground_truth_evaluator":
48 | try:
49 | score = min(op["scores"])
50 | solved = any(op["problem_solved"])
51 | except:
52 | continue
53 | if "cost" in op:
54 | cost = op["cost"]
55 | prompt_tokens = op["prompt_tokens"]
56 | completion_tokens = op["completion_tokens"]
57 | scores[method].append(
58 | [result["key"], score, solved, prompt_tokens, completion_tokens, cost]
59 | )
60 | scores[method] = sorted(scores[method], key=lambda x: x[0])
61 | return scores
62 |
63 |
64 | def get_final_scores_doc_merge(results_complete):
65 | scores = {}
66 | for method in results_complete.keys():
67 | scores[method] = []
68 | for result in results_complete[method]:
69 | score = 0
70 | solved = False
71 | cost = 1
72 | prompt_tokens = 0
73 | completion_tokens = 0
74 | for op in reversed(result["data"]):
75 | if "cost" in op:
76 | cost = op["cost"]
77 | prompt_tokens = op["prompt_tokens"]
78 | completion_tokens = op["completion_tokens"]
79 | if "operation" in op and op["operation"] == "score":
80 | try:
81 | score = max(op["scores"])
82 | break
83 | except:
84 | continue
85 | scores[method].append(
86 | [result["key"], score, solved, prompt_tokens, completion_tokens, cost]
87 | )
88 | scores[method] = sorted(scores[method], key=lambda x: x[0])
89 | return scores
90 |
91 |
92 | def get_plotting_data(base_directory, score_method):
93 | results_complete = get_complete_results(base_directory)
94 | scores = score_method(results_complete)
95 | results_plotting = {
96 | method: {
97 | "scores": [x[1] for x in scores[method]],
98 | "solved": sum([1 for x in scores[method] if x[2]]),
99 | "costs": [x[5] for x in scores[method]],
100 | }
101 | for method in scores.keys()
102 | }
103 | return results_plotting
104 |
105 |
106 | def plot_results(
107 | name,
108 | results,
109 | methods_order=["io", "cot", "tot", "tot2", "tog"],
110 | methods_labels=["IO", "CoT", "ToT", "ToT2", "GoT"],
111 | model="GPT-3.5",
112 | length=32,
113 | y_lower=0,
114 | y_upper=16,
115 | cost_upper=1.8,
116 | display_solved=True,
117 | annotation_offset=1,
118 | display_left_ylabel=False,
119 | display_right_ylabel=False,
120 | ):
121 | methods_order = [method for method in methods_order if method in results]
122 | # Extract scores based on the order
123 | if name == "set_intersection":
124 | scores_ordered = [
125 | [min(score, length) for score in results[method]["scores"] if score != 1000]
126 | for method in methods_order
127 | ]
128 | elif name == "sorting":
129 | scores_ordered = [
130 | [
131 | min(score, length)
132 | for score in results[method]["scores"]
133 | if score != 100 and score != 300
134 | ]
135 | for method in methods_order
136 | ]
137 | elif name == "keyword_counting":
138 | scores_ordered = [
139 | [
140 | score
141 | for score in results[method]["scores"]
142 | if score != 100 and score != 300
143 | ]
144 | for method in methods_order
145 | ]
146 | elif name == "document_merging":
147 | scores_ordered = [
148 | [score for score in results[method]["scores"]] for method in methods_order
149 | ]
150 | total_costs = [sum(results[method]["costs"]) for method in methods_order]
151 |
152 | # Create figure and axis
153 | if name == "keyword_counting" or name == "document_merging":
154 | fig, ax = plt.subplots(dpi=150, figsize=(3.75, 5))
155 | else:
156 | fig, ax = plt.subplots(dpi=150, figsize=(2.5, 5))
157 |
158 | # Create boxplots
159 | positions = range(1, len(methods_order) + 1)
160 | ax.boxplot(scores_ordered, positions=positions)
161 |
162 | fig_fontsize = 12
163 |
164 | # Set the ticks and labels
165 | plt.yticks(fontsize=fig_fontsize)
166 | ax.set_xticks(range(1, len(methods_order) + 1))
167 | ax.set_xticks(range(1, len(methods_order) + 1))
168 | if name == "keyword_counting":
169 | ax.set_xticklabels(methods_labels, fontsize=10)
170 | else:
171 | ax.set_xticklabels(methods_labels, fontsize=fig_fontsize)
172 |
173 | if name == "document_merging":
174 | ax.set_ylim(y_lower, 12 if display_solved else 9.75)
175 | else:
176 | ax.set_ylim(y_lower, (y_upper + 2) if display_solved else y_upper + 1)
177 |
178 | if name == "sorting" or name == "set_intersection":
179 | ax1_yticks = range(
180 | y_lower, y_upper + 1, 2 if length < 48 else (4 if length < 96 else 8)
181 | )
182 | ax.set_yticks(ax1_yticks)
183 |
184 | if display_left_ylabel:
185 | if name == "keyword_counting":
186 | ax.set_ylabel(
187 | f"Number of errors; the lower the better", fontsize=fig_fontsize
188 | )
189 | elif name == "document_merging":
190 | ax.set_ylabel(
191 | f"Score (out of 10); the higher the better", fontsize=fig_fontsize
192 | )
193 | else:
194 | ax.set_ylabel(
195 | f"#incorrect elements; the lower the better", fontsize=fig_fontsize
196 | )
197 |
198 | if name == "sorting" or name == "set_intersection":
199 | ax.set_title(f"{length} elements")
200 |
201 | ax2 = ax.twinx()
202 | ax2.bar(positions, total_costs, alpha=0.5, color="blue", label="Total Cost ($)")
203 | ax2.yaxis.set_tick_params(colors="#1919ff", labelsize=fig_fontsize)
204 | ax2.set_ylim(0, cost_upper)
205 | number_of_ticks = len(ax.get_yticks())
206 | tick_interval = cost_upper / (number_of_ticks)
207 | ax2_ticks = [tick_interval * i for i in range(number_of_ticks)]
208 |
209 | # Set custom tick positions for ax2
210 | ax2.set_yticks(ax2_ticks)
211 |
212 | if display_right_ylabel:
213 | ax2.set_ylabel(
214 | "Total Cost ($); the lower the better",
215 | color="#1919ff",
216 | fontsize=fig_fontsize,
217 | )
218 |
219 | if display_solved:
220 | annotation_height = y_upper + annotation_offset
221 | count = 1
222 | for method in methods_order:
223 | if method not in results:
224 | continue
225 | solved = results[method]["solved"]
226 | ax.text(
227 | count,
228 | annotation_height,
229 | f"{solved}",
230 | ha="center",
231 | va="bottom",
232 | fontsize=fig_fontsize,
233 | )
234 | count += 1
235 |
236 | model = model.replace(".", "").replace("-", "").lower()
237 | if name == "keyword_counting" or name == "document_merging":
238 | fig.savefig(f"{name}_{model}.pdf", bbox_inches="tight")
239 | else:
240 | fig.savefig(f"{name}_{model}_{length}.pdf", bbox_inches="tight")
241 |
242 |
243 | plot_results(
244 | "set_intersection",
245 | get_plotting_data("set_intersection_gpt35_032", get_final_scores),
246 | methods_order=["io", "cot", "tot", "tot2", "tog2"],
247 | length=32,
248 | y_upper=19,
249 | cost_upper=2,
250 | display_solved=True,
251 | annotation_offset=0.5,
252 | display_left_ylabel=True,
253 | display_right_ylabel=True,
254 | )
255 |
256 | plot_results(
257 | "set_intersection",
258 | get_plotting_data("set_intersection_gpt35_064", get_final_scores),
259 | methods_order=["io", "cot", "tot", "tot2", "tog2"],
260 | length=64,
261 | y_upper=32,
262 | cost_upper=5.4,
263 | display_solved=True,
264 | annotation_offset=0.2,
265 | display_left_ylabel=True,
266 | display_right_ylabel=True,
267 | )
268 |
269 | plot_results(
270 | "set_intersection",
271 | get_plotting_data("set_intersection_gpt35_128", get_final_scores),
272 | methods_order=["io", "cot", "tot", "tot2", "tog2"],
273 | length=128,
274 | y_upper=94,
275 | cost_upper=12,
276 | display_solved=True,
277 | annotation_offset=-3,
278 | display_left_ylabel=True,
279 | display_right_ylabel=True,
280 | )
281 |
282 | plot_results(
283 | "sorting",
284 | get_plotting_data("sorting_gpt35_032", get_final_scores),
285 | length=32,
286 | display_solved=False,
287 | annotation_offset=0.5,
288 | display_left_ylabel=True,
289 | display_right_ylabel=True,
290 | )
291 |
292 | plot_results(
293 | "sorting",
294 | get_plotting_data("sorting_gpt35_064", get_final_scores),
295 | length=64,
296 | y_upper=64,
297 | cost_upper=5.1,
298 | display_solved=False,
299 | display_left_ylabel=True,
300 | display_right_ylabel=True,
301 | )
302 |
303 | plot_results(
304 | "sorting",
305 | get_plotting_data("sorting_gpt35_128", get_final_scores),
306 | length=128,
307 | y_upper=128,
308 | cost_upper=17,
309 | display_solved=False,
310 | display_left_ylabel=True,
311 | display_right_ylabel=True,
312 | )
313 |
314 | plot_results(
315 | "keyword_counting",
316 | get_plotting_data("keyword_counting_gpt35", get_final_scores),
317 | methods_order=["io", "cot", "tot", "tot2", "gsp4", "gsp8", "gspx"],
318 | methods_labels=["IO", "CoT", "ToT", "ToT2", "GoT4", "GoT8", "GoTx"],
319 | y_upper=35,
320 | cost_upper=9,
321 | display_solved=True,
322 | annotation_offset=-0.3,
323 | display_left_ylabel=True,
324 | display_right_ylabel=True,
325 | )
326 |
327 | plot_results(
328 | "document_merging",
329 | get_plotting_data("document_merging_gpt35_16k", get_final_scores_doc_merge),
330 | methods_order=["io", "cot", "tot", "gsp", "gsp2"],
331 | methods_labels=["IO", "CoT", "ToT", "GoT", "GoT2"],
332 | y_upper=10,
333 | cost_upper=15,
334 | display_solved=False,
335 | display_left_ylabel=True,
336 | display_right_ylabel=True,
337 | )
338 |
--------------------------------------------------------------------------------
/paper/poster.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/spcl/graph-of-thoughts/363421c61c7bc11edf32845a697ae2aaccd75463/paper/poster.pdf
--------------------------------------------------------------------------------
/paper/poster.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/spcl/graph-of-thoughts/363421c61c7bc11edf32845a697ae2aaccd75463/paper/poster.png
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | [build-system]
2 | requires = ["hatchling"]
3 | build-backend = "hatchling.build"
4 |
5 | [project]
6 | name = "graph_of_thoughts"
7 | version = "0.0.3"
8 | authors = [
9 | { name="Maciej Besta", email="maciej.besta@inf.ethz.ch" },
10 | { name="Nils Blach", email="nils.blach@inf.ethz.ch" },
11 | { name="Ales Kubicek", email="akubicek@student.ethz.ch" },
12 | { name="Robert Gerstenberger", email="gerstenberger.robert@gmail.com" },
13 | ]
14 | description = "Python package for Graph of Thoughts that enables solving elaborate problems with Large Language Models"
15 | readme = "README.md"
16 | license = {file = "LICENSE"}
17 | requires-python = ">=3.8"
18 | classifiers = [
19 | "Programming Language :: Python :: 3",
20 | "Operating System :: OS Independent",
21 | ]
22 | dependencies = [
23 | "backoff>=2.2.1,<3.0.0",
24 | "openai>=1.0.0,<2.0.0",
25 | "matplotlib>=3.7.1,<4.0.0",
26 | "numpy>=1.24.3,<2.0.0",
27 | "pandas>=2.0.3,<3.0.0",
28 | "sympy>=1.12,<2.0",
29 | "torch>=2.0.1,<3.0.0",
30 | "transformers>=4.31.0,<5.0.0",
31 | "accelerate>=0.21.0,<1.0.0",
32 | "bitsandbytes>=0.41.0,<1.0.0",
33 | "scipy>=1.10.1,<2.0.0",
34 | ]
35 |
36 | [project.urls]
37 | Homepage = "https://github.com/spcl/graph-of-thoughts"
38 |
39 | [project.scripts]
40 |
--------------------------------------------------------------------------------