15 |
14 |
15 |
22 | 
23 |
32 |
33 |
36 |
37 | ## Leaderboard
38 |
39 | I was able to solve the big puzzle using roughly 2000 tokens (input/output). Can you do better? I'll post the best ones here.
40 |
41 | ## Other Puzzles
42 |
43 | If you find this fun also check out.
44 |
45 | * https://github.com/srush/GPU-Puzzles
46 |
--------------------------------------------------------------------------------
/app.py:
--------------------------------------------------------------------------------
1 | import os
2 | import sys
3 | import gradio as gr
4 | from dataclasses import dataclass
5 | from chalk import *
6 | from colour import Color
7 | import inspect
8 | import os
9 | import openai
10 | from typing import List, Tuple, Optional
11 | from enum import Enum
12 | import io
13 | from contextlib import redirect_stdout
14 | import imageio
15 | import tiktoken
16 | import time
17 | import pandas as pd
18 | import csv
19 | from huggingface_hub import HfApi, Repository
20 | DATASET_REPO_URL = "https://huggingface.co/datasets/srush/gptworld-leaderboard"
21 | HF_TOKEN = os.environ.get("HF_API")
22 | DATA_FILENAME = "data.csv"
23 | DATA_FILE = os.path.join("data", DATA_FILENAME)
24 |
25 | openai.api_key = ""
26 | tab = " "
27 | repo = Repository(git_user="srush",
28 | local_dir="data", clone_from=DATASET_REPO_URL, use_auth_token=HF_TOKEN
29 | )
30 | repo.git_pull()
31 | def start2(prompt, board, api_key):
32 | out = ""
33 | # for chunk in openai.ChatCompletion.create(
34 | # model="gpt-4",
35 | # messages=[{
36 | # "role": "user",
37 | # "content": prompt,
38 |
39 | # }],
40 | # stream=True,
41 | # temperature= 0
42 | # ):
43 | board = board#Game(boundary=(9, 9), key=(1, 1), flag=(2, 2), init=(0, 0), walls=[(2, 0)])
44 | actions = [Actions.DOWNRIGHT, Actions.RIGHT, Actions.DOWNRIGHT, Actions.PICKUP, Actions.DOWNRIGHT]
45 | contents = example(board, actions)
46 | print(contents)
47 | # encoding = tiktoken.encoding_for_model("gpt-4")
48 | # num_tokens = encoding.encode(string)
49 |
50 | for content in contents:
51 | time.sleep(0.005)
52 | content = content
53 | if content is not None:
54 | out += content
55 | print(content, end="")
56 | yield out
57 | yield out
58 |
59 | def start(prompt, board, api_key):
60 | out = ""
61 | # encoding = tiktoken.encoding_for_model("gpt-4")
62 | # num_tokens = encoding.encode(string)
63 | content = ""
64 | openai.api_key = api_key
65 | for chunk in openai.ChatCompletion.create(
66 | model="gpt-4",
67 | messages=[{
68 | "role": "user",
69 | "content": prompt,
70 |
71 | }],
72 | stream=True,
73 | temperature= 0
74 | ):
75 |
76 | # for content in contents:
77 | time.sleep(0.005)
78 | content = chunk["choices"][0].get("delta", {}).get("content")
79 | if content is not None:
80 | out += content
81 | print(content, end="")
82 | yield out
83 | yield out
84 |
85 | def num_tokens_from_string(string: str, encoding_name: str="gpt-4") -> int:
86 | """Returns the number of tokens in a text string."""
87 | encoding = tiktoken.encoding_for_model(encoding_name)
88 | num_tokens = len(encoding.encode(string))
89 | return num_tokens
90 |
91 |
92 | # + [markdown] id="LMTjwXdD7v-I"
93 | # ## Game Code
94 | #
95 | # This code creates a mini-game to play. It takes place on a hexagon. You are represented by a circle. You need to first pick up a key represented by a triangle. You finally need to make it to the cross to finish the game. The actions show each of the directions you can move.
96 | #
97 | #
98 |
99 | # + id="Fv3eTRKiV2ZB" cellView="form"
100 | #@title Game Code
101 |
102 | # Possible Actions
103 | class Actions(Enum):
104 | UPRIGHT = "UR"
105 | RIGHT = "R"
106 | DOWNRIGHT = "DR"
107 | DOWNLEFT = "DL"
108 | LEFT = "L"
109 | UPLEFT = "UL"
110 | PICKUP = "Pickup"
111 |
112 | # Movements
113 | change = {
114 | Actions.UPRIGHT : (-1, 1),
115 | Actions.RIGHT : (0, 2),
116 | Actions.DOWNRIGHT : (1, 1),
117 | Actions.DOWNLEFT : (1, -1),
118 | Actions.LEFT : (0, -2),
119 | Actions.UPLEFT : (-1, -1),
120 | Actions.PICKUP : (0, 0),
121 | }
122 | change_str = {action.value: change[action] for action in Actions}
123 | def add(a, b):
124 | return a[0] + b[0], a[1] + b[1]
125 |
126 | @dataclass
127 | class Board:
128 | grid: List[str]
129 | player_pos: Tuple[int, int]
130 | flag_pos: Tuple[int, int]
131 | wall_pos:List[Tuple[int, int]]
132 | key_pos:Optional[Tuple[int, int]]
133 |
134 | def move(self, action: Actions) -> 'Board':
135 | "Move by creating a new board."
136 | d_m = change[action]
137 | if action == Actions.PICKUP:
138 | if self.player_pos == self.key_pos:
139 | return Board(self.grid, self.player_pos, self.flag_pos, self.wall_pos, None)
140 | else:
141 | return self
142 |
143 | new_player_pos = add(self.player_pos, d_m)
144 | # Out of bounds
145 | if new_player_pos[0] < 0 or new_player_pos[0] >= len(self.grid):
146 | return self
147 | if new_player_pos[1] < 0 or new_player_pos[1] >= len(self.grid[0]):
148 | return self
149 | # Can't move through walls
150 | if self.grid[new_player_pos[0]][new_player_pos[1]] == 'W':
151 | return self
152 |
153 | new_grid = [row[:] for row in self.grid] # Create a copy of the grid
154 | new_grid[self.player_pos[0]][self.player_pos[1]] = '.'
155 | new_grid[new_player_pos[0]][new_player_pos[1]] = '@'
156 | return Board(new_grid, new_player_pos, self.flag_pos, self.wall_pos, self.key_pos)
157 |
158 | def __str__(self) -> str:
159 | return '\n'.join(''.join(row) for i, row in enumerate(self.grid))
160 |
161 | @classmethod
162 | def create_empty_board(cls, size: Tuple[int, int], key_pos, flag_pos, init, wall_pos) -> 'Board':
163 | grid = [['.' if i % 2 == j % 2 else " " for i in range(size[1])] for j in range(size[0])]
164 | player_pos = init
165 | flag_pos = flag_pos
166 | grid[player_pos[0]][player_pos[1]] = '@'
167 | grid[flag_pos[0]][flag_pos[1]] = 'P'
168 | grid[key_pos[0]][key_pos[1]] = 'K'
169 | for pos in wall_pos:
170 | grid[pos[0]][pos[1]] = 'W'
171 | return cls(grid, player_pos, flag_pos, wall_pos, key_pos)
172 |
173 | class Game:
174 | def __init__(self, init, flag, walls, key, boundary):
175 | "Create the version of the game that the AI sees."
176 | self.boundary = boundary
177 | self.board = Board.create_empty_board(boundary, key, flag, init, walls)
178 | self.original = self.board
179 | self.actions = []
180 |
181 | def move(self, action):
182 | self.board = self.board.move(action)
183 | self.actions.append(action)
184 |
185 | @property
186 | def walls(self):
187 | return self.board.wall_pos
188 |
189 | def won(self):
190 | final = self.board
191 | return final.key_pos is None and final.player_pos == final.flag_pos
192 |
193 | def __repr__(self) -> str:
194 | walls = ",".join(map(str, self.board.wall_pos))
195 | return f"Game(init={self.board.player_pos}, flag={self.board.flag_pos}, walls={self.board.wall_pos}, boundary={self.boundary}, key={self.board.key_pos})"
196 |
197 | # This is the version of move that the AI can see.
198 | def move(game, action, old_pos):
199 | # ACTIONS (must be legal)
200 | game.move(Actions(action))
201 | offset = change_str[action]
202 | pos = (old_pos[0] + offset[0], old_pos[1] + offset[1])
203 |
204 | assert 0 <= pos[0] < game.boundary[0], "Row position out of bounds"
205 | assert 0 <= pos[1] < game.boundary[1], "Col position out of bounds"
206 | assert pos not in game.walls, f"Walked into wall {pos}"
207 | if action == "PU":
208 | assert pos == game.key, f"Not over key"
209 | return pos
210 |
211 |
212 | # + [markdown] id="PDOcPiQq8u_Y"
213 | # We can look at the board by drawing it.
214 |
215 | # + colab={"base_uri": "https://localhost:8080/", "height": 221} id="Ic7WgOTi8uF1" outputId="4dc07cb9-9e5f-4d28-d4ea-470ad4b13141"
216 | #@title Drawing code
217 | def draw_board(grid, num=0):
218 | hex = regular_polygon(6, 1).rotate_by(1/12).line_width(0.5).fill_color(Color("white"))
219 | w = hex.get_envelope().width
220 | canvas = empty()
221 | for r, b in enumerate(grid):
222 | def show(v):
223 | if v == ".":
224 | return hex
225 | if v == "@":
226 | return hex + circle(0.35).fill_color(Color("red"))
227 | if v == "P":
228 | x = rectangle(0.25, 0.7).fill_color(Color("blue")).line_width(0)
229 | return hex + (x.rotate_by(0.25/2) + x.rotate_by(-0.25/2))
230 | if v == "K":
231 | return hex + triangle(0.75).fill_color(Color("purple"))
232 | if v == "W":
233 | return hex.fill_color(Color("black"))
234 | if v ==" ":
235 | return hex
236 | row = hcat(show(v) for i, v in enumerate(b[1 if r %2 else 0::2]))
237 | canvas += row.translate(w * 0.5 if r%2 else 0, 1.5 * r)
238 | canvas = canvas.center_xy().frame(0.5)
239 | canvas = rectangle(canvas.get_envelope().width, canvas.get_envelope().height).line_width(0.5).fill_color(Color("orange")) + canvas
240 | canvas.render_svg(f"pic{num}.svg", 256)
241 | canvas.render(f"pic{num}.png", 256)
242 | return canvas
243 |
244 |
245 |
246 | # + colab={"base_uri": "https://localhost:8080/", "height": 424} id="nqgPKLu0AMhU" outputId="19e4c6d0-b792-4a34-f4c4-81902974c346"
247 | # game = Game(boundary=(5, 5), key=(0, 2), flag=(4, 4), init=(0, 0), walls=[(2, 2)])
248 | # display(draw_board(game.board.grid))
249 | # move(game, "DR", (0,0))
250 | # display(draw_board(game.board.grid))
251 |
252 |
253 | # + [markdown] id="PhqF9af5_jvh"
254 | # ## Prompt Code
255 | #
256 | # The puzzle is to write prompt code to make the model accomplish this task. We have provided some scaffolding code for you. The code creates:
257 | #
258 | # * A header for describing the game.
259 | # * A function `make_fun` that shows the AI how to move in code.
260 | # * A footer to describe the final game board that you want the mode to solve.
261 | #
262 | # You can fill this in a watch how the model moves around.
263 |
264 | # + id="jFf7TCOJaVHX"
265 | #@title Make the Prompt
266 | def make_fun(board, actions):
267 | "This function generates python code for few-shot examples"
268 | out = tab + "p = " + str(board.player_pos)
269 | for i, action in enumerate(actions):
270 | new_board = board.move(action)
271 | out += f"""
272 | # TODO ADD CODE
273 | p = move(b, "{action.value}", p) # TODO ADD CODE"""
274 | board = new_board
275 | return out
276 |
277 | def example(game, actions):
278 | """
279 | This code makes a few shot example. You don't need to edit it.
280 | """
281 | return f"""
282 | def my_example():
283 | b = {repr(game)}
284 | {make_fun(game.board, actions)}
285 | return b
286 | """
287 |
288 |
289 | ex = 0
290 | def prompt(game):
291 | """
292 | You should fill these sections out to teach the AI how to play the game.
293 |
294 | Or you may do your own thing :)
295 | """
296 | print(f"""
297 | # TODO: DESCRIBE THE GAME
298 |
299 | # TODO: DESCRIBE THE ACTIONS
300 | change_str = {change_str}
301 |
302 | {inspect.getsource(move)}
303 | """)
304 |
305 | def example(game, actions):
306 | """
307 | This code makes a few shot example. You don't need to edit it.
308 | """
309 | global ex
310 | ex += 1
311 | print(f"""
312 | def example{ex}():
313 | b = {repr(game)}
314 | {make_fun(game.board, actions)}
315 | return b
316 | # ------------
317 | """)
318 |
319 | # Create a few shot example (you may not need this)
320 | board = Game(boundary=(3, 3), key=(1, 1), flag=(2, 2), init=(0, 0), walls=[(2, 0)])
321 | actions = [Actions.DOWNRIGHT, Actions.PICKUP, Actions.DOWNRIGHT]
322 | example(board, actions)
323 |
324 | # Test case
325 | print(f"""
326 | # ----
327 | # TODO: ADD any custom example code
328 | #---
329 | # TODO: FINAL description.
330 |
331 | # Contraints for this function:", {repr(game)}
332 | # Please fill this in with code like the examples above (do not provide a description):
333 | #
334 | # The following function `my_example` instantiates a GameBoard called b with these constraints.
335 |
336 | """)
337 |
338 |
339 |
340 | # + [markdown] id="-iecyV7nAbFT"
341 | # This code lets you make a game and see the output for a prompt for that game. There are easy, medium, and hard games.
342 |
343 | # + colab={"base_uri": "https://localhost:8080/"} id="cOneYFok_OMe" outputId="97080186-7322-4ba9-b500-095fb39071aa"
344 | # Easy
345 | easy_game = Game(boundary=(3, 3), key=(1, 1), flag=(2, 2), init=(0, 0), walls=[])
346 |
347 | # Medium
348 | medium_game = Game(boundary=(5, 5), key=(3, 1), flag=(4, 4), init=(0, 0), walls=[(1, 1)])
349 |
350 | # Hard (This is the main one)
351 | hard_game = Game(boundary=(8, 15), key=(3, 1), flag=(7, 13), init=(0, 0), walls=[(2, 2), (1, 1), (5, 3), (1, 11), (5, 5), (6, 6), (6, 10), (2, 6), (4, 12)])
352 |
353 | # Evil
354 | evil_game = Game(boundary=(8, 15), key=(5, 1), flag=(7, 13), init=(0, 0), walls=[(2, 2), (3, 3), (4, 2), (1, 1), (2, 4), (7, 11), (5, 3), (1, 11), (5, 5), (6, 6), (6, 10), (2, 6), (4, 12)])
355 |
356 | games = {"Easy": easy_game, "Medium": medium_game, "Hard": hard_game, "Evil": evil_game}
357 |
358 | # Anima
359 | def animate(game):
360 | cur = game.original
361 | i = 0
362 | images = []
363 | draw_board(cur.grid, i)
364 | images.append(imageio.v2.imread(f"pic{i}.png"))
365 | for act in game.actions:
366 | cur = cur.move(act)
367 | i += 1
368 | draw_board(cur.grid, i)
369 | images.append(imageio.v2.imread(f"pic{i}.png"))
370 |
371 | return imageio.v2.mimsave('movie.gif', images, **{ 'duration': 1000, 'loop': 100})
372 |
373 |
374 | def load(inp):
375 | if inp in games:
376 | board = games[inp]
377 | else:
378 | board = eval(inp)
379 | draw_board(board.board.grid, 0).render_svg("tmp.svg")
380 |
381 | return ["tmp.svg"], repr(board)
382 |
383 | draw_board(hard_game.board.grid, 0).render_svg("hard.svg")
384 | draw_board(easy_game.board.grid, 0).render_svg("easy.svg")
385 | with gr.Blocks() as app:
386 |
387 | # test = gr.Code(label="test")
388 | # im2 = gr.Gallery()
389 | # im2.style(preview=True)
390 | gr.HTML("""
391 | 395 | GPTWorld is a prompting game. Your goal is to get an LLM to complete a maze. You are the red dot (🔴) need to first get the key (▲) and then reach the exit (x). The game takes place on a hexagonal grid with walls [Even rows are labeled (0,0), (0, 2), (0,4) and odd rows are labeled (1, 1), (1, 3), (1, 5)]. You play by prompting GPT to write code which solves the game on the right. 396 |
397 |