├── .env.template
├── types.d.ts
├── app
    ├── client.tsx
    ├── App.tsx
    ├── components
    │   ├── Chat.module.css
    │   ├── AssistantTranscript.tsx
    │   ├── DebugWorld.tsx
    │   ├── Debug.tsx
    │   ├── AssistantTranscriptEntry.tsx
    │   ├── DebugLocation.tsx
    │   ├── Chat.tsx
    │   ├── DebugFunctions.tsx
    │   └── Assistant.tsx
    └── styles.css
├── .vscode
    ├── launch.json
    └── settings.json
├── partykit.json
├── party
    ├── messages.ts
    ├── test.ts
    ├── openai.ts
    ├── world.ts
    ├── functions.ts
    ├── server.ts
    └── assistant.ts
├── package.json
├── public
    ├── index.html
    └── normalize.css
├── LICENSE
├── .gitignore
├── README.md
└── tsconfig.json


/.env.template:
--------------------------------------------------------------------------------
1 | OPENAI_API_KEY=sk-...
2 | OPENAI_API_ORGANIZATION=# Leave empty for no organization
3 | 


--------------------------------------------------------------------------------
/types.d.ts:
--------------------------------------------------------------------------------
1 | declare module "*.module.css" {
2 |   declare const styles: Record<string, string>;
3 |   export = styles;
4 | }
5 | 


--------------------------------------------------------------------------------
/app/client.tsx:
--------------------------------------------------------------------------------
1 | import "./styles.css";
2 | import App from "./App";
3 | import { createRoot } from "react-dom/client";
4 | 
5 | createRoot(document.getElementById("app")!).render(<App />);
6 | 


--------------------------------------------------------------------------------
/.vscode/launch.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "configurations": [
 3 |     {
 4 |       "type": "node",
 5 |       "request": "attach",
 6 |       "name": "PartyKit debugger",
 7 |       "address": "localhost",
 8 |       "port": 9229
 9 |     }
10 |   ]
11 | }
12 | 


--------------------------------------------------------------------------------
/.vscode/settings.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "files.associations": {
 3 |     "partykit.json": "jsonc"
 4 |   },
 5 |   "json.schemas": [
 6 |     {
 7 |       "fileMatch": ["partykit.json"],
 8 |       "url": "https://www.partykit.io/schema.json"
 9 |     }
10 |   ]
11 | }
12 | 


--------------------------------------------------------------------------------
/app/App.tsx:
--------------------------------------------------------------------------------
 1 | import Debug from "./components/Debug";
 2 | import Assistant from "./components/Assistant";
 3 | import Chat from "./components/Chat";
 4 | 
 5 | export default function App() {
 6 |   return (
 7 |     <main>
 8 |       <Debug />
 9 |       <Assistant />
10 |       {/*<Chat />*/}
11 |     </main>
12 |   );
13 | }
14 | 


--------------------------------------------------------------------------------
/partykit.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "$schema": "https://www.partykit.io/schema.json",
 3 |   "name": "lares",
 4 |   "main": "party/server.ts",
 5 |   "compatibilityDate": "2024-03-15",
 6 |   "parties": {
 7 |     "assistant": "party/assistant.ts"
 8 |   },
 9 |   "serve": {
10 |     "path": "public",
11 |     "build": "app/client.tsx"
12 |   }
13 | }
14 | 


--------------------------------------------------------------------------------
/party/messages.ts:
--------------------------------------------------------------------------------
 1 | import type { World } from "./world";
 2 | 
 3 | export type SyncMessage = {
 4 |   type: "sync";
 5 |   state: World;
 6 | };
 7 | 
 8 | export type AskUserMessage = {
 9 |   type: "askUser";
10 |   toolCallId: string;
11 |   question: string;
12 | };
13 | 
14 | export type AskUserResponseMessage = {
15 |   type: "askUserResponse";
16 |   toolCallId: string;
17 |   answer: string;
18 | };
19 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "sketch-home-sim",
 3 |   "version": "0.0.0",
 4 |   "private": true,
 5 |   "scripts": {
 6 |     "dev": "partykit dev --live",
 7 |     "deploy": "partykit deploy"
 8 |   },
 9 |   "dependencies": {
10 |     "openai": "^4.29.0",
11 |     "partysocket": "^1.0.0",
12 |     "react": "^18.2.0",
13 |     "react-dom": "^18.2.0",
14 |     "zod-to-json-schema": "^3.22.4"
15 |   },
16 |   "devDependencies": {
17 |     "@types/react": "^18.2.60",
18 |     "@types/react-dom": "^18.2.19",
19 |     "partykit": "^0.0.96",
20 |     "typescript": "^5.3.3"
21 |   }
22 | }
23 | 


--------------------------------------------------------------------------------
/app/components/Chat.module.css:
--------------------------------------------------------------------------------
 1 | .listItem {
 2 |   display: flex;
 3 |   justify-content: start;
 4 |   align-items: end;
 5 |   gap: 0.5rem;
 6 | }
 7 | 
 8 | .listItem.reversed {
 9 |   justify-content: end;
10 |   flex-direction: row-reverse;
11 | }
12 | 
13 | .user {
14 |   border-radius: 50%;
15 |   background-color: lightblue;
16 |   width: 3rem;
17 |   height: 3rem;
18 |   display: flex;
19 |   justify-content: center;
20 |   align-items: center;
21 |   font-weight: 600;
22 | }
23 | 
24 | .message {
25 |   border-radius: 0.5rem;
26 |   background-color: rgb(231, 229, 228);
27 |   padding: 0.25rem 0.5rem;
28 |   max-width: calc(100% - 7rem);
29 | }
30 | 


--------------------------------------------------------------------------------
/app/components/AssistantTranscript.tsx:
--------------------------------------------------------------------------------
 1 | import { useRef, useEffect } from "react";
 2 | import type { OpenAIMessage } from "../../party/openai";
 3 | import AssistantTranscriptEntry from "./AssistantTranscriptEntry";
 4 | 
 5 | export default function AssistantTranscript({
 6 |   transcript,
 7 | }: {
 8 |   transcript: OpenAIMessage[];
 9 | }) {
10 |   return (
11 |     <div
12 |       style={{
13 |         display: "flex",
14 |         flexDirection: "column-reverse",
15 |         gap: "0.5rem",
16 |         width: "100%",
17 |       }}
18 |     >
19 |       {transcript.map((entry: any, index: number) => (
20 |         <AssistantTranscriptEntry
21 |           key={`entry-${index}`}
22 |           entry={entry}
23 |           index={index}
24 |         />
25 |       ))}
26 |     </div>
27 |   );
28 | }
29 | 


--------------------------------------------------------------------------------
/app/components/DebugWorld.tsx:
--------------------------------------------------------------------------------
 1 | import type { World } from "../../party/world";
 2 | import DebugLocation from "./DebugLocation";
 3 | export default function DebugWorld({ world }: { world: World | null }) {
 4 |   return (
 5 |     <div>
 6 |       <div style={{ display: "flex", gap: "1rem", flexWrap: "wrap" }}>
 7 |         {world?.locations.map((location) => (
 8 |           <DebugLocation
 9 |             key={location.id}
10 |             location={location}
11 |             moveableItems={world?.moveableItems.filter(
12 |               (item) => item.location === location.id
13 |             )}
14 |           />
15 |         ))}
16 |       </div>
17 |       <details style={{ display: "none" }}>
18 |         <summary>Debug</summary>
19 |         <pre>{JSON.stringify(world, null, 2)}</pre>
20 |       </details>
21 |     </div>
22 |   );
23 | }
24 | 


--------------------------------------------------------------------------------
/public/index.html:
--------------------------------------------------------------------------------
 1 | <!doctype html>
 2 | <html lang="en">
 3 |   <head>
 4 |     <meta charset="utf-8" />
 5 |     <meta
 6 |       name="viewport"
 7 |       content="width=device-width, initial-scale=1.0, shrink-to-fit=no"
 8 |     />
 9 |     <meta http-equiv="X-UA-Compatible" content="IE=edge" />
10 |     <title>Lares: Smart home sim with an AI agent</title>
11 |     <!-- Primary Meta Tags -->
12 |     <meta name="title" content="Lares" />
13 |     <meta name="description" content="Smart home sim with an AI agent" />
14 |     <meta name="author" content="Matt Webb (interconnected.org)" />
15 |     <!-- Theme Colour -->
16 |     <meta name="theme-color" content="#ffffff" />
17 | 
18 |     <link rel="stylesheet" href="/normalize.css" />
19 |     <link rel="stylesheet" href="/dist/client.css" />
20 |   </head>
21 |   <body style="background-color: #eee;">
22 |     <div id="app"></div>
23 |     <script type="module" src="/dist/client.js"></script>
24 |   </body>
25 | </html>
26 | 


--------------------------------------------------------------------------------
/app/styles.css:
--------------------------------------------------------------------------------
 1 | /* 
 2 |   We've already included normalize.css. 
 3 | 
 4 |   But we'd like a modern looking boilerplate. 
 5 |   Clean type, sans-serif, and a nice color palette. 
 6 |   
 7 | */
 8 | 
 9 | * {
10 |   box-sizing: border-box;
11 | }
12 | 
13 | body {
14 |   font-family: sans-serif;
15 |   font-size: 14px;
16 |   line-height: 1.5;
17 |   color: #333;
18 |   min-height: 100dvh;
19 | }
20 | 
21 | h1,
22 | h2,
23 | h3,
24 | h4,
25 | h5,
26 | h6 {
27 |   font-family: sans-serif;
28 |   font-weight: 600;
29 |   line-height: 1.25;
30 |   margin-top: 0;
31 |   margin-bottom: 0.5rem;
32 | }
33 | 
34 | #app {
35 |   height: 100%;
36 |   width: 100%;
37 | }
38 | 
39 | main {
40 |   display: flex;
41 |   justify-content: stretch;
42 |   align-items: start;
43 |   width: 100%;
44 |   max-width: 100%;
45 | }
46 | 
47 | main > div {
48 |   width: 50%;
49 |   padding: 1rem;
50 |   max-width: 50%;
51 |   min-height: 100dvh;
52 | }
53 | 
54 | code {
55 |   font-family: monospace;
56 |   font-size: 0.85rem;
57 |   font-weight: normal;
58 | }


--------------------------------------------------------------------------------
/app/components/Debug.tsx:
--------------------------------------------------------------------------------
 1 | import { useState } from "react";
 2 | import usePartySocket from "partysocket/react";
 3 | import DebugWorld from "./DebugWorld";
 4 | import DebugFunctions from "./DebugFunctions";
 5 | import type { World } from "../../party/world";
 6 | 
 7 | const DEFAULT_ROOM = "default";
 8 | 
 9 | export default function Debug() {
10 |   const [world, setWorld] = useState<World | null>(null);
11 | 
12 |   const socket = usePartySocket({
13 |     room: DEFAULT_ROOM,
14 |     onMessage: (evt) => {
15 |       const data = JSON.parse(evt.data);
16 |       if (data.type === "sync") {
17 |         setWorld(data.state);
18 |       }
19 |     },
20 |   });
21 | 
22 |   return (
23 |     <div
24 |       style={{
25 |         display: "flex",
26 |         flexDirection: "column",
27 |         gap: "2rem",
28 |         backgroundColor: "#eee",
29 |       }}
30 |     >
31 |       <h1>World</h1>
32 |       <DebugWorld world={world} />
33 |       <DebugFunctions host={socket.host} room={DEFAULT_ROOM} />
34 |     </div>
35 |   );
36 | }
37 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2023 PartyKit, inc.
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/party/test.ts:
--------------------------------------------------------------------------------
 1 | import type { OpenAIMessage } from "./openai";
 2 | 
 3 | export const TEST_INSTRUCTION =
 4 |   "turn the light on in the office according to whether there is daylight";
 5 | 
 6 | export const TEST_TRANSCRIPT: OpenAIMessage[] = [
 7 |   {
 8 |     role: "assistant",
 9 |     content:
10 |       "I have decided to call the function askUser because: First, I need to determine the current level of natural daylight outside to decide whether the office light needs to be turned on.",
11 |   },
12 |   {
13 |     role: "assistant",
14 |     content: null,
15 |     tool_calls: [
16 |       {
17 |         id: "call_u5DlhskWvxPwwleVEdNwN9Hw",
18 |         type: "function",
19 |         function: {
20 |           name: "askUser",
21 |           arguments: '{"question":"Is there daylight outside right now?"}',
22 |         },
23 |       },
24 |     ],
25 |   },
26 |   {
27 |     tool_call_id: "call_u5DlhskWvxPwwleVEdNwN9Hw",
28 |     role: "tool",
29 |     name: "askUser",
30 |     content: '{"success":"yes"}',
31 |   },
32 |   {
33 |     role: "assistant",
34 |     content:
35 |       "I have decided to call the function halt because: Since there is daylight outside, turning on the light in the office is unnecessary. Therefore, no action is needed regarding the office light.",
36 |   },
37 |   {
38 |     role: "assistant",
39 |     content: null,
40 |     tool_calls: [
41 |       {
42 |         id: "call_5UPNleP3KWh21VM7Kpj7UrfI",
43 |         type: "function",
44 |         function: {
45 |           name: "halt",
46 |           arguments:
47 |             '{"messageToUser":"No need to turn on the office light since there\'s daylight outside."}',
48 |         },
49 |       },
50 |     ],
51 |   },
52 | ];
53 | 


--------------------------------------------------------------------------------
/app/components/AssistantTranscriptEntry.tsx:
--------------------------------------------------------------------------------
 1 | import type { OpenAIMessage } from "../../party/openai";
 2 | 
 3 | export default function AssistantTranscriptEntry({
 4 |   entry,
 5 |   index,
 6 | }: {
 7 |   entry: OpenAIMessage;
 8 |   index: number;
 9 | }) {
10 |   let payload = null;
11 |   if (entry.role === "assistant" && entry.tool_calls) {
12 |     payload = JSON.parse(entry.tool_calls[0].function.arguments);
13 |   } else if (entry.role === "tool") {
14 |     payload = JSON.parse(entry.content);
15 |   }
16 | 
17 |   return (
18 |     <div
19 |       style={{
20 |         position: "relative",
21 |         width: "100%",
22 |         borderRadius: "0.25rem",
23 |         border: "1px solid #cfc",
24 |         padding: "0.5rem",
25 |         display: "flex",
26 |         flexDirection: "column",
27 |         gap: "0.5rem",
28 |         justifyItems: "start",
29 |         alignItems: "start",
30 |       }}
31 |     >
32 |       <div
33 |         style={{
34 |           border: "1px solid #999",
35 |           borderRadius: "9999px",
36 |           padding: "0.25rem 0.5rem",
37 |         }}
38 |       >
39 |         {entry.role}
40 |       </div>
41 |       <div
42 |         style={{
43 |           position: "absolute",
44 |           top: "0.25rem",
45 |           right: "0.25rem",
46 |           fontWeight: "bold",
47 |           color: "#999",
48 |         }}
49 |       >
50 |         #{index + 1}
51 |       </div>
52 |       {entry.role === "assistant" && !entry.tool_calls && (
53 |         <div>{entry.content}</div>
54 |       )}
55 |       {entry.role === "assistant" && entry.tool_calls && (
56 |         <div>
57 |           {entry.tool_calls[0].function.name}(
58 |           <code>{JSON.stringify(payload)}</code>)
59 |         </div>
60 |       )}
61 |       {entry.role === "tool" && (
62 |         <div>
63 |           {entry.name} {payload.success ? "success" : "error"}:{" "}
64 |           <code>{payload.success || payload.error}</code>
65 |         </div>
66 |       )}
67 |     </div>
68 |   );
69 | }
70 | 


--------------------------------------------------------------------------------
/party/openai.ts:
--------------------------------------------------------------------------------
 1 | import OpenAI from "openai";
 2 | 
 3 | export type OpenAIMessage = OpenAI.Chat.Completions.ChatCompletionMessageParam;
 4 | 
 5 | export async function getChatCompletionResponse({
 6 |   messages,
 7 |   tool,
 8 | }: {
 9 |   messages: OpenAIMessage[];
10 |   tool: any;
11 | }) {
12 |   // If no organization is set, usage will count against the default key owner
13 |   if (!process.env.OPENAI_API_ORGANIZATION) {
14 |     console.info(
15 |       "No OPENAI_API_ORGANIZATION set, usage will count against the default key owner"
16 |     );
17 |   }
18 | 
19 |   const openai = new OpenAI({
20 |     apiKey: process.env.OPENAI_API_KEY,
21 |     organization: process.env.OPENAI_API_ORGANIZATION,
22 |   });
23 | 
24 |   /*const prompt = [
25 |     {
26 |       role: "system",
27 |       content:
28 |         "You are a helpful AI assistant. Your responses are always accurate and extremely brief.",
29 |     } satisfies OpenAIMessage,
30 |     ...messages,
31 |   ];*/
32 | 
33 |   const response = await openai.chat.completions.create({
34 |     model: "gpt-4o",
35 |     messages,
36 |     tools: [{ type: "function", function: tool.getSignature() }],
37 |     tool_choice: { type: "function", function: { name: tool.name } },
38 |   });
39 | 
40 |   console.log("response", JSON.stringify(response, null, 2));
41 | 
42 |   // Reply with the first message that includes a tool call
43 |   const toolCallMessage = response.choices.find(
44 |     (choice) => choice.message.tool_calls
45 |   );
46 |   if (toolCallMessage) {
47 |     return toolCallMessage.message as OpenAI.Chat.Completions.ChatCompletionMessage;
48 |   }
49 | 
50 |   return;
51 | 
52 |   /*let tool_calls: OpenAI.Chat.Completions.ChatCompletionMessageToolCall[] = [];
53 |   let response_messages: OpenAI.Chat.Completions.ChatCompletionMessage[] = [];
54 |   response.choices.map((choice) => {
55 |     if (choice.message.tool_calls) {
56 |       tool_calls.push(...choice.message.tool_calls);
57 |     }
58 |     if (choice.message.content) {
59 |       response_messages.push(choice.message);
60 |     }
61 |   });
62 | 
63 |   // Ignore response_messages
64 |   //return { response_messages, tool_calls };
65 |   return tool_calls;*/
66 | }
67 | 


--------------------------------------------------------------------------------
/app/components/DebugLocation.tsx:
--------------------------------------------------------------------------------
 1 | import type { Location, MoveableItem } from "../../party/world";
 2 | 
 3 | export default function DebugLocation({
 4 |   location,
 5 |   moveableItems,
 6 | }: {
 7 |   location: Location;
 8 |   moveableItems: MoveableItem[];
 9 | }) {
10 |   // Make list of all items in location with type 'light'
11 |   const lights = location.contents.filter((item) => item.type === "light");
12 |   // Do any of the lights have state.on === true?
13 |   const isLit = lights.some((light) => light.state.on);
14 | 
15 |   return (
16 |     <div
17 |       style={{
18 |         width: "100%",
19 |         maxWidth: "12rem",
20 |         border: "0px solid #666",
21 |         borderRadius: "0.25rem",
22 |         padding: "0.5rem",
23 |         background: isLit ? "#fde68a" : "#c7d2fe",
24 |         transition: "background-color 0.5s",
25 |       }}
26 |     >
27 |       <h2>
28 |         {location.name} <code>&lt;{location.id}&gt;</code>
29 |       </h2>
30 |       <div
31 |         style={{
32 |           textTransform: "uppercase",
33 |           fontSize: "0.8rem",
34 |           fontWeight: "bold",
35 |         }}
36 |       >
37 |         Devices
38 |       </div>
39 |       {location.contents.map((item) => (
40 |         <div key={item.id}>
41 |           <strong>{item.name}</strong> <code>&lt;{item.id}&gt;</code>
42 |           <br />
43 |           type: <code>{item.type}</code>
44 |           <br />
45 |           state: <code>{JSON.stringify(item.state)}</code>
46 |         </div>
47 |       ))}
48 | 
49 |       <div
50 |         style={{
51 |           textTransform: "uppercase",
52 |           fontSize: "0.8rem",
53 |           fontWeight: "bold",
54 |           marginTop: "1rem",
55 |         }}
56 |       >
57 |         Exits
58 |       </div>
59 |       <div>
60 |         {location.exits.map((exit) => (
61 |           <span key={exit}>
62 |             {" "}
63 |             <code>&lt;{exit}&gt;</code>
64 |           </span>
65 |         ))}
66 |       </div>
67 |       {moveableItems.length > 0 && (
68 |         <>
69 |           <div
70 |             style={{
71 |               textTransform: "uppercase",
72 |               fontSize: "0.8rem",
73 |               fontWeight: "bold",
74 |               marginTop: "1rem",
75 |             }}
76 |           >
77 |             Also here
78 |           </div>
79 | 
80 |           <ul style={{ marginTop: "0.25rem" }}>
81 |             {moveableItems.map((item) => (
82 |               <li key={item.id}>
83 |                 <strong style={{ background: "white" }}>{item.name}</strong>{" "}
84 |                 <code>&lt;{item.id}&gt;</code>
85 |                 <br />
86 |                 type: <code>{item.type}</code>
87 |               </li>
88 |             ))}
89 |           </ul>
90 |         </>
91 |       )}
92 |     </div>
93 |   );
94 | }
95 | 


--------------------------------------------------------------------------------
/app/components/Chat.tsx:
--------------------------------------------------------------------------------
 1 | import styles from "./Chat.module.css";
 2 | import type { OpenAIMessage } from "../../party/openai";
 3 | import { useState } from "react";
 4 | import usePartySocket from "partysocket/react";
 5 | 
 6 | // Another example is:
 7 | // tell me where my dog is. you'll know it's my dog when you see it
 8 | const DEFAULT_INSTRUCTION = "turn on the light in the lounge";
 9 | 
10 | /* IN PROGRESS: Alternate chat view */
11 | 
12 | export default function Chat() {
13 |   const [assistant, setAssistant] = useState<any>(null);
14 |   const [input, setInput] = useState<string>(DEFAULT_INSTRUCTION);
15 | 
16 |   const socket = usePartySocket({
17 |     party: "assistant",
18 |     room: "default",
19 |     onMessage: (evt) => {
20 |       const data = JSON.parse(evt.data);
21 |       if (data.type === "sync") {
22 |         setAssistant(data.state);
23 |       } else if (data.type === "askUser") {
24 |         socket.send(
25 |           JSON.stringify({
26 |             type: "askUserResponse",
27 |             toolCallId: data.toolCallId,
28 |             answer: prompt(data.question),
29 |           })
30 |         );
31 |       }
32 |     },
33 |   });
34 | 
35 |   const transcript = [
36 |     ...(assistant?.instruction
37 |       ? [{ role: "user", content: assistant.instruction } as OpenAIMessage]
38 |       : []),
39 |     ...(assistant?.transcript || []),
40 |   ];
41 | 
42 |   return (
43 |     <div
44 |       style={{
45 |         display: "flex",
46 |         width: "100%",
47 |         flexDirection: "column-reverse",
48 |         justifyContent: "end",
49 |         alignContent: "start",
50 |         gap: "1rem",
51 |         backgroundColor: "red",
52 |         fontSize: "1.2rem",
53 |       }}
54 |     >
55 |       <form
56 |         style={{
57 |           display: "flex",
58 |           justifyContent: "stretch",
59 |           gap: "0.5rem",
60 |           width: "100%",
61 |         }}
62 |       >
63 |         <input
64 |           type="text"
65 |           name="instruction"
66 |           value={input}
67 |           onChange={(e) => setInput(e.target.value)}
68 |           style={{ flex: 1 }}
69 |         />
70 |         <button type="submit">Send</button>
71 |       </form>
72 |       {transcript.map((entry: OpenAIMessage, index: number) => {
73 |         if (entry.role === "user") {
74 |           return (
75 |             <div
76 |               key={index}
77 |               className={`${styles.listItem} ${styles.reversed}`}
78 |             >
79 |               <div className={styles.user}>You</div>
80 |               <div className={styles.message}>{entry.content as string}</div>
81 |             </div>
82 |           );
83 |         } else {
84 |           return <div key={index}>{JSON.stringify(entry, null, 2)}</div>;
85 |         }
86 |       })}
87 |     </div>
88 |   );
89 | }
90 | 


--------------------------------------------------------------------------------
/party/world.ts:
--------------------------------------------------------------------------------
  1 | type FixedItemType = "light";
  2 | 
  3 | type MoveableItemType = "robot" | "dog" | "person";
  4 | 
  5 | type FixedItem = {
  6 |   id: string;
  7 |   name: string;
  8 |   type: FixedItemType;
  9 |   state: any;
 10 | };
 11 | 
 12 | export type FixedItemLight = FixedItem & {
 13 |   type: "light";
 14 |   state: { on: boolean };
 15 | };
 16 | 
 17 | export type MoveableItem = {
 18 |   id: string;
 19 |   name: string;
 20 |   type: MoveableItemType;
 21 |   location: string;
 22 | };
 23 | 
 24 | export type Location = {
 25 |   id: string;
 26 |   name: string;
 27 |   contents: FixedItem[];
 28 |   exits: string[];
 29 | };
 30 | 
 31 | export type World = {
 32 |   locations: Location[];
 33 |   moveableItems: MoveableItem[];
 34 | };
 35 | 
 36 | export const DEFAULT_WORLD: World = {
 37 |   locations: [
 38 |     <Location>{
 39 |       id: "lounge",
 40 |       name: "Lounge",
 41 |       contents: [
 42 |         <FixedItemLight>{
 43 |           id: "lounge-light",
 44 |           name: "Light",
 45 |           type: "light",
 46 |           state: { on: false },
 47 |         },
 48 |       ],
 49 |       exits: ["hallway", "office"],
 50 |     },
 51 |     <Location>{
 52 |       id: "hallway",
 53 |       name: "Hallway",
 54 |       contents: [
 55 |         <FixedItemLight>{
 56 |           id: "hallway-light",
 57 |           name: "Light",
 58 |           type: "light",
 59 |           state: { on: true },
 60 |         },
 61 |       ],
 62 |       exits: ["lounge", "kitchen", "bedroom"],
 63 |     },
 64 |     <Location>{
 65 |       id: "kitchen",
 66 |       name: "Kitchen",
 67 |       contents: [
 68 |         <FixedItemLight>{
 69 |           id: "kitchen-light",
 70 |           name: "Light",
 71 |           type: "light",
 72 |           state: { on: false },
 73 |         },
 74 |       ],
 75 |       exits: ["hallway"],
 76 |     },
 77 |     <Location>{
 78 |       id: "bedroom",
 79 |       name: "Bedroom",
 80 |       contents: [
 81 |         <FixedItemLight>{
 82 |           id: "bedroom-light",
 83 |           name: "Light",
 84 |           type: "light",
 85 |           state: { on: false },
 86 |         },
 87 |       ],
 88 |       exits: ["hallway"],
 89 |     },
 90 |     <Location>{
 91 |       id: "office",
 92 |       name: "Office",
 93 |       contents: [
 94 |         <FixedItemLight>{
 95 |           id: "office-light",
 96 |           name: "Light",
 97 |           type: "light",
 98 |           state: { on: false },
 99 |         },
100 |       ],
101 |       exits: ["lounge"],
102 |     },
103 |   ],
104 |   moveableItems: [
105 |     <MoveableItem>{
106 |       id: "robot",
107 |       name: "Robot",
108 |       type: "robot",
109 |       location: "hallway",
110 |     },
111 |     <MoveableItem>{
112 |       id: "bruno",
113 |       name: "Bruno",
114 |       type: "dog",
115 |       location: "bedroom",
116 |     },
117 |     <MoveableItem>{
118 |       id: "sally",
119 |       name: "Sally",
120 |       type: "person",
121 |       location: "office",
122 |     },
123 |   ],
124 | };
125 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # Logs
  2 | 
  3 | logs
  4 | _.log
  5 | npm-debug.log_
  6 | yarn-debug.log*
  7 | yarn-error.log*
  8 | lerna-debug.log*
  9 | .pnpm-debug.log*
 10 | 
 11 | # Diagnostic reports (https://nodejs.org/api/report.html)
 12 | 
 13 | report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
 14 | 
 15 | # Runtime data
 16 | 
 17 | pids
 18 | _.pid
 19 | _.seed
 20 | \*.pid.lock
 21 | 
 22 | # Directory for instrumented libs generated by jscoverage/JSCover
 23 | 
 24 | lib-cov
 25 | 
 26 | # Coverage directory used by tools like istanbul
 27 | 
 28 | coverage
 29 | \*.lcov
 30 | 
 31 | # nyc test coverage
 32 | 
 33 | .nyc_output
 34 | 
 35 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
 36 | 
 37 | .grunt
 38 | 
 39 | # Bower dependency directory (https://bower.io/)
 40 | 
 41 | bower_components
 42 | 
 43 | # node-waf configuration
 44 | 
 45 | .lock-wscript
 46 | 
 47 | # Compiled binary addons (https://nodejs.org/api/addons.html)
 48 | 
 49 | build/Release
 50 | 
 51 | # Dependency directories
 52 | 
 53 | node_modules/
 54 | jspm_packages/
 55 | 
 56 | # Snowpack dependency directory (https://snowpack.dev/)
 57 | 
 58 | web_modules/
 59 | 
 60 | # TypeScript cache
 61 | 
 62 | \*.tsbuildinfo
 63 | 
 64 | # Optional npm cache directory
 65 | 
 66 | .npm
 67 | 
 68 | # Optional eslint cache
 69 | 
 70 | .eslintcache
 71 | 
 72 | # Optional stylelint cache
 73 | 
 74 | .stylelintcache
 75 | 
 76 | # Microbundle cache
 77 | 
 78 | .rpt2_cache/
 79 | .rts2_cache_cjs/
 80 | .rts2_cache_es/
 81 | .rts2_cache_umd/
 82 | 
 83 | # Optional REPL history
 84 | 
 85 | .node_repl_history
 86 | 
 87 | # Output of 'npm pack'
 88 | 
 89 | \*.tgz
 90 | 
 91 | # Yarn Integrity file
 92 | 
 93 | .yarn-integrity
 94 | 
 95 | # dotenv environment variable files
 96 | 
 97 | .env
 98 | .env.development.local
 99 | .env.test.local
100 | .env.production.local
101 | .env.local
102 | 
103 | # parcel-bundler cache (https://parceljs.org/)
104 | 
105 | .cache
106 | .parcel-cache
107 | 
108 | # Next.js build output
109 | 
110 | .next
111 | out
112 | 
113 | # Nuxt.js build / generate output
114 | 
115 | .nuxt
116 | dist
117 | 
118 | # Gatsby files
119 | 
120 | .cache/
121 | 
122 | # Comment in the public line in if your project uses Gatsby and not Next.js
123 | 
124 | # https://nextjs.org/blog/next-9-1#public-directory-support
125 | 
126 | # public
127 | 
128 | # vuepress build output
129 | 
130 | .vuepress/dist
131 | 
132 | # vuepress v2.x temp and cache directory
133 | 
134 | .temp
135 | .cache
136 | 
137 | # Docusaurus cache and generated files
138 | 
139 | .docusaurus
140 | 
141 | # Serverless directories
142 | 
143 | .serverless/
144 | 
145 | # FuseBox cache
146 | 
147 | .fusebox/
148 | 
149 | # DynamoDB Local files
150 | 
151 | .dynamodb/
152 | 
153 | # TernJS port file
154 | 
155 | .tern-port
156 | 
157 | # Stores VSCode versions used for testing VSCode extensions
158 | 
159 | .vscode-test
160 | 
161 | # yarn v2
162 | 
163 | .yarn/cache
164 | .yarn/unplugged
165 | .yarn/build-state.yml
166 | .yarn/install-state.gz
167 | .pnp.\*
168 | 
169 | .partykit
170 | .DS_Store
171 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Lares
 2 | 
 3 | Read: [Lares smart home assistant: A toy AI agent demonstrating emergent behavior](https://interconnected.org/more/2024/lares/)
 4 | 
 5 | ## What is this?
 6 | 
 7 | What you'll find here:
 8 | 
 9 | - A simple simulation of a smart home system
10 | - An assistant that takes instructions from the user
11 | - The assistant using OpenAI function calling to interaction with the simulation, to achieve its goal
12 | 
13 | When you run the sim using `npm run dev` there's a web view:
14 | 
15 | - Left side: A debug view that shows the entire world state (including information unavailable to the assistant, such as the location of movable items like pets and people)
16 | - Right side: The assistant view, including a text input for the user to give instructions, and a view of the assistant's current state (its transcript of messages with OpenAI)
17 | 
18 | ## Development
19 | 
20 | ```bash
21 | cp .env.template .env # populate with OPENAI_API_KEY
22 | npm install
23 | npm run dev
24 | ```
25 | 
26 | ## Navigating around the repo
27 | 
28 | Looking in `party/`...
29 | 
30 | There are two parties:
31 | 
32 | - `server.ts` -- the world sim. Contains world state, and the endpoints for the assistant to interact with the world using functions. Includes a 'sync' WebSocket message so the debug view can see the world state
33 | - `assistant.ts` -- takes an instruction, and includes the function calling logic
34 | 
35 | There are also utility functions:
36 | 
37 | - `openai.ts` -- communicates with OpenAI, constraining the response to a single tool call (tool == function)
38 | - `functions.ts` -- the functions that the assistant can call via OpenAI, including an `intentFunction` which is used to decide which actual function to call
39 | - `world.ts` and `messages.ts` -- types and constants
40 | 
41 | ## The function calling loop
42 | 
43 | When the assistant gets an instruction, it loops around the following steps:
44 | 
45 | - get the latest world state (the assistant always has access to the layout of the house, the devices in each rooms, and their current state. It does not have access to moveable items, i.e. un-networked pets and people)
46 | - call `intentFunction` using OpenAI to decide which function to call, given the user's instructions
47 | - call the decided function using OpenAI to get the function arguments
48 | - add this function call to the message transcript
49 | - (if the decided function is 'halt', stop here)
50 | - using the function name and arguments, make an HTTP request to the sim in `server.ts` to mutate the world state and get the result
51 | - adds the function call _result_ to the assistant's transcript -- this is essentially the assistant's memory, something that is additional to the world state.
52 | 
53 | The loop continues until:
54 | 
55 | - the assistant receives a 'halt' function call -- this also includes the ability to report back to the user
56 | - the assistant hits the maximum number of function calls (currently set to 10)
57 | 
58 | ## Next steps
59 | 
60 | - Real world integration: could the world sim represent _my house?_ Could the function calls be actual HomeKit calls?
61 | 


--------------------------------------------------------------------------------
/app/components/DebugFunctions.tsx:
--------------------------------------------------------------------------------
 1 | import { useState } from "react";
 2 | import PartySocket from "partysocket";
 3 | 
 4 | const DEBUG_LIGHTS = [
 5 |   "lounge-light",
 6 |   "hallway-light",
 7 |   "kitchen-light",
 8 |   "bedroom-light",
 9 |   "office-light",
10 | ];
11 | 
12 | const DEBUG_ROOMS = ["lounge", "hallway", "kitchen", "bedroom", "office"];
13 | 
14 | export default function DebugFunctions({
15 |   host,
16 |   room,
17 | }: {
18 |   host: string;
19 |   room: string;
20 | }) {
21 |   const [response, setResponse] = useState<string | null>(null);
22 | 
23 |   const fetch = async (fn: string, args: any) => {
24 |     const res = await PartySocket.fetch(
25 |       { host, room, path: "call" },
26 |       { method: "POST", body: JSON.stringify({ fn, args }) }
27 |     );
28 |     setResponse(await res.text());
29 |   };
30 | 
31 |   const handleToggleLightSubmit = (evt: React.FormEvent<HTMLFormElement>) => {
32 |     evt.preventDefault();
33 |     const light = evt.currentTarget.elements.namedItem(
34 |       "light"
35 |     ) as HTMLSelectElement;
36 |     fetch("toggleLight", { lightId: light.value });
37 |   };
38 | 
39 |   const handleLookWithRobotSubmit = (evt: React.FormEvent<HTMLFormElement>) => {
40 |     evt.preventDefault();
41 |     fetch("lookWithRobot", {});
42 |   };
43 | 
44 |   const handleMoveRobotSubmit = (evt: React.FormEvent<HTMLFormElement>) => {
45 |     evt.preventDefault();
46 |     const room = evt.currentTarget.elements.namedItem(
47 |       "room"
48 |     ) as HTMLSelectElement;
49 |     fetch("moveRobot", { destinationRoomId: room.value });
50 |   };
51 | 
52 |   return (
53 |     <details>
54 |       <summary
55 |         style={{
56 |           fontSize: "1rem",
57 |           fontWeight: "medium",
58 |           marginBottom: "1rem",
59 |           opacity: 0.5,
60 |         }}
61 |       >
62 |         Debug: Manual function caller
63 |       </summary>
64 |       <div style={{ display: "flex", flexDirection: "column", gap: "1rem" }}>
65 |         <form
66 |           onSubmit={handleToggleLightSubmit}
67 |           style={{ display: "flex", gap: "0.5rem" }}
68 |         >
69 |           <select name="light">
70 |             {DEBUG_LIGHTS.map((light) => (
71 |               <option key={light} value={light}>
72 |                 {light}
73 |               </option>
74 |             ))}
75 |           </select>
76 |           <button type="submit">toggleLight</button>
77 |         </form>
78 |         <form onSubmit={handleLookWithRobotSubmit}>
79 |           <button>lookWithRobot</button>
80 |         </form>
81 |         <form
82 |           onSubmit={handleMoveRobotSubmit}
83 |           style={{ display: "flex", gap: "0.5rem" }}
84 |         >
85 |           <select name="room">
86 |             {DEBUG_ROOMS.map((room) => (
87 |               <option key={room} value={room}>
88 |                 {room}
89 |               </option>
90 |             ))}
91 |           </select>
92 |           <button>moveRobot</button>
93 |         </form>
94 |         <div>Latest response: {response ? <pre>{response}</pre> : "None"}</div>
95 |       </div>
96 |     </details>
97 |   );
98 | }
99 | 


--------------------------------------------------------------------------------
/party/functions.ts:
--------------------------------------------------------------------------------
  1 | import { z } from "zod";
  2 | import { zodToJsonSchema } from "zod-to-json-schema";
  3 | 
  4 | type OpenAIFunctionType = {
  5 |   name: string;
  6 |   description: string;
  7 |   schema: z.ZodObject<any, any>;
  8 |   dispatchType: "REMOTE" | "LOCAL";
  9 |   getSignature: () => {
 10 |     name: string;
 11 |     description: string;
 12 |     parameters: any;
 13 |   };
 14 | };
 15 | 
 16 | class OpenAIFunction implements OpenAIFunctionType {
 17 |   name: string;
 18 |   description: string;
 19 |   schema: z.ZodObject<any, any>;
 20 |   dispatchType: "REMOTE" | "LOCAL";
 21 | 
 22 |   constructor(
 23 |     name: string,
 24 |     description: string,
 25 |     schema: z.ZodObject<any, any>,
 26 |     dispatchType: "REMOTE" | "LOCAL"
 27 |   ) {
 28 |     this.name = name;
 29 |     this.description = description;
 30 |     this.schema = schema;
 31 |     this.dispatchType = dispatchType;
 32 |   }
 33 | 
 34 |   getSignature() {
 35 |     return {
 36 |       name: this.name,
 37 |       description: this.description,
 38 |       parameters: zodToJsonSchema(this.schema, { target: "openApi3" }),
 39 |       dispatchType: this.dispatchType,
 40 |     };
 41 |   }
 42 | }
 43 | 
 44 | const toggleLightFunction = new OpenAIFunction(
 45 |   "toggleLight",
 46 |   "Toggle the light in a room, making it light or dark. You can see what's in a room when it's light.",
 47 |   z
 48 |     .object({
 49 |       lightId: z.string().describe("ID of the light to toggle"),
 50 |     })
 51 |     .describe("Toggle light"),
 52 |   "REMOTE"
 53 | );
 54 | 
 55 | const moveRobotFunction = new OpenAIFunction(
 56 |   "moveRobot",
 57 |   "Move the robot to an adjacent room. The robot can only move to rooms that are connected to its current location by an exit.",
 58 |   z
 59 |     .object({
 60 |       destinationRoomId: z
 61 |         .string()
 62 |         .describe("ID of the adjacent room to move the robot to"),
 63 |     })
 64 |     .describe("Move robot"),
 65 |   "REMOTE"
 66 | );
 67 | 
 68 | const lookWithRobotFunction = new OpenAIFunction(
 69 |   "lookWithRobot",
 70 |   "Look into the robot's current room, returning the fixed and moving items (only if the room is light).",
 71 |   z.object({}).describe("List the contents of the room the robot is in"),
 72 |   "REMOTE"
 73 | );
 74 | 
 75 | const askUserFunction = new OpenAIFunction(
 76 |   "askUser",
 77 |   "Ask the user a question. Use this to get more information or clarify something. Only use this as a last resort",
 78 |   z.object({ question: z.string() }).describe("The question to ask the user"),
 79 |   "LOCAL"
 80 | );
 81 | 
 82 | const haltFunction = new OpenAIFunction(
 83 |   "halt",
 84 |   "Stop operations because you've achieved the goal or can't go any further",
 85 |   z
 86 |     .object({
 87 |       messageToUser: z
 88 |         .string()
 89 |         .describe(
 90 |           "Report back to the user with a message. Remember that you may have performed a number of functions, and the current state of the home is not the state when the user made their request. Never use this to ask for more information."
 91 |         ),
 92 |     })
 93 |     .describe("Stop operations and report back to the user"),
 94 |   "LOCAL"
 95 | );
 96 | 
 97 | export const intentFunction = new OpenAIFunction(
 98 |   "decideBestFunction",
 99 |   "Decide which function to call next based on how to best respond to the user",
100 | 
101 |   z
102 |     .object({
103 |       reasoning: z.string(),
104 |       bestFunction: z.enum([
105 |         toggleLightFunction.name,
106 |         moveRobotFunction.name,
107 |         lookWithRobotFunction.name,
108 |         askUserFunction.name,
109 |         haltFunction.name,
110 |       ]),
111 |     })
112 |     .describe("Function to call"),
113 |   "LOCAL"
114 | );
115 | 
116 | export const allFunctions = [
117 |   toggleLightFunction,
118 |   moveRobotFunction,
119 |   lookWithRobotFunction,
120 |   askUserFunction,
121 |   haltFunction,
122 | ];
123 | 


--------------------------------------------------------------------------------
/party/server.ts:
--------------------------------------------------------------------------------
  1 | import type * as Party from "partykit/server";
  2 | import { DEFAULT_WORLD } from "./world";
  3 | import type { World, FixedItemLight } from "./world";
  4 | import { allFunctions } from "./functions";
  5 | 
  6 | export default class WorldServer implements Party.Server {
  7 |   world: World = DEFAULT_WORLD;
  8 | 
  9 |   constructor(readonly room: Party.Room) {}
 10 | 
 11 |   onConnect(conn: Party.Connection, ctx: Party.ConnectionContext) {
 12 |     conn.send(JSON.stringify({ type: "sync", state: this.world }));
 13 |   }
 14 | 
 15 |   onMessage(message: string, sender: Party.Connection) {}
 16 | 
 17 |   async onRequest(req: Party.Request) {
 18 |     const path = new URL(req.url).pathname;
 19 | 
 20 |     if (req.method === "POST") {
 21 |       const body = await req.json();
 22 |       if (path.endsWith("/call")) {
 23 |         const { fn, args } = body as any;
 24 |         console.log("fn", fn, "args", args);
 25 |         if (!allFunctions.map((f) => f.name).includes(fn)) {
 26 |           return new Response("Unknown Function", { status: 500 });
 27 |         }
 28 | 
 29 |         const result = this.handleFunctionCall(fn, args);
 30 | 
 31 |         this.room.broadcast(
 32 |           JSON.stringify({ type: "sync", state: this.world })
 33 |         );
 34 |         return new Response(JSON.stringify(result, null, 2));
 35 |       }
 36 | 
 37 |       return new Response("Not Found", { status: 404 });
 38 |     } else if (req.method === "GET") {
 39 |       if (path.endsWith("/debug")) {
 40 |         return new Response(JSON.stringify({ world: this.world }, null, 2));
 41 |       } else if (path.endsWith("/world")) {
 42 |         // A static version of this.world which omits moveableItems
 43 |         return new Response(
 44 |           JSON.stringify({ world: this.world.locations }, null, 2)
 45 |         );
 46 |       }
 47 | 
 48 |       return new Response("Not Found", { status: 404 });
 49 |     }
 50 | 
 51 |     return new Response("Method Not Allowed", { status: 405 });
 52 |   }
 53 | 
 54 |   handleFunctionCall(fn: string, args: any) {
 55 |     const robot = this.world.moveableItems.find((i) => i.type === "robot");
 56 |     const robotLocation = this.world.locations.find(
 57 |       (l) => l.id === robot?.location
 58 |     );
 59 |     switch (fn) {
 60 |       case "toggleLight":
 61 |         // Get the location and the item to toggle using args.lightId
 62 |         const location = this.world.locations.find((l) =>
 63 |           l.contents.some((i) => i.type === "light" && i.id === args.lightId)
 64 |         );
 65 |         if (!location) {
 66 |           return { error: "No light found with that ID" };
 67 |         }
 68 |         const light = location.contents.find(
 69 |           (i) => i.type === "light" && i.id === args.lightId
 70 |         ) as FixedItemLight;
 71 |         light.state.on = !light.state.on;
 72 |         return {
 73 |           success: `Light with id <${light.id}> in room with id <${
 74 |             location.id
 75 |           }> is now: ${light.state.on ? "on" : "off"}`,
 76 |         };
 77 |       case "moveRobot":
 78 |         // Check for errors
 79 |         if (!robot) {
 80 |           return { error: "No robot found" };
 81 |         }
 82 |         // Get the current location of the robot
 83 |         if (!robotLocation) {
 84 |           return { error: "Robot is in an unknown location" };
 85 |         }
 86 |         // Check if the destination is a valid exit
 87 |         if (!robotLocation.exits.includes(args.destinationRoomId)) {
 88 |           return {
 89 |             error: `Destination room with ID <${args.destinationRoomId}> is not adjacent to current room with ID <${robotLocation.id}>`,
 90 |           };
 91 |         }
 92 |         // Move the robot
 93 |         robot.location = args.destinationRoomId;
 94 |         return {
 95 |           success: `Robot moved to room with ID <${args.destinationRoomId}>`,
 96 |         };
 97 |       case "lookWithRobot":
 98 |         // Check for errors
 99 |         if (!robot) {
100 |           return { error: "No robot found" };
101 |         }
102 |         // Get the current location of the robot
103 |         if (!robotLocation) {
104 |           return { error: "Robot is in an unknown location" };
105 |         }
106 |         // Is it dark? If so, return an error
107 |         const robotLocationLight = robotLocation.contents.find(
108 |           (i) => i.type === "light"
109 |         ) as FixedItemLight;
110 |         if (!robotLocationLight.state.on) {
111 |           return { error: "It's too dark to see anything" };
112 |         }
113 |         // Return the contents of the location including movable items
114 |         // The format is:
115 |         // ```The robot is in Hallway (ID <hallway>). It can see: Light (ID <hallway-light>), [etc]```
116 |         // The list of items includes fixed items and movable items (excluding the robot itself)
117 |         const fixedItems = robotLocation.contents;
118 |         const movableItems = this.world.moveableItems.filter(
119 |           (i) => i.location === robotLocation.id && i.id !== robot.id
120 |         );
121 |         return {
122 |           success: `The robot is in ${robotLocation.name} (ID <${
123 |             robotLocation.id
124 |           }>). It can see: ${fixedItems
125 |             .map((i) => `${i.name} (ID <${i.id}>, type: ${i.type})`)
126 |             .join(", ")}, ${movableItems
127 |             .map((i) => `${i.name} (ID <${i.id}>, type: ${i.type})`)
128 |             .join(", ")}`,
129 |         };
130 |     }
131 |   }
132 | }
133 | 
134 | WorldServer satisfies Party.Worker;
135 | 


--------------------------------------------------------------------------------
/app/components/Assistant.tsx:
--------------------------------------------------------------------------------
  1 | import { useState } from "react";
  2 | import usePartySocket from "partysocket/react";
  3 | import AssistantTranscript from "./AssistantTranscript";
  4 | 
  5 | // Another example is:
  6 | // tell me where my dog is. you'll know it's my dog when you see it
  7 | const DEFAULT_INSTRUCTION = "turn on the light in the lounge";
  8 | 
  9 | export default function Debug() {
 10 |   const [assistant, setAssistant] = useState<any>(null);
 11 |   const [input, setInput] = useState<string>(DEFAULT_INSTRUCTION);
 12 | 
 13 |   const socket = usePartySocket({
 14 |     party: "assistant",
 15 |     room: "default",
 16 |     onMessage: (evt) => {
 17 |       const data = JSON.parse(evt.data);
 18 |       if (data.type === "sync") {
 19 |         setAssistant(data.state);
 20 |       } else if (data.type === "askUser") {
 21 |         socket.send(
 22 |           JSON.stringify({
 23 |             type: "askUserResponse",
 24 |             toolCallId: data.toolCallId,
 25 |             answer: prompt(data.question),
 26 |           })
 27 |         );
 28 |       }
 29 |     },
 30 |   });
 31 | 
 32 |   const handleSubmit = (e: React.FormEvent<HTMLFormElement>) => {
 33 |     e.preventDefault();
 34 |     socket.send(JSON.stringify({ type: "instruct", instruction: input }));
 35 |     setInput("");
 36 |   };
 37 | 
 38 |   // If the final message in the assistant.transcript...
 39 |   // - has .role === assistant
 40 |   // - has .tool_calls
 41 |   // - has .tool_calls[0].function.name === "halt"
 42 |   // then the assistant has finished its task. So parse:
 43 |   // `.tool_calls[0].function.arguments` and get the value of 'messageToUser'
 44 |   let messageToUser = null;
 45 |   if (
 46 |     assistant && // if assistant is not null
 47 |     assistant.transcript && // if assistant.transcript is not null
 48 |     assistant.transcript.length > 0
 49 |   ) {
 50 |     const last = assistant.transcript[assistant.transcript.length - 1];
 51 |     if (
 52 |       last.role === "assistant" &&
 53 |       last.tool_calls &&
 54 |       last.tool_calls[0].function.name === "halt"
 55 |     ) {
 56 |       const payload = JSON.parse(last.tool_calls[0].function.arguments);
 57 |       messageToUser = payload.messageToUser;
 58 |     }
 59 |   }
 60 | 
 61 |   return (
 62 |     <div
 63 |       style={{
 64 |         display: "flex",
 65 |         flexDirection: "column",
 66 |         justifyContent: "stretch",
 67 |         gap: "2rem",
 68 |         backgroundColor: "#fff",
 69 |       }}
 70 |     >
 71 |       <h1>Agent</h1>
 72 |       {!assistant?.instruction && (
 73 |         <>
 74 |           <form
 75 |             onSubmit={handleSubmit}
 76 |             style={{
 77 |               display: "flex",
 78 |               justifyContent: "stretch",
 79 |               gap: "0.5rem",
 80 |               width: "100%",
 81 |               fontSize: "1.25rem",
 82 |             }}
 83 |           >
 84 |             <input
 85 |               type="text"
 86 |               name="instruction"
 87 |               value={input}
 88 |               onChange={(e) => setInput(e.target.value)}
 89 |               style={{ flex: 1 }}
 90 |             />
 91 |             <button type="submit">Instruct</button>
 92 |           </form>
 93 |           <div>
 94 |             <h2>Try these examples</h2>
 95 |             <ul>
 96 |               <li>turn on the light in the kitchen</li>
 97 |               <li>turn off all the lights</li>
 98 |               <li>turn on the light for sally</li>
 99 |               <li>find my dog</li>
100 |               <li>turn the lights in the office to something appropriate</li>
101 |             </ul>
102 |           </div>
103 |         </>
104 |       )}
105 |       {assistant?.instruction && (
106 |         <>
107 |           <div
108 |             style={{
109 |               fontSize: "1.25rem",
110 |               backgroundColor: "#eee",
111 |               padding: "1rem",
112 |               display: "flex",
113 |               flexDirection: "column",
114 |               justifyContent: "start",
115 |               alignItems: "start",
116 |               gap: "0.5rem",
117 |             }}
118 |           >
119 |             <div>
120 |               <strong>User</strong>
121 |               <br />
122 |               {assistant.instruction}
123 |             </div>
124 |             <div>
125 |               <strong>Agent</strong>
126 |               <br />
127 |               {messageToUser !== null ? (
128 |                 <span style={{ backgroundColor: "#86efac" }}>
129 |                   {messageToUser}
130 |                 </span>
131 |               ) : (
132 |                 <span>
133 |                   <i>Working...</i>
134 |                 </span>
135 |               )}
136 |             </div>
137 |             {messageToUser && (
138 |               <div>
139 |                 <a
140 |                   href="#"
141 |                   style={{
142 |                     color: "#666",
143 |                     textDecoration: "underline",
144 |                     fontSize: "1rem",
145 |                   }}
146 |                   onClick={() => setAssistant(null)}
147 |                 >
148 |                   Reset
149 |                 </a>
150 |               </div>
151 |             )}
152 |           </div>
153 |           <div style={{ flexGrow: 1 }}>
154 |             <h2>Activity</h2>
155 |             <p>
156 |               <strong>Tool calls remaining before cut-off:</strong>{" "}
157 |               {assistant.remaining}
158 |             </p>
159 |             <AssistantTranscript transcript={assistant.transcript} />
160 |           </div>
161 |           <details>
162 |             <summary
163 |               style={{
164 |                 fontSize: "1rem",
165 |                 marginBottom: "1rem",
166 |                 opacity: 0.5,
167 |               }}
168 |             >
169 |               Transcript [debug]
170 |             </summary>
171 |             <pre
172 |               style={{
173 |                 fontSize: "0.85rem",
174 |                 whiteSpace: "pre-wrap",
175 |                 wordWrap: "break-word",
176 |               }}
177 |             >
178 |               {JSON.stringify(assistant.transcript, null, 2)}
179 |             </pre>
180 |           </details>
181 |         </>
182 |       )}
183 |     </div>
184 |   );
185 | }
186 | 


--------------------------------------------------------------------------------
/public/normalize.css:
--------------------------------------------------------------------------------
  1 | /*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */
  2 | 
  3 | /* Document
  4 |    ========================================================================== */
  5 | 
  6 | /**
  7 |  * 1. Correct the line height in all browsers.
  8 |  * 2. Prevent adjustments of font size after orientation changes in iOS.
  9 |  */
 10 | 
 11 | html {
 12 |   line-height: 1.15; /* 1 */
 13 |   -webkit-text-size-adjust: 100%; /* 2 */
 14 | }
 15 | 
 16 | /* Sections
 17 |    ========================================================================== */
 18 | 
 19 | /**
 20 |  * Remove the margin in all browsers.
 21 |  */
 22 | 
 23 | body {
 24 |   margin: 0;
 25 | }
 26 | 
 27 | /**
 28 |  * Render the `main` element consistently in IE.
 29 |  */
 30 | 
 31 | main {
 32 |   display: block;
 33 | }
 34 | 
 35 | /**
 36 |  * Correct the font size and margin on `h1` elements within `section` and
 37 |  * `article` contexts in Chrome, Firefox, and Safari.
 38 |  */
 39 | 
 40 | h1 {
 41 |   font-size: 2em;
 42 |   margin: 0.67em 0;
 43 | }
 44 | 
 45 | /* Grouping content
 46 |    ========================================================================== */
 47 | 
 48 | /**
 49 |  * 1. Add the correct box sizing in Firefox.
 50 |  * 2. Show the overflow in Edge and IE.
 51 |  */
 52 | 
 53 | hr {
 54 |   box-sizing: content-box; /* 1 */
 55 |   height: 0; /* 1 */
 56 |   overflow: visible; /* 2 */
 57 | }
 58 | 
 59 | /**
 60 |  * 1. Correct the inheritance and scaling of font size in all browsers.
 61 |  * 2. Correct the odd `em` font sizing in all browsers.
 62 |  */
 63 | 
 64 | pre {
 65 |   font-family: monospace, monospace; /* 1 */
 66 |   font-size: 1em; /* 2 */
 67 | }
 68 | 
 69 | /* Text-level semantics
 70 |    ========================================================================== */
 71 | 
 72 | /**
 73 |  * Remove the gray background on active links in IE 10.
 74 |  */
 75 | 
 76 | a {
 77 |   background-color: transparent;
 78 | }
 79 | 
 80 | /**
 81 |  * 1. Remove the bottom border in Chrome 57-
 82 |  * 2. Add the correct text decoration in Chrome, Edge, IE, Opera, and Safari.
 83 |  */
 84 | 
 85 | abbr[title] {
 86 |   border-bottom: none; /* 1 */
 87 |   text-decoration: underline; /* 2 */
 88 |   text-decoration: underline dotted; /* 2 */
 89 | }
 90 | 
 91 | /**
 92 |  * Add the correct font weight in Chrome, Edge, and Safari.
 93 |  */
 94 | 
 95 | b,
 96 | strong {
 97 |   font-weight: bolder;
 98 | }
 99 | 
100 | /**
101 |  * 1. Correct the inheritance and scaling of font size in all browsers.
102 |  * 2. Correct the odd `em` font sizing in all browsers.
103 |  */
104 | 
105 | code,
106 | kbd,
107 | samp {
108 |   font-family: monospace, monospace; /* 1 */
109 |   font-size: 1em; /* 2 */
110 | }
111 | 
112 | /**
113 |  * Add the correct font size in all browsers.
114 |  */
115 | 
116 | small {
117 |   font-size: 80%;
118 | }
119 | 
120 | /**
121 |  * Prevent `sub` and `sup` elements from affecting the line height in
122 |  * all browsers.
123 |  */
124 | 
125 | sub,
126 | sup {
127 |   font-size: 75%;
128 |   line-height: 0;
129 |   position: relative;
130 |   vertical-align: baseline;
131 | }
132 | 
133 | sub {
134 |   bottom: -0.25em;
135 | }
136 | 
137 | sup {
138 |   top: -0.5em;
139 | }
140 | 
141 | /* Embedded content
142 |    ========================================================================== */
143 | 
144 | /**
145 |  * Remove the border on images inside links in IE 10.
146 |  */
147 | 
148 | img {
149 |   border-style: none;
150 | }
151 | 
152 | /* Forms
153 |    ========================================================================== */
154 | 
155 | /**
156 |  * 1. Change the font styles in all browsers.
157 |  * 2. Remove the margin in Firefox and Safari.
158 |  */
159 | 
160 | button,
161 | input,
162 | optgroup,
163 | select,
164 | textarea {
165 |   font-family: inherit; /* 1 */
166 |   font-size: 100%; /* 1 */
167 |   line-height: 1.15; /* 1 */
168 |   margin: 0; /* 2 */
169 | }
170 | 
171 | /**
172 |  * Show the overflow in IE.
173 |  * 1. Show the overflow in Edge.
174 |  */
175 | 
176 | button,
177 | input {
178 |   /* 1 */
179 |   overflow: visible;
180 | }
181 | 
182 | /**
183 |  * Remove the inheritance of text transform in Edge, Firefox, and IE.
184 |  * 1. Remove the inheritance of text transform in Firefox.
185 |  */
186 | 
187 | button,
188 | select {
189 |   /* 1 */
190 |   text-transform: none;
191 | }
192 | 
193 | /**
194 |  * Correct the inability to style clickable types in iOS and Safari.
195 |  */
196 | 
197 | button,
198 | [type="button"],
199 | [type="reset"],
200 | [type="submit"] {
201 |   -webkit-appearance: button;
202 | }
203 | 
204 | /**
205 |  * Remove the inner border and padding in Firefox.
206 |  */
207 | 
208 | button::-moz-focus-inner,
209 | [type="button"]::-moz-focus-inner,
210 | [type="reset"]::-moz-focus-inner,
211 | [type="submit"]::-moz-focus-inner {
212 |   border-style: none;
213 |   padding: 0;
214 | }
215 | 
216 | /**
217 |  * Restore the focus styles unset by the previous rule.
218 |  */
219 | 
220 | button:-moz-focusring,
221 | [type="button"]:-moz-focusring,
222 | [type="reset"]:-moz-focusring,
223 | [type="submit"]:-moz-focusring {
224 |   outline: 1px dotted ButtonText;
225 | }
226 | 
227 | /**
228 |  * Correct the padding in Firefox.
229 |  */
230 | 
231 | fieldset {
232 |   padding: 0.35em 0.75em 0.625em;
233 | }
234 | 
235 | /**
236 |  * 1. Correct the text wrapping in Edge and IE.
237 |  * 2. Correct the color inheritance from `fieldset` elements in IE.
238 |  * 3. Remove the padding so developers are not caught out when they zero out
239 |  *    `fieldset` elements in all browsers.
240 |  */
241 | 
242 | legend {
243 |   box-sizing: border-box; /* 1 */
244 |   color: inherit; /* 2 */
245 |   display: table; /* 1 */
246 |   max-width: 100%; /* 1 */
247 |   padding: 0; /* 3 */
248 |   white-space: normal; /* 1 */
249 | }
250 | 
251 | /**
252 |  * Add the correct vertical alignment in Chrome, Firefox, and Opera.
253 |  */
254 | 
255 | progress {
256 |   vertical-align: baseline;
257 | }
258 | 
259 | /**
260 |  * Remove the default vertical scrollbar in IE 10+.
261 |  */
262 | 
263 | textarea {
264 |   overflow: auto;
265 | }
266 | 
267 | /**
268 |  * 1. Add the correct box sizing in IE 10.
269 |  * 2. Remove the padding in IE 10.
270 |  */
271 | 
272 | [type="checkbox"],
273 | [type="radio"] {
274 |   box-sizing: border-box; /* 1 */
275 |   padding: 0; /* 2 */
276 | }
277 | 
278 | /**
279 |  * Correct the cursor style of increment and decrement buttons in Chrome.
280 |  */
281 | 
282 | [type="number"]::-webkit-inner-spin-button,
283 | [type="number"]::-webkit-outer-spin-button {
284 |   height: auto;
285 | }
286 | 
287 | /**
288 |  * 1. Correct the odd appearance in Chrome and Safari.
289 |  * 2. Correct the outline style in Safari.
290 |  */
291 | 
292 | [type="search"] {
293 |   -webkit-appearance: textfield; /* 1 */
294 |   outline-offset: -2px; /* 2 */
295 | }
296 | 
297 | /**
298 |  * Remove the inner padding in Chrome and Safari on macOS.
299 |  */
300 | 
301 | [type="search"]::-webkit-search-decoration {
302 |   -webkit-appearance: none;
303 | }
304 | 
305 | /**
306 |  * 1. Correct the inability to style clickable types in iOS and Safari.
307 |  * 2. Change font properties to `inherit` in Safari.
308 |  */
309 | 
310 | ::-webkit-file-upload-button {
311 |   -webkit-appearance: button; /* 1 */
312 |   font: inherit; /* 2 */
313 | }
314 | 
315 | /* Interactive
316 |    ========================================================================== */
317 | 
318 | /*
319 |  * Add the correct display in Edge, IE 10+, and Firefox.
320 |  */
321 | 
322 | details {
323 |   display: block;
324 | }
325 | 
326 | /*
327 |  * Add the correct display in all browsers.
328 |  */
329 | 
330 | summary {
331 |   display: list-item;
332 | }
333 | 
334 | /* Misc
335 |    ========================================================================== */
336 | 
337 | /**
338 |  * Add the correct display in IE 10+.
339 |  */
340 | 
341 | template {
342 |   display: none;
343 | }
344 | 
345 | /**
346 |  * Add the correct display in IE 10.
347 |  */
348 | 
349 | [hidden] {
350 |   display: none;
351 | }
352 | 


--------------------------------------------------------------------------------
/party/assistant.ts:
--------------------------------------------------------------------------------
  1 | import type * as Party from "partykit/server";
  2 | import { intentFunction, allFunctions } from "./functions";
  3 | import { getChatCompletionResponse } from "./openai";
  4 | import type { OpenAIMessage } from "./openai";
  5 | import type {
  6 |   SyncMessage,
  7 |   AskUserMessage,
  8 |   AskUserResponseMessage,
  9 | } from "./messages";
 10 | import { TEST_INSTRUCTION, TEST_TRANSCRIPT } from "./test";
 11 | 
 12 | const MAX_INSTRUCTIONS = 20;
 13 | 
 14 | export default class AssistantServer implements Party.Server {
 15 |   instruction: string | null = null; //TEST_INSTRUCTION;
 16 |   remaining = MAX_INSTRUCTIONS;
 17 |   transcript: OpenAIMessage[] = []; //TEST_TRANSCRIPT;
 18 |   halt: boolean = false;
 19 |   suspended: boolean = false;
 20 | 
 21 |   constructor(readonly room: Party.Room) {}
 22 | 
 23 |   onConnect(conn: Party.Connection) {
 24 |     this.broadcastSync();
 25 |   }
 26 | 
 27 |   async onMessage(message: string, conn: Party.Connection) {
 28 |     const data = JSON.parse(message);
 29 |     if (data.type === "instruct") {
 30 |       this.instruction = data.instruction;
 31 |       this.remaining = MAX_INSTRUCTIONS;
 32 |       this.transcript = [];
 33 |       this.halt = false;
 34 |       this.broadcastSync();
 35 |       await this.run();
 36 |     } else if (data.type === "askUserResponse") {
 37 |       if (this.suspended) {
 38 |         this.transcript = [
 39 |           ...this.transcript,
 40 |           {
 41 |             tool_call_id: data.toolCallId,
 42 |             role: "tool",
 43 |             name: "askUser",
 44 |             content: JSON.stringify({ success: data.answer }),
 45 |           },
 46 |         ];
 47 |         this.suspended = false;
 48 |         await this.run();
 49 |       }
 50 |     }
 51 |   }
 52 | 
 53 |   async run() {
 54 |     while (this.remaining > 0 && !this.halt && !this.suspended) {
 55 |       await this.tick();
 56 |       this.remaining--;
 57 |       this.broadcastSync();
 58 |     }
 59 |   }
 60 | 
 61 |   broadcastSync() {
 62 |     this.room.broadcast(
 63 |       JSON.stringify({
 64 |         type: "sync",
 65 |         state: {
 66 |           instruction: this.instruction,
 67 |           remaining: this.remaining,
 68 |           transcript: this.transcript,
 69 |         },
 70 |       })
 71 |     );
 72 |   }
 73 | 
 74 |   async tick() {
 75 |     // Decide on a function and then perform it
 76 |     // In order, this is what we'll do:
 77 |     // - Fetch the current world state
 78 |     // - Build a transcript which includes:
 79 |     //    - The world state
 80 |     //    - The instruction
 81 |     //    - A list of functions performed so far and their results
 82 |     // - Run this transcript using intentFunction
 83 |     // - Perform the returned function
 84 |     // - Add to the transcript and return the result
 85 |     const res = await this.room.context.parties.main
 86 |       .get(this.room.id)
 87 |       .fetch("/world", { method: "GET" });
 88 |     const world = await res.json();
 89 | 
 90 |     const intro = [
 91 |       {
 92 |         role: "system",
 93 |         content:
 94 |           "You are a helpful AI assistant controlling a smart home. When the user refers to 'you' they either mean the entire home or the moveable robot that you control. You always know the layout of the home, the devices in it (which includes the robot), and their current states. You do not know the position of moveable items such as people, animals, and objects that might be carried, and you know even know their names initially. You can build your knowledge by using various devices. You call functions to control the devices in the home.",
 95 |       },
 96 |       {
 97 |         role: "system",
 98 |         content: `The current state of the home follows. The state of devices is up to date with your most recent functions. Consult the transcript for any other discoveries you have made:
 99 |         
100 |         ${JSON.stringify(world, null, 2)}`,
101 |       },
102 |       {
103 |         role: "system",
104 |         content:
105 |           "The user's instruction follows. Your goal is to fulfil it to the best of your ability. It may take a sequence of many instructions to achieve your goal, and you may have to deliberately build knowledge so you know enough to reach the goal. At each step, call the best function to move you closer to your goal. When you're done, call the halt function.",
106 |       },
107 |       {
108 |         role: "user",
109 |         content: `Instruction: ${this.instruction}`,
110 |       },
111 |     ] as OpenAIMessage[];
112 | 
113 |     const intentMessage = await getChatCompletionResponse({
114 |       messages: [
115 |         ...intro,
116 |         ...this.transcript,
117 |         {
118 |           role: "system",
119 |           content:
120 |             "You now have the opportunity to decide how best to respond. Choose which function will be best.",
121 |         },
122 |         {
123 |           role: "user",
124 |           content:
125 |             "Which of these functions will best move you closer to your goal?\n\n" +
126 |             allFunctions
127 |               .map((f) => `- ${f.name} -- ${f.description}`)
128 |               .join("\n"),
129 |         },
130 |       ],
131 |       tool: intentFunction,
132 |     });
133 | 
134 |     if (!intentMessage || !intentMessage.tool_calls) {
135 |       console.error("No intent message");
136 |       return;
137 |     }
138 | 
139 |     const intentArgs = JSON.parse(
140 |       intentMessage.tool_calls[0].function.arguments
141 |     );
142 |     const intentFunctionName = intentArgs.bestFunction;
143 | 
144 |     // Get the function
145 |     const functionToCall = allFunctions.find(
146 |       (f) => f.name === intentFunctionName
147 |     );
148 |     const reasoning = intentArgs.reasoning;
149 | 
150 |     if (!functionToCall) {
151 |       console.error("Unknown function", intentFunctionName);
152 |       return;
153 |     }
154 | 
155 |     this.transcript = [
156 |       ...this.transcript,
157 |       {
158 |         role: "assistant",
159 |         content: `I have decided to call the function ${functionToCall.name} because: ${reasoning}`,
160 |       },
161 |     ];
162 |     this.broadcastSync();
163 | 
164 |     // Clarify any arguments for the function, and get the call itself
165 |     const toolCallMessage = await getChatCompletionResponse({
166 |       messages: [
167 |         ...intro,
168 |         ...this.transcript,
169 |         {
170 |           role: "system",
171 |           content:
172 |             "Call the available function to move closer to your goal. You may have to add arguments. Only use a function call. Do not reply with a text message.",
173 |         },
174 |       ],
175 |       tool: functionToCall,
176 |     });
177 | 
178 |     if (!toolCallMessage || !toolCallMessage.tool_calls) {
179 |       console.error("No tool call message");
180 |       return;
181 |     }
182 | 
183 |     // Add the tool call to the transcript, before performing it
184 |     this.transcript = [...this.transcript, toolCallMessage];
185 |     this.broadcastSync();
186 | 
187 |     if (functionToCall.dispatchType === "LOCAL") {
188 |       // We need to handle the function right here, instead of calling the remote world simulator
189 | 
190 |       // Abort without calling if the function is 'halt'
191 |       if (functionToCall.name === "halt") {
192 |         this.halt = true;
193 |         return;
194 |       }
195 | 
196 |       if (functionToCall.name === "askUser") {
197 |         // If the function is 'askUser', we need to ask the user for input
198 |         // We'll need to handle this in a different way
199 |         this.room.broadcast(
200 |           JSON.stringify(<AskUserMessage>{
201 |             type: "askUser",
202 |             question: JSON.parse(
203 |               toolCallMessage.tool_calls[0].function.arguments
204 |             ).question,
205 |             toolCallId: toolCallMessage.tool_calls[0].id,
206 |           })
207 |         );
208 |         this.suspended = true;
209 |         return;
210 |       }
211 |     }
212 | 
213 |     // It's a REMOTE dispatchType. Go ahead and call the function
214 | 
215 |     const toolCallArgs = JSON.parse(
216 |       toolCallMessage.tool_calls[0].function.arguments
217 |     );
218 | 
219 |     // Perform the function by calling the function on the server
220 |     const callResult = await this.room.context.parties.main
221 |       .get(this.room.id)
222 |       .fetch("/call", {
223 |         method: "POST",
224 |         body: JSON.stringify({
225 |           fn: functionToCall.name,
226 |           args: toolCallArgs,
227 |         }),
228 |       });
229 |     const result = await callResult.json();
230 | 
231 |     // Add the tool call and the result to the transcript
232 |     this.transcript = [
233 |       ...this.transcript,
234 |       {
235 |         tool_call_id: toolCallMessage.tool_calls[0].id,
236 |         role: "tool",
237 |         name: functionToCall.name,
238 |         content: JSON.stringify(result, null, 2),
239 |       },
240 |     ];
241 |     this.broadcastSync();
242 |   }
243 | }
244 | 
245 | AssistantServer satisfies Party.Worker;
246 | 


--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
  1 | {
  2 |   "compilerOptions": {
  3 |     /* Visit https://aka.ms/tsconfig to read more about this file */
  4 | 
  5 |     /* Projects */
  6 |     // "incremental": true,                              /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
  7 |     // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
  8 |     // "tsBuildInfoFile": "./.tsbuildinfo",              /* Specify the path to .tsbuildinfo incremental compilation file. */
  9 |     // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects. */
 10 |     // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
 11 |     // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
 12 | 
 13 |     /* Language and Environment */
 14 |     "target": "es2020" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
 15 |     // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
 16 |     "jsx": "react-jsx" /* Specify what JSX code is generated. */,
 17 |     // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */
 18 |     // "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
 19 |     // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
 20 |     // "jsxFragmentFactory": "",                         /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
 21 |     // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
 22 |     // "reactNamespace": "",                             /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
 23 |     // "noLib": true,                                    /* Disable including any library files, including the default lib.d.ts. */
 24 |     // "useDefineForClassFields": true,                  /* Emit ECMAScript-standard-compliant class fields. */
 25 |     // "moduleDetection": "auto",                        /* Control what method is used to detect module-format JS files. */
 26 | 
 27 |     /* Modules */
 28 |     "module": "ES2020" /* Specify what module code is generated. */,
 29 |     // "rootDir": "./",                                  /* Specify the root folder within your source files. */
 30 |     "moduleResolution": "Bundler" /* Specify how TypeScript looks up a file from a given module specifier. */,
 31 |     // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
 32 |     // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
 33 |     // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
 34 |     // "typeRoots": [],                                  /* Specify multiple folders that act like './node_modules/@types'. */
 35 |     // "types": [],                                      /* Specify type package names to be included without being referenced in a source file. */
 36 |     // "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
 37 |     // "moduleSuffixes": [],                             /* List of file name suffixes to search when resolving a module. */
 38 |     // "allowImportingTsExtensions": true,               /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
 39 |     // "resolvePackageJsonExports": true,                /* Use the package.json 'exports' field when resolving package imports. */
 40 |     // "resolvePackageJsonImports": true,                /* Use the package.json 'imports' field when resolving imports. */
 41 |     // "customConditions": [],                           /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
 42 |     "resolveJsonModule": true /* Enable importing .json files. */,
 43 |     // "allowArbitraryExtensions": true,                 /* Enable importing files with any extension, provided a declaration file is present. */
 44 |     // "noResolve": true,                                /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
 45 | 
 46 |     /* JavaScript Support */
 47 |     // "allowJs": true,                                  /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
 48 |     // "checkJs": true,                                  /* Enable error reporting in type-checked JavaScript files. */
 49 |     // "maxNodeModuleJsDepth": 1,                        /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
 50 | 
 51 |     /* Emit */
 52 |     // "declaration": true,                              /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
 53 |     // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
 54 |     // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
 55 |     // "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
 56 |     // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */
 57 |     // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
 58 |     // "outDir": "./",                                   /* Specify an output folder for all emitted files. */
 59 |     // "removeComments": true,                           /* Disable emitting comments. */
 60 |     "noEmit": true /* Disable emitting files from a compilation. */,
 61 |     // "importHelpers": true,                            /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
 62 |     // "importsNotUsedAsValues": "remove",               /* Specify emit/checking behavior for imports that are only used for types. */
 63 |     // "downlevelIteration": true,                       /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
 64 |     // "sourceRoot": "",                                 /* Specify the root path for debuggers to find the reference source code. */
 65 |     // "mapRoot": "",                                    /* Specify the location where debugger should locate map files instead of generated locations. */
 66 |     // "inlineSources": true,                            /* Include source code in the sourcemaps inside the emitted JavaScript. */
 67 |     // "emitBOM": true,                                  /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
 68 |     // "newLine": "crlf",                                /* Set the newline character for emitting files. */
 69 |     // "stripInternal": true,                            /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
 70 |     // "noEmitHelpers": true,                            /* Disable generating custom helper functions like '__extends' in compiled output. */
 71 |     // "noEmitOnError": true,                            /* Disable emitting files if any type checking errors are reported. */
 72 |     // "preserveConstEnums": true,                       /* Disable erasing 'const enum' declarations in generated code. */
 73 |     // "declarationDir": "./",                           /* Specify the output directory for generated declaration files. */
 74 |     // "preserveValueImports": true,                     /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
 75 | 
 76 |     /* Interop Constraints */
 77 |     // "isolatedModules": true,                          /* Ensure that each file can be safely transpiled without relying on other imports. */
 78 |     "verbatimModuleSyntax": true /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */,
 79 |     // "allowSyntheticDefaultImports": true,             /* Allow 'import x from y' when a module doesn't have a default export. */
 80 |     "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
 81 |     // "preserveSymlinks": true,                         /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
 82 |     "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
 83 | 
 84 |     /* Type Checking */
 85 |     "strict": true /* Enable all strict type-checking options. */,
 86 |     // "noImplicitAny": true,                            /* Enable error reporting for expressions and declarations with an implied 'any' type. */
 87 |     // "strictNullChecks": true,                         /* When type checking, take into account 'null' and 'undefined'. */
 88 |     // "strictFunctionTypes": true,                      /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
 89 |     // "strictBindCallApply": true,                      /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
 90 |     // "strictPropertyInitialization": true,             /* Check for class properties that are declared but not set in the constructor. */
 91 |     // "noImplicitThis": true,                           /* Enable error reporting when 'this' is given the type 'any'. */
 92 |     // "useUnknownInCatchVariables": true,               /* Default catch clause variables as 'unknown' instead of 'any'. */
 93 |     // "alwaysStrict": true,                             /* Ensure 'use strict' is always emitted. */
 94 |     // "noUnusedLocals": true,                           /* Enable error reporting when local variables aren't read. */
 95 |     // "noUnusedParameters": true,                       /* Raise an error when a function parameter isn't read. */
 96 |     // "exactOptionalPropertyTypes": true,               /* Interpret optional property types as written, rather than adding 'undefined'. */
 97 |     // "noImplicitReturns": true,                        /* Enable error reporting for codepaths that do not explicitly return in a function. */
 98 |     // "noFallthroughCasesInSwitch": true,               /* Enable error reporting for fallthrough cases in switch statements. */
 99 |     // "noUncheckedIndexedAccess": true,                 /* Add 'undefined' to a type when accessed using an index. */
100 |     // "noImplicitOverride": true,                       /* Ensure overriding members in derived classes are marked with an override modifier. */
101 |     // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type. */
102 |     // "allowUnusedLabels": true,                        /* Disable error reporting for unused labels. */
103 |     // "allowUnreachableCode": true,                     /* Disable error reporting for unreachable code. */
104 | 
105 |     /* Completeness */
106 |     // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
107 |     "skipLibCheck": true /* Skip type checking all .d.ts files. */,
108 |   },
109 | }
110 | 


--------------------------------------------------------------------------------