├── src
├── app
│ ├── favicon.ico
│ ├── layout.tsx
│ ├── api
│ │ └── copilotkit
│ │ │ └── route.ts
│ ├── globals.css
│ └── page.tsx
├── lib
│ ├── types.ts
│ └── chat-input-context.tsx
└── components
│ ├── CustomChatInput.tsx
│ ├── ApiKeyInput.tsx
│ └── ArtifactPanel.tsx
├── agent
├── .gitignore
├── langgraph.json
├── requirements.txt
├── railway.json
├── .dockerignore
├── Dockerfile
├── server.py
├── pyproject.toml
├── DEPLOY.md
└── agent.py
├── postcss.config.mjs
├── public
├── vercel.svg
├── window.svg
├── file.svg
├── globe.svg
└── next.svg
├── next.config.ts
├── eslint.config.mjs
├── .gitignore
├── tsconfig.json
├── LICENSE
├── package.json
└── README.md
/src/app/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CopilotKit/scene-creator-copilot/HEAD/src/app/favicon.ico
--------------------------------------------------------------------------------
/agent/.gitignore:
--------------------------------------------------------------------------------
1 | venv/
2 | __pycache__/
3 | *.pyc
4 | .env
5 | .vercel
6 |
7 | # python
8 | .venv/
9 | .langgraph_api/
--------------------------------------------------------------------------------
/postcss.config.mjs:
--------------------------------------------------------------------------------
1 | const config = {
2 | plugins: ["@tailwindcss/postcss"],
3 | };
4 |
5 | export default config;
6 |
--------------------------------------------------------------------------------
/public/vercel.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/next.config.ts:
--------------------------------------------------------------------------------
1 | import type { NextConfig } from "next";
2 |
3 | const nextConfig: NextConfig = {
4 | /* config options here */
5 | };
6 |
7 | export default nextConfig;
8 |
--------------------------------------------------------------------------------
/agent/langgraph.json:
--------------------------------------------------------------------------------
1 | {
2 | "python_version": "3.12",
3 | "dockerfile_lines": [],
4 | "dependencies": ["."],
5 | "graphs": {
6 | "sample_agent": "./agent.py:graph"
7 | },
8 | "env": ".env",
9 | "http": {
10 | "app": "./server.py:app"
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/agent/requirements.txt:
--------------------------------------------------------------------------------
1 | langchain>=0.3.28
2 | langgraph>=0.6.6
3 | langsmith>=0.4.23
4 | langchain-google-genai>=3.1.0
5 | langchain-core>=1.0.5
6 | httpx>=0.28.0
7 | fastapi>=0.115.5,<1.0.0
8 | uvicorn>=0.29.0,<1.0.0
9 | python-dotenv>=1.0.0,<2.0.0
10 | langgraph-cli[inmem]>=0.3.3
11 |
--------------------------------------------------------------------------------
/agent/railway.json:
--------------------------------------------------------------------------------
1 | {
2 | "$schema": "https://railway.app/railway.schema.json",
3 | "build": {
4 | "builder": "DOCKERFILE",
5 | "dockerfilePath": "Dockerfile"
6 | },
7 | "deploy": {
8 | "startCommand": "python -m langgraph_cli dev --port 8000 --host 0.0.0.0",
9 | "restartPolicyType": "ON_FAILURE",
10 | "restartPolicyMaxRetries": 10
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/public/window.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/public/file.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/eslint.config.mjs:
--------------------------------------------------------------------------------
1 | import nextCoreWebVitals from "eslint-config-next/core-web-vitals";
2 | import nextTypescript from "eslint-config-next/typescript";
3 |
4 | const eslintConfig = [
5 | ...nextCoreWebVitals,
6 | ...nextTypescript,
7 | {
8 | ignores: [
9 | "node_modules/**",
10 | ".next/**",
11 | "out/**",
12 | "build/**",
13 | "next-env.d.ts",
14 | "agent",
15 | ],
16 | },
17 | ];
18 |
19 | export default eslintConfig;
20 |
--------------------------------------------------------------------------------
/agent/.dockerignore:
--------------------------------------------------------------------------------
1 | # Python
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 | *.so
6 | .Python
7 | venv/
8 | env/
9 | ENV/
10 | .venv
11 | pip-log.txt
12 | pip-delete-this-directory.txt
13 |
14 | # IDEs
15 | .vscode/
16 | .idea/
17 | *.swp
18 | *.swo
19 | *~
20 |
21 | # OS
22 | .DS_Store
23 | Thumbs.db
24 |
25 | # Git
26 | .git/
27 | .gitignore
28 |
29 | # Testing
30 | .pytest_cache/
31 | .coverage
32 | htmlcov/
33 |
34 | # Development
35 | .env.local
36 | .env.development
37 | *.log
38 |
--------------------------------------------------------------------------------
/agent/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3.12-slim
2 | WORKDIR /app
3 |
4 | # Install build dependencies and uv
5 | RUN apt-get update && apt-get install -y curl gcc && rm -rf /var/lib/apt/lists/*
6 | RUN curl -LsSf https://astral.sh/uv/install.sh | sh
7 | ENV PATH="/root/.local/bin:${PATH}"
8 |
9 | # Copy dependency files
10 | COPY requirements.txt pyproject.toml /app/
11 |
12 | # Install dependencies using uv
13 | RUN uv pip install --system -r requirements.txt
14 |
15 | # Copy application files
16 | COPY . /app
17 |
18 | # Expose port (Railway will use PORT env var)
19 | EXPOSE 8000
20 |
21 | # Start LangGraph dev server
22 | CMD ["python", "-m", "langgraph_cli", "dev", "--port", "8000", "--host", "0.0.0.0"]
23 |
--------------------------------------------------------------------------------
/agent/server.py:
--------------------------------------------------------------------------------
1 | """
2 | Custom routes to extend the LangGraph API server.
3 |
4 | This mounts a /generated route for serving generated images.
5 | Configure in langgraph.json via http.app setting.
6 | """
7 | from pathlib import Path
8 | from fastapi import FastAPI
9 | from fastapi.staticfiles import StaticFiles
10 |
11 | # Create the generated images directory
12 | GENERATED_DIR = Path(__file__).parent / "generated"
13 | GENERATED_DIR.mkdir(exist_ok=True)
14 |
15 | # Custom FastAPI app to mount alongside LangGraph routes
16 | app = FastAPI(title="Custom Routes")
17 |
18 | # Mount static files for generated images
19 | app.mount("/generated", StaticFiles(directory=str(GENERATED_DIR)), name="generated")
20 |
--------------------------------------------------------------------------------
/agent/pyproject.toml:
--------------------------------------------------------------------------------
1 | # Python dependencies for the Scene Creator agent
2 | # This file documents the project dependencies in a standard format
3 | # For installation, use: uv pip install -r requirements.txt
4 |
5 | [project]
6 | name = "scene-creator-agent"
7 | version = "0.1.0"
8 | description = "LangGraph agent for scene creation with Gemini 3 and Nano Banana"
9 | requires-python = ">=3.10"
10 | dependencies = [
11 | "langchain>=0.3.28",
12 | "langgraph>=0.6.6",
13 | "langsmith>=0.4.23",
14 | "langchain-google-genai>=3.1.0",
15 | "langchain-core>=1.0.5",
16 | "httpx>=0.28.0",
17 | "fastapi>=0.115.5,<1.0.0",
18 | "uvicorn>=0.29.0,<1.0.0",
19 | "python-dotenv>=1.0.0,<2.0.0",
20 | "langgraph-cli[inmem]>=0.3.3",
21 | ]
22 |
--------------------------------------------------------------------------------
/src/lib/types.ts:
--------------------------------------------------------------------------------
1 | // Types for scene generation artifacts
2 |
3 | export interface Character {
4 | id: string;
5 | name: string;
6 | description: string;
7 | imageUrl?: string;
8 | prompt?: string;
9 | }
10 |
11 | export interface Background {
12 | id: string;
13 | name: string;
14 | description: string;
15 | imageUrl?: string;
16 | prompt?: string;
17 | }
18 |
19 | export interface Scene {
20 | id: string;
21 | name: string;
22 | description: string;
23 | characterIds: string[];
24 | backgroundId: string;
25 | imageUrl?: string;
26 | prompt?: string;
27 | }
28 |
29 | // Agent state matching Python AgentState
30 | export interface AgentState {
31 | characters: Character[];
32 | backgrounds: Background[];
33 | scenes: Scene[];
34 | apiKey?: string; // Dynamic API key from frontend
35 | }
36 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
2 |
3 | # dependencies
4 | /node_modules
5 | /.pnp
6 | .pnp.*
7 | .yarn/*
8 | !.yarn/patches
9 | !.yarn/plugins
10 | !.yarn/releases
11 | !.yarn/versions
12 |
13 | # testing
14 | /coverage
15 |
16 | # next.js
17 | /.next/
18 | /out/
19 |
20 | # production
21 | /build
22 |
23 | # misc
24 | .DS_Store
25 | *.pem
26 |
27 | # debug
28 | npm-debug.log*
29 | yarn-debug.log*
30 | yarn-error.log*
31 | .pnpm-debug.log*
32 |
33 | # env files (can opt-in for committing if needed)
34 | .env*
35 |
36 | # vercel
37 | .vercel
38 |
39 | # typescript
40 | *.tsbuildinfo
41 | next-env.d.ts
42 |
43 | # lock files
44 | package-lock.json
45 | yarn.lock
46 | pnpm-lock.yaml
47 | bun.lockb
48 |
49 | # LangGraph API
50 | .langgraph_api
51 |
52 | .claude
53 | .meridian
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "target": "ES2017",
4 | "lib": [
5 | "dom",
6 | "dom.iterable",
7 | "esnext"
8 | ],
9 | "allowJs": true,
10 | "skipLibCheck": true,
11 | "strict": true,
12 | "noEmit": true,
13 | "esModuleInterop": true,
14 | "module": "esnext",
15 | "moduleResolution": "bundler",
16 | "resolveJsonModule": true,
17 | "isolatedModules": true,
18 | "jsx": "react-jsx",
19 | "incremental": true,
20 | "plugins": [
21 | {
22 | "name": "next"
23 | }
24 | ],
25 | "paths": {
26 | "@/*": [
27 | "./src/*"
28 | ]
29 | }
30 | },
31 | "include": [
32 | "next-env.d.ts",
33 | "**/*.ts",
34 | "**/*.tsx",
35 | ".next/types/**/*.ts",
36 | ".next/dev/types/**/*.ts"
37 | ],
38 | "exclude": [
39 | "node_modules"
40 | ]
41 | }
42 |
--------------------------------------------------------------------------------
/public/globe.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/src/app/layout.tsx:
--------------------------------------------------------------------------------
1 | import type { Metadata } from "next";
2 | import { Space_Mono } from "next/font/google";
3 | import { CopilotKit } from "@copilotkit/react-core";
4 | import "./globals.css";
5 | import "@copilotkit/react-ui/styles.css";
6 |
7 | const spaceMono = Space_Mono({
8 | weight: ["400", "700"],
9 | subsets: ["latin"],
10 | variable: "--font-space-mono",
11 | });
12 |
13 | export const metadata: Metadata = {
14 | title: "Scene Creator - CopilotKit + Gemini 3 Demo",
15 | description: "Create scenes with AI-generated characters and backgrounds",
16 | };
17 |
18 | export default function RootLayout({
19 | children,
20 | }: Readonly<{
21 | children: React.ReactNode;
22 | }>) {
23 | return (
24 |
25 |
26 |
27 | {children}
28 |
29 |
30 |
31 | );
32 | }
33 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | The MIT License
2 |
3 | Copyright (c) Atai Barkai
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in
13 | all copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 | THE SOFTWARE.
22 |
--------------------------------------------------------------------------------
/src/lib/chat-input-context.tsx:
--------------------------------------------------------------------------------
1 | "use client";
2 |
3 | import { createContext, useContext, useState, useCallback, ReactNode } from "react";
4 |
5 | interface ChatInputContextType {
6 | inputValue: string;
7 | setInputValue: (value: string) => void;
8 | inputRef: React.RefObject | null;
9 | setInputRef: (ref: React.RefObject) => void;
10 | }
11 |
12 | const ChatInputContext = createContext(null);
13 |
14 | export function ChatInputProvider({ children }: { children: ReactNode }) {
15 | const [inputValue, setInputValue] = useState("");
16 | const [inputRef, setInputRef] = useState | null>(null);
17 |
18 | return (
19 |
20 | {children}
21 |
22 | );
23 | }
24 |
25 | export function useChatInput() {
26 | const context = useContext(ChatInputContext);
27 | if (!context) {
28 | throw new Error("useChatInput must be used within ChatInputProvider");
29 | }
30 | return context;
31 | }
32 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "langgraph-python-starter",
3 | "version": "0.1.0",
4 | "private": true,
5 | "scripts": {
6 | "dev": "concurrently \"npm run dev:ui\" \"npm run dev:agent\" --names ui,agent --prefix-colors blue,green --kill-others",
7 | "dev:debug": "LOG_LEVEL=debug npm run dev",
8 | "dev:agent": "cd agent && npx @langchain/langgraph-cli dev --port 8123 --no-browser",
9 | "dev:ui": "next dev --turbopack",
10 | "build": "next build",
11 | "start": "next start",
12 | "lint": "eslint ."
13 | },
14 | "dependencies": {
15 | "@ag-ui/langgraph": "0.0.18",
16 | "@copilotkit/react-core": "1.10.6",
17 | "@copilotkit/react-ui": "1.10.6",
18 | "@copilotkit/runtime": "1.10.6",
19 | "next": "16.0.1",
20 | "react": "^19.2.0",
21 | "react-dom": "^19.2.0",
22 | "zod": "^3.24.4"
23 | },
24 | "devDependencies": {
25 | "@langchain/langgraph-cli": "0.0.40",
26 | "@tailwindcss/postcss": "^4",
27 | "@types/node": "^20",
28 | "@types/react": "^19",
29 | "@types/react-dom": "^19",
30 | "concurrently": "^9.1.2",
31 | "eslint": "^9",
32 | "eslint-config-next": "16.0.1",
33 | "tailwindcss": "^4",
34 | "typescript": "^5"
35 | }
36 | }
--------------------------------------------------------------------------------
/src/app/api/copilotkit/route.ts:
--------------------------------------------------------------------------------
1 | import {
2 | CopilotRuntime,
3 | ExperimentalEmptyAdapter,
4 | copilotRuntimeNextJSAppRouterEndpoint,
5 | } from "@copilotkit/runtime";
6 |
7 | import { LangGraphAgent } from "@ag-ui/langgraph"
8 | import { NextRequest } from "next/server";
9 |
10 | // 1. Use EmptyAdapter since we're in agent lock mode (LangGraph handles all LLM calls)
11 | // Suggestions will be set programmatically via useCopilotChatSuggestions with static values
12 | const serviceAdapter = new ExperimentalEmptyAdapter();
13 |
14 | // 2. Create the CopilotRuntime instance and utilize the LangGraph AG-UI
15 | // integration to setup the connection.
16 | const runtime = new CopilotRuntime({
17 | agents: {
18 | "sample_agent": new LangGraphAgent({
19 | deploymentUrl: process.env.LANGGRAPH_DEPLOYMENT_URL || "http://localhost:8123",
20 | graphId: "sample_agent",
21 | langsmithApiKey: process.env.LANGSMITH_API_KEY || "",
22 | }),
23 | }
24 | });
25 |
26 | // 3. Build a Next.js API route that handles the CopilotKit runtime requests.
27 | export const POST = async (req: NextRequest) => {
28 | const { handleRequest } = copilotRuntimeNextJSAppRouterEndpoint({
29 | runtime,
30 | serviceAdapter,
31 | endpoint: "/api/copilotkit",
32 | });
33 |
34 | return handleRequest(req);
35 | };
--------------------------------------------------------------------------------
/public/next.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/src/components/CustomChatInput.tsx:
--------------------------------------------------------------------------------
1 | "use client";
2 |
3 | import { type InputProps } from "@copilotkit/react-ui";
4 | import { useChatInput } from "@/lib/chat-input-context";
5 | import { useEffect, useRef } from "react";
6 |
7 | export function CustomChatInput({ inProgress, onSend }: InputProps) {
8 | const { inputValue, setInputValue, setInputRef } = useChatInput();
9 | const inputRef = useRef(null);
10 |
11 | // Register the input ref with context so other components can focus it
12 | useEffect(() => {
13 | setInputRef(inputRef as any);
14 | }, [setInputRef]);
15 |
16 | // Focus and move cursor to end when value changes externally
17 | useEffect(() => {
18 | if (inputValue && inputRef.current) {
19 | inputRef.current.focus();
20 | // Move cursor to end
21 | const length = inputValue.length;
22 | inputRef.current.setSelectionRange(length, length);
23 | }
24 | }, [inputValue]);
25 |
26 | const handleSubmit = () => {
27 | if (inputValue.trim()) {
28 | onSend(inputValue);
29 | setInputValue("");
30 | }
31 | };
32 |
33 | const handleKeyDown = (e: React.KeyboardEvent) => {
34 | if (e.key === "Enter" && !e.shiftKey) {
35 | e.preventDefault();
36 | handleSubmit();
37 | }
38 | };
39 |
40 | return (
41 |
42 |
61 | );
62 | }
63 |
--------------------------------------------------------------------------------
/src/app/globals.css:
--------------------------------------------------------------------------------
1 | @import "tailwindcss";
2 |
3 | :root {
4 | /* Neo-Brutalist Palette */
5 | --bg-primary: #F2F0E9; /* Warm off-white */
6 | --bg-secondary: #E6E4DD;
7 | --bg-card: #FFFFFF;
8 |
9 | --fg-primary: #121212;
10 | --fg-secondary: #555555;
11 |
12 | --accent-red: #FF3333;
13 | --accent-blue: #3355FF;
14 | --accent-yellow: #FFCC00;
15 |
16 | --border-color: #121212;
17 |
18 | --shadow-hard: 4px 4px 0px 0px var(--border-color);
19 | --shadow-hard-hover: 6px 6px 0px 0px var(--border-color);
20 | --shadow-hard-sm: 2px 2px 0px 0px var(--border-color);
21 | }
22 |
23 | body {
24 | background-color: var(--bg-primary);
25 | color: var(--fg-primary);
26 | }
27 |
28 | /* Custom Scrollbar */
29 | ::-webkit-scrollbar {
30 | width: 12px;
31 | background: var(--bg-secondary);
32 | border-left: 2px solid var(--border-color);
33 | }
34 |
35 | ::-webkit-scrollbar-thumb {
36 | background: var(--fg-primary);
37 | border: 2px solid var(--bg-secondary);
38 | }
39 |
40 | ::-webkit-scrollbar-thumb:hover {
41 | background: var(--accent-red);
42 | }
43 |
44 | /* Utilities */
45 | .brutalist-card {
46 | background-color: var(--bg-card);
47 | border: 2px solid var(--border-color);
48 | box-shadow: var(--shadow-hard);
49 | transition: all 0.1s ease;
50 | }
51 |
52 | .brutalist-card:hover {
53 | transform: translate(-2px, -2px);
54 | box-shadow: var(--shadow-hard-hover);
55 | }
56 |
57 | .brutalist-btn {
58 | background-color: var(--bg-card);
59 | border: 2px solid var(--border-color);
60 | box-shadow: var(--shadow-hard-sm);
61 | font-weight: 700;
62 | text-transform: uppercase;
63 | transition: all 0.1s ease;
64 | }
65 |
66 | .brutalist-btn:hover:not(:disabled) {
67 | transform: translate(-1px, -1px);
68 | box-shadow: 3px 3px 0px 0px var(--border-color);
69 | background-color: var(--accent-yellow);
70 | }
71 |
72 | .brutalist-btn:active:not(:disabled) {
73 | transform: translate(1px, 1px);
74 | box-shadow: 1px 1px 0px 0px var(--border-color);
75 | }
76 |
77 | .brutalist-btn:disabled {
78 | opacity: 0.5;
79 | cursor: not-allowed;
80 | box-shadow: none;
81 | transform: translate(2px, 2px);
82 | }
83 |
84 | .brutalist-input {
85 | background-color: var(--bg-card);
86 | border: 2px solid var(--border-color);
87 | box-shadow: var(--shadow-hard-sm);
88 | outline: none;
89 | transition: all 0.1s ease;
90 | }
91 |
92 | .brutalist-input:focus {
93 | border-color: var(--accent-blue);
94 | box-shadow: 4px 4px 0px 0px var(--accent-blue);
95 | }
96 |
97 | /* Copilot Kit Overrides */
98 | .copilot-kit-sidebar {
99 | border-left: 2px solid var(--border-color) !important;
100 | background-color: var(--bg-primary) !important;
101 | }
102 |
103 | /* Pattern Background Utility */
104 | .bg-grid-pattern {
105 | background-image:
106 | linear-gradient(var(--border-color) 1px, transparent 1px),
107 | linear-gradient(90deg, var(--border-color) 1px, transparent 1px);
108 | background-size: 40px 40px;
109 | background-position: center center;
110 | opacity: 0.5;
111 | }
--------------------------------------------------------------------------------
/src/components/ApiKeyInput.tsx:
--------------------------------------------------------------------------------
1 | "use client";
2 |
3 | import { useState } from "react";
4 |
5 | interface ApiKeyInputProps {
6 | currentKey: string;
7 | onSave: (key: string) => void;
8 | onClear: () => void;
9 | }
10 |
11 | export function ApiKeyInput({ currentKey, onSave, onClear }: ApiKeyInputProps) {
12 | const [input, setInput] = useState("");
13 | const [isEditing, setIsEditing] = useState(!currentKey);
14 |
15 | const handleSave = () => {
16 | if (input.trim()) {
17 | onSave(input.trim());
18 | setInput("");
19 | setIsEditing(false);
20 | }
21 | };
22 |
23 | const handleClear = () => {
24 | onClear();
25 | setInput("");
26 | setIsEditing(true);
27 | };
28 |
29 | if (!isEditing && currentKey) {
30 | return (
31 |
32 |
33 |
🔑
34 |
35 | {currentKey.substring(0, 8)}...
36 |
37 |
38 |
39 |
45 |
51 |
52 |
53 | );
54 | }
55 |
56 | return (
57 |
58 |
62 |
63 | Enter your Google AI API key
64 |
65 |
66 |
setInput(e.target.value)}
70 | onKeyDown={(e) => e.key === "Enter" && handleSave()}
71 | placeholder="AIza..."
72 | className="w-full px-2 py-1.5 border-2 border-black bg-white font-mono text-xs focus:outline-none focus:shadow-[3px_3px_0px_0px_black]"
73 | autoFocus
74 | />
75 |
76 |
83 | {currentKey && (
84 |
90 | )}
91 |
92 |
93 |
94 | );
95 | }
96 |
--------------------------------------------------------------------------------
/agent/DEPLOY.md:
--------------------------------------------------------------------------------
1 | # Deploying the Agent to Railway
2 |
3 | This guide covers deploying the LangGraph agent to Railway.
4 |
5 | ## Prerequisites
6 |
7 | - [Railway account](https://railway.app/) (free tier available)
8 | - Railway CLI (optional but recommended): `npm install -g @railway/cli`
9 | - Google AI API key from [Google AI Studio](https://makersuite.google.com/app/apikey)
10 |
11 | ## Method 1: Deploy via Railway CLI (Recommended)
12 |
13 | 1. **Install Railway CLI** (if not already installed):
14 | ```bash
15 | npm install -g @railway/cli
16 | ```
17 |
18 | 2. **Login to Railway**:
19 | ```bash
20 | railway login
21 | ```
22 |
23 | 3. **Initialize Railway project** (from the agent directory):
24 | ```bash
25 | cd agent
26 | railway init
27 | ```
28 |
29 | 4. **Set environment variables**:
30 | ```bash
31 | railway variables set GOOGLE_API_KEY=your-google-ai-api-key-here
32 | ```
33 |
34 | 5. **Deploy**:
35 | ```bash
36 | railway up
37 | ```
38 |
39 | 6. **Get the deployment URL**:
40 | ```bash
41 | railway domain
42 | ```
43 |
44 | ## Method 2: Deploy via Railway Dashboard
45 |
46 | 1. **Go to [Railway Dashboard](https://railway.app/dashboard)**
47 |
48 | 2. **Create New Project**:
49 | - Click "New Project"
50 | - Select "Deploy from GitHub repo"
51 | - Connect your GitHub account if not already connected
52 | - Select this repository
53 |
54 | 3. **Configure the service**:
55 | - Railway will auto-detect the Dockerfile
56 | - Root directory: `/agent`
57 | - Port: 8000 (automatically detected)
58 |
59 | 4. **Set Environment Variables**:
60 | - Go to the "Variables" tab
61 | - Add: `GOOGLE_API_KEY` = your Google AI API key
62 | - (Optional) Add LangSmith variables for tracing
63 |
64 | 5. **Deploy**:
65 | - Railway will automatically build and deploy
66 | - Wait for the build to complete (usually 2-3 minutes)
67 |
68 | 6. **Get the URL**:
69 | - Go to "Settings" > "Networking"
70 | - Click "Generate Domain"
71 | - Your agent will be available at: `https://your-project.up.railway.app`
72 |
73 | ## Method 3: Deploy via Railway Button
74 |
75 | Add this to your repository README:
76 |
77 | ```markdown
78 | [](https://railway.app/template/your-template-id)
79 | ```
80 |
81 | ## Updating the Frontend
82 |
83 | After deploying the agent, update your Next.js frontend to use the Railway URL:
84 |
85 | 1. Open `src/app/api/copilotkit/route.ts`
86 | 2. Update the agent URL:
87 | ```typescript
88 | const agent = new LangGraphAgent({
89 | agentUrl: process.env.AGENT_URL || "https://your-project.up.railway.app",
90 | });
91 | ```
92 |
93 | 3. Add to your `.env.local`:
94 | ```
95 | AGENT_URL=https://your-project.up.railway.app
96 | ```
97 |
98 | ## Health Check
99 |
100 | Once deployed, verify the agent is running:
101 |
102 | ```bash
103 | curl https://your-project.up.railway.app/health
104 | ```
105 |
106 | ## Monitoring
107 |
108 | - View logs: `railway logs`
109 | - View metrics: Railway Dashboard > Metrics tab
110 | - Add LangSmith for tracing: Set `LANGCHAIN_TRACING_V2=true` and `LANGCHAIN_API_KEY`
111 |
112 | ## Troubleshooting
113 |
114 | ### Build fails with "No space left on device"
115 | - Railway free tier has limited build space
116 | - Try removing unused dependencies from requirements.txt
117 |
118 | ### Agent timeout errors
119 | - Railway free tier has request timeouts
120 | - For production, upgrade to Railway Pro
121 |
122 | ### Environment variables not working
123 | - Ensure variables are set in Railway dashboard or via CLI
124 | - Restart the deployment after adding variables
125 |
126 | ## Cost Optimization
127 |
128 | Railway pricing:
129 | - Free tier: $5 credit/month
130 | - Pro: $20/month + usage-based pricing
131 | - Image generation with Gemini uses Google AI quota, not Railway resources
132 |
133 | For production deployment, consider:
134 | - Monitoring image generation usage
135 | - Implementing rate limiting
136 | - Caching generated images
137 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Scene Creator - CopilotKit + LangGraph + Gemini 3 Demo
2 |
3 | A demo app showcasing [CopilotKit](https://copilotkit.ai) integration with [LangGraph](https://www.langchain.com/langgraph) and Google's Gemini 3 models. Generate AI-powered scenes by creating characters, backgrounds, and combining them together.
4 |
5 | https://github.com/user-attachments/assets/3c60c2b8-5ccd-42f0-817f-0e5e22398a48
6 |
7 | ## What This Demo Shows
8 |
9 | - **CopilotKit + LangGraph Integration** - Connect a Python LangGraph agent to a Next.js frontend
10 | - **Shared State Pattern** - Bidirectional state sync between frontend and agent
11 | - **Human-in-the-Loop (HITL)** - Approve/reject AI actions before execution
12 | - **Generative UI** - Real-time tool execution feedback in chat
13 | - **Dynamic API Keys** - Pass API keys from frontend to agent at runtime
14 | - **Image Generation** - Using Gemini 3 and Nano Banana (gemini-2.5-flash-image)
15 |
16 | ## Demo Features
17 |
18 | | Feature | Description |
19 | |---------|-------------|
20 | | Character Generation | Create characters with AI-generated images |
21 | | Background Generation | Generate environments and scenes |
22 | | Scene Composition | Combine characters and backgrounds |
23 | | Image Editing | Modify generated images with natural language |
24 | | HITL Approval | Review and approve image prompts before generation |
25 |
26 | ## Quick Start
27 |
28 | ### Prerequisites
29 | - Node.js 18+
30 | - Python 3.10+
31 | - Google AI API Key ([get one here](https://makersuite.google.com/app/apikey))
32 |
33 | ### Setup
34 |
35 | ```bash
36 | # 1. Install dependencies
37 | npm install
38 |
39 | # 2. Set up your API key
40 | echo 'GOOGLE_API_KEY=your-key-here' > agent/.env
41 |
42 | # 3. Start the app
43 | npm run dev
44 | ```
45 |
46 | Open [http://localhost:3000](http://localhost:3000) and start creating!
47 |
48 | ## Project Structure
49 |
50 | ```
51 | ├── src/
52 | │ ├── app/
53 | │ │ ├── page.tsx # Main UI with CopilotKit integration
54 | │ │ └── api/copilotkit/ # CopilotKit API route
55 | │ ├── components/
56 | │ │ ├── ArtifactPanel.tsx # Display generated artifacts
57 | │ │ ├── ApiKeyInput.tsx # Dynamic API key management
58 | │ │ └── CustomChatInput.tsx
59 | │ └── lib/
60 | │ └── types.ts # Shared TypeScript types
61 | ├── agent/
62 | │ ├── agent.py # LangGraph agent with tools
63 | │ ├── server.py # Custom routes (static files)
64 | │ └── langgraph.json # LangGraph configuration
65 | ```
66 |
67 | ## Key CopilotKit Patterns
68 |
69 | ### 1. Shared State (Frontend ↔ Agent)
70 |
71 | ```typescript
72 | // Frontend: src/app/page.tsx
73 | const { state, setState } = useCoAgent({
74 | name: "sample_agent",
75 | initialState: { characters: [], backgrounds: [], scenes: [] },
76 | });
77 |
78 | // Update state from frontend
79 | setState((prev) => ({ ...prev, apiKey: newKey }));
80 | ```
81 |
82 | ```python
83 | # Agent: agent/agent.py
84 | class AgentState(MessagesState):
85 | characters: List[dict] = []
86 | backgrounds: List[dict] = []
87 | scenes: List[dict] = []
88 | apiKey: str = ""
89 |
90 | # Read state in agent
91 | api_key = state.get("apiKey", "")
92 | ```
93 |
94 | ### 2. Human-in-the-Loop (HITL)
95 |
96 | ```typescript
97 | // Frontend: Enable HITL for specific tool
98 | useCopilotAction({
99 | name: "approve_image_prompt",
100 | disabled: true, // Agent calls this, not user
101 | handler: async ({ prompt }) => {
102 | // Show approval UI, return approved/rejected
103 | },
104 | });
105 | ```
106 |
107 | ### 3. Generative UI
108 |
109 | ```typescript
110 | // Show real-time tool progress
111 | useCopilotAction({
112 | name: "create_character",
113 | render: ({ status, result }) => (
114 |
115 | ),
116 | });
117 | ```
118 |
119 | ### 4. LangGraph Agent Tools
120 |
121 | ```python
122 | # agent/agent.py
123 | @tool
124 | async def create_character(
125 | name: str,
126 | description: str,
127 | prompt: str,
128 | state: Annotated[dict, InjectedState] # Access shared state
129 | ) -> dict:
130 | api_key = state.get("apiKey", "")
131 | image_url = await generate_image(prompt, api_key=api_key)
132 | return {"name": name, "description": description, "imageUrl": image_url}
133 | ```
134 |
135 | ## Deployment
136 |
137 | Deploy the agent to Railway:
138 |
139 | ```bash
140 | cd agent
141 | railway link
142 | railway up
143 | railway variables --set "AGENT_URL=https://your-app.up.railway.app"
144 | railway variables --set "GOOGLE_API_KEY=your-key"
145 | ```
146 |
147 | See [agent/DEPLOY.md](agent/DEPLOY.md) for detailed deployment guide.
148 |
149 | ## Tech Stack
150 |
151 | | Layer | Technology |
152 | |-------|------------|
153 | | Frontend | Next.js 16, React 19, Tailwind CSS 4 |
154 | | AI Integration | CopilotKit 1.10.6 |
155 | | Agent | Python, LangGraph 0.6.6 |
156 | | LLM | Gemini 3 Pro Preview |
157 | | Image Gen | Nano Banana (gemini-2.5-flash-image) |
158 |
159 | ## Learn More
160 |
161 | - [CopilotKit Docs](https://docs.copilotkit.ai) - Full CopilotKit documentation
162 | - [LangGraph + CopilotKit Guide](https://docs.copilotkit.ai/coagents/langgraph/langgraph-native-python) - Integration guide
163 | - [Shared State Pattern](https://docs.copilotkit.ai/coagents/langgraph/shared-state) - State synchronization
164 |
165 | ## License
166 |
167 | MIT
168 |
169 | Built by Mark Morgan
170 |
--------------------------------------------------------------------------------
/src/components/ArtifactPanel.tsx:
--------------------------------------------------------------------------------
1 | "use client";
2 |
3 | import { Character, Background, Scene } from "@/lib/types";
4 | import { useChatInput } from "@/lib/chat-input-context";
5 |
6 | interface ArtifactPanelProps {
7 | characters: Character[];
8 | backgrounds: Background[];
9 | scenes: Scene[];
10 | }
11 |
12 | export function ArtifactPanel({ characters, backgrounds, scenes }: ArtifactPanelProps) {
13 | const { setInputValue } = useChatInput();
14 |
15 | const handleEdit = (type: string, id: string) => {
16 | setInputValue(`[EDIT ${type} ${id}]: `);
17 | };
18 | const hasArtifacts = characters.length > 0 || backgrounds.length > 0 || scenes.length > 0;
19 |
20 | return (
21 |
22 | {!hasArtifacts ? (
23 |
24 | ) : (
25 |
26 | {scenes.length > 0 && (
27 |
28 |
29 | {scenes.map((scene) => (
30 |
handleEdit("scene", scene.id)}>
31 | {scene.imageUrl ? (
32 |
33 | ) : (
34 |
35 | // Generating_Scene_Data...
36 |
37 | )}
38 |
39 | ))}
40 |
41 |
42 | )}
43 |
44 | {characters.length > 0 && (
45 |
46 |
47 | {characters.map((character) => (
48 |
handleEdit("character", character.id)}>
49 | {character.imageUrl ? (
50 |
51 | ) : (
52 |
53 | [Loading...]
54 |
55 | )}
56 |
57 | ))}
58 |
59 |
60 | )}
61 |
62 | {backgrounds.length > 0 && (
63 |
64 |
65 | {backgrounds.map((background) => (
66 |
handleEdit("background", background.id)}>
67 | {background.imageUrl ? (
68 |
69 | ) : (
70 |
71 | [Loading...]
72 |
73 | )}
74 |
75 | ))}
76 |
77 |
78 | )}
79 |
80 | )}
81 |
82 | );
83 | }
84 |
85 | function EmptyState() {
86 | return (
87 |
88 |
89 | ?
90 |
91 |
No Data Found
92 |
93 | Initiate sequence by requesting a character or background generation from the terminal.
94 |
95 |
96 | );
97 | }
98 |
99 | function ArtifactSection({ title, count, children }: { title: string; count: number; children: React.ReactNode }) {
100 | return (
101 |
102 |
103 |
{title}
104 |
105 | [{count.toString().padStart(2, '0')}]
106 |
107 |
108 | {children}
109 |
110 | );
111 | }
112 |
113 | function ArtifactCard({ title, type, children, onEdit }: { title: string; type: string; children: React.ReactNode; onEdit?: () => void }) {
114 | return (
115 |
116 |
117 | {type}
118 |
119 | {children}
120 |
121 |
{title}
122 | {onEdit && (
123 |
130 | )}
131 |
132 |
133 | );
134 | }
--------------------------------------------------------------------------------
/src/app/page.tsx:
--------------------------------------------------------------------------------
1 | "use client";
2 |
3 | import { useCoAgent, useCopilotAction, useCopilotReadable, useHumanInTheLoop } from "@copilotkit/react-core";
4 | import { CopilotSidebar } from "@copilotkit/react-ui";
5 | import { ArtifactPanel } from "@/components/ArtifactPanel";
6 | import { CustomChatInput } from "@/components/CustomChatInput";
7 | import { ApiKeyInput } from "@/components/ApiKeyInput";
8 | import { ChatInputProvider } from "@/lib/chat-input-context";
9 | import { AgentState } from "@/lib/types";
10 | import { useRef, useMemo, useState, useEffect } from "react";
11 |
12 | const API_KEY_STORAGE_KEY = "google_api_key";
13 |
14 | export default function SceneCreatorPage() {
15 | // API key state with localStorage persistence
16 | const [apiKey, setApiKeyState] = useState("");
17 |
18 | // Load API key from localStorage on mount
19 | useEffect(() => {
20 | const stored = localStorage.getItem(API_KEY_STORAGE_KEY);
21 | if (stored) {
22 | setApiKeyState(stored);
23 | }
24 | }, []);
25 |
26 | // Shared state with the LangGraph agent
27 | const { state, setState, running } = useCoAgent({
28 | name: "sample_agent",
29 | initialState: {
30 | characters: [],
31 | backgrounds: [],
32 | scenes: [],
33 | apiKey: "",
34 | },
35 | });
36 |
37 | // Sync API key to agent state when it changes
38 | useEffect(() => {
39 | if (apiKey && apiKey !== state.apiKey) {
40 | setState((prevState) => ({
41 | characters: prevState?.characters || [],
42 | backgrounds: prevState?.backgrounds || [],
43 | scenes: prevState?.scenes || [],
44 | apiKey,
45 | }));
46 | }
47 | }, [apiKey, state.apiKey, setState]);
48 |
49 | // Save API key to localStorage and agent state
50 | const saveApiKey = (key: string) => {
51 | localStorage.setItem(API_KEY_STORAGE_KEY, key);
52 | setApiKeyState(key);
53 | setState((prevState) => ({
54 | characters: prevState?.characters || [],
55 | backgrounds: prevState?.backgrounds || [],
56 | scenes: prevState?.scenes || [],
57 | apiKey: key,
58 | }));
59 | };
60 |
61 | // Clear API key from localStorage and agent state
62 | const clearApiKey = () => {
63 | localStorage.removeItem(API_KEY_STORAGE_KEY);
64 | setApiKeyState("");
65 | setState((prevState) => ({
66 | characters: prevState?.characters || [],
67 | backgrounds: prevState?.backgrounds || [],
68 | scenes: prevState?.scenes || [],
69 | apiKey: "",
70 | }));
71 | };
72 |
73 | // Keep a reference to the last valid state to prevent flickering during requests
74 | const lastValidState = useRef({
75 | characters: [],
76 | backgrounds: [],
77 | scenes: [],
78 | });
79 |
80 | // Update reference when we have actual data
81 | const displayState = useMemo(() => {
82 | const hasData =
83 | (state.characters && state.characters.length > 0) ||
84 | (state.backgrounds && state.backgrounds.length > 0) ||
85 | (state.scenes && state.scenes.length > 0);
86 |
87 | if (hasData) {
88 | lastValidState.current = {
89 | characters: state.characters || [],
90 | backgrounds: state.backgrounds || [],
91 | scenes: state.scenes || [],
92 | };
93 | }
94 |
95 | // During loading, show the last known state if current is empty
96 | if (running && !hasData && (
97 | lastValidState.current.characters.length > 0 ||
98 | lastValidState.current.backgrounds.length > 0 ||
99 | lastValidState.current.scenes.length > 0
100 | )) {
101 | return lastValidState.current;
102 | }
103 |
104 | return {
105 | characters: state.characters || [],
106 | backgrounds: state.backgrounds || [],
107 | scenes: state.scenes || [],
108 | };
109 | }, [state, running]);
110 |
111 | // Make artifact data readable to the Copilot for better context awareness
112 | useCopilotReadable({
113 | description: "Available characters that can be used in scenes",
114 | value: displayState.characters.map(c => ({ id: c.id, name: c.name, description: c.description })),
115 | });
116 |
117 | useCopilotReadable({
118 | description: "Available backgrounds that can be used in scenes",
119 | value: displayState.backgrounds.map(b => ({ id: b.id, name: b.name, description: b.description })),
120 | });
121 |
122 | useCopilotReadable({
123 | description: "Created scenes combining characters and backgrounds",
124 | value: displayState.scenes.map(s => ({
125 | id: s.id,
126 | name: s.name,
127 | characterIds: s.characterIds,
128 | backgroundId: s.backgroundId
129 | })),
130 | });
131 |
132 | // Human-in-the-loop prompt approval before image generation
133 | useHumanInTheLoop({
134 | name: "approve_image_prompt",
135 | description: "Request user approval for an image generation prompt before creating the image. Call this BEFORE calling create_character, create_background, or create_scene.",
136 | parameters: [
137 | {
138 | name: "artifact_type",
139 | type: "string",
140 | description: "Type of artifact: 'character', 'background', or 'scene'",
141 | required: true,
142 | },
143 | {
144 | name: "name",
145 | type: "string",
146 | description: "Name of the artifact being created",
147 | required: true,
148 | },
149 | {
150 | name: "prompt",
151 | type: "string",
152 | description: "The image generation prompt to be approved",
153 | required: true,
154 | },
155 | ],
156 | render: ({ args, status, respond, result }) => {
157 | if (status === "executing" && respond) {
158 | return (
159 | respond({ approved: true, prompt: finalPrompt })}
164 | onCancel={() => respond({ approved: false })}
165 | />
166 | );
167 | }
168 |
169 | if (status === "complete" && result) {
170 | const res = result as { approved: boolean; prompt?: string };
171 | return (
172 |
173 |
174 | {res.approved ? (
175 | <>
176 | ✓
177 | Prompt approved
178 | >
179 | ) : (
180 | <>
181 | ✕
182 | Generation cancelled
183 | >
184 | )}
185 |
186 |
187 | );
188 | }
189 |
190 | return <>>;
191 | },
192 | });
193 |
194 | // Generative UI for create_character tool
195 | useCopilotAction({
196 | name: "create_character",
197 | available: "disabled",
198 | render: ({ status, args, result }) => (
199 |
206 | ),
207 | });
208 |
209 | // Generative UI for create_background tool
210 | useCopilotAction({
211 | name: "create_background",
212 | available: "disabled",
213 | render: ({ status, args, result }) => (
214 |
221 | ),
222 | });
223 |
224 | // Generative UI for create_scene tool
225 | useCopilotAction({
226 | name: "create_scene",
227 | available: "disabled",
228 | render: ({ status, args, result }) => (
229 |
236 | ),
237 | });
238 |
239 | // Generative UI for edit_character tool
240 | useCopilotAction({
241 | name: "edit_character",
242 | available: "disabled",
243 | render: ({ status, args, result }) => (
244 |
251 | ),
252 | });
253 |
254 | // Generative UI for edit_background tool
255 | useCopilotAction({
256 | name: "edit_background",
257 | available: "disabled",
258 | render: ({ status, args, result }) => (
259 |
266 | ),
267 | });
268 |
269 | // Generative UI for edit_scene tool
270 | useCopilotAction({
271 | name: "edit_scene",
272 | available: "disabled",
273 | render: ({ status, args, result }) => (
274 |
281 | ),
282 | });
283 |
284 | // Show only API key input if no key is set
285 | if (!apiKey) {
286 | return (
287 |
288 |
289 |
290 |
Scene Creator
291 |
292 | AI-powered scene generation with Gemini 3 & Nano Banana
293 |
294 |
295 |
300 |
301 |
302 | );
303 | }
304 |
305 | return (
306 |
307 |
308 | {/* Floating API Key Tooltip - Top Left */}
309 |
316 |
317 | {/* Main artifact display panel */}
318 |
323 |
324 | {/* Chat sidebar */}
325 |
343 |
344 |
345 | );
346 | }
347 |
348 | // Tool progress card component for Generative UI
349 | function ToolCard({
350 | icon,
351 | title,
352 | status,
353 | description,
354 | result,
355 | }: {
356 | icon: string;
357 | title: string;
358 | status: string;
359 | description?: string;
360 | result?: string;
361 | }) {
362 | const isComplete = status === "complete";
363 | const isExecuting = status === "executing" || status === "inProgress";
364 |
365 | return (
366 |
367 |
368 |
369 | {icon}
370 |
371 |
372 |
373 | {title}
374 | {isExecuting && (
375 |
376 | PROCESSING
377 |
378 | )}
379 | {isComplete && (
380 | DONE
381 | )}
382 |
383 | {description && (
384 |
385 | {description}
386 |
387 | )}
388 | {isComplete && result && (
389 |
390 | → {result}
391 |
392 | )}
393 |
394 |
395 |
396 | );
397 | }
398 |
399 | // Prompt approval card component for HITL
400 | function PromptApprovalCard({
401 | artifactType,
402 | name,
403 | prompt,
404 | onApprove,
405 | onCancel,
406 | }: {
407 | artifactType: string;
408 | name: string;
409 | prompt: string;
410 | onApprove: (prompt: string) => void;
411 | onCancel: () => void;
412 | }) {
413 | const [editedPrompt, setEditedPrompt] = useState(prompt);
414 | const [isEditing, setIsEditing] = useState(false);
415 |
416 | const icon = artifactType === "character" ? "👤" : artifactType === "background" ? "🏞️" : "🎬";
417 |
418 | return (
419 |
420 |
421 | {icon}
422 |
423 | APPROVE {artifactType}
424 |
425 |
426 |
427 |
428 |
Target: {name}
429 | {isEditing ? (
430 |
443 |
444 |
445 |
451 |
457 |
463 |
464 |
465 | );
466 | }
467 |
--------------------------------------------------------------------------------
/agent/agent.py:
--------------------------------------------------------------------------------
1 | """
2 | Main entry point for the LangGraph agent.
3 | Uses Gemini 3 (gemini-3-pro-preview) for text generation.
4 | Main agent writes all prompts directly (no subagents).
5 | """
6 |
7 | import os
8 | import uuid
9 | import httpx
10 | import base64
11 | import asyncio
12 | import time
13 | from pathlib import Path
14 | from typing import Any, List, Annotated
15 | from typing_extensions import Literal
16 | from langchain_google_genai import ChatGoogleGenerativeAI
17 | from langchain_core.messages import SystemMessage
18 | from langchain_core.runnables import RunnableConfig
19 | from langchain.tools import tool
20 | from langgraph.graph import StateGraph, END
21 | from langgraph.types import Command
22 | from langgraph.graph import MessagesState
23 | from langgraph.prebuilt import ToolNode, InjectedState
24 |
25 |
26 | # === Generated images directory ===
27 | GENERATED_DIR = Path(__file__).parent / "generated"
28 | GENERATED_DIR.mkdir(exist_ok=True)
29 |
30 |
31 | def get_agent_url() -> str:
32 | """Get the agent's base URL for serving static files."""
33 | return os.getenv("AGENT_URL", "http://localhost:8000")
34 |
35 |
36 | def get_image_path(image_url: str) -> Path:
37 | """Convert image URL to local file path."""
38 | # Strip query parameters (e.g., ?t=123456 cache busting)
39 | base_url = image_url.split("?")[0]
40 |
41 | # Handle absolute URLs from agent
42 | agent_url = get_agent_url()
43 | if base_url.startswith(agent_url):
44 | base_url = base_url[len(agent_url):]
45 |
46 | # Handle relative /generated/ URLs
47 | if base_url.startswith("/generated/"):
48 | filename = base_url.replace("/generated/", "")
49 | return GENERATED_DIR / filename
50 |
51 | # Fallback to direct path
52 | return Path(base_url)
53 |
54 |
55 | # === State definition ===
56 |
57 | class AgentState(MessagesState):
58 | """Agent state with scene generation artifacts."""
59 | characters: List[dict] = []
60 | backgrounds: List[dict] = []
61 | scenes: List[dict] = []
62 | tools: List[Any] # CopilotKit tools
63 | apiKey: str = "" # Dynamic API key from frontend
64 |
65 |
66 | def get_model(api_key: str = None):
67 | """Get configured Gemini 3 model."""
68 | kwargs = {
69 | "model": os.getenv("GEMINI_MODEL", "gemini-3-pro-preview"),
70 | "temperature": 1.0,
71 | }
72 | if api_key:
73 | kwargs["google_api_key"] = api_key
74 | return ChatGoogleGenerativeAI(**kwargs)
75 |
76 |
77 | async def generate_image(prompt: str, input_images: List[str] = None, api_key: str = None) -> str:
78 | """Generate an image using Nano Banana (gemini-2.5-flash-image) via HTTP.
79 |
80 | Args:
81 | prompt: The image generation prompt
82 | input_images: Optional list of image file paths to include for composition
83 | api_key: Google API key (from state or env)
84 |
85 | Returns:
86 | URL path to the generated image (e.g., /generated/abc123.png)
87 | """
88 | if not api_key:
89 | api_key = os.getenv("GOOGLE_API_KEY")
90 | url = f"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-image:generateContent"
91 |
92 | # Build parts array
93 | parts = []
94 |
95 | # Add input images first (for composition)
96 | if input_images:
97 | for img_path in input_images:
98 | # Strip query parameters (e.g., ?t=123456 cache busting)
99 | base_img_path = img_path.split("?")[0]
100 |
101 | # Convert URL to file path
102 | file_path = get_image_path(base_img_path)
103 |
104 | if file_path.exists():
105 | # Read image and encode as base64
106 | def read_image(fp):
107 | return fp.read_bytes()
108 |
109 | image_bytes = await asyncio.to_thread(read_image, file_path)
110 | image_base64 = base64.b64encode(image_bytes).decode("utf-8")
111 |
112 | parts.append({
113 | "inline_data": {
114 | "mime_type": "image/png",
115 | "data": image_base64
116 | }
117 | })
118 |
119 | # Add text prompt
120 | parts.append({"text": prompt})
121 |
122 | payload = {
123 | "contents": [{
124 | "parts": parts
125 | }],
126 | "generationConfig": {
127 | "responseModalities": ["TEXT", "IMAGE"]
128 | }
129 | }
130 |
131 | headers = {
132 | "Content-Type": "application/json",
133 | "x-goog-api-key": api_key
134 | }
135 |
136 | async with httpx.AsyncClient(timeout=60.0) as client:
137 | response = await client.post(url, json=payload, headers=headers)
138 | response.raise_for_status()
139 | data = response.json()
140 |
141 | # Extract image data from response and save to disk
142 | if "candidates" in data and len(data["candidates"]) > 0:
143 | parts = data["candidates"][0].get("content", {}).get("parts", [])
144 | for part in parts:
145 | if "inlineData" in part:
146 | image_data = part["inlineData"]["data"]
147 | mime_type = part["inlineData"].get("mimeType", "image/png")
148 |
149 | # Determine file extension
150 | ext = "png" if "png" in mime_type else "jpg"
151 |
152 | # Generate unique filename
153 | filename = f"{uuid.uuid4()}.{ext}"
154 |
155 | # Save to agent's generated directory
156 | output_path = GENERATED_DIR / filename
157 |
158 | # Decode and save (using to_thread for async compatibility)
159 | image_bytes = base64.b64decode(image_data)
160 |
161 | def save_image():
162 | output_path.write_bytes(image_bytes)
163 |
164 | await asyncio.to_thread(save_image)
165 |
166 | # Return absolute URL that frontend can use
167 | return f"{get_agent_url()}/generated/{filename}"
168 |
169 | return None
170 |
171 |
172 | async def edit_image(image_url: str, edit_prompt: str, api_key: str = None) -> str:
173 | """Edit an existing image using Nano Banana.
174 |
175 | Args:
176 | image_url: URL to the existing image (absolute or relative)
177 | edit_prompt: Description of the changes to make
178 | api_key: Google API key (from state or env)
179 |
180 | Returns:
181 | URL to the edited image (overwrites the original)
182 | """
183 | if not api_key:
184 | api_key = os.getenv("GOOGLE_API_KEY")
185 | url = f"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-image:generateContent"
186 |
187 | # Convert URL to file path
188 | file_path = get_image_path(image_url)
189 |
190 | if not file_path.exists():
191 | return None
192 |
193 | # Read and encode image
194 | def read_image(fp):
195 | return fp.read_bytes()
196 |
197 | image_bytes = await asyncio.to_thread(read_image, file_path)
198 | image_base64 = base64.b64encode(image_bytes).decode("utf-8")
199 |
200 | # Build request with image and edit prompt
201 | payload = {
202 | "contents": [{
203 | "parts": [
204 | {
205 | "inline_data": {
206 | "mime_type": "image/png",
207 | "data": image_base64
208 | }
209 | },
210 | {"text": edit_prompt}
211 | ]
212 | }],
213 | "generationConfig": {
214 | "responseModalities": ["TEXT", "IMAGE"]
215 | }
216 | }
217 |
218 | headers = {
219 | "Content-Type": "application/json",
220 | "x-goog-api-key": api_key
221 | }
222 |
223 | async with httpx.AsyncClient(timeout=60.0) as client:
224 | response = await client.post(url, json=payload, headers=headers)
225 | response.raise_for_status()
226 | data = response.json()
227 |
228 | # Extract and save the edited image (overwrite original)
229 | if "candidates" in data and len(data["candidates"]) > 0:
230 | parts = data["candidates"][0].get("content", {}).get("parts", [])
231 | for part in parts:
232 | if "inlineData" in part:
233 | image_data = part["inlineData"]["data"]
234 | new_image_bytes = base64.b64decode(image_data)
235 |
236 | def save_image():
237 | file_path.write_bytes(new_image_bytes)
238 |
239 | await asyncio.to_thread(save_image)
240 |
241 | # Return absolute URL with cache-busting timestamp
242 | filename = file_path.name
243 | return f"{get_agent_url()}/generated/{filename}?t={int(time.time() * 1000)}"
244 |
245 | return None
246 |
247 |
248 | # === Backend tools for the main agent ===
249 |
250 | @tool
251 | async def create_character(
252 | name: str,
253 | description: str,
254 | prompt: str,
255 | state: Annotated[dict, InjectedState]
256 | ) -> dict:
257 | """Create a new character with an AI-generated image.
258 |
259 | Args:
260 | name: Name of the character
261 | description: Brief description for the user (1 sentence)
262 | prompt: Detailed image generation prompt (50-100 words, include visual details, art style, pose, lighting)
263 |
264 | Returns:
265 | Character data including id, name, description, prompt, and imageUrl
266 | """
267 | # Get API key from state
268 | api_key = state.get("apiKey", "")
269 |
270 | # Generate the character image using Nano Banana
271 | image_url = await generate_image(prompt, api_key=api_key)
272 |
273 | character_id = str(uuid.uuid4())
274 |
275 | return {
276 | "id": character_id,
277 | "name": name,
278 | "description": description,
279 | "prompt": prompt,
280 | "imageUrl": image_url
281 | }
282 |
283 |
284 | @tool
285 | async def create_background(
286 | name: str,
287 | description: str,
288 | prompt: str,
289 | state: Annotated[dict, InjectedState]
290 | ) -> dict:
291 | """Create a new background/environment with an AI-generated image.
292 |
293 | Args:
294 | name: Name of the background/environment
295 | description: Brief description for the user (1 sentence)
296 | prompt: Detailed image generation prompt (50-100 words, include environment details, lighting, atmosphere)
297 |
298 | Returns:
299 | Background data including id, name, description, prompt, and imageUrl
300 | """
301 | # Get API key from state
302 | api_key = state.get("apiKey", "")
303 |
304 | # Generate the background image using Nano Banana
305 | image_url = await generate_image(prompt, api_key=api_key)
306 |
307 | background_id = str(uuid.uuid4())
308 |
309 | return {
310 | "id": background_id,
311 | "name": name,
312 | "description": description,
313 | "prompt": prompt,
314 | "imageUrl": image_url
315 | }
316 |
317 |
318 | @tool
319 | async def create_scene(
320 | name: str,
321 | description: str,
322 | prompt: str,
323 | character_ids: List[str],
324 | background_id: str,
325 | state: Annotated[dict, InjectedState]
326 | ) -> dict:
327 | """Create a scene by composing characters with a background.
328 |
329 | Args:
330 | name: Name of the scene
331 | description: Brief description for the user (1 sentence)
332 | prompt: Detailed image generation prompt for the composed scene (75-125 words)
333 | character_ids: List of character IDs to include in the scene
334 | background_id: ID of the background to use
335 |
336 | Returns:
337 | Scene data including id, name, description, prompt, and imageUrl
338 | """
339 | # Get characters and backgrounds from state
340 | characters = state.get("characters", [])
341 | backgrounds = state.get("backgrounds", [])
342 |
343 | # Collect images for composition
344 | input_images = []
345 |
346 | # Validate and collect character images
347 | for char_id in character_ids:
348 | char = next((c for c in characters if c["id"] == char_id), None)
349 | if not char:
350 | return {"error": f"Character with id {char_id} not found"}
351 | if not char.get("imageUrl"):
352 | return {"error": f"Character '{char.get('name', char_id)}' has no image"}
353 | input_images.append(char["imageUrl"])
354 |
355 | # Validate and collect background image
356 | bg = next((b for b in backgrounds if b["id"] == background_id), None)
357 | if not bg:
358 | return {"error": f"Background with id {background_id} not found"}
359 | if not bg.get("imageUrl"):
360 | return {"error": f"Background '{bg.get('name', background_id)}' has no image"}
361 | input_images.append(bg["imageUrl"])
362 |
363 | # Get API key from state
364 | api_key = state.get("apiKey", "")
365 |
366 | # Generate the scene image using Nano Banana with character/background images
367 | image_url = await generate_image(prompt, input_images, api_key=api_key)
368 |
369 | scene_id = str(uuid.uuid4())
370 |
371 | return {
372 | "id": scene_id,
373 | "name": name,
374 | "description": description,
375 | "characterIds": character_ids,
376 | "backgroundId": background_id,
377 | "prompt": prompt,
378 | "imageUrl": image_url
379 | }
380 |
381 |
382 | @tool
383 | async def edit_character(
384 | character_id: str,
385 | edit_description: str,
386 | state: Annotated[dict, InjectedState]
387 | ) -> dict:
388 | """Edit an existing character's image based on user description.
389 |
390 | Args:
391 | character_id: ID of the character to edit
392 | edit_description: Description of the changes to make
393 |
394 | Returns:
395 | Updated character data
396 | """
397 | # Find the character from state
398 | characters = state.get("characters", [])
399 | char = next((c for c in characters if c["id"] == character_id), None)
400 | if not char:
401 | return {"error": f"Character with id {character_id} not found"}
402 |
403 | if not char.get("imageUrl"):
404 | return {"error": "Character has no image to edit"}
405 |
406 | # Get API key from state
407 | api_key = state.get("apiKey", "")
408 |
409 | # Edit the image
410 | edited_url = await edit_image(
411 | char["imageUrl"],
412 | f"Edit this character image: {edit_description}. Keep the same character but apply the requested changes.",
413 | api_key=api_key
414 | )
415 |
416 | if not edited_url:
417 | return {"error": "Failed to edit image"}
418 |
419 | return {
420 | "id": char["id"],
421 | "name": char["name"],
422 | "description": char["description"],
423 | "prompt": char.get("prompt", ""),
424 | "imageUrl": edited_url,
425 | "edited": True
426 | }
427 |
428 |
429 | @tool
430 | async def edit_background(
431 | background_id: str,
432 | edit_description: str,
433 | state: Annotated[dict, InjectedState]
434 | ) -> dict:
435 | """Edit an existing background's image based on user description.
436 |
437 | Args:
438 | background_id: ID of the background to edit
439 | edit_description: Description of the changes to make
440 |
441 | Returns:
442 | Updated background data
443 | """
444 | # Find the background from state
445 | backgrounds = state.get("backgrounds", [])
446 | bg = next((b for b in backgrounds if b["id"] == background_id), None)
447 | if not bg:
448 | return {"error": f"Background with id {background_id} not found"}
449 |
450 | if not bg.get("imageUrl"):
451 | return {"error": "Background has no image to edit"}
452 |
453 | # Get API key from state
454 | api_key = state.get("apiKey", "")
455 |
456 | # Edit the image
457 | edited_url = await edit_image(
458 | bg["imageUrl"],
459 | f"Edit this background image: {edit_description}. Keep the same environment but apply the requested changes.",
460 | api_key=api_key
461 | )
462 |
463 | if not edited_url:
464 | return {"error": "Failed to edit image"}
465 |
466 | return {
467 | "id": bg["id"],
468 | "name": bg["name"],
469 | "description": bg["description"],
470 | "prompt": bg.get("prompt", ""),
471 | "imageUrl": edited_url,
472 | "edited": True
473 | }
474 |
475 |
476 | @tool
477 | async def edit_scene(
478 | scene_id: str,
479 | edit_description: str,
480 | regenerate_from_sources: bool,
481 | state: Annotated[dict, InjectedState],
482 | new_character_ids: List[str] = None,
483 | new_background_id: str = None
484 | ) -> dict:
485 | """Edit an existing scene's image.
486 |
487 | Args:
488 | scene_id: ID of the scene to edit
489 | edit_description: Description of the changes to make (write full composition prompt for regenerate_from_sources=True)
490 | regenerate_from_sources: If True, regenerate scene from current character/background images (use after editing a character or background, or adding new characters). If False, edit the scene image directly (use for composition changes).
491 | new_character_ids: Optional new list of character IDs (use when adding/removing characters from the scene)
492 | new_background_id: Optional new background ID (use when changing the scene's background)
493 |
494 | Returns:
495 | Updated scene data
496 | """
497 | # Find the scene from state
498 | scenes = state.get("scenes", [])
499 | scene = next((s for s in scenes if s["id"] == scene_id), None)
500 | if not scene:
501 | return {"error": f"Scene with id {scene_id} not found"}
502 |
503 | if regenerate_from_sources:
504 | # Regenerate scene from updated character/background images
505 | characters = state.get("characters", [])
506 | backgrounds = state.get("backgrounds", [])
507 |
508 | # Use new IDs if provided, otherwise use existing
509 | char_ids = new_character_ids if new_character_ids is not None else scene.get("characterIds", [])
510 | bg_id = new_background_id if new_background_id is not None else scene.get("backgroundId", "")
511 |
512 | input_images = []
513 |
514 | # Collect character images
515 | for char_id in char_ids:
516 | char = next((c for c in characters if c["id"] == char_id), None)
517 | if char and char.get("imageUrl"):
518 | input_images.append(char["imageUrl"])
519 |
520 | # Collect background image
521 | bg = next((b for b in backgrounds if b["id"] == bg_id), None)
522 | if bg and bg.get("imageUrl"):
523 | input_images.append(bg["imageUrl"])
524 |
525 | if not input_images:
526 | return {"error": "No source images found for regeneration"}
527 |
528 | # Get API key from state
529 | api_key = state.get("apiKey", "")
530 |
531 | # Generate new scene with updated sources
532 | new_url = await generate_image(edit_description, input_images, api_key=api_key)
533 |
534 | if not new_url:
535 | return {"error": "Failed to regenerate scene"}
536 |
537 | return {
538 | "id": scene["id"],
539 | "name": scene["name"],
540 | "description": scene["description"],
541 | "characterIds": char_ids,
542 | "backgroundId": bg_id,
543 | "prompt": edit_description,
544 | "imageUrl": new_url,
545 | "edited": True
546 | }
547 | else:
548 | # Edit the existing scene image directly (for composition changes)
549 | if not scene.get("imageUrl"):
550 | return {"error": "Scene has no image to edit"}
551 |
552 | # Get API key from state
553 | api_key = state.get("apiKey", "")
554 |
555 | edited_url = await edit_image(
556 | scene["imageUrl"],
557 | f"Edit this scene image: {edit_description}. Keep the same composition but apply the requested changes.",
558 | api_key=api_key
559 | )
560 |
561 | if not edited_url:
562 | return {"error": "Failed to edit image"}
563 |
564 | return {
565 | "id": scene["id"],
566 | "name": scene["name"],
567 | "description": scene["description"],
568 | "characterIds": scene.get("characterIds", []),
569 | "backgroundId": scene.get("backgroundId", ""),
570 | "prompt": scene.get("prompt", ""),
571 | "imageUrl": edited_url,
572 | "edited": True
573 | }
574 |
575 |
576 | # Backend tools list
577 | backend_tools = [create_character, create_background, create_scene, edit_character, edit_background, edit_scene]
578 | backend_tool_names = [tool.name for tool in backend_tools]
579 |
580 |
581 | # === Main agent nodes ===
582 |
583 | async def chat_node(state: AgentState, config: RunnableConfig) -> Command[Literal["tool_node", "__end__"]]:
584 | """Main agent that handles user requests and writes prompts directly."""
585 |
586 | # Extract API key from shared state (passed from frontend via setState)
587 | api_key = state.get("apiKey", "") or os.getenv("GOOGLE_API_KEY", "")
588 |
589 | # Use to_thread to avoid blocking the event loop during model initialization
590 | model = await asyncio.to_thread(get_model, api_key)
591 |
592 | # Bind both CopilotKit tools and backend tools
593 | all_tools = [*state.get("tools", []), *backend_tools]
594 | model_with_tools = model.bind_tools(all_tools, parallel_tool_calls=False)
595 |
596 | # Build context about current artifacts
597 | chars = state.get("characters", [])
598 | bgs = state.get("backgrounds", [])
599 | scenes = state.get("scenes", [])
600 |
601 | char_list = "\n".join([f" - {c['name']} (id: {c['id']}): {c['description']}" for c in chars]) or " None yet"
602 | bg_list = "\n".join([f" - {b['name']} (id: {b['id']}): {b['description']}" for b in bgs]) or " None yet"
603 | scene_list = "\n".join([f" - {s['name']} (id: {s['id']}): {s['description']}" for s in scenes]) or " None yet"
604 |
605 | system_message = SystemMessage(content=f"""You are a creative assistant helping users create scenes with AI-generated characters and backgrounds.
606 |
607 | ## Your Capabilities
608 | You have tools to create and edit characters, backgrounds, and scenes. When calling these tools, YOU write the image generation prompts directly.
609 |
610 | **Tools available:**
611 | - **approve_image_prompt(artifact_type, name, prompt)**: REQUIRED before creating! Gets user approval for the prompt
612 | - **create_character(name, description, prompt)**: Create a character image
613 | - **create_background(name, description, prompt)**: Create a background image
614 | - **create_scene(name, description, prompt, character_ids, background_id)**: Compose a scene from characters + background
615 | - **edit_character/edit_background/edit_scene**: Edit existing images
616 |
617 | ## CRITICAL: Human-in-the-Loop Approval
618 | **Before calling create_character, create_background, or create_scene, you MUST first call approve_image_prompt.**
619 |
620 | Workflow:
621 | 1. Call approve_image_prompt with artifact_type ("character"/"background"/"scene"), name, and your proposed prompt
622 | 2. Wait for user to approve (they may edit the prompt)
623 | 3. If approved, the result will contain the final prompt - use THAT prompt when calling create_*
624 | 4. If cancelled, do NOT call the create tool
625 |
626 | Example flow:
627 | - User: "Create a warrior character"
628 | - You: Call approve_image_prompt(artifact_type="character", name="Warrior", prompt="A fierce warrior...")
629 | - [User approves with maybe edited prompt]
630 | - You: Call create_character(name="Warrior", description="...", prompt="")
631 |
632 | ## Current Session State
633 | Characters:
634 | {char_list}
635 |
636 | Backgrounds:
637 | {bg_list}
638 |
639 | Scenes:
640 | {scene_list}
641 |
642 | ## Prompt Writing Guidelines
643 | Keep prompts SIMPLE and SHORT. Nano Banana works better with minimal constraints.
644 |
645 | **For characters:**
646 | - Keep it simple: "Create a photo of [character description]"
647 | - IMPORTANT: Always add "on a plain white background" or "studio photo" to get clean images for compositing
648 | - Example: "Create a photo of CJ from GTA San Andreas on a plain white background"
649 |
650 | **For backgrounds:**
651 | - Keep it simple: "[environment description]"
652 | - Example: "Grove Street neighborhood in Los Santos"
653 |
654 | **For scenes:**
655 | - Just describe how to place the characters: "Place these characters in this environment naturally"
656 | - Add activity if needed: "Place these characters in this environment, they are walking together"
657 | - Keep it SHORT - don't over-describe
658 |
659 | ## Workflow Guidelines
660 | 1. When creating artifacts, write creative names, brief descriptions, and detailed prompts
661 | 2. For scenes, ensure user has at least one character and one background first
662 | 3. When editing, the edit_description should clearly state what changes to make
663 | 4. Be creative and helpful - suggest ideas if user is unsure
664 | 5. **Adding elements to existing scenes**: If user asks to add a character to an existing scene:
665 | - Do NOT create a new scene
666 | - Use edit_scene with regenerate_from_sources=True
667 | - Update the scene's character_ids to include the new character
668 | - Write a composition prompt that includes ALL characters (existing + new)
669 |
670 | ## Important: Cascading Edits (SEQUENTIAL - ONE TOOL AT A TIME)
671 | - When user edits a character or background, you must update scenes containing them
672 | - **CRITICAL: Call only ONE tool at a time.** Wait for each tool to complete before calling the next.
673 | - Sequence: First edit_character/edit_background → wait for result → then edit_scene for each affected scene
674 | - Do NOT call multiple tools in the same response - the scene edit needs the updated character/background image
675 | - Example: User says "make the character's shirt red" → call edit_character ONLY, then in next turn call edit_scene
676 |
677 | ## edit_scene: regenerate_from_sources parameter
678 | - **regenerate_from_sources=True**: Use after editing a character or background. This sends ONLY the character/background images to Nano Banana (NOT the old scene).
679 | - **CRITICAL**: Write a FULL scene composition prompt as if creating a new scene!
680 | - Do NOT write "regenerate" or "update" - Nano Banana has no memory of the previous scene
681 | - Write: "Naturally integrate this character into this environment at proper scale. The character should be walking down the street..."
682 | - NOT: "Regenerate the scene to show the character with..."
683 | - **regenerate_from_sources=False**: Use for direct scene edits (like "move character to the left"). This edits the existing scene image.
684 |
685 | ## Edit Priority
686 | - Prefer editing the source element (character/background) over editing scenes directly
687 | - If user asks to change something in a scene (e.g., "add more trees to the scene"), edit the background first, then edit the scene
688 | - Only edit a scene directly if the user wants to change composition (e.g., "move the character to the left", "change the character's pose in this scene")
689 |
690 | ## Response Style
691 | - Be friendly and encouraging
692 | - Describe what you're creating before calling tools
693 | - After creation, summarize what was made
694 | - Suggest next steps""")
695 |
696 | response = await model_with_tools.ainvoke([
697 | system_message,
698 | *state["messages"],
699 | ], config)
700 |
701 | # Check if we need to route to tool node
702 | tool_calls = getattr(response, "tool_calls", None)
703 | if tool_calls:
704 | # Check if any tool call is a backend tool
705 | for tool_call in tool_calls:
706 | if tool_call.get("name") in backend_tool_names:
707 | return Command(
708 | goto="tool_node",
709 | update={"messages": [response], "apiKey": api_key}
710 | )
711 |
712 | # No backend tool calls, end the conversation turn
713 | return Command(
714 | goto=END,
715 | update={"messages": [response]}
716 | )
717 |
718 |
719 | async def process_tool_results(state: AgentState, config: RunnableConfig) -> Command[Literal["chat_node"]]:
720 | """Process tool results and update state with new artifacts."""
721 | import json
722 |
723 | # Get the messages
724 | messages = state["messages"]
725 | new_characters = list(state.get("characters", []))
726 | new_backgrounds = list(state.get("backgrounds", []))
727 | new_scenes = list(state.get("scenes", []))
728 |
729 | # Look for tool messages with results
730 | for msg in messages:
731 | if hasattr(msg, "name") and hasattr(msg, "content"):
732 | tool_name = msg.name
733 | try:
734 | # Parse the tool result
735 | if isinstance(msg.content, str):
736 | result = json.loads(msg.content)
737 | else:
738 | result = msg.content
739 |
740 | # Update appropriate collection
741 | if tool_name == "create_character" and isinstance(result, dict) and "id" in result:
742 | if not any(c["id"] == result["id"] for c in new_characters):
743 | new_characters.append(result)
744 | elif tool_name == "create_background" and isinstance(result, dict) and "id" in result:
745 | if not any(b["id"] == result["id"] for b in new_backgrounds):
746 | new_backgrounds.append(result)
747 | elif tool_name == "create_scene" and isinstance(result, dict) and "id" in result:
748 | if not any(s["id"] == result["id"] for s in new_scenes):
749 | new_scenes.append(result)
750 | # Handle edit tools - update existing items
751 | elif tool_name == "edit_character" and isinstance(result, dict) and "id" in result and not result.get("error"):
752 | for i, c in enumerate(new_characters):
753 | if c["id"] == result["id"]:
754 | new_characters[i] = result
755 | break
756 | elif tool_name == "edit_background" and isinstance(result, dict) and "id" in result and not result.get("error"):
757 | for i, b in enumerate(new_backgrounds):
758 | if b["id"] == result["id"]:
759 | new_backgrounds[i] = result
760 | break
761 | elif tool_name == "edit_scene" and isinstance(result, dict) and "id" in result and not result.get("error"):
762 | for i, s in enumerate(new_scenes):
763 | if s["id"] == result["id"]:
764 | new_scenes[i] = result
765 | break
766 |
767 | except (json.JSONDecodeError, TypeError):
768 | pass # Not a JSON result, skip
769 |
770 | return Command(
771 | goto="chat_node",
772 | update={
773 | "characters": new_characters,
774 | "backgrounds": new_backgrounds,
775 | "scenes": new_scenes,
776 | }
777 | )
778 |
779 |
780 | # === Build the graph ===
781 |
782 | workflow = StateGraph(AgentState)
783 |
784 | # Add nodes
785 | workflow.add_node("chat_node", chat_node)
786 | workflow.add_node("tool_node", ToolNode(tools=backend_tools))
787 | workflow.add_node("process_results", process_tool_results)
788 |
789 | # Set entry point
790 | workflow.set_entry_point("chat_node")
791 |
792 | # Add edges
793 | workflow.add_edge("tool_node", "process_results")
794 |
795 | # Compile the graph
796 | graph = workflow.compile()
797 |
--------------------------------------------------------------------------------