& {
46 | asChild?: boolean;
47 | }) {
48 | const Comp = asChild ? Slot : "button";
49 |
50 | return (
51 |
20 |
21 | An Error Occurred
22 |
23 |
24 | {error || "An unexpected error occurred."}
25 |
26 |
27 |
28 | Retry Load
29 |
30 |
31 | Start New Recording
32 |
33 |
34 |
37 | 🖱️
38 |
39 | Click on {s.elementTag}
40 | {s.elementText && `: "${s.elementText}"`}
41 |
42 |
43 | );
44 | }
45 | case "input": {
46 | const s = step as InputStep;
47 | return (
48 |
49 | ⌨️
50 |
51 | Input into {s.elementTag} : "{s.value}"
52 |
53 |
54 | );
55 | }
56 | case "key_press": {
57 | const s = step as KeyPressStep;
58 | return (
59 |
60 | 🔑
61 |
62 | Press {s.key} on {s.elementTag || "document"}
63 |
64 |
65 | );
66 | }
67 | case "navigation": {
68 | const s = step as NavigationStep;
69 | return (
70 |
71 | 🧭
72 | Navigate: {s.url}
73 |
74 | );
75 | }
76 | case "scroll": {
77 | const s = step as ScrollStep;
78 | return (
79 |
80 | ↕️
81 |
82 | Scroll to ({s.scrollX}, {s.scrollY})
83 |
84 |
85 | );
86 | }
87 | default:
88 | return <>{(step as any).type}; // Fallback
89 | }
90 | };
91 |
92 | // --- Step Detail Renderer (Collapsible section or part of card body) ---
93 | const renderStepDetailsContent = (step: Step) => {
94 | const baseInfo = (
95 | <>
96 |
97 | Timestamp: {" "}
98 | {new Date(step.timestamp).toLocaleString()}
99 |
100 | {step.url && (
101 |
102 | URL: {step.url}
103 |
104 | )}
105 | {/* Tab ID might be less relevant now, could remove */}
106 | {/* Tab ID: {step.tabId}
*/}
107 |
108 | );
109 |
110 | let specificInfo = null;
111 |
112 | switch (step.type) {
113 | case "click":
114 | case "input":
115 | case "key_press": {
116 | const s = step as ClickStep | InputStep | KeyPressStep; // Union type
117 | specificInfo = (
118 | <>
119 | {(s as ClickStep | InputStep).frameUrl &&
120 | (s as ClickStep | InputStep).frameUrl !== s.url && (
121 |
122 | Frame URL: {" "}
123 | {(s as ClickStep | InputStep).frameUrl}
124 |
125 | )}
126 | {s.xpath && (
127 |
128 | XPath: {s.xpath}
129 |
130 | )}
131 | {s.cssSelector && (
132 |
133 | CSS: {s.cssSelector}
134 |
135 | )}
136 | {s.elementTag && (
137 |
138 | Element: {s.elementTag}
139 |
140 | )}
141 | {(s as ClickStep).elementText && (
142 |
143 | Text: {(s as ClickStep).elementText}
144 |
145 | )}
146 | {(s as InputStep).value && (
147 |
148 | Value: {(s as InputStep).value}
149 |
150 | )}
151 | {(s as KeyPressStep).key && (
152 |
153 | Key: {(s as KeyPressStep).key}
154 |
155 | )}
156 |
157 | );
158 | break;
159 | }
160 | case "navigation": {
161 | // Base info already has URL
162 | break;
163 | }
164 | case "scroll": {
165 | const s = step as ScrollStep;
166 | specificInfo = (
167 | <>
168 |
169 | Target ID: {s.targetId}
170 |
171 |
172 | Scroll X: {s.scrollX}
173 |
174 |
175 | Scroll Y: {s.scrollY}
176 |
177 |
178 | );
179 | break;
180 | }
181 | default:
182 | specificInfo = (
183 | Details not available for type: {(step as any).type}
184 | );
185 | }
186 |
187 | return (
188 |
189 | {baseInfo}
190 | {specificInfo}
191 |
192 | );
193 | };
194 |
195 | return (
196 |
208 | {/* Card Content using Flexbox */}
209 |
210 | {/* Left side: Summary and Details */}
211 |
212 |
213 | {renderStepSummary(step)}
214 |
215 | {renderStepDetailsContent(step)}
216 |
217 |
218 | {/* Right side: Screenshot (if available) */}
219 | {canShowScreenshot && screenshot && (
220 |
221 |
227 |
228 | )}
229 |
230 |
(null); // Ref for the scrollable div
240 |
241 | // Effect to scroll selected card into view
242 | useEffect(() => {
243 | if (recordingStatus !== "recording") {
244 | // Only scroll selection when not recording
245 | const element = document.getElementById(
246 | `event-item-${currentEventIndex}`
247 | );
248 | element?.scrollIntoView({ behavior: "smooth", block: "center" });
249 | }
250 | }, [currentEventIndex, recordingStatus]); // Add recordingStatus dependency
251 |
252 | // Effect to scroll to bottom when new steps are added during recording
253 | useEffect(() => {
254 | if (recordingStatus === "recording" && scrollContainerRef.current) {
255 | const { current: container } = scrollContainerRef;
256 | // Use setTimeout to allow DOM update before scrolling
257 | setTimeout(() => {
258 | container.scrollTop = container.scrollHeight;
259 | console.log("Scrolled to bottom due to new event during recording");
260 | }, 0);
261 | }
262 | // Depend on the number of steps and recording status
263 | }, [steps.length, recordingStatus]);
264 |
265 | if (!workflow || !workflow.steps || workflow.steps.length === 0) {
266 | return (
267 |
268 | No events recorded yet.
269 |
270 | );
271 | }
272 |
273 | return (
274 | // Assign the ref to the scrollable container
275 |
276 | {" "}
277 | {/* Single scrollable container */}
278 | {steps.map((step, index) => (
279 | selectEvent(index)}
285 | />
286 | ))}
287 |
288 | );
289 | };
290 |
--------------------------------------------------------------------------------
/extension/src/entrypoints/sidepanel/components/initial-view.tsx:
--------------------------------------------------------------------------------
1 | import React from "react";
2 | import { useWorkflow } from "../context/workflow-provider";
3 | import { Button } from "@/components/ui/button"; // Reverted to alias path
4 |
5 | export const InitialView: React.FC = () => {
6 | const { startRecording } = useWorkflow();
7 |
8 | return (
9 |
10 |
Record a Workflow
11 |
12 | Start Recording
13 |
14 | );
15 | };
16 |
--------------------------------------------------------------------------------
/extension/src/entrypoints/sidepanel/components/logina-view.tsx:
--------------------------------------------------------------------------------
1 | import React from "react";
2 |
3 | // Simple loading spinner component
4 | const Spinner: React.FC = () => (
5 |
11 |
25 | );
26 |
27 | export const LoadingView: React.FC = () => {
28 | return (
29 |
30 |
31 |
Loading workflow data...
32 |
12 |
13 |
14 |
15 |
18 |
19 | Recording ({stepCount} steps)
20 |
21 |
22 |
23 | Stop Recording
24 |
25 |
26 |
27 | {/* EventViewer will now take full available space within this div */}
28 |
30 |
38 |
39 |
Recording Finished
40 |
41 |
42 | Discard & Start New
43 |
44 |
51 | Download JSON
52 |
53 |
54 |
55 |
56 |
58 |
(
28 | undefined
29 | );
30 |
31 | interface WorkflowProviderProps {
32 | children: ReactNode;
33 | }
34 |
35 | const POLLING_INTERVAL = 2000; // Fetch every 2 seconds during recording
36 |
37 | export const WorkflowProvider: React.FC = ({
38 | children,
39 | }) => {
40 | const [workflow, setWorkflow] = useState(null);
41 | const [recordingStatus, setRecordingStatus] = useState("idle"); // 'idle', 'recording', 'stopped', 'error'
42 | const [currentEventIndex, setCurrentEventIndex] = useState(0);
43 | const [isLoading, setIsLoading] = useState(true);
44 | const [error, setError] = useState(null);
45 |
46 | const fetchWorkflowData = useCallback(async (isPolling: boolean = false) => {
47 | if (!isPolling) {
48 | setIsLoading(true);
49 | }
50 | try {
51 | const data = await chrome.runtime.sendMessage({
52 | type: "GET_RECORDING_DATA",
53 | });
54 | console.log(
55 | "Received workflow data from background (polling=" + isPolling + "):",
56 | data
57 | );
58 | if (data && data.workflow && data.recordingStatus) {
59 | // Always update workflow when fetching (polling or not)
60 | setWorkflow(data.workflow);
61 |
62 | // If NOT polling, update status and index based on fetched data
63 | // If polling, we primarily rely on the broadcast message for status changes
64 | if (!isPolling) {
65 | setRecordingStatus(data.recordingStatus);
66 | setCurrentEventIndex(
67 | data.workflow.steps ? data.workflow.steps.length - 1 : 0
68 | );
69 | } else {
70 | // If polling, ensure index is valid (e.g., after step deletion)
71 | // Only adjust index based on current workflow state from polling
72 | setCurrentEventIndex((prevIndex) =>
73 | Math.min(prevIndex, (data.workflow.steps?.length || 1) - 1)
74 | );
75 | // Do NOT set recordingStatus from polling fetch, wait for broadcast message
76 | }
77 | // Clear error on successful fetch
78 | setError(null);
79 | } else {
80 | console.warn(
81 | "Received invalid/incomplete data structure from GET_RECORDING_DATA"
82 | );
83 | if (!isPolling) {
84 | setWorkflow(null);
85 | setRecordingStatus("idle");
86 | setCurrentEventIndex(0);
87 | }
88 | }
89 | } catch (err: any) {
90 | console.error(
91 | "Error fetching workflow data (polling=" + isPolling + "):",
92 | err
93 | );
94 | // Only set error state if it wasn't a background poll error
95 | // and the status isn't already error
96 | if (!isPolling) {
97 | setError(`Failed to load workflow data: ${err.message}`);
98 | setRecordingStatus("error");
99 | setWorkflow(null);
100 | }
101 | } finally {
102 | if (!isPolling) {
103 | setIsLoading(false);
104 | }
105 | }
106 | // Remove state dependencies to stabilize the function reference
107 | }, []);
108 |
109 | useEffect(() => {
110 | // Initial fetch on mount
111 | fetchWorkflowData(false);
112 |
113 | // Listener for status updates pushed from the background script
114 | const messageListener = (message: any, sender: any, sendResponse: any) => {
115 | console.log("Sidepanel received message:", message);
116 | if (message.type === "recording_status_updated") {
117 | console.log(
118 | "Recording status updated message received:",
119 | message.payload
120 | );
121 | const newStatus = message.payload.status;
122 | // Use functional update to get previous status reliably
123 | setRecordingStatus((prevStatus) => {
124 | // If status changed from non-stopped/idle to stopped or idle, fetch final data
125 | if (
126 | newStatus !== prevStatus &&
127 | (newStatus === "stopped" || newStatus === "idle")
128 | ) {
129 | fetchWorkflowData(false); // Fetch final data, show loading
130 | }
131 | return newStatus; // Return the new status to update the state
132 | });
133 | }
134 | };
135 | chrome.runtime.onMessage.addListener(messageListener);
136 |
137 | // --- Polling Logic ---
138 | let pollingIntervalId: NodeJS.Timeout | null = null;
139 | if (recordingStatus === "recording") {
140 | pollingIntervalId = setInterval(() => {
141 | fetchWorkflowData(true); // Fetch updates in the background (polling)
142 | }, POLLING_INTERVAL);
143 | console.log(`Polling started (Interval ID: ${pollingIntervalId})`);
144 | }
145 | // --- End Polling Logic ---
146 |
147 | // Cleanup listener and interval
148 | return () => {
149 | chrome.runtime.onMessage.removeListener(messageListener);
150 | if (pollingIntervalId) {
151 | clearInterval(pollingIntervalId);
152 | console.log(
153 | `Polling stopped (Cleared Interval ID: ${pollingIntervalId})`
154 | );
155 | }
156 | };
157 | // Keep dependencies: fetchWorkflowData is stable now,
158 | // recordingStatus dependency correctly handles interval setup/teardown.
159 | }, [fetchWorkflowData, recordingStatus]);
160 |
161 | // startRecording, stopRecording, discardAndStartNew, selectEvent remain largely the same
162 | // but ensure isLoading is handled appropriately
163 | const startRecording = useCallback(() => {
164 | setError(null);
165 | setIsLoading(true);
166 | chrome.runtime.sendMessage({ type: "START_RECORDING" }, (response) => {
167 | // Loading state will be turned off by the fetchWorkflowData triggered
168 | // by the recording_status_updated message, or on error here.
169 | if (chrome.runtime.lastError) {
170 | console.error("Error starting recording:", chrome.runtime.lastError);
171 | setError(
172 | `Failed to start recording: ${chrome.runtime.lastError.message}`
173 | );
174 | setRecordingStatus("error");
175 | setIsLoading(false); // Stop loading on error
176 | } else {
177 | console.log("Start recording acknowledged by background.");
178 | // State updates happen via broadcast + fetch
179 | }
180 | });
181 | }, []); // No dependencies needed
182 |
183 | const stopRecording = useCallback(() => {
184 | setError(null);
185 | setIsLoading(true);
186 | chrome.runtime.sendMessage({ type: "STOP_RECORDING" }, (response) => {
187 | // Loading state will be turned off by the fetchWorkflowData triggered
188 | // by the recording_status_updated message, or on error here.
189 | if (chrome.runtime.lastError) {
190 | console.error("Error stopping recording:", chrome.runtime.lastError);
191 | setError(
192 | `Failed to stop recording: ${chrome.runtime.lastError.message}`
193 | );
194 | setRecordingStatus("error");
195 | setIsLoading(false); // Stop loading on error
196 | } else {
197 | console.log("Stop recording acknowledged by background.");
198 | // State updates happen via broadcast + fetch
199 | }
200 | });
201 | }, []); // No dependencies needed
202 |
203 | const discardAndStartNew = useCallback(() => {
204 | startRecording();
205 | }, [startRecording]);
206 |
207 | const selectEvent = useCallback((index: number) => {
208 | setCurrentEventIndex(index);
209 | }, []);
210 |
211 | const value = {
212 | workflow,
213 | recordingStatus,
214 | currentEventIndex,
215 | isLoading,
216 | error,
217 | startRecording,
218 | stopRecording,
219 | discardAndStartNew,
220 | selectEvent,
221 | fetchWorkflowData,
222 | };
223 |
224 | return (
225 |
226 | {children}
227 |
228 | );
229 | };
230 |
231 | export const useWorkflow = (): WorkflowContextType => {
232 | const context = useContext(WorkflowContext);
233 | if (context === undefined) {
234 | throw new Error("useWorkflow must be used within a WorkflowProvider");
235 | }
236 | return context;
237 | };
238 |
--------------------------------------------------------------------------------
/extension/src/entrypoints/sidepanel/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Default Popup Title
7 |
15 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/extension/src/entrypoints/sidepanel/index.tsx:
--------------------------------------------------------------------------------
1 | import React from "react";
2 | import ReactDOM from "react-dom/client";
3 |
4 | // import vite tailwind css
5 | import "@/assets/tailwind.css";
6 |
7 | import { ErrorView } from "./components/error-view";
8 | import { InitialView } from "./components/initial-view";
9 | import { LoadingView } from "./components/logina-view";
10 | import { RecordingView } from "./components/recording-view";
11 | import { StoppedView } from "./components/stopped-view";
12 | import { WorkflowProvider, useWorkflow } from "./context/workflow-provider";
13 |
14 | const AppContent: React.FC = () => {
15 | const { recordingStatus, isLoading, error } = useWorkflow();
16 |
17 | if (isLoading) {
18 | return
39 |
40 |
45 |
46 |
47 | );
48 | };
49 |
50 | const rootElement = document.getElementById("root");
51 | if (!rootElement) {
52 | throw new Error("Root element not found");
53 | }
54 |
55 | const root = ReactDOM.createRoot(rootElement);
56 | root.render(
2 |
3 |
4 |
6 |
7 |
8 |
9 |
11 |
12 |
14 |
15 |
16 |
--------------------------------------------------------------------------------
/extension/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "extends": "./.wxt/tsconfig.json",
3 | "compilerOptions": {
4 | "allowImportingTsExtensions": true,
5 | "jsx": "react-jsx",
6 | "baseUrl": ".",
7 | "paths": {
8 | "@/*": [
9 | "./src/*"
10 | ]
11 | }
12 | }
13 | }
--------------------------------------------------------------------------------
/extension/vite.config.ts:
--------------------------------------------------------------------------------
1 | // vite.config.ts
2 | import { defineConfig } from "vite";
3 | import path from "path";
4 | import tailwindcss from "@tailwindcss/vite";
5 |
6 | export default defineConfig({
7 | plugins: [tailwindcss()],
8 | resolve: {
9 | alias: {
10 | "@": path.resolve(__dirname, "./src/"),
11 | },
12 | },
13 | });
14 |
--------------------------------------------------------------------------------
/extension/wxt.config.ts:
--------------------------------------------------------------------------------
1 | // wxt.config.ts
2 | import { defineConfig } from "wxt";
3 | import baseViteConfig from "./vite.config";
4 |
5 | import { mergeConfig } from "vite";
6 |
7 | // See https://wxt.dev/api/config.html
8 | export default defineConfig({
9 | modules: ["@wxt-dev/module-react"],
10 | srcDir: "src",
11 | vite: () =>
12 | mergeConfig(baseViteConfig, {
13 | // WXT-specific overrides (optional)
14 | }),
15 | manifest: {
16 | permissions: ["tabs", "sidePanel", ""],
17 | host_permissions: ["http://127.0.0.1/*"],
18 | // options_page: "options.html",
19 | // action: {
20 | // default_popup: "popup.html",
21 | // },
22 | },
23 | });
24 |
--------------------------------------------------------------------------------
/static/workflow-use.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/browser-use/workflow-use/fc823f70f1996c12f18a818a9676aac674daa154/static/workflow-use.png
--------------------------------------------------------------------------------
/ui/.eslintrc.cjs:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | root: true,
3 | env: { browser: true, es2020: true },
4 | extends: [
5 | 'eslint:recommended',
6 | 'plugin:@typescript-eslint/recommended',
7 | 'plugin:react-hooks/recommended',
8 | ],
9 | ignorePatterns: ['dist', '.eslintrc.cjs'],
10 | parser: '@typescript-eslint/parser',
11 | plugins: ['react-refresh'],
12 | rules: {
13 | 'react-refresh/only-export-components': [
14 | 'warn',
15 | { allowConstantExport: true },
16 | ],
17 | },
18 | }
19 |
--------------------------------------------------------------------------------
/ui/.github/dependabot.yml:
--------------------------------------------------------------------------------
1 | # docs:
2 | # https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
3 |
4 | version: 2
5 | updates:
6 | - package-ecosystem: 'npm'
7 | directory: '/'
8 | schedule:
9 | interval: 'daily'
10 | allow:
11 | - dependency-name: '@xyflow/react'
12 |
--------------------------------------------------------------------------------
/ui/.gitignore:
--------------------------------------------------------------------------------
1 | # Logs
2 | logs
3 | *.log
4 | npm-debug.log*
5 | yarn-debug.log*
6 | yarn-error.log*
7 | pnpm-debug.log*
8 | lerna-debug.log*
9 |
10 | node_modules
11 | dist
12 | dist-ssr
13 | *.local
14 |
15 | # Editor directories and files
16 | .vscode/*
17 | !.vscode/extensions.json
18 | .idea
19 | .DS_Store
20 | *.suo
21 | *.ntvs*
22 | *.njsproj
23 | *.sln
24 | *.sw?
25 |
26 | src/lib/api/apigen.d.ts
27 |
--------------------------------------------------------------------------------
/ui/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Browser Use Workflow Visualizer
8 |
9 |
10 |
11 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/ui/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "workflow-use-ui",
3 | "version": "0.0.0",
4 | "type": "module",
5 | "scripts": {
6 | "dev": "vite",
7 | "build": "tsc && vite build",
8 | "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
9 | "preview": "vite preview",
10 | "get-openapi": "curl http://localhost:${BACKEND_PORT:-8000}/openapi.json -o ./src/lib/api/openapi.json",
11 | "type-gen": "npx openapi-typescript ./src/lib/api/openapi.json -o ./src/lib/api/apigen.d.ts",
12 | "type-gen-update": "rm -rf ./src/lib/api/apigen.d.ts && npm run get-openapi && npm run type-gen",
13 | "postinstall": "test -n \"$NOYARNPOSTINSTALL\" || npm run type-gen"
14 | },
15 | "dependencies": {
16 | "@tailwindcss/vite": "^4.1.7",
17 | "@xyflow/react": "^12.5.1",
18 | "openapi-fetch": "^0.14.0",
19 | "openapi-react-query": "^0.5.0",
20 | "react": "^18.2.0",
21 | "react-dom": "^18.2.0",
22 | "tailwindcss": "^4.1.7",
23 | "zod": "^3.24.4"
24 | },
25 | "devDependencies": {
26 | "@types/node": "^22.15.19",
27 | "@types/react": "^18.2.53",
28 | "@types/react-dom": "^18.2.18",
29 | "@typescript-eslint/eslint-plugin": "^6.20.0",
30 | "@typescript-eslint/parser": "^6.20.0",
31 | "@vitejs/plugin-react": "^4.2.1",
32 | "eslint": "^8.56.0",
33 | "eslint-plugin-react-hooks": "^4.6.0",
34 | "eslint-plugin-react-refresh": "^0.4.5",
35 | "openapi-typescript": "^7.8.0",
36 | "typescript": "^5.3.3",
37 | "vite": "^5.0.12"
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/ui/public/browseruse.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/browser-use/workflow-use/fc823f70f1996c12f18a818a9676aac674daa154/ui/public/browseruse.png
--------------------------------------------------------------------------------
/ui/public/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/browser-use/workflow-use/fc823f70f1996c12f18a818a9676aac674daa154/ui/public/favicon.ico
--------------------------------------------------------------------------------
/ui/src/App.tsx:
--------------------------------------------------------------------------------
1 | import { ReactFlowProvider } from "@xyflow/react";
2 | import "@xyflow/react/dist/style.css";
3 | import WorkflowLayout from "./components/workflow-layout";
4 | import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
5 |
6 | // Create a client
7 | const queryClient = new QueryClient();
8 |
9 | export default function App() {
10 | return (
11 |
12 |
13 |
15 |
16 | );
17 | }
18 |
--------------------------------------------------------------------------------
/ui/src/components/log-viewer.tsx:
--------------------------------------------------------------------------------
1 | import React, { useState, useEffect, useRef } from "react";
2 | import { LogViewerProps } from "../types/log-viewer.types";
3 |
4 | const LogViewer: React.FC = ({
5 | taskId,
6 | initialPosition,
7 | onStatusChange,
8 | onError,
9 | onCancel,
10 | onClose,
11 | }) => {
12 | const [isCancelling, setIsCancelling] = useState(false);
13 | const [logs, setLogs] = useState([]);
14 | const [position, setPosition] = useState(initialPosition);
15 | const [status, setStatus] = useState("running");
16 | const [error, setError] = useState(null);
17 | const [polling, setPolling] = useState(true);
18 |
19 | const logContainerRef = useRef(null);
20 | const pollingIntervalRef = useRef(null);
21 |
22 | useEffect(() => {
23 | if (logContainerRef.current) {
24 | logContainerRef.current.scrollTop = logContainerRef.current.scrollHeight;
25 | }
26 | }, [logs]);
27 |
28 | useEffect(() => {
29 | return () => {
30 | setPolling(false);
31 | };
32 | }, []);
33 |
34 | useEffect(() => {
35 | if (!taskId) return;
36 |
37 | const pollLogs = async () => {
38 | try {
39 | const response = await fetch(
40 | `http://127.0.0.1:8000/api/workflows/logs/${taskId}?position=${position}`
41 | );
42 |
43 | if (!response.ok) {
44 | throw new Error(`Error fetching logs: ${response.status}`);
45 | }
46 |
47 | const data = await response.json();
48 |
49 | if (data.logs && data.logs.length > 0) {
50 | setLogs((prevLogs) => [...prevLogs, ...data.logs]);
51 | }
52 |
53 | setPosition(data.log_position);
54 |
55 | if (data.status && data.status !== status) {
56 | setStatus(data.status);
57 | onStatusChange?.(data.status);
58 |
59 | if (data.status === "failed" && data.error) {
60 | setError(data.error);
61 | onError?.(data.error);
62 | }
63 | }
64 | } catch (err) {
65 | console.error("Error polling logs:", err);
66 | }
67 | };
68 |
69 | pollLogs();
70 |
71 | pollingIntervalRef.current = setInterval(() => {
72 | if (polling) {
73 | pollLogs();
74 | }
75 | }, 2000);
76 |
77 | return () => {
78 | if (pollingIntervalRef.current) {
79 | clearInterval(pollingIntervalRef.current);
80 | }
81 | };
82 | }, [taskId, position, status, polling, onStatusChange, onError]);
83 |
84 | const cancelWorkflow = async () => {
85 | if (!taskId || isCancelling || status !== "running") return;
86 |
87 | setIsCancelling(true);
88 |
89 | try {
90 | const response = await fetch(
91 | `http://127.0.0.1:8000/api/workflows/tasks/${taskId}/cancel`,
92 | {
93 | method: "POST",
94 | headers: {
95 | "Content-Type": "application/json",
96 | },
97 | }
98 | );
99 |
100 | const data = await response.json();
101 |
102 | if (response.ok && data.success) {
103 | // The status will be updated through the polling mechanism
104 | if (onCancel) {
105 | onCancel();
106 | }
107 | } else {
108 | console.error(
109 | "Failed to cancel workflow:",
110 | data.message || "Unknown error"
111 | );
112 | }
113 | } catch (err) {
114 | console.error("Error cancelling workflow:", err);
115 | } finally {
116 | setIsCancelling(false);
117 | }
118 | };
119 |
120 | const downloadLogs = () => {
121 | if (logs.length === 0) return;
122 |
123 | const textContent = logs.join("");
124 |
125 | const blob = new Blob([textContent], { type: "text/plain" });
126 |
127 | const url = URL.createObjectURL(blob);
128 |
129 | const link = document.createElement("a");
130 | link.href = url;
131 | link.download = taskId
132 | ? `workflow-logs-${taskId}.txt`
133 | : "workflow-logs.txt";
134 |
135 | document.body.appendChild(link);
136 | link.click();
137 | document.body.removeChild(link);
138 |
139 | URL.revokeObjectURL(url);
140 | };
141 |
142 | const formatLog = (log: string, index: number) => {
143 | const timestampMatch = log.match(/^\[(.*?)\]/);
144 |
145 | if (timestampMatch) {
146 | const timestamp = timestampMatch[0];
147 | const message = log.substring(timestamp.length);
148 |
149 | return (
150 |
151 | {timestamp}
152 | {message}
153 |
154 | );
155 | }
156 |
157 | return (
158 |
159 | {log}
160 |
161 | );
162 | };
163 |
164 | return (
165 |
166 |
167 |
Workflow Execution Logs
168 |
169 |
170 | {status === "running" && (
171 |
179 |
191 |
195 | {isCancelling ? "Cancelling..." : "Cancel"}
196 |
197 | )}
198 | {logs.length > 0 && (
199 |
204 |
216 |
220 | Download
221 |
222 | )}
223 |
224 |
239 | Status: {status.charAt(0).toUpperCase() + status.slice(1)}
240 |
241 |
242 |
243 |
244 |
248 | {logs.length > 0 ? (
249 | logs.map((log, index) => formatLog(log, index))
250 | ) : (
251 |
252 | Waiting for logs...
253 |
254 | )}
255 |
256 | {error && (
257 |
258 | Error: {error}
259 |
260 | )}
261 |
262 |
263 | {/* Close button at the bottom */}
264 |
265 |
270 | Close
271 |
272 |
273 |
6 |
11 |
No Workflows Found
12 |
13 | To get started with Workflow Use, you need to first create a workflow
14 | or place an existing workflow file in the workflows/tmp folder.
15 |
16 |
17 | Once you've added a workflow file, refresh this page to visualize and interact with it. For more information, checkout out the documentation below.
18 |
19 |
25 | Learn More
26 |
27 |
= ({
6 | workflowName,
7 | workflowMetadata,
8 | }) => {
9 | const [showModal, setShowModal] = useState(false);
10 | const [showLogViewer, setShowLogViewer] = useState(false);
11 | const [isRunning, setIsRunning] = useState(false);
12 | const [error, setError] = useState(null);
13 | const [inputFields, setInputFields] = useState([]);
14 | const [taskId, setTaskId] = useState(null);
15 | const [logPosition, setLogPosition] = useState(0);
16 | const [workflowStatus, setWorkflowStatus] = useState("idle");
17 |
18 | const openModal = () => {
19 | if (!workflowName) return;
20 |
21 | setShowModal(true);
22 | setError(null);
23 |
24 | if (workflowMetadata && workflowMetadata.input_schema) {
25 | const fields = workflowMetadata.input_schema.map((input: any) => ({
26 | name: input.name,
27 | type: input.type,
28 | required: input.required,
29 | value: input.type === "boolean" ? false : "",
30 | }));
31 | setInputFields(fields);
32 | } else {
33 | setInputFields([]);
34 | }
35 | };
36 |
37 | const closeModal = () => {
38 | setShowModal(false);
39 |
40 | if (!isRunning) {
41 | resetState();
42 | }
43 | };
44 |
45 | const closeLogViewer = () => {
46 | setShowLogViewer(false);
47 | resetState();
48 | };
49 |
50 | const resetState = () => {
51 | setIsRunning(false);
52 | setError(null);
53 | setInputFields([]);
54 | setTaskId(null);
55 | setLogPosition(0);
56 | setWorkflowStatus("idle");
57 | };
58 |
59 | const handleInputChange = (index: number, value: any) => {
60 | const updatedFields = [...inputFields];
61 | if (updatedFields[index]) {
62 | updatedFields[index].value = value;
63 | setInputFields(updatedFields);
64 | }
65 | };
66 |
67 | const executeWorkflow = async () => {
68 | if (!workflowName) return;
69 |
70 | const missingInputs = inputFields.filter(
71 | (field) => field.required && !field.value
72 | );
73 | if (missingInputs.length > 0) {
74 | setError(
75 | `Missing required inputs: ${missingInputs
76 | .map((f) => f.name)
77 | .join(", ")}`
78 | );
79 | return;
80 | }
81 |
82 | setIsRunning(true);
83 | setError(null);
84 | setTaskId(null);
85 | setLogPosition(0);
86 | setWorkflowStatus("idle");
87 |
88 | try {
89 | const inputs: Record = {};
90 | inputFields.forEach((field) => {
91 | inputs[field.name] = field.value;
92 | });
93 |
94 | const response = await fetch(
95 | "http://127.0.0.1:8000/api/workflows/execute",
96 | {
97 | method: "POST",
98 | headers: {
99 | "Content-Type": "application/json",
100 | },
101 | body: JSON.stringify({
102 | name: workflowName,
103 | inputs,
104 | }),
105 | }
106 | );
107 |
108 | const data = await response.json();
109 | setTaskId(data.task_id);
110 | setLogPosition(data.log_position);
111 | setIsRunning(true);
112 | setShowLogViewer(true);
113 | setShowModal(false);
114 | } catch (err) {
115 | console.error("Failed to execute workflow:", err);
116 | setError("An error occurred while executing the workflow");
117 | }
118 | };
119 |
120 | const handleStatusChange = (status: string) => {
121 | setWorkflowStatus(status);
122 |
123 | if (
124 | status === "completed" ||
125 | status === "failed" ||
126 | status === "cancelled"
127 | ) {
128 | setIsRunning(false);
129 | }
130 | };
131 |
132 | const handleCancelWorkflow = () => {
133 | setWorkflowStatus("cancelling");
134 | };
135 |
136 | const handleWorkflowError = (errorMessage: string) => {
137 | setError(errorMessage);
138 | };
139 |
140 | if (!workflowName) return null;
141 |
142 | return (
143 |
144 | {/* play button */}
145 |
150 | ▶
151 |
152 |
153 | {/* parameter‑input modal */}
154 | {showModal && (
155 |
156 |
157 | {/* header */}
158 |
159 |
160 | Execute Workflow: {workflowMetadata?.name || workflowName}
161 |
162 |
166 | ×
167 |
168 |
169 |
170 | {/* content */}
171 |
172 | {error && (
173 |
174 | {error}
175 |
176 | )}
177 |
178 | {inputFields.length ? (
179 |
215 | ) : (
216 |
No input parameters required for this workflow.
217 | )}
218 |
219 |
220 |
224 | Cancel
225 |
226 |
231 | Execute Workflow
232 |
233 |
234 |
235 |
236 |
237 | )}
238 |
239 | {/* log viewer */}
240 | {showLogViewer && taskId && (
241 |
242 |
243 |
244 |
245 | Workflow Execution
246 | {workflowStatus !== "running" &&
247 | ` (${workflowStatus
248 | .charAt(0)
249 | .toUpperCase()}${workflowStatus.slice(1)})`}
250 |
251 |
252 |
253 |
254 |
263 |
264 |
265 | )}
266 |
;
11 | allWorkflowsMetadata?: Record;
12 | }
13 |
14 | export const Sidebar: React.FC = ({
15 | workflows,
16 | onSelect,
17 | selected,
18 | workflowMetadata,
19 | onUpdateMetadata,
20 | allWorkflowsMetadata = {},
21 | }) => (
22 |
23 | {/* logo */}
24 |
25 |
30 |
Workflows
33 |
34 |
35 | {workflows.map((id) => (
36 |
50 |
51 | );
52 |
53 | export default Sidebar;
54 |
--------------------------------------------------------------------------------
/ui/src/components/workflow-item.tsx:
--------------------------------------------------------------------------------
1 | import React, { useState, ChangeEvent } from "react";
2 | import {
3 | WorkflowMetadata,
4 | WorkflowItemProps,
5 | } from "../types/workflow-layout.types";
6 |
7 | const WorkflowItem: React.FC = ({
8 | id,
9 | selected,
10 | metadata,
11 | onSelect,
12 | onUpdateMetadata,
13 | }) => {
14 | const [isEditing, setIsEditing] = useState(false);
15 | const [edited, setEdited] = useState(null);
16 | const [submitting, setSubmitting] = useState(false);
17 |
18 | const displayName = () => {
19 | if (metadata)
20 | return (
21 | <>
22 | {metadata.name}
23 | {metadata.version && (
24 |
25 | v{metadata.version}
26 |
27 | )}
28 |
29 | );
30 |
31 | return (
32 |
33 | Loading workflow…
34 | {id}
35 |
36 | );
37 | };
38 |
39 | const change =
40 | (field: keyof WorkflowMetadata) =>
41 | (e: ChangeEvent) =>
42 | edited && setEdited({ ...edited, [field]: e.target.value });
43 |
44 | const save = async () => {
45 | if (!edited) return;
46 | setSubmitting(true);
47 | await onUpdateMetadata(edited).finally(() => setSubmitting(false));
48 | setIsEditing(false);
49 | };
50 |
51 | const editForm = metadata && (
52 |
53 | {(["name", "version"] as (keyof WorkflowMetadata)[]).map((f) => (
54 |
55 |
56 | {f[0]?.toUpperCase() + f.slice(1)}
57 |
58 |
64 | ))}
65 |
66 |
67 | Description
68 |
73 |
74 |
75 |
76 | setIsEditing(false)}
78 | className="bg-[#444] text-white py-1.5 px-3 rounded text-xs"
79 | >
80 | Cancel
81 |
82 |
89 | {submitting ? "Saving…" : "Save"}
90 |
91 |
92 |
98 |
Description
99 |
{metadata.description}
100 |
Input Parameters
105 |
106 | {metadata.input_schema.map((p) => (
107 |
108 | {p.name}
109 | ({p.type})
110 | {p.required && * }
111 |
112 | ))}
113 |
114 |
115 | )}
116 |
117 | );
118 |
119 | return (
120 | <>
121 | {/* row button */}
122 |
123 | onSelect(id)}
128 | >
129 | {displayName()}
130 |
131 |
132 |
133 | {/* details panel */}
134 | {selected && metadata && (
135 |
136 |
137 |
Details
138 | {!isEditing && (
139 | {
141 | setEdited({ ...metadata });
142 | setIsEditing(true);
143 | }}
144 | className="bg-[#444] text-white py-1 px-2 rounded text-xs"
145 | >
146 | Edit
147 |
148 | )}
149 |
150 | {isEditing ? editForm : readOnly}
151 |
152 | )}
153 |
154 | );
155 | };
156 |
157 | export default WorkflowItem;
158 |
--------------------------------------------------------------------------------
/ui/src/components/workflow-layout.tsx:
--------------------------------------------------------------------------------
1 | import React, {
2 | useState,
3 | useCallback,
4 | useLayoutEffect,
5 | MouseEvent,
6 | useEffect,
7 | } from "react";
8 | import {
9 | ReactFlow,
10 | Background,
11 | Controls,
12 | MiniMap,
13 | addEdge,
14 | useNodesState,
15 | useEdgesState,
16 | type OnConnect,
17 | useReactFlow,
18 | } from "@xyflow/react";
19 | import { type Node } from "@xyflow/react";
20 | import { NodeData } from "../types/node-config-menu.types";
21 | import { jsonToFlow } from "../utils/json-to-flow";
22 | import { type WorkflowMetadata } from "../types/workflow-layout.types";
23 | import Sidebar from "./sidebar";
24 | import { NodeConfigMenu } from "./node-config-menu";
25 | import { PlayButton } from "./play-button";
26 | import NoWorkflowsMessage from "./no-workflow-message";
27 | import { $api } from "../lib/api";
28 |
29 | const WorkflowLayout: React.FC = () => {
30 | const [selected, setSelected] = useState(null);
31 | const [nodes, setNodes, onNodesChange] = useNodesState([]);
32 | const [edges, setEdges, onEdgesChange] = useEdgesState([]);
33 | const [selectedNode, setSelectedNode] = useState | null>(null);
34 | const [workflowMetadata, setWorkflowMetadata] =
35 | useState(null);
36 | const [savedNodePositions, setSavedNodePositions] = useState<
37 | Record>
38 | >({});
39 | const { fitView } = useReactFlow();
40 |
41 | // ----- Queries using $api -----
42 | // Fetch all workflows
43 | const { data: workflowsResponse, isLoading: isLoadingWorkflows } =
44 | $api.useQuery("get", "/api/workflows");
45 |
46 | const workflows: string[] = workflowsResponse?.workflows ?? [];
47 |
48 | // Fetch a specific workflow (enabled only when selected is truthy)
49 | const { data: selectedWorkflow, isLoading: isLoadingSelectedWorkflow } =
50 | $api.useQuery(
51 | "get",
52 | "/api/workflows/{name}",
53 | selected
54 | ? {
55 | params: { path: { name: selected } },
56 | }
57 | : ({} as any),
58 | {
59 | enabled: !!selected,
60 | }
61 | );
62 |
63 | // Mutation for updating workflow metadata
64 | const updateMetadataMutation = $api.useMutation(
65 | "post",
66 | "/api/workflows/update-metadata"
67 | );
68 |
69 | const updateWorkflowMetadata = useCallback(
70 | async (name: string, metadata: WorkflowMetadata) => {
71 | await updateMetadataMutation.mutateAsync({
72 | body: { name, metadata } as any,
73 | });
74 | },
75 | [updateMetadataMutation]
76 | );
77 |
78 | const isUpdating = updateMetadataMutation.isPending;
79 |
80 | const onConnect: OnConnect = useCallback(
81 | (connection) => setEdges((edges) => addEdge(connection, edges)),
82 | [setEdges]
83 | );
84 |
85 | // Handle node click to show configuration
86 | const onNodeClick = useCallback((_: MouseEvent, node: Node) => {
87 | setSelectedNode(node);
88 | }, []);
89 |
90 | // Close the node configuration menu
91 | const closeNodeMenu = useCallback(() => {
92 | setSelectedNode(null);
93 | }, []);
94 |
95 | // Fit view when nodes change
96 | useLayoutEffect(() => {
97 | if (nodes.length > 0) {
98 | window.requestAnimationFrame(() => {
99 | fitView({ padding: 0.2 });
100 | });
101 | }
102 | }, [nodes.length, fitView]);
103 |
104 | // Update nodes and edges when selected workflow changes
105 | useEffect(() => {
106 | if (selectedWorkflow) {
107 | const flowData = jsonToFlow(selectedWorkflow);
108 |
109 | // Apply saved positions if available
110 | if (selected && savedNodePositions[selected]) {
111 | const savedPositionsForWorkflow = savedNodePositions[selected] || {};
112 | const nodesWithSavedPositions = flowData.nodes.map((node: any) => {
113 | const savedPosition = savedPositionsForWorkflow[node.id];
114 | if (savedPosition) {
115 | return {
116 | ...node,
117 | position: savedPosition,
118 | };
119 | }
120 | return node;
121 | });
122 | setNodes(nodesWithSavedPositions as any);
123 | } else {
124 | setNodes(flowData.nodes as any);
125 | }
126 |
127 | setEdges(flowData.edges as any);
128 | setWorkflowMetadata(flowData.metadata);
129 | }
130 | }, [selectedWorkflow, selected, savedNodePositions, setNodes, setEdges]);
131 |
132 | // Handle node drag stop to save positions
133 | const onNodeDragStop = useCallback(
134 | (_: React.MouseEvent, node: Node) => {
135 | if (selected) {
136 | setSavedNodePositions((prev) => {
137 | const workflowPositions = prev[selected] || {};
138 | return {
139 | ...prev,
140 | [selected]: {
141 | ...workflowPositions,
142 | [node.id]: { x: node.position.x, y: node.position.y },
143 | },
144 | };
145 | });
146 | }
147 | },
148 | [selected]
149 | );
150 |
151 | // Auto-select first workflow if none selected
152 | useEffect(() => {
153 | if (workflows.length > 0 && !selected) {
154 | setSelected(workflows[0]!);
155 | }
156 | }, [workflows, selected]);
157 |
158 | const isLoading = isLoadingWorkflows || isLoadingSelectedWorkflow;
159 |
160 | if (isLoading) {
161 | return (
162 |
163 |
168 |
Loading workflows...
169 |
177 |
{
183 | if (selected) {
184 | await updateWorkflowMetadata(selected, metadata);
185 | }
186 | }}
187 | />
188 |
189 |
190 | {nodes.length ? (
191 | <>
192 |
203 |
207 |
208 | {/* actions */}
209 |
210 |
214 |
215 |
{
219 | if (selected && workflowMetadata) {
220 | await updateWorkflowMetadata(selected, workflowMetadata);
221 | }
222 | }}
223 | disabled={isUpdating}
224 | >
225 | {/* hero‑icons refresh */}
226 |
236 |
238 |
239 |
240 |
241 | ) : (
242 |
243 | Select a workflow to visualize
244 |
245 | )}
246 |
247 | {selectedNode && (
248 |
253 | )}
254 |
({
6 | baseUrl: "http://localhost:8000",
7 | });
8 |
9 | export const $api = createClient(fetchClient);
10 |
11 | export default $api;
12 |
--------------------------------------------------------------------------------
/ui/src/lib/api/openapi.json:
--------------------------------------------------------------------------------
1 | {"openapi":"3.1.0","info":{"title":"Workflow Execution Service","version":"0.1.0"},"paths":{"/api/workflows":{"get":{"summary":"List Workflows","operationId":"list_workflows_api_workflows_get","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/WorkflowListResponse"}}}}}}},"/api/workflows/{name}":{"get":{"summary":"Get Workflow","operationId":"get_workflow_api_workflows__name__get","parameters":[{"name":"name","in":"path","required":true,"schema":{"type":"string","title":"Name"}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"type":"string","title":"Response Get Workflow Api Workflows Name Get"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/workflows/update":{"post":{"summary":"Update Workflow","operationId":"update_workflow_api_workflows_update_post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/WorkflowUpdateRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/WorkflowResponse"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/workflows/update-metadata":{"post":{"summary":"Update Workflow Metadata","operationId":"update_workflow_metadata_api_workflows_update_metadata_post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/WorkflowMetadataUpdateRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/WorkflowResponse"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/workflows/execute":{"post":{"summary":"Execute Workflow","operationId":"execute_workflow_api_workflows_execute_post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/WorkflowExecuteRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/WorkflowExecuteResponse"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/workflows/logs/{task_id}":{"get":{"summary":"Get Logs","operationId":"get_logs_api_workflows_logs__task_id__get","parameters":[{"name":"task_id","in":"path","required":true,"schema":{"type":"string","title":"Task Id"}},{"name":"position","in":"query","required":false,"schema":{"type":"integer","default":0,"title":"Position"}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/WorkflowLogsResponse"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/workflows/tasks/{task_id}/status":{"get":{"summary":"Get Task Status","operationId":"get_task_status_api_workflows_tasks__task_id__status_get","parameters":[{"name":"task_id","in":"path","required":true,"schema":{"type":"string","title":"Task Id"}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/WorkflowStatusResponse"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/api/workflows/tasks/{task_id}/cancel":{"post":{"summary":"Cancel Workflow","operationId":"cancel_workflow_api_workflows_tasks__task_id__cancel_post","parameters":[{"name":"task_id","in":"path","required":true,"schema":{"type":"string","title":"Task Id"}}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/WorkflowCancelResponse"}}}},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}}},"components":{"schemas":{"HTTPValidationError":{"properties":{"detail":{"items":{"$ref":"#/components/schemas/ValidationError"},"type":"array","title":"Detail"}},"type":"object","title":"HTTPValidationError"},"ValidationError":{"properties":{"loc":{"items":{"anyOf":[{"type":"string"},{"type":"integer"}]},"type":"array","title":"Location"},"msg":{"type":"string","title":"Message"},"type":{"type":"string","title":"Error Type"}},"type":"object","required":["loc","msg","type"],"title":"ValidationError"},"WorkflowCancelResponse":{"properties":{"success":{"type":"boolean","title":"Success"},"message":{"type":"string","title":"Message"}},"type":"object","required":["success","message"],"title":"WorkflowCancelResponse"},"WorkflowExecuteRequest":{"properties":{"name":{"type":"string","title":"Name"},"inputs":{"type":"object","title":"Inputs"}},"type":"object","required":["name","inputs"],"title":"WorkflowExecuteRequest"},"WorkflowExecuteResponse":{"properties":{"success":{"type":"boolean","title":"Success"},"task_id":{"type":"string","title":"Task Id"},"workflow":{"type":"string","title":"Workflow"},"log_position":{"type":"integer","title":"Log Position"},"message":{"type":"string","title":"Message"}},"type":"object","required":["success","task_id","workflow","log_position","message"],"title":"WorkflowExecuteResponse"},"WorkflowListResponse":{"properties":{"workflows":{"items":{"type":"string"},"type":"array","title":"Workflows"}},"type":"object","required":["workflows"],"title":"WorkflowListResponse"},"WorkflowLogsResponse":{"properties":{"logs":{"items":{"type":"string"},"type":"array","title":"Logs"},"position":{"type":"integer","title":"Position"},"log_position":{"type":"integer","title":"Log Position"},"status":{"type":"string","title":"Status"},"result":{"anyOf":[{"items":{"type":"object"},"type":"array"},{"type":"null"}],"title":"Result"},"error":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Error"}},"type":"object","required":["logs","position","log_position","status"],"title":"WorkflowLogsResponse"},"WorkflowMetadataUpdateRequest":{"properties":{"name":{"type":"string","title":"Name"},"metadata":{"type":"object","title":"Metadata"}},"type":"object","required":["name","metadata"],"title":"WorkflowMetadataUpdateRequest"},"WorkflowResponse":{"properties":{"success":{"type":"boolean","title":"Success"},"error":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Error"}},"type":"object","required":["success"],"title":"WorkflowResponse"},"WorkflowStatusResponse":{"properties":{"task_id":{"type":"string","title":"Task Id"},"status":{"type":"string","title":"Status"},"workflow":{"type":"string","title":"Workflow"},"result":{"anyOf":[{"items":{"type":"object"},"type":"array"},{"type":"null"}],"title":"Result"},"error":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Error"}},"type":"object","required":["task_id","status","workflow"],"title":"WorkflowStatusResponse"},"WorkflowUpdateRequest":{"properties":{"filename":{"type":"string","title":"Filename"},"nodeId":{"type":"integer","title":"Nodeid"},"stepData":{"type":"object","title":"Stepdata"}},"type":"object","required":["filename","nodeId","stepData"],"title":"WorkflowUpdateRequest"}}}}
--------------------------------------------------------------------------------
/ui/src/main.tsx:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import ReactDOM from 'react-dom/client';
3 |
4 | import App from './App';
5 |
6 | import './index.css';
7 |
8 | ReactDOM.createRoot(document.getElementById('root')!).render(
9 |
10 |
12 | );
13 |
--------------------------------------------------------------------------------
/ui/src/types/log-viewer.types.ts:
--------------------------------------------------------------------------------
1 | export interface LogViewerProps {
2 | taskId: string | null;
3 | initialPosition: number;
4 | onStatusChange?: (status: string) => void;
5 | onError?: (error: string) => void;
6 | onCancel?: () => void;
7 | onClose?: () => void;
8 | }
--------------------------------------------------------------------------------
/ui/src/types/node-config-menu.types.ts:
--------------------------------------------------------------------------------
1 | import type { Node } from '@xyflow/react';
2 |
3 | export interface StepData {
4 | description: string;
5 | output: any | null;
6 | timestamp: number | null;
7 | tabId: number | null;
8 | type: 'navigation' | 'click' | 'select_change' | 'input';
9 | url?: string;
10 | cssSelector?: string;
11 | xpath?: string;
12 | elementTag?: string;
13 | elementText?: string;
14 | selectedText?: string;
15 | value?: string;
16 | }
17 |
18 | export interface NodeData extends Record {
19 | label: string;
20 | stepData: StepData;
21 | }
22 |
23 | export interface NodeConfigMenuProps {
24 | node: Node | null;
25 | onClose: () => void;
26 | workflowFilename: string | null;
27 | }
--------------------------------------------------------------------------------
/ui/src/types/play-button.types.ts:
--------------------------------------------------------------------------------
1 | import { WorkflowMetadata } from './workflow-layout.types';
2 |
3 | export interface PlayButtonProps {
4 | workflowName: string | null;
5 | workflowMetadata: WorkflowMetadata | null;
6 | }
7 |
8 | export interface InputField {
9 | name: string;
10 | type: string;
11 | required: boolean;
12 | value: any;
13 | }
--------------------------------------------------------------------------------
/ui/src/types/sidebar.types.ts:
--------------------------------------------------------------------------------
1 | import { type WorkflowMetadata } from './workflow-layout.types';
2 |
3 | export interface SidebarProps {
4 | workflows: string[];
5 | onSelect: (wf: string) => void;
6 | selected: string | null;
7 | workflowMetadata: WorkflowMetadata | null;
8 | onUpdateMetadata: (metadata: WorkflowMetadata) => Promise;
9 | allWorkflowsMetadata?: Record;
10 | }
--------------------------------------------------------------------------------
/ui/src/types/workflow-layout.types.ts:
--------------------------------------------------------------------------------
1 | import { z } from 'zod';
2 | import type { Node, BuiltInNode } from '@xyflow/react';
3 |
4 | export type PositionLoggerNode = Node<{ label: string }, 'position-logger'>;
5 | export type AppNode = BuiltInNode | PositionLoggerNode;
6 |
7 | /* ── Input field definition ────────────────────────────────────────── */
8 | const inputFieldSchema = z.object({
9 | name: z.string(),
10 | type: z.enum(['string', 'number', 'boolean']),
11 | required: z.boolean(),
12 | });
13 |
14 | /* ── Step definition ───────────────────────────────────────────────── */
15 | const stepSchema = z.object({
16 | /* core fields */
17 | description: z.string(),
18 | output: z.unknown().nullable(),
19 | timestamp: z.number().int().nullable(),
20 | tabId: z.number().int().nullable(),
21 | type: z.enum(['navigation', 'click', 'select_change', 'input']),
22 |
23 | /* optional fields (vary by step type) */
24 | url: z.string().url().optional(),
25 | cssSelector: z.string().optional(),
26 | xpath: z.string().optional(),
27 | elementTag: z.string().optional(),
28 | elementText: z.string().optional(),
29 | selectedText: z.string().optional(),
30 | value: z.string().optional(),
31 | });
32 |
33 | /* ── Workflow wrapper ──────────────────────────────────────────────── */
34 | export const workflowSchema = z.object({
35 | workflow_analysis: z.string(),
36 | name: z.string(),
37 | description: z.string(),
38 | version: z.string(),
39 | steps: z.array(stepSchema),
40 | input_schema: z.array(inputFieldSchema),
41 | });
42 |
43 | /* ── Inferred TypeScript type ───────────────────────────────────────– */
44 | export type Workflow = z.infer;
45 |
46 | export interface WorkflowStep {
47 | description: string;
48 | type: string;
49 | [key: string]: any;
50 | }
51 |
52 | export interface WorkflowMetadata {
53 | name: string;
54 | description: string;
55 | version: string;
56 | input_schema: any[];
57 | workflow_analysis?: string;
58 | }
59 |
60 | export interface WorkflowItemProps {
61 | id: string;
62 | selected: boolean;
63 | metadata?: WorkflowMetadata;
64 | onSelect: (id: string) => void;
65 | onUpdateMetadata: (m: WorkflowMetadata) => Promise;
66 | }
67 |
--------------------------------------------------------------------------------
/ui/src/utils/json-to-flow.ts:
--------------------------------------------------------------------------------
1 | import { Edge } from '@xyflow/react';
2 | import { Workflow, WorkflowStep, WorkflowMetadata, AppNode } from '../types/workflow-layout.types';
3 |
4 | export function jsonToFlow(workflow: string): {
5 | nodes: AppNode[];
6 | edges: Edge[];
7 | metadata: WorkflowMetadata;
8 | } {
9 | const parsedWorkflow = JSON.parse(workflow) as Workflow;
10 | const nodes: AppNode[] = parsedWorkflow.steps.map((step: WorkflowStep, idx: number) => ({
11 | id: String(idx),
12 | data: {
13 | label: `${step.description}`,
14 | stepData: step,
15 | workflowName: parsedWorkflow.name
16 | },
17 | position: { x: 0, y: idx * 100 }
18 | }));
19 |
20 | const edges: Edge[] = parsedWorkflow.steps.slice(1).map((_, idx) => ({
21 | id: `e${idx}-${idx + 1}`,
22 | source: String(idx),
23 | target: String(idx + 1),
24 | animated: true,
25 | }));
26 |
27 | const metadata: WorkflowMetadata = {
28 | name: parsedWorkflow.name,
29 | description: parsedWorkflow.description,
30 | version: parsedWorkflow.version,
31 | input_schema: parsedWorkflow.input_schema,
32 | workflow_analysis: parsedWorkflow.workflow_analysis
33 | };
34 |
35 | return { nodes, edges, metadata };
36 | }
--------------------------------------------------------------------------------
/ui/src/vite-env.d.ts:
--------------------------------------------------------------------------------
1 | ///
11 | [Image Data]
12 |
13 | Follow these rules when generating the output JSON:
14 | 0. The first thing you will output is the "workflow_analysis". First analyze the original workflow recording, what it is about and create a general analysis of the workflow. Also think about which variables are going to be needed for the workflow.
15 | 1. Top-level keys: "workflow_analysis", "name", "description", "input_schema", "steps" and "version".
16 | - "input_schema" - MUST follow JSON-Schema draft-7 subset semantics:
17 | [
18 | {{"name": "foo", "type": "string", "required": true}},
19 | {{"name": "bar", "type": "number"}},
20 | ...
21 | ]
22 | - Always aim to include at least one input in "input_schema" unless the workflow is explicitly static (e.g., always navigates to a fixed URL with no user-driven variability). Base inputs on the user goal, event parameters (e.g., search queries, form inputs), or potential reusable values. For example, if the workflow searches for a term, include an input like {{"name": "search_term", "type": "string", "required": true}}.
23 | - Only use an empty "input_schema" if no dynamic inputs are relevant after careful analysis. Justify this choice in the "workflow_analysis".
24 | 2. "steps" is an array of dictionaries executed sequentially.
25 | - Each dictionary MUST include a `"type"` field.
26 | - **Agentic Steps ("type": "agent")**:
27 | - Use `"type": "agent"` for tasks where the user must interact with or select from frequently changing content, even if the website’s structure is consistent. Examples include choosing an item from a dynamic list (e.g., a restaurant from search results) or selecting a specific value from a variable set (e.g., a date from a calendar that changes with the month).
28 | - **MUST** include a `"task"` string describing the user’s goal for the step from their perspective (e.g., "Select the restaurant named {{restaurant_name}} from the search results").
29 | - Include a `"description"` explaining why agentic reasoning is needed (e.g., "The list of restaurants varies with each search, requiring the agent to find the specified one").
30 | - Optionally include `"max_steps"` (defaults to 5) to limit agent exploration.
31 | - **Replace deterministic steps with agentic steps** when the task involves:
32 | - Selecting from a list or set of options that changes frequently (e.g., restaurants, products, or search results).
33 | - Interacting with time-sensitive or context-dependent elements (e.g., picking a date from a calendar or a time slot from a schedule).
34 | - Evaluating content to match user input (e.g., finding a specific item based on its name or attributes).
35 | - Break complex tasks into multiple specific agentic steps rather than one broad task.
36 | - **Use the user’s goal (if provided) or inferred intent from the recording** to identify where agentic steps are needed for dynamic content, even if the recording uses deterministic steps.
37 | - **extract_page_content** - Use this type when you want to extract data from the page. If the task is simply extracting data from the page, use this instead of agentic steps (never create agentic step for simple data extraction).
38 | - **Deterministic events** → keep the original recorder event structure. The
39 | value of `"type"` MUST match **exactly** one of the available action
40 | names listed below; all additional keys are interpreted as parameters for
41 | that action.
42 | - For each step you create also add a very short description that describes what the step tries to achieve.
43 | - sometimes navigating to a certain url is a side effects of another action (click, submit, key press, etc.). In that case choose either (if you think navigating to the url is the best option) or don't add the step at all.
44 | 3. When referencing workflow inputs inside event parameters or agent tasks use
45 | the placeholder syntax `{{input_name}}` (e.g. "cssSelector": "#msg-{{row}}")
46 | – do *not* use any prefix like "input.". Decide the inputs dynamically based on the user's
47 | goal.
48 | 4. Quote all placeholder values to ensure the JSON parser treats them as
49 | strings.
50 | 5. In the events you will find all the selectors relative to a particular action, replicate all of them in the workflow.
51 | 6. For many workflows steps you can go directly to certain url and skip the initial clicks (for example searching for something).
52 |
53 |
54 | High-level task description provided by the user (may be empty):
55 | {goal}
56 |
57 | Available actions:
58 | {actions}
59 |
60 | Input session events will follow one-by-one in subsequent messages.
61 | """
62 |
--------------------------------------------------------------------------------
/workflows/workflow_use/builder/service.py:
--------------------------------------------------------------------------------
1 | import base64
2 | import json
3 | import logging
4 | import re
5 | from pathlib import Path
6 | from typing import Any, Dict, List, Optional, cast
7 |
8 | from langchain_core.exceptions import OutputParserException
9 | from langchain_core.language_models import BaseChatModel
10 | from langchain_core.messages import HumanMessage
11 | from langchain_core.prompts import PromptTemplate
12 | from pydantic import ValidationError
13 |
14 | from workflow_use.builder.prompts import WORKFLOW_BUILDER_PROMPT_TEMPLATE
15 | from workflow_use.controller.service import WorkflowController
16 | from workflow_use.schema.views import WorkflowDefinitionSchema
17 |
18 | logger = logging.getLogger(__name__)
19 |
20 |
21 | class BuilderService:
22 | """
23 | Service responsible for building executable workflow JSON definitions
24 | from recorded browser session events using an LLM.
25 | """
26 |
27 | def __init__(self, llm: BaseChatModel):
28 | """
29 | Initializes the BuilderService.
30 |
31 | Args:
32 | llm: A LangChain BaseChatModel instance configured for use.
33 | It should ideally support vision capabilities if screenshots are used.
34 | """
35 | if llm is None:
36 | raise ValueError('A BaseChatModel instance must be provided.')
37 |
38 | # Configure the LLM to return structured output based on the Pydantic model
39 | try:
40 | # Specify method="function_calling" for better compatibility
41 | self.llm_structured = llm.with_structured_output(WorkflowDefinitionSchema, method='function_calling')
42 | except NotImplementedError:
43 | logger.warning('LLM does not support structured output natively. Falling back.')
44 | # Basic LLM call if structured output is not supported
45 | # Output parsing will be handled manually later
46 | self.llm_structured = llm # Store the original llm
47 |
48 | self.prompt_template = PromptTemplate.from_template(WORKFLOW_BUILDER_PROMPT_TEMPLATE)
49 | self.actions_markdown = self._get_available_actions_markdown()
50 | logger.info('BuilderService initialized.')
51 |
52 | def _get_available_actions_markdown(self) -> str:
53 | """Return a markdown list of available actions and their schema."""
54 | controller = WorkflowController()
55 | lines: List[str] = []
56 | for action in controller.registry.registry.actions.values():
57 | # Only include deterministic actions relevant for building from recordings
58 | # Exclude agent-specific or meta-actions if necessary
59 | # Based on schema/views.py, the recorder types seem to map directly
60 | # to controller action *names*, but the prompt uses the event `type` field.
61 | # Let's assume the prompt template correctly lists the *event types* expected.
62 | # This function provides the detailed schema for the LLM.
63 | schema_info = action.param_model.model_json_schema()
64 | # Simplify schema representation for the prompt if too verbose
65 | param_details = []
66 | props = schema_info.get('properties', {})
67 | required = schema_info.get('required', [])
68 | for name, details in props.items():
69 | req_star = '*' if name in required else ''
70 | param_details.append(f'`{name}`{req_star} ({details.get("type", "any")})')
71 |
72 | lines.append(f'- **`{action.name}`**: {action.description}') # Using action name from controller
73 | if param_details:
74 | lines.append(f' - Parameters: {", ".join(param_details)}')
75 |
76 | # Add descriptions for agent/extract_content types manually if not in controller
77 | if 'agent' not in [a.name for a in controller.registry.registry.actions.values()]:
78 | lines.append('- **`agent`**: Executes a task using an autonomous agent.')
79 | lines.append(' - Parameters: `task`* (string), `description` (string), `max_steps` (integer)')
80 | # if "extract_content" not in [
81 | # a.name for a in controller.registry.registry.actions.values()
82 | # ]:
83 | # lines.append(
84 | # "- **`extract_content`**: Uses an LLM to extract specific information from the current page."
85 | # )
86 | # lines.append(
87 | # " - Parameters: `goal`* (string), `description` (string), `should_strip_link_urls` (boolean)"
88 | # )
89 |
90 | logger.debug(f'Generated actions markdown:\n{lines}')
91 | return '\n'.join(lines)
92 |
93 | @staticmethod
94 | def _find_first_user_interaction_url(events: List[Dict[str, Any]]) -> Optional[str]:
95 | """Finds the URL of the first recorded user interaction."""
96 | return next(
97 | (
98 | evt.get('frameUrl')
99 | for evt in events
100 | if evt.get('type')
101 | in [
102 | 'input',
103 | 'click',
104 | 'scroll',
105 | 'select_change',
106 | 'key_press',
107 | ] # Added more types
108 | ),
109 | None,
110 | )
111 |
112 | def _parse_llm_output_to_workflow(self, llm_content: str) -> WorkflowDefinitionSchema:
113 | """Attempts to parse the LLM string output into a WorkflowDefinitionSchema."""
114 | logger.debug(f'Raw LLM Output:\n{llm_content}')
115 | content_to_parse = llm_content
116 |
117 | # Heuristic cleanup: Extract JSON from markdown code blocks
118 | if '```json' in content_to_parse:
119 | match = re.search(r'```json\s*([\s\S]*?)\s*```', content_to_parse, re.DOTALL)
120 | if match:
121 | content_to_parse = match.group(1).strip()
122 | logger.debug('Extracted JSON from ```json block.')
123 | elif content_to_parse.strip().startswith('{') and content_to_parse.strip().endswith('}'):
124 | # Assume it's already JSON if it looks like it
125 | content_to_parse = content_to_parse.strip()
126 | logger.debug('Assuming raw output is JSON.')
127 | else:
128 | logger.warning('Could not reliably extract JSON from LLM output, attempting parse anyway.')
129 |
130 | try:
131 | # Try parsing directly first (might work with structured output)
132 | workflow_data = WorkflowDefinitionSchema.model_validate_json(content_to_parse)
133 | logger.info('Successfully parsed LLM output into WorkflowDefinitionSchema.')
134 | return workflow_data
135 | except (json.JSONDecodeError, ValidationError) as e:
136 | logger.error(f'Failed to parse LLM output into WorkflowDefinitionSchema: {e}')
137 | logger.debug(f'Content attempted parsing:\n{content_to_parse}')
138 | raise ValueError(f'LLM output could not be parsed into a valid Workflow schema. Error: {e}') from e
139 |
140 | async def build_workflow(
141 | self,
142 | input_workflow: WorkflowDefinitionSchema,
143 | user_goal: str,
144 | use_screenshots: bool = False,
145 | max_images: int = 20,
146 | ) -> WorkflowDefinitionSchema:
147 | """
148 | Generates an enhanced Workflow definition from an input workflow object using an LLM.
149 |
150 | Args:
151 | input_workflow: The initial WorkflowDefinitionSchema object containing steps to process.
152 | user_goal: Optional high-level description of the workflow's purpose.
153 | If None, the user might be prompted interactively.
154 | use_screenshots: Whether to include screenshots as visual context for the LLM (if available in steps).
155 | max_images: Maximum number of screenshots to include (to manage cost/tokens).
156 |
157 | Returns:
158 | A new WorkflowDefinitionSchema object generated by the LLM.
159 |
160 | Raises:
161 | ValueError: If the input workflow is invalid or the LLM output cannot be parsed.
162 | Exception: For other LLM or processing errors.
163 | """
164 | # Validate input slightly
165 | if not input_workflow or not isinstance(input_workflow.steps, list):
166 | raise ValueError('Invalid input_workflow object provided.')
167 |
168 | # Handle user goal
169 | goal = user_goal
170 | if goal is None:
171 | try:
172 | goal = input('Please describe the high-level task for the workflow (optional, press Enter to skip): ').strip()
173 | except EOFError: # Handle non-interactive environments
174 | goal = ''
175 | goal = goal or 'Automate the recorded browser actions.' # Default goal if empty
176 |
177 | # Format the main instruction prompt
178 | prompt_str = self.prompt_template.format(
179 | actions=self.actions_markdown,
180 | goal=goal,
181 | )
182 |
183 | # Prepare the vision messages list
184 | vision_messages: List[Dict[str, Any]] = [{'type': 'text', 'text': prompt_str}]
185 |
186 | # Integrate message preparation logic here
187 | images_used = 0
188 | for step in input_workflow.steps:
189 | step_messages: List[Dict[str, Any]] = [] # Messages for this specific step
190 |
191 | # 1. Text representation (JSON dump)
192 | step_dict = step.model_dump(mode='json', exclude_none=True)
193 | screenshot_data = step_dict.pop('screenshot', None) # Pop potential screenshot
194 | step_messages.append({'type': 'text', 'text': json.dumps(step_dict, indent=2)})
195 |
196 | # 2. Optional screenshot
197 | attach_image = use_screenshots and images_used < max_images
198 | step_type = getattr(step, 'type', step_dict.get('type'))
199 |
200 | if attach_image and step_type != 'input': # Don't attach for inputs
201 | # Re-retrieve screenshot data if it wasn't popped (e.g., nested under 'data')
202 | # This assumes screenshot might still be in the original step model or dict
203 | # A bit redundant, ideally screenshot handling is consistent
204 | screenshot = screenshot_data or getattr(step, 'screenshot', None) or step_dict.get('data', {}).get('screenshot')
205 |
206 | if screenshot:
207 | if isinstance(screenshot, str) and screenshot.startswith('data:'):
208 | screenshot = screenshot.split(',', 1)[-1]
209 |
210 | # Validate base64 payload
211 | try:
212 | base64.b64decode(cast(str, screenshot), validate=True)
213 | meta = f""
214 | step_messages.append({'type': 'text', 'text': meta})
215 | step_messages.append(
216 | {
217 | 'type': 'image_url',
218 | 'image_url': {'url': f'data:image/png;base64,{screenshot}'},
219 | }
220 | )
221 | images_used += 1 # Increment image count *only* if successfully added
222 | except (TypeError, ValueError, Exception) as e:
223 | logger.warning(
224 | f"Invalid or missing screenshot for step type '{step_type}' "
225 | f'@ {step_dict.get("timestamp", "")}. Error: {e}'
226 | )
227 | # Don't add image messages if invalid
228 |
229 | # Add the messages for this step to the main list
230 | vision_messages.extend(step_messages)
231 |
232 | logger.info(f'Prepared {len(vision_messages)} total message parts, including {images_used} images.')
233 |
234 | # Invoke the LLM (structured output preferred)
235 | try:
236 | # Invoke the LLM (structured output preferred)
237 | # Need to handle cases where structured output isn't truly supported
238 | if hasattr(self.llm_structured, 'output_schema'): # Check if it seems like structured output model
239 | llm_response = await self.llm_structured.ainvoke([HumanMessage(content=cast(Any, vision_messages))])
240 | # If structured output worked, llm_response is the Pydantic object
241 | if isinstance(llm_response, WorkflowDefinitionSchema):
242 | workflow_data = llm_response
243 | else:
244 | # It might have returned a message or dict, try parsing its content
245 | content = getattr(llm_response, 'content', str(llm_response))
246 | workflow_data = self._parse_llm_output_to_workflow(str(content))
247 | else:
248 | # Fallback to basic LLM call and manual parsing
249 | llm_response = await self.llm_structured.ainvoke([HumanMessage(content=cast(Any, vision_messages))])
250 | llm_content = str(getattr(llm_response, 'content', llm_response)) # Get string content
251 | workflow_data = self._parse_llm_output_to_workflow(llm_content)
252 |
253 | except OutputParserException as ope:
254 | logger.error(f'LLM output parsing failed (OutputParserException): {ope}')
255 | # Try to parse the raw output as a fallback
256 | raw_output = getattr(ope, 'llm_output', str(ope))
257 | logger.info('Attempting to parse raw output as fallback...')
258 | try:
259 | workflow_data = self._parse_llm_output_to_workflow(raw_output)
260 | except ValueError as ve_fallback:
261 | raise ValueError(
262 | f'LLM structured output failed, and fallback parsing also failed. Error: {ve_fallback}'
263 | ) from ve_fallback
264 | except Exception as e:
265 | logger.exception(f'An error occurred during LLM invocation or processing: {e}')
266 | raise # Re-raise other unexpected errors
267 |
268 | # Return the workflow data object directly
269 | return workflow_data
270 |
271 | # path handlers
272 | async def build_workflow_from_path(self, path: Path, user_goal: str) -> WorkflowDefinitionSchema:
273 | """Build a workflow from a JSON file path."""
274 | with open(path, 'r') as f:
275 | workflow_data = json.load(f)
276 |
277 | workflow_data_schema = WorkflowDefinitionSchema.model_validate(workflow_data)
278 | return await self.build_workflow(workflow_data_schema, user_goal)
279 |
280 | async def save_workflow_to_path(self, workflow: WorkflowDefinitionSchema, path: Path):
281 | """Save a workflow to a JSON file path."""
282 | with open(path, 'w') as f:
283 | json.dump(workflow.model_dump(mode='json'), f, indent=2)
284 |
--------------------------------------------------------------------------------
/workflows/workflow_use/builder/tests/build_workflow.py:
--------------------------------------------------------------------------------
1 | import asyncio
2 | from pathlib import Path
3 |
4 | # Ensure langchain-openai is installed and OPENAI_API_KEY is set
5 | from langchain_openai import ChatOpenAI
6 |
7 | from workflow_use.builder.service import BuilderService
8 |
9 | # Instantiate the LLM and the service directly
10 | llm_instance = ChatOpenAI(model='gpt-4o') # Or your preferred model
11 | builder_service = BuilderService(llm=llm_instance)
12 |
13 |
14 | async def test_build_workflow_from_path():
15 | """
16 | Tests that the workflow is built correctly from a JSON file path.
17 | """
18 | path = Path(__file__).parent / 'tmp' / 'recording.json'
19 | workflow_definition = await builder_service.build_workflow_from_path(
20 | path,
21 | 'go to apple.com and extract the price of the iphone XY (where XY is a variable)',
22 | )
23 |
24 | print(workflow_definition)
25 |
26 | output_path = path.with_suffix('.workflow.json')
27 | await builder_service.save_workflow_to_path(workflow_definition, output_path)
28 |
29 |
30 | if __name__ == '__main__':
31 | asyncio.run(test_build_workflow_from_path())
32 |
--------------------------------------------------------------------------------
/workflows/workflow_use/controller/service.py:
--------------------------------------------------------------------------------
1 | import asyncio
2 | import logging
3 |
4 | from browser_use import Browser
5 | from browser_use.agent.views import ActionResult
6 | from browser_use.controller.service import Controller
7 | from langchain_core.language_models.chat_models import BaseChatModel
8 | from langchain_core.prompts import PromptTemplate
9 |
10 | from workflow_use.controller.utils import get_best_element_handle, truncate_selector
11 | from workflow_use.controller.views import (
12 | ClickElementDeterministicAction,
13 | InputTextDeterministicAction,
14 | KeyPressDeterministicAction,
15 | NavigationAction,
16 | PageExtractionAction,
17 | ScrollDeterministicAction,
18 | SelectDropdownOptionDeterministicAction,
19 | )
20 |
21 | logger = logging.getLogger(__name__)
22 |
23 | DEFAULT_ACTION_TIMEOUT_MS = 1000
24 |
25 | # List of default actions from browser_use.controller.service.Controller to disable
26 | # todo: come up with a better way to filter out the actions (filter IN the actions would be much nicer in this case)
27 | DISABLED_DEFAULT_ACTIONS = [
28 | 'done',
29 | 'search_google',
30 | 'go_to_url', # I am using this action from the main controller to avoid duplication
31 | 'go_back',
32 | 'wait',
33 | 'click_element_by_index',
34 | 'input_text',
35 | 'save_pdf',
36 | 'switch_tab',
37 | 'open_tab',
38 | 'close_tab',
39 | 'extract_content',
40 | 'scroll_down',
41 | 'scroll_up',
42 | 'send_keys',
43 | 'scroll_to_text',
44 | 'get_dropdown_options',
45 | 'select_dropdown_option',
46 | 'drag_drop',
47 | 'get_sheet_contents',
48 | 'select_cell_or_range',
49 | 'get_range_contents',
50 | 'clear_selected_range',
51 | 'input_selected_cell_text',
52 | 'update_range_contents',
53 | ]
54 |
55 |
56 | class WorkflowController(Controller):
57 | def __init__(self, *args, **kwargs):
58 | # Pass the list of actions to exclude to the base class constructor
59 | super().__init__(*args, exclude_actions=DISABLED_DEFAULT_ACTIONS, **kwargs)
60 | self.__register_actions()
61 |
62 | def __register_actions(self):
63 | # Navigate to URL ------------------------------------------------------------
64 | @self.registry.action('Manually navigate to URL', param_model=NavigationAction)
65 | async def navigation(params: NavigationAction, browser_session: Browser) -> ActionResult:
66 | """Navigate to the given URL."""
67 | page = await browser_session.get_current_page()
68 | await page.goto(params.url)
69 | await page.wait_for_load_state()
70 |
71 | msg = f'🔗 Navigated to URL: {params.url}'
72 | logger.info(msg)
73 | return ActionResult(extracted_content=msg, include_in_memory=True)
74 |
75 | # Click element by CSS selector --------------------------------------------------
76 |
77 | @self.registry.action(
78 | 'Click element by all available selectors',
79 | param_model=ClickElementDeterministicAction,
80 | )
81 | async def click(params: ClickElementDeterministicAction, browser_session: Browser) -> ActionResult:
82 | """Click the first element matching *params.cssSelector* with fallback mechanisms."""
83 | page = await browser_session.get_current_page()
84 | original_selector = params.cssSelector
85 |
86 | try:
87 | locator, selector_used = await get_best_element_handle(
88 | page,
89 | params.cssSelector,
90 | params,
91 | timeout_ms=DEFAULT_ACTION_TIMEOUT_MS,
92 | )
93 | await locator.click(force=True)
94 |
95 | msg = f'🖱️ Clicked element with CSS selector: {truncate_selector(selector_used)} (original: {truncate_selector(original_selector)})'
96 | logger.info(msg)
97 | return ActionResult(extracted_content=msg, include_in_memory=True)
98 | except Exception as e:
99 | error_msg = f'Failed to click element. Original selector: {truncate_selector(original_selector)}. Error: {str(e)}'
100 | logger.error(error_msg)
101 | raise Exception(error_msg)
102 |
103 | # Input text into element --------------------------------------------------------
104 | @self.registry.action(
105 | 'Input text into an element by all available selectors',
106 | param_model=InputTextDeterministicAction,
107 | )
108 | async def input(
109 | params: InputTextDeterministicAction,
110 | browser_session: Browser,
111 | has_sensitive_data: bool = False,
112 | ) -> ActionResult:
113 | """Fill text into the element located with *params.cssSelector*."""
114 | page = await browser_session.get_current_page()
115 | original_selector = params.cssSelector
116 |
117 | try:
118 | locator, selector_used = await get_best_element_handle(
119 | page,
120 | params.cssSelector,
121 | params,
122 | timeout_ms=DEFAULT_ACTION_TIMEOUT_MS,
123 | )
124 |
125 | # Check if it's a SELECT element
126 | is_select = await locator.evaluate('(el) => el.tagName === "SELECT"')
127 | if is_select:
128 | return ActionResult(
129 | extracted_content='Ignored input into select element',
130 | include_in_memory=True,
131 | )
132 |
133 | # Add a small delay and click to ensure the element is focused
134 | await locator.fill(params.value)
135 | await asyncio.sleep(0.5)
136 | await locator.click(force=True)
137 | await asyncio.sleep(0.5)
138 |
139 | msg = f'⌨️ Input "{params.value}" into element with CSS selector: {truncate_selector(selector_used)} (original: {truncate_selector(original_selector)})'
140 | logger.info(msg)
141 | return ActionResult(extracted_content=msg, include_in_memory=True)
142 | except Exception as e:
143 | error_msg = f'Failed to input text. Original selector: {truncate_selector(original_selector)}. Error: {str(e)}'
144 | logger.error(error_msg)
145 | raise Exception(error_msg)
146 |
147 | # Select dropdown option ---------------------------------------------------------
148 | @self.registry.action(
149 | 'Select dropdown option by all available selectors and visible text',
150 | param_model=SelectDropdownOptionDeterministicAction,
151 | )
152 | async def select_change(params: SelectDropdownOptionDeterministicAction, browser_session: Browser) -> ActionResult:
153 | """Select dropdown option whose visible text equals *params.value*."""
154 | page = await browser_session.get_current_page()
155 | original_selector = params.cssSelector
156 |
157 | try:
158 | locator, selector_used = await get_best_element_handle(
159 | page,
160 | params.cssSelector,
161 | params,
162 | timeout_ms=DEFAULT_ACTION_TIMEOUT_MS,
163 | )
164 |
165 | await locator.select_option(label=params.selectedText)
166 |
167 | msg = f'Selected option "{params.selectedText}" in dropdown {truncate_selector(selector_used)} (original: {truncate_selector(original_selector)})'
168 | logger.info(msg)
169 | return ActionResult(extracted_content=msg, include_in_memory=True)
170 | except Exception as e:
171 | error_msg = f'Failed to select option. Original selector: {truncate_selector(original_selector)}. Error: {str(e)}'
172 | logger.error(error_msg)
173 | raise Exception(error_msg)
174 |
175 | # Key press action ------------------------------------------------------------
176 | @self.registry.action(
177 | 'Press key on element by all available selectors',
178 | param_model=KeyPressDeterministicAction,
179 | )
180 | async def key_press(params: KeyPressDeterministicAction, browser_session: Browser) -> ActionResult:
181 | """Press *params.key* on the element identified by *params.cssSelector*."""
182 | page = await browser_session.get_current_page()
183 | original_selector = params.cssSelector
184 |
185 | try:
186 | locator, selector_used = await get_best_element_handle(page, params.cssSelector, params, timeout_ms=5000)
187 |
188 | await locator.press(params.key)
189 |
190 | msg = f"🔑 Pressed key '{params.key}' on element with CSS selector: {truncate_selector(selector_used)} (original: {truncate_selector(original_selector)})"
191 | logger.info(msg)
192 | return ActionResult(extracted_content=msg, include_in_memory=True)
193 | except Exception as e:
194 | error_msg = f'Failed to press key. Original selector: {truncate_selector(original_selector)}. Error: {str(e)}'
195 | logger.error(error_msg)
196 | raise Exception(error_msg)
197 |
198 | # Scroll action --------------------------------------------------------------
199 | @self.registry.action('Scroll page', param_model=ScrollDeterministicAction)
200 | async def scroll(params: ScrollDeterministicAction, browser_session: Browser) -> ActionResult:
201 | """Scroll the page by the given x/y pixel offsets."""
202 | page = await browser_session.get_current_page()
203 | await page.evaluate(f'window.scrollBy({params.scrollX}, {params.scrollY});')
204 | msg = f'📜 Scrolled page by (x={params.scrollX}, y={params.scrollY})'
205 | logger.info(msg)
206 | return ActionResult(extracted_content=msg, include_in_memory=True)
207 |
208 | # Extract content ------------------------------------------------------------
209 |
210 | @self.registry.action(
211 | 'Extract page content to retrieve specific information from the page, e.g. all company names, a specific description, all information about, links with companies in structured format or simply links',
212 | param_model=PageExtractionAction,
213 | )
214 | async def extract_page_content(
215 | params: PageExtractionAction, browser_session: Browser, page_extraction_llm: BaseChatModel
216 | ):
217 | page = await browser_session.get_current_page()
218 | import markdownify
219 |
220 | strip = ['a', 'img']
221 |
222 | content = markdownify.markdownify(await page.content(), strip=strip)
223 |
224 | # manually append iframe text into the content so it's readable by the LLM (includes cross-origin iframes)
225 | for iframe in page.frames:
226 | if iframe.url != page.url and not iframe.url.startswith('data:'):
227 | content += f'\n\nIFRAME {iframe.url}:\n'
228 | content += markdownify.markdownify(await iframe.content())
229 |
230 | prompt = 'Your task is to extract the content of the page. You will be given a page and a goal and you should extract all relevant information around this goal from the page. If the goal is vague, summarize the page. Respond in json format. Extraction goal: {goal}, Page: {page}'
231 | template = PromptTemplate(input_variables=['goal', 'page'], template=prompt)
232 | try:
233 | output = await page_extraction_llm.ainvoke(template.format(goal=params.goal, page=content))
234 | msg = f'📄 Extracted from page\n: {output.content}\n'
235 | logger.info(msg)
236 | return ActionResult(extracted_content=msg, include_in_memory=True)
237 | except Exception as e:
238 | logger.debug(f'Error extracting content: {e}')
239 | msg = f'📄 Extracted from page\n: {content}\n'
240 | logger.info(msg)
241 | return ActionResult(extracted_content=msg)
242 |
--------------------------------------------------------------------------------
/workflows/workflow_use/controller/utils.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import re
3 |
4 | logger = logging.getLogger(__name__)
5 |
6 |
7 | def truncate_selector(selector: str, max_length: int = 35) -> str:
8 | """Truncate a CSS selector to a maximum length, adding ellipsis if truncated."""
9 | return selector if len(selector) <= max_length else f'{selector[:max_length]}...'
10 |
11 |
12 | async def get_best_element_handle(page, selector, params=None, timeout_ms=500):
13 | """Find element using stability-ranked selector strategies."""
14 | original_selector = selector
15 |
16 | # Generate stability-ranked fallback selectors
17 | fallbacks = generate_stable_selectors(selector, params)
18 |
19 | # Try all selectors with exponential backoff for timeouts
20 | selectors_to_try = [original_selector] + fallbacks
21 |
22 | for try_selector in selectors_to_try:
23 | try:
24 | logger.info(f'Trying selector: {truncate_selector(try_selector)}')
25 | locator = page.locator(try_selector)
26 | await locator.wait_for(state='visible', timeout=timeout_ms)
27 | logger.info(f'Found element with selector: {truncate_selector(try_selector)}')
28 | return locator, try_selector
29 | except Exception as e:
30 | logger.error(f'Selector failed: {truncate_selector(try_selector)} with error: {e}')
31 |
32 | # Try XPath as last resort
33 | if params and getattr(params, 'xpath', None):
34 | xpath = params.xpath
35 | try:
36 | # Generate stable XPath alternatives
37 | xpath_alternatives = [xpath] + generate_stable_xpaths(xpath, params)
38 |
39 | for try_xpath in xpath_alternatives:
40 | xpath_selector = f'xpath={try_xpath}'
41 | logger.info(f'Trying XPath: {truncate_selector(xpath_selector)}')
42 | locator = page.locator(xpath_selector)
43 | await locator.wait_for(state='visible', timeout=timeout_ms)
44 | return locator, xpath_selector
45 | except Exception as e:
46 | logger.error(f'All XPaths failed with error: {e}')
47 |
48 | raise Exception(f'Failed to find element. Original: {original_selector}')
49 |
50 |
51 | def generate_stable_selectors(selector, params=None):
52 | """Generate selectors from most to least stable based on selector patterns."""
53 | fallbacks = []
54 |
55 | # 1. Extract attribute-based selectors (most stable)
56 | attributes_to_check = [
57 | 'placeholder',
58 | 'aria-label',
59 | 'name',
60 | 'title',
61 | 'role',
62 | 'data-testid',
63 | ]
64 | for attr in attributes_to_check:
65 | attr_pattern = rf'\[{attr}\*?=[\'"]([^\'"]*)[\'"]'
66 | attr_match = re.search(attr_pattern, selector)
67 | if attr_match:
68 | attr_value = attr_match.group(1)
69 | element_tag = extract_element_tag(selector, params)
70 | if element_tag:
71 | fallbacks.append(f'{element_tag}[{attr}*="{attr_value}"]')
72 |
73 | # 2. Combine tag + class + one attribute (good stability)
74 | element_tag = extract_element_tag(selector, params)
75 | classes = extract_stable_classes(selector)
76 | for attr in attributes_to_check:
77 | attr_pattern = rf'\[{attr}\*?=[\'"]([^\'"]*)[\'"]'
78 | attr_match = re.search(attr_pattern, selector)
79 | if attr_match and classes and element_tag:
80 | attr_value = attr_match.group(1)
81 | class_selector = '.'.join(classes)
82 | fallbacks.append(f'{element_tag}.{class_selector}[{attr}*="{attr_value}"]')
83 |
84 | # 3. Tag + class combination (less stable but often works)
85 | if element_tag and classes:
86 | class_selector = '.'.join(classes)
87 | fallbacks.append(f'{element_tag}.{class_selector}')
88 |
89 | # 4. Remove dynamic parts (IDs, state classes)
90 | if '[id=' in selector:
91 | fallbacks.append(re.sub(r'\[id=[\'"].*?[\'"]\]', '', selector))
92 |
93 | for state in ['.focus-visible', '.hover', '.active', '.focus', ':focus']:
94 | if state in selector:
95 | fallbacks.append(selector.replace(state, ''))
96 |
97 | # 5. Use text-based selector if we have element tag and text
98 | if params and getattr(params, 'elementTag', None) and getattr(params, 'elementText', None) and params.elementText.strip():
99 | fallbacks.append(f"{params.elementTag}:has-text('{params.elementText}')")
100 |
101 | return list(dict.fromkeys(fallbacks)) # Remove duplicates while preserving order
102 |
103 |
104 | def extract_element_tag(selector, params=None):
105 | """Extract element tag from selector or params."""
106 | # Try to get from selector first
107 | tag_match = re.match(r'^([a-zA-Z][a-zA-Z0-9]*)', selector)
108 | if tag_match:
109 | return tag_match.group(1).lower()
110 |
111 | # Fall back to params
112 | if params and getattr(params, 'elementTag', None):
113 | return params.elementTag.lower()
114 |
115 | return ''
116 |
117 |
118 | def extract_stable_classes(selector):
119 | """Extract classes that appear to be stable (not state-related)."""
120 | class_pattern = r'\.([a-zA-Z0-9_-]+)'
121 | classes = re.findall(class_pattern, selector)
122 |
123 | # Filter out likely state classes
124 | stable_classes = [
125 | cls
126 | for cls in classes
127 | if not any(state in cls.lower() for state in ['focus', 'hover', 'active', 'selected', 'checked', 'disabled'])
128 | ]
129 |
130 | return stable_classes
131 |
132 |
133 | def generate_stable_xpaths(xpath, params=None):
134 | """Generate stable XPath alternatives."""
135 | alternatives = []
136 |
137 | # Handle "id()" XPath pattern which is brittle
138 | if 'id(' in xpath:
139 | element_tag = getattr(params, 'elementTag', '').lower()
140 | if element_tag:
141 | # Create XPaths based on attributes from params
142 | if params and getattr(params, 'cssSelector', None):
143 | for attr in ['placeholder', 'aria-label', 'title', 'name']:
144 | attr_pattern = rf'\[{attr}\*?=[\'"]([^\'"]*)[\'"]'
145 | attr_match = re.search(attr_pattern, params.cssSelector)
146 | if attr_match:
147 | attr_value = attr_match.group(1)
148 | alternatives.append(f"//{element_tag}[contains(@{attr}, '{attr_value}')]")
149 |
150 | return alternatives
151 |
--------------------------------------------------------------------------------
/workflows/workflow_use/controller/views.py:
--------------------------------------------------------------------------------
1 | from typing import Literal, Optional
2 |
3 | from pydantic import BaseModel
4 |
5 |
6 | # Shared config allowing extra fields so recorder payloads pass through
7 | class _BaseExtra(BaseModel):
8 | """Base model ignoring unknown fields."""
9 |
10 | class Config:
11 | extra = 'ignore'
12 |
13 |
14 | # Mixin for shared step metadata (timestamp and tab context)
15 | class StepMeta(_BaseExtra):
16 | timestamp: int
17 | tabId: int
18 |
19 |
20 | # Common optional fields present in recorder events
21 | class RecorderBase(StepMeta):
22 | xpath: Optional[str] = None
23 | elementTag: Optional[str] = None
24 | elementText: Optional[str] = None
25 | frameUrl: Optional[str] = None
26 | screenshot: Optional[str] = None
27 |
28 |
29 | class ClickElementDeterministicAction(RecorderBase):
30 | """Parameters for clicking an element identified by CSS selector."""
31 |
32 | type: Literal['click']
33 | cssSelector: str
34 |
35 |
36 | class InputTextDeterministicAction(RecorderBase):
37 | """Parameters for entering text into an input field identified by CSS selector."""
38 |
39 | type: Literal['input']
40 | cssSelector: str
41 | value: str
42 |
43 |
44 | class SelectDropdownOptionDeterministicAction(RecorderBase):
45 | """Parameters for selecting a dropdown option identified by *selector* and *text*."""
46 |
47 | type: Literal['select_change']
48 | cssSelector: str
49 | selectedValue: str
50 | selectedText: str
51 |
52 |
53 | class KeyPressDeterministicAction(RecorderBase):
54 | """Parameters for pressing a key on an element identified by CSS selector."""
55 |
56 | type: Literal['key_press']
57 | cssSelector: str
58 | key: str
59 |
60 |
61 | class NavigationAction(_BaseExtra):
62 | """Parameters for navigating to a URL."""
63 |
64 | type: Literal['navigation']
65 | url: str
66 |
67 |
68 | class ScrollDeterministicAction(_BaseExtra):
69 | """Parameters for scrolling the page by x/y offsets (pixels)."""
70 |
71 | type: Literal['scroll']
72 | scrollX: int = 0
73 | scrollY: int = 0
74 | targetId: Optional[int] = None
75 |
76 |
77 | class PageExtractionAction(_BaseExtra):
78 | """Parameters for extracting content from the page."""
79 |
80 | type: Literal['extract_page_content']
81 | goal: str
82 |
--------------------------------------------------------------------------------
/workflows/workflow_use/mcp/service.py:
--------------------------------------------------------------------------------
1 | import json as _json
2 | from inspect import Parameter, Signature
3 | from pathlib import Path
4 | from typing import Any
5 |
6 | from fastmcp import FastMCP
7 | from langchain_core.language_models.chat_models import BaseChatModel
8 |
9 | from workflow_use.schema.views import WorkflowDefinitionSchema
10 | from workflow_use.workflow.service import Workflow
11 |
12 |
13 | def get_mcp_server(
14 | llm_instance: BaseChatModel,
15 | page_extraction_llm: BaseChatModel | None = None,
16 | workflow_dir: str = './tmp',
17 | name: str = 'WorkflowService',
18 | description: str = 'Exposes workflows as MCP tools.',
19 | ):
20 | mcp_app = FastMCP(name=name, description=description)
21 |
22 | _setup_workflow_tools(mcp_app, llm_instance, page_extraction_llm, workflow_dir)
23 | return mcp_app
24 |
25 |
26 | def _setup_workflow_tools(
27 | mcp_app: FastMCP, llm_instance: BaseChatModel, page_extraction_llm: BaseChatModel | None, workflow_dir: str
28 | ):
29 | """
30 | Scans a directory for workflow.json files, loads them, and registers them as tools
31 | with the FastMCP instance by dynamically setting function signatures.
32 | """
33 | workflow_files = list(Path(workflow_dir).glob('*.workflow.json'))
34 | print(f"[FastMCP Service] Found workflow files in '{workflow_dir}': {len(workflow_files)}")
35 |
36 | for wf_file_path in workflow_files:
37 | try:
38 | print(f'[FastMCP Service] Loading workflow from: {wf_file_path}')
39 | schema = WorkflowDefinitionSchema.load_from_json(str(wf_file_path))
40 |
41 | # Instantiate the workflow
42 | workflow = Workflow(
43 | workflow_schema=schema, llm=llm_instance, page_extraction_llm=page_extraction_llm, browser=None, controller=None
44 | )
45 |
46 | params_for_signature = []
47 | annotations_for_runner = {}
48 |
49 | if hasattr(workflow._input_model, 'model_fields'):
50 | for field_name, model_field in workflow._input_model.model_fields.items():
51 | param_annotation = model_field.annotation if model_field.annotation is not None else Any
52 |
53 | param_default = Parameter.empty
54 | if not model_field.is_required():
55 | param_default = model_field.default if model_field.default is not None else None
56 |
57 | params_for_signature.append(
58 | Parameter(
59 | name=field_name,
60 | kind=Parameter.POSITIONAL_OR_KEYWORD,
61 | default=param_default,
62 | annotation=param_annotation,
63 | )
64 | )
65 | annotations_for_runner[field_name] = param_annotation
66 |
67 | dynamic_signature = Signature(params_for_signature)
68 |
69 | # Sanitize workflow name for the function name
70 | safe_workflow_name_for_func = ''.join(c if c.isalnum() else '_' for c in schema.name)
71 | dynamic_func_name = f'tool_runner_{safe_workflow_name_for_func}_{schema.version.replace(".", "_")}'
72 |
73 | # Define the actual function that will be called by FastMCP
74 | # It uses a closure to capture the specific 'workflow' instance
75 | def create_runner(wf_instance: Workflow):
76 | async def actual_workflow_runner(**kwargs):
77 | # kwargs will be populated by FastMCP based on the dynamic_signature
78 | raw_result = await wf_instance.run(inputs=kwargs)
79 | try:
80 | return _json.dumps(raw_result, default=str)
81 | except Exception:
82 | return str(raw_result)
83 |
84 | return actual_workflow_runner
85 |
86 | runner_func_impl = create_runner(workflow)
87 |
88 | # Set the dunder attributes that FastMCP will inspect
89 | runner_func_impl.__name__ = dynamic_func_name
90 | runner_func_impl.__doc__ = schema.description
91 | runner_func_impl.__signature__ = dynamic_signature
92 | runner_func_impl.__annotations__ = annotations_for_runner
93 |
94 | # Tool name and description for FastMCP registration
95 | unique_tool_name = f'{schema.name.replace(" ", "_")}_{schema.version}'
96 | tool_description = schema.description
97 |
98 | tool_decorator = mcp_app.tool(name=unique_tool_name, description=tool_description)
99 | tool_decorator(runner_func_impl)
100 |
101 | param_names_for_log = list(dynamic_signature.parameters.keys())
102 | print(
103 | f"[FastMCP Service] Registered tool (via signature): '{unique_tool_name}' for '{schema.name}'. Params: {param_names_for_log}"
104 | )
105 |
106 | except Exception as e:
107 | print(f'[FastMCP Service] Failed to load or register workflow from {wf_file_path}: {e}')
108 | import traceback
109 |
110 | traceback.print_exc()
111 |
--------------------------------------------------------------------------------
/workflows/workflow_use/mcp/tests/test_tools.py:
--------------------------------------------------------------------------------
1 | from langchain_openai import ChatOpenAI
2 |
3 | from workflow_use.mcp.service import get_mcp_server
4 |
5 | # async def main():
6 | if __name__ == '__main__':
7 | llm_instance = ChatOpenAI(model='gpt-4o', temperature=0)
8 |
9 | print('[FastMCP Server] Starting MCP server...')
10 | # This will run the FastMCP server, typically using stdio transport by default.
11 | # For CLI execution like `fastmcp run workflow_use.mcp.server:mcp_app`,
12 | # this __main__ block might be bypassed by FastMCP's runner,
13 | # but it's good practice for direct Python execution.
14 | mcp = get_mcp_server(llm_instance, workflow_dir='./tmp')
15 | mcp.run(
16 | transport='sse',
17 | host='0.0.0.0',
18 | port=8008,
19 | )
20 |
21 | # async with Client(mcp) as client:
22 | # tools = await client.list_tools()
23 | # print(f'Available tools: {tools}')
24 |
25 | # result = await client.call_tool(
26 | # 'Government_Form_Submission_1.0',
27 | # {
28 | # 'first_name': 'John',
29 | # 'last_name': 'Smith',
30 | # 'social_security_last4': '1234',
31 | # 'gender': 'male',
32 | # 'marital_status': 'single',
33 | # },
34 | # )
35 |
36 | # print(result)
37 |
38 |
39 | # if __name__ == '__main__':
40 | # asyncio.run(main())
41 |
--------------------------------------------------------------------------------
/workflows/workflow_use/recorder/recorder.py:
--------------------------------------------------------------------------------
1 | # Example for your new main runner (e.g., in recorder.py or a new script)
2 | import asyncio
3 |
4 | from workflow_use.recorder.service import RecordingService # Adjust import path if necessary
5 |
6 |
7 | async def run_recording():
8 | service = RecordingService()
9 | print('Starting recording session via service...')
10 | workflow_schema = await service.capture_workflow()
11 |
12 | if workflow_schema:
13 | print('\n--- MAIN SCRIPT: CAPTURED WORKFLOW ---')
14 | try:
15 | print(workflow_schema.model_dump_json(indent=2))
16 | except AttributeError:
17 | # Fallback if model_dump_json isn't available (e.g. if it's a dict)
18 | import json
19 |
20 | print(json.dumps(workflow_schema, indent=2)) # Ensure schema is serializable
21 | print('------------------------------------')
22 | else:
23 | print('MAIN SCRIPT: No workflow was captured.')
24 |
25 |
26 | if __name__ == '__main__':
27 | try:
28 | asyncio.run(run_recording())
29 | except KeyboardInterrupt:
30 | print('Main recording script interrupted.')
31 | except Exception as e:
32 | print(f'An error occurred in the main recording script: {e}')
33 |
--------------------------------------------------------------------------------
/workflows/workflow_use/recorder/service.py:
--------------------------------------------------------------------------------
1 | import asyncio
2 | import json
3 | import pathlib
4 | from typing import Optional
5 |
6 | import uvicorn
7 | from browser_use import Browser
8 | from browser_use.browser.profile import BrowserProfile
9 | from fastapi import FastAPI
10 | from patchright.async_api import async_playwright as patchright_async_playwright
11 |
12 | # Assuming views.py is correctly located for this import path
13 | from workflow_use.recorder.views import (
14 | HttpRecordingStoppedEvent,
15 | HttpWorkflowUpdateEvent,
16 | RecorderEvent,
17 | WorkflowDefinitionSchema, # This is the expected output type
18 | )
19 |
20 | # Path Configuration (should be identical to recorder.py if run from the same context)
21 | SCRIPT_DIR = pathlib.Path(__file__).resolve().parent
22 | EXT_DIR = SCRIPT_DIR.parent.parent.parent / 'extension' / '.output' / 'chrome-mv3'
23 | USER_DATA_DIR = SCRIPT_DIR / 'user_data_dir'
24 |
25 |
26 | class RecordingService:
27 | def __init__(self):
28 | self.event_queue: asyncio.Queue[RecorderEvent] = asyncio.Queue()
29 | self.last_workflow_update_event: Optional[HttpWorkflowUpdateEvent] = None
30 | self.browser: Browser
31 |
32 | self.final_workflow_output: Optional[WorkflowDefinitionSchema] = None
33 | self.recording_complete_event = asyncio.Event()
34 | self.final_workflow_processed_lock = asyncio.Lock()
35 | self.final_workflow_processed_flag = False
36 |
37 | self.app = FastAPI(title='Temporary Recording Event Server')
38 | self.app.add_api_route('/event', self._handle_event_post, methods=['POST'], status_code=202)
39 | # -- DEBUGGING --
40 | # Turn this on to debug requests
41 | # @self.app.middleware("http")
42 | # async def log_requests(request: Request, call_next):
43 | # print(f"[Debug] Incoming request: {request.method} {request.url}")
44 | # try:
45 | # # Read request body
46 | # body = await request.body()
47 | # print(f"[Debug] Request body: {body.decode('utf-8', errors='replace')}")
48 | # response = await call_next(request)
49 | # print(f"[Debug] Response status: {response.status_code}")
50 | # return response
51 | # except Exception as e:
52 | # print(f"[Error] Error processing request: {str(e)}")
53 |
54 | self.uvicorn_server_instance: Optional[uvicorn.Server] = None
55 | self.server_task: Optional[asyncio.Task] = None
56 | self.browser_task: Optional[asyncio.Task] = None
57 | self.event_processor_task: Optional[asyncio.Task] = None
58 |
59 | async def _handle_event_post(self, event_data: RecorderEvent):
60 | if isinstance(event_data, HttpWorkflowUpdateEvent):
61 | self.last_workflow_update_event = event_data
62 | await self.event_queue.put(event_data)
63 | return {'status': 'accepted', 'message': 'Event queued for processing'}
64 |
65 | async def _process_event_queue(self):
66 | print('[Service] Event processing task started.')
67 | try:
68 | while True:
69 | event = await self.event_queue.get()
70 | print(f'[Service] Event Received: {event.type}')
71 | if isinstance(event, HttpWorkflowUpdateEvent):
72 | # self.last_workflow_update_event is already updated in _handle_event_post
73 | pass
74 | elif isinstance(event, HttpRecordingStoppedEvent):
75 | print('[Service] RecordingStoppedEvent received, processing final workflow...')
76 | await self._capture_and_signal_final_workflow('RecordingStoppedEvent')
77 | self.event_queue.task_done()
78 | except asyncio.CancelledError:
79 | print('[Service] Event processing task cancelled.')
80 | except Exception as e:
81 | print(f'[Service] Error in event processing task: {e}')
82 |
83 | async def _capture_and_signal_final_workflow(self, trigger_reason: str):
84 | processed_this_call = False
85 | async with self.final_workflow_processed_lock:
86 | if not self.final_workflow_processed_flag and self.last_workflow_update_event:
87 | print(f'[Service] Capturing final workflow (Trigger: {trigger_reason}).')
88 | self.final_workflow_output = self.last_workflow_update_event.payload
89 | self.final_workflow_processed_flag = True
90 | processed_this_call = True
91 |
92 | if processed_this_call:
93 | print('[Service] Final workflow captured. Setting recording_complete_event.')
94 | self.recording_complete_event.set() # Signal completion to the main method
95 |
96 | # If processing was due to RecordingStoppedEvent, also try to close the browser
97 | if trigger_reason == 'RecordingStoppedEvent' and self.browser:
98 | print('[Service] Attempting to close browser due to RecordingStoppedEvent...')
99 | try:
100 | await self.browser.close()
101 | print('[Service] Browser close command issued.')
102 | except Exception as e_close:
103 | print(f'[Service] Error closing browser on recording stop: {e_close}')
104 |
105 | async def _launch_browser_and_wait(self):
106 | print(f'[Service] Attempting to load extension from: {EXT_DIR}')
107 | if not EXT_DIR.exists() or not EXT_DIR.is_dir():
108 | print(f'[Service] ERROR: Extension directory not found: {EXT_DIR}')
109 | self.recording_complete_event.set() # Signal failure
110 | return
111 |
112 | # Ensure user data dir exists
113 | USER_DATA_DIR.mkdir(parents=True, exist_ok=True)
114 | print(f'[Service] Using browser user data directory: {USER_DATA_DIR}')
115 |
116 | try:
117 | # Create browser profile with extension support
118 | profile = BrowserProfile(
119 | headless=False,
120 | user_data_dir=str(USER_DATA_DIR.resolve()),
121 | args=[
122 | f'--disable-extensions-except={str(EXT_DIR.resolve())}',
123 | f'--load-extension={str(EXT_DIR.resolve())}',
124 | '--no-default-browser-check',
125 | '--no-first-run',
126 | ],
127 | keep_alive=True,
128 | )
129 |
130 | # Create and configure browser
131 | playwright = await patchright_async_playwright().start()
132 | self.browser = Browser(browser_profile=profile, playwright=playwright)
133 |
134 | print('[Service] Starting browser with extensions...')
135 | await self.browser.start()
136 |
137 | print('[Service] Browser launched. Waiting for close or recording stop...')
138 |
139 | # Wait for browser to be closed manually or recording to stop
140 | # We'll implement a simple polling mechanism to check if browser is still running
141 | while True:
142 | try:
143 | # Check if browser is still running by trying to get current page
144 | await self.browser.get_current_page()
145 | await asyncio.sleep(1) # Poll every second
146 | except Exception:
147 | # Browser is likely closed
148 | print('[Service] Browser appears to be closed or inaccessible.')
149 | break
150 |
151 | except asyncio.CancelledError:
152 | print('[Service] Browser task cancelled.')
153 | if self.browser:
154 | try:
155 | await self.browser.close()
156 | except:
157 | pass # Best effort
158 | raise # Re-raise to be caught by gather
159 | except Exception as e:
160 | print(f'[Service] Error in browser task: {e}')
161 | finally:
162 | print('[Service] Browser task finalization.')
163 | # self.browser = None
164 | # This call ensures that if browser is closed manually, we still try to capture.
165 | await self._capture_and_signal_final_workflow('BrowserTaskEnded')
166 |
167 | async def capture_workflow(self) -> Optional[WorkflowDefinitionSchema]:
168 | print('[Service] Starting capture_workflow session...')
169 | # Reset state for this session
170 | self.last_workflow_update_event = None
171 | self.final_workflow_output = None
172 | self.recording_complete_event.clear()
173 | self.final_workflow_processed_flag = False
174 |
175 | # Start background tasks
176 | self.event_processor_task = asyncio.create_task(self._process_event_queue())
177 | self.browser_task = asyncio.create_task(self._launch_browser_and_wait())
178 |
179 | # Configure and start Uvicorn server
180 | config = uvicorn.Config(self.app, host='127.0.0.1', port=7331, log_level='warning', loop='asyncio')
181 | self.uvicorn_server_instance = uvicorn.Server(config)
182 | self.server_task = asyncio.create_task(self.uvicorn_server_instance.serve())
183 | print('[Service] Uvicorn server task started.')
184 |
185 | try:
186 | print('[Service] Waiting for recording to complete...')
187 | await self.recording_complete_event.wait()
188 | print('[Service] Recording complete event received. Proceeding to cleanup.')
189 | except asyncio.CancelledError:
190 | print('[Service] capture_workflow task was cancelled externally.')
191 | finally:
192 | print('[Service] Starting cleanup phase...')
193 |
194 | # 1. Stop Uvicorn server
195 | if self.uvicorn_server_instance and self.server_task and not self.server_task.done():
196 | print('[Service] Signaling Uvicorn server to shut down...')
197 | self.uvicorn_server_instance.should_exit = True
198 | try:
199 | await asyncio.wait_for(self.server_task, timeout=5) # Give server time to shut down
200 | except asyncio.TimeoutError:
201 | print('[Service] Uvicorn server shutdown timed out. Cancelling task.')
202 | self.server_task.cancel()
203 | except asyncio.CancelledError: # If capture_workflow itself was cancelled
204 | pass
205 | except Exception as e_server_shutdown:
206 | print(f'[Service] Error during Uvicorn server shutdown: {e_server_shutdown}')
207 |
208 | # 2. Stop browser task (and ensure browser is closed)
209 | if self.browser_task and not self.browser_task.done():
210 | print('[Service] Cancelling browser task...')
211 | self.browser_task.cancel()
212 | try:
213 | await self.browser_task
214 | except asyncio.CancelledError:
215 | pass
216 | except Exception as e_browser_cancel:
217 | print(f'[Service] Error awaiting cancelled browser task: {e_browser_cancel}')
218 |
219 | if self.browser: # Final check to close browser if still open
220 | print('[Service] Ensuring browser is closed in cleanup...')
221 | try:
222 | self.browser.browser_profile.keep_alive = False
223 | await self.browser.close()
224 | except Exception as e_browser_close:
225 | print(f'[Service] Error closing browser in final cleanup: {e_browser_close}')
226 | # self.browser = None
227 |
228 | # 3. Stop event processor task
229 | if self.event_processor_task and not self.event_processor_task.done():
230 | print('[Service] Cancelling event processor task...')
231 | self.event_processor_task.cancel()
232 | try:
233 | await self.event_processor_task
234 | except asyncio.CancelledError:
235 | pass
236 | except Exception as e_ep_cancel:
237 | print(f'[Service] Error awaiting cancelled event processor task: {e_ep_cancel}')
238 |
239 | print('[Service] Cleanup phase complete.')
240 |
241 | if self.final_workflow_output:
242 | print('[Service] Returning captured workflow.')
243 | else:
244 | print('[Service] No workflow captured or an error occurred.')
245 | return self.final_workflow_output
246 |
247 |
248 | async def main_service_runner(): # Example of how to run the service
249 | service = RecordingService()
250 | workflow_data = await service.capture_workflow()
251 | if workflow_data:
252 | print('\n--- CAPTURED WORKFLOW DATA (from main_service_runner) ---')
253 | # Assuming WorkflowDefinitionSchema has model_dump_json or similar
254 | try:
255 | print(workflow_data.model_dump_json(indent=2))
256 | except AttributeError:
257 | print(json.dumps(workflow_data, indent=2)) # Fallback for plain dicts if model_dump_json not present
258 | print('-----------------------------------------------------')
259 | else:
260 | print('No workflow data was captured by the service.')
261 |
262 |
263 | if __name__ == '__main__':
264 | # This allows running service.py directly for testing
265 | try:
266 | asyncio.run(main_service_runner())
267 | except KeyboardInterrupt:
268 | print('Service runner interrupted by user.')
269 |
--------------------------------------------------------------------------------
/workflows/workflow_use/recorder/views.py:
--------------------------------------------------------------------------------
1 | from typing import Literal, Union
2 |
3 | from pydantic import BaseModel
4 |
5 | from workflow_use.schema.views import WorkflowDefinitionSchema
6 |
7 | # --- Event Payloads ---
8 |
9 |
10 | class RecordingStatusPayload(BaseModel):
11 | message: str
12 |
13 |
14 | # --- Main Event Models (mirroring HttpEvent types from message-bus-types.ts) ---
15 |
16 |
17 | class BaseHttpEvent(BaseModel):
18 | timestamp: int
19 |
20 |
21 | class HttpWorkflowUpdateEvent(BaseHttpEvent):
22 | type: Literal['WORKFLOW_UPDATE'] = 'WORKFLOW_UPDATE'
23 | payload: WorkflowDefinitionSchema
24 |
25 |
26 | class HttpRecordingStartedEvent(BaseHttpEvent):
27 | type: Literal['RECORDING_STARTED'] = 'RECORDING_STARTED'
28 | payload: RecordingStatusPayload
29 |
30 |
31 | class HttpRecordingStoppedEvent(BaseHttpEvent):
32 | type: Literal['RECORDING_STOPPED'] = 'RECORDING_STOPPED'
33 | payload: RecordingStatusPayload
34 |
35 |
36 | # Union of all possible event types received by the recorder
37 | RecorderEvent = Union[
38 | HttpWorkflowUpdateEvent,
39 | HttpRecordingStartedEvent,
40 | HttpRecordingStoppedEvent,
41 | ]
42 |
--------------------------------------------------------------------------------
/workflows/workflow_use/schema/views.py:
--------------------------------------------------------------------------------
1 | from typing import List, Literal, Optional, Union
2 |
3 | from pydantic import BaseModel, Field
4 |
5 |
6 | # --- Base Step Model ---
7 | # Common fields for all step types
8 | class BaseWorkflowStep(BaseModel):
9 | description: Optional[str] = Field(None, description="Optional description/comment about the step's purpose.")
10 | output: Optional[str] = Field(None, description='Context key to store step output under.')
11 | # Allow other fields captured from raw events but not explicitly modeled
12 | model_config = {'extra': 'allow'}
13 |
14 |
15 | # --- Timestamped Step Mixin (for deterministic actions) ---
16 | class TimestampedWorkflowStep(BaseWorkflowStep):
17 | timestamp: Optional[int] = Field(None, description='Timestamp from recording (informational).')
18 | tabId: Optional[int] = Field(None, description='Browser tab ID from recording (informational).')
19 |
20 |
21 | # --- Agent Step ---
22 | class AgentTaskWorkflowStep(BaseWorkflowStep):
23 | type: Literal['agent']
24 | task: str = Field(..., description='The objective or task description for the agent.')
25 | max_steps: Optional[int] = Field(
26 | None,
27 | description='Maximum number of iterations for the agent (default handled in code).',
28 | )
29 | # Agent steps might also have 'params' for other configs, handled by extra='allow'
30 |
31 |
32 | # --- Deterministic Action Steps (based on controllers and examples) ---
33 |
34 |
35 | # Actions from src/workflows/controller/service.py & Examples
36 | class NavigationStep(TimestampedWorkflowStep):
37 | """Navigates using the 'navigation' action (likely maps to go_to_url)."""
38 |
39 | type: Literal['navigation'] # As seen in examples
40 | url: str = Field(..., description='Target URL to navigate to. Can use {context_var}.')
41 |
42 |
43 | class ClickStep(TimestampedWorkflowStep):
44 | """Clicks an element using 'click' (maps to workflow controller's click)."""
45 |
46 | type: Literal['click'] # As seen in examples
47 | cssSelector: str = Field(..., description='CSS selector for the target element.')
48 | xpath: Optional[str] = Field(None, description='XPath selector (often informational).')
49 | elementTag: Optional[str] = Field(None, description='HTML tag (informational).')
50 | elementText: Optional[str] = Field(None, description='Element text (informational).')
51 |
52 |
53 | class InputStep(TimestampedWorkflowStep):
54 | """Inputs text using 'input' (maps to workflow controller's input)."""
55 |
56 | type: Literal['input'] # As seen in examples
57 | cssSelector: str = Field(..., description='CSS selector for the target input element.')
58 | value: str = Field(..., description='Value to input. Can use {context_var}.')
59 | xpath: Optional[str] = Field(None, description='XPath selector (informational).')
60 | elementTag: Optional[str] = Field(None, description='HTML tag (informational).')
61 |
62 |
63 | class SelectChangeStep(TimestampedWorkflowStep):
64 | """Selects a dropdown option using 'select_change' (maps to workflow controller's select_change)."""
65 |
66 | type: Literal['select_change'] # Assumed type for workflow controller's select_change
67 | cssSelector: str = Field(..., description='CSS selector for the target select element.')
68 | selectedText: str = Field(..., description='Visible text of the option to select. Can use {context_var}.')
69 | xpath: Optional[str] = Field(None, description='XPath selector (informational).')
70 | elementTag: Optional[str] = Field(None, description='HTML tag (informational).')
71 |
72 |
73 | class KeyPressStep(TimestampedWorkflowStep):
74 | """Presses a key using 'key_press' (maps to workflow controller's key_press)."""
75 |
76 | type: Literal['key_press'] # As seen in examples
77 | cssSelector: str = Field(..., description='CSS selector for the target element.')
78 | key: str = Field(..., description="The key to press (e.g., 'Tab', 'Enter').")
79 | xpath: Optional[str] = Field(None, description='XPath selector (informational).')
80 | elementTag: Optional[str] = Field(None, description='HTML tag (informational).')
81 |
82 |
83 | class ScrollStep(TimestampedWorkflowStep):
84 | """Scrolls the page using 'scroll' (maps to workflow controller's scroll)."""
85 |
86 | type: Literal['scroll'] # Assumed type for workflow controller's scroll
87 | scrollX: int = Field(..., description='Horizontal scroll pixels.')
88 | scrollY: int = Field(..., description='Vertical scroll pixels.')
89 |
90 |
91 | class PageExtractionStep(TimestampedWorkflowStep):
92 | """Extracts text from the page using 'page_extraction' (maps to workflow controller's page_extraction)."""
93 |
94 | type: Literal['extract_page_content'] # Assumed type for workflow controller's page_extraction
95 | goal: str = Field(..., description='The goal of the page extraction.')
96 |
97 |
98 | # --- Union of all possible step types ---
99 | # This Union defines what constitutes a valid step in the "steps" list.
100 | DeterministicWorkflowStep = Union[
101 | NavigationStep,
102 | ClickStep,
103 | InputStep,
104 | SelectChangeStep,
105 | KeyPressStep,
106 | ScrollStep,
107 | PageExtractionStep,
108 | ]
109 |
110 | AgenticWorkflowStep = AgentTaskWorkflowStep
111 |
112 |
113 | WorkflowStep = Union[
114 | # Pure workflow
115 | DeterministicWorkflowStep,
116 | # Agentic
117 | AgenticWorkflowStep,
118 | ]
119 |
120 | allowed_controller_actions = []
121 |
122 |
123 | # --- Input Schema Definition ---
124 | # (Remains the same)
125 | class WorkflowInputSchemaDefinition(BaseModel):
126 | name: str = Field(
127 | ...,
128 | description='The name of the property. This will be used as the key in the input schema.',
129 | )
130 | type: Literal['string', 'number', 'bool']
131 | required: Optional[bool] = Field(
132 | default=None,
133 | description='None if the property is optional, True if the property is required.',
134 | )
135 |
136 |
137 | # --- Top-Level Workflow Definition File ---
138 | # Uses the Union WorkflowStep type
139 |
140 |
141 | class WorkflowDefinitionSchema(BaseModel):
142 | """Pydantic model representing the structure of the workflow JSON file."""
143 |
144 | workflow_analysis: Optional[str] = Field(
145 | None,
146 | description='A chain of thought reasoning analysis of the original workflow recording.',
147 | )
148 |
149 | name: str = Field(..., description='The name of the workflow.')
150 | description: str = Field(..., description='A human-readable description of the workflow.')
151 | version: str = Field(..., description='The version identifier for this workflow definition.')
152 | steps: List[WorkflowStep] = Field(
153 | ...,
154 | min_length=1,
155 | description='An ordered list of steps (actions or agent tasks) to be executed.',
156 | )
157 | input_schema: list[WorkflowInputSchemaDefinition] = Field(
158 | # default=WorkflowInputSchemaDefinition(),
159 | description='List of input schema definitions.',
160 | )
161 |
162 | # Add loader from json file
163 | @classmethod
164 | def load_from_json(cls, json_path: str):
165 | with open(json_path, 'r') as f:
166 | return cls.model_validate_json(f.read())
167 |
--------------------------------------------------------------------------------
/workflows/workflow_use/workflow/prompts.py:
--------------------------------------------------------------------------------
1 | WORKFLOW_FALLBACK_PROMPT_TEMPLATE = (
2 | 'While executing step {step_index}/{total_steps} in the workflow:\n\n'
3 | # "{workflow_details}\n\n"
4 | "The deterministic action '{action_type}' failed with the following context:\n"
5 | '{fail_details}\n\n'
6 | 'The intended target or expected value for this step was: {failed_value}\n\n'
7 | 'IMPORTANT: Your task is to ONLY complete this specific step ({step_index}) and nothing more. '
8 | "The step's purpose is described as: '{step_description}'.\n"
9 | 'Do not retry the same action that failed. Instead, choose a different suitable action(s) to accomplish the same goal. '
10 | 'For example, if a click failed, consider navigating to a URL, inputting text, or selecting an option. '
11 | 'However, ONLY perform the minimum action needed to complete this specific step. '
12 | 'If the step requires clicking a button, ONLY click that button. If it requires navigation, ONLY navigate. '
13 | 'Do not perform any additional actions beyond what is strictly necessary for this step. '
14 | 'Once the objective of step {step_index} is reached, call the Done action to complete the step. '
15 | 'Do not proceed to the next step or perform any actions beyond this specific step.'
16 | )
17 |
18 | STRUCTURED_OUTPUT_PROMPT = """
19 | You are a data extraction expert. Your task is to extract structured information from the provided content.
20 |
21 | The content may contain various pieces of information from different sources. You need to analyze this content and extract the relevant information according to the output schema provided below.
22 |
23 | Only extract information that is explicitly present in the content. Be precise and follow the schema exactly.
24 | """
25 |
--------------------------------------------------------------------------------
/workflows/workflow_use/workflow/tests/run_workflow.py:
--------------------------------------------------------------------------------
1 | import asyncio
2 | from pathlib import Path
3 |
4 | # Ensure langchain-openai is installed and OPENAI_API_KEY is set
5 | from langchain_openai import ChatOpenAI
6 |
7 | from workflow_use.builder.service import BuilderService
8 | from workflow_use.workflow.service import Workflow
9 |
10 | # Instantiate the LLM and the service directly
11 | llm_instance = ChatOpenAI(model='gpt-4o') # Or your preferred model
12 | builder_service = BuilderService(llm=llm_instance)
13 |
14 |
15 | async def test_run_workflow():
16 | """
17 | Tests that the workflow is built correctly from a JSON file path.
18 | """
19 | path = Path(__file__).parent / 'tmp' / 'recording.workflow.json'
20 |
21 | workflow = Workflow.load_from_file(path)
22 | result = await workflow.run({'model': '12'})
23 | print(result)
24 |
25 |
26 | if __name__ == '__main__':
27 | asyncio.run(test_run_workflow())
28 |
--------------------------------------------------------------------------------
/workflows/workflow_use/workflow/tests/test_extract.py:
--------------------------------------------------------------------------------
1 | import asyncio
2 | from pathlib import Path
3 |
4 | # Ensure langchain-openai is installed and OPENAI_API_KEY is set
5 | from langchain_openai import ChatOpenAI
6 | from pydantic import BaseModel
7 |
8 | from workflow_use.workflow.service import Workflow
9 |
10 | # Instantiate the LLM and the service directly
11 | llm_instance = ChatOpenAI(model='gpt-4o') # Or your preferred model
12 | page_extraction_llm = ChatOpenAI(model='gpt-4o-mini')
13 |
14 |
15 | class OutputModel(BaseModel):
16 | api_key: str
17 |
18 |
19 | async def test_run_workflow():
20 | """
21 | Tests that the workflow is built correctly from a JSON file path.
22 | """
23 | path = Path(__file__).parent / 'tmp' / 'extract.workflow.json'
24 |
25 | workflow = Workflow.load_from_file(path, llm=llm_instance, page_extraction_llm=page_extraction_llm)
26 |
27 | result = await workflow.run({'api_key_name': 'test key'}, output_model=OutputModel)
28 |
29 | assert result.output_model is not None
30 |
31 | print(result.output_model.api_key)
32 |
33 |
34 | if __name__ == '__main__':
35 | asyncio.run(test_run_workflow())
36 |
--------------------------------------------------------------------------------
/workflows/workflow_use/workflow/views.py:
--------------------------------------------------------------------------------
1 | from typing import Any, Dict, Generic, List, Optional, TypeVar
2 |
3 | from browser_use.agent.views import ActionResult, AgentHistoryList
4 | from pydantic import BaseModel, Field
5 |
6 | T = TypeVar('T', bound=BaseModel)
7 |
8 |
9 | class WorkflowRunOutput(BaseModel, Generic[T]):
10 | """Output of a workflow run"""
11 |
12 | step_results: List[ActionResult | AgentHistoryList]
13 | output_model: Optional[T] = None
14 |
15 |
16 | class StructuredWorkflowOutput(BaseModel):
17 | """Base model for structured workflow outputs.
18 |
19 | This can be used as a parent class for custom output models that
20 | will be filled by convert_results_to_output_model method.
21 | """
22 |
23 | raw_data: Dict[str, Any] = Field(default_factory=dict, description='Raw extracted data from workflow execution')
24 |
25 | status: str = Field(default='success', description='Overall status of the workflow execution')
26 |
27 | error_message: Optional[str] = Field(default=None, description='Error message if the workflow failed')
28 |
--------------------------------------------------------------------------------
3 | 
11 |