├── .eslintrc.json ├── .github └── image.png ├── public └── favicon.ico ├── postcss.config.js ├── next.config.js ├── components ├── Label.tsx ├── Generation.ts ├── AppBar.tsx ├── Editor.tsx ├── Nodes │ ├── RandomNumber.tsx │ ├── Concat.tsx │ ├── RegexReplace.tsx │ ├── LoadImage.tsx │ ├── Interrogate.tsx │ ├── Transformer.tsx │ ├── Image.tsx │ └── index.tsx ├── Bar.tsx └── Node.tsx ├── pages ├── _app.tsx ├── _document.tsx └── index.tsx ├── tailwind.config.js ├── .gitignore ├── tsconfig.json ├── README.md ├── package.json └── styles └── globals.css /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals" 3 | } 4 | -------------------------------------------------------------------------------- /.github/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KAJdev/node-diffusion/HEAD/.github/image.png -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KAJdev/node-diffusion/HEAD/public/favicon.ico -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | const nextConfig = { 3 | reactStrictMode: true, 4 | } 5 | 6 | module.exports = nextConfig 7 | -------------------------------------------------------------------------------- /components/Label.tsx: -------------------------------------------------------------------------------- 1 | export function Label({ children }: { children?: React.ReactNode }) { 2 | return ( 3 |

{children}

4 | ); 5 | } 6 | -------------------------------------------------------------------------------- /pages/_app.tsx: -------------------------------------------------------------------------------- 1 | import '../styles/globals.css' 2 | import type { AppProps } from 'next/app' 3 | 4 | export default function App({ Component, pageProps }: AppProps) { 5 | return 6 | } 7 | -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | module.exports = { 3 | content: [ 4 | "./pages/**/*.{js,ts,jsx,tsx}", 5 | "./components/**/*.{js,ts,jsx,tsx}", 6 | ], 7 | theme: { 8 | extend: {}, 9 | }, 10 | plugins: [], 11 | } 12 | -------------------------------------------------------------------------------- /pages/_document.tsx: -------------------------------------------------------------------------------- 1 | import { Html, Head, Main, NextScript } from 'next/document' 2 | 3 | export default function Document() { 4 | return ( 5 | 6 | 7 | 8 |
9 | 10 | 11 | 12 | ) 13 | } 14 | -------------------------------------------------------------------------------- /components/Generation.ts: -------------------------------------------------------------------------------- 1 | export namespace Generation { 2 | export async function RenderImage( 3 | prompt: string, 4 | cfg_scale: number, 5 | steps: number, 6 | init?: Blob, 7 | mask?: Blob 8 | ): Promise { 9 | return new Blob(); 10 | } 11 | 12 | export async function PredictText( 13 | prompt: string, 14 | temperature: number 15 | ): Promise { 16 | return ""; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # next.js 12 | /.next/ 13 | /out/ 14 | 15 | # production 16 | /build 17 | 18 | # misc 19 | .DS_Store 20 | *.pem 21 | 22 | # debug 23 | npm-debug.log* 24 | yarn-debug.log* 25 | yarn-error.log* 26 | .pnpm-debug.log* 27 | 28 | # local env files 29 | .env*.local 30 | 31 | # vercel 32 | .vercel 33 | 34 | # typescript 35 | *.tsbuildinfo 36 | next-env.d.ts 37 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": ["dom", "dom.iterable", "esnext"], 5 | "allowJs": true, 6 | "skipLibCheck": true, 7 | "strict": true, 8 | "forceConsistentCasingInFileNames": true, 9 | "noEmit": true, 10 | "esModuleInterop": true, 11 | "module": "esnext", 12 | "moduleResolution": "node", 13 | "resolveJsonModule": true, 14 | "isolatedModules": true, 15 | "jsx": "preserve", 16 | "incremental": true 17 | }, 18 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], 19 | "exclude": ["node_modules"] 20 | } 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Node Diffusion (working name) 2 | 3 | Welcome to another one of my weekend projects exploring how humans can interact with state of the art AI models. This week features a node-graph editor inspired by shader/material editors found in many other creative software. Built using React Flow, zustand, nextjs, cloudflare workers, StabilityAI API, and OpenAI API. 4 | 5 | > You can see a live version of the site at 6 | 7 | ![](https://github.com/KAJdev/node-diffusion/blob/main/.github/image.png?raw=true) 8 | 9 | ## Contributing 10 | 11 | Just make a PR and try to follow the functional paradigm. All skill levels welcome. Let's build AI interactions together :). 12 | -------------------------------------------------------------------------------- /pages/index.tsx: -------------------------------------------------------------------------------- 1 | import Head from "next/head"; 2 | import { ReactFlowProvider } from "reactflow"; 3 | import { AppBar } from "../components/AppBar"; 4 | import { Editor } from "../components/Editor"; 5 | 6 | export default function Home() { 7 | return ( 8 | <> 9 | 10 | Node Diffusion 11 | 12 | 13 | 14 | 15 |
16 | 17 | 18 | 19 | 20 |
21 | 22 | ); 23 | } 24 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "node-diffusion", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "@next/font": "13.0.7", 13 | "@types/node": "18.11.16", 14 | "@types/react": "18.0.26", 15 | "@types/react-dom": "18.0.9", 16 | "eslint": "8.30.0", 17 | "eslint-config-next": "13.0.7", 18 | "lucide-react": "^0.105.0-alpha.3", 19 | "next": "13.0.7", 20 | "react": "18.2.0", 21 | "react-dom": "18.2.0", 22 | "reactflow": "^11.3.3", 23 | "typescript": "4.9.4", 24 | "zustand": "^4.1.5" 25 | }, 26 | "devDependencies": { 27 | "autoprefixer": "^10.4.13", 28 | "postcss": "^8.4.20", 29 | "tailwindcss": "^3.2.4" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /components/AppBar.tsx: -------------------------------------------------------------------------------- 1 | import { Save } from "lucide-react"; 2 | 3 | export function AppBar() { 4 | return ( 5 |
6 |
7 |
8 | 13 |
14 |
15 | 18 |
19 |
20 | ); 21 | } 22 | -------------------------------------------------------------------------------- /styles/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | :root { 6 | --font-mono: ui-monospace, Menlo, Monaco, 'Cascadia Mono', 'Segoe UI Mono', 7 | 'Roboto Mono', 'Oxygen Mono', 'Ubuntu Monospace', 'Source Code Pro', 8 | 'Fira Mono', 'Droid Sans Mono', 'Courier New', monospace; 9 | } 10 | 11 | html, 12 | body { 13 | max-width: 100vw; 14 | overflow-x: hidden; 15 | color: white; 16 | } 17 | 18 | a { 19 | color: inherit; 20 | text-decoration: none; 21 | } 22 | 23 | .react-flow__minimap { 24 | @apply bg-neutral-800 fill-neutral-500; 25 | } 26 | .react-flow__minimap .react-flow__minimap-mask { 27 | @apply fill-neutral-800; 28 | } 29 | .react-flow__minimap .react-flow__minimap-node { 30 | @apply fill-neutral-700; 31 | stroke: none; 32 | } 33 | .react-flow__minimap path { 34 | @apply fill-neutral-700; 35 | } 36 | 37 | .react-flow__controls { 38 | @apply bg-neutral-800 rounded overflow-hidden; 39 | } 40 | .react-flow__controls button { 41 | @apply bg-neutral-800 text-white border-white/10 last-of-type:border-transparent p-2 duration-200 hover:bg-white/10 active:bg-white/20; 42 | } 43 | .react-flow__controls path { 44 | @apply fill-white; 45 | } 46 | 47 | .border-animate { 48 | background-image: linear-gradient(90deg, #3c3c3c 50%, transparent 50%), linear-gradient(90deg, #3c3c3c 50%, transparent 50%), linear-gradient(0deg, #3c3c3c 50%, transparent 50%), linear-gradient(0deg, #3c3c3c 50%, transparent 50%); 49 | background-repeat: repeat-x, repeat-x, repeat-y, repeat-y; 50 | background-size: 15px 2px, 15px 2px, 2px 15px, 2px 15px; 51 | background-position: left top, right bottom, left bottom, right top; 52 | animation: border-dance 1s infinite linear; 53 | } 54 | 55 | @keyframes border-dance { 56 | 0% { 57 | background-position: left top, right bottom, left bottom, right top; 58 | } 59 | 60 | 100% { 61 | background-position: left 15px top, right 15px bottom, left bottom 15px, right top 15px; 62 | } 63 | } -------------------------------------------------------------------------------- /components/Editor.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | Connection, 3 | ConnectionLineType, 4 | EdgeChange, 5 | MarkerType, 6 | NodeChange, 7 | useReactFlow, 8 | } from "@reactflow/core"; 9 | import React, { useCallback, useState } from "react"; 10 | import ReactFlow, { 11 | addEdge, 12 | applyEdgeChanges, 13 | applyNodeChanges, 14 | Background, 15 | Controls, 16 | Edge, 17 | MiniMap, 18 | Node, 19 | } from "reactflow"; 20 | import "reactflow/dist/style.css"; 21 | import { Bar } from "./Bar"; 22 | import shallow from "zustand/shallow"; 23 | 24 | import { Nodes, NodesState } from "./Nodes"; 25 | 26 | const selector = (state: NodesState) => ({ 27 | nodes: state.nodes, 28 | edges: state.edges, 29 | onNodesChange: state.onNodesChange, 30 | onEdgesChange: state.onEdgesChange, 31 | onConnect: state.onConnect, 32 | addNode: state.addNode, 33 | }); 34 | 35 | export function Editor() { 36 | const { nodes, edges, onNodesChange, onEdgesChange, onConnect, addNode } = 37 | Nodes.use(selector, shallow); 38 | 39 | return ( 40 | <> 41 | 70 | 71 | 72 | {/* */} 73 | 74 | { 76 | addNode({ 77 | ...newNode, 78 | data: { 79 | ...newNode.data, 80 | locked: false, 81 | running: false, 82 | repeating: false, 83 | }, 84 | id: Math.random().toString(), 85 | }); 86 | }} 87 | /> 88 | 89 | ); 90 | } 91 | 92 | export namespace Editor {} 93 | -------------------------------------------------------------------------------- /components/Nodes/RandomNumber.tsx: -------------------------------------------------------------------------------- 1 | /* eslint-disable @next/next/no-img-element */ 2 | import { Dice2, Trash2, Lock, Unlock } from "lucide-react"; 3 | import { memo } from "react"; 4 | import { Handle, Position, NodeProps, NodeToolbar, Node } from "reactflow"; 5 | import "reactflow/dist/style.css"; 6 | import { Nodes } from "."; 7 | import { Label } from "../Label"; 8 | 9 | import { 10 | NumberVariable, 11 | Output, 12 | Outputs, 13 | Panel, 14 | Toolbar, 15 | ToolButton, 16 | Variables, 17 | } from "../Node"; 18 | 19 | export type RandomNumber = NodeProps<{ 20 | locked: boolean; 21 | running: boolean; 22 | repeating: boolean; 23 | input: { 24 | min: number; 25 | max: number; 26 | }; 27 | output: { 28 | number: number; 29 | }; 30 | }>; 31 | 32 | export function RandomNumber(node: RandomNumber) { 33 | const { editNode, deleteNode } = Nodes.use((state) => ({ 34 | editNode: state.editNode, 35 | deleteNode: state.deleteNode, 36 | })); 37 | 38 | return ( 39 | 44 | 45 | 47 | editNode(node.id, { 48 | locked: !node.data.locked, 49 | }) 50 | } 51 | active={node.data.locked} 52 | > 53 | {node.data.locked ? : } 54 | 55 | deleteNode(node.id)}> 56 | 57 | 58 | 59 | 60 | 61 | 67 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | ); 80 | } 81 | 82 | export namespace RandomNumber { 83 | export async function run(node: Node): Promise { 84 | const { min, max } = node.data.input; 85 | return { 86 | number: Math.random() * (max - min) + min, 87 | }; 88 | } 89 | 90 | export const Memo = memo(RandomNumber); 91 | } 92 | -------------------------------------------------------------------------------- /components/Nodes/Concat.tsx: -------------------------------------------------------------------------------- 1 | /* eslint-disable @next/next/no-img-element */ 2 | import { Play, Trash2, Unlock, Lock } from "lucide-react"; 3 | import { memo, useEffect } from "react"; 4 | import { 5 | Handle, 6 | Position, 7 | NodeProps, 8 | NodeToolbar, 9 | getIncomers, 10 | getConnectedEdges, 11 | useUpdateNodeInternals, 12 | Node, 13 | } from "reactflow"; 14 | import "reactflow/dist/style.css"; 15 | import { Nodes } from "."; 16 | import { Label } from "../Label"; 17 | import { 18 | Output, 19 | Outputs, 20 | Panel, 21 | TextVariable, 22 | Toolbar, 23 | ToolButton, 24 | Variables, 25 | } from "../Node"; 26 | 27 | export type Concat = NodeProps<{ 28 | locked: boolean; 29 | running: boolean; 30 | repeating: boolean; 31 | input: { 32 | first: string; 33 | second: string; 34 | }; 35 | output: { 36 | final: string; 37 | }; 38 | }>; 39 | 40 | export function Concat(node: Concat) { 41 | const { editNode, deleteNode, nodes, edges } = Nodes.use((state) => ({ 42 | editNode: state.editNode, 43 | deleteNode: state.deleteNode, 44 | nodes: state.nodes, 45 | edges: state.edges, 46 | })); 47 | 48 | return ( 49 | 50 | 51 | 53 | editNode(node.id, { 54 | locked: !node.data.locked, 55 | }) 56 | } 57 | active={node.data.locked} 58 | > 59 | {node.data.locked ? : } 60 | 61 | deleteNode(node.id)}> 62 | 63 | 64 | 65 | 66 | 67 | 73 | 79 | 80 | 81 | 82 | 88 | 89 | 90 | ); 91 | } 92 | 93 | export namespace Concat { 94 | export async function run(node: Node): Promise { 95 | return { 96 | final: node.data.input.first + node.data.input.second, 97 | }; 98 | } 99 | 100 | export const Memo = memo(Concat); 101 | } 102 | -------------------------------------------------------------------------------- /components/Nodes/RegexReplace.tsx: -------------------------------------------------------------------------------- 1 | /* eslint-disable @next/next/no-img-element */ 2 | import { Play, Trash2, Unlock, Lock } from "lucide-react"; 3 | import { memo, useEffect } from "react"; 4 | import { 5 | Handle, 6 | Position, 7 | NodeProps, 8 | NodeToolbar, 9 | getIncomers, 10 | getConnectedEdges, 11 | useUpdateNodeInternals, 12 | Node, 13 | } from "reactflow"; 14 | import "reactflow/dist/style.css"; 15 | import { Nodes } from "."; 16 | import { Label } from "../Label"; 17 | import { 18 | Output, 19 | Outputs, 20 | Panel, 21 | TextVariable, 22 | Toolbar, 23 | ToolButton, 24 | Variables, 25 | } from "../Node"; 26 | 27 | export type RegexReplace = NodeProps<{ 28 | locked: boolean; 29 | running: boolean; 30 | repeating: boolean; 31 | input: { 32 | expression: string; 33 | replacement: string; 34 | text: string; 35 | }; 36 | output: { 37 | final: string; 38 | }; 39 | }>; 40 | 41 | export function RegexReplace(node: RegexReplace) { 42 | const { editNode, deleteNode, nodes, edges } = Nodes.use((state) => ({ 43 | editNode: state.editNode, 44 | deleteNode: state.deleteNode, 45 | nodes: state.nodes, 46 | edges: state.edges, 47 | })); 48 | 49 | return ( 50 | 55 | 56 | 58 | editNode(node.id, { 59 | locked: !node.data.locked, 60 | }) 61 | } 62 | active={node.data.locked} 63 | > 64 | {node.data.locked ? : } 65 | 66 | deleteNode(node.id)}> 67 | 68 | 69 | 70 | 71 | 72 | 78 | 84 | 90 | 91 | 92 | 93 | 100 | 101 | 102 | ); 103 | } 104 | 105 | export namespace RegexReplace { 106 | export async function run(node: Node): Promise { 107 | return { 108 | final: node.data.input.text.replace( 109 | new RegExp(node.data.input.expression, "g"), 110 | node.data.input.replacement 111 | ), 112 | }; 113 | } 114 | 115 | export const Memo = memo(RegexReplace); 116 | } 117 | -------------------------------------------------------------------------------- /components/Nodes/LoadImage.tsx: -------------------------------------------------------------------------------- 1 | /* eslint-disable @next/next/no-img-element */ 2 | import { ImageIcon, Lock, Trash2, Unlock } from "lucide-react"; 3 | import { memo, useCallback, useEffect, useState } from "react"; 4 | import { NodeProps, Node } from "reactflow"; 5 | import "reactflow/dist/style.css"; 6 | import { Nodes } from "."; 7 | import { Content, Output, Outputs, Panel, Toolbar, ToolButton } from "../Node"; 8 | 9 | export type LoadImage = NodeProps<{ 10 | locked: boolean; 11 | running: boolean; 12 | repeating: boolean; 13 | input: {}; 14 | output: { 15 | image: string; 16 | }; 17 | }>; 18 | 19 | export function LoadImage(node: LoadImage) { 20 | const { editNode, deleteNode } = Nodes.use((state) => ({ 21 | editNode: state.editNode, 22 | deleteNode: state.deleteNode, 23 | })); 24 | 25 | const [loaded, setLoaded] = useState(false); 26 | 27 | useEffect(() => { 28 | if (!node.data.output.image) { 29 | setLoaded(false); 30 | } 31 | }, [node.data.output.image]); 32 | 33 | const loadImage = useCallback(() => { 34 | const input = document.createElement("input"); 35 | input.type = "file"; 36 | input.accept = "image/png"; 37 | input.onchange = (e) => { 38 | if (e.target) { 39 | const file = (e.target as HTMLInputElement).files![0]; 40 | // create blob url 41 | const url = URL.createObjectURL(file); 42 | // set image 43 | editNode(node.id, { 44 | output: { 45 | image: url, 46 | }, 47 | }); 48 | } 49 | }; 50 | input.click(); 51 | }, [editNode, node.id]); 52 | 53 | return ( 54 | 59 | 60 | 62 | editNode(node.id, { 63 | locked: !node.data.locked, 64 | }) 65 | } 66 | active={node.data.locked} 67 | > 68 | {node.data.locked ? : } 69 | 70 | deleteNode(node.id)}> 71 | 72 | 73 | 74 | 75 | 76 | {node.data.output.image && ( 77 | graph image setLoaded(true)} 82 | onClick={loadImage} 83 | /> 84 | )} 85 | {!node.data.output.image && !loaded && ( 86 |
90 | 91 |

Choose Image

92 |
93 | )} 94 | 95 | 96 | 97 | 98 | 99 | ); 100 | } 101 | 102 | export namespace LoadImage { 103 | export async function run(node: Node): Promise { 104 | return { 105 | image: node.data.output.image, 106 | }; 107 | } 108 | 109 | export const Memo = memo(LoadImage); 110 | } 111 | -------------------------------------------------------------------------------- /components/Nodes/Interrogate.tsx: -------------------------------------------------------------------------------- 1 | /* eslint-disable @next/next/no-img-element */ 2 | import { ImageIcon, Lock, Play, Repeat, Trash2, Unlock } from "lucide-react"; 3 | import React from "react"; 4 | import { memo, useEffect, useMemo, useRef, useState } from "react"; 5 | import { 6 | Handle, 7 | Position, 8 | NodeProps, 9 | NodeToolbar, 10 | getIncomers, 11 | getConnectedEdges, 12 | Node, 13 | } from "reactflow"; 14 | import "reactflow/dist/style.css"; 15 | import { Nodes } from "."; 16 | import { Label } from "../Label"; 17 | import { 18 | Content, 19 | ImageVariable, 20 | NumberVariable, 21 | Output, 22 | Outputs, 23 | Panel, 24 | TextVariable, 25 | Toolbar, 26 | ToolButton, 27 | Variables, 28 | } from "../Node"; 29 | 30 | export type Interrogate = NodeProps<{ 31 | locked: boolean; 32 | running: boolean; 33 | repeating: boolean; 34 | input: { 35 | image: string; 36 | }; 37 | output: { 38 | prompt: string; 39 | }; 40 | }>; 41 | 42 | export function Interrogate(node: Interrogate) { 43 | const { editNode, deleteNode } = Nodes.use((state) => ({ 44 | editNode: state.editNode, 45 | deleteNode: state.deleteNode, 46 | })); 47 | 48 | const [loaded, setLoaded] = useState(false); 49 | 50 | return ( 51 | 56 | 57 | {!node.data.running && ( 58 | { 60 | Nodes.resolveNode(node.id); 61 | }} 62 | > 63 | 64 | 65 | )} 66 | { 68 | if (node.data.repeating) { 69 | editNode(node.id, { 70 | repeating: false, 71 | }); 72 | } else { 73 | Nodes.resolveNode(node.id, true); 74 | } 75 | }} 76 | active={node.data.repeating} 77 | > 78 | 79 | 80 | 82 | editNode(node.id, { 83 | locked: !node.data.locked, 84 | }) 85 | } 86 | active={node.data.locked} 87 | > 88 | {node.data.locked ? : } 89 | 90 | deleteNode(node.id)}> 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 106 | 107 | 108 | ); 109 | } 110 | 111 | export namespace Interrogate { 112 | export async function run(node: Node): Promise { 113 | const { image } = node.data.input; 114 | 115 | if (!image) { 116 | if (node.data.repeating) { 117 | // disable repeating 118 | Nodes.use.getState().editNode(node.id, { 119 | repeating: false, 120 | }); 121 | } 122 | return { 123 | prompt: node.data.output.prompt, 124 | }; 125 | } 126 | 127 | // image is probably a dataurl or a regular image. Need to fetch and convert to base64 128 | const img_d = await fetch(image); 129 | const img_b = await img_d.blob(); 130 | const img_b64 = await new Promise((resolve) => { 131 | const reader = new FileReader(); 132 | reader.onloadend = () => { 133 | resolve(reader.result); 134 | }; 135 | reader.readAsDataURL(img_b); 136 | }); 137 | 138 | const data = await fetch("https://api.prototyped.ai/interrogate", { 139 | method: "POST", 140 | body: JSON.stringify({ 141 | image: img_b64, 142 | }), 143 | }).then((res) => res.json()); 144 | 145 | return { 146 | prompt: data.prompt, 147 | }; 148 | } 149 | 150 | export const Memo = memo(Interrogate); 151 | } 152 | -------------------------------------------------------------------------------- /components/Nodes/Transformer.tsx: -------------------------------------------------------------------------------- 1 | /* eslint-disable @next/next/no-img-element */ 2 | import { Play, Trash2, Lock, Unlock, Repeat } from "lucide-react"; 3 | import { memo } from "react"; 4 | import { 5 | Handle, 6 | Position, 7 | NodeProps, 8 | NodeToolbar, 9 | getIncomers, 10 | getConnectedEdges, 11 | useUpdateNodeInternals, 12 | Node, 13 | } from "reactflow"; 14 | import "reactflow/dist/style.css"; 15 | import { Nodes } from "."; 16 | import { Label } from "../Label"; 17 | import { 18 | NumberVariable, 19 | Output, 20 | Outputs, 21 | Panel, 22 | TextVariable, 23 | Toolbar, 24 | ToolButton, 25 | Variables, 26 | } from "../Node"; 27 | 28 | export type Transformer = NodeProps<{ 29 | locked: boolean; 30 | running: boolean; 31 | repeating: boolean; 32 | input: { 33 | prompt: string; 34 | temperature: number; 35 | top_p: number; 36 | frequency_penalty: number; 37 | }; 38 | output: { 39 | prediction: string; 40 | }; 41 | }>; 42 | 43 | export function Transformer(node: Transformer) { 44 | const { editNode, deleteNode } = Nodes.use((state) => ({ 45 | editNode: state.editNode, 46 | deleteNode: state.deleteNode, 47 | })); 48 | 49 | return ( 50 | 51 | 52 | {!node.data.running && ( 53 | { 55 | Nodes.resolveNode(node.id); 56 | }} 57 | > 58 | 59 | 60 | )} 61 | { 63 | if (node.data.repeating) { 64 | editNode(node.id, { 65 | repeating: false, 66 | }); 67 | } else { 68 | Nodes.resolveNode(node.id, true); 69 | } 70 | }} 71 | active={node.data.repeating} 72 | > 73 | 74 | 75 | 77 | editNode(node.id, { 78 | locked: !node.data.locked, 79 | }) 80 | } 81 | active={node.data.locked} 82 | > 83 | {node.data.locked ? : } 84 | 85 | deleteNode(node.id)}> 86 | 87 | 88 | 89 | 90 | 91 | 97 | 103 | 109 | 115 | 116 | 117 | 118 | 125 | 126 | 127 | ); 128 | } 129 | 130 | export namespace Transformer { 131 | export async function run(node: Node): Promise { 132 | const { prompt, temperature, top_p, frequency_penalty } = node.data.input; 133 | 134 | if (!prompt) { 135 | if (node.data.repeating) { 136 | // disable repeating 137 | Nodes.use.getState().editNode(node.id, { 138 | repeating: false, 139 | }); 140 | } 141 | return { 142 | prediction: "", 143 | }; 144 | } 145 | 146 | const response = await fetch("https://api.prototyped.ai/text", { 147 | method: "POST", 148 | headers: { 149 | "Content-Type": "application/json", 150 | }, 151 | body: JSON.stringify({ 152 | prompt, 153 | temperature, 154 | top_p, 155 | frequency_penalty, 156 | }), 157 | }); 158 | 159 | const json = await response.json(); 160 | 161 | return { 162 | prediction: json.choices.pop().text, 163 | }; 164 | } 165 | 166 | export const Memo = memo(Transformer); 167 | } 168 | -------------------------------------------------------------------------------- /components/Bar.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | Dice2, 3 | File, 4 | FileType2, 5 | ImagePlus, 6 | Regex, 7 | TextCursorInput, 8 | View, 9 | } from "lucide-react"; 10 | import { useReactFlow } from "reactflow"; 11 | 12 | export type Bar = { 13 | onCreateNode: (newNode: { type: string; data: any; position: any }) => void; 14 | }; 15 | 16 | export function Bar({ onCreateNode }: Bar) { 17 | const flow = useReactFlow(); 18 | 19 | return ( 20 |
21 | 45 | 69 | 91 | 113 | 136 | 155 | 176 |
177 | ); 178 | } 179 | 180 | function Button({ 181 | children, 182 | active, 183 | onClick, 184 | }: { 185 | children: React.ReactNode; 186 | active?: boolean; 187 | onClick?: () => void; 188 | }) { 189 | return ( 190 | 198 | ); 199 | } 200 | -------------------------------------------------------------------------------- /components/Nodes/Image.tsx: -------------------------------------------------------------------------------- 1 | /* eslint-disable @next/next/no-img-element */ 2 | import { ImageIcon, Lock, Play, Repeat, Trash2, Unlock } from "lucide-react"; 3 | import { memo, useEffect, useMemo, useRef, useState } from "react"; 4 | import { 5 | Handle, 6 | Position, 7 | NodeProps, 8 | NodeToolbar, 9 | getIncomers, 10 | getConnectedEdges, 11 | Node, 12 | } from "reactflow"; 13 | import "reactflow/dist/style.css"; 14 | import { Nodes } from "."; 15 | import { Label } from "../Label"; 16 | import { 17 | Content, 18 | ImageVariable, 19 | NumberVariable, 20 | Output, 21 | Outputs, 22 | Panel, 23 | TextVariable, 24 | Toolbar, 25 | ToolButton, 26 | Variables, 27 | } from "../Node"; 28 | 29 | export type Image = NodeProps<{ 30 | locked: boolean; 31 | running: boolean; 32 | repeating: boolean; 33 | input: { 34 | init: string; 35 | prompt: string; 36 | steps: number; 37 | cfg_scale: number; 38 | }; 39 | output: { 40 | image: string; 41 | }; 42 | }>; 43 | 44 | export function Image(node: Image) { 45 | const { editNode, deleteNode } = Nodes.use((state) => ({ 46 | editNode: state.editNode, 47 | deleteNode: state.deleteNode, 48 | })); 49 | 50 | const [loaded, setLoaded] = useState(false); 51 | 52 | useEffect(() => { 53 | if (!node.data.output.image) { 54 | setLoaded(false); 55 | } 56 | }, [node.data.output.image]); 57 | 58 | return ( 59 | 64 | 65 | {!node.data.running && ( 66 | { 68 | Nodes.resolveNode(node.id); 69 | }} 70 | > 71 | 72 | 73 | )} 74 | { 76 | if (node.data.repeating) { 77 | editNode(node.id, { 78 | repeating: false, 79 | }); 80 | } else { 81 | Nodes.resolveNode(node.id, true); 82 | } 83 | }} 84 | active={node.data.repeating} 85 | > 86 | 87 | 88 | 90 | editNode(node.id, { 91 | locked: !node.data.locked, 92 | }) 93 | } 94 | active={node.data.locked} 95 | > 96 | {node.data.locked ? : } 97 | 98 | deleteNode(node.id)}> 99 | 100 | 101 | 102 | 103 | 104 | {node.data.output.image && ( 105 | graph image setLoaded(true)} 110 | /> 111 | )} 112 | {!node.data.output.image && !loaded && ( 113 |
114 | 115 |

Image

116 |
117 | )} 118 | 119 | 120 | 121 | 122 | 128 | 134 | 140 | 141 | 142 | 143 | 144 | 145 | ); 146 | } 147 | 148 | export namespace Image { 149 | export async function run(node: Node): Promise { 150 | const { input } = node.data; 151 | const { init, steps, cfg_scale, prompt } = input; 152 | 153 | if (!prompt) { 154 | if (node.data.repeating) { 155 | // disable repeating 156 | Nodes.use.getState().editNode(node.id, { 157 | repeating: false, 158 | }); 159 | } 160 | return { 161 | image: node.data.output.image, 162 | }; 163 | } 164 | 165 | let data: any = {}; 166 | if (init && init.length > 0) { 167 | // init is probably a dataurl or a regular image. Need to fetch and convert to base64 168 | const init_d = await fetch(init); 169 | const init_b = await init_d.blob(); 170 | const init_b64 = await new Promise((resolve) => { 171 | const reader = new FileReader(); 172 | reader.onloadend = () => { 173 | resolve(reader.result); 174 | }; 175 | reader.readAsDataURL(init_b); 176 | }); 177 | 178 | data = await fetch("https://api.prototyped.ai/init", { 179 | method: "POST", 180 | body: JSON.stringify({ 181 | steps, 182 | scale: cfg_scale, 183 | prompt, 184 | count: 1, 185 | init: init_b64, 186 | }), 187 | }).then((res) => res.json()); 188 | } else { 189 | data = await fetch("https://api.prototyped.ai/image", { 190 | method: "POST", 191 | body: JSON.stringify({ 192 | steps, 193 | scale: cfg_scale, 194 | prompt, 195 | count: 1, 196 | }), 197 | }).then((res) => res.json()); 198 | } 199 | 200 | return { 201 | image: data[0].image, 202 | }; 203 | } 204 | 205 | export const Memo = memo(Image); 206 | } 207 | -------------------------------------------------------------------------------- /components/Nodes/index.tsx: -------------------------------------------------------------------------------- 1 | import { Image } from "./Image"; 2 | import { Transformer } from "./Transformer"; 3 | import { RandomNumber } from "./RandomNumber"; 4 | import { Concat } from "./Concat"; 5 | import { RegexReplace } from "./RegexReplace"; 6 | import { LoadImage } from "./LoadImage"; 7 | import { Interrogate } from "./Interrogate"; 8 | import create from "zustand"; 9 | import { 10 | Connection, 11 | Edge, 12 | EdgeChange, 13 | Node, 14 | NodeChange, 15 | addEdge, 16 | OnNodesChange, 17 | OnEdgesChange, 18 | OnConnect, 19 | applyNodeChanges, 20 | applyEdgeChanges, 21 | getIncomers, 22 | getConnectedEdges, 23 | } from "reactflow"; 24 | 25 | export type NodesState = { 26 | nodes: Node[]; 27 | edges: Edge[]; 28 | onNodesChange: OnNodesChange; 29 | onEdgesChange: OnEdgesChange; 30 | onConnect: OnConnect; 31 | addNode: (node: Node) => void; 32 | editNode: (id: string, newData: any) => void; 33 | deleteNode: (id: string) => void; 34 | setEdgeAnimationState: (id: string, state: boolean) => void; 35 | }; 36 | 37 | export declare namespace Nodes { 38 | export { 39 | Image, 40 | Transformer, 41 | RandomNumber, 42 | Concat, 43 | RegexReplace, 44 | LoadImage, 45 | Interrogate, 46 | }; 47 | } 48 | 49 | export namespace Nodes { 50 | Nodes.Image = Image; 51 | Nodes.Transformer = Transformer; 52 | Nodes.RandomNumber = RandomNumber; 53 | Nodes.Concat = Concat; 54 | Nodes.RegexReplace = RegexReplace; 55 | Nodes.LoadImage = LoadImage; 56 | Nodes.Interrogate = Interrogate; 57 | 58 | export const nodeTypes = { 59 | Image: Nodes.Image.Memo, 60 | Transformer: Nodes.Transformer.Memo, 61 | RandomNumber: Nodes.RandomNumber.Memo, 62 | Concat: Nodes.Concat.Memo, 63 | RegexReplace: Nodes.RegexReplace.Memo, 64 | LoadImage: Nodes.LoadImage.Memo, 65 | Interrogate: Nodes.Interrogate.Memo, 66 | }; 67 | 68 | export const use = create((set, get) => ({ 69 | nodes: [], 70 | edges: [], 71 | onNodesChange: (changes: NodeChange[]) => { 72 | set({ 73 | nodes: applyNodeChanges(changes, get().nodes), 74 | }); 75 | }, 76 | onEdgesChange: (changes: EdgeChange[]) => { 77 | set({ 78 | edges: applyEdgeChanges(changes, get().edges), 79 | }); 80 | }, 81 | onConnect: (connection: Connection) => { 82 | set({ 83 | edges: addEdge(connection, get().edges), 84 | }); 85 | }, 86 | addNode: (node: Node) => { 87 | set({ 88 | nodes: [...get().nodes, node], 89 | }); 90 | }, 91 | editNode: (id: string, newData: any) => { 92 | set({ 93 | nodes: get().nodes.map((node) => { 94 | if (node.id === id) { 95 | return { 96 | ...node, 97 | data: { 98 | ...node.data, 99 | ...newData, 100 | }, 101 | }; 102 | } 103 | return node; 104 | }), 105 | }); 106 | }, 107 | deleteNode: (id: string) => { 108 | set({ 109 | nodes: get().nodes.filter((node) => node.id !== id), 110 | edges: get().edges.filter( 111 | (edge) => edge.source !== id && edge.target !== id 112 | ), 113 | }); 114 | }, 115 | setEdgeAnimationState: (id: string, state: boolean) => { 116 | set({ 117 | edges: get().edges.map((edge) => { 118 | if (edge.id === id) { 119 | return { 120 | ...edge, 121 | animated: state, 122 | }; 123 | } 124 | return edge; 125 | }), 126 | }); 127 | }, 128 | })); 129 | 130 | export async function runNode(node: Node): Promise { 131 | switch (node.type) { 132 | case "Image": 133 | return Nodes.Image.run(node); 134 | case "Transformer": 135 | return Nodes.Transformer.run(node); 136 | case "RandomNumber": 137 | return Nodes.RandomNumber.run(node); 138 | case "Concat": 139 | return Nodes.Concat.run(node); 140 | case "RegexReplace": 141 | return Nodes.RegexReplace.run(node); 142 | case "LoadImage": 143 | return Nodes.LoadImage.run(node); 144 | case "Interrogate": 145 | return Nodes.Interrogate.run(node); 146 | default: 147 | throw new Error(`Node type ${node.type} not found`); 148 | } 149 | } 150 | 151 | export async function resolveNode( 152 | nodeid: string, 153 | repeat?: boolean 154 | ): Promise { 155 | const node = Nodes.use.getState().nodes.find((node) => node.id === nodeid); 156 | 157 | if (!node) { 158 | throw new Error(`Node ${nodeid} not found`); 159 | } 160 | 161 | // set node repeating to true 162 | if (repeat) { 163 | Nodes.use.getState().editNode(nodeid, { repeating: true }); 164 | 165 | while (true) { 166 | await resolveSingleNode(nodeid); 167 | 168 | // check if node is still repeating 169 | const node = Nodes.use 170 | .getState() 171 | .nodes.find((node) => node.id === nodeid); 172 | 173 | if (!node || !node.data.repeating) { 174 | break; 175 | } 176 | } 177 | } else { 178 | resolveSingleNode(nodeid); 179 | } 180 | } 181 | 182 | export async function resolveSingleNode(nodeid: string): Promise { 183 | // get the node 184 | const node = Nodes.use.getState().nodes.find((node) => node.id === nodeid); 185 | 186 | if (!node) { 187 | throw new Error(`Node ${nodeid} not found`); 188 | } 189 | 190 | const { nodes, edges } = Nodes.use.getState(); 191 | 192 | let sourceNodes: Node[] = []; 193 | try { 194 | sourceNodes = getIncomers(node, nodes, edges); 195 | } catch (e) { 196 | if (e instanceof RangeError) { 197 | console.log( 198 | `Node loop detected. Please check your connections and try again.` 199 | ); 200 | } 201 | return; 202 | } 203 | 204 | // check if we are in an infinite loop 205 | if (sourceNodes.find((node) => node.id === nodeid)) { 206 | console.log( 207 | `Node loop detected. Please check your connections and try again.` 208 | ); 209 | return; 210 | } 211 | 212 | const sourcePromises = sourceNodes.map((node) => { 213 | console.log(`Resolving source node ${node.id}`); 214 | return resolveSingleNode(node.id); 215 | }); 216 | 217 | if (sourcePromises.length) { 218 | console.log(`Waiting for ${sourcePromises} sources to resolve`); 219 | await Promise.all(sourcePromises); 220 | console.log(`Sources resolved`); 221 | } 222 | 223 | // probably need to refetch sources here, in case they changed 224 | const newSources = Nodes.use.getState(); 225 | const sourceNodes2 = getIncomers(node, newSources.nodes, newSources.edges); 226 | const sourceEdges2 = getConnectedEdges([node], newSources.edges); 227 | 228 | // populate the node's input with the output of the sources 229 | const input = sourceEdges2.reduce((acc, edge) => { 230 | const sourceNode = sourceNodes2.find((node) => node.id === edge.source); 231 | if (sourceNode) { 232 | // @ts-ignore 233 | acc[edge.targetHandle.split("-").pop()] = 234 | // @ts-ignore 235 | sourceNode.data.output[edge.sourceHandle.split("-").pop()]; 236 | } 237 | return acc; 238 | }, {}); 239 | 240 | // set the node's input 241 | Nodes.use.getState().editNode(nodeid, { 242 | input: { 243 | ...node.data.input, 244 | ...input, 245 | }, 246 | running: true, 247 | }); 248 | 249 | // animate the edges 250 | sourceEdges2.forEach((edge) => { 251 | Nodes.use.getState().setEdgeAnimationState(edge.id, true); 252 | }); 253 | 254 | // get the node again 255 | const node2 = Nodes.use.getState().nodes.find((node) => node.id === nodeid); 256 | 257 | // run the node 258 | const output = 259 | node2 && !node2.data.locked ? await runNode(node2) : node.data.output; 260 | 261 | // stop animating the edges 262 | sourceEdges2.forEach((edge) => { 263 | Nodes.use.getState().setEdgeAnimationState(edge.id, false); 264 | }); 265 | 266 | // update the node's output 267 | Nodes.use.getState().editNode(nodeid, { output, running: false }); 268 | 269 | console.log( 270 | "returning output for node", 271 | nodeid, 272 | output, 273 | "had input", 274 | input 275 | ); 276 | 277 | return output; 278 | } 279 | } 280 | -------------------------------------------------------------------------------- /components/Node.tsx: -------------------------------------------------------------------------------- 1 | import { Trash2 } from "lucide-react"; 2 | import { useEffect, useRef } from "react"; 3 | import { 4 | getConnectedEdges, 5 | Handle, 6 | NodeToolbar, 7 | Position, 8 | useUpdateNodeInternals, 9 | } from "reactflow"; 10 | import { Label } from "./Label"; 11 | import { Nodes } from "./Nodes"; 12 | 13 | export function Panel({ 14 | children, 15 | name, 16 | running, 17 | selected, 18 | }: { 19 | children?: React.ReactNode; 20 | name: string; 21 | running?: boolean; 22 | selected?: boolean; 23 | }) { 24 | return ( 25 |
34 |
35 |

{name}

36 |
37 | {children} 38 |
39 | ); 40 | } 41 | 42 | export function Variables({ children }: { children?: React.ReactNode }) { 43 | return ( 44 |
45 |
46 |

Variables

47 |
48 |
{children}
49 |
50 | ); 51 | } 52 | 53 | export function Outputs({ children }: { children?: React.ReactNode }) { 54 | return ( 55 |
56 |
57 |

Outputs

58 |
59 |
{children}
60 |
61 | ); 62 | } 63 | 64 | export function Content({ children }: { children?: React.ReactNode }) { 65 | return
{children}
; 66 | } 67 | 68 | export function Toolbar({ 69 | children, 70 | show, 71 | }: { 72 | children?: React.ReactNode; 73 | show: boolean; 74 | }) { 75 | return ( 76 | 77 |
78 | {children} 79 |
80 | 81 | ); 82 | } 83 | 84 | export function ToolButton({ 85 | children, 86 | onClick, 87 | active, 88 | }: { 89 | children: React.ReactNode; 90 | onClick: () => void; 91 | active?: boolean; 92 | }) { 93 | return ( 94 | 102 | ); 103 | } 104 | 105 | export function NumberVariable({ 106 | value, 107 | nodeID, 108 | label, 109 | name, 110 | }: { 111 | value: number; 112 | nodeID: string; 113 | label: string; 114 | name: string; 115 | }) { 116 | const { nodes, editNode, edges } = Nodes.use((state) => { 117 | return { 118 | nodes: state.nodes, 119 | editNode: state.editNode, 120 | edges: state.edges, 121 | }; 122 | }); 123 | 124 | const node = nodes.find((node) => node.id === nodeID); 125 | 126 | const updateNodeInternals = useUpdateNodeInternals(); 127 | 128 | const labelRef = useRef(null); 129 | const handleRef = useRef(null); 130 | 131 | useEffect(() => { 132 | if (labelRef.current && handleRef.current) { 133 | // move the location of the handle to be relative to the label 134 | handleRef.current.style.position = "abolute"; 135 | handleRef.current.style.top = `0.6rem`; 136 | handleRef.current.style.left = `-1rem`; 137 | 138 | // update the node internals 139 | updateNodeInternals(nodeID); 140 | } 141 | }, [labelRef, handleRef, updateNodeInternals, nodeID]); 142 | 143 | // get connections 144 | const edgeConnections = node ? getConnectedEdges([node], edges) : []; 145 | 146 | // check if its connected 147 | const isConnected = 148 | edgeConnections.find( 149 | (edge) => edge.targetHandle === `input-${name}` && edge.target === nodeID 150 | ) !== undefined; 151 | 152 | return ( 153 |
157 |