├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── generator9000_app ├── .eslintrc.json ├── .gitignore ├── app │ ├── actions.ts │ ├── components │ │ ├── data_actions.ts │ │ ├── datafield.tsx │ │ ├── export_modal.tsx │ │ ├── generation_menu.tsx │ │ ├── generation_pod.tsx │ │ ├── inspect.tsx │ │ ├── inspect_weaviate.tsx │ │ ├── navbar.tsx │ │ ├── object_card.tsx │ │ ├── prompt_menu.tsx │ │ ├── server_actions.ts │ │ ├── settings_modal.tsx │ │ ├── types.ts │ │ └── weaviate_connector.tsx │ ├── globals.css │ ├── icon.png │ ├── layout.tsx │ └── page.tsx ├── next.config.mjs ├── package-lock.json ├── package.json ├── postcss.config.js ├── public │ ├── catwork.gif │ ├── gatitos-gatos.gif │ ├── generator_9000.riv │ ├── justacat.gif │ ├── maxwell.wav │ ├── next.svg │ ├── vercel.svg │ └── weaviate_logo.svg ├── tailwind.config.ts └── tsconfig.json └── img └── gen9000.gif /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # v0.2.0 (Open Source Release) 2 | 3 | ## Added 4 | - Clear API Key 5 | - Easter Egg 6 | - Add Weaviate Connector 7 | - Inspect Weaviate Objects 8 | - Add data to existing collections 9 | - Add new collections 10 | - Improved secure cookie handling 11 | - Improved environment variable handling 12 | 13 | ## Fixed 14 | 15 | - Improve Error Messaging 16 | 17 | ## Changed 18 | - UI Changes for improved UX 19 | - Fix colors 20 | - Transform settings into dropdown 21 | - File Input as Button 22 | - Make everything a bit smaller 23 | 24 | # v0.1.0 (Open Source Release) 25 | 26 | ## Added 27 | - Open Source release! 28 | 29 | ## Fixed 30 | 31 | ## Changed -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020-2024, Weaviate B.V. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of the copyright holder nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🐈 Welcome to Generator 9000 2 | ## ✨ Your Starting Point for AI-Synthetic Data Generation 3 | 4 | [![Start generating](https://img.shields.io/badge/Check%20out%20the%20app!-yellow?&style=flat-square&logo=react&logoColor=white)](https://www.gen9000.co/?template=Empty) 5 | 6 | ![Demo of Generator9000](https://github.com/weaviate/generator9000/blob/main/img/gen9000.gif) 7 | 8 | Welcome to Generator9000! It's an open-source web app that's all about creating synthetic data objects tailored to your use case. We use Generative AI (GPT4 from OpenAI) to not only generate data objects with specific fields but also to create images based on these data objects. 🚀 9 | 10 | This tool is perfect if you're looking to build new, high-quality datasets for demos or Proof of Concept (PoC) projects, especially in these fast-evolving times. It's user-friendly, giving you full control over the prompts for both data and images, and other settings that directly affect the Large Language Models (LLM) you're working with. 11 | 12 | You can tinker with all the settings, generate data objects, evaluate them, and then save your creations either on disk or in Weaviate. 📁✨ 13 | 14 | **Access Generator9000 here:** https://www.gen9000.co/ 15 | 16 | > 🔑 You can enter your OpenAI key right in the frontend. It'll be saved as a httpOnly and secure cookie, making it super easy to pick up right where you left off next time you fire up the app. If you don't feel comfortable giving random websites your credentials, you can also host Generator9000 locally! 17 | 18 | # 🚀 Getting Started 19 | 20 | Before you dive in, make sure your machine is running Node `v21.3.0` or higher. Here’s how you can get Generator9000 up and running: 21 | 22 | 1. Clone the repository: 23 | 24 | ``` git clone https://github.com/weaviate/generator9000 ``` 25 | 26 | 2. Navigate to the app directory: 27 | 28 | ``` cd generator9000_app ``` 29 | 30 | 3. Install the necessary packages: 31 | 32 | ``` npm install ``` 33 | 34 | 4. Create a `.env.local` file and add your OpenAI API key. If you want to use Weaviate, you can also add Weaviate variables: 35 | 36 | ``` OPENAI_API_KEY=YOUR_KEY ``` 37 | ``` WEAVIATE_URL=YOUR_URL ``` 38 | ``` WEAVIATE_API_KEY=YOUR_KEY ``` 39 | 40 | 5. Start the development server and start generating over 9000: 41 | 42 | ``` npm run dev ``` 43 | 44 | 45 | That's it! 🎉 Create, experiment, and have fun creating your very own synthetic datasets. Happy generating! 🌟 46 | -------------------------------------------------------------------------------- /generator9000_app/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals" 3 | } 4 | -------------------------------------------------------------------------------- /generator9000_app/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | .yarn/install-state.gz 8 | 9 | # testing 10 | /coverage 11 | 12 | # next.js 13 | /.next/ 14 | /out/ 15 | 16 | # production 17 | /build 18 | 19 | # misc 20 | .DS_Store 21 | *.pem 22 | 23 | # debug 24 | npm-debug.log* 25 | yarn-debug.log* 26 | yarn-error.log* 27 | 28 | # local env files 29 | .env*.local 30 | 31 | # vercel 32 | .vercel 33 | 34 | # typescript 35 | *.tsbuildinfo 36 | next-env.d.ts 37 | -------------------------------------------------------------------------------- /generator9000_app/app/actions.ts: -------------------------------------------------------------------------------- 1 | 'use server' 2 | 3 | import weaviate, { WeaviateClient, ObjectsBatcher, ApiKey } from 'weaviate-ts-client'; 4 | import { Template, DataField } from './components/types' 5 | import { cookies } from 'next/headers' 6 | 7 | export async function clear_weaviate_cookies() { 8 | 9 | cookies().delete('weaviate_url') 10 | cookies().delete('weaviate_key') 11 | 12 | } 13 | 14 | export async function get_API_Status() { 15 | 16 | const cookieStore = cookies() 17 | const api_cookie = cookieStore.get('Generator9000_APIKEY')?.value 18 | 19 | if (api_cookie) { 20 | return true 21 | } else { 22 | return !!process.env.OPENAI_API_KEY; 23 | } 24 | } 25 | 26 | export async function set_api_key(api: string) { 27 | cookies().set({ 28 | name: 'Generator9000_APIKEY', 29 | value: api, 30 | httpOnly: true, 31 | secure: true, 32 | path: '/', 33 | sameSite: "strict", 34 | maxAge: 3600 35 | }) 36 | } 37 | 38 | export async function remove_api_key() { 39 | cookies().delete('Generator9000_APIKEY') 40 | } 41 | 42 | export async function connect_weaviate(_url: string, _key: string) { 43 | try { 44 | 45 | const cookieStore = cookies() 46 | const openai_key = cookieStore.get('Generator9000_APIKEY')?.value || process.env.OPENAI_API_KEY || "" 47 | const cookie_url = cookieStore.get('weaviate_url')?.value || process.env.WEAVIATE_URL || "" 48 | const cookie_key = cookieStore.get('weaviate_key')?.value || process.env.WEAVIATE_API_KEY || "" 49 | 50 | let choosen_url = _url 51 | let choosen_key = _key 52 | 53 | if (_key === "" || _url === "") { 54 | if (cookie_url && cookie_key) { 55 | choosen_url = cookie_url 56 | choosen_key = cookie_key 57 | } else { 58 | return { "templates": [], "error": "", "connected": false }; 59 | } 60 | } 61 | 62 | const client: WeaviateClient = weaviate.client({ 63 | scheme: 'https', 64 | host: choosen_url, 65 | apiKey: new ApiKey(choosen_key), 66 | headers: { 'X-OpenAI-Api-Key': openai_key }, 67 | }); 68 | 69 | let allClassDefinitions = await client.schema.getter().do(); 70 | 71 | // Initialize templates array with the "Create new" template at the beginning 72 | let templates: Template[] = [{ 73 | name: "New Collection", 74 | prompt: "", 75 | imagePrompt: "", 76 | datafields: [] 77 | }]; 78 | 79 | if (allClassDefinitions.classes && allClassDefinitions.classes.length > 0) { 80 | // Generate templates for all class definitions 81 | const classTemplates = await Promise.all( 82 | allClassDefinitions.classes.map(classDef => createTemplate(client, classDef.class)) 83 | ); 84 | // Append the generated templates after the "Create new" template 85 | templates = templates.concat(classTemplates); 86 | } 87 | 88 | if (_key != "" || _url != "") { 89 | cookies().set({ 90 | name: 'weaviate_url', 91 | value: _url, 92 | httpOnly: true, 93 | secure: true, 94 | path: '/', 95 | sameSite: "strict", 96 | maxAge: 3600 97 | 98 | }) 99 | cookies().set({ 100 | name: 'weaviate_key', 101 | value: _key, 102 | httpOnly: true, 103 | secure: true, 104 | path: '/', 105 | sameSite: "strict", 106 | maxAge: 3600 107 | }) 108 | } 109 | 110 | return { templates, "error": "", "connected": true }; 111 | } catch (e) { 112 | console.error("Error fetching: " + e); 113 | return { "templates": [], "error": "Error fetching: " + e, "connected": false }; 114 | } 115 | } 116 | 117 | export async function get_weaviate_data(collectionName: string, dataFields: DataField[], page: number) { 118 | try { 119 | 120 | const cookieStore = cookies() 121 | const openai_key = cookieStore.get('Generator9000_APIKEY')?.value || process.env.OPENAI_API_KEY || "" 122 | const _url = cookieStore.get('weaviate_url')?.value || process.env.WEAVIATE_URL || "" 123 | const _key = cookieStore.get('weaviate_key')?.value || process.env.WEAVIATE_API_KEY || "" 124 | 125 | const offset = 12 * (page - 1) 126 | const fields = dataFields.map(field => field.name).join(' '); 127 | 128 | const client: WeaviateClient = weaviate.client({ 129 | scheme: 'https', 130 | host: _url, 131 | apiKey: new ApiKey(_key), 132 | headers: { 'X-OpenAI-Api-Key': openai_key }, 133 | }); 134 | 135 | const query = await client.graphql.get() 136 | .withClassName(collectionName) 137 | .withFields(fields + " _additional { id }") 138 | .withOffset(offset) 139 | .withLimit(12) 140 | .do() 141 | 142 | const count_result = await client 143 | .graphql 144 | .aggregate() 145 | .withClassName(collectionName) 146 | .withFields('meta { count }') 147 | .do(); 148 | 149 | if (query && count_result) { 150 | 151 | const data = query["data"]["Get"][collectionName] 152 | const count = count_result["data"]["Aggregate"][collectionName][0]["meta"]["count"] 153 | 154 | data.forEach((item: any) => { 155 | // Check if the item does not have an 'id' key or if the 'id' key is somehow falsy 156 | if (!item.id) { 157 | // If 'id' key is missing or falsy, set it using the '_additional' dictionary's 'id' 158 | item.id = item._additional.id; 159 | } 160 | }); 161 | 162 | 163 | if (data) { 164 | return { data: data, "error": "", count: count }; 165 | } else { 166 | return { data: [], "error": "No data", count: 0 }; 167 | } 168 | } else { 169 | return { data: [], "error": "No data", count: 0 }; 170 | } 171 | 172 | } catch (e) { 173 | console.error("Error fetching: " + e); 174 | return { "data": [], "error": "Error fetching: " + e, count: 0 }; 175 | } 176 | } 177 | 178 | function createPropertiesList(dictionary: any, selected_image_key: string) { 179 | let properties = []; 180 | 181 | for (const [key, value] of Object.entries(dictionary)) { 182 | let property: any = { 183 | name: key, 184 | dataType: getDataType(value), // Call a function to determine the dataType 185 | }; 186 | 187 | // Check if the current key is the selected_image_key 188 | if (key === selected_image_key) { 189 | property.moduleConfig = { 190 | 'text2vec-openai': { skip: true }, 191 | }; 192 | // Ensure dataType is ['text'] for the selected_image_key 193 | property.dataType = ['text']; 194 | } 195 | 196 | properties.push(property); 197 | } 198 | 199 | return properties; 200 | } 201 | 202 | function getDataType(value: any) { 203 | if (Array.isArray(value)) { 204 | // Check the first element to determine the type inside the array 205 | const elementType = typeof value[0]; 206 | return elementType === 'string' ? ['text[]'] : ['number[]']; 207 | } else { 208 | return typeof value === 'string' ? ['text'] : ['number']; 209 | } 210 | } 211 | 212 | export async function import_weaviate_data(collectionName: string, data: any, selectedTemplate: string, selected_img: string, prompt: string, imagePrompt: string) { 213 | try { 214 | 215 | const cookieStore = cookies() 216 | const openai_key = cookieStore.get('Generator9000_APIKEY')?.value || process.env.OPENAI_API_KEY || "" 217 | const _url = cookieStore.get('weaviate_url')?.value || process.env.WEAVIATE_URL || "" 218 | const _key = cookieStore.get('weaviate_key')?.value || process.env.WEAVIATE_API_KEY || "" 219 | 220 | const client: WeaviateClient = weaviate.client({ 221 | scheme: 'https', 222 | host: _url, 223 | apiKey: new ApiKey(_key), 224 | headers: { 'X-OpenAI-Api-Key': openai_key }, 225 | }); 226 | 227 | const schema = await client.schema.exists(collectionName); 228 | 229 | if (schema && selectedTemplate != "New Collection") { 230 | const result = await client.data 231 | .creator() 232 | .withClassName(collectionName) 233 | .withProperties(data) 234 | .do(); 235 | 236 | 237 | if (result) { 238 | return { "error": "" } 239 | } else { 240 | return { "error": "Could not create new data object" } 241 | } 242 | } else if (!schema && selectedTemplate != collectionName) { 243 | 244 | const properties = createPropertiesList(data, selected_img) 245 | 246 | const new_schema = { 247 | class: collectionName, 248 | description: prompt + " / " + imagePrompt, 249 | vectorizer: 'text2vec-openai', 250 | properties: properties 251 | } 252 | 253 | const res = await client 254 | .schema.classCreator() 255 | .withClass(new_schema) 256 | .do(); 257 | 258 | if (res) { 259 | const result = await client.data 260 | .creator() 261 | .withClassName(collectionName) 262 | .withProperties(data) 263 | .do(); 264 | 265 | 266 | if (result) { 267 | return { "error": "" } 268 | } else { 269 | return { "error": "Could not create new data object" } 270 | } 271 | } 272 | 273 | return { "error": "Could not create a new schema" } 274 | 275 | } else { 276 | return { "error": "Schema " + collectionName + " already exists" }; 277 | } 278 | 279 | 280 | 281 | } catch (e) { 282 | console.error("Error at importing: " + e); 283 | return { "error": "Error ingesting: " + e }; 284 | } 285 | } 286 | 287 | async function createTemplate(client: WeaviateClient, className: any): Promise