├── .gitattributes ├── .gitignore ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── Troubleshooting.md ├── backend ├── .gitignore ├── .pre-commit-config.yaml ├── Dockerfile ├── README.md ├── access_token.py ├── build.sh ├── config.py ├── evals │ ├── __init__.py │ ├── config.py │ ├── core.py │ └── utils.py ├── image_generation.py ├── llm.py ├── main.py ├── mock_llm.py ├── poetry.lock ├── prompts │ ├── __init__.py │ ├── imported_code_prompts.py │ ├── screenshot_system_prompts.py │ ├── test_prompts.py │ └── types.py ├── pyproject.toml ├── pyrightconfig.json ├── routes │ ├── evals.py │ ├── generate_code.py │ ├── home.py │ └── screenshot.py ├── run_evals.py ├── start.py └── utils.py ├── design-docs.md ├── docker-compose.yml ├── frontend ├── .env.example ├── .eslintrc.cjs ├── .gitignore ├── Dockerfile ├── components.json ├── index.html ├── package.json ├── postcss.config.js ├── public │ └── brand │ │ └── twitter-summary-card.png ├── src │ ├── App.tsx │ ├── components │ │ ├── CodeMirror.tsx │ │ ├── CodePreview.tsx │ │ ├── CodeTab.tsx │ │ ├── ImageUpload.tsx │ │ ├── ImportCodeSection.tsx │ │ ├── OnboardingNote.tsx │ │ ├── OutputSettingsSection.tsx │ │ ├── PicoBadge.tsx │ │ ├── Preview.tsx │ │ ├── SettingsDialog.tsx │ │ ├── Spinner.tsx │ │ ├── TermsOfServiceDialog.tsx │ │ ├── UrlInputSection.tsx │ │ ├── evals │ │ │ ├── EvalsPage.tsx │ │ │ └── RatingPicker.tsx │ │ ├── history │ │ │ ├── HistoryDisplay.tsx │ │ │ ├── history_types.ts │ │ │ ├── utils.test.ts │ │ │ └── utils.ts │ │ ├── settings │ │ │ └── AccessCodeSection.tsx │ │ └── ui │ │ │ ├── accordion.tsx │ │ │ ├── alert-dialog.tsx │ │ │ ├── badge.tsx │ │ │ ├── button.tsx │ │ │ ├── checkbox.tsx │ │ │ ├── collapsible.tsx │ │ │ ├── dialog.tsx │ │ │ ├── hover-card.tsx │ │ │ ├── input.tsx │ │ │ ├── label.tsx │ │ │ ├── popover.tsx │ │ │ ├── progress.tsx │ │ │ ├── scroll-area.tsx │ │ │ ├── select.tsx │ │ │ ├── separator.tsx │ │ │ ├── switch.tsx │ │ │ ├── tabs.tsx │ │ │ └── textarea.tsx │ ├── config.ts │ ├── constants.ts │ ├── generateCode.ts │ ├── hooks │ │ ├── usePersistedState.ts │ │ └── useThrottle.ts │ ├── index.css │ ├── lib │ │ ├── stacks.ts │ │ └── utils.ts │ ├── main.tsx │ ├── types.ts │ └── vite-env.d.ts ├── tailwind.config.js ├── tsconfig.json ├── tsconfig.node.json ├── vite.config.ts └── yarn.lock └── sweep.yaml /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .aider* 2 | 3 | # Project-related files 4 | 5 | # Run logs 6 | backend/run_logs/* 7 | 8 | # Weird Docker setup related files 9 | backend/backend/* 10 | 11 | # Env vars 12 | frontend/.env.local 13 | .env 14 | 15 | # Mac files 16 | .DS_Store 17 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.analysis.typeCheckingMode": "strict" 3 | } 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Abi Raja 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # screenshot-to-code 2 | 3 | This simple app converts a screenshot to code (HTML/Tailwind CSS, or React or Bootstrap or Vue). It uses GPT-4 Vision to generate the code and DALL-E 3 to generate similar-looking images. You can now also enter a URL to clone a live website! 4 | 5 | https://github.com/abi/screenshot-to-code/assets/23818/6cebadae-2fe3-4986-ac6a-8fb9db030045 6 | 7 | See the [Examples](#-examples) section below for more demos. 8 | 9 | ## 🚀 Try It Out! 10 | 11 | 🆕 [Try it here](https://screenshottocode.com) (bring your own OpenAI key - **your key must have access to GPT-4 Vision. See [FAQ](#%EF%B8%8F-faqs) section below for details**). Or see [Getting Started](#-getting-started) below for local install instructions. 12 | 13 | 14 | ## 🛠 Getting Started 15 | 16 | The app has a React/Vite frontend and a FastAPI backend. You will need an OpenAI API key with access to the GPT-4 Vision API. 17 | 18 | Run the backend (I use Poetry for package management - `pip install poetry` if you don't have it): 19 | 20 | ```bash 21 | cd backend 22 | echo "OPENAI_API_KEY=sk-your-key" > .env 23 | poetry install 24 | poetry shell 25 | poetry run uvicorn main:app --reload --port 7001 26 | ``` 27 | 28 | Run the frontend: 29 | 30 | ```bash 31 | cd frontend 32 | yarn 33 | yarn dev 34 | ``` 35 | 36 | Open http://localhost:5173 to use the app. 37 | 38 | If you prefer to run the backend on a different port, update VITE_WS_BACKEND_URL in `frontend/.env.local` 39 | 40 | For debugging purposes, if you don't want to waste GPT4-Vision credits, you can run the backend in mock mode (which streams a pre-recorded response): 41 | 42 | ```bash 43 | MOCK=true poetry run uvicorn main:app --reload --port 7001 44 | ``` 45 | 46 | ## Configuration 47 | 48 | - You can configure the OpenAI base URL if you need to use a proxy: Set OPENAI_BASE_URL in the `backend/.env` or directly in the UI in the settings dialog 49 | 50 | ## Docker 51 | 52 | If you have Docker installed on your system, in the root directory, run: 53 | 54 | ```bash 55 | echo "OPENAI_API_KEY=sk-your-key" > .env 56 | docker-compose up -d --build 57 | ``` 58 | 59 | The app will be up and running at http://localhost:5173. Note that you can't develop the application with this setup as the file changes won't trigger a rebuild. 60 | 61 | ## 🙋‍♂️ FAQs 62 | 63 | - **I'm running into an error when setting up the backend. How can I fix it?** [Try this](https://github.com/abi/screenshot-to-code/issues/3#issuecomment-1814777959). If that still doesn't work, open an issue. 64 | - **How do I get an OpenAI API key?** See https://github.com/abi/screenshot-to-code/blob/main/Troubleshooting.md 65 | - **How can I provide feedback?** For feedback, feature requests and bug reports, open an issue or ping me on [Twitter](https://twitter.com/_abi_). 66 | 67 | ## 📚 Examples 68 | 69 | **NYTimes** 70 | 71 | | Original | Replica | 72 | | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | 73 | | Screenshot 2023-11-20 at 12 54 03 PM | Screenshot 2023-11-20 at 12 59 56 PM | 74 | 75 | **Instagram page (with not Taylor Swift pics)** 76 | 77 | https://github.com/abi/screenshot-to-code/assets/23818/503eb86a-356e-4dfc-926a-dabdb1ac7ba1 78 | 79 | **Hacker News** but it gets the colors wrong at first so we nudge it 80 | 81 | https://github.com/abi/screenshot-to-code/assets/23818/3fec0f77-44e8-4fb3-a769-ac7410315e5d 82 | 83 | ## 🌍 Hosted Version 84 | 85 | 🆕 [Try it here](https://screenshottocode.com) (bring your own OpenAI key - **your key must have access to GPT-4 Vision. See [FAQ](#%EF%B8%8F-faqs) section for details**). Or see [Getting Started](#-getting-started) for local install instructions. 86 | 87 | [!["Buy Me A Coffee"](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://www.buymeacoffee.com/abiraja) 88 | -------------------------------------------------------------------------------- /Troubleshooting.md: -------------------------------------------------------------------------------- 1 | ### Getting an OpenAI API key with GPT4-Vision model access 2 | 3 | You don't need a ChatGPT Pro account. Screenshot to code uses API keys from your OpenAI developer account. In order to get access to the GPT4 Vision model, log into your OpenAI account and then, follow these instructions: 4 | 5 | 1. Open [OpenAI Dashboard](https://platform.openai.com/) 6 | 1. Go to Settings > Billing 7 | 1. Click at the Add payment details 8 | 285636868-c80deb92-ab47-45cd-988f-deee67fbd44d 9 | 4. You have to buy some credits. The minimum is $5. 10 | 11 | 5. Go to Settings > Limits and check at the bottom of the page, your current tier has to be "Tier 1" to have GPT4 access 12 | 285636973-da38bd4d-8a78-4904-8027-ca67d729b933 13 | 6. Go to Screenshot to code and paste it in the Settings dialog under OpenAI key (gear icon). Your key is only stored in your browser. Never stored on our servers. 14 | 15 | Some users have also reported that it can take upto 30 minutes after your credit purchase for the GPT4 vision model to be activated. 16 | 17 | If you've followed these steps, and it still doesn't work, feel free to open a Github issue. 18 | -------------------------------------------------------------------------------- /backend/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 105 | __pypackages__/ 106 | 107 | # Celery stuff 108 | celerybeat-schedule 109 | celerybeat.pid 110 | 111 | # SageMath parsed files 112 | *.sage.py 113 | 114 | # Environments 115 | .env 116 | .venv 117 | env/ 118 | venv/ 119 | ENV/ 120 | env.bak/ 121 | venv.bak/ 122 | 123 | # Spyder project settings 124 | .spyderproject 125 | .spyproject 126 | 127 | # Rope project settings 128 | .ropeproject 129 | 130 | # mkdocs documentation 131 | /site 132 | 133 | # mypy 134 | .mypy_cache/ 135 | .dmypy.json 136 | dmypy.json 137 | 138 | # Pyre type checker 139 | .pyre/ 140 | 141 | # pytype static type analyzer 142 | .pytype/ 143 | 144 | # Cython debug symbols 145 | cython_debug/ 146 | 147 | # PyCharm 148 | # JetBrains specific template is maintainted in a separate JetBrains.gitignore that can 149 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 150 | # and can be added to the global gitignore or merged into this file. For a more nuclear 151 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 152 | #.idea/ 153 | 154 | 155 | # Temporary eval output 156 | evals_data 157 | -------------------------------------------------------------------------------- /backend/.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v3.2.0 6 | hooks: 7 | - id: end-of-file-fixer 8 | - id: check-yaml 9 | - id: check-added-large-files 10 | - repo: local 11 | hooks: 12 | - id: poetry-pytest 13 | name: Run pytest with Poetry 14 | entry: poetry run --directory backend pytest 15 | language: system 16 | pass_filenames: false 17 | always_run: true 18 | files: ^backend/ 19 | -------------------------------------------------------------------------------- /backend/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.12-slim-bullseye 2 | 3 | ENV POETRY_VERSION 1.4.1 4 | 5 | # Install system dependencies 6 | RUN pip install "poetry==$POETRY_VERSION" 7 | 8 | # Set work directory 9 | WORKDIR /app 10 | 11 | # Copy only requirements to cache them in docker layer 12 | COPY poetry.lock pyproject.toml /app/ 13 | 14 | # Disable the creation of virtual environments 15 | RUN poetry config virtualenvs.create false 16 | 17 | # Install dependencies 18 | RUN poetry install 19 | 20 | # Copy the current directory contents into the container at /app 21 | COPY ./ /app/ 22 | -------------------------------------------------------------------------------- /backend/README.md: -------------------------------------------------------------------------------- 1 | # Run the type checker 2 | 3 | poetry run pyright 4 | 5 | # Run tests 6 | 7 | poetry run pytest 8 | -------------------------------------------------------------------------------- /backend/access_token.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import httpx 4 | 5 | 6 | async def validate_access_token(access_code: str): 7 | async with httpx.AsyncClient() as client: 8 | url = ( 9 | "https://backend.buildpicoapps.com/screenshot_to_code/validate_access_token" 10 | ) 11 | data = json.dumps( 12 | { 13 | "access_code": access_code, 14 | "secret": os.environ.get("PICO_BACKEND_SECRET"), 15 | } 16 | ) 17 | headers = {"Content-Type": "application/json"} 18 | 19 | response = await client.post(url, content=data, headers=headers) 20 | response_data = response.json() 21 | return response_data 22 | -------------------------------------------------------------------------------- /backend/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # exit on error 3 | set -o errexit 4 | 5 | echo "Installing the latest version of poetry..." 6 | pip install --upgrade pip 7 | pip install poetry==1.4.1 8 | 9 | rm poetry.lock 10 | poetry lock 11 | python -m poetry install 12 | -------------------------------------------------------------------------------- /backend/config.py: -------------------------------------------------------------------------------- 1 | # Useful for debugging purposes when you don't want to waste GPT4-Vision credits 2 | # Setting to True will stream a mock response instead of calling the OpenAI API 3 | # TODO: Should only be set to true when value is 'True', not any abitrary truthy value 4 | import os 5 | 6 | 7 | SHOULD_MOCK_AI_RESPONSE = bool(os.environ.get("MOCK", False)) 8 | 9 | # Set to True when running in production (on the hosted version) 10 | # Used as a feature flag to enable or disable certain features 11 | IS_PROD = os.environ.get("IS_PROD", False) 12 | -------------------------------------------------------------------------------- /backend/evals/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hacksider/screenshot-to-code/1bc26b616fc6ec9c8fd749a002fe256a72f272f9/backend/evals/__init__.py -------------------------------------------------------------------------------- /backend/evals/config.py: -------------------------------------------------------------------------------- 1 | EVALS_DIR = "./evals_data" 2 | -------------------------------------------------------------------------------- /backend/evals/core.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from llm import stream_openai_response 4 | from prompts import assemble_prompt 5 | from prompts.types import Stack 6 | 7 | 8 | async def generate_code_core(image_url: str, stack: Stack) -> str: 9 | prompt_messages = assemble_prompt(image_url, stack) 10 | openai_api_key = os.environ.get("OPENAI_API_KEY") 11 | openai_base_url = None 12 | 13 | async def process_chunk(content: str): 14 | pass 15 | 16 | if not openai_api_key: 17 | raise Exception("OpenAI API key not found") 18 | 19 | completion = await stream_openai_response( 20 | prompt_messages, 21 | api_key=openai_api_key, 22 | base_url=openai_base_url, 23 | callback=lambda x: process_chunk(x), 24 | ) 25 | 26 | return completion 27 | -------------------------------------------------------------------------------- /backend/evals/utils.py: -------------------------------------------------------------------------------- 1 | import base64 2 | 3 | 4 | async def image_to_data_url(filepath: str): 5 | with open(filepath, "rb") as image_file: 6 | encoded_string = base64.b64encode(image_file.read()).decode() 7 | return f"data:image/png;base64,{encoded_string}" 8 | -------------------------------------------------------------------------------- /backend/image_generation.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import re 3 | from typing import Dict, List, Union 4 | from openai import AsyncOpenAI 5 | from bs4 import BeautifulSoup 6 | 7 | 8 | async def process_tasks(prompts: List[str], api_key: str, base_url: str): 9 | tasks = [generate_image(prompt, api_key, base_url) for prompt in prompts] 10 | results = await asyncio.gather(*tasks, return_exceptions=True) 11 | 12 | processed_results: List[Union[str, None]] = [] 13 | for result in results: 14 | if isinstance(result, Exception): 15 | print(f"An exception occurred: {result}") 16 | processed_results.append(None) 17 | else: 18 | processed_results.append(result) 19 | 20 | return processed_results 21 | 22 | 23 | async def generate_image(prompt: str, api_key: str, base_url: str): 24 | client = AsyncOpenAI(api_key=api_key, base_url=base_url) 25 | image_params: Dict[str, Union[str, int]] = { 26 | "model": "dall-e-3", 27 | "quality": "standard", 28 | "style": "natural", 29 | "n": 1, 30 | "size": "1024x1024", 31 | "prompt": prompt, 32 | } 33 | res = await client.images.generate(**image_params) 34 | await client.close() 35 | return res.data[0].url 36 | 37 | 38 | def extract_dimensions(url: str): 39 | # Regular expression to match numbers in the format '300x200' 40 | matches = re.findall(r"(\d+)x(\d+)", url) 41 | 42 | if matches: 43 | width, height = matches[0] # Extract the first match 44 | width = int(width) 45 | height = int(height) 46 | return (width, height) 47 | else: 48 | return (100, 100) 49 | 50 | 51 | def create_alt_url_mapping(code: str) -> Dict[str, str]: 52 | soup = BeautifulSoup(code, "html.parser") 53 | images = soup.find_all("img") 54 | 55 | mapping: Dict[str, str] = {} 56 | 57 | for image in images: 58 | if not image["src"].startswith("https://placehold.co"): 59 | mapping[image["alt"]] = image["src"] 60 | 61 | return mapping 62 | 63 | 64 | async def generate_images( 65 | code: str, api_key: str, base_url: Union[str, None], image_cache: Dict[str, str] 66 | ): 67 | # Find all images 68 | soup = BeautifulSoup(code, "html.parser") 69 | images = soup.find_all("img") 70 | 71 | # Extract alt texts as image prompts 72 | alts = [] 73 | for img in images: 74 | # Only include URL if the image starts with https://placehold.co 75 | # and it's not already in the image_cache 76 | if ( 77 | img["src"].startswith("https://placehold.co") 78 | and image_cache.get(img.get("alt")) is None 79 | ): 80 | alts.append(img.get("alt", None)) 81 | 82 | # Exclude images with no alt text 83 | alts = [alt for alt in alts if alt is not None] 84 | 85 | # Remove duplicates 86 | prompts = list(set(alts)) 87 | 88 | # Return early if there are no images to replace 89 | if len(prompts) == 0: 90 | return code 91 | 92 | # Generate images 93 | results = await process_tasks(prompts, api_key, base_url) 94 | 95 | # Create a dict mapping alt text to image URL 96 | mapped_image_urls = dict(zip(prompts, results)) 97 | 98 | # Merge with image_cache 99 | mapped_image_urls = {**mapped_image_urls, **image_cache} 100 | 101 | # Replace old image URLs with the generated URLs 102 | for img in images: 103 | # Skip images that don't start with https://placehold.co (leave them alone) 104 | if not img["src"].startswith("https://placehold.co"): 105 | continue 106 | 107 | new_url = mapped_image_urls[img.get("alt")] 108 | 109 | if new_url: 110 | # Set width and height attributes 111 | width, height = extract_dimensions(img["src"]) 112 | img["width"] = width 113 | img["height"] = height 114 | # Replace img['src'] with the mapped image URL 115 | img["src"] = new_url 116 | else: 117 | print("Image generation failed for alt text:" + img.get("alt")) 118 | 119 | # Return the modified HTML 120 | # (need to prettify it because BeautifulSoup messes up the formatting) 121 | return soup.prettify() 122 | -------------------------------------------------------------------------------- /backend/llm.py: -------------------------------------------------------------------------------- 1 | from typing import Awaitable, Callable, List 2 | from openai import AsyncOpenAI 3 | from openai.types.chat import ChatCompletionMessageParam, ChatCompletionChunk 4 | 5 | MODEL_GPT_4_VISION = "gpt-4-vision-preview" 6 | 7 | 8 | async def stream_openai_response( 9 | messages: List[ChatCompletionMessageParam], 10 | api_key: str, 11 | base_url: str | None, 12 | callback: Callable[[str], Awaitable[None]], 13 | ) -> str: 14 | client = AsyncOpenAI(api_key=api_key, base_url=base_url) 15 | 16 | model = MODEL_GPT_4_VISION 17 | 18 | # Base parameters 19 | params = {"model": model, "messages": messages, "stream": True, "timeout": 600} 20 | 21 | # Add 'max_tokens' only if the model is a GPT4 vision model 22 | if model == MODEL_GPT_4_VISION: 23 | params["max_tokens"] = 4096 24 | params["temperature"] = 0 25 | 26 | stream = await client.chat.completions.create(**params) # type: ignore 27 | full_response = "" 28 | async for chunk in stream: # type: ignore 29 | assert isinstance(chunk, ChatCompletionChunk) 30 | content = chunk.choices[0].delta.content or "" 31 | full_response += content 32 | await callback(content) 33 | 34 | await client.close() 35 | 36 | return full_response 37 | -------------------------------------------------------------------------------- /backend/main.py: -------------------------------------------------------------------------------- 1 | # Load environment variables first 2 | from dotenv import load_dotenv 3 | 4 | load_dotenv() 5 | 6 | 7 | from fastapi import FastAPI 8 | from fastapi.middleware.cors import CORSMiddleware 9 | from routes import screenshot, generate_code, home, evals 10 | 11 | app = FastAPI(openapi_url=None, docs_url=None, redoc_url=None) 12 | 13 | # Configure CORS settings 14 | app.add_middleware( 15 | CORSMiddleware, 16 | allow_origins=["*"], 17 | allow_credentials=True, 18 | allow_methods=["*"], 19 | allow_headers=["*"], 20 | ) 21 | 22 | # Add routes 23 | app.include_router(generate_code.router) 24 | app.include_router(screenshot.router) 25 | app.include_router(home.router) 26 | app.include_router(evals.router) 27 | -------------------------------------------------------------------------------- /backend/mock_llm.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from typing import Awaitable, Callable 3 | 4 | 5 | async def mock_completion(process_chunk: Callable[[str], Awaitable[None]]) -> str: 6 | code_to_return = NO_IMAGES_NYTIMES_MOCK_CODE 7 | 8 | for i in range(0, len(code_to_return), 10): 9 | await process_chunk(code_to_return[i : i + 10]) 10 | await asyncio.sleep(0.01) 11 | 12 | return code_to_return 13 | 14 | 15 | APPLE_MOCK_CODE = """ 16 | 17 | 18 | 19 | Product Showcase 20 | 21 | 22 | 27 | 28 | 29 | 51 | 52 |
53 |
54 |
55 | Brand Logo 56 |

WATCH SERIES 9

57 |

Smarter. Brighter. Mightier.

58 |
59 | Learn more > 60 | Buy > 61 |
62 |
63 |
64 | Product image of a smartwatch with a pink band and a circular interface displaying various health metrics. 65 | Product image of a smartwatch with a blue band and a square interface showing a classic analog clock face. 66 |
67 |
68 |
69 | 70 | """ 71 | 72 | NYTIMES_MOCK_CODE = """ 73 | 74 | 75 | 76 | 77 | The New York Times - News 78 | 79 | 80 | 81 | 86 | 87 | 88 |
89 |
90 |
91 |
92 | 93 | 94 |
Tuesday, November 14, 2023
Today's Paper
95 |
96 |
97 | The New York Times Logo 98 |
99 |
100 | 101 |
Account
102 |
103 |
104 | 114 |
115 |
116 |
117 |
118 |
119 |
120 |

Israeli Military Raids Gaza’s Largest Hospital

121 |

Israeli troops have entered the Al-Shifa Hospital complex, where conditions have grown dire and Israel says Hamas fighters are embedded.

122 | See more updates 123 |
124 | 125 |
126 |
127 |
128 | Flares and plumes of smoke over the northern Gaza skyline on Tuesday. 129 |

From Elvis to Elopements, the Evolution of the Las Vegas Wedding

130 |

The glittering city that attracts thousands of couples seeking unconventional nuptials has grown beyond the drive-through wedding.

131 | 8 MIN READ 132 |
133 | 134 |
135 |
136 |
137 |
138 |
139 | 140 | 141 | """ 142 | 143 | NO_IMAGES_NYTIMES_MOCK_CODE = """ 144 | 145 | 146 | 147 | 148 | The New York Times - News 149 | 150 | 151 | 152 | 157 | 158 | 159 |
160 |
161 |
162 |
163 | 164 | 165 |
Tuesday, November 14, 2023
Today's Paper
166 |
167 |
168 | 169 |
Account
170 |
171 |
172 | 182 |
183 |
184 |
185 |
186 |
187 |
188 |

Israeli Military Raids Gaza’s Largest Hospital

189 |

Israeli troops have entered the Al-Shifa Hospital complex, where conditions have grown dire and Israel says Hamas fighters are embedded.

190 | See more updates 191 |
192 | 193 |
194 |
195 |
196 |

From Elvis to Elopements, the Evolution of the Las Vegas Wedding

197 |

The glittering city that attracts thousands of couples seeking unconventional nuptials has grown beyond the drive-through wedding.

198 | 8 MIN READ 199 |
200 | 201 |
202 |
203 |
204 |
205 |
206 | 207 | 208 | """ 209 | -------------------------------------------------------------------------------- /backend/prompts/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import List, NoReturn, Union 2 | 3 | from openai.types.chat import ChatCompletionMessageParam, ChatCompletionContentPartParam 4 | 5 | from prompts.imported_code_prompts import IMPORTED_CODE_SYSTEM_PROMPTS 6 | from prompts.screenshot_system_prompts import SYSTEM_PROMPTS 7 | from prompts.types import Stack 8 | 9 | 10 | USER_PROMPT = """ 11 | Generate code for a web page that looks exactly like this. 12 | """ 13 | 14 | SVG_USER_PROMPT = """ 15 | Generate code for a SVG that looks exactly like this. 16 | """ 17 | 18 | 19 | def assemble_imported_code_prompt( 20 | code: str, stack: Stack, result_image_data_url: Union[str, None] = None 21 | ) -> List[ChatCompletionMessageParam]: 22 | system_content = IMPORTED_CODE_SYSTEM_PROMPTS[stack] 23 | 24 | user_content = ( 25 | "Here is the code of the app: " + code 26 | if stack != "svg" 27 | else "Here is the code of the SVG: " + code 28 | ) 29 | return [ 30 | { 31 | "role": "system", 32 | "content": system_content, 33 | }, 34 | { 35 | "role": "user", 36 | "content": user_content, 37 | }, 38 | ] 39 | # TODO: Use result_image_data_url 40 | 41 | 42 | def assemble_prompt( 43 | image_data_url: str, 44 | stack: Stack, 45 | result_image_data_url: Union[str, None] = None, 46 | ) -> List[ChatCompletionMessageParam]: 47 | system_content = SYSTEM_PROMPTS[stack] 48 | user_prompt = USER_PROMPT if stack != "svg" else SVG_USER_PROMPT 49 | 50 | user_content: List[ChatCompletionContentPartParam] = [ 51 | { 52 | "type": "image_url", 53 | "image_url": {"url": image_data_url, "detail": "high"}, 54 | }, 55 | { 56 | "type": "text", 57 | "text": user_prompt, 58 | }, 59 | ] 60 | 61 | # Include the result image if it exists 62 | if result_image_data_url: 63 | user_content.insert( 64 | 1, 65 | { 66 | "type": "image_url", 67 | "image_url": {"url": result_image_data_url, "detail": "high"}, 68 | }, 69 | ) 70 | return [ 71 | { 72 | "role": "system", 73 | "content": system_content, 74 | }, 75 | { 76 | "role": "user", 77 | "content": user_content, 78 | }, 79 | ] 80 | -------------------------------------------------------------------------------- /backend/prompts/imported_code_prompts.py: -------------------------------------------------------------------------------- 1 | from prompts.types import SystemPrompts 2 | 3 | 4 | IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT = """ 5 | You are an expert Tailwind developer. 6 | 7 | - Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. 8 | - Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. 9 | - For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. 10 | 11 | In terms of libraries, 12 | 13 | - Use this script to include Tailwind: 14 | - You can use Google Fonts 15 | - Font Awesome for icons: 16 | 17 | Return only the full code in tags. 18 | Do not include markdown "```" or "```html" at the start or end. 19 | """ 20 | 21 | IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT = """ 22 | You are an expert React/Tailwind developer 23 | 24 | - Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. 25 | - Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. 26 | - For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. 27 | 28 | In terms of libraries, 29 | 30 | - Use these script to include React so that it can run on a standalone page: 31 | 32 | 33 | 34 | - Use this script to include Tailwind: 35 | - You can use Google Fonts 36 | - Font Awesome for icons: 37 | 38 | Return only the full code in tags. 39 | Do not include markdown "```" or "```html" at the start or end. 40 | """ 41 | 42 | IMPORTED_CODE_BOOTSTRAP_SYSTEM_PROMPT = """ 43 | You are an expert Bootstrap developer. 44 | 45 | - Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. 46 | - Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. 47 | - For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. 48 | 49 | In terms of libraries, 50 | 51 | - Use this script to include Bootstrap: 52 | - You can use Google Fonts 53 | - Font Awesome for icons: 54 | 55 | Return only the full code in tags. 56 | Do not include markdown "```" or "```html" at the start or end. 57 | """ 58 | 59 | IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT = """ 60 | You are an expert Ionic/Tailwind developer. 61 | 62 | - Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. 63 | - Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. 64 | - For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. 65 | 66 | In terms of libraries, 67 | 68 | - Use these script to include Ionic so that it can run on a standalone page: 69 | 70 | 71 | 72 | - Use this script to include Tailwind: 73 | - You can use Google Fonts 74 | - ionicons for icons, add the following 78 | 79 | 80 | 81 | Return only the full code in tags. 82 | Do not include markdown "```" or "```html" at the start or end. 83 | """ 84 | 85 | IMPORTED_CODE_VUE_TAILWIND_SYSTEM_PROMPT = """ 86 | You are an expert Vue/Tailwind developer. 87 | 88 | - Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. 89 | - Repeat elements as needed. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. 90 | - For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. 91 | 92 | In terms of libraries, 93 | 94 | - Use these script to include Vue so that it can run on a standalone page: 95 | 96 | - Use Vue using the global build like so: 97 |
{{ message }}
98 | 109 | - Use this script to include Tailwind: 110 | - You can use Google Fonts 111 | - Font Awesome for icons: 112 | 113 | Return only the full code in tags. 114 | Do not include markdown "```" or "```html" at the start or end. 115 | The return result must only include the code.""" 116 | 117 | IMPORTED_CODE_SVG_SYSTEM_PROMPT = """ 118 | You are an expert at building SVGs. 119 | 120 | - Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. 121 | - Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. 122 | - For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. 123 | - You can use Google Fonts 124 | 125 | Return only the full code in tags. 126 | Do not include markdown "```" or "```svg" at the start or end. 127 | """ 128 | 129 | IMPORTED_CODE_SYSTEM_PROMPTS = SystemPrompts( 130 | html_tailwind=IMPORTED_CODE_TAILWIND_SYSTEM_PROMPT, 131 | react_tailwind=IMPORTED_CODE_REACT_TAILWIND_SYSTEM_PROMPT, 132 | bootstrap=IMPORTED_CODE_BOOTSTRAP_SYSTEM_PROMPT, 133 | ionic_tailwind=IMPORTED_CODE_IONIC_TAILWIND_SYSTEM_PROMPT, 134 | vue_tailwind=IMPORTED_CODE_VUE_TAILWIND_SYSTEM_PROMPT, 135 | svg=IMPORTED_CODE_SVG_SYSTEM_PROMPT, 136 | ) 137 | -------------------------------------------------------------------------------- /backend/prompts/screenshot_system_prompts.py: -------------------------------------------------------------------------------- 1 | from prompts.types import SystemPrompts 2 | 3 | 4 | HTML_TAILWIND_SYSTEM_PROMPT = """ 5 | You are an expert Tailwind developer 6 | You take screenshots of a reference web page from the user, and then build single page apps 7 | using Tailwind, HTML and JS. 8 | You might also be given a screenshot(The second image) of a web page that you have already built, and asked to 9 | update it to look more like the reference image(The first image). 10 | 11 | - Make sure the app looks exactly like the screenshot. 12 | - Pay close attention to background color, text color, font size, font family, 13 | padding, margin, border, etc. Match the colors and sizes exactly. 14 | - Use the exact text from the screenshot. 15 | - Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. 16 | - Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. 17 | - For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. 18 | 19 | In terms of libraries, 20 | 21 | - Use this script to include Tailwind: 22 | - You can use Google Fonts 23 | - Font Awesome for icons: 24 | 25 | Return only the full code in tags. 26 | Do not include markdown "```" or "```html" at the start or end. 27 | """ 28 | 29 | BOOTSTRAP_SYSTEM_PROMPT = """ 30 | You are an expert Bootstrap developer 31 | You take screenshots of a reference web page from the user, and then build single page apps 32 | using Bootstrap, HTML and JS. 33 | You might also be given a screenshot(The second image) of a web page that you have already built, and asked to 34 | update it to look more like the reference image(The first image). 35 | 36 | - Make sure the app looks exactly like the screenshot. 37 | - Pay close attention to background color, text color, font size, font family, 38 | padding, margin, border, etc. Match the colors and sizes exactly. 39 | - Use the exact text from the screenshot. 40 | - Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. 41 | - Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. 42 | - For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. 43 | 44 | In terms of libraries, 45 | 46 | - Use this script to include Bootstrap: 47 | - You can use Google Fonts 48 | - Font Awesome for icons: 49 | 50 | Return only the full code in tags. 51 | Do not include markdown "```" or "```html" at the start or end. 52 | """ 53 | 54 | REACT_TAILWIND_SYSTEM_PROMPT = """ 55 | You are an expert React/Tailwind developer 56 | You take screenshots of a reference web page from the user, and then build single page apps 57 | using React and Tailwind CSS. 58 | You might also be given a screenshot(The second image) of a web page that you have already built, and asked to 59 | update it to look more like the reference image(The first image). 60 | 61 | - Make sure the app looks exactly like the screenshot. 62 | - Pay close attention to background color, text color, font size, font family, 63 | padding, margin, border, etc. Match the colors and sizes exactly. 64 | - Use the exact text from the screenshot. 65 | - Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. 66 | - Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. 67 | - For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. 68 | 69 | In terms of libraries, 70 | 71 | - Use these script to include React so that it can run on a standalone page: 72 | 73 | 74 | 75 | - Use this script to include Tailwind: 76 | - You can use Google Fonts 77 | - Font Awesome for icons: 78 | 79 | Return only the full code in tags. 80 | Do not include markdown "```" or "```html" at the start or end. 81 | """ 82 | 83 | IONIC_TAILWIND_SYSTEM_PROMPT = """ 84 | You are an expert Ionic/Tailwind developer 85 | You take screenshots of a reference web page from the user, and then build single page apps 86 | using Ionic and Tailwind CSS. 87 | You might also be given a screenshot(The second image) of a web page that you have already built, and asked to 88 | update it to look more like the reference image(The first image). 89 | 90 | - Make sure the app looks exactly like the screenshot. 91 | - Pay close attention to background color, text color, font size, font family, 92 | padding, margin, border, etc. Match the colors and sizes exactly. 93 | - Use the exact text from the screenshot. 94 | - Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. 95 | - Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. 96 | - For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. 97 | 98 | In terms of libraries, 99 | 100 | - Use these script to include Ionic so that it can run on a standalone page: 101 | 102 | 103 | 104 | - Use this script to include Tailwind: 105 | - You can use Google Fonts 106 | - ionicons for icons, add the following 110 | 111 | 112 | 113 | Return only the full code in tags. 114 | Do not include markdown "```" or "```html" at the start or end. 115 | """ 116 | 117 | VUE_TAILWIND_SYSTEM_PROMPT = """ 118 | You are an expert Vue/Tailwind developer 119 | You take screenshots of a reference web page from the user, and then build single page apps 120 | using Vue and Tailwind CSS. 121 | You might also be given a screenshot(The second image) of a web page that you have already built, and asked to 122 | update it to look more like the reference image(The first image). 123 | 124 | - Make sure the app looks exactly like the screenshot. 125 | - Pay close attention to background color, text color, font size, font family, 126 | padding, margin, border, etc. Match the colors and sizes exactly. 127 | - Use the exact text from the screenshot. 128 | - Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. 129 | - Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. 130 | - For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. 131 | - Use Vue using the global build like so: 132 | 133 |
{{ message }}
134 | 145 | 146 | In terms of libraries, 147 | 148 | - Use these script to include Vue so that it can run on a standalone page: 149 | 150 | - Use this script to include Tailwind: 151 | - You can use Google Fonts 152 | - Font Awesome for icons: 153 | 154 | Return only the full code in tags. 155 | Do not include markdown "```" or "```html" at the start or end. 156 | The return result must only include the code. 157 | """ 158 | 159 | 160 | SVG_SYSTEM_PROMPT = """ 161 | You are an expert at building SVGs. 162 | You take screenshots of a reference web page from the user, and then build a SVG that looks exactly like the screenshot. 163 | 164 | - Make sure the SVG looks exactly like the screenshot. 165 | - Pay close attention to background color, text color, font size, font family, 166 | padding, margin, border, etc. Match the colors and sizes exactly. 167 | - Use the exact text from the screenshot. 168 | - Do not add comments in the code such as "" and "" in place of writing the full code. WRITE THE FULL CODE. 169 | - Repeat elements as needed to match the screenshot. For example, if there are 15 items, the code should have 15 items. DO NOT LEAVE comments like "" or bad things will happen. 170 | - For images, use placeholder images from https://placehold.co and include a detailed description of the image in the alt text so that an image generation AI can generate the image later. 171 | - You can use Google Fonts 172 | 173 | Return only the full code in tags. 174 | Do not include markdown "```" or "```svg" at the start or end. 175 | """ 176 | 177 | 178 | SYSTEM_PROMPTS = SystemPrompts( 179 | html_tailwind=HTML_TAILWIND_SYSTEM_PROMPT, 180 | react_tailwind=REACT_TAILWIND_SYSTEM_PROMPT, 181 | bootstrap=BOOTSTRAP_SYSTEM_PROMPT, 182 | ionic_tailwind=IONIC_TAILWIND_SYSTEM_PROMPT, 183 | vue_tailwind=VUE_TAILWIND_SYSTEM_PROMPT, 184 | svg=SVG_SYSTEM_PROMPT, 185 | ) 186 | -------------------------------------------------------------------------------- /backend/prompts/types.py: -------------------------------------------------------------------------------- 1 | from typing import Literal, TypedDict 2 | 3 | 4 | class SystemPrompts(TypedDict): 5 | html_tailwind: str 6 | react_tailwind: str 7 | bootstrap: str 8 | ionic_tailwind: str 9 | vue_tailwind: str 10 | svg: str 11 | 12 | 13 | Stack = Literal[ 14 | "html_tailwind", 15 | "react_tailwind", 16 | "bootstrap", 17 | "ionic_tailwind", 18 | "vue_tailwind", 19 | "svg", 20 | ] 21 | -------------------------------------------------------------------------------- /backend/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "backend" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Abi Raja "] 6 | license = "MIT" 7 | 8 | [tool.poetry.dependencies] 9 | python = "^3.10" 10 | fastapi = "^0.95.0" 11 | uvicorn = "^0.25.0" 12 | websockets = "^12.0" 13 | openai = "^1.2.4" 14 | python-dotenv = "^1.0.0" 15 | beautifulsoup4 = "^4.12.2" 16 | httpx = "^0.25.1" 17 | pre-commit = "^3.6.2" 18 | 19 | [tool.poetry.group.dev.dependencies] 20 | pytest = "^7.4.3" 21 | pyright = "^1.1.345" 22 | 23 | [build-system] 24 | requires = ["poetry-core"] 25 | build-backend = "poetry.core.masonry.api" 26 | -------------------------------------------------------------------------------- /backend/pyrightconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "exclude": ["image_generation.py"] 3 | } 4 | -------------------------------------------------------------------------------- /backend/routes/evals.py: -------------------------------------------------------------------------------- 1 | import os 2 | from fastapi import APIRouter 3 | from pydantic import BaseModel 4 | from evals.utils import image_to_data_url 5 | from evals.config import EVALS_DIR 6 | 7 | 8 | router = APIRouter() 9 | 10 | 11 | class Eval(BaseModel): 12 | input: str 13 | output: str 14 | 15 | 16 | @router.get("/evals") 17 | async def get_evals(): 18 | # Get all evals from EVALS_DIR 19 | input_dir = EVALS_DIR + "/inputs" 20 | output_dir = EVALS_DIR + "/outputs" 21 | 22 | evals: list[Eval] = [] 23 | for file in os.listdir(input_dir): 24 | if file.endswith(".png"): 25 | input_file_path = os.path.join(input_dir, file) 26 | input_file = await image_to_data_url(input_file_path) 27 | 28 | # Construct the corresponding output file name 29 | output_file_name = file.replace(".png", ".html") 30 | output_file_path = os.path.join(output_dir, output_file_name) 31 | 32 | # Check if the output file exists 33 | if os.path.exists(output_file_path): 34 | with open(output_file_path, "r") as f: 35 | output_file_data = f.read() 36 | else: 37 | output_file_data = "Output file not found." 38 | 39 | evals.append( 40 | Eval( 41 | input=input_file, 42 | output=output_file_data, 43 | ) 44 | ) 45 | 46 | return evals 47 | -------------------------------------------------------------------------------- /backend/routes/generate_code.py: -------------------------------------------------------------------------------- 1 | import os 2 | import traceback 3 | from fastapi import APIRouter, WebSocket 4 | import openai 5 | from config import IS_PROD, SHOULD_MOCK_AI_RESPONSE 6 | from llm import stream_openai_response 7 | from openai.types.chat import ChatCompletionMessageParam 8 | from mock_llm import mock_completion 9 | from typing import Dict, List, cast, get_args 10 | from image_generation import create_alt_url_mapping, generate_images 11 | from prompts import assemble_imported_code_prompt, assemble_prompt 12 | from access_token import validate_access_token 13 | from datetime import datetime 14 | import json 15 | from prompts.types import Stack 16 | 17 | from utils import pprint_prompt # type: ignore 18 | 19 | 20 | router = APIRouter() 21 | 22 | 23 | def write_logs(prompt_messages: List[ChatCompletionMessageParam], completion: str): 24 | # Get the logs path from environment, default to the current working directory 25 | logs_path = os.environ.get("LOGS_PATH", os.getcwd()) 26 | 27 | # Create run_logs directory if it doesn't exist within the specified logs path 28 | logs_directory = os.path.join(logs_path, "run_logs") 29 | if not os.path.exists(logs_directory): 30 | os.makedirs(logs_directory) 31 | 32 | print("Writing to logs directory:", logs_directory) 33 | 34 | # Generate a unique filename using the current timestamp within the logs directory 35 | filename = datetime.now().strftime(f"{logs_directory}/messages_%Y%m%d_%H%M%S.json") 36 | 37 | # Write the messages dict into a new file for each run 38 | with open(filename, "w") as f: 39 | f.write(json.dumps({"prompt": prompt_messages, "completion": completion})) 40 | 41 | 42 | @router.websocket("/generate-code") 43 | async def stream_code(websocket: WebSocket): 44 | await websocket.accept() 45 | 46 | print("Incoming websocket connection...") 47 | 48 | async def throw_error( 49 | message: str, 50 | ): 51 | await websocket.send_json({"type": "error", "value": message}) 52 | await websocket.close() 53 | 54 | # TODO: Are the values always strings? 55 | params: Dict[str, str] = await websocket.receive_json() 56 | 57 | print("Received params") 58 | 59 | # Read the code config settings from the request. Fall back to default if not provided. 60 | generated_code_config = "" 61 | if "generatedCodeConfig" in params and params["generatedCodeConfig"]: 62 | generated_code_config = params["generatedCodeConfig"] 63 | print(f"Generating {generated_code_config} code") 64 | 65 | # Get the OpenAI API key from the request. Fall back to environment variable if not provided. 66 | # If neither is provided, we throw an error. 67 | openai_api_key = None 68 | if "accessCode" in params and params["accessCode"]: 69 | print("Access code - using platform API key") 70 | res = await validate_access_token(params["accessCode"]) 71 | if res["success"]: 72 | openai_api_key = os.environ.get("PLATFORM_OPENAI_API_KEY") 73 | else: 74 | await websocket.send_json( 75 | { 76 | "type": "error", 77 | "value": res["failure_reason"], 78 | } 79 | ) 80 | return 81 | else: 82 | if params["openAiApiKey"]: 83 | openai_api_key = params["openAiApiKey"] 84 | print("Using OpenAI API key from client-side settings dialog") 85 | else: 86 | openai_api_key = os.environ.get("OPENAI_API_KEY") 87 | if openai_api_key: 88 | print("Using OpenAI API key from environment variable") 89 | 90 | if not openai_api_key: 91 | print("OpenAI API key not found") 92 | await websocket.send_json( 93 | { 94 | "type": "error", 95 | "value": "No OpenAI API key found. Please add your API key in the settings dialog or add it to backend/.env file. If you add it to .env, make sure to restart the backend server.", 96 | } 97 | ) 98 | return 99 | 100 | # Validate the generated code config 101 | if not generated_code_config in get_args(Stack): 102 | await throw_error(f"Invalid generated code config: {generated_code_config}") 103 | return 104 | # Cast the variable to the Stack type 105 | valid_stack = cast(Stack, generated_code_config) 106 | 107 | # Get the OpenAI Base URL from the request. Fall back to environment variable if not provided. 108 | openai_base_url = None 109 | # Disable user-specified OpenAI Base URL in prod 110 | if not os.environ.get("IS_PROD"): 111 | if "openAiBaseURL" in params and params["openAiBaseURL"]: 112 | openai_base_url = params["openAiBaseURL"] 113 | print("Using OpenAI Base URL from client-side settings dialog") 114 | else: 115 | openai_base_url = os.environ.get("OPENAI_BASE_URL") 116 | if openai_base_url: 117 | print("Using OpenAI Base URL from environment variable") 118 | 119 | if not openai_base_url: 120 | print("Using official OpenAI URL") 121 | 122 | # Get the image generation flag from the request. Fall back to True if not provided. 123 | should_generate_images = ( 124 | params["isImageGenerationEnabled"] 125 | if "isImageGenerationEnabled" in params 126 | else True 127 | ) 128 | 129 | print("generating code...") 130 | await websocket.send_json({"type": "status", "value": "Generating code..."}) 131 | 132 | async def process_chunk(content: str): 133 | await websocket.send_json({"type": "chunk", "value": content}) 134 | 135 | # Image cache for updates so that we don't have to regenerate images 136 | image_cache: Dict[str, str] = {} 137 | 138 | # If this generation started off with imported code, we need to assemble the prompt differently 139 | if params.get("isImportedFromCode") and params["isImportedFromCode"]: 140 | original_imported_code = params["history"][0] 141 | prompt_messages = assemble_imported_code_prompt( 142 | original_imported_code, valid_stack 143 | ) 144 | for index, text in enumerate(params["history"][1:]): 145 | if index % 2 == 0: 146 | message: ChatCompletionMessageParam = { 147 | "role": "user", 148 | "content": text, 149 | } 150 | else: 151 | message: ChatCompletionMessageParam = { 152 | "role": "assistant", 153 | "content": text, 154 | } 155 | prompt_messages.append(message) 156 | else: 157 | # Assemble the prompt 158 | try: 159 | if params.get("resultImage") and params["resultImage"]: 160 | prompt_messages = assemble_prompt( 161 | params["image"], valid_stack, params["resultImage"] 162 | ) 163 | else: 164 | prompt_messages = assemble_prompt(params["image"], valid_stack) 165 | except: 166 | await websocket.send_json( 167 | { 168 | "type": "error", 169 | "value": "Error assembling prompt. Contact support at support@picoapps.xyz", 170 | } 171 | ) 172 | await websocket.close() 173 | return 174 | 175 | if params["generationType"] == "update": 176 | # Transform the history tree into message format 177 | # TODO: Move this to frontend 178 | for index, text in enumerate(params["history"]): 179 | if index % 2 == 0: 180 | message: ChatCompletionMessageParam = { 181 | "role": "assistant", 182 | "content": text, 183 | } 184 | else: 185 | message: ChatCompletionMessageParam = { 186 | "role": "user", 187 | "content": text, 188 | } 189 | prompt_messages.append(message) 190 | 191 | image_cache = create_alt_url_mapping(params["history"][-2]) 192 | 193 | pprint_prompt(prompt_messages) 194 | 195 | if SHOULD_MOCK_AI_RESPONSE: 196 | completion = await mock_completion(process_chunk) 197 | else: 198 | try: 199 | completion = await stream_openai_response( 200 | prompt_messages, 201 | api_key=openai_api_key, 202 | base_url=openai_base_url, 203 | callback=lambda x: process_chunk(x), 204 | ) 205 | except openai.AuthenticationError as e: 206 | print("[GENERATE_CODE] Authentication failed", e) 207 | error_message = ( 208 | "Incorrect OpenAI key. Please make sure your OpenAI API key is correct, or create a new OpenAI API key on your OpenAI dashboard." 209 | + ( 210 | " Alternatively, you can purchase code generation credits directly on this website." 211 | if IS_PROD 212 | else "" 213 | ) 214 | ) 215 | return await throw_error(error_message) 216 | except openai.NotFoundError as e: 217 | print("[GENERATE_CODE] Model not found", e) 218 | error_message = ( 219 | e.message 220 | + ". Please make sure you have followed the instructions correctly to obtain an OpenAI key with GPT vision access: https://github.com/abi/screenshot-to-code/blob/main/Troubleshooting.md" 221 | + ( 222 | " Alternatively, you can purchase code generation credits directly on this website." 223 | if IS_PROD 224 | else "" 225 | ) 226 | ) 227 | return await throw_error(error_message) 228 | except openai.RateLimitError as e: 229 | print("[GENERATE_CODE] Rate limit exceeded", e) 230 | error_message = ( 231 | "OpenAI error - 'You exceeded your current quota, please check your plan and billing details.'" 232 | + ( 233 | " Alternatively, you can purchase code generation credits directly on this website." 234 | if IS_PROD 235 | else "" 236 | ) 237 | ) 238 | return await throw_error(error_message) 239 | 240 | # Write the messages dict into a log so that we can debug later 241 | write_logs(prompt_messages, completion) 242 | 243 | try: 244 | if should_generate_images: 245 | await websocket.send_json( 246 | {"type": "status", "value": "Generating images..."} 247 | ) 248 | updated_html = await generate_images( 249 | completion, 250 | api_key=openai_api_key, 251 | base_url=openai_base_url, 252 | image_cache=image_cache, 253 | ) 254 | else: 255 | updated_html = completion 256 | await websocket.send_json({"type": "setCode", "value": updated_html}) 257 | await websocket.send_json( 258 | {"type": "status", "value": "Code generation complete."} 259 | ) 260 | except Exception as e: 261 | traceback.print_exc() 262 | print("Image generation failed", e) 263 | # Send set code even if image generation fails since that triggers 264 | # the frontend to update history 265 | await websocket.send_json({"type": "setCode", "value": completion}) 266 | await websocket.send_json( 267 | {"type": "status", "value": "Image generation failed but code is complete."} 268 | ) 269 | 270 | await websocket.close() 271 | -------------------------------------------------------------------------------- /backend/routes/home.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter 2 | from fastapi.responses import HTMLResponse 3 | 4 | 5 | router = APIRouter() 6 | 7 | 8 | @router.get("/") 9 | async def get_status(): 10 | return HTMLResponse( 11 | content="

Your backend is running correctly. Please open the front-end URL (default is http://localhost:5173) to use screenshot-to-code.

" 12 | ) 13 | -------------------------------------------------------------------------------- /backend/routes/screenshot.py: -------------------------------------------------------------------------------- 1 | import base64 2 | from fastapi import APIRouter 3 | from pydantic import BaseModel 4 | import httpx 5 | 6 | router = APIRouter() 7 | 8 | 9 | def bytes_to_data_url(image_bytes: bytes, mime_type: str) -> str: 10 | base64_image = base64.b64encode(image_bytes).decode("utf-8") 11 | return f"data:{mime_type};base64,{base64_image}" 12 | 13 | 14 | async def capture_screenshot( 15 | target_url: str, api_key: str, device: str = "desktop" 16 | ) -> bytes: 17 | api_base_url = "https://api.screenshotone.com/take" 18 | 19 | params = { 20 | "access_key": api_key, 21 | "url": target_url, 22 | "full_page": "true", 23 | "device_scale_factor": "1", 24 | "format": "png", 25 | "block_ads": "true", 26 | "block_cookie_banners": "true", 27 | "block_trackers": "true", 28 | "cache": "false", 29 | "viewport_width": "342", 30 | "viewport_height": "684", 31 | } 32 | 33 | if device == "desktop": 34 | params["viewport_width"] = "1280" 35 | params["viewport_height"] = "832" 36 | 37 | async with httpx.AsyncClient(timeout=60) as client: 38 | response = await client.get(api_base_url, params=params) 39 | if response.status_code == 200 and response.content: 40 | return response.content 41 | else: 42 | raise Exception("Error taking screenshot") 43 | 44 | 45 | class ScreenshotRequest(BaseModel): 46 | url: str 47 | apiKey: str 48 | 49 | 50 | class ScreenshotResponse(BaseModel): 51 | url: str 52 | 53 | 54 | @router.post("/api/screenshot") 55 | async def app_screenshot(request: ScreenshotRequest): 56 | # Extract the URL from the request body 57 | url = request.url 58 | api_key = request.apiKey 59 | 60 | # TODO: Add error handling 61 | image_bytes = await capture_screenshot(url, api_key=api_key) 62 | 63 | # Convert the image bytes to a data url 64 | data_url = bytes_to_data_url(image_bytes, "image/png") 65 | 66 | return ScreenshotResponse(url=data_url) 67 | -------------------------------------------------------------------------------- /backend/run_evals.py: -------------------------------------------------------------------------------- 1 | # Load environment variables first 2 | from dotenv import load_dotenv 3 | 4 | load_dotenv() 5 | 6 | import os 7 | from typing import Any, Coroutine 8 | import asyncio 9 | 10 | from evals.config import EVALS_DIR 11 | from evals.core import generate_code_core 12 | from evals.utils import image_to_data_url 13 | 14 | STACK = "html_tailwind" 15 | 16 | 17 | async def main(): 18 | INPUT_DIR = EVALS_DIR + "/inputs" 19 | OUTPUT_DIR = EVALS_DIR + "/outputs" 20 | 21 | # Get all the files in the directory (only grab pngs) 22 | evals = [f for f in os.listdir(INPUT_DIR) if f.endswith(".png")] 23 | 24 | tasks: list[Coroutine[Any, Any, str]] = [] 25 | for filename in evals: 26 | filepath = os.path.join(INPUT_DIR, filename) 27 | data_url = await image_to_data_url(filepath) 28 | task = generate_code_core(data_url, STACK) 29 | tasks.append(task) 30 | 31 | results = await asyncio.gather(*tasks) 32 | 33 | os.makedirs(OUTPUT_DIR, exist_ok=True) 34 | 35 | for filename, content in zip(evals, results): 36 | # File name is derived from the original filename in evals 37 | output_filename = f"{os.path.splitext(filename)[0]}.html" 38 | output_filepath = os.path.join(OUTPUT_DIR, output_filename) 39 | with open(output_filepath, "w") as file: 40 | file.write(content) 41 | 42 | 43 | asyncio.run(main()) 44 | -------------------------------------------------------------------------------- /backend/start.py: -------------------------------------------------------------------------------- 1 | import uvicorn 2 | 3 | if __name__ == "__main__": 4 | uvicorn.run("main:app", port=7001, reload=True) 5 | -------------------------------------------------------------------------------- /backend/utils.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import json 3 | from typing import List 4 | from openai.types.chat import ChatCompletionMessageParam 5 | 6 | 7 | def pprint_prompt(prompt_messages: List[ChatCompletionMessageParam]): 8 | print(json.dumps(truncate_data_strings(prompt_messages), indent=4)) 9 | 10 | 11 | def truncate_data_strings(data: List[ChatCompletionMessageParam]): # type: ignore 12 | # Deep clone the data to avoid modifying the original object 13 | cloned_data = copy.deepcopy(data) 14 | 15 | if isinstance(cloned_data, dict): 16 | for key, value in cloned_data.items(): # type: ignore 17 | # Recursively call the function if the value is a dictionary or a list 18 | if isinstance(value, (dict, list)): 19 | cloned_data[key] = truncate_data_strings(value) # type: ignore 20 | # Truncate the string if it it's long and add ellipsis and length 21 | elif isinstance(value, str): 22 | cloned_data[key] = value[:40] # type: ignore 23 | if len(value) > 40: 24 | cloned_data[key] += "..." + f" ({len(value)} chars)" # type: ignore 25 | 26 | elif isinstance(cloned_data, list): # type: ignore 27 | # Process each item in the list 28 | cloned_data = [truncate_data_strings(item) for item in cloned_data] # type: ignore 29 | 30 | return cloned_data # type: ignore 31 | -------------------------------------------------------------------------------- /design-docs.md: -------------------------------------------------------------------------------- 1 | ## Version History 2 | 3 | Version history is stored as a tree on the client-side. 4 | 5 | ![Screenshot to Code](https://github.com/abi/screenshot-to-code/assets/23818/e35644aa-b90a-4aa7-8027-b8732796fd7c) 6 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.9' 2 | 3 | services: 4 | backend: 5 | build: 6 | context: ./backend 7 | dockerfile: Dockerfile 8 | 9 | env_file: 10 | - .env 11 | 12 | # or 13 | # environment: 14 | #- BACKEND_PORT=7001 # if you change the port, make sure to also change the VITE_WS_BACKEND_URL at frontend/.env.local 15 | # - OPENAI_API_KEY=your_openai_api_key 16 | 17 | ports: 18 | - "${BACKEND_PORT:-7001}:${BACKEND_PORT:-7001}" 19 | 20 | command: poetry run uvicorn main:app --host 0.0.0.0 --port ${BACKEND_PORT:-7001} 21 | 22 | frontend: 23 | build: 24 | context: ./frontend 25 | dockerfile: Dockerfile 26 | ports: 27 | - "5173:5173" 28 | -------------------------------------------------------------------------------- /frontend/.env.example: -------------------------------------------------------------------------------- 1 | VITE_WS_BACKEND_URL=ws://127.0.0.1:7001 2 | -------------------------------------------------------------------------------- /frontend/.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | env: { browser: true, es2020: true }, 4 | extends: [ 5 | 'eslint:recommended', 6 | 'plugin:@typescript-eslint/recommended', 7 | 'plugin:react-hooks/recommended', 8 | ], 9 | ignorePatterns: ['dist', '.eslintrc.cjs'], 10 | parser: '@typescript-eslint/parser', 11 | plugins: ['react-refresh'], 12 | rules: { 13 | 'react-refresh/only-export-components': [ 14 | 'warn', 15 | { allowConstantExport: true }, 16 | ], 17 | }, 18 | } 19 | -------------------------------------------------------------------------------- /frontend/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | 26 | # Env files 27 | .env* 28 | -------------------------------------------------------------------------------- /frontend/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:20.9-bullseye-slim 2 | 3 | # Set the working directory in the container 4 | WORKDIR /app 5 | 6 | # Copy package.json and yarn.lock 7 | COPY package.json yarn.lock /app/ 8 | 9 | # Install dependencies 10 | RUN yarn install 11 | 12 | # Copy the current directory contents into the container at /app 13 | COPY ./ /app/ 14 | 15 | # Expose port 5173 to access the server 16 | EXPOSE 5173 17 | 18 | # Command to run the application 19 | CMD ["yarn", "dev", "--host", "0.0.0.0"] 20 | -------------------------------------------------------------------------------- /frontend/components.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://ui.shadcn.com/schema.json", 3 | "style": "new-york", 4 | "rsc": false, 5 | "tsx": true, 6 | "tailwind": { 7 | "config": "tailwind.config.js", 8 | "css": "src/index.css", 9 | "baseColor": "slate", 10 | "cssVariables": true 11 | }, 12 | "aliases": { 13 | "components": "@/components", 14 | "utils": "@/lib/utils" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /frontend/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 10 | 11 | 12 | 13 | 14 | 15 | 19 | 20 | 21 | <%- injectHead %> 22 | 23 | Screenshot to Code 24 | 25 | 26 | 27 | 31 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 48 | 52 | 53 | 54 |
55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "screenshot-to-code", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite", 8 | "dev-hosted": "vite --mode prod", 9 | "build": "tsc && vite build", 10 | "build-hosted": "tsc && vite build --mode prod", 11 | "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", 12 | "preview": "vite preview", 13 | "test": "vitest" 14 | }, 15 | "dependencies": { 16 | "@codemirror/lang-html": "^6.4.6", 17 | "@radix-ui/react-accordion": "^1.1.2", 18 | "@radix-ui/react-alert-dialog": "^1.0.5", 19 | "@radix-ui/react-checkbox": "^1.0.4", 20 | "@radix-ui/react-collapsible": "^1.0.3", 21 | "@radix-ui/react-dialog": "^1.0.5", 22 | "@radix-ui/react-hover-card": "^1.0.7", 23 | "@radix-ui/react-icons": "^1.3.0", 24 | "@radix-ui/react-label": "^2.0.2", 25 | "@radix-ui/react-popover": "^1.0.7", 26 | "@radix-ui/react-progress": "^1.0.3", 27 | "@radix-ui/react-scroll-area": "^1.0.5", 28 | "@radix-ui/react-select": "^2.0.0", 29 | "@radix-ui/react-separator": "^1.0.3", 30 | "@radix-ui/react-slot": "^1.0.2", 31 | "@radix-ui/react-switch": "^1.0.3", 32 | "@radix-ui/react-tabs": "^1.0.4", 33 | "class-variance-authority": "^0.7.0", 34 | "classnames": "^2.3.2", 35 | "clsx": "^2.0.0", 36 | "codemirror": "^6.0.1", 37 | "copy-to-clipboard": "^3.3.3", 38 | "html2canvas": "^1.4.1", 39 | "react": "^18.2.0", 40 | "react-dom": "^18.2.0", 41 | "react-dropzone": "^14.2.3", 42 | "react-hot-toast": "^2.4.1", 43 | "react-icons": "^4.12.0", 44 | "react-router-dom": "^6.20.1", 45 | "tailwind-merge": "^2.0.0", 46 | "tailwindcss-animate": "^1.0.7", 47 | "thememirror": "^2.0.1", 48 | "vite-plugin-checker": "^0.6.2" 49 | }, 50 | "devDependencies": { 51 | "@types/node": "^20.9.0", 52 | "@types/react": "^18.2.15", 53 | "@types/react-dom": "^18.2.7", 54 | "@typescript-eslint/eslint-plugin": "^6.0.0", 55 | "@typescript-eslint/parser": "^6.0.0", 56 | "@vitejs/plugin-react": "^4.0.3", 57 | "autoprefixer": "^10.4.16", 58 | "eslint": "^8.45.0", 59 | "eslint-plugin-react-hooks": "^4.6.0", 60 | "eslint-plugin-react-refresh": "^0.4.3", 61 | "postcss": "^8.4.31", 62 | "tailwindcss": "^3.3.5", 63 | "typescript": "^5.0.2", 64 | "vite": "^4.4.5", 65 | "vite-plugin-html": "^3.2.0", 66 | "vitest": "^1.0.1" 67 | }, 68 | "engines": { 69 | "node": ">=14.18.0" 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /frontend/postcss.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /frontend/public/brand/twitter-summary-card.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hacksider/screenshot-to-code/1bc26b616fc6ec9c8fd749a002fe256a72f272f9/frontend/public/brand/twitter-summary-card.png -------------------------------------------------------------------------------- /frontend/src/App.tsx: -------------------------------------------------------------------------------- 1 | import { useEffect, useRef, useState } from "react"; 2 | import ImageUpload from "./components/ImageUpload"; 3 | import CodePreview from "./components/CodePreview"; 4 | import Preview from "./components/Preview"; 5 | import { generateCode } from "./generateCode"; 6 | import Spinner from "./components/Spinner"; 7 | import classNames from "classnames"; 8 | import { 9 | FaCode, 10 | FaDesktop, 11 | FaDownload, 12 | FaMobile, 13 | FaUndo, 14 | } from "react-icons/fa"; 15 | 16 | import { Switch } from "./components/ui/switch"; 17 | import { Button } from "@/components/ui/button"; 18 | import { Textarea } from "@/components/ui/textarea"; 19 | import { Tabs, TabsContent, TabsList, TabsTrigger } from "./components/ui/tabs"; 20 | import SettingsDialog from "./components/SettingsDialog"; 21 | import { AppState, CodeGenerationParams, EditorTheme, Settings } from "./types"; 22 | import { IS_RUNNING_ON_CLOUD } from "./config"; 23 | import { PicoBadge } from "./components/PicoBadge"; 24 | import { OnboardingNote } from "./components/OnboardingNote"; 25 | import { usePersistedState } from "./hooks/usePersistedState"; 26 | import { UrlInputSection } from "./components/UrlInputSection"; 27 | import TermsOfServiceDialog from "./components/TermsOfServiceDialog"; 28 | import html2canvas from "html2canvas"; 29 | import { USER_CLOSE_WEB_SOCKET_CODE } from "./constants"; 30 | import CodeTab from "./components/CodeTab"; 31 | import OutputSettingsSection from "./components/OutputSettingsSection"; 32 | import { History } from "./components/history/history_types"; 33 | import HistoryDisplay from "./components/history/HistoryDisplay"; 34 | import { extractHistoryTree } from "./components/history/utils"; 35 | import toast from "react-hot-toast"; 36 | import ImportCodeSection from "./components/ImportCodeSection"; 37 | import { Stack } from "./lib/stacks"; 38 | 39 | const IS_OPENAI_DOWN = false; 40 | 41 | function App() { 42 | const [appState, setAppState] = useState(AppState.INITIAL); 43 | const [generatedCode, setGeneratedCode] = useState(""); 44 | 45 | const [referenceImages, setReferenceImages] = useState([]); 46 | const [executionConsole, setExecutionConsole] = useState([]); 47 | const [updateInstruction, setUpdateInstruction] = useState(""); 48 | const [isImportedFromCode, setIsImportedFromCode] = useState(false); 49 | 50 | // Settings 51 | const [settings, setSettings] = usePersistedState( 52 | { 53 | openAiApiKey: null, 54 | openAiBaseURL: null, 55 | screenshotOneApiKey: null, 56 | isImageGenerationEnabled: true, 57 | editorTheme: EditorTheme.COBALT, 58 | generatedCodeConfig: Stack.HTML_TAILWIND, 59 | // Only relevant for hosted version 60 | isTermOfServiceAccepted: false, 61 | accessCode: null, 62 | }, 63 | "setting" 64 | ); 65 | 66 | // App history 67 | const [appHistory, setAppHistory] = useState([]); 68 | // Tracks the currently shown version from app history 69 | const [currentVersion, setCurrentVersion] = useState(null); 70 | 71 | const [shouldIncludeResultImage, setShouldIncludeResultImage] = 72 | useState(false); 73 | 74 | const wsRef = useRef(null); 75 | 76 | // When the user already has the settings in local storage, newly added keys 77 | // do not get added to the settings so if it's falsy, we populate it with the default 78 | // value 79 | useEffect(() => { 80 | if (!settings.generatedCodeConfig) { 81 | setSettings((prev) => ({ 82 | ...prev, 83 | generatedCodeConfig: Stack.HTML_TAILWIND, 84 | })); 85 | } 86 | }, [settings.generatedCodeConfig, setSettings]); 87 | 88 | const takeScreenshot = async (): Promise => { 89 | const iframeElement = document.querySelector( 90 | "#preview-desktop" 91 | ) as HTMLIFrameElement; 92 | if (!iframeElement?.contentWindow?.document.body) { 93 | return ""; 94 | } 95 | 96 | const canvas = await html2canvas(iframeElement.contentWindow.document.body); 97 | const png = canvas.toDataURL("image/png"); 98 | return png; 99 | }; 100 | 101 | const downloadCode = () => { 102 | // Create a blob from the generated code 103 | const blob = new Blob([generatedCode], { type: "text/html" }); 104 | const url = URL.createObjectURL(blob); 105 | 106 | // Create an anchor element and set properties for download 107 | const a = document.createElement("a"); 108 | a.href = url; 109 | a.download = "index.html"; // Set the file name for download 110 | document.body.appendChild(a); // Append to the document 111 | a.click(); // Programmatically click the anchor to trigger download 112 | 113 | // Clean up by removing the anchor and revoking the Blob URL 114 | document.body.removeChild(a); 115 | URL.revokeObjectURL(url); 116 | }; 117 | 118 | const reset = () => { 119 | setAppState(AppState.INITIAL); 120 | setGeneratedCode(""); 121 | setReferenceImages([]); 122 | setExecutionConsole([]); 123 | setUpdateInstruction(""); 124 | setIsImportedFromCode(false); 125 | setAppHistory([]); 126 | setCurrentVersion(null); 127 | setShouldIncludeResultImage(false); 128 | }; 129 | 130 | const cancelCodeGeneration = () => { 131 | wsRef.current?.close?.(USER_CLOSE_WEB_SOCKET_CODE); 132 | // make sure stop can correct the state even if the websocket is already closed 133 | cancelCodeGenerationAndReset(); 134 | }; 135 | 136 | const cancelCodeGenerationAndReset = () => { 137 | // When this is the first version, reset the entire app state 138 | if (currentVersion === null) { 139 | reset(); 140 | } else { 141 | // Otherwise, revert to the last version 142 | setGeneratedCode(appHistory[currentVersion].code); 143 | setAppState(AppState.CODE_READY); 144 | } 145 | }; 146 | 147 | function doGenerateCode( 148 | params: CodeGenerationParams, 149 | parentVersion: number | null 150 | ) { 151 | setExecutionConsole([]); 152 | setAppState(AppState.CODING); 153 | 154 | // Merge settings with params 155 | const updatedParams = { ...params, ...settings }; 156 | 157 | generateCode( 158 | wsRef, 159 | updatedParams, 160 | // On change 161 | (token) => setGeneratedCode((prev) => prev + token), 162 | // On set code 163 | (code) => { 164 | setGeneratedCode(code); 165 | if (params.generationType === "create") { 166 | setAppHistory([ 167 | { 168 | type: "ai_create", 169 | parentIndex: null, 170 | code, 171 | inputs: { image_url: referenceImages[0] }, 172 | }, 173 | ]); 174 | setCurrentVersion(0); 175 | } else { 176 | setAppHistory((prev) => { 177 | // Validate parent version 178 | if (parentVersion === null) { 179 | toast.error( 180 | "No parent version set. Contact support or open a Github issue." 181 | ); 182 | return prev; 183 | } 184 | 185 | const newHistory: History = [ 186 | ...prev, 187 | { 188 | type: "ai_edit", 189 | parentIndex: parentVersion, 190 | code, 191 | inputs: { 192 | prompt: updateInstruction, 193 | }, 194 | }, 195 | ]; 196 | setCurrentVersion(newHistory.length - 1); 197 | return newHistory; 198 | }); 199 | } 200 | }, 201 | // On status update 202 | (line) => setExecutionConsole((prev) => [...prev, line]), 203 | // On cancel 204 | () => { 205 | cancelCodeGenerationAndReset(); 206 | }, 207 | // On complete 208 | () => { 209 | setAppState(AppState.CODE_READY); 210 | } 211 | ); 212 | } 213 | 214 | // Initial version creation 215 | function doCreate(referenceImages: string[]) { 216 | // Reset any existing state 217 | reset(); 218 | 219 | setReferenceImages(referenceImages); 220 | if (referenceImages.length > 0) { 221 | doGenerateCode( 222 | { 223 | generationType: "create", 224 | image: referenceImages[0], 225 | }, 226 | currentVersion 227 | ); 228 | } 229 | } 230 | 231 | // Subsequent updates 232 | async function doUpdate() { 233 | if (currentVersion === null) { 234 | toast.error( 235 | "No current version set. Contact support or open a Github issue." 236 | ); 237 | return; 238 | } 239 | 240 | let historyTree; 241 | try { 242 | historyTree = extractHistoryTree(appHistory, currentVersion); 243 | } catch { 244 | toast.error( 245 | "Version history is invalid. This shouldn't happen. Please contact support or open a Github issue." 246 | ); 247 | return; 248 | } 249 | 250 | const updatedHistory = [...historyTree, updateInstruction]; 251 | 252 | if (shouldIncludeResultImage) { 253 | const resultImage = await takeScreenshot(); 254 | doGenerateCode( 255 | { 256 | generationType: "update", 257 | image: referenceImages[0], 258 | resultImage: resultImage, 259 | history: updatedHistory, 260 | isImportedFromCode, 261 | }, 262 | currentVersion 263 | ); 264 | } else { 265 | doGenerateCode( 266 | { 267 | generationType: "update", 268 | image: referenceImages[0], 269 | history: updatedHistory, 270 | isImportedFromCode, 271 | }, 272 | currentVersion 273 | ); 274 | } 275 | 276 | setGeneratedCode(""); 277 | setUpdateInstruction(""); 278 | } 279 | 280 | const handleTermDialogOpenChange = (open: boolean) => { 281 | setSettings((s) => ({ 282 | ...s, 283 | isTermOfServiceAccepted: !open, 284 | })); 285 | }; 286 | 287 | function setStack(stack: Stack) { 288 | setSettings((prev) => ({ 289 | ...prev, 290 | generatedCodeConfig: stack, 291 | })); 292 | } 293 | 294 | function importFromCode(code: string, stack: Stack) { 295 | setIsImportedFromCode(true); 296 | 297 | // Set up this project 298 | setGeneratedCode(code); 299 | setStack(stack); 300 | setAppHistory([ 301 | { 302 | type: "code_create", 303 | parentIndex: null, 304 | code, 305 | inputs: { code }, 306 | }, 307 | ]); 308 | setCurrentVersion(0); 309 | 310 | setAppState(AppState.CODE_READY); 311 | } 312 | 313 | return ( 314 |
315 | {IS_RUNNING_ON_CLOUD && } 316 | {IS_RUNNING_ON_CLOUD && ( 317 | 321 | )} 322 |
323 |
324 |
325 |

Screenshot to Code

326 | 327 |
328 | 329 | setStack(config)} 332 | shouldDisableUpdates={ 333 | appState === AppState.CODING || appState === AppState.CODE_READY 334 | } 335 | /> 336 | 337 | {IS_RUNNING_ON_CLOUD && 338 | !(settings.openAiApiKey || settings.accessCode) && ( 339 | 340 | )} 341 | 342 | {IS_OPENAI_DOWN && ( 343 |
344 | OpenAI API is currently down. Try back in 30 minutes or later. We 345 | apologize for the inconvenience. 346 |
347 | )} 348 | 349 | {(appState === AppState.CODING || 350 | appState === AppState.CODE_READY) && ( 351 | <> 352 | {/* Show code preview only when coding */} 353 | {appState === AppState.CODING && ( 354 |
355 |
356 | 357 | {executionConsole.slice(-1)[0]} 358 |
359 |
360 | 366 |
367 | 368 |
369 | )} 370 | 371 | {appState === AppState.CODE_READY && ( 372 |
373 |
374 |