├── AGENTS.md ├── CLAUDE.md ├── GEMINI.md ├── .env.sample ├── apps └── ui_component_eval │ ├── src │ ├── vite-env.d.ts │ ├── assets │ │ ├── gemini-logo.png │ │ ├── anthropic-logo-white.png │ │ └── vue.svg │ ├── main.ts │ ├── components │ │ └── HelloWorld.vue │ ├── style.css │ └── App.vue │ ├── tsconfig.json │ ├── vite.config.ts │ ├── .gitignore │ ├── index.html │ ├── tsconfig.app.json │ ├── package.json │ ├── tsconfig.node.json │ ├── public │ └── vite.svg │ ├── README.md │ └── bun.lock ├── scripts └── deforest.sh ├── .claude ├── settings.json └── commands │ └── trees.md ├── .gitignore └── README.md /AGENTS.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /CLAUDE.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /GEMINI.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.env.sample: -------------------------------------------------------------------------------- 1 | GEMINI_API_KEY= 2 | ANTHROPIC_API_KEY= 3 | OPENAI_API_KEY= -------------------------------------------------------------------------------- /apps/ui_component_eval/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /apps/ui_component_eval/src/assets/gemini-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disler/agentic-coding-tool-eval/HEAD/apps/ui_component_eval/src/assets/gemini-logo.png -------------------------------------------------------------------------------- /apps/ui_component_eval/src/main.ts: -------------------------------------------------------------------------------- 1 | import { createApp } from 'vue' 2 | import './style.css' 3 | import App from './App.vue' 4 | 5 | createApp(App).mount('#app') 6 | -------------------------------------------------------------------------------- /apps/ui_component_eval/src/assets/anthropic-logo-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disler/agentic-coding-tool-eval/HEAD/apps/ui_component_eval/src/assets/anthropic-logo-white.png -------------------------------------------------------------------------------- /apps/ui_component_eval/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": [], 3 | "references": [ 4 | { "path": "./tsconfig.app.json" }, 5 | { "path": "./tsconfig.node.json" } 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /apps/ui_component_eval/vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite' 2 | import vue from '@vitejs/plugin-vue' 3 | 4 | // https://vite.dev/config/ 5 | export default defineConfig({ 6 | plugins: [vue()], 7 | }) 8 | -------------------------------------------------------------------------------- /scripts/deforest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Remove git worktrees directory 4 | rm -rf trees/ 5 | 6 | # Clean up stale worktree references 7 | git worktree prune 8 | 9 | # Create trees directory 10 | mkdir -p trees/ -------------------------------------------------------------------------------- /apps/ui_component_eval/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | -------------------------------------------------------------------------------- /apps/ui_component_eval/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Vite + Vue + TS 8 | 9 | 10 |
11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /apps/ui_component_eval/src/assets/vue.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /apps/ui_component_eval/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@vue/tsconfig/tsconfig.dom.json", 3 | "compilerOptions": { 4 | "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo", 5 | 6 | /* Linting */ 7 | "strict": true, 8 | "noUnusedLocals": true, 9 | "noUnusedParameters": true, 10 | "erasableSyntaxOnly": true, 11 | "noFallthroughCasesInSwitch": true, 12 | "noUncheckedSideEffectImports": true 13 | }, 14 | "include": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.vue"] 15 | } 16 | -------------------------------------------------------------------------------- /apps/ui_component_eval/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ui_component_eval", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite", 8 | "build": "vue-tsc -b && vite build", 9 | "preview": "vite preview" 10 | }, 11 | "dependencies": { 12 | "vue": "^3.5.17" 13 | }, 14 | "devDependencies": { 15 | "@vitejs/plugin-vue": "^6.0.0", 16 | "@vue/tsconfig": "^0.7.0", 17 | "typescript": "~5.8.3", 18 | "vite": "^7.0.0", 19 | "vue-tsc": "^2.2.10" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /.claude/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "permissions": { 3 | "allow": [ 4 | "Bash(mkdir:*)", 5 | "Bash(uv:*)", 6 | "Bash(find:*)", 7 | "Bash(mv:*)", 8 | "Bash(grep:*)", 9 | "Bash(npm:*)", 10 | "Bash(ls:*)", 11 | "Bash(cp:*)", 12 | "Bash(git:*)", 13 | "Write", 14 | "Edit", 15 | "Bash(./scripts/copy_dot_env.sh:*)", 16 | "Bash(chmod:*)", 17 | "Bash(touch:*)" 18 | ], 19 | "deny": [ 20 | "Bash(git push --force:*)", 21 | "Bash(git push -f:*)", 22 | "Bash(rm -rf:*)" 23 | ] 24 | } 25 | } -------------------------------------------------------------------------------- /.claude/commands/trees.md: -------------------------------------------------------------------------------- 1 | # Create git worktrees 2 | 3 | Read the `Variables` then process the `Run` commands and then `Report` the results. 4 | 5 | ## Variables 6 | 7 | comma_separated_branch_names: $ARGUMENTS 8 | tree_directory: `trees/` 9 | 10 | ## Run 11 | 12 | For each branch name in `comma_separated_branch_names`, create a new git worktree in the `tree_directory` with the respective branch name. 13 | 14 | Look for .env in the root directory and copy it to the worktree directory for each branch. 15 | 16 | ## Report 17 | 18 | Report the results of the `Run` commands with a path to the worktree directory, the branch name, and the path to each .env file copied to the worktree directory. -------------------------------------------------------------------------------- /apps/ui_component_eval/tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo", 4 | "target": "ES2023", 5 | "lib": ["ES2023"], 6 | "module": "ESNext", 7 | "skipLibCheck": true, 8 | 9 | /* Bundler mode */ 10 | "moduleResolution": "bundler", 11 | "allowImportingTsExtensions": true, 12 | "verbatimModuleSyntax": true, 13 | "moduleDetection": "force", 14 | "noEmit": true, 15 | 16 | /* Linting */ 17 | "strict": true, 18 | "noUnusedLocals": true, 19 | "noUnusedParameters": true, 20 | "erasableSyntaxOnly": true, 21 | "noFallthroughCasesInSwitch": true, 22 | "noUncheckedSideEffectImports": true 23 | }, 24 | "include": ["vite.config.ts"] 25 | } 26 | -------------------------------------------------------------------------------- /apps/ui_component_eval/src/components/HelloWorld.vue: -------------------------------------------------------------------------------- 1 | 8 | 9 | 36 | 37 | 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Python 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | *.so 6 | .Python 7 | build/ 8 | develop-eggs/ 9 | dist/ 10 | downloads/ 11 | eggs/ 12 | .eggs/ 13 | lib/ 14 | lib64/ 15 | parts/ 16 | sdist/ 17 | var/ 18 | wheels/ 19 | *.egg-info/ 20 | .installed.cfg 21 | *.egg 22 | MANIFEST 23 | .pytest_cache/ 24 | .coverage 25 | .tox/ 26 | .mypy_cache/ 27 | .dmypy.json 28 | dmypy.json 29 | .pyre/ 30 | *.log 31 | venv/ 32 | env/ 33 | ENV/ 34 | .venv 35 | .env 36 | 37 | # TypeScript 38 | node_modules/ 39 | dist/ 40 | *.tsbuildinfo 41 | .npm 42 | .eslintcache 43 | *.tgz 44 | .yarn-integrity 45 | .yarn/cache 46 | .yarn/unplugged 47 | .yarn/build-state.yml 48 | .yarn/install-state.gz 49 | .pnp.* 50 | 51 | # IDE 52 | .vscode/ 53 | .idea/ 54 | *.swp 55 | *.swo 56 | *~ 57 | .DS_Store 58 | 59 | # Environment variables 60 | .env 61 | .env.local 62 | .env.*.local 63 | 64 | # Logs 65 | logs/ 66 | *.log 67 | npm-debug.log* 68 | yarn-debug.log* 69 | yarn-error.log* 70 | 71 | # Testing 72 | coverage/ 73 | .nyc_output 74 | 75 | # Temporary files 76 | *.tmp 77 | *.temp 78 | .cache/ 79 | 80 | # Git worktrees 81 | trees/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Agentic Coding Tool Evals 2 | > Micro apps to compare and evaluate Agentic Coding Tools in a hands on way. 3 | 4 | ## Apps 5 | 6 | `apps/ui_component_eval/*` - Which Agentic Coding Tool can adhere to the prompts (`apps/ui_component_eval/README.md`) and build the best UI component? 7 | 8 | ## Setup 9 | 10 | Using Claude Code, run the custom slash command `/trees "claude_code,gemini_cli"` to create the worktrees. 11 | 12 | ## Requirements 13 | 14 | - Claude Code (or setup git worktrees manually) 15 | - Git 16 | - bun | npm | yarn | pnpm 17 | - .env (copy .env.sample to .env) 18 | 19 | --- 20 | 21 | ## Agentic Coding Tools 22 | > Tools we want to evaluate 23 | > 24 | > Beware, we're running these tools will run in permisssionless mode. 25 | 26 | Claude Code - `claude --dangerously-skip-permissions` - https://docs.anthropic.com/en/docs/claude-code/overview 27 | 28 | Gemini CLI - `gemini --yolo` - https://github.com/google-gemini/gemini-cli 29 | 30 | Codex CLI - `codex --dangerously-auto-approve-everything` - https://github.com/openai/codex 31 | 32 | ## Master AI Coding 33 | Learn to code with AI with foundational [Principles of AI Coding](https://agenticengineer.com/principled-ai-coding?y=agtooleval) 34 | 35 | Follow the [IndyDevDan youtube channel](https://www.youtube.com/@indydevdan) for more AI coding tips and tricks. -------------------------------------------------------------------------------- /apps/ui_component_eval/public/vite.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /apps/ui_component_eval/src/style.css: -------------------------------------------------------------------------------- 1 | :root { 2 | font-family: system-ui, Avenir, Helvetica, Arial, sans-serif; 3 | line-height: 1.5; 4 | font-weight: 400; 5 | 6 | color-scheme: light dark; 7 | color: rgba(255, 255, 255, 0.87); 8 | background-color: #242424; 9 | 10 | font-synthesis: none; 11 | text-rendering: optimizeLegibility; 12 | -webkit-font-smoothing: antialiased; 13 | -moz-osx-font-smoothing: grayscale; 14 | } 15 | 16 | a { 17 | font-weight: 500; 18 | color: #646cff; 19 | text-decoration: inherit; 20 | } 21 | a:hover { 22 | color: #535bf2; 23 | } 24 | 25 | body { 26 | margin: 0; 27 | display: flex; 28 | place-items: center; 29 | min-width: 320px; 30 | min-height: 100vh; 31 | } 32 | 33 | h1 { 34 | font-size: 3.2em; 35 | line-height: 1.1; 36 | } 37 | 38 | button { 39 | border-radius: 8px; 40 | border: 1px solid transparent; 41 | padding: 0.6em 1.2em; 42 | font-size: 1em; 43 | font-weight: 500; 44 | font-family: inherit; 45 | background-color: #1a1a1a; 46 | cursor: pointer; 47 | transition: border-color 0.25s; 48 | } 49 | button:hover { 50 | border-color: #646cff; 51 | } 52 | button:focus, 53 | button:focus-visible { 54 | outline: 4px auto -webkit-focus-ring-color; 55 | } 56 | 57 | .card { 58 | padding: 2em; 59 | } 60 | 61 | #app { 62 | max-width: 1280px; 63 | margin: 0 auto; 64 | padding: 2rem; 65 | text-align: center; 66 | } 67 | 68 | @media (prefers-color-scheme: light) { 69 | :root { 70 | color: #213547; 71 | background-color: #ffffff; 72 | } 73 | a:hover { 74 | color: #747bff; 75 | } 76 | button { 77 | background-color: #f9f9f9; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /apps/ui_component_eval/src/App.vue: -------------------------------------------------------------------------------- 1 | 2 | 3 | 44 | 45 | 111 | -------------------------------------------------------------------------------- /apps/ui_component_eval/README.md: -------------------------------------------------------------------------------- 1 | # UI Component Evaluation 2 | 3 | This project evaluates the UI building capabilities of different agentic coding tools. Below are UI prompts organized by complexity level to test component creation abilities. 4 | 5 | ## Evaluation Criteria 6 | - Visual accuracy 7 | - Code quality 8 | - Component reusability 9 | - Performance 10 | - Accessibility 11 | 12 | ## UI Prompts by Complexity Level 13 | 14 | ### Level 1: Basic Static Components (No Props) 15 | 16 | 1. **Hero Banner** 17 | `` 18 | - Total components: 1 (single component, no sub-components) 19 | - Create a hero banner with a gradient background (purple to blue) 20 | - Center-aligned headline "Welcome to Our Platform" 21 | - Subtext "Building the future, one component at a time" 22 | - Fixed height of 400px 23 | 24 | 2. **Stats Card** 25 | `` 26 | - Total components: 1-2 (main card, optional StatItem sub-component) 27 | - Create a stats display card with 3 metrics 28 | - Show "1.2M Users", "99.9% Uptime", "24/7 Support" 29 | - Each stat should have an icon placeholder 30 | - Use a clean card design with subtle shadow 31 | 32 | 3. **Feature Grid** 33 | `` 34 | - Total components: 1-2 (main grid, optional FeatureCard sub-component) 35 | - Create a 3-column feature grid 36 | - Each feature has an emoji icon, title, and description 37 | - Features: "⚡ Fast" (Lightning quick performance), "🔒 Secure" (Bank-level security), "🎨 Beautiful" (Stunning designs) 38 | - Responsive design that stacks on mobile 39 | 40 | ### Level 2: Simple Props & Basic Interactivity 41 | 42 | 1. **Button Component** 43 | `