├── .eslintrc.json ├── public ├── favicon.ico ├── og-image.jpg ├── x.svg ├── github.svg ├── info-icon.svg └── favicon.svg ├── app ├── lib │ ├── constants.ts │ ├── utils.ts │ ├── definitions.ts │ ├── metrics.ts │ ├── db.ts │ ├── data.ts │ └── actions.ts ├── components │ ├── fonts.ts │ ├── globals.css │ ├── optimize │ │ ├── buttons.tsx │ │ ├── passwordPrompt.tsx │ │ ├── floating.tsx │ │ ├── table.tsx │ │ └── instructions.tsx │ ├── footer.tsx │ └── index │ │ ├── buttons.tsx │ │ ├── evalControls.tsx │ │ ├── floating.tsx │ │ ├── table.tsx │ │ ├── instructions-original.tsx │ │ └── instructions.tsx ├── api │ ├── site-metrics │ │ └── route.tsx │ ├── update │ │ └── route.tsx │ ├── upload │ │ └── route.tsx │ ├── optimize │ │ └── route.tsx │ └── evaluate │ │ └── route.tsx ├── layout.tsx ├── optimize │ └── page.tsx └── page.tsx ├── postcss.config.mjs ├── next.config.mjs ├── tailwind.config.ts ├── .gitignore ├── .env.local ├── tsconfig.json ├── package.json └── README.md /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["next/core-web-vitals", "next/typescript"] 3 | } 4 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugeneyan/align-app/HEAD/public/favicon.ico -------------------------------------------------------------------------------- /public/og-image.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugeneyan/align-app/HEAD/public/og-image.jpg -------------------------------------------------------------------------------- /app/lib/constants.ts: -------------------------------------------------------------------------------- 1 | export const MAX_TRIALS = process.env.NEXT_PUBLIC_MAX_TRIALS ? parseInt(process.env.NEXT_PUBLIC_MAX_TRIALS, 10) : 10; 2 | 3 | -------------------------------------------------------------------------------- /postcss.config.mjs: -------------------------------------------------------------------------------- 1 | /** @type {import('postcss-load-config').Config} */ 2 | const config = { 3 | plugins: { 4 | tailwindcss: {}, 5 | }, 6 | }; 7 | 8 | export default config; 9 | -------------------------------------------------------------------------------- /app/components/fonts.ts: -------------------------------------------------------------------------------- 1 | import { Inter, Tektur } from "next/font/google"; 2 | // Other fonts considered: Roboto, Open_Sans, Pixelify_Sans, Orbitron, Quantico, Geo 3 | 4 | export const inter = Inter({ subsets: ["latin"] }); 5 | export const tektur = Tektur({ weight: "400", subsets: ["latin"] }); 6 | -------------------------------------------------------------------------------- /next.config.mjs: -------------------------------------------------------------------------------- 1 | // next.config.js 2 | /** @type {import('next').NextConfig} */ 3 | const nextConfig = { 4 | reactStrictMode: true, 5 | experimental: { 6 | serverActions: { 7 | allowedOrigins: ["*"], 8 | }, 9 | }, 10 | } 11 | 12 | export default nextConfig -------------------------------------------------------------------------------- /app/lib/utils.ts: -------------------------------------------------------------------------------- 1 | function getTableName(fileName: string): string { 2 | return fileName 3 | .toLowerCase() 4 | .replace(/\.[^/.]+$/, "") // Remove file extension 5 | .replace(/[^a-z0-9]/g, '_') // Replace non-alphanumeric characters with underscore 6 | } 7 | 8 | export { getTableName } -------------------------------------------------------------------------------- /tailwind.config.ts: -------------------------------------------------------------------------------- 1 | import type { Config } from "tailwindcss"; 2 | 3 | const config: Config = { 4 | content: [ 5 | "./pages/**/*.{js,ts,jsx,tsx,mdx}", 6 | "./components/**/*.{js,ts,jsx,tsx,mdx}", 7 | "./app/**/*.{js,ts,jsx,tsx,mdx}", 8 | ], 9 | theme: { 10 | extend: {}, 11 | }, 12 | plugins: [], 13 | darkMode: 'class', // This enables dark mode, but it won't be used by default 14 | }; 15 | export default config; 16 | -------------------------------------------------------------------------------- /public/x.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | .yarn/install-state.gz 8 | 9 | # testing 10 | /coverage 11 | 12 | # next.js 13 | /.next/ 14 | /out/ 15 | 16 | # production 17 | /build 18 | 19 | # misc 20 | .DS_Store 21 | *.pem 22 | 23 | # debug 24 | npm-debug.log* 25 | yarn-debug.log* 26 | yarn-error.log* 27 | 28 | # local env files 29 | .env*.local 30 | 31 | # vercel 32 | .vercel 33 | 34 | # typescript 35 | *.tsbuildinfo 36 | next-env.d.ts 37 | 38 | # database 39 | database.sqlite -------------------------------------------------------------------------------- /.env.local: -------------------------------------------------------------------------------- 1 | OPENAI_API_KEY= 2 | ANTHROPIC_API_KEY= 3 | OPENAI_4O_MINI=gpt-4o-mini 4 | ANTHROPIC_HAIKU3=claude-3-haiku-20240307 5 | 6 | MIN_LABELS_FOR_EVALUATION=20 7 | MIN_LABELS_FOR_OPTIMIZATION=50 8 | NEXT_PUBLIC_MIN_LABELS_FOR_EVALUATION=20 9 | NEXT_PUBLIC_MIN_LABELS_FOR_OPTIMIZATION=50 10 | LABEL_HPO_URL=http://localhost:8000 11 | MAX_UPLOAD_ROWS=100 12 | NEXT_PUBLIC_MAX_UPLOAD_ROWS=100 13 | MAX_TRIALS=10 14 | NEXT_PUBLIC_MAX_TRIALS=10 15 | HPO_SAMPLES=25 16 | NEXT_PUBLIC_HPO_SAMPLES=25 17 | NEXT_PUBLIC_DELETE_OPTIMIZATION_PASSWORD= 18 | 19 | RAILWAY_ENVIRONMENT=true 20 | DATABASE_URL=postgresql://postgres:@localhost:5432/postgres 21 | 22 | -------------------------------------------------------------------------------- /app/lib/definitions.ts: -------------------------------------------------------------------------------- 1 | export type InputOutput = { 2 | id: number; 3 | input: string; 4 | output: string; 5 | } 6 | 7 | export type Eval = { 8 | id: number; 9 | label: string; 10 | explanation: string; 11 | prediction: string; 12 | } 13 | 14 | export type OptimizationResult = { 15 | id: number; 16 | run_number: number; 17 | prompt: string; 18 | precision: number; 19 | recall: number; 20 | f1: number; 21 | cohens_kappa: number; 22 | true_positives: number; 23 | false_positives: number; 24 | true_negatives: number; 25 | false_negatives: number; 26 | split_type: string; 27 | evaluation_fields: string; 28 | evaluation_model: string; 29 | } 30 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["dom", "dom.iterable", "esnext"], 4 | "allowJs": true, 5 | "skipLibCheck": true, 6 | "strict": true, 7 | "noEmit": true, 8 | "esModuleInterop": true, 9 | "module": "esnext", 10 | "moduleResolution": "bundler", 11 | "resolveJsonModule": true, 12 | "isolatedModules": true, 13 | "jsx": "preserve", 14 | "incremental": true, 15 | "plugins": [ 16 | { 17 | "name": "next" 18 | } 19 | ], 20 | "paths": { 21 | "@/*": ["./*"] 22 | } 23 | }, 24 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], 25 | "exclude": ["node_modules"] 26 | } 27 | -------------------------------------------------------------------------------- /app/components/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | :root { 6 | --foreground-rgb: 0, 0, 0; 7 | --background-start-rgb: 214, 219, 220; 8 | --background-end-rgb: 255, 255, 255; 9 | } 10 | 11 | body { 12 | color: rgb(var(--foreground-rgb)); 13 | background: linear-gradient( 14 | to bottom, 15 | transparent, 16 | rgb(var(--background-end-rgb)) 17 | ) 18 | rgb(var(--background-start-rgb)); 19 | } 20 | 21 | /* Add this to ensure container-fluid works correctly */ 22 | .container-fluid { 23 | width: 100%; 24 | padding-right: 30px; 25 | padding-left: 30px; 26 | margin-right: auto; 27 | margin-left: auto; 28 | } 29 | 30 | .footer-icon { 31 | fill: #808080; /* Medium grey */ 32 | } 33 | 34 | @layer utilities { 35 | .text-balance { 36 | text-wrap: balance; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /app/components/optimize/buttons.tsx: -------------------------------------------------------------------------------- 1 | interface BackToMainButtonProps { 2 | onClick: () => void; 3 | } 4 | 5 | export const BackToMainButton = ({ onClick }: BackToMainButtonProps) => { 6 | return ( 7 | 13 | ); 14 | }; 15 | 16 | interface DeleteOptimizationsButtonProps { 17 | onClick: () => void; 18 | } 19 | 20 | export const DeleteOptimizationsButton = ({ onClick }: DeleteOptimizationsButtonProps) => { 21 | return ( 22 | 28 | ); 29 | }; 30 | -------------------------------------------------------------------------------- /public/github.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /app/components/footer.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Image from 'next/image'; 3 | import Link from 'next/link'; 4 | 5 | const Footer: React.FC = () => { 6 | return ( 7 | 20 | ); 21 | }; 22 | 23 | export default Footer; 24 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "label", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "@anthropic-ai/sdk": "^0.27.3", 13 | "@heroicons/react": "^2.1.5", 14 | "@types/file-saver": "^2.0.7", 15 | "ai": "^3.4.0", 16 | "csv-parse": "^5.5.6", 17 | "csv-stringify": "^6.5.1", 18 | "file-saver": "^2.0.5", 19 | "next": "14.2.13", 20 | "openai": "^4.63.0", 21 | "pg": "^8.13.0", 22 | "react": "^18", 23 | "react-dom": "^18", 24 | "react-responsive": "^10.0.0", 25 | "sqlite": "^5.1.1", 26 | "sqlite3": "^5.1.7", 27 | "zod": "^3.23.8" 28 | }, 29 | "devDependencies": { 30 | "@types/node": "^20", 31 | "@types/pg": "^8.11.10", 32 | "@types/react": "^18", 33 | "@types/react-dom": "^18", 34 | "eslint": "^8", 35 | "eslint-config-next": "14.2.13", 36 | "postcss": "^8", 37 | "tailwindcss": "^3.4.1", 38 | "typescript": "^5" 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ALIGN Eval Frontend 2 | 3 | This is the frontend code for [ALIGN Eval](https://aligneval.com), a prototype game to help you build and optimize LLM evaluators. Read about the workflow and how it was built [here](https://eugeneyan.com/writing/aligneval/). 4 | 5 | ## What is ALIGN Eval? 6 | 7 | ALIGN Eval helps you build better LLM evaluators through: 8 | - 🎮 A prototype game that makes building LLM-evaluators easy and fun 9 | - 📊 Tools to evaluate your prompts against labeled data 10 | - ✨ Semi-automatic optimization to improve your LLM-evaluators 11 | - 🔄 An iterative workflow to align annotators with AI output, and AI with annotator input 12 | 13 | While framed as a game to build LLM evaluators, you can use ALIGN Eval to craft and optimize any prompt that does binary classification! 14 | 15 | ## Getting Started 16 | 17 | First, run the development server: 18 | 19 | ```bash 20 | npm run dev 21 | # or 22 | yarn dev 23 | # or 24 | pnpm dev 25 | # or 26 | bun dev 27 | ``` 28 | 29 | Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. 30 | 31 | ## Tech Stack 32 | 33 | - **Framework**: Next.js with TypeScript 34 | - **Styling**: Tailwind CSS 35 | - **Fonts**: Uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to optimize and load [Geist](https://vercel.com/font) 36 | 37 | ## Contributing 38 | 39 | ALIGN Eval is currently in beta. Please share constructive feedback and report bugs on [GitHub](https://github.com/eugeneyan/align-app) or [X](https://x.com/eugeneyan). 40 | -------------------------------------------------------------------------------- /app/components/index/buttons.tsx: -------------------------------------------------------------------------------- 1 | import React, { useRef } from 'react'; 2 | 3 | interface FileUploadButtonProps { 4 | fileName: string | null; 5 | onFileSelect: (file: File | null) => void; 6 | } 7 | 8 | export const FileUploadButton = ({ fileName, onFileSelect }: FileUploadButtonProps) => { 9 | const fileInputRef = useRef(null); 10 | 11 | const handleClick = () => { 12 | fileInputRef.current?.click(); 13 | }; 14 | 15 | const handleChange = (e: React.ChangeEvent) => { 16 | const file = e.target.files?.[0] || null; 17 | onFileSelect(file); 18 | // Reset the input value to allow selecting the same file again 19 | if (fileInputRef.current) fileInputRef.current.value = ''; 20 | }; 21 | 22 | return ( 23 | <> 24 | 31 | 37 | 38 | ); 39 | }; 40 | 41 | interface ButtonProps { 42 | onClick: () => void; 43 | } 44 | 45 | export const DownloadButton = ({ onClick }: ButtonProps) => ( 46 | 52 | ); 53 | 54 | export const DeleteButton = ({ onClick }: ButtonProps) => ( 55 | 61 | ); 62 | -------------------------------------------------------------------------------- /public/info-icon.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/components/optimize/passwordPrompt.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useRef } from 'react'; 2 | 3 | interface PasswordPromptProps { 4 | onConfirm: (password: string) => Promise; 5 | onCancel: () => void; 6 | } 7 | 8 | export const PasswordPrompt = ({ onConfirm, onCancel }: PasswordPromptProps) => { 9 | const [errorMessage, setErrorMessage] = useState(''); 10 | const passwordRef = useRef(null); 11 | 12 | const handleConfirm = async () => { 13 | const password = passwordRef.current?.value || ''; 14 | const isSuccess = await onConfirm(password); 15 | if (!isSuccess) { 16 | setErrorMessage('Incorrect password!'); 17 | } 18 | }; 19 | 20 | const handleKeyDown = (e: React.KeyboardEvent) => { 21 | if (e.key === 'Enter') { 22 | handleConfirm(); 23 | } 24 | }; 25 | 26 | return ( 27 |
28 |
29 |

Enter Password to Clear Optimizations

30 |

31 | Hint: It's the Twitter handle of the person who made this. 32 |

33 | 40 | {errorMessage && ( 41 |

{errorMessage}

42 | )} 43 |
44 | 50 | 56 |
57 |
58 |
59 | ); 60 | }; 61 | -------------------------------------------------------------------------------- /app/components/optimize/floating.tsx: -------------------------------------------------------------------------------- 1 | // Add console.log to debug HPO_SAMPLES value 2 | const HPO_SAMPLES = Number(process.env.NEXT_PUBLIC_HPO_SAMPLES); 3 | 4 | interface ProgressBarProps { 5 | current: number | null; 6 | message: string; 7 | } 8 | 9 | const ProgressBar = ({ current, message }: ProgressBarProps) => { 10 | 11 | let percentage = 0; 12 | if (current) { 13 | percentage = Math.min((current / HPO_SAMPLES) * 100, 100); 14 | } 15 | 16 | const optimizationCompleted = message.toLowerCase().includes('optimization completed') || message.toLowerCase().includes('evaluating on test') 17 | const barColor = optimizationCompleted ? 'bg-blue-500' : 'bg-green-500'; 18 | const textColor = optimizationCompleted ? 'text-blue-700' : 'text-green-700'; 19 | 20 | return ( 21 |
22 |
23 |
27 |
28 |

29 | Processed {current || 0} of {HPO_SAMPLES} rows 30 |

31 |
32 | ); 33 | }; 34 | 35 | interface FloatingMessageProps { 36 | message: string; 37 | rowCount: number | 0; 38 | } 39 | 40 | export const FloatingMessage = ({ message, rowCount }: FloatingMessageProps) => { 41 | 42 | const getMessageColor = (message: string): string => { 43 | if (message.toLowerCase().includes('starting')) return 'bg-yellow-100 border-l-4 border-yellow-500 text-yellow-700'; 44 | if (message.toLowerCase().includes('completed') || message.toLowerCase().includes('ran')) return 'bg-blue-100 border-l-4 border-blue-500 text-blue-700'; 45 | if (message.toLowerCase().includes('running')) return 'bg-green-100 border-l-4 border-green-500 text-green-700'; 46 | return 'bg-gray-100 border-l-4 border-gray-500 text-gray-700'; // default color 47 | }; 48 | 49 | return ( 50 |
51 |

{message}

52 | {(rowCount !== null || message.toLowerCase().includes('optimization completed')) && ( 53 | 54 | )} 55 |
56 | ); 57 | }; 58 | -------------------------------------------------------------------------------- /app/lib/metrics.ts: -------------------------------------------------------------------------------- 1 | import type { Eval } from '@/app/lib/definitions'; 2 | 3 | interface Metrics { 4 | precision: number; 5 | recall: number; 6 | f1: number; 7 | cohensKappa: number; 8 | sampleSize: number; 9 | truePositives: number; 10 | trueNegatives: number; 11 | falsePositives: number; 12 | falseNegatives: number; 13 | } 14 | 15 | export const computeMetrics = (data: Eval[]): Metrics => { 16 | if (data.length === 0) return { 17 | precision: 0, 18 | recall: 0, 19 | f1: 0, 20 | cohensKappa: 0, 21 | sampleSize: 0, 22 | truePositives: 0, 23 | trueNegatives: 0, 24 | falsePositives: 0, 25 | falseNegatives: 0 26 | }; 27 | 28 | let truePositives = 0; 29 | let trueNegatives = 0; 30 | let falsePositives = 0; 31 | let falseNegatives = 0; 32 | let totalAgreement = 0; 33 | let totalItems = 0; 34 | 35 | data.forEach(row => { 36 | const prediction = String(row.prediction).trim(); 37 | const label = String(row.label).trim(); 38 | 39 | if (prediction !== '' && label !== '' && prediction !== 'null' && label !== 'null') { 40 | totalItems++; 41 | if (prediction === label) totalAgreement++; 42 | 43 | if (prediction === '1' && label === '1') { 44 | truePositives++; 45 | } else if (prediction === '1' && label === '0') { 46 | falsePositives++; 47 | } else if (prediction === '0' && label === '1') { 48 | falseNegatives++; 49 | } else if (prediction === '0' && label === '0') { 50 | trueNegatives++; 51 | } 52 | } 53 | }); 54 | 55 | const precision = truePositives / (truePositives + falsePositives) || 0; 56 | const recall = truePositives / (truePositives + falseNegatives) || 0; 57 | const f1 = 2 * (precision * recall) / (precision + recall) || 0; 58 | 59 | // Calculate Cohen's Kappa 60 | const observedAgreement = totalAgreement / totalItems; 61 | const expectedAgreement = ( 62 | ((truePositives + falsePositives) * (truePositives + falseNegatives) + 63 | (trueNegatives + falsePositives) * (trueNegatives + falseNegatives)) / 64 | (totalItems * totalItems) 65 | ); 66 | const cohensKappa = (observedAgreement - expectedAgreement) / (1 - expectedAgreement); 67 | 68 | return { 69 | precision, 70 | recall, 71 | f1, 72 | cohensKappa, 73 | sampleSize: totalItems, 74 | truePositives, 75 | trueNegatives, 76 | falsePositives, 77 | falseNegatives 78 | }; 79 | }; 80 | 81 | -------------------------------------------------------------------------------- /app/api/site-metrics/route.tsx: -------------------------------------------------------------------------------- 1 | import { NextRequest, NextResponse } from 'next/server' 2 | import { getDb } from '@/app/lib/db' 3 | 4 | // Initialize metrics table if it doesn't exist 5 | async function initializeMetricsTable() { 6 | const db = await getDb() 7 | await db.run(` 8 | CREATE TABLE IF NOT EXISTS site_metrics ( 9 | id SERIAL PRIMARY KEY, 10 | metric_name VARCHAR(50) NOT NULL, 11 | metric_value BIGINT DEFAULT 0, 12 | last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 13 | UNIQUE(metric_name) 14 | ) 15 | `) 16 | 17 | // Initialize metrics if they don't exist 18 | const metrics = [ 19 | 'upload_file_count', 20 | 'upload_row_count', 21 | 'labeled_file_count', 22 | 'labeled_row_count', 23 | 'evaluate_file_count', 24 | 'evaluate_row_count', 25 | 'optimized_file_count', 26 | 'optimized_trial_count' 27 | ] 28 | 29 | for (const metric of metrics) { 30 | await db.run(` 31 | INSERT INTO site_metrics (metric_name, metric_value) 32 | VALUES ($1, 0) 33 | ON CONFLICT (metric_name) DO NOTHING 34 | `, [metric]) 35 | } 36 | } 37 | 38 | // Increment a specific metric 39 | async function incrementMetric(metricName: string, incrementBy: number = 1) { 40 | const db = await getDb() 41 | console.log(`INCREMENT ${metricName} +${incrementBy}`) 42 | await db.run(` 43 | UPDATE site_metrics 44 | SET 45 | metric_value = metric_value + $1, 46 | last_updated = CURRENT_TIMESTAMP 47 | WHERE metric_name = $2 48 | `, [incrementBy, metricName]) 49 | } 50 | 51 | // Get all metrics 52 | export async function GET() { 53 | try { 54 | await initializeMetricsTable() 55 | const db = await getDb() 56 | const metrics = await db.all('SELECT * FROM site_metrics') 57 | return NextResponse.json(metrics) 58 | } catch (error) { 59 | console.error('Error fetching metrics:', error) 60 | return NextResponse.json({ error: 'Error fetching metrics' }, { status: 500 }) 61 | } 62 | } 63 | 64 | // Update metrics 65 | export async function POST(req: NextRequest) { 66 | try { 67 | const { metric_name, increment_by = 1 } = await req.json() 68 | 69 | if (!metric_name) { 70 | return NextResponse.json({ error: 'metric_name is required' }, { status: 400 }) 71 | } 72 | 73 | await initializeMetricsTable() 74 | await incrementMetric(metric_name, increment_by) 75 | 76 | return NextResponse.json({ success: true }) 77 | } catch (error) { 78 | console.error('Error updating metrics:', error) 79 | return NextResponse.json({ error: 'Error updating metrics' }, { status: 500 }) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /app/api/update/route.tsx: -------------------------------------------------------------------------------- 1 | import { NextRequest, NextResponse } from 'next/server' 2 | import { updateEvalRow, clearEvals, dropEvalTable } from '@/app/lib/actions' 3 | 4 | // Update a row in the database 5 | export async function PUT(req: NextRequest) { 6 | const { id, fileName, ...updates } = await req.json() 7 | 8 | if (!fileName) { 9 | return NextResponse.json({ error: 'File name is required' }, { status: 400 }) 10 | } 11 | 12 | // Update metrics 13 | const protocol = process.env.NODE_ENV === 'development' ? 'http' : 'https' 14 | const host = req.headers.get('host') || 'localhost:3000' 15 | const baseUrl = `${protocol}://${host}` 16 | 17 | fetch(`${baseUrl}/api/site-metrics`, { 18 | method: 'POST', 19 | headers: { 'Content-Type': 'application/json' }, 20 | body: JSON.stringify({ 21 | metric_name: 'labeled_row_count', 22 | increment_by: 1 23 | }) 24 | }).catch(error => console.error('Error updating row count metric:', error)) 25 | 26 | try { 27 | const result = await updateEvalRow(fileName, id, updates); 28 | return NextResponse.json({ message: result.message }) 29 | } catch (error) { 30 | console.error('Error updating row:', error) 31 | return NextResponse.json({ error: 'Failed to update row' }, { status: 500 }) 32 | } 33 | } 34 | 35 | // Clear explanations and predictions column 36 | export async function POST(req: NextRequest) { 37 | try { 38 | const { action, fileName } = await req.json() 39 | 40 | if (!fileName) { 41 | return NextResponse.json({ error: 'File name is required' }, { status: 400 }) 42 | } 43 | 44 | if (action === 'clearEvaluations') { 45 | const result = await clearEvals(fileName); 46 | return NextResponse.json({ message: result.message }, { status: 200 }) 47 | } else { 48 | return NextResponse.json({ error: 'Invalid action' }, { status: 400 }) 49 | } 50 | } catch (error) { 51 | console.error('Error clearing evaluations:', error) 52 | return NextResponse.json({ error: 'Error clearing evaluations' }, { status: 500 }) 53 | } 54 | } 55 | 56 | // Delete a table from the database 57 | export async function DELETE(req: NextRequest) { 58 | const fileName = req.nextUrl.searchParams.get('fileName') 59 | 60 | if (!fileName) { 61 | return NextResponse.json({ error: 'File name is required' }, { status: 400 }) 62 | } 63 | 64 | try { 65 | const result = await dropEvalTable(fileName); 66 | return NextResponse.json({ message: result.message }, { status: 200 }) 67 | } catch (error) { 68 | console.error('Error deleting table:', error) 69 | return NextResponse.json({ error: 'Error deleting table' }, { status: 500 }) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /app/lib/db.ts: -------------------------------------------------------------------------------- 1 | import sqlite3 from 'sqlite3' 2 | import { open, Database as SQLiteDatabase } from 'sqlite' 3 | import { Pool, QueryResult } from 'pg' 4 | 5 | // Define a generic type for database rows 6 | type DbRow = Record; 7 | 8 | export interface DbWrapper { 9 | all: (sql: string, params?: unknown[]) => Promise; 10 | get: (sql: string, params?: unknown[]) => Promise; 11 | run: (sql: string, params?: unknown[]) => Promise; 12 | close: () => Promise; 13 | } 14 | 15 | class SQLiteWrapper implements DbWrapper { 16 | private db: SQLiteDatabase; 17 | 18 | constructor(db: SQLiteDatabase) { 19 | this.db = db; 20 | } 21 | 22 | async all(sql: string, params?: unknown[]): Promise { 23 | return this.db.all(sql, params); 24 | } 25 | 26 | async get(sql: string, params?: unknown[]): Promise { 27 | return this.db.get(sql, params); 28 | } 29 | 30 | async run(sql: string, params?: unknown[]): Promise { 31 | await this.db.run(sql, params); 32 | } 33 | 34 | async close(): Promise { 35 | await this.db.close(); 36 | } 37 | } 38 | 39 | class PostgresWrapper implements DbWrapper { 40 | private pool: Pool; 41 | 42 | constructor(pool: Pool) { 43 | this.pool = pool; 44 | } 45 | 46 | async all(sql: string, params?: unknown[]): Promise { 47 | const client = await this.pool.connect(); 48 | try { 49 | const result: QueryResult = await client.query(sql, params); 50 | return result.rows; 51 | } finally { 52 | client.release(); 53 | } 54 | } 55 | 56 | async get(sql: string, params?: unknown[]): Promise { 57 | const client = await this.pool.connect(); 58 | try { 59 | const result: QueryResult = await client.query(sql, params); 60 | return result.rows[0]; 61 | } finally { 62 | client.release(); 63 | } 64 | } 65 | 66 | async run(sql: string, params?: unknown[]): Promise { 67 | const client = await this.pool.connect(); 68 | try { 69 | await client.query(sql, params); 70 | } finally { 71 | client.release(); 72 | } 73 | } 74 | 75 | async close(): Promise { 76 | await this.pool.end(); 77 | } 78 | } 79 | 80 | export async function getDb(): Promise { 81 | if (process.env.RAILWAY_ENVIRONMENT) { 82 | const pool = new Pool({ 83 | connectionString: process.env.DATABASE_URL, 84 | }); 85 | return new PostgresWrapper(pool); 86 | } else { 87 | const db = await open({ 88 | filename: '../database.sqlite', 89 | driver: sqlite3.Database 90 | }); 91 | return new SQLiteWrapper(db); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /app/layout.tsx: -------------------------------------------------------------------------------- 1 | import '@/app/components/globals.css'; 2 | import { inter } from "@/app/components/fonts"; 3 | import Footer from '@/app/components/footer'; 4 | 5 | import type { Metadata } from "next"; 6 | import Script from 'next/script'; 7 | import { Suspense } from 'react'; 8 | 9 | export const metadata: Metadata = { 10 | title: "AlignEval: Making Evals Easy, Fun, and Semi-Automated", 11 | description: "A prototype tool/game to help you look at your data, label it, evaluate output, and optimize evaluators.", 12 | icons: { 13 | icon: '/favicon.svg', 14 | }, 15 | openGraph: { 16 | title: 'AlignEval: Making Evals Easy, Fun, and Semi-Automated', 17 | description: 'A prototype tool/game to help you look at your data, label it, evaluate output, and optimize evaluators.', 18 | url: 'https://aligneval.com', 19 | siteName: 'AlignEval', 20 | images: [{ 21 | url: 'https://aligneval.com/og-image.jpg', 22 | secureUrl: 'https://aligneval.com/og-image.jpg', 23 | width: 1200, 24 | height: 630, 25 | alt: 'AlignEval: Upload, Label, Evaluate, Optimize', 26 | }], 27 | locale: 'en_US', 28 | type: 'website', 29 | }, 30 | twitter: { 31 | card: 'summary_large_image', 32 | title: 'AlignEval: Making Evals Easy, Fun, and Semi-Automated', 33 | description: 'A prototype tool/game to help you look at your data, label it, evaluate output, and optimize evaluators.', 34 | creator: '@eugeneyan', 35 | images: { 36 | url: 'https://aligneval.com/og-image.jpg', 37 | secureUrl: 'https://aligneval.com/og-image.jpg', 38 | alt: 'AlignEval: Upload, Label, Evaluate, Optimize', 39 | }, 40 | }, 41 | }; 42 | 43 | export default function RootLayout({ 44 | children, 45 | }: Readonly<{ 46 | children: React.ReactNode; 47 | }>) { 48 | return ( 49 | 50 | 51 |
52 | 54 |
Loading...
55 |
56 | }> 57 | {children} 58 | 59 | 60 |