├── .prettierignore ├── .eslintrc.json ├── public ├── paper.pdf ├── favicon.ico ├── favicon.zip ├── rubric_eval.png ├── twitter-card.png ├── examples │ ├── proof_1.png │ ├── proof_2.png │ ├── symbolic.png │ ├── math_symbolic_2.png │ ├── physics_symbolic_1.png │ └── physics_symbolic_2.png ├── parsed_results.png ├── rubric_example.png ├── types_of_problems.png ├── favicon │ ├── favicon-16x16.png │ ├── favicon-32x32.png │ ├── apple-touch-icon.png │ ├── android-chrome-192x192.png │ ├── android-chrome-512x512.png │ ├── about.txt │ └── site.webmanifest ├── sample_model_response.png └── fonts.css ├── src ├── screens │ ├── ARB │ │ ├── index.tsx │ │ └── Arb.tsx │ ├── Home │ │ ├── index.tsx │ │ ├── Components │ │ │ └── Mathjax.tsx │ │ └── Home.tsx │ ├── AnonDocs │ │ ├── index.tsx │ │ └── Docs.tsx │ └── Documentation │ │ ├── index.tsx │ │ └── Documentation.tsx ├── pages │ ├── home.tsx │ ├── index.tsx │ ├── docs.tsx │ ├── documentation.tsx │ ├── api │ │ ├── lawProblem.ts │ │ ├── mathProblem.ts │ │ ├── mcatReadingProblem.ts │ │ ├── mcatScienceProblem.ts │ │ ├── physicsProblem.ts │ │ ├── physicsImgProblem.ts │ │ ├── mcatScienceImageProblem.ts │ │ └── lib │ │ │ ├── law │ │ │ ├── index.ts │ │ │ └── [id].ts │ │ │ ├── math │ │ │ ├── numerical │ │ │ │ ├── index.ts │ │ │ │ └── [id].ts │ │ │ ├── prooflike │ │ │ │ ├── index.ts │ │ │ │ └── [id].ts │ │ │ └── symbolic │ │ │ │ ├── index.ts │ │ │ │ └── [id].ts │ │ │ ├── physics │ │ │ ├── numerical │ │ │ │ ├── noimg │ │ │ │ │ ├── index.ts │ │ │ │ │ └── [id].ts │ │ │ │ └── img │ │ │ │ │ ├── index.ts │ │ │ │ │ └── [id].ts │ │ │ └── symbolic │ │ │ │ ├── noimg │ │ │ │ ├── index.ts │ │ │ │ └── [id].ts │ │ │ │ └── img │ │ │ │ ├── index.ts │ │ │ │ └── [id].ts │ │ │ ├── mcatReading │ │ │ ├── val │ │ │ │ ├── index.ts │ │ │ │ └── [id].ts │ │ │ └── test │ │ │ │ ├── index.ts │ │ │ │ └── [id].ts │ │ │ ├── mcatScience │ │ │ ├── val │ │ │ │ ├── noimg │ │ │ │ │ ├── index.ts │ │ │ │ │ └── [id].ts │ │ │ │ └── img │ │ │ │ │ ├── index.ts │ │ │ │ │ └── [id].ts │ │ │ └── test │ │ │ │ ├── noimg │ │ │ │ ├── index.ts │ │ │ │ └── [id].ts │ │ │ │ └── img │ │ │ │ ├── index.ts │ │ │ │ └── [id].ts │ │ │ └── stats │ │ │ ├── math │ │ │ ├── numerical │ │ │ │ ├── index.ts │ │ │ │ └── [id].ts │ │ │ └── symbolic │ │ │ │ ├── index.ts │ │ │ │ └── [id].ts │ │ │ └── physics │ │ │ ├── numerical │ │ │ ├── index.ts │ │ │ └── [id].ts │ │ │ └── symbolic │ │ │ ├── index.ts │ │ │ └── [id].ts │ ├── _document.tsx │ └── _app.tsx ├── types │ ├── mathjax.d.ts │ └── Problem.ts ├── server │ ├── utils │ │ ├── request.ts │ │ └── middleware.ts │ └── mongodb │ │ ├── connectDb.js │ │ ├── models │ │ ├── stats_models.js │ │ ├── law_problem.js │ │ ├── mcat_problem.js │ │ └── numerical_problem.js │ │ └── actions │ │ ├── lawProblem.js │ │ ├── stats.js │ │ ├── mcatProblem.js │ │ └── numericalProblem.js └── styles │ └── globals.css ├── .husky └── pre-commit ├── postcss.config.js ├── next.config.js ├── .gitignore ├── tailwind.config.js ├── .github └── workflows │ └── lint.yml ├── tsconfig.json ├── LICENSE.txt ├── package.json └── README.md /.prettierignore: -------------------------------------------------------------------------------- 1 | **/.next -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals" 3 | } 4 | -------------------------------------------------------------------------------- /public/paper.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/paper.pdf -------------------------------------------------------------------------------- /src/screens/ARB/index.tsx: -------------------------------------------------------------------------------- 1 | import Arb from "./Arb"; 2 | 3 | export default Arb; 4 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/favicon.ico -------------------------------------------------------------------------------- /public/favicon.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/favicon.zip -------------------------------------------------------------------------------- /src/pages/home.tsx: -------------------------------------------------------------------------------- 1 | import Home from "../screens/Home"; 2 | 3 | export default Home; 4 | -------------------------------------------------------------------------------- /src/pages/index.tsx: -------------------------------------------------------------------------------- 1 | import Arb from "../screens/ARB"; 2 | 3 | export default Arb; 4 | -------------------------------------------------------------------------------- /src/screens/Home/index.tsx: -------------------------------------------------------------------------------- 1 | import Home from "./Home"; 2 | 3 | export default Home; 4 | -------------------------------------------------------------------------------- /public/rubric_eval.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/rubric_eval.png -------------------------------------------------------------------------------- /public/twitter-card.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/twitter-card.png -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | . "$(dirname -- "$0")/_/husky.sh" 3 | 4 | yarn lint-staged 5 | -------------------------------------------------------------------------------- /public/examples/proof_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/examples/proof_1.png -------------------------------------------------------------------------------- /public/examples/proof_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/examples/proof_2.png -------------------------------------------------------------------------------- /public/parsed_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/parsed_results.png -------------------------------------------------------------------------------- /public/rubric_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/rubric_example.png -------------------------------------------------------------------------------- /public/examples/symbolic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/examples/symbolic.png -------------------------------------------------------------------------------- /public/types_of_problems.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/types_of_problems.png -------------------------------------------------------------------------------- /public/favicon/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/favicon/favicon-16x16.png -------------------------------------------------------------------------------- /public/favicon/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/favicon/favicon-32x32.png -------------------------------------------------------------------------------- /public/sample_model_response.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/sample_model_response.png -------------------------------------------------------------------------------- /src/pages/docs.tsx: -------------------------------------------------------------------------------- 1 | import Documentation from "../screens/AnonDocs"; 2 | 3 | export default Documentation; 4 | -------------------------------------------------------------------------------- /src/screens/AnonDocs/index.tsx: -------------------------------------------------------------------------------- 1 | import Documentation from "./Docs"; 2 | 3 | export default Documentation; 4 | -------------------------------------------------------------------------------- /public/fonts.css: -------------------------------------------------------------------------------- 1 | @import url("https://fonts.googleapis.com/css2?family=Ubuntu:wght@300;400;500;700&display=swap"); 2 | -------------------------------------------------------------------------------- /public/examples/math_symbolic_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/examples/math_symbolic_2.png -------------------------------------------------------------------------------- /public/favicon/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/favicon/apple-touch-icon.png -------------------------------------------------------------------------------- /src/screens/Documentation/index.tsx: -------------------------------------------------------------------------------- 1 | import Documentation from "./Documentation"; 2 | 3 | export default Documentation; 4 | -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | }; 7 | -------------------------------------------------------------------------------- /public/examples/physics_symbolic_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/examples/physics_symbolic_1.png -------------------------------------------------------------------------------- /public/examples/physics_symbolic_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/examples/physics_symbolic_2.png -------------------------------------------------------------------------------- /src/pages/documentation.tsx: -------------------------------------------------------------------------------- 1 | import Documentation from "../screens/Documentation"; 2 | 3 | export default Documentation; 4 | -------------------------------------------------------------------------------- /public/favicon/android-chrome-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/favicon/android-chrome-192x192.png -------------------------------------------------------------------------------- /public/favicon/android-chrome-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TheDuckAI/arb/HEAD/public/favicon/android-chrome-512x512.png -------------------------------------------------------------------------------- /next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | const nextConfig = { 3 | reactStrictMode: true, 4 | }; 5 | 6 | module.exports = nextConfig; 7 | -------------------------------------------------------------------------------- /src/types/mathjax.d.ts: -------------------------------------------------------------------------------- 1 | interface MathJax { 2 | typesetPromise: () => Promise; 3 | } 4 | 5 | interface Window { 6 | MathJax?: MathJax; 7 | } 8 | -------------------------------------------------------------------------------- /public/favicon/about.txt: -------------------------------------------------------------------------------- 1 | This favicon was generated using the following graphics from Twitter Twemoji: 2 | 3 | - Graphics Title: 1f9d0.svg 4 | - Graphics Author: Copyright 2020 Twitter, Inc and other contributors (https://github.com/twitter/twemoji) 5 | - Graphics Source: https://github.com/twitter/twemoji/blob/master/assets/svg/1f9d0.svg 6 | - Graphics License: CC-BY 4.0 (https://creativecommons.org/licenses/by/4.0/) 7 | -------------------------------------------------------------------------------- /public/favicon/site.webmanifest: -------------------------------------------------------------------------------- 1 | { 2 | "name": "", 3 | "short_name": "", 4 | "icons": [ 5 | { 6 | "src": "/android-chrome-192x192.png", 7 | "sizes": "192x192", 8 | "type": "image/png" 9 | }, 10 | { 11 | "src": "/android-chrome-512x512.png", 12 | "sizes": "512x512", 13 | "type": "image/png" 14 | } 15 | ], 16 | "theme_color": "#ffffff", 17 | "background_color": "#ffffff", 18 | "display": "standalone" 19 | } 20 | -------------------------------------------------------------------------------- /src/pages/api/lawProblem.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getRandomLawProblem } from "../../server/mongodb/actions/lawProblem"; 3 | import requestWrapper from "../../server/utils/middleware"; 4 | 5 | const handler = async (req: NextApiRequest, res: NextApiResponse) => { 6 | const randomProblem = await getRandomLawProblem(); 7 | res.status(200).json({ randomProblem }); 8 | }; 9 | 10 | export default requestWrapper(handler, "GET"); 11 | -------------------------------------------------------------------------------- /src/pages/api/mathProblem.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getRandomMathProblem } from "../../server/mongodb/actions/numericalProblem"; 3 | import requestWrapper from "../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const randomProblem = await getRandomMathProblem(); 7 | res.status(200).json({ randomProblem }); 8 | } 9 | 10 | export default requestWrapper(handler, "GET"); 11 | -------------------------------------------------------------------------------- /src/pages/api/mcatReadingProblem.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getRandomReadingProblem } from "../../server/mongodb/actions/mcatProblem"; 3 | import requestWrapper from "../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const randomProblem = await getRandomReadingProblem(); 7 | res.status(200).json({ randomProblem }); 8 | } 9 | 10 | export default requestWrapper(handler, "GET"); 11 | -------------------------------------------------------------------------------- /src/pages/api/mcatScienceProblem.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getRandomScienceProblem } from "../../server/mongodb/actions/mcatProblem"; 3 | import requestWrapper from "../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const randomProblem = await getRandomScienceProblem(); 7 | res.status(200).json({ randomProblem }); 8 | } 9 | 10 | export default requestWrapper(handler, "GET"); 11 | -------------------------------------------------------------------------------- /src/pages/api/physicsProblem.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getRandomPhysicsProblem } from "../../server/mongodb/actions/numericalProblem"; 3 | import requestWrapper from "../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const randomProblem = await getRandomPhysicsProblem(); 7 | res.status(200).json({ randomProblem }); 8 | } 9 | 10 | export default requestWrapper(handler, "GET"); 11 | -------------------------------------------------------------------------------- /src/pages/api/physicsImgProblem.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getRandomPhysicsImgProblem } from "../../server/mongodb/actions/numericalProblem"; 3 | import requestWrapper from "../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const randomProblem = await getRandomPhysicsImgProblem(); 7 | res.status(200).json({ randomProblem }); 8 | } 9 | 10 | export default requestWrapper(handler, "GET"); 11 | -------------------------------------------------------------------------------- /src/pages/api/mcatScienceImageProblem.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getRandomScienceImagesProblem } from "../../server/mongodb/actions/mcatProblem"; 3 | import requestWrapper from "../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const randomProblem = await getRandomScienceImagesProblem(); 7 | res.status(200).json({ randomProblem }); 8 | } 9 | 10 | export default requestWrapper(handler, "GET"); 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | bun.lockb 8 | 9 | # testing 10 | /coverage 11 | 12 | # next.js 13 | /.next/ 14 | /out/ 15 | 16 | # production 17 | /build 18 | 19 | # misc 20 | .DS_Store 21 | *.pem 22 | 23 | # debug 24 | npm-debug.log* 25 | yarn-debug.log* 26 | yarn-error.log* 27 | 28 | # local env files 29 | .env 30 | 31 | # vercel 32 | .vercel 33 | 34 | # typescript 35 | *.tsbuildinfo 36 | next-env.d.ts -------------------------------------------------------------------------------- /src/server/utils/request.ts: -------------------------------------------------------------------------------- 1 | const sendRequest = async ( 2 | route: string, 3 | method: "GET" | "POST" | "PUT" | "DELETE" 4 | ): Promise => { 5 | let result: Response; 6 | if (method === "GET") { 7 | result = await fetch(route); 8 | } else { 9 | console.log("Invalid Method"); 10 | throw new Error("Invalid Method"); 11 | } 12 | 13 | if (!result.ok) { 14 | throw new Error(`Request failed with status ${result.status}`); 15 | } 16 | 17 | return result.json(); 18 | }; 19 | 20 | export default sendRequest; 21 | -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | module.exports = { 3 | content: [ 4 | "./src/pages/**/*.{js,ts,jsx,tsx,mdx}", 5 | "./src/components/**/*.{js,ts,jsx,tsx,mdx}", 6 | "./src/app/**/*.{js,ts,jsx,tsx,mdx}", 7 | ], 8 | theme: { 9 | extend: { 10 | backgroundImage: { 11 | "gradient-radial": "radial-gradient(var(--tw-gradient-stops))", 12 | "gradient-conic": 13 | "conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))", 14 | }, 15 | }, 16 | }, 17 | plugins: [], 18 | }; 19 | -------------------------------------------------------------------------------- /src/pages/api/lib/law/index.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getAllLawProblems } from "../../../../server/mongodb/actions/lawProblem"; 3 | import requestWrapper from "../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const allProblems = await getAllLawProblems(); 7 | 8 | const formattedResponse = JSON.stringify(allProblems, null, 2); 9 | 10 | res.setHeader("Content-Type", "application/json"); 11 | res.status(200).send(formattedResponse); 12 | } 13 | 14 | export default requestWrapper(handler, "GET"); 15 | -------------------------------------------------------------------------------- /src/pages/_document.tsx: -------------------------------------------------------------------------------- 1 | import Document, { Html, Head, Main, NextScript } from "next/document"; 2 | 3 | class MyDocument extends Document { 4 | render() { 5 | return ( 6 | 7 | 8 | 14 | 15 | 16 |
17 | 18 | 19 | 20 | ); 21 | } 22 | } 23 | 24 | export default MyDocument; 25 | -------------------------------------------------------------------------------- /src/pages/api/lib/math/numerical/index.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getAllMathProblems } from "../../../../../server/mongodb/actions/numericalProblem"; 3 | import requestWrapper from "../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const allProblems = await getAllMathProblems(); 7 | 8 | const formattedResponse = JSON.stringify(allProblems, null, 2); 9 | 10 | res.setHeader("Content-Type", "application/json"); 11 | res.status(200).send(formattedResponse); 12 | } 13 | 14 | export default requestWrapper(handler, "GET"); 15 | -------------------------------------------------------------------------------- /src/pages/api/lib/physics/numerical/noimg/index.ts: -------------------------------------------------------------------------------- 1 | import { getAllPhysicsProblems } from "@/server/mongodb/actions/numericalProblem"; 2 | import type { NextApiRequest, NextApiResponse } from "next"; 3 | import requestWrapper from "../../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const allProblems = await getAllPhysicsProblems(); 7 | 8 | const formattedResponse = JSON.stringify(allProblems, null, 2); 9 | 10 | res.setHeader("Content-Type", "application/json"); 11 | res.status(200).send(formattedResponse); 12 | } 13 | 14 | export default requestWrapper(handler, "GET"); 15 | -------------------------------------------------------------------------------- /src/pages/api/lib/math/prooflike/index.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getAllMathProofProblems } from "../../../../../server/mongodb/actions/numericalProblem"; 3 | import requestWrapper from "../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const allProblems = await getAllMathProofProblems(); 7 | 8 | const formattedResponse = JSON.stringify(allProblems, null, 2); 9 | 10 | res.setHeader("Content-Type", "application/json"); 11 | res.status(200).send(formattedResponse); 12 | } 13 | 14 | export default requestWrapper(handler, "GET"); 15 | -------------------------------------------------------------------------------- /src/pages/api/lib/mcatReading/val/index.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getAllReadingTestProblems } from "../../../../../server/mongodb/actions/mcatProblem"; 3 | import requestWrapper from "../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const allProblems = await getAllReadingTestProblems(); 7 | 8 | const formattedResponse = JSON.stringify(allProblems, null, 2); 9 | 10 | res.setHeader("Content-Type", "application/json"); 11 | res.status(200).send(formattedResponse); 12 | } 13 | 14 | export default requestWrapper(handler, "GET"); 15 | -------------------------------------------------------------------------------- /src/pages/api/lib/math/symbolic/index.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getAllMathSymbolicProblems } from "../../../../../server/mongodb/actions/numericalProblem"; 3 | import requestWrapper from "../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const allProblems = await getAllMathSymbolicProblems(); 7 | 8 | const formattedResponse = JSON.stringify(allProblems, null, 2); 9 | 10 | res.setHeader("Content-Type", "application/json"); 11 | res.status(200).send(formattedResponse); 12 | } 13 | 14 | export default requestWrapper(handler, "GET"); 15 | -------------------------------------------------------------------------------- /src/pages/api/lib/mcatReading/test/index.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getAllReadingTestProblems } from "../../../../../server/mongodb/actions/mcatProblem"; 3 | import requestWrapper from "../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const allProblems = await getAllReadingTestProblems(); 7 | 8 | const formattedResponse = JSON.stringify(allProblems, null, 2); 9 | 10 | res.setHeader("Content-Type", "application/json"); 11 | res.status(200).send(formattedResponse); 12 | } 13 | 14 | export default requestWrapper(handler, "GET"); 15 | -------------------------------------------------------------------------------- /src/pages/api/lib/mcatScience/val/noimg/index.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getAllScienceProblems } from "../../../../../../server/mongodb/actions/mcatProblem"; 3 | import requestWrapper from "../../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const allProblems = await getAllScienceProblems(); 7 | 8 | const formattedResponse = JSON.stringify(allProblems, null, 2); 9 | 10 | res.setHeader("Content-Type", "application/json"); 11 | res.status(200).send(formattedResponse); 12 | } 13 | 14 | export default requestWrapper(handler, "GET"); 15 | -------------------------------------------------------------------------------- /src/pages/api/lib/physics/symbolic/noimg/index.ts: -------------------------------------------------------------------------------- 1 | import { getAllPhysicsTestProblems } from "@/server/mongodb/actions/numericalProblem"; 2 | import type { NextApiRequest, NextApiResponse } from "next"; 3 | import requestWrapper from "../../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const allProblems = await getAllPhysicsTestProblems(); 7 | 8 | const formattedResponse = JSON.stringify(allProblems, null, 2); 9 | 10 | res.setHeader("Content-Type", "application/json"); 11 | res.status(200).send(formattedResponse); 12 | } 13 | 14 | export default requestWrapper(handler, "GET"); 15 | -------------------------------------------------------------------------------- /src/pages/api/lib/stats/math/numerical/index.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | 3 | import requestWrapper from "@/server/utils/middleware"; 4 | import { getStatsForMathNumericalProblems } from "../../../../../../server/mongodb/actions/stats"; 5 | 6 | async function handler(req: NextApiRequest, res: NextApiResponse) { 7 | const stats = await getStatsForMathNumericalProblems(); 8 | 9 | const formattedResponse = JSON.stringify(stats, null, 2); 10 | 11 | res.setHeader("Content-Type", "application/json"); 12 | res.status(200).send(formattedResponse); 13 | } 14 | 15 | export default requestWrapper(handler, "GET"); 16 | -------------------------------------------------------------------------------- /src/pages/api/lib/stats/math/symbolic/index.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | 3 | import requestWrapper from "@/server/utils/middleware"; 4 | import { getStatsForMathSymbolicProblems } from "../../../../../../server/mongodb/actions/stats"; 5 | 6 | async function handler(req: NextApiRequest, res: NextApiResponse) { 7 | const stats = await getStatsForMathSymbolicProblems(); 8 | 9 | const formattedResponse = JSON.stringify(stats, null, 2); 10 | 11 | res.setHeader("Content-Type", "application/json"); 12 | res.status(200).send(formattedResponse); 13 | } 14 | 15 | export default requestWrapper(handler, "GET"); 16 | -------------------------------------------------------------------------------- /src/pages/api/lib/mcatScience/test/noimg/index.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getAllScienceTestProblems } from "../../../../../../server/mongodb/actions/mcatProblem"; 3 | import requestWrapper from "../../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const allProblems = await getAllScienceTestProblems(); 7 | 8 | const formattedResponse = JSON.stringify(allProblems, null, 2); 9 | 10 | res.setHeader("Content-Type", "application/json"); 11 | res.status(200).send(formattedResponse); 12 | } 13 | 14 | export default requestWrapper(handler, "GET"); 15 | -------------------------------------------------------------------------------- /src/pages/api/lib/mcatScience/val/img/index.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getAllScienceImagesProblems } from "../../../../../../server/mongodb/actions/mcatProblem"; 3 | import requestWrapper from "../../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const allProblems = await getAllScienceImagesProblems(); 7 | 8 | const formattedResponse = JSON.stringify(allProblems, null, 2); 9 | 10 | res.setHeader("Content-Type", "application/json"); 11 | res.status(200).send(formattedResponse); 12 | } 13 | 14 | export default requestWrapper(handler, "GET"); 15 | -------------------------------------------------------------------------------- /src/pages/api/lib/physics/numerical/img/index.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getAllPhysicsImgProblems } from "../../../../../../server/mongodb/actions/numericalProblem"; 3 | import requestWrapper from "../../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const allProblems = await getAllPhysicsImgProblems(); 7 | 8 | const formattedResponse = JSON.stringify(allProblems, null, 2); 9 | 10 | res.setHeader("Content-Type", "application/json"); 11 | res.status(200).send(formattedResponse); 12 | } 13 | 14 | export default requestWrapper(handler, "GET"); 15 | -------------------------------------------------------------------------------- /src/pages/api/lib/stats/physics/numerical/index.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | 3 | import requestWrapper from "@/server/utils/middleware"; 4 | import { getStatsForPhysicsNumericalProblems } from "../../../../../../server/mongodb/actions/stats"; 5 | 6 | async function handler(req: NextApiRequest, res: NextApiResponse) { 7 | const stats = await getStatsForPhysicsNumericalProblems(); 8 | 9 | const formattedResponse = JSON.stringify(stats, null, 2); 10 | 11 | res.setHeader("Content-Type", "application/json"); 12 | res.status(200).send(formattedResponse); 13 | } 14 | 15 | export default requestWrapper(handler, "GET"); 16 | -------------------------------------------------------------------------------- /src/pages/api/lib/stats/physics/symbolic/index.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | 3 | import requestWrapper from "@/server/utils/middleware"; 4 | import { getStatsForPhysicsSymbolicProblems } from "../../../../../../server/mongodb/actions/stats"; 5 | 6 | async function handler(req: NextApiRequest, res: NextApiResponse) { 7 | const stats = await getStatsForPhysicsSymbolicProblems(); 8 | 9 | const formattedResponse = JSON.stringify(stats, null, 2); 10 | 11 | res.setHeader("Content-Type", "application/json"); 12 | res.status(200).send(formattedResponse); 13 | } 14 | 15 | export default requestWrapper(handler, "GET"); 16 | -------------------------------------------------------------------------------- /src/pages/api/lib/mcatScience/test/img/index.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getAllScienceImagesTestProblems } from "../../../../../../server/mongodb/actions/mcatProblem"; 3 | import requestWrapper from "../../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const allProblems = await getAllScienceImagesTestProblems(); 7 | 8 | const formattedResponse = JSON.stringify(allProblems, null, 2); 9 | 10 | res.setHeader("Content-Type", "application/json"); 11 | res.status(200).send(formattedResponse); 12 | } 13 | 14 | export default requestWrapper(handler, "GET"); 15 | -------------------------------------------------------------------------------- /src/pages/api/lib/physics/symbolic/img/index.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { getAllPhysicsImgTestProblems } from "../../../../../../server/mongodb/actions/numericalProblem"; 3 | import requestWrapper from "../../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | const allProblems = await getAllPhysicsImgTestProblems(); 7 | 8 | const formattedResponse = JSON.stringify(allProblems, null, 2); 9 | 10 | res.setHeader("Content-Type", "application/json"); 11 | res.status(200).send(formattedResponse); 12 | } 13 | 14 | export default requestWrapper(handler, "GET"); 15 | -------------------------------------------------------------------------------- /src/types/Problem.ts: -------------------------------------------------------------------------------- 1 | export interface LawProblem { 2 | "Problem Statement": string; 3 | "Problem Number": string; 4 | Topic: string; 5 | Source: string; 6 | "Answer Candidates": string[]; 7 | "Output Format Instructions": string; 8 | Solution: string; 9 | "Final Answer": string; 10 | Images: string[]; 11 | "Problem Type": string; 12 | _id: string; 13 | } 14 | 15 | export interface numericalProblem { 16 | Problem_Statement: string; 17 | "Problem Type": string; 18 | Topic: string; 19 | Solution: string; 20 | "Final Answer": string; 21 | Images: string[]; 22 | "Output Format Instructions": string; 23 | rubric: string; 24 | rubric_template: string; 25 | _id: string; 26 | } 27 | -------------------------------------------------------------------------------- /src/pages/_app.tsx: -------------------------------------------------------------------------------- 1 | import "@/styles/globals.css"; 2 | import type { AppProps } from "next/app"; 3 | import { ChakraProvider } from "@chakra-ui/react"; 4 | 5 | import { Lato } from "@next/font/google"; 6 | 7 | import { extendTheme } from "@chakra-ui/react"; 8 | 9 | const theme = extendTheme({ 10 | fonts: { 11 | heading: "Ubuntu", 12 | }, 13 | }); 14 | 15 | export const roboto = Lato({ 16 | subsets: ["latin"], 17 | weight: "300", 18 | }); 19 | 20 | export default function App({ Component, pageProps }: AppProps) { 21 | return ( 22 | 23 |
24 | 25 |
26 | 27 | ); 28 | } 29 | -------------------------------------------------------------------------------- /src/pages/api/lib/law/[id].ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { findLawProblemById } from "../../../../server/mongodb/actions/lawProblem"; 3 | import requestWrapper from "../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findLawProblemById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/styles/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | @import "../../public/fonts.css"; 5 | 6 | :root { 7 | --foreground-rgb: 0, 0, 0; 8 | --background-start-rgb: 214, 219, 220; 9 | --background-end-rgb: 255, 255, 255; 10 | } 11 | 12 | @media (prefers-color-scheme: dark) { 13 | :root { 14 | --foreground-rgb: 255, 255, 255; 15 | --background-start-rgb: 0, 0, 0; 16 | --background-end-rgb: 0, 0, 0; 17 | } 18 | } 19 | 20 | body { 21 | font-family: "Ubuntu", sans-serif; 22 | color: rgb(var(--foreground-rgb)); 23 | background: linear-gradient( 24 | to bottom, 25 | transparent, 26 | rgb(var(--background-end-rgb)) 27 | ) 28 | rgb(var(--background-start-rgb)); 29 | } 30 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | run-linters: 13 | name: Run linters 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - name: Check out Git repository 18 | uses: actions/checkout@v2 19 | 20 | - name: Set up Node.js 21 | uses: actions/setup-node@v1 22 | with: 23 | node-version: 16 24 | 25 | - name: Install Node.js dependencies 26 | run: yarn install --immutable --immutable-cache --check-cache 27 | 28 | - name: Run linters 29 | uses: wearerequired/lint-action@v2 30 | with: 31 | eslint: true 32 | prettier: false 33 | -------------------------------------------------------------------------------- /src/pages/api/lib/math/numerical/[id].ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { findMathProblemById } from "../../../../../server/mongodb/actions/numericalProblem"; 3 | import requestWrapper from "../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findMathProblemById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/pages/api/lib/mcatReading/val/[id].ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { findReadingProblemById } from "../../../../../server/mongodb/actions/mcatProblem"; 3 | import requestWrapper from "../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findReadingProblemById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/pages/api/lib/math/prooflike/[id].ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { findMathProofProblemById } from "../../../../../server/mongodb/actions/numericalProblem"; 3 | import requestWrapper from "../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findMathProofProblemById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/pages/api/lib/mcatReading/test/[id].ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { findReadingTestProblemById } from "../../../../../server/mongodb/actions/mcatProblem"; 3 | import requestWrapper from "../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findReadingTestProblemById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/pages/api/lib/mcatScience/val/noimg/[id].ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { findScienceProblemById } from "../../../../../../server/mongodb/actions/mcatProblem"; 3 | import requestWrapper from "../../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findScienceProblemById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/server/mongodb/connectDb.js: -------------------------------------------------------------------------------- 1 | import mongoose from "mongoose"; 2 | import dotenv from "dotenv"; 3 | 4 | dotenv.config(); 5 | 6 | const connectDb = async () => { 7 | const MONGO_URI = process.env.MONGO_URI; 8 | 9 | if (!MONGO_URI) { 10 | return { 11 | success: false, 12 | message: "MongoDB connection string is not set", 13 | }; 14 | } 15 | 16 | if (!global.cache) { 17 | try { 18 | global.cache = await mongoose.connect(MONGO_URI); 19 | return { success: true }; 20 | } catch (error) { 21 | console.log(error); 22 | return { 23 | success: false, 24 | message: "Error connecting to MongoDB", 25 | }; 26 | } 27 | } 28 | return { success: true }; 29 | }; 30 | 31 | export default connectDb; 32 | -------------------------------------------------------------------------------- /src/pages/api/lib/math/symbolic/[id].ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { findMathSymbolicProblemById } from "../../../../../server/mongodb/actions/numericalProblem"; 3 | import requestWrapper from "../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findMathSymbolicProblemById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/pages/api/lib/mcatScience/test/noimg/[id].ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { findScienceTestProblemById } from "../../../../../../server/mongodb/actions/mcatProblem"; 3 | import requestWrapper from "../../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findScienceTestProblemById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/pages/api/lib/mcatScience/val/img/[id].ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { findScienceImagesProblemById } from "../../../../../../server/mongodb/actions/mcatProblem"; 3 | import requestWrapper from "../../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findScienceImagesProblemById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/pages/api/lib/physics/numerical/noimg/[id].ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { findPhysicsProblemById } from "../../../../../../server/mongodb/actions/numericalProblem"; 3 | import requestWrapper from "../../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findPhysicsProblemById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/pages/api/lib/stats/math/symbolic/[id].ts: -------------------------------------------------------------------------------- 1 | import { findStatsForMathSymbolicProblemsById } from "@/server/mongodb/actions/stats"; 2 | import requestWrapper from "@/server/utils/middleware"; 3 | import type { NextApiRequest, NextApiResponse } from "next"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findStatsForMathSymbolicProblemsById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/pages/api/lib/physics/numerical/img/[id].ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { findPhysicsImgProblemById } from "../../../../../../server/mongodb/actions/numericalProblem"; 3 | import requestWrapper from "../../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findPhysicsImgProblemById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/pages/api/lib/physics/symbolic/noimg/[id].ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { findPhysicsTestProblemById } from "../../../../../../server/mongodb/actions/numericalProblem"; 3 | import requestWrapper from "../../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findPhysicsTestProblemById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/pages/api/lib/stats/math/numerical/[id].ts: -------------------------------------------------------------------------------- 1 | import { findStatsForMathNumericalProblemsById } from "@/server/mongodb/actions/stats"; 2 | import requestWrapper from "@/server/utils/middleware"; 3 | import type { NextApiRequest, NextApiResponse } from "next"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findStatsForMathNumericalProblemsById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/pages/api/lib/mcatScience/test/img/[id].ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { findScienceImagesTestProblemById } from "../../../../../../server/mongodb/actions/mcatProblem"; 3 | import requestWrapper from "../../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findScienceImagesTestProblemById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/pages/api/lib/physics/symbolic/img/[id].ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import { findPhysicsImgTestProblemById } from "../../../../../../server/mongodb/actions/numericalProblem"; 3 | import requestWrapper from "../../../../../../server/utils/middleware"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findPhysicsImgTestProblemById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/pages/api/lib/stats/physics/symbolic/[id].ts: -------------------------------------------------------------------------------- 1 | import { findStatsForPhysicsSymbolicProblemsById } from "@/server/mongodb/actions/stats"; 2 | import requestWrapper from "@/server/utils/middleware"; 3 | import type { NextApiRequest, NextApiResponse } from "next"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findStatsForPhysicsSymbolicProblemsById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /src/pages/api/lib/stats/physics/numerical/[id].ts: -------------------------------------------------------------------------------- 1 | import { findStatsForPhysicsNumericalProblemsById } from "@/server/mongodb/actions/stats"; 2 | import requestWrapper from "@/server/utils/middleware"; 3 | import type { NextApiRequest, NextApiResponse } from "next"; 4 | 5 | async function handler(req: NextApiRequest, res: NextApiResponse) { 6 | if (req.method !== "GET") { 7 | res.status(405).end(); 8 | return; 9 | } 10 | const { id } = req.query; 11 | const problem = await findStatsForPhysicsNumericalProblemsById(id); 12 | 13 | const formattedResponse = JSON.stringify(problem, null, 2); 14 | 15 | res.setHeader("Content-Type", "application/json"); 16 | res.status(200).send(formattedResponse); 17 | } 18 | 19 | export default requestWrapper(handler, "GET"); 20 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": ["dom", "dom.iterable", "esnext"], 5 | "allowJs": true, 6 | "skipLibCheck": true, 7 | "strict": true, 8 | "forceConsistentCasingInFileNames": true, 9 | "noEmit": true, 10 | "esModuleInterop": true, 11 | "module": "esnext", 12 | "moduleResolution": "node", 13 | "resolveJsonModule": true, 14 | "isolatedModules": true, 15 | "jsx": "preserve", 16 | "typeRoots": ["./types", "./node_modules/@types"], 17 | "incremental": true, 18 | "paths": { 19 | "@/*": ["./src/*"] 20 | } 21 | }, 22 | "include": [ 23 | "next-env.d.ts", 24 | "**/*.ts", 25 | "**/*.tsx", 26 | "src/pages/api/loadData.js", 27 | "src/pages/api/lawProblem.ts" 28 | ], 29 | "exclude": ["node_modules"] 30 | } 31 | -------------------------------------------------------------------------------- /src/server/mongodb/models/stats_models.js: -------------------------------------------------------------------------------- 1 | import mongoose from "mongoose"; 2 | 3 | const statsSchema = new Schema({ 4 | _id: mongoose.Schema.Types.ObjectId, 5 | numberOfQuestions: { 6 | type: String, 7 | required: true, 8 | }, 9 | percentage: { 10 | type: String, 11 | required: true, 12 | }, 13 | subjectName: { 14 | type: String, 15 | required: true, 16 | } 17 | }); 18 | 19 | export const mathNumericalStatsModel = mongoose.models.mathNumericalStats || mongoose.model("mathNumericalStats", statsSchema, "stats_numerical"); 20 | 21 | export const mathSymbolicStatsModel = mongoose.models.mathSymbolicStats || mongoose.model("mathSymbolicStats", statsSchema, "stats_symbolic"); 22 | 23 | export const physicsNumericalStatsModel = mongoose.models.physicsNumericalStats || mongoose.model("physicsNumericalStats", statsSchema, "stats_physics_numerical"); 24 | 25 | export const physicsSymbolicStatsModel = mongoose.models.physicsSymbolicStats || mongoose.model("physicsSymbolicStats", statsSchema, "stats_physics_symbolic"); -------------------------------------------------------------------------------- /src/server/mongodb/models/law_problem.js: -------------------------------------------------------------------------------- 1 | import mongoose from "mongoose"; 2 | 3 | const lawProblemSchema = new mongoose.Schema({ 4 | _id: mongoose.Schema.Types.ObjectId, 5 | problemStatement: { 6 | type: String, 7 | required: true, 8 | }, 9 | problemNumber: { 10 | type: String, 11 | required: false, 12 | }, 13 | topic: { 14 | type: String, 15 | required: false, 16 | }, 17 | source: { 18 | type: String, 19 | required: false, 20 | }, 21 | answerCandidates: { 22 | type: [String], 23 | required: true, 24 | }, 25 | outputFormatInstructions: { 26 | type: String, 27 | required: false, 28 | }, 29 | solution: { 30 | type: String, 31 | required: false, 32 | }, 33 | finalAnswer: { 34 | type: String, 35 | required: true, 36 | }, 37 | images: { 38 | type: [String], 39 | required: false, 40 | }, 41 | problemType: { 42 | type: String, 43 | }, 44 | }); 45 | 46 | export const lawProblemModel = 47 | mongoose.models.lawProblem || 48 | mongoose.model("lawProblem", lawProblemSchema, "law"); 49 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2023 Scott Chacon and others 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /src/server/mongodb/actions/lawProblem.js: -------------------------------------------------------------------------------- 1 | import { lawProblemModel } from "../models/law_problem"; 2 | 3 | import { ObjectId } from "mongodb"; 4 | 5 | async function findOneById(model, id, projection) { 6 | if (!ObjectId.isValid(id)) { 7 | throw new Error("Invalid MongoDB ID format"); 8 | } 9 | return await model.findById(id, projection); 10 | } 11 | 12 | async function getRandomLawProblem() { 13 | const randomDocument = await lawProblemModel.aggregate([ 14 | { $sample: { size: 1 } }, 15 | ]); 16 | return randomDocument[0]; 17 | } 18 | 19 | async function getAllLawProblems() { 20 | return await lawProblemModel.find( 21 | {}, 22 | { 23 | "Problem Statement": 1, 24 | "Problem Number": 1, 25 | "Answer Candidates": 1, 26 | "Final Answer": 1, 27 | "Problem Type": 1, 28 | } 29 | ); 30 | } 31 | 32 | async function findLawProblemById(id) { 33 | return await findOneById(lawProblemModel, id, { 34 | "Problem Statement": 1, 35 | "Answer Candidates": 1, 36 | "Final Answer": 1, 37 | }); 38 | } 39 | 40 | export { findLawProblemById, getRandomLawProblem, getAllLawProblems }; 41 | -------------------------------------------------------------------------------- /src/server/utils/middleware.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from "next"; 2 | import connectDb from "../mongodb/connectDb"; 3 | 4 | type RequestMethod = "GET" | "POST" | "PUT" | "DELETE"; 5 | 6 | export default function requestWrapper( 7 | handler: (req: NextApiRequest, res: NextApiResponse) => void | Promise, 8 | method: RequestMethod 9 | ) { 10 | return async (req: NextApiRequest, res: NextApiResponse) => { 11 | if (req.method !== method) { 12 | return res.status(400).json({ 13 | success: false, 14 | message: "Request Failure: Invalid method for request", 15 | }); 16 | } 17 | 18 | if (method !== "GET") { 19 | try { 20 | req.body = JSON.parse(req.body); 21 | } catch (error) { 22 | console.log(error); 23 | return res.status(400).json({ 24 | success: false, 25 | message: "Invalid request body", 26 | }); 27 | } 28 | } 29 | 30 | try { 31 | const mongoRes = await connectDb(); 32 | if (!mongoRes.success) { 33 | return res.status(500).json(mongoRes); 34 | } 35 | } catch (error) { 36 | console.error(error); 37 | return res.status(500).json({ 38 | success: false, 39 | message: "Server Error", 40 | }); 41 | } 42 | 43 | return handler(req, res); 44 | }; 45 | } 46 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "arb-jsonl-web-interface", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "@chakra-ui/react": "^2.8.0", 13 | "@emotion/react": "^11.11.1", 14 | "@emotion/styled": "^11.11.0", 15 | "@next/font": "^13.4.12", 16 | "@types/node": "20.4.1", 17 | "@types/react": "18.2.14", 18 | "@types/react-dom": "18.2.6", 19 | "autoprefixer": "10.4.14", 20 | "axios": "^1.4.0", 21 | "dotenv": "^16.3.1", 22 | "eslint": "8.44.0", 23 | "eslint-config-next": "13.4.9", 24 | "extract-mongo-schema": "^0.2.11", 25 | "framer-motion": "^10.13.0", 26 | "husky": "^8.0.3", 27 | "lint-staged": "^13.2.3", 28 | "lodash": "^4.17.21", 29 | "mongoose": "^7.3.3", 30 | "next": "^13.4.9", 31 | "postcss": "8.4.25", 32 | "prettier": "^3.0.0", 33 | "react": "18.2.0", 34 | "react-dom": "18.2.0", 35 | "react-icons": "^4.10.1", 36 | "tailwindcss": "3.3.2", 37 | "typescript": "5.1.6" 38 | }, 39 | "husky": { 40 | "hooks": { 41 | "pre-commit": "lint-staged" 42 | } 43 | }, 44 | "lint-staged": { 45 | "*.{js,ts,tsx}": [ 46 | "eslint --fix", 47 | "prettier --write" 48 | ] 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/screens/Home/Components/Mathjax.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useState } from "react"; 2 | 3 | interface MathJaxProps { 4 | problemStatement: string; 5 | } 6 | 7 | const MathJaxComponent: React.FC = ({ problemStatement }) => { 8 | const [parsedContent, setParsedContent] = useState(""); 9 | 10 | useEffect(() => { 11 | const regex = /\$\$(.*?)\$\$|\$(.*?)\$|\\\((.*?)\\\)|\\\[(.*?)\\\]/g; 12 | let result; 13 | 14 | let parsedString = problemStatement; 15 | 16 | while ((result = regex.exec(parsedString)) !== null) { 17 | if (result[1]) { 18 | parsedString = parsedString.replace(result[0], `\\[${result[1]}\\]`); 19 | } else if (result[2]) { 20 | parsedString = parsedString.replace(result[0], `\\(${result[2]}\\)`); 21 | } 22 | } 23 | 24 | setParsedContent(parsedString); 25 | }, [problemStatement]); 26 | 27 | useEffect(() => { 28 | if (typeof window !== "undefined" && window.MathJax && parsedContent) { 29 | window.MathJax.typesetPromise() 30 | .then(() => { 31 | console.log("Typesetting complete."); 32 | }) 33 | .catch((err) => { 34 | console.warn("MathJax typesetting failed:", err); 35 | }); 36 | } 37 | }, [parsedContent]); 38 | 39 | return ; 40 | }; 41 | 42 | export default MathJaxComponent; 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | duckai logo 3 |
4 | Advanced Reasoning Benchmark 5 |
6 |

7 | 8 |

9 | arXiv 10 | Lint Status 11 | 12 |

13 | 14 |
A DuckAI project in collaboration with the Georgia Institute of Technology, ETH Zürich, Nomos AI, Stanford University Center for Legal Informatics, and the Mila - Quebec AI Institute
15 | 16 | ### Abstract 17 | 18 | ARB is a novel benchmark dataset composed of advanced reasoning problems designed to evaluate LLMs on text comprehension and expert domain reasoning, presenting a more challenging test than prior benchmarks, featuring questions that test deeper knowledge of mathematics, physics, biology, chemistry, and law. 19 | 20 | ### API Usage 21 | 22 | Endpoint url: https://advanced-reasoning-benchmark.netlify.app/api/ 23 |
24 | The documentation for the complete REST API of the ARB dataset is [here](https://advanced-reasoning-benchmark.netlify.app/api/). 25 | 26 | Copyright © 2023 [DuckAI](https://github.com/TheDuckAI) 27 | -------------------------------------------------------------------------------- /src/server/mongodb/models/mcat_problem.js: -------------------------------------------------------------------------------- 1 | import mongoose from "mongoose"; 2 | 3 | const mcatSchema = new mongoose.Schema({ 4 | _id: mongoose.Schema.Types.ObjectId, 5 | problemStatement: { 6 | type: String, 7 | required: true, 8 | }, 9 | problemNumber: { 10 | type: String, 11 | required: false, 12 | }, 13 | answerCandidates: { 14 | type: [String], 15 | required: true, 16 | }, 17 | images: { 18 | type: [String], 19 | required: false, 20 | }, 21 | topic: { 22 | type: String, 23 | required: false, 24 | }, 25 | source: { 26 | type: String, 27 | required: false, 28 | }, 29 | solution: { 30 | type: String, 31 | required: false, 32 | }, 33 | finalAnswer: { 34 | type: String, 35 | required: true, 36 | }, 37 | outputFormatInstructions: { 38 | type: String, 39 | required: false, 40 | }, 41 | subject: { 42 | type: String, 43 | required: true, 44 | }, 45 | problemType: { 46 | type: String, 47 | required: true, 48 | enum: ["Multiple choice", "Numerical", "Boolean", "Text"], 49 | }, 50 | }); 51 | 52 | export const mcatReadingValModel = 53 | mongoose.models.mcatReadingVal || 54 | mongoose.model("mcatReadingVal", mcatSchema, "mcat_reading_val"); 55 | export const mcatScienceImgModel = 56 | mongoose.models.mcatScienceImg || 57 | mongoose.model("mcatScienceImg", mcatSchema, "mcat_science_images_val"); 58 | export const mcatScienceValModel = 59 | mongoose.models.mcatScienceVal || 60 | mongoose.model("mcatScienceVal", mcatSchema, "mcat_science_val"); 61 | 62 | export const mcatReadingTestModel = 63 | mongoose.models.mcatReadingTest || 64 | mongoose.model("mcatReadingTest", mcatSchema, "mcat_reading_test"); 65 | 66 | export const mcatScienceTestModel = 67 | mongoose.models.mcatScienceTest || 68 | mongoose.model("mcatScienceTest", mcatSchema, "mcat_science_test"); 69 | 70 | export const mcatScienceImgTestModel = 71 | mongoose.models.mcatScienceImgTest || 72 | mongoose.model("mcatScienceImgTest", mcatSchema, "mcat_science_images_test"); 73 | -------------------------------------------------------------------------------- /src/server/mongodb/models/numerical_problem.js: -------------------------------------------------------------------------------- 1 | import mongoose from "mongoose"; 2 | 3 | const numericalProblemSchema = new mongoose.Schema({ 4 | _id: mongoose.Schema.Types.ObjectId, 5 | Problem_Statement: { 6 | type: String, 7 | required: true, 8 | }, 9 | Topic: { 10 | type: String, 11 | required: true, 12 | }, 13 | Source: String, 14 | Answer_Candidates: [String], 15 | Images: [String], 16 | Output_Format_Instructions: String, 17 | Solution: { 18 | type: String, 19 | required: true, 20 | }, 21 | Final_Answer: { 22 | type: String, 23 | required: true, 24 | }, 25 | Problem_Type: { 26 | type: String, 27 | required: true, 28 | }, 29 | Answer_Choices: [String], 30 | rubric: { 31 | type: String, 32 | required: true, 33 | }, 34 | rubric_template: { 35 | type: String, 36 | required: true, 37 | }, 38 | }); 39 | 40 | export const mathNumericalModel = 41 | mongoose.models.mathNumericalProblem || 42 | mongoose.model( 43 | "mathNumericalProblem", 44 | numericalProblemSchema, 45 | "math_numerical" 46 | ); 47 | export const physicsNumericalModel = 48 | mongoose.models.physicsNumericalProblem || 49 | mongoose.model( 50 | "physicsNumericalProblem", 51 | numericalProblemSchema, 52 | "physics_numerical" 53 | ); 54 | export const physicsImgModel = 55 | mongoose.models.physicsImgProblem || 56 | mongoose.model( 57 | "physicsImgProblem", 58 | numericalProblemSchema, 59 | "physics_numerical_images" 60 | ); 61 | 62 | export const mathProofsModel = 63 | mongoose.models.mathProofsProblem || 64 | mongoose.model("mathProofsProblem", numericalProblemSchema, "mathProofs"); 65 | 66 | export const mathSymbolicModel = 67 | mongoose.models.mathSymbolicModel || 68 | mongoose.model("mathSymbolicModel", numericalProblemSchema, "mathSymbolic"); 69 | 70 | export const physicsTestModel = 71 | mongoose.models.physicsTestProblem || 72 | mongoose.model("physicsTestProblem", numericalProblemSchema, "physics_test"); 73 | 74 | export const physicsTestImgModel = 75 | mongoose.models.physicsTestImgProblem || 76 | mongoose.model( 77 | "physicsTestImgProblem", 78 | numericalProblemSchema, 79 | "physics_test_img" 80 | ); 81 | -------------------------------------------------------------------------------- /src/server/mongodb/actions/stats.js: -------------------------------------------------------------------------------- 1 | import { 2 | mathNumericalStatsModel, 3 | mathSymbolicStatsModel, 4 | physicsNumericalStatsModel, 5 | physicsSymbolicStatsModel 6 | } from "../models/stats_models.js"; 7 | 8 | async function findOneByMongoId(model, id, projection) { 9 | if (!ObjectId.isValid(id)) { 10 | throw new Error("Invalid MongoDB ID format"); 11 | } 12 | return await model.findById(id, projection); 13 | } 14 | 15 | async function getStatsForMathNumericalProblems() { 16 | return await mathNumericalStatsModel.find({}, { 17 | numberOfQuestions: 1, 18 | percentage: 1, 19 | subjectName: 1, 20 | }); 21 | } 22 | 23 | async function getStatsForMathSymbolicProblems() { 24 | return await mathSymbolicStatsModel.find({}, { 25 | numberOfQuestions: 1, 26 | percentage: 1, 27 | subjectName: 1, 28 | }); 29 | } 30 | 31 | async function getStatsForPhysicsNumericalProblems() { 32 | return await physicsNumericalStatsModel.find({}, { 33 | numberOfQuestions: 1, 34 | percentage: 1, 35 | subjectName: 1, 36 | }); 37 | } 38 | 39 | async function getStatsForPhysicsSymbolicProblems() { 40 | return await physicsSymbolicStatsModel.find({}, { 41 | numberOfQuestions: 1, 42 | percentage: 1, 43 | subjectName: 1, 44 | }); 45 | } 46 | 47 | async function findStatsForMathNumericalProblemsById(id) { 48 | return findOneByMongoId(mathNumericalStatsModel, id, { 49 | numberOfQuestions: 1, 50 | percentage: 1, 51 | subjectName: 1, 52 | }); 53 | } 54 | 55 | async function findStatsForMathSymbolicProblemsById(id) { 56 | return findOneByMongoId(mathSymbolicStatsModel, id, { 57 | numberOfQuestions: 1, 58 | percentage: 1, 59 | subjectName: 1, 60 | }); 61 | } 62 | 63 | async function findStatsForPhysicsNumericalProblemsById(id) { 64 | return findOneByMongoId(physicsNumericalStatsModel, id, { 65 | numberOfQuestions: 1, 66 | percentage: 1, 67 | subjectName: 1, 68 | }); 69 | } 70 | 71 | async function findStatsForPhysicsSymbolicProblemsById(id) { 72 | return findOneByMongoId(physicsSymbolicStatsModel, id, { 73 | numberOfQuestions: 1, 74 | percentage: 1, 75 | subjectName: 1, 76 | }); 77 | } 78 | 79 | export { 80 | findStatsForMathNumericalProblemsById, 81 | findStatsForMathSymbolicProblemsById, 82 | findStatsForPhysicsNumericalProblemsById, 83 | findStatsForPhysicsSymbolicProblemsById, 84 | getStatsForMathNumericalProblems, 85 | getStatsForMathSymbolicProblems, 86 | getStatsForPhysicsNumericalProblems, 87 | getStatsForPhysicsSymbolicProblems 88 | }; 89 | 90 | -------------------------------------------------------------------------------- /src/server/mongodb/actions/mcatProblem.js: -------------------------------------------------------------------------------- 1 | import { 2 | mcatReadingTestModel, 3 | mcatReadingValModel, 4 | mcatScienceImgModel, 5 | mcatScienceImgTestModel, 6 | mcatScienceTestModel, 7 | mcatScienceValModel, 8 | } from "../models/mcat_problem"; 9 | 10 | import _ from "lodash"; 11 | 12 | import { ObjectId } from "mongodb"; 13 | 14 | async function findOneById(model, id, projection) { 15 | if (!ObjectId.isValid(id)) { 16 | throw new Error("Invalid MongoDB ID format"); 17 | } 18 | return await model.findById(id, projection); 19 | } 20 | 21 | async function getRandomReadingProblem() { 22 | const randomDocuments = await mcatReadingValModel.aggregate([ 23 | { $sample: { size: 5 } }, 24 | ]); 25 | return _.sample(randomDocuments); 26 | } 27 | 28 | async function getRandomScienceProblem() { 29 | const randomDocuments = await mcatScienceValModel.aggregate([ 30 | { $sample: { size: 5 } }, 31 | ]); 32 | return _.sample(randomDocuments); 33 | } 34 | 35 | async function getRandomScienceImagesProblem() { 36 | const randomDocuments = await mcatScienceImgModel.aggregate([ 37 | { $sample: { size: 5 } }, 38 | ]); 39 | return _.sample(randomDocuments); 40 | } 41 | 42 | async function getAllReadingProblems() { 43 | return await mcatReadingValModel.find( 44 | {}, 45 | { 46 | "Problem Statement": 1, 47 | "Answer Candidates": 1, 48 | "Problem Number": 1, 49 | Topic: 1, 50 | Solution: 1, 51 | "Final Answer": 1, 52 | Subject: 1, 53 | "Problem Type": 1, 54 | } 55 | ); 56 | } 57 | 58 | async function getAllScienceProblems() { 59 | return await mcatScienceValModel.find( 60 | {}, 61 | { 62 | "Problem Statement": 1, 63 | "Answer Candidates": 1, 64 | "Problem Number": 1, 65 | Topic: 1, 66 | Solution: 1, 67 | "Final Answer": 1, 68 | Subject: 1, 69 | "Problem Type": 1, 70 | } 71 | ); 72 | } 73 | 74 | async function getAllScienceImagesProblems() { 75 | return await mcatScienceImgModel.find( 76 | {}, 77 | { 78 | "Problem Statement": 1, 79 | Images: 1, 80 | "Answer Candidates": 1, 81 | "Problem Number": 1, 82 | Topic: 1, 83 | Solution: 1, 84 | "Final Answer": 1, 85 | Subject: 1, 86 | "Problem Type": 1, 87 | } 88 | ); 89 | } 90 | 91 | async function getAllScienceImagesTestProblems() { 92 | return await mcatScienceImgTestModel.find( 93 | {}, 94 | { 95 | "Problem Statement": 1, 96 | Images: 1, 97 | "Answer Candidates": 1, 98 | "Problem Number": 1, 99 | Topic: 1, 100 | Solution: 1, 101 | "Final Answer": 1, 102 | Subject: 1, 103 | "Problem Type": 1, 104 | } 105 | ); 106 | } 107 | 108 | async function getAllScienceTestProblems() { 109 | return await mcatScienceTestModel.find( 110 | {}, 111 | { 112 | "Problem Statement": 1, 113 | "Answer Candidates": 1, 114 | "Problem Number": 1, 115 | Topic: 1, 116 | Solution: 1, 117 | "Final Answer": 1, 118 | Subject: 1, 119 | "Problem Type": 1, 120 | } 121 | ); 122 | } 123 | 124 | async function getAllReadingTestProblems() { 125 | return await mcatReadingTestModel.find( 126 | {}, 127 | { 128 | "Problem Statement": 1, 129 | "Answer Candidates": 1, 130 | "Problem Number": 1, 131 | Topic: 1, 132 | Solution: 1, 133 | "Final Answer": 1, 134 | Subject: 1, 135 | "Problem Type": 1, 136 | } 137 | ); 138 | } 139 | 140 | async function findReadingProblemById(id) { 141 | return await findOneById(mcatReadingValModel, id, { 142 | "Problem Statement": 1, 143 | "Answer Candidates": 1, 144 | Solution: 1, 145 | "Final Answer": 1, 146 | }); 147 | } 148 | 149 | async function findReadingTestProblemById(id) { 150 | return await findOneById(mcatReadingTestModel, id, { 151 | "Problem Statement": 1, 152 | "Answer Candidates": 1, 153 | Solution: 1, 154 | "Final Answer": 1, 155 | }); 156 | } 157 | 158 | async function findScienceProblemById(id) { 159 | return await findOneById(mcatScienceValModel, id, { 160 | "Problem Statement": 1, 161 | "Answer Candidates": 1, 162 | Solution: 1, 163 | "Final Answer": 1, 164 | }); 165 | } 166 | 167 | async function findScienceTestProblemById(id) { 168 | return await findOneById(mcatScienceTestModel, id, { 169 | "Problem Statement": 1, 170 | "Answer Candidates": 1, 171 | Solution: 1, 172 | "Final Answer": 1, 173 | }); 174 | } 175 | 176 | async function findScienceImagesProblemById(id) { 177 | return await findOneById(mcatScienceImgModel, id, { 178 | "Problem Statement": 1, 179 | Images: 1, 180 | "Answer Candidates": 1, 181 | Solution: 1, 182 | "Final Answer": 1, 183 | }); 184 | } 185 | 186 | async function findScienceImagesTestProblemById(id) { 187 | return await findOneById(mcatScienceImgTestModel, id, { 188 | "Problem Statement": 1, 189 | Images: 1, 190 | "Answer Candidates": 1, 191 | Solution: 1, 192 | "Final Answer": 1, 193 | }); 194 | } 195 | 196 | export { 197 | findReadingProblemById, findReadingTestProblemById, findScienceImagesProblemById, findScienceImagesTestProblemById, findScienceProblemById, findScienceTestProblemById, getAllReadingProblems, getAllReadingTestProblems, getAllScienceImagesProblems, getAllScienceImagesTestProblems, getAllScienceProblems, getAllScienceTestProblems, getRandomReadingProblem, getRandomScienceImagesProblem, getRandomScienceProblem 198 | }; 199 | -------------------------------------------------------------------------------- /src/server/mongodb/actions/numericalProblem.js: -------------------------------------------------------------------------------- 1 | import { 2 | mathNumericalModel, 3 | mathProofsModel, 4 | mathSymbolicModel, 5 | physicsImgModel, 6 | physicsNumericalModel, 7 | physicsTestImgModel, 8 | physicsTestModel, 9 | } from "../models/numerical_problem"; 10 | 11 | import { ObjectId } from "mongodb"; 12 | 13 | async function findOneByMongoId(model, id, projection) { 14 | if (!ObjectId.isValid(id)) { 15 | throw new Error("Invalid MongoDB ID format"); 16 | } 17 | return await model.findById(id, projection); 18 | } 19 | 20 | async function getRandomMathProblem() { 21 | const randomDocument = await mathNumericalModel.aggregate([ 22 | { $sample: { size: 1 } }, 23 | ]); 24 | return randomDocument[0]; 25 | } 26 | 27 | async function getRandomPhysicsProblem() { 28 | const randomDocument = await physicsNumericalModel.aggregate([ 29 | { $sample: { size: 1 } }, 30 | ]); 31 | return randomDocument[0]; 32 | } 33 | 34 | async function getRandomPhysicsImgProblem() { 35 | const randomDocument = await physicsImgModel.aggregate([ 36 | { $sample: { size: 1 } }, 37 | ]); 38 | return randomDocument[0]; 39 | } 40 | 41 | async function getAllMathProblems() { 42 | return await mathNumericalModel.find( 43 | {}, 44 | { 45 | _id: 1, 46 | Problem_Statement: 1, 47 | Topic: 1, 48 | "Output Format Instructions": 1, 49 | Solution: 1, 50 | "Final Answer": 1, 51 | "Problem Type": 1, 52 | rubric: 1, 53 | rubric_template: 1, 54 | } 55 | ); 56 | } 57 | 58 | async function getAllMathProofProblems() { 59 | return await mathProofsModel.find( 60 | {}, 61 | { 62 | _id: 1, 63 | Problem_Statement: 1, 64 | Topic: 1, 65 | "Output Format Instructions": 1, 66 | Solution: 1, 67 | "Final Answer": 1, 68 | "Problem Type": 1, 69 | rubric: 1, 70 | rubric_template: 1, 71 | } 72 | ); 73 | } 74 | 75 | async function getAllMathSymbolicProblems() { 76 | return await mathSymbolicModel.find( 77 | {}, 78 | { 79 | _id: 1, 80 | Problem_Statement: 1, 81 | Topic: 1, 82 | "Output Format Instructions": 1, 83 | Solution: 1, 84 | "Final Answer": 1, 85 | "Problem Type": 1, 86 | rubric: 1, 87 | rubric_template: 1, 88 | } 89 | ); 90 | } 91 | 92 | async function getAllPhysicsProblems() { 93 | return await physicsNumericalModel.find( 94 | {}, 95 | { 96 | _id: 1, 97 | Problem_Statement: 1, 98 | Topic: 1, 99 | "Output Format Instructions": 1, 100 | Solution: 1, 101 | "Final Answer": 1, 102 | "Problem Type": 1, 103 | rubric: 1, 104 | rubric_template: 1, 105 | } 106 | ); 107 | } 108 | 109 | async function getAllPhysicsTestProblems() { 110 | return await physicsTestModel.find( 111 | {}, 112 | { 113 | _id: 1, 114 | Problem_Statement: 1, 115 | Topic: 1, 116 | "Output Format Instructions": 1, 117 | Solution: 1, 118 | "Final Answer": 1, 119 | "Problem Type": 1, 120 | rubric: 1, 121 | rubric_template: 1, 122 | } 123 | ); 124 | } 125 | 126 | async function getAllPhysicsImgProblems() { 127 | return await physicsImgModel.find( 128 | {}, 129 | { 130 | _id: 1, 131 | Problem_Statement: 1, 132 | Topic: 1, 133 | Images: 1, 134 | "Output Format Instructions": 1, 135 | Solution: 1, 136 | "Final Answer": 1, 137 | "Problem Type": 1, 138 | rubric: 1, 139 | rubric_template: 1, 140 | } 141 | ); 142 | } 143 | 144 | async function getAllPhysicsImgTestProblems() { 145 | return await physicsTestImgModel.find( 146 | {}, 147 | { 148 | _id: 1, 149 | Problem_Statement: 1, 150 | Topic: 1, 151 | Images: 1, 152 | "Output Format Instructions": 1, 153 | Solution: 1, 154 | "Final Answer": 1, 155 | "Problem Type": 1, 156 | rubric: 1, 157 | rubric_template: 1, 158 | } 159 | ); 160 | } 161 | 162 | async function findMathProblemById(id) { 163 | return findOneByMongoId(mathNumericalModel, id, { 164 | Problem_Statement: 1, 165 | Solution: 1, 166 | "Final Answer": 1, 167 | }); 168 | } 169 | 170 | async function findMathProofProblemById(id) { 171 | return findOneByMongoId(mathProofsModel, id, { 172 | Problem_Statement: 1, 173 | Solution: 1, 174 | "Final Answer": 1, 175 | }); 176 | } 177 | 178 | async function findMathSymbolicProblemById(id) { 179 | return findOneByMongoId(mathSymbolicModel, id, { 180 | Problem_Statement: 1, 181 | Solution: 1, 182 | "Final Answer": 1, 183 | }); 184 | } 185 | 186 | async function findPhysicsProblemById(id) { 187 | return findOneByMongoId(physicsNumericalModel, id, { 188 | Problem_Statement: 1, 189 | Solution: 1, 190 | "Final Answer": 1, 191 | }); 192 | } 193 | 194 | async function findPhysicsTestProblemById(id) { 195 | return findOneByMongoId(physicsTestModel, id, { 196 | Problem_Statement: 1, 197 | Solution: 1, 198 | "Final Answer": 1, 199 | }); 200 | } 201 | 202 | async function findPhysicsImgProblemById(id) { 203 | return findOneByMongoId(physicsImgModel, id, { 204 | Problem_Statement: 1, 205 | Images: 1, 206 | Solution: 1, 207 | "Final Answer": 1, 208 | }); 209 | } 210 | 211 | async function findPhysicsImgTestProblemById(id) { 212 | return findOneByMongoId(physicsTestImgModel, id, { 213 | Problem_Statement: 1, 214 | Images: 1, 215 | Solution: 1, 216 | "Final Answer": 1, 217 | }); 218 | } 219 | 220 | export { 221 | findMathProblemById, findMathProofProblemById, 222 | findMathSymbolicProblemById, findPhysicsImgProblemById, findPhysicsImgTestProblemById, findPhysicsProblemById, findPhysicsTestProblemById, getAllMathProblems, 223 | getAllMathProofProblems, 224 | getAllMathSymbolicProblems, getAllPhysicsImgProblems, 225 | getAllPhysicsImgTestProblems, getAllPhysicsProblems, getAllPhysicsTestProblems, getRandomMathProblem, 226 | getRandomPhysicsImgProblem, 227 | getRandomPhysicsProblem 228 | }; 229 | 230 | -------------------------------------------------------------------------------- /src/screens/Home/Home.tsx: -------------------------------------------------------------------------------- 1 | import { useEffect, useState } from "react"; 2 | import { Image } from "@chakra-ui/react"; 3 | import { Center } from "@chakra-ui/react"; 4 | 5 | import { 6 | Box, 7 | Button, 8 | Flex, 9 | Heading, 10 | Icon, 11 | Link, 12 | Select, 13 | Text, 14 | } from "@chakra-ui/react"; 15 | import MathJaxComponent from "./Components/Mathjax"; 16 | import { FaArrowCircleLeft } from "react-icons/fa"; 17 | import { LawProblem, numericalProblem } from "@/types/Problem"; 18 | 19 | function isNumericalProblem( 20 | problem: LawProblem | numericalProblem 21 | ): problem is numericalProblem { 22 | return (problem as numericalProblem).Problem_Statement !== undefined; 23 | } 24 | 25 | function isLawProblem( 26 | problem: LawProblem | numericalProblem 27 | ): problem is LawProblem { 28 | return (problem as LawProblem).Solution == ""; 29 | } 30 | 31 | export default function Home() { 32 | const [problem, setProblem] = useState( 33 | null 34 | ); 35 | const [randomEndpoint, setRandomEndpoint] = useState(""); 36 | const [problemType, setProblemType] = useState(""); 37 | const [problemStatement, setProblemStatement] = useState(""); 38 | const [finalAnswer, setFinalAnswer] = useState(""); 39 | const [solution, setSolution] = useState(""); 40 | const [answerCandidates, setAnswerCandidates] = useState([]); 41 | 42 | const [showSolution, setShowSolution] = useState(false); 43 | const [showFinalAnswer, setShowFinalAnswer] = useState(false); 44 | 45 | const endpoints = { 46 | "Law Problem": "/api/lawProblem", 47 | "Math Numerical Problem": "/api/mathProblem", 48 | "MCAT Reading Problem": "/api/mcatReadingProblem", 49 | "MCAT Science Problem": "/api/mcatScienceProblem", 50 | "MCAT Science Image Problem": "/api/mcatScienceImageProblem", 51 | "Physics Numerical Problem": "/api/physicsProblem", 52 | "Physics Numerical Image Problem": "/api/physicsImgProblem", 53 | }; 54 | 55 | const fetchProblem = (endpoint: string) => { 56 | fetch(endpoint) 57 | .then((response) => response.json()) 58 | .then((data) => { 59 | console.log(data); 60 | setProblem(data["randomProblem"]); 61 | }); 62 | }; 63 | useEffect(() => { 64 | if (problem) { 65 | if (isNumericalProblem(problem)) { 66 | setProblemStatement(problem.Problem_Statement); 67 | setSolution(problem["Solution"]); 68 | setFinalAnswer(problem["Final Answer"]); 69 | } else { 70 | setProblemStatement(problem["Problem Statement"]); 71 | setSolution(problem["Solution"]); 72 | setAnswerCandidates(problem["Answer Candidates"]); 73 | setFinalAnswer(problem["Final Answer"]); 74 | } 75 | } 76 | }, [problem]); 77 | 78 | useEffect(() => { 79 | const randomProblemType = 80 | Object.keys(endpoints)[ 81 | Math.floor(Math.random() * Object.keys(endpoints).length) 82 | ]; 83 | setProblemType(randomProblemType); 84 | const randomEndpoint = (endpoints as { [key: string]: string })[ 85 | randomProblemType 86 | ]; 87 | setRandomEndpoint(randomEndpoint); 88 | fetchProblem(randomEndpoint); 89 | }, []); 90 | 91 | const handleProblemTypeChange = ( 92 | event: React.ChangeEvent 93 | ) => { 94 | const selectedProblemType = event.target.value; 95 | setProblemType(selectedProblemType); 96 | const selectedEndpoint = (endpoints as { [key: string]: string })[ 97 | selectedProblemType 98 | ]; 99 | setRandomEndpoint(selectedEndpoint); 100 | fetchProblem(selectedEndpoint); 101 | }; 102 | 103 | const toggleSolutionVisibility = () => { 104 | setShowSolution((prevState) => !prevState); 105 | }; 106 | 107 | const toggleFinalAnswerVisibility = () => { 108 | setShowFinalAnswer((prevState) => !prevState); 109 | }; 110 | 111 | if (!problem) { 112 | return ( 113 | 120 | 121 | Loading... 122 | 123 | 124 | ); 125 | } 126 | 127 | console.log("Problem Statement:", problemStatement); 128 | 129 | console.log("Final Answer:", finalAnswer); 130 | 131 | return ( 132 | <> 133 | 145 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 167 | Advanced Reasoning Benchmark 168 | 169 | 170 | an interactive problem sampler* 171 | 172 | 173 | 180 | 181 | 184 | 185 | 186 | 187 | 188 | 189 | {randomEndpoint === "/api/mcatScienceImageProblem" && ( 190 | 191 | {problem.Images && 192 | problem.Images.map((imageName, index) => { 193 | const imageUrl = `https://storage.googleapis.com/images_problems/${imageName}`; 194 | return ( 195 |
196 | {`Problem 204 |
205 | ); 206 | })} 207 | 208 | )} 209 | {randomEndpoint === "/api/physicsImgProblem" && ( 210 | 211 | {problem.Images && 212 | problem.Images.map((imageUrl, index) => { 213 | return ( 214 |
215 | {`Problem 223 |
224 | ); 225 | })} 226 | 227 | )} 228 | 229 | {!isNumericalProblem(problem) && ( 230 | 231 | 232 | Answer Candidates: 233 | 234 | {answerCandidates && 235 | answerCandidates.map((answer, index) => { 236 | let prefix = String.fromCharCode(index + 65); 237 | return ( 238 | 239 | {`${prefix}. `} 240 | 241 | 242 | ); 243 | })} 244 | 245 | )} 246 | 247 | {!isLawProblem(problem) && ( 248 | 251 | )} 252 | {!isLawProblem(problem) && showSolution && ( 253 | 254 | 255 | Solution: 256 | 257 | {solution && } 258 | 259 | )} 260 |
261 | 262 | 265 | 266 | {showFinalAnswer && ( 267 | 268 | 269 | Final Answer: 270 | 271 | 272 | 273 | )} 274 | 275 | 276 | 277 | * To prevent data contamination in the training data of future models, 278 | only the validation split has been made available on the web 279 | interface.
For details, please refer to section C of the 280 | appendix in the paper. 281 | 282 | 283 | 284 | 292 | 293 | 294 | ); 295 | } 296 | -------------------------------------------------------------------------------- /src/screens/ARB/Arb.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { 3 | Box, 4 | Flex, 5 | Link, 6 | Heading, 7 | Text, 8 | IconButton, 9 | Button, 10 | Container, 11 | Image, 12 | Icon, 13 | Code, 14 | } from "@chakra-ui/react"; 15 | import { 16 | FaArrowCircleRight, 17 | FaFilePdf, 18 | FaGithub, 19 | FaBook, 20 | FaImages, 21 | } from "react-icons/fa"; 22 | import { AiFillFile } from "react-icons/ai"; 23 | 24 | 25 | const Arb: React.FC = () => { 26 | const example_problems = ["/proof_1.png", "/proof_2.png"]; 27 | const overview = [ 28 | "/parsed_results.png", 29 | "/rubric_eval.png", 30 | "/rubric_example.png", 31 | "sample_model_response.png", 32 | ]; 33 | 34 | return ( 35 | 36 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | ARB: Advanced Reasoning Benchmark for Large Language Models 55 | 56 | 57 | 58 | 59 | 67 | 75 | 76 | 77 | 85 | 93 | 94 | 102 | 110 | 111 | 112 | 119 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | Abstract 138 | 139 | 140 | 141 | 142 | Large Language Models (LLMs) have demonstrated remarkable 143 | performance on various quantitative reasoning and knowledge 144 | benchmarks, such as MMLU and MATH. However, many of these 145 | benchmarks are losing utility as LLMs get increasingly high 146 | scores, despite not yet achieving expert level performance in 147 | these domains. We introduce ARB, a novel benchmark composed of 148 | advanced reasoning problems designed to evaluate LLMs on text 149 | comprehension and expert domain reasoning. ARB presents a more 150 | challenging test than prior benchmarks, featuring questions that 151 | test deeper knowledge of mathematics, physics, biology, 152 | chemistry, and law. 153 | 154 | 155 | 156 | As a subset of ARB, we introduce a challenging set of math and 157 | physics problems which require advanced symbolic reasoning and 158 | domain knowledge. In order to improve both automatic and 159 | assisted symbolic evaluation capabilities, we introduce a 160 | rubric-based self-evaluation approach, allowing GPT-4 to score 161 | its own intermediate reasoning steps. 162 | 163 | 164 | 165 | We evaluated recent models such as GPT-4 and Claude on ARB and 166 | demonstrated that even with Chain-of-Thought prompting methods, 167 | current models score well below 50% on more demanding expert 168 | tasks. Further, we conducted a human evaluation of the symbolic 169 | subset of ARB, finding close agreement between annotators and 170 | GPT-4 self-evaluation scores. 171 | 172 | 173 | 174 | 175 |
176 |
177 | 178 | 179 | 180 | Sample Problems 181 | 182 | 183 | 184 | Math Symbolic 185 | 186 | 187 | Symbolic 193 | 194 | 195 | 196 | Math Proof-like 197 | 198 | 199 | Proof-like 205 | 206 | 207 | 208 | Physics Symbolic 209 | 210 | 211 | Physics Symbolic 217 | 218 | 219 | 220 | ... see more in the interface! 221 | 222 | 223 | 224 | 225 | 226 | 227 |
228 |
229 | 230 | 231 | 232 | Evaluation Results 233 | 234 | 235 | Our evaluation of current large language models (LLMs) focuses on 236 | text-only problems, with no multimodal tasks, using models 237 | including ChatGPT, GPT 3.5, GPT-4, and Claude. Each question type 238 | is assessed with task-specific instructions and chain of thought; 239 | for multiple-choice questions, the model's choice is compared 240 | with the correct answer, while numerical, symbolic, and proof-like 241 | problems require extraction and parsing of the model's 242 | answer, often requiring mathematical libraries and manual grading 243 | due to their complexity. We also tested two model-based approaches 244 | for grading, including GPT-4's ability to grade equivalence 245 | of two symbolic expressions and a rubric-based evaluation method, 246 | which showed promising results, facilitating the evaluation of 247 | increasingly unstructured answers. 248 | 249 | 250 | Eval Result 256 | 257 | 258 | 259 |
260 |
261 | 262 | 263 | 264 | Model-based Rubric Evaluation 265 | 266 | 267 | As the complexity of reasoning tasks for language learning models 268 | (LLMs) grows, reliable evaluation becomes challenging due to 269 | difficulties in grading symbolic answers and assessing 270 | intermediate reasoning steps. We propose an approach where the 271 | model generates and uses rubrics to evaluate solutions, based on 272 | reference solutions and examples of human-crafted rubrics. Our 273 | evaluation revealed that GPT-4 creates effective rubrics, covering 274 | key solution steps well but struggling with point allocation, 275 | outperforming its predecessor, GPT-3.5-turbo. 276 | 277 | 278 | Rubric Eval 284 | Rubric Example 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | ); 298 | }; 299 | 300 | export default Arb; 301 | -------------------------------------------------------------------------------- /src/screens/AnonDocs/Docs.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { 3 | Box, 4 | Flex, 5 | Link, 6 | Heading, 7 | Text, 8 | Container, 9 | Image, 10 | Icon, 11 | Code, 12 | } from "@chakra-ui/react"; 13 | import { 14 | FaArrowCircleRight 15 | } from "react-icons/fa"; 16 | 17 | const Documentation: React.FC = () => { 18 | 19 | return ( 20 | 21 | 32 | 37 | 38 | 39 | 40 | 41 | 42 | API Documentation 43 | 44 | 45 | We provide a simple API to access the Advanced Reasoning Benchmark (ARB). The API currently supports standard HTTP GET requests. 46 | 47 | 48 | {/* 49 | API Calls 50 | 51 | 52 | We have three different types of API calls for retrieving problems. See here and select ARB API server as the mock server to see what the outputs look like. 53 | */} 54 | 55 | 56 | 57 | 58 | 59 | 60 | Categories without Images. 61 | 62 | 63 | For law, problems are retrieved according to the following: 64 | 65 | 66 | 67 | import requests 68 |
69 | response = 70 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/law/") 71 |
72 | data = response.json() 73 | 74 | 75 | 76 | 77 | For math, you can retrieve the problems according to the following template: 78 | 79 | 80 | 81 | https://advanced-reasoning-benchmark.netlify.app/api/lib/math/{"{answer-type}"}/ 82 | 83 | 84 | 85 | where acceptable values for `answer-type` are `numerical`, `symbolic`, and `prooflike`. 86 | 87 | 88 | Numerical 89 | 90 | 91 | 92 | import requests 93 |
94 | response = 95 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/math/numerical") 96 |
97 | data = response.json() 98 | 99 | 100 | 101 | Symbolic 102 | 103 | 104 | 105 | import requests 106 |
107 | response = 108 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/math/symbolic") 109 |
110 | data = response.json() 111 | 112 | 113 | 114 | Proof-like 115 | 116 | 117 | 118 | import requests 119 |
120 | response = 121 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/math/prooflike") 122 |
123 | data = response.json() 124 | 125 | 126 | 127 | 128 | 129 | For MCAT Reading, you can retrieve the problems according to the following template: 130 | 131 | 132 | 133 | https://advanced-reasoning-benchmark.netlify.app/api/lib/mcatReading/{"{split}"} 134 | 135 | 136 | 137 | where acceptable values for `split` are `val` and `test`. 138 | 139 | 140 | MCAT Reading Validation Split: 141 | 142 | 143 | 144 | import requests 145 |
146 | response = 147 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/mcatReading/val") 148 |
149 | data = response.json() 150 | 151 | 152 | 153 | MCAT Reading Test Split: 154 | 155 | 156 | 157 | import requests 158 |
159 | response = 160 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/mcatReading/test") 161 |
162 | data = response.json() 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | Categories without Images. 174 | 175 | 176 | For physics and MCAT Science, you can specify whether you want the problems with or without images. 177 | 178 | 179 | 180 | 181 | https://advanced-reasoning-benchmark.netlify.app/api/lib/{"{category}"}/{"{answer-type}"}/{"{modality}"} 182 | 183 | 184 | 185 | For physics, acceptable values for `{"{answer-type}"}` are `numerical` and `symbolic`. Acceptable values for `{"{modality}"}` are `img` and `noimg`. 186 | 187 | 188 | Physics Numerical without Images 189 | 190 | 191 | 192 | import requests 193 |
194 | response = 195 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/physics/numerical/noimg") 196 |
197 | data = response.json() 198 | 199 | 200 | 201 | Physics Numerical with Images 202 | 203 | 204 | 205 | import requests 206 |
207 | response = 208 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/physics/numerical/img") 209 |
210 | data = response.json() 211 | 212 | 213 | 214 | Physics Symbolic without Images 215 | 216 | 217 | 218 | import requests 219 |
220 | response = 221 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/physics/symbolic/noimg") 222 |
223 | data = response.json() 224 | 225 | 226 | 227 | Physics Symbolic with Images 228 | 229 | 230 | 231 | import requests 232 |
233 | response = 234 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/physics/symbolic/img") 235 |
236 | data = response.json() 237 | 238 | 239 | 240 | 241 | For MCAT Science, `{"{answer-type}"}` should be filled in with the splits, namely `val` and `test`. Acceptable values for `{"{modality}"}` are `img` and `noimg`. For example, if you want the MCAT Science from the validation split without images, you can use the following template: 242 | 243 | 244 | MCAT Science Validation Split without Image 245 | 246 | 247 | 248 | import requests 249 |
250 | response = 251 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/mcatScience/val/noimg") 252 |
253 | data = response.json() 254 | 255 | 256 | 257 | MCAT Science Validation Split with Image 258 | 259 | 260 | 261 | import requests 262 |
263 | response = 264 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/mcatScience/val/img") 265 |
266 | data = response.json() 267 | 268 | 269 | 270 | MCAT Science Test Split without Image 271 | 272 | 273 | 274 | import requests 275 |
276 | response = 277 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/mcatScience/test/noimg") 278 |
279 | data = response.json() 280 | 281 | 282 | 283 | MCAT Science Test Split with Image 284 | 285 | 286 | 287 | import requests 288 |
289 | response = 290 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/mcatScience/test/img") 291 |
292 | data = response.json() 293 | 294 | 295 | 296 | 297 | 298 | Get a specific problem by id within category 299 | 300 | 301 | For any of the above, if you terminate the API call with `/{"id"}`, then you can get a specific problem with that ID. 302 | 303 | Some examples: 304 | 305 | 306 | 307 | 308 | https://advanced-reasoning-benchmark.netlify.app/api/lib/mcatScience/test/noimg/64cdbc1978a46c58407af688 309 | 310 | 311 | 312 | 313 | 314 | 315 | https://advanced-reasoning-benchmark.netlify.app/api/lib/math/numerical/64ade9c30b1afac21d212df7 316 | 317 | 318 | 319 | 320 | 328 | 329 | 330 | Copyright © 2023 ARB Team 331 | 332 | 333 | 334 | ); 335 | }; 336 | 337 | export default Documentation; 338 | -------------------------------------------------------------------------------- /src/screens/Documentation/Documentation.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { 3 | Box, 4 | Flex, 5 | Link, 6 | Heading, 7 | Text, 8 | Container, 9 | Image, 10 | Icon, 11 | Code, 12 | } from "@chakra-ui/react"; 13 | import { 14 | FaArrowCircleRight 15 | } from "react-icons/fa"; 16 | 17 | const Documentation: React.FC = () => { 18 | 19 | return ( 20 | 21 | 32 | 37 | 38 | 39 | 40 | 41 | 42 | API Documentation 43 | 44 | 45 | We provide a simple API to access the Advanced Reasoning Benchmark (ARB). The API currently supports standard HTTP GET requests. 46 | 47 | 48 | {/* 49 | API Calls 50 | 51 | 52 | We have three different types of API calls for retrieving problems. See here and select ARB API server as the mock server to see what the outputs look like. 53 | */} 54 | 55 | 56 | 57 | 58 | 59 | 60 | Categories without Images. 61 | 62 | 63 | For law, problems are retrieved according to the following: 64 | 65 | 66 | 67 | import requests 68 |
69 | response = 70 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/law/") 71 |
72 | data = response.json() 73 | 74 | 75 | 76 | 77 | For math, you can retrieve the problems according to the following template: 78 | 79 | 80 | 81 | https://advanced-reasoning-benchmark.netlify.app/api/lib/math/{"{answer-type}"}/ 82 | 83 | 84 | 85 | where acceptable values for `answer-type` are `numerical`, `symbolic`, and `prooflike`. 86 | 87 | 88 | Numerical 89 | 90 | 91 | 92 | import requests 93 |
94 | response = 95 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/math/numerical") 96 |
97 | data = response.json() 98 | 99 | 100 | 101 | Symbolic 102 | 103 | 104 | 105 | import requests 106 |
107 | response = 108 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/math/symbolic") 109 |
110 | data = response.json() 111 | 112 | 113 | 114 | Proof-like 115 | 116 | 117 | 118 | import requests 119 |
120 | response = 121 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/math/prooflike") 122 |
123 | data = response.json() 124 | 125 | 126 | 127 | 128 | 129 | For MCAT Reading, you can retrieve the problems according to the following template: 130 | 131 | 132 | 133 | https://advanced-reasoning-benchmark.netlify.app/api/lib/mcatReading/{"{split}"} 134 | 135 | 136 | 137 | where acceptable values for `split` are `val` and `test`. 138 | 139 | 140 | MCAT Reading Validation Split: 141 | 142 | 143 | 144 | import requests 145 |
146 | response = 147 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/mcatReading/val") 148 |
149 | data = response.json() 150 | 151 | 152 | 153 | MCAT Reading Test Split: 154 | 155 | 156 | 157 | import requests 158 |
159 | response = 160 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/mcatReading/test") 161 |
162 | data = response.json() 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | Categories without Images. 174 | 175 | 176 | For physics and MCAT Science, you can specify whether you want the problems with or without images. 177 | 178 | 179 | 180 | 181 | https://advanced-reasoning-benchmark.netlify.app/api/lib/{"{category}"}/{"{answer-type}"}/{"{modality}"} 182 | 183 | 184 | 185 | For physics, acceptable values for `{"{answer-type}"}` are `numerical` and `symbolic`. Acceptable values for `{"{modality}"}` are `img` and `noimg`. 186 | 187 | 188 | Physics Numerical without Images 189 | 190 | 191 | 192 | import requests 193 |
194 | response = 195 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/physics/numerical/noimg") 196 |
197 | data = response.json() 198 | 199 | 200 | 201 | Physics Numerical with Images 202 | 203 | 204 | 205 | import requests 206 |
207 | response = 208 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/physics/numerical/img") 209 |
210 | data = response.json() 211 | 212 | 213 | 214 | Physics Symbolic without Images 215 | 216 | 217 | 218 | import requests 219 |
220 | response = 221 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/physics/symbolic/noimg") 222 |
223 | data = response.json() 224 | 225 | 226 | 227 | Physics Symbolic with Images 228 | 229 | 230 | 231 | import requests 232 |
233 | response = 234 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/physics/symbolic/img") 235 |
236 | data = response.json() 237 | 238 | 239 | 240 | 241 | For MCAT Science, `{"{answer-type}"}` should be filled in with the splits, namely `val` and `test`. Acceptable values for `{"{modality}"}` are `img` and `noimg`. For example, if you want the MCAT Science from the validation split without images, you can use the following template: 242 | 243 | 244 | MCAT Science Validation Split without Image 245 | 246 | 247 | 248 | import requests 249 |
250 | response = 251 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/mcatScience/val/noimg") 252 |
253 | data = response.json() 254 | 255 | 256 | 257 | MCAT Science Validation Split with Image 258 | 259 | 260 | 261 | import requests 262 |
263 | response = 264 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/mcatScience/val/img") 265 |
266 | data = response.json() 267 | 268 | 269 | 270 | MCAT Science Test Split without Image 271 | 272 | 273 | 274 | import requests 275 |
276 | response = 277 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/mcatScience/test/noimg") 278 |
279 | data = response.json() 280 | 281 | 282 | 283 | MCAT Science Test Split with Image 284 | 285 | 286 | 287 | import requests 288 |
289 | response = 290 | requests.get("https://advanced-reasoning-benchmark.netlify.app/api/lib/mcatScience/test/img") 291 |
292 | data = response.json() 293 | 294 | 295 | 296 | 297 | 298 | Get a specific problem by id within category 299 | 300 | 301 | For any of the above, if you terminate the API call with `/{"id"}`, then you can get a specific problem with that ID. 302 | 303 | Some examples: 304 | 305 | 306 | 307 | 308 | https://advanced-reasoning-benchmark.netlify.app/api/lib/mcatScience/test/noimg/64cdbc1978a46c58407af688 309 | 310 | 311 | 312 | 313 | 314 | 315 | https://advanced-reasoning-benchmark.netlify.app/api/lib/math/numerical/64ade9c30b1afac21d212df7 316 | 317 | 318 | 319 | 320 | 328 | 329 | 330 | Copyright © 2023 ARB Team 331 | 332 | 333 | 334 | ); 335 | }; 336 | 337 | export default Documentation; 338 | --------------------------------------------------------------------------------