├── .gitignore ├── LICENSE ├── README.md ├── api ├── .env ├── README.md ├── Test_Inference.ipynb ├── __pycache__ │ ├── evaluator_app.cpython-38.pyc │ └── text_utils.cpython-38.pyc ├── docs │ ├── gpt3 │ │ ├── 2005.14165.pdf │ │ └── gpt3-eval.csv │ ├── karpathy-lex-pod │ │ ├── karpathy-pod-eval.csv │ │ └── karpathy-pod.txt │ └── transformers-challenge │ │ ├── 2005.14165.pdf │ │ ├── 2112.04426.pdf │ │ ├── 2203.15556.pdf │ │ ├── 2205.06175.pdf │ │ ├── 2302.13971.pdf │ │ └── transformers-eval.csv ├── evaluator_app.py ├── logging.conf ├── railway.json ├── requirements.txt └── text_utils.py ├── nextjs ├── .env.local ├── .gitignore ├── components │ ├── Demo.tsx │ ├── ExperimentSummaryTable.tsx │ ├── HeaderEvaluator.tsx │ ├── PersonCard.tsx │ ├── Playground.tsx │ ├── Sidebar.tsx │ ├── SummaryChart.tsx │ ├── TestFileUploadZone.tsx │ └── tables │ │ ├── ExperimentResultTable.tsx │ │ └── FilesTable.tsx ├── next-env.d.ts ├── next.config.js ├── package.json ├── pages │ ├── _app.tsx │ ├── about │ │ └── index.tsx │ ├── index.tsx │ └── playground │ │ └── index.tsx ├── public │ ├── favicon │ │ ├── about.txt │ │ ├── android-chrome-192x192.png │ │ ├── android-chrome-512x512.png │ │ ├── apple-touch-icon.png │ │ ├── favicon-16x16.png │ │ ├── favicon-32x32.png │ │ ├── favicon.ico │ │ └── site.webmanifest │ ├── github-mark.svg │ ├── slack-mark.svg │ ├── testData │ │ ├── experiments.json │ │ ├── karpathy-pod.json │ │ ├── results.json │ │ └── testDataset.json │ └── twitter-black.svg ├── styles │ ├── global.css │ └── utils.module.css ├── tsconfig.json ├── utils │ ├── renderPassFail.ts │ ├── types.ts │ └── variables.ts └── yarn.lock └── streamlit ├── README.md ├── auto-evaluator.py ├── eval_sets └── lex-pod-eval.json ├── img └── diagnostic.jpg ├── kor_retriever_lex.py ├── prompts.py ├── requirements.txt └── self_query_retriever_lex.py /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .vercel 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/README.md -------------------------------------------------------------------------------- /api/.env: -------------------------------------------------------------------------------- 1 | ENVIRONMENT=development -------------------------------------------------------------------------------- /api/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/README.md -------------------------------------------------------------------------------- /api/Test_Inference.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/Test_Inference.ipynb -------------------------------------------------------------------------------- /api/__pycache__/evaluator_app.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/__pycache__/evaluator_app.cpython-38.pyc -------------------------------------------------------------------------------- /api/__pycache__/text_utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/__pycache__/text_utils.cpython-38.pyc -------------------------------------------------------------------------------- /api/docs/gpt3/2005.14165.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/docs/gpt3/2005.14165.pdf -------------------------------------------------------------------------------- /api/docs/gpt3/gpt3-eval.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/docs/gpt3/gpt3-eval.csv -------------------------------------------------------------------------------- /api/docs/karpathy-lex-pod/karpathy-pod-eval.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/docs/karpathy-lex-pod/karpathy-pod-eval.csv -------------------------------------------------------------------------------- /api/docs/karpathy-lex-pod/karpathy-pod.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/docs/karpathy-lex-pod/karpathy-pod.txt -------------------------------------------------------------------------------- /api/docs/transformers-challenge/2005.14165.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/docs/transformers-challenge/2005.14165.pdf -------------------------------------------------------------------------------- /api/docs/transformers-challenge/2112.04426.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/docs/transformers-challenge/2112.04426.pdf -------------------------------------------------------------------------------- /api/docs/transformers-challenge/2203.15556.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/docs/transformers-challenge/2203.15556.pdf -------------------------------------------------------------------------------- /api/docs/transformers-challenge/2205.06175.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/docs/transformers-challenge/2205.06175.pdf -------------------------------------------------------------------------------- /api/docs/transformers-challenge/2302.13971.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/docs/transformers-challenge/2302.13971.pdf -------------------------------------------------------------------------------- /api/docs/transformers-challenge/transformers-eval.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/docs/transformers-challenge/transformers-eval.csv -------------------------------------------------------------------------------- /api/evaluator_app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/evaluator_app.py -------------------------------------------------------------------------------- /api/logging.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/logging.conf -------------------------------------------------------------------------------- /api/railway.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/railway.json -------------------------------------------------------------------------------- /api/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/requirements.txt -------------------------------------------------------------------------------- /api/text_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/api/text_utils.py -------------------------------------------------------------------------------- /nextjs/.env.local: -------------------------------------------------------------------------------- 1 | NEXT_PUBLIC_API_URL=http://localhost:8000 -------------------------------------------------------------------------------- /nextjs/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/.gitignore -------------------------------------------------------------------------------- /nextjs/components/Demo.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/components/Demo.tsx -------------------------------------------------------------------------------- /nextjs/components/ExperimentSummaryTable.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/components/ExperimentSummaryTable.tsx -------------------------------------------------------------------------------- /nextjs/components/HeaderEvaluator.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/components/HeaderEvaluator.tsx -------------------------------------------------------------------------------- /nextjs/components/PersonCard.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/components/PersonCard.tsx -------------------------------------------------------------------------------- /nextjs/components/Playground.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/components/Playground.tsx -------------------------------------------------------------------------------- /nextjs/components/Sidebar.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/components/Sidebar.tsx -------------------------------------------------------------------------------- /nextjs/components/SummaryChart.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/components/SummaryChart.tsx -------------------------------------------------------------------------------- /nextjs/components/TestFileUploadZone.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/components/TestFileUploadZone.tsx -------------------------------------------------------------------------------- /nextjs/components/tables/ExperimentResultTable.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/components/tables/ExperimentResultTable.tsx -------------------------------------------------------------------------------- /nextjs/components/tables/FilesTable.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/components/tables/FilesTable.tsx -------------------------------------------------------------------------------- /nextjs/next-env.d.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/next-env.d.ts -------------------------------------------------------------------------------- /nextjs/next.config.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/next.config.js -------------------------------------------------------------------------------- /nextjs/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/package.json -------------------------------------------------------------------------------- /nextjs/pages/_app.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/pages/_app.tsx -------------------------------------------------------------------------------- /nextjs/pages/about/index.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/pages/about/index.tsx -------------------------------------------------------------------------------- /nextjs/pages/index.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/pages/index.tsx -------------------------------------------------------------------------------- /nextjs/pages/playground/index.tsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/pages/playground/index.tsx -------------------------------------------------------------------------------- /nextjs/public/favicon/about.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/public/favicon/about.txt -------------------------------------------------------------------------------- /nextjs/public/favicon/android-chrome-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/public/favicon/android-chrome-192x192.png -------------------------------------------------------------------------------- /nextjs/public/favicon/android-chrome-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/public/favicon/android-chrome-512x512.png -------------------------------------------------------------------------------- /nextjs/public/favicon/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/public/favicon/apple-touch-icon.png -------------------------------------------------------------------------------- /nextjs/public/favicon/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/public/favicon/favicon-16x16.png -------------------------------------------------------------------------------- /nextjs/public/favicon/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/public/favicon/favicon-32x32.png -------------------------------------------------------------------------------- /nextjs/public/favicon/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/public/favicon/favicon.ico -------------------------------------------------------------------------------- /nextjs/public/favicon/site.webmanifest: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/public/favicon/site.webmanifest -------------------------------------------------------------------------------- /nextjs/public/github-mark.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/public/github-mark.svg -------------------------------------------------------------------------------- /nextjs/public/slack-mark.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/public/slack-mark.svg -------------------------------------------------------------------------------- /nextjs/public/testData/experiments.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/public/testData/experiments.json -------------------------------------------------------------------------------- /nextjs/public/testData/karpathy-pod.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/public/testData/karpathy-pod.json -------------------------------------------------------------------------------- /nextjs/public/testData/results.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/public/testData/results.json -------------------------------------------------------------------------------- /nextjs/public/testData/testDataset.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/public/testData/testDataset.json -------------------------------------------------------------------------------- /nextjs/public/twitter-black.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/public/twitter-black.svg -------------------------------------------------------------------------------- /nextjs/styles/global.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/styles/global.css -------------------------------------------------------------------------------- /nextjs/styles/utils.module.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/styles/utils.module.css -------------------------------------------------------------------------------- /nextjs/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/tsconfig.json -------------------------------------------------------------------------------- /nextjs/utils/renderPassFail.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/utils/renderPassFail.ts -------------------------------------------------------------------------------- /nextjs/utils/types.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/utils/types.ts -------------------------------------------------------------------------------- /nextjs/utils/variables.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/utils/variables.ts -------------------------------------------------------------------------------- /nextjs/yarn.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/nextjs/yarn.lock -------------------------------------------------------------------------------- /streamlit/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/streamlit/README.md -------------------------------------------------------------------------------- /streamlit/auto-evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/streamlit/auto-evaluator.py -------------------------------------------------------------------------------- /streamlit/eval_sets/lex-pod-eval.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/streamlit/eval_sets/lex-pod-eval.json -------------------------------------------------------------------------------- /streamlit/img/diagnostic.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/streamlit/img/diagnostic.jpg -------------------------------------------------------------------------------- /streamlit/kor_retriever_lex.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/streamlit/kor_retriever_lex.py -------------------------------------------------------------------------------- /streamlit/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/streamlit/prompts.py -------------------------------------------------------------------------------- /streamlit/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/streamlit/requirements.txt -------------------------------------------------------------------------------- /streamlit/self_query_retriever_lex.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/auto-evaluator/HEAD/streamlit/self_query_retriever_lex.py --------------------------------------------------------------------------------