├── .gitignore ├── LICENSE ├── README.md ├── assets ├── LDB_paper.pdf ├── ldb-logo5.webp └── overview-ldb.png ├── input_data ├── humaneval │ ├── dataset │ │ └── probs.jsonl │ ├── seed │ │ ├── codellama │ │ │ └── seed.jsonl │ │ ├── gpt-3.5-turbo-0613 │ │ │ └── seed.jsonl │ │ ├── gpt-4-1106-preview │ │ │ └── seed.jsonl │ │ ├── reflexion │ │ │ └── seed.jsonl │ │ └── starcoder │ │ │ └── seed.jsonl │ └── test │ │ └── tests.jsonl ├── mbpp │ ├── dataset │ │ └── probs.jsonl │ ├── seed │ │ ├── codellama │ │ │ └── seed.jsonl │ │ ├── gpt-3.5-turbo-0613 │ │ │ └── seed.jsonl │ │ └── starcoder │ │ │ └── seed.jsonl │ └── test │ │ └── tests.jsonl └── transcoder │ ├── dataset │ └── probs.jsonl │ ├── seed │ ├── codellama │ │ └── seed.jsonl │ ├── gpt-3.5-turbo-0613 │ │ └── seed.jsonl │ └── starcoder │ │ └── seed.jsonl │ └── test │ └── tests.jsonl ├── programming ├── __init__.py ├── executors │ ├── __init__.py │ ├── executor_types.py │ ├── executor_utils.py │ ├── factory.py │ └── py_executor.py ├── generators │ ├── __init__.py │ ├── factory.py │ ├── model.py │ ├── parse.py │ ├── prompt.py │ └── py_generate.py ├── ldb.py ├── main.py ├── repeat_simple.py ├── run_ldb.sh ├── run_simple.sh ├── simple.py ├── tracing │ ├── __init__.py │ ├── staticfg │ │ ├── LICENSE │ │ ├── __init__.py │ │ ├── builder.py │ │ └── model.py │ └── tracer.py └── utils.py ├── requirements.txt └── results └── ldb ├── humaneval ├── codellama │ └── results.jsonl ├── gpt-3.5-turbo-0613-function-level │ └── results.jsonl ├── gpt-3.5-turbo-0613-line-level │ └── results.jsonl ├── gpt-3.5-turbo-0613 │ └── results.jsonl ├── gpt-4-seed │ └── results.jsonl ├── level_function │ └── results.jsonl ├── level_line │ └── results.jsonl ├── reflexion │ ├── gpt-3.5-turbo-0613.jsonl │ ├── gpt-4-1106-preview.jsonl │ └── gpt-4o-2024-05-13.jsonl └── starcoder │ └── results.jsonl ├── mbpp ├── codellama │ └── results.jsonl ├── gpt-3.5-turbo-0613 │ └── results.jsonl └── starcoder │ └── results.jsonl └── transcoder ├── codellama └── results.jsonl ├── gpt-3.5-turbo-0613 └── results.jsonl └── starcoder └── results.jsonl /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/README.md -------------------------------------------------------------------------------- /assets/LDB_paper.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/assets/LDB_paper.pdf -------------------------------------------------------------------------------- /assets/ldb-logo5.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/assets/ldb-logo5.webp -------------------------------------------------------------------------------- /assets/overview-ldb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/assets/overview-ldb.png -------------------------------------------------------------------------------- /input_data/humaneval/dataset/probs.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/humaneval/dataset/probs.jsonl -------------------------------------------------------------------------------- /input_data/humaneval/seed/codellama/seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/humaneval/seed/codellama/seed.jsonl -------------------------------------------------------------------------------- /input_data/humaneval/seed/gpt-3.5-turbo-0613/seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/humaneval/seed/gpt-3.5-turbo-0613/seed.jsonl -------------------------------------------------------------------------------- /input_data/humaneval/seed/gpt-4-1106-preview/seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/humaneval/seed/gpt-4-1106-preview/seed.jsonl -------------------------------------------------------------------------------- /input_data/humaneval/seed/reflexion/seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/humaneval/seed/reflexion/seed.jsonl -------------------------------------------------------------------------------- /input_data/humaneval/seed/starcoder/seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/humaneval/seed/starcoder/seed.jsonl -------------------------------------------------------------------------------- /input_data/humaneval/test/tests.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/humaneval/test/tests.jsonl -------------------------------------------------------------------------------- /input_data/mbpp/dataset/probs.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/mbpp/dataset/probs.jsonl -------------------------------------------------------------------------------- /input_data/mbpp/seed/codellama/seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/mbpp/seed/codellama/seed.jsonl -------------------------------------------------------------------------------- /input_data/mbpp/seed/gpt-3.5-turbo-0613/seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/mbpp/seed/gpt-3.5-turbo-0613/seed.jsonl -------------------------------------------------------------------------------- /input_data/mbpp/seed/starcoder/seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/mbpp/seed/starcoder/seed.jsonl -------------------------------------------------------------------------------- /input_data/mbpp/test/tests.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/mbpp/test/tests.jsonl -------------------------------------------------------------------------------- /input_data/transcoder/dataset/probs.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/transcoder/dataset/probs.jsonl -------------------------------------------------------------------------------- /input_data/transcoder/seed/codellama/seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/transcoder/seed/codellama/seed.jsonl -------------------------------------------------------------------------------- /input_data/transcoder/seed/gpt-3.5-turbo-0613/seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/transcoder/seed/gpt-3.5-turbo-0613/seed.jsonl -------------------------------------------------------------------------------- /input_data/transcoder/seed/starcoder/seed.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/transcoder/seed/starcoder/seed.jsonl -------------------------------------------------------------------------------- /input_data/transcoder/test/tests.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/input_data/transcoder/test/tests.jsonl -------------------------------------------------------------------------------- /programming/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /programming/executors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/executors/__init__.py -------------------------------------------------------------------------------- /programming/executors/executor_types.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/executors/executor_types.py -------------------------------------------------------------------------------- /programming/executors/executor_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/executors/executor_utils.py -------------------------------------------------------------------------------- /programming/executors/factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/executors/factory.py -------------------------------------------------------------------------------- /programming/executors/py_executor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/executors/py_executor.py -------------------------------------------------------------------------------- /programming/generators/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/generators/__init__.py -------------------------------------------------------------------------------- /programming/generators/factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/generators/factory.py -------------------------------------------------------------------------------- /programming/generators/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/generators/model.py -------------------------------------------------------------------------------- /programming/generators/parse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/generators/parse.py -------------------------------------------------------------------------------- /programming/generators/prompt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/generators/prompt.py -------------------------------------------------------------------------------- /programming/generators/py_generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/generators/py_generate.py -------------------------------------------------------------------------------- /programming/ldb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/ldb.py -------------------------------------------------------------------------------- /programming/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/main.py -------------------------------------------------------------------------------- /programming/repeat_simple.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/repeat_simple.py -------------------------------------------------------------------------------- /programming/run_ldb.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/run_ldb.sh -------------------------------------------------------------------------------- /programming/run_simple.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/run_simple.sh -------------------------------------------------------------------------------- /programming/simple.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/simple.py -------------------------------------------------------------------------------- /programming/tracing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/tracing/__init__.py -------------------------------------------------------------------------------- /programming/tracing/staticfg/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/tracing/staticfg/LICENSE -------------------------------------------------------------------------------- /programming/tracing/staticfg/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/tracing/staticfg/__init__.py -------------------------------------------------------------------------------- /programming/tracing/staticfg/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/tracing/staticfg/builder.py -------------------------------------------------------------------------------- /programming/tracing/staticfg/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/tracing/staticfg/model.py -------------------------------------------------------------------------------- /programming/tracing/tracer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/tracing/tracer.py -------------------------------------------------------------------------------- /programming/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/programming/utils.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/requirements.txt -------------------------------------------------------------------------------- /results/ldb/humaneval/codellama/results.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/humaneval/codellama/results.jsonl -------------------------------------------------------------------------------- /results/ldb/humaneval/gpt-3.5-turbo-0613-function-level/results.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/humaneval/gpt-3.5-turbo-0613-function-level/results.jsonl -------------------------------------------------------------------------------- /results/ldb/humaneval/gpt-3.5-turbo-0613-line-level/results.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/humaneval/gpt-3.5-turbo-0613-line-level/results.jsonl -------------------------------------------------------------------------------- /results/ldb/humaneval/gpt-3.5-turbo-0613/results.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/humaneval/gpt-3.5-turbo-0613/results.jsonl -------------------------------------------------------------------------------- /results/ldb/humaneval/gpt-4-seed/results.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/humaneval/gpt-4-seed/results.jsonl -------------------------------------------------------------------------------- /results/ldb/humaneval/level_function/results.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/humaneval/level_function/results.jsonl -------------------------------------------------------------------------------- /results/ldb/humaneval/level_line/results.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/humaneval/level_line/results.jsonl -------------------------------------------------------------------------------- /results/ldb/humaneval/reflexion/gpt-3.5-turbo-0613.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/humaneval/reflexion/gpt-3.5-turbo-0613.jsonl -------------------------------------------------------------------------------- /results/ldb/humaneval/reflexion/gpt-4-1106-preview.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/humaneval/reflexion/gpt-4-1106-preview.jsonl -------------------------------------------------------------------------------- /results/ldb/humaneval/reflexion/gpt-4o-2024-05-13.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/humaneval/reflexion/gpt-4o-2024-05-13.jsonl -------------------------------------------------------------------------------- /results/ldb/humaneval/starcoder/results.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/humaneval/starcoder/results.jsonl -------------------------------------------------------------------------------- /results/ldb/mbpp/codellama/results.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/mbpp/codellama/results.jsonl -------------------------------------------------------------------------------- /results/ldb/mbpp/gpt-3.5-turbo-0613/results.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/mbpp/gpt-3.5-turbo-0613/results.jsonl -------------------------------------------------------------------------------- /results/ldb/mbpp/starcoder/results.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/mbpp/starcoder/results.jsonl -------------------------------------------------------------------------------- /results/ldb/transcoder/codellama/results.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/transcoder/codellama/results.jsonl -------------------------------------------------------------------------------- /results/ldb/transcoder/gpt-3.5-turbo-0613/results.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/transcoder/gpt-3.5-turbo-0613/results.jsonl -------------------------------------------------------------------------------- /results/ldb/transcoder/starcoder/results.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/FloridSleeves/LLMDebugger/HEAD/results/ldb/transcoder/starcoder/results.jsonl --------------------------------------------------------------------------------