├── LICENSE ├── README.md ├── baselines ├── baselines.py ├── cot.sh ├── direct.sh ├── evaluate.py ├── execution.py ├── icl.sh ├── rag.sh └── utils.py ├── pie └── src │ ├── __init__.py │ ├── codenet_eval │ ├── __init__.py │ ├── __pycache__ │ │ ├── evalconfig.cpython-37.pyc │ │ ├── evalconfig.cpython-39.pyc │ │ ├── sandbox.cpython-37.pyc │ │ └── sandbox.cpython-39.pyc │ ├── a.out │ ├── cpp_examples │ │ ├── fast_but_wrong.cpp │ │ ├── fast_num.cpp │ │ └── slow_num.cpp │ ├── e.cpp │ ├── eval_config │ │ └── sample_eval_config.yaml │ ├── eval_submissions │ │ ├── __init__.py │ │ ├── all_submissions_time_eval.py │ │ ├── parse_runtimes.py │ │ └── prep_for_eval.py │ ├── evalconfig.py │ ├── run_eval.py │ ├── run_eval_feedback.py │ └── sandbox.py │ ├── make_splits.py │ └── utils │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-39.pyc │ ├── diff_utils.cpython-39.pyc │ ├── format_utils.cpython-39.pyc │ └── name_utils.cpython-39.pyc │ ├── diff_utils.py │ ├── format_utils.py │ ├── name_utils.py │ ├── parallel_utils.py │ ├── print_diffs.py │ ├── remove_seps.py │ └── stat_utils.py ├── processed_data └── README.md ├── requirement.txt └── sbllm ├── evaluate.py ├── evol_query.py ├── execution.py ├── initial.py ├── merge.py ├── run.sh ├── run_cpp.sh └── utils.py /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/README.md -------------------------------------------------------------------------------- /baselines/baselines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/baselines/baselines.py -------------------------------------------------------------------------------- /baselines/cot.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/baselines/cot.sh -------------------------------------------------------------------------------- /baselines/direct.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/baselines/direct.sh -------------------------------------------------------------------------------- /baselines/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/baselines/evaluate.py -------------------------------------------------------------------------------- /baselines/execution.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/baselines/execution.py -------------------------------------------------------------------------------- /baselines/icl.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/baselines/icl.sh -------------------------------------------------------------------------------- /baselines/rag.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/baselines/rag.sh -------------------------------------------------------------------------------- /baselines/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/baselines/utils.py -------------------------------------------------------------------------------- /pie/src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pie/src/codenet_eval/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pie/src/codenet_eval/__pycache__/evalconfig.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/__pycache__/evalconfig.cpython-37.pyc -------------------------------------------------------------------------------- /pie/src/codenet_eval/__pycache__/evalconfig.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/__pycache__/evalconfig.cpython-39.pyc -------------------------------------------------------------------------------- /pie/src/codenet_eval/__pycache__/sandbox.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/__pycache__/sandbox.cpython-37.pyc -------------------------------------------------------------------------------- /pie/src/codenet_eval/__pycache__/sandbox.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/__pycache__/sandbox.cpython-39.pyc -------------------------------------------------------------------------------- /pie/src/codenet_eval/a.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/a.out -------------------------------------------------------------------------------- /pie/src/codenet_eval/cpp_examples/fast_but_wrong.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/cpp_examples/fast_but_wrong.cpp -------------------------------------------------------------------------------- /pie/src/codenet_eval/cpp_examples/fast_num.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/cpp_examples/fast_num.cpp -------------------------------------------------------------------------------- /pie/src/codenet_eval/cpp_examples/slow_num.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/cpp_examples/slow_num.cpp -------------------------------------------------------------------------------- /pie/src/codenet_eval/e.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/e.cpp -------------------------------------------------------------------------------- /pie/src/codenet_eval/eval_config/sample_eval_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/eval_config/sample_eval_config.yaml -------------------------------------------------------------------------------- /pie/src/codenet_eval/eval_submissions/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pie/src/codenet_eval/eval_submissions/all_submissions_time_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/eval_submissions/all_submissions_time_eval.py -------------------------------------------------------------------------------- /pie/src/codenet_eval/eval_submissions/parse_runtimes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/eval_submissions/parse_runtimes.py -------------------------------------------------------------------------------- /pie/src/codenet_eval/eval_submissions/prep_for_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/eval_submissions/prep_for_eval.py -------------------------------------------------------------------------------- /pie/src/codenet_eval/evalconfig.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/evalconfig.py -------------------------------------------------------------------------------- /pie/src/codenet_eval/run_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/run_eval.py -------------------------------------------------------------------------------- /pie/src/codenet_eval/run_eval_feedback.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/run_eval_feedback.py -------------------------------------------------------------------------------- /pie/src/codenet_eval/sandbox.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/codenet_eval/sandbox.py -------------------------------------------------------------------------------- /pie/src/make_splits.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/make_splits.py -------------------------------------------------------------------------------- /pie/src/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pie/src/utils/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/utils/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /pie/src/utils/__pycache__/diff_utils.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/utils/__pycache__/diff_utils.cpython-39.pyc -------------------------------------------------------------------------------- /pie/src/utils/__pycache__/format_utils.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/utils/__pycache__/format_utils.cpython-39.pyc -------------------------------------------------------------------------------- /pie/src/utils/__pycache__/name_utils.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/utils/__pycache__/name_utils.cpython-39.pyc -------------------------------------------------------------------------------- /pie/src/utils/diff_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/utils/diff_utils.py -------------------------------------------------------------------------------- /pie/src/utils/format_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/utils/format_utils.py -------------------------------------------------------------------------------- /pie/src/utils/name_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/utils/name_utils.py -------------------------------------------------------------------------------- /pie/src/utils/parallel_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/utils/parallel_utils.py -------------------------------------------------------------------------------- /pie/src/utils/print_diffs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/utils/print_diffs.py -------------------------------------------------------------------------------- /pie/src/utils/remove_seps.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/utils/remove_seps.py -------------------------------------------------------------------------------- /pie/src/utils/stat_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/pie/src/utils/stat_utils.py -------------------------------------------------------------------------------- /processed_data/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/processed_data/README.md -------------------------------------------------------------------------------- /requirement.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/requirement.txt -------------------------------------------------------------------------------- /sbllm/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/sbllm/evaluate.py -------------------------------------------------------------------------------- /sbllm/evol_query.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/sbllm/evol_query.py -------------------------------------------------------------------------------- /sbllm/execution.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/sbllm/execution.py -------------------------------------------------------------------------------- /sbllm/initial.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/sbllm/initial.py -------------------------------------------------------------------------------- /sbllm/merge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/sbllm/merge.py -------------------------------------------------------------------------------- /sbllm/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/sbllm/run.sh -------------------------------------------------------------------------------- /sbllm/run_cpp.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/sbllm/run_cpp.sh -------------------------------------------------------------------------------- /sbllm/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shuzhenggao/sbllm/HEAD/sbllm/utils.py --------------------------------------------------------------------------------