├── .env.template ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── analyze_results_and_post_to_slack.py ├── auto_error_analysis.ipynb ├── correct_sql_instructions.ipynb ├── data ├── idk.csv ├── idk_bigquery.csv ├── instruct_advanced_bigquery.csv ├── instruct_advanced_mysql.csv ├── instruct_advanced_postgres.csv ├── instruct_advanced_sqlite.csv ├── instruct_advanced_tsql.csv ├── instruct_basic_bigquery.csv ├── instruct_basic_mysql.csv ├── instruct_basic_postgres.csv ├── instruct_basic_sqlite.csv ├── instruct_basic_tsql.csv ├── questions_gen_bigquery.csv ├── questions_gen_mysql.csv ├── questions_gen_postgres.csv ├── questions_gen_snowflake.csv ├── questions_gen_sqlite.csv └── questions_gen_tsql.csv ├── eval └── eval.py ├── gcs_eval.py ├── gcs_eval_checkpoints.py ├── main.py ├── prompts ├── README.md ├── prompt.md ├── prompt_anthropic.md ├── prompt_cot.md ├── prompt_cot_postgres.md ├── prompt_cot_sqlite.md ├── prompt_experimental.md ├── prompt_gemini.md ├── prompt_mistral.md ├── prompt_openai.json ├── prompt_openai_o1.json ├── prompt_qwen.json └── prompt_together.json ├── requirements.txt ├── requirements_test.txt ├── results_fn_bigquery ├── .env.yaml.template ├── main.py └── requirements.txt ├── results_fn_postgres ├── .env.yaml.template ├── create.sql ├── main.py └── requirements.txt ├── run_checkpoints.sh ├── run_checkpoints_adapters.sh ├── run_checkpoints_cot.sh ├── run_model_cot.sh ├── run_qwen.sh ├── runners ├── anthropic_runner.py ├── api_runner.py ├── bedrock_runner.py ├── deepseek_runner.py ├── gemini_runner.py ├── mistral_runner.py ├── openai_runner.py └── together_runner.py ├── tests ├── __init__.py ├── local_db_tests.py ├── test_eval.py └── test_utils_pruning.py ├── translate_sql_dialect.py ├── upload_wandb.py └── utils ├── aliases.py ├── api_server.py ├── asyncio_helpers.py ├── creds.py ├── dialects.py ├── gen_prompt.py ├── llm.py ├── pruning.py ├── questions.py ├── reporting.py └── upload_report_gcloud.py /.env.template: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/.env.template -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/.gitignore -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/README.md -------------------------------------------------------------------------------- /analyze_results_and_post_to_slack.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/analyze_results_and_post_to_slack.py -------------------------------------------------------------------------------- /auto_error_analysis.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/auto_error_analysis.ipynb -------------------------------------------------------------------------------- /correct_sql_instructions.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/correct_sql_instructions.ipynb -------------------------------------------------------------------------------- /data/idk.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/idk.csv -------------------------------------------------------------------------------- /data/idk_bigquery.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/idk_bigquery.csv -------------------------------------------------------------------------------- /data/instruct_advanced_bigquery.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/instruct_advanced_bigquery.csv -------------------------------------------------------------------------------- /data/instruct_advanced_mysql.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/instruct_advanced_mysql.csv -------------------------------------------------------------------------------- /data/instruct_advanced_postgres.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/instruct_advanced_postgres.csv -------------------------------------------------------------------------------- /data/instruct_advanced_sqlite.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/instruct_advanced_sqlite.csv -------------------------------------------------------------------------------- /data/instruct_advanced_tsql.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/instruct_advanced_tsql.csv -------------------------------------------------------------------------------- /data/instruct_basic_bigquery.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/instruct_basic_bigquery.csv -------------------------------------------------------------------------------- /data/instruct_basic_mysql.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/instruct_basic_mysql.csv -------------------------------------------------------------------------------- /data/instruct_basic_postgres.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/instruct_basic_postgres.csv -------------------------------------------------------------------------------- /data/instruct_basic_sqlite.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/instruct_basic_sqlite.csv -------------------------------------------------------------------------------- /data/instruct_basic_tsql.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/instruct_basic_tsql.csv -------------------------------------------------------------------------------- /data/questions_gen_bigquery.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/questions_gen_bigquery.csv -------------------------------------------------------------------------------- /data/questions_gen_mysql.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/questions_gen_mysql.csv -------------------------------------------------------------------------------- /data/questions_gen_postgres.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/questions_gen_postgres.csv -------------------------------------------------------------------------------- /data/questions_gen_snowflake.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/questions_gen_snowflake.csv -------------------------------------------------------------------------------- /data/questions_gen_sqlite.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/questions_gen_sqlite.csv -------------------------------------------------------------------------------- /data/questions_gen_tsql.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/data/questions_gen_tsql.csv -------------------------------------------------------------------------------- /eval/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/eval/eval.py -------------------------------------------------------------------------------- /gcs_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/gcs_eval.py -------------------------------------------------------------------------------- /gcs_eval_checkpoints.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/gcs_eval_checkpoints.py -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/main.py -------------------------------------------------------------------------------- /prompts/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/prompts/README.md -------------------------------------------------------------------------------- /prompts/prompt.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/prompts/prompt.md -------------------------------------------------------------------------------- /prompts/prompt_anthropic.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/prompts/prompt_anthropic.md -------------------------------------------------------------------------------- /prompts/prompt_cot.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/prompts/prompt_cot.md -------------------------------------------------------------------------------- /prompts/prompt_cot_postgres.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/prompts/prompt_cot_postgres.md -------------------------------------------------------------------------------- /prompts/prompt_cot_sqlite.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/prompts/prompt_cot_sqlite.md -------------------------------------------------------------------------------- /prompts/prompt_experimental.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/prompts/prompt_experimental.md -------------------------------------------------------------------------------- /prompts/prompt_gemini.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/prompts/prompt_gemini.md -------------------------------------------------------------------------------- /prompts/prompt_mistral.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/prompts/prompt_mistral.md -------------------------------------------------------------------------------- /prompts/prompt_openai.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/prompts/prompt_openai.json -------------------------------------------------------------------------------- /prompts/prompt_openai_o1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/prompts/prompt_openai_o1.json -------------------------------------------------------------------------------- /prompts/prompt_qwen.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/prompts/prompt_qwen.json -------------------------------------------------------------------------------- /prompts/prompt_together.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/prompts/prompt_together.json -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/requirements.txt -------------------------------------------------------------------------------- /requirements_test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/requirements_test.txt -------------------------------------------------------------------------------- /results_fn_bigquery/.env.yaml.template: -------------------------------------------------------------------------------- 1 | BQ_PROJECT: 2 | BQ_TABLE: 3 | CREDENTIALS_PATH: -------------------------------------------------------------------------------- /results_fn_bigquery/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/results_fn_bigquery/main.py -------------------------------------------------------------------------------- /results_fn_bigquery/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/results_fn_bigquery/requirements.txt -------------------------------------------------------------------------------- /results_fn_postgres/.env.yaml.template: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/results_fn_postgres/.env.yaml.template -------------------------------------------------------------------------------- /results_fn_postgres/create.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/results_fn_postgres/create.sql -------------------------------------------------------------------------------- /results_fn_postgres/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/results_fn_postgres/main.py -------------------------------------------------------------------------------- /results_fn_postgres/requirements.txt: -------------------------------------------------------------------------------- 1 | functions_framework 2 | pandas 3 | psycopg2 -------------------------------------------------------------------------------- /run_checkpoints.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/run_checkpoints.sh -------------------------------------------------------------------------------- /run_checkpoints_adapters.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/run_checkpoints_adapters.sh -------------------------------------------------------------------------------- /run_checkpoints_cot.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/run_checkpoints_cot.sh -------------------------------------------------------------------------------- /run_model_cot.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/run_model_cot.sh -------------------------------------------------------------------------------- /run_qwen.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/run_qwen.sh -------------------------------------------------------------------------------- /runners/anthropic_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/runners/anthropic_runner.py -------------------------------------------------------------------------------- /runners/api_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/runners/api_runner.py -------------------------------------------------------------------------------- /runners/bedrock_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/runners/bedrock_runner.py -------------------------------------------------------------------------------- /runners/deepseek_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/runners/deepseek_runner.py -------------------------------------------------------------------------------- /runners/gemini_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/runners/gemini_runner.py -------------------------------------------------------------------------------- /runners/mistral_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/runners/mistral_runner.py -------------------------------------------------------------------------------- /runners/openai_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/runners/openai_runner.py -------------------------------------------------------------------------------- /runners/together_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/runners/together_runner.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/local_db_tests.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/tests/local_db_tests.py -------------------------------------------------------------------------------- /tests/test_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/tests/test_eval.py -------------------------------------------------------------------------------- /tests/test_utils_pruning.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /translate_sql_dialect.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/translate_sql_dialect.py -------------------------------------------------------------------------------- /upload_wandb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/upload_wandb.py -------------------------------------------------------------------------------- /utils/aliases.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/utils/aliases.py -------------------------------------------------------------------------------- /utils/api_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/utils/api_server.py -------------------------------------------------------------------------------- /utils/asyncio_helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/utils/asyncio_helpers.py -------------------------------------------------------------------------------- /utils/creds.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/utils/creds.py -------------------------------------------------------------------------------- /utils/dialects.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/utils/dialects.py -------------------------------------------------------------------------------- /utils/gen_prompt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/utils/gen_prompt.py -------------------------------------------------------------------------------- /utils/llm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/utils/llm.py -------------------------------------------------------------------------------- /utils/pruning.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/questions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/utils/questions.py -------------------------------------------------------------------------------- /utils/reporting.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/utils/reporting.py -------------------------------------------------------------------------------- /utils/upload_report_gcloud.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/defog-ai/sql-eval/HEAD/utils/upload_report_gcloud.py --------------------------------------------------------------------------------