├── .github └── workflows │ └── close_stale_ticket.yml ├── .gitmodules ├── CodeQueries_preparation ├── README.md ├── aggregate_file_level.py ├── data_ingestion │ ├── create_raw_codeql_queryset.py │ ├── create_raw_programs_dataset.py │ ├── download_and_serialize_dataset.sh │ ├── queries.csv │ ├── raw_codeql_queryset.proto │ ├── raw_programs_dataset.proto │ ├── run_create_raw_codeql_queryset.py │ ├── run_create_raw_programs_dataset.py │ ├── test_create_raw_codeql_queryset.py │ └── test_create_raw_programs_dataset.py └── data_preparation │ ├── commands.md │ ├── contexts │ ├── basecontexts.py │ ├── conflictingattributesinbaseclasses.py │ ├── defineequalswhenaddingattributes.py │ ├── distributable.py │ ├── equalsorhash.py │ ├── flaskdebug.py │ ├── get_builtin_stub.py │ ├── get_context.py │ ├── get_mro.py │ ├── incompleteordering.py │ ├── incorrectcomparisonusingis.py │ ├── initcallssubclassmethod.py │ ├── iterreturnsnoniterator.py │ ├── missingcalltoinit.py │ ├── my-languages.so │ ├── noncallablecalled.py │ ├── signatureoverriddenmethod.py │ ├── test_conflictingattributesinbaseclasses.py │ ├── test_data │ │ ├── test__aux_res.csv │ │ └── test_file_content.py │ ├── test_defineequalswhenaddingattributes.py │ ├── test_distributable.py │ ├── test_equalsorhash.py │ ├── test_flaskdebug.py │ ├── test_incompleteordering.py │ ├── test_incorrectcomparisonusingis.py │ ├── test_initcallssubclassmethod.py │ ├── test_iterreturnsnoniterator.py │ ├── test_missingcalltoinit.py │ ├── test_noncallablecalled.py │ ├── test_run.sh │ ├── test_signatureoverriddenmethod.py │ ├── test_unusedimport.py │ ├── test_useimplicitnonereturnvalue.py │ ├── test_wrongnumberargumentsincall.py │ ├── test_wrongnumberargumentsinclassinstantiation.py │ ├── unusedimport.py │ ├── useimplicitnonereturnvalue.py │ ├── wrongnumberargumentsincall.py │ └── wrongnumberargumentsinclassinstantiation.py │ ├── create_block_subtokens_labels.py │ ├── create_blocks_labels_dataset.py │ ├── create_blocks_relevance_labels_dataset.py │ ├── create_groupwise_prediction_dataset.py │ ├── create_query_result.py │ ├── create_relevance_prediction_examples.py │ ├── create_single_example.py │ ├── create_span_prediction_training_examples.py │ ├── create_tokenized_files_labels.py │ ├── dataset_with_context.proto │ ├── merge_negative_positive_examples.py │ ├── my-languages.so │ ├── run_create_block_subtokens_labels.py │ ├── run_create_blocks_labels_dataset.py │ ├── run_create_groupwise_prediction_dataset.py │ ├── run_create_query_result.py │ ├── run_create_relevance_prediction_examples.py │ ├── run_create_span_prediction_training_examples.py │ ├── run_create_tokenized_files_labels.py │ ├── test_create_block_labels_dataset.py │ ├── test_create_cubert_model_examples.py │ ├── test_create_query_result.py │ ├── test_create_subtokens_labels.py │ ├── test_create_tokenized_files_labels.py │ └── vocab.txt ├── Codequeries_Statistics.pdf ├── LICENSE ├── README.md ├── analyze_classified_spans.py ├── evaluate_generated_spans.py ├── evaluate_relevance.py ├── evaluate_spanprediction.py ├── figures └── QA_Task.png ├── generate_spans.py ├── get_sampled_data.py ├── pretrained_model_configs ├── README.md ├── config_1024.json ├── config_512.json └── vocab.txt ├── prompt_templates ├── ex_with_sf.j2 ├── ex_wo_sf.j2 ├── span_highlight_0shot.j2 ├── span_highlight_fewshot.j2 └── span_highlight_fewshot_sf.j2 ├── requirements.txt ├── resources ├── codequeries_meta.json ├── query_folderName_map.pkl ├── sampled_test_data.pkl └── sampled_train_all_data.pkl ├── setup.sh ├── train_relevanceprediction.py ├── train_spanprediction.py ├── using_CodeQueries.ipynb ├── utils.py └── utils_openai.py /.github/workflows/close_stale_ticket.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/.github/workflows/close_stale_ticket.yml -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/.gitmodules -------------------------------------------------------------------------------- /CodeQueries_preparation/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/README.md -------------------------------------------------------------------------------- /CodeQueries_preparation/aggregate_file_level.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/aggregate_file_level.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_ingestion/create_raw_codeql_queryset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_ingestion/create_raw_codeql_queryset.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_ingestion/create_raw_programs_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_ingestion/create_raw_programs_dataset.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_ingestion/download_and_serialize_dataset.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_ingestion/download_and_serialize_dataset.sh -------------------------------------------------------------------------------- /CodeQueries_preparation/data_ingestion/queries.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_ingestion/queries.csv -------------------------------------------------------------------------------- /CodeQueries_preparation/data_ingestion/raw_codeql_queryset.proto: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_ingestion/raw_codeql_queryset.proto -------------------------------------------------------------------------------- /CodeQueries_preparation/data_ingestion/raw_programs_dataset.proto: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_ingestion/raw_programs_dataset.proto -------------------------------------------------------------------------------- /CodeQueries_preparation/data_ingestion/run_create_raw_codeql_queryset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_ingestion/run_create_raw_codeql_queryset.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_ingestion/run_create_raw_programs_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_ingestion/run_create_raw_programs_dataset.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_ingestion/test_create_raw_codeql_queryset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_ingestion/test_create_raw_codeql_queryset.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_ingestion/test_create_raw_programs_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_ingestion/test_create_raw_programs_dataset.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/commands.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/commands.md -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/basecontexts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/basecontexts.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/conflictingattributesinbaseclasses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/conflictingattributesinbaseclasses.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/defineequalswhenaddingattributes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/defineequalswhenaddingattributes.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/distributable.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/distributable.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/equalsorhash.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/equalsorhash.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/flaskdebug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/flaskdebug.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/get_builtin_stub.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/get_builtin_stub.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/get_context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/get_context.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/get_mro.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/get_mro.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/incompleteordering.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/incompleteordering.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/incorrectcomparisonusingis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/incorrectcomparisonusingis.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/initcallssubclassmethod.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/initcallssubclassmethod.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/iterreturnsnoniterator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/iterreturnsnoniterator.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/missingcalltoinit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/missingcalltoinit.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/my-languages.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/my-languages.so -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/noncallablecalled.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/noncallablecalled.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/signatureoverriddenmethod.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/signatureoverriddenmethod.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_conflictingattributesinbaseclasses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_conflictingattributesinbaseclasses.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_data/test__aux_res.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_data/test__aux_res.csv -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_data/test_file_content.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_data/test_file_content.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_defineequalswhenaddingattributes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_defineequalswhenaddingattributes.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_distributable.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_distributable.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_equalsorhash.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_equalsorhash.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_flaskdebug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_flaskdebug.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_incompleteordering.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_incompleteordering.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_incorrectcomparisonusingis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_incorrectcomparisonusingis.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_initcallssubclassmethod.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_initcallssubclassmethod.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_iterreturnsnoniterator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_iterreturnsnoniterator.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_missingcalltoinit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_missingcalltoinit.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_noncallablecalled.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_noncallablecalled.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_run.sh -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_signatureoverriddenmethod.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_signatureoverriddenmethod.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_unusedimport.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_unusedimport.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_useimplicitnonereturnvalue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_useimplicitnonereturnvalue.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_wrongnumberargumentsincall.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_wrongnumberargumentsincall.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/test_wrongnumberargumentsinclassinstantiation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/test_wrongnumberargumentsinclassinstantiation.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/unusedimport.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/unusedimport.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/useimplicitnonereturnvalue.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/useimplicitnonereturnvalue.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/wrongnumberargumentsincall.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/wrongnumberargumentsincall.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/contexts/wrongnumberargumentsinclassinstantiation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/contexts/wrongnumberargumentsinclassinstantiation.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/create_block_subtokens_labels.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/create_block_subtokens_labels.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/create_blocks_labels_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/create_blocks_labels_dataset.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/create_blocks_relevance_labels_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/create_blocks_relevance_labels_dataset.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/create_groupwise_prediction_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/create_groupwise_prediction_dataset.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/create_query_result.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/create_query_result.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/create_relevance_prediction_examples.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/create_relevance_prediction_examples.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/create_single_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/create_single_example.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/create_span_prediction_training_examples.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/create_span_prediction_training_examples.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/create_tokenized_files_labels.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/create_tokenized_files_labels.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/dataset_with_context.proto: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/dataset_with_context.proto -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/merge_negative_positive_examples.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/merge_negative_positive_examples.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/my-languages.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/my-languages.so -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/run_create_block_subtokens_labels.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/run_create_block_subtokens_labels.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/run_create_blocks_labels_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/run_create_blocks_labels_dataset.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/run_create_groupwise_prediction_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/run_create_groupwise_prediction_dataset.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/run_create_query_result.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/run_create_query_result.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/run_create_relevance_prediction_examples.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/run_create_relevance_prediction_examples.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/run_create_span_prediction_training_examples.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/run_create_span_prediction_training_examples.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/run_create_tokenized_files_labels.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/run_create_tokenized_files_labels.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/test_create_block_labels_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/test_create_block_labels_dataset.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/test_create_cubert_model_examples.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/test_create_cubert_model_examples.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/test_create_query_result.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/test_create_query_result.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/test_create_subtokens_labels.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/test_create_subtokens_labels.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/test_create_tokenized_files_labels.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/test_create_tokenized_files_labels.py -------------------------------------------------------------------------------- /CodeQueries_preparation/data_preparation/vocab.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/CodeQueries_preparation/data_preparation/vocab.txt -------------------------------------------------------------------------------- /Codequeries_Statistics.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/Codequeries_Statistics.pdf -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/README.md -------------------------------------------------------------------------------- /analyze_classified_spans.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/analyze_classified_spans.py -------------------------------------------------------------------------------- /evaluate_generated_spans.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/evaluate_generated_spans.py -------------------------------------------------------------------------------- /evaluate_relevance.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/evaluate_relevance.py -------------------------------------------------------------------------------- /evaluate_spanprediction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/evaluate_spanprediction.py -------------------------------------------------------------------------------- /figures/QA_Task.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/figures/QA_Task.png -------------------------------------------------------------------------------- /generate_spans.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/generate_spans.py -------------------------------------------------------------------------------- /get_sampled_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/get_sampled_data.py -------------------------------------------------------------------------------- /pretrained_model_configs/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/pretrained_model_configs/README.md -------------------------------------------------------------------------------- /pretrained_model_configs/config_1024.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/pretrained_model_configs/config_1024.json -------------------------------------------------------------------------------- /pretrained_model_configs/config_512.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/pretrained_model_configs/config_512.json -------------------------------------------------------------------------------- /pretrained_model_configs/vocab.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/pretrained_model_configs/vocab.txt -------------------------------------------------------------------------------- /prompt_templates/ex_with_sf.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/prompt_templates/ex_with_sf.j2 -------------------------------------------------------------------------------- /prompt_templates/ex_wo_sf.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/prompt_templates/ex_wo_sf.j2 -------------------------------------------------------------------------------- /prompt_templates/span_highlight_0shot.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/prompt_templates/span_highlight_0shot.j2 -------------------------------------------------------------------------------- /prompt_templates/span_highlight_fewshot.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/prompt_templates/span_highlight_fewshot.j2 -------------------------------------------------------------------------------- /prompt_templates/span_highlight_fewshot_sf.j2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/prompt_templates/span_highlight_fewshot_sf.j2 -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | transformers==4.10 2 | tqdm 3 | absl-py 4 | sklearn 5 | datasets 6 | rank_bm25 -------------------------------------------------------------------------------- /resources/codequeries_meta.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/resources/codequeries_meta.json -------------------------------------------------------------------------------- /resources/query_folderName_map.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/resources/query_folderName_map.pkl -------------------------------------------------------------------------------- /resources/sampled_test_data.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/resources/sampled_test_data.pkl -------------------------------------------------------------------------------- /resources/sampled_train_all_data.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/resources/sampled_train_all_data.pkl -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/setup.sh -------------------------------------------------------------------------------- /train_relevanceprediction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/train_relevanceprediction.py -------------------------------------------------------------------------------- /train_spanprediction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/train_spanprediction.py -------------------------------------------------------------------------------- /using_CodeQueries.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/using_CodeQueries.ipynb -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/utils.py -------------------------------------------------------------------------------- /utils_openai.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thepurpleowl/codequeries-benchmark/HEAD/utils_openai.py --------------------------------------------------------------------------------