├── AI_ETHICS.md ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.md ├── SECURITY.md ├── data ├── mini_val │ ├── competition │ │ ├── cache-6022f0f10d984b45_00000_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00001_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00002_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00003_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00004_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00005_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00006_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00007_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00008_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00009_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00010_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00011_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00012_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00013_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00014_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00015_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00016_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00017_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00018_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00019_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00020_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00021_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00022_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00023_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00024_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00025_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00026_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00027_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00028_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00029_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00030_of_00032.arrow │ │ ├── cache-6022f0f10d984b45_00031_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00000_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00001_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00002_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00003_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00004_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00005_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00006_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00007_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00008_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00009_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00010_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00011_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00012_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00013_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00014_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00015_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00016_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00017_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00018_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00019_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00020_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00021_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00022_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00023_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00024_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00025_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00026_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00027_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00028_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00029_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00030_of_00032.arrow │ │ ├── cache-ced9542cf73d68e1_00031_of_00032.arrow │ │ ├── data-00000-of-00001.arrow │ │ ├── dataset_info.json │ │ └── state.json │ ├── dataset_dict.json │ ├── interview │ │ ├── cache-6097a52e1350d851_00000_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00001_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00002_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00003_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00004_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00005_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00006_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00007_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00008_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00009_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00010_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00011_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00012_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00013_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00014_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00015_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00016_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00017_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00018_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00019_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00020_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00021_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00022_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00023_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00024_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00025_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00026_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00027_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00028_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00029_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00030_of_00032.arrow │ │ ├── cache-6097a52e1350d851_00031_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00000_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00001_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00002_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00003_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00004_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00005_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00006_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00007_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00008_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00009_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00010_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00011_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00012_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00013_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00014_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00015_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00016_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00017_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00018_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00019_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00020_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00021_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00022_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00023_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00024_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00025_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00026_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00027_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00028_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00029_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00030_of_00032.arrow │ │ ├── cache-6a9138e60b7da468_00031_of_00032.arrow │ │ ├── data-00000-of-00001.arrow │ │ ├── dataset_info.json │ │ └── state.json │ └── introductory │ │ ├── cache-08efd176dfa05db8_00000_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00001_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00002_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00003_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00004_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00005_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00006_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00007_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00008_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00009_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00010_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00011_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00012_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00013_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00014_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00015_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00016_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00017_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00018_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00019_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00020_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00021_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00022_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00023_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00024_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00025_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00026_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00027_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00028_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00029_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00030_of_00032.arrow │ │ ├── cache-08efd176dfa05db8_00031_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00000_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00001_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00002_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00003_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00004_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00005_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00006_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00007_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00008_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00009_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00010_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00011_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00012_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00013_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00014_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00015_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00016_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00017_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00018_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00019_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00020_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00021_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00022_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00023_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00024_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00025_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00026_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00027_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00028_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00029_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00030_of_00032.arrow │ │ ├── cache-4bcfb8043179f39d_00031_of_00032.arrow │ │ ├── data-00000-of-00001.arrow │ │ ├── dataset_info.json │ │ └── state.json ├── mini_val_example_tests.pkl └── test_example_tests.pkl ├── images └── code llm agent - clustering_v2.svg ├── install.sh ├── prompts ├── codechain_gen.txt ├── codechain_revise.txt └── direct_gen.txt ├── scripts ├── evaluate_codechain.sh ├── evaluate_direct_gen.sh ├── run_codechain.sh └── run_direct_gen.sh └── src ├── .DS_Store ├── clustering.py ├── configs ├── .DS_Store ├── config_clustering.py ├── config_evaluate.py ├── config_generate.py └── config_processing.py ├── embedding ├── constants.py ├── datasets_loader.py ├── distributed_utils.py ├── encoder.py ├── preprocessing_utils.py └── utils.py ├── evaluate.py ├── generate.py ├── processing.py └── utils ├── .DS_Store ├── utils.py ├── utils_evaluate.py └── utils_execute.py /AI_ETHICS.md: -------------------------------------------------------------------------------- 1 | ## Ethics disclaimer for Salesforce AI models, data, code 2 | 3 | This release is for research purposes only in support of an academic 4 | paper. Our models, datasets, and code are not specifically designed or 5 | evaluated for all downstream purposes. We strongly recommend users 6 | evaluate and address potential concerns related to accuracy, safety, and 7 | fairness before deploying this model. We encourage users to consider the 8 | common limitations of AI, comply with applicable laws, and leverage best 9 | practices when selecting use cases, particularly for high-risk scenarios 10 | where errors or misuse could significantly impact people’s lives, rights, 11 | or safety. For further guidance on use cases, refer to our standard 12 | [AUP](https://www.salesforce.com/content/dam/web/en_us/www/documents/legal/Agreements/policies/ExternalFacing_Services_Policy.pdf) 13 | and [AI AUP](https://www.salesforce.com/content/dam/web/en_us/www/documents/legal/Agreements/policies/ai-acceptable-use-policy.pdf). 14 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Comment line immediately above ownership line is reserved for related other information. Please be careful while editing. 2 | #ECCN:Open Source 5D002 3 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Salesforce Open Source Community Code of Conduct 2 | 3 | ## About the Code of Conduct 4 | 5 | Equality is a core value at Salesforce. We believe a diverse and inclusive 6 | community fosters innovation and creativity, and are committed to building a 7 | culture where everyone feels included. 8 | 9 | Salesforce open-source projects are committed to providing a friendly, safe, and 10 | welcoming environment for all, regardless of gender identity and expression, 11 | sexual orientation, disability, physical appearance, body size, ethnicity, nationality, 12 | race, age, religion, level of experience, education, socioeconomic status, or 13 | other similar personal characteristics. 14 | 15 | The goal of this code of conduct is to specify a baseline standard of behavior so 16 | that people with different social values and communication styles can work 17 | together effectively, productively, and respectfully in our open source community. 18 | It also establishes a mechanism for reporting issues and resolving conflicts. 19 | 20 | All questions and reports of abusive, harassing, or otherwise unacceptable behavior 21 | in a Salesforce open-source project may be reported by contacting the Salesforce 22 | Open Source Conduct Committee at ossconduct@salesforce.com. 23 | 24 | ## Our Pledge 25 | 26 | In the interest of fostering an open and welcoming environment, we as 27 | contributors and maintainers pledge to making participation in our project and 28 | our community a harassment-free experience for everyone, regardless of gender 29 | identity and expression, sexual orientation, disability, physical appearance, 30 | body size, ethnicity, nationality, race, age, religion, level of experience, education, 31 | socioeconomic status, or other similar personal characteristics. 32 | 33 | ## Our Standards 34 | 35 | Examples of behavior that contributes to creating a positive environment 36 | include: 37 | 38 | * Using welcoming and inclusive language 39 | * Being respectful of differing viewpoints and experiences 40 | * Gracefully accepting constructive criticism 41 | * Focusing on what is best for the community 42 | * Showing empathy toward other community members 43 | 44 | Examples of unacceptable behavior by participants include: 45 | 46 | * The use of sexualized language or imagery and unwelcome sexual attention or 47 | advances 48 | * Personal attacks, insulting/derogatory comments, or trolling 49 | * Public or private harassment 50 | * Publishing, or threatening to publish, others' private information—such as 51 | a physical or electronic address—without explicit permission 52 | * Other conduct which could reasonably be considered inappropriate in a 53 | professional setting 54 | * Advocating for or encouraging any of the above behaviors 55 | 56 | ## Our Responsibilities 57 | 58 | Project maintainers are responsible for clarifying the standards of acceptable 59 | behavior and are expected to take appropriate and fair corrective action in 60 | response to any instances of unacceptable behavior. 61 | 62 | Project maintainers have the right and responsibility to remove, edit, or 63 | reject comments, commits, code, wiki edits, issues, and other contributions 64 | that are not aligned with this Code of Conduct, or to ban temporarily or 65 | permanently any contributor for other behaviors that they deem inappropriate, 66 | threatening, offensive, or harmful. 67 | 68 | ## Scope 69 | 70 | This Code of Conduct applies both within project spaces and in public spaces 71 | when an individual is representing the project or its community. Examples of 72 | representing a project or community include using an official project email 73 | address, posting via an official social media account, or acting as an appointed 74 | representative at an online or offline event. Representation of a project may be 75 | further defined and clarified by project maintainers. 76 | 77 | ## Enforcement 78 | 79 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 80 | reported by contacting the Salesforce Open Source Conduct Committee 81 | at ossconduct@salesforce.com. All complaints will be reviewed and investigated 82 | and will result in a response that is deemed necessary and appropriate to the 83 | circumstances. The committee is obligated to maintain confidentiality with 84 | regard to the reporter of an incident. Further details of specific enforcement 85 | policies may be posted separately. 86 | 87 | Project maintainers who do not follow or enforce the Code of Conduct in good 88 | faith may face temporary or permanent repercussions as determined by other 89 | members of the project's leadership and the Salesforce Open Source Conduct 90 | Committee. 91 | 92 | ## Attribution 93 | 94 | This Code of Conduct is adapted from the [Contributor Covenant][contributor-covenant-home], 95 | version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html. 96 | It includes adaptions and additions from [Go Community Code of Conduct][golang-coc], 97 | [CNCF Code of Conduct][cncf-coc], and [Microsoft Open Source Code of Conduct][microsoft-coc]. 98 | 99 | This Code of Conduct is licensed under the [Creative Commons Attribution 3.0 License][cc-by-3-us]. 100 | 101 | [contributor-covenant-home]: https://www.contributor-covenant.org (https://www.contributor-covenant.org/) 102 | [golang-coc]: https://golang.org/conduct 103 | [cncf-coc]: https://github.com/cncf/foundation/blob/master/code-of-conduct.md 104 | [microsoft-coc]: https://opensource.microsoft.com/codeofconduct/ 105 | [cc-by-3-us]: https://creativecommons.org/licenses/by/3.0/us/ -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 2 | # Contributing Guide For CodeChain 3 | 4 | This page lists the operational governance model of this project, as well as the recommendations and requirements for how to best contribute to CodeChain. We strive to obey these as best as possible. As always, thanks for contributing – we hope these guidelines make it easier and shed some light on our approach and processes. 5 | 6 | # Governance Model 7 | 8 | For this project, please follow the community-based governance model below. 9 | 10 | ## Community Based 11 | 12 | The intent and goal of open sourcing this project is to increase the contributor and user base. The governance model is one where new project leads (`admins`) will be added to the project based on their contributions and efforts, a so-called "do-acracy" or "meritocracy" similar to that used by all Apache Software Foundation projects. 13 | 14 | 15 | # Issues, requests & ideas 16 | 17 | Use GitHub Issues page to submit issues, enhancement requests and discuss ideas. 18 | 19 | ### Bug Reports and Fixes 20 | - If you find a bug, please search for it in the [Issues](https://github.com/SalesforceAIResearch/CodeChain/issues), and if it isn't already tracked, 21 | [create a new issue](https://github.com/SalesforceAIResearch/CodeChain/issues/new). Fill out the "Bug Report" section of the issue template. Even if an Issue is closed, feel free to comment and add details, it will still 22 | be reviewed. 23 | - Issues that have already been identified as a bug (note: able to reproduce) will be labelled `bug`. 24 | - If you'd like to submit a fix for a bug, [send a Pull Request](#creating_a_pull_request) and mention the Issue number. 25 | - Include tests that isolate the bug and verifies that it was fixed. 26 | 27 | ### New Features 28 | - If you'd like to add new functionality to this project, describe the problem you want to solve in a [new Issue](https://github.com/SalesforceAIResearch/CodeChain/issues/new). 29 | - Issues that have been identified as a feature request will be labelled `enhancement`. 30 | - If you'd like to implement the new feature, please wait for feedback from the project 31 | maintainers before spending too much time writing the code. In some cases, `enhancement`s may 32 | not align well with the project objectives at the time. 33 | 34 | ### Tests, Documentation, Miscellaneous 35 | - If you'd like to improve the tests, you want to make the documentation clearer, you have an 36 | alternative implementation of something that may have advantages over the way its currently 37 | done, or you have any other change, we would be happy to hear about it! 38 | - If its a trivial change, go ahead and [send a Pull Request](#creating_a_pull_request) with the changes you have in mind. 39 | - If not, [open an Issue](https://github.com/SalesforceAIResearch/CodeChain/issues/new) to discuss the idea first. 40 | 41 | If you're new to our project and looking for some way to make your first contribution, look for 42 | Issues labelled `good first contribution`. 43 | 44 | # Contribution Checklist 45 | 46 | - [x] Clean, simple, well styled code 47 | - [x] Commits should be atomic and messages must be descriptive. Related issues should be mentioned by Issue number. 48 | - [x] Comments 49 | - Module-level & function-level comments. 50 | - Comments on complex blocks of code or algorithms (include references to sources). 51 | - [x] Tests 52 | - The test suite, if provided, must be complete and pass 53 | - Increase code coverage, not versa. 54 | - Use any of our testkits that contains a bunch of testing facilities you would need. For example: `import com.salesforce.op.test._` and borrow inspiration from existing tests. 55 | - [x] Dependencies 56 | - Minimize number of dependencies. 57 | - Prefer Apache 2.0, BSD3, MIT, ISC and MPL licenses. 58 | - [x] Reviews 59 | - Changes must be approved via peer code review 60 | 61 | # Creating a Pull Request 62 | 63 | 1. **Ensure the bug/feature was not already reported** by searching on GitHub under Issues. If none exists, create a new issue so that other contributors can keep track of what you are trying to add/fix and offer suggestions (or let you know if there is already an effort in progress). 64 | 3. **Clone** the forked repo to your machine. 65 | 4. **Create** a new branch to contain your work (e.g. `git br fix-issue-11`) 66 | 4. **Commit** changes to your own branch. 67 | 5. **Push** your work back up to your fork. (e.g. `git push fix-issue-11`) 68 | 6. **Submit** a Pull Request against the `main` branch and refer to the issue(s) you are fixing. Try not to pollute your pull request with unintended changes. Keep it simple and small. 69 | 7. **Sign** the Salesforce CLA (you will be prompted to do so when submitting the Pull Request) 70 | 71 | > **NOTE**: Be sure to [sync your fork](https://help.github.com/articles/syncing-a-fork/) before making a pull request. 72 | 73 | 74 | # Code of Conduct 75 | Please follow our [Code of Conduct](CODE_OF_CONDUCT.md). 76 | 77 | # License 78 | By contributing your code, you agree to license your contribution under the terms of our project [LICENSE](LICENSE.txt) and to sign the [Salesforce CLA](https://cla.salesforce.com/sign-cla) 79 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License Version 2.0 2 | 3 | Copyright (c) 2023 Salesforce, Inc. 4 | All rights reserved. 5 | 6 | Apache License 7 | Version 2.0, January 2004 8 | http://www.apache.org/licenses/ 9 | 10 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 11 | 12 | 1. Definitions. 13 | 14 | "License" shall mean the terms and conditions for use, reproduction, 15 | and distribution as defined by Sections 1 through 9 of this document. 16 | 17 | "Licensor" shall mean the copyright owner or entity authorized by 18 | the copyright owner that is granting the License. 19 | 20 | "Legal Entity" shall mean the union of the acting entity and all 21 | other entities that control, are controlled by, or are under common 22 | control with that entity. For the purposes of this definition, 23 | "control" means (i) the power, direct or indirect, to cause the 24 | direction or management of such entity, whether by contract or 25 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 26 | outstanding shares, or (iii) beneficial ownership of such entity. 27 | 28 | "You" (or "Your") shall mean an individual or Legal Entity 29 | exercising permissions granted by this License. 30 | 31 | "Source" form shall mean the preferred form for making modifications, 32 | including but not limited to software source code, documentation 33 | source, and configuration files. 34 | 35 | "Object" form shall mean any form resulting from mechanical 36 | transformation or translation of a Source form, including but 37 | not limited to compiled object code, generated documentation, 38 | and conversions to other media types. 39 | 40 | "Work" shall mean the work of authorship, whether in Source or 41 | Object form, made available under the License, as indicated by a 42 | copyright notice that is included in or attached to the work 43 | (an example is provided in the Appendix below). 44 | 45 | "Derivative Works" shall mean any work, whether in Source or Object 46 | form, that is based on (or derived from) the Work and for which the 47 | editorial revisions, annotations, elaborations, or other modifications 48 | represent, as a whole, an original work of authorship. For the purposes 49 | of this License, Derivative Works shall not include works that remain 50 | separable from, or merely link (or bind by name) to the interfaces of, 51 | the Work and Derivative Works thereof. 52 | 53 | "Contribution" shall mean any work of authorship, including 54 | the original version of the Work and any modifications or additions 55 | to that Work or Derivative Works thereof, that is intentionally 56 | submitted to Licensor for inclusion in the Work by the copyright owner 57 | or by an individual or Legal Entity authorized to submit on behalf of 58 | the copyright owner. For the purposes of this definition, "submitted" 59 | means any form of electronic, verbal, or written communication sent 60 | to the Licensor or its representatives, including but not limited to 61 | communication on electronic mailing lists, source code control systems, 62 | and issue tracking systems that are managed by, or on behalf of, the 63 | Licensor for the purpose of discussing and improving the Work, but 64 | excluding communication that is conspicuously marked or otherwise 65 | designated in writing by the copyright owner as "Not a Contribution." 66 | 67 | "Contributor" shall mean Licensor and any individual or Legal Entity 68 | on behalf of whom a Contribution has been received by Licensor and 69 | subsequently incorporated within the Work. 70 | 71 | 2. Grant of Copyright License. Subject to the terms and conditions of 72 | this License, each Contributor hereby grants to You a perpetual, 73 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 74 | copyright license to reproduce, prepare Derivative Works of, 75 | publicly display, publicly perform, sublicense, and distribute the 76 | Work and such Derivative Works in Source or Object form. 77 | 78 | 3. Grant of Patent License. Subject to the terms and conditions of 79 | this License, each Contributor hereby grants to You a perpetual, 80 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 81 | (except as stated in this section) patent license to make, have made, 82 | use, offer to sell, sell, import, and otherwise transfer the Work, 83 | where such license applies only to those patent claims licensable 84 | by such Contributor that are necessarily infringed by their 85 | Contribution(s) alone or by combination of their Contribution(s) 86 | with the Work to which such Contribution(s) was submitted. If You 87 | institute patent litigation against any entity (including a 88 | cross-claim or counterclaim in a lawsuit) alleging that the Work 89 | or a Contribution incorporated within the Work constitutes direct 90 | or contributory patent infringement, then any patent licenses 91 | granted to You under this License for that Work shall terminate 92 | as of the date such litigation is filed. 93 | 94 | 4. Redistribution. You may reproduce and distribute copies of the 95 | Work or Derivative Works thereof in any medium, with or without 96 | modifications, and in Source or Object form, provided that You 97 | meet the following conditions: 98 | 99 | (a) You must give any other recipients of the Work or 100 | Derivative Works a copy of this License; and 101 | 102 | (b) You must cause any modified files to carry prominent notices 103 | stating that You changed the files; and 104 | 105 | (c) You must retain, in the Source form of any Derivative Works 106 | that You distribute, all copyright, patent, trademark, and 107 | attribution notices from the Source form of the Work, 108 | excluding those notices that do not pertain to any part of 109 | the Derivative Works; and 110 | 111 | (d) If the Work includes a "NOTICE" text file as part of its 112 | distribution, then any Derivative Works that You distribute must 113 | include a readable copy of the attribution notices contained 114 | within such NOTICE file, excluding those notices that do not 115 | pertain to any part of the Derivative Works, in at least one 116 | of the following places: within a NOTICE text file distributed 117 | as part of the Derivative Works; within the Source form or 118 | documentation, if provided along with the Derivative Works; or, 119 | within a display generated by the Derivative Works, if and 120 | wherever such third-party notices normally appear. The contents 121 | of the NOTICE file are for informational purposes only and 122 | do not modify the License. You may add Your own attribution 123 | notices within Derivative Works that You distribute, alongside 124 | or as an addendum to the NOTICE text from the Work, provided 125 | that such additional attribution notices cannot be construed 126 | as modifying the License. 127 | 128 | You may add Your own copyright statement to Your modifications and 129 | may provide additional or different license terms and conditions 130 | for use, reproduction, or distribution of Your modifications, or 131 | for any such Derivative Works as a whole, provided Your use, 132 | reproduction, and distribution of the Work otherwise complies with 133 | the conditions stated in this License. 134 | 135 | 5. Submission of Contributions. Unless You explicitly state otherwise, 136 | any Contribution intentionally submitted for inclusion in the Work 137 | by You to the Licensor shall be under the terms and conditions of 138 | this License, without any additional terms or conditions. 139 | Notwithstanding the above, nothing herein shall supersede or modify 140 | the terms of any separate license agreement you may have executed 141 | with Licensor regarding such Contributions. 142 | 143 | 6. Trademarks. This License does not grant permission to use the trade 144 | names, trademarks, service marks, or product names of the Licensor, 145 | except as required for reasonable and customary use in describing the 146 | origin of the Work and reproducing the content of the NOTICE file. 147 | 148 | 7. Disclaimer of Warranty. Unless required by applicable law or 149 | agreed to in writing, Licensor provides the Work (and each 150 | Contributor provides its Contributions) on an "AS IS" BASIS, 151 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 152 | implied, including, without limitation, any warranties or conditions 153 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 154 | PARTICULAR PURPOSE. You are solely responsible for determining the 155 | appropriateness of using or redistributing the Work and assume any 156 | risks associated with Your exercise of permissions under this License. 157 | 158 | 8. Limitation of Liability. In no event and under no legal theory, 159 | whether in tort (including negligence), contract, or otherwise, 160 | unless required by applicable law (such as deliberate and grossly 161 | negligent acts) or agreed to in writing, shall any Contributor be 162 | liable to You for damages, including any direct, indirect, special, 163 | incidental, or consequential damages of any character arising as a 164 | result of this License or out of the use or inability to use the 165 | Work (including but not limited to damages for loss of goodwill, 166 | work stoppage, computer failure or malfunction, or any and all 167 | other commercial damages or losses), even if such Contributor 168 | has been advised of the possibility of such damages. 169 | 170 | 9. Accepting Warranty or Additional Liability. While redistributing 171 | the Work or Derivative Works thereof, You may choose to offer, 172 | and charge a fee for, acceptance of support, warranty, indemnity, 173 | or other liability obligations and/or rights consistent with this 174 | License. However, in accepting such obligations, You may act only 175 | on Your own behalf and on Your sole responsibility, not on behalf 176 | of any other Contributor, and only if You agree to indemnify, 177 | defend, and hold each Contributor harmless for any liability 178 | incurred by, or claims asserted against, such Contributor by reason 179 | of your accepting any such warranty or additional liability. 180 | 181 | END OF TERMS AND CONDITIONS 182 | 183 | APPENDIX: How to apply the Apache License to your work. 184 | 185 | To apply the Apache License to your work, attach the following 186 | boilerplate notice, with the fields enclosed by brackets "{}" 187 | replaced with your own identifying information. (Don't include 188 | the brackets!) The text should be enclosed in the appropriate 189 | comment syntax for the file format. We also recommend that a 190 | file or class name and description of purpose be included on the 191 | same "printed page" as the copyright notice for easier 192 | identification within third-party archives. 193 | 194 | Copyright {yyyy} {name of copyright owner} 195 | 196 | Licensed under the Apache License, Version 2.0 (the "License"); 197 | you may not use this file except in compliance with the License. 198 | You may obtain a copy of the License at 199 | 200 | http://www.apache.org/licenses/LICENSE-2.0 201 | 202 | Unless required by applicable law or agreed to in writing, software 203 | distributed under the License is distributed on an "AS IS" BASIS, 204 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 205 | See the License for the specific language governing permissions and 206 | limitations under the License. 207 | 208 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CodeChain: Towards Modular Code Generation Through Chain of Self-revisions with Representative Sub-modules 2 | 3 | This is the official code for the paper "CodeChain: Towards Modular Code Generation Through Chain of Self-revisions with Representative Sub-modules". Check out our [blog](https://blog.salesforceairesearch.com/codechain/) and [paper](https://arxiv.org/abs/2310.08992) for more details. 4 | 5 | Authors: [Hung Le](https://scholar.google.com/citations?user=jnYI1UgAAAAJ&hl=en), [Hailin Chen](https://aclanthology.org/people/h/hailin-chen/), [Amrita Saha](https://scholar.google.co.uk/citations?user=3Zb5Y2YAAAAJ&hl=en), [Akash Gokul](https://scholar.google.com/citations?user=MYRUJkUAAAAJ&hl=en), [Doyen Sahoo](https://scholar.google.com.sg/citations?user=A61jJD4AAAAJ&hl=en), [Shafiq Joty](https://scholar.google.com/citations?user=hR249csAAAAJ&hl=en) 6 | 7 | ### Contents: 8 | * [x] [CodeChain Overview](#codechain-overview) 9 | * [x] [Setup Datasets and Models](#setup-datasets-and-models) 10 | * [x] [Generate and evaluate code](#generate-and-evaluate-code) 11 | * [x] [Citation](#citation) 12 | * [x] [License](#license) 13 | 14 | ## CodeChain Overview 15 |

16 | 17 |

18 | A pretrained LLM is first instructed with chain-of-thought prompting to generate a set of modularised solutions. Generated sub-modules are then extracted from potentially correct solutions and grouped into different semantic clusters. The cluster centroids are selected as representative sub-modules to condition the next self-revision round. The model is instructed to reuse or adapt these modules into its revised solutions. 19 | 20 | ## Setup Datasets and Models 21 | 22 | If you have conda ready, just run to install dependencies: 23 | 24 | ``` 25 | ./install.sh 26 | ``` 27 | 28 | ### Datasets 29 | 30 | We use the main benchmark [APPS](https://huggingface.co/datasets/codeparrot/apps) in our experiments. Please follow the instructions to donwload the datasets. 31 | 32 | To run the code, place the dataset into the `data` folder. For APPS, parition the data into sub-folder by the difficulty level e.g. competition, interview, introductory. We uploaded the `data/mini_val` for the APPS validation split we used in the paper. 33 | 34 | We also uploaded the example test cases we extracted from the problem description on APPS. See `*_example_tests.pkl` files in the `data` folder. 35 | 36 | ### Models 37 | 38 | Put a file `openaiapikey.txt` for querying GPT models. The file should just contain your OpenAI api key. By default, our code is compatible to run with GPT3.5 or GPT4. To run on open source models, we recommend to set up following the [VLLM framework](https://github.com/vllm-project/vllm) which can speed the generation significantly. The inference code on this framework is similar to how we run on a GPT model in this repo. 39 | 40 | 41 | ## Generate and Evaluate Code 42 | ### Direct Generation 43 | 44 | ``` 45 | ./scripts/run_direct_gen.sh 46 | ``` 47 | 48 | Run this script to direct generate code in a normal setting (without CoT prompting and self-revision). 49 | This process uses the prompt `prompts/direct_gen.txt` to instruct the model to generate code. 50 | Refer to `src/configs/config_generate.py` for a list of parameters to be configured. 51 | The output code is saved into a sub-folder under `outputs` folder. 52 | 53 | ### Code Chain 54 | 55 | ``` 56 | ./scripts/run_codechain.sh 57 | ``` 58 | 59 | Run this script to generate and revise code iteratively in with CodeChain framework. 60 | This process uses the prompt `prompts/codechain_gen.txt` to instruct the model to generate code with CoT prompting and `prompts/codechain_revise.txt` to instruct the model to self-revise the code with selected represenatative sub-modules. 61 | This process contains 4 major sub-processes: generating code, evaluating code with example test cases, post-processing, and clustering. 62 | Refer to the corresponding `src/configs/config_.py` for a list of parameters to be configured. 63 | All outputs (including code, processed code, extracted modules and clustered samples) are saved with a template of the `outputs/*` (e.g. `outputs/_data.csv`, `outputs/_xxclusters.csv`). 64 | 65 | 66 | ### Evaluation 67 | ``` 68 | ./scripts/evaluate_codechain.sh 69 | ``` 70 | 71 | Finally, to evaluate the code on hidden test cases by pass@1, run the evaluation script. This will evaluate the generated code on all generation and revision rounds. 72 | 73 | 74 | ## Citation 75 | If you find the paper or the source code useful to your projects, please cite the following bibtex: 76 | 77 | 
78 | @misc{le2023codechain,
79 |       title={CodeChain: Towards Modular Code Generation Through Chain of Self-revisions with Representative Sub-modules}, 
80 |       author={Hung Le and Hailin Chen and Amrita Saha and Akash Gokul and Doyen Sahoo and Shafiq Joty},
81 |       year={2023},
82 |       eprint={2310.08992},
83 |       archivePrefix={arXiv},
84 |       primaryClass={cs.AI}
85 | }
86 |
87 | 88 | ## License 89 | 90 | The code is released under Apache License Version 2.0 - see `LICENSE.txt` for details. 91 | 92 | The code is developed from other open source projects: including [APPS](https://github.com/hendrycks/apps/), [BigCode Encoder](https://github.com/bigcode-project/bigcode-encoder), and [BigCode Evaluation Harness](https://github.com/bigcode-project/bigcode-evaluation-harness). We thank the original contributors of these works for open-sourcing their valuable source codes. 93 | 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | ## Security 2 | 3 | Please report any security issue to [security@salesforce.com](mailto:security@salesforce.com) 4 | as soon as it is discovered. This library limits its runtime dependencies in 5 | order to reduce the total cost of ownership as much as can be, but all consumers 6 | should remain vigilant and have their security stakeholders review all third-party 7 | products (3PP) like this one and their dependencies. -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00000_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00000_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00001_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00001_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00002_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00002_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00003_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00003_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00004_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00004_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00005_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00005_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00006_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00006_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00007_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00007_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00008_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00008_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00009_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00009_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00010_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00010_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00011_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00011_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00012_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00012_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00013_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00013_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00014_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00014_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00015_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00015_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00016_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00016_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00017_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00017_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00018_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00018_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00019_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00019_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00020_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00020_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00021_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00021_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00022_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00022_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00023_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00023_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00024_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00024_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00025_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00025_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00026_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00026_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00027_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00027_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00028_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00028_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00029_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00029_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00030_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00030_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-6022f0f10d984b45_00031_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-6022f0f10d984b45_00031_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00000_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00000_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00001_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00001_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00002_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00002_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00003_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00003_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00004_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00004_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00005_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00005_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00006_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00006_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00007_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00007_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00008_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00008_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00009_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00009_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00010_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00010_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00011_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00011_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00012_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00012_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00013_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00013_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00014_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00014_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00015_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00015_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00016_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00016_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00017_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00017_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00018_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00018_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00019_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00019_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00020_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00020_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00021_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00021_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00022_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00022_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00023_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00023_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00024_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00024_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00025_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00025_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00026_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00026_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00027_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00027_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00028_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00028_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00029_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00029_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00030_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00030_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/cache-ced9542cf73d68e1_00031_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/cache-ced9542cf73d68e1_00031_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/data-00000-of-00001.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/competition/data-00000-of-00001.arrow -------------------------------------------------------------------------------- /data/mini_val/competition/dataset_info.json: -------------------------------------------------------------------------------- 1 | { 2 | "citation": "", 3 | "description": "", 4 | "features": { 5 | "problem_id": { 6 | "dtype": "int64", 7 | "_type": "Value" 8 | }, 9 | "question": { 10 | "dtype": "string", 11 | "_type": "Value" 12 | }, 13 | "solutions": { 14 | "dtype": "string", 15 | "_type": "Value" 16 | }, 17 | "input_output": { 18 | "dtype": "string", 19 | "_type": "Value" 20 | }, 21 | "difficulty": { 22 | "dtype": "string", 23 | "_type": "Value" 24 | }, 25 | "url": { 26 | "dtype": "string", 27 | "_type": "Value" 28 | }, 29 | "starter_code": { 30 | "dtype": "string", 31 | "_type": "Value" 32 | }, 33 | "valid": { 34 | "dtype": "bool", 35 | "_type": "Value" 36 | } 37 | }, 38 | "homepage": "", 39 | "license": "" 40 | } -------------------------------------------------------------------------------- /data/mini_val/competition/state.json: -------------------------------------------------------------------------------- 1 | { 2 | "_data_files": [ 3 | { 4 | "filename": "data-00000-of-00001.arrow" 5 | } 6 | ], 7 | "_fingerprint": "e30b79776ce31019", 8 | "_format_columns": null, 9 | "_format_kwargs": {}, 10 | "_format_type": null, 11 | "_output_all_columns": false, 12 | "_split": null 13 | } -------------------------------------------------------------------------------- /data/mini_val/dataset_dict.json: -------------------------------------------------------------------------------- 1 | {"splits": ["introductory", "interview", "competition"]} -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00000_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00000_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00001_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00001_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00002_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00002_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00003_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00003_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00004_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00004_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00005_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00005_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00006_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00006_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00007_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00007_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00008_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00008_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00009_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00009_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00010_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00010_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00011_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00011_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00012_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00012_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00013_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00013_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00014_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00014_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00015_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00015_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00016_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00016_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00017_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00017_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00018_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00018_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00019_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00019_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00020_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00020_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00021_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00021_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00022_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00022_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00023_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00023_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00024_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00024_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00025_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00025_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00026_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00026_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00027_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00027_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00028_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00028_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00029_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00029_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00030_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00030_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6097a52e1350d851_00031_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6097a52e1350d851_00031_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00000_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00000_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00001_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00001_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00002_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00002_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00003_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00003_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00004_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00004_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00005_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00005_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00006_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00006_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00007_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00007_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00008_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00008_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00009_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00009_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00010_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00010_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00011_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00011_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00012_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00012_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00013_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00013_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00014_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00014_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00015_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00015_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00016_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00016_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00017_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00017_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00018_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00018_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00019_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00019_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00020_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00020_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00021_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00021_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00022_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00022_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00023_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00023_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00024_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00024_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00025_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00025_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00026_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00026_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00027_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00027_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00028_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00028_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00029_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00029_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00030_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00030_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/cache-6a9138e60b7da468_00031_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/cache-6a9138e60b7da468_00031_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/data-00000-of-00001.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/interview/data-00000-of-00001.arrow -------------------------------------------------------------------------------- /data/mini_val/interview/dataset_info.json: -------------------------------------------------------------------------------- 1 | { 2 | "citation": "", 3 | "description": "", 4 | "features": { 5 | "problem_id": { 6 | "dtype": "int64", 7 | "_type": "Value" 8 | }, 9 | "question": { 10 | "dtype": "string", 11 | "_type": "Value" 12 | }, 13 | "solutions": { 14 | "dtype": "string", 15 | "_type": "Value" 16 | }, 17 | "input_output": { 18 | "dtype": "string", 19 | "_type": "Value" 20 | }, 21 | "difficulty": { 22 | "dtype": "string", 23 | "_type": "Value" 24 | }, 25 | "url": { 26 | "dtype": "string", 27 | "_type": "Value" 28 | }, 29 | "starter_code": { 30 | "dtype": "string", 31 | "_type": "Value" 32 | }, 33 | "valid": { 34 | "dtype": "bool", 35 | "_type": "Value" 36 | } 37 | }, 38 | "homepage": "", 39 | "license": "" 40 | } -------------------------------------------------------------------------------- /data/mini_val/interview/state.json: -------------------------------------------------------------------------------- 1 | { 2 | "_data_files": [ 3 | { 4 | "filename": "data-00000-of-00001.arrow" 5 | } 6 | ], 7 | "_fingerprint": "b6e65dc124a91636", 8 | "_format_columns": null, 9 | "_format_kwargs": {}, 10 | "_format_type": null, 11 | "_output_all_columns": false, 12 | "_split": null 13 | } -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00000_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00000_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00001_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00001_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00002_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00002_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00003_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00003_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00004_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00004_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00005_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00005_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00006_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00006_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00007_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00007_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00008_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00008_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00009_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00009_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00010_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00010_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00011_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00011_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00012_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00012_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00013_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00013_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00014_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00014_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00015_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00015_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00016_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00016_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00017_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00017_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00018_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00018_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00019_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00019_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00020_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00020_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00021_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00021_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00022_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00022_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00023_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00023_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00024_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00024_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00025_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00025_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00026_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00026_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00027_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00027_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00028_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00028_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00029_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00029_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00030_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00030_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-08efd176dfa05db8_00031_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-08efd176dfa05db8_00031_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00000_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00000_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00001_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00001_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00002_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00002_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00003_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00003_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00004_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00004_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00005_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00005_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00006_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00006_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00007_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00007_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00008_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00008_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00009_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00009_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00010_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00010_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00011_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00011_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00012_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00012_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00013_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00013_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00014_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00014_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00015_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00015_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00016_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00016_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00017_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00017_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00018_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00018_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00019_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00019_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00020_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00020_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00021_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00021_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00022_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00022_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00023_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00023_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00024_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00024_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00025_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00025_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00026_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00026_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00027_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00027_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00028_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00028_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00029_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00029_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00030_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00030_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/cache-4bcfb8043179f39d_00031_of_00032.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/cache-4bcfb8043179f39d_00031_of_00032.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/data-00000-of-00001.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val/introductory/data-00000-of-00001.arrow -------------------------------------------------------------------------------- /data/mini_val/introductory/dataset_info.json: -------------------------------------------------------------------------------- 1 | { 2 | "citation": "", 3 | "description": "", 4 | "features": { 5 | "problem_id": { 6 | "dtype": "int64", 7 | "_type": "Value" 8 | }, 9 | "question": { 10 | "dtype": "string", 11 | "_type": "Value" 12 | }, 13 | "solutions": { 14 | "dtype": "string", 15 | "_type": "Value" 16 | }, 17 | "input_output": { 18 | "dtype": "string", 19 | "_type": "Value" 20 | }, 21 | "difficulty": { 22 | "dtype": "string", 23 | "_type": "Value" 24 | }, 25 | "url": { 26 | "dtype": "string", 27 | "_type": "Value" 28 | }, 29 | "starter_code": { 30 | "dtype": "string", 31 | "_type": "Value" 32 | }, 33 | "valid": { 34 | "dtype": "bool", 35 | "_type": "Value" 36 | } 37 | }, 38 | "homepage": "", 39 | "license": "" 40 | } -------------------------------------------------------------------------------- /data/mini_val/introductory/state.json: -------------------------------------------------------------------------------- 1 | { 2 | "_data_files": [ 3 | { 4 | "filename": "data-00000-of-00001.arrow" 5 | } 6 | ], 7 | "_fingerprint": "bee0ba3e5e514e82", 8 | "_format_columns": null, 9 | "_format_kwargs": {}, 10 | "_format_type": null, 11 | "_output_all_columns": false, 12 | "_split": null 13 | } -------------------------------------------------------------------------------- /data/mini_val_example_tests.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/mini_val_example_tests.pkl -------------------------------------------------------------------------------- /data/test_example_tests.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/data/test_example_tests.pkl -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | if conda info --envs | grep -q codechain; then echo "Skip! codechain already exists" 2 | else conda create -n codechain python=3.7 -y 3 | fi 4 | source activate codechain 5 | pip install --upgrade pip 6 | pip install datasets transformers timeout_decorator pyext openai flask scipy scikit-learn 7 | # install torch for 1.10 for cuda 11.1 8 | pip install torch==1.10.1+cu111 torchvision==0.11.2+cu111 torchaudio==0.10.1 -f https://download.pytorch.org/whl/cu111/torch_stable.html -------------------------------------------------------------------------------- /prompts/codechain_gen.txt: -------------------------------------------------------------------------------- 1 | Below is a competitive programming question. Read the question carefully. 2 | 3 | *Instruction* 4 | Develop a well-structured Python solution for the provided problem that obeys the constraints and passes the example test cases. Ensure modularity and considering potential edge cases and failures. Start by outlining the required code modules, including function headers and signatures. Subsequently, proceed to implement each module to create the final code. 5 | 6 | In simpler terms, create a clean and organized Python solution for the given problem. Break it down into smaller parts (modules) with clear function names and input/output specifications. Once the structure is ready, write the actual code for each module to complete the solution. 7 | 8 | The output code needs to <>. Please wrap your code answer using ```. 9 | 10 | ### Example 1 11 | ### TASK: 12 | Polycarp has $n$ different binary words. A word called binary if it contains only characters '0' and '1'. For example, these words are binary: "0001", "11", "0" and "0011100". 13 | 14 | Polycarp wants to offer his set of $n$ binary words to play a game "words". In this game, players name words and each next word (starting from the second) must start with the last character of the previous word. The first word can be any. For example, these sequence of words can be named during the game: "0101", "1", "10", "00", "00001". 15 | 16 | Word reversal is the operation of reversing the order of the characters. For example, the word "0111" after the reversal becomes "1110", the word "11010" after the reversal becomes "01011". 17 | 18 | Probably, Polycarp has such a set of words that there is no way to put them in the order correspondent to the game rules. In this situation, he wants to reverse some words from his set so that: the final set of $n$ words still contains different words (i.e. all words are unique); there is a way to put all words of the final set of words in the order so that the final sequence of $n$ words is consistent with the game rules. 19 | 20 | Polycarp wants to reverse minimal number of words. Please, help him. 21 | 22 | 23 | -----Input----- 24 | 25 | The first line of the input contains one integer $t$ ($1 \le t \le 10^4$) — the number of test cases in the input. Then $t$ test cases follow. 26 | 27 | The first line of a test case contains one integer $n$ ($1 \le n \le 2\cdot10^5$) — the number of words in the Polycarp's set. Next $n$ lines contain these words. All of $n$ words aren't empty and contains only characters '0' and '1'. The sum of word lengths doesn't exceed $4\cdot10^6$. All words are different. 28 | 29 | Guaranteed, that the sum of $n$ for all test cases in the input doesn't exceed $2\cdot10^5$. Also, guaranteed that the sum of word lengths for all test cases in the input doesn't exceed $4\cdot10^6$. 30 | 31 | 32 | -----Output----- 33 | 34 | Print answer for all of $t$ test cases in the order they appear. 35 | 36 | If there is no answer for the test case, print -1. Otherwise, the first line of the output should contain $k$ ($0 \le k \le n$) — the minimal number of words in the set which should be reversed. The second line of the output should contain $k$ distinct integers — the indexes of the words in the set which should be reversed. Words are numerated from $1$ to $n$ in the order they appear. If $k=0$ you can skip this line (or you can print an empty line). If there are many answers you can print any of them. 37 | 38 | 39 | -----Example----- 40 | Input 41 | 4 42 | 4 43 | 0001 44 | 1000 45 | 0011 46 | 0111 47 | 3 48 | 010 49 | 101 50 | 0 51 | 2 52 | 00000 53 | 00001 54 | 4 55 | 01 56 | 001 57 | 0001 58 | 00001 59 | 60 | Output 61 | 1 62 | 3 63 | -1 64 | 0 65 | 66 | 2 67 | 1 2 68 | 69 | ### RESPONSE: 70 | 71 | STEP 1: GENERATE SUB-MODULES: 72 | ```module 73 | def count_start_end_chars(words): 74 | """ 75 | Description: This function counts the number of words that start and end with each character. 76 | Input: 77 | words (list): A list of binary words. 78 | Output: 79 | start_count (defaultdict): A dictionary containing the count of words that start with each character. 80 | end_count (defaultdict): A dictionary containing the count of words that end with each character. 81 | """ 82 | ``` 83 | 84 | ```module 85 | def reverse_words(words, reversed_indices): 86 | """ 87 | Description: This function reverses the specified words in the given list. 88 | Input: 89 | words (list): A list of binary words. 90 | reversed_indices (list): A list of indices indicating the words to be reversed. 91 | Output: 92 | reversed_words (list): A new list of words with the specified words reversed. 93 | """ 94 | ``` 95 | 96 | STEP 2: GENERATE PYTHON CODE 97 | ```python 98 | import collections 99 | from utils import * 100 | 101 | def count_start_end_chars(words): 102 | start_count = collections.defaultdict(int) 103 | end_count = collections.defaultdict(int) 104 | for word in words: 105 | start_count[word[0]] += 1 106 | end_count[word[-1]] += 1 107 | return start_count, end_count 108 | 109 | def reverse_words(words, reversed_indices): 110 | reversed_words = [] 111 | for i, word in enumerate(words): 112 | if i in reversed_indices: 113 | reversed_words.append(word[::-1]) 114 | else: 115 | reversed_words.append(word) 116 | return reversed_words 117 | 118 | def solve_task(words): 119 | start_count, end_count = count_start_end_chars(words) 120 | 121 | characters_with_difference = [] 122 | for char in start_count: 123 | if abs(start_count[char] - end_count[char]) > 1: 124 | characters_with_difference.append(char) 125 | 126 | reversed_indices = [] 127 | for char in characters_with_difference: 128 | difference = abs(start_count[char] - end_count[char]) 129 | reverse_count = difference // 2 130 | if start_count[char] < end_count[char]: 131 | indices = [i for i, word in enumerate(words) if word.startswith(char)] 132 | reversed_indices.extend(indices[:reverse_count]) 133 | else: 134 | indices = [i for i, word in enumerate(words) if word.endswith(char)] 135 | reversed_indices.extend(indices[:reverse_count]) 136 | 137 | reversed_words = reverse_words(words, reversed_indices) 138 | 139 | total_reversed = len(reversed_indices) 140 | return total_reversed, reversed_words 141 | 142 | t = int(input()) 143 | for _ in range(t): 144 | n = int(input()) 145 | words = [] 146 | for _ in range(n): 147 | words.append(input()) 148 | total_reversed, reversed_words = solve_task(words) 149 | print(total_reversed) 150 | if total_reversed != 0: 151 | print(*reversed_words) 152 | ``` 153 | 154 | ----------------- 155 | 156 | ### Example 2 157 | ### TASK: 158 | <> 159 | 160 | <> 161 | 162 | ### RESPONSE: 163 | -------------------------------------------------------------------------------- /prompts/codechain_revise.txt: -------------------------------------------------------------------------------- 1 | Below is a competitive programming question. Read the question carefully. 2 | 3 | *Instruction* 4 | Develop a well-structured Python solution for the provided problem that obeys the constraints and passes the example test cases. Ensure modularity and considering potential edge cases and failures. Given a set of related utility Python functions, try to reuse or adapt them as much as possible into your solution (create new unique functions if needed). Start by outlining the required code modules, including function headers and signatures. Subsequently, proceed to implement each module to create the final code. 5 | 6 | In simpler terms, create a clean and organized Python solution for the given problem. Break it down into smaller parts (modules) with clear function names and input/output specifications. Once the structure is ready, write the actual code for each module to complete the solution. 7 | 8 | The output code needs to <>. Please wrap your code answer using ```. 9 | 10 | ### Example 1 11 | ### TASK: 12 | Polycarp has $n$ different binary words. A word called binary if it contains only characters '0' and '1'. For example, these words are binary: "0001", "11", "0" and "0011100". 13 | 14 | Polycarp wants to offer his set of $n$ binary words to play a game "words". In this game, players name words and each next word (starting from the second) must start with the last character of the previous word. The first word can be any. For example, these sequence of words can be named during the game: "0101", "1", "10", "00", "00001". 15 | 16 | Word reversal is the operation of reversing the order of the characters. For example, the word "0111" after the reversal becomes "1110", the word "11010" after the reversal becomes "01011". 17 | 18 | Probably, Polycarp has such a set of words that there is no way to put them in the order correspondent to the game rules. In this situation, he wants to reverse some words from his set so that: the final set of $n$ words still contains different words (i.e. all words are unique); there is a way to put all words of the final set of words in the order so that the final sequence of $n$ words is consistent with the game rules. 19 | 20 | Polycarp wants to reverse minimal number of words. Please, help him. 21 | 22 | 23 | -----Input----- 24 | 25 | The first line of the input contains one integer $t$ ($1 \le t \le 10^4$) — the number of test cases in the input. Then $t$ test cases follow. 26 | 27 | The first line of a test case contains one integer $n$ ($1 \le n \le 2\cdot10^5$) — the number of words in the Polycarp's set. Next $n$ lines contain these words. All of $n$ words aren't empty and contains only characters '0' and '1'. The sum of word lengths doesn't exceed $4\cdot10^6$. All words are different. 28 | 29 | Guaranteed, that the sum of $n$ for all test cases in the input doesn't exceed $2\cdot10^5$. Also, guaranteed that the sum of word lengths for all test cases in the input doesn't exceed $4\cdot10^6$. 30 | 31 | 32 | -----Output----- 33 | 34 | Print answer for all of $t$ test cases in the order they appear. 35 | 36 | If there is no answer for the test case, print -1. Otherwise, the first line of the output should contain $k$ ($0 \le k \le n$) — the minimal number of words in the set which should be reversed. The second line of the output should contain $k$ distinct integers — the indexes of the words in the set which should be reversed. Words are numerated from $1$ to $n$ in the order they appear. If $k=0$ you can skip this line (or you can print an empty line). If there are many answers you can print any of them. 37 | 38 | 39 | -----Example----- 40 | Input 41 | 4 42 | 4 43 | 0001 44 | 1000 45 | 0011 46 | 0111 47 | 3 48 | 010 49 | 101 50 | 0 51 | 2 52 | 00000 53 | 00001 54 | 4 55 | 01 56 | 001 57 | 0001 58 | 00001 59 | 60 | Output 61 | 1 62 | 3 63 | -1 64 | 0 65 | 66 | 2 67 | 1 2 68 | 69 | ### RELEVANT FUNCTIONS: 70 | 71 | ```module 72 | def reverse_words(string): 73 | """ 74 | description: this function reverses each word in the given string. 75 | input: 76 | string (str): the input string. 77 | output: 78 | reversed_string (str): the reversed string with each word reversed. 79 | """ 80 | return ' '.join(s[::-1] for s in string.split(' ')) 81 | ``` 82 | 83 | ```module 84 | def count_start_end_chars(words): 85 | """ 86 | Description: This function counts the number of words that start and end with each character. 87 | Input: 88 | words (list): A list of binary words. 89 | Output: 90 | start_count (defaultdict): A dictionary containing the count of words that start with each character. 91 | end_count (defaultdict): A dictionary containing the count of words that end with each character. 92 | """ 93 | start_count = collections.defaultdict(int) 94 | end_count = collections.defaultdict(int) 95 | for word in words: 96 | start_count[word[0]] += 1 97 | end_count[word[-1]] += 1 98 | return start_count, end_count 99 | ``` 100 | 101 | ### RESPONSE: 102 | 103 | STEP 1: GENERATE SUB-MODULES: 104 | ```module 105 | def count_start_end_chars(words): 106 | """ 107 | Description: This function counts the number of words that start and end with each character. 108 | Input: 109 | words (list): A list of binary words. 110 | Output: 111 | start_count (defaultdict): A dictionary containing the count of words that start with each character. 112 | end_count (defaultdict): A dictionary containing the count of words that end with each character. 113 | """ 114 | ``` 115 | 116 | ```module 117 | def reverse_words(words, reversed_indices): 118 | """ 119 | Description: This function reverses the specified words in the given list. 120 | Input: 121 | words (list): A list of binary words. 122 | reversed_indices (list): A list of indices indicating the words to be reversed. 123 | Output: 124 | reversed_words (list): A new list of words with the specified words reversed. 125 | """ 126 | ``` 127 | 128 | STEP 2: GENERATE PYTHON CODE 129 | ```python 130 | import collections 131 | 132 | def count_start_end_chars(words): 133 | start_count = collections.defaultdict(int) 134 | end_count = collections.defaultdict(int) 135 | for word in words: 136 | start_count[word[0]] += 1 137 | end_count[word[-1]] += 1 138 | return start_count, end_count 139 | 140 | def reverse_words(words, reversed_indices): 141 | reversed_words = [] 142 | for i, word in enumerate(words): 143 | if i in reversed_indices: 144 | reversed_words.append(word[::-1]) 145 | else: 146 | reversed_words.append(word) 147 | return reversed_words 148 | 149 | def solve_task(words): 150 | start_count, end_count = count_start_end_chars(words) 151 | 152 | characters_with_difference = [] 153 | for char in start_count: 154 | if abs(start_count[char] - end_count[char]) > 1: 155 | characters_with_difference.append(char) 156 | 157 | reversed_indices = [] 158 | for char in characters_with_difference: 159 | difference = abs(start_count[char] - end_count[char]) 160 | reverse_count = difference // 2 161 | if start_count[char] < end_count[char]: 162 | indices = [i for i, word in enumerate(words) if word.startswith(char)] 163 | reversed_indices.extend(indices[:reverse_count]) 164 | else: 165 | indices = [i for i, word in enumerate(words) if word.endswith(char)] 166 | reversed_indices.extend(indices[:reverse_count]) 167 | 168 | reversed_words = reverse_words(words, reversed_indices) 169 | 170 | total_reversed = len(reversed_indices) 171 | return total_reversed, reversed_words 172 | 173 | t = int(input()) 174 | for _ in range(t): 175 | n = int(input()) 176 | words = [] 177 | for _ in range(n): 178 | words.append(input()) 179 | total_reversed, reversed_words = solve_task(words) 180 | print(total_reversed) 181 | if total_reversed != 0: 182 | print(*reversed_words) 183 | 184 | ``` 185 | 186 | ----------------- 187 | 188 | ### Example 2 189 | ### TASK: 190 | <> 191 | 192 | <> 193 | 194 | ### RELEVANT FUNCTIONS: 195 | <> 196 | 197 | ### RESPONSE: 198 | -------------------------------------------------------------------------------- /prompts/direct_gen.txt: -------------------------------------------------------------------------------- 1 | Below is a competitive programming question. Read the question carefully. 2 | 3 | *Instruction* 4 | Develop a well-structured Python solution for the provided problem that obeys the constraints and passes the example test cases. 5 | 6 | The output code needs to <>. Please wrap your code answer using ```. 7 | 8 | ### Example 1 9 | 10 | Polycarp has $n$ different binary words. A word called binary if it contains only characters '0' and '1'. For example, these words are binary: "0001", "11", "0" and "0011100". 11 | 12 | Polycarp wants to offer his set of $n$ binary words to play a game "words". In this game, players name words and each next word (starting from the second) must start with the last character of the previous word. The first word can be any. For example, these sequence of words can be named during the game: "0101", "1", "10", "00", "00001". 13 | 14 | Word reversal is the operation of reversing the order of the characters. For example, the word "0111" after the reversal becomes "1110", the word "11010" after the reversal becomes "01011". 15 | 16 | Probably, Polycarp has such a set of words that there is no way to put them in the order correspondent to the game rules. In this situation, he wants to reverse some words from his set so that: the final set of $n$ words still contains different words (i.e. all words are unique); there is a way to put all words of the final set of words in the order so that the final sequence of $n$ words is consistent with the game rules. 17 | 18 | Polycarp wants to reverse minimal number of words. Please, help him. 19 | 20 | 21 | -----Input----- 22 | 23 | The first line of the input contains one integer $t$ ($1 \le t \le 10^4$) — the number of test cases in the input. Then $t$ test cases follow. 24 | 25 | The first line of a test case contains one integer $n$ ($1 \le n \le 2\cdot10^5$) — the number of words in the Polycarp's set. Next $n$ lines contain these words. All of $n$ words aren't empty and contains only characters '0' and '1'. The sum of word lengths doesn't exceed $4\cdot10^6$. All words are different. 26 | 27 | Guaranteed, that the sum of $n$ for all test cases in the input doesn't exceed $2\cdot10^5$. Also, guaranteed that the sum of word lengths for all test cases in the input doesn't exceed $4\cdot10^6$. 28 | 29 | 30 | -----Output----- 31 | 32 | Print answer for all of $t$ test cases in the order they appear. 33 | 34 | If there is no answer for the test case, print -1. Otherwise, the first line of the output should contain $k$ ($0 \le k \le n$) — the minimal number of words in the set which should be reversed. The second line of the output should contain $k$ distinct integers — the indexes of the words in the set which should be reversed. Words are numerated from $1$ to $n$ in the order they appear. If $k=0$ you can skip this line (or you can print an empty line). If there are many answers you can print any of them. 35 | 36 | 37 | -----Example----- 38 | Input 39 | 4 40 | 4 41 | 0001 42 | 1000 43 | 0011 44 | 0111 45 | 3 46 | 010 47 | 101 48 | 0 49 | 2 50 | 00000 51 | 00001 52 | 4 53 | 01 54 | 001 55 | 0001 56 | 00001 57 | 58 | Output 59 | 1 60 | 3 61 | -1 62 | 0 63 | 64 | 2 65 | 1 2 66 | 67 | ### RESPONSE: 68 | 69 | ```python 70 | for _ in range(int(input())): 71 | n = int(input()) 72 | mass = [] 73 | zo = 0 74 | oz = 0 75 | zz = 0 76 | oo = 0 77 | ozs = [] 78 | zos = [] 79 | ozss = set() 80 | zoss = set() 81 | for j in range(n): 82 | k = input() 83 | mass.append(k) 84 | if k[0] == '0' and k[-1] == '1': 85 | zoss.add(k) 86 | zos.append(j + 1) 87 | zo += 1 88 | elif k[0] == '1' and k[-1] == '0': 89 | ozss.add(k) 90 | ozs.append(j + 1) 91 | oz += 1 92 | elif k[0] == '0' and k[-1] == '0': 93 | zz += 1 94 | else: 95 | oo += 1 96 | if zz and oo and not oz and not zo: 97 | print(-1) 98 | continue 99 | else: 100 | if zo > oz: 101 | print((zo - oz) // 2) 102 | ans = [] 103 | need = (zo - oz) // 2 104 | i = 0 105 | while need: 106 | zzz = mass[zos[i] - 1][len(mass[zos[i] - 1]) - 1:: -1] 107 | if zzz not in ozss: 108 | ans.append(zos[i]) 109 | need -= 1 110 | i += 1 111 | print(*ans) 112 | else: 113 | print((oz - zo) // 2) 114 | ans = [] 115 | need = (oz - zo) // 2 116 | i = 0 117 | while need: 118 | zzz = mass[ozs[i] - 1][len(mass[ozs[i] - 1]) - 1:: -1] 119 | if zzz not in zoss: 120 | ans.append(ozs[i]) 121 | need -= 1 122 | i += 1 123 | print(*ans) 124 | ``` 125 | 126 | ----------------- 127 | 128 | ### Example 2 129 | 130 | <> 131 | 132 | <> 133 | 134 | ### RESPONSE: 135 | -------------------------------------------------------------------------------- /scripts/evaluate_codechain.sh: -------------------------------------------------------------------------------- 1 | split=test # test OR mini_val 2 | model=gpt3.5 # gpt3.5 or gpt4 3 | start=0 4 | end=10 5 | num_gen_samples=20 6 | start_round=1 7 | end_round=5 8 | 9 | ## ROUND 0 10 | prompt=prompts/codechain_gen.txt 11 | round=round0 12 | exp_name=${model}_${split} 13 | output_path=outputs/${exp_name}_$round 14 | num_clusters=5 15 | 16 | # Test by hidden test cases 17 | python src/evaluate.py --save_gen_path $output_path --eval_split $split 18 | 19 | ## REVISION ROUNDS 20 | for (( round_number = $start_round; round_number <= $end_round; round_number++ )) 21 | do 22 | echo "REVISION ROUND $round_number" 23 | round=round${round_number} 24 | output_path=outputs/${exp_name}_$round 25 | echo "OUTPUT PATH: $output_path" 26 | 27 | # Test by hidden test cases 28 | python src/evaluate.py --save_gen_path $output_path --eval_split $split --original_gen_path outputs/${exp_name}_round0 29 | 30 | done 31 | -------------------------------------------------------------------------------- /scripts/evaluate_direct_gen.sh: -------------------------------------------------------------------------------- 1 | split=test # test OR mini_val 2 | model=gpt3.5 # gpt3.5 or gpt4 3 | start=0 4 | end=10 5 | num_gen_samples=20 6 | 7 | prompt=prompts/direct_gen.txt 8 | exp_name=${model}_${split} 9 | output_path=outputs/${exp_name}_directgen 10 | 11 | python src/evaluate.py --save_gen_path $output_path --eval_split $split -------------------------------------------------------------------------------- /scripts/run_codechain.sh: -------------------------------------------------------------------------------- 1 | split=test # test OR mini_val 2 | model=gpt3.5 # gpt3.5 or gpt4 3 | start=0 4 | end=10 5 | num_gen_samples=20 6 | start_round=1 7 | end_round=5 8 | 9 | ## ROUND 0 10 | prompt=prompts/codechain_gen.txt 11 | round=round0 12 | exp_name=${model}_${split} 13 | output_path=outputs/${exp_name}_$round 14 | num_clusters=5 15 | 16 | # Generate code 17 | python src/generate.py --output_path $output_path --prompt_file $prompt --split $split --model $model --start $start --end $end --num_gen_samples $num_gen_samples 18 | 19 | # Test by example test cases 20 | python src/evaluate.py --save_gen_path $output_path --example_test_path data/${split}_example_tests.pkl --eval_split $split --save_results_path ${output_path}_results_exampletests.json 21 | 22 | # Postprocessing 23 | python src/processing.py --output_path $output_path --output_file ${output_path}_data.csv --result_path ${output_path}_results_exampletests.json 24 | 25 | # Clustering 26 | echo "Clustering N_CLUSTERS: $num_clusters" 27 | python src/clustering.py --data_path ${output_path}_data.csv --n_cluster ${num_clusters} --output_embed_file ${output_path}_embeds.npy --output_file ${output_path}_${num_clusters}clusters.csv 28 | 29 | 30 | ## REVISION ROUNDS 31 | prompt=prompts/codechain_revise.txt 32 | 33 | for (( round_number = $start_round; round_number <= $end_round; round_number++ )) 34 | do 35 | echo "REVISION ROUND $round_number" 36 | 37 | prior_output_path=$output_path 38 | prior_num_clusters=$num_clusters 39 | prior_result_path=${prior_output_path}_data.csv_all_results.json 40 | modules_file=${prior_output_path}_${prior_num_clusters}clusters.csv 41 | echo "MODULE FILE $modules_file" 42 | 43 | round=round${round_number} 44 | output_path=outputs/${exp_name}_$round 45 | num_clusters=$(( 5 - $round_number )) 46 | echo "OUTPUT PATH: $output_path" 47 | 48 | # generate code 49 | python src/generate.py --output_path $output_path --prompt_file $prompt --split $split --model $model --start $start --end $end --num_gen_samples $num_gen_samples --modules_file $modules_file --num_clusters ${prior_num_clusters} 50 | 51 | if [ $round_number -ne $end_round ] 52 | then 53 | # Test by example test cases 54 | python src/evaluate.py --save_gen_path $output_path --example_test_path data/${split}_example_tests.pkl --eval_split $split --save_results_path ${output_path}_results_exampletests.json 55 | 56 | # Postprocessing 57 | python src/processing.py --output_path $output_path --output_file ${output_path}_data.csv --result_path ${output_path}_results_exampletests.json --past_result_path $prior_result_path 58 | 59 | # Clustering 60 | echo "Clustering N_CLUSTERS: $num_clusters" 61 | python src/clustering.py --data_path ${output_path}_data.csv --n_cluster ${num_clusters} --output_embed_file ${output_path}_embeds.npy --output_file ${output_path}_${num_clusters}clusters.csv 62 | fi 63 | done 64 | -------------------------------------------------------------------------------- /scripts/run_direct_gen.sh: -------------------------------------------------------------------------------- 1 | split=test # test OR mini_val 2 | model=gpt3.5 # gpt3.5 or gpt4 3 | start=0 4 | end=10 5 | num_gen_samples=20 6 | 7 | prompt=prompts/direct_gen.txt 8 | exp_name=${model}_${split} 9 | output_path=outputs/${exp_name}_directgen 10 | 11 | python src/generate.py --output_path $output_path --prompt_file $prompt --split $split --model $model --start $start --end $end --num_gen_samples $num_gen_samples -------------------------------------------------------------------------------- /src/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/src/.DS_Store -------------------------------------------------------------------------------- /src/clustering.py: -------------------------------------------------------------------------------- 1 | import json 2 | from glob import glob 3 | from tqdm import tqdm 4 | from multiprocessing import Pool 5 | from functools import partial 6 | import numpy as np 7 | import re 8 | import datasets 9 | import pickle as pkl 10 | import pandas as pd 11 | import argparse, pprint 12 | from collections import Counter 13 | from scipy.stats import describe 14 | from sklearn.cluster import KMeans 15 | from scipy.spatial.distance import cdist 16 | from sklearn.metrics import silhouette_score 17 | 18 | from configs.config_clustering import parse_args 19 | from utils.utils import get_embedding 20 | 21 | args = parse_args() 22 | argsdict = vars(args) 23 | print(pprint.pformat(argsdict)) 24 | 25 | data = pd.read_csv(args.data_path) 26 | embeds = get_embedding(args.output_embed_file, data, args.embedding_model) 27 | 28 | # Cluster by embeddings 29 | print("Clustering...") 30 | groups_by_problem = data.groupby(['file']).groups 31 | funcs = data['func'].tolist() 32 | func_codes = data['func_code'].tolist() 33 | all_files = [] 34 | all_centroids = [] 35 | all_centroid_codes = [] 36 | num_funcs = [] 37 | all_distances = [] 38 | 39 | all_num_clusters = [] 40 | for group, indices in tqdm(groups_by_problem.items()): 41 | num_funcs.append(len(indices)) 42 | group_embeds = embeds[indices] 43 | 44 | num_clusters = args.n_clusters 45 | all_num_clusters.append(num_clusters) 46 | 47 | kmeans_model = KMeans(n_clusters=min(len(indices), args.n_clusters), random_state=0) 48 | classes = kmeans_model.fit_predict(group_embeds).tolist() 49 | 50 | min_dist = np.min(cdist(group_embeds, kmeans_model.cluster_centers_, 'euclidean'), axis=1) 51 | 52 | Y = pd.DataFrame(min_dist, index=range(len(classes)), columns=['Center_euclidean_dist']) 53 | Z = pd.DataFrame(classes, index=range(len(classes)), columns=['cluster_ID']) 54 | 55 | PAP = pd.concat([Y,Z], axis=1) 56 | grouped = PAP.groupby(['cluster_ID']) 57 | centroids = grouped.idxmin() 58 | 59 | centroids_mapping = centroids.to_dict()['Center_euclidean_dist'] 60 | for cluster, index in centroids_mapping.items(): 61 | real_index = indices[index] 62 | all_centroids.append(funcs[real_index]) 63 | all_centroid_codes.append(func_codes[real_index]) 64 | all_files.append(group) 65 | 66 | print("Number of functions per problem:") 67 | print(describe(num_funcs)) 68 | print() 69 | data = {} 70 | data['file'] = all_files 71 | data['centroid'] = all_centroids 72 | data['centroid_code'] = all_centroid_codes 73 | 74 | print("number of set clusters:") 75 | print(Counter(all_num_clusters)) 76 | print(describe(all_num_clusters)) 77 | 78 | freqs = Counter(all_files) 79 | print("Number of util functions per problem:") 80 | print(describe(list(freqs.values()))) 81 | 82 | data = pd.DataFrame(data) 83 | if args.output_file is not None: 84 | data.to_csv(args.output_file) -------------------------------------------------------------------------------- /src/configs/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/src/configs/.DS_Store -------------------------------------------------------------------------------- /src/configs/config_clustering.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | def parse_args(): 5 | parser = argparse.ArgumentParser(description="Config for clustering") 6 | 7 | parser.add_argument( 8 | '--embedding_model', 9 | type=str, 10 | default='starencoder', 11 | help="Type of embedding model; default to use StarCoder encoder") 12 | parser.add_argument( 13 | '--data_path', 14 | type=str, 15 | default=None, 16 | help="Path to generated code for clustering") 17 | parser.add_argument( 18 | '--n_clusters', 19 | type=int, 20 | default=5, 21 | help="Number of clusters to run k-means clustering") 22 | parser.add_argument( 23 | '--output_embed_file', 24 | type=str, 25 | default=None, 26 | help="Path to save embedding file") 27 | parser.add_argument( 28 | '--output_file', 29 | type=str, 30 | default=None, 31 | help="Path to save processed data of clustered modular functions") 32 | 33 | args = parser.parse_args() 34 | 35 | return args -------------------------------------------------------------------------------- /src/configs/config_evaluate.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | def parse_args(): 5 | parser = argparse.ArgumentParser(description="Config for evaluation") 6 | 7 | parser.add_argument("--num_proc", default=os.cpu_count(), type=int) 8 | 9 | parser.add_argument( 10 | "--eval_split", 11 | choices=['mini_val', 'test'], 12 | default='mini_val', 13 | help="Which split to evaluate in APPS", 14 | ) 15 | 16 | parser.add_argument( 17 | "--save_gen_path", 18 | type=str, 19 | default=None, 20 | help='Path to generated code to be evaluated' 21 | ) 22 | 23 | parser.add_argument( 24 | "--save_results_path", 25 | type=str, 26 | default=None, 27 | help='Path to save test results' 28 | ) 29 | 30 | parser.add_argument( 31 | "--example_test_path", 32 | type=str, 33 | default=None, 34 | help='Path to example test cases if testing on example test cases; if None, test on hidden test cases' 35 | ) 36 | 37 | parser.add_argument( 38 | "--original_gen_path", 39 | type=str, 40 | default=None, 41 | help='Path to the original generation code e.g. round 0 generation; if None, only test on revised programs' 42 | ) 43 | 44 | args = parser.parse_args() 45 | 46 | return args 47 | -------------------------------------------------------------------------------- /src/configs/config_generate.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | def parse_args(): 5 | parser = argparse.ArgumentParser(description="Config for generation") 6 | 7 | parser.add_argument( 8 | '--model', 9 | type=str, 10 | help="Model name for generation") 11 | parser.add_argument( 12 | '--output_path', 13 | type=str, 14 | help="Output path to save generated code") 15 | parser.add_argument( 16 | '--prompt_file', 17 | type=str, 18 | help="Path to instruction prompt") 19 | parser.add_argument( 20 | '--modules_file', 21 | type=str, 22 | default=None, 23 | help="Path to extracted modules for self-revision") 24 | parser.add_argument( 25 | '--num_gen_samples', 26 | type=int, 27 | default=5, 28 | help="Number of generation samples per problem") 29 | parser.add_argument( 30 | '--split', 31 | type=str, 32 | default='mini_val', 33 | help="name of data split in APPS") 34 | parser.add_argument( 35 | '--num_clusters', 36 | type=int, 37 | default=1, 38 | help="Number of clusters in prior generation round") 39 | parser.add_argument( 40 | '--start', 41 | type=int, 42 | default=0, 43 | help="Star index of dataset") 44 | parser.add_argument( 45 | '--end', 46 | type=int, 47 | default=150, 48 | help="End index of dataset") 49 | 50 | args = parser.parse_args() 51 | 52 | return args 53 | -------------------------------------------------------------------------------- /src/configs/config_processing.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | def parse_args(): 5 | parser = argparse.ArgumentParser(description="Config for post-processing generated code to extract code and modular functions") 6 | 7 | parser.add_argument( 8 | '--output_path', 9 | type=str, 10 | default=None, 11 | help="Path to generated code to be processed") 12 | parser.add_argument( 13 | '--result_path', 14 | type=str, 15 | default=None, 16 | help="Path to test results on example test cases for filtering") 17 | parser.add_argument( 18 | '--past_result_path', 19 | type=str, 20 | default=None, 21 | help="Path to all past test results (compiled from all prior generation rounds) on example test cases") 22 | parser.add_argument( 23 | '--output_file', 24 | type=str, 25 | help="Path to save processed data") 26 | 27 | args = parser.parse_args() 28 | 29 | return args -------------------------------------------------------------------------------- /src/embedding/constants.py: -------------------------------------------------------------------------------- 1 | # Various constants. 2 | 3 | PADDING_ID_FOR_LABELS = -100 # used to convert paddin ids to ignored labels. 4 | MAX_VALID_DATA_ROW_COUNT = 2000 5 | 6 | # Data path: 7 | 8 | GFG_DATA_PATH = "./resources/data/transcoder_evaluation_gfg" 9 | 10 | # Special tokens 11 | MASK_TOKEN = "" 12 | SEPARATOR_TOKEN = "" 13 | PAD_TOKEN = "" 14 | CLS_TOKEN = "" 15 | 16 | # Results notebook name. 17 | RESULTS_FNAME = "results.ipynb" 18 | -------------------------------------------------------------------------------- /src/embedding/datasets_loader.py: -------------------------------------------------------------------------------- 1 | from typing import List, Dict, Union 2 | import torch 3 | from torch.utils.data import Dataset 4 | import datasets 5 | 6 | # Workaround toolkit misreporting available disk space. 7 | datasets.builder.has_sufficient_disk_space = lambda needed_bytes, directory=".": True 8 | from datasets import load_dataset, load_from_disk 9 | from datasets.builder import DatasetBuildError 10 | from transformers import AutoTokenizer 11 | from embedding.preprocessing_utils import ( 12 | perturb_tokens, 13 | get_special_tokens_mask, 14 | get_pooling_mask, 15 | pre_process_codesearchnet_train, 16 | pre_process_codesearchnet_test, 17 | pre_process_gfg, 18 | pre_process_the_stack, 19 | ) 20 | from embedding.constants import MASK_TOKEN, PAD_TOKEN, SEPARATOR_TOKEN, CLS_TOKEN 21 | 22 | DATASET_NAME_TO_PREPROCESSING_FUNCTION = { 23 | "the-stack": { 24 | "train": pre_process_the_stack, 25 | "test": pre_process_the_stack, 26 | }, 27 | "code_search_net": { 28 | "train": pre_process_codesearchnet_train, 29 | "test": pre_process_codesearchnet_test, 30 | }, 31 | "gfg": { 32 | "train": pre_process_gfg, 33 | "test": pre_process_gfg, 34 | }, 35 | } 36 | 37 | 38 | class RandomlyPairedDataset(Dataset): 39 | """Indexed dataset class with randomly picked negative pairs.""" 40 | 41 | def __init__(self, base_dataset: datasets.Dataset) -> None: 42 | """Intanstiates an indexed dataset wrapping a base data source. 43 | We use this class to be able to get examples from the dataset including negative pairs. 44 | 45 | Args: 46 | base_dataset (datasets.Dataset): Base indexed data source. 47 | """ 48 | self.data_source = base_dataset 49 | 50 | def __len__(self) -> int: 51 | """Returns the length of the dataset which matches that of the base data source. 52 | 53 | Returns: 54 | int: Dataset length. 55 | """ 56 | return len(self.data_source) 57 | 58 | def __getitem__(self, i: int) -> List[Dict]: 59 | """Reads from the base dataset and returns an addition random entry that serves as negative example. 60 | 61 | Args: 62 | i (int): Index to be read. 63 | 64 | Returns: 65 | List[Dict]: Pair of examples. The example indexed by i is returned along with a different random point. 66 | """ 67 | rand_idx = torch.randint(0, len(self.data_source), (1,)).item() 68 | while rand_idx == i: 69 | rand_idx = torch.randint(0, len(self.data_source), (1,)).item() 70 | 71 | example = self.data_source[i] 72 | negative_example = self.data_source[rand_idx] 73 | 74 | return example["source"], example["target"], negative_example["source"] 75 | 76 | 77 | class PairedDataset(Dataset): 78 | """Indexed dataset class yielding source/target pairs.""" 79 | 80 | def __init__(self, base_dataset: datasets.Dataset) -> None: 81 | """Intanstiates an indexed dataset wrapping a base data source. 82 | We use this class to be able to get paired examples from the base dataset. 83 | The base dataset must be pre-processed to include the fields 'source' and 'target'. 84 | 85 | Args: 86 | base_dataset (datasets.Dataset): Base indexed pre-processed data source. 87 | """ 88 | self.data_source = base_dataset 89 | 90 | def __len__(self) -> int: 91 | """Returns the length of the dataset which matches that of the base data source. 92 | 93 | Returns: 94 | int: Dataset length. 95 | """ 96 | return len(self.data_source) 97 | 98 | def __getitem__(self, i: int) -> List[Dict]: 99 | """Reads from the base dataset and returns a paier of examples. 100 | 101 | Args: 102 | i (int): Index to be read. 103 | 104 | Returns: 105 | List[Dict]: Pair of examples. The 'source' and 'target' fields of the example indexed by i are returned. 106 | """ 107 | 108 | example = self.data_source[i] 109 | 110 | return example["source"], example["target"] 111 | 112 | 113 | def get_dataset( 114 | dataset_name: str, 115 | path_to_cache: str, 116 | split: str, 117 | maximum_raw_length: int, 118 | force_preprocess: bool = False, 119 | maximum_row_cout: int = None, 120 | ) -> Union[PairedDataset, RandomlyPairedDataset]: 121 | """Get dataset instance. 122 | 123 | Args: 124 | dataset_name (str): Name of the base dataset. 125 | path_to_cache (str): Path to the base dataset. 126 | split (str): data split in {'train', 'valid', 'test'}. 127 | maximum_raw_length (int, optional): Maximum length of the raw entries from the source dataset. 128 | force_preprocess (bool, optional): Whether to force pre-processing. Defaults to False. 129 | maximum_row_cout (int, optional) = Maximum size of the dataset in term of row count. Defaults to None. 130 | 131 | Returns: 132 | dataset: An indexed dataset object. 133 | """ 134 | try: 135 | base_dataset = load_dataset( 136 | dataset_name, 137 | use_auth_token=True, 138 | cache_dir=path_to_cache, 139 | split=split, 140 | ) 141 | except DatasetBuildError: 142 | # Try to specify data files. Specific for The Stack. 143 | base_dataset = load_dataset( 144 | dataset_name, 145 | use_auth_token=True, 146 | cache_dir=path_to_cache, 147 | data_files="sample.parquet", 148 | split=split, 149 | ) 150 | except FileNotFoundError: 151 | # Try to load from disk if above failed. 152 | base_dataset = load_from_disk(path_to_cache) 153 | 154 | if force_preprocess: 155 | base_dataset.cleanup_cache_files() 156 | 157 | base_dataset = base_dataset.shuffle(seed=42) 158 | 159 | if maximum_row_cout is not None: 160 | base_dataset = base_dataset.select( 161 | range(min(len(base_dataset), maximum_row_cout)) 162 | ) 163 | 164 | if "train" in split.lower(): 165 | split_preproc_key = "train" 166 | else: 167 | split_preproc_key = "test" 168 | 169 | try: 170 | pre_proc_fn = DATASET_NAME_TO_PREPROCESSING_FUNCTION[dataset_name][ 171 | split_preproc_key 172 | ] 173 | except KeyError: 174 | for k in DATASET_NAME_TO_PREPROCESSING_FUNCTION.keys(): 175 | if "the-stack" in dataset_name.lower() and "the-stack" in k: 176 | pre_proc_fn = DATASET_NAME_TO_PREPROCESSING_FUNCTION[k][ 177 | split_preproc_key 178 | ] 179 | 180 | base_dataset = base_dataset.map(pre_proc_fn(maximum_raw_length), num_proc=96) 181 | 182 | base_dataset = base_dataset.shuffle(seed=42) 183 | 184 | if "train" in split_preproc_key: 185 | return RandomlyPairedDataset(base_dataset) 186 | else: 187 | return PairedDataset(base_dataset) 188 | 189 | 190 | def prepare_tokenizer(tokenizer_path): 191 | try: 192 | tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) 193 | except OSError: 194 | tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_auth_token=True) 195 | 196 | tokenizer.add_special_tokens({"pad_token": PAD_TOKEN}) 197 | tokenizer.add_special_tokens({"sep_token": SEPARATOR_TOKEN}) 198 | tokenizer.add_special_tokens({"cls_token": CLS_TOKEN}) 199 | tokenizer.add_special_tokens({"mask_token": MASK_TOKEN}) 200 | return tokenizer 201 | 202 | 203 | class TrainCollator: 204 | """Train collator object mapping sequences of items from dataset instance 205 | into batches of IDs and masks used for training models. 206 | """ 207 | 208 | def __init__( 209 | self, 210 | tokenizer_path: str, 211 | maximum_length: int, 212 | mlm_masking_probability: float, 213 | contrastive_masking_probability: float, 214 | ignore_contrastive_loss_data: bool = False, 215 | **kwargs, 216 | ) -> None: 217 | """Creates instance of collator. 218 | 219 | Args: 220 | tokenizer_path (str): Path to tokenizer. 221 | maximum_length (int): Truncating length of token sequences. 222 | mlm_masking_probability (float): Masking probability for MLM objective. 223 | contrastive_masking_probability (float): Masking probability for contrastive objective. 224 | ignore_contrastive_loss_data (bool, optional): Do not add append positive pairs to batch. Defaults to False. 225 | """ 226 | self.mlm_masking_probability = mlm_masking_probability 227 | self.contrastive_masking_probability = contrastive_masking_probability 228 | self.maximum_length = maximum_length 229 | self.ignore_contrastive_loss_data = ignore_contrastive_loss_data 230 | 231 | self.tokenizer = prepare_tokenizer(tokenizer_path) 232 | 233 | self.sep_token_id = self.tokenizer.get_vocab()[self.tokenizer.sep_token] 234 | self.pad_token_id = self.tokenizer.get_vocab()[self.tokenizer.pad_token] 235 | self.mask_token_id = self.tokenizer.get_vocab()[self.tokenizer.mask_token] 236 | self.cls_token_id = self.tokenizer.get_vocab()[self.tokenizer.cls_token] 237 | 238 | def __call__(self, batch: List[Dict]) -> Dict[str, torch.Tensor]: 239 | """Maps list of triplets of examples to batches of token ids, masks, and labels used for training. 240 | The firt two elements in a triplet correspond to neighbor chunkes from the same file. The third 241 | element corresponds to a chunk from a random file. 242 | 243 | Args: 244 | batch (List[Dict]): List of pairs of examples. 245 | 246 | Returns: 247 | Dict[str, torch.Tensor]: Batches of tokens, masks, and labels. 248 | """ 249 | source_list = [ 250 | el[0] for el in batch 251 | ] # el[0] is the first half of a code snippet. 252 | # Following are the labels for the seq relationship loss: 0 -> negative pair, 1 -> positive pair. 253 | seq_relationship_labels = torch.randint(0, 2, (len(batch),)).long() 254 | target_list = [ 255 | # seq_relationship_label==1 -> positive pair -> we take the second half of the code snippet 256 | # seq_relationship_label==0 -> negative pair -> we take a random code snippet given in el[1] 257 | el[1] if seq_relationship_labels[i] == 1 else el[2] 258 | for i, el in enumerate(batch) 259 | ] 260 | 261 | input_examples_list = [ # Combine source and target w/ template: [CLS] SOURCE [SEP] [TARGET] [SEP] 262 | f"{CLS_TOKEN}{source_list[i]}{SEPARATOR_TOKEN}{target_list[i]}{SEPARATOR_TOKEN}" 263 | for i in range(len(batch)) 264 | ] 265 | 266 | input_examples_encoding = self.tokenizer( 267 | input_examples_list, 268 | padding="longest", 269 | max_length=self.maximum_length, 270 | truncation=True, 271 | return_tensors="pt", 272 | ) 273 | 274 | input_examples_ids = input_examples_encoding.input_ids 275 | input_examples_att_mask = ( 276 | input_examples_encoding.attention_mask 277 | ) # Padding masks. 278 | 279 | special_tokens_mask = get_special_tokens_mask( 280 | self.tokenizer, input_examples_ids 281 | ) 282 | 283 | input_examples_ids, mlm_labels = perturb_tokens( 284 | input_examples_ids, 285 | special_tokens_mask, 286 | self.mlm_masking_probability, 287 | self.mask_token_id, 288 | len(self.tokenizer), 289 | ) # Dynamically perturbs input tokens and generates corresponding mlm labels. 290 | 291 | if not self.ignore_contrastive_loss_data: 292 | positive_examples_ids, positive_mlm_labels = perturb_tokens( 293 | input_examples_ids, 294 | special_tokens_mask, 295 | self.contrastive_masking_probability, 296 | self.mask_token_id, 297 | len(self.tokenizer), 298 | ) # Positve examples are independently perturbed versions of the source, used for the contrastive loss. 299 | 300 | input_ids = torch.cat([input_examples_ids, positive_examples_ids], 0) 301 | attention_mask = torch.cat( 302 | [input_examples_att_mask, input_examples_att_mask.clone()], 0 303 | ) 304 | pooling_mask = get_pooling_mask( 305 | input_ids, self.sep_token_id 306 | ) # Pooling masks indicate the first [SEP] occurrence, used for seq embedding. 307 | labels = torch.cat([mlm_labels, positive_mlm_labels], 0) 308 | next_sentence_label = torch.cat( 309 | [seq_relationship_labels, seq_relationship_labels.clone()], 0 310 | ) 311 | else: 312 | input_ids = input_examples_ids 313 | attention_mask = input_examples_att_mask 314 | pooling_mask = get_pooling_mask( 315 | input_ids, self.sep_token_id 316 | ) # Pooling masks indicate the first [SEP] occurrence, used for seq embedding. 317 | labels = mlm_labels 318 | next_sentence_label = seq_relationship_labels 319 | 320 | return { 321 | "input_ids": input_ids, 322 | "attention_mask": attention_mask, 323 | "pooling_mask": pooling_mask, 324 | "labels": labels, 325 | "next_sentence_label": next_sentence_label, 326 | } 327 | 328 | 329 | class TestCollator: 330 | """Test collator object mapping sequences of items from dataset instance 331 | into batches of IDs and masks used for training models. 332 | """ 333 | 334 | def __init__(self, tokenizer_path: str, maximum_length: int, **kwargs) -> None: 335 | """Creates instance of collator. 336 | 337 | Args: 338 | tokenizer_path (str): Path to tokenizer. 339 | maximum_length (int): Truncating length of token sequences. 340 | """ 341 | self.maximum_length = maximum_length 342 | 343 | self.tokenizer = prepare_tokenizer(tokenizer_path) 344 | 345 | self.sep_token_id = self.tokenizer.get_vocab()[self.tokenizer.sep_token] 346 | self.pad_token_id = self.tokenizer.get_vocab()[self.tokenizer.pad_token] 347 | self.mask_token_id = self.tokenizer.get_vocab()[self.tokenizer.mask_token] 348 | self.cls_token_id = self.tokenizer.get_vocab()[self.tokenizer.cls_token] 349 | 350 | def __call__(self, batch: List[Dict]) -> Dict[str, torch.Tensor]: 351 | """Maps list of pairs of examples to batches of token ids, masks, and labels used for training. 352 | 353 | Args: 354 | batch (List[Dict]): List of pairs of examples. 355 | 356 | Returns: 357 | Dict[str, torch.Tensor]: Batches of tokens and masks. 358 | """ 359 | source_list = [el[0] for el in batch] 360 | target_list = [el[1] for el in batch] 361 | 362 | source_examples_list = [ 363 | f"{CLS_TOKEN}{source_list[i]}{SEPARATOR_TOKEN}" for i in range(len(batch)) 364 | ] 365 | target_examples_list = [ 366 | f"{CLS_TOKEN}{target_list[i]}{SEPARATOR_TOKEN}" for i in range(len(batch)) 367 | ] 368 | 369 | source_target_examples_encoding = self.tokenizer( 370 | source_examples_list + target_examples_list, 371 | padding="longest", 372 | max_length=self.maximum_length, 373 | truncation=True, 374 | return_tensors="pt", 375 | ) 376 | 377 | source_target_examples_ids = source_target_examples_encoding.input_ids 378 | source_target_examples_att_mask = source_target_examples_encoding.attention_mask 379 | 380 | return { 381 | "source_target_ids": source_target_examples_ids, 382 | "source_target_att_mask": source_target_examples_att_mask, 383 | "return_loss": True, # This is so Trainer.prediction_step() will call Trainer.compute_loss during eval() 384 | } 385 | 386 | 387 | class Collator: 388 | """Object wrapping both train and test collators. 389 | Decides which collator to use depending on the configuration of the batch. 390 | Pair of examples --> test instance 391 | Triplet of examples --> train instances 392 | """ 393 | 394 | def __init__( 395 | self, 396 | tokenizer_path: str, 397 | maximum_length: int, 398 | mlm_masking_probability: float = 0.5, 399 | contrastive_masking_probability: float = 0.5, 400 | ignore_contrastive_loss_data: bool = False, 401 | ) -> None: 402 | """Creates instance of collator. 403 | 404 | Args: 405 | tokenizer_path (str): Path to tokenizer. 406 | maximum_length (int): Truncating length of token sequences. 407 | mlm_masking_probability (float, optional): Masking probability for MLM objective. Defaults to 0.5. 408 | contrastive_masking_probability (float, optional): Masking probability for contrastive objective. Defaults to 0.5. 409 | ignore_contrastive_loss_data (bool, optional): Do not add append positive pairs to batch. Defaults to False. 410 | """ 411 | self.train_collator = TrainCollator( 412 | tokenizer_path=tokenizer_path, 413 | maximum_length=maximum_length, 414 | mlm_masking_probability=mlm_masking_probability, 415 | contrastive_masking_probability=contrastive_masking_probability, 416 | ignore_contrastive_loss_data=ignore_contrastive_loss_data, 417 | ) 418 | self.test_collator = TestCollator( 419 | tokenizer_path=tokenizer_path, 420 | maximum_length=maximum_length, 421 | ) 422 | 423 | self.vocabulary_size = len(self.train_collator.tokenizer.vocab) 424 | self.pad_token_id = self.train_collator.pad_token_id 425 | 426 | def __call__(self, batch: List[Dict]) -> Dict[str, torch.Tensor]: 427 | """Maps list of pairs of examples to batches of token ids, masks, and labels used for training. 428 | 429 | Args: 430 | batch (List[Dict]): List of pairs of examples. 431 | 432 | Returns: 433 | Dict[str, torch.Tensor]: Batches of tokens and masks. 434 | """ 435 | 436 | if len(batch[0]) == 3: 437 | return self.train_collator(batch) 438 | elif len(batch[0]) == 2: 439 | return self.test_collator(batch) 440 | else: 441 | raise AttributeError("Unknown batch configuration.") 442 | -------------------------------------------------------------------------------- /src/embedding/distributed_utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.distributed as dist 3 | 4 | 5 | class AllGather(torch.autograd.Function): 6 | """ 7 | all_gather with gradient back-propagation 8 | Adapted from https://github.com/Lightning-AI/lightning-bolts/blob/5577453a6d7072724d9ae24184daf8f45d4baff7/pl_bolts/models/self_supervised/simclr/simclr_module.py#L20-L40 9 | """ 10 | 11 | @staticmethod 12 | def forward(ctx, tensor): 13 | ctx.batch_size = tensor.shape[0] 14 | 15 | gathered_tensor = [ 16 | torch.zeros_like(tensor) for _ in range(torch.distributed.get_world_size()) 17 | ] 18 | 19 | torch.distributed.all_gather(gathered_tensor, tensor) 20 | gathered_tensor = torch.cat(gathered_tensor, 0) 21 | 22 | return gathered_tensor 23 | 24 | @staticmethod 25 | def backward(ctx, grad_output): 26 | grad_input = grad_output.clone() 27 | torch.distributed.all_reduce( 28 | grad_input, op=torch.distributed.ReduceOp.SUM, async_op=False 29 | ) 30 | 31 | idx_from = torch.distributed.get_rank() * ctx.batch_size 32 | idx_to = (torch.distributed.get_rank() + 1) * ctx.batch_size 33 | return grad_input[idx_from:idx_to] 34 | 35 | 36 | all_gather = AllGather.apply 37 | -------------------------------------------------------------------------------- /src/embedding/encoder.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Dict 3 | import torch 4 | from transformers import AutoModel, AutoTokenizer 5 | from embedding.constants import SEPARATOR_TOKEN, CLS_TOKEN 6 | from embedding.utils import pool_and_normalize 7 | from embedding.datasets_loader import prepare_tokenizer 8 | from embedding.preprocessing_utils import truncate_sentences 9 | import pdb 10 | from tqdm import tqdm 11 | 12 | def set_device(inputs: Dict[str, torch.Tensor], device: str) -> Dict[str, torch.Tensor]: 13 | output_data = {} 14 | for k, v in inputs.items(): 15 | output_data[k] = v.to(device) 16 | 17 | return output_data 18 | 19 | class BaseEncoder(torch.nn.Module): #, ABC): 20 | 21 | def __init__(self, device, max_input_len, maximum_token_len, model_name): 22 | super().__init__() 23 | 24 | self.model_name = model_name 25 | self.tokenizer = prepare_tokenizer(model_name) 26 | if model_name == 'Salesforce/codet5p-110m-embedding': 27 | self.encoder = AutoModel.from_pretrained(model_name, trust_remote_code=True).to(device).eval() 28 | else: 29 | self.encoder = AutoModel.from_pretrained(model_name, use_auth_token=True).to(device).eval() 30 | self.device = device 31 | self.max_input_len = max_input_len 32 | self.maximum_token_len = maximum_token_len 33 | 34 | #@abstractmethod 35 | #def forward(self,): 36 | # pass 37 | 38 | def encode(self, input_sentences, batch_size=32, **kwargs): 39 | truncated_input_sentences = truncate_sentences(input_sentences, self.max_input_len) 40 | 41 | n_batches = len(truncated_input_sentences) // batch_size + int(len(truncated_input_sentences) % batch_size > 0) 42 | 43 | embedding_batch_list = [] 44 | 45 | for i in tqdm(range(n_batches)): 46 | start_idx = i*batch_size 47 | end_idx = min((i+1)*batch_size, len(truncated_input_sentences)) 48 | 49 | with torch.no_grad(): 50 | embedding_batch_list.append( 51 | self.forward(truncated_input_sentences[start_idx:end_idx]).detach().cpu() 52 | ) 53 | 54 | input_sentences_embedding = torch.cat(embedding_batch_list) 55 | return [emb.squeeze() for emb in input_sentences_embedding] 56 | 57 | #return [emb.squeeze().numpy() for emb in input_sentences_embedding] 58 | 59 | class StarEncoder(BaseEncoder): 60 | 61 | def __init__(self, device, max_input_len, maximum_token_len): 62 | super().__init__(device, max_input_len, maximum_token_len, model_name = "bigcode/starencoder") 63 | 64 | def forward(self, input_sentences): 65 | 66 | inputs = self.tokenizer( 67 | [f"{CLS_TOKEN}{sentence}{SEPARATOR_TOKEN}" for sentence in input_sentences], 68 | padding="longest", 69 | max_length=self.maximum_token_len, 70 | truncation=True, 71 | return_tensors="pt", 72 | ) 73 | 74 | outputs = self.encoder(**set_device(inputs, self.device)) 75 | embedding = pool_and_normalize(outputs.hidden_states[-1], inputs.attention_mask) 76 | 77 | return embedding 78 | 79 | class CodeT5Plus(BaseEncoder): 80 | def __init__(self, device, max_input_len, maximum_token_len): 81 | super().__init__(device, max_input_len, maximum_token_len, model_name = "Salesforce/codet5p-110m-embedding") 82 | 83 | self.tokenizer = AutoTokenizer.from_pretrained("Salesforce/codet5p-110m-embedding", trust_remote_code=True) 84 | 85 | def forward(self, input_sentences): 86 | 87 | inputs = self.tokenizer( 88 | [f"{CLS_TOKEN}{sentence}{SEPARATOR_TOKEN}" for sentence in input_sentences], 89 | padding="longest", 90 | max_length=self.maximum_token_len, 91 | truncation=True, 92 | return_tensors="pt", 93 | ) 94 | 95 | embedding = self.encoder(**set_device(inputs, self.device)) 96 | 97 | return embedding 98 | 99 | 100 | class CodeBERT(BaseEncoder): 101 | 102 | def __init__(self, device, max_input_len, maximum_token_len): 103 | super().__init__(device, max_input_len, maximum_token_len, model_name = "microsoft/codebert-base") 104 | 105 | self.tokenizer = AutoTokenizer.from_pretrained("microsoft/codebert-base") 106 | 107 | def forward(self, input_sentences): 108 | 109 | inputs = self.tokenizer( 110 | [sentence for sentence in input_sentences], 111 | padding="longest", 112 | max_length=self.maximum_token_len, 113 | truncation=True, 114 | return_tensors="pt", 115 | ) 116 | 117 | inputs = set_device(inputs, self.device) 118 | 119 | #print(inputs["input_ids"].shape, inputs["attention_mask"].shape) 120 | outputs = self.encoder(inputs["input_ids"], inputs["attention_mask"]) 121 | 122 | embedding = outputs["pooler_output"] 123 | 124 | return torch.cat([torch.nn.functional.normalize(torch.Tensor(el)[None, :]) for el in embedding]) 125 | 126 | -------------------------------------------------------------------------------- /src/embedding/preprocessing_utils.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, List, Union 2 | import torch 3 | from embedding.constants import PADDING_ID_FOR_LABELS 4 | from transformers.tokenization_utils_base import PreTrainedTokenizerBase 5 | 6 | 7 | def perturb_tokens( 8 | input_ids: torch.Tensor, 9 | special_tokens_masks: torch.Tensor, 10 | masking_fraction: float, 11 | masking_token_id: Union[int, float], 12 | vocab_size: Union[int, float], 13 | ) -> List[torch.Tensor]: 14 | """Perturb tokens in preparation for MLM loss computation. 15 | Adapted from: 16 | https://github.com/huggingface/transformers/blob/d4306daea1f68d8e854b7b3b127878a5fbd53489/src/transformers/data/data_collator.py#L750 17 | 18 | Args: 19 | input_ids (torch.Tensor): Batch of input tokens IDs. 20 | special_tokens_masks (torch.Tensor): Masks for special tokens that shouldn't be perturberd. 21 | masking_fraction (float): Probability of masking ou a given token. 22 | masking_token_id (Union[int, float]): Token id for masks. 23 | vocab_size (Union[int, float]): vocab_size used to replace inputs for random words. 24 | 25 | Returns: 26 | List[torch.Tensor]: Perturbed ids along with label ids. 27 | """ 28 | 29 | perturbed_input_ids = input_ids.clone() 30 | labels = input_ids.clone() 31 | # We sample a few tokens in each sequence for MLM training (with probability masking_fraction) 32 | probability_matrix = torch.full(labels.shape, masking_fraction) 33 | 34 | probability_matrix.masked_fill_(special_tokens_masks, value=0.0) 35 | masked_indices = torch.bernoulli(probability_matrix).bool() 36 | labels[ 37 | ~masked_indices 38 | ] = PADDING_ID_FOR_LABELS # We only compute loss on masked tokens 39 | 40 | # 80% of the time, we replace masked input tokens with masking_token_id 41 | indices_replaced = ( 42 | torch.bernoulli(torch.full(labels.shape, 0.8)).bool() & masked_indices 43 | ) 44 | perturbed_input_ids[indices_replaced] = masking_token_id 45 | 46 | # 10% of the time, we replace masked input tokens with random word 47 | indices_random = ( 48 | torch.bernoulli(torch.full(labels.shape, 0.5)).bool() 49 | & masked_indices 50 | & ~indices_replaced 51 | ) 52 | random_words = torch.randint(vocab_size, labels.shape, dtype=torch.long) 53 | perturbed_input_ids[indices_random] = random_words[indices_random] 54 | 55 | # The rest of the time (10% of the time) we keep the masked input tokens unchanged 56 | return perturbed_input_ids, labels 57 | 58 | 59 | def get_special_tokens_mask( 60 | tokenizer: PreTrainedTokenizerBase, token_ids: torch.Tensor 61 | ) -> torch.Tensor: 62 | """Get masks indicating the positions of special tokens in a batch of token ids. 63 | Adapted from: 64 | https://github.com/huggingface/transformers/blob/d4306daea1f68d8e854b7b3b127878a5fbd53489/src/transformers/data/data_collator.py#L759 65 | 66 | Args: 67 | tokenizer (PreTrainedTokenizerBase): Tokenizer used to gnerate encoding in token_ids. 68 | token_ids (torch.Tensor): batch of token ids. 69 | 70 | Returns: 71 | torch.Tensor: Batch of masks with 1 wherever a special token appears and 0 otherwise. 72 | """ 73 | 74 | special_tokens_mask = [ 75 | tokenizer.get_special_tokens_mask(val, already_has_special_tokens=True) 76 | for val in token_ids.tolist() 77 | ] 78 | special_tokens_mask = torch.tensor(special_tokens_mask, dtype=torch.bool) 79 | 80 | return special_tokens_mask 81 | 82 | 83 | def truncate_sentences( 84 | sentence_list: List[str], maximum_length: Union[int, float] 85 | ) -> List[str]: 86 | """Truncates list of sentences to a maximum length. 87 | 88 | Args: 89 | sentence_list (List[str]): List of sentences to be truncated. 90 | maximum_length (Union[int, float]): Maximum length of any output sentence. 91 | 92 | Returns: 93 | List[str]: List of truncated sentences. 94 | """ 95 | 96 | truncated_sentences = [] 97 | 98 | for sentence in sentence_list: 99 | truncated_sentences.append(sentence[:maximum_length]) 100 | 101 | return truncated_sentences 102 | 103 | 104 | def split_sentence( 105 | sentence: str, maximum_length: Union[int, float] = None 106 | ) -> List[str]: 107 | """Truncates and splits a given sentence. 108 | 109 | Args: 110 | sentence (str): Input sentence. 111 | maximum_length (Union[int, float], optional): Maximum length. Defaults to None. 112 | 113 | Returns: 114 | List[str]: List of pair of sentences, each being a half of the input after truncation. 115 | """ 116 | 117 | if maximum_length is None: 118 | maximum_length = len(sentence) 119 | else: 120 | maximum_length = min(maximum_length, len(sentence)) 121 | 122 | half_length = maximum_length // 2 123 | 124 | return sentence[:half_length], sentence[half_length:maximum_length] 125 | 126 | 127 | def get_pooling_mask( 128 | input_ids: torch.Tensor, sep_token_id: Union[int, float] 129 | ) -> torch.Tensor: 130 | """Gets pooling masks. For a sequence of input tokens, the mask will be 131 | a sequence of ones up until the first [SEP] occurrence, and 0 after that. 132 | 133 | Args: 134 | input_ids (torch.Tensor): Batch of input ids with shape [B, T]. 135 | sep_token_id (Union[int, float]): Id for [SEP] token. 136 | 137 | Returns: 138 | torch.Tensor: Batch of pooling masks with shape [B, T] 139 | """ 140 | # idx indicates the first occurrence of sep_token_id per along dim 0 of input_ids 141 | idx = (input_ids == sep_token_id).float().argmax(1) 142 | 143 | repeated_idx = idx.unsqueeze(1).repeat(1, input_ids.size(1)) 144 | 145 | ranges = torch.arange(input_ids.size(1)).repeat(input_ids.size(0), 1) 146 | 147 | pooling_mask = (repeated_idx >= ranges).long() 148 | 149 | return pooling_mask 150 | 151 | 152 | class pre_process_codesearchnet_train: 153 | def __init__(self, maximum_length: int) -> None: 154 | """Pre process code search net data by truncating and splitting code snippets. 155 | 156 | Args: 157 | maximum_length (int): Max length of code snippets. 158 | """ 159 | self.maximum_length = maximum_length 160 | 161 | def __call__(self, example: Dict) -> Dict: 162 | """Reads code string, truncates it and splits in two pieces. 163 | 164 | Args: 165 | example (Dict): Input data example. 166 | 167 | Returns: 168 | Dict: Pre-processed example. 169 | """ 170 | code_str = example["func_code_string"] 171 | code_str_source, code_str_target = split_sentence(code_str, self.maximum_length) 172 | example.update({"source": code_str_source, "target": code_str_target}) 173 | return example 174 | 175 | 176 | class pre_process_codesearchnet_test: 177 | def __init__(self, maximum_length: int) -> None: 178 | """Pre process code search net data by truncating and pairing code and docstring. 179 | 180 | Args: 181 | maximum_length (int): Max length of code snippets. 182 | """ 183 | self.maximum_length = maximum_length 184 | 185 | def __call__(self, example: Dict) -> Dict: 186 | """Reads and truncates code and doc strings. 187 | 188 | Args: 189 | example (Dict): Input data example. 190 | 191 | Returns: 192 | Dict: Pre-processed example. 193 | """ 194 | source = example["func_documentation_tokens"] 195 | source = (" ").join(source)[: self.maximum_length] 196 | target = example["func_code_string"][: self.maximum_length] 197 | example.update({"source": source, "target": target}) 198 | return example 199 | 200 | 201 | class pre_process_gfg: 202 | def __init__(self, maximum_length: int) -> None: 203 | """Pre process Python-Java Geeks4Geeks data by truncating and pairing code snippets. 204 | 205 | Args: 206 | maximum_length (int): Max length of code snippets. 207 | """ 208 | self.maximum_length = maximum_length 209 | 210 | def __call__(self, example: Dict) -> Dict: 211 | """Reads and truncates code strings. 212 | 213 | Args: 214 | example (Dict): Input data example. 215 | 216 | Returns: 217 | Dict: Pre-processed example. 218 | """ 219 | 220 | source = example["python_func"][: self.maximum_length] 221 | target = example["java_func"][: self.maximum_length] 222 | 223 | example.update({"source": source, "target": target}) 224 | 225 | return example 226 | 227 | 228 | class pre_process_the_stack: 229 | def __init__(self, maximum_length: int) -> None: 230 | """Pre process The Stack data by truncating and splitting code files. 231 | 232 | Args: 233 | maximum_length (int): Max length of code snippets. 234 | """ 235 | self.maximum_length = maximum_length 236 | 237 | def __call__(self, example: Dict) -> Dict: 238 | """Reads, truncates, and splits code strings. 239 | 240 | Args: 241 | example (Dict): Input data example. 242 | 243 | Returns: 244 | Dict: Pre-processed example. 245 | """ 246 | 247 | code_str = example["content"] 248 | code_str_source, code_str_target = split_sentence(code_str, self.maximum_length) 249 | example.update({"source": code_str_source, "target": code_str_target}) 250 | return example 251 | -------------------------------------------------------------------------------- /src/embedding/utils.py: -------------------------------------------------------------------------------- 1 | from typing import List, Union, Optional 2 | import torch 3 | from embedding.distributed_utils import all_gather 4 | 5 | 6 | class TempCoef(torch.nn.Module): 7 | """Module wrapping a temperature coeficient used to compute contrastive losses.""" 8 | 9 | def __init__(self, initial_value: float) -> None: 10 | """Constructs TempCoef instance. 11 | 12 | Args: 13 | initial_value (float): Startting value of the temperature. 14 | """ 15 | super().__init__() 16 | self.temp_coef = torch.nn.Parameter(torch.Tensor([initial_value])) 17 | 18 | def forward(self, logits_matrix: torch.Tensor) -> torch.Tensor: 19 | """Forward pass of the module: Multiply input tensor by the temperature value. 20 | 21 | Args: 22 | logits_matrix (torch.Tensor): Input tensor 23 | 24 | Returns: 25 | torch.Tensor: Logits matrix multiplied by temp. 26 | """ 27 | # Apply learnable temperature factor on similarities 28 | # Clamping after op to avoid numerical instabilities 29 | logits_matrix = logits_matrix * self.temp_coef.clamp(1e-4, 30.0) 30 | 31 | return logits_matrix 32 | 33 | def get_temp_coef(self) -> float: 34 | """Get temperature value. 35 | 36 | Returns: 37 | float: temperature value. 38 | """ 39 | return self.temp_coef.data.detach() 40 | 41 | 42 | def gather_embeddings( 43 | embedding_1: torch.Tensor, embedding_2: torch.Tensor 44 | ) -> List[torch.Tensor]: 45 | """Gathers embeddings across devices in distributed settings. 46 | 47 | Args: 48 | embedding_1 (torch.Tensor): First batch of embeddings. Expected shape: [n, d] 49 | embedding_2 (torch.Tensor): Second batch of embeddings. Expected shape: [n, d] 50 | 51 | Returns: 52 | torch.Tensor: List of tensors concatenated from all devices, each with shape [n*NUM_DEVICES, d]. 53 | """ 54 | 55 | # Creates extra dimension for concatenating batches into single tensor. 56 | embedding_1 = embedding_1.unsqueeze(1) 57 | embedding_2 = embedding_2.unsqueeze(1) 58 | 59 | embedding = torch.cat( 60 | ( 61 | embedding_1, 62 | embedding_2, 63 | ), 64 | 1, 65 | ) 66 | 67 | # Gather embeddings across devices 68 | embedding_dist = all_gather(embedding) 69 | 70 | embedding_1_dist = embedding_dist[:, 0, :] 71 | embedding_2_dist = embedding_dist[:, 1, :] 72 | 73 | return embedding_1_dist, embedding_2_dist 74 | 75 | 76 | def clip_contrastive_loss( 77 | emb_1: torch.Tensor, 78 | emb_2: torch.Tensor, 79 | temperature_coef: TempCoef, 80 | local_loss: bool = False, 81 | ) -> torch.Tensor: 82 | """Computes contrastive CLIP-style loss. 83 | 84 | Args: 85 | emb_1 (torch.Tensor): Input embeddings. 86 | emb_2 (torch.Tensor): Embedding of positive pairs (perturbed inputs) 87 | temperature_coef (TempCoef): Module wrapping trainable temperature parameter. 88 | local_loss (bool, optional): If set, contrastive loss will only use data in current device. Defaults to False. 89 | 90 | Returns: 91 | torch.Tensor: Contrastive loss. 92 | """ 93 | 94 | if local_loss: 95 | emb_1_dist, emb_2_dist = emb_1, emb_2 96 | else: 97 | # Gathers embeddings across devices. 98 | emb_1_dist, emb_2_dist = gather_embeddings(emb_1, emb_2) 99 | 100 | # Compute similarity matrix 101 | similarities = emb_1_dist @ emb_2_dist.T 102 | 103 | # Multiply similarity matrix by temperature 104 | similarities = temperature_coef(similarities) 105 | 106 | # Matching representations of positive pairs assumed to be located at the main 107 | # dioagonal of the similarity matrix if targets are not given 108 | ce_labels = torch.arange(similarities.size(0)).long().to(similarities.device) 109 | 110 | # We use a cross-entropy criterion to increase the similarities between 111 | # matching representations of source and target 112 | # We follow CLIP and apply the loss across columns and rows. 113 | sim_loss = 0.5 * ( 114 | torch.nn.functional.cross_entropy(similarities, ce_labels) 115 | + torch.nn.functional.cross_entropy(similarities.T, ce_labels) 116 | ) 117 | 118 | return sim_loss 119 | 120 | 121 | def pool_and_normalize( 122 | features_sequence: torch.Tensor, 123 | attention_masks: torch.Tensor, 124 | return_norms: bool = False, 125 | ) -> Union[torch.Tensor, List[torch.Tensor]]: 126 | """Temporal ooling of sequences of vectors and projection onto the unit sphere. 127 | 128 | Args: 129 | features_sequence (torch.Tensor): Inpute features with shape [B, T, F]. 130 | attention_masks (torch.Tensor): Pooling masks with shape [B, T, F]. 131 | return_norms (bool, optional): Whether to additionally return the norms. Defaults to False. 132 | 133 | Returns: 134 | Union[torch.Tensor, List[torch.Tensor]]: Pooled and normalized vectors with shape [B, F]. 135 | """ 136 | 137 | pooled_embeddings = pooling(features_sequence, attention_masks) 138 | embedding_norms = pooled_embeddings.norm(dim=1) 139 | 140 | normalizing_factor = torch.where( # Only normalize embeddings with norm > 1.0. 141 | embedding_norms > 1.0, embedding_norms, torch.ones_like(embedding_norms) 142 | ) 143 | 144 | pooled_normalized_embeddings = pooled_embeddings / normalizing_factor[:, None] 145 | 146 | if return_norms: 147 | return pooled_normalized_embeddings, embedding_norms 148 | else: 149 | return pooled_normalized_embeddings 150 | 151 | 152 | def pooling(x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor: 153 | """Pools a batch of vector sequences into a batch of vector global representations. 154 | It does so by taking the last vector in the sequence, as indicated by the mask. 155 | 156 | Args: 157 | x (torch.Tensor): Batch of vector sequences with shape [B, T, F]. 158 | mask (torch.Tensor): Batch of masks with shape [B, T]. 159 | 160 | Returns: 161 | torch.Tensor: Pooled version of the input batch with shape [B, F]. 162 | """ 163 | 164 | eos_idx = mask.sum(1) - 1 165 | batch_idx = torch.arange(len(eos_idx), device=x.device) 166 | 167 | mu = x[batch_idx, eos_idx, :] 168 | 169 | return mu 170 | 171 | 172 | def retrieval_eval( 173 | x_source: torch.Tensor, x_target: torch.Tensor, return_similarities: Optional[bool] = False 174 | ) -> List[torch.Tensor]: 175 | """Performs retrieval evaluation given paired embeddings of source and target data. 176 | 177 | Args: 178 | x_source (torch.Tensor): Source batch of embeddings with shape [B, emb_dim]. 179 | x_target (torch.Tensor): Target batch of embeddings with shape [B, emb_dim]. 180 | return_similarities (Optional[bool]): Whether to return similarity matrix. Defaults to False. 181 | 182 | Returns: 183 | List[torch.Tensor]: Various retrieval metrics: R@1, R@5, and MRR. 184 | """ 185 | 186 | # Compute similarity matrix 187 | similarities = x_source @ x_target.T 188 | 189 | topk_indices = torch.topk(similarities, k=similarities.size(1), dim=1)[1] 190 | 191 | ce_labels = torch.arange(similarities.size(0)).long().view(similarities.size(0), 1) 192 | 193 | # Bool tensor indicating which rows contain the idx corresponding to the main diag. of the sim matrix 194 | results = topk_indices.eq(ce_labels) 195 | 196 | r_at_1 = results[:, :1].sum() / float(similarities.size(0)) 197 | r_at_5 = results[:, :5].sum() / float(similarities.size(0)) 198 | 199 | ranks = results.nonzero()[:, 1].float() + 1.0 200 | mrr = (1 / ranks).mean() 201 | 202 | if return_similarities: 203 | return r_at_1, r_at_5, mrr, similarities 204 | else: 205 | return r_at_1, r_at_5, mrr -------------------------------------------------------------------------------- /src/evaluate.py: -------------------------------------------------------------------------------- 1 | import tempfile, shutil, os 2 | import json 3 | import pprint 4 | import pickle as pkl 5 | 6 | from datetime import timedelta 7 | import time 8 | from copy import copy 9 | import random 10 | from copy import deepcopy 11 | from functools import partial 12 | import shutil 13 | from glob import glob 14 | from tqdm import tqdm 15 | from scipy.stats import describe 16 | import random 17 | import pdb 18 | 19 | from datasets import load_dataset, load_from_disk, Dataset, DatasetDict 20 | 21 | from configs.config_evaluate import parse_args 22 | from utils.utils import extract_code 23 | from utils.utils_evaluate import safe_eval_answer_from_agent, compute_pass_at_ks 24 | 25 | args = parse_args() 26 | argsdict = vars(args) 27 | print(pprint.pformat(argsdict)) 28 | 29 | val_dataset = load_from_disk(f'data/{args.eval_split}') 30 | 31 | if args.example_test_path is not None: 32 | example_tests = pkl.load(open(args.example_test_path, 'rb')) 33 | 34 | num_outputs = [] 35 | original_samples = 0 36 | processed_val_dataset = DatasetDict() # empty DatasetDict 37 | for level, subset_data in val_dataset.items(): 38 | new_data = [] 39 | for sample in subset_data: 40 | problem_id = sample['problem_id'] 41 | 42 | starter_code = sample['starter_code'] 43 | gen_file = args.save_gen_path + '/{}.json'.format(problem_id) 44 | 45 | if not os.path.exists(gen_file) and args.original_gen_path is not None: 46 | gen_file = args.original_gen_path + '/{}.json'.format(problem_id) 47 | original_samples += 1 48 | 49 | if os.path.exists(gen_file): 50 | data = json.load(open(gen_file, 'r')) 51 | output = data['output'] 52 | prompt = data['prompt'] 53 | final_code = [extract_code(c) for c in output] 54 | 55 | else: 56 | continue 57 | 58 | if len(final_code)==0: 59 | final_code = [''] 60 | 61 | num_outputs.append(len(final_code)) 62 | sample['gpt_codes'] = final_code 63 | 64 | sample['gen_file'] = gen_file 65 | 66 | if args.example_test_path is not None: 67 | tests = example_tests[sample['problem_id']] 68 | if tests is None: 69 | continue 70 | else: 71 | sample['input_output'] = tests 72 | 73 | new_data.append(sample) 74 | 75 | 76 | processed_val_dataset[level] = Dataset.from_list(new_data) 77 | 78 | 79 | print("Evaluation dataset:") 80 | print(processed_val_dataset) 81 | print("# Samples from original (Round 0): {}".format(original_samples)) 82 | 83 | # generate answer for each difficulty 84 | res_strs = [] 85 | res_strs_by_passk = {} 86 | pass_results_all = {} 87 | for level, new_subset_data in processed_val_dataset.items(): 88 | new_subset_data = new_subset_data.map(safe_eval_answer_from_agent, num_proc=args.num_proc) 89 | processed_val_dataset[level] = Dataset.from_list(new_subset_data) 90 | 91 | # calculate metrics 92 | pass_results = [example['gpt_pass_flags'] for example in processed_val_dataset[level] if example['gpt_pass_flags']] 93 | 94 | for example in processed_val_dataset[level]: 95 | pass_results_all[example['problem_id']] = {'result': example['gpt_pass_flags'], 'file': example['gen_file']} 96 | 97 | pass_at_ks = compute_pass_at_ks(pass_results, ks=[1]) 98 | 99 | cost = sum([dp['gpt_cost'] if 'gpt_cost' in dp else 0 for dp in processed_val_dataset[level]]) 100 | res_str = f"Difficulty level: {level}, pass@k: {pass_at_ks}, num of examples: {len(pass_results)}, total_cost: {cost}" 101 | print(res_str) 102 | res_strs.append(res_str) 103 | for k,v in pass_at_ks.items(): 104 | if k not in res_strs_by_passk: 105 | res_strs_by_passk[k] = [] 106 | res_strs_by_passk[k].append(v) 107 | print("="*20) 108 | print('\n'.join(res_strs)) 109 | 110 | print("Number of outputs distributions:") 111 | print(describe(num_outputs)) 112 | 113 | for k,v in res_strs_by_passk.items(): 114 | v = [str(i) for i in v] 115 | print('pass@{}'.format(k), ' '.join(v)) 116 | 117 | if args.save_results_path is not None: 118 | json.dump(pass_results_all, open(args.save_results_path, 'w')) -------------------------------------------------------------------------------- /src/generate.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import json 4 | import pprint 5 | from tqdm import tqdm 6 | import pickle as pkl 7 | import pandas as pd 8 | import pdb 9 | 10 | import openai 11 | from datasets import load_dataset, load_from_disk 12 | import torch 13 | 14 | from configs.config_generate import parse_args 15 | from utils.utils import get_util_functions_self_cluster 16 | 17 | 18 | args = parse_args() 19 | argsdict = vars(args) 20 | print(pprint.pformat(argsdict)) 21 | 22 | if torch.cuda.is_available(): 23 | device = "cuda" 24 | else: 25 | device = "cpu" 26 | 27 | model_mapping = {"gpt3.5": "gpt-3.5-turbo-16k", 28 | "gpt4": "gpt-4"} 29 | 30 | API_KEY_FILE = 'openaiapikey.txt' 31 | with open(API_KEY_FILE, 'r', encoding='utf-8') as infile: 32 | openai.api_key = infile.read() 33 | if not openai.api_key: 34 | print("Could not find API key value at {}".format(API_KEY_FILE)) 35 | sys.exit(1) 36 | 37 | if args.split is not None and args.split == 'mini_val': 38 | apps = load_from_disk(f'data/{args.split}') 39 | apps_problem_ls = [] 40 | for level in ['introductory', 'interview', 'competition']: 41 | apps_problem_ls += list(apps[level]) 42 | else: 43 | apps = load_dataset("codeparrot/apps")[args.split] 44 | 45 | with open(args.prompt_file, 'r', encoding='utf-8') as infile: 46 | prompt = infile.read() 47 | 48 | if not os.path.exists(args.output_path): 49 | os.makedirs(args.output_path, exist_ok=True) 50 | 51 | if args.modules_file is not None: 52 | if '.csv' in args.modules_file: 53 | modules = pd.read_csv(args.modules_file) 54 | else: 55 | modules = pkl.load(open(args.modules_file, 'rb')) 56 | 57 | modules = get_util_functions_self_cluster(modules, num_clusters=args.num_clusters) 58 | print("Util functions for {} problems".format(len(modules))) 59 | 60 | lens = [] 61 | for idx in tqdm(range(args.start, args.end), total=args.end-args.start): 62 | if args.split is not None and args.split == 'mini_val': 63 | problem = apps_problem_ls[idx] 64 | else: 65 | problem = apps[idx] 66 | problem_id = problem['problem_id'] 67 | 68 | if os.path.exists(args.output_path + '/{}.json'.format(problem_id)): 69 | continue 70 | 71 | question = problem['question'] 72 | curr_prompt = prompt.replace("<>", question) 73 | 74 | if '<>' in prompt: 75 | starter_code = problem['starter_code'] 76 | curr_prompt = curr_prompt.replace("<>", starter_code) 77 | 78 | if '<>' in prompt: 79 | starter_code = problem['starter_code'] 80 | if len(starter_code)>0: 81 | starter_code_prompt = f"Notes:\nThe final python function should begin with: \n```python\n{starter_code}\n```" 82 | else: 83 | starter_code_prompt = '' 84 | curr_prompt = curr_prompt.replace("<>", starter_code_prompt) 85 | 86 | if '<>' in prompt: 87 | starter_code = problem['starter_code'] 88 | if len(starter_code)>0: 89 | question_guide = 'use the provided function signature' 90 | else: 91 | question_guide = 'read from and write to standard IO' 92 | curr_prompt = curr_prompt.replace("<>", question_guide) 93 | 94 | if '<>' in curr_prompt: 95 | if problem_id not in modules: continue 96 | curr_modules = list(modules[problem_id]) 97 | module_seq = '' 98 | for module in curr_modules: 99 | module_seq += "\n```module\n" + module.strip() + "\n```\n" 100 | curr_prompt = curr_prompt.replace('<>', module_seq) 101 | 102 | success = False 103 | start = time.time() 104 | responses = [] 105 | if args.num_gen_samples==1: 106 | num_loops = 1 107 | else: 108 | num_loops = int(args.num_gen_samples/5) 109 | 110 | for i in tqdm(range(num_loops), leave=False): 111 | while time.time() - start < 80: 112 | try: 113 | response = openai.ChatCompletion.create( 114 | model=model_mapping[args.model], 115 | messages=[ 116 | {"role": "system", 117 | "content": "You are a helpful AI assistant to help developers to solve challenging coding problems."}, 118 | {"role": "user", 119 | "content": curr_prompt} 120 | ], 121 | n=5 if args.num_gen_samples > 1 else 1, 122 | ) 123 | success = True 124 | responses.append(response) 125 | break 126 | except openai.error.InvalidRequestError as e: 127 | print( 128 | f"Invalid request error: {e}" 129 | ) 130 | time.sleep(10) 131 | continue 132 | 133 | except openai.error.RateLimitError as e: 134 | print( 135 | f"Rate limit error: {e}" 136 | ) 137 | time.sleep(10) 138 | continue 139 | 140 | except openai.error.APIError as e: 141 | print( 142 | f"HTTP code 502 API error: {e}" 143 | ) 144 | time.sleep(10) 145 | continue 146 | 147 | if not success: 148 | print("Failure to generate! skipp this sample problem id {}".format(problem_id)) 149 | continue 150 | 151 | if args.num_gen_samples == 1: 152 | result = response.choices[0].message['content'] 153 | else: 154 | result = [] 155 | for response in responses: 156 | result += [choice.message['content'] for choice in response.choices] 157 | 158 | curr_output = {} 159 | curr_output['prompt'] = curr_prompt 160 | curr_output['output'] = result 161 | json.dump(curr_output, open(args.output_path + '/{}.json'.format(problem_id), 'w')) -------------------------------------------------------------------------------- /src/processing.py: -------------------------------------------------------------------------------- 1 | import json 2 | from glob import glob 3 | from tqdm import tqdm 4 | from multiprocessing import Pool 5 | from functools import partial 6 | import numpy as np 7 | import re 8 | import argparse, pprint 9 | import datasets 10 | import os 11 | from collections import Counter 12 | import pandas as pd 13 | 14 | import tokenize 15 | from io import BytesIO 16 | from collections import deque 17 | from scipy.stats import describe 18 | import itertools 19 | 20 | from configs.config_processing import parse_args 21 | from utils.utils import extract_func, get_func_codes, extract_code, is_in_final_code, udpate_code_by_all_past_results 22 | 23 | import pdb 24 | 25 | args = parse_args() 26 | argsdict = vars(args) 27 | print(pprint.pformat(argsdict)) 28 | 29 | files = sorted(glob(args.output_path + '/*.json')) 30 | print("Number of data files: {}".format(len(files))) 31 | 32 | if args.result_path is not None: 33 | results = json.load(open(args.result_path, 'r')) 34 | if args.past_result_path is not None: 35 | past_results = json.load(open(args.past_result_path, 'r')) 36 | results, files = udpate_code_by_all_past_results(results, past_results, files) 37 | 38 | all_funcs = [] 39 | all_func_list = [] 40 | all_files = [] 41 | all_func_codes = [] 42 | 43 | empty_reasoning_count = 0 44 | empty_funcs_count = 0 45 | empty_code_count = 0 46 | empty_final_funcs_count = 0 47 | 48 | num_outputs = [] 49 | 50 | it = tqdm(files, total=len(files)) 51 | for sample in it: 52 | data = json.load(open(sample, 'r')) 53 | if 'output' not in data: pdb.set_trace() 54 | output = data['output'] 55 | problem_id = sample.split('/')[-1].replace('.json','') 56 | 57 | if args.result_path is not None: 58 | if str(problem_id) not in results: 59 | continue 60 | 61 | curr_results = results[str(problem_id)]['result'] 62 | if len(curr_results) != len(output): pdb.set_trace() 63 | output = [o for i, o in enumerate(output) if curr_results[i]==True] 64 | 65 | num_outputs.append(len(output)) 66 | 67 | final_code = [extract_code(o) for o in output] 68 | final_code = [o for o in final_code if len(o)>0] 69 | 70 | funcs = [extract_func(o) for o in output] 71 | func_codes = [get_func_codes(f) for f in final_code] 72 | 73 | assert len(func_codes) == len(funcs) 74 | 75 | temp = [is_in_final_code(f, fc) for f, fc in zip(funcs, func_codes)] 76 | funcs = [f[0] for f in temp] 77 | funcs = list(itertools.chain.from_iterable(funcs)) 78 | full_funcs = [f[1] for f in temp] 79 | full_funcs = list(itertools.chain.from_iterable(full_funcs)) 80 | 81 | # Add functions from final code too 82 | if len(funcs)==0 and len(output)!=0: 83 | full_funcs = list(itertools.chain.from_iterable(func_codes)) 84 | funcs = ['No docstring'] * len(full_funcs) 85 | if len(funcs)==0: 86 | empty_final_funcs_count += 1 87 | 88 | all_funcs += funcs 89 | all_func_codes += full_funcs 90 | all_func_list.append(funcs) 91 | 92 | all_files += [sample] * len(funcs) 93 | 94 | assert len(all_funcs) == len(all_func_codes) 95 | 96 | 97 | print("Number of generation samples: ") 98 | print(describe(num_outputs)) 99 | print(Counter(num_outputs)) 100 | print() 101 | functions_per_problem = [len(i) for i in all_func_list] 102 | print("Average func per problem: {}".format(sum(functions_per_problem)/len(functions_per_problem))) 103 | print(describe(functions_per_problem)) 104 | print(Counter(functions_per_problem)) 105 | 106 | if args.output_file is not None: 107 | json.dump(results, open(args.output_file + '_all_results.json', 'w')) 108 | 109 | processed_data = pd.DataFrame( 110 | {'file': all_files, 111 | 'func': all_funcs, 112 | 'func_code': all_func_codes 113 | }) 114 | 115 | print("Saving processed data to {}".format(args.output_file)) 116 | processed_data.to_csv(args.output_file) 117 | 118 | -------------------------------------------------------------------------------- /src/utils/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SalesforceAIResearch/CodeChain/a738bedd8d925cfc9abf2a794f6faf609f6f7612/src/utils/.DS_Store -------------------------------------------------------------------------------- /src/utils/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | from glob import glob 3 | import pdb 4 | from tqdm import tqdm 5 | from multiprocessing import Pool 6 | from functools import partial 7 | import numpy as np 8 | import re 9 | import argparse, pprint 10 | import datasets 11 | import os 12 | from collections import Counter 13 | import pandas as pd 14 | import random 15 | import tokenize 16 | from io import BytesIO 17 | from collections import deque 18 | from scipy.stats import describe 19 | import copy 20 | import torch 21 | 22 | def extract_code_segment(result, keyword, all_segments=True): 23 | regex = '\`\`\`\s*{}((?:.|\n)*?)\`\`\`'.format(keyword) 24 | codes = re.findall(regex, result) 25 | if len(codes)==0: 26 | regex = '\`\`\`\s*{}'.format(keyword) 27 | indices = [(m.start(0), m.end(0)) for m in re.finditer(regex, result)] 28 | if len(indices) == 0: 29 | return '' 30 | last_end_index = indices[-1][1] 31 | code = result[last_end_index:] 32 | return code 33 | if all_segments: 34 | return '\n'.join(codes) 35 | return codes[-1] 36 | 37 | def extract_code(result, keyword='python'): 38 | code = extract_code_segment(result, keyword) 39 | if len(code)==0: 40 | code = extract_code_segment(result, '', False) 41 | 42 | return code 43 | 44 | def extract_func(result, keyword='module'): 45 | regex = '\`\`\`\s*{}((?:.|\n)*?)\`\`\`'.format(keyword) 46 | codes = re.findall(regex, result) 47 | 48 | if len(codes)==0: 49 | if '\nSTEP 2' in result: 50 | index = result.index('\nSTEP 2') 51 | regex = '\`\`\`\s*python((?:.|\n)*?)\`\`\`' 52 | codes = re.findall(regex, result[:index]) 53 | 54 | codes = [o for o in codes if 'class ' not in o and 'def main(' not in o] 55 | 56 | new_outputs = [] 57 | for output in codes: 58 | indices = [m.start() for m in re.finditer('def ', output)] 59 | if len(indices)>1: 60 | funcs = [] 61 | for i, index in enumerate(indices[:-1]): 62 | func = output[index: indices[i+1]] 63 | funcs.append(func) 64 | func = output[indices[-1]:] 65 | funcs.append(func) 66 | new_outputs += funcs 67 | elif len(indices)==0: 68 | continue 69 | else: 70 | new_outputs.append(output) 71 | 72 | return new_outputs 73 | 74 | def get_func_codes(code_string): 75 | code_string = code_string.strip() 76 | file = BytesIO(code_string.encode()) 77 | tokens = None 78 | try: 79 | tokens = deque(tokenize.tokenize(file.readline)) 80 | except Exception as e: 81 | print("Error parsing function code: " + str(e)) 82 | pass 83 | if tokens is None: 84 | return [] 85 | lines = [] 86 | while tokens: 87 | token = tokens.popleft() 88 | if token.type == tokenize.NAME and token.string == 'def': 89 | start_line, _ = token.start 90 | last_token = token 91 | while tokens: 92 | token = tokens.popleft() 93 | if token.type == tokenize.NEWLINE: 94 | break 95 | last_token = token 96 | if last_token.type == tokenize.OP and last_token.string == ':': 97 | indents = 0 98 | while tokens: 99 | token = tokens.popleft() 100 | if token.type == tokenize.NL: 101 | continue 102 | if token.type == tokenize.INDENT: 103 | indents += 1 104 | elif token.type == tokenize.DEDENT: 105 | indents -= 1 106 | if not indents: 107 | break 108 | else: 109 | last_token = token 110 | lines.append((start_line, last_token.end[0])) 111 | code_lines = code_string.split('\n') 112 | outputs = [] 113 | for line in lines: 114 | start, end = line 115 | function = '\n'.join(code_lines[start-1:end]) 116 | if len(function.strip())>0: 117 | outputs.append(function) 118 | return outputs 119 | 120 | def is_in_final_code(funcs, func_codes): 121 | output_funcs = [] 122 | output_funcs_codes = [] 123 | for func in funcs: 124 | lines = func.split('\n') 125 | for line in lines: 126 | if 'def ' in line: 127 | for func_code in func_codes: 128 | if line.strip() in func_code: 129 | output_funcs.append(func) 130 | output_funcs_codes.append(func_code) 131 | break 132 | break 133 | assert len(output_funcs) == len(output_funcs_codes) 134 | return output_funcs, output_funcs_codes 135 | 136 | def get_embedding(output_embed_file, data, embedding_model, func_type='func'): 137 | from embedding.encoder import CodeBERT, StarEncoder, CodeT5Plus 138 | 139 | if func_type == 'func': 140 | seqs = [] 141 | for x, y in zip(data['func'].tolist(), data['func_code'].tolist()): 142 | if x != 'No docstring': 143 | seqs.append(x + '\n' + y) 144 | else: 145 | seqs.append(y) 146 | 147 | elif func_type == 'centroid': 148 | seqs = [] 149 | clusters = [] 150 | docs = [] 151 | codes = [] 152 | for row in data.iterrows(): 153 | cluster = row[1]['cluster'] 154 | if cluster not in clusters: 155 | clusters.append(cluster) 156 | doc = row[1]['centroid'] 157 | code = row[1]['centroid_code'] 158 | docs.append(doc) 159 | codes.append(code) 160 | if doc != 'No docstring': 161 | seqs.append(doc + '\n' + code) 162 | else: 163 | seqs.append(y) 164 | 165 | if os.path.exists(output_embed_file): 166 | print("Loading embedding from {}".format(output_embed_file)) 167 | embeds = np.load(output_embed_file) 168 | print("Embedding of shape {}".format(embeds.shape)) 169 | else: 170 | DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu') 171 | MAX_INPUT_LEN = 10000 172 | MAX_TOKEN_LEN = 512 if embedding_model == 'codebert' else 1024 173 | 174 | if embedding_model == 'codebert': 175 | encoder = CodeBERT(DEVICE, MAX_INPUT_LEN, MAX_TOKEN_LEN) 176 | elif embedding_model == 'starencoder': 177 | encoder = StarEncoder(DEVICE, MAX_INPUT_LEN, MAX_TOKEN_LEN) 178 | elif embedding_model == 'codet5': 179 | encoder = CodeT5Plus(DEVICE, MAX_INPUT_LEN, MAX_TOKEN_LEN) 180 | 181 | print("Obtain embeddings...") 182 | embeddings = encoder.encode(seqs) 183 | embeds = np.stack(embeddings, axis=0) 184 | print("Embedding of shape {}".format(embeds.shape)) 185 | np.save(output_embed_file, embeds) 186 | print("Saved embedding to {}".format(output_embed_file)) 187 | 188 | if func_type == 'centroid': 189 | return embeds, docs, codes 190 | 191 | return embeds 192 | 193 | def create_func_prompt(doc, code): 194 | if doc == 'No docstring': 195 | return code 196 | else: 197 | code_lines = code.split('\n') 198 | cutoff_line_idx = -1 199 | for line_idx, line in enumerate(code_lines): 200 | if 'def ' in line: 201 | cutoff_line_idx = line_idx 202 | break 203 | code = '\n'.join(code_lines[cutoff_line_idx+1:]) 204 | return doc + '\n' + code 205 | 206 | def get_util_functions_self_cluster(data, num_clusters=1, problem_id_type=int): 207 | outputs = {} 208 | for row in data.iterrows(): 209 | file = row[1]['file'] 210 | problem_id = problem_id_type(file.split('/')[-1].replace('.json', '')) 211 | centroid = row[1]['centroid_code'] 212 | centroid_doc = row[1]['centroid'] 213 | 214 | if problem_id not in outputs: 215 | outputs[problem_id] = [] 216 | 217 | func_str = create_func_prompt(centroid_doc, centroid) 218 | if func_str not in outputs[problem_id]: 219 | outputs[problem_id].append(func_str) 220 | 221 | new_outputs = {} 222 | for k,v in outputs.items(): 223 | sampled = random.sample(v, min(num_clusters, len(v))) 224 | new_outputs[k] = sampled 225 | 226 | lens = [len(i) for i in new_outputs.values()] 227 | print("Distribution of number of utils:") 228 | print(describe(lens)) 229 | print(Counter(lens)) 230 | return new_outputs 231 | 232 | def udpate_code_by_all_past_results(results, past_results, files): 233 | new_files = [] 234 | for k,v in past_results.items(): 235 | past_result = v['result'] 236 | 237 | if k in results: 238 | curr_result = results[k]['result'] 239 | 240 | # if no passed code in this round, check past results 241 | if True not in curr_result and True in past_result: 242 | results[k] = past_results[k] 243 | 244 | elif True in past_result: 245 | results[k] = past_results[k] 246 | 247 | if k in results: 248 | new_files.append(results[k]['file']) 249 | 250 | return results, new_files -------------------------------------------------------------------------------- /src/utils/utils_evaluate.py: -------------------------------------------------------------------------------- 1 | import gzip 2 | import io 3 | import itertools 4 | import json 5 | import pprint 6 | import numpy as np 7 | import re 8 | import sys 9 | import timeout_decorator 10 | import traceback 11 | from collections import Counter 12 | from io import StringIO 13 | import sys 14 | from collections import defaultdict 15 | from datasets import concatenate_datasets, load_dataset 16 | from multiprocessing import Process, Queue 17 | import multiprocessing 18 | from tqdm import tqdm 19 | from typing import Dict, List, Union 20 | import os 21 | import ast 22 | import random 23 | import subprocess 24 | import tempfile, shutil, os 25 | from pyext import RuntimeModule 26 | from copy import deepcopy, copy 27 | from functools import wraps 28 | import time 29 | import contextlib 30 | import pdb 31 | 32 | from utils.utils_execute import run_test 33 | 34 | GLOBAL_TIMEOUT = 10 # TIMEOUT for one solution 35 | 36 | def safe_eval_answer_from_agent(example): 37 | def _temp_run(code, tests, result): 38 | try: 39 | flag, outcomes = verify_code_official(tests, code) 40 | result.append(flag) 41 | except Exception as e: 42 | pass 43 | 44 | example['gpt_pass_flags'] = [] 45 | try: 46 | if type(example['input_output'])==str: 47 | tests = json.loads(example['input_output']) 48 | else: 49 | tests = example['input_output'] 50 | except: 51 | print(f"Failed to get unit tests for problem {example['problem_id']} with {example['input_output']}") 52 | return example 53 | 54 | for code in example['gpt_codes']: 55 | manager = multiprocessing.Manager() 56 | result = manager.list() 57 | start = time.time() 58 | p = multiprocessing.Process(target=_temp_run, args=(code, tests, result)) 59 | p.start() 60 | p.join(timeout=GLOBAL_TIMEOUT + 1) 61 | time_used = time.time() - start 62 | if p.is_alive(): 63 | p.kill() 64 | if not result: 65 | result = [-1] 66 | 67 | if result[0] == True: 68 | example['gpt_pass_flags'] += [True] 69 | else: 70 | example['gpt_pass_flags'] += [False] 71 | 72 | return example 73 | 74 | def verify_code_official(tests, solution, debug=False, return_output=False): 75 | ''' verify if code passes all tests, using apps official implementation (https://github.com/hendrycks/apps/blob/main/eval/testing_util.py#L122) 76 | ''' 77 | tests = deepcopy(tests) 78 | # suppress the stdout of solution execution 79 | # todo: suppress stderr as well 80 | with contextlib.redirect_stdout(io.StringIO()): 81 | results = run_test(tests, solution, debug=debug, return_output=return_output) 82 | if return_output: 83 | all_outputs = results[1] 84 | results = results[0] 85 | if all([res == True for res in results]): 86 | if return_output: 87 | return True, results, all_outputs 88 | return True, results 89 | else: 90 | if return_output: 91 | return False, results, all_outputs 92 | return False, results 93 | 94 | 95 | def estimate_pass_at_k( 96 | num_samples: Union[int, List[int], np.ndarray], 97 | num_correct: Union[List[int], np.ndarray], 98 | k: int, 99 | ) -> np.ndarray: 100 | """ 101 | Estimates pass@k of each problem and returns them in an array. 102 | Taken from https://github.com/openai/human-eval/blob/master/human_eval/evaluation.py#L13. 103 | """ 104 | def estimator(n: int, c: int, k: int) -> float: 105 | """ 106 | Calculates 1 - comb(n - c, k) / comb(n, k). 107 | """ 108 | if n - c < k: 109 | return 1.0 110 | return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1)) 111 | 112 | if isinstance(num_samples, int): 113 | num_samples_it = itertools.repeat(num_samples, len(num_correct)) 114 | else: 115 | assert len(num_samples) == len(num_correct) 116 | num_samples_it = iter(num_samples) 117 | 118 | return np.array( 119 | [estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)] 120 | ) 121 | 122 | def compute_pass_at_ks(results, ks): 123 | output = { 124 | k: estimate_pass_at_k( 125 | [len(x) for x in results], 126 | [sum([i == True for i in x]) for x in results], 127 | k, 128 | ).mean() 129 | for k in ks 130 | } 131 | return output 132 | --------------------------------------------------------------------------------