├── .github ├── README.md └── workflows │ ├── build-jetbrains.yml │ ├── release-jetbrains.yml │ └── release-vsc.yml ├── .gitignore ├── LICENSE.md ├── code4me-jetbrains-plugin ├── .run │ ├── Run IDE for UI Tests.run.xml │ ├── Run IDE with Plugin.run.xml │ ├── Run Plugin Tests.run.xml │ ├── Run Plugin Verification.run.xml │ └── Run Qodana.run.xml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── build.gradle.kts ├── gradle.properties ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle.kts └── src │ └── main │ ├── java │ └── me │ │ └── code4me │ │ └── plugin │ │ ├── Code4MeBundle.java │ │ ├── Code4MeIcons.java │ │ ├── Code4MeScheduledThreadPoolExecutor.java │ │ ├── actions │ │ └── CodeCompleteAction.java │ │ ├── api │ │ ├── Code4MeErrorResponse.java │ │ ├── Code4MeResponse.java │ │ ├── PredictionAutocompleteRequest.java │ │ ├── PredictionAutocompleteResponse.java │ │ ├── PredictionVerifyRequest.java │ │ └── PredictionVerifyResponse.java │ │ ├── completions │ │ └── Code4MeCompletionContributor.java │ │ ├── dialogs │ │ └── Code4MeDialogWrapper.java │ │ ├── exceptions │ │ ├── AlreadyAutocompletingException.java │ │ └── ApiServerException.java │ │ ├── listeners │ │ ├── Code4MeDocumentListener.java │ │ ├── Code4MeDynamicPluginListener.java │ │ └── Code4MeProjectManagerListener.java │ │ ├── services │ │ ├── Code4MeApiService.java │ │ ├── Code4MeDocumentListenerService.java │ │ ├── Code4MeSettingsService.java │ │ └── Code4MeTriggerPointsService.java │ │ └── util │ │ └── Code4MeUtil.java │ └── resources │ ├── META-INF │ ├── plugin.xml │ └── pluginIcon.svg │ ├── icons │ └── pluginIcon.svg │ ├── messages │ └── Code4MeBundle.properties │ └── triggerPoints.json ├── code4me-server ├── Dockerfile ├── README.md ├── markdowns │ └── index.md ├── requirements.txt ├── src │ ├── api.py │ ├── app.py │ ├── codegpt.py │ ├── evaluation.py │ ├── incoder.py │ ├── limiter.py │ ├── model.py │ ├── modeling_jonberta.py │ ├── query_filter.py │ ├── unixcoder.py │ ├── unixcoder_wrapper.py │ ├── user_study.py │ └── util.py ├── static │ ├── android-chrome-192x192.png │ ├── android-chrome-512x512.png │ ├── apple-touch-icon.png │ ├── browserconfig.xml │ ├── css │ │ └── retro.css │ ├── favicon-16x16.png │ ├── favicon-32x32.png │ ├── favicon.ico │ ├── fonts │ │ └── JetBrainsMono-Regular.woff2 │ ├── mstile-144x144.png │ ├── mstile-150x150.png │ ├── mstile-310x150.png │ ├── mstile-310x310.png │ ├── mstile-70x70.png │ ├── safari-pinned-tab.svg │ └── site.webmanifest └── templates │ └── index.html ├── code4me-vsc-plugin ├── .eslintignore ├── .eslintrc.js ├── .gitignore ├── .vscode │ ├── launch.json │ └── tasks.json ├── .vscodeignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── package-lock.json ├── package.json ├── pluginIcon.png ├── src │ └── extension.ts └── tsconfig.json └── docker-compose.yml /.github/README.md: -------------------------------------------------------------------------------- 1 | ../code4me-server/markdowns/index.md -------------------------------------------------------------------------------- /.github/workflows/build-jetbrains.yml: -------------------------------------------------------------------------------- 1 | # GitHub Actions Workflow is created for testing and preparing the plugin release in the following steps: 2 | # - validate Gradle Wrapper, 3 | # - run 'test' and 'verifyPlugin' tasks, 4 | # - run Qodana inspections, 5 | # - run 'buildPlugin' task and prepare artifact for the further tests, 6 | # - run 'runPluginVerifier' task, 7 | # - create a draft release. 8 | # 9 | # Workflow is triggered on push and pull_request events. 10 | # 11 | # GitHub Actions reference: https://help.github.com/en/actions 12 | # 13 | ## JBIJPPTPL 14 | 15 | name: Build JetBrains Plugin 16 | on: 17 | # Trigger the workflow on pushes to only the 'main' branch (this avoids duplicate checks being run e.g. for dependabot pull requests) 18 | push: 19 | branches: [main] 20 | paths: 21 | - code4me-jetbrains-plugin/** 22 | # Trigger the workflow on any pull request 23 | pull_request: 24 | paths: 25 | - code4me-jetbrains-plugin/** 26 | 27 | env: 28 | WORKING_DIRECTORY: code4me-jetbrains-plugin 29 | 30 | jobs: 31 | 32 | # Run Gradle Wrapper Validation Action to verify the wrapper's checksum 33 | # Run verifyPlugin, IntelliJ Plugin Verifier, and test Gradle tasks 34 | # Build plugin and provide the artifact for the next workflow jobs 35 | build: 36 | name: Build 37 | runs-on: ubuntu-latest 38 | outputs: 39 | version: ${{ steps.properties.outputs.version }} 40 | changelog: ${{ steps.properties.outputs.changelog }} 41 | steps: 42 | 43 | # Free GitHub Actions Environment Disk Space 44 | - name: Maximize Build Space 45 | run: | 46 | sudo rm -rf /usr/share/dotnet 47 | sudo rm -rf /usr/local/lib/android 48 | sudo rm -rf /opt/ghc 49 | # Check out current repository 50 | - name: Fetch Sources 51 | uses: actions/checkout@v3 52 | 53 | # Validate wrapper 54 | - name: Gradle Wrapper Validation 55 | uses: gradle/wrapper-validation-action@v1.0.5 56 | 57 | # Setup Java 11 environment for the next steps 58 | - name: Setup Java 59 | uses: actions/setup-java@v3 60 | with: 61 | distribution: zulu 62 | java-version: 11 63 | cache: gradle 64 | 65 | # Set environment variables 66 | - name: Export Properties 67 | id: properties 68 | shell: bash 69 | working-directory: ${{ env.WORKING_DIRECTORY }} 70 | run: | 71 | PROPERTIES="$(./gradlew properties --console=plain -q)" 72 | VERSION="$(echo "$PROPERTIES" | grep "^version:" | cut -f2- -d ' ')" 73 | NAME="$(echo "$PROPERTIES" | grep "^pluginName:" | cut -f2- -d ' ')" 74 | CHANGELOG="$(./gradlew getChangelog --unreleased --no-header --console=plain -q)" 75 | echo "version=$VERSION" >> $GITHUB_OUTPUT 76 | echo "name=$NAME" >> $GITHUB_OUTPUT 77 | echo "pluginVerifierHomeDir=~/.pluginVerifier" >> $GITHUB_OUTPUT 78 | 79 | echo "changelog<> $GITHUB_OUTPUT 80 | echo "$CHANGELOG" >> $GITHUB_OUTPUT 81 | echo "EOF" >> $GITHUB_OUTPUT 82 | ./gradlew listProductsReleases # prepare list of IDEs for Plugin Verifier 83 | 84 | # Run tests 85 | - name: Run Tests 86 | working-directory: ${{ env.WORKING_DIRECTORY }} 87 | run: ./gradlew check 88 | 89 | # Collect Tests Result of failed tests 90 | - name: Collect Tests Result 91 | if: ${{ failure() }} 92 | uses: actions/upload-artifact@v3 93 | with: 94 | name: tests-result 95 | path: ${{ github.workspace }}/${{ env.WORKING_DIRECTORY }}/build/reports/tests 96 | 97 | # Cache Plugin Verifier IDEs 98 | - name: Setup Plugin Verifier IDEs Cache 99 | uses: actions/cache@v3 100 | with: 101 | path: ${{ steps.properties.outputs.pluginVerifierHomeDir }}/ides 102 | key: ${{ format('plugin-verifier-{0}-{1}', hashFiles(format('{0}/build/listProductsReleases.txt', env.WORKING_DIRECTORY)), vars.CACHE_KEY) }} 103 | 104 | # Run Verify Plugin task and IntelliJ Plugin Verifier tool 105 | - name: Run Plugin Verification tasks 106 | working-directory: ${{ env.WORKING_DIRECTORY }} 107 | run: ./gradlew runPluginVerifier -Pplugin.verifier.home.dir=${{ steps.properties.outputs.pluginVerifierHomeDir }} 108 | 109 | # Collect Plugin Verifier Result 110 | - name: Collect Plugin Verifier Result 111 | if: ${{ always() }} 112 | uses: actions/upload-artifact@v3 113 | with: 114 | name: pluginVerifier-result 115 | path: ${{ github.workspace }}/${{ env.WORKING_DIRECTORY }}/build/reports/pluginVerifier 116 | 117 | # Prepare plugin archive content for creating artifact 118 | - name: Prepare Plugin Artifact 119 | id: artifact 120 | shell: bash 121 | run: | 122 | cd ${{ github.workspace }}/${{ env.WORKING_DIRECTORY }}/build/distributions 123 | FILENAME=`ls *.zip` 124 | unzip "$FILENAME" -d content 125 | echo "filename=${FILENAME:0:-4}" >> $GITHUB_OUTPUT 126 | # Store already-built plugin as an artifact for downloading 127 | - name: Upload artifact 128 | uses: actions/upload-artifact@v3 129 | with: 130 | name: ${{ steps.artifact.outputs.filename }} 131 | path: ${{ github.workspace }}/${{ env.WORKING_DIRECTORY }}/build/distributions/content/*/* 132 | 133 | # Prepare a draft release for GitHub Releases page for the manual verification 134 | # If accepted and published, release workflow would be triggered 135 | releaseDraft: 136 | name: Release Draft 137 | if: github.event_name != 'pull_request' 138 | needs: build 139 | runs-on: ubuntu-latest 140 | permissions: 141 | contents: write 142 | steps: 143 | 144 | # Check out current repository 145 | - name: Fetch Sources 146 | uses: actions/checkout@v3 147 | 148 | # Remove old release drafts by using the curl request for the available releases with a draft flag 149 | - name: Remove Old Release Drafts 150 | env: 151 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 152 | working-directory: ${{ env.WORKING_DIRECTORY }} 153 | run: | 154 | gh api repos/{owner}/{repo}/releases \ 155 | --jq '.[] | select(.draft == true) | .id' \ 156 | | xargs -I '{}' gh api -X DELETE repos/{owner}/{repo}/releases/{} 157 | # Create a new release draft which is not publicly visible and requires manual acceptance 158 | - name: Create Release Draft 159 | env: 160 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 161 | working-directory: ${{ env.WORKING_DIRECTORY }} 162 | run: | 163 | gh release create v${{ needs.build.outputs.version }} \ 164 | --draft \ 165 | --title "v${{ needs.build.outputs.version }}" \ 166 | --notes "$(cat << 'EOM' 167 | ${{ needs.build.outputs.changelog }} 168 | EOM 169 | )" 170 | -------------------------------------------------------------------------------- /.github/workflows/release-jetbrains.yml: -------------------------------------------------------------------------------- 1 | # GitHub Actions Workflow created for handling the release process based on the draft release prepared with the Build workflow. 2 | # Running the publishPlugin task requires all following secrets to be provided: PUBLISH_TOKEN, PRIVATE_KEY, PRIVATE_KEY_PASSWORD, CERTIFICATE_CHAIN. 3 | # See https://plugins.jetbrains.com/docs/intellij/plugin-signing.html for more information. 4 | 5 | name: Release JetBrains Plugin 6 | on: 7 | release: 8 | types: [prereleased, released] 9 | 10 | env: 11 | WORKING_DIRECTORY: code4me-jetbrains-plugin 12 | 13 | jobs: 14 | 15 | # Prepare and publish the plugin to the Marketplace repository 16 | release: 17 | name: Publish Plugin 18 | runs-on: ubuntu-latest 19 | permissions: 20 | contents: write 21 | pull-requests: write 22 | steps: 23 | 24 | # Check out current repository 25 | - name: Fetch Sources 26 | uses: actions/checkout@v3 27 | with: 28 | ref: ${{ github.event.release.tag_name }} 29 | 30 | # Setup Java 11 environment for the next steps 31 | - name: Setup Java 32 | uses: actions/setup-java@v3 33 | with: 34 | distribution: zulu 35 | java-version: 11 36 | cache: gradle 37 | 38 | # Set environment variables 39 | - name: Export Properties 40 | id: properties 41 | shell: bash 42 | working-directory: ${{ env.WORKING_DIRECTORY }} 43 | run: | 44 | CHANGELOG="$(cat << 'EOM' | sed -e 's/^[[:space:]]*$//g' -e '/./,$!d' 45 | ${{ github.event.release.body }} 46 | EOM 47 | )" 48 | 49 | CHANGELOG="${CHANGELOG//'%'/'%25'}" 50 | CHANGELOG="${CHANGELOG//$'\n'/'%0A'}" 51 | CHANGELOG="${CHANGELOG//$'\r'/'%0D'}" 52 | echo "changelog=$CHANGELOG" >> $GITHUB_OUTPUT 53 | # Update Unreleased section with the current release note 54 | - name: Patch Changelog 55 | if: ${{ steps.properties.outputs.changelog != '' }} 56 | env: 57 | CHANGELOG: ${{ steps.properties.outputs.changelog }} 58 | working-directory: ${{ env.WORKING_DIRECTORY }} 59 | run: | 60 | ./gradlew patchChangelog --release-note="$CHANGELOG" 61 | 62 | # Publish the plugin to the Marketplace 63 | - name: Publish Plugin 64 | env: 65 | PUBLISH_TOKEN: ${{ secrets.PUBLISH_TOKEN }} 66 | CERTIFICATE_CHAIN: ${{ secrets.CERTIFICATE_CHAIN }} 67 | PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} 68 | PRIVATE_KEY_PASSWORD: ${{ secrets.PRIVATE_KEY_PASSWORD }} 69 | working-directory: ${{ env.WORKING_DIRECTORY }} 70 | run: ./gradlew clean publishPlugin 71 | 72 | # Upload artifact as a release asset 73 | - name: Upload Release Asset 74 | env: 75 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 76 | working-directory: ${{ env.WORKING_DIRECTORY }} 77 | run: gh release upload ${{ github.event.release.tag_name }} ./build/distributions/* 78 | 79 | # Create pull request 80 | - name: Create Pull Request 81 | if: ${{ steps.properties.outputs.changelog != '' }} 82 | env: 83 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 84 | working-directory: ${{ env.WORKING_DIRECTORY }} 85 | run: | 86 | VERSION="${{ github.event.release.tag_name }}" 87 | BRANCH="changelog-update-$VERSION" 88 | LABEL="release changelog" 89 | git config user.email "action@github.com" 90 | git config user.name "GitHub Action" 91 | git checkout -b $BRANCH 92 | git commit -am "Changelog update - $VERSION" 93 | git push --set-upstream origin $BRANCH 94 | 95 | gh label create "$LABEL" \ 96 | --description "Pull requests with release changelog update" \ 97 | || true 98 | gh pr create \ 99 | --title "Changelog update - \`$VERSION\`" \ 100 | --body "Current pull request contains patched \`CHANGELOG.md\` file for the \`$VERSION\` version." \ 101 | --label "$LABEL" \ 102 | --head $BRANCH 103 | -------------------------------------------------------------------------------- /.github/workflows/release-vsc.yml: -------------------------------------------------------------------------------- 1 | name: Release VSC Plugin 2 | on: 3 | push: 4 | branches: [main] 5 | paths: 6 | - code4me-vsc-plugin/** 7 | 8 | env: 9 | WORKING_DIRECTORY: code4me-vsc-plugin 10 | 11 | jobs: 12 | release: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - name: Check Commit Message 17 | id: check-commit 18 | run: | 19 | if [[ "${{ github.event.head_commit.message }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(\-[a-zA-Z]+)?$ ]]; then 20 | echo ::set-output name=isSemver::true 21 | fi 22 | - uses: actions/checkout@v3 23 | if: steps.check-commit.outputs.isSemver == 'true' 24 | - uses: actions/setup-node@v3 25 | if: steps.check-commit.outputs.isSemver == 'true' 26 | with: 27 | node-version: 16 28 | cache: 'npm' 29 | cache-dependency-path: code4me-vsc-plugin/package-lock.json 30 | 31 | - name: Install the dependencies 32 | if: steps.check-commit.outputs.isSemver == 'true' 33 | working-directory: ${{ env.WORKING_DIRECTORY }} 34 | run: npm i 35 | 36 | - name: Install vsce 37 | if: steps.check-commit.outputs.isSemver == 'true' 38 | working-directory: ${{ env.WORKING_DIRECTORY }} 39 | run: npm i -g vsce 40 | 41 | - name: Publish 42 | if: steps.check-commit.outputs.isSemver == 'true' 43 | working-directory: ${{ env.WORKING_DIRECTORY }} 44 | run: vsce publish -p ${{ secrets.VSCE_PAT }} 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .gradle 2 | .idea 3 | build 4 | .java-version 5 | venv 6 | __pycache__ 7 | users*.json 8 | data_aral 9 | 10 | models 11 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/.run/Run IDE for UI Tests.run.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 12 | 15 | 17 | true 18 | true 19 | false 20 | 21 | 22 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/.run/Run IDE with Plugin.run.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 12 | 17 | 19 | true 20 | true 21 | false 22 | 23 | 24 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/.run/Run Plugin Tests.run.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 12 | 17 | 19 | true 20 | true 21 | false 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/.run/Run Plugin Verification.run.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 12 | 17 | 19 | true 20 | true 21 | false 22 | 23 | 25 | 26 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/.run/Run Qodana.run.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 9 | 16 | 19 | 21 | true 22 | true 23 | false 24 | 25 | 26 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Code4Me Changelog 4 | 5 | ## [Unreleased] 6 | 7 | ### Changed 8 | - Update version for new jetbrains release 9 | 10 | ## [1.0.13] 11 | 12 | ### Changed 13 | - Update dependencies for new jetbrains release 14 | 15 | ## [1.0.12] 16 | 17 | ### Changed 18 | - Update build configurations 19 | 20 | ## [1.0.11] 21 | 22 | ### Changed 23 | - Allowing storing of context 24 | - Increase max context length 25 | - Fixed redundant bracket when completing 26 | 27 | ## [1.0.10] 28 | 29 | ### Changed 30 | - Send plugin version to server 31 | - Add support for unknown trigger points 32 | - Added except keyword 33 | 34 | ## [1.0.9] 35 | 36 | ### Changed 37 | - Increased max context length 38 | 39 | ## [1.0.8] 40 | 41 | ### Added 42 | - Survey 43 | - UniXCoder 44 | 45 | ## [1.0.7] 46 | 47 | ### Added 48 | - Latest IJ environment 49 | 50 | ### Fixed 51 | - Dynamic plugin reload 52 | 53 | ## [1.0.6] 54 | 55 | ### Added 56 | - Extra trigger points 57 | - Multiple prediction support 58 | 59 | ### Fixed 60 | - Large document completion 61 | - API consistency 62 | - Pycharm CE compatibility 63 | - InCoder suggestion 64 | 65 | ## [1.0.5-beta] 66 | 67 | ### Added 68 | - Changelog 69 | 70 | ### Fixed 71 | - Updated vendor 72 | 73 | [Unreleased]: https://github.com/code4me-me/code4me/compare/vvalueof(GradlePropertyValueSource)...HEAD 74 | 75 | [1.0.12]: https://github.com/code4me-me/code4me/compare/v1.0.11...v1.0.12 76 | 77 | [1.0.11]: https://github.com/code4me-me/code4me/compare/v1.0.10...v1.0.11 78 | 79 | [1.0.10]: https://github.com/code4me-me/code4me/compare/v1.0.9...v1.0.10 80 | 81 | [1.0.9]: https://github.com/code4me-me/code4me/compare/v1.0.8...v1.0.9 82 | 83 | [1.0.8]: https://github.com/code4me-me/code4me/compare/v1.0.7...v1.0.8 84 | 85 | [1.0.7]: https://github.com/code4me-me/code4me/compare/v1.0.6...v1.0.7 86 | 87 | [1.0.6]: https://github.com/code4me-me/code4me/compare/v1.0.5-beta...v1.0.6 88 | 89 | [1.0.5-beta]: https://github.com/code4me-me/code4me/commits/v1.0.5-beta 90 | 91 | [valueof(GradlePropertyValueSource)]: https://github.com/code4me-me/code4me/compare/v1.0.12...vvalueof(GradlePropertyValueSource) 92 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2000-2021 JetBrains s.r.o. 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/README.md: -------------------------------------------------------------------------------- 1 | # IntelliJ Platform Code4Me Plugin 2 | 3 | 4 | **Code4Me** is a plugin that autocompletes code in a TU Delft research project for the CSE3000 course given in the Computer Science and Engineering bachelor. 5 | 6 | The plugin has triggerpoints on which it will suggest autocompletions, and a keybind which will trigger the autocompletion wherever you want. 7 | 8 | For more information, visit the [Code4Me](https://code4me.me) website. 9 | 10 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/build.gradle.kts: -------------------------------------------------------------------------------- 1 | import org.jetbrains.changelog.Changelog 2 | import org.jetbrains.changelog.markdownToHTML 3 | 4 | fun properties(key: String) = providers.gradleProperty(key) 5 | fun environment(key: String) = providers.environmentVariable(key) 6 | 7 | plugins { 8 | // Java support 9 | id("java") 10 | // Kotlin support 11 | id("org.jetbrains.kotlin.jvm") version "1.8.10" 12 | // Gradle IntelliJ Plugin 13 | id("org.jetbrains.intellij") version "1.13.0" 14 | // Gradle Changelog Plugin 15 | id("org.jetbrains.changelog") version "2.0.0" 16 | } 17 | 18 | group = properties("pluginGroup").get() 19 | version = properties("pluginVersion").get() 20 | 21 | // Configure project's dependencies 22 | repositories { 23 | mavenCentral() 24 | } 25 | 26 | // Configure Gradle IntelliJ Plugin - read more: https://plugins.jetbrains.com/docs/intellij/tools-gradle-intellij-plugin.html 27 | intellij { 28 | pluginName.set(properties("pluginName")) 29 | version.set(properties("platformVersion")) 30 | type.set(properties("platformType")) 31 | 32 | // Plugin Dependencies. Uses `platformPlugins` property from the gradle.properties file. 33 | plugins.set(properties("platformPlugins").map { it.split(',').map(String::trim).filter(String::isNotEmpty) }) 34 | } 35 | 36 | // Configure Gradle Changelog Plugin - read more: https://github.com/JetBrains/gradle-changelog-plugin 37 | changelog { 38 | groups.empty() 39 | repositoryUrl.set(properties("pluginRepositoryUrl")) 40 | } 41 | 42 | tasks { 43 | // Set the JVM compatibility versions 44 | properties("javaVersion").get().let { 45 | withType { 46 | sourceCompatibility = it 47 | targetCompatibility = it 48 | } 49 | } 50 | 51 | wrapper { 52 | gradleVersion = properties("gradleVersion").get() 53 | } 54 | 55 | patchPluginXml { 56 | version.set(properties("pluginVersion")) 57 | sinceBuild.set(properties("pluginSinceBuild")) 58 | untilBuild.set(properties("pluginUntilBuild")) 59 | 60 | // Extract the section from README.md and provide for the plugin's manifest 61 | pluginDescription.set(providers.fileContents(layout.projectDirectory.file("README.md")).asText.map { 62 | val start = "" 63 | val end = "" 64 | 65 | with (it.lines()) { 66 | if (!containsAll(listOf(start, end))) { 67 | throw GradleException("Plugin description section not found in README.md:\n$start ... $end") 68 | } 69 | subList(indexOf(start) + 1, indexOf(end)).joinToString("\n").let(::markdownToHTML) 70 | } 71 | }) 72 | 73 | val changelog = project.changelog // local variable for configuration cache compatibility 74 | // Get the latest available change notes from the changelog file 75 | changeNotes.set(properties("pluginVersion").map { pluginVersion -> 76 | with(changelog) { 77 | renderItem( 78 | getOrNull(pluginVersion) 79 | ?: runCatching { getLatest() }.getOrElse { getUnreleased() }, 80 | Changelog.OutputType.HTML, 81 | ) 82 | } 83 | }) 84 | } 85 | 86 | signPlugin { 87 | certificateChain.set(environment("CERTIFICATE_CHAIN")) 88 | privateKey.set(environment("PRIVATE_KEY")) 89 | password.set(environment("PRIVATE_KEY_PASSWORD")) 90 | } 91 | 92 | publishPlugin { 93 | dependsOn("patchChangelog") 94 | token.set(environment("PUBLISH_TOKEN")) 95 | // The pluginVersion is based on the SemVer (https://semver.org) and supports pre-release labels, like 2.1.7-alpha.3 96 | // Specify pre-release label to publish the plugin in a custom Release Channel automatically. Read more: 97 | // https://plugins.jetbrains.com/docs/intellij/deployment.html#specifying-a-release-channel 98 | channels.set(properties("pluginVersion").map { listOf(it.split('-').getOrElse(1) { "default" }.split('.').first()) }) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/gradle.properties: -------------------------------------------------------------------------------- 1 | # IntelliJ Platform Artifacts Repositories -> https://plugins.jetbrains.com/docs/intellij/intellij-artifacts.html 2 | 3 | pluginGroup = me.code4me.plugin 4 | pluginName = Code4Me 5 | pluginRepositoryUrl = https://github.com/code4me-me/code4me 6 | # SemVer format -> https://semver.org 7 | pluginVersion = 1.0.14 8 | 9 | # Supported build number ranges and IntelliJ Platform versions -> https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html 10 | pluginSinceBuild = 212 11 | pluginUntilBuild = 232.* 12 | 13 | # IntelliJ Platform Properties -> https://plugins.jetbrains.com/docs/intellij/tools-gradle-intellij-plugin.html#configuration-intellij-extension 14 | platformType = PY 15 | platformVersion = 2023.2.1 16 | 17 | # Plugin Dependencies -> https://plugins.jetbrains.com/docs/intellij/plugin-dependencies.html 18 | # Example: platformPlugins = com.intellij.java, com.jetbrains.php:203.4449.22 19 | platformPlugins = 20 | 21 | # Java language level used to compile sources and to generate the files for - Java 11 is required since 2020.3 22 | javaVersion = 11 23 | 24 | # Gradle Releases -> https://github.com/gradle/gradle/releases 25 | gradleVersion = 7.6 26 | 27 | # Opt-out flag for bundling Kotlin standard library -> https://plugins.jetbrains.com/docs/intellij/kotlin.html#kotlin-standard-library 28 | # suppress inspection "UnusedProperty" 29 | kotlin.stdlib.default.dependency = false 30 | 31 | # Enable Gradle Configuration Cache -> https://docs.gradle.org/current/userguide/configuration_cache.html 32 | # suppress inspection "UnusedProperty" 33 | org.gradle.unsafe.configuration-cache = true 34 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code4me-me/code4me/d8429d41c42ac16957659b95380908458a7c2980/code4me-jetbrains-plugin/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip 4 | networkTimeout=10000 5 | zipStoreBase=GRADLE_USER_HOME 6 | zipStorePath=wrapper/dists 7 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | # This is normally unused 84 | # shellcheck disable=SC2034 85 | APP_BASE_NAME=${0##*/} 86 | APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit 87 | 88 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 89 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 90 | 91 | # Use the maximum available, or set MAX_FD != -1 to use that value. 92 | MAX_FD=maximum 93 | 94 | warn () { 95 | echo "$*" 96 | } >&2 97 | 98 | die () { 99 | echo 100 | echo "$*" 101 | echo 102 | exit 1 103 | } >&2 104 | 105 | # OS specific support (must be 'true' or 'false'). 106 | cygwin=false 107 | msys=false 108 | darwin=false 109 | nonstop=false 110 | case "$( uname )" in #( 111 | CYGWIN* ) cygwin=true ;; #( 112 | Darwin* ) darwin=true ;; #( 113 | MSYS* | MINGW* ) msys=true ;; #( 114 | NONSTOP* ) nonstop=true ;; 115 | esac 116 | 117 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 118 | 119 | 120 | # Determine the Java command to use to start the JVM. 121 | if [ -n "$JAVA_HOME" ] ; then 122 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 123 | # IBM's JDK on AIX uses strange locations for the executables 124 | JAVACMD=$JAVA_HOME/jre/sh/java 125 | else 126 | JAVACMD=$JAVA_HOME/bin/java 127 | fi 128 | if [ ! -x "$JAVACMD" ] ; then 129 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 130 | 131 | Please set the JAVA_HOME variable in your environment to match the 132 | location of your Java installation." 133 | fi 134 | else 135 | JAVACMD=java 136 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | 142 | # Increase the maximum file descriptors if we can. 143 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 144 | case $MAX_FD in #( 145 | max*) 146 | # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. 147 | # shellcheck disable=SC3045 148 | MAX_FD=$( ulimit -H -n ) || 149 | warn "Could not query maximum file descriptor limit" 150 | esac 151 | case $MAX_FD in #( 152 | '' | soft) :;; #( 153 | *) 154 | # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. 155 | # shellcheck disable=SC3045 156 | ulimit -n "$MAX_FD" || 157 | warn "Could not set maximum file descriptor limit to $MAX_FD" 158 | esac 159 | fi 160 | 161 | # Collect all arguments for the java command, stacking in reverse order: 162 | # * args from the command line 163 | # * the main class name 164 | # * -classpath 165 | # * -D...appname settings 166 | # * --module-path (only if needed) 167 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 168 | 169 | # For Cygwin or MSYS, switch paths to Windows format before running java 170 | if "$cygwin" || "$msys" ; then 171 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 172 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 173 | 174 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 175 | 176 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 177 | for arg do 178 | if 179 | case $arg in #( 180 | -*) false ;; # don't mess with options #( 181 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 182 | [ -e "$t" ] ;; #( 183 | *) false ;; 184 | esac 185 | then 186 | arg=$( cygpath --path --ignore --mixed "$arg" ) 187 | fi 188 | # Roll the args list around exactly as many times as the number of 189 | # args, so each arg winds up back in the position where it started, but 190 | # possibly modified. 191 | # 192 | # NB: a `for` loop captures its iteration list before it begins, so 193 | # changing the positional parameters here affects neither the number of 194 | # iterations, nor the values presented in `arg`. 195 | shift # remove old arg 196 | set -- "$@" "$arg" # push replacement arg 197 | done 198 | fi 199 | 200 | # Collect all arguments for the java command; 201 | # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of 202 | # shell script including quotes and variable substitutions, so put them in 203 | # double quotes to make sure that they get re-expanded; and 204 | # * put everything else in single quotes, so that it's not re-expanded. 205 | 206 | set -- \ 207 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 208 | -classpath "$CLASSPATH" \ 209 | org.gradle.wrapper.GradleWrapperMain \ 210 | "$@" 211 | 212 | # Stop when "xargs" is not available. 213 | if ! command -v xargs >/dev/null 2>&1 214 | then 215 | die "xargs is not available" 216 | fi 217 | 218 | # Use "xargs" to parse quoted args. 219 | # 220 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 221 | # 222 | # In Bash we could simply go: 223 | # 224 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 225 | # set -- "${ARGS[@]}" "$@" 226 | # 227 | # but POSIX shell has neither arrays nor command substitution, so instead we 228 | # post-process each arg (as a line of input to sed) to backslash-escape any 229 | # character that might be a shell metacharacter, then use eval to reverse 230 | # that process (while maintaining the separation between arguments), and wrap 231 | # the whole thing up as a single "set" statement. 232 | # 233 | # This will of course break if any of these variables contains a newline or 234 | # an unmatched quote. 235 | # 236 | 237 | eval "set -- $( 238 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 239 | xargs -n1 | 240 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 241 | tr '\n' ' ' 242 | )" '"$@"' 243 | 244 | exec "$JAVACMD" "$@" 245 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%"=="" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%"=="" set DIRNAME=. 29 | @rem This is normally unused 30 | set APP_BASE_NAME=%~n0 31 | set APP_HOME=%DIRNAME% 32 | 33 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 34 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 35 | 36 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 37 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 38 | 39 | @rem Find java.exe 40 | if defined JAVA_HOME goto findJavaFromJavaHome 41 | 42 | set JAVA_EXE=java.exe 43 | %JAVA_EXE% -version >NUL 2>&1 44 | if %ERRORLEVEL% equ 0 goto execute 45 | 46 | echo. 47 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 48 | echo. 49 | echo Please set the JAVA_HOME variable in your environment to match the 50 | echo location of your Java installation. 51 | 52 | goto fail 53 | 54 | :findJavaFromJavaHome 55 | set JAVA_HOME=%JAVA_HOME:"=% 56 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 57 | 58 | if exist "%JAVA_EXE%" goto execute 59 | 60 | echo. 61 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 62 | echo. 63 | echo Please set the JAVA_HOME variable in your environment to match the 64 | echo location of your Java installation. 65 | 66 | goto fail 67 | 68 | :execute 69 | @rem Setup the command line 70 | 71 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 72 | 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if %ERRORLEVEL% equ 0 goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | set EXIT_CODE=%ERRORLEVEL% 85 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 86 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 87 | exit /b %EXIT_CODE% 88 | 89 | :mainEnd 90 | if "%OS%"=="Windows_NT" endlocal 91 | 92 | :omega 93 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/settings.gradle.kts: -------------------------------------------------------------------------------- 1 | rootProject.name = "Code4Me" 2 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/Code4MeBundle.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin; 2 | 3 | import com.intellij.DynamicBundle; 4 | import org.jetbrains.annotations.Nls; 5 | import org.jetbrains.annotations.NotNull; 6 | import org.jetbrains.annotations.PropertyKey; 7 | 8 | import java.util.function.Supplier; 9 | 10 | public class Code4MeBundle extends DynamicBundle { 11 | 12 | private static final String PATH_TO_BUNDLE = "messages.Code4MeBundle"; 13 | private static final Code4MeBundle instance = new Code4MeBundle(); 14 | 15 | public static @Nls String message( 16 | @NotNull @PropertyKey(resourceBundle = PATH_TO_BUNDLE) String key, 17 | Object... params 18 | ) { 19 | return instance.getMessage(key, params); 20 | } 21 | 22 | @NotNull 23 | public static Supplier messagePointer( 24 | @NotNull @PropertyKey(resourceBundle = PATH_TO_BUNDLE) String key, 25 | Object... params 26 | ) { 27 | return instance.getLazyMessage(key, params); 28 | } 29 | 30 | private Code4MeBundle() { 31 | super(PATH_TO_BUNDLE); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/Code4MeIcons.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin; 2 | 3 | import com.intellij.openapi.util.IconLoader; 4 | 5 | import javax.swing.Icon; 6 | 7 | public interface Code4MeIcons { 8 | Icon PLUGIN_ICON = IconLoader.getIcon("/icons/pluginIcon.svg", Code4MeIcons.class); 9 | } 10 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/Code4MeScheduledThreadPoolExecutor.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.concurrent.Callable; 6 | import java.util.concurrent.ScheduledFuture; 7 | import java.util.concurrent.ScheduledThreadPoolExecutor; 8 | import java.util.concurrent.TimeUnit; 9 | 10 | public class Code4MeScheduledThreadPoolExecutor extends ScheduledThreadPoolExecutor { 11 | 12 | private static final Code4MeScheduledThreadPoolExecutor instance = new Code4MeScheduledThreadPoolExecutor(); 13 | 14 | public static Code4MeScheduledThreadPoolExecutor getInstance() { 15 | return instance; 16 | } 17 | 18 | private Code4MeScheduledThreadPoolExecutor() { 19 | super(1); 20 | } 21 | 22 | @Override 23 | public ScheduledFuture schedule(@NotNull Runnable command, long delay, @NotNull TimeUnit unit) { 24 | return super.schedule(() -> { 25 | try { 26 | command.run(); 27 | } catch (Throwable th) { 28 | th.printStackTrace(); 29 | throw th; 30 | } 31 | }, delay, unit); 32 | } 33 | 34 | @Override 35 | public ScheduledFuture schedule(@NotNull Callable callable, long delay, @NotNull TimeUnit unit) { 36 | return super.schedule(() -> { 37 | try { 38 | return callable.call(); 39 | } catch (Throwable th) { 40 | th.printStackTrace(); 41 | throw th; 42 | } 43 | }, delay, unit); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/actions/CodeCompleteAction.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.actions; 2 | 3 | import com.intellij.openapi.actionSystem.AnAction; 4 | import com.intellij.openapi.actionSystem.AnActionEvent; 5 | import com.intellij.openapi.editor.CaretModel; 6 | import com.intellij.openapi.editor.Document; 7 | import com.intellij.openapi.editor.Editor; 8 | import com.intellij.openapi.fileEditor.FileEditorManager; 9 | import com.intellij.openapi.project.Project; 10 | import me.code4me.plugin.completions.Code4MeCompletionContributor; 11 | import org.jetbrains.annotations.NotNull; 12 | 13 | public class CodeCompleteAction extends AnAction { 14 | 15 | @Override 16 | public void actionPerformed(@NotNull AnActionEvent event) { 17 | Project project = event.getProject(); 18 | if (project == null) return; 19 | 20 | Editor editor = FileEditorManager.getInstance(event.getProject()).getSelectedTextEditor(); 21 | if (editor == null) return; 22 | 23 | Document doc = editor.getDocument(); 24 | String text = doc.getText(); 25 | CaretModel caretModel = editor.getCaretModel(); 26 | int offset = caretModel.getOffset(); 27 | 28 | Code4MeCompletionContributor.suggestCompletion(project, editor, doc, text, offset, null); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/api/Code4MeErrorResponse.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.api; 2 | 3 | public class Code4MeErrorResponse extends Code4MeResponse { 4 | 5 | private final String error; 6 | 7 | public Code4MeErrorResponse(String error, int statusCode) { 8 | super(statusCode); 9 | this.error = error; 10 | } 11 | 12 | public String getError() { 13 | return error; 14 | } 15 | 16 | @Override 17 | public boolean isError() { 18 | return true; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/api/Code4MeResponse.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.api; 2 | 3 | public abstract class Code4MeResponse { 4 | 5 | private final int statusCode; 6 | public Code4MeResponse(int statusCode) { 7 | this.statusCode = statusCode; 8 | } 9 | 10 | public int getStatusCode() { 11 | return statusCode; 12 | } 13 | 14 | public boolean isError() { 15 | return false; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/api/PredictionAutocompleteRequest.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.api; 2 | 3 | import com.intellij.ide.plugins.PluginManagerCore; 4 | import com.intellij.openapi.extensions.PluginId; 5 | import com.intellij.openapi.project.Project; 6 | import me.code4me.plugin.services.Code4MeSettingsService; 7 | import me.code4me.plugin.services.Code4MeTriggerPointsService; 8 | import org.jetbrains.annotations.Nullable; 9 | 10 | public class PredictionAutocompleteRequest { 11 | 12 | private static final int MAX_CHARACTERS = 7984; 13 | 14 | private final String leftContext; 15 | private final String rightContext; 16 | private final String triggerPoint; 17 | private final String language; 18 | private final String ide; 19 | private final boolean keybind; 20 | private final String pluginVersion; 21 | private final boolean storeContext; 22 | 23 | private PredictionAutocompleteRequest( 24 | String leftContext, 25 | String rightContext, 26 | String triggerPoint, 27 | String language, 28 | String ide, 29 | boolean keybind, 30 | boolean storeContext 31 | ) { 32 | this.leftContext = leftContext; 33 | this.rightContext = rightContext; 34 | this.triggerPoint = triggerPoint; 35 | this.language = language; 36 | this.ide = ide; 37 | this.keybind = keybind; 38 | this.pluginVersion = PluginManagerCore.getPlugin(PluginId.getId("me.code4me.plugin")).getVersion(); 39 | this.storeContext = storeContext; 40 | } 41 | 42 | public static PredictionAutocompleteRequest of( 43 | String text, 44 | int offset, 45 | @Nullable String triggerPoint, 46 | String language, 47 | String ide, 48 | Project project 49 | ) { 50 | Code4MeTriggerPointsService triggerPointsService = project.getService(Code4MeTriggerPointsService.class); 51 | String leftContext = text.substring(0, offset); 52 | String rightContext = text.substring(offset); 53 | String fixedLeftContext = leftContext.substring(Math.max(0, leftContext.length() - MAX_CHARACTERS)); 54 | String fixedRightContext = rightContext.substring(0, Math.min(MAX_CHARACTERS, rightContext.length())); 55 | boolean storeContext = project.getService(Code4MeSettingsService.class).getSettings().isStoreContext(); 56 | boolean keybind = triggerPoint == null; 57 | 58 | if (keybind) { 59 | if (fixedLeftContext.contains("\n")) { 60 | String line = fixedLeftContext.substring(fixedLeftContext.lastIndexOf('\n') + 1).trim(); 61 | if (line.contains(" ")) { 62 | String token = line.substring(line.lastIndexOf(' ') + 1); 63 | triggerPoint = token; 64 | 65 | int max = Math.min(triggerPointsService.getMaxNoSpaceTriggerPointLength(), token.length()); 66 | for (int i = max; i >= 1; i--) { 67 | String lastChars = token.substring(token.length() - i); 68 | if (Boolean.FALSE.equals(triggerPointsService.getTriggerPoint(lastChars))) { 69 | triggerPoint = lastChars; 70 | break; 71 | } 72 | } 73 | } 74 | } 75 | } 76 | 77 | return new PredictionAutocompleteRequest( 78 | fixedLeftContext, 79 | fixedRightContext, 80 | triggerPoint, 81 | language, 82 | ide, 83 | keybind, 84 | storeContext 85 | ); 86 | } 87 | 88 | public String getLeftContext() { 89 | return leftContext; 90 | } 91 | 92 | public String getRightContext() { 93 | return rightContext; 94 | } 95 | 96 | public String getTriggerPoint() { 97 | return triggerPoint; 98 | } 99 | 100 | public String getLanguage() { 101 | return language; 102 | } 103 | 104 | public String getIde() { 105 | return ide; 106 | } 107 | 108 | public boolean getKeybind() { 109 | return keybind; 110 | } 111 | 112 | public String getPluginVersion() { 113 | return pluginVersion; 114 | } 115 | 116 | public boolean isStoreContext() { 117 | return storeContext; 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/api/PredictionAutocompleteResponse.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.api; 2 | 3 | public class PredictionAutocompleteResponse extends Code4MeResponse { 4 | 5 | private final String[] predictions; 6 | private final String verifyToken; 7 | private final boolean survey; 8 | 9 | public PredictionAutocompleteResponse(String[] predictions, String verifyToken, boolean survey) { 10 | super(200); 11 | this.predictions = predictions; 12 | this.verifyToken = verifyToken; 13 | this.survey = survey; 14 | } 15 | 16 | public String[] getPredictions() { 17 | return predictions; 18 | } 19 | 20 | public String getVerifyToken() { 21 | return verifyToken; 22 | } 23 | 24 | public boolean isSurvey() { 25 | return survey; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/api/PredictionVerifyRequest.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.api; 2 | 3 | public class PredictionVerifyRequest { 4 | 5 | private final String verifyToken; 6 | private final String chosenPrediction; 7 | private final String groundTruth; 8 | 9 | public PredictionVerifyRequest(String verifyToken, String chosenPrediction, String groundTruth) { 10 | this.verifyToken = verifyToken; 11 | this.chosenPrediction = chosenPrediction; 12 | this.groundTruth = groundTruth; 13 | } 14 | 15 | public String getVerifyToken() { 16 | return verifyToken; 17 | } 18 | 19 | public String getChosenPrediction() { 20 | return chosenPrediction; 21 | } 22 | 23 | public String getGroundTruth() { 24 | return groundTruth; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/api/PredictionVerifyResponse.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.api; 2 | 3 | public class PredictionVerifyResponse extends Code4MeResponse { 4 | 5 | public PredictionVerifyResponse() { 6 | super(200); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/completions/Code4MeCompletionContributor.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.completions; 2 | 3 | import com.intellij.codeInsight.completion.CodeCompletionHandlerBase; 4 | import com.intellij.codeInsight.completion.CompletionContributor; 5 | import com.intellij.codeInsight.completion.CompletionParameters; 6 | import com.intellij.codeInsight.completion.CompletionProvider; 7 | import com.intellij.codeInsight.completion.CompletionResultSet; 8 | import com.intellij.codeInsight.completion.CompletionType; 9 | import com.intellij.codeInsight.completion.PrioritizedLookupElement; 10 | import com.intellij.codeInsight.hint.HintManager; 11 | import com.intellij.codeInsight.lookup.LookupElement; 12 | import com.intellij.codeInsight.lookup.LookupElementBuilder; 13 | import com.intellij.notification.NotificationAction; 14 | import com.intellij.notification.NotificationGroupManager; 15 | import com.intellij.notification.NotificationType; 16 | import com.intellij.openapi.application.ApplicationManager; 17 | import com.intellij.openapi.application.ModalityState; 18 | import com.intellij.openapi.editor.Document; 19 | import com.intellij.openapi.editor.Editor; 20 | import com.intellij.openapi.editor.event.DocumentEvent; 21 | import com.intellij.openapi.editor.event.DocumentListener; 22 | import com.intellij.openapi.project.Project; 23 | import com.intellij.util.ProcessingContext; 24 | import com.intellij.patterns.PlatformPatterns; 25 | import me.code4me.plugin.Code4MeIcons; 26 | import me.code4me.plugin.Code4MeBundle; 27 | import me.code4me.plugin.Code4MeScheduledThreadPoolExecutor; 28 | import me.code4me.plugin.api.PredictionAutocompleteRequest; 29 | import me.code4me.plugin.api.PredictionVerifyRequest; 30 | import me.code4me.plugin.api.Code4MeErrorResponse; 31 | import me.code4me.plugin.exceptions.ApiServerException; 32 | import me.code4me.plugin.services.Code4MeApiService; 33 | import me.code4me.plugin.services.Code4MeSettingsService; 34 | import me.code4me.plugin.util.Code4MeUtil; 35 | import org.jetbrains.annotations.NotNull; 36 | import java.awt.EventQueue; 37 | import java.util.ArrayList; 38 | import java.util.Arrays; 39 | import java.util.concurrent.ScheduledFuture; 40 | import java.util.concurrent.TimeUnit; 41 | import java.util.concurrent.atomic.AtomicInteger; 42 | import java.util.function.Supplier; 43 | 44 | public class Code4MeCompletionContributor extends CompletionContributor { 45 | 46 | private static final CompletionCache completionCache = new CompletionCache(); 47 | 48 | public Code4MeCompletionContributor() { 49 | extend(CompletionType.BASIC, PlatformPatterns.psiElement(), new Code4MeCompletionProvider()); 50 | } 51 | 52 | public static void suggestCompletion( 53 | Project project, 54 | Editor editor, 55 | Document doc, 56 | String text, 57 | int offset, 58 | String triggerPoint 59 | ) { 60 | PredictionAutocompleteRequest request = PredictionAutocompleteRequest.of( 61 | text, 62 | offset, 63 | triggerPoint, 64 | Code4MeUtil.getLanguage(project, doc), 65 | "jetbrains", 66 | project 67 | ); 68 | 69 | project.getService(Code4MeApiService.class).fetchAutoCompletion(project, request).thenAccept(res -> { 70 | String[] predictions = res.getPredictions(); 71 | 72 | Code4MeSettingsService settingsService = project.getService(Code4MeSettingsService.class); 73 | Code4MeSettingsService.Settings settings = settingsService.getSettings(); 74 | if (!settings.isIgnoringSurvey() && res.isSurvey()) { 75 | NotificationGroupManager.getInstance() 76 | .getNotificationGroup("Code4Me Notifications") 77 | .createNotification( 78 | Code4MeBundle.message("survey-title"), 79 | Code4MeBundle.message("survey-content"), 80 | NotificationType.INFORMATION 81 | ).addAction(NotificationAction.createSimple( 82 | Code4MeBundle.message("survey-redirect-action"), 83 | () -> project.getService(Code4MeApiService.class).redirectToCode4MeSurvey(project)) 84 | ).addAction(NotificationAction.createSimple( 85 | Code4MeBundle.message("survey-cancel-action"), 86 | () -> { 87 | settings.setIgnoringSurvey(true); 88 | settingsService.save(); 89 | }) 90 | ).notify(project); 91 | } 92 | 93 | EventQueue.invokeLater(() -> { 94 | if (predictions == null || Arrays.stream(predictions).allMatch(String::isBlank)) { 95 | HintManager.getInstance().showInformationHint( 96 | editor, 97 | Code4MeBundle.message("no-suggestions-available") 98 | ); 99 | } else { 100 | String verifyToken = res.getVerifyToken(); 101 | completionCache.setPredictions(predictions); 102 | completionCache.setOffset(offset); 103 | completionCache.setVerifyToken(verifyToken); 104 | completionCache.setTimeoutSupplier(() -> checkCodeChanges(project, verifyToken, null, offset, doc)); 105 | completionCache.setEmpty(false); 106 | 107 | ApplicationManager.getApplication().invokeLater(() -> { 108 | CodeCompletionHandlerBase handler = CodeCompletionHandlerBase.createHandler( 109 | CompletionType.BASIC, 110 | false, 111 | false, 112 | false 113 | ); 114 | handler.invokeCompletion(project, editor, 0, false); 115 | }, ModalityState.current()); 116 | } 117 | }); 118 | }).exceptionally(th -> { 119 | showError(project, th.getCause()); 120 | return null; 121 | }); 122 | } 123 | 124 | private static ScheduledFuture checkCodeChanges( 125 | Project project, 126 | String verifyToken, 127 | String chosenPrediction, 128 | int offset, 129 | Document doc 130 | ) { 131 | AtomicInteger atomicOffset = new AtomicInteger(offset); 132 | DocumentListener listener = new DocumentListener() { 133 | @Override 134 | public void documentChanged(@NotNull DocumentEvent event) { 135 | if (event.getOffset() < atomicOffset.get()) { 136 | atomicOffset.addAndGet(event.getNewLength() - event.getOldLength()); 137 | } 138 | } 139 | }; 140 | doc.addDocumentListener(listener); 141 | 142 | return Code4MeScheduledThreadPoolExecutor.getInstance().schedule(() -> { 143 | doc.removeDocumentListener(listener); 144 | String[] lines = doc.getText().substring(atomicOffset.get()).split("\n"); 145 | String groundTruth = lines.length == 0 ? "" : lines[0]; 146 | project.getService(Code4MeApiService.class).sendCompletionData( 147 | project, 148 | new PredictionVerifyRequest(verifyToken, chosenPrediction, groundTruth) 149 | ).exceptionally(th -> { 150 | showError(project, th.getCause()); 151 | return null; 152 | }); 153 | }, 30, TimeUnit.SECONDS); 154 | } 155 | 156 | private static void showError(Project project, Throwable th) { 157 | if (th instanceof ApiServerException) { 158 | Code4MeErrorResponse response = ((ApiServerException) th).getResponse(); 159 | String message = response == null ? th.getMessage() : response.getError(); 160 | EventQueue.invokeLater(() -> NotificationGroupManager.getInstance() 161 | .getNotificationGroup("Code4Me Notifications") 162 | .createNotification( 163 | Code4MeBundle.message("project-opened-title"), 164 | message, 165 | NotificationType.ERROR 166 | ).notify(project)); 167 | } 168 | } 169 | 170 | private static class Code4MeCompletionProvider extends CompletionProvider { 171 | 172 | @Override 173 | protected void addCompletions( 174 | @NotNull CompletionParameters parameters, 175 | @NotNull ProcessingContext context, 176 | @NotNull CompletionResultSet result 177 | ) { 178 | if (completionCache.isEmpty()) { 179 | return; 180 | } 181 | 182 | ScheduledFuture dataRequest = completionCache.getTimeoutSupplier().get(); 183 | 184 | // Need to add all elements in one go to avoid split 185 | ArrayList elements = new ArrayList<>(); 186 | for (String prediction : completionCache.getPredictions()) { 187 | elements.add(prioritize(LookupElementBuilder.create(prediction) 188 | .withIcon(Code4MeIcons.PLUGIN_ICON) 189 | .withInsertHandler((cxt, item) -> { 190 | if (!dataRequest.cancel(true)) return; 191 | checkCodeChanges( 192 | cxt.getProject(), 193 | completionCache.getVerifyToken(), 194 | prediction, 195 | completionCache.getOffset(), 196 | cxt.getDocument() 197 | ); 198 | String insertion = item.getLookupString(); 199 | String[] lines = cxt.getDocument().getText().substring(completionCache.getOffset()).split("\n"); 200 | String line = lines.length == 0 ? "" : lines[0]; 201 | 202 | if (!insertion.equals(line)) { 203 | if (line.endsWith(")") || line.endsWith("}") || line.endsWith("]") || line.endsWith(">")) { 204 | cxt.getDocument().deleteString(completionCache.getOffset() + insertion.length(), completionCache.getOffset() + insertion.length() + 1); 205 | } 206 | } 207 | }) 208 | .withTypeText("Code4Me") 209 | )); 210 | } 211 | result.addAllElements(elements); 212 | 213 | completionCache.setEmpty(true); 214 | } 215 | } 216 | 217 | private static LookupElement prioritize(LookupElement element) { 218 | return PrioritizedLookupElement.withGrouping( 219 | PrioritizedLookupElement.withExplicitProximity( 220 | PrioritizedLookupElement.withPriority( 221 | element, 222 | Double.MAX_VALUE - 1 223 | ), 224 | Integer.MAX_VALUE - 1 225 | ), 226 | Integer.MAX_VALUE - 1 227 | ); 228 | } 229 | 230 | private static class CompletionCache { 231 | 232 | private String[] predictions; 233 | private int offset; 234 | private String verifyToken; 235 | private boolean empty; 236 | private Supplier> timeoutSupplier; 237 | 238 | public CompletionCache() { 239 | this.predictions = new String[0]; 240 | this.offset = -1; 241 | this.verifyToken = ""; 242 | this.empty = true; 243 | this.timeoutSupplier = null; 244 | } 245 | 246 | public void setPredictions(String[] predictions) { 247 | this.predictions = predictions; 248 | } 249 | 250 | public String[] getPredictions() { 251 | return predictions; 252 | } 253 | 254 | public void setOffset(int offset) { 255 | this.offset = offset; 256 | } 257 | 258 | public int getOffset() { 259 | return offset; 260 | } 261 | 262 | public void setVerifyToken(String verifyToken) { 263 | this.verifyToken = verifyToken; 264 | } 265 | 266 | public String getVerifyToken() { 267 | return verifyToken; 268 | } 269 | 270 | public void setEmpty(boolean empty) { 271 | this.empty = empty; 272 | } 273 | 274 | public boolean isEmpty() { 275 | return empty; 276 | } 277 | 278 | public void setTimeoutSupplier(Supplier> timeoutSupplier) { 279 | this.timeoutSupplier = timeoutSupplier; 280 | } 281 | 282 | public Supplier> getTimeoutSupplier() { 283 | return timeoutSupplier; 284 | } 285 | } 286 | } 287 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/dialogs/Code4MeDialogWrapper.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.dialogs; 2 | 3 | import com.intellij.openapi.ui.DialogWrapper; 4 | import me.code4me.plugin.Code4MeBundle; 5 | 6 | import javax.annotation.Nullable; 7 | import javax.swing.Box; 8 | import javax.swing.JCheckBox; 9 | import javax.swing.JComponent; 10 | import javax.swing.JLabel; 11 | import javax.swing.JPanel; 12 | import java.awt.BorderLayout; 13 | import java.awt.Dimension; 14 | 15 | public class Code4MeDialogWrapper extends DialogWrapper { 16 | 17 | private static final int WIDTH = 330; 18 | private static final int HEIGHT = 270; 19 | 20 | private final JPanel dialogPanel = new JPanel(new BorderLayout()); 21 | private final JLabel contentLabel = new JLabel(); 22 | private final JCheckBox triggerPoints = new JCheckBox(); 23 | private final JCheckBox storeContext = new JCheckBox(); 24 | 25 | public Code4MeDialogWrapper() { 26 | super(true); 27 | setTitle(Code4MeBundle.message("settings-title")); 28 | init(); 29 | pack(); 30 | setResizable(false); 31 | } 32 | 33 | @Nullable 34 | @Override 35 | protected JComponent createCenterPanel() { 36 | Dimension dimension = new Dimension(WIDTH, HEIGHT); 37 | dialogPanel.setPreferredSize(dimension); 38 | dialogPanel.setMinimumSize(dimension); 39 | dialogPanel.setMaximumSize(dimension); 40 | 41 | contentLabel.setText("" + Code4MeBundle.message("settings-content") + ""); 42 | triggerPoints.setText(Code4MeBundle.message("settings-trigger-points")); 43 | storeContext.setText(Code4MeBundle.message("settings-store-context")); 44 | 45 | Box box = Box.createVerticalBox(); 46 | box.add(contentLabel); 47 | box.add(triggerPoints); 48 | box.add(storeContext); 49 | 50 | dialogPanel.add(box); 51 | 52 | return dialogPanel; 53 | } 54 | 55 | public boolean isTriggerPointsSelected() { 56 | return triggerPoints.isSelected(); 57 | } 58 | 59 | public void setTriggerPointsSelected(boolean selected) { 60 | triggerPoints.setSelected(selected); 61 | } 62 | 63 | public boolean isStoreContextSelected() { 64 | return storeContext.isSelected(); 65 | } 66 | 67 | public void setStoreContextSelected(boolean selected) { 68 | storeContext.setSelected(selected); 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/exceptions/AlreadyAutocompletingException.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.exceptions; 2 | 3 | import me.code4me.plugin.Code4MeBundle; 4 | 5 | public class AlreadyAutocompletingException extends RuntimeException { 6 | 7 | public AlreadyAutocompletingException() { 8 | super(Code4MeBundle.message("already-auto-completing")); 9 | } 10 | 11 | public AlreadyAutocompletingException(String message) { 12 | super(message); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/exceptions/ApiServerException.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.exceptions; 2 | 3 | import me.code4me.plugin.api.Code4MeErrorResponse; 4 | import org.jetbrains.annotations.Nullable; 5 | 6 | public class ApiServerException extends RuntimeException { 7 | 8 | private @Nullable final Code4MeErrorResponse response; 9 | 10 | private ApiServerException(@Nullable Code4MeErrorResponse response, String message) { 11 | super(message); 12 | this.response = response; 13 | } 14 | 15 | public ApiServerException(Code4MeErrorResponse response) { 16 | this(response, response.getError()); 17 | } 18 | 19 | public ApiServerException(String message) { 20 | this(null, message); 21 | } 22 | 23 | public @Nullable Code4MeErrorResponse getResponse() { 24 | return response; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/listeners/Code4MeDocumentListener.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.listeners; 2 | 3 | import com.intellij.openapi.editor.Document; 4 | import com.intellij.openapi.editor.Editor; 5 | import com.intellij.openapi.editor.event.DocumentEvent; 6 | import com.intellij.openapi.editor.event.DocumentListener; 7 | import com.intellij.openapi.fileEditor.FileDocumentManager; 8 | import com.intellij.openapi.fileEditor.FileEditorManager; 9 | import com.intellij.openapi.project.Project; 10 | import com.intellij.openapi.vfs.VirtualFile; 11 | import me.code4me.plugin.completions.Code4MeCompletionContributor; 12 | import me.code4me.plugin.services.Code4MeTriggerPointsService; 13 | import org.jetbrains.annotations.NotNull; 14 | 15 | public class Code4MeDocumentListener implements DocumentListener { 16 | 17 | private final Project project; 18 | private final Code4MeTriggerPointsService triggerPointsService; 19 | 20 | public Code4MeDocumentListener(Project project) { 21 | this.project = project; 22 | this.triggerPointsService = project.getService(Code4MeTriggerPointsService.class); 23 | } 24 | 25 | 26 | @Override 27 | public void documentChanged(@NotNull DocumentEvent event) { 28 | Document doc = event.getDocument(); 29 | VirtualFile file = FileDocumentManager.getInstance().getFile(doc); 30 | if (file == null || !file.isInLocalFileSystem()) return; 31 | 32 | Editor editor = FileEditorManager.getInstance(project).getSelectedTextEditor(); 33 | if (editor == null) return; 34 | 35 | String text = doc.getText(); 36 | if (text.isBlank()) return; 37 | 38 | int offset = editor.getCaretModel().getOffset(); 39 | if (offset >= text.length()) return; 40 | 41 | int offsetPlusOne = offset + 1; 42 | if (offsetPlusOne < text.length() && text.charAt(offsetPlusOne) != '\n') return; 43 | 44 | char[] word = new char[triggerPointsService.getMaxTriggerPointLength()]; 45 | int i = 0; 46 | boolean initSpaces = text.charAt(offset) == ' '; 47 | int spaces = 0; 48 | int j; 49 | while (i < word.length && (j = offset - spaces - i) >= 0) { 50 | char c = text.charAt(j); 51 | if (c == ' ') { 52 | if (initSpaces) { 53 | spaces++; 54 | continue; 55 | } 56 | break; 57 | } else if (c == '\n') { 58 | break; 59 | } else { 60 | initSpaces = false; 61 | } 62 | 63 | word[word.length - 1 - i] = c; 64 | String triggerPoint = new String(word).trim(); 65 | Boolean trailingSpace = triggerPointsService.getTriggerPoint(triggerPoint); 66 | if (trailingSpace != null && (!trailingSpace || spaces > 0)) { 67 | Code4MeCompletionContributor.suggestCompletion(project, editor, doc, text, offsetPlusOne, triggerPoint); 68 | break; 69 | } 70 | i++; 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/listeners/Code4MeDynamicPluginListener.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.listeners; 2 | 3 | import com.intellij.ide.DataManager; 4 | import com.intellij.ide.plugins.DynamicPluginListener; 5 | import com.intellij.ide.plugins.IdeaPluginDescriptor; 6 | import com.intellij.openapi.actionSystem.CommonDataKeys; 7 | import com.intellij.openapi.project.Project; 8 | import me.code4me.plugin.services.Code4MeDocumentListenerService; 9 | import org.jetbrains.annotations.NotNull; 10 | 11 | public class Code4MeDynamicPluginListener implements DynamicPluginListener { 12 | 13 | @Override 14 | public void pluginLoaded(@NotNull IdeaPluginDescriptor pluginDescriptor) { 15 | DataManager.getInstance().getDataContextFromFocusAsync().then(dataContext -> { 16 | Project project = dataContext.getData(CommonDataKeys.PROJECT); 17 | if (project != null) { 18 | project.getService(Code4MeDocumentListenerService.class).addDocumentListenerForProject(project); 19 | } 20 | return dataContext; 21 | }); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/listeners/Code4MeProjectManagerListener.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.listeners; 2 | 3 | import com.intellij.notification.NotificationAction; 4 | import com.intellij.notification.NotificationGroupManager; 5 | import com.intellij.notification.NotificationType; 6 | import com.intellij.openapi.project.Project; 7 | import com.intellij.openapi.project.ProjectManagerListener; 8 | import me.code4me.plugin.Code4MeBundle; 9 | import me.code4me.plugin.dialogs.Code4MeDialogWrapper; 10 | import me.code4me.plugin.services.Code4MeDocumentListenerService; 11 | import me.code4me.plugin.services.Code4MeSettingsService; 12 | import org.jetbrains.annotations.NotNull; 13 | 14 | public class Code4MeProjectManagerListener implements ProjectManagerListener { 15 | 16 | @Override 17 | public void projectOpened(@NotNull Project project) { 18 | NotificationGroupManager.getInstance() 19 | .getNotificationGroup("Code4Me Notifications") 20 | .createNotification( 21 | Code4MeBundle.message("project-opened-title"), 22 | Code4MeBundle.message("project-opened-content"), 23 | NotificationType.INFORMATION 24 | ).addAction(NotificationAction.createSimple( 25 | Code4MeBundle.message("project-opened-settings-action"), 26 | () -> openSettingsDialog(project)) 27 | ).notify(project); 28 | project.getService(Code4MeDocumentListenerService.class).addDocumentListenerForProject(project); 29 | } 30 | 31 | private void openSettingsDialog(Project project) { 32 | Code4MeSettingsService settingsService = project.getService(Code4MeSettingsService.class); 33 | Code4MeSettingsService.Settings settings = settingsService.getSettings(); 34 | 35 | Code4MeDialogWrapper dialog = new Code4MeDialogWrapper(); 36 | dialog.setTriggerPointsSelected(settings.isTriggerPoints()); 37 | dialog.setStoreContextSelected(settings.isStoreContext()); 38 | if (dialog.showAndGet()) { 39 | settings.setTriggerPoints(dialog.isTriggerPointsSelected()); 40 | settings.setStoreContext(dialog.isStoreContextSelected()); 41 | settingsService.save(); 42 | 43 | Code4MeDocumentListenerService service = project.getService(Code4MeDocumentListenerService.class); 44 | if (dialog.isTriggerPointsSelected()) { 45 | service.addDocumentListenerForProject(project); 46 | } else { 47 | service.removeDocumentListenerForProject(project); 48 | } 49 | } 50 | } 51 | 52 | @Override 53 | public void projectClosed(@NotNull Project project) { 54 | project.getService(Code4MeDocumentListenerService.class).removeDocumentListenerForProject(project); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/services/Code4MeApiService.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.services; 2 | 3 | import com.google.gson.Gson; 4 | import com.intellij.ide.BrowserUtil; 5 | import com.intellij.openapi.progress.ProgressIndicator; 6 | import com.intellij.openapi.progress.ProgressManager; 7 | import com.intellij.openapi.progress.Task; 8 | import com.intellij.openapi.project.Project; 9 | import me.code4me.plugin.Code4MeBundle; 10 | import me.code4me.plugin.api.PredictionAutocompleteRequest; 11 | import me.code4me.plugin.api.PredictionAutocompleteResponse; 12 | import me.code4me.plugin.api.PredictionVerifyRequest; 13 | import me.code4me.plugin.api.PredictionVerifyResponse; 14 | import me.code4me.plugin.api.Code4MeErrorResponse; 15 | import me.code4me.plugin.api.Code4MeResponse; 16 | import me.code4me.plugin.exceptions.AlreadyAutocompletingException; 17 | import me.code4me.plugin.exceptions.ApiServerException; 18 | import org.apache.http.Header; 19 | import org.apache.http.HttpEntity; 20 | import org.apache.http.client.methods.CloseableHttpResponse; 21 | import org.apache.http.client.methods.HttpPost; 22 | import org.apache.http.entity.ContentType; 23 | import org.apache.http.entity.StringEntity; 24 | import org.apache.http.impl.client.CloseableHttpClient; 25 | import org.apache.http.impl.client.HttpClients; 26 | import org.jetbrains.annotations.NotNull; 27 | 28 | import java.io.IOException; 29 | import java.io.InputStream; 30 | import java.io.InputStreamReader; 31 | import java.nio.charset.StandardCharsets; 32 | import java.util.concurrent.CompletableFuture; 33 | import java.util.concurrent.ForkJoinPool; 34 | import java.util.concurrent.atomic.AtomicBoolean; 35 | 36 | public class Code4MeApiService { 37 | 38 | private static final Gson gson = new Gson(); 39 | private static final String BASE_URL = "https://code4me.me/api/v1"; 40 | private static final String PREDICTION_AUTOCOMPLETE_ENDPOINT = "/prediction/autocomplete"; 41 | private static final String PREDICTION_VERIFY_ENDPOINT = "/prediction/verify"; 42 | private static final String SURVEY_ENDPOINT = "/survey?user_id=%s"; 43 | 44 | private final AtomicBoolean lock = new AtomicBoolean(false); 45 | 46 | public Code4MeApiService() { 47 | 48 | } 49 | 50 | public CompletableFuture fetchAutoCompletion( 51 | Project project, 52 | PredictionAutocompleteRequest request 53 | ) { 54 | CompletableFuture future = new CompletableFuture<>(); 55 | 56 | if (lock.getAndSet(true)) { 57 | future.completeExceptionally(new AlreadyAutocompletingException()); 58 | return future; 59 | } 60 | 61 | String token = project.getService(Code4MeSettingsService.class).getSettings().getUserToken(); 62 | 63 | String autocompleting = Code4MeBundle.message("auto-completing"); 64 | ProgressManager.getInstance().run(new Task.Backgroundable(project, autocompleting, false) { 65 | public void run(@NotNull ProgressIndicator indicator) { 66 | indicator.setText(autocompleting); 67 | 68 | try (CloseableHttpClient client = HttpClients.createDefault()) { 69 | HttpPost httpPost = new HttpPost(BASE_URL + PREDICTION_AUTOCOMPLETE_ENDPOINT); 70 | httpPost.addHeader("Authorization", "Bearer " + token); 71 | httpPost.setEntity(new StringEntity( 72 | gson.toJson(request), 73 | ContentType.create( 74 | ContentType.APPLICATION_JSON.getMimeType(), 75 | StandardCharsets.UTF_8 76 | ) 77 | )); 78 | 79 | try (CloseableHttpResponse res = client.execute(httpPost)) { 80 | Code4MeResponse response = parseResponseBody( 81 | res.getEntity(), 82 | PredictionAutocompleteResponse.class, 83 | res.getStatusLine().getStatusCode() 84 | ); 85 | 86 | if (response instanceof PredictionAutocompleteResponse) { 87 | future.complete((PredictionAutocompleteResponse) response); 88 | } else if (response instanceof Code4MeErrorResponse) { 89 | future.completeExceptionally(new ApiServerException((Code4MeErrorResponse) response)); 90 | } else { 91 | future.completeExceptionally(new RuntimeException("Unknown Code4MeResponse " + response)); 92 | } 93 | } 94 | } catch (IOException ex) { 95 | future.completeExceptionally(ex); 96 | } 97 | lock.set(false); 98 | } 99 | }); 100 | return future; 101 | } 102 | 103 | public CompletableFuture sendCompletionData( 104 | Project project, 105 | PredictionVerifyRequest request 106 | ) { 107 | CompletableFuture future = new CompletableFuture<>(); 108 | 109 | String token = project.getService(Code4MeSettingsService.class).getSettings().getUserToken(); 110 | 111 | ForkJoinPool.commonPool().execute(() -> { 112 | try (CloseableHttpClient client = HttpClients.createDefault()) { 113 | HttpPost httpPost = new HttpPost(BASE_URL + PREDICTION_VERIFY_ENDPOINT); 114 | httpPost.addHeader("Authorization", "Bearer " + token); 115 | httpPost.setEntity(new StringEntity( 116 | gson.toJson(request), 117 | ContentType.create( 118 | ContentType.APPLICATION_JSON.getMimeType(), 119 | StandardCharsets.UTF_8 120 | ) 121 | )); 122 | 123 | try (CloseableHttpResponse res = client.execute(httpPost)) { 124 | Code4MeResponse response = parseResponseBody( 125 | res.getEntity(), 126 | PredictionVerifyResponse.class, 127 | res.getStatusLine().getStatusCode() 128 | ); 129 | 130 | if (response instanceof PredictionVerifyResponse) { 131 | future.complete((PredictionVerifyResponse) response); 132 | } else if (response instanceof Code4MeErrorResponse) { 133 | future.completeExceptionally( 134 | new RuntimeException(((Code4MeErrorResponse) response).getError()) 135 | ); 136 | } else { 137 | future.completeExceptionally(new RuntimeException("Unknown Code4MeResponse " + response)); 138 | } 139 | } 140 | } catch (IOException ex) { 141 | future.completeExceptionally(ex); 142 | } 143 | }); 144 | return future; 145 | } 146 | 147 | private Code4MeResponse parseResponseBody( 148 | HttpEntity entity, 149 | Class clazz, 150 | int statusCode 151 | ) throws IOException { 152 | try ( 153 | InputStream in = entity.getContent(); 154 | InputStreamReader reader = new InputStreamReader(in) 155 | ) { 156 | Header contentType = entity.getContentType(); 157 | if (contentType != null && contentType.getValue().contains("application/json") && statusCode >= 200 && statusCode < 300) { 158 | return gson.fromJson(reader, clazz); 159 | } else { 160 | return new Code4MeErrorResponse(new String(in.readAllBytes(), StandardCharsets.UTF_8), statusCode); 161 | } 162 | } 163 | } 164 | 165 | public void redirectToCode4MeSurvey(Project project) { 166 | String userToken = project.getService(Code4MeSettingsService.class).getSettings().getUserToken(); 167 | BrowserUtil.browse(BASE_URL + String.format(SURVEY_ENDPOINT, userToken)); 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/services/Code4MeDocumentListenerService.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.services; 2 | 3 | import com.intellij.openapi.editor.EditorFactory; 4 | import com.intellij.openapi.project.Project; 5 | import me.code4me.plugin.listeners.Code4MeDocumentListener; 6 | 7 | import java.util.HashMap; 8 | import java.util.Map; 9 | 10 | public class Code4MeDocumentListenerService { 11 | 12 | private final Map documentListenerMap = new HashMap<>(); 13 | 14 | public void addDocumentListenerForProject(Project project) { 15 | if (!documentListenerMap.containsKey(project)) { 16 | Code4MeDocumentListener documentListener = new Code4MeDocumentListener(project); 17 | documentListenerMap.put(project, documentListener); 18 | EditorFactory.getInstance().getEventMulticaster().addDocumentListener(documentListener, () -> {}); 19 | } 20 | } 21 | 22 | public void removeDocumentListenerForProject(Project project) { 23 | Code4MeDocumentListener documentListener = documentListenerMap.remove(project); 24 | if (documentListener != null) { 25 | EditorFactory.getInstance().getEventMulticaster().removeDocumentListener(documentListener); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/services/Code4MeSettingsService.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.services; 2 | 3 | import com.google.gson.Gson; 4 | import com.intellij.credentialStore.CredentialAttributes; 5 | import com.intellij.credentialStore.CredentialAttributesKt; 6 | import com.intellij.ide.passwordSafe.PasswordSafe; 7 | 8 | import java.util.UUID; 9 | 10 | public class Code4MeSettingsService { 11 | 12 | private static final Gson gson = new Gson(); 13 | private static final String SERVICE_NAME = "Code4MeSettings"; 14 | private static final String SETTINGS_KEY = "Settings"; 15 | 16 | private final CredentialAttributes credentialAttributes; 17 | private final Settings settings; 18 | 19 | public Code4MeSettingsService() { 20 | this.credentialAttributes = new CredentialAttributes(CredentialAttributesKt.generateServiceName( 21 | SERVICE_NAME, 22 | SETTINGS_KEY 23 | )); 24 | 25 | String settingsJson = PasswordSafe.getInstance().getPassword(credentialAttributes); 26 | if (settingsJson == null) { 27 | this.settings = new Settings(generateToken(), true, false, false); 28 | this.save(); 29 | } else { 30 | this.settings = gson.fromJson(settingsJson, Settings.class); 31 | if (this.settings.getUserToken() == null) { 32 | this.settings.setUserToken(generateToken()); 33 | this.save(); 34 | } 35 | } 36 | } 37 | 38 | public Settings getSettings() { 39 | return settings; 40 | } 41 | 42 | private String generateToken() { 43 | return UUID.randomUUID().toString().replace("-", ""); 44 | } 45 | 46 | public void save() { 47 | PasswordSafe.getInstance().setPassword(credentialAttributes, gson.toJson(settings)); 48 | } 49 | 50 | public static class Settings { 51 | 52 | private String userToken; 53 | private boolean triggerPoints; 54 | private Boolean ignoringSurvey; 55 | private Boolean storeContext; 56 | 57 | public Settings(String userToken, boolean triggerPoints, boolean ignoringSurvey, boolean storeContext) { 58 | this.userToken = userToken; 59 | this.triggerPoints = triggerPoints; 60 | this.ignoringSurvey = ignoringSurvey; 61 | this.storeContext = storeContext; 62 | } 63 | 64 | public void setUserToken(String userToken) { 65 | this.userToken = userToken; 66 | } 67 | 68 | public String getUserToken() { 69 | return userToken; 70 | } 71 | 72 | public boolean isTriggerPoints() { 73 | return triggerPoints; 74 | } 75 | 76 | public void setTriggerPoints(boolean triggerPoints) { 77 | this.triggerPoints = triggerPoints; 78 | } 79 | 80 | public boolean isIgnoringSurvey() { 81 | return Boolean.TRUE.equals(ignoringSurvey); 82 | } 83 | 84 | public void setIgnoringSurvey(boolean ignoringSurvey) { 85 | this.ignoringSurvey = ignoringSurvey; 86 | } 87 | 88 | public boolean isStoreContext() { 89 | return Boolean.TRUE.equals(storeContext); 90 | } 91 | 92 | public void setStoreContext(boolean storeContext) { 93 | this.storeContext = storeContext; 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/services/Code4MeTriggerPointsService.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.services; 2 | 3 | import com.google.gson.Gson; 4 | 5 | import java.io.IOException; 6 | import java.io.InputStream; 7 | import java.io.InputStreamReader; 8 | import java.util.Arrays; 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | import java.util.Objects; 12 | 13 | public class Code4MeTriggerPointsService { 14 | 15 | private static final Gson gson = new Gson(); 16 | private final Map triggerPointMap = new HashMap<>(); 17 | private final int maxTriggerPointLength; 18 | private final int maxNoSpaceTriggerPointLength; 19 | 20 | public Code4MeTriggerPointsService() throws IOException { 21 | int maxTriggerPointLength; 22 | int maxNoSpaceTriggerPointLength; 23 | try ( 24 | InputStream in = getClass().getResourceAsStream("/triggerPoints.json"); 25 | InputStreamReader reader = new InputStreamReader(Objects.requireNonNull(in)) 26 | ) { 27 | TriggerPoints points = gson.fromJson(reader, TriggerPoints.class); 28 | Arrays.stream(points.enforceSpace).forEach(keyword -> triggerPointMap.put(keyword, true)); 29 | Arrays.stream(points.noSpace).forEach(keyword -> triggerPointMap.put(keyword, false)); 30 | maxTriggerPointLength = triggerPointMap.keySet().stream() 31 | .mapToInt(String::length) 32 | .max() 33 | .orElse(0); 34 | maxNoSpaceTriggerPointLength = triggerPointMap.entrySet().stream() 35 | .filter(entry -> Boolean.FALSE.equals(entry.getValue())) 36 | .map(Map.Entry::getKey) 37 | .mapToInt(String::length) 38 | .max() 39 | .orElse(0); 40 | } 41 | this.maxTriggerPointLength = maxTriggerPointLength; 42 | this.maxNoSpaceTriggerPointLength = maxNoSpaceTriggerPointLength; 43 | } 44 | 45 | public Boolean getTriggerPoint(String keyword) { 46 | return triggerPointMap.get(keyword); 47 | } 48 | 49 | public int getMaxTriggerPointLength() { 50 | return maxTriggerPointLength; 51 | } 52 | 53 | public int getMaxNoSpaceTriggerPointLength() { 54 | return maxNoSpaceTriggerPointLength; 55 | } 56 | 57 | private static class TriggerPoints { 58 | 59 | private String[] enforceSpace; 60 | private String[] noSpace; 61 | 62 | private TriggerPoints(String[] enforceSpace, String[] noSpace) { 63 | this.enforceSpace = enforceSpace; 64 | this.noSpace = noSpace; 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/java/me/code4me/plugin/util/Code4MeUtil.java: -------------------------------------------------------------------------------- 1 | package me.code4me.plugin.util; 2 | 3 | import com.intellij.openapi.editor.Document; 4 | import com.intellij.openapi.project.Project; 5 | import com.intellij.psi.PsiDocumentManager; 6 | import com.intellij.psi.PsiFile; 7 | 8 | public class Code4MeUtil { 9 | 10 | public static String getLanguage(Project project, Document doc) { 11 | PsiFile psiFile = PsiDocumentManager.getInstance(project).getPsiFile(doc); 12 | if (psiFile == null) return "unknown"; 13 | return psiFile.getFileType().getDefaultExtension(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/resources/META-INF/plugin.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | me.code4me.plugin 4 | Code4Me 5 | Code4Me 7 | 8 | com.intellij.modules.platform 9 | 10 | messages.Code4MeBundle 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 21 | 22 | 23 | 24 | 26 | 28 | 29 | 30 | 31 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/resources/META-INF/pluginIcon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/resources/icons/pluginIcon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/resources/messages/Code4MeBundle.properties: -------------------------------------------------------------------------------- 1 | name=Code4Me 2 | settings-title=Code4Me Plugin Setup 3 | settings-content=Thanks for using the Code4Me Plugin.

You can use the alt + shift + K keybind to query an autocompletion.

In addition to that, leave the following checkbox enabled to get real-time suggestions on key points.

To perform a failure analysis & improve code4me, we'd like to store the close context around completions. This data will be stored anonymously and removed after 3 months. Would you like to participate in our study? We deeply appreciate your contribution.

4 | settings-trigger-points=Use trigger points 5 | settings-store-context=Allow storing of completion context 6 | no-suggestions-available=No Code4Me Suggestions available 7 | already-auto-completing=You are already autocompleting! 8 | auto-completing=Autocompleting... 9 | project-opened-title=Code4Me plugin 10 | project-opened-content=Thanks for using the Code4Me Plugin.
You can use the alt + shift + K keybind to query an autocompletion.
Open the settings menu to disable trigger point completion or enable context storage to help with our research. 11 | project-opened-settings-action=Open settings 12 | survey-title=Code4me survey 13 | survey-content=Thanks for using the Code4Me plugin. We would greatly appreciate it if you could help our study by filling in the survey. 14 | survey-redirect-action=Fill in survey 15 | survey-cancel-action=Do not show again 16 | -------------------------------------------------------------------------------- /code4me-jetbrains-plugin/src/main/resources/triggerPoints.json: -------------------------------------------------------------------------------- 1 | { 2 | "enforceSpace": [ 3 | "await", 4 | "assert", 5 | "raise", 6 | "del", 7 | "lambda", 8 | "yield", 9 | "return", 10 | "while", 11 | "for", 12 | "if", 13 | "elif", 14 | "else", 15 | "global", 16 | "in", 17 | "and", 18 | "not", 19 | "or", 20 | "is", 21 | "with", 22 | "except" 23 | ], 24 | "noSpace": [ 25 | ".", 26 | "+", 27 | "-", 28 | "*", 29 | "/", 30 | "%", 31 | "**", 32 | "<<", 33 | ">>", 34 | "&", 35 | "|", 36 | "^", 37 | "==", 38 | "!=", 39 | "<=", 40 | ">=", 41 | "+=", 42 | "-=", 43 | "=", 44 | "<", 45 | ">", 46 | ";", 47 | ",", 48 | "[", 49 | "(", 50 | "{", 51 | "~" 52 | ] 53 | } 54 | -------------------------------------------------------------------------------- /code4me-server/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pytorch/pytorch:latest 2 | ARG UID=999 3 | 4 | RUN groupadd -g $UID codeforme 5 | RUN useradd -r -u $UID -g codeforme -d /codeforme codeforme 6 | RUN mkdir -p /codeforme 7 | RUN chown -R codeforme:codeforme /codeforme 8 | USER codeforme 9 | 10 | WORKDIR /codeforme 11 | COPY requirements.txt ./ 12 | RUN pip3 install -r requirements.txt 13 | COPY src/ . 14 | CMD ["python3", "./app.py"] 15 | -------------------------------------------------------------------------------- /code4me-server/README.md: -------------------------------------------------------------------------------- 1 | # Code4Me Python Server 2 | This repo contains the python backend for the Code4Me plugin(s). 3 | Below you can find the api specification for the endpoints. 4 | 5 | ## Base URL 6 | The Base URL for the (currently) deployed server is: 7 | ``` 8 | https://code4me.me/ 9 | ``` 10 | 11 | ## Authentication 12 | Each request must contain the following Authorization header: 13 | ``` 14 | Authorization: Bearer 15 | ``` 16 | 17 | This token must be created once on the client, as a UUID v4, without dashes. 18 | 19 | ## /api/v1 20 | The current version of the API. 21 | 22 | ### `POST` /api/v1/prediction/autocomplete 23 | The autocompletion endpoint. 24 | 25 | #### Request headers 26 | ``` 27 | Content-Type: application/json 28 | ``` 29 | 30 | #### Request body 31 | ``` 32 | { 33 | "leftContext": string, 34 | "rightContext": string, 35 | "triggerPoint": string, 36 | "language": string, 37 | "ide": string, 38 | "keybind": boolean, 39 | "pluginVersion": string, 40 | "storeContext": boolean 41 | } 42 | ``` 43 | - `leftContext`: the context left of the prediction 44 | - `rightContext`: the context right of the prediction 45 | - `triggerPoint`: the trigger keyword in case a trigger point was used, null otherwise 46 | - `language`: language of the source file 47 | - `ide`: the ide the request was fired from 48 | - `keybind`: a boolean indicating whether the keybind was used to autocomplete 49 | - `pluginVersion`: the version of the plugin that made the request 50 | - `storeContext`: a boolean indicating whether the user allows to store the context 51 | 52 | #### Response body 53 | ``` 54 | { 55 | "predictions": string[], 56 | "verifyToken": string, 57 | "survey: boolean 58 | } 59 | ``` 60 | - `predictions`: the suggestion(s) made by the server 61 | - `verifyToken`: a token to be used for the `/api/v1/prediction/verify` endpoint. 62 | - `survey`: whether a survey should be prompted to the user 63 | 64 | ### `POST` /api/v1/prediction/verify 65 | The verification endpoint. 66 | Called after 30 seconds from the client, such that the server knows the ground truth of the prediction. 67 | 68 | #### Request headers 69 | ``` 70 | Content-Type: application/json 71 | ``` 72 | 73 | #### Request body 74 | ``` 75 | { 76 | "verifyToken": string, 77 | "chosenPrediction": string | null, 78 | "groundTruth": string 79 | } 80 | ``` 81 | - `verifyToken`: the token from the response of `/api/v1/prediction/autocomplete` 82 | - `chosenPrediction`: the chosen prediction from the client 83 | - `groundTruth`: the ground truth of the prediction (the line from the same offset as the completion after 30s) 84 | 85 | #### Response body 86 | N/A 87 | -------------------------------------------------------------------------------- /code4me-server/markdowns/index.md: -------------------------------------------------------------------------------- 1 | # Code4Me 2 | Code4Me provides automatic intelligent code completion based on large pre-trained language models. Code4Me predicts statement (line) completion and is available for both PyCharm (also other JetBrains IDEs) and Visual Studio Code. The code suggestions from Code4Me can be recognised by the logo in the suggestion menu. Code4Me **automatically** triggers *on specific trigger characters* or the user can prompt it manually by pressing the keybind. The keybind differs per IDE: 3 | 4 | - Jetbrains: **ALT + SHIFT + K** 5 | - VSC: Your keybind for the **`triggerSuggest`** command (likely **CTRL + Space**). 6 | 7 | Code4Me does not hinder native auto completion or other extensions. However, Code4Me could be hindered by other extensions. If this is the case, please disable the other (autocomplete) extensions. Please report the extension that causes trouble on our github [repository](https://github.com/code4me-me/code4me) by creating an [issue](https://github.com/code4me-me/code4me/issues/new). 8 | 9 | ## Installation 10 | The plugin is available on both marketplaces. You can download them by either clicking the link in your browser or looking up 'Code4Me' in the integrated marketplace of your IDE. 11 | 12 | ### JetBrains 13 | The Code4Me plugin can be found in the [JetBrains Marketplace](https://plugins.jetbrains.com/plugin/19200-code4me). 14 | 15 | ### VSC 16 | The Code4Me plugin can be found in the [Visual Studio Code Marketplace](https://marketplace.visualstudio.com/items?itemName=Code4Me.code4me-plugin). 17 | 18 | ## Data Collection 19 | The plugin works with a remote API in order to suggest autocompletions. After triggering the autocompletion (automatically or manually), the client's request is sent to the server. Running the ML-based model, the server returns a prediction in response to the client. The remote API requires a segment of the current document (close left context at the trigger point). This limited segment is sent to the backend server only for prediction and will not be stored on our server. 20 | For the purpose of evaluating the models' predictions, we only store the suggestion made by the ML-based models at each trigger point and compare it against the accepted prediction by users (collected after 30 seconds). 21 | 22 | The plugin does not collect personal data nor the segment sent by default. The plugin does collect the following data: 23 | 24 | * Suggested autocompletion. 25 | * Verification of the autocompletion. 26 | * The plugin tracks the line the code was inserted and sends that line to the server after 30 seconds. 27 | * Inference time of completion. 28 | 29 | One of our research goals is to identify where and why code autocompletion fails and how it can be improved. Hence, we would like to conduct a failure analysis on the completions provided by code4me. To this end, we need to study the code context fed to our models based on which the predictions are made. This context includes a number of code tokens before and after the completion point. This is an **optional** setting and this data will be collected anonymously. The data will only be stored for three months to perform the failure analysis. After this period, we *completely* remove this data. 30 | 31 | Furthermore, Code4Me is in full compliance with the GDPR and all data is anonymous. The data collected will remain on the servers of TU Delft until the end of the study. By using Code4Me you give permission for the data collection. 32 | 33 | ## Code4Me is not working! 34 | Code4Me should automatically trigger on specific triggers points such as a period. Additionally, it can be manually triggered by the keybinds mentioned. If the manual trigger does not prompt a code suggestion it could be the case that the ML-based model did not find a completion. To make sure that this is not the case, please attempt to trigger the plugin with enough context (code). 35 | 36 | If Code4Me still does not prompt autocomplete suggestions, please make sure that the suggestions have not dropped in ranking by scrolling down the suggestion list. 37 | 38 | If there is no suggestion in the whole completion list than it is likely that another (autocomplete) extension is interferring with Code4Me. Please try disabling the other extensions. 39 | 40 | If it still does not work, please create an [issue](https://github.com/code4me-me/code4me/issues/new) on our github [repository](https://github.com/code4me-me/code4me). 41 | 42 | ## Source Code 43 | The source code for the IDE plugins and the API webserver can be found at our GitHub [repository](https://github.com/code4me-me/code4me). -------------------------------------------------------------------------------- /code4me-server/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask~=2.2.2 2 | Flask-Limiter~=3.1.0 3 | torch~=1.13.1 4 | transformers~=4.21.0 5 | rouge-score~=0.1.2 6 | Levenshtein~=0.20.9 7 | nltk~=3.8.1 8 | datasets~=2.9.0 9 | markdown~=3.4.1 10 | joblib~=1.2.0 11 | safetensors 12 | -------------------------------------------------------------------------------- /code4me-server/src/api.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | import os, time, random, json, uuid, glob, torch, traceback 3 | 4 | from enum import Enum 5 | from typing import List, Tuple 6 | from model import Model 7 | from datetime import datetime 8 | from joblib import Parallel, delayed 9 | from flask import Blueprint, request, Response, redirect, current_app 10 | from limiter import limiter 11 | 12 | from user_study import ( 13 | filter_request, 14 | store_completion_request, 15 | should_prompt_survey, 16 | USER_STUDY_DIR, 17 | ) 18 | 19 | v1 = Blueprint("v1", __name__) 20 | v2 = Blueprint("v2", __name__) 21 | 22 | os.makedirs("data", exist_ok=True) 23 | 24 | def authorise(req) -> str: 25 | ''' Authorise the request. Raise ValueError if the request is not authorised. ''' 26 | 27 | auth = req.authorization.token 28 | if auth is None: 29 | raise ValueError("Missing bearer token") 30 | return auth 31 | 32 | def get_predictions(completion_request: dict) -> Tuple[float, dict[str, str]]: 33 | ''' Return a list of predictions. ''' 34 | 35 | prefix = completion_request['prefix'].rstrip() 36 | suffix = completion_request['suffix'] 37 | 38 | def predict_model(model: Model) -> str: 39 | try: 40 | return model.value[1](prefix, suffix)[0] 41 | except torch.cuda.OutOfMemoryError: 42 | exit(1) 43 | 44 | t0 = datetime.now() 45 | predictions = Parallel(n_jobs=os.cpu_count(), prefer="threads")(delayed(predict_model)(model) for model in Model) 46 | time = (datetime.now() - t0).total_seconds() * 1000 47 | 48 | predictions = {model.name: prediction for model, prediction in zip(Model, predictions)} 49 | return time, predictions 50 | 51 | @v2.route("/prediction/autocomplete", methods=["POST"]) 52 | @limiter.limit("4000/hour") 53 | def autocomplete_v2(): 54 | 55 | try: 56 | # TODO: As we want every request to be authorised, this can be extracted into a decorator 57 | user_uuid = authorise(request) 58 | request_json = request.json 59 | 60 | # TODO: add a None filter type for baseline comparison 61 | filter_time, filter_type, should_filter = filter_request(user_uuid, request_json) 62 | 63 | predict_time, predictions = get_predictions(request_json) \ 64 | if (not should_filter) or (request_json['trigger'] == 'manual') \ 65 | else (None, {}) 66 | 67 | log_filter = f'\033[1m{"filter" if should_filter else "predict"}\033[0m' 68 | log_context = f'{request_json["prefix"][-10:]}•{request_json["suffix"][:5]}' 69 | current_app.logger.warning(f'{log_filter} {log_context} \t{filter_type} {[v[:10] for v in predictions.values()]}') 70 | 71 | verify_token = uuid.uuid4().hex if not should_filter else '' 72 | prompt_survey = should_prompt_survey(user_uuid) if not should_filter else False 73 | 74 | store_completion_request(user_uuid, verify_token, { 75 | **request_json, 76 | 'timestamp': datetime.now().isoformat(), 77 | 'filter_type': filter_type, 78 | 'filter_time': filter_time, 79 | 'should_filter': should_filter, 80 | 'predict_time': predict_time, 81 | 'predictions': predictions, 82 | 'survey': prompt_survey, 83 | 'study_version': '0.0.1' 84 | }) 85 | 86 | return { 87 | 'predictions': predictions, 88 | 'verifyToken': verify_token, 89 | 'survey': prompt_survey 90 | } 91 | 92 | except Exception as e: 93 | 94 | error_uuid = uuid.uuid4().hex 95 | current_app.logger.warning(f''' 96 | Error {error_uuid} for {user_uuid if user_uuid is not None else "unauthenticated user"} 97 | {request.json if request.is_json else "no request json found"} 98 | ''') 99 | traceback.print_exc() 100 | 101 | return response({ "error": error_uuid }, status=400) 102 | 103 | @v2.route("/prediction/verify", methods=["POST"]) 104 | @limiter.limit("4000/hour") 105 | def verify_v2(): 106 | 107 | user_uuid = authorise(request) 108 | verify_json = request.json 109 | 110 | # current_app.logger.info(verify_json) 111 | 112 | verify_token = verify_json['verifyToken'] 113 | file_path = os.path.join(USER_STUDY_DIR, user_uuid, f'{verify_token}.json') 114 | 115 | with open(file_path, 'r+') as completion_file: 116 | completion_json = json.load(completion_file) 117 | 118 | if 'ground_truth' in completion_json: 119 | return response({ 120 | "error": "Already used verify token" 121 | }, status=400) 122 | 123 | completion_json.update(verify_json) 124 | 125 | completion_file.seek(0) 126 | completion_file.write(json.dumps(completion_json)) 127 | completion_file.truncate() 128 | 129 | return response({'success': True}) 130 | 131 | 132 | ##### NOTE: OLD IMPLEMENTATION KEPT FOR JETBRAINS USERS #### 133 | # (and, those that have turned of auto-update for vsc extensions) 134 | 135 | @v1.route("/prediction/autocomplete", methods=["POST"]) 136 | @limiter.limit("1000/hour") 137 | def autocomplete(): 138 | user_token, res = authorize_user() 139 | if res is not None: 140 | return res 141 | 142 | values, res = get_body_values( 143 | request.get_json(), 144 | [ 145 | ("leftContext", str, False), 146 | ("rightContext", str, False), 147 | ("triggerPoint", str, True), 148 | ("language", str, False), 149 | ("ide", str, False), 150 | ("keybind", bool, True), 151 | ("pluginVersion", str, True), 152 | ("storeContext", bool, True) 153 | ], 154 | ) 155 | if res is not None: 156 | return res 157 | 158 | # remove trailing whitespace from left context - tokens usually include a leading space, so this should improve accuracy 159 | left_context = values["leftContext"] or "" 160 | stripped_left_context = left_context.rstrip() 161 | right_context = values["rightContext"] 162 | store_context = values.get("storeContext", False) is True 163 | 164 | t_before = datetime.now() 165 | predictions = {} 166 | unique_predictions_set = set() 167 | 168 | def predict_model(model: Model) -> List[str]: 169 | try: 170 | return model.value[1](stripped_left_context, right_context) 171 | except torch.cuda.OutOfMemoryError: 172 | exit(1) 173 | 174 | results = Parallel(n_jobs=os.cpu_count(), prefer="threads")(delayed(predict_model)(model) for model in Model) 175 | for model, model_predictions in zip(Model, results): 176 | predictions[model.name] = model_predictions 177 | unique_predictions_set.update(model_predictions) 178 | 179 | t_after = datetime.now() 180 | unique_predictions = list(unique_predictions_set) 181 | random.shuffle(unique_predictions) 182 | 183 | verify_token = uuid.uuid4().hex 184 | 185 | with open(f"data/{user_token}-{verify_token}.json", "w+") as f: 186 | f.write(json.dumps({ 187 | "completionTimestamp": datetime.now().isoformat(), 188 | "triggerPoint": values["triggerPoint"], 189 | "language": values["language"].lower(), 190 | "ide": values["ide"].lower(), 191 | "modelPredictions": predictions, 192 | "predictions": unique_predictions, 193 | "inferenceTime": (t_after - t_before).total_seconds() * 1000, 194 | "leftContextLength": len(left_context), 195 | "rightContextLength": len(right_context), 196 | "keybind": values["keybind"], 197 | "pluginVersion": values["pluginVersion"], 198 | "leftContext": left_context if store_context else None, 199 | "rightContext": right_context if store_context else None 200 | })) 201 | 202 | # # # TODO: disabled surveys temporarily, as we are currently looking through >1M files on every request. 203 | # n_suggestions = len(glob.glob(f"data/{user_token}*.json")) 204 | # survey = n_suggestions >= 100 and n_suggestions % 50 == 0 205 | survey = False 206 | 207 | return response({ 208 | "predictions": unique_predictions, 209 | "verifyToken": verify_token, 210 | "survey": survey 211 | }) 212 | 213 | 214 | @v1.route("/prediction/verify", methods=["POST"]) 215 | @limiter.limit("1000/hour") 216 | def verify(): 217 | user_token, res = authorize_user() 218 | if res is not None: 219 | return res 220 | 221 | values, res = get_body_values( 222 | request.get_json(), 223 | [ 224 | ("verifyToken", str, False), 225 | ("chosenPrediction", str, True), 226 | ("groundTruth", str, False), 227 | ], 228 | ) 229 | if res is not None: 230 | return res 231 | 232 | verify_token = values["verifyToken"] 233 | file_path = f"data/{user_token}-{verify_token}.json" 234 | if not os.path.exists(file_path): 235 | return response({ 236 | "error": "Invalid verify token" 237 | }, status=400) 238 | 239 | with open(file_path, "r+") as completion_file: 240 | prediction_data = json.load(completion_file) 241 | if "groundTruth" in prediction_data: 242 | return response({ 243 | "error": "Already used verify token" 244 | }, status=400) 245 | 246 | prediction_data["chosenPrediction"] = values["chosenPrediction"] 247 | prediction_data["groundTruth"] = values["groundTruth"] 248 | 249 | completion_file.seek(0) 250 | completion_file.write(json.dumps(prediction_data)) 251 | completion_file.truncate() 252 | 253 | return response({ 254 | "success": True 255 | }) 256 | 257 | 258 | @v1.route("/survey") 259 | def survey(): 260 | user_id = request.args.get("user_id", default="", type=str) 261 | return redirect(os.getenv("SURVEY_LINK").replace("{user_id}", user_id), code=302) 262 | 263 | 264 | def authorize_user(): 265 | authorization = request.headers["Authorization"] 266 | if not authorization.startswith("Bearer "): 267 | return None, response({ 268 | "error": "Missing bearer token" 269 | }, status=401) 270 | 271 | user_token = authorization[len("Bearer "):] 272 | return user_token, None 273 | 274 | 275 | def get_body_values(body, keys): 276 | values = {} 277 | for key, obj, optional in keys: 278 | value, res = get_body_value(body, key, obj, optional) 279 | if res is not None: 280 | return None, res 281 | values[key] = value 282 | return values, None 283 | 284 | 285 | def get_body_value(body, key, obj, optional=False): 286 | if not optional and key not in body: 287 | return None, response({ 288 | "error": f"Missing key '{key}' in request body" 289 | }, status=400) 290 | value = body.get(key, None) 291 | if value is not None and not isinstance(value, obj): 292 | return None, response({ 293 | "error": f"Key '{key}' is not of type '{obj.__name__}'" 294 | }, status=400) 295 | return value, None 296 | 297 | 298 | def response(body, status=200): 299 | return Response(json.dumps(body, indent=2), mimetype="application/json", status=status) 300 | -------------------------------------------------------------------------------- /code4me-server/src/app.py: -------------------------------------------------------------------------------- 1 | import markdown, os 2 | 3 | from pathlib import Path 4 | from flask import Flask, jsonify, render_template 5 | from api import v1, v2 6 | from limiter import limiter 7 | 8 | app = Flask(__name__, static_folder="static", template_folder="templates") 9 | limiter.init_app(app) 10 | app.register_blueprint(v1, url_prefix='/api/v1') 11 | app.register_blueprint(v2, url_prefix='/api/v2') 12 | 13 | markdown_path = 'markdowns/index.md' 14 | index_md = markdown.markdown(Path(markdown_path).read_text()) 15 | 16 | 17 | @app.errorhandler(429) 18 | def page_not_found(e): 19 | return jsonify(error=429, text=str(e)), 429 20 | 21 | 22 | @app.route("/") 23 | @app.route("/index.html") 24 | def home(): 25 | return render_template("index.html", md=index_md) 26 | 27 | 28 | if __name__ == "__main__": 29 | app.run(host='0.0.0.0', port=3000) 30 | -------------------------------------------------------------------------------- /code4me-server/src/codegpt.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from transformers import GPT2LMHeadModel, GPT2Config, GPT2Tokenizer 3 | import os 4 | import torch 5 | 6 | # env variable for local testing 7 | CODE4ME_TEST = os.environ.get("CODE4ME_TEST", "false") == "true" 8 | 9 | checkpoint_path = "gpt2" # default checkpoint is the non-finetuned gpt2 model 10 | 11 | # if CODEGPT_CHECKPOINT_PATH is set, use that checkpoint 12 | if os.environ.get("CODEGPT_CHECKPOINT_PATH"): 13 | checkpoint_path = os.environ.get("CODEGPT_CHECKPOINT_PATH") 14 | 15 | if not os.path.exists(checkpoint_path) and not CODE4ME_TEST: 16 | raise ValueError(f"Invalid checkpoint path: '{checkpoint_path}'") 17 | 18 | config = GPT2Config 19 | tokenizer = GPT2Tokenizer.from_pretrained(checkpoint_path, do_lower_case=False, sep_token='', 20 | bos_token='', eos_token='', pad_token='', 21 | unk_token='<|UNKNOWN|>') 22 | model = GPT2LMHeadModel.from_pretrained(checkpoint_path) 23 | model.resize_token_embeddings(len(tokenizer)) 24 | 25 | 26 | class Beam(object): 27 | def __init__(self, size, sos, eos): 28 | self.size = size 29 | self.tt = torch.cuda if torch.cuda.is_available() else torch 30 | # The score for each translation on the beam. 31 | self.scores = self.tt.FloatTensor(size).zero_().to(device) 32 | # The backpointers at each time-step. 33 | self.prevKs = [] 34 | # The outputs at each time-step. 35 | self.nextYs = [self.tt.LongTensor(size) 36 | .fill_(0).to(device)] 37 | self.nextYs[0][:] = sos 38 | # Has EOS topped the beam yet. 39 | self._eos = eos 40 | self.eosTop = False 41 | # Time and k pair for finished. 42 | self.finished = [] 43 | 44 | def getCurrentState(self): 45 | "Get the outputs for the current timestep." 46 | batch = self.tt.LongTensor(self.nextYs[-1]).view(-1, 1) 47 | return batch 48 | 49 | def getCurrentOrigin(self): 50 | "Get the backpointers for the current timestep." 51 | return self.prevKs[-1] 52 | 53 | def advance(self, wordLk): 54 | """ 55 | Given prob over words for every last beam `wordLk` and attention 56 | `attnOut`: Compute and update the beam search. 57 | 58 | Parameters: 59 | 60 | * `wordLk`- probs of advancing from the last step (K x words) 61 | * `attnOut`- attention at the last step 62 | 63 | Returns: True if beam search is complete. 64 | """ 65 | numWords = wordLk.size(1) 66 | 67 | # Sum the previous scores. 68 | if len(self.prevKs) > 0: 69 | beamLk = wordLk + self.scores.unsqueeze(1).expand_as(wordLk) 70 | 71 | # Don't let EOS have children. 72 | for i in range(self.nextYs[-1].size(0)): 73 | if self.nextYs[-1][i] in self._eos: 74 | beamLk[i] = -1e20 75 | else: 76 | beamLk = wordLk[0] 77 | flatBeamLk = beamLk.view(-1) 78 | bestScores, bestScoresId = flatBeamLk.topk(self.size, 0, True, True) 79 | 80 | self.scores = bestScores 81 | 82 | # bestScoresId is flattened beam x word array, so calculate which 83 | # word and beam each score came from 84 | prevK = torch.div(bestScoresId, numWords, rounding_mode='trunc') 85 | self.prevKs.append(prevK) 86 | self.nextYs.append((bestScoresId - prevK * numWords)) 87 | 88 | for i in range(self.nextYs[-1].size(0)): 89 | if self.nextYs[-1][i] in self._eos: 90 | s = self.scores[i] 91 | self.finished.append((s, len(self.nextYs) - 1, i)) 92 | 93 | # End condition is when top-of-beam is EOS and no global score. 94 | if self.nextYs[-1][0] in self._eos: 95 | self.eosTop = True 96 | 97 | def done(self): 98 | return self.eosTop and len(self.finished) >= self.size 99 | 100 | def getFinal(self): 101 | if len(self.finished) == 0: 102 | self.finished.append((self.scores[0], len(self.nextYs) - 1, 0)) 103 | self.finished.sort(key=lambda a: -a[0]) 104 | if len(self.finished) != self.size: 105 | unfinished = [] 106 | for i in range(self.nextYs[-1].size(0)): 107 | if self.nextYs[-1][i] not in self._eos: 108 | s = self.scores[i] 109 | unfinished.append((s, len(self.nextYs) - 1, i)) 110 | unfinished.sort(key=lambda a: -a[0]) 111 | self.finished += unfinished[:self.size - len(self.finished)] 112 | return self.finished[:self.size] 113 | 114 | def getHyp(self, beam_res): 115 | """ 116 | Walk back to construct the full hypothesis. 117 | """ 118 | hyps = [] 119 | for _, timestep, k in beam_res: 120 | hyp = [] 121 | for j in range(len(self.prevKs[:timestep]) - 1, -1, -1): 122 | hyp.append(self.nextYs[j + 1][k]) 123 | k = self.prevKs[j][k] 124 | hyps.append(hyp[::-1]) 125 | return hyps 126 | 127 | def buildTargetTokens(self, preds): 128 | sentence = [] 129 | for pred in preds: 130 | tokens = [] 131 | for tok in pred: 132 | tokens.append(tok) 133 | if tok in self._eos: 134 | break 135 | sentence.append(tokens) 136 | return sentence 137 | 138 | 139 | def DecodeIds(idxs): 140 | codes = "" 141 | for idx in idxs: 142 | to_add = tokenizer.convert_ids_to_tokens(idx) 143 | if to_add[0] == '\u0120': 144 | if not codes.endswith(" "): 145 | codes += " " + to_add[1:] 146 | else: 147 | codes += to_add[1:] 148 | elif ( 149 | idx in [tokenizer.bos_token_id, tokenizer.eos_token_id, tokenizer.sep_token_id, 150 | tokenizer.pad_token_id] 151 | ): 152 | codes += " " + to_add + " " 153 | else: 154 | codes += to_add 155 | return codes.strip(" ") 156 | 157 | 158 | device_name = os.environ.get("CODEGPT_DEVICE", "cuda:0" if torch.cuda.is_available() else "cpu") 159 | device = torch.device(device_name) 160 | model.to(device) 161 | model.eval() 162 | break_ids = [tokenizer.sep_token_id] 163 | 164 | m = torch.nn.LogSoftmax(dim=-1).to(device) 165 | # I presume the .cuda. is not necessary here if it is moved to the CUDA device immediately, but not risking it. 166 | zero = torch.cuda.LongTensor(1).fill_(0).to(device) if not CODE4ME_TEST else torch.LongTensor(1).fill_(0).to(device) 167 | 168 | def codegpt_predict(left_context: str, right_context: str) -> List[str]: 169 | left_context = left_context.replace("\n", "") 170 | input_size = 960 171 | predict_size = 64 172 | block_size = input_size + predict_size 173 | 174 | # pre-truncate to ensure we do not tokenize too much (takes too long) 175 | avg_chars_per_token = 4.26 176 | avg_chars_per_token += 1 # add some margin to be safe 177 | input = left_context[-int(input_size * avg_chars_per_token):] 178 | 179 | tokens = tokenizer.encode(input)[-(input_size - 1):] 180 | # print tokens as strings 181 | # prepend with 182 | tokens = [tokenizer.bos_token_id] + tokens 183 | inputs = torch.tensor(tokens, device=device).unsqueeze(0) 184 | with torch.no_grad(): 185 | beam_size = 1 186 | outputs = model(inputs)[1] 187 | p = [] 188 | for i in range(inputs.shape[0]): 189 | past = [torch.cat([x[0].unsqueeze(0), x[1].unsqueeze(0)], dim=0) if type(x) == tuple else x for x in 190 | outputs] 191 | past_hidden = [x[:, i:i + 1].expand(-1, beam_size, -1, -1, -1) for x in past] 192 | beam = Beam(beam_size, inputs[i][-1].data, break_ids) 193 | input_ids = None 194 | for _ in range(predict_size): 195 | if beam.done(): 196 | break 197 | input_ids = beam.getCurrentState() 198 | outputs = model(input_ids, past_key_values=past_hidden) 199 | out = m(outputs[0][:, -1, :]).data 200 | beam.advance(out) 201 | past = [torch.cat([x[0].unsqueeze(0), x[1].unsqueeze(0)], dim=0) if type(x) == tuple else x for x in 202 | outputs[1]] 203 | past_hidden = [x.data.index_select(1, beam.getCurrentOrigin()) for x in past] 204 | hyp = beam.getHyp(beam.getFinal()) 205 | pred = beam.buildTargetTokens(hyp)[:beam_size] 206 | 207 | pred = [torch.cat([x.view(-1) for x in p] + [zero] * (100 - len(p))).view(1, -1) for p in pred] 208 | p.append(torch.cat(pred, 0).unsqueeze(0)) 209 | p = torch.cat(p, 0) 210 | for pred in p: 211 | t = pred[0].cpu().numpy() 212 | t = t.tolist() 213 | if 0 in t: 214 | t = t[:t.index(0)] 215 | if tokenizer.eos_token_id in t: 216 | t = t[:t.index(tokenizer.eos_token_id)] 217 | if tokenizer.sep_token_id in t: 218 | t = t[:t.index(tokenizer.sep_token_id)] 219 | text = DecodeIds(t).strip() 220 | return [text] 221 | return [] 222 | -------------------------------------------------------------------------------- /code4me-server/src/evaluation.py: -------------------------------------------------------------------------------- 1 | import re 2 | from datetime import datetime 3 | 4 | import Levenshtein as Levenshtein 5 | from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction 6 | from nltk.translate.meteor_score import meteor_score 7 | from rouge_score import rouge_scorer 8 | 9 | 10 | def compute_rouge(line: str, completion: str): 11 | scorer = rouge_scorer.RougeScorer(["rougeL"], use_stemmer=True) 12 | scores = scorer.score(line, completion) 13 | score = scores["rougeL"] 14 | 15 | return { 16 | "precision": score.precision, 17 | "recall": score.recall, 18 | "f1measure": score.fmeasure 19 | } 20 | 21 | 22 | def tokenize_code(code): 23 | tokens = [ 24 | x 25 | for x in re.split('("""(.|\n)*"""|"(.|\n)*"|#.*|!=|\*\*|<<|>>|==|>=|<=| +|\W)', code) 26 | if x and not x.isspace() 27 | ] 28 | return tokens, " ".join(tokens) 29 | 30 | 31 | def compute(line: str, completion: str): 32 | tokenized_line, tokenized_line_str = tokenize_code(line) 33 | tokenized_completion, tokenized_completion_str = tokenize_code(completion) 34 | return { 35 | "bleu": sentence_bleu([tokenized_line], tokenized_completion, smoothing_function=SmoothingFunction().method2), 36 | "exactMatch": float(line == completion), 37 | "levenshtein": Levenshtein.ratio(line, completion), 38 | "meteor": meteor_score(references=[tokenized_line], hypothesis=tokenized_completion), 39 | "rouge": compute_rouge(tokenized_line_str, tokenized_completion_str), 40 | "statisticTimestamp": datetime.now().isoformat() 41 | } 42 | -------------------------------------------------------------------------------- /code4me-server/src/incoder.py: -------------------------------------------------------------------------------- 1 | import os 2 | import util 3 | from typing import List 4 | 5 | import torch 6 | from transformers import AutoModelForCausalLM, AutoTokenizer, StoppingCriteriaList, StoppingCriteria 7 | 8 | model_name = "facebook/incoder-1B" 9 | model = AutoModelForCausalLM.from_pretrained(model_name) 10 | 11 | device_name = os.environ.get("INCODER_DEVICE", "cuda:0" if torch.cuda.is_available() else "cpu") 12 | device = torch.device(device_name) 13 | model.to(device) 14 | 15 | tokenizer = AutoTokenizer.from_pretrained(model_name) 16 | 17 | CUDA = os.getenv("CODE4ME_CUDA", "False") == "True" 18 | if CUDA: 19 | model = model.half() 20 | 21 | # signals the start of a document 22 | BOS = "<|endoftext|>" 23 | # signals the end of a generated infill 24 | EOM = "<|endofmask|>" 25 | # signals the end of a file 26 | EOF = "<|/ file |>" 27 | # Until the end of the line 28 | stop_tokens = [205, 284, 353, 536, 994, 3276, 4746, 15471, 16027, 28602, 40289, 43275, 50517] 29 | 30 | 31 | def make_sentinel(i): 32 | # signals (1) a location to insert an infill and (2) the start of the infill generation 33 | return f"<|mask:{i}|>" 34 | 35 | 36 | class StatementStoppingCriteria(StoppingCriteria): 37 | 38 | def __init__(self, init_length: int, stop_tokens: List[int]): 39 | self.init_length = init_length 40 | self.stop_tokens = stop_tokens 41 | 42 | def __contains_stop_token(self, tokens): 43 | for token in tokens: 44 | if token in self.stop_tokens: 45 | return True 46 | return False 47 | 48 | def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool: 49 | return self.__contains_stop_token(input_ids[0][self.init_length:]) 50 | 51 | 52 | def decode(tokens): 53 | return tokenizer.decode( 54 | tokens, 55 | clean_up_tokenization_spaces=False, 56 | skip_special_tokens=True 57 | ) 58 | 59 | 60 | def generate(left_context: str, right_context: str): 61 | left_context_tokenised = tokenizer(left_context, return_tensors="pt").to(device).input_ids[0] 62 | right_context_tokenised = tokenizer(right_context, return_tensors="pt").to(device).input_ids[0] 63 | 64 | left_context = decode(util.truncate_left_context( 65 | left_context_tokenised, 66 | max(1000, 2000 - len(right_context_tokenised)) 67 | )) 68 | 69 | right_context = decode(util.truncate_right_context( 70 | right_context_tokenised, 71 | max(1000, 2000 - len(left_context_tokenised)) 72 | )) 73 | 74 | prompt = left_context + make_sentinel(0) + right_context + EOF + make_sentinel(1) + make_sentinel(0) 75 | tokens = tokenizer(prompt, return_tensors="pt") 76 | if CUDA: 77 | tokens = tokens.to(device) 78 | token_count = len(tokens.input_ids[0]) 79 | 80 | stopping_criteria = StoppingCriteriaList() 81 | stopping_criteria.append(StatementStoppingCriteria(token_count, stop_tokens)) 82 | 83 | with torch.no_grad(): 84 | completion = model.generate( 85 | **tokens, 86 | do_sample=True, 87 | top_p=0.95, 88 | temperature=0.2, 89 | max_length=min(2048, token_count + 48), 90 | stopping_criteria=stopping_criteria 91 | )[0][token_count:] 92 | 93 | decoded_completion = decode(completion).strip().split("\n")[0] 94 | return [decoded_completion] 95 | -------------------------------------------------------------------------------- /code4me-server/src/limiter.py: -------------------------------------------------------------------------------- 1 | from flask_limiter import Limiter 2 | from flask_limiter.util import get_remote_address 3 | 4 | limiter = Limiter(key_func=get_remote_address) 5 | -------------------------------------------------------------------------------- /code4me-server/src/model.py: -------------------------------------------------------------------------------- 1 | import os 2 | from enum import Enum 3 | from typing import Callable 4 | 5 | # NOTE: Convenient for testing, use preset generate functions 6 | # if os.getenv("CODE4ME_TEST", "false") == "true": 7 | # print(''' 8 | # \033[1m WARNING: RUNNING IN TEST MODE \033[0m 9 | # ''') 10 | # # if the env variable TEST_MODE is set to True, then remap model.generate to lambda: 'model_name' 11 | 12 | # incoder = type("InCoder", (object,), {}) 13 | # unixcoder_wrapper = type("UniXCoder", (object,), {}) 14 | # import codegpt 15 | # # codegpt = type("CodeGPT", (object,), {}) 16 | 17 | # incoder.generate = lambda left, right: ['predict_incoder'] 18 | # unixcoder_wrapper.generate = lambda left, right: [' predict_unixcoder'] 19 | 20 | # # codegpt.codegpt_predict = lambda left, right: [' (predict_codegpt'] 21 | # else: 22 | # # ooh yeah, import statements in an else stmt; i see new things every day 23 | import incoder 24 | import unixcoder_wrapper 25 | import codegpt 26 | 27 | class Model(Enum): 28 | InCoder = (0, incoder.generate) 29 | UniXCoder = (1, unixcoder_wrapper.generate) 30 | CodeGPT = (2, codegpt.codegpt_predict) 31 | 32 | @classmethod 33 | def _missing_(cls, value): 34 | if isinstance(value, int): 35 | for item in cls: 36 | if item.value[0] == value: 37 | return item 38 | return super()._missing_(value) 39 | -------------------------------------------------------------------------------- /code4me-server/src/query_filter.py: -------------------------------------------------------------------------------- 1 | import os, math, enum, torch, numpy as np 2 | 3 | from modeling_jonberta import JonbertaForSequenceClassification, add_features_to_model 4 | from transformers import TextClassificationPipeline, AutoTokenizer, AutoConfig 5 | from safetensors import safe_open 6 | 7 | MODELS_DIR = 'models' 8 | DEVICE = 1 if torch.cuda.is_available() else -1 9 | 10 | 11 | intercept, coef = 3.73303724, np.array([ 0.00860799, -0.03679135, -0.06289737, 0.4488578 , -0.40977991, -0.57503621, -0.41543147, 0.02215769, -0.56694562, 0.62073879, -0.26658544, -0.33758971, -0.19398661, 0.10083877, 0.29011958, 0.01642904, 0.082694 , -0.45812433, 0.19563108, 1.11585148, -0.12549902, -0.03319017, 0. , 0.37221593, 0.20887294, 0.59667318, -0.76727645, -2.23206534, 0. , 0. , 0. , 0. , -0.52622741, -1.80321186, -0.65761382, -0.66972758, 0. , -2.12369698, -3.08559028, -2.64399433, -2.17775627, -0.72525643, -1.94062537, -0.64899621, 0. , 0.07055691, 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , -4.80829315, -2.20680964, -3.35584853, -3.23677452, 0. , 0. , 0.16874269, 0.46803166, 0.6497761 , 0.52477345, 0.5324576 , 0.51661321, 0.33516685, 0.27858223, 0.39369077, 0.1905836 , 0.11973277, 0.3743934 , 0.40315233, 0.48388634, 0.32372177, 0.6324842 , 0.09022166, 0.38000563, 0.4746545 , 0.54397314, 0.22015718, 0.11972259, 0.33946541, 0.29087561, 0.16096189, 0.18354135, -1.20029481, 0.03437284, 0.08835093, -1.75083818, 0.97368022, 0. , 1.54601348, 0.72473379, 1.00326585, 1.8238706 , 2.44167387, 1.74815122, 0.79420007, 1.53473857, 1.08563755, 0.53734968, 0.55176486, 0.98191938, 0.90612076, 1.81525461, 1.21869578, 1.07433351, 0.40708646, 2.276902 , 1.85239634, 2.01438915, 0.77927204, 0.67669704, 0.69432173, 0.72461073, 0.75737211, 0.27126203, -2.08431261, -1.47177109, 0.02996505, -0.47417774, 0. , 0. , 0. , 0. , 0. , -0.964373 , -0.84868705, -0.65761382, -1.42460126, 0. , -1.47293568, -0.94525298, -0.60052356, -1.12780257, -1.92249699, -1.66530837, -0.64899621, 0. , 0.07055691, 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , -1.35681768, -0.80897361, -0.16270093, -0.69864107, 0. , 0. , 0.16874269, 0.46803166, 0.6497761 , 0.52477345, 0.5324576 , 0.51661321, 0.33516685, 0.27858223, 0.39369077, 0.1905836 , 0.11973277, 0.3743934 , 0.40315233, 0.48388634, -0.0571159 , 0.6324842 , 0.09022166, 0.38000563, 0.4746545 , 0.54397314, 0.22015718, 0.11972259, 0.33946541, 0.29087561, 0.16096189, 0.18354135, -1.79744913, 0.03437284, 0.08835093, -1.75083818, 0.97368022, 0. , 0.33769289, 0.72473379, 1.00326585, -0.47593682, -0.28913642, -0.47461482, 0.79420007, -1.07146562, 1.08563755, 0.53734968, 0.55176486, 1.25787508, 0.90612076, -0.05355035, 0.74789048, 1.07433351, 0.40708646, -0.71501723, -0.04197237, 0.10833025, 0.77927204, 0.67669704, 0.75031618, 0.72461073, 0.75737211, 0.27126203, -1.3740823 , -1.18380704, 0.02996505, -0.47417774]) 12 | tokenizer = AutoTokenizer.from_pretrained('huggingface/CodeBERTa-small-v1') 13 | 14 | from transformers import set_seed 15 | import random 16 | def set_all_seeds(seed=42): 17 | set_seed(seed) 18 | random.seed(seed) 19 | np.random.seed(seed) 20 | torch.manual_seed(seed) 21 | torch.cuda.manual_seed_all(seed) 22 | # torch.distributed.barrier() 23 | 24 | 25 | class Logres: 26 | 27 | SUPPORTED_LANGS = [ 28 | 'javascript', 'typescript', 'typescriptreact', 'python', 'vue', 'php', 'dart', 29 | 'javascriptreact', 'go', 'css', 'cpp', 'html', 'scss', 'markdown', 'csharp', 30 | 'java', 'json', 'rust', 'ruby', 'c', 31 | ] 32 | 33 | def __init__(self, weights, intercept): 34 | self.weights = weights 35 | self.intercept = intercept 36 | 37 | @classmethod 38 | def character_map(cls, char) -> list: 39 | ''' Converts a character code (32-127) to one-hot vector ''' 40 | char_code = ord(char) 41 | return [1 if char_code == i else 0 for i in range(32, 127)] 42 | 43 | @classmethod 44 | def lang_map(cls, query_lang) -> list: 45 | ''' Converts a supported language to one-hot vector ''' 46 | return [1 if query_lang == lang else 0 for lang in cls.SUPPORTED_LANGS] 47 | 48 | def _preprocess(self, X: dict) -> np.array: 49 | ''' Preprocess a query into a vector of features ''' 50 | 51 | document_length = len(X['prefix']) + len(X['suffix']) 52 | offset = len(X['prefix']) 53 | offset_percentage = offset / document_length 54 | whitespace_after_cursor = 1 if (len(X['suffix']) >= 1 and X['suffix'][0] == ' ') else 0 55 | last_prefix_line = X['prefix'].split('\n')[-1] 56 | last_prefix_line_stripped = last_prefix_line.rstrip() 57 | 58 | return np.array([ 59 | math.log(1 + X['time_since_last_completion']), 60 | math.log(1 + document_length), 61 | math.log(1 + offset), 62 | offset_percentage, 63 | *self.lang_map(X['language']), 64 | whitespace_after_cursor, 65 | math.log(1 + len(last_prefix_line)), 66 | math.log(1 + len(last_prefix_line_stripped)), 67 | *self.character_map(last_prefix_line[-1] if len(last_prefix_line) > 0 else chr(0)), 68 | *self.character_map(last_prefix_line_stripped[-1] if len(last_prefix_line_stripped) > 0 else chr(0)), 69 | ]) 70 | 71 | def predict(self, X: dict) -> bool: 72 | X = self._preprocess(X) 73 | should_filter = X @ self.weights + self.intercept > 0 # True means positive class 74 | # important to wrap this in bool for json serialisation! 75 | return not bool(should_filter) # True means to filter out 76 | 77 | def get_nontextual_features(query) -> list: 78 | ''' Get the features that could otherwise not be extracted from the context alone ''' 79 | 80 | offset = len(query['prefix']) 81 | document_length = offset + len(query['suffix']) 82 | 83 | return [ 84 | 1 if query['ide'] == 'jetbrains' else 0, 85 | 1 if query['ide'] == 'vsc' else 0, 86 | math.log(1 + query['time_since_last_completion']), 87 | math.log(1 + document_length), 88 | math.log(1 + offset), 89 | offset / document_length, 90 | *Logres.lang_map(query['language']), 91 | ] 92 | 93 | def tokenize_joint_sample(sample, max_suffix_tokens=128): 94 | ''' For a single sample, tokenize prefix and suffix, separating by sep token. 95 | Set max_suffix_tokens to maximal amount of suffix to include, when it exists. ''' 96 | 97 | max_length = tokenizer.model_max_length # 512 98 | 99 | # figure out how many suffix tokens we have (128 max) 100 | tokenizer.truncation_side = 'right' 101 | suffix = tokenizer(sample['suffix'], padding='do_not_pad', truncation=True, return_tensors='pt', 102 | max_length = max_suffix_tokens + 1) # to accomodate removal of 103 | 104 | n_suffix_tokens = len(suffix['input_ids'][0]) - 1 105 | 106 | tokenizer.truncation_side = 'left' 107 | prefix = tokenizer(sample['prefix'], padding='do_not_pad', truncation=True, return_tensors='pt', 108 | max_length = max_length - n_suffix_tokens) 109 | 110 | n_prefix_tokens = len(prefix['input_ids'][0]) 111 | tokenizer.truncation_side = 'right' 112 | suffix = tokenizer(sample['suffix'], padding='max_length', truncation=True, return_tensors='pt', 113 | max_length = max_length - n_prefix_tokens + 1) # to accomodate removal of 114 | 115 | suffix['input_ids'] = suffix['input_ids'][:, 1:] 116 | suffix['attention_mask'] = suffix['attention_mask'][:, 1:] 117 | 118 | sample.update({k: torch.cat((prefix[k], suffix[k]), dim=1) for k in prefix}) 119 | return sample 120 | 121 | 122 | class MyPipeline(TextClassificationPipeline): 123 | ''' oh yeah custom pipeline because of the custom tokenisation! 124 | how convenient huggingface ill hug your face extra hard next time i see you ''' 125 | 126 | def __init__(self, *args, incl_features=True, preprocess_fn=tokenize_joint_sample, model_name=None, **kwargs): 127 | if 'device' in kwargs and 'model' in kwargs: 128 | print(f'\tusing device \033[1m{kwargs["device"]}\033[0m for model \033[1m{model_name}\033[0m') 129 | super().__init__(*args, **kwargs) 130 | self.incl_features = incl_features 131 | self.preprocess_fn = preprocess_fn 132 | 133 | def _sanitize_parameters(self, **kwargs): 134 | preprocess_kwargs = {} 135 | if 'preprocess_fn' in kwargs: 136 | preprocess_kwargs['preprocess_fn'] = kwargs.pop('preprocess_fn') 137 | return preprocess_kwargs, {}, {} 138 | 139 | def preprocess(self, inputs, preprocess_fn=None): 140 | inputs = { 141 | 'prefix': inputs['prefix'], 142 | 'suffix': inputs['suffix'], 143 | 'encoder_hidden_states': get_nontextual_features(inputs) 144 | } 145 | inputs = preprocess_fn(inputs) if self.preprocess_fn is None else self.preprocess_fn(inputs) 146 | if 'prefix' in inputs: del inputs['prefix'] 147 | if 'suffix' in inputs: del inputs['suffix'] 148 | # given that pipeline is used in sequential eval, we neeed to add a batch dimension for the model to not throw a tantrum 149 | if self.incl_features: 150 | inputs['encoder_hidden_states'] = torch.tensor(inputs['encoder_hidden_states'], dtype=torch.float32).unsqueeze(0) 151 | elif 'encoder_hidden_states' in inputs: 152 | del inputs['encoder_hidden_states'] 153 | return inputs 154 | 155 | def _forward(self, model_inputs): 156 | return self.model(**model_inputs) 157 | 158 | def postprocess(self, model_outputs) -> bool: 159 | prediction = model_outputs.logits.argmax(-1).item() == 1 # 1 is the positive class 160 | return not bool(prediction) # True means to filter out 161 | 162 | def get_model(model_name): 163 | model_dir = os.path.join(MODELS_DIR, model_name) 164 | config = AutoConfig.from_pretrained(model_dir) 165 | 166 | model = JonbertaForSequenceClassification(config) 167 | if hasattr(config, 'add_head') and config.add_head: 168 | add_features_to_model(model, config) 169 | 170 | # ah yes huggingface is a 5 BILLION dollar company now 171 | state_dict = {} 172 | with safe_open(os.path.join(model_dir, 'model.safetensors'), framework='pt') as f: 173 | for key in f.keys(): 174 | state_dict[key] = f.get_tensor(key) 175 | new_layers = model.load_state_dict(state_dict, strict=False) 176 | print(f'''incompatible keys during loading: {new_layers}. 177 | I don\'t know why this happens, I can't reproduce it locally 178 | As long as it's just embedding position ids, it should be fine.''') 179 | 180 | return model 181 | 182 | class Filter(enum.Enum): 183 | NO_FILTER = 'no_filter' 184 | FEATURE = 'feature' 185 | CONTEXT = 'context' 186 | JOINT_H = 'joint_h' 187 | JOINT_A = 'joint_a' 188 | 189 | no_filter = lambda request_json: True 190 | logres = Logres(coef, intercept) 191 | set_all_seeds() # just in case 192 | context_filter = MyPipeline( device=DEVICE, task='text-classification', 193 | model=get_model('12_codeberta-biased-2e-05lr--0'), incl_features=True, 194 | model_name='12_codeberta-biased-2e-05lr--0' ) 195 | set_all_seeds() 196 | joint_h_filter = MyPipeline( device=DEVICE, task='text-classification', 197 | model=get_model('-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD-dense--reinit)-2e-05lr-1'), incl_features=True, 198 | model_name='-13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(HEAD-dense--reinit)-2e-05lr-1' ) 199 | set_all_seeds() 200 | joint_a_filter = MyPipeline( device=DEVICE, task='text-classification', 201 | model=get_model('13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0]L)-2e-05lr--4'), incl_features=True, 202 | model_name='13_jonberta-biased-12_codeberta-biased-2e-05lr--0-(ATTN-208C_f-[0]L)-2e-05lr--4' ) 203 | 204 | filters = { 205 | Filter.NO_FILTER: no_filter, 206 | Filter.FEATURE: logres.predict, 207 | Filter.CONTEXT: context_filter, 208 | Filter.JOINT_H: joint_h_filter, 209 | Filter.JOINT_A: joint_a_filter, 210 | } 211 | -------------------------------------------------------------------------------- /code4me-server/src/unixcoder.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | import torch 5 | import torch.nn as nn 6 | from transformers import RobertaTokenizer, RobertaModel, RobertaConfig 7 | 8 | class UniXcoder(nn.Module): 9 | def __init__(self, model_name): 10 | """ 11 | Build UniXcoder. 12 | 13 | Parameters: 14 | 15 | * `model_name`- huggingface model card name. e.g. microsoft/unixcoder-base 16 | """ 17 | super(UniXcoder, self).__init__() 18 | self.tokenizer = RobertaTokenizer.from_pretrained(model_name) 19 | self.config = RobertaConfig.from_pretrained(model_name) 20 | self.config.is_decoder = True 21 | self.model = RobertaModel.from_pretrained(model_name, config=self.config) 22 | 23 | self.register_buffer("bias", torch.tril(torch.ones((1024, 1024), dtype=torch.uint8)).view(1,1024, 1024)) 24 | self.lm_head = nn.Linear(self.config.hidden_size, self.config.vocab_size, bias=False) 25 | self.lm_head.weight = self.model.embeddings.word_embeddings.weight 26 | self.lsm = nn.LogSoftmax(dim=-1) 27 | 28 | self.tokenizer.add_tokens([""],special_tokens=True) 29 | 30 | def tokenize(self, inputs, mode="", max_length=512, padding=False): 31 | """ 32 | Convert string to token ids 33 | 34 | Parameters: 35 | 36 | * `inputs`- list of input strings. 37 | * `max_length`- The maximum total source sequence length after tokenization. 38 | * `padding`- whether to pad source sequence length to max_length. 39 | * `mode`- which mode the sequence will use. i.e. , , 40 | """ 41 | assert mode in ["", "", ""] 42 | 43 | tokenizer = self.tokenizer 44 | 45 | tokens_ids = [] 46 | for x in inputs: 47 | tokens = tokenizer.tokenize(x) 48 | if mode == "": 49 | tokens = tokens[:max_length-4] 50 | tokens = [tokenizer.cls_token,mode,tokenizer.sep_token] + tokens + [tokenizer.sep_token] 51 | elif mode == "": 52 | tokens = tokens[-(max_length-3):] 53 | tokens = [tokenizer.cls_token,mode,tokenizer.sep_token] + tokens 54 | else: 55 | tokens = tokens[:max_length-5] 56 | tokens = [tokenizer.cls_token,mode,tokenizer.sep_token] + tokens + [tokenizer.sep_token] 57 | 58 | tokens_id = tokenizer.convert_tokens_to_ids(tokens) 59 | if padding: 60 | tokens_id = tokens_id + [self.config.pad_token_id] * (max_length-len(tokens_id)) 61 | tokens_ids.append(tokens_id) 62 | return tokens_ids 63 | 64 | def get_tokenizer(self): 65 | return self.tokenizer 66 | 67 | def decode(self, source_ids): 68 | """ Convert token ids to string """ 69 | predictions = [] 70 | for x in source_ids: 71 | prediction = [] 72 | for y in x: 73 | t = y.cpu().numpy() 74 | t = list(t) 75 | if 0 in t: 76 | t = t[:t.index(0)] 77 | text = self.tokenizer.decode(t,clean_up_tokenization_spaces=False) 78 | prediction.append(text) 79 | predictions.append(prediction) 80 | return predictions 81 | 82 | def forward(self, source_ids): 83 | """ Obtain token embeddings and sentence embeddings """ 84 | mask = source_ids.ne(self.config.pad_token_id) 85 | token_embeddings = self.model(source_ids,attention_mask = mask.unsqueeze(1) * mask.unsqueeze(2))[0] 86 | sentence_embeddings = (token_embeddings * mask.unsqueeze(-1)).sum(1) / mask.sum(-1).unsqueeze(-1) 87 | return token_embeddings, sentence_embeddings 88 | 89 | def generate(self, source_ids, decoder_only=True, eos_id=None, beam_size=5, max_length=64, stop_tokens=None): 90 | """ Generate sequence given context (source_ids) """ 91 | 92 | if stop_tokens is None: 93 | stop_tokens = [] 94 | 95 | # Set encoder mask attention matrix: bidirectional for , unirectional for 96 | if decoder_only: 97 | mask = self.bias[:,:source_ids.size(-1),:source_ids.size(-1)] 98 | else: 99 | mask = source_ids.ne(self.config.pad_token_id) 100 | mask = mask.unsqueeze(1) * mask.unsqueeze(2) 101 | 102 | if eos_id is None: 103 | eos_id = self.config.eos_token_id 104 | 105 | device = source_ids.device 106 | 107 | # Decoding using beam search 108 | preds = [] 109 | zero = torch.LongTensor(1).fill_(0).to(device) 110 | source_len = list(source_ids.ne(1).sum(-1).cpu().numpy()) 111 | length = source_ids.size(-1) 112 | encoder_output = self.model(source_ids,attention_mask=mask) 113 | for i in range(source_ids.shape[0]): 114 | context = [[x[i:i+1,:,:source_len[i]].repeat(beam_size,1,1,1) for x in y] 115 | for y in encoder_output.past_key_values] 116 | beam = Beam(beam_size,eos_id,device) 117 | input_ids = beam.getCurrentState().clone() 118 | context_ids = source_ids[i:i+1,:source_len[i]].repeat(beam_size,1) 119 | out = encoder_output.last_hidden_state[i:i+1,:source_len[i]].repeat(beam_size,1,1) 120 | for _ in range(max_length): 121 | if beam.done(): 122 | break 123 | if _ == 0: 124 | hidden_states = out[:,-1,:] 125 | out = self.lsm(self.lm_head(hidden_states)).data 126 | beam.advance(out) 127 | input_ids.data.copy_(input_ids.data.index_select(0, beam.getCurrentOrigin())) 128 | input_ids = beam.getCurrentState().clone() 129 | else: 130 | length = context_ids.size(-1)+input_ids.size(-1) 131 | out = self.model(input_ids,attention_mask=self.bias[:,context_ids.size(-1):length,:length], 132 | past_key_values=context).last_hidden_state 133 | hidden_states = out[:,-1,:] 134 | out = self.lsm(self.lm_head(hidden_states)).data 135 | beam.advance(out) 136 | input_ids.data.copy_(input_ids.data.index_select(0, beam.getCurrentOrigin())) 137 | input_ids = torch.cat((input_ids,beam.getCurrentState().clone()),-1) 138 | 139 | tokens = beam.buildTargetTokens([[beam.getLastHyp(beam.getFinalNonModifying())]]) 140 | if len(tokens) > 0: 141 | tokens = tokens[0] 142 | if len(tokens) > 0: 143 | if tokens[0] in stop_tokens: 144 | break 145 | 146 | hyp = beam.getHyp(beam.getFinal()) 147 | pred = beam.buildTargetTokens(hyp)[:beam_size] 148 | pred = [torch.cat([x.view(-1) for x in p]+[zero]*(max_length-len(p))).view(1,-1) for p in pred] 149 | preds.append(torch.cat(pred,0).unsqueeze(0)) 150 | 151 | preds = torch.cat(preds,0) 152 | 153 | return preds 154 | 155 | 156 | 157 | class Beam(object): 158 | def __init__(self, size, eos, device): 159 | self.size = size 160 | self.device = device 161 | # The score for each translation on the beam. 162 | self.scores = torch.FloatTensor(size).zero_().to(device) 163 | # The backpointers at each time-step. 164 | self.prevKs = [] 165 | # The outputs at each time-step. 166 | self.nextYs = [torch.LongTensor(size).fill_(0).to(device)] 167 | # Has EOS topped the beam yet. 168 | self._eos = eos 169 | self.eosTop = False 170 | # Time and k pair for finished. 171 | self.finished = [] 172 | 173 | def getCurrentState(self): 174 | "Get the outputs for the current timestep." 175 | batch = self.nextYs[-1].view(-1, 1) 176 | return batch 177 | 178 | def getCurrentOrigin(self): 179 | "Get the backpointers for the current timestep." 180 | return self.prevKs[-1] 181 | 182 | def advance(self, wordLk): 183 | """ 184 | Given prob over words for every last beam `wordLk` and attention 185 | `attnOut`: Compute and update the beam search. 186 | 187 | Parameters: 188 | 189 | * `wordLk`- probs of advancing from the last step (K x words) 190 | * `attnOut`- attention at the last step 191 | 192 | Returns: True if beam search is complete. 193 | """ 194 | numWords = wordLk.size(1) 195 | 196 | # Sum the previous scores. 197 | if len(self.prevKs) > 0: 198 | beamLk = wordLk + self.scores.unsqueeze(1).expand_as(wordLk) 199 | 200 | # Don't let EOS have children. 201 | for i in range(self.nextYs[-1].size(0)): 202 | if self.nextYs[-1][i] == self._eos: 203 | beamLk[i] = -1e20 204 | else: 205 | beamLk = wordLk[0] 206 | flatBeamLk = beamLk.view(-1) 207 | bestScores, bestScoresId = flatBeamLk.topk(self.size, 0, True, True) 208 | 209 | self.scores = bestScores 210 | 211 | # bestScoresId is flattened beam x word array, so calculate which 212 | # word and beam each score came from 213 | prevK = torch.div(bestScoresId, numWords, rounding_mode='floor') 214 | self.prevKs.append(prevK) 215 | self.nextYs.append((bestScoresId - prevK * numWords)) 216 | 217 | 218 | for i in range(self.nextYs[-1].size(0)): 219 | if self.nextYs[-1][i] == self._eos: 220 | s = self.scores[i] 221 | self.finished.append((s, len(self.nextYs) - 1, i)) 222 | 223 | # End condition is when top-of-beam is EOS and no global score. 224 | if self.nextYs[-1][0] == self._eos: 225 | self.eosTop = True 226 | 227 | def done(self): 228 | return self.eosTop and len(self.finished) >= self.size 229 | 230 | def getFinal(self): 231 | if len(self.finished) == 0: 232 | self.finished.append((self.scores[0], len(self.nextYs) - 1, 0)) 233 | self.finished.sort(key=lambda a: -a[0]) 234 | if len(self.finished) != self.size: 235 | unfinished=[] 236 | for i in range(self.nextYs[-1].size(0)): 237 | if self.nextYs[-1][i] != self._eos: 238 | s = self.scores[i] 239 | unfinished.append((s, len(self.nextYs) - 1, i)) 240 | unfinished.sort(key=lambda a: -a[0]) 241 | self.finished+=unfinished[:self.size-len(self.finished)] 242 | return self.finished[:self.size] 243 | 244 | def getFinalNonModifying(self): 245 | finished = self.finished.copy() 246 | if len(finished) == 0: 247 | finished.append((self.scores[0], len(self.nextYs) - 1, 0)) 248 | finished.sort(key=lambda a: -a[0]) 249 | if len(finished) != self.size: 250 | unfinished=[] 251 | for i in range(self.nextYs[-1].size(0)): 252 | if self.nextYs[-1][i] != self._eos: 253 | s = self.scores[i] 254 | unfinished.append((s, len(self.nextYs) - 1, i)) 255 | unfinished.sort(key=lambda a: -a[0]) 256 | finished+=unfinished[:self.size-len(finished)] 257 | return finished[:self.size] 258 | 259 | def getLastHyp(self, beam_res): 260 | for _, timestep, k in beam_res: 261 | for j in range(len(self.prevKs[:timestep]) - 1, -1, -1): 262 | return self.nextYs[j + 1][k] 263 | return None 264 | 265 | def getHyp(self, beam_res): 266 | """ 267 | Walk back to construct the full hypothesis. 268 | """ 269 | hyps=[] 270 | for _,timestep, k in beam_res: 271 | hyp = [] 272 | for j in range(len(self.prevKs[:timestep]) - 1, -1, -1): 273 | hyp.append(self.nextYs[j+1][k]) 274 | k = self.prevKs[j][k] 275 | hyps.append(hyp[::-1]) 276 | return hyps 277 | 278 | def buildTargetTokens(self, preds): 279 | sentence=[] 280 | for pred in preds: 281 | tokens = [] 282 | for tok in pred: 283 | if tok==self._eos: 284 | break 285 | tokens.append(tok) 286 | sentence.append(tokens) 287 | return sentence 288 | 289 | 290 | -------------------------------------------------------------------------------- /code4me-server/src/unixcoder_wrapper.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | import os 3 | 4 | import torch 5 | from unixcoder import UniXcoder 6 | 7 | device_name = os.environ.get("UNIXCODER_DEVICE", "cuda:0" if torch.cuda.is_available() else "cpu") 8 | device = torch.device(device_name) 9 | model = UniXcoder("microsoft/unixcoder-base") 10 | model.to(device) 11 | 12 | stop_tokens = [ 13 | 317, 1022, 2094, 2357, 2830, 2941, 3425, 4148, 4226, 14 | 7204, 7675, 7995, 8292, 12494, 13440, 18149, 18533, 15 | 19134, 19260, 21261, 23648, 29837, 33034, 33593, 33815, 16 | 34180, 35780, 37120, 39622, 41345, 41640, 42768, 47720 17 | ] 18 | 19 | 20 | def generate(left_context: str, right_context: str) -> List[str]: 21 | tokens_ids = model.tokenize([left_context], max_length=936, mode="") 22 | source_ids = torch.tensor(tokens_ids).to(device) 23 | prediction_ids = model.generate(source_ids, decoder_only=True, beam_size=1, max_length=128, stop_tokens=stop_tokens) 24 | predictions = model.decode(prediction_ids) 25 | return [prediction.strip().split("\n")[0] for prediction in predictions[0]] 26 | -------------------------------------------------------------------------------- /code4me-server/src/user_study.py: -------------------------------------------------------------------------------- 1 | import os, enum, random, json, pickle 2 | 3 | from dataclasses import dataclass 4 | from datetime import datetime 5 | from typing import Tuple, Callable 6 | from query_filter import Filter, filters 7 | 8 | SESSION_TIMEOUT = 1800 9 | MAX_CACHE_SIZE = 30 10 | STUDY_VERSION = '0.0.1' 11 | 12 | USER_STUDY_DIR = 'data_aral' 13 | os.makedirs(USER_STUDY_DIR, exist_ok=True) 14 | 15 | 16 | # Cache of user_uuid -> (last_access, filter_type) 17 | # which allows us to retrieve the Filter predict function via filters[filter_type] 18 | cache = {} # We only have like 100 concurrent users at a time, so in-memory it is 19 | 20 | 21 | def get_request_filter(user_uuid: str, time: datetime) -> Callable[[dict], bool]: 22 | ''' A user is assigned the same filter while in a session, i.e. if it is no longer than 23 | 30 mins since the last completion. Otherwise, they are assigned a filter at random ''' 24 | 25 | if user_uuid in cache and (time - cache[user_uuid][0]).seconds < SESSION_TIMEOUT: 26 | filter_type, last_access = cache[user_uuid][1], (time - cache[user_uuid][0]).total_seconds() 27 | else: 28 | filter_type, last_access = random.choice(list(filters.keys())), 0.0 29 | 30 | cache[user_uuid] = (time, filter_type) 31 | if len(cache) > MAX_CACHE_SIZE: prune_cache(time) 32 | 33 | return filter_type, last_access 34 | 35 | def prune_cache(time: datetime): 36 | ''' Prune cache of users with expired sessions, or update MAX_CACHE_SIZE to grow 37 | proportionally. I.e. minimise memory while ensuring all users are kept track of ''' 38 | 39 | global MAX_CACHE_SIZE 40 | 41 | for user_uuid, (last_access, _) in cache.items(): 42 | if (time - last_access).seconds > SESSION_TIMEOUT: 43 | del cache[user_uuid] 44 | 45 | if len(cache) > MAX_CACHE_SIZE: 46 | new_size = len(cache) + MAX_CACHE_SIZE 47 | print(f'Growing cache to size {new_size}') 48 | MAX_CACHE_SIZE = new_size 49 | elif len(cache) < MAX_CACHE_SIZE // 2: 50 | new_size = MAX_CACHE_SIZE // 2 51 | print(f'Shrinking cache to size {new_size}') 52 | MAX_CACHE_SIZE = new_size 53 | else: 54 | print(f"Pruned cache to size {len(cache)}") 55 | 56 | def filter_request(user_uuid: str, completion_request: dict) -> Tuple[float, Filter, bool]: 57 | ''' Call the request filter (point of this study), returning the time taken and 58 | whether the request should be filtered. ''' 59 | 60 | t0 = datetime.now() 61 | filter_type, last_access = get_request_filter(user_uuid, t0) 62 | 63 | completion_request['time_since_last_completion'] = last_access 64 | 65 | filter_fn = filters[filter_type] 66 | should_filter = filter_fn(completion_request) \ 67 | if (len(completion_request['prefix']) + len(completion_request['suffix'])) >= 10\ 68 | else True 69 | time = (datetime.now() - t0).total_seconds() * 1000 70 | 71 | return time, filter_type.value, should_filter 72 | 73 | 74 | def store_completion_request(user_uuid: str, verify_token: str, completion_request: dict): 75 | ''' Store the completion request in USER_STUDY_DIR/user_uuid/verify_token.json ''' 76 | 77 | user_dir = os.path.join(USER_STUDY_DIR, user_uuid) 78 | json_file = f'{verify_token}.json' 79 | os.makedirs(user_dir, exist_ok=True) 80 | 81 | with open(os.path.join(user_dir, json_file), 'w') as f: 82 | f.write(json.dumps(completion_request)) 83 | 84 | def should_prompt_survey(user_uuid: str): 85 | ''' Return whether to prompt the user with survey. I re-specify it here, 86 | as it depends on `USER_STUDY_DIR` ''' 87 | 88 | user_dir = os.path.join(USER_STUDY_DIR, user_uuid) 89 | if not os.path.exists(user_dir): return False 90 | 91 | n_suggestions = len(os.listdir(user_dir)) 92 | return n_suggestions >= 100 and n_suggestions % 50 == 0 93 | -------------------------------------------------------------------------------- /code4me-server/src/util.py: -------------------------------------------------------------------------------- 1 | def truncate_left_context(context, max_length): 2 | return context[max(0, len(context) - max_length):] 3 | 4 | 5 | def truncate_right_context(context, max_length): 6 | return context[:min(max_length, len(context))] 7 | -------------------------------------------------------------------------------- /code4me-server/static/android-chrome-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code4me-me/code4me/d8429d41c42ac16957659b95380908458a7c2980/code4me-server/static/android-chrome-192x192.png -------------------------------------------------------------------------------- /code4me-server/static/android-chrome-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code4me-me/code4me/d8429d41c42ac16957659b95380908458a7c2980/code4me-server/static/android-chrome-512x512.png -------------------------------------------------------------------------------- /code4me-server/static/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code4me-me/code4me/d8429d41c42ac16957659b95380908458a7c2980/code4me-server/static/apple-touch-icon.png -------------------------------------------------------------------------------- /code4me-server/static/browserconfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | #da532c 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /code4me-server/static/css/retro.css: -------------------------------------------------------------------------------- 1 | /** 2 | Credits: https://github.com/markdowncss/retro 3 | */ 4 | @font-face { 5 | font-family: "JetBrainsMono"; 6 | src: url('/fonts/JetBrainsMono-Regular.woff2') format('woff2'); 7 | } 8 | 9 | pre, 10 | code { 11 | font-family: "JetBrainsMono", Menlo, Monaco, "Courier New", monospace; 12 | } 13 | 14 | pre { 15 | padding: .5rem; 16 | line-height: 1.25; 17 | overflow-x: scroll; 18 | } 19 | 20 | @media print { 21 | *, 22 | *:before, 23 | *:after { 24 | background: transparent !important; 25 | color: #000 !important; 26 | box-shadow: none !important; 27 | text-shadow: none !important; 28 | } 29 | 30 | a, 31 | a:visited { 32 | text-decoration: underline; 33 | } 34 | 35 | a[href]:after { 36 | content: " (" attr(href) ")"; 37 | } 38 | 39 | abbr[title]:after { 40 | content: " (" attr(title) ")"; 41 | } 42 | 43 | a[href^="#"]:after, 44 | a[href^="javascript:"]:after { 45 | content: ""; 46 | } 47 | 48 | pre, 49 | blockquote { 50 | border: 1px solid #999; 51 | page-break-inside: avoid; 52 | } 53 | 54 | thead { 55 | display: table-header-group; 56 | } 57 | 58 | tr, 59 | img { 60 | page-break-inside: avoid; 61 | } 62 | 63 | img { 64 | max-width: 100% !important; 65 | } 66 | 67 | p, 68 | h2, 69 | h3 { 70 | orphans: 3; 71 | widows: 3; 72 | } 73 | 74 | h2, 75 | h3 { 76 | page-break-after: avoid; 77 | } 78 | } 79 | 80 | a, 81 | a:visited { 82 | color: #01ff70; 83 | } 84 | 85 | a:hover, 86 | a:focus, 87 | a:active { 88 | color: #2ecc40; 89 | } 90 | 91 | .retro-no-decoration { 92 | text-decoration: none; 93 | } 94 | 95 | html { 96 | font-size: 12px; 97 | } 98 | 99 | @media screen and (min-width: 32rem) and (max-width: 48rem) { 100 | html { 101 | font-size: 15px; 102 | } 103 | } 104 | 105 | @media screen and (min-width: 48rem) { 106 | html { 107 | font-size: 16px; 108 | } 109 | } 110 | 111 | body { 112 | line-height: 1.85; 113 | } 114 | 115 | p, 116 | .retro-p { 117 | font-size: 1rem; 118 | margin-bottom: 1.3rem; 119 | } 120 | 121 | h1, 122 | .retro-h1, 123 | h2, 124 | .retro-h2, 125 | h3, 126 | .retro-h3, 127 | h4, 128 | .retro-h4 { 129 | margin: 1.414rem 0 .5rem; 130 | font-weight: inherit; 131 | line-height: 1.42; 132 | } 133 | 134 | h1, 135 | .retro-h1 { 136 | margin-top: 0; 137 | font-size: 3.998rem; 138 | } 139 | 140 | h2, 141 | .retro-h2 { 142 | font-size: 2.827rem; 143 | } 144 | 145 | h3, 146 | .retro-h3 { 147 | font-size: 1.999rem; 148 | } 149 | 150 | h4, 151 | .retro-h4 { 152 | font-size: 1.414rem; 153 | } 154 | 155 | h5, 156 | .retro-h5 { 157 | font-size: 1.121rem; 158 | } 159 | 160 | h6, 161 | .retro-h6 { 162 | font-size: .88rem; 163 | } 164 | 165 | small, 166 | .retro-small { 167 | font-size: .707em; 168 | } 169 | 170 | /* https://github.com/mrmrs/fluidity */ 171 | 172 | img, 173 | canvas, 174 | iframe, 175 | video, 176 | svg, 177 | select, 178 | textarea { 179 | max-width: 100%; 180 | } 181 | 182 | html, 183 | body { 184 | background-color: #222; 185 | min-height: 100%; 186 | } 187 | 188 | html { 189 | font-size: 18px; 190 | } 191 | 192 | body { 193 | color: #fafafa; 194 | font-family: "JetBrainsMono", monospace; 195 | line-height: 1.45; 196 | margin: 6rem auto 1rem; 197 | max-width: 48rem; 198 | padding: .25rem; 199 | } 200 | 201 | pre { 202 | background-color: #333; 203 | } 204 | 205 | blockquote { 206 | border-left: 3px solid #01ff70; 207 | padding-left: 1rem; 208 | } 209 | -------------------------------------------------------------------------------- /code4me-server/static/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code4me-me/code4me/d8429d41c42ac16957659b95380908458a7c2980/code4me-server/static/favicon-16x16.png -------------------------------------------------------------------------------- /code4me-server/static/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code4me-me/code4me/d8429d41c42ac16957659b95380908458a7c2980/code4me-server/static/favicon-32x32.png -------------------------------------------------------------------------------- /code4me-server/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code4me-me/code4me/d8429d41c42ac16957659b95380908458a7c2980/code4me-server/static/favicon.ico -------------------------------------------------------------------------------- /code4me-server/static/fonts/JetBrainsMono-Regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code4me-me/code4me/d8429d41c42ac16957659b95380908458a7c2980/code4me-server/static/fonts/JetBrainsMono-Regular.woff2 -------------------------------------------------------------------------------- /code4me-server/static/mstile-144x144.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code4me-me/code4me/d8429d41c42ac16957659b95380908458a7c2980/code4me-server/static/mstile-144x144.png -------------------------------------------------------------------------------- /code4me-server/static/mstile-150x150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code4me-me/code4me/d8429d41c42ac16957659b95380908458a7c2980/code4me-server/static/mstile-150x150.png -------------------------------------------------------------------------------- /code4me-server/static/mstile-310x150.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code4me-me/code4me/d8429d41c42ac16957659b95380908458a7c2980/code4me-server/static/mstile-310x150.png -------------------------------------------------------------------------------- /code4me-server/static/mstile-310x310.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code4me-me/code4me/d8429d41c42ac16957659b95380908458a7c2980/code4me-server/static/mstile-310x310.png -------------------------------------------------------------------------------- /code4me-server/static/mstile-70x70.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code4me-me/code4me/d8429d41c42ac16957659b95380908458a7c2980/code4me-server/static/mstile-70x70.png -------------------------------------------------------------------------------- /code4me-server/static/safari-pinned-tab.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 7 | 8 | Created by potrace 1.14, written by Peter Selinger 2001-2017 9 | 10 | 12 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /code4me-server/static/site.webmanifest: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Code4Me", 3 | "short_name": "Code4Me", 4 | "icons": [ 5 | { 6 | "src": "/static/android-chrome-192x192.png", 7 | "sizes": "192x192", 8 | "type": "image/png" 9 | }, 10 | { 11 | "src": "/static/android-chrome-512x512.png", 12 | "sizes": "512x512", 13 | "type": "image/png" 14 | } 15 | ], 16 | "theme_color": "#ffffff", 17 | "background_color": "#ffffff", 18 | "display": "standalone" 19 | } 20 | -------------------------------------------------------------------------------- /code4me-server/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Code4Me 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | {{ md|safe }} 19 | 20 | 21 | -------------------------------------------------------------------------------- /code4me-vsc-plugin/.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules/** -------------------------------------------------------------------------------- /code4me-vsc-plugin/.eslintrc.js: -------------------------------------------------------------------------------- 1 | /**@type {import('eslint').Linter.Config} */ 2 | // eslint-disable-next-line no-undef 3 | module.exports = { 4 | root: true, 5 | parser: '@typescript-eslint/parser', 6 | plugins: [ 7 | '@typescript-eslint', 8 | ], 9 | extends: [ 10 | 'eslint:recommended', 11 | 'plugin:@typescript-eslint/recommended', 12 | ], 13 | rules: { 14 | 'semi': [2, "always"], 15 | '@typescript-eslint/no-unused-vars': 0, 16 | '@typescript-eslint/no-explicit-any': 0, 17 | '@typescript-eslint/explicit-module-boundary-types': 0, 18 | '@typescript-eslint/no-non-null-assertion': 0, 19 | } 20 | }; -------------------------------------------------------------------------------- /code4me-vsc-plugin/.gitignore: -------------------------------------------------------------------------------- 1 | out 2 | node_modules 3 | .vscode-test 4 | vscode.d.test 5 | vscode.d.ts 6 | vscode.proposed.inlineCompletions.d -------------------------------------------------------------------------------- /code4me-vsc-plugin/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | // A launch configuration that compiles the extension and then opens it inside a new window 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | { 6 | "version": "0.2.0", 7 | "configurations": [ 8 | { 9 | "name": "Run Extension", 10 | "type": "extensionHost", 11 | "request": "launch", 12 | "runtimeExecutable": "${execPath}", 13 | "args": [ 14 | "--extensionDevelopmentPath=${workspaceFolder}" 15 | ], 16 | "outFiles": [ 17 | "${workspaceFolder}/out/**/*.js" 18 | ], 19 | "preLaunchTask": "${defaultBuildTask}" 20 | }, 21 | { 22 | "name": "Extension Tests", 23 | "type": "extensionHost", 24 | "request": "launch", 25 | "runtimeExecutable": "${execPath}", 26 | "args": [ 27 | "--extensionDevelopmentPath=${workspaceFolder}", 28 | "--extensionTestsPath=${workspaceFolder}/out/test/suite/index" 29 | ], 30 | "outFiles": [ 31 | "${workspaceFolder}/out/test/**/*.js" 32 | ], 33 | "preLaunchTask": "${defaultBuildTask}" 34 | } 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /code4me-vsc-plugin/.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | // { 2 | // "version": "2.0.0", 3 | // "tasks": [ 4 | // { 5 | // "type": "npm", 6 | // "script": "compile", 7 | // "group": { 8 | // "kind": "build", 9 | // "isDefault": true 10 | // }, 11 | // "problemMatcher": [], 12 | // "label": "npm: compile", 13 | // "detail": "tsc -p ./" 14 | // } 15 | // ] 16 | // } 17 | 18 | // NOTE: Not sure how the above came about. 19 | // It prevents the plugin from hot-reloading when developing. 20 | // The below is the correct configuration for the tasks.json file according to vscode themselves. 21 | // Specifically, `"script": "compile"` should be `"script": "watch"` to, you guessed it, watch for changes! 22 | 23 | // See https://go.microsoft.com/fwlink/?LinkId=733558 24 | // for the documentation about the tasks.json format 25 | { 26 | "version": "2.0.0", 27 | "tasks": [ 28 | { 29 | "type": "npm", 30 | "script": "watch", 31 | "problemMatcher": "$tsc-watch", 32 | "isBackground": true, 33 | "presentation": { 34 | "reveal": "never" 35 | }, 36 | "group": { 37 | "kind": "build", 38 | "isDefault": true 39 | } 40 | } 41 | ] 42 | } 43 | -------------------------------------------------------------------------------- /code4me-vsc-plugin/.vscodeignore: -------------------------------------------------------------------------------- 1 | .vscode/** 2 | **/*.ts 3 | **/*.map 4 | .gitignore 5 | **/tsconfig.json 6 | **/tsconfig.base.json 7 | contributing.md 8 | .travis.yml -------------------------------------------------------------------------------- /code4me-vsc-plugin/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | ## [v1.0.9] - 2022-08-17 3 | ### Fixed 4 | - Fix improper syntax upon suggestion ending in '),'. 5 | 6 | ### Changed 7 | - Send along ext (excluding the dot). 8 | 9 | ## [v1.0.8] - 2022-07-19 10 | ### Added 11 | - **Optional** data storage of context code option, by default false. 12 | - Settings page. 13 | - Send along .ext instead of natural language of ext. 14 | - Receive multiple Code4Me suggestions. 15 | 16 | ### Fixed 17 | - Syntax breaking suggestions with duplicate brackets at the end. 18 | 19 | ## [v1.0.7] - 2022-06-15 20 | ### Added 21 | - Identifying of manual or automatic trigger. 22 | - Sends version of extension to API. 23 | - Add two keywords: `with` and `except`. 24 | 25 | ## [v1.0.6] - 2022-06-11 26 | ### Changed 27 | - Code4Me now works on all languages. 28 | 29 | ## [v1.0.5] - 2022-06-10 30 | ### Changed 31 | - Edited readme. 32 | - Changed data destruction from at the end of the thesis to at the end of the study. 33 | 34 | ## [v1.0.4] - 2022-06-09 35 | ### Added 36 | - Proper survey link. 37 | 38 | ## [v1.0.3] - 2022-06-09 39 | ### Added 40 | - Prompts a window when request limit has been reached. 41 | - Prompts a window if an unknown error occurs. 42 | 43 | ## [v1.0.2] - 2022-06-07 44 | ### Changed 45 | - 2048 character transmitted to 3992 characters (average token length represented in characters). 46 | 47 | ## [v1.0.1] - 2022-06-06 48 | ### Added 49 | - Survey prompt 50 | - Implicit verification 51 | 52 | ### Changed 53 | - Readme adjusted 54 | 55 | ### Fixed 56 | - Different indentations could cause the plugin to error. 57 | 58 | ## [v1.0.0] - 2022-05-27 59 | - Initial release -------------------------------------------------------------------------------- /code4me-vsc-plugin/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2000-2021 JetBrains s.r.o. 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /code4me-vsc-plugin/README.md: -------------------------------------------------------------------------------- 1 | ## Code4Me 2 | 3 | 4 | **AI code completion**, seamlessly integrated with IntelliSense. Manually trigger completions with `^Space`, and cycle with `^N` and `^P` for the next and previous completion. 5 | 6 | The code suggestions from Code4Me are displayed with a unicode "❮／❯" representation of the logo. Code4Me does not hinder native auto completion or other extensions. For more information, visit the [Code4Me](https://code4me.me) website. 7 | 8 | 9 | ### Goal 10 | Code4Me exists for research purposes at the [Delft University of Technology](https://www.tudelft.nl/). Code4Me is a supportive tool to gather data about developers' interactions with code-completion LLMs. 11 | 12 | ### Data Collection 13 | The plugin does **not** collect identifiable data. The plugin does collect the following *anonymised* usage data: 14 | 15 | - Close context around the cursor. 16 | - The generated completion, and whether it was accepted. 17 | - Verified insertion. 18 | - The plugin tracks the line the code was inserted and sends that line to the server after a timeout. 19 | - Time of completion. 20 | 21 | Code4Me is in full compliance with the GDPR. The data collected will remain on the servers of TU Delft until the end of the study; and will not be published. By using Code4Me, you give permission for data collection. Thank you for supporting open-source research! 22 | -------------------------------------------------------------------------------- /code4me-vsc-plugin/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "code4me-plugin", 3 | "displayName": "Code4Me", 4 | "description": "Language model code completion.", 5 | "author": "Code4Me", 6 | "license": "Apache-2.0", 7 | "version": "1.1.0", 8 | "categories": [ 9 | "Machine Learning", 10 | "Programming Languages", 11 | "Education" 12 | ], 13 | "keywords": [ 14 | "AutoComplete", 15 | "Code Completion", 16 | "Python", 17 | "Language Model" 18 | ], 19 | "repository": { 20 | "type": "git", 21 | "url": "https://github.com/code4me-me/code4me" 22 | }, 23 | "icon": "pluginIcon.png", 24 | "publisher": "Code4Me", 25 | "engines": { 26 | "vscode": "^1.47.0" 27 | }, 28 | "activationEvents": [ 29 | "onStartupFinished" 30 | ], 31 | "main": "./out/extension.js", 32 | "contributes": { 33 | "commands": [ 34 | { 35 | "command": "code4me.action.triggerSuggest", 36 | "title": "Trigger Suggest (code4me)" 37 | } 38 | ], 39 | "keybindings": [ 40 | { 41 | "command": "code4me.action.triggerSuggest", 42 | "key": "ctrl+space", 43 | "mac": "ctrl+space", 44 | "when": "editorTextFocus && !suggestWidgetVisible" 45 | } 46 | ], 47 | "configuration": { 48 | "title": "Code4Me", 49 | "properties": { 50 | "code4me.storeContext": { 51 | "type": "boolean", 52 | "default": false, 53 | "description": "Allow the context of your code to be stored anonymously onto the servers of TU Delft for research purposes. This data will be removed after 3 months.", 54 | "scope": "window" 55 | }, 56 | "code4me.promptDataStorage": { 57 | "type": "boolean", 58 | "default": true, 59 | "description": "Prompt the data storage request upon start-up.", 60 | "scope": "window" 61 | }, 62 | "code4me.promptSurvey": { 63 | "type": "boolean", 64 | "default": true, 65 | "description": "Prompt the survey once in a while.", 66 | "scope": "window" 67 | } 68 | } 69 | } 70 | }, 71 | "scripts": { 72 | "vscode:prepublish": "npm run compile", 73 | "compile": "tsc -p ./", 74 | "lint": "eslint . --ext .ts,.tsx", 75 | "watch": "tsc -watch -p ./" 76 | }, 77 | "devDependencies": { 78 | "@types/csprng": "^0.1.2", 79 | "@types/node": "^12.12.0", 80 | "@types/node-fetch": "^2.6.1", 81 | "@types/uuid": "^8.3.4", 82 | "@types/vscode": "^1.47.0", 83 | "@typescript-eslint/eslint-plugin": "^5.19.0", 84 | "@typescript-eslint/parser": "^5.19.0", 85 | "eslint": "^8.13.0", 86 | "typescript": "^4.6.3", 87 | "vscode-dts": "^0.3.3" 88 | }, 89 | "dependencies": { 90 | "csprng": "^0.1.2", 91 | "node-fetch": "^2.6.7", 92 | "uuid": "^8.3.2" 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /code4me-vsc-plugin/pluginIcon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code4me-me/code4me/d8429d41c42ac16957659b95380908458a7c2980/code4me-vsc-plugin/pluginIcon.png -------------------------------------------------------------------------------- /code4me-vsc-plugin/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "es2020", 5 | "lib": ["es2020"], 6 | "outDir": "out", 7 | "sourceMap": true, 8 | "strict": true, 9 | "rootDir": "src", 10 | "esModuleInterop": true 11 | }, 12 | "exclude": [ 13 | "node_modules", 14 | ".vscode-test" 15 | ] 16 | } -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | services: 3 | nginx: 4 | image: nginx:latest 5 | restart: always 6 | extra_hosts: 7 | - "host.docker.internal:host-gateway" 8 | depends_on: 9 | - codeforme 10 | volumes: 11 | - ./nuhginks/nginx.conf:/etc/nginx/conf.d/default.conf 12 | - ./nuhginks/cloudflare.conf:/etc/nginx/cloudflare.conf 13 | - ./nuhginks/certs:/etc/nginx/certs 14 | ports: 15 | - "443:443" 16 | codeforme: 17 | build: 18 | context: ./code4me-server 19 | dockerfile: Dockerfile 20 | args: 21 | UID: "${UID}" 22 | restart: always 23 | deploy: 24 | resources: 25 | reservations: 26 | devices: 27 | - capabilities: [gpu] 28 | volumes: 29 | - ./.cache:/codeforme/.cache 30 | - ./code4me-server/users.json:/codeforme/users.json 31 | - ./code4me-server/nltk_data:/codeforme/nltk_data 32 | - ./code4me-server/data:/codeforme/data 33 | - ./code4me-server/markdowns:/codeforme/markdowns 34 | - ./code4me-server/templates:/codeforme/templates 35 | - ./code4me-server/static:/codeforme/static 36 | - ./codegpt_checkpoint:/codeforme/codegpt_checkpoint 37 | - ./code4me-server/models:/codeforme/models 38 | - ./code4me-server/data_aral:/codeforme/data_aral 39 | expose: 40 | - 3000 41 | environment: 42 | - CODE4ME_CUDA=True 43 | - "SURVEY_LINK=https://docs.google.com/forms/d/1uES5o6etbWEZVNpUc0TGisDJXIIuj5hIwa9tF7_FQuw/?entry.1566855902={user_id}" 44 | - CODEGPT_CHECKPOINT_PATH=/codeforme/codegpt_checkpoint 45 | - UNIXCODER_DEVICE=cuda:1 46 | - INCODER_DEVICE=cuda:0 47 | - CODEGPT_DEVICE=cuda:1 48 | --------------------------------------------------------------------------------