├── .github └── workflows │ └── release.yaml ├── .gitignore ├── DEBUG.md ├── Dockerfile ├── LICENSE ├── README.ch.md ├── README.md ├── README.rst ├── build-image.sh ├── go ├── cmd │ └── server.go └── pkg │ └── tabnine │ └── tabnine.go ├── images ├── chrome-console.png ├── chrome-inspect.png ├── demo.gif ├── remote-server-url-config.jpg └── show-original-complete.gif ├── setup.cfg ├── setup.py ├── src └── jupyter_tabnine │ ├── __init__.py │ ├── _version.py │ ├── handler.py │ ├── static │ ├── README.md │ ├── main.css │ ├── main.js │ └── tabnine.yaml │ └── tabnine.py ├── start-server.sh └── stop-server.sh /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Create Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*.*.*' 7 | jobs: 8 | release: 9 | name: Create Release 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout code 13 | uses: actions/checkout@v2 14 | - name: Setup Python 15 | uses: actions/setup-python@v2 16 | with: 17 | python-version: '3.x' 18 | - name: Tag Version 19 | id: tag-version 20 | run: echo ::set-output name=RELEASE_VERSION::$(echo ${GITHUB_REF:10}) 21 | - name: Package Version 22 | id: setup-version 23 | run: echo ::set-output name=SETUP_VERSION::$(python3 setup.py --version) 24 | - name: Tag & setup.py Versions Not Matching 25 | if: ${{ format('v{0}', steps.setup-version.outputs.SETUP_VERSION) != steps.tag-version.outputs.RELEASE_VERSION}} 26 | run: exit 1 27 | - name: Build package 28 | id: build_package 29 | run: | 30 | pip3 install wheel && \ 31 | python3 setup.py sdist bdist_wheel 32 | if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') 33 | - name: Create Release 34 | id: create_release 35 | uses: actions/create-release@v1 36 | env: 37 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 38 | with: 39 | tag_name: ${{ github.ref }} 40 | release_name: Release ${{ github.ref }} 41 | draft: false 42 | prerelease: false 43 | - name: Publish package 44 | if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') 45 | uses: codota/gh-action-pypi-publish@master 46 | with: 47 | user: ${{ secrets.PYPI_USERNAME }} 48 | password: ${{ secrets.PYPI_TOKEN }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .ipynb_checkpoints 3 | build 4 | dist 5 | *.egg-info 6 | *.ipynb 7 | go/cmd/binaries 8 | .idea 9 | -------------------------------------------------------------------------------- /DEBUG.md: -------------------------------------------------------------------------------- 1 | # How To Debug 2 | 3 | A lot of users release issues reporting problems with the autofill proposition, here's some steps to get more information about what's wrong. 4 | 5 | * Open Chrome Inspect 6 | 7 | ![Chrome Inspect](images/chrome-inspect.png) 8 | 9 | * Open the console and see the logs( All logs of this plugin start with `[nbextensions/jupyter_tabnine/main]` ) 10 | 11 | ![Chrome Console](images/chrome-console.png) 12 | 13 | * If you can't figure out what's wrong, please do not hesitate to release an issue and report the detail of logs in that issue. 14 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-alpine3.11 2 | COPY go/cmd/server /usr/local/bin/tabnine-server 3 | RUN apk add build-base && pip install python-language-server 4 | RUN chmod 777 /usr/local/bin/tabnine-server 5 | EXPOSE 8080 6 | ENTRYPOINT ["/usr/local/bin/tabnine-server", "-libBaseDir=/usr/local/lib", "-port=8080"] 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 WU WENMIN 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.ch.md: -------------------------------------------------------------------------------- 1 | # Jupyter Notebook的TabNine自动补全插件 2 | ![jupyter-tabnine](images/demo.gif) 3 | 4 | *Read this in other languages: [English](README.md), [中文](README.ch.md)* 5 | 6 | Jupyter Notebook上基于TabNine的自动补全插件，实现基于深度学习的代码自动补全功能。 7 | 8 | 其他TabNine插件的实现版本通过在client端启动一个TabNine子进程，并通过管道读写来和TabNine子进程通信。这在Jupyter Notebook上是没法直接实现的， 9 | 因为Jupyter Notebook客户端插件不支持安装第三方库，原有的库又不支持启动子进程。 10 | 11 | 本项目通过分别实现一个Jupyter Notebook插件和一个Jupyter Server插件来解决这个问题。客户端和服务器通过HTTP来通信。 12 | 基于`JavaScript`的客户端插件根据文件内容构造请求数据，并向Server插件发送请求。基于`Python`的server插件在初始化时启动一个子进程来执行`TabNine`二进制文件， 13 | 在收到客户端请求后，将请求通过管道发送给`TabNine`并将TabNine的返回结果发送给客户端。客户端收到结果后解析生成HTML组件并响应键盘事件。 14 | 15 | ## 安装 16 | 整个安装步骤分为：安装python包、安装客户端插件和`enable`客户端、服务器插件，所有安装步骤都可以通过`pip`和`jupyter`命令完成。 17 | 18 | ### 1. 安装python包（以下任选其一） 19 | 20 | * 从github repo 安装： `pip3 install https://github.com/wenmin-wu/jupyter-tabnine/archive/master.zip [--user][--upgrade]` 21 | * 或者 从`pypi` 安装： `pip3 install jupyter-tabnine [--user][--upgrade]` 22 | * 或者 从源码安装： 23 | ```Bash 24 | git clone https://github.com/wenmin-wu/jupyter-tabnine.git 25 | python3 setup.py install 26 | ``` 27 | 28 | ### 2. 安装Notebook插件 29 | `jupyter nbextension install --py jupyter_tabnine [--user|--sys-prefix|--system]` 30 | 31 | ### 3. enable Notebook 和 Server 插件 32 | ```Bash 33 | jupyter nbextension enable --py jupyter_tabnine [--user|--sys-prefix|--system] 34 | jupyter serverextension enable --py jupyter_tabnine [--user|--sys-prefix|--system] 35 | ``` 36 | 37 | --- 38 | 如果你的Jupyter版本在4.2以前，因为`--py`在4.2版本以前的jupyter上没法用，所以步骤1以后操作比较tricky。你需要先找到`jupyter_tabnine`的安装路径， 39 | 然后手动安装。安装路径可以通过下面的命令找到： 40 | ```Python 41 | python -c "import os.path as p; from jupyter_tabnine import __file__ as f, _jupyter_nbextension_paths as n; print(p.normpath(p.join(p.dirname(f), n()[0]['src'])))" 42 | ``` 43 | 然后执行： 44 | ```Bash 45 | jupyter nbextension install 46 | jupyter nbextension enable jupyter_tabnine/main 47 | jupyter serverextension enable 48 | ``` 49 | `` 是第一个`python`命令的输出结果。 50 | 51 | ## 使用说明 52 | 在安装完成后，该插件自动处于激活状态，任何字母、操作符、数字键入都会trigger自动补全。 53 | 54 | * 有时你可能想临时查看Jupyter原来的补全结果，可以按`Shift` + `空格` 55 | ![显示原来补全Demo](images/show-original-complete.gif) 56 | * 如果你想关掉TabNine自动补全，既可以在Notebook nbextension 的页面 disable Jupyter TabNine。也可以点击 `Help` 在弹框中找到 Jupyter TabNine把它点掉。 57 | * 远程补全服务器也是支持的。如果你想部署个Server来处理客户端插件请求，或者你们公司在`Kubernetes`上部署一个Server Cluter请看下面章节了解如何部署自动补全Server。 58 | 59 | ## 开源许可证 60 | 61 | [MIT License](LICENSE) 62 | 63 | ## 部署自动补全Server (可选) 64 | 65 | 很快添加 66 | 67 | ## Star 趋势 68 | 69 | [![Stargazers over time](https://starchart.cc/wenmin-wu/jupyter-tabnine.svg)](https://starchart.cc/wenmin-wu/jupyter-tabnine) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TabNine for Jupyter Notebook 2 | 3 | **This plugin has been created by [wenmin-wu](https://github.com/wenmin-wu).** 4 | 5 | **This plugin has been tested on MacOS, Linux and Windows, it support all these systems. For browsers it supports Chrome and Safari but not IE** 6 | 7 | If you found this plugin doesn't work for you, please debug according to [How to Debug](DEBUG.md). And if you can't figure out what's wrong, please release an issue and report the logs in detail. 8 | 9 | Thanks for using this plugin! Have fun! :) 10 | 11 | *Read this in other languages: [English](README.md), [中文](README.ch.md)* 12 | 13 | ![jupyter-tabnine](images/demo.gif) 14 | 15 | This extension for Jupyter Notebook enables the use of coding auto-completion based on Deep Learning. 16 | 17 | Other client plugins of TabNine require starting a child process for TabNine binary and using Pipe for communication. This can’t be done with Jupyter Notebook, since child process can’t be created with JQuery and Jupyter Notebook doesn’t provide any way for adding third-part js libs to plugins. 18 | 19 | In this repository, it is achived by developing a client plugin and a server plugin for Jupyter Notebook. The client plugin generate request info and send http request to the server plugin. The server plugin pass the request info to it’s client process (TabNine) and return the request to client plugin. 20 | 21 | ## Installation 22 | 23 | I saw a lot users came across problems due to didn't install and configure this plugin correctly, the simplest way to install and configure this plugin is by issuing following command: 24 | 25 | ``` 26 | pip3 install jupyter-tabnine --user 27 | jupyter nbextension install --py jupyter_tabnine --user 28 | jupyter nbextension enable --py jupyter_tabnine --user 29 | jupyter serverextension enable --py jupyter_tabnine --user 30 | ``` 31 | 32 | If you want to install and configure in a customized way, you can refer to following: 33 | 34 | The extension consists of a pypi package that includes a javascript 35 | notebook extension, along with a python jupyter server extension. Since Jupyter 4.2, pypi is the recommended way to distribute nbextensions. The extension can be installed: 36 | 37 | * from the master version on the github repo (this will be always the most recent version) 38 | 39 | * via pip for the version hosted on pypi 40 | 41 | From the github repo or from Pypi, 42 | 1. install the package 43 | * `pip3 install https://github.com/wenmin-wu/jupyter-tabnine/archive/master.zip [--user][--upgrade]` 44 | * or `pip3 install jupyter-tabnine [--user][--upgrade]` 45 | * or clone the repo and install 46 | 47 | `git clone https://github.com/wenmin-wu/jupyter-tabnine.git` 48 | 49 | `python3 setup.py install` 50 | 2. install the notebook extension 51 | `jupyter nbextension install --py jupyter_tabnine [--user|--sys-prefix|--system]` 52 | 53 | 3. and enable notebook extension and server extension 54 | ```Bash 55 | jupyter nbextension enable --py jupyter_tabnine [--user|--sys-prefix|--system] 56 | jupyter serverextension enable --py jupyter_tabnine [--user|--sys-prefix|--system] 57 | ``` 58 | --- 59 | For **Jupyter versions before 4.2**, the situation after step 1 is more tricky, since the --py option isn’t available, so you will have to find the location of the source files manually as follows (instructions adapted from [@jcb91](https://github.com/jcb91)’s jupyter_highlight_selected_word). Execute 60 | 61 | ```Python 62 | python -c "import os.path as p; from jupyter_tabnine import __file__ as f, _jupyter_nbextension_paths as n; print(p.normpath(p.join(p.dirname(f), n()[0]['src'])))" 63 | ``` 64 | Then, issue 65 | ```Bash 66 | jupyter nbextension install 67 | jupyter nbextension enable jupyter_tabnine/main 68 | jupyter serverextension enable 69 | ``` 70 | where `` is the output of the first python command. 71 | 72 | ## Usage 73 | 74 | * Jupyter TabNine will be active after being installed. Sometimes, you may want to show the Jupyter original complete temporally, then click `shift` + `space`. 75 | 76 | ![show original complete demo](images/show-original-complete.gif) 77 | * Remote auto-completion server is also supported. You may want this to speed up the completion request handing. Or maybe your company want to deploy a compeltion server cluster that services everyone. Read following to learn how to deploy remote server. 78 | 79 | ## Uninstallation 80 | To uninstall TabNine plugin from mac/linux run the following commands: 81 | ```Bash 82 | jupyter nbextension uninstall --py jupyter_tabnine 83 | pip3 uninstall jupyter-tabnine 84 | ``` 85 | 86 | ## Contributing 87 | 88 | Pull requests are welcome. For major changes, please open an issue first to discuss what you would like to change. 89 | 90 | Please make sure to update tests as appropriate. 91 | 92 | ## License 93 | 94 | [MIT License](LICENSE) 95 | 96 | ## Remote Completion Server Deployment 97 | It's useful to deploy a remote tabnine server if you don't want to occupy too much local resources. You can build, deploy and config a remote tabnine server according to the following steps. 98 | 99 | **NOTE:** You need to install jupyter-tabnine with `pip3 install https://github.com/wenmin-wu/jupyter-tabnine/archive/master.zip`, because the version which fix this plugin with remote server problem haven't been relased to PyPi. 100 | ### Build Server Image 101 | **I have uploaded an image to Docker Hub, skip this section if you prefer to use it directly.** 102 | * Install the golang (recommended version is 1.13 - 1.14) 103 | * Issue `go get -v github.com/wenmin-wu/jupyter-tabnine/go/cmd` 104 | * Issue `cd $HOME/go/src/github.com/wenmin-wu/jupyter-tabnine` 105 | * Issue `bash ./build-image.sh` 106 | ### Start Server 107 | **Change the image name in this bash script to `wuwenmin1991/tabnine-server:1.0` if you did't build your own image** 108 | * Simply issue `bash start-server.sh` 109 | 110 | ### Configure Under Nbextensions 111 | * Please [install Nbextensions](https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/install.html) if you haven't installed. 112 | * Open Jupyter Notebook and go to the Nbextensions setting page, click **Jupyter TabNine**, scroll down and fill in the remote server url, e.g. 113 | ![remote-server-url-config](images/remote-server-url-config.jpg) 114 | ### Stop Server 115 | * Simply issue `bash stop-server.sh` 116 | 117 | ## Stargazers over time 118 | 119 | [![Stargazers over time](https://starchart.cc/wenmin-wu/jupyter-tabnine.svg)](https://starchart.cc/wenmin-wu/jupyter-tabnine) 120 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | TabNine for Jupyter Notebook 2 | ============================================== 3 | This extension for Jupyter Notebook enables the use of 4 | coding auto-completion based on Deep Learning. 5 | 6 | Other client plugins of TabNine require starting a child process for TabNine binary 7 | and using Pipe for communication. This can't be done with Jupyter Notebook, since child process 8 | can't be created with JQuery and Jupyter Notebook doesn't provide any way for adding third-part js libs to plugins. 9 | 10 | In this repository, it is achived by developing a client plugin and a server plugin for Jupyter Notebook. 11 | The client plugin generate request info and send http request to the server plugin. 12 | The server plugin pass the request info to it's client process (TabNine) and return the request to client plugin. 13 | 14 | Installation 15 | ------------ 16 | The extension consists of a pypi package that includes a javascript 17 | notebook extension, along with a python jupyter server extension. 18 | Since Jupyter 4.2, pypi is the recommended way to distribute nbextensions. 19 | The extension can be installed 20 | 21 | - from the master version on the github repo (this will be always the most recent version) 22 | - via pip for the version hosted on pypi 23 | 24 | From the github repo or from Pypi, 25 | 26 | 1. install the package 27 | 28 | - ``pip3 install https://github.com/wenmin-wu/jupyter-tabnine/archive/master.zip [--user][--upgrade]`` 29 | - or ``pip3 install jupyter-tabnine [--user][--upgrade]`` 30 | - or clone the repo and install 31 | ``git clone https://github.com/wenmin-wu/jupyter-tabnine.git`` 32 | 33 | ``python3 setup.py install`` 34 | 35 | 2. install the notebook extension 36 | 37 | :: 38 | 39 | jupyter nbextension install --py jupyter_tabnine [--user|--sys-prefix|--system] 40 | 41 | 3. and enable notebook extension and server extension 42 | 43 | :: 44 | 45 | jupyter nbextension enable --py jupyter_tabnine [--user|--sys-prefix|--system] 46 | jupyter serverextension enable --py jupyter_tabnine [--user|--sys-prefix|--system] 47 | 48 | ------------ 49 | 50 | For Jupyter versions before 4.2, the situation after step 1 is more 51 | tricky, since the ``--py`` option isn't available, so you will have to 52 | find the location of the source files manually as follows (instructions 53 | adapted from [@jcb91](https://github.com/jcb91)'s 54 | `jupyter\_highlight\_selected\_word `__). 55 | Execute 56 | 57 | :: 58 | 59 | python -c "import os.path as p; from jupyter_tabnine import __file__ as f, _jupyter_nbextension_paths as n; print(p.normpath(p.join(p.dirname(f), n()[0]['src'])))" 60 | 61 | Then, issue 62 | 63 | :: 64 | 65 | jupyter nbextension install 66 | jupyter nbextension enable jupyter_tabnine/jupyter_tabnine 67 | 68 | where ```` is the output of the first python 69 | command. 70 | 71 | Tips 72 | ------------ 73 | - A shortcut is added to let you switch between Jupyter raw completion and TabNine auto-competion. Just enter ``shift`` + ``space`` when you want raw completion of Jupyter :) 74 | - Remote auto-completion server is also supported. You may want this to speed up the completion request handing. Or maybe your company want to deploy a compeltion server cluster that services everyone. Refer https://github.com/wenmin-wu/jupyter-tabnine to learn how to deploy remote server. 75 | -------------------------------------------------------------------------------- /build-image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | if ! type "docker" > /dev/null; then 4 | echo "Please install docker first!" 5 | fi 6 | 7 | wd=$(pwd) 8 | container_wd=${wd#${HOME}} 9 | echo "container working directory: ${container_wd}" 10 | docker run --rm -v ${HOME}/go:/go golang:1.14-alpine3.11 \ 11 | go build -o ${container_wd}/go/cmd/server ${container_wd}/go/cmd/server.go 12 | 13 | docker build -t tabnine-server:latest . 14 | -------------------------------------------------------------------------------- /go/cmd/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | "net/url" 9 | "os" 10 | "os/signal" 11 | "strings" 12 | "syscall" 13 | 14 | "github.com/wenmin-wu/jupyter-tabnine/go/pkg/tabnine" 15 | ) 16 | 17 | func main() { 18 | var libBaseDir string 19 | 20 | var port int 21 | 22 | flag.StringVar(&libBaseDir, "libBaseDir", "./", "base directory of tabnine binaries") 23 | flag.IntVar(&port, "port", 9999, "Server port") 24 | flag.Parse() 25 | 26 | tn, err := tabnine.NewTabNine(libBaseDir) 27 | if err != nil { 28 | log.Fatal(err) 29 | } 30 | 31 | http.HandleFunc("/tabnine", func(w http.ResponseWriter, r *http.Request) { 32 | // fix cross-domain request problem 33 | w.Header().Set("Access-Control-Allow-Origin", "*") 34 | urlStr, _ := url.QueryUnescape(r.URL.String()) 35 | index := strings.Index(urlStr, "=") 36 | data := []byte(urlStr[index+1:]) 37 | _, err = w.Write(tn.Request(data)) 38 | }) 39 | 40 | numSignals := 3 41 | ch := make(chan os.Signal, numSignals) 42 | 43 | signal.Notify(ch, os.Interrupt, syscall.SIGTERM, syscall.SIGINT) 44 | 45 | go func() { 46 | signalType := <-ch 47 | signal.Stop(ch) 48 | tn.Close() 49 | log.Printf("Signal Type: %s\n", signalType) 50 | os.Exit(0) 51 | }() 52 | log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil)) 53 | } 54 | -------------------------------------------------------------------------------- /go/pkg/tabnine/tabnine.go: -------------------------------------------------------------------------------- 1 | package tabnine 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "log" 10 | "net/http" 11 | "os" 12 | "os/exec" 13 | "path/filepath" 14 | "runtime" 15 | "strings" 16 | "sync" 17 | 18 | "github.com/coreos/go-semver/semver" 19 | ) 20 | 21 | type TabNine struct { 22 | baseDir string 23 | cmd *exec.Cmd 24 | outReader *bufio.Reader 25 | mux sync.Mutex 26 | inPipeWriter *io.PipeWriter 27 | outPipeWriter *io.PipeWriter 28 | inPipeReader *io.PipeReader 29 | outPipeReader *io.PipeReader 30 | completeRes *AutocompleteResult 31 | emptyRes []byte 32 | } 33 | 34 | type AutocompleteResult struct { 35 | OldPrefix string `json:"old_prefix"` 36 | Results []*ResultEntry `json:"results"` 37 | UserMessage []string `json:"user_message"` 38 | } 39 | 40 | type ResultEntry struct { 41 | NewPrefix string `json:"new_prefix"` 42 | OldSuffix string `json:"old_suffix"` 43 | NewSuffix string `json:"new_suffix"` 44 | Details string `json:"detail"` 45 | } 46 | 47 | const ( 48 | updateVersionUrl = "https://update.tabnine.com/version" 49 | downloadUrlPrefix = "https://update.tabnine.com" 50 | ) 51 | 52 | var systemMap = map[string]string{ 53 | "darwin": "apple-darwin", 54 | "linux": "unknown-linux-gnu", 55 | "windows": "pc-windows-gnu", 56 | } 57 | 58 | func NewTabNine(baseDir string) (*TabNine, error) { 59 | empty := AutocompleteResult{} 60 | emptyRes, _ := json.Marshal(empty) 61 | tabnine := &TabNine{ 62 | baseDir: baseDir, 63 | completeRes: &empty, 64 | emptyRes: emptyRes, 65 | } 66 | err := tabnine.init() 67 | return tabnine, err 68 | } 69 | 70 | func (t *TabNine) init() (err error) { 71 | log.Println("TabNine Initializing") 72 | // download if needed 73 | var binaryPath string 74 | var wg sync.WaitGroup 75 | wg.Add(1) 76 | go func(wg *sync.WaitGroup) { 77 | binaryPath, err = t.getBinaryPath() 78 | wg.Done() 79 | }(&wg) 80 | t.inPipeReader, t.inPipeWriter = io.Pipe() 81 | t.outPipeReader, t.outPipeWriter = io.Pipe() 82 | wg.Wait() 83 | if err == nil { 84 | t.cmd = exec.Command(binaryPath, "--client=jupyter-server") 85 | t.cmd.Stdin = t.inPipeReader 86 | t.cmd.Stdout = t.outPipeWriter 87 | t.outReader = bufio.NewReader(t.outPipeReader) 88 | err = t.cmd.Start() 89 | // go t.cmd.Wait() 90 | } 91 | log.Println("TabNine Initialized") 92 | return 93 | } 94 | 95 | func (t *TabNine) downloadBinary(url, binaryPath string) (err error) { 96 | binaryDir := filepath.Dir(binaryPath) 97 | isExist, isDir := checkDir(binaryDir) 98 | if isExist && !isDir { 99 | err = os.RemoveAll(binaryDir) 100 | if err != nil { 101 | return 102 | } 103 | } 104 | 105 | if !isExist { 106 | err = os.MkdirAll(binaryDir, os.ModePerm) 107 | if err != nil { 108 | return 109 | } 110 | } 111 | 112 | resp, err := http.Get(url) 113 | if err != nil { 114 | return 115 | } 116 | 117 | if resp.StatusCode < 200 || resp.StatusCode > 299 { 118 | err = fmt.Errorf("Request update version error: %s", resp.Status) 119 | return 120 | } 121 | defer resp.Body.Close() 122 | 123 | out, err := os.Create(binaryPath) 124 | if err != nil { 125 | return 126 | } 127 | defer out.Close() 128 | _, err = io.Copy(out, resp.Body) 129 | return 130 | } 131 | 132 | func (t *TabNine) getBinaryPath() (binaryPath string, err error) { 133 | binaryDir := t.baseDir + "/binaries" 134 | if err != nil { 135 | return 136 | } 137 | needCreateDir := true 138 | isExist, isDir := checkDir(binaryDir) 139 | if isExist && isDir { 140 | needCreateDir = false 141 | } 142 | if isExist && !isDir { 143 | err = os.RemoveAll(binaryDir) 144 | if err != nil { 145 | return 146 | } 147 | } 148 | 149 | if needCreateDir { 150 | os.MkdirAll(binaryDir, os.ModePerm) 151 | } 152 | 153 | dirs, err := ioutil.ReadDir(binaryDir) 154 | if err != nil { 155 | return 156 | } 157 | 158 | var versions []*semver.Version 159 | 160 | for _, d := range dirs { 161 | versions = append(versions, semver.New(d.Name())) 162 | } 163 | semver.Sort(versions) 164 | arch := parseArch(runtime.GOARCH) 165 | sys := systemMap[strings.ToLower(runtime.GOOS)] 166 | exeName := "TabNine" 167 | if strings.ToLower(runtime.GOOS) == "windows" { 168 | exeName += ".exe" 169 | } 170 | triple := fmt.Sprintf("%s-%s", arch, sys) 171 | for _, v := range versions { 172 | binaryPath = filepath.Join(binaryDir, v.String(), triple, exeName) 173 | if isFile(binaryPath) { 174 | err = os.Chmod(binaryPath, 0755) 175 | return 176 | } 177 | } 178 | // need download 179 | resp, err := http.Get(updateVersionUrl) 180 | if err != nil { 181 | return 182 | } 183 | defer resp.Body.Close() 184 | if resp.StatusCode < 200 || resp.StatusCode > 299 { 185 | err = fmt.Errorf("Request update version error: %s", resp.Status) 186 | return 187 | } 188 | body, err := ioutil.ReadAll(resp.Body) 189 | if err != nil { 190 | return 191 | } 192 | log.Println("Binary doesn't exist, starting download.'") 193 | latestVersion := strings.TrimSpace(string(body)) 194 | log.Printf("Latest version: %s\n", latestVersion) 195 | subPath := filepath.Join(latestVersion, triple, exeName) 196 | binaryPath = filepath.Join(binaryDir, subPath) 197 | downloadUrl := fmt.Sprintf("%s/%s", downloadUrlPrefix, subPath) 198 | log.Printf("Download url: %s, Binary path: %s", downloadUrl, binaryPath) 199 | err = t.downloadBinary(downloadUrl, binaryPath) 200 | if err != nil { 201 | log.Fatal("Download failed ", err) 202 | return 203 | } 204 | err = os.Chmod(binaryPath, 0755) 205 | log.Println("Download finished.") 206 | return 207 | } 208 | 209 | func (t *TabNine) Request(data []byte) (res []byte) { 210 | t.mux.Lock() 211 | t.inPipeWriter.Write(data) 212 | t.inPipeWriter.Write([]byte("\n")) 213 | bytes, err := t.outReader.ReadBytes('\n') 214 | t.mux.Unlock() 215 | if err != nil { 216 | res = t.emptyRes 217 | return 218 | } 219 | // remove useless fields 220 | err = json.Unmarshal(bytes, t.completeRes) 221 | if err != nil { 222 | res = t.emptyRes 223 | return 224 | } 225 | res, err = json.Marshal(t.completeRes) 226 | return 227 | } 228 | 229 | func (t *TabNine) Close() { 230 | log.Println("tabnine closing... cleaning up...") 231 | t.cmd.Process.Kill() 232 | t.inPipeWriter.Close() 233 | t.outPipeWriter.Close() 234 | t.inPipeReader.Close() 235 | t.outPipeReader.Close() 236 | } 237 | 238 | func checkDir(path string) (isExist, isDir bool) { 239 | info, err := os.Stat(path) 240 | isExist = false 241 | if os.IsNotExist(err) { 242 | return 243 | } 244 | isExist = true 245 | isDir = info.IsDir() 246 | return 247 | } 248 | 249 | func isFile(path string) bool { 250 | isExist, isDir := checkDir(path) 251 | return isExist && !isDir 252 | } 253 | 254 | func isDir(path string) bool { 255 | isExist, isDir := checkDir(path) 256 | return isExist && isDir 257 | } 258 | 259 | func parseArch(arch string) string { 260 | if strings.ToLower(arch) == "amd64" { 261 | return "x86_64" 262 | } 263 | return arch 264 | } 265 | -------------------------------------------------------------------------------- /images/chrome-console.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codota/jupyter-tabnine/f05d3b00c1339f4ea8de73ee460ef3616e0d50cf/images/chrome-console.png -------------------------------------------------------------------------------- /images/chrome-inspect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codota/jupyter-tabnine/f05d3b00c1339f4ea8de73ee460ef3616e0d50cf/images/chrome-inspect.png -------------------------------------------------------------------------------- /images/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codota/jupyter-tabnine/f05d3b00c1339f4ea8de73ee460ef3616e0d50cf/images/demo.gif -------------------------------------------------------------------------------- /images/remote-server-url-config.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codota/jupyter-tabnine/f05d3b00c1339f4ea8de73ee460ef3616e0d50cf/images/remote-server-url-config.jpg -------------------------------------------------------------------------------- /images/show-original-complete.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codota/jupyter-tabnine/f05d3b00c1339f4ea8de73ee460ef3616e0d50cf/images/show-original-complete.gif -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.rst 3 | license_file = LICENSE 4 | 5 | [bdist_wheel] 6 | universal = 1 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | import os 3 | import codecs 4 | from glob import glob 5 | 6 | 7 | def read(rel_path): 8 | here = os.path.abspath(os.path.dirname(__file__)) 9 | with codecs.open(os.path.join(here, rel_path), "r") as fp: 10 | return fp.read() 11 | 12 | 13 | def get_version(rel_path): 14 | for line in read(rel_path).splitlines(): 15 | if line.startswith("__version__"): 16 | delim = '"' if '"' in line else "'" 17 | return line.split(delim)[1] 18 | else: 19 | raise RuntimeError("Unable to find version string.") 20 | 21 | 22 | with open("./README.rst") as f: 23 | readme = f.read() 24 | 25 | setuptools.setup( 26 | name="jupyter_tabnine", 27 | version=get_version("src/jupyter_tabnine/_version.py"), 28 | url="https://github.com/wenmin-wu/jupyter-tabnine", 29 | author="Wenmin Wu", 30 | long_description=readme, 31 | long_description_content_type="text/x-rst", 32 | author_email="wuwenmin1991@gmail.com", 33 | license="MIT", 34 | description="Jupyter notebook extension which support coding auto-completion based on Deep Learning", 35 | packages=setuptools.find_packages("src"), 36 | package_dir={"": "src"}, 37 | data_files=[("static", glob("src/jupyter_tabnine/static/*"))], 38 | install_requires=[ 39 | "ipython", 40 | "jupyter_core", 41 | "nbconvert", 42 | "notebook >=4.2", 43 | ], 44 | python_requires=">=3.5", 45 | classifiers=[ 46 | "Framework :: Jupyter", 47 | ], 48 | include_package_data=True, 49 | zip_safe=False, 50 | ) 51 | -------------------------------------------------------------------------------- /src/jupyter_tabnine/__init__.py: -------------------------------------------------------------------------------- 1 | from notebook.utils import url_path_join as ujoin 2 | from .handler import TabnineHandler 3 | from .tabnine import Tabnine 4 | 5 | # Jupyter Extension points 6 | def _jupyter_server_extension_paths(): 7 | return [{"module": "jupyter_tabnine",}] 8 | 9 | 10 | def _jupyter_nbextension_paths(): 11 | return [ 12 | { 13 | "section": "notebook", 14 | "dest": "jupyter_tabnine", 15 | "src": "static", 16 | "require": "jupyter_tabnine/main", 17 | } 18 | ] 19 | 20 | 21 | def load_jupyter_server_extension(nb_server_app): 22 | """ 23 | Called when the extension is loaded. 24 | 25 | Args: 26 | nb_server_app (NotebookWebApplication): handle to the Notebook webserver instance. 27 | """ 28 | web_app = nb_server_app.web_app 29 | host_pattern = ".*$" 30 | route_pattern = ujoin(web_app.settings["base_url"], "/tabnine") 31 | tabnine = Tabnine() 32 | web_app.add_handlers( 33 | host_pattern, [(route_pattern, TabnineHandler, {"tabnine": tabnine})] 34 | ) 35 | -------------------------------------------------------------------------------- /src/jupyter_tabnine/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "1.2.3" 2 | -------------------------------------------------------------------------------- /src/jupyter_tabnine/handler.py: -------------------------------------------------------------------------------- 1 | from tornado import web 2 | from urllib.parse import unquote 3 | from notebook.base.handlers import IPythonHandler 4 | 5 | 6 | class TabnineHandler(IPythonHandler): 7 | def initialize(self, tabnine): 8 | self.tabnine = tabnine 9 | 10 | @web.authenticated 11 | async def get(self): 12 | url_params = self.request.uri 13 | request_data = unquote(url_params[url_params.index("=") + 1 :]) 14 | response = self.tabnine.request(request_data) 15 | if response: 16 | self.write(response) 17 | -------------------------------------------------------------------------------- /src/jupyter_tabnine/static/README.md: -------------------------------------------------------------------------------- 1 | Jupyter TabNine 2 | ========== 3 | This extension provides code auto-completion based on deep learning. 4 | 5 | * Author: Wenmin Wu 6 | * Repository: https://github.com/wenmin-wu/jupyter-tabnine 7 | * Email: wuwenmin1991@gmail.com 8 | 9 | Options 10 | ------- 11 | 12 | * `jupytertabnine.before_line_limit`: 13 | maximum number of lines before for context generation, 14 | too many lines will slow down the request. -1 means Infinity, 15 | thus the lines will equal to number of lines before current line. 16 | 17 | * `jupytertabnine.after_line_limit`: 18 | maximum number of lines after for context generation, 19 | too many lines will slow down the request. -1 means Infinity, 20 | thus the lines will equal to number of lines after current line. 21 | 22 | * `jupytertabnine.options_limit`: 23 | maximum number of options that will be shown 24 | 25 | * `jupytertabnine.assist_active`: 26 | Enable continuous code auto-completion when notebook is first opened, or 27 | if false, only when selected from extensions menu. 28 | 29 | * `jupytertabnine.assist_delay`: 30 | delay in milliseconds between keypress & completion request. 31 | 32 | * `jupyter_tabnine.remote_server_url`: 33 | remote server url, you may want to use a remote server to handle client request. 34 | This can spped up the request handling depending on the server configuration. Refer to https://github.com/wenmin-wu/jupyter-tabnine to see how to deploy remote server. -------------------------------------------------------------------------------- /src/jupyter_tabnine/static/main.css: -------------------------------------------------------------------------------- 1 | .complete-container { 2 | display: flex; 3 | flex-wrap: wrap; 4 | justify-content: space-between; 5 | vertical-align: middle; 6 | font-family: monospace, monospace; 7 | } 8 | 9 | .complete-block { 10 | min-width: 200px; 11 | margin: 2px; 12 | padding: 1px; 13 | display: inline-block; 14 | } 15 | 16 | .user-message { 17 | margin: 2px; 18 | padding: 1px; 19 | display: list-item; 20 | } 21 | 22 | .user-message span { 23 | color: #CD5C5C; 24 | } 25 | 26 | 27 | 28 | .complete-word { 29 | width: auto; 30 | text-align: left; 31 | } 32 | 33 | .complete-detail { 34 | text-align: right; 35 | } 36 | 37 | 38 | .complete-dropdown-content { 39 | background-color: rgb(255, 255, 255, 0.85); 40 | box-shadow: 0px 8px 16px 0px rgba(0, 0, 0, 0.5); 41 | } 42 | -------------------------------------------------------------------------------- /src/jupyter_tabnine/static/main.js: -------------------------------------------------------------------------------- 1 | define([ 2 | 'base/js/namespace', 3 | 'base/js/keyboard', 4 | 'base/js/utils', 5 | 'jquery', 6 | 'module', 7 | 'notebook/js/cell', 8 | 'notebook/js/codecell', 9 | 'notebook/js/completer', 10 | 'require' 11 | ], function ( 12 | Jupyter, 13 | keyboard, 14 | utils, 15 | $, 16 | module, 17 | cell, 18 | codecell, 19 | completer, 20 | requirejs 21 | ) { 22 | 'use strict'; 23 | 24 | const N_LINES_BEFORE = 50; 25 | const N_LINES_AFTER = 50; 26 | 27 | var assistActive; 28 | 29 | var config = { 30 | assist_active: true, 31 | options_limit: 10, 32 | assist_delay: 0, 33 | before_line_limit: -1, 34 | after_line_limit: -1, 35 | remote_server_url: '', 36 | } 37 | 38 | var logPrefix = '[' + module.id + ']'; 39 | var baseUrl = utils.get_body_data('baseUrl'); 40 | var requestInfo = { 41 | "version": "1.0.7", 42 | "request": { 43 | "Autocomplete": { 44 | "filename": Jupyter.notebook.notebook_path.replace('.ipynb', '.py'), 45 | "before": "", 46 | "after": "", 47 | "region_includes_beginning": false, 48 | "region_includes_end": false, 49 | "max_num_results": config.options_limit, 50 | } 51 | } 52 | } 53 | 54 | var Cell = cell.Cell; 55 | var CodeCell = codecell.CodeCell; 56 | var Completer = completer.Completer; 57 | var keycodes = keyboard.keycodes; 58 | var specials = [ 59 | keycodes.enter, 60 | keycodes.esc, 61 | keycodes.backspace, 62 | keycodes.tab, 63 | keycodes.up, 64 | keycodes.down, 65 | keycodes.left, 66 | keycodes.right, 67 | keycodes.shift, 68 | keycodes.ctrl, 69 | keycodes.alt, 70 | keycodes.meta, 71 | keycodes.capslock, 72 | // keycodes.space, 73 | keycodes.pageup, 74 | keycodes.pagedown, 75 | keycodes.end, 76 | keycodes.home, 77 | keycodes.insert, 78 | keycodes.delete, 79 | keycodes.numlock, 80 | keycodes.f1, 81 | keycodes.f2, 82 | keycodes.f3, 83 | keycodes.f4, 84 | keycodes.f5, 85 | keycodes.f6, 86 | keycodes.f7, 87 | keycodes.f8, 88 | keycodes.f9, 89 | keycodes.f10, 90 | keycodes.f11, 91 | keycodes.f12, 92 | keycodes.f13, 93 | keycodes.f14, 94 | keycodes.f15 95 | ]; 96 | 97 | function loadCss(name) { 98 | $('').attr({ 99 | type: 'text/css', 100 | rel: 'stylesheet', 101 | href: requirejs.toUrl(name) 102 | }).appendTo('head'); 103 | } 104 | 105 | 106 | function onlyModifierEvent(event) { 107 | var key = keyboard.inv_keycodes[event.which]; 108 | return ( 109 | (event.altKey || event.ctrlKey || event.metaKey || event.shiftKey) && 110 | (key === 'alt' || key === 'ctrl' || key === 'meta' || key === 'shift') 111 | ); 112 | } 113 | 114 | function requestComplterServer(requestData, isAsync, handleResData) { 115 | var serverUrl = config.remote_server_url ? config.remote_server_url : baseUrl; 116 | if (serverUrl.charAt(serverUrl.length - 1) == '/') { 117 | serverUrl += 'tabnine'; 118 | } else { 119 | serverUrl += '/tabnine'; 120 | } 121 | // use get to solve post redirecting too many times 122 | $.get(serverUrl, { 'data': JSON.stringify(requestData) }) 123 | .done(function (data) { 124 | if (typeof data === 'string') { 125 | data = JSON.parse(data); 126 | } 127 | handleResData(data); 128 | }).fail(function (error) { 129 | console.log(logPrefix, ' get error: ', error); 130 | }); 131 | } 132 | 133 | function isValidCodeLine(line) { 134 | // comment line is valid, since we want to get completions 135 | if (line.length === 0 || 136 | line.charAt(0) === '!') { 137 | return false; 138 | } 139 | return true; 140 | } 141 | 142 | // A Deep Completer which extends Completer 143 | const DeepCompleter = function (cell, events) { 144 | Completer.call(this, cell, events); 145 | } 146 | DeepCompleter.prototype = Object.create(Completer.prototype); 147 | DeepCompleter.prototype.constructor = DeepCompleter; 148 | DeepCompleter.prototype.finish_completing = function (msg) { 149 | var optionsLimit = config.options_limit; 150 | var beforeLineLimit = config.before_line_limit > 0 ? config.before_line_limit : Infinity; 151 | var afterLineLimit = config.after_line_limit > 0 ? config.after_line_limit : Infinity; 152 | if (this.visible && $('#complete').length) { 153 | console.info(logPrefix, 'complete is visible, ignore by just return'); 154 | return; 155 | } 156 | 157 | var currEditor = this.editor; 158 | var currCell = this.cell; 159 | // check whether current cell satisfies line before and line after 160 | var cursor = currEditor.getCursor(); 161 | var currCellLines = currEditor.getValue().split("\n"); 162 | var before = []; 163 | var after = []; 164 | var currLine = currCellLines[cursor.line]; 165 | if (isValidCodeLine(currLine)) { 166 | before.push(currLine.slice(0, cursor.ch)); 167 | after.push(currLine.slice(cursor.ch, currLine.length)); 168 | } 169 | 170 | var i = cursor.line - 1; 171 | for (; i >= 0; before.length < beforeLineLimit, i--) { 172 | if (isValidCodeLine(currCellLines[i])) { 173 | before.push(currCellLines[i]); 174 | } 175 | } 176 | requestInfo.request.Autocomplete.region_includes_beginning = (i < 0); 177 | 178 | i = cursor.line + 1; 179 | for (; i < currCellLines.length && after.length < afterLineLimit; i++) { 180 | if (isValidCodeLine(currCellLines[i])) { 181 | after.push(currCellLines[i]); 182 | } 183 | } 184 | 185 | var cells = Jupyter.notebook.get_cells(); 186 | var index; 187 | for (index = cells.length - 1; index >= 0 && cells[index] != currCell; index--); 188 | var regionIncludesBeginning = requestInfo.request.Autocomplete.region_includes_beginning; 189 | requestInfo.request.Autocomplete.region_includes_beginning = regionIncludesBeginning && (index == 0); 190 | requestInfo.request.Autocomplete.region_includes_end = (i == currCellLines.length) 191 | && (index == cells.length - 1); 192 | // need lookup other cells 193 | if (before.length < beforeLineLimit || after.length < afterLineLimit) { 194 | i = index - 1; 195 | // always use for loop instead of while loop if poosible. 196 | // since I always foget to describe/increase i in while loop 197 | var atLineBeginning = true; // set true in case of three is no more lines before 198 | for (; i >= 0 && before.length < beforeLineLimit; i--) { 199 | var cellLines = cells[i].get_text().split("\n"); 200 | var j = cellLines.length - 1; 201 | atLineBeginning = false; 202 | for (; j >= 0 && before.length < beforeLineLimit; j--) { 203 | if (isValidCodeLine(cellLines[j])) { 204 | before.push(cellLines[j]); 205 | } 206 | } 207 | atLineBeginning = (j < 0); 208 | } 209 | // at the first cell and at the first line of that cell 210 | requestInfo.request.Autocomplete.region_includes_beginning = (i < 0) && atLineBeginning; 211 | 212 | i = index + 1; 213 | var atLineEnd = true; // set true in case of three is no more liens left 214 | for (; i < cells.length && after.length < afterLineLimit; i++) { 215 | var cellLines = cells[i].get_text().split("\n"); 216 | j = 0; 217 | atLineEnd = false; 218 | for (; j < cellLines.length && after.length < afterLineLimit; j++) { 219 | if (isValidCodeLine(cellLines[j])) { 220 | after.push(cellLines[j]); 221 | } 222 | } 223 | atLineEnd = (j == cellLines.length); 224 | } 225 | // at the last cell and at the last line of that cell 226 | requestInfo.request.Autocomplete.region_includes_end = (i == cells.length) && atLineEnd; 227 | } 228 | before.reverse(); 229 | this.before = before; 230 | this.after = after; 231 | 232 | before = before.slice(Math.max(0, before.length - N_LINES_BEFORE), before.length); 233 | after = after.slice(0, N_LINES_AFTER); 234 | requestInfo.request.Autocomplete.before = before.join("\n"); 235 | requestInfo.request.Autocomplete.after = after.join("\n"); 236 | 237 | this.complete = $('
').addClass('completions complete-dropdown-content'); 238 | this.complete.attr('id', 'complete'); 239 | $('body').append(this.complete); 240 | this.visible = true; 241 | // fix page flickering 242 | this.start = currEditor.indexFromPos(cursor); 243 | this.complete.css({ 244 | 'display': 'none', 245 | }); 246 | 247 | var that = this; 248 | requestComplterServer(requestInfo, true, function (data) { 249 | var complete = that.complete; 250 | if (data.results.length == 0) { 251 | that.close(); 252 | return; 253 | } 254 | that.completions = data.results.slice(0, optionsLimit); 255 | that.completions.forEach(function (res) { 256 | var completeContainer = generateCompleteContainer(res); 257 | complete.append(completeContainer); 258 | }); 259 | that.add_user_msg(data.user_message); 260 | that.set_location(data.old_prefix); 261 | that.add_keyevent_listeners() 262 | }); 263 | return true; 264 | } 265 | 266 | DeepCompleter.prototype.add_user_msg = function (user_messages) { 267 | var that = this; 268 | if (user_messages) { 269 | user_messages.forEach(function (user_message) { 270 | var msgLine = $('
').addClass('user-message'); 271 | $('').text(user_message).appendTo(msgLine); 272 | that.complete.append(msgLine); 273 | }); 274 | } 275 | } 276 | 277 | DeepCompleter.prototype.update = function () { 278 | // In this case, only current line have been changed. 279 | // so we can use cached other lines and this line to 280 | // generate before and after 281 | var optionsLimit = config.options_limit; 282 | if (!this.complete) { 283 | return; 284 | } 285 | var cursor = this.editor.getCursor(); 286 | this.start = this.editor.indexFromPos(cursor); // get current cursor 287 | var currLineText = this.editor.getLineHandle(cursor.line).text; 288 | var currLineBefore = currLineText.slice(0, cursor.ch); 289 | var currLineAfter = currLineText.slice(cursor.ch, currLineText.length); 290 | if (this.before.length > 0) { 291 | this.before[this.before.length - 1] = currLineBefore; 292 | } else { 293 | this.before.push(currLineBefore); 294 | } 295 | if (this.after.length > 0) { 296 | this.after[0] = currLineAfter; 297 | } else { 298 | this.after.push(currLineAfter); 299 | } 300 | var before = this.before.slice(Math.max(0, this.before.length - N_LINES_BEFORE), this.before.length); 301 | var after = this.after.slice(0, N_LINES_AFTER); 302 | requestInfo.request.Autocomplete.before = before.join('\n'); 303 | requestInfo.request.Autocomplete.after = after.join('\n'); 304 | var that = this; 305 | requestComplterServer(requestInfo, true, function (data) { 306 | if (data.results.length == 0) { 307 | that.close(); 308 | return; 309 | } 310 | var results = data.results; 311 | var completeContainers = $("#complete").find('.complete-container'); 312 | var i; 313 | that.completions = results.slice(0, optionsLimit); 314 | // replace current options first 315 | for (i = 0; i < that.completions.length && i < completeContainers.length; i++) { 316 | $(completeContainers[i]).find('.complete-word').text(results[i].new_prefix); 317 | $(completeContainers[i]).find('.complete-detail').text(results[i].detail); 318 | } 319 | // add 320 | for (; i < that.completions.length; i++) { 321 | var completeContainer = generateCompleteContainer(results[i]); 322 | that.complete.append(completeContainer); 323 | } 324 | // remove 325 | for (; i < completeContainers.length; i++) { 326 | $(completeContainers[i]).remove(); 327 | } 328 | 329 | var userMessages = $('#complete').find('.user-message'); 330 | if (userMessages) { 331 | if (userMessages instanceof Array) { 332 | for (var i = 0; i < userMessages.length; i++) { 333 | $(userMessages[i]).remove(); 334 | } 335 | } else { 336 | $(userMessages).remove(); 337 | } 338 | } 339 | that.add_user_msg(data.user_message); 340 | 341 | that.set_location(data.old_prefix); 342 | that.editor.off('keydown', that._handle_keydown); 343 | that.editor.off('keyup', that._handle_keyup); 344 | that.add_keyevent_listeners(); 345 | }); 346 | }; 347 | 348 | DeepCompleter.prototype.close = function () { 349 | this.done = true; 350 | $('#complete').remove(); 351 | this.editor.off('keydown', this._handle_keydown); 352 | this.visible = false; 353 | this.completions = null; 354 | this.completeFrom = null; 355 | this.complete = null; 356 | // before are copied from completer.js 357 | this.editor.off('keyup', this._handle_key_up); 358 | }; 359 | 360 | DeepCompleter.prototype.set_location = function (oldPrefix) { 361 | if (!this.complete) { 362 | return; 363 | } 364 | var start = this.start; 365 | this.completeFrom = this.editor.posFromIndex(start); 366 | if (oldPrefix) { 367 | oldPrefix = oldPrefix; 368 | this.completeFrom.ch -= oldPrefix.length; 369 | // this.completeFrom.ch = Math.max(this.completeFrom.ch, 0); 370 | } 371 | var pos = this.editor.cursorCoords( 372 | this.completeFrom 373 | ); 374 | 375 | var left = pos.left - 3; 376 | var top; 377 | var cheight = this.complete.height(); 378 | var wheight = $(window).height(); 379 | if (pos.bottom + cheight + 5 > wheight) { 380 | top = pos.top - cheight - 4; 381 | } else { 382 | top = pos.bottom + 1; 383 | } 384 | this.complete.css({ 385 | 'left': left + 'px', 386 | 'top': top + 'px', 387 | 'display': 'initial' 388 | }); 389 | }; 390 | 391 | DeepCompleter.prototype.add_keyevent_listeners = function () { 392 | var options = $("#complete").find('.complete-container'); 393 | var editor = this.editor; 394 | var currIndex = -1; 395 | var preIndex; 396 | this.isKeyupFired = true; // make keyup only fire once 397 | var that = this; 398 | this._handle_keydown = function (comp, event) { // define as member method to handle close 399 | // since some opration is async, it's better to check whether complete is existing or not. 400 | if (!$('#complete').length || !that.completions) { 401 | // editor.off('keydown', this._handle_keydown); 402 | // editor.off('keyup', this._handle_handle_keyup); 403 | return; 404 | } 405 | that.isKeyupFired = false; 406 | if (event.keyCode == keycodes.up || event.keyCode == keycodes.tab 407 | || event.keyCode == keycodes.down || event.keyCode == keycodes.enter) { 408 | event.codemirrorIgnore = true; 409 | event._ipkmIgnore = true; 410 | event.preventDefault(); 411 | // it's better to prevent enter key when completions being shown 412 | if (event.keyCode == keycodes.enter) { 413 | that.close(); 414 | return; 415 | } 416 | preIndex = currIndex; 417 | currIndex = event.keyCode == keycodes.up ? currIndex - 1 : currIndex + 1; 418 | currIndex = currIndex < 0 ? 419 | options.length - 1 420 | : (currIndex >= options.length ? 421 | currIndex - options.length 422 | : currIndex); 423 | $(options[currIndex]).css('background', 'lightblue'); 424 | var end = editor.getCursor(); 425 | if (that.completions[currIndex].old_suffix) { 426 | end.ch += that.completions[currIndex].old_suffix.length; 427 | } 428 | var replacement = that.completions[currIndex].new_prefix; 429 | replacement += that.completions[currIndex].new_suffix; 430 | editor.replaceRange(replacement, that.completeFrom, end); 431 | if (preIndex != -1) { 432 | $(options[preIndex]).css('background', ''); 433 | } 434 | } else if (needUpdateComplete(event.keyCode)) { 435 | // Let this be handled by keyup, since it can get current pressed key. 436 | } else { 437 | that.close(); 438 | } 439 | } 440 | 441 | var that = this; 442 | this._handle_keyup = function (cmp, event) { 443 | if (!that.isKeyupFired && !event.altKey && 444 | !event.ctrlKey && !event.metaKey && needUpdateComplete(event.keyCode)) { 445 | that.update(); 446 | that.isKeyupFired = true; 447 | }; 448 | }; 449 | 450 | editor.on('keydown', this._handle_keydown); 451 | editor.on('keyup', this._handle_keyup); 452 | }; 453 | 454 | function generateCompleteContainer(responseComplete) { 455 | var completeContainer = $('
') 456 | .addClass('complete-container'); 457 | var wordContainer = $('
') 458 | .addClass('complete-block') 459 | .addClass('complete-word') 460 | .text(responseComplete.new_prefix); 461 | completeContainer.append(wordContainer); 462 | var probContainer = $('
') 463 | .addClass('complete-block') 464 | .addClass('complete-detail') 465 | .text(responseComplete.detail) 466 | completeContainer.append(probContainer); 467 | return completeContainer; 468 | } 469 | 470 | function isAlphabeticKeyCode(keyCode) { 471 | return keyCode >= 65 && keyCode <= 90; 472 | } 473 | 474 | function isNumberKeyCode(keyCode) { 475 | return (keyCode >= 48 && keyCode <= 57) || (keyCode >= 96 && keyCode <= 105); 476 | } 477 | 478 | function isOperatorKeyCode(keyCode) { 479 | return (keyCode >= 106 && keyCode <= 111) || 480 | (keyCode >= 186 && keyCode <= 192) || 481 | (keyCode >= 219 && keyCode <= 222); 482 | } 483 | 484 | function needUpdateComplete(keyCode) { 485 | return isAlphabeticKeyCode(keyCode) || isNumberKeyCode(keyCode) || isOperatorKeyCode(keyCode); 486 | } 487 | 488 | function patchCellKeyevent() { 489 | var origHandleCodemirrorKeyEvent = Cell.prototype.handle_codemirror_keyevent; 490 | Cell.prototype.handle_codemirror_keyevent = function (editor, event) { 491 | if (!this.base_completer) { 492 | console.log(logPrefix, ' new base completer'); 493 | this.base_completer = new Completer(this, this.events); 494 | } 495 | 496 | if (!this.deep_completer) { 497 | console.log(logPrefix, ' new deep completer'); 498 | this.deep_completer = new DeepCompleter(this, this.events) 499 | } 500 | 501 | if (assistActive && !event.altKey && !event.metaKey && !event.ctrlKey 502 | && (this instanceof CodeCell) && !onlyModifierEvent(event)) { 503 | this.tooltip.remove_and_cancel_tooltip(); 504 | if (!editor.somethingSelected() && 505 | editor.getSelections().length <= 1 && 506 | !this.completer.visible && 507 | specials.indexOf(event.keyCode) == -1) { 508 | var cell = this; 509 | if (event.keyCode == keycodes.space && event.shiftKey) { 510 | event.preventDefault(); 511 | console.log(logPrefix, ' call base completer....'); 512 | cell.completer = cell.base_completer; 513 | } else { 514 | console.log(logPrefix, ' call deep completer....'); 515 | cell.completer = cell.deep_completer; 516 | } 517 | setTimeout(function () { 518 | cell.completer.startCompletion(); 519 | }, config.assist_delay); 520 | } 521 | } 522 | return origHandleCodemirrorKeyEvent.apply(this, arguments); 523 | }; 524 | } 525 | 526 | function setAssistState(newState) { 527 | assistActive = newState; 528 | $('.assistant-toggle > .fa').toggleClass('fa-check', assistActive); 529 | console.log(logPrefix, 'continuous autocompletion', assistActive ? 'on' : 'off'); 530 | } 531 | 532 | function toggleAutocompletion() { 533 | setAssistState(!assistActive); 534 | } 535 | 536 | function addMenuItem() { 537 | if ($('#help_menu').find('.assistant-toggle').length > 0) { 538 | return; 539 | } 540 | var menuItem = $('
  • ').insertAfter('#keyboard_shortcuts'); 541 | var menuLink = $('').text('Jupyter TabNine') 542 | .addClass('assistant-toggle') 543 | .attr('title', 'Provide continuous code autocompletion') 544 | .on('click', toggleAutocompletion) 545 | .appendTo(menuItem); 546 | $('').addClass('fa menu-icon pull-right').prependTo(menuLink); 547 | } 548 | 549 | 550 | function load_notebook_extension() { 551 | return Jupyter.notebook.config.loaded.then(function on_success() { 552 | $.extend(true, config, Jupyter.notebook.config.data.jupyter_tabnine); 553 | loadCss('./main.css'); 554 | }, function on_error(err) { 555 | console.warn(logPrefix, 'error loading config:', err); 556 | }).then(function on_success() { 557 | // patchCellCreatElement(); 558 | patchCellKeyevent(); 559 | addMenuItem(); 560 | setAssistState(config.assist_active); 561 | }); 562 | } 563 | return { 564 | load_ipython_extension: load_notebook_extension, 565 | load_jupyter_extension: load_notebook_extension 566 | }; 567 | }); 568 | -------------------------------------------------------------------------------- /src/jupyter_tabnine/static/tabnine.yaml: -------------------------------------------------------------------------------- 1 | Type: Jupyter Notebook Extension 2 | Main: main.js 3 | Name: Jupyter TabNine 4 | Link: README.md 5 | Description: | 6 | Provide code auto-completion with deep learning for every keypress. 7 | Compatibility: 4.x, 5.x 8 | Parameters: 9 | - name: jupyter_tabnine.before_line_limit 10 | description: | 11 | maximum number of lines before for context generation, 12 | too many lines will slow down the request. -1 means Infinity, 13 | thus the lines will equal to number of lines before current line. 14 | input_type: number 15 | default: 10 16 | - name: jupyter_tabnine.after_line_limit 17 | description: | 18 | maximum number of lines after for context generation, 19 | too many lines will slow down the request. -1 means Infinity, 20 | thus the lines will equal to number of lines after current line. 21 | input_type: number 22 | default: 10 23 | - name: jupyter_tabnine.options_limit 24 | description: | 25 | maximum number of options that will be shown 26 | input_type: number 27 | default: 10 28 | - name: jupyter_tabnine.assist_active 29 | description: | 30 | Enable continuous code auto-completion when notebook is first opened, or 31 | if false, only when selected from extensions menu. 32 | input_type: checkbox 33 | default: true 34 | - name: jupyter_tabnine.assist_delay 35 | description: | 36 | delay in milliseconds between keypress & completion request. 37 | input_type: number 38 | min: 0 39 | step: 1 40 | default: 0 41 | - name: jupyter_tabnine.remote_server_url 42 | description: | 43 | remote server url, you may want to use a remote server to handle client request. 44 | this can spped up the request handling depending on the server configuration. 45 | refer to https://github.com/wenmin-wu/jupyter-tabnine to see how to deploy remote server. 46 | input_type: string 47 | default: '' 48 | -------------------------------------------------------------------------------- /src/jupyter_tabnine/tabnine.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import platform 5 | import subprocess 6 | import stat 7 | import threading 8 | import zipfile 9 | import notebook 10 | 11 | from urllib.request import urlopen, urlretrieve 12 | from urllib.error import HTTPError 13 | from ._version import __version__ 14 | 15 | if platform.system() == "Windows": 16 | try: 17 | from colorama import init 18 | 19 | init(convert=True) 20 | except ImportError: 21 | try: 22 | import pip 23 | 24 | pip.main(["install", "--user", "colorama"]) 25 | from colorama import init 26 | 27 | init(convert=True) 28 | except Exception: 29 | logger = logging.getLogger("ImportError") 30 | logger.error( 31 | "Install colorama failed. Install it manually to enjoy colourful log." 32 | ) 33 | 34 | 35 | logging.basicConfig( 36 | level=logging.INFO, 37 | format="\x1b[1m\x1b[33m[%(levelname)s %(asctime)s.%(msecs)03d %(name)s]\x1b[0m: %(message)s", 38 | datefmt="%Y-%m-%d %H:%M:%S", 39 | ) 40 | 41 | _TABNINE_SERVER_URL = "https://update.tabnine.com/bundles" 42 | _TABNINE_EXECUTABLE = "TabNine" 43 | 44 | 45 | class TabnineDownloader(threading.Thread): 46 | def __init__(self, download_url, output_dir, tabnine): 47 | threading.Thread.__init__(self) 48 | self.download_url = download_url 49 | self.output_dir = output_dir 50 | self.logger = logging.getLogger(self.__class__.__name__) 51 | self.tabnine = tabnine 52 | 53 | def run(self): 54 | try: 55 | self.logger.info( 56 | "Begin to download Tabnine Binary from %s", self.download_url 57 | ) 58 | if not os.path.isdir(self.output_dir): 59 | os.makedirs(self.output_dir) 60 | zip_path, _ = urlretrieve(self.download_url) 61 | with zipfile.ZipFile(zip_path, "r") as zf: 62 | for filename in zf.namelist(): 63 | zf.extract(filename, self.output_dir) 64 | target = os.path.join(self.output_dir, filename) 65 | add_execute_permission(target) 66 | self.logger.info("Finish download Tabnine Binary to %s", self.output_dir) 67 | sem_complete_on(self.tabnine) 68 | except Exception as e: 69 | self.logger.error("Download failed, error: %s", e) 70 | 71 | 72 | def sem_complete_on(tabnine): 73 | SEM_ON_REQ_DATA = { 74 | "version": "1.0.7", 75 | "request": { 76 | "Autocomplete": { 77 | "filename": "test.py", 78 | "before": "tabnine::sem", 79 | "after": "", 80 | "region_includes_beginning": True, 81 | "region_includes_end": True, 82 | "max_num_results": 10, 83 | } 84 | }, 85 | } 86 | res = tabnine.request(json.dumps(SEM_ON_REQ_DATA)) 87 | try: 88 | tabnine.logger.info( 89 | f' {res["results"][0]["new_prefix"]}{res["results"][0]["new_suffix"]}' 90 | ) 91 | except Exception: 92 | tabnine.logger.warning(" wrong response of turning on semantic completion") 93 | 94 | 95 | class Tabnine(object): 96 | def __init__(self): 97 | self.name = "tabnine" 98 | self._proc = None 99 | self._response = None 100 | self.logger = logging.getLogger(self.__class__.__name__) 101 | self._install_dir = os.path.dirname(os.path.realpath(__file__)) 102 | self._binary_dir = os.path.join(self._install_dir, "binaries") 103 | self.logger.info(" install dir: %s", self._install_dir) 104 | self.download_if_needed() 105 | 106 | def request(self, data): 107 | proc = self._get_running_tabnine() 108 | if proc is None: 109 | return 110 | try: 111 | proc.stdin.write((data + "\n").encode("utf8")) 112 | proc.stdin.flush() 113 | except BrokenPipeError: 114 | self._restart() 115 | return 116 | 117 | output = proc.stdout.readline().decode("utf8") 118 | try: 119 | return json.loads(output) 120 | except json.JSONDecodeError: 121 | self.logger.debug("Tabnine output is corrupted: " + output) 122 | 123 | def _restart(self): 124 | if self._proc is not None: 125 | self._proc.terminate() 126 | self._proc = None 127 | path = get_tabnine_path(self._binary_dir) 128 | if path is None: 129 | self.logger.error("no Tabnine binary found") 130 | return 131 | self._proc = subprocess.Popen( 132 | [ 133 | path, 134 | "--client", 135 | "jupyter", 136 | "--log-file-path", 137 | os.path.join(self._install_dir, "tabnine.log"), 138 | "--client-metadata", 139 | "pluginVersion={}".format(__version__), 140 | "clientVersion={}".format(notebook.__version__), 141 | ], 142 | stdin=subprocess.PIPE, 143 | stdout=subprocess.PIPE, 144 | stderr=subprocess.DEVNULL, 145 | ) 146 | 147 | def _get_running_tabnine(self): 148 | if self._proc is None: 149 | self._restart() 150 | if self._proc is not None and self._proc.poll(): 151 | self.logger.error( 152 | "Tabnine exited with code {}".format(self._proc.returncode) 153 | ) 154 | self._restart() 155 | return self._proc 156 | 157 | def download_if_needed(self): 158 | if os.path.isdir(self._binary_dir): 159 | tabnine_path = get_tabnine_path(self._binary_dir) 160 | if tabnine_path is not None: 161 | add_execute_permission(tabnine_path) 162 | self.logger.info( 163 | "Tabnine binary already exists in %s ignore downloading", 164 | tabnine_path, 165 | ) 166 | sem_complete_on(self) 167 | return 168 | self._download() 169 | 170 | def _download(self): 171 | version = get_tabnine_version() 172 | distro = get_distribution_name() 173 | download_url = "{}/{}/{}/{}.zip".format( 174 | _TABNINE_SERVER_URL, version, distro, _TABNINE_EXECUTABLE 175 | ) 176 | output_dir = os.path.join(self._binary_dir, version, distro) 177 | TabnineDownloader(download_url, output_dir, self).start() 178 | 179 | 180 | def get_tabnine_version(): 181 | version_url = "{}/{}".format(_TABNINE_SERVER_URL, "version") 182 | 183 | try: 184 | return urlopen(version_url).read().decode("UTF-8").strip() 185 | except HTTPError: 186 | return None 187 | 188 | 189 | arch_translations = { 190 | "arm64": "aarch64", 191 | "AMD64": "x86_64", 192 | } 193 | 194 | 195 | def get_distribution_name(): 196 | sysinfo = platform.uname() 197 | sys_architecture = sysinfo.machine 198 | 199 | if sys_architecture in arch_translations: 200 | sys_architecture = arch_translations[sys_architecture] 201 | 202 | if sysinfo.system == "Windows": 203 | sys_platform = "pc-windows-gnu" 204 | 205 | elif sysinfo.system == "Darwin": 206 | sys_platform = "apple-darwin" 207 | 208 | elif sysinfo.system == "Linux": 209 | sys_platform = "unknown-linux-musl" 210 | 211 | elif sysinfo.system == "FreeBSD": 212 | sys_platform = "unknown-freebsd" 213 | 214 | else: 215 | raise RuntimeError( 216 | "Platform was not recognized as any of " "Windows, macOS, Linux, FreeBSD" 217 | ) 218 | 219 | return "{}-{}".format(sys_architecture, sys_platform) 220 | 221 | 222 | def get_tabnine_path(binary_dir): 223 | distro = get_distribution_name() 224 | versions = os.listdir(binary_dir) 225 | versions.sort(key=parse_semver, reverse=True) 226 | for version in versions: 227 | path = os.path.join( 228 | binary_dir, version, distro, executable_name(_TABNINE_EXECUTABLE) 229 | ) 230 | if os.path.isfile(path): 231 | return path 232 | 233 | 234 | def parse_semver(s): 235 | try: 236 | return [int(x) for x in s.split(".")] 237 | except ValueError: 238 | return [] 239 | 240 | 241 | def add_execute_permission(path): 242 | st = os.stat(path) 243 | new_mode = st.st_mode | stat.S_IEXEC 244 | if new_mode != st.st_mode: 245 | os.chmod(path, new_mode) 246 | 247 | 248 | def executable_name(name): 249 | if platform.system() == "Windows": 250 | return name + ".exe" 251 | else: 252 | return name 253 | -------------------------------------------------------------------------------- /start-server.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | if ! type "docker" > /dev/null; then 4 | echo "Please install docker first!" 5 | fi 6 | 7 | IMAGE_NAME="tabnine-server:latest" 8 | docker run --rm --name jupyter-tabnine-server \ 9 | -p 9999:8080 -d ${IMAGE_NAME} 10 | echo "Please start your jupyter notebook and enjoy :)" 11 | -------------------------------------------------------------------------------- /stop-server.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | docker rm -f jupyter-tabnine-server 3 | --------------------------------------------------------------------------------