├── .gitignore ├── LICENSE ├── README.md ├── dockerfiles ├── Dockerfile.common ├── Dockerfile.cpu ├── Dockerfile.gpu └── README.md ├── jupyter-text2code-demo.gif ├── jupyter_text2code ├── __init__.py ├── etc │ └── jupyter-text2code-extension.json ├── jupyter_text2code.css ├── jupyter_text2code.yaml ├── jupyter_text2code_lib.py ├── jupyter_text2code_serverextension │ ├── __init__.py │ ├── data │ │ ├── .gitkeep │ │ ├── awesome-notebooks.csv │ │ ├── generated_intents.csv │ │ ├── intent_lookup.csv │ │ └── ner_templates.csv │ └── models │ │ ├── .gitkeep │ │ ├── intent_index.idx │ │ ├── model-best │ │ ├── config.cfg │ │ ├── meta.json │ │ ├── ner │ │ │ ├── cfg │ │ │ ├── model │ │ │ └── moves │ │ ├── tok2vec │ │ │ ├── cfg │ │ │ └── model │ │ ├── tokenizer │ │ └── vocab │ │ │ ├── key2row │ │ │ ├── lookups.bin │ │ │ ├── strings.json │ │ │ ├── vectors │ │ │ └── vectors.cfg │ │ └── model-last │ │ ├── config.cfg │ │ ├── meta.json │ │ ├── ner │ │ ├── cfg │ │ ├── model │ │ └── moves │ │ ├── tok2vec │ │ ├── cfg │ │ └── model │ │ ├── tokenizer │ │ └── vocab │ │ ├── key2row │ │ ├── lookups.bin │ │ ├── strings.json │ │ ├── vectors │ │ └── vectors.cfg └── main.js ├── notebooks ├── Code Generator.ipynb ├── Episodes.csv ├── Generate Training data NER.ipynb └── ctds.ipynb ├── scripts ├── README.md ├── config.cfg ├── create_intent_index.py ├── create_lookup_file.py ├── data │ ├── awesome-notebooks.csv │ ├── awesome-notebooks.pkl │ ├── st_naas_intent_index.idx │ └── tf_naas_intent_index.idx ├── eval_models_performance.ipynb ├── generate_training_data.py ├── process_awesome_notebooks.py └── train_spacy_v3_ner.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything 2 | *.pyc 3 | .ipynb_checkpoints/ 4 | 5 | # whitelist 6 | # --------- 7 | 8 | !.gitignore 9 | 10 | 11 | # Ignore following 12 | .idea 13 | jupyter_text2code.egg-info/ 14 | .pickle 15 | 16 | ner_train_data.pickle 17 | 18 | build/ 19 | dist/ 20 | venv -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Deepklarity Technologies Pvt. Ltd. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Text2Code for Jupyter notebook 2 | ### A proof-of-concept jupyter extension which converts english queries into relevant python code. 3 | 4 | 5 | ![](jupyter-text2code-demo.gif) 6 | 7 | ### Blog post with more details: 8 | #### [Data analysis made easy: Text2Code for Jupyter notebook](https://towardsdatascience.com/data-analysis-made-easy-text2code-for-jupyter-notebook-5380e89bb493?source=friends_link&sk=2c46fff2c31f7fe59b667350e4596b18) 9 | 10 | ### Demo Video: 11 | #### [Text2Code for Jupyter notebook](https://www.youtube.com/watch?v=3gZ7_9W-TJs) 12 | 13 | ## Supported Operating Systems: 14 | - Ubuntu 15 | - macOS 16 | 17 | ## Installation 18 | 19 | ### NOTE: We have renamed the plugin from mopp to jupyter-text2code. Uninstall mopp before installing new jupyter-text2code version. 20 | ``` 21 | pip uninstall mopp 22 | ``` 23 | 24 | #### CPU-only install: 25 | For Mac and other Ubuntu installations not having a nvidia GPU, we need to explicitly set an environment variable at time of install. 26 | ``` 27 | export JUPYTER_TEXT2CODE_MODE="cpu" 28 | 29 | ``` 30 | 31 | #### GPU install dependencies: 32 | ``` 33 | sudo apt-get install libopenblas-dev libomp-dev 34 | ``` 35 | 36 | #### Installation commands: 37 | 38 | ``` 39 | git clone https://github.com/deepklarity/jupyter-text2code.git 40 | cd jupyter-text2code 41 | pip install . 42 | jupyter nbextension enable jupyter-text2code/main 43 | 44 | ``` 45 | 46 | ## Uninstallation: 47 | ``` 48 | pip uninstall jupyter-text2code 49 | ``` 50 | 51 | ## Usage Instructions: 52 | 53 | - Start Jupyter notebook server by running the following command: ``` jupyter notebook ``` 54 | - If you don't see ``` Nbextensions``` tab in Jupyter notebook run the following command:``` jupyter contrib nbextension install --user ``` 55 | - You can open the sample ``` notebooks/ctds.ipynb``` notebook for testing 56 | - If installation happened successfully, then for the first time, Universal Sentence Encoder model will be downloaded from `tensorflow_hub` 57 | - Click on the `Terminal` Icon which appears on the menu (to activate the extension) 58 | - Type "help" to see a list of currently supported commands in the repo 59 | - Watch [Demo video](https://www.youtube.com/watch?v=3gZ7_9W-TJs) for some examples 60 | 61 | ## Docker containers for jupyter-text2code (old version) 62 | 63 | We have published CPU and GPU images to docker hub with all dependencies pre-installed. 64 | ##### Visit https://hub.docker.com/r/deepklarity/jupyter-text2code/ to download the images and usage instructions. 65 | 66 | ##### CPU image size: ``` 1.51 GB ``` 67 | ##### GPU image size: ``` 2.56 GB ``` 68 | 69 | ## Model training: 70 | The plugin now supports pandas commands + quick snippet insertion of available snippets from [awesome-notebooks](https://github.com/jupyter-naas/awesome-notebooks). With this change, we can now get snippets for most popular integrations from within the jupyter tab. eg: 71 | - Get followers count from twitter 72 | - Get stats about a story from instagram 73 | The detailed training steps are available in [scripts README](scripts/README.md) where we also evaluated performance of different models and ended up selecting SentenceTransformers `paraphrase-MiniLM-L6-v2` 74 | 75 | 76 | ### Steps to add more intents: 77 | - Add more templates in `ner_templates` with a new intent_id 78 | - Generate training data. Modify `generate_training_data.py` if different generation techniques are needed or if introducing a new entity. 79 | - Train intent index 80 | - Train NER model 81 | - modify `jupyter_text2code/jupyter_text2code_serverextension/__init__.py` with new intent's condition and add actual code for the intent 82 | - Reinstall plugin by running: `pip install .` 83 | 84 | ### TODO: 85 | - [] Add Ollama support to work with local LLMs 86 | - [x] Publish Docker image 87 | - [X] Refactor code and make it mode modular, remove duplicate code, etc 88 | - [X] Add support for more commands 89 | - [X] Improve intent detection and NER 90 | - [ ] Add support for Windows 91 | - [ ] Explore sentence Paraphrasing to generate higher-quality training data 92 | - [ ] Gather real-world variable names, library names as opposed to randomly generating them 93 | - [ ] Try NER with a transformer-based model 94 | - [ ] With enough data, train a language model to directly do English->code like GPT-3 does, instead of having separate stages in the pipeline 95 | - [ ] Create a survey to collect linguistic data 96 | - [ ] Add Speech2Code support 97 | 98 | #### Authored By: 99 | 100 | - [Deepak Rawat](https://twitter.com/dsr_ai) 101 | - [Kartik Godawat](https://twitter.com/kartik_godawat) 102 | - [Abdullah Meda](https://www.linkedin.com/in/abdmeda/) 103 | -------------------------------------------------------------------------------- /dockerfiles/Dockerfile.common: -------------------------------------------------------------------------------- 1 | ARG WORKDIR=/home/deepklarity/jupyter-text2code 2 | 3 | WORKDIR $WORKDIR 4 | 5 | ENV PYTHONUNBUFFERED=1 TFHUB_CACHE_DIR=/root/.cache/tfhub_modules 6 | 7 | RUN --mount=type=cache,mode=0777,target=/var/cache/apt --mount=type=cache,mode=0777,target=/var/lib/apt \ 8 | --mount=type=cache,mode=0777,target=/root/.cache/pip \ 9 | apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y git \ 10 | && git clone https://github.com/deepklarity/jupyter-text2code.git $WORKDIR \ 11 | && apt-get purge --auto-remove -y git \ 12 | && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ 13 | && python -m pip --no-cache-dir install -U --force-reinstall pip && pip --no-cache-dir install -U . \ 14 | && jupyter contrib nbextension install --user && jupyter nbextension enable jupyter-text2code/main \ 15 | && python -c 'import tensorflow_hub as hub; hub.load("https://tfhub.dev/google/universal-sentence-encoder/4")' 16 | 17 | CMD jupyter notebook --ip 0.0.0.0 --port 8888 --no-browser --allow-root 18 | -------------------------------------------------------------------------------- /dockerfiles/Dockerfile.cpu: -------------------------------------------------------------------------------- 1 | # syntax = edrevo/dockerfile-plus 2 | FROM python:3.7-slim 3 | 4 | ENV JUPYTER_TEXT2CODE_MODE="cpu" 5 | 6 | INCLUDE+ dockerfiles/Dockerfile.common 7 | -------------------------------------------------------------------------------- /dockerfiles/Dockerfile.gpu: -------------------------------------------------------------------------------- 1 | # syntax = edrevo/dockerfile-plus 2 | FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04 3 | 4 | RUN --mount=type=cache,mode=0777,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \ 5 | apt-get -y update && DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common libopenblas-dev libomp-dev \ 6 | && add-apt-repository ppa:deadsnakes/ppa && apt-get -y update && apt-get -y install python3.7 python3-pip \ 7 | && update-alternatives --install /usr/bin/python python /usr/bin/python3 10 8 | 9 | INCLUDE+ dockerfiles/Dockerfile.common 10 | -------------------------------------------------------------------------------- /dockerfiles/README.md: -------------------------------------------------------------------------------- 1 | # About jupyter-text2code 2 | 3 | ### A proof-of-concept jupyter extension which converts english queries into relevant python code. 4 | 5 | #### Github Repository: https://github.com/deepklarity/jupyter-text2code 6 | 7 | ### Blog post with more details: 8 | #### [Data analysis made easy: Text2Code for Jupyter notebook](https://towardsdatascience.com/data-analysis-made-easy-text2code-for-jupyter-notebook-5380e89bb493?source=friends_link&sk=2c46fff2c31f7fe59b667350e4596b18) 9 | 10 | ### Demo Video: 11 | #### [Text2Code for Jupyter notebook](https://www.youtube.com/watch?v=3gZ7_9W-TJs) 12 | 13 | # How to Use the Images 14 | 15 | ### Install docker from: https://docs.docker.com/engine/install/ 16 | 17 | ### CPU image: 18 | 19 | 1. Pull the docker image 20 | ``` 21 | docker pull deepklarity/jupyter-text2code:latest 22 | ``` 23 | 2. Run the Docker image 24 | ``` 25 | docker run -it -p 8888:8888 deepklarity/jupyter-text2code:latest 26 | ``` 27 | 28 | ### GPU image: 29 | 1. Pull the docker image 30 | ``` 31 | docker pull docker pull deepklarity/jupyter-text2code:latest-gpu 32 | ``` 33 | 2. Run the Docker image 34 | ``` 35 | docker run -it --gpus all -p 8888:8888 deepklarity/jupyter-text2code:latest-gpu 36 | ``` 37 | 38 | ### Open Jupyter Notebook: 39 | 40 | #### Once the container is running, you will see a URL with a token in the terminal/console. Open that URL in your browser. 41 | 42 | Example url: ``` http://127.0.0.1:8888/?token=48c6ea28c1cbce210c008f1ef8dab8fa91ad77420922e259 ``` 43 | 44 | ### Usage Instructions: 45 | 46 | - You can open the sample ``` notebooks/ctds.ipynb``` notebook for testing 47 | - Click on the `Terminal` Icon which appears on the menu (to activate the extension) 48 | - Type "help" to see a list of currently supported commands in the repo 49 | - Watch [Demo video](https://www.youtube.com/watch?v=3gZ7_9W-TJs) for some examples 50 | -------------------------------------------------------------------------------- /jupyter-text2code-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter-text2code-demo.gif -------------------------------------------------------------------------------- /jupyter_text2code/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/__init__.py -------------------------------------------------------------------------------- /jupyter_text2code/etc/jupyter-text2code-extension.json: -------------------------------------------------------------------------------- 1 | { 2 | "NotebookApp": { 3 | "load_extensions": { 4 | "jupyter-text2code/main": true 5 | }, 6 | "nbserver_extensions": { 7 | "jupyter_text2code.jupyter_text2code_serverextension": true 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code.css: -------------------------------------------------------------------------------- 1 | .notebook_app { 2 | /*background:red !important;*/ 3 | } 4 | 5 | #jupyter_text2code_editor { 6 | position: fixed; 7 | bottom: 0; 8 | width: 61%; 9 | z-index: 999; 10 | background: lightblue; 11 | padding: 10px 35px; 12 | margin-left: -14px; 13 | } 14 | 15 | #jupyter_text2code_editor_history { 16 | position: fixed; 17 | top: 150px; 18 | right: 10px; 19 | background: lightblue; 20 | padding: 5px; 21 | width: 15%; 22 | } 23 | 24 | #jupyter_text2code_preset_wrapper { 25 | max-height: 250px; 26 | overflow-y: scroll; 27 | } 28 | 29 | #jupyter_text2code_history_wrapper { 30 | max-height: 250px; 31 | overflow-y: scroll; 32 | } 33 | 34 | #jupyter_text2code_query { 35 | height: 50px; 36 | width: 100%; 37 | font-size: 20px; 38 | } 39 | #jupyter_text2code_history { 40 | padding: 3px; 41 | } 42 | 43 | .jupyter_text2code_history_item { 44 | background: lightcyan; 45 | margin: 10px 0px; 46 | padding: 5px; 47 | padding-left: 10px; 48 | cursor: pointer; 49 | } 50 | 51 | .jupyter_text2code_preset_item { 52 | background: lightcyan; 53 | margin: 10px 0px; 54 | padding: 5px; 55 | padding-left: 10px; 56 | cursor: pointer; 57 | } 58 | 59 | .jupyter_text2code_sub_heading { 60 | padding: 3px; 61 | font-size: 14px; 62 | font-weight: bold; 63 | } 64 | 65 | #jupyter_text2code_preset_content { 66 | padding: 3px; 67 | } 68 | 69 | .jupyter_text2code_what_heading { 70 | font-weight: bold; 71 | padding: 5px; 72 | } 73 | 74 | #jupyter_text2code_submit { 75 | margin: 10px; 76 | } 77 | 78 | .jupyter_text2code_spinner { 79 | font-size: 22px; 80 | margin-left: 10px; 81 | } -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code.yaml: -------------------------------------------------------------------------------- 1 | Type: IPython Notebook Extension 2 | Compatibility: 3.x, 4.x, 5.x, 6.x 3 | Main: main.js 4 | Name: jupyter-text2code 5 | Icon: icon.jpg 6 | Description: "jupyter-text2code plugin with pandas and plotly support" 7 | -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_lib.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from IPython import get_ipython 4 | from IPython.core.magics.namespace import NamespaceMagics 5 | 6 | _nms = NamespaceMagics() 7 | _Jupyter = get_ipython() 8 | _nms.shell = _Jupyter.kernel.shell 9 | 10 | 11 | def dataframes_info(): 12 | values = _nms.who_ls() 13 | info = {v: (eval(v).columns.tolist()) for v in values if type(eval(v)).__name__ == 'DataFrame'} 14 | return json.dumps(info) 15 | -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import json 4 | from abc import ABC 5 | from itertools import groupby 6 | 7 | import faiss 8 | import spacy 9 | import numpy as np 10 | import pandas as pd 11 | from notebook.utils import url_path_join 12 | from notebook.base.handlers import IPythonHandler 13 | from sentence_transformers import SentenceTransformer 14 | 15 | home = os.path.dirname(__file__) 16 | 17 | SPACY_MODEL_DIR = os.path.join(home, "models/model-best") 18 | FAISS_INDEX_PATH = os.path.join(home, "models/intent_index.idx") 19 | INTENT_DF_PATH = os.path.join(home, "data/intent_lookup.csv") 20 | HELP_LIST = ['Import all libraries - Example Usage: import all libraries', 21 | 'Use plotly dark theme - Example Usage: use dark theme', 22 | 'Load file into a dataframe - Example Usage: Load train.csv in df', 23 | 'Show n rows of dataframe - Example Usage: Show 10 rows from df', 24 | 'Shape of dataframe - Example Usage: Show shape of df', 25 | 'Describe dataframe - Example Usage: Describe dataframe df', 26 | 'List columns of dataframe - Example Usage: Show columns from df', 27 | 'Correlation matrix of dataframe - Example Usage: Display corelation matrix of df', 28 | 'Histogram of column in dataframe - Example Usage: Plot histogram of category from df', 29 | 'Bar chart of columns from dataframe - Example Usage: Show bar chart of product and amount from df', 30 | 'Pie chart of column - Example Usage: Make pie chart of fruits from df', 31 | 'Group by aggregations of columns in dataframe - Example Usage: group df by country and show sum and mean of population', 32 | 'Line chart of columns in dataframe - Example Usage: Line chart of price and sale from df', 33 | 'Scatter plot of columns in dataframe - Example Usage: Show scatter plot of youtube_likes and episode_duration from df', 34 | 'Heatmap of columns in dataframe - Example Usage: from df make heat map of recording_time and youtube_views', 35 | 'List all files in current directory - Example Usage: List all files in current directory' 36 | ] 37 | HELP_TEXT = "\n".join([f"# {s}" for s in HELP_LIST]) 38 | 39 | 40 | class CodeGenerator: 41 | 42 | def __init__(self): 43 | self.nlp = spacy.load(SPACY_MODEL_DIR) 44 | self.embedding_model = SentenceTransformer("paraphrase-MiniLM-L6-v2") 45 | self.intent_index = faiss.read_index(FAISS_INDEX_PATH) 46 | self.intent_df = pd.read_csv(INTENT_DF_PATH) 47 | self.intent_df = self.intent_df.set_index('intent_id') 48 | 49 | def _get_embedding(self, command): 50 | command = re.sub('[^A-Za-z0-9 ]+', '', command).lower() 51 | return list(np.array(self.embedding_model.encode([command])[0])) 52 | 53 | def _get_intent(self, query, k_nearest=1): 54 | query_vector = np.array([self._get_embedding(query)]).astype(np.float32) 55 | faiss.normalize_L2(query_vector) 56 | similarities, similarities_ids = self.intent_index.search(query_vector, k_nearest) 57 | return similarities_ids[0][0], self.intent_df['code'][similarities_ids[0][0]] 58 | 59 | def generate_code(self, query, df_info_dict={}, debug=False): 60 | intent_id, intent_code = self._get_intent(query) 61 | if 0 <= intent_id < 10000: # Existing 62 | doc = self.nlp(query) 63 | entities = {key: list(g) for key, g in groupby(sorted(doc.ents, key=lambda x: x.label_), lambda x: x.label_)} 64 | for entity, labels in entities.items(): 65 | intent_code = re.sub(fr'\${entity.lower()}', lambda _: next(iter(map(lambda x: x.text, labels))), intent_code) 66 | elif 10000 <= intent_id < 20000: # Naas 67 | print("Nothing yet") 68 | 69 | return re.sub(r'\$\w+', 'xxx', intent_code) 70 | 71 | 72 | print("*" * 20) 73 | print("*" * 20) 74 | print("Loading_jupyter_server_extension. First install will download SentenceTransformers, please wait...") 75 | print("*" * 20) 76 | print("*" * 20) 77 | CG = CodeGenerator() 78 | 79 | 80 | class JupyterText2CodeHandler(IPythonHandler, ABC): 81 | def __init__(self, application, request, **kwargs): 82 | super(JupyterText2CodeHandler, self).__init__(application, request, **kwargs) 83 | 84 | # TODO: Add logger 85 | def get(self): 86 | query = self.get_argument('query') 87 | 88 | try: 89 | status = "success" 90 | if query.lower() == 'help': 91 | command = HELP_TEXT 92 | else: 93 | df_info = self.get_argument('dataframes_info') 94 | df_info_dict = json.loads(df_info[1:-1]) 95 | command = CG.generate_code(query, df_info_dict, debug=True) 96 | 97 | response = {"status": status, "message": command} 98 | except Exception as e: 99 | response = {"status": "error", "message": str(e)} 100 | 101 | response["message"] = f"#Query: {query}\n\n{response['message']}" 102 | self.finish(json.dumps(response)) 103 | 104 | 105 | def load_jupyter_server_extension(nb_server_app): 106 | """ 107 | Called when the extension is loaded. 108 | 109 | Args: 110 | nb_server_app (NotebookWebApplication): handle to the Notebook webserver instance. 111 | """ 112 | web_app = nb_server_app.web_app 113 | host_pattern = '.*$' 114 | route_pattern = url_path_join(web_app.settings['base_url'], '/jupyter-text2code') 115 | web_app.add_handlers(host_pattern, [(route_pattern, JupyterText2CodeHandler)]) 116 | print("loaded_jupyter_server_extension: jupyter-text2code") 117 | -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/data/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/data/.gitkeep -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/data/ner_templates.csv: -------------------------------------------------------------------------------- 1 | intent_id,template,code 2 | 0,import $libname,import $libname 3 | 1,import all libraries,"import pandas as pd 4 | import numpy as np 5 | import os 6 | import plotly.express as px 7 | import matplotlib.pyplot as plt 8 | pd.options.plotting.backend = 'plotly'" 9 | 2,load $fname,$varname = pd.read_csv('$fname') 10 | 2,load $fname in $varname,$varname = pd.read_csv('$fname') 11 | 3,show $cardinal rows from $varname,$varname.head($cardinal) 12 | 3,show $cardinal rows of $varname,$varname.head($cardinal) 13 | 3,print $cardinal rows from $varname,$varname.head($cardinal) 14 | 3,print $cardinal rows of $varname,$varname.head($cardinal) 15 | 3,print $varname head,$varname.head() 16 | 4,plot histogram of $colname column in $varname,$varname.plot.hist(x='$colname') 17 | 4,plot histogram of $colname in $varname,$varname.plot.hist(x='$colname') 18 | 4,draw histogram of $colname column in $varname,$varname.plot.hist(x='$colname') 19 | 4,get histogram of $colname in $varname,$varname.plot.hist(x='$colname') 20 | 5,get correlation matrix of $varname,$varname.corr() 21 | 6,print $varname shape,$varname.shape 22 | 6,print shape of $varname,$varname.shape 23 | 6,get size of $varname,$varname.shape 24 | 7,barplot $colname and $colname columns of $varname,"px.bar(x='$colname', y='$colname', data_frame=$varname, title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 25 | 7,plot $colname and $colname columns of $varname in a bar plot,"px.bar(x='$colname', y='$colname', data_frame=$varname, title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 26 | 7,bar plot $colname and $colname column of $varname,"px.bar(x='$colname', y='$colname', data_frame=$varname, title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 27 | 7,bar plot $colname and $colname in $varname,"px.bar(x='$colname', y='$colname', data_frame=$varname, title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 28 | 7,plot $colname and $colname of $varname in a bar plot,"px.bar(x='$colname', y='$colname', data_frame=$varname, title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 29 | 7,plot $colname $colname of $varname in a bar plot,"px.bar(x='$colname', y='$colname', data_frame=$varname, title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 30 | 7,show a bar plot with $colname on x axis over $colname in $varname,"px.bar(x='$colname', y='$colname', data_frame=$varname, title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 31 | 7,show a bar plot with $colname on x axis and $colname on y axis in $varname,"px.bar(x='$colname', y='$colname', data_frame=$varname, title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 32 | 7,show a bar plot with $colname on y axis and $colname on x axis,"px.bar(x='$colname', y='$colname', data_frame=$varname, title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 33 | 8,piechart of $colname column in $varname grouped by $colname column,"tmp = $varname['$colname'].value_counts(dropna=False) 34 | px.pie(tmp,values=tmp.values,names=tmp.index,title='CustomTitle')" 35 | 8,piechart of $colname in $varname grouped by $colname,"tmp = $varname['$colname'].value_counts(dropna=False) 36 | px.pie(tmp,values=tmp.values,names=tmp.index,title='CustomTitle')" 37 | 8,pie chart of $colname column of $varname grouped by $colname column,"tmp = $varname['$colname'].value_counts(dropna=False) 38 | px.pie(tmp,values=tmp.values,names=tmp.index,title='CustomTitle')" 39 | 8,pie chart of column $colname in $varname grouped by column $colname,"tmp = $varname['$colname'].value_counts(dropna=False) 40 | px.pie(tmp,values=tmp.values,names=tmp.index,title='CustomTitle')" 41 | 8,pie chart of $colname of $varname coloured by $colname,"tmp = $varname['$colname'].value_counts(dropna=False) 42 | px.pie(tmp,values=tmp.values,names=tmp.index,title='CustomTitle')" 43 | 9,install $libname,!pip install $libname 44 | 10,list columns of $varname,$varname.columns 45 | 10,list all columns of $varname,$varname.columns 46 | 11,describe $varname,$varname.describe() 47 | 12,group $varname by $colname and get $function of $colname,# Not supported in the current release :( 48 | 12,group the $varname by $colname and get $function of $colname,# Not supported in the current release :( 49 | 12,$varname group by $colname $function by $colname,# Not supported in the current release :( 50 | 12,find $function of $colname group by $colname from $varname,# Not supported in the current release :( 51 | 12,$function $colname group by $colname from $varname,# Not supported in the current release :( 52 | 12,$function $colname group by $colname from $varname,# Not supported in the current release :( 53 | 13,display a line plot showing $colname vs $colname in $varname,"$varname.plot.line(x='$colname', y='$colname', color=None, title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 54 | 13,display a line plot showing $colname on y-axis and $colname on x-axis from $varname,"$varname.plot.line(x='$colname', y='$colname', color=None, title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 55 | 13,display a line plot of $colname versus $colname in $varname,"$varname.plot.line(x='$colname', y='$colname', color=None, title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 56 | 13,line plot of $colname and $colname in $varname,"$varname.plot.line(x='$colname', y='$colname', color=None, title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 57 | 14,show a scatter plot of $colname over $colname in $varname,"$varname.plot.scatter(x='$colname', y='$colname', color=None, size=None, title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 58 | 14,scatter plot of $colname and $colname in $varname,"$varname.plot.scatter(x='$colname', y='$colname', color=None, size=None, title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 59 | 15,show a heatmap with $colname on x axis and $colname on y axis in $varname,"$varname.plot(kind='density_heatmap', x='$colname', y='$colname', title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 60 | 15,display a heatmap of $colname on y axis and $colname on x axis in $varname,"$varname.plot(kind='density_heatmap', x='$colname', y='$colname', title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 61 | 15,heatmap of $colname and $colname in $varname,"$varname.plot(kind='density_heatmap', x='$colname', y='$colname', title='CustomTitle', labels={'$colname':'$colname', '$colname':'$colname'})" 62 | 16,list all files in current directory,!ls . 63 | 17,switch to dark theme,"import plotly.io as pio 64 | pio.templates.default = 'plotly_dark'" 65 | -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/models/.gitkeep -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/intent_index.idx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/models/intent_index.idx -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-best/config.cfg: -------------------------------------------------------------------------------- 1 | [paths] 2 | train = "corpus/train.spacy" 3 | dev = "corpus/dev.spacy" 4 | vectors = "en_core_web_sm" 5 | init_tok2vec = null 6 | 7 | [system] 8 | gpu_allocator = null 9 | seed = 0 10 | 11 | [nlp] 12 | lang = "en" 13 | pipeline = ["tok2vec","ner"] 14 | batch_size = 1000 15 | disabled = [] 16 | before_creation = null 17 | after_creation = null 18 | after_pipeline_creation = null 19 | tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"} 20 | 21 | [components] 22 | 23 | [components.ner] 24 | factory = "ner" 25 | incorrect_spans_key = null 26 | moves = null 27 | scorer = {"@scorers":"spacy.ner_scorer.v1"} 28 | update_with_oracle_cut_size = 100 29 | 30 | [components.ner.model] 31 | @architectures = "spacy.TransitionBasedParser.v2" 32 | state_type = "ner" 33 | extra_state_tokens = false 34 | hidden_width = 64 35 | maxout_pieces = 2 36 | use_upper = true 37 | nO = null 38 | 39 | [components.ner.model.tok2vec] 40 | @architectures = "spacy.Tok2VecListener.v1" 41 | width = ${components.tok2vec.model.encode.width} 42 | upstream = "*" 43 | 44 | [components.tok2vec] 45 | factory = "tok2vec" 46 | 47 | [components.tok2vec.model] 48 | @architectures = "spacy.Tok2Vec.v2" 49 | 50 | [components.tok2vec.model.embed] 51 | @architectures = "spacy.MultiHashEmbed.v2" 52 | width = ${components.tok2vec.model.encode.width} 53 | attrs = ["NORM","PREFIX","SUFFIX","SHAPE"] 54 | rows = [5000,2500,2500,2500] 55 | include_static_vectors = true 56 | 57 | [components.tok2vec.model.encode] 58 | @architectures = "spacy.MaxoutWindowEncoder.v2" 59 | width = 256 60 | depth = 8 61 | window_size = 1 62 | maxout_pieces = 3 63 | 64 | [corpora] 65 | 66 | [corpora.dev] 67 | @readers = "spacy.Corpus.v1" 68 | path = ${paths.dev} 69 | max_length = 0 70 | gold_preproc = false 71 | limit = 0 72 | augmenter = null 73 | 74 | [corpora.train] 75 | @readers = "spacy.Corpus.v1" 76 | path = ${paths.train} 77 | max_length = 0 78 | gold_preproc = false 79 | limit = 0 80 | augmenter = null 81 | 82 | [training] 83 | dev_corpus = "corpora.dev" 84 | train_corpus = "corpora.train" 85 | seed = ${system.seed} 86 | gpu_allocator = ${system.gpu_allocator} 87 | dropout = 0.1 88 | accumulate_gradient = 1 89 | patience = 1600 90 | max_epochs = 0 91 | max_steps = 20000 92 | eval_frequency = 200 93 | frozen_components = [] 94 | annotating_components = [] 95 | before_to_disk = null 96 | 97 | [training.batcher] 98 | @batchers = "spacy.batch_by_words.v1" 99 | discard_oversize = false 100 | tolerance = 0.2 101 | get_length = null 102 | 103 | [training.batcher.size] 104 | @schedules = "compounding.v1" 105 | start = 100 106 | stop = 1000 107 | compound = 1.001 108 | t = 0.0 109 | 110 | [training.logger] 111 | @loggers = "spacy.ConsoleLogger.v1" 112 | progress_bar = false 113 | 114 | [training.optimizer] 115 | @optimizers = "Adam.v1" 116 | beta1 = 0.9 117 | beta2 = 0.999 118 | L2_is_weight_decay = true 119 | L2 = 0.01 120 | grad_clip = 1.0 121 | use_averages = false 122 | eps = 0.00000001 123 | learn_rate = 0.001 124 | 125 | [training.score_weights] 126 | ents_f = 1.0 127 | ents_p = 0.0 128 | ents_r = 0.0 129 | ents_per_type = null 130 | 131 | [pretraining] 132 | 133 | [initialize] 134 | vectors = ${paths.vectors} 135 | init_tok2vec = ${paths.init_tok2vec} 136 | vocab_data = null 137 | lookups = null 138 | before_init = null 139 | after_init = null 140 | 141 | [initialize.components] 142 | 143 | [initialize.tokenizer] -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-best/meta.json: -------------------------------------------------------------------------------- 1 | { 2 | "lang":"en", 3 | "name":"pipeline", 4 | "version":"0.0.0", 5 | "spacy_version":">=3.3.0,<3.4.0", 6 | "description":"", 7 | "author":"", 8 | "email":"", 9 | "url":"", 10 | "license":"", 11 | "spacy_git_version":"497a708c7", 12 | "vectors":{ 13 | "width":0, 14 | "vectors":0, 15 | "keys":0, 16 | "name":null, 17 | "mode":"default" 18 | }, 19 | "labels":{ 20 | "tok2vec":[ 21 | 22 | ], 23 | "ner":[ 24 | "CARDINAL", 25 | "COLNAME", 26 | "FNAME", 27 | "FUNCTION", 28 | "LIBNAME", 29 | "VARNAME" 30 | ] 31 | }, 32 | "pipeline":[ 33 | "tok2vec", 34 | "ner" 35 | ], 36 | "components":[ 37 | "tok2vec", 38 | "ner" 39 | ], 40 | "disabled":[ 41 | 42 | ], 43 | "performance":{ 44 | "ents_f":1.0, 45 | "ents_p":1.0, 46 | "ents_r":1.0, 47 | "ents_per_type":{ 48 | "COLNAME":{ 49 | "p":1.0, 50 | "r":1.0, 51 | "f":1.0 52 | }, 53 | "VARNAME":{ 54 | "p":1.0, 55 | "r":1.0, 56 | "f":1.0 57 | }, 58 | "CARDINAL":{ 59 | "p":1.0, 60 | "r":1.0, 61 | "f":1.0 62 | }, 63 | "FUNCTION":{ 64 | "p":1.0, 65 | "r":1.0, 66 | "f":1.0 67 | }, 68 | "FNAME":{ 69 | "p":1.0, 70 | "r":1.0, 71 | "f":1.0 72 | }, 73 | "LIBNAME":{ 74 | "p":1.0, 75 | "r":1.0, 76 | "f":1.0 77 | } 78 | }, 79 | "tok2vec_loss":6.8340121594, 80 | "ner_loss":1014.5475574388 81 | } 82 | } -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-best/ner/cfg: -------------------------------------------------------------------------------- 1 | { 2 | "moves":null, 3 | "update_with_oracle_cut_size":100, 4 | "multitasks":[ 5 | 6 | ], 7 | "min_action_freq":1, 8 | "learn_tokens":false, 9 | "beam_width":1, 10 | "beam_density":0.0, 11 | "beam_update_prob":0.0, 12 | "incorrect_spans_key":null 13 | } -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-best/ner/model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/models/model-best/ner/model -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-best/ner/moves: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/models/model-best/ner/moves -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-best/tok2vec/cfg: -------------------------------------------------------------------------------- 1 | { 2 | 3 | } -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-best/tok2vec/model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/models/model-best/tok2vec/model -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-best/tokenizer: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/models/model-best/tokenizer -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-best/vocab/key2row: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/models/model-best/vocab/key2row -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-best/vocab/lookups.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/models/model-best/vocab/lookups.bin -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-best/vocab/vectors: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/models/model-best/vocab/vectors -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-best/vocab/vectors.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "mode":"default" 3 | } -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-last/config.cfg: -------------------------------------------------------------------------------- 1 | [paths] 2 | train = "corpus/train.spacy" 3 | dev = "corpus/dev.spacy" 4 | vectors = "en_core_web_sm" 5 | init_tok2vec = null 6 | 7 | [system] 8 | gpu_allocator = null 9 | seed = 0 10 | 11 | [nlp] 12 | lang = "en" 13 | pipeline = ["tok2vec","ner"] 14 | batch_size = 1000 15 | disabled = [] 16 | before_creation = null 17 | after_creation = null 18 | after_pipeline_creation = null 19 | tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"} 20 | 21 | [components] 22 | 23 | [components.ner] 24 | factory = "ner" 25 | incorrect_spans_key = null 26 | moves = null 27 | scorer = {"@scorers":"spacy.ner_scorer.v1"} 28 | update_with_oracle_cut_size = 100 29 | 30 | [components.ner.model] 31 | @architectures = "spacy.TransitionBasedParser.v2" 32 | state_type = "ner" 33 | extra_state_tokens = false 34 | hidden_width = 64 35 | maxout_pieces = 2 36 | use_upper = true 37 | nO = null 38 | 39 | [components.ner.model.tok2vec] 40 | @architectures = "spacy.Tok2VecListener.v1" 41 | width = ${components.tok2vec.model.encode.width} 42 | upstream = "*" 43 | 44 | [components.tok2vec] 45 | factory = "tok2vec" 46 | 47 | [components.tok2vec.model] 48 | @architectures = "spacy.Tok2Vec.v2" 49 | 50 | [components.tok2vec.model.embed] 51 | @architectures = "spacy.MultiHashEmbed.v2" 52 | width = ${components.tok2vec.model.encode.width} 53 | attrs = ["NORM","PREFIX","SUFFIX","SHAPE"] 54 | rows = [5000,2500,2500,2500] 55 | include_static_vectors = true 56 | 57 | [components.tok2vec.model.encode] 58 | @architectures = "spacy.MaxoutWindowEncoder.v2" 59 | width = 256 60 | depth = 8 61 | window_size = 1 62 | maxout_pieces = 3 63 | 64 | [corpora] 65 | 66 | [corpora.dev] 67 | @readers = "spacy.Corpus.v1" 68 | path = ${paths.dev} 69 | max_length = 0 70 | gold_preproc = false 71 | limit = 0 72 | augmenter = null 73 | 74 | [corpora.train] 75 | @readers = "spacy.Corpus.v1" 76 | path = ${paths.train} 77 | max_length = 0 78 | gold_preproc = false 79 | limit = 0 80 | augmenter = null 81 | 82 | [training] 83 | dev_corpus = "corpora.dev" 84 | train_corpus = "corpora.train" 85 | seed = ${system.seed} 86 | gpu_allocator = ${system.gpu_allocator} 87 | dropout = 0.1 88 | accumulate_gradient = 1 89 | patience = 1600 90 | max_epochs = 0 91 | max_steps = 20000 92 | eval_frequency = 200 93 | frozen_components = [] 94 | annotating_components = [] 95 | before_to_disk = null 96 | 97 | [training.batcher] 98 | @batchers = "spacy.batch_by_words.v1" 99 | discard_oversize = false 100 | tolerance = 0.2 101 | get_length = null 102 | 103 | [training.batcher.size] 104 | @schedules = "compounding.v1" 105 | start = 100 106 | stop = 1000 107 | compound = 1.001 108 | t = 0.0 109 | 110 | [training.logger] 111 | @loggers = "spacy.ConsoleLogger.v1" 112 | progress_bar = false 113 | 114 | [training.optimizer] 115 | @optimizers = "Adam.v1" 116 | beta1 = 0.9 117 | beta2 = 0.999 118 | L2_is_weight_decay = true 119 | L2 = 0.01 120 | grad_clip = 1.0 121 | use_averages = false 122 | eps = 0.00000001 123 | learn_rate = 0.001 124 | 125 | [training.score_weights] 126 | ents_f = 1.0 127 | ents_p = 0.0 128 | ents_r = 0.0 129 | ents_per_type = null 130 | 131 | [pretraining] 132 | 133 | [initialize] 134 | vectors = ${paths.vectors} 135 | init_tok2vec = ${paths.init_tok2vec} 136 | vocab_data = null 137 | lookups = null 138 | before_init = null 139 | after_init = null 140 | 141 | [initialize.components] 142 | 143 | [initialize.tokenizer] -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-last/meta.json: -------------------------------------------------------------------------------- 1 | { 2 | "lang":"en", 3 | "name":"pipeline", 4 | "version":"0.0.0", 5 | "spacy_version":">=3.3.0,<3.4.0", 6 | "description":"", 7 | "author":"", 8 | "email":"", 9 | "url":"", 10 | "license":"", 11 | "spacy_git_version":"497a708c7", 12 | "vectors":{ 13 | "width":0, 14 | "vectors":0, 15 | "keys":0, 16 | "name":null, 17 | "mode":"default" 18 | }, 19 | "labels":{ 20 | "tok2vec":[ 21 | 22 | ], 23 | "ner":[ 24 | "CARDINAL", 25 | "COLNAME", 26 | "FNAME", 27 | "FUNCTION", 28 | "LIBNAME", 29 | "VARNAME" 30 | ] 31 | }, 32 | "pipeline":[ 33 | "tok2vec", 34 | "ner" 35 | ], 36 | "components":[ 37 | "tok2vec", 38 | "ner" 39 | ], 40 | "disabled":[ 41 | 42 | ], 43 | "performance":{ 44 | "ents_f":0.9992156863, 45 | "ents_p":1.0, 46 | "ents_r":0.9984326019, 47 | "ents_per_type":{ 48 | "COLNAME":{ 49 | "p":1.0, 50 | "r":0.9970457903, 51 | "f":0.9985207101 52 | }, 53 | "VARNAME":{ 54 | "p":1.0, 55 | "r":1.0, 56 | "f":1.0 57 | }, 58 | "CARDINAL":{ 59 | "p":1.0, 60 | "r":1.0, 61 | "f":1.0 62 | }, 63 | "FUNCTION":{ 64 | "p":1.0, 65 | "r":1.0, 66 | "f":1.0 67 | }, 68 | "FNAME":{ 69 | "p":1.0, 70 | "r":1.0, 71 | "f":1.0 72 | }, 73 | "LIBNAME":{ 74 | "p":1.0, 75 | "r":1.0, 76 | "f":1.0 77 | } 78 | }, 79 | "tok2vec_loss":0.0000003981, 80 | "ner_loss":0.0000006017 81 | } 82 | } -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-last/ner/cfg: -------------------------------------------------------------------------------- 1 | { 2 | "moves":null, 3 | "update_with_oracle_cut_size":100, 4 | "multitasks":[ 5 | 6 | ], 7 | "min_action_freq":1, 8 | "learn_tokens":false, 9 | "beam_width":1, 10 | "beam_density":0.0, 11 | "beam_update_prob":0.0, 12 | "incorrect_spans_key":null 13 | } -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-last/ner/model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/models/model-last/ner/model -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-last/ner/moves: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/models/model-last/ner/moves -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-last/tok2vec/cfg: -------------------------------------------------------------------------------- 1 | { 2 | 3 | } -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-last/tok2vec/model: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/models/model-last/tok2vec/model -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-last/tokenizer: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/models/model-last/tokenizer -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-last/vocab/key2row: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/models/model-last/vocab/key2row -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-last/vocab/lookups.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/models/model-last/vocab/lookups.bin -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-last/vocab/vectors: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deepklarity/jupyter-text2code/f8f01f4ed3eee935f3ebbd696c9fdbd743cbecc1/jupyter_text2code/jupyter_text2code_serverextension/models/model-last/vocab/vectors -------------------------------------------------------------------------------- /jupyter_text2code/jupyter_text2code_serverextension/models/model-last/vocab/vectors.cfg: -------------------------------------------------------------------------------- 1 | { 2 | "mode":"default" 3 | } -------------------------------------------------------------------------------- /jupyter_text2code/main.js: -------------------------------------------------------------------------------- 1 | define([ 2 | 'jquery', 3 | 'require', 4 | 'base/js/namespace', 5 | 'base/js/dialog', 6 | 'base/js/events', 7 | ], function ( 8 | $, 9 | requirejs, 10 | Jupyter, 11 | dialog, 12 | events 13 | ) { 14 | "use strict"; 15 | 16 | var jupyter_text2code_lib = {} 17 | jupyter_text2code_lib.code_init = ""; 18 | 19 | // define default values for config parameters 20 | var params = { 21 | jupyter_text2code_it_default_to_public: false, 22 | }; 23 | 24 | var extension_state = { 25 | is_open: false, 26 | data: { 27 | query: "import all libraries", 28 | history: [], 29 | presets: [ 30 | "help", 31 | "use dark theme", 32 | "import all libraries", 33 | "load (xxx.csv) in (df)", 34 | "pie plot of (column) in (df)", 35 | "bar plot of columns (column) & (column) in (df)", 36 | "list all columns of (df)", 37 | "show (x) rows of (df)" 38 | ] 39 | } 40 | }; 41 | 42 | function code_exec_callback(query, response) { 43 | var generated_code = JSON.parse(response)["message"] 44 | 45 | extension_state.data.history.push({"query": query, "code": generated_code}); 46 | update_history_display(query); 47 | 48 | var cur_cell = Jupyter.notebook.get_selected_cell(); 49 | if (cur_cell.get_text() == ""){ 50 | var command_cell = cur_cell; 51 | }else{ 52 | var command_cell = Jupyter.notebook.insert_cell_below('code'); 53 | } 54 | command_cell.select(); 55 | command_cell.set_text(generated_code); 56 | command_cell.execute(); 57 | Jupyter.notebook.insert_cell_below(); 58 | Jupyter.notebook.select_next(); 59 | } 60 | 61 | function jupyter_text2code_lib_callback(out_data) { 62 | if (out_data.msg_type === "execute_result"){ 63 | var query = $("#jupyter_text2code_query").val(); 64 | $.get({ 65 | url: '/jupyter-text2code', 66 | data: {"query": query, "dataframes_info": out_data.content.data['text/plain']}, 67 | beforeSend: function(){ 68 | $("#jupyter_text2code_loader").show(); 69 | }, 70 | success: function(response) { 71 | code_exec_callback(query, response); 72 | }, 73 | error: handle_jupyter_text2code_error, 74 | complete: function(){ 75 | $("#jupyter_text2code_loader").hide(); 76 | }, 77 | }); 78 | } 79 | } 80 | 81 | function read_code_init(lib) { 82 | var libName = Jupyter.notebook.base_url + "nbextensions/jupyter-text2code/" + lib; 83 | $.get(libName).done(function(data) { 84 | jupyter_text2code_lib.code_init = data; 85 | requirejs( 86 | [], 87 | function() { 88 | Jupyter.notebook.kernel.execute(jupyter_text2code_lib.code_init, { iopub: { output: jupyter_text2code_lib_callback } }, { silent: false }); 89 | }) 90 | console.log(libName + ' loaded library'); 91 | }).fail(function() { 92 | console.log(libName + 'failed to load ' + lib + ' library') 93 | }); 94 | } 95 | 96 | var initialize = function () { 97 | Jupyter.toolbar.add_buttons_group([ 98 | Jupyter.keyboard_manager.actions.register ({ 99 | help : 'Launch jupyter-text2code', 100 | icon : 'fa-terminal', 101 | handler: toggle_jupyter_text2code_editor 102 | }, 'create-jupyter-text2code-from-notebook', 'Text2Code') 103 | ]); 104 | read_code_init("jupyter_text2code_lib.py"); 105 | }; 106 | 107 | function toggle_jupyter_text2code_editor() { 108 | if(extension_state.is_open) { 109 | extension_state.is_open = false; 110 | $(".jupyter_text2code_editor_display").hide(); 111 | } 112 | else { 113 | if($('#jupyter_text2code_editor').length == 0) { 114 | build_jupyter_text2code_editor(); 115 | } 116 | extension_state.is_open = true; 117 | $(".jupyter_text2code_editor_display").show(); 118 | } 119 | } 120 | 121 | function build_alert(alert_class) { 122 | return $('
') 123 | .addClass('alert alert-dismissable') 124 | .addClass(alert_class) 125 | .append( 126 | $('