├── .editorconfig ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── docs.yml │ └── test.yml ├── .gitignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── docs ├── mkdocs.yml └── src │ ├── CONTRIBUTING.md │ ├── addendum.md │ ├── comparison_with_ipystata.md │ ├── getting_started.md │ ├── helpers │ └── helpers.js │ ├── img │ ├── atom.png │ ├── atom_help_magic.png │ ├── browse_atom.png │ ├── browse_notebook.png │ ├── esttab-html-file.png │ ├── esttab-html-jupyterlab.png │ ├── esttab-latex-jupyterlab.png │ ├── esttab-latex-pdf.png │ ├── hydrogen-watch-browse.gif │ ├── hydrogen-watch-graph.gif │ ├── jupyter_console.png │ ├── jupyter_notebook.png │ ├── jupyter_notebook_example.gif │ ├── jupyterlab_autocompletion.png │ ├── mobaxterm-local-port-forwarding.png │ ├── notebook_help_magic.png │ ├── nteract.pdf │ ├── nteract.png │ ├── starting_jupyter_notebook.gif │ ├── stata_kernel_example.gif │ └── subscribe-to-releases.png │ ├── index.md │ ├── license.md │ ├── using_jupyter │ ├── atom.md │ ├── console.md │ ├── intro.md │ ├── lab.md │ ├── notebook.md │ ├── nteract.md │ ├── qtconsole.md │ └── remote.md │ └── using_stata_kernel │ ├── configuration.md │ ├── intro.md │ ├── limitations.md │ ├── magics.md │ └── troubleshooting.md ├── examples └── Example.ipynb ├── poetry.lock ├── poetry.toml ├── pyproject.toml ├── setup.cfg ├── stata_kernel ├── __init__.py ├── __main__.py ├── ado │ ├── _StataKernelCompletions.ado │ ├── _StataKernelHead.ado │ ├── _StataKernelLog.ado │ ├── _StataKernelResetRC.ado │ └── _StataKernelTail.ado ├── code_manager.py ├── codemirror │ └── stata.js ├── completions.py ├── config.py ├── css │ └── _StataKernelHelpDefault.css ├── docs │ ├── css │ │ └── pandoc.css │ ├── index.html │ ├── index.txt │ ├── logo-64x64.png │ ├── make.sh │ ├── make_href │ └── using_stata_kernel │ │ ├── magics.html │ │ └── magics.txt ├── install.py ├── kernel.py ├── pygments │ ├── _mata_builtins.py │ └── stata.py ├── stata_lexer.py ├── stata_magics.py ├── stata_session.py └── utils.py └── tests ├── test_data └── auto.dta ├── test_kernel_completions.py ├── test_kernel_display_data.py ├── test_kernel_stdout.py ├── test_mata_lexer.py ├── test_stata_lexer.py └── utils.py /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: https://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Defaults for all files 7 | [*] 8 | end_of_line = lf 9 | insert_final_newline = true 10 | charset = utf-8 11 | trim_trailing_whitespace = true 12 | indent_style = space 13 | indent_size = 4 14 | 15 | # 2 space indentation 16 | [*.{js,R,r,yml,json}] 17 | indent_size = 2 18 | 19 | # Tab indentation (no size specified) 20 | [Makefile] 21 | indent_style = tab 22 | trim_trailing_whitespace = false 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | 5 | --- 6 | 7 | #### Problem description 8 | 9 | - This should explain **why** the current behavior is a problem and why the expected output is a better solution. 10 | - **Note**: Many problems can be resolved by simply upgrading `stata_kernel` to the latest version. Before submitting, please try: 11 | 12 | ``` 13 | pip install stata_kernel --upgrade 14 | ``` 15 | and check if your issue is fixed. 16 | 17 | #### Debugging log 18 | 19 | If possible, attach the text file located at 20 | 21 | ``` 22 | $HOME/.stata_kernel_cache/console_debug.log 23 | ``` 24 | where `$HOME` is your home directory. This will help us debug your problem quicker. 25 | 26 | **NOTE: This file includes a history of your session. If you work with restricted data, do not include this file.** 27 | 28 | #### Code Sample 29 | 30 | Especially if you cannot attach the debugging log, please include a [minimal, complete, and verifiable example.](https://stackoverflow.com/help/mcve) 31 | 32 | ```stata 33 | // Your code here 34 | 35 | ``` 36 | 37 | #### Expected Output 38 | 39 | #### Other information 40 | 41 | If you didn't attach the debugging log, please provide: 42 | 43 | - Operating System 44 | - Stata version 45 | - Package version (You can find this by running `pip show stata_kernel` in your terminal.) 46 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | 5 | --- 6 | 7 | **Is your feature request related to a problem? Please describe.** 8 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 9 | 10 | **Describe the solution you'd like** 11 | A clear and concise description of what you want to happen. 12 | 13 | **Describe alternatives you've considered** 14 | A clear and concise description of any alternative solutions or features you've considered. 15 | 16 | **Additional context** 17 | Add any other context or screenshots about the feature request here. 18 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | - [ ] closes #xxxx 2 | - [ ] tests added / passed 3 | - [ ] passes `git diff upstream/master -u -- "*.py" | flake8 --diff` 4 | - [ ] whatsnew entry 5 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: Docs 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | docs: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - uses: actions/setup-python@v2 14 | with: 15 | python-version: 3.x 16 | - run: pip install mkdocs mkdocs-material 17 | - run: cd docs && mkdocs gh-deploy --force 18 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | # On every pull request, but only on push to master 4 | on: 5 | push: 6 | branches: 7 | - master 8 | tags: 9 | - "*" 10 | pull_request: 11 | 12 | jobs: 13 | tests: 14 | runs-on: ubuntu-20.04 15 | strategy: 16 | matrix: 17 | python-version: [3.6, 3.7, 3.8, 3.9] 18 | 19 | steps: 20 | - uses: actions/checkout@v2 21 | 22 | - name: Set up Python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v2 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | 27 | - name: Set up Poetry 28 | uses: snok/install-poetry@v1 29 | with: 30 | version: 1.1.10 31 | 32 | - name: Install dependencies 33 | run: | 34 | poetry install 35 | poetry run python -m stata_kernel.install 36 | 37 | - name: Run tests 38 | # TODO: We should be running other tests and not just these two files 39 | run: | 40 | poetry run pytest tests/test_mata_lexer.py tests/test_stata_lexer.py 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .stata_kernel_images 3 | docs/html/mkdocs/ 4 | 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # Jupyter Notebook 77 | .ipynb_checkpoints 78 | 79 | # pyenv 80 | .python-version 81 | 82 | # celery beat schedule file 83 | celerybeat-schedule 84 | 85 | # SageMath parsed files 86 | *.sage.py 87 | 88 | # Environments 89 | .env 90 | .venv 91 | env/ 92 | venv/ 93 | ENV/ 94 | env.bak/ 95 | venv.bak/ 96 | 97 | # Spyder project settings 98 | .spyderproject 99 | .spyproject 100 | 101 | # Rope project settings 102 | .ropeproject 103 | 104 | # mkdocs documentation 105 | /site 106 | 107 | # mypy 108 | .mypy_cache/ 109 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [1.12.3] - 2021-02-25 4 | 5 | - Simplify kernel installation for conda environments 6 | 7 | ## [1.12.2] - 2020-11-30 8 | 9 | - File spaces in cache dir for graph export (#371) 10 | - spacing in cache dir for head/tail magics (#369) 11 | 12 | ## [1.12.1] - 2020-10-19 13 | 14 | - Fix install on linux when unable to find stata path (#363) 15 | 16 | ## [1.12.0] - 2020-09-21 17 | 18 | - Fix including spaces in filename (#356) 19 | - Add environmental variable option for config global and user locations (#353) 20 | - Support eps graphics via user-specified convert program (#346) 21 | 22 | ## [1.11.2] - 2020-02-21 23 | 24 | - Fix bug where `pdf_dup` was not defined when `graph_format` was not `svg` or `png`. #332 25 | 26 | ## [1.11.1] - 2020-02-10 27 | 28 | - Update search path to find Stata 16 path automatically on Windows. #330 29 | 30 | ## [1.11.0] - 2020-02-10 31 | 32 | - Allow for a global configuration file on Linux. #327 33 | - Fix bug with spaces in log file. #318 34 | - Fix deprecation notice. #324 35 | 36 | ## [1.10.5] - 2019-01-30 37 | 38 | - Fix bug where the return code (`_rc`) was reset by internal code after every user command. #288 39 | 40 | ## [1.10.4] - 2019-01-29 41 | 42 | - _Actually_ fix bug that prevented long commands on Windows from working. #284 43 | 44 | ## [1.10.3] - 2019-01-28 45 | 46 | - Fix "Code editor out of sync" Jupyterlab message. #270 47 | 48 | ## [1.10.2] - 2019-01-28 49 | 50 | - Fix bug that prevented long commands on Windows from working. #284 51 | - Make sure to always find correct Stata path on Mac when using Automation. 52 | - Several documentation updates 53 | 54 | ## [1.10.1] - 2019-01-08 55 | 56 | - Fix bug that prevented from working on Windows. #281 57 | 58 | ## [1.10.0] - 2019-01-04 59 | 60 | - Fix `%help` magic in JupyterLab. #273 61 | - Added example Jupyter Notebook file to documentation. #275 62 | - Throws custom error if the Automation type library isn't registered on Windows. #276 63 | - Adds program names to autocompletion. #280 64 | - Don't sort file paths before variable names. #261 65 | 66 | ## [1.9.0] - 2018-12-19 67 | 68 | - Add `%html` and `%latex` magics for displaying formatted output. #267 69 | - `r()` is no longer cleared between commands. #266 70 | - Add tests 71 | 72 | ## [1.8.1] - 2018-12-17 73 | 74 | - Fix log cleaning for programs and for loops with > 10 lines. #257 75 | - Fix autocomplete to show globals upon typing `${`. #253 76 | - Add kernel tests. #254 77 | - JupyterLab syntax highlighting now exists. Run `jupyter labextension install jupyterlab-stata-highlight` 78 | - Fix bug where incorrect files were suggested. #262 79 | - Allow use of bracketed globals in file path autocompletion 80 | 81 | ## [1.8.0] - 2018-11-28 82 | 83 | - Don't launch Stata with WinExec on Windows. This should have the benefit that 1) a new Stata window isn't created unless it needs to be, and 2) graphs don't show up unnecessarily. #249 84 | - Add magics to show and hide the Stata GUI. #251 85 | - Fix displaying image in QtConsole. #246 86 | - Fix completions bug. #247 87 | 88 | ## [1.7.4] - 2018-11-21 89 | 90 | - Fix finding Stata path on Windows when key not in registry. #242 91 | 92 | ## [1.7.3] - 2018-11-20 93 | 94 | - Fix log cleaning on Windows. #241 95 | 96 | ## [1.7.2] - 2018-11-20 97 | 98 | - Fix install. Add _mata_builtins to MANIFEST.in. #239 99 | 100 | ## [1.7.1] - 2018-11-19 101 | 102 | - Fix `%browse`, `%head`, `%tail` display issues. #237 103 | 104 | ## [1.7.0] - 2018-11-19 105 | 106 | - Add Mata mode. #116 107 | - Allow program to function without a configuration file. #222 108 | - Fixed Ctrl+C behavior. 109 | - Hide most Stata kernel output from user logs #228 110 | - Add completions on word boundaries. #229 111 | - Wrap SVGs in iframe tags to prevent cross-image issues. #235 112 | - Don't overwrite configuration file if it already exists. 113 | 114 | ## [1.6.2] - 2018-10-25 115 | 116 | - Revert closing graph window automatically. #219 117 | - Refactor `%browse` to use the same code as `%head` internally. #217 118 | 119 | ## [1.6.1] - 2018-10-24 120 | 121 | - Remove `regex` package as a dependency. #212 122 | - Fix `cap`/`noi`/`qui` completions with scalars and matrices. #213 123 | - Close Stata graph window after saving graph. #214 124 | - Fix regex to hide "note: graph.svg not found" 125 | 126 | ## [1.6.0] - 2018-10-24 127 | 128 | - File path autocompletions. Currently only works when files have no spaces in them. #195 129 | - Only export graph after successful command. #210 130 | - Display `--more--` in frontend when it stops the display. #198 131 | - Fix `scatter` in graph regex. Only `sc` and `scatter` produce plots when not 132 | preceded by `twoway`. #205 133 | - Give matrix and scalar autocompletions when using shortened command names. 134 | #206 135 | 136 | ## [1.5.9] - 2018-10-16 137 | 138 | - Fix bugs with Python 3.5. #203 139 | 140 | ## [1.5.8] - 2018-10-11 141 | 142 | - Fix incorrect regex escaping. #201 143 | 144 | ## [1.5.7] - 2018-10-11 145 | 146 | - Fix bug that parsed multiple `///` on a single line incorrectly. #200 147 | 148 | ## [1.5.6] - 2018-10-09 149 | 150 | - Fix bug that prevented `set rmsg on` from working. #199 151 | 152 | ## [1.5.5] - 2018-10-05 153 | 154 | - Add `user_graph_keywords` setting to allow graphs to be displayed for third-party commands. 155 | 156 | ## [1.5.4] - 2018-09-21 157 | 158 | - Force utf-8 encoding when writing `include` code to file. #196 159 | - Catch `EOF` when waiting for the PDF copy of graph. #192 160 | 161 | ## [1.5.3] - 2018-09-20 162 | 163 | - Set pexpect terminal size to 255 columns. #190 164 | 165 | ## [1.5.2] - 2018-09-19 166 | 167 | - Add pywin32 as a pip dependency on Windows, thus making installation easier. 168 | - Add jupyter 1.0.0 metapackage as a dependency, so that installs from Miniconda also install all of Jupyter. 169 | 170 | ## [1.5.1] - 2018-09-17 171 | 172 | - Fix issues with `--more--`. #103 173 | - PDF Graph redundancy. This improves ease of export to PDF via LaTeX. 174 | - Catch PermissionsError when copying syntax highlighting files 175 | - Add Stata logo for Jupyter Notebook 176 | - Autoclose local macro quotes in Jupyter Notebook 177 | - Highlight /// as comments in Jupyter Notebook 178 | - Highlight macros in Jupyter Notebook 179 | - Check latest PyPi package version and add alert to banner if newer 180 | - Simplify `%set` magic 181 | - Set default linesize to 255 for now to improve image handling. #177 182 | 183 | ## [1.5.0] - 2018-09-14 184 | 185 | - Add CodeMirror syntax highlighting for Jupyter Notebook 186 | - Improve Pygments syntax highlighting for highlighting of Jupyter QtConsole, Jupyter Console, and Notebook outputs in HTML and PDF format. 187 | - Restore PDF graph support. Although it doesn't display within the Jupyter Notebook for security (or maybe just practical) reasons, it's helpful when exporting a Notebook to a PDF via LaTeX. 188 | - Temporarily fix encoding error from CJK characters being split by a line break. #167 189 | 190 | ## [1.4.8] - 2018-09-12 191 | 192 | - Fix use of `which` in install script 193 | - Redirect `xstata` to `stata` on Linux. #149 194 | - Fix hiding code lines when there are hard tab characters (`\t`). #153 195 | - Make HTML help links open in new tab. #158 196 | - Open log files with utf-8 encoding. https://github.com/kylebarron/language-stata/issues/98 197 | 198 | ## [1.4.7] - 2018-09-08 199 | 200 | - Fix pypi upload. Need to use `python setup.py sdist bdist_wheel` and not `python setup.py sdist bdist`. The latter creates two source packages, and only one source package can ever be uploaded to Pypi per release. 201 | 202 | ## [1.4.6] - 2018-09-08 203 | 204 | - Fix `install.py`; previously it had unmatched `{` and `}` 205 | - Fix display of whitespace when entire result is whitespace. #111 206 | 207 | ## [1.4.5] - 2018-09-07 208 | 209 | - Don't embed images in HTML help; link to them. #140 210 | - Fix blocking for line continuation when string is before `{` #139 211 | - Fix hiding of code lines with leading whitespace. #120 212 | - Remove `stata_kernel_graph_counter` from globals suggestions. #109 213 | - Always use UTF-8 encoding when loading SVGs. #130 214 | - Add download count and Atom gif to README. Try to fix images for Pypi page. 215 | 216 | ## [1.4.4] - 2018-09-06 217 | 218 | - Fully hide Stata GUI on Windows. Always export log file, even on Windows and Mac Automation. 219 | - Set more off within ado files. Should fix #132. 220 | - Use bumpversion for easy version number updating. 221 | - Add `%help kernel` and `%help magics` options 222 | - Add general debugging information (like OS/Stata version/package version) to log 223 | - Add help links to Jupyter Notebook's Help dropdown UI 224 | - Various docs fixes 225 | 226 | ## [1.4.3] - 2018-09-04 227 | 228 | - Release to pypi again because 1.4.2 didn't upload correctly. Apparently only a 229 | Mac version was uploaded, and even that didn't work. 230 | 231 | ## [1.4.2] - 2018-08-21 232 | 233 | - Fix line cleaning for loops/programs of more than 9 lines 234 | - Remove pexpect timeout 235 | - Provide error message upon incomplete input sent to `do_execute` 236 | 237 | ## [1.4.1] - 2018-08-21 238 | 239 | - Add `%head` and `%tail` magics 240 | - Change `%set plot` to `%set graph` 241 | 242 | ## [1.4.0] - 2018-08-21 243 | 244 | - Return results as Stata returns them, not when command finishes 245 | - More stable method of knowing when a command finishes by looking for the text's MD5 hash 246 | - Finds Stata executable during install 247 | - Automatically show graphs after graph commands 248 | - Add %help and %browse magics 249 | - Allow for graph scaling factors 250 | - Fix Windows locals issue 251 | - Fix image spacing 252 | 253 | ## [1.3.1] - 2018-08-13 254 | 255 | - Fix pip installation by adding CHANGELOG and requirements files to `MANIFEST.in`. 256 | 257 | ## [1.3.0] - 2018-08-13 258 | 259 | - Context-aware autocompletions 260 | - Support for #delimit; blocks interactively 261 | - Better parsing for when a user-provided block is complete or not. Typing `2 + ///` will prompt for the next line. 262 | - Split lexer into two lexers. This is helpful to first remove comments and convert #delimit; blocks to cr-delimited blocks. 263 | - Fix svg aspect ratio 264 | - Magics for plotting, retrieving locals and globals, timing commands, seeing current delimiter. 265 | - Add documentation website 266 | 267 | ## [1.2.0] - 2018-08-11 268 | 269 | - Support for `if`, `else`, `else if`, `cap`, `qui`, `noi`, `program`, `input` blocks #28, #27, #30 270 | - Support different graph formats #21 271 | - Heavily refactor codebase into hopefully more stable API #32 272 | - Correctly parse long, text wrapped lines from log file or console #41 273 | - Use a single cache directory, configurable by the user #43 274 | - Correctly remove comments, using a tokenizer #38, #25, #29 275 | 276 | 277 | ## [1.1.0] - 2018-08-06 278 | 279 | **Initial release!** This would ordinarily be something like version 0.1.0, but the Echo kernel framework that I made this from was marked as 1.1 internally, and I forgot to change that before people started downloading this. I don't want to move my number down to 0.1 and have people who already installed not be able to upgrade. 280 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | All types of contributions are welcome. You can: 4 | 5 | - [Submit a bug report](#bug-reports) 6 | - [Update or add new documentation or examples](#updating-the-docs) 7 | - [Add automated tests](#tests) 8 | - Submit a pull request for a new feature 9 | 10 | ## Bug reports 11 | 12 | The best way to get your issue solved is to provide a [minimal, complete, verifiable example.](https://stackoverflow.com/help/mcve) In order to submit a bug report, [click here](https://github.com/kylebarron/stata_kernel/issues/new/choose) and fill out the template. 13 | 14 | ## Debugging 15 | 16 | The following seems to be the easiest way to debug internals: 17 | 18 | ```py 19 | from stata_kernel.kernel import StataKernel 20 | from stata_kernel.code_manager import CodeManager 21 | 22 | kernel = StataKernel() 23 | session = kernel.stata 24 | 25 | # If on windows, may be helpful 26 | session.show_gui() 27 | 28 | code = 'sysuse auto, clear' 29 | cm = CodeManager(code) 30 | text_to_run, md5, text_to_exclude = cm.get_text(kernel.conf, session) 31 | rc, res = session.do( 32 | text_to_run, md5, text_to_exclude=text_to_exclude, display=False) 33 | ``` 34 | 35 | ## Tests 36 | 37 | ### Adding tests 38 | 39 | Tests are contained in the Python files in the `tests/` folder. The `test_stata_lexer.py` and `test_mata_lexer.py` files run automated tests on the code Stata kernel uses to parse user input. 40 | 41 | ### Running tests 42 | 43 | To run the tests, you need to install `pytest` and `jupyter_kernel_test`: 44 | ``` 45 | pip install pytest jupyter_kernel_test 46 | ``` 47 | 48 | From the project root, to run all tests, run 49 | 50 | ``` 51 | pytest tests/ 52 | ``` 53 | 54 | To run just the non-automated tests that depend on having Stata available locally, run 55 | 56 | ``` 57 | pytest tests/test_kernel.py 58 | ``` 59 | 60 | For each of the above, if you get a `ModuleNotFound` error, you may need to use `python -m pytest tests/`. 61 | 62 | ## Updating the docs 63 | 64 | First install `mkdocs`: 65 | 66 | ``` 67 | pip install mkdocs mkdocs-material 68 | ``` 69 | 70 | Then `cd` to the docs folder: 71 | ``` 72 | cd docs/ 73 | ``` 74 | 75 | Then to serve the documentation website in real time, run 76 | ``` 77 | mkdocs serve 78 | ``` 79 | This starts a web server on localhost, usually on port 8000. So you can open your web browser and type in `localhost:8000`, click Enter, and you should see the website. This will update in real time as you write more documentation. 80 | 81 | To create a static website, run: 82 | ``` 83 | mkdocs build 84 | ``` 85 | 86 | To publish the website to the documentation website (if you have repository push access) run: 87 | ``` 88 | mkdocs gh-deploy 89 | ``` 90 | 91 | 92 | ## Releasing new versions 93 | 94 | To increment version numbers, run one of: 95 | ``` 96 | bumpversion major 97 | bumpversion minor 98 | bumpversion patch 99 | ``` 100 | in the project's root directory. This will also automatically create a git commit and tag of the version. Then push with: 101 | 102 | ``` 103 | git push origin master --tags 104 | ``` 105 | 106 | so that Github sees the newest tag. 107 | 108 | Then to release: 109 | 110 | ``` 111 | python setup.py sdist bdist_wheel 112 | python -m twine upload dist/stata_kernel-VERSION* 113 | ``` 114 | and put in the PyPI username and password. 115 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include CHANGELOG.md 3 | include requirements.txt 4 | include requirements_dev.txt 5 | include LICENSE 6 | include stata_kernel/ado/*.ado 7 | include stata_kernel/css/*.css 8 | include stata_kernel/docs/logo-64x64.png 9 | include stata_kernel/docs/*html 10 | include stata_kernel/docs/*txt 11 | include stata_kernel/docs/using_stata_kernel/*html 12 | include stata_kernel/docs/using_stata_kernel/*txt 13 | include stata_kernel/codemirror/stata.js 14 | include stata_kernel/pygments/stata.py 15 | include stata_kernel/pygments/_mata_builtins.py 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # stata_kernel 2 | 3 | [![Build Status](https://travis-ci.org/kylebarron/stata_kernel.svg?branch=master)](https://travis-ci.org/kylebarron/stata_kernel) [![Downloads](https://pepy.tech/badge/stata-kernel)](https://pepy.tech/project/stata-kernel) [![Downloads/Month](https://pepy.tech/badge/stata-kernel/month)](https://pepy.tech/project/stata-kernel) 4 | 5 | `stata_kernel` is a Jupyter kernel for Stata; It works on Windows, macOS, and 6 | Linux. 7 | 8 | To see an example Jupyter Notebook, [click here.](https://nbviewer.jupyter.org/github/kylebarron/stata_kernel/blob/master/examples/Example.ipynb) 9 | 10 | For documentation and more information, see: [https://kylebarron.dev/stata_kernel](https://kylebarron.dev/stata_kernel) 11 | 12 | #### Jupyter Notebook 13 | ![Jupyter Notebook](https://raw.githubusercontent.com/kylebarron/stata_kernel/master/docs/src/img/jupyter_notebook_example.gif) 14 | 15 | #### Atom 16 | ![Atom](https://raw.githubusercontent.com/kylebarron/stata_kernel/master/docs/src/img/stata_kernel_example.gif) 17 | -------------------------------------------------------------------------------- /docs/mkdocs.yml: -------------------------------------------------------------------------------- 1 | # Project Information 2 | site_name: 'stata_kernel' 3 | site_description: 'A Jupyter Kernel for Stata. Works with Windows, macOS, and Linux.' 4 | site_author: 'Kyle Barron' 5 | 6 | docs_dir: 'src' 7 | site_dir: 'html/mkdocs' 8 | 9 | # Repository 10 | repo_name: 'kylebarron/stata_kernel' 11 | repo_url: 'https://github.com/kylebarron/stata_kernel' 12 | edit_uri: 'blob/master/docs/src/' 13 | site_url: 'https://kylebarron.dev/stata_kernel/' 14 | 15 | # Social links 16 | extra: 17 | social: 18 | - icon: 'fontawesome/brands/github' 19 | link: 'https://github.com/kylebarron' 20 | - icon: 'fontawesome/brands/twitter' 21 | link: 'https://twitter.com/kylebarron2' 22 | 23 | # Layout 24 | nav: 25 | - Home: 'index.md' 26 | - Getting Started: 'getting_started.md' 27 | - Jupyter Notebook Example: 'https://nbviewer.jupyter.org/github/kylebarron/stata_kernel/blob/master/examples/Example.ipynb' 28 | - Using Jupyter: 29 | - Introduction: 'using_jupyter/intro.md' 30 | - JupyterLab: 'using_jupyter/lab.md' 31 | - Hydrogen in Atom: 'using_jupyter/atom.md' 32 | - Jupyter Notebook: 'using_jupyter/notebook.md' 33 | - Nteract: 'using_jupyter/nteract.md' 34 | - Jupyter QtConsole: 'using_jupyter/qtconsole.md' 35 | - Jupyter Console: 'using_jupyter/console.md' 36 | - Remote Servers: 'using_jupyter/remote.md' 37 | - Using the Stata Kernel: 38 | - Introduction: 'using_stata_kernel/intro.md' 39 | - Configuration: 'using_stata_kernel/configuration.md' 40 | - Magics: 'using_stata_kernel/magics.md' 41 | - Limitations: 'using_stata_kernel/limitations.md' 42 | - Troubleshooting: 'using_stata_kernel/troubleshooting.md' 43 | - Comparison with IPyStata: 'comparison_with_ipystata.md' 44 | - Contributing: 'CONTRIBUTING.md' 45 | - Addendum: 'addendum.md' 46 | - License: 'license.md' 47 | 48 | google_analytics: 49 | - 'UA-125422577-1' 50 | - 'auto' 51 | 52 | # Theme 53 | theme: 54 | feature: 55 | tabs: false 56 | icon: 57 | logo: 'material/home' 58 | repo: 'fontawesome/brands/github' 59 | name: 'material' 60 | language: 'en' 61 | palette: 62 | primary: 'blue' 63 | accent: 'light blue' 64 | font: 65 | text: 'Nunito Sans' 66 | code: 'Fira Code' 67 | 68 | # Uncomment if I use math in the docs in the future 69 | # extra_javascript: 70 | # - helpers/helpers.js 71 | # - https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.4/MathJax.js?config=TeX-AMS-MML_HTMLorMML 72 | 73 | # These extensions are chosen to be a superset of Pandoc's Markdown. 74 | # This way, I can write in Pandoc's Markdown and have it be supported here. 75 | # https://pandoc.org/MANUAL.html 76 | markdown_extensions: 77 | - admonition 78 | - attr_list 79 | - codehilite: 80 | guess_lang: false 81 | - def_list 82 | - footnotes 83 | - pymdownx.arithmatex 84 | - pymdownx.betterem 85 | - pymdownx.caret: 86 | insert: false 87 | - pymdownx.details 88 | - pymdownx.emoji 89 | - pymdownx.escapeall: 90 | hardbreak: true 91 | nbsp: true 92 | - pymdownx.magiclink: 93 | hide_protocol: true 94 | repo_url_shortener: true 95 | - pymdownx.smartsymbols 96 | - pymdownx.superfences 97 | - pymdownx.tasklist: 98 | custom_checkbox: true 99 | - pymdownx.tilde 100 | - toc: 101 | permalink: true 102 | -------------------------------------------------------------------------------- /docs/src/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ../../CONTRIBUTING.md -------------------------------------------------------------------------------- /docs/src/addendum.md: -------------------------------------------------------------------------------- 1 | # Addendum 2 | 3 | As an ardent open-source advocate and someone who actively dislikes using Stata, 4 | it somewhat pains me that my work creates value for a proprietary, closed-source 5 | program. I hope that this program improves research in a utilitarian way, and 6 | shows to new users the scope of the open-source tools that have existed for 7 | upwards of a _decade_. 8 | 9 | ## Contributors 10 | 11 | - [Kyle Barron](https://github.com/kylebarron) 12 | - [Mauricio Cáceres](https://github.com/mcaceresb) 13 | - [Full list of contributors](https://github.com/kylebarron/stata_kernel/graphs/contributors) 14 | -------------------------------------------------------------------------------- /docs/src/comparison_with_ipystata.md: -------------------------------------------------------------------------------- 1 | # Comparison with [IPyStata](https://github.com/TiesdeKok/ipystata) 2 | 3 | ## `stata_kernel` is faster with larger datasets 4 | 5 | `stata_kernel` takes a different approach to communication with Stata. With `IPyStata` on macOS and Linux, to run each segment of code 6 | 7 | 1. Your data has to be moved from Python to Stata 8 | 2. Run the commands in Stata 9 | 3. Return the data to Python to save it for the next command 10 | 11 | This process is prohibitive with larger amounts of data. In contrast, `stata_kernel` controls Stata directly, so it generally is no slower than using the Stata program itself. 12 | 13 | ## `stata_kernel` provides more features 14 | 15 | `stata_kernel` is a pure Jupyter kernel, whereas IPyStata is a Jupyter _magic_ within the Python kernel. This means that with `stata_kernel` 16 | 17 | - You don't have to include `%%stata` at the beginning of every cell. 18 | - You get features like autocompletion and being able to use `;` as a delimiter. 19 | - You see intermediate results of long-running commands without waiting for the entire command to have finished. 20 | - You can create multiple graphs in the same cell without having to name each of them individually. (Order of the graphs is also guaranteed). 21 | - You don't have to have any knowledge whatsoever of Python [^1]. 22 | 23 | [^1]: Python is amazing language, and if you want to move on to bigger data, I highly recommend learning Python. Now that `stata_kernel` is installed, if you want to start a Python notebook instead of a Stata notebook, just choose New > Python 3 in the dropdown menu. 24 | -------------------------------------------------------------------------------- /docs/src/getting_started.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | 3 | It doesn't take much to get `stata_kernel` up and running. Here's how: 4 | 5 | ## Prerequisites 6 | 7 | - **Stata**. A currently-licensed version of Stata must already be installed. `stata_kernel` has been reported to work with at least Stata 13+, and may work with Stata 12. 8 | - **Python**. In order to install the kernel, Python 3.5, 3.6, or 3.7 needs to be installed on the computer on which Stata is running. 9 | 10 | I suggest installing the [Anaconda 11 | distribution](https://www.anaconda.com/download/). This doesn't require 12 | administrator privileges, and is the simplest way to install Python and many of the most popular scientific packages. 13 | 14 | The full Anaconda installation is quite large, and includes many libraries for Python that 15 | `stata_kernel` doesn't use. If you don't plan to use Python and want to use 16 | less disk space, install [Miniconda](https://conda.io/miniconda.html), a bare-bones version of Anaconda. Then when [installing the package](#package-install) any other necessary dependencies will be 17 | downloaded automatically. 18 | 19 | ???+ note "Windows-specific steps" 20 | 21 | In order to let `stata_kernel` talk to Stata, you need to [link the Stata Automation library](https://www.stata.com/automation/#install): 22 | 23 | 1. In the installation directory (most likely `C:\Program Files (x86)\Stata15` or similar), right-click on the Stata executable, for example, `StataSE.exe`. Choose `Create Shortcut`. Placing it on the Desktop is fine. 24 | 2. Right-click on the newly created `Shortcut to StataSE.exe`, choose `Property`, and append `/Register` to the end of the Target field. So if the target is currently `"C:\Program Files\Stata15\StataSE.exe"`, change it to `"C:\Program Files\Stata15\StataSE.exe" /Register`. Click `OK`. 25 | 3. Right-click on the updated `Shortcut to StataSE.exe`; choose `Run as administrator`. 26 | 27 | ## Package Install 28 | 29 | If you use Anaconda or Miniconda, from the Anaconda Prompt run: 30 | 31 | ```bash 32 | conda install -c conda-forge stata_kernel 33 | python -m stata_kernel.install 34 | ``` 35 | 36 | If you do not use Anaconda/Miniconda, from a terminal or command prompt run: 37 | 38 | ```bash 39 | pip install stata_kernel 40 | python -m stata_kernel.install 41 | ``` 42 | 43 | If Python 2 is the default version of Python on your system, you may need to use 44 | `python3` instead of `python` for the `python3 -m stata_kernel.install` step. 45 | 46 | ### Jupyter 47 | 48 | If you chose to install Anaconda you already have [Jupyter Notebook](https://jupyter-notebook.readthedocs.io/en/stable/notebook.html) and [Jupyter Lab](https://jupyterlab.readthedocs.io/en/stable/getting_started/overview.html) installed. 49 | 50 | Otherwise, you need to install Jupyter Notebook or Jupyter Lab. I recommend the latter as it is a similar but more modern environment. If you have Miniconda, open the Anaconda Prompt and run: 51 | 52 | ```bash 53 | conda install jupyterlab 54 | ``` 55 | 56 | If you use pip, you can install it via: 57 | 58 | ```bash 59 | pip install jupyterlab 60 | ``` 61 | 62 | If you would not like to install Jupyter Lab and only need the Notebook, you can install it by running 63 | 64 | ```bash 65 | conda install notebook 66 | ``` 67 | 68 | or 69 | 70 | ```bash 71 | pip install notebook 72 | ``` 73 | 74 | depending on your package manager. 75 | 76 | In order to get syntax highlighting in Jupyter Lab, run: 77 | ```bash 78 | conda install -c conda-forge nodejs -y 79 | jupyter labextension install jupyterlab-stata-highlight 80 | ``` 81 | 82 | If you didn't install Python from Anaconda/Miniconda, the `conda` command won't work and you'll need to install [Node.js](https://nodejs.org/en/download/) directly before running `jupyter labextension install`. 83 | 84 | ### Upgrading 85 | 86 | To upgrade from a previous version of `stata_kernel`, from a terminal or command prompt run 87 | 88 | ```bash 89 | conda update stata_kernel -y 90 | ``` 91 | in the case of Anaconda/Miniconda or 92 | 93 | ```bash 94 | pip install stata_kernel --upgrade 95 | ``` 96 | otherwise. 97 | 98 | When upgrading, you don't have to run `python -m stata_kernel.install` again. 99 | 100 | ## Using 101 | 102 | Next, read more about [Jupyter and its different 103 | interfaces](using_jupyter/intro.md) or about [how to use the Stata 104 | kernel](using_stata_kernel/intro.md), specifically. 105 | -------------------------------------------------------------------------------- /docs/src/helpers/helpers.js: -------------------------------------------------------------------------------- 1 | MathJax.Hub.Config({ 2 | TeX: { 3 | equationNumbers: { 4 | autoNumber: "AMS" 5 | } 6 | }, 7 | tex2jax: { 8 | inlineMath: [ ['$','$'], ['\\(', '\\)'] ], 9 | displayMath: [ ['$$','$$'] ], 10 | processEscapes: true, 11 | } 12 | }); 13 | -------------------------------------------------------------------------------- /docs/src/img/atom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/atom.png -------------------------------------------------------------------------------- /docs/src/img/atom_help_magic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/atom_help_magic.png -------------------------------------------------------------------------------- /docs/src/img/browse_atom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/browse_atom.png -------------------------------------------------------------------------------- /docs/src/img/browse_notebook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/browse_notebook.png -------------------------------------------------------------------------------- /docs/src/img/esttab-html-file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/esttab-html-file.png -------------------------------------------------------------------------------- /docs/src/img/esttab-html-jupyterlab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/esttab-html-jupyterlab.png -------------------------------------------------------------------------------- /docs/src/img/esttab-latex-jupyterlab.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/esttab-latex-jupyterlab.png -------------------------------------------------------------------------------- /docs/src/img/esttab-latex-pdf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/esttab-latex-pdf.png -------------------------------------------------------------------------------- /docs/src/img/hydrogen-watch-browse.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/hydrogen-watch-browse.gif -------------------------------------------------------------------------------- /docs/src/img/hydrogen-watch-graph.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/hydrogen-watch-graph.gif -------------------------------------------------------------------------------- /docs/src/img/jupyter_console.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/jupyter_console.png -------------------------------------------------------------------------------- /docs/src/img/jupyter_notebook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/jupyter_notebook.png -------------------------------------------------------------------------------- /docs/src/img/jupyter_notebook_example.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/jupyter_notebook_example.gif -------------------------------------------------------------------------------- /docs/src/img/jupyterlab_autocompletion.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/jupyterlab_autocompletion.png -------------------------------------------------------------------------------- /docs/src/img/mobaxterm-local-port-forwarding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/mobaxterm-local-port-forwarding.png -------------------------------------------------------------------------------- /docs/src/img/notebook_help_magic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/notebook_help_magic.png -------------------------------------------------------------------------------- /docs/src/img/nteract.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/nteract.pdf -------------------------------------------------------------------------------- /docs/src/img/nteract.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/nteract.png -------------------------------------------------------------------------------- /docs/src/img/starting_jupyter_notebook.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/starting_jupyter_notebook.gif -------------------------------------------------------------------------------- /docs/src/img/stata_kernel_example.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/stata_kernel_example.gif -------------------------------------------------------------------------------- /docs/src/img/subscribe-to-releases.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/docs/src/img/subscribe-to-releases.png -------------------------------------------------------------------------------- /docs/src/index.md: -------------------------------------------------------------------------------- 1 | # stata_kernel 2 | 3 | `stata_kernel` is a Jupyter kernel for Stata. It works on Windows, macOS, and 4 | Linux. 5 | 6 | ## What is Jupyter? 7 | 8 | Jupyter is an open-source ecosystem for interactive data science. Originally 9 | developed around the Python programming language, Jupyter has grown to interface 10 | with dozens of programming languages. 11 | 12 | `stata_kernel` is the bridge that interactively connects Stata to all the 13 | elements in the ecosystem. 14 | 15 | - [**JupyterLab**](using_jupyter/lab.md) is a web-based interactive editor that allows for interweaving of code, text, and results. 16 | 17 | - Splice models in LaTeX math mode with the code that implements them and the graphs depicting their output. 18 | - [Click here](https://nbviewer.jupyter.org/github/kylebarron/stata_kernel/blob/master/examples/Example.ipynb) to see an example Jupyter Notebook file using `stata_kernel`. 19 | - Jupyter Notebooks can be exported as PDFs or HTML, and are as good for teaching new students as they are for displaying research results. 20 | 21 | ![Jupyter Notebook](img/jupyter_notebook_example.gif) 22 | 23 | - [**Hydrogen**](using_jupyter/atom.md) is a package for the [Atom text editor](https://atom.io) that connects with Jupyter kernels to display results interactively in your text editor. 24 | - The [**Jupyter console**](using_jupyter/console.md) is an enhanced interactive console. Its features include enhanced autocompletion, better searching of history, syntax highlighting, among others. The similar [QtConsole](using_jupyter/qtconsole.md) even allows displaying plots within the terminal. 25 | - [Enhanced remote work](using_jupyter/remote.md). You can set up Jupyter to run computations remotely but to show results locally. Since the only data passing over the network are the text inputs and outputs from Stata, communcation happens much faster than loading `xstata`, especially on slower networks. Being able to use Jupyter Notebook or Hydrogen vastly enhances productivity compared to working with the Stata console through a remote terminal. 26 | 27 | ## `stata_kernel` Features 28 | 29 | - [x] Supports Windows, macOS, and Linux. 30 | - [x] Use any type of comments in your code, not just `*`. 31 | - [x] [Autocompletions](using_stata_kernel/intro#autocompletion) as you type based on the variables, macros, scalars, and matrices currently in memory. As of version 1.6.0 it also suggests file paths for autocompletion. 32 | - [x] [Display graphs](using_stata_kernel/intro/#displaying-graphs). 33 | - [x] Receive results as they appear, not after the entire command finishes. 34 | - [x] [Pull up interactive help files within the kernel](using_stata_kernel/magics#help). 35 | - [x] [Browse data interactively](using_stata_kernel/magics#browse). 36 | - [x] [`#delimit ;` interactive support](using_stata_kernel/intro#delimit-mode) 37 | - [x] Work with a [remote session of Stata](using_jupyter/remote). 38 | - [x] Mata interactive support 39 | - [ ] Cross-session history file 40 | 41 | ## Screenshots 42 | 43 | **Atom** 44 | 45 | ![Atom](img/stata_kernel_example.gif) 46 | -------------------------------------------------------------------------------- /docs/src/using_jupyter/atom.md: -------------------------------------------------------------------------------- 1 | # Hydrogen in Atom 2 | 3 | Hydrogen is a package for the Atom text editor that connects with Jupyter kernels, such as `stata_kernel`, to display results interactively inside the text editor. 4 | 5 | I'll go over how to install Atom and Hydrogen, and then provide a quick overview of Hydrogen's capabilities. For more information on how to use Hydrogen, see [Hydrogen's documentation](https://nteract.gitbooks.io/hydrogen/docs/Usage/GettingStarted.html). 6 | 7 | ## Installation 8 | 9 | Atom and Hydrogen are both free and open source software, just like `stata_kernel`. The download and install is free and easy. 10 | 11 | ### Atom 12 | 13 | Go to [atom.io](https://atom.io), choose the installer for your operating system, then double click the downloaded file. 14 | 15 | ### Hydrogen 16 | 17 | Next you'll need to install a couple add-on packages: 18 | 19 | - [Hydrogen](https://atom.io/packages/hydrogen): this connects to Jupyter kernels and allows you to view results in-line next to your code. You can use it with Python, R, and Julia, as well as Stata. 20 | - [language-stata](https://atom.io/packages/language-stata): this provides syntax highlighting for Stata code and is necessary so that Hydrogen knows that a file with extension `.do` is a Stata file. 21 | 22 | To install these, go to the Atom Settings. You can get there by clicking Preferences > Settings in the menus or by using the keyboard shortcut Ctrl+, (Cmd-, on macOS). Then click _Install_ on the menu in the left, and type in `Hydrogen`, and `language-stata`, and click `Install`. 23 | 24 | Once those are installed, open a do-file and run Ctrl-Enter (Cmd-Enter on macOS) to start the Stata kernel. 25 | 26 | ## Using Atom 27 | 28 | If you've never used Atom before, you're in luck, because it's quite simple and intuitive to use. 29 | [Read here](https://flight-manual.atom.io/getting-started/sections/atom-basics/) for some basics about how to use Atom. 30 | 31 | The most important thing to know is that you can access every command available to you in Atom by using the keyboard shortcut Ctrl+Shift+P (Cmd+Shift+P on macOS). This brings up a menu, called the _Command Palette_, where you can find any command that any package provides. 32 | 33 | ## Running code 34 | 35 | There are three main ways to run code using Hydrogen: 36 | 37 | - Selection. Manually select/highlight the lines you want to send to Stata and then run `Hydrogen: Run`, which is usually bound to Ctrl-Enter. 38 | - Cursor block. When `Hydrogen: Run` is called and no code is selected, Hydrogen runs the current line. If code following the present line is more indented than the current line, Hydrogen will run the entire indented block. 39 | - Cell. A cell is a block of lines to be executed at once. They are defined using `%%` inside comments. See [here](https://nteract.gitbooks.io/hydrogen/docs/Usage/GettingStarted.html#hydrogen-run-cell) for more information. 40 | 41 | ### Output 42 | 43 | Output will display directly beside or beneath the block of code that was 44 | selected to be run. Code that does not produce any output will show a check mark 45 | when completed. 46 | 47 | ### Watch Expressions 48 | 49 | Hydrogen allows for [_watch expressions_](https://nteract.gitbooks.io/hydrogen/docs/Usage/GettingStarted.html#watch-expressions). These are expressions that are automatically re-run after every command sent to the kernel. This is convenient for viewing regression output, graphs, or the data set after changing a parameter. 50 | 51 | Note that since the watch expression is run after _every_ command, it shouldn't be something that changes the state of the data. For example, `replace mpg = mpg * 2` would be unwise to set as a watch expression because that would change the column's data after every command. 52 | 53 | Using watch expressions with a graph: 54 | ![](../img/hydrogen-watch-graph.gif) 55 | 56 | Using watch expressions to browse data: 57 | ![](../img/hydrogen-watch-browse.gif) 58 | 59 | ## Indentation and for loops 60 | 61 | Stata `foreach` loops and programs must be sent as a whole to the kernel. If you 62 | send only part of a loop, you'll receive a reply that insufficient input was 63 | provided. 64 | 65 | The easiest way to make sure that this happens is to indent all code pertaining 66 | to a block. This will ensure all lines of the block are sent to `stata_kernel`, 67 | even if the ending `}` has the same indentation as the initial line. 68 | 69 | If the cursor is anywhere on the first line in the segment below, and you run 70 | Ctrl-Enter or Shift-Enter (which 71 | moves your cursor to the next line), it will include the final `}`. 72 | 73 | ```stata 74 | foreach i in 1 2 3 4 { 75 | display "`i'" 76 | } 77 | ``` 78 | -------------------------------------------------------------------------------- /docs/src/using_jupyter/console.md: -------------------------------------------------------------------------------- 1 | # Jupyter Console 2 | 3 | To use it as a console, in your terminal or command prompt run: 4 | ``` 5 | jupyter console --kernel stata 6 | ``` 7 | 8 | Example: 9 | 10 | ![Jupyter Notebook](../img/jupyter_console.png) 11 | -------------------------------------------------------------------------------- /docs/src/using_jupyter/intro.md: -------------------------------------------------------------------------------- 1 | # Introduction to Jupyter 2 | 3 | There are many different ways to use Jupyter. I'll briefly explain several ways of using Jupyter and provide links to more documentation. The information in this section is common to all languages Jupyter supports. [Using the Stata Kernel](../using_stata_kernel/magics.md) has information specific to using Stata with Jupyter. 4 | 5 | If you're unsure which to use, choose [JupyterLab](lab.md). 6 | 7 | - [JupyterLab](lab.md): This is an interactive web-based editor that improves upon the classic Notebook by making it easy to work with several files at the same time in the same window. Users familiar with RStudio may like this. 8 | - [Hydrogen for Atom](atom.md): This is a plugin for the Atom text editor that displays results in line with your code. It's my personal favorite. 9 | - [Jupyter Notebook](notebook.md): This is the classic interactive web-based editor. 10 | - [Nteract](nteract.md): This is a desktop application to work with Jupyter Notebook files. Some may prefer it to the classic web interface of Jupyter Notebooks. 11 | - [Jupyter Console](console.md): This is an enhanced REPL that lives in the console. 12 | - [Jupyter QtConsole](qtconsole.md): An enhanced console that additionally supports graphs and other rich-text displays. 13 | - [Working remotely](remote.md): Any of these tools can be used to edit code on your local computer and have the code run on a remote Unix/Linux server. 14 | -------------------------------------------------------------------------------- /docs/src/using_jupyter/lab.md: -------------------------------------------------------------------------------- 1 | # JupyterLab 2 | 3 | Jupyter Lab is the successor to [Jupyter Notebook](notebook.md), and allows for having multiple documents side-by-side. 4 | 5 | ### Starting JupyterLab 6 | 7 | You can start JupyterLab by running: 8 | 9 | ``` 10 | jupyter lab 11 | ``` 12 | 13 | in your terminal or command prompt. Just like the Notebook, this should open up a page in your browser, where you can open a new Stata notebook or console. 14 | 15 | ### Syntax highlighting 16 | 17 | To enable syntax highlighting for Stata with JupyterLab, you need to run (only once): 18 | 19 | ```bash 20 | conda install -c conda-forge nodejs -y 21 | jupyter labextension install jupyterlab-stata-highlight 22 | ``` 23 | 24 | If you didn't install Python from Anaconda, the `conda` command won't work and you'll need to install [Node.js](https://nodejs.org/en/download/) directly before running `jupyter labextension install`. 25 | 26 | ### Plugins 27 | 28 | One of the benefits of JupyterLab over the Notebook is that it was designed for extensibility. There's a growing list of plugins that can be used with JupyterLab. Here's an unofficial list: 29 | 30 | ### More info 31 | 32 | Project documentation website: 33 | 34 | 35 | ![](https://jupyterlab.readthedocs.io/en/stable/_images/jupyterlab.png) 36 | -------------------------------------------------------------------------------- /docs/src/using_jupyter/notebook.md: -------------------------------------------------------------------------------- 1 | # Jupyter Notebook 2 | 3 | You can start the Jupyter Notebook server by running 4 | 5 | ``` 6 | jupyter notebook 7 | ``` 8 | 9 | in your terminal or command prompt. That should open up your browser to the 10 | Jupyter home screen. Click the *New* drop down menu in the top right and choose 11 | `Stata` from the list to start a new notebook using Stata as your default 12 | kernel. 13 | 14 | Below is a gif that shows each step of this process. 15 | 16 | [Click here for more documentation.](https://jupyter-notebook.readthedocs.io/en/stable/notebook.html) 17 | 18 | ![](../img/starting_jupyter_notebook.gif) 19 | -------------------------------------------------------------------------------- /docs/src/using_jupyter/nteract.md: -------------------------------------------------------------------------------- 1 | # Nteract 2 | 3 | Nteract is a desktop-based computing environment to work with Jupyter Notebook files. It can load and save Jupyter Notebook files and export to PDFs, including output. 4 | 5 | You can download the software from . 6 | 7 | (It's helpful to know that you can use Ctrl+Space to trigger autocompletions in Nteract.) 8 | 9 | Below is an example screenshot of using Nteract, followed by its PDF output. 10 | 11 | ![](../img/nteract.png) 12 | 13 | 15 | -------------------------------------------------------------------------------- /docs/src/using_jupyter/qtconsole.md: -------------------------------------------------------------------------------- 1 | # Jupyter QtConsole 2 | 3 | 4 | > The Qt console is a very lightweight application that largely feels like a terminal, but provides a number of enhancements only possible in a GUI, such as inline figures, proper multi-line editing with syntax highlighting, graphical calltips, and much more. The Qt console can use any Jupyter kernel. 5 | 6 | Project documentation website: 7 | https://qtconsole.readthedocs.io/en/stable/ 8 | 9 | 10 | ![](https://qtconsole.readthedocs.io/en/stable/_images/qtconsole.png) 11 | -------------------------------------------------------------------------------- /docs/src/using_jupyter/remote.md: -------------------------------------------------------------------------------- 1 | # Remote Unix/Linux Servers 2 | 3 | ## Introduction 4 | 5 | There are two ways to use Jupyter on a remote Unix/Linux server. 6 | 7 | - The less-usable way, but which is simpler to get started with, is simply using [Jupyter Console](console.md) within a remote terminal. This gives you many—but not all—of the features of `stata_kernel`, including syntax highlighting, free-flowing comments, autocompletions, and `#delimit ;` support. To do this, all you need to do is install `stata_kernel` on the remote server (Stata must also be installed on that remote server). 8 | 9 | - The much more powerful way to run Jupyter on a remote server is to set it up so that you're working with [JupyterLab](lab.md) in your local web browser, or with [Hydrogen](atom.md) in Atom running locally, but where all the computation happens on the remote server. This is a little more involved to set up, but, once running, makes working on a remote server effortless. 10 | 11 | The rest of this document explains how to set up the latter method. 12 | 13 | !!! warning 14 | 15 | It's important to understand the security implications of allowing remote 16 | access to your machine. An improperly configured server or a Jupyter 17 | Notebook server without a password could allow an attacker to run arbitrary 18 | code on your computer, conceivably accessing private data or deleting files. 19 | 20 | ## Overview 21 | 22 | This setup is one of my favorite parts of Jupyter. Since very little information is sent back and forth, this setup has very low latency, which means that it's fast even on slower networks. With this setup, if you're waiting on output, it's likely because Stata is slow, not the connection. 23 | 24 | When using a program like `tmux` or `screen` on the server, you can reconnect _to the same Jupyter kernel_ (and thus the same Stata session) after becoming disconnected from the network (i.e. if a VPN times out). In contrast, with a normal console or GUI Stata session, there is no way to reconnect to the running Stata session, and you'd have to recreate the state of the environment from scratch. 25 | 26 | To understand how to set this up, it's helpful to know a bit about how Jupyter works. 27 | Jupyter has a "client-server model": 28 | 29 | - the _client_ (i.e. the Notebook, Hydrogen, or console) accepts user input and displays output. 30 | - the _server_ (the Jupyter internals) receives input from the client, forwards it to Stata, retrieves the results, and sends them back to the client. 31 | 32 | Usually, both the client and server run on the same computer. 33 | However, this is not a requirement; they just need to be able to communicate with each other. 34 | 35 | When working with remote servers, you'll instead run the client on the computer you're physically at and the server on the computer on which computation should happen. 36 | 37 | ### Network Ports 38 | 39 | A network [_port_](https://en.wikipedia.org/wiki/Port_(computer_networking)) is a numbered entity that defines what type of communication to access. 40 | 41 | For example, HTTP which defines how websites are loaded in your browser, runs on a specific port (80). So going to in your browser is the same as going to [example.com:80](http://example.com:80). The `:80` is just usually omitted. 42 | 43 | Instead of loading web pages, we need to configure it so that your browser loads the data from _Jupyter_. 44 | 45 | By default, the Jupyter server process runs on port `8888`. This is why, when you run Jupyter Notebook on your local machine, you'll usually see `http://localhost:8888` in the address bar. This means that the Jupyter server process is emitting data on port `8888` on the local computer. 46 | 47 | When working with Jupyter remotely, you'll have to know what port Jupyter is running on so that you can _forward_ the remote port to your local computer during the SSH connection. This is the core of the step that connects the server process on the remote computer to the client process on the local computer. 48 | 49 | ## Set up 50 | 51 | There are two options for what to use on the server: 52 | 53 | - [Jupyter Notebook Server](https://jupyter-notebook.readthedocs.io/en/stable/public_server.html): This is relatively simple to set up and does not need administrator privileges. Though not designed for use on a multi-user server, it is possible to use on one. 54 | - [JupyterHub](https://github.com/jupyterhub/jupyterhub): This is Jupyter's official solution for servers with multiple users. This is much more difficult to set up (and might need administrator privileges) and I won't go into details here. 55 | 56 | The rest of this guide assumes you're installing the Jupyter Notebook server. [Full documentation is here](https://jupyter-notebook.readthedocs.io/en/stable/public_server.html). 57 | 58 | !!! info 59 | 60 | `stata_kernel` must be installed on the server, the computer that is doing the computations. Neither Jupyter nor `stata_kernel` (nor Stata) needs to be installed on the client computer. 61 | 62 | The following instructions assume you are able to connect to the remote server 63 | through SSH. Setting up an SSH server is outside the scope of this guide. 64 | 65 | ### Creating the configuration file 66 | 67 | On the _server_ computer run in a terminal 68 | 69 | ``` 70 | jupyter notebook --generate-config 71 | ``` 72 | 73 | This creates a configuration file at one of three locations on the remote computer: 74 | 75 | - Windows: `C:\Users\USERNAME\.jupyter\jupyter_notebook_config.py` 76 | - OS X: `/Users/USERNAME/.jupyter/jupyter_notebook_config.py` 77 | - Linux: `/home/USERNAME/.jupyter/jupyter_notebook_config.py` 78 | 79 | 80 | ### Connecting with SSH 81 | 82 | #### Linux and macOS 83 | 84 | A usual SSH connection can be created from the terminal by running: 85 | 86 | ``` 87 | ssh username@host 88 | ``` 89 | 90 | In order to connect to the remote Jupyter session, we need to forward the port that Jupyter is running on from the remote computer to the local computer. If Jupyter is running on port `8888`, we can forward the remote port to the same local port by running 91 | 92 | ``` 93 | ssh username@host -L 8888:localhost:8888 94 | ``` 95 | 96 | #### Windows 97 | 98 | ![Mobaxterm local port forwarding](../img/mobaxterm-local-port-forwarding.png) 99 | 100 | [Here's information](https://blog.mobatek.net/post/ssh-tunnels-and-port-forwarding/#simple-explanation-of-ssh-tunnels-and-port-forwarding:b8ebdf9b2cb412a3a77c16c73c0d31ed) for how to forward a remote port using Mobaxterm. In MobaXterm, go to Tools > MobaxtermTunnel > New and select `Local port forwarding`. The most common configuration, clockwise from left, is: 101 | 102 | - `8888` 103 | - `localhost` 104 | - `8888` 105 | - the server hostname, i.e. `server.org` or its full IP address 106 | - your username, i.e. `john_doe` 107 | - `22` 108 | 109 | Full explanation, clockwise from left 110 | 111 | - The "My computer with MobaXterm" box on the left (`12345` in the image) is customarily `8888`, though it can be any 4- or 5-digit number. This is the number you'll type into your local web browser, i.e. `http://localhost:8888`. 112 | - The "Remote server" box in the top right should usually have `localhost` and then the port on which Jupyter is running. Usually this is `8888`. 113 | - The "SSH server" box in the bottom right should have your server's hostname and username, and the port on which SSH is running (usually `22`). 114 | 115 | ### Starting Jupyter Notebook 116 | 117 | Run `jupyter notebook --no-browser --port=8888` in the remote terminal. 118 | 119 | On your local computer, go to `http://localhost:8888`. It should ask you for a token to sign in, which normally will be printed in the remote console once Jupyter Notebook starts. 120 | 121 | Then you can start a Jupyter Notebook session with the Stata kernel like usual. 122 | 123 | Here's [more documentation](https://nteract.gitbooks.io/hydrogen/docs/Usage/RemoteKernelConnection.html) for connecting to a remote kernel from Hydrogen. You can even connect to multiple servers at the same time. 124 | -------------------------------------------------------------------------------- /docs/src/using_stata_kernel/configuration.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | The configuration file is a plain text file named `.stata_kernel.conf` and is 4 | located in your home directory, or defined by the environmental variable 5 | `STATA_KERNEL_USER_CONFIG_PATH`. Settings must be under the heading 6 | `[stata_kernel]`. You can change any of the package's settings by 7 | opening the file and changing the `value` of any line of the form 8 | 9 | ``` 10 | configuration_setting_name = value 11 | ``` 12 | 13 | You can also make changes to the configuration while the kernel is running with the [%set magic](magics.md#set). For example: 14 | 15 | ``` 16 | %set autocomplete_closing_symbol False 17 | %set graph_format png 18 | ``` 19 | 20 | If you want these changes to be stored permanently, add `--permanently`: 21 | ``` 22 | %set graph_format png --permanently 23 | ``` 24 | 25 | !!! info "System wide configuration file for JupyterHub" 26 | 27 | If you are installing `stata_kernel` in Jupyter Hub you must create a system 28 | wide configuration file to provide default values. The default location 29 | is in `/etc/stata_kernel.conf`, or defined by the environmental variable 30 | `STATA_KERNEL_GLOBAL_CONFIG_PATH`. 31 | 32 | ## General settings 33 | 34 | ### `stata_path` 35 | 36 | A string; the path on your file system to your Stata executable. Usually this can be found automatically during the [install step](../getting_started.md#package-install), but sometimes may need to be set manually. This cannot be changed while using `%set`, and must be edited in the configuration file before starting Jupyter. 37 | 38 | ### `cache_directory` 39 | 40 | A string; the directory for the kernel to store temporary log files and graphs. By default, this is `~/.stata_kernel_cache`, where `~` means your home directory. You may wish to change this location if, for example, you're working under a Data Use Agreement where all related files must be stored in a specific directory. 41 | 42 | ### `execution_mode` 43 | 44 | **macOS only**, a string of either `"automation"` or `"console"`. 45 | 46 | This is the method through which `stata_kernel` communicates with Stata. `automation` uses [Stata Automation](https://www.stata.com/automation/) while `console` controls the console version of Stata. 47 | 48 | `console` is the default because it allows for multiple independent sessions 49 | of Stata to run at the same time, and can be faster. `automation` supports running `browse`, to bring up the Stata data explorer, however the [`%browse` magic](magics.md#browse) can also be used to inspect data within Jupyter (with either execution mode). 50 | 51 | On Windows, all communication with Stata happens through Stata Automation, because Stata console doesn't exist for Windows. On Linux/Unix all communication happens through the console, because Stata Automation doesn't exist for Linux/Unix. 52 | 53 | ???+ warning "Notice for StataIC Mac users" 54 | 55 | The main way that `stata_kernel` communicates with the running Stata session 56 | on macOS and Linux is with the _console version_ of Stata. This runs in a 57 | terminal instead of with the Stata GUI. For no good reason StataCorp decided 58 | not to ship the console program with StataIC on macOS. 59 | 60 | To work around this, StataIC Mac users must use `automation` execution mode. 61 | 62 | On macOS, using Automation is slower than using console mode, but there's 63 | nothing I can do about it. I asked StataCorp why they don't ship a console 64 | version with StataIC on Mac, when they do on Linux. Basically you're not a 65 | "power user". 66 | 67 | > Unix operating systems often have an optional graphical user interface so we 68 | > need to include console versions of Stata for all flavors of Stata on those 69 | > systems. 70 | > 71 | > The Mac operating system always has a graphical user interface so the console 72 | > version of Stata on the Mac is a special tool that is included for power users. 73 | > The Stata/IC for Mac is designed for regular Stata users and does not include a 74 | > console version. 75 | > 76 | > Originally the Mac versions of Stata were just like the Windows versions and 77 | > did not have any console support. 78 | 79 | \- Stata Technical Support 80 | 81 | ### `autocomplete_closing_symbol` 82 | 83 | either `True` or `False`; whether autocompletion suggestions should include the closing symbol (i.e. ``'`` for a local macro or `}` if the global starts with `${`). This is `False` by default. 84 | 85 | ## Graph settings 86 | 87 | These settings determine how graphs are displayed internally. [Read here](intro.md#displaying-graphs) for more information about how `stata_kernel` displays graphs. 88 | 89 | ### `graph_format` 90 | 91 | `svg`, `png` or `eps`; the format to export graphs. By default this is `svg` for most operating systems and versions of Stata, but is `png` by default for Windows on Stata 14 and below. Note `eps` cannot be displayed by kernel front-ends and requires converting to `png` via `graph_epstopng_program`. 92 | 93 | ### `graph_epstopng_program` 94 | 95 | With `graph_format = eps`, a program to convert `eps` figures to `png` so they can be displayed by the kernel front-end. For example, on Linux the user can specify `graph_epstopng_program = convert -density 300 {0} -resize '900x600' {1}`. (Note the name of the temporary graph files are passed to the program and the user must take that into account.) 96 | 97 | ### `graph_scale` 98 | 99 | a decimal number. This scales equally the width and height of plots displayed. By default, plots are 600 pixels wide. 100 | 101 | ### `graph_width` 102 | 103 | an integer. This is the width in pixels of graphs displayed. If no `graph_height` is set, Stata will determine the optimal height for the specific image. 104 | 105 | ### `graph_height` 106 | 107 | an integer. This is the height in pixels of graphs displayed. 108 | 109 | ### `user_graph_keywords` 110 | 111 | a string. `stata_kernel` [displays graphs](intro.md#displaying-graphs) by quietly inserting a `graph export` command after any command that creates a graph, and then loading and displaying the saved file. By default, it only looks for the base list of graph commands. 112 | 113 | If you use third party commands that generate figures, this option allows you to provide a list of commands that will also display graphs. Provide multiple graph names as a comma-delimited string, e.g. in the configuration file add: 114 | 115 | ``` 116 | user_graph_keywords = vioplot,coefplot 117 | ``` 118 | 119 | Note that when using the [`%set` magic](magics.md#set), the list of comma-delimited keywords must not have any spaces in it. For example, you must run 120 | 121 | ``` 122 | %set user_graph_keywords vioplot,coefplot 123 | ``` 124 | 125 | and not 126 | 127 | ``` 128 | %set user_graph_keywords vioplot, coefplot 129 | ``` 130 | 131 | ### `graph_svg_redundancy` 132 | 133 | Whether to provide redundant PDF images when `svg` is the display format. `True` by default. 134 | For more information about what _Graph Redundancy_ is, [read here](intro.md#graph-redundancy). 135 | 136 | ### `graph_png_redundancy` 137 | 138 | Whether to provide redundant PDF images when `png` is the display format. `False` by default. 139 | For more information about what _Graph Redundancy_ is, [read here](intro.md#graph-redundancy). 140 | 141 | ## Example config file 142 | 143 | An example config file: 144 | 145 | ``` ini 146 | [stata_kernel] 147 | stata_path = "C:\Program Files\Stata16\StataMP-64.exe" 148 | execution_mode = automation 149 | cache_directory = ~/.stata_kernel_cache 150 | autocomplete_closing_symbol = False 151 | graph_format = svg 152 | graph_scale = 1 153 | user_graph_keywords = coefplot,vioplot 154 | ``` 155 | -------------------------------------------------------------------------------- /docs/src/using_stata_kernel/intro.md: -------------------------------------------------------------------------------- 1 | # Using the Stata Kernel 2 | 3 | `stata_kernel` is the bridge between Stata and the Jupyter ecosystem. It will work with any of the tools outlined in [Using Jupyter](../using_jupyter/intro.md). After [installing](../getting_started.md) and optionally [configuring](configuration.md) `stata_kernel`, it should be ready for use. 4 | 5 | I recommend browsing [this example Jupyter Notebook file](https://nbviewer.jupyter.org/github/kylebarron/stata_kernel/blob/master/examples/Example.ipynb) to see many of `stata_kernel`'s features in action. If you [download that file](https://raw.githubusercontent.com/kylebarron/stata_kernel/master/examples/Example.ipynb) and load it into [JupyterLab](../using_jupyter/lab.md), you can edit the cells to try it out interactively. 6 | 7 | ## Displaying graphs 8 | 9 | `stata_kernel` displays graphs by quietly inserting a `graph export` command after any command that creates a graph, and then loading and displaying the saved file. The advantage of this approach is that it will display _all_ graphs created, even inside a loop or program, as long as that program was defined in text you run with the kernel. 10 | 11 | To minimize false positives, the graph keyword must appear at the beginning of a line. To hide the display of a graph, just prefix `graph` with [`quietly`](https://www.stata.com/help.cgi?quietly). 12 | 13 | ### Graph not displaying? 14 | 15 | `stata_kernel` looks for _graph commands_ in your code, and requires that these be the first non-whitespace characters on a given line. This means that if you prefix `scatter` with `quietly`, `noisily`, or `capture`, the graph won't be displayed. 16 | 17 | In order to force display of a graph, you can run: 18 | ```do 19 | graph display 20 | ``` 21 | 22 | `stata_kernel` checks your command against a list of graph commands, and only tries to export a graph if one matches. This means that some user-created commands that export graphs (e.g., `coefplot` or `vioplot`) won't work out of the box. 23 | 24 | To display graphs from user created commands, add the command name to the 25 | [`user_graph_keywords`](../../getting_started#user_graph_keywords) setting. You 26 | can do this either in the configuration file before starting the session or with 27 | ``` 28 | %set user_graph_keywords command1,command2,... 29 | ``` 30 | during the session. 31 | 32 | ### Graph display format 33 | 34 | `stata_kernel` must export the image in some format in order to load and display it. Sadly, there are considerable pros and cons to each image format: 35 | 36 | - `png` files can't be created with the console version of Stata, and thus are off limits to the Linux and Mac (console) modes of `stata_kernel`. Additionally, they can look pixelated when scaled up. 37 | - `svg` files can be created by Stata 14 and 15 on all platforms, and look crisp at all sizes, but can't be used with LaTeX PDF output. 38 | - `pdf` files can be created by all recent versions of Stata on all platforms, look crisp at all sizes, and work in LaTeX PDF output, but [JupyterLab](../../using_jupyter/lab) is the only front end in which they'll display correctly. 39 | - `tif` files are too large. 40 | 41 | `stata_kernel` lets you set the display format to be `png`, `svg`, or `pdf`. 42 | 43 | ### Graph redundancy 44 | 45 | One of the many amazing things about Jupyter Notebooks is that with a single click, you can [export the notebook](https://nbconvert.readthedocs.io/en/latest/) to an aesthetic PDF using LaTeX. 46 | 47 | Except on Windows using Stata 14 or earlier, `stata_kernel` displays images in `svg` format by default. However, as [noted above](#graph-display-format), these images can't be included in a LaTeX PDF export without conversion. 48 | 49 | To solve this problem, `stata_kernel` has the ability to hand Jupyter _both_ the `svg` or `png` version of an image _and_ the `pdf` version of the same image. While only the former will be displayed, the `pdf` version of the image will be stored in the Notebook and used when exported to PDF. 50 | 51 | While ease of use with LaTeX on macOS and Linux is a significant benefit, this redundancy does make `stata_kernel` delay slightly when displaying an image and enlarges the Jupyter Notebook file size (because two formats of every image will be stored within the Jupyter Notebook file). 52 | 53 | To turn off graph redundancy, change both configuration options to False: 54 | 55 | - `graph_svg_redundancy`: Whether to provide redundant PDF images when `svg` is the display format. `True` by default. 56 | - `graph_png_redundancy`: Whether to provide redundant PDF images when `png` is the display format. `False` by default. 57 | 58 | Because both image formats will be stored within the Jupyter Notebook file, `stata_kernel` will warn you if graph redundancy is on and an image is larger than 2 megabytes. To turn off this warning, set `graph_redundancy_warning` to `False`. 59 | 60 | ## Autocompletion 61 | 62 | Based on the current Stata environment, `stata_kernel` will autocomplete variables, locals, globals, scalars, matrices, _and file paths_ (as of version 1.6.0). 63 | 64 | As of version 1.6.0, file paths will only generate suggestions if there are no spaces in what you've typed. In the future I hope to relax this restriction, so that quoted file paths with spaces will still allow autocomplete. 65 | 66 | By default, autocomplete does not include the trailing character (such as a `'` for a local macro) when you select a suggestion. This is because front ends like [Hydrogen](../../using_jupyter/atom) already autocomplete the `'` for you after you type a `` ` ``. If you're using a different front end, you can turn on the [`autocomplete_closing_symbol`](../../getting_started#autocomplete_closing_symbol) setting so that locals include the ending `'`. 67 | 68 | ## `#delimit ;` mode 69 | 70 | Stata lets you use [`;` as a command 71 | delimiter](https://www.stata.com/help.cgi?delimit) in a do file after a line 72 | `#delimit ;`. This can be helpful when writing very long commands, like 73 | complicated graphs, as it allows for more free-flowing line wrapping. But Stata 74 | doesn't allow for running `;`-delimited code interactively, which makes 75 | debugging code difficult. 76 | 77 | `stata_kernel` lets you code interactively with semicolons as the delimiter for commands within the `#delimit ;` mode. When activated, it first removes the extra line breaks in the input code and then removes the semicolon, resulting in the code's carriage return-delimited equivalent, which can then be sent to Stata. 78 | 79 | To turn this mode on, just run `#delimit ;`. To turn it off, run `#delimit cr`. To check the current delimiter, run `%delimit`. Code can switch back and forth between delimiters several times with no issue. 80 | 81 | Note that when the setting is enabled, sending code to Stata without a `;` at the end will be returned as invalid. 82 | -------------------------------------------------------------------------------- /docs/src/using_stata_kernel/limitations.md: -------------------------------------------------------------------------------- 1 | # Limitations 2 | 3 | Due to the architecture of `stata_kernel`, there is some usual functionality that won't work. 4 | 5 | ### Log files 6 | 7 | #### Log files have extra code in them 8 | 9 | In order to provide extra functionality like magics and autocomplete, 10 | `stata_kernel` runs a few extra commands in Stata after your command has 11 | completed. The downside of this is that all those extra commands and their 12 | output show up in user-created log files. 13 | 14 | In general, I recommend using `stata_kernel` to create Jupyter Notebooks, rather than using Stata's log file exporting. 15 | 16 | #### Can't run `log close _all` on Windows 17 | 18 | Some people have `log close _all` as a standard command at the top of each 19 | script. On Windows and on macOS using [Automation 20 | mode](http://localhost:8000/getting_started/#general-settings), this will break 21 | `stata_kernel` functionality, because that line closes the log file that it uses 22 | to receive communications. 23 | -------------------------------------------------------------------------------- /docs/src/using_stata_kernel/magics.md: -------------------------------------------------------------------------------- 1 | # Magics 2 | 3 | **Magics** are programs provided by `stata_kernel` that enhance the experience 4 | of working with Stata in Jupyter. 5 | 6 | All magics are special commands that start with `%`. They must be the first word 7 | of the cell or selection, otherwise they won't be intercepted and will be sent 8 | to Stata. 9 | 10 | For most of the magics listed, you can add `--help` to see a help menu in the kernel. For example, 11 | ``` 12 | In [1]: %locals --help 13 | usage: %locals [-h] [-v] [REGEX [REGEX ...]] 14 | 15 | positional arguments: 16 | REGEX regex to match 17 | 18 | optional arguments: 19 | -h, --help show this help message and exit 20 | -v, --verbose Verbose output (print full contents of matched locals). 21 | ``` 22 | 23 | The magics that respond with richly formatted text, namely `%browse` and 24 | `%help`, will not work with Jupyter Console or Jupyter QtConsole, since they 25 | don't support displaying HTML. 26 | 27 | ## `%browse`, `%head`, `%tail` 28 | 29 | **Interactively view your dataset** 30 | 31 | This can optionally be provided with a `varlist`, `N`, or `if`: 32 | ``` 33 | %browse [-h] [N] [varlist] [if] 34 | %head [-h] [N] [varlist] [if] 35 | %tail [-h] [N] [varlist] [if] 36 | ``` 37 | 38 | By default: 39 | 40 | - `%browse` displays the first 200 rows 41 | - `%head` displays the first 10 rows 42 | - `%tail` displays the last 10 rows 43 | 44 | If you're using Windows or macOS with Automation mode, you can also run `browse` 45 | (without the `%`) and it will open the usual Stata data explorer. 46 | 47 | | | | 48 | |:--------------------:|:-----------------------------------------------:| 49 | | **Atom** | ![Atom](../img/browse_atom.png) | 50 | | **Jupyter Notebook** | ![Jupyter Notebook](../img/browse_notebook.png) | 51 | 52 | ## `%delimit` 53 | 54 | **Print the current delimiter** 55 | 56 | This takes no arguments; it prints the delimiter currently set: either `cr` or 57 | `;`. If you want to change the delimiter, use `#delimit ;` or `#delimit cr`. The 58 | delimiter will remain set until changed. 59 | 60 | ``` 61 | In [1]: %delimit 62 | The delimiter is currently: cr 63 | 64 | In [2]: #delimit ; 65 | delimiter now ; 66 | In [3]: %delimit 67 | The delimiter is currently: ; 68 | 69 | In [4]: #delimit cr 70 | delimiter now cr 71 | ``` 72 | 73 | ## `%help` 74 | 75 | **Display a help file in rich text** 76 | 77 | ``` 78 | %help [-h] command_or_topic_name 79 | ``` 80 | 81 | Add the term you want to search for after `%help`: 82 | ``` 83 | In [1]: %help histogram 84 | ``` 85 | 86 | The terms in italics (Atom) or underlined (Jupyter Notebook) are _links_. Click 87 | on them to see another help menu. 88 | 89 | 90 | | | | 91 | |:--------------------:|:-----------------------------------------------:| 92 | | **Atom** | ![Atom](../img/atom_help_magic.png) | 93 | | **Jupyter Notebook** | ![Jupyter Notebook](../img/notebook_help_magic.png) | 94 | 95 | ## `%html`, `%latex` 96 | 97 | **Display command output as HTML or LaTeX** 98 | 99 | This can be useful when creating regression tables with `esttab`, for example. The images below are run following 100 | 101 | ```stata 102 | sysuse auto 103 | eststo: qui regress price weight mpg 104 | eststo: qui regress price weight mpg foreign 105 | ``` 106 | 107 | An HTML table will display correctly both inside JupyterLab and as a saved HTML file. 108 | 109 | | | | 110 | |:-------------------:|:------------------------------------------------------------:| 111 | | **JupyterLab** | ![esttab-html-jupyterlab](../img/esttab-html-jupyterlab.png) | 112 | | **Saved HTML file** | ![esttab-html-file](../img/esttab-html-file.png) | 113 | 114 | 115 | A LaTeX table will not display correctly within JupyterLab (it only supports the math subset of LaTeX) **but it _will_** render correctly upon export to a PDF (which happens through LaTeX). 116 | 117 | | | | 118 | |:-------------------:|:--------------------------------------------------------------:| 119 | | **Saved PDF file** | ![esttab-latex-pdf](../img/esttab-latex-pdf.png) | 120 | | **Saved HTML file** | ![esttab-latex-jupyterlab](../img/esttab-latex-jupyterlab.png) | 121 | 122 | 123 | 124 | 125 | ## `%locals`, `%globals` 126 | 127 | **List local or global macros** 128 | 129 | ``` 130 | %locals [-h] [-v] [REGEX [REGEX ...]] 131 | %globals [-h] [-v] [REGEX [REGEX ...]] 132 | ``` 133 | 134 | These take two optional arguments: 135 | 136 | 1. a regular expression for filtering the locals or globals displayed 137 | 2. a `--verbose` flag 138 | 139 | ``` 140 | In [1]: %globals S_ 141 | (note: showing first line of global values; run with --verbose) 142 | 143 | S_ADO: BASE;SITE;.;PERSONAL;PLUS;OLDPLACE;`"/home/kyle/github/stata/sta 144 | S_level: 95 145 | S_CONSOLE: console 146 | S_FLAVOR: Intercooled 147 | S_OS: Unix 148 | S_MACH: PC (64-bit x86-64) 149 | 150 | In [2]: %globals S_ --verbose 151 | S_ADO: BASE;SITE;.;PERSONAL;PLUS;OLDPLACE;`"/home/kyle/github/stata/sta 152 | > ta_kernel/stata_kernel/ado"' 153 | S_level: 95 154 | S_CONSOLE: console 155 | S_FLAVOR: Intercooled 156 | S_OS: Unix 157 | S_MACH: PC (64-bit x86-64) 158 | ``` 159 | 160 | 161 | ## `%set` 162 | 163 | **Set configuration value** 164 | 165 | Usage: 166 | ``` 167 | %set [-h] [--permanently] [--reset] key value 168 | ``` 169 | 170 | - `key`: Configuration key name. The full list of configuration options is shown on the [Configuration](configuration.md) page. 171 | - `value`: Value to set. 172 | - `--permanently`: Store settings permanently. 173 | - `--reset`: Restore default settings. 174 | 175 | As an example, you can change the graph settings like so: 176 | 177 | ``` 178 | %set graph_format svg --permanently 179 | %set graph_scale 1 180 | %set graph_width 500 181 | %set graph_height 300 182 | ``` 183 | 184 | ## `%show_gui`, `%hide_gui` 185 | 186 | Show/hide the Stata Graphical User Interface (GUI). Only works on Windows (and Mac if using automation execution mode)" 187 | 188 | ## `%status` 189 | 190 | Print information about: 191 | 192 | - Stata kernel version 193 | - Whether you're in Stata/Mata 194 | - Current delimiter 195 | 196 | 205 | -------------------------------------------------------------------------------- /docs/src/using_stata_kernel/troubleshooting.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting 2 | 3 | ## Installation 4 | 5 | - If the `pip install` step gives you an error like "DEPRECATION: Uninstalling a distutils installed project (pexpect) has been deprecated", try 6 | 7 | ``` 8 | pip install stata_kernel --ignore-install pexpect 9 | ``` 10 | 11 | - If you have multiple installations of Python on your machine, make sure you run `python -m stata_kernel.install` during installation with the same Python executable as the one you usually use. This matters especially when using several Python virtual environments. You'll need to install `stata_kernel` within each environment you use. 12 | 13 | ## Graphs won't display 14 | 15 | - If you're using a user-written command to generate your graph, you'll need to add that command to the [list of graph keywords](configuration.md#user_graph_keywords). 16 | - If you're on Windows and using Edge as your browser, SVG images won't work. This is a known issue. 17 | 18 | Easy solutions: 19 | 20 | - Don't use Internet Explorer/Edge 21 | - Set the graph format to PNG instead of SVG. Run one of the following to [change the graph's storage format](configuration.md#graph_format): 22 | 23 | ``` 24 | %set graph_format png 25 | %set graph_format png --permanently 26 | ``` 27 | -------------------------------------------------------------------------------- /poetry.toml: -------------------------------------------------------------------------------- 1 | [virtualenvs] 2 | in-project = true 3 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "stata_kernel" 3 | version = "1.13.0" 4 | description = "A Jupyter kernel for Stata. Works with Windows, macOS, and Linux." 5 | authors = [ 6 | "Kyle Barron ", 7 | "Mauricio Caceres Bravo ", 8 | ] 9 | license = "GPLv3" 10 | classifiers = [ 11 | 'Development Status :: 3 - Alpha', 12 | 'Environment :: Console', 13 | 'Environment :: Web Environment', 14 | 'Intended Audience :: Developers', 15 | 'Intended Audience :: Education', 16 | 'Intended Audience :: Science/Research', 17 | 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', 18 | 'Operating System :: MacOS', 19 | 'Operating System :: Microsoft :: Windows', 20 | 'Operating System :: POSIX :: Linux', 21 | 'Operating System :: Unix', 22 | 'Programming Language :: Python :: 3.6', 23 | 'Programming Language :: Python :: 3.7', 24 | 'Programming Language :: Python :: 3.8', 25 | 'Programming Language :: Python :: 3.9', 26 | ] 27 | keywords = ["stata"] 28 | repository = "https://github.com/kylebarron/stata_kernel" 29 | documentation = "https://kylebarron.dev/stata_kernel/" 30 | 31 | [tool.poetry.dependencies] 32 | python = "^3.6.1" 33 | beautifulsoup4 = "^4.6.3" 34 | pandas = "^1.0" 35 | pygments = "^2.2" 36 | pexpect = "^4.6.0" 37 | requests = "^2.19.1" 38 | jupyter-client = "^5.2.3" 39 | IPython = "^7.16.3" 40 | ipykernel = "^4.8.2" 41 | packaging = "^17.1" 42 | pillow = ">=5.2.0" 43 | pywin32 = { version = ">=223", platform = "Windows" } 44 | 45 | [tool.poetry.dev-dependencies] 46 | bumpversion = "^0.6" 47 | mkdocs-material = ">=3.0.3" 48 | mkdocs = ">=1.0" 49 | yapf = ">=0.20.2" 50 | pytest = ">=3.7.1" 51 | # importlib-metadata==0.23 52 | # jupyter_kernel_test==0.3.0 53 | 54 | [build-system] 55 | requires = ["poetry-core>=1.1.10"] 56 | build-backend = "poetry.core.masonry.api" 57 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 1.13.0 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:pyproject.toml] 7 | search = version = "{current_version}" 8 | replace = version = "{new_version}" 9 | 10 | [bumpversion:file:stata_kernel/kernel.py] 11 | search = implementation_version = '{current_version}' 12 | replace = implementation_version = '{new_version}' 13 | 14 | [bumpversion:file:stata_kernel/__init__.py] 15 | search = __version__ = '{current_version}' 16 | replace = __version__ = '{new_version}' 17 | 18 | [pycodestyle] 19 | max-line-length = 80 20 | 21 | [flake8] 22 | max-line-length = 80 23 | 24 | [yapf] 25 | align_closing_bracket_with_visual_indent = False 26 | allow_multiline_dictionary_keys = False 27 | allow_multiline_lambdas = False 28 | allow_split_before_dict_value = True 29 | blank_line_before_class_docstring = False 30 | blank_line_before_nested_class_or_def = False 31 | coalesce_brackets = True 32 | column_limit = 80 33 | continuation_indent_width = 4 34 | dedent_closing_brackets = False 35 | each_dict_entry_on_separate_line = True 36 | i18n_comment = 37 | i18n_function_call = 38 | indent_dictionary_value = True 39 | indent_width = 4 40 | join_multiple_lines = True 41 | no_spaces_around_selected_binary_operators = set() 42 | spaces_around_default_or_named_assign = False 43 | spaces_around_power_operator = True 44 | spaces_before_comment = 2 45 | space_between_ending_comma_and_closing_bracket = True 46 | split_arguments_when_comma_terminated = False 47 | split_before_bitwise_operator = True 48 | split_before_closing_bracket = False 49 | split_before_dict_set_generator = True 50 | split_before_expression_after_opening_paren = False 51 | split_before_first_argument = True 52 | split_before_logical_operator = True 53 | split_before_named_assigns = False 54 | split_complex_comprehension = True 55 | split_penalty_after_opening_bracket = 30 56 | split_penalty_after_unary_operator = 10000 57 | split_penalty_before_if_expr = 30 58 | split_penalty_bitwise_operator = 300 59 | split_penalty_comprehension = 80 60 | split_penalty_excess_character = 4500 61 | split_penalty_for_added_line_split = 30 62 | split_penalty_import_names = 0 63 | split_penalty_logical_operator = 300 64 | use_tabs = False 65 | 66 | -------------------------------------------------------------------------------- /stata_kernel/__init__.py: -------------------------------------------------------------------------------- 1 | """An example Jupyter kernel""" 2 | 3 | __version__ = '1.13.0' 4 | -------------------------------------------------------------------------------- /stata_kernel/__main__.py: -------------------------------------------------------------------------------- 1 | from ipykernel.kernelapp import IPKernelApp 2 | 3 | import traceback 4 | try: 5 | from .kernel import StataKernel 6 | except: 7 | print('Cannot import kernel') 8 | traceback.print_exc() 9 | 10 | IPKernelApp.launch_instance(kernel_class=StataKernel) 11 | -------------------------------------------------------------------------------- /stata_kernel/ado/_StataKernelCompletions.ado: -------------------------------------------------------------------------------- 1 | capture program drop _StataKernelCompletions 2 | program _StataKernelCompletions 3 | set more off 4 | set trace off 5 | syntax [varlist] 6 | disp "%mata%" 7 | mata mata desc 8 | disp "%varlist%" 9 | disp `"`varlist'"' 10 | disp "%globals%" 11 | disp `"`:all globals'"' 12 | * NOTE: This only works for globals; locals are, well, local ): 13 | * disp "%locals%" 14 | * mata : invtokens(st_dir("local", "macro", "*")') 15 | disp "%logfiles%" 16 | qui log query _all 17 | if ( `"`r(numlogs)'"' != "" ) { 18 | forvalues l = 1 / `r(numlogs)' { 19 | * Skip stata automation log 20 | if ( `"`r(name`l')'"' != "stata_kernel_log" ) { 21 | disp r(filename`l') 22 | } 23 | } 24 | } 25 | disp "%scalars%" 26 | disp `"`:all scalars'"' 27 | disp "%programs%" 28 | program dir 29 | disp "%matrices%" 30 | disp `"`:all matrices'"' 31 | end 32 | -------------------------------------------------------------------------------- /stata_kernel/ado/_StataKernelHead.ado: -------------------------------------------------------------------------------- 1 | capture program drop _StataKernelHead 2 | program _StataKernelHead 3 | syntax [anything] [if] using, [n_default(int 10) *] 4 | set more off 5 | set trace off 6 | if ( !`=_N > 0' ) error 2000 7 | 8 | * First, parse the number of lines to print. Either the first 10 or 9 | * the number specified by the user. 10 | 11 | if ( regexm(`"`anything'"', "^[ ]*([0-9]+)") ) { 12 | local n1 = regexs(1) 13 | gettoken n2 anything: anything 14 | cap assert `n1' == `n2' 15 | if ( _rc ) error 198 16 | local n = `n1' 17 | } 18 | else local n = `n_default' 19 | 20 | * Number of rows must be positive 21 | if ( `n' <= 0 ) { 22 | disp as err "n must be > 0" 23 | exit 198 24 | } 25 | 26 | * Parse varlist and if condition 27 | local 0 `anything' `if' `using', `options' 28 | syntax [varlist] [if/] using, [*] 29 | 30 | * Apply if condition then get the first n matching condition 31 | qui if ( "`if'" != "" ) { 32 | local stype = cond(`=_N' < maxlong(), "long", "double") 33 | tempvar touse sumtouse index 34 | gen byte `touse' = `if' 35 | gen `stype' `sumtouse' = sum(`touse') 36 | gen `stype' `index' = _n 37 | local last = `=`sumtouse'[_N]' 38 | if ( `n' == 1 ) { 39 | local ifin if (`sumtouse' == `n') & `touse' == 1 40 | } 41 | else { 42 | local ifin if (`sumtouse' >= 1) & (`sumtouse' <= `n') & `touse' == 1 43 | } 44 | } 45 | else { 46 | local ifin in 1 / `=min(`n', _N)' 47 | } 48 | 49 | qui export delimited `index' `varlist' `using' `ifin', replace `options' 50 | list `varlist' `ifin' 51 | end 52 | -------------------------------------------------------------------------------- /stata_kernel/ado/_StataKernelLog.ado: -------------------------------------------------------------------------------- 1 | capture program drop _StataKernelLog 2 | program _StataKernelLog 3 | 4 | * Hold r() results 5 | if ( `"`0'"' == "off" ) { 6 | cap _return drop _StataKernelR 7 | _return hold _StataKernelR 8 | } 9 | 10 | * Loop through log files and close or reopen them all 11 | set more off 12 | set trace off 13 | qui log query _all 14 | local lognames 15 | if ( `"`r(numlogs)'"' != "" ) { 16 | qui forvalues l = 1 / `r(numlogs)' { 17 | local lognames `lognames' `r(name`l')' 18 | } 19 | } 20 | if ( `"`0'"' == "off" ) { 21 | qui foreach logname of local lognames { 22 | if ( `"`logname'"' == "stata_kernel_log" ) { 23 | * Skip stata automation log 24 | } 25 | else if ( `"`logname'"' == "" ) { 26 | log off 27 | } 28 | else { 29 | log off `logname' 30 | } 31 | } 32 | } 33 | else if ( `"`0'"' == "on" ) { 34 | qui foreach logname of local lognames { 35 | if ( `"`logname'"' == "stata_kernel_log" ) { 36 | * Skip stata automation log 37 | } 38 | else if ( `"`logname'"' == "" ) { 39 | log on 40 | } 41 | else { 42 | log on `logname' 43 | } 44 | } 45 | } 46 | else { 47 | disp as err "Can only switch logs -on- or -off-" 48 | exit 198 49 | } 50 | 51 | * Restore r() results 52 | if ( `"`0'"' == "on" ) { 53 | cap _return restore _StataKernelR 54 | cap _return drop _StataKernelR 55 | } 56 | end 57 | -------------------------------------------------------------------------------- /stata_kernel/ado/_StataKernelResetRC.ado: -------------------------------------------------------------------------------- 1 | capture program drop _StataKernelResetRC 2 | program _StataKernelResetRC 3 | syntax, num(int) 4 | cap exit `num' 5 | end 6 | -------------------------------------------------------------------------------- /stata_kernel/ado/_StataKernelTail.ado: -------------------------------------------------------------------------------- 1 | capture program drop _StataKernelTail 2 | program _StataKernelTail 3 | syntax [anything] [if] using, [*] 4 | set more off 5 | set trace off 6 | if ( !`=_N > 0' ) error 2000 7 | 8 | * First, parse the number of lines to print. Either the last 10 or 9 | * the number specified by the user. 10 | 11 | if ( regexm(`"`anything'"', "^[ ]*([0-9]+)") ) { 12 | local n1 = regexs(1) 13 | gettoken n2 anything: anything 14 | cap assert `n1' == `n2' 15 | if ( _rc ) error 198 16 | local n = `n1' 17 | } 18 | else local n = 10 19 | 20 | * Number of rows must be positive 21 | if ( `n' <= 0 ) { 22 | disp as err "n must be > 0" 23 | exit 198 24 | } 25 | 26 | * Parse varlist and if condition 27 | local 0 `anything' `if' `using', `options' 28 | syntax [varlist] [if/] using, [*] 29 | 30 | * Apply if condition then get the last n matching condition 31 | qui if ( "`if'" != "" ) { 32 | local stype = cond(`=_N' < maxlong(), "long", "double") 33 | tempvar touse sumtouse index 34 | gen byte `touse' = `if' 35 | gen `stype' `sumtouse' = sum(`touse') 36 | gen `stype' `index' = _n 37 | local last = `=`sumtouse'[_N]' 38 | if ( `n' == 1 ) { 39 | local ifin if (`sumtouse' == `last') & `touse' == 1 40 | } 41 | else { 42 | local f = `last' - `n' 43 | local ifin if (`sumtouse' >= `f') & (`sumtouse' <= `last') & `touse' == 1 44 | } 45 | } 46 | else { 47 | local ifin in -`n'/l 48 | } 49 | 50 | qui export delimited `index' `varlist' `using' `ifin', replace `options' 51 | list `varlist' `ifin' 52 | if ( "`if'" == "" ) disp _N 53 | end 54 | -------------------------------------------------------------------------------- /stata_kernel/code_manager.py: -------------------------------------------------------------------------------- 1 | import re 2 | import platform 3 | import hashlib 4 | 5 | from pygments import lex 6 | from textwrap import dedent 7 | 8 | from .stata_lexer import StataLexer 9 | from .stata_lexer import CommentAndDelimitLexer 10 | from .config import config 11 | 12 | base_graph_keywords = [ 13 | r'gr(a|ap|aph)?' + r'(?!\s+' + r'(save|replay|print|export|dir|set|' + 14 | r'des(c|cr|cri|crib|cribe)?|rename|copy|drop|close|q(u|ue|uer|uery)?))', 15 | r'tw(o|ow|owa|oway)?', r'sc(atter)?', r'line', 16 | r'hist(o|og|ogr|ogra|ogram)?', r'kdensity', r'lowess', r'lpoly', 17 | r'tsr?line', r'symplot', r'quantile', r'qnorm', r'pnorm', r'qchi', r'pchi', 18 | r'qqplot', r'gladder', r'qladder', r'rvfplot', r'avplot', r'avplots', 19 | r'cprplot', r'acprplot', r'rvpplot', r'lvr2plot', r'ac', r'pac', r'pergram', 20 | r'cumsp', r'xcorr', r'wntestb', r'estat\s+acplot', r'estat\s+aroots', 21 | r'estat\s+sbcusum', r'fcast\s+graph', r'varstable', r'vecstable', 22 | r'irf\s+graph', r'irf\s+ograph', r'irf\s+cgraph', r'xtline' 23 | r'sts\s+graph', r'strate', r'ltable', r'stci', r'stphplot', r'stcoxkm', 24 | r'estat phtest', r'stcurve', r'roctab', r'rocplot', r'roccomp', 25 | r'rocregplot', r'lroc', r'lsens', r'biplot', r'irtgraph\s+icc', 26 | r'irtgraph\s+tcc', r'irtgraph\s+iif', r'irtgraph\s+tif', r'biplot', 27 | r'cluster dendrogram', r'screeplot', r'scoreplot', r'loadingplot', 28 | r'procoverlay', r'cabiplot', r'caprojection', r'mcaplot', r'mcaprojection', 29 | r'mdsconfig', r'mdsshepard', r'cusum', r'cchart', r'pchart', r'rchart', 30 | r'xchart', r'shewhart', r'serrbar', r'marginsplot', r'bayesgraph', 31 | r'tabodds', r'teffects\s+overlap', r'npgraph', r'grmap', r'pkexamine'] 32 | 33 | 34 | class CodeManager(): 35 | """Class to deal with text before sending to Stata 36 | """ 37 | 38 | def __init__(self, code, semicolon_delimit=False, mata_mode=False): 39 | code = re.sub(r'\r\n', r'\n', code) 40 | # Hard tabs in input are not shown in output and mess up removing lines 41 | code = re.sub(r'\t', ' ', code) 42 | self.input = code 43 | if semicolon_delimit: 44 | if mata_mode: 45 | code = 'mata;\n' + code 46 | 47 | code = '#delimit ;\n' + code 48 | elif mata_mode: 49 | code = 'mata\n' + code 50 | 51 | # First use the Comment and Delimiting lexer 52 | self.tokens_fp_all = self.tokenize_first_pass(code) 53 | self.tokens_fp_no_comments = self.remove_comments(self.tokens_fp_all) 54 | 55 | if not self.tokens_fp_no_comments: 56 | self.tokens_fp_no_comments = [('Token.Text', '')] 57 | 58 | self.ends_sc = str(self.tokens_fp_no_comments[-1][0]) in [ 59 | 'Token.TextInSemicolonBlock', 'Token.SemicolonDelimiter'] 60 | 61 | tokens_nl_delim = self.convert_delimiter(self.tokens_fp_no_comments) 62 | text = ''.join([x[1] for x in tokens_nl_delim]) 63 | self.tokens_final = self.tokenize_second_pass(text) 64 | 65 | # NOTE: Consider wrapping mata call for include in mata and 66 | # end. Do not include end in the include file if the result of 67 | # this loop is. False Instead, send end before the include file 68 | # is done. 69 | 70 | self.mata_mode = False 71 | self.mata_open = False 72 | self.mata_error = False 73 | if mata_mode: 74 | self.mata_open = True 75 | self.mata_mode = True 76 | self.tokens_final = self.tokens_final[1:] 77 | 78 | self.mata_closed = False 79 | for token, chunk in self.tokens_final: 80 | if str(token) == 'Token.Mata.Close': 81 | self.mata_closed = True 82 | self.mata_mode = False 83 | elif str(token).startswith('Token.Mata.Open'): 84 | self.mata_closed = False 85 | self.mata_mode = True 86 | if str(token) == 'Token.Mata.OpenError': 87 | self.mata_error = True 88 | 89 | self.is_complete = self._is_complete() 90 | 91 | def tokenize_first_pass(self, code): 92 | """Tokenize input code for Comments and Delimit blocks 93 | 94 | Args: 95 | code (str): 96 | Input string. Should use `\\n` for end of lines. 97 | 98 | Return: 99 | (List[Tuple[Token, str]]): 100 | List of token tuples. The only token types currently used in the 101 | lexer are: 102 | - Text (plain text) 103 | - Comment.Single (// and *) 104 | - Comment.Special (///) 105 | - Comment.Multiline (/* */) 106 | - Keyword.Namespace (code inside #delimit ; block) 107 | - Keyword.Reserved (; delimiter) 108 | """ 109 | comment_lexer = CommentAndDelimitLexer(stripall=False, stripnl=False) 110 | return [x for x in lex(code, comment_lexer)] 111 | 112 | def remove_comments(self, tokens): 113 | """Remove comments from tokens 114 | 115 | Return: 116 | (List[Tuple[Token, str]]): 117 | list of non-comment tokens 118 | """ 119 | return [x for x in tokens if not str(x[0]).startswith('Token.Comment')] 120 | 121 | def convert_delimiter(self, tokens): 122 | """If parts of tokens are `;`-delimited, convert to `\\n`-delimited 123 | 124 | - If there are no ;-delimiters, return 125 | - Else, replace newlines with spaces, see https://github.com/kylebarron/stata_kernel/pull/70#issuecomment-412399978 126 | - Then change the ; delimiters to newlines 127 | """ 128 | 129 | # If all tokens are newline-delimited, return 130 | if 'Token.TextInSemicolonBlock' not in [str(x[0]) for x in tokens]: 131 | return tokens 132 | 133 | # Replace newlines in `;`-delimited blocks with spaces 134 | tokens = [ 135 | ('Space instead of newline', ' ') if 136 | (str(x[0]) == 'Token.TextInSemicolonBlock') and x[1] == '\n' else x 137 | for x in tokens[:-1]] 138 | 139 | # Change the ; delimiters to \n 140 | tokens = [ 141 | ('Newline delimiter', '\n') if 142 | (str(x[0]) == 'Token.SemicolonDelimiter') and x[1] == ';' else x 143 | for x in tokens] 144 | return tokens 145 | 146 | def tokenize_second_pass(self, code): 147 | """Tokenize clean code for syntactic blocks 148 | 149 | Args: 150 | code (str): 151 | Input string. Should have `\\n` as the delimiter. Should have no 152 | comments. Should use `\\n` for end of lines. 153 | 154 | Return: 155 | (List[Tuple[Token, str]]): 156 | List of token tuples. Some of the token types: 157 | lexer are: 158 | - Text (plain text) 159 | - Comment.Single (// and *) 160 | - Comment.Special (///) 161 | - Comment.Multiline (/* */) 162 | - Keyword.Namespace (code inside #delimit ; block) 163 | - Keyword.Reserved (; delimiter) 164 | """ 165 | block_lexer = StataLexer(stripall=False, stripnl=False) 166 | return [x for x in lex(code, block_lexer)] 167 | 168 | def _is_complete(self): 169 | """Determine whether the code provided is complete 170 | 171 | Ways in which code entered is not complete: 172 | - If in the middle of a block construct, i.e. foreach, program, input 173 | - If the last token provided is inside a line-continuation comment, i.e. 174 | `di 2 + ///` or `di 2 + /*`. 175 | - If in a #delimit ; block and there are non-whitespace characters after 176 | the last semicolon. 177 | 178 | Special case for code to be complete: 179 | - %magics 180 | """ 181 | 182 | magic_regex = re.compile( 183 | r'\A%(?P.+?)(?P\s+.*)?\Z', 184 | flags=re.DOTALL + re.MULTILINE) 185 | if magic_regex.search(self.input): 186 | return True 187 | 188 | # block constructs 189 | if str(self.tokens_final[-1][0]).startswith('Token.TextBlock'): 190 | return False 191 | 192 | # last token a line-continuation comment 193 | if str(self.tokens_fp_all[-1][0]) in ['Token.Comment.Multiline', 194 | 'Token.Comment.Special']: 195 | return False 196 | 197 | if self.ends_sc: 198 | # Find indices of `;` 199 | inds = [ 200 | ind for ind, x in enumerate(self.tokens_fp_no_comments) 201 | if (str(x[0]) == 'Token.SemicolonDelimiter') and x[1] == ';'] 202 | 203 | if not inds: 204 | inds = [0] 205 | 206 | # Check if there's non whitespace text after the last semicolon 207 | # If so, then it's not complete 208 | tr_text = ''.join([ 209 | x[1] 210 | for x in self.tokens_fp_no_comments[max(inds) + 1:]]).strip() 211 | if tr_text: 212 | return False 213 | 214 | return True 215 | 216 | def get_text(self, stata=None): 217 | """Get valid, executable text 218 | 219 | For any text longer than one line, I save the text to a do file and send 220 | `include path_to_do_file` to Stata. I insert `graph export` after 221 | _every_ graph keyword. This way, even if the graph is created within a 222 | loop or program, I can still see that it was created and I can grab it. 223 | 224 | I create an md5 of the lines of text that I run, and then add that as 225 | `md5' so that I can definitively know when Stata has finished with the 226 | code I sent it. 227 | 228 | Args: 229 | stata: instance of Stata session 230 | 231 | Returns: 232 | (str, str, str): 233 | (Text to run in kernel, md5 to expect for, code lines to remove from output) 234 | """ 235 | 236 | tokens = self.tokens_final 237 | 238 | text = ''.join([x[1] for x in tokens]).strip() 239 | lines = text.split('\n') 240 | 241 | # Remove empty lines. This is important because there are often extra 242 | # newlines from removed comments. And they can confuse the code that 243 | # removes code lines from the log output. 244 | lines = [x for x in lines if x.strip() != ''] 245 | 246 | has_block = bool([x for x in tokens if str(x[0]) == 'Token.TextBlock']) 247 | 248 | use_include = has_block 249 | cap_re = re.compile(r'\bcap(t|tu|tur|ture)?\b').search 250 | qui_re = re.compile(r'\bqui(e|et|etl|etly)?\b').search 251 | noi_re = re.compile(r'\bn(o|oi|ois|oisi|oisil|oisily)?\b').search 252 | if cap_re(text) or qui_re(text) or noi_re(text): 253 | use_include = True 254 | 255 | if len(lines) > 1: 256 | use_include = True 257 | 258 | if stata: 259 | use_include = use_include and not stata.mata_open 260 | use_include = use_include and not stata.mata_mode 261 | 262 | # Insert `graph export` 263 | graph_fmt = config.get('graph_format', 'svg') 264 | graph_scale = float(config.get('graph_scale', '1')) 265 | graph_width = int(config.get('graph_width', '600')) 266 | graph_height = config.get('graph_height') 267 | cache_dir = config.get('cache_dir') 268 | if graph_fmt == 'svg': 269 | pdf_dup = config.get('graph_svg_redundancy', 'True') 270 | pdf_dup = pdf_dup.lower() == 'true' 271 | elif graph_fmt == 'png': 272 | pdf_dup = config.get('graph_png_redundancy', 'False') 273 | pdf_dup = pdf_dup.lower() == 'true' 274 | elif graph_fmt == 'eps': 275 | pdf_dup = config.get('graph_eps_redundancy', 'False') 276 | pdf_dup = pdf_dup.lower() == 'true' 277 | else: 278 | pdf_dup = False 279 | 280 | dim_str = " width({})".format(int(graph_width * graph_scale)) 281 | if graph_height: 282 | graph_height = int(graph_height) 283 | dim_str += " height({})".format(int(graph_height * graph_scale)) 284 | if graph_fmt == 'pdf': 285 | dim_str = '' 286 | if graph_fmt == 'eps': 287 | dim_str = '' 288 | 289 | cache_dir_str = str(cache_dir) 290 | if platform.system() == 'Windows': 291 | cache_dir_str = re.sub(r'\\', '/', cache_dir_str) 292 | gph_cnt = 'stata_kernel_graph_counter' 293 | 294 | # yapf: disable 295 | if not pdf_dup: 296 | g_exp = dedent(""" 297 | if _rc == 0 {{ 298 | noi gr export `"{0}/graph${1}.{2}"',{3} replace 299 | global {1} = ${1} + 1 300 | }}\ 301 | """.format(cache_dir_str, gph_cnt, graph_fmt, dim_str)) 302 | else: 303 | g_exp = dedent(""" 304 | if _rc == 0 {{ 305 | noi gr export `"{0}/graph${1}.{2}"',{3} replace 306 | noi gr export `"{0}/graph${1}.pdf"', replace 307 | global {1} = ${1} + 1 308 | }}\ 309 | """.format(cache_dir_str, gph_cnt, graph_fmt, dim_str)) 310 | 311 | if stata: 312 | g_exp = stata._mata_escape(g_exp) 313 | # yapf: enable 314 | 315 | user_graph_keywords = config.get( 316 | 'user_graph_keywords', 'coefplot,vioplot') 317 | user_graph_keywords = [ 318 | re.sub(r'\s+', '\\\\s+', x.strip()) 319 | for x in user_graph_keywords.split(',')] 320 | graph_keywords = r'^\s*\b({})\b'.format( 321 | '|'.join([*base_graph_keywords, *user_graph_keywords])) 322 | lines = [ 323 | 'cap noi ' + x + g_exp if re.match(graph_keywords, x) else x 324 | for x in lines] 325 | 326 | text = '\n'.join(lines) 327 | hash_text = hashlib.md5(text.encode('utf-8')).hexdigest() 328 | text_to_exclude = text 329 | if use_include: 330 | with (cache_dir / 'include.do').open('w', encoding='utf-8') as f: 331 | f.write(text + '\n') 332 | text = 'include "{}/include.do"'.format(cache_dir_str) 333 | text_to_exclude = text + '\n' + text_to_exclude 334 | 335 | text += "\n`{}'".format(hash_text) 336 | return text, hash_text, text_to_exclude 337 | -------------------------------------------------------------------------------- /stata_kernel/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import platform 4 | 5 | from pathlib import Path 6 | from textwrap import dedent 7 | from tempfile import TemporaryDirectory 8 | from configparser import ConfigParser, NoSectionError 9 | 10 | from .utils import find_path 11 | 12 | GLOBAL_PATH = '/etc/stata_kernel.conf' 13 | USER_PATH = '~/.stata_kernel.conf' 14 | 15 | GLOBAL_PATH_ENVVAR_NAME = 'STATA_KERNEL_GLOBAL_CONFIG_PATH' 16 | USER_PATH_ENVVAR_NAME = 'STATA_KERNEL_USER_CONFIG_PATH' 17 | 18 | 19 | class Config(): 20 | all_settings = [ 21 | 'autocomplete_closing_symbol', 22 | 'cache_directory', 23 | 'execution_mode', 24 | 'graph_format', 25 | 'graph_height', 26 | 'graph_png_redundancy', 27 | 'graph_redundancy_warning', 28 | 'graph_scale', 29 | 'graph_svg_redundancy', 30 | 'graph_width', 31 | 'stata_path', 32 | 'user_graph_keywords', ] # yapf: ignore 33 | 34 | def __init__(self): 35 | """ 36 | Load config both from a potential system-wide config file or from a 37 | user-defined one if present. 38 | 39 | We first load from the system-wide location if the file is present and 40 | then we load the user-defined location, updating entries. Thus the 41 | user-location takes precedence, but either file can be missing. 42 | 43 | The user-defined path defaults to `~/.stata_kernel.conf` or can be read 44 | from the environmental variable `STATA_KERNEL_USER_CONFIG_PATH`. 45 | 46 | The system-wide config file was added to facilitate deployments on 47 | systems like Jupyter Hub; it defaults to `/etc/stata_kernel.conf` or 48 | read from the environmental variable `STATA_KERNEL_GLOBAL_CONFIG_PATH`, 49 | and we do not otherwise keep a reference to it because it is likely 50 | non-writable. Setting any config option should go to the user-config. 51 | 52 | Example config file:: 53 | 54 | [stata_kernel] 55 | stata_path = "C:/Program Files/Stata16/StataMP-64.exe" 56 | execution_mode = automation 57 | cache_directory = ~/.stata_kernel_cache 58 | autocomplete_closing_symbol = False 59 | graph_format = svg 60 | graph_scale = 1 61 | user_graph_keywords = coefplot,vioplot 62 | """ 63 | _global_path = os.environ.get(GLOBAL_PATH_ENVVAR_NAME, GLOBAL_PATH) 64 | _user_path = os.environ.get(USER_PATH_ENVVAR_NAME, USER_PATH) 65 | 66 | global_config = ConfigParser() 67 | global_config.read(_global_path) 68 | 69 | self.config_path = Path(_user_path).expanduser() 70 | self.config = ConfigParser() 71 | self.config.read(str(self.config_path)) 72 | 73 | self.env = {} 74 | 75 | for c in (global_config, self.config): 76 | try: 77 | self.env.update(dict(c.items('stata_kernel'))) 78 | except NoSectionError: 79 | pass 80 | 81 | cache_par_dir = Path(self.get('cache_directory', 82 | '~/.stata_kernel_cache')).expanduser() 83 | cache_par_dir.mkdir(parents=True, exist_ok=True) 84 | self._cache_temp_dir = TemporaryDirectory(dir=str(cache_par_dir)) 85 | cache_dir = Path(self._cache_temp_dir.name) 86 | 87 | stata_path = self.get('stata_path', find_path()) 88 | if not stata_path: 89 | self.raise_config_error('stata_path') 90 | 91 | if platform.system() == 'Darwin': 92 | stata_path = self.get_mac_stata_path_variant(stata_path) 93 | execution_mode = self.get('execution_mode', 'console') 94 | if execution_mode not in ['console', 'automation']: 95 | self.raise_config_error('execution_mode') 96 | elif platform.system() == 'Windows': 97 | execution_mode = 'automation' 98 | else: 99 | execution_mode = 'console' 100 | stata_path = self.get_linux_stata_path_variant(stata_path) 101 | 102 | self.set('cache_dir', cache_dir) 103 | self.set('stata_path', stata_path) 104 | self.set('execution_mode', execution_mode) 105 | if not self.get('stata_path'): 106 | self.raise_config_error('stata_path') 107 | 108 | def get(self, key, backup=None): 109 | return self.env.get(key, backup) 110 | 111 | def set(self, key, val, permanent=False): 112 | if key.startswith('cache_dir'): 113 | val = Path(val).expanduser() 114 | val.mkdir(parents=True, exist_ok=True) 115 | 116 | self.env[key] = val 117 | 118 | if permanent: 119 | if key.startswith('cache_dir'): 120 | key = 'cache_directory' 121 | val = str(val) 122 | 123 | if key.startswith('graph_'): 124 | val = str(val) 125 | 126 | try: 127 | self.config['stata_kernel'] 128 | except KeyError: 129 | self.config['stata_kernel'] = {} 130 | 131 | self.config.set('stata_kernel', key, val) 132 | with self.config_path.open('w') as f: 133 | self.config.write(f) 134 | 135 | def get_mac_stata_path_variant(self, stata_path): 136 | path = Path(stata_path) 137 | if self.get('execution_mode') == 'automation': 138 | d = {'stata': 'Stata', 'stata-se': 'StataSE', 'stata-mp': 'StataMP'} 139 | else: 140 | d = {'Stata': 'stata', 'StataSE': 'stata-se', 'StataMP': 'stata-mp'} 141 | 142 | bin_name = d.get(path.name, path.name) 143 | return str(path.parent / bin_name) 144 | 145 | def get_linux_stata_path_variant(self, stata_path): 146 | d = { 147 | 'xstata': 'stata', 148 | 'xstata-se': 'stata-se', 149 | 'xstata-mp': 'stata-mp'} 150 | for xname, name in d.items(): 151 | if stata_path.endswith(xname): 152 | stata_path = re.sub(r'{}$'.format(xname), name, stata_path) 153 | break 154 | 155 | return stata_path 156 | 157 | def raise_config_error(self, option): 158 | msg = """\ 159 | {} option in configuration file is missing or invalid 160 | Refer to the documentation to see how to set it manually: 161 | 162 | https://kylebarron.dev/stata_kernel/using_stata_kernel/configuration/ 163 | """.format(option) 164 | raise ValueError(dedent(msg)) 165 | 166 | def _remove_unsafe(self, key, permanent=False): 167 | self.env.pop(key, None) 168 | if permanent: 169 | self.config.remove_option(option=key, section='stata_kernel') 170 | with self.config_path.open('w') as f: 171 | self.config.write(f) 172 | 173 | 174 | config = Config() 175 | -------------------------------------------------------------------------------- /stata_kernel/css/_StataKernelHelpDefault.css: -------------------------------------------------------------------------------- 1 | h2 { 2 | font-family: Arial,Helvetica,Helv,sans-serif; 3 | color: #000000; 4 | } 5 | 6 | pre { 7 | margin: 10px; 8 | } 9 | 10 | table { 11 | background-color: transparent; 12 | border-color: transparent; 13 | bgcolor: transparent; 14 | } 15 | 16 | tr { 17 | background-color: transparent; 18 | border-color: transparent; 19 | bgcolor: transparent; 20 | } 21 | 22 | body { 23 | background-color: transparent; 24 | border-color: transparent; 25 | margin: 0px; 26 | } 27 | 28 | div { 29 | background-color: transparent; 30 | border-color: transparent; 31 | } 32 | -------------------------------------------------------------------------------- /stata_kernel/docs/css/pandoc.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: 'Monospace'; 3 | } 4 | 5 | blockquote { 6 | /* color: #666666; */ 7 | margin: 0; 8 | padding-left: 3em; 9 | border-left: 0.5em solid /*#EEE*/; 10 | } 11 | 12 | hr { 13 | display: block; 14 | height: 2px; 15 | border: 0; 16 | border-top: 1px solid /* #aaa */; 17 | border-bottom: 1px solid /* #eee */; 18 | margin: 1em 0; 19 | padding: 0; 20 | } 21 | 22 | p { 23 | margin: 1em 0; 24 | } 25 | 26 | img { 27 | max-width: 75%; 28 | } 29 | 30 | h1, h2, h3, h4, h5, h6 { 31 | /* color: #111; */ 32 | line-height: 125%; 33 | margin-top: 0em; 34 | font-weight: normal; 35 | } 36 | 37 | h4, h5, h6 { 38 | font-weight: bold; 39 | } 40 | 41 | h1 { 42 | font-size: 2.5em; 43 | } 44 | 45 | h2 { 46 | font-size: 2em; 47 | } 48 | 49 | h3 { 50 | font-size: 1.5em; 51 | } 52 | 53 | h4 { 54 | font-size: 1.2em; 55 | } 56 | 57 | h5 { 58 | font-size: 1em; 59 | } 60 | 61 | h6 { 62 | font-size: 0.9em; 63 | } 64 | 65 | pre, code, kbd, samp { 66 | /* color: #000; */ 67 | font-family: monospace, monospace; 68 | _font-family: 'courier new', monospace; 69 | font-size: 0.91em; 70 | } 71 | 72 | pre { 73 | white-space: pre; 74 | white-space: pre-wrap; 75 | word-wrap: break-word; 76 | } 77 | 78 | b, strong { 79 | font-weight: bold; 80 | } 81 | 82 | dfn { 83 | font-style: italic; 84 | } 85 | 86 | ins { 87 | /* background: #ff9; */ 88 | color: #000; 89 | text-decoration: none; 90 | } 91 | 92 | mark { 93 | /* background: #ff0; */ 94 | /* color: #000; */ 95 | font-style: italic; 96 | font-weight: bold; 97 | } 98 | 99 | sub, sup { 100 | font-size: 75%; 101 | line-height: 0; 102 | position: relative; 103 | vertical-align: baseline; 104 | } 105 | 106 | sup { 107 | top: -0.5em; 108 | } 109 | 110 | sub { 111 | bottom: -0.25em; 112 | } 113 | 114 | ul, ol { 115 | margin: 1em 0; 116 | padding: 0 0 0 2em; 117 | } 118 | 119 | li p:last-child { 120 | margin-bottom: 0; 121 | } 122 | 123 | ul ul, ol ol { 124 | margin: .3em 0; 125 | } 126 | 127 | dl { 128 | margin-bottom: 1em; 129 | } 130 | 131 | dt { 132 | font-weight: bold; 133 | margin-bottom: .8em; 134 | } 135 | 136 | dd { 137 | margin: 0 0 .8em 2em; 138 | } 139 | 140 | dd:last-child { 141 | margin-bottom: 0; 142 | } 143 | 144 | img { 145 | border: 0; 146 | -ms-interpolation-mode: bicubic; 147 | vertical-align: middle; 148 | } 149 | 150 | figure { 151 | display: block; 152 | text-align: center; 153 | margin: 1em 0; 154 | } 155 | 156 | figure img { 157 | border: none; 158 | margin: 0 auto; 159 | } 160 | 161 | figcaption { 162 | font-size: 0.8em; 163 | font-style: italic; 164 | margin: 0 0 .8em; 165 | } 166 | -------------------------------------------------------------------------------- /stata_kernel/docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | index 9 | 18 | 21 | 188 | 189 | 190 |

stata_kernel

191 |

stata_kernel is a Jupyter kernel for Stata. It works on Windows, macOS, and Linux.

192 | Online documentation is available at 193 | 194 | kylebarron.dev/stata_kernel 195 |
196 |

197 |

What is Jupyter?

198 |

Jupyter is an open-source ecosystem for interactive data science. Originally developed around the Python programming language, Jupyter has grown to interface with dozens of programming languages.

199 |

stata_kernel is the bridge that interactively connects Stata to all the elements in the ecosystem.

200 |
    201 |

  • JupyterLab is a web-based interactive editor that allows for interweaving of code, text, and results.

    202 |
      203 |
    • Splice models in LaTeX math mode with the code that implements them and the graphs depicting their output.
      • 204 |
    • Click here to see an example Jupyter Notebook file using stata_kernel.
      • 205 |
    • Jupyter Notebooks can be exported as PDFs or HTML, and are as good for teaching new students as they are for displaying research results.
      • 206 |
    207 |
    208 | Jupyter Notebook 209 |
    • 210 |

  • Hydrogen is a package for the Atom text editor that connects with Jupyter kernels to display results interactively in your text editor.

    • 211 |

  • The Jupyter console is an enhanced interactive console. Its features include enhanced autocompletion, better searching of history, syntax highlighting, among others. The similar QtConsole even allows displaying plots within the terminal.

    • 212 |

  • Enhanced remote work. You can set up Jupyter to run computations remotely but to show results locally. Since the only data passing over the network are the text inputs and outputs from Stata, communcation happens much faster than loading xstata, especially on slower networks. Being able to use Jupyter Notebook or Hydrogen vastly enhances productivity compared to working with the Stata console through a remote terminal.

    • 213 |
214 |

stata_kernel Features

215 | 239 |

Screenshots

240 |

Atom

241 |
242 | Atom 243 |
244 | 245 | 246 | -------------------------------------------------------------------------------- /stata_kernel/docs/index.txt: -------------------------------------------------------------------------------- 1 | stata_kernel 2 | 3 | stata_kernel is a Jupyter kernel for Stata. It works on Windows, macOS, 4 | and Linux. 5 | 6 | What is Jupyter? 7 | 8 | Jupyter is an open-source ecosystem for interactive data science. 9 | Originally developed around the Python programming language, Jupyter has 10 | grown to interface with dozens of programming languages. 11 | 12 | stata_kernel is the bridge that interactively connects Stata to all the 13 | elements in the ecosystem. 14 | 15 | - JupyterLab is a web-based interactive editor that allows for 16 | interweaving of code, text, and results. 17 | 18 | - Splice models in LaTeX math mode with the code that implements 19 | them and the graphs depicting their output. 20 | - Click here to see an example Jupyter Notebook file using 21 | stata_kernel. 22 | - Jupyter Notebooks can be exported as PDFs or HTML, and are as 23 | good for teaching new students as they are for displaying 24 | research results. 25 | 26 | [Jupyter Notebook] 27 | 28 | - Hydrogen is a package for the Atom text editor that connects with 29 | Jupyter kernels to display results interactively in your text 30 | editor. 31 | 32 | - The Jupyter console is an enhanced interactive console. Its features 33 | include enhanced autocompletion, better searching of history, syntax 34 | highlighting, among others. The similar QtConsole even allows 35 | displaying plots within the terminal. 36 | 37 | - Enhanced remote work. You can set up Jupyter to run computations 38 | remotely but to show results locally. Since the only data passing 39 | over the network are the text inputs and outputs from Stata, 40 | communcation happens much faster than loading xstata, especially on 41 | slower networks. Being able to use Jupyter Notebook or Hydrogen 42 | vastly enhances productivity compared to working with the Stata 43 | console through a remote terminal. 44 | 45 | stata_kernel Features 46 | 47 | - ☒ Supports Windows, macOS, and Linux. 48 | - ☒ Use any type of comments in your code, not just *. 49 | - ☒ Autocompletions as you type based on the variables, macros, 50 | scalars, and matrices currently in memory. As of version 1.6.0 it 51 | also suggests file paths for autocompletion. 52 | - ☒ Display graphs. 53 | - ☒ Receive results as they appear, not after the entire command 54 | finishes. 55 | - ☒ Pull up interactive help files within the kernel. 56 | - ☒ Browse data interactively. 57 | - ☒ #delimit ; interactive support 58 | - ☒ Work with a remote session of Stata. 59 | - ☒ Mata interactive support 60 | - ☐ Cross-session history file 61 | 62 | Screenshots 63 | 64 | Atom 65 | 66 | [Atom] 67 | -------------------------------------------------------------------------------- /stata_kernel/docs/logo-64x64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/stata_kernel/docs/logo-64x64.png -------------------------------------------------------------------------------- /stata_kernel/docs/make.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | css=stata_kernel/docs/css/pandoc.css 4 | kernel=stata_kernel/docs/index 5 | magics=stata_kernel/docs/using_stata_kernel/magics 6 | 7 | # index.md 8 | cd ../../docs/src 9 | echo '' >> _tmp 12 | 13 | pandoc -H ./_tmp -t html5 -o ../../${kernel}.html index.md 14 | pandoc -t plain -o ../../${kernel}.txt index.md 15 | rm _tmp 16 | 17 | # using_stata_kernel/magics.md 18 | cd using_stata_kernel 19 | echo '' >> _tmp 22 | 23 | pandoc -H _tmp -t html5 -o ../../../${magics}.html magics.md 24 | pandoc -t plain -o ../../../${magics}.txt magics.md 25 | rm _tmp 26 | 27 | # clean pandoc html 28 | cd ../../../stata_kernel/docs 29 | python make_href 30 | -------------------------------------------------------------------------------- /stata_kernel/docs/make_href: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from bs4 import BeautifulSoup as bs 5 | import urllib 6 | import re 7 | import os 8 | 9 | help_msg_html = """\ 10 |

11 | Online documentation is available at 12 | 13 | kylebarron.dev/stata_kernel 14 |
15 |

16 | """ 17 | 18 | # (file, include help message) 19 | files = [('index.html', True), ('using_stata_kernel/magics.html', False)] 20 | html_root = "https://kylebarron.dev/stata_kernel/" 21 | 22 | 23 | def main(): 24 | for (fpath, fmsg) in files: 25 | cleanHTML(fpath, fmsg) 26 | 27 | 28 | def cleanHTML(html_path, help_msg): 29 | html_relpath = os.path.dirname(html_path) 30 | html_base = urllib.parse.urljoin( 31 | html_root, 32 | '{0}/'.format(html_relpath)) if html_relpath != '' else html_root 33 | 34 | with open(html_path, 'r') as f: 35 | soup = bs(f.read(), 'html.parser') 36 | 37 | if help_msg: 38 | p = soup.find("body").find("p") 39 | p.insert(3, bs(help_msg_html, 'lxml').html.body.p) 40 | 41 | for a in soup.find_all('a', href=True): 42 | href = a.get('href') 43 | if not href.startswith('http'): 44 | href = re.subn(r'\.md(?=\b)', '', href, 1)[0] 45 | print(urllib.parse.urljoin(html_base, href)) 46 | a['href'] = urllib.parse.urljoin(html_base, href) 47 | 48 | for img in soup.find_all('img', src=True): 49 | src = img.get('src') 50 | if not src.startswith('http'): 51 | src = re.subn(r'\.md(?=\b)', '', src, 1)[0] 52 | print(urllib.parse.urljoin(html_base, src)) 53 | img['src'] = urllib.parse.urljoin(html_base, src) 54 | 55 | with open(html_path, 'w') as f: 56 | f.write(str(soup)) 57 | 58 | 59 | if __name__ == "__main__": 60 | main() 61 | -------------------------------------------------------------------------------- /stata_kernel/docs/using_stata_kernel/magics.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | magics 9 | 80 | 83 | 250 | 251 | 252 |

Magics

253 |

Magics are programs provided by stata_kernel that enhance the experience of working with Stata in Jupyter.

254 |

All magics are special commands that start with %. They must be the first word of the cell or selection, otherwise they won’t be intercepted and will be sent to Stata.

255 |

For most of the magics listed, you can add --help to see a help menu in the kernel. For example,

256 | 
In [1]: %locals --help
257 | usage: %locals [-h] [-v] [REGEX [REGEX ...]]
258 | 
259 | positional arguments:
260 |   REGEX          regex to match
261 | 
262 | optional arguments:
263 |   -h, --help     show this help message and exit
264 |   -v, --verbose  Verbose output (print full contents of matched locals).
265 |

The magics that respond with richly formatted text, namely %browse and %help, will not work with Jupyter Console or Jupyter QtConsole, since they don’t support displaying HTML.

266 |

%browse, %head, %tail

267 |

Interactively view your dataset

268 |

This can optionally be provided with a varlist, N, or if:

269 | 
%browse [-h] [N] [varlist] [if]
270 | %head [-h] [N] [varlist] [if]
271 | %tail [-h] [N] [varlist] [if]
272 |

By default:

273 |
    274 |
  • %browse displays the first 200 rows
    • 275 |
  • %head displays the first 10 rows
    • 276 |
  • %tail displays the last 10 rows
    • 277 |
278 |

If you’re using Windows or macOS with Automation mode, you can also run browse (without the %) and it will open the usual Stata data explorer.

279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 |
AtomAtom
Jupyter NotebookJupyter Notebook
291 |

%delimit

292 |

Print the current delimiter

293 |

This takes no arguments; it prints the delimiter currently set: either cr or ;. If you want to change the delimiter, use #delimit ; or #delimit cr. The delimiter will remain set until changed.

294 | 
In [1]: %delimit
295 | The delimiter is currently: cr
296 | 
297 | In [2]: #delimit ;
298 | delimiter now ;
299 | In [3]: %delimit
300 | The delimiter is currently: ;
301 | 
302 | In [4]: #delimit cr
303 | delimiter now cr
304 |

%help

305 |

Display a help file in rich text

306 | 
%help [-h] command_or_topic_name
307 |

Add the term you want to search for after %help:

308 | 
In [1]: %help histogram
309 |

The terms in italics (Atom) or underlined (Jupyter Notebook) are links. Click on them to see another help menu.

310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 |
AtomAtom
Jupyter NotebookJupyter Notebook
322 |

%html, %latex

323 |

Display command output as HTML or LaTeX

324 |

This can be useful when creating regression tables with esttab, for example. The images below are run following

325 | 
sysuse auto
326 | eststo: qui regress price weight mpg
327 | eststo: qui regress price weight mpg foreign
328 |

An HTML table will display correctly both inside JupyterLab and as a saved HTML file.

329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | 338 | 339 | 340 |
JupyterLabesttab-html-jupyterlab
Saved HTML fileesttab-html-file
341 |

A LaTeX table will not display correctly within JupyterLab (it only supports the math subset of LaTeX) but it will render correctly upon export to a PDF (which happens through LaTeX).

342 | 343 | 344 | 345 | 346 | 347 | 348 | 349 | 350 | 351 | 352 | 353 |
Saved PDF fileesttab-latex-pdf
Saved HTML fileesttab-latex-jupyterlab
354 |

%locals, %globals

355 |

List local or global macros

356 | 
%locals [-h] [-v] [REGEX [REGEX ...]]
357 | %globals [-h] [-v] [REGEX [REGEX ...]]
358 |

These take two optional arguments:

359 |
    360 |
  1. a regular expression for filtering the locals or globals displayed
    2. 361 |
  2. a --verbose flag
    3. 362 |
363 | 
In [1]: %globals S_
364 | (note: showing first line of global values; run with --verbose)
365 | 
366 | S_ADO:     BASE;SITE;.;PERSONAL;PLUS;OLDPLACE;`"/home/kyle/github/stata/sta
367 | S_level:   95
368 | S_CONSOLE: console
369 | S_FLAVOR:  Intercooled
370 | S_OS:      Unix
371 | S_MACH:    PC (64-bit x86-64)
372 | 
373 | In [2]: %globals S_ --verbose
374 | S_ADO:     BASE;SITE;.;PERSONAL;PLUS;OLDPLACE;`"/home/kyle/github/stata/sta
375 |            > ta_kernel/stata_kernel/ado"'
376 | S_level:   95
377 | S_CONSOLE: console
378 | S_FLAVOR:  Intercooled
379 | S_OS:      Unix
380 | S_MACH:    PC (64-bit x86-64)
381 |

%set

382 |

Set configuration value

383 |

Usage:

384 | 
%set [-h] [--permanently] [--reset] key value
385 |
    386 |
  • key: Configuration key name. The full list of configuration options is shown on the Configuration page.
    • 387 |
  • value: Value to set.
    • 388 |
  • --permanently: Store settings permanently.
    • 389 |
  • --reset: Restore default settings.
    • 390 |
391 |

As an example, you can change the graph settings like so:

392 | 
%set graph_format svg --permanently
393 | %set graph_scale 1
394 | %set graph_width 500
395 | %set graph_height 300
396 |

%show_gui, %hide_gui

397 |

Show/hide the Stata Graphical User Interface (GUI). Only works on Windows (and Mac if using automation execution mode)"

398 |

%status

399 |

Print information about:

400 |
    401 |
  • Stata kernel version
    • 402 |
  • Whether you’re in Stata/Mata
    • 403 |
  • Current delimiter
    • 404 |
405 | 414 | 415 | 416 | -------------------------------------------------------------------------------- /stata_kernel/docs/using_stata_kernel/magics.txt: -------------------------------------------------------------------------------- 1 | Magics 2 | 3 | Magics are programs provided by stata_kernel that enhance the experience 4 | of working with Stata in Jupyter. 5 | 6 | All magics are special commands that start with %. They must be the 7 | first word of the cell or selection, otherwise they won’t be intercepted 8 | and will be sent to Stata. 9 | 10 | For most of the magics listed, you can add --help to see a help menu in 11 | the kernel. For example, 12 | 13 | In [1]: %locals --help 14 | usage: %locals [-h] [-v] [REGEX [REGEX ...]] 15 | 16 | positional arguments: 17 | REGEX regex to match 18 | 19 | optional arguments: 20 | -h, --help show this help message and exit 21 | -v, --verbose Verbose output (print full contents of matched locals). 22 | 23 | The magics that respond with richly formatted text, namely %browse and 24 | %help, will not work with Jupyter Console or Jupyter QtConsole, since 25 | they don’t support displaying HTML. 26 | 27 | %browse, %head, %tail 28 | 29 | Interactively view your dataset 30 | 31 | This can optionally be provided with a varlist, N, or if: 32 | 33 | %browse [-h] [N] [varlist] [if] 34 | %head [-h] [N] [varlist] [if] 35 | %tail [-h] [N] [varlist] [if] 36 | 37 | By default: 38 | 39 | - %browse displays the first 200 rows 40 | - %head displays the first 10 rows 41 | - %tail displays the last 10 rows 42 | 43 | If you’re using Windows or macOS with Automation mode, you can also run 44 | browse (without the %) and it will open the usual Stata data explorer. 45 | 46 | ------------------ -------------------- 47 | Atom [Atom] 48 | Jupyter Notebook [Jupyter Notebook] 49 | ------------------ -------------------- 50 | 51 | %delimit 52 | 53 | Print the current delimiter 54 | 55 | This takes no arguments; it prints the delimiter currently set: either 56 | cr or ;. If you want to change the delimiter, use #delimit ; or 57 | #delimit cr. The delimiter will remain set until changed. 58 | 59 | In [1]: %delimit 60 | The delimiter is currently: cr 61 | 62 | In [2]: #delimit ; 63 | delimiter now ; 64 | In [3]: %delimit 65 | The delimiter is currently: ; 66 | 67 | In [4]: #delimit cr 68 | delimiter now cr 69 | 70 | %help 71 | 72 | Display a help file in rich text 73 | 74 | %help [-h] command_or_topic_name 75 | 76 | Add the term you want to search for after %help: 77 | 78 | In [1]: %help histogram 79 | 80 | The terms in italics (Atom) or underlined (Jupyter Notebook) are links. 81 | Click on them to see another help menu. 82 | 83 | ------------------ -------------------- 84 | Atom [Atom] 85 | Jupyter Notebook [Jupyter Notebook] 86 | ------------------ -------------------- 87 | 88 | %html, %latex 89 | 90 | Display command output as HTML or LaTeX 91 | 92 | This can be useful when creating regression tables with esttab, for 93 | example. The images below are run following 94 | 95 | sysuse auto 96 | eststo: qui regress price weight mpg 97 | eststo: qui regress price weight mpg foreign 98 | 99 | An HTML table will display correctly both inside JupyterLab and as a 100 | saved HTML file. 101 | 102 | ----------------- -------------------------- 103 | JupyterLab [esttab-html-jupyterlab] 104 | Saved HTML file [esttab-html-file] 105 | ----------------- -------------------------- 106 | 107 | A LaTeX table will not display correctly within JupyterLab (it only 108 | supports the math subset of LaTeX) but it will render correctly upon 109 | export to a PDF (which happens through LaTeX). 110 | 111 | ----------------- --------------------------- 112 | Saved PDF file [esttab-latex-pdf] 113 | Saved HTML file [esttab-latex-jupyterlab] 114 | ----------------- --------------------------- 115 | 116 | %locals, %globals 117 | 118 | List local or global macros 119 | 120 | %locals [-h] [-v] [REGEX [REGEX ...]] 121 | %globals [-h] [-v] [REGEX [REGEX ...]] 122 | 123 | These take two optional arguments: 124 | 125 | 1. a regular expression for filtering the locals or globals displayed 126 | 2. a --verbose flag 127 | 128 | In [1]: %globals S_ 129 | (note: showing first line of global values; run with --verbose) 130 | 131 | S_ADO: BASE;SITE;.;PERSONAL;PLUS;OLDPLACE;`"/home/kyle/github/stata/sta 132 | S_level: 95 133 | S_CONSOLE: console 134 | S_FLAVOR: Intercooled 135 | S_OS: Unix 136 | S_MACH: PC (64-bit x86-64) 137 | 138 | In [2]: %globals S_ --verbose 139 | S_ADO: BASE;SITE;.;PERSONAL;PLUS;OLDPLACE;`"/home/kyle/github/stata/sta 140 | > ta_kernel/stata_kernel/ado"' 141 | S_level: 95 142 | S_CONSOLE: console 143 | S_FLAVOR: Intercooled 144 | S_OS: Unix 145 | S_MACH: PC (64-bit x86-64) 146 | 147 | %set 148 | 149 | Set configuration value 150 | 151 | Usage: 152 | 153 | %set [-h] [--permanently] [--reset] key value 154 | 155 | - key: Configuration key name. The full list of configuration options 156 | is shown on the Configuration page. 157 | - value: Value to set. 158 | - --permanently: Store settings permanently. 159 | - --reset: Restore default settings. 160 | 161 | As an example, you can change the graph settings like so: 162 | 163 | %set graph_format svg --permanently 164 | %set graph_scale 1 165 | %set graph_width 500 166 | %set graph_height 300 167 | 168 | %show_gui, %hide_gui 169 | 170 | Show/hide the Stata Graphical User Interface (GUI). Only works on 171 | Windows (and Mac if using automation execution mode)" 172 | 173 | %status 174 | 175 | Print information about: 176 | 177 | - Stata kernel version 178 | - Whether you’re in Stata/Mata 179 | - Current delimiter 180 | -------------------------------------------------------------------------------- /stata_kernel/install.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import json 4 | import argparse 5 | import platform 6 | 7 | from shutil import copyfile 8 | from pathlib import Path 9 | from textwrap import dedent 10 | from pkg_resources import resource_filename 11 | from IPython.utils.tempdir import TemporaryDirectory 12 | from jupyter_client.kernelspec import KernelSpecManager 13 | 14 | 15 | kernel_json = { 16 | "argv": [sys.executable, "-m", "stata_kernel", "-f", "{connection_file}"], 17 | "display_name": "Stata", 18 | "language": "stata", } 19 | 20 | if sys.version_info[0] < 3: 21 | raise ImportError('Python < 3 is unsupported.') 22 | 23 | if sys.version_info[1] < 5: 24 | raise ImportError('Python < 3.5 is unsupported.') 25 | 26 | 27 | def install_my_kernel_spec(user=True, prefix=None): 28 | with TemporaryDirectory() as td: 29 | os.chmod(td, 0o755) # Starts off as 700, not user readable 30 | with open(os.path.join(td, 'kernel.json'), 'w') as f: 31 | json.dump(kernel_json, f, sort_keys=True) 32 | 33 | # Copy logo to tempdir to be installed with kernelspec 34 | logo_path = resource_filename('stata_kernel', 'docs/logo-64x64.png') 35 | copyfile(logo_path, os.path.join(td, 'logo-64x64.png')) 36 | 37 | print('Installing Jupyter kernel spec') 38 | KernelSpecManager().install_kernel_spec( 39 | td, 'stata', user=user, prefix=prefix) 40 | 41 | 42 | def install_conf(conf_file): 43 | if platform.system() == 'Windows': 44 | execution_mode = 'automation' 45 | else: 46 | execution_mode = 'console' 47 | 48 | # By avoiding an import of .utils until we need it, we can 49 | # complete the installation process in virtual environments 50 | # without needing this submodule nor its downstream imports. 51 | from .utils import find_path 52 | stata_path = find_path() 53 | if not stata_path: 54 | msg = """\ 55 | WARNING: Could not find Stata path. 56 | Refer to the documentation to see how to set it manually: 57 | 58 | https://kylebarron.dev/stata_kernel/using_stata_kernel/configuration 59 | 60 | """ 61 | print(dedent(msg)) 62 | elif 'IC.app' in stata_path: 63 | execution_mode = 'automation' 64 | 65 | conf_default = dedent( 66 | """\ 67 | [stata_kernel] 68 | 69 | # Path to stata executable. If you type this in your terminal, it should 70 | # start the Stata console 71 | stata_path = {} 72 | 73 | # **macOS only** 74 | # The manner in which the kernel connects to Stata. Either 'console' or 75 | # 'automation'. 'console' is the default because it allows multiple 76 | # independent sessions of Stata at the same time. 77 | execution_mode = {} 78 | 79 | # Directory to hold temporary images and log files 80 | cache_directory = ~/.stata_kernel_cache 81 | 82 | # Whether autocompletion suggestions should include the closing symbol 83 | # (i.e. ``'`` for a local macro or `}}` if the global starts with `${{`) 84 | autocomplete_closing_symbol = False 85 | 86 | # Extension and format for images 87 | graph_format = svg 88 | 89 | # Scaling factor for graphs 90 | graph_scale = 1 91 | 92 | # List of user-created keywords that produce graphs. 93 | # Should be comma-delimited. 94 | user_graph_keywords = coefplot,vioplot 95 | """.format(stata_path, execution_mode)) 96 | 97 | with conf_file.open('w') as f: 98 | f.write(conf_default) 99 | 100 | 101 | def _is_root(): 102 | try: 103 | return os.geteuid() == 0 104 | except AttributeError: 105 | return False # assume not an admin on non-Unix platforms 106 | 107 | 108 | def main(argv=None): 109 | ap = argparse.ArgumentParser() 110 | ap.add_argument( 111 | '--user', action='store_true', 112 | help="Install to the per-user kernels registry. Default if not root.") 113 | ap.add_argument( 114 | '--sys-prefix', action='store_true', 115 | help="Install to sys.prefix (e.g. a virtualenv or conda env)") 116 | ap.add_argument( 117 | '--prefix', help="Install to the given prefix. " 118 | "Kernelspec will be installed in {PREFIX}/share/jupyter/kernels/") 119 | ap.add_argument( 120 | '--no-conf-file', action='store_true', 121 | help="Skip the creation of a default user configuration file.") 122 | args = ap.parse_args(argv) 123 | 124 | if args.sys_prefix: 125 | args.prefix = sys.prefix 126 | if not args.prefix and not _is_root(): 127 | args.user = True 128 | 129 | install_my_kernel_spec(user=args.user, prefix=args.prefix) 130 | if not args.no_conf_file: 131 | conf_file = Path('~/.stata_kernel.conf').expanduser() 132 | if not conf_file.is_file(): 133 | install_conf(conf_file) 134 | 135 | 136 | if __name__ == '__main__': 137 | main() 138 | -------------------------------------------------------------------------------- /stata_kernel/kernel.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import base64 4 | import shutil 5 | import platform 6 | import html 7 | 8 | from PIL import Image 9 | from pathlib import Path 10 | from textwrap import dedent 11 | from datetime import datetime 12 | from xml.etree import ElementTree as ET 13 | from pkg_resources import resource_filename 14 | from ipykernel.kernelbase import Kernel 15 | 16 | from .config import config 17 | from .completions import CompletionsManager 18 | from .code_manager import CodeManager 19 | from .stata_session import StataSession 20 | from .stata_magics import StataMagics 21 | 22 | 23 | class StataKernel(Kernel): 24 | implementation = 'stata_kernel' 25 | implementation_version = '1.13.0' 26 | language = 'stata' 27 | language_info = { 28 | 'name': 'stata', 29 | 'mimetype': 'text/x-stata', 30 | 'codemirror_mode': 'stata', 31 | 'file_extension': '.do', 32 | 'version': '15.1'} 33 | help_links = [ 34 | {'text': 'stata_kernel Help', 'url': 'https://kylebarron.dev/stata_kernel/'}, 35 | {'text': 'Stata Help', 'url': 'https://www.stata.com/features/documentation/'} 36 | ] # yapf: disable 37 | 38 | def __init__(self, *args, **kwargs): 39 | # Copy syntax highlighting files 40 | from_paths = [ 41 | Path(resource_filename('stata_kernel', 'pygments/stata.py')), 42 | Path(resource_filename('stata_kernel', 'codemirror/stata.js'))] 43 | to_paths = [ 44 | Path(resource_filename('pygments', 'lexers/stata.py')), 45 | Path( 46 | resource_filename( 47 | 'notebook', 48 | 'static/components/codemirror/mode/stata/stata.js'))] 49 | 50 | for from_path, to_path in zip(from_paths, to_paths): 51 | copy = False 52 | if to_path.is_file(): 53 | to_path_dt = datetime.fromtimestamp(to_path.stat().st_mtime) 54 | from_path_dt = datetime.fromtimestamp(from_path.stat().st_mtime) 55 | if from_path_dt > to_path_dt: 56 | copy = True 57 | else: 58 | copy = True 59 | 60 | if copy: 61 | try: 62 | to_path.parents[0].mkdir(parents=True, exist_ok=True) 63 | shutil.copy(str(from_path), str(to_path)) 64 | except OSError: 65 | pass 66 | 67 | super(StataKernel, self).__init__(*args, **kwargs) 68 | 69 | self.graph_formats = ['svg', 'png', 'pdf', 'eps'] 70 | self.sc_delimit_mode = False 71 | self.stata = StataSession(self) 72 | self.banner = self.stata.banner 73 | self.language_version = self.stata.stata_version 74 | self.magics = StataMagics(self) 75 | self.completions = CompletionsManager(self) 76 | self.quickdo('cap di "Set _rc to 0 initially"') 77 | 78 | def do_execute( 79 | self, code, silent, store_history=True, user_expressions=None, 80 | allow_stdin=False): 81 | """Execute user code. 82 | 83 | This is the function that Jupyter calls to run code. Must return a 84 | dictionary as described here: 85 | https://jupyter-client.readthedocs.io/en/stable/messaging.html#execution-results 86 | """ 87 | invalid_input_msg = """\ 88 | stata_kernel error: code entered was incomplete. 89 | 90 | This usually means that a loop or program was not correctly terminated. 91 | This can also happen if you are in `#delimit ;` mode and did not end the 92 | command with `;`. Use `%delimit` to see the current delimiter mode and 93 | use `#delimit cr` to switch back to the default mode where `;` is 94 | unnecessary. 95 | """ 96 | if not self.is_complete(code): 97 | self.send_response( 98 | self.iopub_socket, 'stream', { 99 | 'text': dedent(invalid_input_msg), 100 | 'name': 'stderr'}) 101 | 102 | return { 103 | 'status': 'error', 104 | 'ename': 'error_exception_name', 105 | 'evalue': 'exception_value', 106 | 'traceback': [''], 107 | 'execution_count': self.execution_count} 108 | 109 | # Search for magics in the code 110 | code = self.magics.magic(code, self) 111 | 112 | # If the magic executed, bail out early 113 | if self.magics.quit_early: 114 | return self.magics.quit_early 115 | 116 | # Tokenize code and return code chunks 117 | cm = CodeManager(code, self.sc_delimit_mode, self.stata.mata_mode) 118 | self.stata._mata_refresh(cm) 119 | text_to_run, md5, text_to_exclude = cm.get_text(self.stata) 120 | 121 | # Execute code chunk 122 | rc, res = self.stata.do( 123 | text_to_run, md5, text_to_exclude=text_to_exclude) 124 | res = self.stata._mata_restart(rc, res) 125 | 126 | # Post magic results, if applicable 127 | self.magics.post(self) 128 | self.post_do_hook() 129 | 130 | # Alert if delimiter changed. NOTE: This compares the delimiter at the 131 | # end of the code block with that at the end of the previous code block. 132 | if (not silent) and (cm.ends_sc != self.sc_delimit_mode): 133 | delim = ';' if cm.ends_sc else 'cr' 134 | self.send_response( 135 | self.iopub_socket, 'stream', { 136 | 'text': 'delimiter now {}'.format(delim), 137 | 'name': 'stdout'}) 138 | self.sc_delimit_mode = cm.ends_sc 139 | 140 | # The base class increments the execution count 141 | return_obj = {'execution_count': self.execution_count} 142 | if rc: 143 | return_obj['status'] = 'error' 144 | return_obj['ename'] = 'error_exception_name' 145 | return_obj['evalue'] = 'exception_value' 146 | return_obj['traceback'] = [''] 147 | else: 148 | return_obj['status'] = 'ok' 149 | return_obj['payload'] = [] 150 | return_obj['user_expressions'] = {} 151 | return return_obj 152 | 153 | def post_do_hook(self): 154 | """Things to do after running commands in Stata 155 | """ 156 | 157 | store_rc = """\ 158 | tempname __user_rc 159 | local `__user_rc' = _rc 160 | """ 161 | self.quickdo(dedent(store_rc)) 162 | _rc, _res = self.cleanLogs("off") 163 | 164 | self.stata.linesize = int(self.quickdo("di `c(linesize)'")) 165 | self.stata.cwd = self.quickdo("pwd") 166 | self.completions.refresh(self) 167 | 168 | _rc, _res = self.cleanLogs("on") 169 | 170 | # Restore _rc 171 | restore_rc = """\ 172 | _StataKernelResetRC, num(``__user_rc'') 173 | macro drop _`__user_rc' 174 | macro drop ___user_rc 175 | """ 176 | self.quickdo(dedent(restore_rc)) 177 | 178 | def quickdo(self, code): 179 | code = self.stata._mata_escape(code) 180 | cm = CodeManager(code) 181 | text_to_run, md5, text_to_exclude = cm.get_text() 182 | rc, res = self.stata.do( 183 | text_to_run, md5, text_to_exclude=text_to_exclude, display=False) 184 | 185 | if not rc: 186 | # Remove rmsg lines when rmsg is on 187 | rmsg_regex = r'r(\(\d+\))?;\s+t=\d*\.\d*\s*\d*:\d*:\d*' 188 | res = [ 189 | x for x in res.split('\n') 190 | if not re.search(rmsg_regex, x.strip())] 191 | res = '\n'.join(res).strip() 192 | if self.stata.mata_open: 193 | res = re.sub( 194 | r'^([:\>]) ??(\{\})?$', '', res, 195 | flags=re.MULTILINE).strip() 196 | 197 | return res 198 | 199 | def cleanLogs(self, what): 200 | code = self.stata._mata_escape("_StataKernelLog {0}".format(what)) 201 | cm = CodeManager(code) 202 | text_to_run, md5, text_to_exclude = cm.get_text() 203 | rc, res = self.stata.do( 204 | text_to_run, md5, text_to_exclude=text_to_exclude, display=False) 205 | 206 | if what == 'off': 207 | code = self.stata._mata_escape('_StataKernelLog {0}'.format(what)) 208 | self.cleanTail(code, self.stata.prompt_dot) 209 | return rc, res 210 | 211 | def send_image(self, graph_paths): 212 | """Load graph and send to frontend 213 | 214 | This supports SVG, PNG, and PDF formats. While PDF display isn't 215 | supported in Atom or Jupyter, the data can be stored within the Jupyter 216 | Notebook file and makes exporting images to PDF through LaTeX easier. 217 | 218 | Args: 219 | graph_paths (List[str]): path to exported graph 220 | """ 221 | 222 | no_display_msg = 'This front-end cannot display the desired image type.' 223 | content = {'data': {'text/plain': no_display_msg}, 'metadata': {}} 224 | warn = False 225 | for graph_path in graph_paths: 226 | file_size = Path(graph_path).stat().st_size 227 | if (file_size > 2 * (1024 ** 3)) & (len(graph_paths) >= 2): 228 | warn = True 229 | 230 | # run user-specified eps convert, if available 231 | graph_epstopng = False 232 | graph_epstopdf = False 233 | if graph_path.endswith('.eps'): 234 | epstopdf_program = config.get('graph_epstopdf_program') 235 | epstopng_program = config.get('graph_epstopng_program') 236 | if epstopng_program: 237 | base_path = graph_path 238 | graph_path = Path(graph_path).with_suffix('.png') 239 | os.system(epstopng_program.format(base_path, graph_path)) 240 | graph_epstopng = True 241 | elif epstopdf_program: 242 | base_path = graph_path 243 | graph_path = Path(graph_path).with_suffix('.pdf') 244 | os.system(epstopdf_program.format(base_path, graph_path)) 245 | graph_epstopdf = True 246 | else: 247 | graph_path = Path(graph_path) 248 | else: 249 | graph_path = Path(graph_path) 250 | 251 | # display graph in front-end 252 | if graph_path.suffix == '.svg': 253 | with graph_path.open('r', encoding='utf-8') as f: 254 | img = f.read() 255 | e = ET.ElementTree(ET.fromstring(img)) 256 | root = e.getroot() 257 | 258 | width = int(root.attrib['width'][:-2]) 259 | height = int(root.attrib['height'][:-2]) 260 | # Wrap SVG in iframe. See #234. 261 | iframe = """\ 262 | 264 | """.format(height, width, html.escape(img)) 265 | content['data']['text/html'] = dedent(iframe) 266 | content['metadata']['text/html'] = { 267 | 'width': width, 268 | 'height': height} 269 | 270 | content['data']['image/svg+xml'] = img 271 | content['metadata']['image/svg+xml'] = { 272 | 'width': int(root.attrib['width'][:-2]), 273 | 'height': int(root.attrib['height'][:-2])} 274 | 275 | elif graph_path.suffix == '.png' or graph_epstopng: 276 | im = Image.open(graph_path) 277 | width = im.size[0] 278 | height = im.size[1] 279 | 280 | # On my Mac, the width is double what I told Stata to export. 281 | # This is not true on my Windows test VM 282 | if platform.system() == 'Darwin': 283 | width /= 2 284 | height /= 2 285 | with graph_path.open('rb') as f: 286 | img = base64.b64encode(f.read()).decode('utf-8') 287 | 288 | content['data']['image/png'] = img 289 | content['metadata']['image/png'] = { 290 | 'width': width, 291 | 'height': height} 292 | 293 | elif graph_path.suffix == '.pdf' or graph_epstopdf: 294 | with graph_path.open('rb') as f: 295 | pdf = base64.b64encode(f.read()).decode('utf-8') 296 | content['data']['application/pdf'] = pdf 297 | 298 | elif graph_path.suffix == '.eps': 299 | with graph_path.open('rb') as f: 300 | eps = base64.b64encode(f.read()).decode('utf-8') 301 | content['data']['application/eps'] = eps 302 | 303 | msg = """\ 304 | **`stata_kernel` Warning**: One of your image files is larger than 2MB 305 | and you have Graph Redundancy on. If you don't plan to export the 306 | Jupyter Notebook file to PDF, you can save space by running: 307 | 308 | ``` 309 | %set graph_svg_redundancy false [--permanently] 310 | %set graph_png_redundancy false [--permanently] 311 | ``` 312 | 313 | To turn off this warning, run: 314 | 315 | ``` 316 | %set graph_redundancy_warning false [--permanently] 317 | ``` 318 | 319 | For more information, see: 320 | 321 | """ 322 | msg = dedent(msg) 323 | warn_setting = config.get('graph_redundancy_warning', 'True') 324 | if warn and (warn_setting.lower() == 'true'): 325 | self.send_response( 326 | self.iopub_socket, 'display_data', { 327 | 'data': { 328 | 'text/plain': msg, 329 | 'text/markdown': msg}, 330 | 'metadata': {}}) 331 | self.send_response(self.iopub_socket, 'display_data', content) 332 | 333 | def do_shutdown(self, restart): 334 | """Shutdown the Stata session 335 | 336 | Shutdown the kernel. You only need to handle your own clean up - the 337 | kernel machinery will take care of cleaning up its own things before 338 | stopping. 339 | """ 340 | self.stata.shutdown() 341 | return {'restart': restart} 342 | 343 | def do_is_complete(self, code): 344 | """Decide if command has completed""" 345 | if self.is_complete(code): 346 | return {'status': 'complete'} 347 | 348 | return {'status': 'incomplete', 'indent': ' '} 349 | 350 | def do_complete(self, code, cursor_pos): 351 | """Provide context-aware suggestions 352 | """ 353 | env, pos, chunk, rcomp = self.completions.get_env( 354 | code[:cursor_pos], code[cursor_pos:(cursor_pos + 2)], 355 | self.sc_delimit_mode, self.stata.mata_mode) 356 | 357 | return { 358 | 'status': 'ok', 359 | 'cursor_start': pos, 360 | 'cursor_end': cursor_pos, 361 | 'matches': self.completions.get(chunk, env, rcomp)} 362 | 363 | def is_complete(self, code): 364 | return CodeManager( 365 | code, self.sc_delimit_mode, self.stata.mata_mode).is_complete 366 | 367 | def cleanTail(self, tail, rprompt): 368 | """ 369 | Search from the end of all open log files for a kernel marker 370 | specified by tail, typically 371 | 372 | . `md5 hash' 373 | 374 | rprompt is a regex for the prompt, typically a dot but it could 375 | be a `>` or a `:` (e.g. in mata). We only search up to 10 chars 376 | past the length of the marker for log files (unless it is a smcl 377 | file, in which case we search up to 100 chars back). 378 | """ 379 | ltail = len(tail) 380 | rtail = re.escape(tail[::-1]) + ' {0,2}' 381 | for logfile in self.completions.suggestions['logfiles']: 382 | lcmp = '' 383 | fname, fext = os.path.splitext(logfile) 384 | with open(logfile, 'r+', encoding='utf-8') as fh: 385 | fh.seek(0, os.SEEK_END) 386 | pos = fh.tell() - 1 387 | # Note the search is inverted because we read from the end 388 | if fext == '.smcl': 389 | maxread = pos - ltail - 100 390 | rfind = rtail + '({0}|}}moc{{|[\\r\\n])'.format(rprompt) 391 | else: 392 | rfind = rtail + rprompt 393 | maxread = pos - ltail - 10 394 | while (pos > maxread) and (re.search(rfind, lcmp) is None): 395 | lcmp += fh.read(1) 396 | pos -= 1 397 | fh.seek(pos, os.SEEK_SET) 398 | 399 | if pos > maxread: 400 | fh.seek(pos + 1, os.SEEK_SET) 401 | fh.truncate() 402 | 403 | def do_inspect(self, code, cursor_pos, detail_level=0): 404 | inspect_keyword = re.compile( 405 | r'\b(?P\w+)\(?\s*$', flags=re.MULTILINE).search 406 | 407 | pre = ( 408 | r'\b(cap(t|tu|tur|ture)?' 409 | r'|qui(e|et|etl|etly)?' 410 | r'|n(o|oi|ois|oisi|oisil|oisily)?)\b') 411 | 412 | inspect_mata = re.compile( 413 | r'^(\s*{0})*(?P\w+)\b'.format(pre), 414 | flags=re.MULTILINE).search 415 | 416 | inspect_not_found = re.compile(r'help for \w+ not found').search 417 | 418 | ismata = False 419 | context = inspect_mata(code) 420 | if context: 421 | ismata = context.groupdict()['context'].strip() == 'mata' 422 | ismata = ismata and code.strip() != 'mata' 423 | 424 | found = False 425 | data = {} 426 | match = inspect_keyword(code) 427 | if match: 428 | keyword = match.groupdict()['keyword'] 429 | if ismata: 430 | keyword = 'mf_' + keyword 431 | 432 | cm = CodeManager('help ' + keyword) 433 | text_to_run, md5, text_to_exclude = cm.get_text() 434 | rc, res = self.stata.do( 435 | text_to_run, md5, text_to_exclude=text_to_exclude, 436 | display=False) 437 | 438 | if inspect_not_found(res) is None: 439 | found = True 440 | data = {'text/plain': res} 441 | 442 | content = { 443 | 'status': 'ok', 444 | # found should be true if an object was found, false otherwise 445 | 'found': found, 446 | # data can be empty if nothing is found 447 | 'data': data, 448 | 'metadata': {}} 449 | 450 | return content 451 | -------------------------------------------------------------------------------- /stata_kernel/pygments/stata.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # yapf: disable 3 | """ 4 | pygments.lexers.stata 5 | ~~~~~~~~~~~~~~~~~~~~~ 6 | Lexer for Stata 7 | :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. 8 | :license: BSD, see LICENSE for details. 9 | """ 10 | 11 | import re 12 | from pygments.lexer import RegexLexer, include, words 13 | from pygments.token import Comment, Keyword, Name, Number, \ 14 | String, Text, Operator 15 | 16 | from pygments.lexers._stata_builtins import builtins_base, builtins_functions 17 | 18 | __all__ = ['StataLexer'] 19 | 20 | 21 | class StataLexer(RegexLexer): 22 | """ 23 | For `Stata `_ do files. 24 | .. versionadded:: 2.2 25 | """ 26 | # Syntax based on 27 | # - http://fmwww.bc.edu/RePEc/bocode/s/synlightlist.ado 28 | # - http://github.com/isagalaev/highlight.js/blob/master/src/languages/stata.js 29 | # - http://github.com/jpitblado/vim-stata/blob/master/syntax/stata.vim 30 | 31 | name = 'Stata' 32 | aliases = ['stata', 'do'] 33 | filenames = ['*.do', '*.ado'] 34 | mimetypes = ['text/x-stata', 'text/stata', 'application/x-stata'] 35 | flags = re.MULTILINE | re.DOTALL 36 | 37 | tokens = { 38 | 'root': [ 39 | include('comments'), 40 | include('strings'), 41 | include('macros'), 42 | include('numbers'), 43 | include('keywords'), 44 | include('operators'), 45 | include('format'), 46 | (r'.', Text), 47 | ], 48 | # Comments are a complicated beast in Stata because they can be 49 | # nested and there are a few corner cases with that. See: 50 | # - github.com/kylebarron/language-stata/issues/90 51 | # - statalist.org/forums/forum/general-stata-discussion/general/1448244 52 | 'comments': [ 53 | (r'(^//|(?<=\s)//)(?!/)', Comment.Single, 'comments-double-slash'), 54 | (r'^\s*\*', Comment.Single, 'comments-star'), 55 | (r'/\*', Comment.Multiline, 'comments-block'), 56 | (r'(^///|(?<=\s)///)', Comment.Special, 'comments-triple-slash') 57 | ], 58 | 'comments-block': [ 59 | (r'/\*', Comment.Multiline, '#push'), 60 | # this ends and restarts a comment block. but need to catch this so 61 | # that it doesn\'t start _another_ level of comment blocks 62 | (r'\*/\*', Comment.Multiline), 63 | (r'(\*/\s+\*(?!/)[^\n]*)|(\*/)', Comment.Multiline, '#pop'), 64 | # Match anything else as a character inside the comment 65 | (r'.', Comment.Multiline), 66 | ], 67 | 'comments-star': [ 68 | (r'///.*?\n', Comment.Single, 69 | ('#pop', 'comments-triple-slash')), 70 | (r'(^//|(?<=\s)//)(?!/)', Comment.Single, 71 | ('#pop', 'comments-double-slash')), 72 | (r'/\*', Comment.Multiline, 'comments-block'), 73 | (r'.(?=\n)', Comment.Single, '#pop'), 74 | (r'.', Comment.Single), 75 | ], 76 | 'comments-triple-slash': [ 77 | (r'\n', Comment.Special, '#pop'), 78 | # A // breaks out of a comment for the rest of the line 79 | (r'//.*?(?=\n)', Comment.Single, '#pop'), 80 | (r'.', Comment.Special), 81 | ], 82 | 'comments-double-slash': [ 83 | (r'\n', Text, '#pop'), 84 | (r'.', Comment.Single), 85 | ], 86 | # `"compound string"' and regular "string"; note the former are 87 | # nested. 88 | 'strings': [ 89 | (r'`"', String, 'string-compound'), 90 | (r'(?=|<|>|&|!=', Operator), 154 | (r'\*|\+|\^|/|!|~|==|~=', Operator) 155 | ], 156 | # Stata numbers 157 | 'numbers': [ 158 | # decimal number 159 | (r'\b[+-]?([0-9]+(\.[0-9]+)?|\.[0-9]+|\.)([eE][+-]?[0-9]+)?[i]?\b', 160 | Number), 161 | ], 162 | # Stata formats 163 | 'format': [ 164 | (r'%-?\d{1,2}(\.\d{1,2})?[gfe]c?', Name.Other), 165 | (r'%(21x|16H|16L|8H|8L)', Name.Other), 166 | (r'%-?(tc|tC|td|tw|tm|tq|th|ty|tg)\S{0,32}', Name.Other), 167 | (r'%[-~]?\d{1,4}s', Name.Other), 168 | ] 169 | } 170 | -------------------------------------------------------------------------------- /stata_kernel/stata_lexer.py: -------------------------------------------------------------------------------- 1 | import re 2 | from pygments.lexer import RegexLexer, include 3 | from pygments.token import Comment, Text, Token 4 | 5 | 6 | # yapf: disable 7 | class StataLexer(RegexLexer): 8 | """Modified Pygments Stata Lexer 9 | 10 | I only need accurate handling of comment, string, and line-continuation 11 | environments. 12 | 13 | When in #delimit ; mode, always add #delimit ; to the first line, then run 14 | through this same lexer. 15 | 16 | I have to use the Pygments token types, which don't really let me express what I want to express. 17 | 18 | Make a `delimiter` type. Then mark `\\n` and `;` as the delimiter in the respective zones. Then I can use the standard Text or Block keywords in both places. Take out any \\n from Token.Text first, because that's a newline in the ;-delimited block. 19 | 20 | For #delimit; text, do two passes, once to remove comments. Then take out ; and extra text \\n and put in delimiting \\n. Then put through once more to find blocks. 21 | 22 | Text: Arbitrary text 23 | Token.SemicolonDelimiter: Delimiter (either \\n or ;), depending on the block 24 | 25 | For mata, we use: 26 | Token.Mata.Open 27 | Token.Mata.OpenError 28 | Token.Mata.Close 29 | Token.TextBlockParen 30 | 31 | If mata was opened with a colon, :, an error will close mata. We 32 | take into account when determining if mata was left open. Further, 33 | we make sure parenthesis behave as blocks, {}, to force the user to 34 | close them. 35 | """ 36 | flags = re.MULTILINE | re.DOTALL 37 | tokens = { 38 | 'root': [ 39 | (r'`"', Text, 'string-compound'), 40 | (r'(? version.parse(stata_kernel_version): 17 | msg = """\ 18 | NOTE: A newer version of stata_kernel exists. Run 19 | 20 | \tpip install stata_kernel --upgrade 21 | 22 | to install the latest version. 23 | """ 24 | return dedent(msg) 25 | except requests.exceptions.RequestException: 26 | return 27 | 28 | 29 | def find_path(): 30 | if os.getenv('CONTINUOUS_INTEGRATION'): 31 | print('WARNING: Running as CI; Stata path not set correctly') 32 | return 'stata' 33 | if platform.system() == 'Windows': 34 | return win_find_path() 35 | elif platform.system() == 'Darwin': 36 | return mac_find_path() 37 | else: 38 | for i in ['stata-mp', 'stata-se', 'stata']: 39 | stata_path = which(i) 40 | if stata_path: 41 | break 42 | 43 | return stata_path 44 | 45 | 46 | def win_find_path(): 47 | import winreg 48 | reg = winreg.ConnectRegistry(None, winreg.HKEY_CLASSES_ROOT) 49 | subkeys = [ 50 | r'Stata16Do\shell\do\command', r'Stata15Do\shell\do\command', 51 | r'Stata14Do\shell\do\command', r'Stata13Do\shell\do\command', 52 | r'Stata12Do\shell\do\command'] 53 | 54 | fpath = '' 55 | for subkey in subkeys: 56 | try: 57 | key = winreg.OpenKey(reg, subkey) 58 | fpath = winreg.QueryValue(key, None).split('"')[1] 59 | except FileNotFoundError: 60 | pass 61 | if fpath: 62 | break 63 | 64 | return fpath 65 | 66 | 67 | def mac_find_path(): 68 | """Attempt to find Stata path on macOS when not on user's PATH 69 | 70 | Returns: 71 | (str): Path to Stata. Empty string if not found. 72 | """ 73 | path = Path('/Applications/Stata') 74 | if not path.exists(): 75 | return '' 76 | 77 | dirs = [ 78 | x for x in path.iterdir() if re.search(r'Stata(SE|MP)?\.app', x.name)] 79 | if not dirs: 80 | return '' 81 | 82 | if len(dirs) > 1: 83 | for ext in ['MP.app', 'SE.app', 'IC.app', '.app']: 84 | name = [x for x in dirs if x.name.endswith(ext)] 85 | if name: 86 | dirs = name 87 | break 88 | 89 | path = dirs[0] / 'Contents' / 'MacOS' 90 | if not path.exists(): 91 | return '' 92 | 93 | binaries = [x for x in path.iterdir()] 94 | for pref in ['stata-mp', 'stata-se', 'stata', 'StataIC']: 95 | name = [x for x in binaries if x.name == pref] 96 | if name: 97 | binaries = name 98 | break 99 | 100 | return str(binaries[0]) 101 | -------------------------------------------------------------------------------- /tests/test_data/auto.dta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kylebarron/stata_kernel/f7be69b81ff83ca9cde27ed1a85f2a93d386d616/tests/test_data/auto.dta -------------------------------------------------------------------------------- /tests/test_kernel_completions.py: -------------------------------------------------------------------------------- 1 | from utils import StataKernelTestFramework 2 | from stata_kernel.config import Config 3 | 4 | class TestKernelCompletions(StataKernelTestFramework): 5 | 6 | # samples for the autocompletion functionality 7 | # for each dictionary, `text` is the input to try and complete, and 8 | # `matches` the list of all complete matching strings which should be found 9 | completion_samples = [ 10 | # Magics 11 | { 12 | 'text': '%', 13 | 'matches': {'browse', 'delimit', 'globals', 'head', 'help', 14 | 'hide_gui', 'locals', 'set', 'show_gui', 'status', 15 | 'tail', 'html', 'latex'} 16 | }, 17 | { 18 | 'text': '%b', 19 | 'matches': {'browse'} 20 | }, 21 | { 22 | 'text': '%set ', 23 | 'matches': set(Config().all_settings) 24 | }, 25 | { 26 | 'text': '%set au', 27 | 'matches': {x for x in Config().all_settings if x.startswith('au')} 28 | } 29 | ] # yapf: disable 30 | 31 | def test_stata_global_completion(self): 32 | code = f"""\ 33 | global abcd "helloworld" 34 | global abdef "foo" 35 | global aef "bar" 36 | """ 37 | self._run(code) 38 | 39 | text = 'di $a' 40 | matches = {'abcd', 'abdef', 'aef'} 41 | self._test_completion(text, matches, exact=True) 42 | 43 | text = 'di ${a' 44 | matches = {'abcd', 'abdef', 'aef'} 45 | self._test_completion(text, matches, exact=True) 46 | 47 | def test_stata_path_completion(self): 48 | text = 'use "' 49 | matches = { 50 | 'CHANGELOG.md', 'CONTRIBUTING.md', 'LICENSE', 'MANIFEST.in', 51 | 'README.md'} 52 | self._test_completion(text, matches, exact=False) 53 | 54 | def test_stata_path_completion1(self): 55 | text = 'use "' 56 | self._test_completion(text) 57 | 58 | def test_stata_path_completion2(self): 59 | text = 'use "./' 60 | matches = { 61 | './CHANGELOG.md', './CONTRIBUTING.md', './LICENSE', './MANIFEST.in', 62 | './README.md'} 63 | self._test_completion(text, matches, exact=False) 64 | 65 | def test_stata_path_completion3(self): 66 | text = 'use "./CHANGE' 67 | matches = {'./CHANGELOG.md'} 68 | self._test_completion(text, matches, exact=True) 69 | 70 | def test_stata_path_completion4(self): 71 | text = 'use "./stata_kernel/' 72 | matches = {'./stata_kernel/completions.py', './stata_kernel/code_manager.py', './stata_kernel/config.py'} 73 | self._test_completion(text, matches, exact=False) 74 | 75 | def test_stata_path_completion5(self): 76 | text = 'use "./stata_kernel/c' 77 | matches = {'./stata_kernel/completions.py', './stata_kernel/code_manager.py', './stata_kernel/config.py'} 78 | self._test_completion(text, matches, exact=False) 79 | 80 | def test_stata_path_completion6(self): 81 | text = 'use "CHANGE' 82 | matches = {'CHANGELOG.md'} 83 | self._test_completion(text, matches, exact=True) 84 | 85 | def test_stata_path_completion7(self): 86 | text = 'use "stata_kernel/' 87 | matches = {'stata_kernel/completions.py', 'stata_kernel/code_manager.py', 'stata_kernel/config.py'} 88 | self._test_completion(text, matches, exact=False) 89 | 90 | def test_stata_path_completion8(self): 91 | text = 'use "stata_kernel/c' 92 | matches = {'stata_kernel/completions.py', 'stata_kernel/code_manager.py', 'stata_kernel/config.py'} 93 | self._test_completion(text, matches, exact=False) 94 | 95 | def test_stata_path_completion9(self): 96 | self._run('global datadir "stata_kernel"') 97 | text = 'use "$datadir/' 98 | matches = {'$datadir/completions.py', '$datadir/code_manager.py', '$datadir/config.py'} 99 | self._test_completion(text, matches, exact=False) 100 | 101 | def test_stata_path_completion10(self): 102 | self._run('global datadir "stata_kernel"') 103 | text = 'use "$datadir/c' 104 | matches = {'$datadir/completions.py', '$datadir/code_manager.py', '$datadir/config.py'} 105 | self._test_completion(text, matches, exact=False) 106 | 107 | def test_stata_path_completion11(self): 108 | self._run('global datadir "stata_kernel"') 109 | text = 'use "${datadir}/' 110 | matches = {'${datadir}/completions.py', '${datadir}/code_manager.py', '${datadir}/config.py'} 111 | self._test_completion(text, matches, exact=False) 112 | 113 | def test_stata_path_completion12(self): 114 | text = 'use "${datadir}/c' 115 | matches = {'${datadir}/completions.py', '${datadir}/code_manager.py', '${datadir}/config.py'} 116 | self._test_completion(text, matches, exact=False) 117 | 118 | 119 | def test_stata_varlist_completion(self): 120 | self._run() 121 | text = 'list ' 122 | matches = [ 123 | 'make', 'price', 'mpg', 'rep78', 'headroom', 'trunk', 'weight', 124 | 'length', 'turn', 'displacement', 'gear_ratio', 'foreign'] 125 | self._test_completion(text, matches, exact=False) 126 | 127 | def test_stata_varlist_completion1(self): 128 | self._run() 129 | text = 'list m' 130 | matches = ['make', 'mpg'] 131 | self._test_completion(text, matches, exact=False) 132 | -------------------------------------------------------------------------------- /tests/test_kernel_display_data.py: -------------------------------------------------------------------------------- 1 | from utils import StataKernelTestFramework 2 | 3 | class TestKernelDisplayData(StataKernelTestFramework): 4 | # Samples of code which should generate a rich display output, and 5 | # the expected MIME type 6 | # code_display_data = [ 7 | # {'code': 'sysuse auto\nscatter price mpg', 'mime': 'image/svg+xml'} 8 | # ] 9 | 10 | def test_stata_display_data(self): 11 | self._test_display_data('scatter price mpg', 'text/html') 12 | self._test_display_data('scatter price mpg', 'image/svg+xml') 13 | self._test_display_data('scatter price mpg', 'application/pdf') 14 | 15 | def test_stata_scatter_regex(self): 16 | """https://github.com/kylebarron/stata_kernel/issues/205""" 17 | self._test_display_data('sc price mpg', 'image/svg+xml') 18 | self._test_display_data('scatter price mpg', 'image/svg+xml') 19 | -------------------------------------------------------------------------------- /tests/test_kernel_stdout.py: -------------------------------------------------------------------------------- 1 | from utils import StataKernelTestFramework 2 | 3 | class TestKernelStdout(StataKernelTestFramework): 4 | # Code in the kernel's language to write "hello, world" to stdout 5 | code_hello_world = 'display "hello, world"' 6 | 7 | def test_stata_stdout(self): 8 | self._test_execute_stdout('di "hi"\ndi "bye"', 'hi\n\nbye') 9 | self._test_execute_stdout('di 6*7', '42') 10 | 11 | def test_stata_more(self): 12 | self._test_execute_stdout('more', '--more--', exact=True) 13 | 14 | def test_stata_display_unicode_letter(self): 15 | """https://github.com/kylebarron/stata_kernel/issues/196""" 16 | code = """\ 17 | local a "ä" 18 | di "`a'"\ 19 | """ 20 | self._test_execute_stdout(code, 'ä', exact=True) 21 | 22 | def test_stata_return_class(self): 23 | """ 24 | Need to make sure r() class doesn't get deleted between runs 25 | https://github.com/kylebarron/stata_kernel/issues/265 26 | """ 27 | self._run('sum mpg', dataset='auto') 28 | self._run('local res = r(max)', dataset=None) 29 | self._test_execute_stdout("di `res'", '41', exact=True) 30 | 31 | def test_long_program(self): 32 | """ 33 | Programs with more than 10 lines should not show line continuation 34 | numbers 35 | """ 36 | code = f"""\ 37 | cap program drop helloworld 38 | program helloworld 39 | di "helloworld" 40 | di "helloworld" 41 | di "helloworld" 42 | di "helloworld" 43 | di "helloworld" 44 | di "helloworld" 45 | di "helloworld" 46 | di "helloworld" 47 | di "helloworld" 48 | di "helloworld" 49 | di "helloworld" 50 | di "helloworld" 51 | di "helloworld" 52 | di "helloworld" 53 | end 54 | """ 55 | self._test_execute_stdout(code, output=None) 56 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | import jupyter_kernel_test 2 | 3 | 4 | class StataKernelTestFramework(jupyter_kernel_test.KernelTests): 5 | # The name identifying an installed kernel to run the tests against 6 | kernel_name = 'stata' 7 | 8 | # language_info.name in a kernel_info_reply should match this 9 | language_name = 'stata' 10 | 11 | def _test_completion(self, text, matches=None, exact=True): 12 | msg_id = self.kc.complete(text) 13 | reply = self.kc.get_shell_msg() 14 | jupyter_kernel_test.messagespec.validate_message( 15 | reply, 'complete_reply', msg_id) 16 | if matches is not None: 17 | if exact: 18 | self.assertEqual(set(reply['content']['matches']), set(matches)) 19 | else: 20 | print(set(matches)) 21 | print(set(reply['content']['matches'])) 22 | self.assertTrue( 23 | set(matches) <= set(reply['content']['matches'])) 24 | else: 25 | self.assertFalse(None) 26 | 27 | def _test_display_data(self, code, mimetype, _not=False): 28 | reply, output_msgs = self._run(code=code) 29 | self.assertEqual(reply['content']['status'], 'ok') 30 | self.assertGreaterEqual(len(output_msgs), 1) 31 | self.assertEqual(output_msgs[0]['msg_type'], 'display_data') 32 | self.assertIn(mimetype, output_msgs[0]['content']['data']) 33 | 34 | def _test_execute_stdout(self, code, output=None, exact=False): 35 | """ 36 | This strings all output sent to stdout together and then checks that 37 | `code` is in `output` 38 | """ 39 | reply, output_msgs = self._run(code=code) 40 | self.assertEqual(reply['content']['status'], 'ok') 41 | if output: 42 | self.assertGreaterEqual(len(output_msgs), 1) 43 | else: 44 | self.assertEqual(len(output_msgs), 0) 45 | return 46 | 47 | all_output = [] 48 | for msg in output_msgs: 49 | if (msg['msg_type'] == 'stream') and ( 50 | msg['content']['name'] == 'stdout'): 51 | all_output.append(msg['content']['text']) 52 | all_output = ''.join(all_output).strip() 53 | if exact: 54 | self.assertEqual(output, all_output) 55 | else: 56 | self.assertIn(output, all_output) 57 | 58 | def _run(self, code=None, dataset='auto', **kwargs): 59 | """Wrapper to run arbitrary code in the Stata kernel 60 | """ 61 | self.flush_channels() 62 | if dataset is not None: 63 | res = self.execute_helper(f'sysuse {dataset}, clear') 64 | if code is not None: 65 | res = self.execute_helper(code=code, **kwargs) 66 | return res 67 | --------------------------------------------------------------------------------