├── .bazelrc ├── .dockerignore ├── .github └── workflows │ └── documentation.yaml ├── .gitignore ├── .swift-format.json ├── BUILD ├── LICENSE ├── Package.swift ├── README.md ├── WORKSPACE ├── bazel └── setup_clang.sh ├── bootup ├── BUILD └── bootup.py ├── display └── JupyterDisplay.swift ├── external ├── PythonKit.BUILD ├── requirements.txt ├── swift-argument-parser.BUILD ├── swift-crypto.BUILD ├── swift-format.BUILD ├── swift-syntax.BUILD ├── swift-system.BUILD └── swift-tools-support-core.BUILD ├── kernels └── swift │ ├── BUILD │ ├── EnableJupyterDisplay.swift │ ├── KernelCommunicator.swift │ ├── matplotlib_swift │ ├── __init__.py │ └── backend_inline.py │ ├── swift_kernel.py │ └── whole_archive.bzl ├── lab.bzl ├── lab └── BUILD ├── notebooks ├── Untitled.ipynb └── _swift-template.ipynb ├── scripts ├── BUILD.bazel ├── bazel │ └── setup_clang.sh ├── black │ ├── BUILD │ ├── format.py │ └── pre-commit ├── buildifier │ ├── buildifier.sh │ └── pre-commit ├── docc.sh ├── install.sh ├── swift-format │ ├── pre-commit │ └── swift-format.sh └── vendors │ └── dispatch └── test ├── __init__.py ├── all_test_docker.py ├── all_test_local.py ├── fast_test.py ├── notebook_tester.py ├── test.py └── tests ├── __init__.py ├── kernel_tests.py ├── notebooks ├── PackageWithC │ ├── Package.swift │ └── Sources │ │ ├── PackageWithC1 │ │ ├── include │ │ │ └── sillyfunction1.h │ │ └── sillyfunction1.c │ │ └── PackageWithC2 │ │ ├── include │ │ └── sillyfunction2.h │ │ └── sillyfunction2.c ├── SimplePackage │ ├── Package.swift │ └── Sources │ │ └── SimplePackage │ │ └── SimplePackage.swift ├── install_package.ipynb ├── install_package_with_c.ipynb ├── install_package_with_user_location.ipynb ├── intentional_compile_error.ipynb ├── intentional_runtime_error.ipynb └── simple_successful.ipynb ├── simple_notebook_tests.py └── tutorial_notebook_tests.py /.bazelrc: -------------------------------------------------------------------------------- 1 | build --disk_cache=.cache 2 | build --features=swift.index_while_building 3 | 4 | try-import %workspace%/clang.bazelrc 5 | try-import %workspace%/.bazelrc.local 6 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .dockerignore 2 | 3 | .idea/ 4 | .git** 5 | screenshots/ 6 | venv/ 7 | 8 | **/*.pyc 9 | -------------------------------------------------------------------------------- /.github/workflows/documentation.yaml: -------------------------------------------------------------------------------- 1 | name: documentation 2 | on: 3 | push: 4 | branches: 5 | - main 6 | jobs: 7 | build: 8 | runs-on: ubuntu-20.04 9 | steps: 10 | - uses: actions/checkout@v2 11 | with: 12 | fetch-depth: 0 13 | 14 | - name: Install bazelisk 15 | run: | 16 | curl -LO "https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64" 17 | mkdir -p "${GITHUB_WORKSPACE}/bin/" 18 | mv bazelisk-linux-amd64 "${GITHUB_WORKSPACE}/bin/bazel" 19 | chmod +x "${GITHUB_WORKSPACE}/bin/bazel" 20 | 21 | - name: Install Swift dependencies 22 | run: | 23 | sudo apt-get install clang libicu-dev 24 | wget https://download.swift.org/swift-5.6.2-release/ubuntu2004/swift-5.6.2-RELEASE/swift-5.6.2-RELEASE-ubuntu20.04.tar.gz 25 | tar xzf swift-5.6.2-RELEASE-ubuntu20.04.tar.gz 26 | echo "$(pwd)/swift-5.6.2-RELEASE-ubuntu20.04/usr/bin" >> $GITHUB_PATH 27 | 28 | - name: Setup clang 29 | run: | 30 | sudo apt -y install clang llvm 31 | ./bazel/setup_clang.sh 32 | echo "build --config=clang" >> "${GITHUB_WORKSPACE}/.bazelrc.local" 33 | 34 | - name: Clean up documentation branch 35 | run: | 36 | git branch -D documentation || true 37 | git checkout -b documentation 38 | 39 | - name: Run docc 40 | run: | 41 | cd "${GITHUB_WORKSPACE}" && ./scripts/docc.sh 42 | 43 | - name: Add and commit documentation 44 | run: | 45 | git config --global user.email "docbot@github.com" 46 | git config --global user.name "docbot" 47 | cd "${GITHUB_WORKSPACE}" && git add "docs/*" && git commit -m "Update docs." 48 | 49 | - name: Push the new branch 50 | run: | 51 | cd "${GITHUB_WORKSPACE}" && git push --force origin documentation:documentation 52 | 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | venv 2 | 3 | **/*.pyc 4 | 5 | bazel-* 6 | 7 | .bazelrc.local 8 | clang.bazelrc 9 | .cache 10 | 11 | *.[oda] 12 | *.~ 13 | *.swp 14 | *.gcno 15 | *.out 16 | 17 | .ipynb_checkpoints 18 | .ibzlnb 19 | 20 | __pycache__ 21 | 22 | .build 23 | .index 24 | -------------------------------------------------------------------------------- /.swift-format.json: -------------------------------------------------------------------------------- 1 | { 2 | "fileScopedDeclarationPrivacy" : { 3 | "accessLevel" : "private" 4 | }, 5 | "indentation" : { 6 | "spaces" : 2 7 | }, 8 | "indentConditionalCompilationBlocks" : true, 9 | "lineBreakAroundMultilineExpressionChainComponents" : false, 10 | "lineBreakBeforeControlFlowKeywords" : false, 11 | "lineBreakBeforeEachArgument" : false, 12 | "lineBreakBeforeEachGenericRequirement" : false, 13 | "lineLength" : 100, 14 | "maximumBlankLines" : 1, 15 | "prioritizeKeepingFunctionOutputTogether" : false, 16 | "respectsExistingLineBreaks" : true, 17 | "rules" : { 18 | "AllPublicDeclarationsHaveDocumentation" : true, 19 | "AlwaysUseLowerCamelCase" : true, 20 | "AmbiguousTrailingClosureOverload" : true, 21 | "BeginDocumentationCommentWithOneLineSummary" : true, 22 | "DoNotUseSemicolons" : true, 23 | "DontRepeatTypeInStaticProperties" : true, 24 | "FileprivateAtFileScope" : true, 25 | "FullyIndirectEnum" : true, 26 | "GroupNumericLiterals" : true, 27 | "IdentifiersMustBeASCII" : true, 28 | "NeverForceUnwrap" : true, 29 | "NeverUseForceTry" : true, 30 | "NeverUseImplicitlyUnwrappedOptionals" : true, 31 | "NoAccessLevelOnExtensionDeclaration" : true, 32 | "NoBlockComments" : true, 33 | "NoCasesWithOnlyFallthrough" : true, 34 | "NoEmptyTrailingClosureParentheses" : true, 35 | "NoLabelsInCasePatterns" : true, 36 | "NoLeadingUnderscores" : true, 37 | "NoParensAroundConditions" : true, 38 | "NoVoidReturnOnFunctionSignature" : true, 39 | "OneCasePerLine" : true, 40 | "OneVariableDeclarationPerLine" : true, 41 | "OnlyOneTrailingClosureArgument" : true, 42 | "OrderedImports" : true, 43 | "ReturnVoidInsteadOfEmptyTuple" : true, 44 | "UseLetInEveryBoundCaseVariable" : true, 45 | "UseShorthandTypeNames" : true, 46 | "UseSingleLinePropertyGetter" : true, 47 | "UseSynthesizedInitializer" : true, 48 | "UseTripleSlashForDocumentationComments" : true, 49 | "ValidateDocumentationComments" : true 50 | }, 51 | "tabWidth" : 2, 52 | "version" : 1 53 | } 54 | -------------------------------------------------------------------------------- /BUILD: -------------------------------------------------------------------------------- 1 | load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") 2 | 3 | swift_library( 4 | name = "JupyterDisplay", 5 | srcs = ["display/JupyterDisplay.swift"], 6 | module_name = "JupyterDisplay", 7 | visibility = ["//visibility:public"], 8 | deps = [], 9 | ) 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.4 2 | // The swift-tools-version declares the minimum version of Swift required to build this package. 3 | 4 | import PackageDescription 5 | 6 | let package = Package( 7 | name: "JupyterDisplay", 8 | platforms: [.macOS(.v10_14), .iOS(.v11), .watchOS(.v3), .tvOS(.v10)], 9 | products: [ 10 | .library(name: "JupyterDisplay", type: .static, targets: ["JupyterDisplay"]), 11 | ], 12 | targets: [ 13 | .target( 14 | name: "JupyterDisplay", 15 | path: "display"), 16 | ] 17 | ) 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Swift-Jupyter 2 | 3 | This is a fork of [google/swift-jupyter](https://github.com/google/swift-jupyter/). It is made 4 | possible to use Jupyterlab (as well as Jupyter Notebook) with most up-to-date Swift toolchain. 5 | 6 | This fork made several significant changes to original repository organization. It biased towards 7 | using [Bazel](https://bazel.build/) as code and dependency management tool. This fork is fully 8 | separated from [Swift for TensorFlow](https://github.com/tensorflow/swift) project and is actively 9 | maintained to keep everything working with up-to-date Swift toolchain. 10 | 11 | Right now, this fork worked well with Swift from 5.4.x to 5.5.x on Linux without any tweaks (if 12 | installed to `/opt/swift`). It is tested with Ubuntu 20.04 distribution, but should work with other 13 | flavors. 14 | 15 | Because both the notebook integration and PythonKit are used actively with one of my side business, 16 | the continuing maintenance of this repository can be assured. 17 | 18 | I do want to make this work on macOS. It should be only a few OS / toolchain detection script away. 19 | 20 | ## Getting Started 21 | 22 | ### Requirements 23 | 24 | Operating system: 25 | 26 | * Ubuntu 20.04 (64-bit); OR 27 | * Other operating systems may work, but you will have to build Swift from 28 | sources. 29 | 30 | Dependencies: 31 | 32 | * Bazel 33 | * Python 3 34 | 35 | ### Installation 36 | 37 | Extract the Swift toolchain to `/opt/swift`. 38 | 39 | Inside the `swift-jupyter` repository: 40 | 41 | ```bash 42 | ./scripts/bazel/setup_clang.sh /usr/local 43 | ``` 44 | 45 | Note: `/usr/local` should be where your `bin/llvm-config` binary resides. 46 | 47 | Now you can run Jupyterlab: 48 | 49 | ```bash 50 | SWIFTPATH=/opt/swift bazel run //lab:bootup -- --notebook-dir $PWD 51 | ``` 52 | 53 | To check if installation is done, run the `notebooks/_swift_template.ipynb` notebook. 54 | 55 | ## Usage Instructions 56 | 57 | ### Rich Output 58 | 59 | This fork removed `EnableIPythonDisplay.swift`. Using [IPython kernel](https://github.com/ipython/ipykernel) 60 | and its interactive shell as in [google/swift-jupyter](https://github.com/google/swift-jupyter/blob/main/swift_shell/__init__.py) 61 | cause some silent crashes with the most up-to-date Swift toolchain. It only happens after many tens 62 | of cell executions. 63 | 64 | Instead, this fork enhanced `EnableJupyterDisplay.swift` to enable rich output from Python (e.g. 65 | Pandas or matplotlib). Because communication with Jupyter notebook requires HMAC computation, this 66 | fork updated `EnableJupyterDisplay.swift` to use [apple/swift-crypto](https://github.com/apple/swift-crypto) 67 | for these cryptography primitives. 68 | 69 | The `EnableJupyterDisplay.swift` now will be automatically included (enabled) once your package 70 | installation is done. 71 | 72 | You can call Python libraries using [PythonKit](https://github.com/liuliu/PythonKit). I've set up 73 | Bazel with both [PythonKit](https://github.com/liuliu/PythonKit) and [swift-crypto](https://github.com/apple/swift-crypto) 74 | support. 75 | 76 | To see these in action, first install these packages with Bazel inside your notebook: 77 | 78 | ```bash 79 | %bazel "@PythonKit//:PythonKit" 80 | %bazel "@SwiftCrypto//:Crypto" 81 | ``` 82 | 83 | Now you should be able to display rich output! For example: 84 | 85 | ```swift 86 | let np = Python.import("numpy") 87 | let plt = Python.import("matplotlib.pyplot") 88 | ``` 89 | 90 | ```swift 91 | let time = np.arange(0, 10, 0.01) 92 | let amplitude = np.exp(-0.1 * time) 93 | let position = amplitude * np.sin(3 * time) 94 | 95 | plt.figure(figsize: [15, 10]) 96 | 97 | plt.plot(time, position) 98 | plt.plot(time, amplitude) 99 | plt.plot(time, -amplitude) 100 | 101 | plt.xlabel("time (s)") 102 | plt.ylabel("position (m)") 103 | plt.title("Oscillations") 104 | 105 | plt.show() 106 | ``` 107 | 108 | ```swift 109 | let pd = Python.import("pandas") 110 | pd.DataFrame.from_records([["col 1": 3, "col 2": 5], ["col 1": 8, "col 2": 2]]).display() 111 | ``` 112 | 113 | ### Build Swift Libraries that Support Jupyter Notebook 114 | 115 | `swift-jupyter` now provides a package called `JupyterDisplay` that you can included in your library 116 | to support rich outputs inside Jupyter Notebook. I've been successfully implemented nontrivial 117 | interactive streaming GUI with this capability. 118 | 119 | If your package has dependency to `JupyterDisplay`, when used inside `swift-jupyter`, your query to 120 | `JupyterDisplay.isEnabled` will be true and you can use `JupyterDisplay.display` functions to send 121 | HTML, PNG images or plain text to Jupyter Notebook. These can be flushed periodically with 122 | `JupyterDisplay.flush()` method. 123 | 124 | ### Code Completion 125 | 126 | This fork supports simple code completion with `sourcekit-lsp` now shipped along Swift toolchain. 127 | The code where uses Swift for TensorFlow's LLDB for code completion is not removed if you choose. I 128 | may remove that support entirely once the `sourcekit-lsp` challenge is resolved: 129 | 130 | Current `sourcekit-lsp` integration relies on appending to a *hypothetical* file on each successful 131 | cell execution. However, REPL execution is a bit different from code in a file because we can shadow 132 | variables without causing any problems. This can eventually leads to a *hypothetical* file that AST 133 | is not well-formed. For these cases, `sourcekit-lsp` based code completion will be degraded into a 134 | token-based code completion engine. 135 | 136 | ## Jupyterlab Integration 137 | 138 | If you manage your repository with Bazel, integrate with `swift-jupyter` provided Jupyterlab is the 139 | best way to run Jupyter Notebook. It keeps Python dependencies clean, can reference to Python packages 140 | either through pip or in-tree Python packages through Bazel. Once it sets up, your repository Jupyterlab 141 | configuration is portable, you don't need to worry about installed extensions not available in a 142 | different computer. 143 | 144 | First, add `swift-jupyter` to your `WORKSPACE` such as: 145 | 146 | ``` 147 | git_repository( 148 | name = "swift-jupyter", 149 | commit = "daf4eef0ea20be0d6eec5306b5b1cfdb11550d1e", 150 | remote = "https://github.com/liuliu/swift-jupyter.git", 151 | shallow_since = "1658788905 -0400", 152 | ) 153 | 154 | # You do need to include Python support package for Bazel, if you haven't already: 155 | 156 | http_archive( 157 | name = "rules_python", 158 | url = "https://github.com/bazelbuild/rules_python/releases/download/0.4.0/rules_python-0.4.0.tar.gz", 159 | sha256 = "954aa89b491be4a083304a2cb838019c8b8c3720a7abb9c4cb81ac7a24230cea", 160 | ) 161 | 162 | load("@rules_python//python:pip.bzl", "pip_install") 163 | 164 | # Install Python dependencies 165 | 166 | pip_install( 167 | requirements = ":requirements.txt", 168 | ) 169 | ``` 170 | 171 | Second, you need to modify / add `requirements.txt` file under `./external/` that includes Jupyterlab 172 | reference. 173 | 174 | Third, just find a place to use the provided `jupyterlab` macro to create a target, for example, this 175 | is what we do inside `./lab/BUILD`: 176 | 177 | ``` 178 | load("@swift-jupyter//:lab.bzl", "jupyterlab") 179 | 180 | jupyterlab( 181 | name = "bootup", 182 | deps = [], 183 | ) 184 | ``` 185 | 186 | You can add more dependencies through `deps` parameter. To launch Jupyterlab, simply do: 187 | ``` 188 | SWIFTPATH=/opt/swift bazel run lab:bootup -- --notebook-dir $PWD 189 | ``` 190 | 191 | ## Troubleshooting 192 | 193 | This fork uses Bazel to setup build and run environment. It helps to maintain a consistent 194 | environment between machines. There are two varying factors could impact the environment on Linux: 195 | 196 | 1. `clang` 197 | 198 | 2. Swift toolchain 199 | 200 | Current script assumes `clang` installed under `/usr/local/bin`, which should be a reasonable 201 | assumption but can vary. If you have LLVM toolchain installed somewhere else, `./scripts/bazel/setup_clang.sh` 202 | run should take a different prefix other than `/usr/local`. 203 | 204 | Bazel doesn't inherent environment variables. Thus, your Swift toolchain location needs to be passed 205 | in through command-line every time. Make sure it is the right location for your Swift toolchain. On 206 | Linux, that path should contain a sub-directory called `usr`. 207 | 208 | ## Compile Swift LLDB with Python Support 209 | 210 | Official packages provided through can be a hit-or-miss. There are 211 | some CI issues that I actively work with Swift members to make sure Swift LLDB is compiled with 212 | Python support. However, they can be missed from time to time. Here is a documented instruction for 213 | how to compile Swift LLDB with Python Support from source: 214 | 215 | 1. Make sure you have enough disk space, and an empty directory for source code from various 216 | repositories, I normally use `/opt/swift`; 217 | 218 | 2. To make sure you will build with Python support, you need to install Python's header file. On 219 | Ubuntu, it is `apt install python3-dev`; 220 | 221 | 3. `git clone` the following repositories to that directory: 222 | 223 | * https://github.com/apple/swift 224 | 225 | * https://github.com/apple/llvm-project 226 | 227 | * https://github.com/apple/swift-llbuild (Please rename to llbuild) 228 | 229 | * https://github.com/apple/swift-cmark (Please rename to cmark) 230 | 231 | * https://github.com/apple/swift-driver 232 | 233 | * https://github.com/apple/swift-tools-support-core 234 | 235 | * https://github.com/apple/swift-corelibs-libdispatch 236 | 237 | * https://github.com/apple/swift-argument-parser 238 | 239 | * https://github.com/jpsim/Yams (Please rename to yams) 240 | 241 | 4. Make sure these repositories are checked out with the matching branch. For example, if you want 242 | to compile for Swift 5.6.x, it should be branch `release/5.6`. For `llvm-project`, it is 243 | `swift/release/5.6`. It is OK to be on the tip for `Yams` and `swift-argument-parser`; 244 | 245 | 5. Create an empty `build` directory at the same level as all other repositories, such as `/opt/swift/build`; 246 | 247 | 6. Go to `swift/utils`, and run `./build-script --release --lldb`; 248 | 249 | 7. You should be able to find the relevant LLDB files under `/opt/swift/build/Ninja-ReleaseAssert/lldb-linux-x86_64`. 250 | -------------------------------------------------------------------------------- /WORKSPACE: -------------------------------------------------------------------------------- 1 | workspace(name = "swift-jupyter") 2 | 3 | load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository") 4 | load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") 5 | 6 | # Bazel dependencies 7 | 8 | git_repository( 9 | name = "build_bazel_rules_swift", 10 | commit = "3bc7bc164020a842ae08e0cf071ed35f0939dd39", 11 | remote = "https://github.com/bazelbuild/rules_swift.git", 12 | shallow_since = "1654173801 -0500", 13 | ) 14 | 15 | load("@build_bazel_rules_swift//swift:repositories.bzl", "swift_rules_dependencies") 16 | 17 | swift_rules_dependencies() 18 | 19 | load("@build_bazel_rules_swift//swift:extras.bzl", "swift_rules_extra_dependencies") 20 | 21 | swift_rules_extra_dependencies() 22 | 23 | http_archive( 24 | name = "rules_python", 25 | url = "https://github.com/bazelbuild/rules_python/releases/download/0.4.0/rules_python-0.4.0.tar.gz", 26 | sha256 = "954aa89b491be4a083304a2cb838019c8b8c3720a7abb9c4cb81ac7a24230cea", 27 | ) 28 | 29 | load("@rules_python//python:pip.bzl", "pip_install") 30 | 31 | # Install Python dependencies 32 | 33 | pip_install( 34 | requirements = ":requirements.txt", 35 | ) 36 | 37 | # 3rd-party open-source dependencies 38 | 39 | new_git_repository( 40 | name = "PythonKit", 41 | build_file = "PythonKit.BUILD", 42 | commit = "99a298f0413b0ac278ac58b7ac9045da920c347d", 43 | remote = "https://github.com/liuliu/PythonKit.git", 44 | shallow_since = "1642703957 -0500", 45 | ) 46 | 47 | new_git_repository( 48 | name = "SwiftArgumentParser", 49 | build_file = "swift-argument-parser.BUILD", 50 | commit = "82905286cc3f0fa8adc4674bf49437cab65a8373", 51 | remote = "https://github.com/apple/swift-argument-parser.git", 52 | shallow_since = "1647436700 -0500", 53 | ) 54 | 55 | new_git_repository( 56 | name = "SwiftSystem", 57 | build_file = "swift-system.BUILD", 58 | commit = "836bc4557b74fe6d2660218d56e3ce96aff76574", 59 | remote = "https://github.com/apple/swift-system.git", 60 | shallow_since = "1638472952 -0800", 61 | ) 62 | 63 | new_git_repository( 64 | name = "SwiftToolsSupportCore", 65 | build_file = "swift-tools-support-core.BUILD", 66 | commit = "b7667f3e266af621e5cc9c77e74cacd8e8c00cb4", 67 | remote = "https://github.com/apple/swift-tools-support-core.git", 68 | shallow_since = "1643831290 -0800", 69 | ) 70 | 71 | new_git_repository( 72 | name = "SwiftSyntax", 73 | build_file = "swift-syntax.BUILD", 74 | commit = "0b6c22b97f8e9320bca62e82cdbee601cf37ad3f", 75 | remote = "https://github.com/apple/swift-syntax.git", 76 | shallow_since = "1647591231 +0100", 77 | ) 78 | 79 | new_git_repository( 80 | name = "SwiftFormat", 81 | build_file = "swift-format.BUILD", 82 | commit = "e6b8c60c7671066d229e30efa1e31acf57be412e", 83 | remote = "https://github.com/apple/swift-format.git", 84 | shallow_since = "1647972246 -0700", 85 | ) 86 | 87 | new_git_repository( 88 | name = "SwiftCrypto", 89 | build_file = "swift-crypto.BUILD", 90 | commit = "a8911e0fadc25aef1071d582355bd1037a176060", 91 | remote = "https://github.com/apple/swift-crypto.git", 92 | shallow_since = "1641892042 +0000", 93 | ) 94 | 95 | # buildifier is written in Go and hence needs rules_go to be built. 96 | # See https://github.com/bazelbuild/rules_go for the up to date setup instructions. 97 | 98 | http_archive( 99 | name = "io_bazel_rules_go", 100 | sha256 = "d1ffd055969c8f8d431e2d439813e42326961d0942bdf734d2c95dc30c369566", 101 | urls = [ 102 | "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.24.5/rules_go-v0.24.5.tar.gz", 103 | "https://github.com/bazelbuild/rules_go/releases/download/v0.24.5/rules_go-v0.24.5.tar.gz", 104 | ], 105 | ) 106 | 107 | load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") 108 | 109 | go_rules_dependencies() 110 | 111 | go_register_toolchains() 112 | 113 | http_archive( 114 | name = "bazel_gazelle", 115 | sha256 = "b85f48fa105c4403326e9525ad2b2cc437babaa6e15a3fc0b1dbab0ab064bc7c", 116 | urls = [ 117 | "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.22.2/bazel-gazelle-v0.22.2.tar.gz", 118 | "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.22.2/bazel-gazelle-v0.22.2.tar.gz", 119 | ], 120 | ) 121 | 122 | load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") 123 | 124 | gazelle_dependencies() 125 | 126 | git_repository( 127 | name = "com_github_bazelbuild_buildtools", 128 | commit = "174cbb4ba7d15a3ad029c2e4ee4f30ea4d76edce", 129 | remote = "https://github.com/bazelbuild/buildtools.git", 130 | shallow_since = "1607975103 +0100", 131 | ) 132 | 133 | -------------------------------------------------------------------------------- /bazel/setup_clang.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BAZELRC_FILE="${BAZELRC_FILE:-$(bazel info workspace)/clang.bazelrc}" 4 | 5 | LLVM_PREFIX=$1 6 | 7 | if [[ ! -e "${LLVM_PREFIX}/bin/llvm-config" ]]; then 8 | echo "Error: cannot find llvm-config in ${LLVM_PREFIX}." 9 | exit 1 10 | fi 11 | 12 | export PATH="$(${LLVM_PREFIX}/bin/llvm-config --bindir):${PATH}" 13 | 14 | RT_LIBRARY_PATH="$(dirname $(find $(llvm-config --libdir) -name libclang_rt.ubsan_standalone_cxx-x86_64.a | head -1))" 15 | 16 | echo "# Generated file, do not edit. If you want to disable clang, just delete this file. 17 | build:clang --action_env='PATH=${PATH}' 18 | build:clang --action_env=CC=clang 19 | build:clang --action_env=CXX=clang++ 20 | build:clang --action_env='LLVM_CONFIG=${LLVM_PREFIX}/bin/llvm-config' 21 | build:clang --repo_env='LLVM_CONFIG=${LLVM_PREFIX}/bin/llvm-config' 22 | build:clang --linkopt='-L$(llvm-config --libdir)' 23 | build:clang --linkopt='-Wl,-rpath,$(llvm-config --libdir)' 24 | 25 | build:clang-asan --action_env=ENVOY_UBSAN_VPTR=1 26 | build:clang-asan --copt=-fsanitize=vptr,function 27 | build:clang-asan --linkopt=-fsanitize=vptr,function 28 | build:clang-asan --linkopt='-L${RT_LIBRARY_PATH}' 29 | build:clang-asan --linkopt=-l:libclang_rt.ubsan_standalone-x86_64.a 30 | build:clang-asan --linkopt=-l:libclang_rt.ubsan_standalone_cxx-x86_64.a 31 | " > ${BAZELRC_FILE} 32 | 33 | -------------------------------------------------------------------------------- /bootup/BUILD: -------------------------------------------------------------------------------- 1 | filegroup( 2 | name = "sources", 3 | srcs = ["bootup.py"], 4 | visibility = ["//visibility:public"], 5 | ) 6 | -------------------------------------------------------------------------------- /bootup/bootup.py: -------------------------------------------------------------------------------- 1 | import re 2 | import sys 3 | import os 4 | import json 5 | import tempfile 6 | 7 | from bazel_tools.tools.python.runfiles import runfiles 8 | from jupyterlab.labapp import main 9 | 10 | r = runfiles.Create() 11 | 12 | if "SWIFTPATH" in os.environ: 13 | SWIFTPATH = os.environ["SWIFTPATH"] 14 | else: 15 | print("Cannot find SWIFTPATH in environment variables. Exiting.") 16 | exit(-1) 17 | 18 | PYTHONPATH = os.environ["PYTHONPATH"] 19 | 20 | 21 | def register_kernels(): 22 | kernel_json = { 23 | "argv": [ 24 | sys.executable, 25 | r.Rlocation("swift-jupyter/kernels/swift/swift_kernel"), 26 | "-f", 27 | "{connection_file}", 28 | ], 29 | "display_name": "Swift", 30 | "language": "swift", 31 | "env": { 32 | "PYTHONPATH": PYTHONPATH 33 | + ":" 34 | + os.path.join( 35 | SWIFTPATH, 36 | "usr/lib/python3/dist-packages", 37 | ), 38 | "REPL_SWIFT_PATH": os.path.join(SWIFTPATH, "usr/bin/repl_swift"), 39 | "LD_LIBRARY_PATH": os.path.join(SWIFTPATH, "usr/lib/swift/linux"), 40 | "SWIFT_BUILD_PATH": os.path.join(SWIFTPATH, "usr/bin/swift-build"), 41 | "SWIFT_PACKAGE_PATH": os.path.join(SWIFTPATH, "usr/bin/swift-package"), 42 | "SOURCEKIT_LSP_PATH": os.path.join(SWIFTPATH, "usr/bin/sourcekit-lsp"), 43 | }, 44 | } 45 | 46 | kernel_code_name = "swift" 47 | 48 | td = tempfile.mkdtemp() 49 | os.environ["JUPYTER_DATA_DIR"] = td 50 | kernel_td = os.path.join(td, "kernels", "swift") 51 | os.makedirs(kernel_td, exist_ok=True) 52 | with open(os.path.join(kernel_td, "kernel.json"), "w") as f: 53 | json.dump(kernel_json, f, indent=2) 54 | 55 | 56 | if __name__ == "__main__": 57 | sys.argv[0] = re.sub(r"(-script\.pyw?|\.exe)?$", "", sys.argv[0]) 58 | jupyterlab = os.path.dirname( 59 | r.Rlocation("pip/pypi__jupyterlab/jupyterlab/__init__.py") 60 | ) 61 | os.environ["JUPYTERLAB_DIR"] = jupyterlab 62 | register_kernels() 63 | sys.exit(main()) 64 | -------------------------------------------------------------------------------- /display/JupyterDisplay.swift: -------------------------------------------------------------------------------- 1 | /// A display protocol for Jupyter notebook. 2 | /// 3 | /// This provides necessary methods for other Swift libraries to introspect whether you can 4 | /// display from within a Jupyter notebook, and methods to send HTML, images or texts over 5 | /// to the notebook. 6 | public protocol JupyterDisplayProtocol { 7 | /// flush display messages to the notebook immediately. By default, these messages are 8 | /// processed at the end of cell execution. 9 | func flush() 10 | /// Display a base64 encoded image in the notebook. 11 | func display(base64EncodedPNG: String, metadata: String?) 12 | /// Display a snippet of HTML in the notebook. 13 | func display(html: String, metadata: String?) 14 | /// Display plain text in the notebook. 15 | func display(text: String, metadata: String?) 16 | /// Check whether the notebook is active and accepting display messages. 17 | var isEnabled: Bool { get } 18 | /// Get the cell number. 19 | var executionCount: Int { get } 20 | } 21 | 22 | extension JupyterDisplayProtocol { 23 | /// Display a base64 encoded image in the notebook. 24 | public func display(base64EncodedPNG: String) { 25 | display(base64EncodedPNG: base64EncodedPNG, metadata: nil) 26 | } 27 | /// Display a snippet of HTML in the notebook. 28 | public func display(html: String) { 29 | display(html: html, metadata: nil) 30 | } 31 | /// Display plain text in the notebook. 32 | public func display(text: String) { 33 | display(text: text, metadata: nil) 34 | } 35 | } 36 | 37 | struct EmptyJupyterDisplay: JupyterDisplayProtocol { 38 | var isEnabled: Bool { false } 39 | var executionCount: Int { 0 } 40 | func flush() {} 41 | func display(base64EncodedPNG: String, metadata: String?) {} 42 | func display(html: String, metadata: String?) {} 43 | func display(text: String, metadata: String?) {} 44 | } 45 | 46 | /// The exposed instance to use for downstream libraries. 47 | public var JupyterDisplay: JupyterDisplayProtocol = EmptyJupyterDisplay() 48 | -------------------------------------------------------------------------------- /external/PythonKit.BUILD: -------------------------------------------------------------------------------- 1 | load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") 2 | 3 | package( 4 | default_visibility = ["//visibility:public"], 5 | ) 6 | 7 | swift_library( 8 | name = "PythonKit", 9 | srcs = glob([ 10 | "PythonKit/**/*.swift", 11 | ]), 12 | module_name = "PythonKit", 13 | ) 14 | -------------------------------------------------------------------------------- /external/requirements.txt: -------------------------------------------------------------------------------- 1 | jupyterlab==3.1.12 2 | pandas==1.3.3 3 | numpy==1.21.2 4 | matplotlib==3.4.3 5 | black==22.6.0 6 | gym==0.25.0 7 | pygame==2.1.2 8 | -------------------------------------------------------------------------------- /external/swift-argument-parser.BUILD: -------------------------------------------------------------------------------- 1 | load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") 2 | 3 | swift_library( 4 | name = "ArgumentParserToolInfo", 5 | srcs = glob([ 6 | "Sources/ArgumentParserToolInfo/**/*.swift", 7 | ]), 8 | module_name = "ArgumentParserToolInfo", 9 | ) 10 | 11 | swift_library( 12 | name = "ArgumentParser", 13 | srcs = glob([ 14 | "Sources/ArgumentParser/**/*.swift", 15 | ]), 16 | module_name = "ArgumentParser", 17 | visibility = ["//visibility:public"], 18 | deps = [ 19 | ":ArgumentParserToolInfo", 20 | ], 21 | ) 22 | -------------------------------------------------------------------------------- /external/swift-crypto.BUILD: -------------------------------------------------------------------------------- 1 | load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") 2 | 3 | cc_library( 4 | name = "CCryptoBoringSSL", 5 | srcs = glob([ 6 | "Sources/CCryptoBoringSSL/crypto/**/*.c", 7 | "Sources/CCryptoBoringSSL/crypto/**/*.h", 8 | "Sources/CCryptoBoringSSL/third_party/**/*.h", 9 | ]) + glob(["Sources/CCryptoBoringSSL/crypto/**/*.linux.x86_64.S"]) + ["Sources/CCryptoBoringSSL/crypto/hrss/asm/poly_rq_mul.S"], 10 | hdrs = glob(["Sources/CCryptoBoringSSL/include/**/*.h"]), 11 | defines = ["WIN32_LEAN_AND_MEAN"], 12 | includes = [ 13 | "Sources/CCryptoBoringSSL/include/", 14 | ], 15 | tags = ["swift_module=CCryptoBoringSSL"], 16 | ) 17 | 18 | cc_library( 19 | name = "CCryptoBoringSSLShims", 20 | srcs = ["Sources/CCryptoBoringSSLShims/shims.c"], 21 | hdrs = ["Sources/CCryptoBoringSSLShims/include/CCryptoBoringSSLShims.h"], 22 | defines = ["CRYPTO_IN_SWIFTPM"], 23 | includes = [ 24 | "Sources/CCryptoBoringSSLShims/include/", 25 | ], 26 | tags = ["swift_module=CCryptoBoringSSLShims"], 27 | deps = [ 28 | ":CCryptoBoringSSL", 29 | ], 30 | ) 31 | 32 | swift_library( 33 | name = "Crypto", 34 | srcs = glob([ 35 | "Sources/Crypto/**/*.swift", 36 | ]), 37 | defines = ["CRYPTO_IN_SWIFTPM"], 38 | module_name = "Crypto", 39 | visibility = ["//visibility:public"], 40 | deps = [ 41 | ":CCryptoBoringSSL", 42 | ":CCryptoBoringSSLShims", 43 | ], 44 | ) 45 | 46 | swift_library( 47 | name = "_CryptoExtras", 48 | srcs = glob([ 49 | "Sources/_CryptoExtras/**/*.swift", 50 | ]), 51 | module_name = "_CryptoExtras", 52 | visibility = ["//visibility:public"], 53 | deps = [ 54 | ":CCrypto", 55 | ":CCryptoBoringSSL", 56 | ":CCryptoBoringSSLShims", 57 | ], 58 | ) 59 | -------------------------------------------------------------------------------- /external/swift-format.BUILD: -------------------------------------------------------------------------------- 1 | load("@build_bazel_rules_swift//swift:swift.bzl", "swift_binary", "swift_library") 2 | 3 | swift_library( 4 | name = "SwiftFormatConfiguration", 5 | srcs = glob([ 6 | "Sources/SwiftFormatConfiguration/**/*.swift", 7 | ]), 8 | module_name = "SwiftFormatConfiguration", 9 | ) 10 | 11 | swift_library( 12 | name = "SwiftFormatCore", 13 | srcs = glob([ 14 | "Sources/SwiftFormatCore/**/*.swift", 15 | ]), 16 | module_name = "SwiftFormatCore", 17 | deps = [ 18 | ":SwiftFormatConfiguration", 19 | "@SwiftSyntax", 20 | ], 21 | ) 22 | 23 | swift_library( 24 | name = "SwiftFormatRules", 25 | srcs = glob([ 26 | "Sources/SwiftFormatRules/**/*.swift", 27 | ]), 28 | module_name = "SwiftFormatRules", 29 | deps = [ 30 | ":SwiftFormatCore", 31 | ], 32 | ) 33 | 34 | swift_library( 35 | name = "SwiftFormatPrettyPrint", 36 | srcs = glob([ 37 | "Sources/SwiftFormatPrettyPrint/**/*.swift", 38 | ]), 39 | module_name = "SwiftFormatPrettyPrint", 40 | deps = [ 41 | ":SwiftFormatCore", 42 | ], 43 | ) 44 | 45 | swift_library( 46 | name = "SwiftFormatWhitespaceLinter", 47 | srcs = glob([ 48 | "Sources/SwiftFormatWhitespaceLinter/**/*.swift", 49 | ]), 50 | module_name = "SwiftFormatWhitespaceLinter", 51 | deps = [ 52 | ":SwiftFormatCore", 53 | ], 54 | ) 55 | 56 | swift_library( 57 | name = "SwiftFormat", 58 | srcs = glob([ 59 | "Sources/SwiftFormat/**/*.swift", 60 | ]), 61 | module_name = "SwiftFormat", 62 | deps = [ 63 | ":SwiftFormatCore", 64 | ":SwiftFormatPrettyPrint", 65 | ":SwiftFormatRules", 66 | ":SwiftFormatWhitespaceLinter", 67 | "@SwiftSyntax//:SwiftSyntaxParser", 68 | ], 69 | ) 70 | 71 | swift_binary( 72 | name = "swift-format", 73 | srcs = glob([ 74 | "Sources/swift-format/**/*.swift", 75 | ]), 76 | visibility = ["//visibility:public"], 77 | deps = [ 78 | ":SwiftFormat", 79 | "@SwiftArgumentParser//:ArgumentParser", 80 | "@SwiftToolsSupportCore//:TSCBasic", 81 | ], 82 | ) 83 | -------------------------------------------------------------------------------- /external/swift-syntax.BUILD: -------------------------------------------------------------------------------- 1 | load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") 2 | load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") 3 | 4 | cc_library( 5 | name = "_CSwiftSyntax", 6 | srcs = ["Sources/_CSwiftSyntax/src/atomic-counter.c"], 7 | hdrs = [ 8 | "Sources/_CSwiftSyntax/include/atomic-counter.h", 9 | "Sources/_CSwiftSyntax/include/c-syntax-nodes.h", 10 | ], 11 | includes = [ 12 | "Sources/_CSwiftSyntax/include/", 13 | ], 14 | tags = ["swift_module=_CSwiftSyntax"], 15 | ) 16 | 17 | swift_library( 18 | name = "SwiftSyntax", 19 | srcs = glob([ 20 | "Sources/SwiftSyntax/**/*.swift", 21 | ]), 22 | module_name = "SwiftSyntax", 23 | visibility = ["//visibility:public"], 24 | deps = [ 25 | ":_CSwiftSyntax", 26 | ], 27 | ) 28 | 29 | swift_library( 30 | name = "SwiftSyntaxBuilder", 31 | srcs = glob([ 32 | "Sources/SwiftSyntaxBuilder/**/*.swift", 33 | ]), 34 | module_name = "SwiftSyntaxBuilder", 35 | visibility = ["//visibility:public"], 36 | deps = [ 37 | ":SwiftSyntax", 38 | ], 39 | ) 40 | 41 | swift_library( 42 | name = "SwiftSyntaxParser", 43 | srcs = glob([ 44 | "Sources/SwiftSyntaxParser/**/*.swift", 45 | ]), 46 | module_name = "SwiftSyntaxParser", 47 | visibility = ["//visibility:public"], 48 | deps = [ 49 | ":SwiftSyntax", 50 | ], 51 | ) 52 | -------------------------------------------------------------------------------- /external/swift-system.BUILD: -------------------------------------------------------------------------------- 1 | load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") 2 | 3 | cc_library( 4 | name = "CSystem", 5 | srcs = ["Sources/CSystem/shims.c"], 6 | hdrs = glob([ 7 | "Sources/CSystem/include/*.h", 8 | ]), 9 | includes = [ 10 | "Sources/CSystem/include/", 11 | ], 12 | tags = ["swift_module=CSystem"], 13 | ) 14 | 15 | swift_library( 16 | name = "SystemPackage", 17 | srcs = glob([ 18 | "Sources/System/**/*.swift", 19 | ]), 20 | defines = [ 21 | "_CRT_SECURE_NO_WARNINGS", 22 | "SYSTEM_PACKAGE", 23 | ], 24 | module_name = "SystemPackage", 25 | visibility = ["//visibility:public"], 26 | deps = [ 27 | ":CSystem", 28 | ], 29 | ) 30 | -------------------------------------------------------------------------------- /external/swift-tools-support-core.BUILD: -------------------------------------------------------------------------------- 1 | load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") 2 | 3 | cc_library( 4 | name = "TSCclibc", 5 | srcs = glob(["Sources/TSCclibc/*.c"]), 6 | hdrs = glob([ 7 | "Sources/TSCclibc/include/*.h", 8 | ]), 9 | includes = [ 10 | "Sources/TSCclibc/include/", 11 | ], 12 | tags = ["swift_module=TSCclibc"], 13 | ) 14 | 15 | swift_library( 16 | name = "TSCLibc", 17 | srcs = glob([ 18 | "Sources/TSCLibc/**/*.swift", 19 | ]), 20 | module_name = "TSCLibc", 21 | deps = [], 22 | ) 23 | 24 | swift_library( 25 | name = "TSCBasic", 26 | srcs = glob([ 27 | "Sources/TSCBasic/**/*.swift", 28 | ]), 29 | module_name = "TSCBasic", 30 | visibility = ["//visibility:public"], 31 | deps = [ 32 | ":TSCLibc", 33 | ":TSCclibc", 34 | "@SwiftSystem//:SystemPackage", 35 | ], 36 | ) 37 | -------------------------------------------------------------------------------- /kernels/swift/BUILD: -------------------------------------------------------------------------------- 1 | load("@pip//:requirements.bzl", "requirement") 2 | 3 | py_library( 4 | name = "matplotlib_swift", 5 | srcs = [ 6 | "matplotlib_swift/__init__.py", 7 | "matplotlib_swift/backend_inline.py", 8 | ], 9 | imports = ["."], 10 | ) 11 | 12 | py_binary( 13 | name = "swift_kernel", 14 | srcs = ["swift_kernel.py"], 15 | data = ["whole_archive.bzl"], 16 | visibility = ["//visibility:public"], 17 | deps = [ 18 | requirement("jupyterlab"), 19 | ":matplotlib_swift", 20 | "@bazel_tools//tools/python/runfiles", 21 | ], 22 | ) 23 | -------------------------------------------------------------------------------- /kernels/swift/EnableJupyterDisplay.swift: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | import Foundation 16 | 17 | #if canImport(Crypto) 18 | import Crypto 19 | #endif 20 | 21 | enum SwiftJupyterDisplay { 22 | 23 | struct Header: Encodable { 24 | let messageID: String 25 | let username: String 26 | let session: String 27 | let date: String 28 | let messageType: String 29 | let version: String 30 | private enum CodingKeys: String, CodingKey { 31 | case messageID = "msg_id" 32 | case messageType = "msg_type" 33 | case username = "username" 34 | case session = "session" 35 | case date = "date" 36 | case version = "version" 37 | } 38 | 39 | init( 40 | messageID: String = UUID().uuidString, 41 | username: String = "kernel", 42 | session: String, 43 | messageType: String = "display_data", 44 | version: String = "5.3" 45 | ) { 46 | self.messageID = messageID 47 | self.username = username 48 | self.session = session 49 | self.messageType = messageType 50 | self.version = version 51 | let currentDate = Date() 52 | let formatter = DateFormatter() 53 | formatter.dateFormat = "yyyy-MM-dd'T'HH:mm:ss.SSSZZZZZ" 54 | formatter.timeZone = TimeZone(secondsFromGMT: 0) 55 | formatter.locale = Locale(identifier: "en_US_POSIX") 56 | self.date = formatter.string(from: currentDate) 57 | } 58 | 59 | var json: String { 60 | let encoder = JSONEncoder() 61 | encoder.outputFormatting = [.sortedKeys] 62 | guard let jsonData = try? encoder.encode(self) else { return "{}" } 63 | let jsonString = String(data: jsonData, encoding: .utf8)! 64 | return jsonString 65 | } 66 | } 67 | 68 | struct Message { 69 | var messageType: String = "display_data" 70 | var delimiter = "" 71 | var key: String = "" 72 | var header: Header 73 | var metadata: String = "{}" 74 | var content: String = "{}" 75 | var hmacSignature: String { 76 | #if canImport(Crypto) 77 | guard let data = (header.json + parentHeader + metadata + content).data(using: .utf8), 78 | let secret = key.data(using: .utf8) 79 | else { return "" } 80 | let authenticationCode = HMAC.authenticationCode( 81 | for: data, using: SymmetricKey(data: secret)) 82 | return authenticationCode.map { String(format: "%02x", $0) }.joined() 83 | #endif 84 | return "" 85 | } 86 | var messageParts: [KernelCommunicator.BytesReference] { 87 | return [ 88 | SwiftJupyterDisplay.bytes(messageType), 89 | SwiftJupyterDisplay.bytes(delimiter), 90 | SwiftJupyterDisplay.bytes(hmacSignature), 91 | SwiftJupyterDisplay.bytes(header.json), 92 | SwiftJupyterDisplay.bytes(parentHeader), 93 | SwiftJupyterDisplay.bytes(metadata), 94 | SwiftJupyterDisplay.bytes(content), 95 | ] 96 | } 97 | 98 | var json: String { 99 | let encoder = JSONEncoder() 100 | let array = [ 101 | messageType, delimiter, hmacSignature, header.json, parentHeader, metadata, content, 102 | ] 103 | guard let jsonData = try? encoder.encode(array) else { return "[]" } 104 | let jsonString = String(data: jsonData, encoding: .utf8)! 105 | return jsonString 106 | } 107 | 108 | init(content: String = "{}") { 109 | header = Header( 110 | username: JupyterKernel.communicator.jupyterSession.username, 111 | session: JupyterKernel.communicator.jupyterSession.id 112 | ) 113 | self.content = content 114 | key = JupyterKernel.communicator.jupyterSession.key 115 | #if !canImport(Crypto) 116 | if !key.isEmpty { 117 | fatalError( 118 | """ 119 | Unable to import Crypto to perform message signing. 120 | Add swift-crypto as a dependency, or disable message signing in Jupyter as follows: 121 | jupyter notebook --Session.key='b\"\"'\n 122 | """) 123 | } 124 | #endif 125 | } 126 | } 127 | 128 | struct PNGImageData: Encodable { 129 | let image: String 130 | let text = "" 131 | init(base64EncodedPNG: String) { 132 | image = base64EncodedPNG 133 | } 134 | private enum CodingKeys: String, CodingKey { 135 | case image = "image/png" 136 | case text = "text/plain" 137 | } 138 | } 139 | 140 | struct HTMLData: Encodable { 141 | let html: String 142 | init(html: String) { 143 | self.html = html 144 | } 145 | private enum CodingKeys: String, CodingKey { 146 | case html = "text/html" 147 | } 148 | } 149 | 150 | struct TextData: Encodable { 151 | let text: String 152 | init(text: String) { 153 | self.text = text 154 | } 155 | private enum CodingKeys: String, CodingKey { 156 | case text = "text/plain" 157 | } 158 | } 159 | 160 | struct MarkdownData: Encodable { 161 | let markdown: String 162 | init(markdown: String) { 163 | self.markdown = markdown 164 | } 165 | private enum CodingKeys: String, CodingKey { 166 | case markdown = "text/markdown" 167 | } 168 | } 169 | 170 | struct JSONData: Encodable { 171 | let json: String 172 | init(json: String) { 173 | self.json = json 174 | } 175 | private enum CodingKeys: String, CodingKey { 176 | case json = "application/json" 177 | } 178 | } 179 | 180 | struct MessageContent: Encodable where Data: Encodable { 181 | let metadata: String 182 | let transient = "{}" 183 | let data: Data 184 | init(metadata: String, data: Data) { 185 | self.metadata = metadata 186 | self.data = data 187 | } 188 | init(data: Data) { 189 | self.metadata = "{}" 190 | self.data = data 191 | } 192 | var json: String { 193 | let encoder = JSONEncoder() 194 | encoder.outputFormatting = [.sortedKeys] 195 | guard let jsonData = try? encoder.encode(self) else { return "{}" } 196 | let jsonString = String(data: jsonData, encoding: .utf8)! 197 | return jsonString 198 | } 199 | } 200 | 201 | static var parentHeader = "{}" 202 | static var messages = [Message]() 203 | 204 | private static func bytes(_ bytes: String) -> KernelCommunicator.BytesReference { 205 | let bytes = bytes.utf8CString.dropLast() 206 | return KernelCommunicator.BytesReference(bytes) 207 | } 208 | 209 | static func enable() { 210 | JupyterKernel.communicator.handleParentMessage(SwiftJupyterDisplay.updateParentMessage) 211 | JupyterKernel.communicator.afterSuccessfulExecution(run: SwiftJupyterDisplay.consumeDisplayMessages) 212 | } 213 | 214 | private static func updateParentMessage(to parentMessage: KernelCommunicator.ParentMessage) { 215 | do { 216 | let jsonData = (parentMessage.json).data(using: .utf8, allowLossyConversion: false) 217 | let jsonDict = try JSONSerialization.jsonObject(with: jsonData!) as? NSDictionary 218 | let headerData = try JSONSerialization.data(withJSONObject: jsonDict!["header"]!) 219 | parentHeader = String(data: headerData, encoding: .utf8)! 220 | } catch { 221 | print("Error in JSON parsing!") 222 | } 223 | } 224 | 225 | private static func consumeDisplayMessages() -> [KernelCommunicator.JupyterDisplayMessage] { 226 | SwiftJupyterDisplay.flushFigures() 227 | var displayMessages = [KernelCommunicator.JupyterDisplayMessage]() 228 | for message in messages { 229 | displayMessages.append(KernelCommunicator.JupyterDisplayMessage(parts: message.messageParts)) 230 | } 231 | messages = [] 232 | return displayMessages 233 | } 234 | 235 | private static func flushFigures() { 236 | #if canImport(PythonKit) 237 | guard matplotlibEnabled else { return } 238 | backend_inline.flush_figures() 239 | #endif 240 | } 241 | 242 | #if canImport(PythonKit) 243 | private static var matplotlibEnabled = false 244 | private static let backend_inline = Python.import("matplotlib_swift.backend_inline") 245 | static func enable_matplotlib() { 246 | guard let matplotlib = try? Python.attemptImport("matplotlib"), 247 | let plt = try? Python.attemptImport("matplotlib.pyplot") 248 | else { return } 249 | matplotlib.interactive(true) 250 | let backend = "module://matplotlib_swift.backend_inline" 251 | matplotlib.rcParams["backend"] = backend.pythonObject 252 | plt.switch_backend(backend) 253 | plt.show._needmain = false 254 | let FigureCanvasBase = Python.import("matplotlib.backend_bases").FigureCanvasBase 255 | let BytesIO = Python.import("io").BytesIO 256 | let b2a_base64 = Python.import("binascii").b2a_base64 257 | let json = Python.import("json") 258 | backend_inline.show._display = 259 | PythonFunction({ (x: [PythonObject]) -> PythonConvertible in 260 | let fig = x[0] 261 | let metadata = x[1] 262 | if fig.canvas == Python.None { 263 | FigureCanvasBase(fig) 264 | } 265 | let bytesIo = BytesIO() 266 | fig.canvas.print_figure( 267 | bytesIo, format: "png", facecolor: fig.get_facecolor(), edgecolor: fig.get_edgecolor(), 268 | dpi: fig.dpi, bbox_inches: "tight") 269 | let base64EncodedPNG = String(b2a_base64(bytesIo.getvalue()).decode("ascii"))! 270 | JupyterDisplay.display( 271 | base64EncodedPNG: base64EncodedPNG, 272 | metadata: metadata == Python.None ? nil : String(json.dumps(metadata))!) 273 | return Python.None 274 | }).pythonObject 275 | matplotlibEnabled = true 276 | } 277 | #endif 278 | } 279 | 280 | struct SwiftJupyterDisplayImpl { 281 | } 282 | 283 | extension SwiftJupyterDisplayImpl { 284 | func display(base64EncodedPNG: String, metadata: String?) { 285 | let pngData = SwiftJupyterDisplay.PNGImageData(base64EncodedPNG: base64EncodedPNG) 286 | let data = SwiftJupyterDisplay.MessageContent(metadata: metadata ?? "{}", data: pngData).json 287 | SwiftJupyterDisplay.messages.append(SwiftJupyterDisplay.Message(content: data)) 288 | } 289 | 290 | func display(html: String, metadata: String?) { 291 | let htmlData = SwiftJupyterDisplay.HTMLData(html: html) 292 | let data = SwiftJupyterDisplay.MessageContent(metadata: metadata ?? "{}", data: htmlData).json 293 | SwiftJupyterDisplay.messages.append(SwiftJupyterDisplay.Message(content: data)) 294 | } 295 | 296 | func display(text: String, metadata: String?) { 297 | let textData = SwiftJupyterDisplay.TextData(text: text) 298 | let data = SwiftJupyterDisplay.MessageContent(metadata: metadata ?? "{}", data: textData).json 299 | SwiftJupyterDisplay.messages.append(SwiftJupyterDisplay.Message(content: data)) 300 | } 301 | 302 | func flush() { 303 | // Encode the messages into JSON array and push out into stdout. The StdoutHandler was enhanced 304 | // to parse special boundary chars and send display message to Jupyter accordingly. 305 | let boundary = "\(String(JupyterKernel.communicator.jupyterSession.id.reversed()))flush" 306 | let payload = 307 | "--\(boundary)[\(SwiftJupyterDisplay.messages.map({ $0.json }).joined(separator: ","))]--\(boundary)--" 308 | guard let data = payload.data(using: .utf8) else { return } 309 | try? FileHandle.standardOutput.write(contentsOf: data) 310 | SwiftJupyterDisplay.messages = [] 311 | } 312 | 313 | var isEnabled: Bool { true } 314 | 315 | var executionCount: Int { JupyterKernel.communicator.executionCount } 316 | } 317 | #if canImport(JupyterDisplay) 318 | import JupyterDisplay 319 | extension SwiftJupyterDisplayImpl: JupyterDisplayProtocol {} 320 | #else 321 | extension SwiftJupyterDisplayImpl { 322 | func display(base64EncodedPNG: String) { 323 | display(base64EncodedPNG: base64EncodedPNG, metadata: nil) 324 | } 325 | 326 | func display(html: String) { 327 | display(html: html, metadata: nil) 328 | } 329 | 330 | func display(text: String) { 331 | display(text: text, metadata: nil) 332 | } 333 | } 334 | #endif 335 | 336 | #if canImport(SwiftPlot) 337 | import SwiftPlot 338 | import AGGRenderer 339 | var __agg_renderer = AGGRenderer() 340 | extension Plot { 341 | func display(size: Size = Size(width: 1000, height: 660)) { 342 | drawGraph(size: size, renderer: __agg_renderer) 343 | JupyterDisplay.display(base64EncodedPNG: __agg_renderer.base64Png()) 344 | } 345 | } 346 | #endif 347 | 348 | #if canImport(PythonKit) 349 | import PythonKit 350 | extension PythonObject { 351 | func display() { 352 | if Bool(Python.hasattr(self, "_repr_html_")) == true { 353 | JupyterDisplay.display(html: String(self[dynamicMember: "_repr_html_"]())!) 354 | return 355 | } 356 | JupyterDisplay.display(text: description) 357 | } 358 | } 359 | #endif 360 | 361 | SwiftJupyterDisplay.enable() 362 | #if canImport(PythonKit) 363 | SwiftJupyterDisplay.enable_matplotlib() 364 | #endif 365 | #if canImport(JupyterDisplay) 366 | JupyterDisplay = SwiftJupyterDisplayImpl() 367 | #else 368 | let JupyterDisplay = SwiftJupyterDisplayImpl() 369 | #endif 370 | -------------------------------------------------------------------------------- /kernels/swift/KernelCommunicator.swift: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /// A struct with functions that the kernel and the code running inside the 16 | /// kernel use to talk to each other. 17 | /// 18 | /// Note that it would be more Jupyter-y for the communication to happen over 19 | /// ZeroMQ. This is not currently possible, because ZeroMQ sends messages 20 | /// asynchronously using IO threads, and LLDB pauses those IO threads, which 21 | /// prevents them from sending the messages. 22 | final class KernelCommunicator { 23 | private var afterSuccessfulExecutionHandlers: [() -> [JupyterDisplayMessage]] 24 | private var parentMessageHandlers: [(ParentMessage) -> Void] 25 | 26 | let jupyterSession: JupyterSession 27 | var executionCount: Int 28 | 29 | private var previousDisplayMessages: [JupyterDisplayMessage] = [] 30 | 31 | init(jupyterSession: JupyterSession) { 32 | self.afterSuccessfulExecutionHandlers = [] 33 | self.parentMessageHandlers = [] 34 | self.jupyterSession = jupyterSession 35 | self.executionCount = 0 36 | } 37 | 38 | /// Register a handler to run after the kernel successfully executes a cell 39 | /// of user code. The handler may return messages. These messages will be 40 | /// sent to the Jupyter client. 41 | func afterSuccessfulExecution( 42 | run handler: @escaping () -> [JupyterDisplayMessage] 43 | ) { 44 | afterSuccessfulExecutionHandlers.append(handler) 45 | } 46 | 47 | /// Register a handler to run when the parent message changes. 48 | func handleParentMessage(_ handler: @escaping (ParentMessage) -> Void) { 49 | parentMessageHandlers.append(handler) 50 | } 51 | 52 | /// The kernel calls this after successfully executing a cell of user code. 53 | /// Returns an array of messages, where each message is returned as an array 54 | /// of parts, where each part is returned as an address to the memory containing the part's 55 | /// bytes and a count of the number of bytes. 56 | func triggerAfterSuccessfulExecution() -> [[(address: UInt, count: Int)]] { 57 | // Keep a reference to the messages, so that their `.unsafeBufferPointer` 58 | // stays valid while the kernel is reading from them. 59 | previousDisplayMessages = afterSuccessfulExecutionHandlers.flatMap { $0() } 60 | return previousDisplayMessages.map { message in 61 | return message.parts.map { part in 62 | let b = part.unsafeBufferPointer 63 | return (address: UInt(bitPattern: b.baseAddress), count: b.count) 64 | } 65 | } 66 | } 67 | 68 | /// The kernel calls this when the parent message changes. 69 | func updateParentMessage(to parentMessage: ParentMessage) { 70 | for parentMessageHandler in parentMessageHandlers { 71 | parentMessageHandler(parentMessage) 72 | } 73 | } 74 | 75 | /// A single serialized display message for the Jupyter client. 76 | /// Corresponds to a ZeroMQ "multipart message". 77 | struct JupyterDisplayMessage { 78 | let parts: [BytesReference] 79 | } 80 | 81 | /// A reference to memory containing bytes. 82 | /// 83 | /// As long as there is a strong reference to an instance, that instance's 84 | /// `unsafeBufferPointer` refers to memory containing the bytes passed to 85 | /// that instance's constructor. 86 | /// 87 | /// We use this so that we can give the kernel a memory location that it can 88 | /// read bytes from. 89 | class BytesReference { 90 | private var bytes: ContiguousArray 91 | 92 | init(_ bytes: S) where S.Element == CChar { 93 | // Construct our own array and copy `bytes` into it, so that no one 94 | // else aliases the underlying memory. 95 | self.bytes = [] 96 | self.bytes.append(contentsOf: bytes) 97 | } 98 | 99 | var unsafeBufferPointer: UnsafeBufferPointer { 100 | // We have tried very hard to make the pointer stay valid outside the 101 | // closure: 102 | // - No one else aliases the underlying memory. 103 | // - The comment on this class reminds users that the memory may become 104 | // invalid after all references to the BytesReference instance are 105 | // released. 106 | return bytes.withUnsafeBufferPointer { $0 } 107 | } 108 | } 109 | 110 | /// ParentMessage identifies the request that causes things to happen. 111 | /// This lets Jupyter, for example, know which cell to display graphics 112 | /// messages in. 113 | struct ParentMessage { 114 | let json: String 115 | } 116 | 117 | /// The data necessary to identify and sign outgoing jupyter messages. 118 | struct JupyterSession { 119 | let id: String 120 | let key: String 121 | let username: String 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /kernels/swift/matplotlib_swift/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuliu/swift-jupyter/534a9515a752d24e24dafb96de21da33f19ee512/kernels/swift/matplotlib_swift/__init__.py -------------------------------------------------------------------------------- /kernels/swift/matplotlib_swift/backend_inline.py: -------------------------------------------------------------------------------- 1 | """A matplotlib backend for publishing figures via display_data""" 2 | 3 | # Copyright (c) IPython Development Team. 4 | # Distributed under the terms of the BSD 3-Clause License. 5 | 6 | import matplotlib 7 | from matplotlib.backends.backend_agg import ( # noqa 8 | new_figure_manager, 9 | FigureCanvasAgg, 10 | new_figure_manager_given_figure, 11 | ) 12 | from matplotlib import colors 13 | from matplotlib._pylab_helpers import Gcf 14 | 15 | 16 | def show(close=None, block=None): 17 | """Show all figures as SVG/PNG payloads sent to the IPython clients. 18 | 19 | Parameters 20 | ---------- 21 | close : bool, optional 22 | If true, a ``plt.close('all')`` call is automatically issued after 23 | sending all the figures. If this is set, the figures will entirely 24 | removed from the internal list of figures. 25 | block : Not used. 26 | The `block` parameter is a Matplotlib experimental parameter. 27 | We accept it in the function signature for compatibility with other 28 | backends. 29 | """ 30 | if close is None: 31 | close = show._close_figures 32 | try: 33 | for figure_manager in Gcf.get_all_fig_managers(): 34 | show._display( 35 | figure_manager.canvas.figure, 36 | _fetch_figure_metadata(figure_manager.canvas.figure), 37 | ) 38 | finally: 39 | show._to_draw = [] 40 | # only call close('all') if any to close 41 | # close triggers gc.collect, which can be slow 42 | if close and Gcf.get_all_fig_managers(): 43 | matplotlib.pyplot.close("all") 44 | 45 | 46 | # This flag will be reset by draw_if_interactive when called 47 | show._draw_called = False 48 | # list of figures to draw when flush_figures is called 49 | show._to_draw = [] 50 | # Close all figures at the end of each cell. 51 | show._close_figures = True 52 | # Placeholder for display function. 53 | show._display = None 54 | 55 | 56 | def draw_if_interactive(): 57 | """ 58 | Is called after every pylab drawing command 59 | """ 60 | # signal that the current active figure should be sent at the end of 61 | # execution. Also sets the _draw_called flag, signaling that there will be 62 | # something to send. At the end of the code execution, a separate call to 63 | # flush_figures() will act upon these values 64 | manager = Gcf.get_active() 65 | if manager is None: 66 | return 67 | fig = manager.canvas.figure 68 | 69 | # Hack: matplotlib FigureManager objects in interacive backends (at least 70 | # in some of them) monkeypatch the figure object and add a .show() method 71 | # to it. This applies the same monkeypatch in order to support user code 72 | # that might expect `.show()` to be part of the official API of figure 73 | # objects. 74 | # For further reference: 75 | # https://github.com/ipython/ipython/issues/1612 76 | # https://github.com/matplotlib/matplotlib/issues/835 77 | 78 | if not hasattr(fig, "show"): 79 | # Queue up `fig` for display 80 | fig.show = lambda *a: show._display(fig, _fetch_figure_metadata(fig)) 81 | 82 | # If matplotlib was manually set to non-interactive mode, this function 83 | # should be a no-op (otherwise we'll generate duplicate plots, since a user 84 | # who set ioff() manually expects to make separate draw/show calls). 85 | if not matplotlib.is_interactive(): 86 | return 87 | 88 | # ensure current figure will be drawn, and each subsequent call 89 | # of draw_if_interactive() moves the active figure to ensure it is 90 | # drawn last 91 | try: 92 | show._to_draw.remove(fig) 93 | except ValueError: 94 | # ensure it only appears in the draw list once 95 | pass 96 | # Queue up the figure for drawing in next show() call 97 | show._to_draw.append(fig) 98 | show._draw_called = True 99 | 100 | 101 | def flush_figures(): 102 | """Send all figures that changed 103 | 104 | This is meant to be called automatically and will call show() if, during 105 | prior code execution, there had been any calls to draw_if_interactive. 106 | 107 | This function is meant to be used as a post_execute callback in IPython, 108 | so user-caused errors are handled with showtraceback() instead of being 109 | allowed to raise. If this function is not called from within IPython, 110 | then these exceptions will raise. 111 | """ 112 | if not show._draw_called: 113 | return 114 | 115 | if show._close_figures: 116 | # ignore the tracking, just draw and close all figures 117 | return show(True) 118 | try: 119 | # exclude any figures that were closed: 120 | active = set([fm.canvas.figure for fm in Gcf.get_all_fig_managers()]) 121 | for fig in [fig for fig in show._to_draw if fig in active]: 122 | show._display(fig, _fetch_figure_metadata(fig)) 123 | finally: 124 | # clear flags for next round 125 | show._to_draw = [] 126 | show._draw_called = False 127 | 128 | 129 | # Changes to matplotlib in version 1.2 requires a mpl backend to supply a default 130 | # figurecanvas. This is set here to a Agg canvas 131 | # See https://github.com/matplotlib/matplotlib/pull/1125 132 | FigureCanvas = FigureCanvasAgg 133 | 134 | 135 | def _fetch_figure_metadata(fig): 136 | """Get some metadata to help with displaying a figure.""" 137 | # determine if a background is needed for legibility 138 | if _is_transparent(fig.get_facecolor()): 139 | # the background is transparent 140 | ticksLight = _is_light( 141 | [ 142 | label.get_color() 143 | for axes in fig.axes 144 | for axis in (axes.xaxis, axes.yaxis) 145 | for label in axis.get_ticklabels() 146 | ] 147 | ) 148 | if ticksLight.size and (ticksLight == ticksLight[0]).all(): 149 | # there are one or more tick labels, all with the same lightness 150 | return {"needs_background": "dark" if ticksLight[0] else "light"} 151 | 152 | return None 153 | 154 | 155 | def _is_light(color): 156 | """Determines if a color (or each of a sequence of colors) is light (as 157 | opposed to dark). Based on ITU BT.601 luminance formula (see 158 | https://stackoverflow.com/a/596241).""" 159 | rgbaArr = colors.to_rgba_array(color) 160 | return rgbaArr[:, :3].dot((0.299, 0.587, 0.114)) > 0.5 161 | 162 | 163 | def _is_transparent(color): 164 | """Determine transparency from alpha.""" 165 | rgba = colors.to_rgba(color) 166 | return rgba[3] < 0.5 167 | 168 | 169 | def set_matplotlib_close(close=True): 170 | """Set whether the inline backend closes all figures automatically or not. 171 | 172 | By default, the inline backend used in the IPython Notebook will close all 173 | matplotlib figures automatically after each cell is run. This means that 174 | plots in different cells won't interfere. Sometimes, you may want to make 175 | a plot in one cell and then refine it in later cells. This can be accomplished 176 | by:: 177 | 178 | In [1]: set_matplotlib_close(False) 179 | 180 | To set this in your config files use the following:: 181 | 182 | c.InlineBackend.close_figures = False 183 | 184 | Parameters 185 | ---------- 186 | close : bool 187 | Should all matplotlib figures be automatically closed after each cell is 188 | run? 189 | """ 190 | show._close_figures = close 191 | -------------------------------------------------------------------------------- /kernels/swift/swift_kernel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import glob 18 | import json 19 | import lldb 20 | import os 21 | import stat 22 | import re 23 | import shlex 24 | import shutil 25 | import signal 26 | import string 27 | import subprocess 28 | import sys 29 | import tempfile 30 | import textwrap 31 | import time 32 | import threading 33 | import sqlite3 34 | import random 35 | 36 | from hashlib import sha256 37 | from ipykernel.kernelbase import Kernel 38 | from jupyter_client.jsonutil import squash_dates 39 | from tornado import ioloop 40 | from bazel_tools.tools.python.runfiles import runfiles 41 | 42 | 43 | class ExecutionResult: 44 | """Base class for the result of executing code.""" 45 | 46 | pass 47 | 48 | 49 | class ExecutionResultSuccess(ExecutionResult): 50 | """Base class for the result of successfully executing code.""" 51 | 52 | pass 53 | 54 | 55 | class ExecutionResultError(ExecutionResult): 56 | """Base class for the result of unsuccessfully executing code.""" 57 | 58 | def description(self): 59 | raise NotImplementedError() 60 | 61 | 62 | class SuccessWithoutValue(ExecutionResultSuccess): 63 | """The code executed successfully, and did not produce a value.""" 64 | 65 | def __repr__(self): 66 | return "SuccessWithoutValue()" 67 | 68 | 69 | class SuccessWithValue(ExecutionResultSuccess): 70 | """The code executed successfully, and produced a value.""" 71 | 72 | def __init__(self, result): 73 | self.result = result # SBValue 74 | 75 | """A description of the value, e.g. 76 | (Int) $R0 = 64""" 77 | 78 | def value_description(self): 79 | stream = lldb.SBStream() 80 | self.result.GetDescription(stream) 81 | return stream.GetData() 82 | 83 | def __repr__(self): 84 | return "SuccessWithValue(result=%s, description=%s)" % ( 85 | repr(self.result), 86 | repr(self.result.description), 87 | ) 88 | 89 | 90 | class PreprocessorError(ExecutionResultError): 91 | """There was an error preprocessing the code.""" 92 | 93 | def __init__(self, exception): 94 | self.exception = exception # PreprocessorException 95 | 96 | def description(self): 97 | return str(self.exception) 98 | 99 | def __repr__(self): 100 | return "PreprocessorError(exception=%s)" % repr(self.exception) 101 | 102 | 103 | class PreprocessorException(Exception): 104 | pass 105 | 106 | 107 | class PackageInstallException(Exception): 108 | pass 109 | 110 | 111 | class SwiftError(ExecutionResultError): 112 | """There was a compile or runtime error.""" 113 | 114 | def __init__(self, result): 115 | self.result = result # SBValue 116 | 117 | def description(self): 118 | return self.result.error.description 119 | 120 | def __repr__(self): 121 | return "SwiftError(result=%s, description=%s)" % ( 122 | repr(self.result), 123 | repr(self.description()), 124 | ) 125 | 126 | 127 | class SIGINTHandler(threading.Thread): 128 | """Interrupts currently-executing code whenever the process receives a 129 | SIGINT.""" 130 | 131 | daemon = True 132 | 133 | def __init__(self, kernel): 134 | super(SIGINTHandler, self).__init__() 135 | self.kernel = kernel 136 | 137 | def run(self): 138 | try: 139 | while True: 140 | signal.sigwait([signal.SIGINT]) 141 | self.kernel.process.SendAsyncInterrupt() 142 | except Exception as e: 143 | self.kernel.log.error("Exception in SIGINTHandler: %s" % str(e)) 144 | 145 | 146 | class StdoutHandler(threading.Thread): 147 | """Collects stdout from the Swift process and sends it to the client.""" 148 | 149 | daemon = True 150 | 151 | def __init__(self, kernel): 152 | super(StdoutHandler, self).__init__() 153 | self.kernel = kernel 154 | self.stop_event = threading.Event() 155 | self.had_stdout = False 156 | self.pending_display_message = None 157 | self.display_boundary_start = "--" + kernel.session.session[::-1] + "flush" 158 | self.display_boundary_end = "--" + kernel.session.session[::-1] + "flush--" 159 | 160 | def _get_stdout(self): 161 | while True: 162 | BUFFER_SIZE = 1000 163 | stdout_buffer = self.kernel.process.GetSTDOUT(BUFFER_SIZE) 164 | if len(stdout_buffer) == 0: 165 | break 166 | yield stdout_buffer 167 | 168 | # Sends stdout to the jupyter client, replacing the ANSI sequence for 169 | # clearing the whole display with a 'clear_output' message to the jupyter 170 | # client. 171 | def _send_stdout(self, stdout): 172 | clear_sequence = "\033[2J" 173 | clear_sequence_index = stdout.find(clear_sequence) 174 | if clear_sequence_index != -1: 175 | self._send_stdout(stdout[:clear_sequence_index]) 176 | self.kernel.send_response( 177 | self.kernel.iopub_socket, "clear_output", {"wait": False} 178 | ) 179 | self._send_stdout(stdout[clear_sequence_index + len(clear_sequence) :]) 180 | else: 181 | self.kernel.send_response( 182 | self.kernel.iopub_socket, "stream", {"name": "stdout", "text": stdout} 183 | ) 184 | 185 | def _send_display_messages(self, display_messages): 186 | display_messages = json.loads(display_messages) 187 | for display_message in display_messages: 188 | display_message = list(map(lambda x: x.encode(), display_message)) 189 | self.kernel.iopub_socket.send_multipart(display_message) 190 | 191 | def _get_and_send_stdout(self): 192 | stdout = "".join([buf for buf in self._get_stdout()]) 193 | while len(stdout) > 0: 194 | if self.pending_display_message is not None: 195 | loc = stdout.find(self.display_boundary_end) 196 | if loc == -1: 197 | self.pending_display_message += stdout 198 | stdout = "" 199 | else: 200 | self.had_stdout = True 201 | self._send_display_messages( 202 | self.pending_display_message + stdout[:loc] 203 | ) 204 | self.pending_display_message = None 205 | stdout = stdout[loc + len(self.display_boundary_end) :] 206 | else: 207 | loc = stdout.find(self.display_boundary_start) 208 | if loc == -1: 209 | self.had_stdout = True 210 | self._send_stdout(stdout) 211 | stdout = "" 212 | else: 213 | if loc > 0: 214 | self.had_stdout = True 215 | self._send_stdout(stdout[:loc]) 216 | self.pending_display_message = "" 217 | stdout = stdout[loc + len(self.display_boundary_start) :] 218 | 219 | def run(self): 220 | try: 221 | while True: 222 | if self.stop_event.wait(0.1): 223 | break 224 | self._get_and_send_stdout() 225 | self._get_and_send_stdout() 226 | except Exception as e: 227 | self.kernel.log.error("Exception in StdoutHandler: %s" % str(e)) 228 | 229 | 230 | def jsonrpc_request(method, stdin, params=None): 231 | if "request_id" not in jsonrpc_request.__dict__: 232 | jsonrpc_request.request_id = 0 233 | jsonrpc_request.request_id += 1 234 | if params is None: 235 | obj = {"jsonrpc": "2.0", "method": method, "id": jsonrpc_request.request_id} 236 | else: 237 | obj = { 238 | "jsonrpc": "2.0", 239 | "method": method, 240 | "id": jsonrpc_request.request_id, 241 | "params": params, 242 | } 243 | jsonstr = json.dumps(obj) 244 | stdin.write( 245 | ("Content-Length: {}\r\n\r\n{}".format(len(jsonstr), jsonstr)).encode("utf-8") 246 | ) 247 | stdin.flush() 248 | return jsonrpc_request.request_id 249 | 250 | 251 | def jsonrpc_notification(method, stdin, params=None): 252 | obj = {"jsonrpc": "2.0", "method": method, "params": params} 253 | jsonstr = json.dumps(obj) 254 | try: 255 | stdin.write( 256 | ("Content-Length: {}\r\n\r\n{}".format(len(jsonstr), jsonstr)).encode( 257 | "utf-8" 258 | ) 259 | ) 260 | stdin.flush() 261 | except Exception as e: 262 | pass 263 | 264 | 265 | def jsonrpc_read_reply(request_id, stdout): 266 | while True: 267 | length = stdout.readline().decode("utf-8").strip() 268 | if length.startswith("Content-Length"): 269 | length = int(length.split(":")[1]) 270 | jsonstr = stdout.read(2 + length).decode("utf-8").strip() 271 | payload = json.loads(jsonstr) 272 | if "id" in payload and payload["id"] == request_id: 273 | return payload 274 | 275 | 276 | class SwiftKernel(Kernel): 277 | implementation = "SwiftKernel" 278 | implementation_version = "0.1" 279 | banner = "" 280 | 281 | language_info = { 282 | "name": "swift", 283 | "mimetype": "text/x-swift", 284 | "file_extension": ".swift", 285 | "version": "", 286 | } 287 | 288 | def __init__(self, **kwargs): 289 | super(SwiftKernel, self).__init__(**kwargs) 290 | 291 | # We don't initialize Swift yet, so that the user has a chance to 292 | # "%install" packages before Swift starts. (See doc comment in 293 | # `_init_swift`). 294 | 295 | # Whether to do code completion. Since the debugger is not yet 296 | # initialized, we can't do code completion yet. 297 | self.lldb_completion_enabled = False 298 | self.lsp_completion_enabled = False 299 | self.sourcekit_lsp = None 300 | self.lsp_code_version = 1 301 | self.lsp_executed_lines = 0 302 | self.lsp_current_cell_code = "" 303 | 304 | def _init_swift(self): 305 | """Initializes Swift so that it's ready to start executing user code. 306 | 307 | This must happen after package installation, because the ClangImporter 308 | does not see modulemap files that appear after it has started.""" 309 | 310 | self._init_repl_process() 311 | self._init_kernel_communicator() 312 | self._init_int_bitwidth() 313 | self._init_sigint_handler() 314 | self._init_sourcekit_lsp() 315 | 316 | # We do completion by default when the toolchain has the 317 | # SBTarget.CompleteCode API. 318 | # The user can disable/enable using "%disableCompletion" and 319 | # "%enableCompletion". 320 | self.lldb_completion_enabled = hasattr(self.target, "CompleteCode") 321 | 322 | def _init_repl_process(self): 323 | self.debugger = lldb.SBDebugger.Create() 324 | if not self.debugger: 325 | raise Exception("Could not start debugger") 326 | self.debugger.SetAsync(False) 327 | 328 | if hasattr(self, "swift_module_search_paths"): 329 | for swift_module_search_path in self.swift_module_search_paths: 330 | self.debugger.HandleCommand( 331 | "settings append target.swift-module-search-paths " 332 | + swift_module_search_path 333 | ) 334 | 335 | # LLDB crashes while trying to load some Python stuff on Mac. Maybe 336 | # something is misconfigured? This works around the problem by telling 337 | # LLDB not to load the Python scripting stuff, which we don't use 338 | # anyways. 339 | self.debugger.SetScriptLanguage(lldb.eScriptLanguageNone) 340 | 341 | repl_swift = os.environ["REPL_SWIFT_PATH"] 342 | self.target = self.debugger.CreateTargetWithFileAndArch( 343 | repl_swift, lldb.LLDB_ARCH_DEFAULT 344 | ) 345 | if not self.target: 346 | raise Exception("Could not create target %s" % repl_swift) 347 | 348 | self.main_bp = self.target.BreakpointCreateByName( 349 | "repl_main", self.target.GetExecutable().GetFilename() 350 | ) 351 | if not self.main_bp: 352 | raise Exception("Could not set breakpoint") 353 | 354 | repl_env = [] 355 | repl_env.append("PYTHONPATH=%s" % os.environ["PYTHONPATH"]) 356 | env_var_blacklist = ["PYTHONPATH", "REPL_SWIFT_PATH"] 357 | for key in os.environ: 358 | if key in env_var_blacklist: 359 | continue 360 | repl_env.append("%s=%s" % (key, os.environ[key])) 361 | 362 | # Turn off "disable ASLR" because it uses the "personality" syscall in 363 | # a way that is forbidden by the default Docker security policy. 364 | launch_info = self.target.GetLaunchInfo() 365 | launch_flags = launch_info.GetLaunchFlags() 366 | launch_info.SetLaunchFlags(launch_flags & ~lldb.eLaunchFlagDisableASLR) 367 | self.target.SetLaunchInfo(launch_info) 368 | 369 | self.process = self.target.LaunchSimple(None, repl_env, os.getcwd()) 370 | if not self.process: 371 | raise Exception("Could not launch process") 372 | 373 | self.expr_opts = lldb.SBExpressionOptions() 374 | self.swift_language = lldb.SBLanguageRuntime.GetLanguageTypeFromString("swift") 375 | self.expr_opts.SetLanguage(self.swift_language) 376 | self.expr_opts.SetREPLMode(True) 377 | self.expr_opts.SetUnwindOnError(False) 378 | self.expr_opts.SetGenerateDebugInfo(True) 379 | 380 | # Sets an infinite timeout so that users can run aribtrarily long 381 | # computations. 382 | self.expr_opts.SetTimeoutInMicroSeconds(0) 383 | 384 | self.main_thread = self.process.GetThreadAtIndex(0) 385 | 386 | def _init_kernel_communicator(self): 387 | result = self._preprocess_and_execute('%include "KernelCommunicator.swift"') 388 | if isinstance(result, ExecutionResultError): 389 | raise Exception("Error initing KernelCommunicator: %s" % result) 390 | 391 | session_key = self.session.key.decode("utf8") 392 | decl_code = """ 393 | enum JupyterKernel { 394 | static var communicator = KernelCommunicator( 395 | jupyterSession: KernelCommunicator.JupyterSession( 396 | id: %s, key: %s, username: %s)) 397 | } 398 | """ % ( 399 | json.dumps(self.session.session), 400 | json.dumps(session_key), 401 | json.dumps(self.session.username), 402 | ) 403 | result = self._preprocess_and_execute(decl_code) 404 | if isinstance(result, ExecutionResultError): 405 | raise Exception("Error declaring JupyterKernel: %s" % result) 406 | 407 | def _init_int_bitwidth(self): 408 | result = self._execute("Int.bitWidth") 409 | if not isinstance(result, SuccessWithValue): 410 | raise Exception("Expected value from Int.bitWidth, but got: %s" % result) 411 | self._int_bitwidth = int( 412 | result.result.GetData().GetSignedInt32(lldb.SBError(), 0) 413 | ) 414 | 415 | def _init_sigint_handler(self): 416 | self.sigint_handler = SIGINTHandler(self) 417 | self.sigint_handler.start() 418 | 419 | def _init_sourcekit_lsp(self): 420 | sourcekit_lsp = os.environ["SOURCEKIT_LSP_PATH"] 421 | if sourcekit_lsp is None: 422 | return 423 | self.sourcekit_lsp = subprocess.Popen( 424 | [sourcekit_lsp], stdin=subprocess.PIPE, stdout=subprocess.PIPE 425 | ) 426 | self.sourcekit_lsp_workspace = tempfile.mkdtemp("-sklsp") 427 | arguments = ["swiftc"] 428 | if hasattr(self, "swift_module_search_paths"): 429 | for swift_module_search_path in self.swift_module_search_paths: 430 | arguments.append("-I" + swift_module_search_path) 431 | arguments.extend([os.path.join(self.sourcekit_lsp_workspace, "main.swift")]) 432 | compile_commands = [ 433 | { 434 | "arguments": arguments, 435 | "directory": self.sourcekit_lsp_workspace, 436 | "file": os.path.join(self.sourcekit_lsp_workspace, "main.swift"), 437 | } 438 | ] 439 | with open( 440 | os.path.join(self.sourcekit_lsp_workspace, "compile_commands.json"), "w+" 441 | ) as f: 442 | json.dump(compile_commands, f, sort_keys=True, indent=2) 443 | jsonrpc_request( 444 | "initialize", 445 | self.sourcekit_lsp.stdin, 446 | { 447 | "rootUri": "file://{}".format(self.sourcekit_lsp_workspace), 448 | "capabilities": { 449 | "textDocument": { 450 | "completion": {"completionItemKind": {"valueSet": [1, 2]}} 451 | } 452 | }, 453 | }, 454 | ) 455 | jsonrpc_notification("initialized", self.sourcekit_lsp.stdin) 456 | jsonrpc_notification( 457 | "textDocument/didOpen", 458 | self.sourcekit_lsp.stdin, 459 | { 460 | "textDocument": { 461 | "uri": "file://{}".format( 462 | os.path.join(self.sourcekit_lsp_workspace, "main.swift") 463 | ), 464 | "languageId": "swift", 465 | "version": 1, 466 | "text": "", 467 | } 468 | }, 469 | ) 470 | self.lsp_completion_enabled = True 471 | 472 | def _init_jupyter_display(self): 473 | result = self._preprocess_and_execute('%include "EnableJupyterDisplay.swift"') 474 | if isinstance(result, ExecutionResultError): 475 | raise Exception("Error initing JupyterDisplay: %s" % result) 476 | 477 | def _file_name_for_source_location(self): 478 | return "" % self.execution_count 479 | 480 | def _lsp_did_change(self, code): 481 | if self.sourcekit_lsp is None: 482 | return 483 | self.lsp_code_version += 1 484 | jsonrpc_notification( 485 | "textDocument/didChange", 486 | self.sourcekit_lsp.stdin, 487 | { 488 | "textDocument": { 489 | "uri": "file://{}".format( 490 | os.path.join(self.sourcekit_lsp_workspace, "main.swift") 491 | ), 492 | "version": self.lsp_code_version, 493 | }, 494 | "contentChanges": [ 495 | { 496 | "range": { 497 | "start": {"line": self.lsp_executed_lines, "character": 0}, 498 | "end": { 499 | "line": self.lsp_executed_lines 500 | + self.lsp_current_cell_code.count("\n"), 501 | "character": len(self.lsp_current_cell_code) 502 | - self.lsp_current_cell_code.rfind("\n") 503 | - 1, 504 | }, 505 | }, 506 | "text": code, 507 | } 508 | ], 509 | }, 510 | ) 511 | self.lsp_current_cell_code = code 512 | 513 | def _lsp_did_execute(self, code): 514 | if self.sourcekit_lsp is None: 515 | return 516 | code = code.rstrip() + "\n" # Add new line if needed. 517 | self._lsp_did_change(code) 518 | self.lsp_executed_lines += code.count("\n") 519 | self.lsp_current_cell_code = "" 520 | 521 | def _preprocess_and_execute(self, code): 522 | try: 523 | preprocessed = self._preprocess(code) 524 | except PreprocessorException as e: 525 | return PreprocessorError(e) 526 | 527 | result = self._execute(preprocessed) 528 | self._lsp_did_execute(preprocessed) 529 | return result 530 | 531 | def _preprocess(self, code): 532 | lines = code.split("\n") 533 | preprocessed_lines = [ 534 | self._preprocess_line(i, line) for i, line in enumerate(lines) 535 | ] 536 | return "\n".join(preprocessed_lines) 537 | 538 | def _handle_disable_completion(self): 539 | self.lldb_completion_enabled = False 540 | self.lsp_completion_enabled = False 541 | self.send_response( 542 | self.iopub_socket, 543 | "stream", 544 | {"name": "stdout", "text": "Completion disabled!\n"}, 545 | ) 546 | 547 | def _handle_enable_completion(self): 548 | if not hasattr(self.target, "CompleteCode") and self.sourcekit_lsp is None: 549 | self.send_response( 550 | self.iopub_socket, 551 | "stream", 552 | { 553 | "name": "stdout", 554 | "text": "Completion NOT enabled because toolchain does not " 555 | + "have CompleteCode API.\n", 556 | }, 557 | ) 558 | return 559 | 560 | if hasattr(self.target, "CompleteCode"): 561 | self.lldb_completion_enabled = True 562 | self.send_response( 563 | self.iopub_socket, 564 | "stream", 565 | {"name": "stdout", "text": "Completion enabled!\n"}, 566 | ) 567 | elif self.sourcekit_lsp is not None: 568 | self.lsp_completion_enabled = True 569 | self.send_response( 570 | self.iopub_socket, 571 | "stream", 572 | {"name": "stdout", "text": "Completion enabled!\n"}, 573 | ) 574 | 575 | def _preprocess_line(self, line_index, line): 576 | """Returns the preprocessed line. 577 | 578 | Does not process "%install" directives, because those need to be 579 | handled before everything else.""" 580 | 581 | include_match = re.match(r"^\s*%include (.*)$", line) 582 | if include_match is not None: 583 | return self._read_include(line_index, include_match.group(1)) 584 | 585 | disable_completion_match = re.match(r"^\s*%disableCompletion\s*$", line) 586 | if disable_completion_match is not None: 587 | self._handle_disable_completion() 588 | return "" 589 | 590 | enable_completion_match = re.match(r"^\s*%enableCompletion\s*$", line) 591 | if enable_completion_match is not None: 592 | self._handle_enable_completion() 593 | return "" 594 | 595 | return line 596 | 597 | def _read_include(self, line_index, rest_of_line): 598 | name_match = re.match(r'^\s*"([^"]+)"\s*$', rest_of_line) 599 | if name_match is None: 600 | raise PreprocessorException( 601 | "Line %d: %%include must be followed by a name in quotes" 602 | % (line_index + 1) 603 | ) 604 | name = name_match.group(1) 605 | 606 | include_paths = [ 607 | os.path.dirname(os.path.realpath(sys.argv[0])), 608 | os.path.realpath("."), 609 | ] 610 | 611 | code = None 612 | for include_path in include_paths: 613 | try: 614 | with open(os.path.join(include_path, name), "r") as f: 615 | code = f.read() 616 | except IOError: 617 | continue 618 | 619 | if code is None: 620 | raise PreprocessorException( 621 | 'Line %d: Could not find "%s". Searched %s.' 622 | % (line_index + 1, name, include_paths) 623 | ) 624 | 625 | return "\n".join( 626 | [ 627 | '#sourceLocation(file: "%s", line: 1)' % name, 628 | code, 629 | '#sourceLocation(file: "%s", line: %d)' 630 | % (self._file_name_for_source_location(), line_index + 1), 631 | "", 632 | ] 633 | ) 634 | 635 | def _process_installs(self, code): 636 | """Handles all "%install" directives, and returns `code` with all 637 | "%install" directives removed.""" 638 | processed_lines = [] 639 | all_packages = [] 640 | all_swiftpm_flags = [] 641 | extra_include_commands = [] 642 | all_bazel_deps = [] 643 | user_install_location = None 644 | for index, line in enumerate(code.split("\n")): 645 | line = self._process_system_command_line(line) 646 | line, install_location = self._process_install_location_line(line) 647 | line, swiftpm_flags = self._process_install_swiftpm_flags_line(line) 648 | all_swiftpm_flags += swiftpm_flags 649 | line, packages = self._process_install_line(index, line) 650 | line, bazel_deps = self._process_bazel_line(index, line) 651 | line, extra_include_command = self._process_extra_include_command_line(line) 652 | if extra_include_command: 653 | extra_include_commands.append(extra_include_command) 654 | processed_lines.append(line) 655 | all_packages += packages 656 | all_bazel_deps += bazel_deps 657 | if install_location: 658 | user_install_location = install_location 659 | 660 | if len(all_packages) > 0 or len(all_swiftpm_flags) > 0: 661 | if len(all_bazel_deps) > 0: 662 | raise PackageInstallException( 663 | "Cannot install SwiftPM packages at the same time with Bazel packages." 664 | ) 665 | self._install_packages( 666 | all_packages, 667 | all_swiftpm_flags, 668 | extra_include_commands, 669 | user_install_location, 670 | ) 671 | self._init_jupyter_display() 672 | elif len(all_bazel_deps) > 0: 673 | if len(extra_include_commands) > 0: 674 | raise PackageInstallException( 675 | "Cannot specify extra include commands %s for Bazel packages." 676 | % extra_include_commands 677 | ) 678 | if user_install_location is not None: 679 | raise PackageInstallException( 680 | "Cannot specify user install location %s for Bazel packages." 681 | % user_install_location 682 | ) 683 | self._install_bazel_deps(all_bazel_deps) 684 | self._init_jupyter_display() 685 | return "\n".join(processed_lines) 686 | 687 | def _process_install_location_line(self, line): 688 | install_location_match = re.match(r"^\s*%install-location (.*)$", line) 689 | if install_location_match is None: 690 | return line, None 691 | 692 | install_location = install_location_match.group(1) 693 | try: 694 | install_location = string.Template(install_location).substitute( 695 | {"cwd": os.getcwd()} 696 | ) 697 | except KeyError as e: 698 | raise PackageInstallException( 699 | "Line %d: Invalid template argument %s" % (line_index + 1, str(e)) 700 | ) 701 | except ValueError as e: 702 | raise PackageInstallException("Line %d: %s" % (line_index + 1, str(e))) 703 | 704 | return "", install_location 705 | 706 | def _process_extra_include_command_line(self, line): 707 | extra_include_command_match = re.match( 708 | r"^\s*%install-extra-include-command (.*)$", line 709 | ) 710 | if extra_include_command_match is None: 711 | return line, None 712 | 713 | extra_include_command = extra_include_command_match.group(1) 714 | 715 | return "", extra_include_command 716 | 717 | def _process_install_swiftpm_flags_line(self, line): 718 | install_swiftpm_flags_match = re.match( 719 | r"^\s*%install-swiftpm-flags (.*)$", line 720 | ) 721 | if install_swiftpm_flags_match is None: 722 | return line, [] 723 | flags = shlex.split(install_swiftpm_flags_match.group(1)) 724 | return "", flags 725 | 726 | def _process_bazel_line(self, line_index, line): 727 | bazel_match = re.match(r"^\s*%bazel\s+(.*)$", line) 728 | if bazel_match is None: 729 | return line, [] 730 | path = bazel_match.group(1).strip("\"'") 731 | return "", [path] 732 | 733 | def _process_install_line(self, line_index, line): 734 | install_match = re.match(r"^\s*%install (.*)$", line) 735 | if install_match is None: 736 | return line, [] 737 | 738 | parsed = shlex.split(install_match.group(1)) 739 | if len(parsed) < 2: 740 | raise PackageInstallException( 741 | "Line %d: %%install usage: SPEC PRODUCT [PRODUCT ...]" 742 | % (line_index + 1) 743 | ) 744 | try: 745 | spec = string.Template(parsed[0]).substitute({"cwd": os.getcwd()}) 746 | except KeyError as e: 747 | raise PackageInstallException( 748 | "Line %d: Invalid template argument %s" % (line_index + 1, str(e)) 749 | ) 750 | except ValueError as e: 751 | raise PackageInstallException("Line %d: %s" % (line_index + 1, str(e))) 752 | 753 | return "", [ 754 | { 755 | "spec": spec, 756 | "products": parsed[1:], 757 | } 758 | ] 759 | 760 | def _process_system_command_line(self, line): 761 | system_match = re.match(r"^\s*%system (.*)$", line) 762 | if system_match is None: 763 | return line 764 | 765 | if hasattr(self, "debugger"): 766 | raise PackageInstallException( 767 | "System commands can only run in the first cell." 768 | ) 769 | 770 | rest_of_line = system_match.group(1) 771 | process = subprocess.Popen( 772 | rest_of_line, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True 773 | ) 774 | process.wait() 775 | command_result = process.stdout.read().decode("utf-8") 776 | self.send_response( 777 | self.iopub_socket, 778 | "stream", 779 | {"name": "stdout", "text": "%s" % command_result}, 780 | ) 781 | return "" 782 | 783 | def _link_extra_includes(self, swift_module_search_path, include_dir): 784 | for include_file in os.listdir(include_dir): 785 | link_name = os.path.join(swift_module_search_path, include_file) 786 | target = os.path.join(include_dir, include_file) 787 | try: 788 | if stat.S_ISLNK(os.lstat(link_name).st_mode): 789 | os.unlink(link_name) 790 | except FileNotFoundError as e: 791 | pass 792 | except Error as e: 793 | raise PackageInstallException( 794 | "Failed to stat scratchwork base path: %s" % str(e) 795 | ) 796 | os.symlink(target, link_name) 797 | 798 | def _dlopen_shared_lib(self, lib_filename): 799 | dynamic_load_code = textwrap.dedent( 800 | """\ 801 | import func Glibc.dlopen 802 | import var Glibc.RTLD_NOW 803 | dlopen(%s, RTLD_NOW) 804 | """ 805 | % json.dumps(lib_filename) 806 | ) 807 | dynamic_load_result = self._execute(dynamic_load_code) 808 | if not isinstance(dynamic_load_result, SuccessWithValue): 809 | raise PackageInstallException( 810 | "Install Error: dlopen error: %s" % str(dynamic_load_result) 811 | ) 812 | if dynamic_load_result.value_description().endswith("nil"): 813 | raise PackageInstallException( 814 | "Install Error: dlopen error. Run " 815 | "`String(cString: dlerror())` to see " 816 | "the error message." 817 | ) 818 | 819 | def _install_packages( 820 | self, packages, swiftpm_flags, extra_include_commands, user_install_location 821 | ): 822 | if len(packages) == 0 and len(swiftpm_flags) == 0: 823 | return 824 | 825 | if hasattr(self, "debugger"): 826 | raise PackageInstallException( 827 | "Install Error: Packages can only be installed during the " 828 | "first cell execution. Restart the kernel to install " 829 | "packages." 830 | ) 831 | 832 | swift_build_path = os.environ.get("SWIFT_BUILD_PATH") 833 | if swift_build_path is None: 834 | raise PackageInstallException( 835 | "Install Error: Cannot install packages because " 836 | "SWIFT_BUILD_PATH is not specified." 837 | ) 838 | 839 | swift_package_path = os.environ.get("SWIFT_PACKAGE_PATH") 840 | if swift_package_path is None: 841 | raise PackageInstallException( 842 | "Install Error: Cannot install packages because " 843 | "SWIFT_PACKAGE_PATH is not specified." 844 | ) 845 | 846 | package_install_scratchwork_base = tempfile.mkdtemp() 847 | package_install_scratchwork_base = os.path.join( 848 | package_install_scratchwork_base, "swift-install" 849 | ) 850 | swift_module_search_path = os.path.join( 851 | package_install_scratchwork_base, "modules" 852 | ) 853 | self.swift_module_search_paths = [swift_module_search_path] 854 | 855 | scratchwork_base_path = os.path.dirname(swift_module_search_path) 856 | package_base_path = os.path.join(scratchwork_base_path, "package") 857 | 858 | # If the user has specified a custom install location, make a link from 859 | # the scratchwork base path to it. 860 | if user_install_location is not None: 861 | # symlink to the specified location 862 | # Remove existing base if it is already a symlink 863 | os.makedirs(user_install_location, exist_ok=True) 864 | try: 865 | if stat.S_ISLNK(os.lstat(scratchwork_base_path).st_mode): 866 | os.unlink(scratchwork_base_path) 867 | except FileNotFoundError as e: 868 | pass 869 | except Error as e: 870 | raise PackageInstallException( 871 | "Failed to stat scratchwork base path: %s" % str(e) 872 | ) 873 | os.symlink( 874 | user_install_location, scratchwork_base_path, target_is_directory=True 875 | ) 876 | 877 | # Make the directory containing our synthesized package. 878 | os.makedirs(package_base_path, exist_ok=True) 879 | 880 | # Make the directory containing our built modules and other includes. 881 | os.makedirs(swift_module_search_path, exist_ok=True) 882 | 883 | # Make links from the install location to extra includes. 884 | for include_command in extra_include_commands: 885 | result = subprocess.run( 886 | include_command, 887 | shell=True, 888 | stdout=subprocess.PIPE, 889 | stderr=subprocess.PIPE, 890 | ) 891 | if result.returncode != 0: 892 | raise PackageInstallException( 893 | "%%install-extra-include-command returned nonzero " 894 | "exit code: %d\nStdout:\n%s\nStderr:\n%s\n" 895 | % ( 896 | result.returncode, 897 | result.stdout.decode("utf8"), 898 | result.stderr.decode("utf8"), 899 | ) 900 | ) 901 | include_dirs = shlex.split(result.stdout.decode("utf8")) 902 | for include_dir in include_dirs: 903 | if include_dir[0:2] != "-I": 904 | self.log.warn( 905 | 'Non "-I" output from ' 906 | "%%install-extra-include-command: %s" % include_dir 907 | ) 908 | continue 909 | include_dir = include_dir[2:] 910 | self._link_extra_includes(swift_module_search_path, include_dir) 911 | 912 | # Summary of how this works: 913 | # - create a SwiftPM package that depends on all the packages that 914 | # the user requested 915 | # - ask SwiftPM to build that package 916 | # - copy all the .swiftmodule and module.modulemap files that SwiftPM 917 | # created to SWIFT_IMPORT_SEARCH_PATH 918 | # - dlopen the .so file that SwiftPM created 919 | 920 | # == Create the SwiftPM package == 921 | 922 | package_swift_template = textwrap.dedent( 923 | """\ 924 | // swift-tools-version:4.2 925 | import PackageDescription 926 | let package = Package( 927 | name: "jupyterInstalledPackages", 928 | products: [ 929 | .library( 930 | name: "jupyterInstalledPackages", 931 | type: .dynamic, 932 | targets: ["jupyterInstalledPackages"]), 933 | ], 934 | dependencies: [%s], 935 | targets: [ 936 | .target( 937 | name: "jupyterInstalledPackages", 938 | dependencies: [%s], 939 | path: ".", 940 | sources: ["jupyterInstalledPackages.swift"]), 941 | ]) 942 | """ 943 | ) 944 | 945 | packages_specs = "" 946 | packages_products = "" 947 | packages_human_description = "" 948 | for package in packages: 949 | packages_specs += "%s,\n" % package["spec"] 950 | packages_human_description += "\t%s\n" % package["spec"] 951 | for target in package["products"]: 952 | packages_products += "%s,\n" % json.dumps(target) 953 | packages_human_description += "\t\t%s\n" % target 954 | 955 | self.send_response( 956 | self.iopub_socket, 957 | "stream", 958 | { 959 | "name": "stdout", 960 | "text": "Installing packages:\n%s" % packages_human_description, 961 | }, 962 | ) 963 | self.send_response( 964 | self.iopub_socket, 965 | "stream", 966 | {"name": "stdout", "text": "With SwiftPM flags: %s\n" % str(swiftpm_flags)}, 967 | ) 968 | self.send_response( 969 | self.iopub_socket, 970 | "stream", 971 | {"name": "stdout", "text": "Working in: %s\n" % scratchwork_base_path}, 972 | ) 973 | 974 | package_swift = package_swift_template % (packages_specs, packages_products) 975 | 976 | with open("%s/Package.swift" % package_base_path, "w") as f: 977 | f.write(package_swift) 978 | with open("%s/jupyterInstalledPackages.swift" % package_base_path, "w") as f: 979 | f.write("// intentionally blank\n") 980 | 981 | # == Ask SwiftPM to build the package == 982 | 983 | # TODO(TF-1179): Remove this workaround after fixing SwiftPM. 984 | swiftpm_env = os.environ 985 | libuuid_path = "/lib/x86_64-linux-gnu/libuuid.so.1" 986 | if os.path.isfile(libuuid_path): 987 | swiftpm_env["LD_PRELOAD"] = libuuid_path 988 | 989 | build_p = subprocess.Popen( 990 | [swift_build_path] + swiftpm_flags, 991 | stdout=subprocess.PIPE, 992 | stderr=subprocess.STDOUT, 993 | cwd=package_base_path, 994 | env=swiftpm_env, 995 | ) 996 | for build_output_line in iter(build_p.stdout.readline, b""): 997 | self.send_response( 998 | self.iopub_socket, 999 | "stream", 1000 | {"name": "stdout", "text": build_output_line.decode("utf8")}, 1001 | ) 1002 | build_returncode = build_p.wait() 1003 | if build_returncode != 0: 1004 | raise PackageInstallException( 1005 | "Install Error: swift-build returned nonzero exit code " 1006 | "%d." % build_returncode 1007 | ) 1008 | 1009 | show_bin_path_result = subprocess.run( 1010 | [swift_build_path, "--show-bin-path"] + swiftpm_flags, 1011 | stdout=subprocess.PIPE, 1012 | stderr=subprocess.PIPE, 1013 | cwd=package_base_path, 1014 | env=swiftpm_env, 1015 | ) 1016 | bin_dir = show_bin_path_result.stdout.decode("utf8").strip() 1017 | lib_filename = os.path.join(bin_dir, "libjupyterInstalledPackages.so") 1018 | 1019 | # == Copy .swiftmodule and modulemap files to SWIFT_IMPORT_SEARCH_PATH == 1020 | 1021 | # Search for build.db. 1022 | build_db_candidates = [ 1023 | os.path.join(bin_dir, "..", "build.db"), 1024 | os.path.join(package_base_path, ".build", "build.db"), 1025 | ] 1026 | build_db_file = next(filter(os.path.exists, build_db_candidates), None) 1027 | if build_db_file is None: 1028 | raise PackageInstallException("build.db is missing") 1029 | 1030 | # Execute swift-package show-dependencies to get all dependencies' paths 1031 | dependencies_result = subprocess.run( 1032 | [swift_package_path, "show-dependencies", "--format", "json"], 1033 | stdout=subprocess.PIPE, 1034 | stderr=subprocess.PIPE, 1035 | cwd=package_base_path, 1036 | env=swiftpm_env, 1037 | ) 1038 | dependencies_json = dependencies_result.stdout.decode("utf8") 1039 | dependencies_obj = json.loads(dependencies_json) 1040 | 1041 | def flatten_deps_paths(dep): 1042 | paths = [] 1043 | paths.append(dep["path"]) 1044 | if dep["dependencies"]: 1045 | for d in dep["dependencies"]: 1046 | paths.extend(flatten_deps_paths(d)) 1047 | return paths 1048 | 1049 | # Make list of paths where we expect .swiftmodule and .modulemap files of dependencies 1050 | dependencies_paths = [package_base_path] 1051 | dependencies_paths = flatten_deps_paths(dependencies_obj) 1052 | dependencies_paths = list(set(dependencies_paths)) 1053 | 1054 | def is_valid_dependency(path): 1055 | for p in dependencies_paths: 1056 | if path.startswith(p): 1057 | return True 1058 | return False 1059 | 1060 | # Query to get build files list from build.db 1061 | # SUBSTR because string starts with "N" (why?) 1062 | SQL_FILES_SELECT = "SELECT SUBSTR(key, 2) FROM 'key_names' WHERE key LIKE ?" 1063 | 1064 | # Connect to build.db 1065 | db_connection = sqlite3.connect(build_db_file) 1066 | cursor = db_connection.cursor() 1067 | 1068 | # Process *.swiftmodules files 1069 | cursor.execute(SQL_FILES_SELECT, ["%.swiftmodule"]) 1070 | swift_modules = [ 1071 | row[0] for row in cursor.fetchall() if is_valid_dependency(row[0]) 1072 | ] 1073 | for filename in swift_modules: 1074 | shutil.copy(filename, swift_module_search_path) 1075 | 1076 | # Process modulemap files 1077 | cursor.execute(SQL_FILES_SELECT, ["%/module.modulemap"]) 1078 | modulemap_files = [ 1079 | row[0] for row in cursor.fetchall() if is_valid_dependency(row[0]) 1080 | ] 1081 | for index, filename in enumerate(modulemap_files): 1082 | # Create a separate directory for each modulemap file because the 1083 | # ClangImporter requires that they are all named 1084 | # "module.modulemap". 1085 | # Use the module name to prevent two modulemaps for the same 1086 | # depndency ending up in multiple directories after several 1087 | # installations, causing the kernel to end up in a bad state. 1088 | # Make all relative header paths in module.modulemap absolute 1089 | # because we copy file to different location. 1090 | 1091 | src_folder, src_filename = os.path.split(filename) 1092 | with open(filename, encoding="utf8") as file: 1093 | modulemap_contents = file.read() 1094 | modulemap_contents = re.sub( 1095 | r'header\s+"(.*?)"', 1096 | lambda m: 'header "%s"' 1097 | % ( 1098 | m.group(1) 1099 | if os.path.isabs(m.group(1)) 1100 | else os.path.abspath(os.path.join(src_folder, m.group(1))) 1101 | ), 1102 | modulemap_contents, 1103 | ) 1104 | 1105 | module_match = re.match(r"module\s+([^\s]+)\s.*{", modulemap_contents) 1106 | module_name = ( 1107 | module_match.group(1) if module_match is not None else str(index) 1108 | ) 1109 | modulemap_dest = os.path.join( 1110 | swift_module_search_path, "modulemap-%s" % module_name 1111 | ) 1112 | os.makedirs(modulemap_dest, exist_ok=True) 1113 | dst_path = os.path.join(modulemap_dest, src_filename) 1114 | 1115 | with open(dst_path, "w", encoding="utf8") as outfile: 1116 | outfile.write(modulemap_contents) 1117 | 1118 | # == dlopen the shared lib == 1119 | 1120 | self.send_response( 1121 | self.iopub_socket, 1122 | "stream", 1123 | {"name": "stdout", "text": "Initializing Swift...\n"}, 1124 | ) 1125 | self._init_swift() 1126 | self._dlopen_shared_lib(lib_filename) 1127 | 1128 | self.send_response( 1129 | self.iopub_socket, 1130 | "stream", 1131 | {"name": "stdout", "text": "Installation complete!\n"}, 1132 | ) 1133 | self.already_installed_packages = True 1134 | 1135 | def _install_bazel_deps(self, lines): 1136 | if len(lines) == 0: 1137 | return 1138 | 1139 | # Identify packages that matches "build --compilation_mode={dbg,opt,fastbuild}" 1140 | packages = [] 1141 | compilation_mode = "opt" # Default to optimized build 1142 | for line in lines: 1143 | compilation_mode_match = re.match( 1144 | r"^\s*build\s+--compilation_mode=(.*)$", line 1145 | ) 1146 | if compilation_mode_match is None: 1147 | packages.append(line) 1148 | continue 1149 | compilation_mode = compilation_mode_match.group(1) 1150 | if not compilation_mode in ["dbg", "opt", "fastbuild"]: 1151 | raise PackageInstallException( 1152 | "Install Error: Unrecognized compilation_mode %s" % compilation_mode 1153 | ) 1154 | 1155 | if hasattr(self, "debugger"): 1156 | raise PackageInstallException( 1157 | "Install Error: Packages can only be installed during the " 1158 | "first cell execution. Restart the kernel to install " 1159 | "packages." 1160 | ) 1161 | 1162 | # Install Bazel packages from local workspace. 1163 | # This follows how we install packages from Swift Package Manager. 1164 | # 1. We create a phantom BUILD target of libjupyterInstalledPackages.so (note 1165 | # that we need to use custom cc_whole_archive_library to make sure all 1166 | # symbols are properly preserved). 1167 | # 2. We query all swift_library targets, and try to find the *.swiftmodule 1168 | # file. If we found, add the path to the swift_module_search_paths. Note 1169 | # that we don't move them around in fear of relative directory path issues. 1170 | # 3. We query all cc_library targets, and try to find module.modulemap file. 1171 | # If we found, add to the swift_module_search_paths. 1172 | # 4. This doesn't cover some paths we simply added through -I during compilation. 1173 | # Hence, we queried the action graph to find the arguments we passed for 1174 | # each swift_library target, and add these paths to swift_module_search_paths 1175 | # as well. 1176 | # 5. After initialize Swift REPL with the swift_module_search_paths, dlopen 1177 | # previous built .so file. 1178 | # 6. And we are done! Now you can `import SwiftDependency` as you want in the 1179 | # notebook. 1180 | 1181 | pwd = os.getcwd() 1182 | workspace = ( 1183 | subprocess.check_output(["bazel", "info", "workspace"]) 1184 | .decode("utf-8") 1185 | .strip() 1186 | ) 1187 | output_base = os.path.join( 1188 | tempfile.gettempdir(), "ibzl", sha256(workspace.encode("utf-8")).hexdigest() 1189 | ) 1190 | bazel_bin_dir = ( 1191 | subprocess.check_output( 1192 | [ 1193 | "bazel", 1194 | "info", 1195 | "bazel-bin", 1196 | "--compilation_mode=%s" % compilation_mode, 1197 | "--swiftcopt=-whole-module-optimization", 1198 | ] 1199 | ) 1200 | .decode("utf-8") 1201 | .strip() 1202 | ) 1203 | execution_root = ( 1204 | subprocess.check_output(["bazel", "info", "execution_root"]) 1205 | .decode("utf-8") 1206 | .strip() 1207 | ) 1208 | # Switch to the workspace root, in this way, we can launch the notebook from whatever 1209 | # subdirectory. 1210 | os.chdir(workspace) 1211 | tmpdir = "".join(random.choice(string.ascii_letters) for i in range(5)) 1212 | bazel_dep_target_path = os.path.join(workspace, ".ibzlnb", tmpdir) 1213 | os.makedirs(bazel_dep_target_path, exist_ok=True) 1214 | self.bazel_dep_target_path = bazel_dep_target_path 1215 | 1216 | # swift_binary doesn't official support build dynamic libraries. Doing so 1217 | # by adding -shared. Note also that we use gold linker to match what 1218 | # Swift Package Manager uses. The default linker has issues with Swift symbols. 1219 | BUILD_template = textwrap.dedent( 1220 | """\ 1221 | load("@build_bazel_rules_swift//swift:swift.bzl", "swift_binary") 1222 | load(":whole_archive.bzl", "cc_whole_archive_library") 1223 | cc_whole_archive_library( 1224 | name = "jupyterInstalledPackages", 1225 | deps = [%s] 1226 | ) 1227 | swift_binary( 1228 | name = "libjupyterInstalledPackages.so", 1229 | linkopts = ["-shared", "-fuse-ld=gold"], 1230 | deps = [":jupyterInstalledPackages"] 1231 | ) 1232 | """ 1233 | ) 1234 | 1235 | packages_dep = "" 1236 | packages_human_description = "" 1237 | for package in packages: 1238 | unquoted_package = package.strip() 1239 | if len(unquoted_package) == 0: 1240 | continue 1241 | packages_dep += '"%s",\n' % unquoted_package 1242 | packages_human_description += "|-%s\n" % unquoted_package 1243 | 1244 | self.send_response( 1245 | self.iopub_socket, 1246 | "stream", 1247 | { 1248 | "name": "stdout", 1249 | "text": "Installing packages:\n%s" % packages_human_description, 1250 | }, 1251 | ) 1252 | self.send_response( 1253 | self.iopub_socket, 1254 | "stream", 1255 | {"name": "stdout", "text": "Working in: %s\n" % bazel_dep_target_path}, 1256 | ) 1257 | 1258 | BUILD = BUILD_template % (packages_dep) 1259 | r = runfiles.Create() 1260 | whole_archive_bzl = r.Rlocation("swift-jupyter/kernels/swift/whole_archive.bzl") 1261 | shutil.copy(whole_archive_bzl, bazel_dep_target_path) 1262 | with open("%s/BUILD.bazel" % bazel_dep_target_path, "w") as f: 1263 | f.write(BUILD) 1264 | subprocess.check_output( 1265 | [ 1266 | "bazel", 1267 | "build", 1268 | "//.ibzlnb/%s:libjupyterInstalledPackages.so" % tmpdir, 1269 | "--compilation_mode=%s" % compilation_mode, 1270 | "--swiftcopt=-whole-module-optimization", 1271 | "--copt=-fPIC", 1272 | ] 1273 | ) 1274 | env = dict(os.environ, CC="clang") 1275 | result = subprocess.check_output( 1276 | [ 1277 | "bazel", 1278 | "--output_base=%s" % output_base, 1279 | "query", 1280 | "kind(swift_library, deps(//.ibzlnb/%s:libjupyterInstalledPackages.so))" 1281 | % tmpdir, 1282 | ], 1283 | env=env, 1284 | ).decode("utf-8") 1285 | # Process *.swiftmodules files 1286 | swift_module_search_paths = set() 1287 | swift_libraries = result.split("\n") 1288 | for dep in swift_libraries: 1289 | dep = dep.strip() 1290 | if len(dep) == 0: 1291 | continue 1292 | result = json.loads( 1293 | subprocess.check_output( 1294 | [ 1295 | "bazel", 1296 | "aquery", 1297 | "--compilation_mode=%s" % compilation_mode, 1298 | "--swiftcopt=-whole-module-optimization", 1299 | "mnemonic(SwiftCompile, %s)" % dep, 1300 | "--output=jsonproto", 1301 | "--include_artifacts=false", 1302 | ], 1303 | env=env, 1304 | ).decode("utf-8") 1305 | ) 1306 | for action in result["actions"]: 1307 | for i, arg in enumerate(action["arguments"]): 1308 | if arg.startswith("-I"): 1309 | swift_module_search_paths.add( 1310 | os.path.join(execution_root, arg[2:]) 1311 | ) 1312 | elif arg.startswith("-iquote"): 1313 | swift_module_search_paths.add( 1314 | os.path.join(execution_root, arg[7:]) 1315 | ) 1316 | elif arg.startswith("-isystem"): 1317 | swift_module_search_paths.add( 1318 | os.path.join(execution_root, arg[8:]) 1319 | ) 1320 | elif arg == "-emit-module-path": 1321 | swift_module = action["arguments"][i + 1] 1322 | swift_module = os.path.join(execution_root, swift_module) 1323 | if os.path.exists(swift_module): 1324 | swift_module_search_paths.add(os.path.dirname(swift_module)) 1325 | self.send_response( 1326 | self.iopub_socket, 1327 | "stream", 1328 | {"name": "stdout", "text": "swiftmodule: %s\n" % swift_module}, 1329 | ) 1330 | result = subprocess.check_output( 1331 | [ 1332 | "bazel", 1333 | "--output_base=%s" % output_base, 1334 | "query", 1335 | 'attr(tags, "swift_module=\\w+", kind(cc_library, deps(//.ibzlnb/%s:libjupyterInstalledPackages.so)))' 1336 | % tmpdir, 1337 | ], 1338 | env=env, 1339 | ).decode("utf-8") 1340 | swift_module_libraries = swift_libraries + list( 1341 | map( 1342 | lambda x: x + ".swift", 1343 | filter(lambda x: len(x.strip()) > 0, result.split("\n")), 1344 | ) 1345 | ) 1346 | # Process *.modulemap 1347 | # First, loop through all module.modulemap from existing swift_module_search_paths, make sure 1348 | # that we don't import them twice. LLDB REPL doesn't treat duplicate modulemap well and will 1349 | # crash with IRGen errors at least in 5.6.2. 1350 | imported_modules = set() 1351 | for swift_module_search_path in swift_module_search_paths: 1352 | for modulemap in glob.glob( 1353 | swift_module_search_path + "/**/module.modulemap", recursive=True 1354 | ): 1355 | with open(modulemap, "r") as r: 1356 | for line in r: 1357 | module_match = re.match(r"module\s+([^\s]+)\s.*{", line) 1358 | if module_match is not None: 1359 | module_name = module_match.group(1).strip('"') 1360 | imported_modules.add(module_name) 1361 | swift_module_search_path = os.path.join(bazel_dep_target_path, "modules") 1362 | os.makedirs(swift_module_search_path, exist_ok=True) 1363 | for dep in swift_module_libraries: 1364 | dep = dep.strip() 1365 | if len(dep) == 0: 1366 | continue 1367 | if dep[0] == "@": 1368 | parts = dep[1:].split("//") 1369 | modulemap = ( 1370 | "/".join( 1371 | filter( 1372 | lambda x: len(x) > 0, 1373 | ["external", parts[0]] + parts[1].split(":"), 1374 | ) 1375 | ) 1376 | + ".modulemap" 1377 | ) 1378 | else: 1379 | parts = dep.split("//") 1380 | modulemap = ( 1381 | "/".join( 1382 | filter(lambda x: len(x) > 0, [parts[0]] + parts[1].split(":")) 1383 | ) 1384 | + ".modulemap" 1385 | ) 1386 | modulemap = os.path.join(bazel_bin_dir, modulemap) 1387 | if os.path.exists(modulemap): 1388 | # Swift REPL cannot understand modulemap with name other than "module.modulemap". 1389 | # Create the directory and symlink to the said location. Need to process relative 1390 | # Paths. 1391 | dirname = os.path.dirname(modulemap) 1392 | module_contents = "" 1393 | with open(modulemap, "r") as r: 1394 | for line in r: 1395 | module_match = re.match(r"module\s+([^\s]+)\s.*{", line) 1396 | if module_match is not None: 1397 | module_name = module_match.group(1).strip('"') 1398 | m = re.search(r'"(\.\.\/.+)"', line) 1399 | while m is not None: 1400 | line = os.path.normpath( 1401 | os.path.join(dirname, m.group(1)) 1402 | ).join([line[: m.start(1)], line[m.end(1) :]]) 1403 | m = re.search(r'"(\.\.\/.+)"', line) 1404 | module_contents += line 1405 | # If it is already imported, don't create the module.modulemap. 1406 | if module_name in imported_modules: 1407 | continue 1408 | module_dirname = os.path.join( 1409 | swift_module_search_path, "modulemap-%s" % module_name 1410 | ) 1411 | module_modulemap = os.path.join(module_dirname, "module.modulemap") 1412 | os.makedirs(module_dirname, exist_ok=True) 1413 | if os.path.exists(module_modulemap): 1414 | os.remove(module_modulemap) 1415 | with open(module_modulemap, "w") as w: 1416 | w.write(module_contents) 1417 | swift_module_search_paths.add(swift_module_search_path) 1418 | lib_filename = os.path.join( 1419 | bazel_bin_dir, ".ibzlnb", tmpdir, "libjupyterInstalledPackages.so" 1420 | ) 1421 | os.environ["RUNFILES_MANIFEST_FILE"] = lib_filename + ".runfiles/MANIFEST" 1422 | os.environ["RUNFILES_DIR"] = lib_filename + ".runfiles" 1423 | self.swift_module_search_paths = list(swift_module_search_paths) 1424 | 1425 | # == dlopen the shared lib == 1426 | 1427 | self.send_response( 1428 | self.iopub_socket, 1429 | "stream", 1430 | {"name": "stdout", "text": "Initializing Swift...\n"}, 1431 | ) 1432 | self._init_swift() 1433 | self._dlopen_shared_lib(lib_filename) 1434 | 1435 | self.send_response( 1436 | self.iopub_socket, 1437 | "stream", 1438 | {"name": "stdout", "text": "Installation complete!\n"}, 1439 | ) 1440 | # Switch back to whatever we previous from. 1441 | os.chdir(pwd) 1442 | self.already_installed_packages = True 1443 | 1444 | def _execute(self, code): 1445 | locationDirective = '#sourceLocation(file: "%s", line: 1)' % ( 1446 | self._file_name_for_source_location() 1447 | ) 1448 | codeWithLocationDirective = locationDirective + "\n" + code 1449 | result = self.target.EvaluateExpression( 1450 | codeWithLocationDirective, self.expr_opts 1451 | ) 1452 | 1453 | if result.error.type == lldb.eErrorTypeInvalid: 1454 | return SuccessWithValue(result) 1455 | elif result.error.type == lldb.eErrorTypeGeneric: 1456 | return SuccessWithoutValue() 1457 | else: 1458 | return SwiftError(result) 1459 | 1460 | def _after_successful_execution(self): 1461 | result = self._execute( 1462 | "JupyterKernel.communicator.triggerAfterSuccessfulExecution()" 1463 | ) 1464 | if not isinstance(result, SuccessWithValue): 1465 | self.log.error( 1466 | "Expected value from triggerAfterSuccessfulExecution(), " 1467 | "but got: %s" % result 1468 | ) 1469 | return 1470 | 1471 | messages = self._read_jupyter_messages(result.result) 1472 | self._send_jupyter_messages(messages) 1473 | 1474 | def _read_jupyter_messages(self, sbvalue): 1475 | return { 1476 | "display_messages": [ 1477 | self._read_display_message(display_message_sbvalue) 1478 | for display_message_sbvalue in sbvalue 1479 | ] 1480 | } 1481 | 1482 | def _read_display_message(self, sbvalue): 1483 | return [self._read_byte_array(part) for part in sbvalue] 1484 | 1485 | def _read_byte_array(self, sbvalue): 1486 | get_address_error = lldb.SBError() 1487 | address = ( 1488 | sbvalue.GetChildMemberWithName("address") 1489 | .GetData() 1490 | .GetAddress(get_address_error, 0) 1491 | ) 1492 | if get_address_error.Fail(): 1493 | raise Exception("getting address: %s" % str(get_address_error)) 1494 | 1495 | get_count_error = lldb.SBError() 1496 | count_data = sbvalue.GetChildMemberWithName("count").GetData() 1497 | if self._int_bitwidth == 32: 1498 | count = count_data.GetSignedInt32(get_count_error, 0) 1499 | elif self._int_bitwidth == 64: 1500 | count = count_data.GetSignedInt64(get_count_error, 0) 1501 | else: 1502 | raise Exception("Unsupported integer bitwidth %d" % self._int_bitwidth) 1503 | if get_count_error.Fail(): 1504 | raise Exception("getting count: %s" % str(get_count_error)) 1505 | 1506 | # ReadMemory requires that count is positive, so early-return an empty 1507 | # byte array when count is 0. 1508 | if count == 0: 1509 | return bytes() 1510 | 1511 | get_data_error = lldb.SBError() 1512 | data = self.process.ReadMemory(address, count, get_data_error) 1513 | if get_data_error.Fail(): 1514 | raise Exception("getting data: %s" % str(get_data_error)) 1515 | 1516 | return data 1517 | 1518 | def _send_jupyter_messages(self, messages): 1519 | for display_message in messages["display_messages"]: 1520 | self.iopub_socket.send_multipart(display_message) 1521 | 1522 | def _set_parent_message(self): 1523 | result = self._execute( 1524 | """ 1525 | JupyterKernel.communicator.updateParentMessage( 1526 | to: KernelCommunicator.ParentMessage(json: %s)) 1527 | """ 1528 | % json.dumps(json.dumps(squash_dates(self.get_parent()))) 1529 | ) 1530 | if isinstance(result, ExecutionResultError): 1531 | raise Exception("Error setting parent message: %s" % result) 1532 | 1533 | def _set_execution_count(self): 1534 | result = self._execute( 1535 | """ 1536 | JupyterKernel.communicator.executionCount = %s 1537 | """ 1538 | % json.dumps(self.execution_count) 1539 | ) 1540 | if isinstance(result, ExecutionResultError): 1541 | raise Exception("Error setting parent message: %s" % result) 1542 | 1543 | def _get_pretty_main_thread_stack_trace(self): 1544 | stack_trace = [] 1545 | for frame in self.main_thread: 1546 | # Do not include frames without source location information. These 1547 | # are frames in libraries and frames that belong to the LLDB 1548 | # expression execution implementation. 1549 | if not frame.line_entry.file: 1550 | continue 1551 | # Do not include frames. These are 1552 | # specializations of library functions. 1553 | if frame.line_entry.file.fullpath == "": 1554 | continue 1555 | stack_trace.append(str(frame)) 1556 | return stack_trace 1557 | 1558 | def _make_execute_reply_error_message(self, traceback): 1559 | return { 1560 | "status": "error", 1561 | "execution_count": self.execution_count, 1562 | "ename": "", 1563 | "evalue": "", 1564 | "traceback": traceback, 1565 | } 1566 | 1567 | def _send_iopub_error_message(self, traceback): 1568 | self.send_response( 1569 | self.iopub_socket, 1570 | "error", 1571 | { 1572 | "ename": "", 1573 | "evalue": "", 1574 | "traceback": traceback, 1575 | }, 1576 | ) 1577 | 1578 | def _send_exception_report(self, while_doing, e): 1579 | self._send_iopub_error_message( 1580 | [ 1581 | "Kernel is in a bad state. Try restarting the kernel.", 1582 | "", 1583 | "Exception in `%s`:" % while_doing, 1584 | str(e), 1585 | ] 1586 | ) 1587 | 1588 | def _execute_cell(self, code): 1589 | self._set_parent_message() 1590 | self._set_execution_count() 1591 | result = self._preprocess_and_execute(code) 1592 | if isinstance(result, ExecutionResultSuccess): 1593 | self._after_successful_execution() 1594 | return result 1595 | 1596 | def do_execute( 1597 | self, code, silent, store_history=True, user_expressions=None, allow_stdin=False 1598 | ): 1599 | 1600 | # Return early if the code is empty or whitespace, to avoid 1601 | # initializing Swift and preventing package installs. 1602 | if len(code) == 0 or code.isspace(): 1603 | return { 1604 | "status": "ok", 1605 | "execution_count": self.execution_count, 1606 | "payload": [], 1607 | "user_expressions": {}, 1608 | } 1609 | 1610 | # Package installs must be done before initializing Swift (see doc 1611 | # comment in `_init_swift`). 1612 | try: 1613 | code = self._process_installs(code) 1614 | except PackageInstallException as e: 1615 | self._send_iopub_error_message([str(e)]) 1616 | return self._make_execute_reply_error_message([str(e)]) 1617 | except Exception as e: 1618 | self._send_exception_report("_process_installs", e) 1619 | raise e 1620 | 1621 | if not hasattr(self, "debugger"): 1622 | self._init_swift() 1623 | 1624 | # Start up a new thread to collect stdout. 1625 | stdout_handler = StdoutHandler(self) 1626 | stdout_handler.start() 1627 | 1628 | # Execute the cell, handle unexpected exceptions, and make sure to 1629 | # always clean up the stdout handler. 1630 | try: 1631 | result = self._execute_cell(code) 1632 | except Exception as e: 1633 | self._send_exception_report("_execute_cell", e) 1634 | raise e 1635 | finally: 1636 | stdout_handler.stop_event.set() 1637 | stdout_handler.join() 1638 | 1639 | # Send values/errors and status to the client. 1640 | if isinstance(result, SuccessWithValue): 1641 | self.send_response( 1642 | self.iopub_socket, 1643 | "execute_result", 1644 | { 1645 | "execution_count": self.execution_count, 1646 | "data": {"text/plain": result.result.description}, 1647 | "metadata": {}, 1648 | }, 1649 | ) 1650 | return { 1651 | "status": "ok", 1652 | "execution_count": self.execution_count, 1653 | "payload": [], 1654 | "user_expressions": {}, 1655 | } 1656 | elif isinstance(result, SuccessWithoutValue): 1657 | return { 1658 | "status": "ok", 1659 | "execution_count": self.execution_count, 1660 | "payload": [], 1661 | "user_expressions": {}, 1662 | } 1663 | elif isinstance(result, ExecutionResultError): 1664 | if not self.process.is_alive: 1665 | self._send_iopub_error_message(["Process killed"]) 1666 | 1667 | # Exit the kernel because there is no way to recover from a 1668 | # killed process. The UI will tell the user that the kernel has 1669 | # died and the UI will automatically restart the kernel. 1670 | # We do the exit in a callback so that this execute request can 1671 | # cleanly finish before the kernel exits. 1672 | loop = ioloop.IOLoop.current() 1673 | loop.add_timeout(time.time() + 0.1, loop.stop) 1674 | 1675 | return self._make_execute_reply_error_message(["Process killed"]) 1676 | 1677 | if stdout_handler.had_stdout: 1678 | # When there is stdout, it is a runtime error. Stdout, which we 1679 | # have already sent to the client, contains the error message 1680 | # (plus some other ugly traceback that we should eventually 1681 | # figure out how to suppress), so this block of code only needs 1682 | # to add a traceback. 1683 | traceback = [] 1684 | traceback.append("Current stack trace:") 1685 | traceback += [ 1686 | "\t%s" % frame 1687 | for frame in self._get_pretty_main_thread_stack_trace() 1688 | ] 1689 | 1690 | self._send_iopub_error_message(traceback) 1691 | return self._make_execute_reply_error_message(traceback) 1692 | 1693 | # There is no stdout, so it must be a compile error. Simply return 1694 | # the error without trying to get a stack trace. 1695 | self._send_iopub_error_message([result.description()]) 1696 | return self._make_execute_reply_error_message([result.description()]) 1697 | 1698 | def _lldb_complete(self, code, cursor_pos): 1699 | code_to_cursor = code[:cursor_pos] 1700 | sbresponse = self.target.CompleteCode(self.swift_language, None, code_to_cursor) 1701 | prefix = sbresponse.GetPrefix() 1702 | insertable_matches = [] 1703 | for i in range(sbresponse.GetNumMatches()): 1704 | sbmatch = sbresponse.GetMatchAtIndex(i) 1705 | insertable_match = prefix + sbmatch.GetInsertable() 1706 | if insertable_match.startswith("_"): 1707 | continue 1708 | insertable_matches.append(insertable_match) 1709 | return { 1710 | "status": "ok", 1711 | "matches": insertable_matches, 1712 | "cursor_start": cursor_pos - len(prefix), 1713 | "cursor_end": cursor_pos, 1714 | } 1715 | 1716 | def _lsp_complete(self, code, cursor_pos): 1717 | if self.sourcekit_lsp is None: 1718 | return { 1719 | "status": "ok", 1720 | "matches": [], 1721 | "cursor_start": cursor_pos, 1722 | "cursor_end": cursor_pos, 1723 | } 1724 | 1725 | self._lsp_did_change(code) 1726 | code_to_cursor = code[:cursor_pos] 1727 | request_id = jsonrpc_request( 1728 | "textDocument/completion", 1729 | self.sourcekit_lsp.stdin, 1730 | { 1731 | "textDocument": { 1732 | "uri": "file://{}".format( 1733 | os.path.join(self.sourcekit_lsp_workspace, "main.swift") 1734 | ) 1735 | }, 1736 | "position": { 1737 | "line": self.lsp_executed_lines + code_to_cursor.count("\n"), 1738 | "character": len(code_to_cursor) - code_to_cursor.rfind("\n") - 1, 1739 | }, 1740 | }, 1741 | ) 1742 | response = jsonrpc_read_reply(request_id, self.sourcekit_lsp.stdout) 1743 | if "result" not in response or "items" not in response["result"]: 1744 | return { 1745 | "status": "ok", 1746 | "matches": [], 1747 | "cursor_start": cursor_pos, 1748 | "cursor_end": cursor_pos, 1749 | } 1750 | items = response["result"]["items"] 1751 | prefix_len = -1 1752 | matches = [] 1753 | for item in items: 1754 | if ( 1755 | "textEdit" not in item 1756 | or item["textEdit"]["range"]["start"]["line"] 1757 | != item["textEdit"]["range"]["end"]["line"] 1758 | ): 1759 | continue 1760 | if ( 1761 | prefix_len >= 0 1762 | and prefix_len 1763 | != item["textEdit"]["range"]["end"]["character"] 1764 | - item["textEdit"]["range"]["start"]["character"] 1765 | ): 1766 | continue 1767 | prefix_len = ( 1768 | item["textEdit"]["range"]["end"]["character"] 1769 | - item["textEdit"]["range"]["start"]["character"] 1770 | ) 1771 | matches.append(item["textEdit"]["newText"]) 1772 | return { 1773 | "status": "ok", 1774 | "matches": matches, 1775 | "cursor_start": cursor_pos - prefix_len, 1776 | "cursor_end": cursor_pos, 1777 | } 1778 | 1779 | def do_complete(self, code, cursor_pos): 1780 | if not self.lldb_completion_enabled and not self.lsp_completion_enabled: 1781 | return { 1782 | "status": "ok", 1783 | "matches": [], 1784 | "cursor_start": cursor_pos, 1785 | "cursor_end": cursor_pos, 1786 | } 1787 | 1788 | if self.lldb_completion_enabled: 1789 | return self._lldb_complete(code, cursor_pos) 1790 | elif self.lsp_completion_enabled: 1791 | return self._lsp_complete(code, cursor_pos) 1792 | 1793 | def do_shutdown(self, restart): 1794 | # Need to cleanup the temporary Bazel dependency path. 1795 | if hasattr(self, "bazel_dep_target_path"): 1796 | shutil.rmtree(self.bazel_dep_target_path) 1797 | if hasattr(self, "process") and self.process is not None: 1798 | self.process.Kill() 1799 | 1800 | 1801 | if __name__ == "__main__": 1802 | # Jupyter sends us SIGINT when the user requests execution interruption. 1803 | # Here, we block all threads from receiving the SIGINT, so that we can 1804 | # handle it in a specific handler thread. 1805 | if hasattr(signal, "pthread_sigmask"): # Not supported in Windows 1806 | signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGINT]) 1807 | 1808 | from ipykernel.kernelapp import IPKernelApp 1809 | 1810 | # We pass the kernel name as a command-line arg, since Jupyter gives those 1811 | # highest priority (in particular overriding any system-wide config). 1812 | IPKernelApp.launch_instance( 1813 | argv=sys.argv + ["--IPKernelApp.kernel_class=__main__.SwiftKernel"] 1814 | ) 1815 | -------------------------------------------------------------------------------- /kernels/swift/whole_archive.bzl: -------------------------------------------------------------------------------- 1 | # -*- python -*- 2 | 3 | # Copyright 2019 The Bazel Authors. All rights reserved. 4 | # Copyright 2019 Toyota Research Institute. All rights reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # This function is forked and modified from bazelbuild/rules_cc as of: 19 | # https://github.com/bazelbuild/rules_cc/blob/262ebec3c2296296526740db4aefce68c80de7fa/cc/find_cc_toolchain.bzl 20 | def _find_cc_toolchain(ctx): 21 | # Check the incompatible flag for toolchain resolution. 22 | if hasattr(cc_common, "is_cc_toolchain_resolution_enabled_do_not_use") and cc_common.is_cc_toolchain_resolution_enabled_do_not_use(ctx = ctx): # noqa 23 | if "//cc:toolchain_type" in ctx.toolchains: 24 | return ctx.toolchains["//cc:toolchain_type"] 25 | fail("In order to use find_cc_toolchain, your rule has to depend on C++ toolchain. See find_cc_toolchain.bzl docs for details.") # noqa 26 | 27 | # Fall back to the legacy implicit attribute lookup. 28 | if hasattr(ctx.attr, "_cc_toolchain"): 29 | return ctx.attr._cc_toolchain[cc_common.CcToolchainInfo] 30 | 31 | # We didn't find anything. 32 | fail("In order to use find_cc_toolchain, your rule has to depend on C++ toolchain. See find_cc_toolchain.bzl docs for details.") # noqa 33 | 34 | # This function is forked and modified from bazelbuild/rules_cc as of: 35 | # https://github.com/bazelbuild/rules_cc/blob/262ebec3c2296296526740db4aefce68c80de7fa/examples/my_c_archive/my_c_archive.bzl 36 | def _cc_whole_archive_library_impl(ctx): 37 | # Find the C++ toolchain. 38 | cc_toolchain = _find_cc_toolchain(ctx) 39 | feature_configuration = cc_common.configure_features( 40 | ctx = ctx, 41 | cc_toolchain = cc_toolchain, 42 | requested_features = ctx.features, 43 | unsupported_features = ctx.disabled_features, 44 | ) 45 | 46 | # Iterate over the transitive list of libraries we want to link, adding 47 | # `alwayslink = True` to each one. 48 | deps_cc_infos = cc_common.merge_cc_infos( 49 | cc_infos = [dep[CcInfo] for dep in ctx.attr.deps], 50 | ) 51 | old_linker_inputs = deps_cc_infos.linking_context.linker_inputs.to_list() 52 | new_linker_inputs = [] 53 | for old_linker_input in old_linker_inputs: 54 | old_libraries_to_link = old_linker_input.libraries # noqa 55 | new_libraries_to_link = [] 56 | for old_library_to_link in old_libraries_to_link: 57 | new_library_to_link = cc_common.create_library_to_link( 58 | actions = ctx.actions, 59 | feature_configuration = feature_configuration, 60 | cc_toolchain = cc_toolchain, 61 | static_library = old_library_to_link.static_library, 62 | pic_static_library = old_library_to_link.pic_static_library, 63 | dynamic_library = old_library_to_link.resolved_symlink_dynamic_library, # noqa 64 | interface_library = old_library_to_link.resolved_symlink_interface_library, # noqa 65 | # This is where the magic happens! 66 | alwayslink = True, 67 | ) 68 | new_libraries_to_link.append(new_library_to_link) 69 | new_linker_inputs.append( 70 | cc_common.create_linker_input( 71 | owner = old_linker_input.owner, 72 | libraries = depset(new_libraries_to_link), 73 | user_link_flags = depset(old_linker_input.user_link_flags), 74 | additional_inputs = depset(old_linker_input.additional_inputs), # noqa 75 | ) 76 | ) 77 | 78 | # Return the CcInfo to pass along to code that wants to link us. 79 | linking_context = cc_common.create_linking_context( 80 | linker_inputs = depset(new_linker_inputs) 81 | ) 82 | return [ 83 | DefaultInfo( 84 | runfiles = ctx.runfiles( 85 | collect_data = True, 86 | collect_default = True, 87 | ), 88 | ), 89 | CcInfo( 90 | compilation_context = deps_cc_infos.compilation_context, 91 | linking_context = linking_context, 92 | ), 93 | ] 94 | 95 | # Forked and modified from bazelbuild/rules_cc as of: 96 | # https://github.com/bazelbuild/rules_cc/blob/262ebec3c2296296526740db4aefce68c80de7fa/examples/my_c_archive/my_c_archive.bzl 97 | cc_whole_archive_library = rule( 98 | implementation = _cc_whole_archive_library_impl, 99 | attrs = { 100 | "deps": attr.label_list(providers = [CcInfo]), 101 | "_cc_toolchain": attr.label( 102 | default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"), 103 | ), 104 | }, 105 | fragments = ["cpp"], 106 | toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], 107 | ) 108 | """Creates an cc_library with `alwayslink = True` added to all of its deps, to 109 | work around https://github.com/bazelbuild/bazel/issues/7362 not providing any 110 | useful way to create shared libraries from multiple cc_library targets unless 111 | you want even statically-linked programs to keep all of their symbols. 112 | """ 113 | -------------------------------------------------------------------------------- /lab.bzl: -------------------------------------------------------------------------------- 1 | load("@pip//:requirements.bzl", "requirement") 2 | load("@rules_python//python:defs.bzl", "py_binary") 3 | 4 | def jupyterlab(name, deps): 5 | py_binary( 6 | name = name, 7 | srcs = ["@swift-jupyter//bootup:sources"], 8 | data = ["@swift-jupyter//kernels/swift:swift_kernel"], 9 | deps = [ 10 | "@bazel_tools//tools/python/runfiles", 11 | requirement("jupyterlab"), 12 | ] + deps, 13 | ) 14 | -------------------------------------------------------------------------------- /lab/BUILD: -------------------------------------------------------------------------------- 1 | load("@pip//:requirements.bzl", "requirement") 2 | load("//:lab.bzl", "jupyterlab") 3 | 4 | py_library( 5 | name = "common", 6 | visibility = ["//lab:__subpackages__"], 7 | deps = [ 8 | requirement("jupyterlab"), 9 | requirement("pandas"), 10 | requirement("numpy"), 11 | requirement("matplotlib"), 12 | requirement("gym"), 13 | requirement("pygame"), 14 | ], 15 | ) 16 | 17 | jupyterlab( 18 | name = "bootup", 19 | deps = [ 20 | ":common", 21 | ], 22 | ) 23 | -------------------------------------------------------------------------------- /notebooks/Untitled.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "id": "b2fd3bc7-8dfc-4a2a-aa28-05b4c623d65d", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [ 10 | "import gym\n", 11 | "import matplotlib.pyplot as plt\n", 12 | "from IPython import display, core\n", 13 | "import IPython" 14 | ] 15 | }, 16 | { 17 | "cell_type": "code", 18 | "execution_count": null, 19 | "id": "6be654a5-7d8c-4e9a-96e5-f10d9237783d", 20 | "metadata": {}, 21 | "outputs": [ 22 | { 23 | "data": { 24 | "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAMi0lEQVR4nO3db2xV9R3H8c+55/6hf2gpLfQPQugoYczVECQuEIci2EQMYpiGDMdYMoNPfOIejKDZ5jOTmUFI9sQsk42xLQwqoFtwDjMmTi26OQYOWDpbLG3pH9rSQnvbe+/57YHQ+ae9XJfe3i/0/XpEes+hv5v23d+5v3PuuZ5zTgDsCeV6AADGRpyAUcQJGEWcgFHECRgVvsHjLOUC2eeN9UVmTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AqHCuB4CJ1dLSomPHjqm4uFjz5s1TdXW1CgsL5ft+roeGL4g4bzFDQ0M6d+6cent71dLSong8rmXLlumRRx5RbW2twuGwPM/L9TCRAc85l+7xtA/CHuecnHMKgkCJREIXLlzQ0aNHdejQIS1evFhPPfWU5s2bR6C2jPnDIM5b3PWfb09Pj3bv3q1XX31VTz/9tO69916FQiw5GEGcU10qldLx48f13HPP6cknn9SDDz5IoDaMGSevOacQ3/d1zz33qKioSNu3b9fMmTO1YsUKDnGNYuacgpxzOnr0qHbs2KHdu3eroqIi10Oa6sb868gxzRTkeZ5WrVql5cuX64UXXlAqlcr1kDAG4pyiwuGwHn/8cb311ltqamrK9XAwBuK8BTjnlBwY0NXGRl05e1bD7e1yqZRu8JJFlZWVWrVqlQ4fPqwgCCZptMgUrzkNGxoakud5mjZt2rjbBCMjuvTnP6vz97//OMpkUuGiIhXfeacqN25UtLx83AUf55xOnTqlZ599Vnv37lV+fn62ngrS4zXnzcQ5pwMHDujdd98dd5sgmdTFgwfV8rOfKX7+vNzIiBQESvb16dLrr6vpJz/RSEfHuDOo53mqqalRMplUW1tbtp4K/k/EaVQ8Hte+ffu0f//+MRdsnHPqe/ttddTXy42MqH9kRL9obNTzp0/rnz09cs7p6rlzav3Vr6Q0h6zRaFTz589XY2NjNp8O/g/EaVRTU5MaGhp05MgRdXR0fO7xxKVLav3lLxXE4xpIJPTD99/XT8+c0b6mJj114oTe6eqSJF0+cUJD58+P+31831d5ebm6rm0PO4jToOvnIePxuHp7e3Xs2LFPHZq6ZFKte/ZopLNTktQ6OKi3rv1bki4nEnrt2mFqMDysIB4f93t5nqfCwkJOpxjEFUIGJRIJFRQUaNOmTZo7d66CIFAymVQkEpEkDfzrX+p7++3R7aOhkGK+r6FPBFZ0bdsbcc4pkUhM7BPAhGDmNCgSiWjz5s264447lJ+fr40bNyoc/vjvaHJgQK27dysYHh7dvrqwUNtqa1UWiynm+7qvslLfXbhQkhSdPVvRNFcAOefU19fHSq1BzJwGeZ6naDSqmpoaHTx4UL7vy/M8OefU/frrGvzMRQOe5+nB227T0tJSDSWTmlNQoGm+L/m+Zq9fr0hJybjfK5lM6sKFC1q7dm22nxa+IOI0rKamRm1tbRoYGFBxcbGGmpvVefDgmKuvnuep6jOzX9GSJSpbsybthe39/f1qb2/XwmszLezgsNawuXPnKhqN6uzZswqGh9W+b58Svb0Z7RsuKlLlo48qlOYCBuec3nvvPVVXV2vGjBkTNGpMFGZOwyKRiNatW6cDBw5o/uXLn1oEupGyujoVfPnLaWfNZDKp/fv3a/369aOvaWEHM6dhnufpgQce0Km//U1v7Nwpl+H1r7E5czRr7Vp5ad5I7ZxTQ0ODuru7tXLlSt7TaRBxGldWWqp1S5Zoz+nTimdwLtILh1X12GOKlJam3a63t1c7d+7U1q1bVVxcPFHDxQQiTsOccxr4xz+0tLlZkVBIB5qblbzB7Fly992a8bWvpb3YPR6Pa8eOHVqwYIHuv/9+Zk2jiNOwZF+f2vbuVTSZ1NZFi3Syt1cvt7QoEQRjXszuT5+uig0bFEpzAUI8HteuXbvU1NSkbdu2jV7YAHuI0ygXBOr+0580+J//SJLKYjF97/bb1dDVpRf//W/1JxKfDtTzVPGNb2javHlj/3/O6eLFi3rmmWf0wQcf6Pnnn9fMmTOZNQ0jToOccxpqblbXH/4w+jXP81SZl6dttbUaTKX0o/ff15udnRpMJuWcU978+R+f0/zEItD1e9j29/ervr5eW7Zs0fTp07Vr1y5VVlYSpnG82dqgYGREH/74x7p84sTnHnPOKeWc/n7pkl46f15DyaSWzZql1Vu36vZ161RcXDx6SV5zc7MaGhp0/PhxlZeX64knntBdd901esURzOC+tTcD55x633xTzbt2ffzm6TTbDQeBPuzv15nKSn0Ujaqrq0uDg4PyPE8FBQUqLy9XbW2t1qxZo0WLFikWixGlTcR5M0j09Ojc9u0abm/PaPtIWZlqfvAD+VVVSiQSo/cC8n1fkUhEkUiEIO3jptLWuVRKrb/+dcZhKhRS+UMPKW/+fHmep1gslt0BYlKxIGSEc059DQ3qPX48433yq6tVet99zIy3KOI0InXlii7W16e9a8EnedGo5mzZonBRUZZHhlwhTgNcEKj7j38cPaeZibK6Ok2vrc3iqJBrxJlj189pdhw6lPYueZ8UKStT+fr18vi06lsaceaYSyR04cUXlezvz2h7z/dV9c1vKjp7dpZHhlwjzhxyzqnvnXc0cPp0xvsULF6skrvvZhFoCiDOHBrp7FTrnj0ZH876hYWqeuyxtHc3wK2DOHPEpVLqfOWV0XvPZqJ01SoVLl7MrDlFEGcOOOc0cOqUuo8ezXifWFWVZj/0UNq7G+DWwk86B1JXrqjtt79VMDiY0fae76vi0UdZBJpiiHOSuSBQ15EjunrmTMb7FH7lKypZsYLD2SmGOCfZ4Icf6mJ9fcbb+wUFmvOd78jPy8viqGARcU6iYGREF/fvVzA0lPE+JStXKn/BgiyOClYR5yRxzqnn2LEx30A9nmm33aaqTZtYBJqi+KlPkuG2NrX/7ndyGX7UnheNqupb3+LC9imMOCeBS6XUcfjwFzqnWbx0qYqXLWMRaAojzixzzqn/5En1/OUvGe8TKSlR5caN8rht5ZRGnFmWunpV7b/5zRdaBJq1dq3yqquZNac44syi6/eevdrYmPE+edXVKqurYxEIxJktzjnFW1vV8dJLGV/Y7kUiqtq0SWE+jg8izqxxiYQu/PznSl6+nPE+xXfeqaKlSzmchSTizArnnLpfe00DJ09mvE+kpERzvv3ttJ9zgqmFOLMg0d2tzpdfzvicpjxPZXV1ilVVZXdguKkQ5wRzzqnnjTc03NGR8T4FCxdq9sMPswiET+G3YYK5REL9J09K6e+kPyqUl6eqzZsVLijI8shwsyHOCeYSCcU/+ijj7Wd+/eua/tWvZnFEuFkRZw5FKypUvmGDxOEsxsBvxUTzfUVnzbrxdqGQyh9+WDE+JxPjIM4JForFNHvdOnnRaJqNQipaskQzV64kTIyLOCeY53masXy5KjZsGPvCdd9XWV2dvvT97ytcWDj5A8RNg8/nzJIgkVDvX/+qriNHFG9pkZxTrLJSpatXq3T1aoX4IFv8Dx+eO9mccwqGhpS6elXOOfkFBfLz84kSn0WcgFFjxslrTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwiTsAo4gSMIk7AKOIEjCJOwCjiBIwK3+Bxb1JGAeBzmDkBo4gTMIo4AaOIEzCKOAGjiBMw6r/i3ttywHHbJQAAAABJRU5ErkJggg==\n", 25 | "text/plain": [ 26 | "
" 27 | ] 28 | }, 29 | "metadata": {}, 30 | "output_type": "display_data" 31 | } 32 | ], 33 | "source": [ 34 | "env = gym.make(\"Pendulum-v1\")\n", 35 | "env.reset()\n", 36 | "for _ in range(10000):\n", 37 | " plt.imshow(env.render(mode='rgb_array'))\n", 38 | " plt.axis('off')\n", 39 | " display.display(plt.gcf()) \n", 40 | " display.clear_output(wait=True)\n", 41 | " action = env.action_space.sample()\n", 42 | " observation, reward, done, info = env.step(action)\n", 43 | " if done:\n", 44 | " observation, info = env.reset(return_info=True)\n", 45 | "env.close()" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": null, 51 | "id": "ab1a9720-24ee-407c-a7bf-3e83a1b630c5", 52 | "metadata": {}, 53 | "outputs": [], 54 | "source": [] 55 | } 56 | ], 57 | "metadata": { 58 | "kernelspec": { 59 | "display_name": "Python 3 (ipykernel)", 60 | "language": "python", 61 | "name": "python3" 62 | }, 63 | "language_info": { 64 | "codemirror_mode": { 65 | "name": "ipython", 66 | "version": 3 67 | }, 68 | "file_extension": ".py", 69 | "mimetype": "text/x-python", 70 | "name": "python", 71 | "nbconvert_exporter": "python", 72 | "pygments_lexer": "ipython3", 73 | "version": "3.8.10" 74 | } 75 | }, 76 | "nbformat": 4, 77 | "nbformat_minor": 5 78 | } 79 | -------------------------------------------------------------------------------- /scripts/BUILD.bazel: -------------------------------------------------------------------------------- 1 | load("@com_github_bazelbuild_buildtools//buildifier:def.bzl", "buildifier") 2 | 3 | buildifier( 4 | name = "buildifier", 5 | ) 6 | -------------------------------------------------------------------------------- /scripts/bazel/setup_clang.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | BAZELRC_FILE="${BAZELRC_FILE:-$(git rev-parse --show-toplevel)/clang.bazelrc}" 6 | 7 | LLVM_PREFIX=$1 8 | 9 | if [[ ! -e "${LLVM_PREFIX}/bin/llvm-config" ]]; then 10 | echo "Error: cannot find llvm-config in ${LLVM_PREFIX}." 11 | exit 1 12 | fi 13 | 14 | export PATH="$(${LLVM_PREFIX}/bin/llvm-config --bindir):${PATH}" 15 | 16 | RT_LIBRARY_PATH="$(dirname $(find $(llvm-config --libdir) -name libclang_rt.ubsan_standalone_cxx-x86_64.a | head -1))" 17 | 18 | echo "# Generated file, do not edit. If you want to disable clang, just delete this file. 19 | build:clang --action_env='PATH=${PATH}' 20 | build:clang --action_env=CC=clang 21 | build:clang --action_env=CXX=clang++ 22 | build:clang --action_env='LLVM_CONFIG=${LLVM_PREFIX}/bin/llvm-config' 23 | build:clang --repo_env='LLVM_CONFIG=${LLVM_PREFIX}/bin/llvm-config' 24 | build:clang --linkopt='-L$(llvm-config --libdir)' 25 | build:clang --linkopt='-Wl,-rpath,$(llvm-config --libdir)' 26 | 27 | build:clang-asan --action_env=ENVOY_UBSAN_VPTR=1 28 | build:clang-asan --copt=-fsanitize=vptr,function 29 | build:clang-asan --linkopt=-fsanitize=vptr,function 30 | build:clang-asan --linkopt='-L${RT_LIBRARY_PATH}' 31 | build:clang-asan --linkopt=-l:libclang_rt.ubsan_standalone-x86_64.a 32 | build:clang-asan --linkopt=-l:libclang_rt.ubsan_standalone_cxx-x86_64.a 33 | " > ${BAZELRC_FILE} 34 | 35 | -------------------------------------------------------------------------------- /scripts/black/BUILD: -------------------------------------------------------------------------------- 1 | load("@pip//:requirements.bzl", "requirement") 2 | 3 | py_binary( 4 | name = "format", 5 | srcs = ["format.py"], 6 | deps = [ 7 | requirement("black"), 8 | ], 9 | ) 10 | -------------------------------------------------------------------------------- /scripts/black/format.py: -------------------------------------------------------------------------------- 1 | from black import patched_main 2 | 3 | if __name__ == "__main__": 4 | patched_main() 5 | -------------------------------------------------------------------------------- /scripts/black/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | FILES=$(git diff --cached --name-only --diff-filter=ACMR "*.py" | sed 's| |\\ |g') 3 | [ -z "$FILES" ] && exit 0 4 | 5 | # Bazel invocation may git clone some repositories, and override these env vars. 6 | 7 | _GIT_INDEX_FILE=$GIT_INDEX_FILE 8 | 9 | unset GIT_INDEX_FILE 10 | 11 | # Prettify all selected files 12 | echo "$FILES" | xargs -I {} bazel run --compilation_mode=opt //scripts/black:format -- `realpath {}` 13 | 14 | export GIT_INDEX_FILE=$_GIT_INDEX_FILE 15 | 16 | # Add back the modified/prettified files to staging 17 | echo "$FILES" | xargs git add 18 | 19 | exit 0 20 | -------------------------------------------------------------------------------- /scripts/buildifier/buildifier.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0) 4 | 5 | bazel run --compilation_mode=opt @com_github_bazelbuild_buildtools//buildifier:buildifier -- $@ 6 | -------------------------------------------------------------------------------- /scripts/buildifier/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | FILES=$(git diff --cached --name-only --diff-filter=ACMR "*BUILD*" | sed 's| |\\ |g') 3 | [ -z "$FILES" ] && exit 0 4 | 5 | # Bazel invocation may git clone some repositories, and override these env vars. 6 | 7 | _GIT_INDEX_FILE=$GIT_INDEX_FILE 8 | 9 | unset GIT_INDEX_FILE 10 | 11 | # Prettify all selected files 12 | echo "$FILES" | xargs -I {} bazel run --compilation_mode=opt @com_github_bazelbuild_buildtools//buildifier:buildifier -- -r `realpath {}` 13 | 14 | export GIT_INDEX_FILE=$_GIT_INDEX_FILE 15 | 16 | # Add back the modified/prettified files to staging 17 | echo "$FILES" | xargs git add 18 | 19 | exit 0 20 | -------------------------------------------------------------------------------- /scripts/docc.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | GIT_ROOT=$(git rev-parse --show-toplevel) 6 | 7 | cd $GIT_ROOT 8 | 9 | # Generate symbol graph 10 | bazel build :JupyterDisplay --features=swift.emit_symbol_graph 11 | # Copy it into a valid bundle 12 | mkdir -p display.docc 13 | cp bazel-bin/JupyterDisplay.symbolgraph/*.json display.docc/ 14 | # Remove all docs 15 | rm -rf docs 16 | # Convert into static hosting document 17 | docc convert display.docc --fallback-display-name="JupyterDisplay" --fallback-bundle-identifier org.liuliu.swift-jupyter --fallback-bundle-version 0.0.1 --output-path docs --hosting-base-path /swift-jupyter --index --transform-for-static-hosting 18 | # Adding auto-redirect index.html 19 | echo '' > docs/index.html 20 | rm -rf display.docc 21 | -------------------------------------------------------------------------------- /scripts/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | GIT_CONFIG=$(git rev-parse --git-dir) 6 | GIT_ROOT=$(git rev-parse --show-toplevel) 7 | 8 | mkdir -p $GIT_CONFIG/hooks/pre-commit.d 9 | 10 | rm -f $GIT_CONFIG/hooks/pre-commit 11 | ln -s $GIT_ROOT/scripts/vendors/dispatch $GIT_CONFIG/hooks/pre-commit 12 | 13 | rm -f $GIT_CONFIG/hooks/pre-commit.d/swift-format 14 | ln -s $GIT_ROOT/scripts/swift-format/pre-commit $GIT_CONFIG/hooks/pre-commit.d/swift-format 15 | 16 | rm -f $GIT_CONFIG/hooks/pre-commit.d/buildifier 17 | ln -s $GIT_ROOT/scripts/buildifier/pre-commit $GIT_CONFIG/hooks/pre-commit.d/buildifier 18 | 19 | rm -f $GIT_CONFIG/hooks/pre-commit.d/black 20 | ln -s $GIT_ROOT/scripts/black/pre-commit $GIT_CONFIG/hooks/pre-commit.d/black 21 | 22 | cd $GIT_ROOT && ./scripts/bazel/setup_clang.sh /usr/local 23 | -------------------------------------------------------------------------------- /scripts/swift-format/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | FILES=$(git diff --cached --name-only --diff-filter=ACMR "*.swift" | sed 's| |\\ |g') 3 | [ -z "$FILES" ] && exit 0 4 | 5 | GIT_ROOT=$(git rev-parse --show-toplevel) 6 | 7 | # Bazel invocation may git clone some repositories, and override these env vars. 8 | 9 | _GIT_INDEX_FILE=$GIT_INDEX_FILE 10 | 11 | unset GIT_INDEX_FILE 12 | 13 | # Prettify all selected files 14 | echo "$FILES" | xargs -I {} bazel run --compilation_mode=opt @SwiftFormat//:swift-format -- format --configuration "$GIT_ROOT/.swift-format.json" -i `realpath {}` 15 | 16 | export GIT_INDEX_FILE=$_GIT_INDEX_FILE 17 | 18 | # Add back the modified/prettified files to staging 19 | echo "$FILES" | xargs git add 20 | 21 | exit 0 22 | 23 | -------------------------------------------------------------------------------- /scripts/swift-format/swift-format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd $(dirname $0) 4 | 5 | bazel run --compilation_mode=opt @SwiftFormat//:swift-format -- $@ 6 | -------------------------------------------------------------------------------- /scripts/vendors/dispatch: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # a modular git hooks dispatcher 5 | # 6 | # Copyright 2014 Michael F. Lamb 7 | # 8 | # This program is free software: you can redistribute it and/or modify it under 9 | # the terms of the GNU General Public License as published by the Free Software 10 | # Foundation, either version 3 of the License, or (at your option) any later 11 | # version. 12 | # 13 | # License: GPLv3 http://www.gnu.org/licenses/gpl.html 14 | # 15 | 16 | # This runs several scripts per git hook instead of one. This lets us organize 17 | # our git hooks into small, discrete, language-agnostic executables. It 18 | # somewhat apes Debian's 'run-parts' in its operation, but does not employ it 19 | # directly. 20 | 21 | # Variable names used in this script: 22 | # 23 | # null_commit - (exported) a magic sha hash value meaningful to git 24 | # empty_tree - (exported) a magic sha hash value meaningful to git 25 | # head_hash - (exported) a sha hash useful to hooks, see details below. 26 | # bad_suffixes - a space-separated list of hook filename suffixes that will make us ignore that hook 27 | # exit_status - the last-encountered failing exit status of a dispatched hook, or 0. 28 | # hook - the filename of the current hook being processed relative to $GIT_DIR/hooks, like 'pre-commit.d/run_linter.sh.optional' 29 | # hook_short - $hook as above, minus any '.optional' suffix, like 'pre-commit.d/run_linter.sh' 30 | # hook_type - the hook name called by git, like 'update', 'pre-commit', 'post-receive', etc. 31 | # run_default - whether the current hook should run, if no configuration mentions it. 32 | # tempfile - filename of a temporary file created to hold standard input to 'pre-push', 'pre-receive', and 'post-receive' hooks. 33 | 34 | # To avoid copypasta this single script checks its own name to see how it is 35 | # being called; the expectation is that each of the git hooks will exist simply 36 | # as a symlink to this script. 37 | hook_type="$(basename $0)" 38 | GIT_DIR="$(git rev-parse --git-dir)" 39 | if [ "$hook_type" = "dispatch" ]; then 40 | echo >&2 "'$hook_type' should not be executed directly; instead, symlink a githook to it and let git invoke it." 41 | exit 1 42 | fi 43 | 44 | # if there's no hooks directory for us to dispatch into, we may trivially exit. 45 | [ -d "$GIT_DIR/hooks/${hook_type}.d" ] || exit 0 46 | 47 | # some magic values used internally by git. 48 | export null_commit="0000000000000000000000000000000000000000" 49 | export empty_tree="4b825dc642cb6eb9a060e54bf8d69288fbee4904" 50 | 51 | # never run hook scripts with these suffixes even if they're marked executable: 52 | # redhat skips these 53 | bad_suffixes=".rpmsave .rpmorig .rpmnew .swp ,v ~ ," 54 | # debian skips these when using --lsbsysinit 55 | bad_suffixes="$bad_suffixes .dpkg-old .dpkg-dist .dpkg-new .dpkg-tmp" 56 | # git skips these 57 | bad_suffixes="$bad_suffixes .sample" 58 | 59 | # Hook scripts are called with the same arguments and data on standard input 60 | # that are passed by git to its githook. 61 | 62 | # In the pre-commit hook one frequently compares (git diff-index) the changes 63 | # marked for commit (--cached) against the most recent commit (HEAD). However, 64 | # there is an edge-case: if this is the first commit on the branch, the HEAD 65 | # reference will not yet exist, so we have to diff against the "secret" 66 | # empty-tree reference. Figure out whether to use HEAD or the secret ref and 67 | # keep that in a variable. Everybody likes to call this variable $AGAINST; I 68 | # think $head_hash is more intuitive. 69 | head_hash=$(git rev-parse --quiet --verify HEAD) || head_hash="$empty_tree" 70 | export head_hash 71 | 72 | # FIXME DEPRECATED bugfix issue #5: previously, this script examined 73 | # "hook.enable" not "hook.enabled", contrary to what it says in the 74 | # documentation. 75 | git config --local --bool --get-regexp '^hook\..*\.enable$' | while read key val; do 76 | if 77 | git config --local --unset "${key}" 78 | then 79 | if 80 | git config --local "${key}d" >/dev/null 81 | then 82 | echo >&2 "Removed deprecated configuration: ${key} = ${val}" 83 | else 84 | git config --local "${key}d" "${val}" 85 | echo >&2 "Renamed deprecated configuration: ${key} -> ${key}d" 86 | fi 87 | else 88 | cat >&2 <<-EOF 89 | $0: Warning: this repository contains the deprecated configuration 90 | ${key}. Could not automatically migrate the configuration for this 91 | repository. You will have to run these commands yourself to configure 92 | hooks properly: 93 | 94 | git config --local --bool ${key}d ${val}; 95 | git config --local --unset ${key} 96 | EOF 97 | break 98 | fi 99 | done 100 | 101 | case "$hook_type" in 102 | pre-push|pre-receive|post-receive|post-rewrite) 103 | # These hooks are unique in that they are provided by git with data on 104 | # standard input. Dump this data into a temporary file, and replay it 105 | # as standard input to each of the hooks. 106 | tempfile="$(mktemp "${TMPDIR:-/tmp}"/tmp.XXXXXXXXXX)" 107 | trap 'rm -f "$tempfile"' EXIT 108 | cat > "$tempfile" 109 | ;; 110 | esac 111 | 112 | # loop over all hooks of our $hook_type 113 | exit_status=0 114 | for hook in "$GIT_DIR/hooks/${hook_type}.d"/*; do 115 | # skip non-executable and non-files 116 | [ -f "$hook" -a -x "$hook" ] || continue 117 | # skip bad suffixes 118 | for ext in $bad_suffixes; do 119 | [ "${hook%$ext}" = "${hook}" ] || continue 2 120 | done 121 | 122 | # now we have a viable candidate. check git config to see if it should be 123 | # run. hooks named .optional default to no, otherwise yes. 124 | hook_short="${hook%.optional}" 125 | if [ "${hook_short}" = "${hook}" ] 126 | then run_default="true" # hook does not end in ".optional" 127 | else run_default="false" # hook ends in ".optional" 128 | fi 129 | hook_short="${hook_short#"$GIT_DIR/hooks/"}" 130 | 131 | # determine if this hook is enabled 132 | # FIXME "hook.*.enable" is deprecated; remove in next release. 133 | [ "$( 134 | git config --bool hook.${hook_short}.enabled || 135 | git config --bool hook.${hook_short##*/}.enabled || 136 | git config --bool hook.${hook_short}.enable || 137 | git config --bool hook.${hook_short##*/}.enable || 138 | echo $run_default)" = "true" ] || continue 139 | 140 | # run the hook with the args git gave us and if $tempfile exists, provide 141 | # it on standard input. 142 | if [ "$tempfile" ]; then 143 | "$hook" "$@" <"$tempfile" 144 | else 145 | "$hook" "$@" 146 | fi || { 147 | exit_status=$? 148 | echo >&2 "$0: $hook_short exited with return code $exit_status" 149 | } 150 | done 151 | 152 | # all the hooks have run; perform some cleanup. 153 | case "$hook_type" in 154 | post-update) 155 | if [ "$exit_status" -ne 0 ]; then 156 | cat >&2 <<- EOF 157 | 158 | Warning: there was a problem running $hook_type githooks. 159 | (However, this has no negative effect on the action you just 160 | attempted.) 161 | EOF 162 | fi 163 | ;; 164 | pre-commit|commit-msg) 165 | if [ $exit_status -ne 0 ]; then 166 | cat >&2 <<- EOF 167 | 168 | Aborting commit from $hook_type due to previous errors. If you 169 | want to override these checks, pass the --no-verify option to 170 | 'git commit'. 171 | 172 | Note: these checks are performed against the contents of your 173 | index (staging area.) If you don't see the problem being 174 | reported, ensure that you've run 'git add' on the fixed lines. 175 | EOF 176 | fi 177 | ;; 178 | update) 179 | if [ $exit_status -ne 0 ]; then 180 | echo >&2 "ERROR: aborting update to $1 due to previous errors." 181 | fi 182 | esac 183 | 184 | # exit with an appropriate success code. 185 | exit $exit_status 186 | 187 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuliu/swift-jupyter/534a9515a752d24e24dafb96de21da33f19ee512/test/__init__.py -------------------------------------------------------------------------------- /test/all_test_docker.py: -------------------------------------------------------------------------------- 1 | """Runs all tests that work in the docker image. 2 | 3 | Specifically, this includes the SwiftKernelTestsPython27 test that requires a 4 | special kernel named 'swift-with-python-2.7'. 5 | """ 6 | 7 | import unittest 8 | 9 | from tests.kernel_tests import * 10 | from tests.simple_notebook_tests import * 11 | from tests.tutorial_notebook_tests import * 12 | 13 | 14 | if __name__ == '__main__': 15 | unittest.main() 16 | -------------------------------------------------------------------------------- /test/all_test_local.py: -------------------------------------------------------------------------------- 1 | """Runs all tests that work locally. 2 | 3 | Specifically, this excludes the SwiftKernelTestsPython27 test that requires a 4 | special kernel named 'swift-with-python-2.7'. 5 | """ 6 | 7 | import unittest 8 | 9 | from tests.kernel_tests import SwiftKernelTests, OwnKernelTests 10 | from tests.simple_notebook_tests import * 11 | from tests.tutorial_notebook_tests import * 12 | 13 | 14 | if __name__ == '__main__': 15 | unittest.main() 16 | -------------------------------------------------------------------------------- /test/fast_test.py: -------------------------------------------------------------------------------- 1 | """Runs fast tests.""" 2 | 3 | import unittest 4 | 5 | from tests.kernel_tests import SwiftKernelTests, OwnKernelTests 6 | from tests.simple_notebook_tests import * 7 | 8 | 9 | if __name__ == '__main__': 10 | unittest.main() 11 | -------------------------------------------------------------------------------- /test/notebook_tester.py: -------------------------------------------------------------------------------- 1 | """Runs notebooks. 2 | 3 | See --help text for more information. 4 | """ 5 | 6 | import argparse 7 | import nbformat 8 | import numpy 9 | import os 10 | import sys 11 | import time 12 | 13 | from collections import defaultdict 14 | from jupyter_client.manager import start_new_kernel 15 | 16 | 17 | # Exception for problems that occur while executing cell. 18 | class ExecuteException(Exception): 19 | def __init__(self, cell_index): 20 | self.cell_index = cell_index 21 | 22 | 23 | # There was an error (that did not crash the kernel) while executing the cell. 24 | class ExecuteError(ExecuteException): 25 | def __init__(self, cell_index, reply, stdout): 26 | super(ExecuteError, self).__init__(cell_index) 27 | self.reply = reply 28 | self.stdout = stdout 29 | 30 | def __str__(self): 31 | return 'ExecuteError at cell %d, reply:\n%s\n\nstdout:\n%s' % ( 32 | self.cell_index, self.reply, self.stdout) 33 | 34 | 35 | # The kernel crashed while executing the cell. 36 | class ExecuteCrash(ExecuteException): 37 | def __init__(self, cell_index): 38 | super(ExecuteCrash, self).__init__(cell_index) 39 | 40 | def __str__(self): 41 | return 'ExecuteCrash at cell %d' % self.cell_index 42 | 43 | 44 | # Exception for problems that occur during a completion request. 45 | class CompleteException(Exception): 46 | def __init__(self, cell_index, char_index): 47 | self.cell_index = cell_index 48 | self.char_index = char_index 49 | 50 | 51 | # There was an error (that did not crash the kernel) while processing a 52 | # completion request. 53 | class CompleteError(CompleteException): 54 | def __init__(self, cell_index, char_index): 55 | super(CompleteError, self).__init__(cell_index, char_index) 56 | 57 | def __str__(self): 58 | return 'CompleteError at cell %d, char %d' % (self.cell_index, 59 | self.char_index) 60 | 61 | 62 | # The kernel crashed while processing a completion request. 63 | class CompleteCrash(CompleteException): 64 | def __init__(self, cell_index, char_index): 65 | super(CompleteCrash, self).__init__(cell_index, char_index) 66 | 67 | def __str__(self): 68 | return 'CompleteCrash at cell %d, char %d' % (self.cell_index, 69 | self.char_index) 70 | 71 | 72 | class NotebookTestRunner: 73 | def __init__(self, notebook, char_step=1, repeat_times=1, 74 | execute_timeout=60, complete_timeout=5, verbose=True): 75 | """ 76 | noteboook - path to a notebook to run the test on 77 | char_step - number of chars to step per completion request. 0 disables 78 | repeat_times - run the notebook this many times, in the same kernel 79 | instance 80 | execute_timeout - number of seconds to wait for cell execution 81 | complete_timeout - number of seconds to wait for completion 82 | verbose - print progress, statistics, and errors 83 | """ 84 | 85 | self.char_step = char_step 86 | self.repeat_times = repeat_times 87 | self.execute_timeout = execute_timeout 88 | self.complete_timeout = complete_timeout 89 | self.verbose = verbose 90 | 91 | notebook_dir = os.path.dirname(notebook) 92 | os.chdir(notebook_dir) 93 | nb = nbformat.read(notebook, as_version=4) 94 | 95 | self.code_cells = [cell for cell in nb.cells 96 | if cell.cell_type == 'code' \ 97 | and not cell.source.startswith('#@title')] 98 | 99 | self.stdout = [] 100 | self.unexpected_errors = [] 101 | 102 | def _execute_cell(self, cell_index): 103 | code = self.code_cells[cell_index].source 104 | self._execute_code(code, cell_index) 105 | 106 | def _execute_code(self, code, cell_index=-1): 107 | self.kc.execute(code) 108 | 109 | # Consume all the iopub messages that the execution produced. 110 | stdout = '' 111 | while True: 112 | try: 113 | reply = self.kc.get_iopub_msg(timeout=self.execute_timeout) 114 | except TimeoutError: 115 | # Timeout usually means that the kernel has crashed. 116 | raise ExecuteCrash(cell_index) 117 | if reply['header']['msg_type'] == 'stream' and \ 118 | reply['content']['name'] == 'stdout': 119 | stdout += reply['content']['text'] 120 | if reply['header']['msg_type'] == 'status' and \ 121 | reply['content']['execution_state'] == 'idle': 122 | break 123 | 124 | # Consume the shell message that the execution produced. 125 | try: 126 | reply = self.kc.get_shell_msg(timeout=self.execute_timeout) 127 | except TimeoutError: 128 | # Timeout usually means that the kernel has crashed. 129 | raise ExecuteCrash(cell_index) 130 | if reply['content']['status'] != 'ok': 131 | raise ExecuteError(cell_index, reply, stdout) 132 | 133 | if cell_index >= 0: 134 | self.stdout.append(stdout) 135 | 136 | return stdout 137 | 138 | def _complete(self, cell_index, char_index): 139 | code = self.code_cells[cell_index].source[:char_index] 140 | try: 141 | reply = self.kc.complete(code, reply=True, timeout=self.complete_timeout) 142 | except TimeoutError: 143 | # Timeout usually means that the kernel has crashed. 144 | raise CompleteCrash(cell_index, char_index) 145 | if reply['content']['status'] != 'ok': 146 | raise CompleteError(cell_index, char_index) 147 | 148 | # Consume all the iopub messages that the completion produced. 149 | while True: 150 | try: 151 | reply = self.kc.get_iopub_msg(timeout=self.execute_timeout) 152 | except TimeoutError: 153 | # Timeout usually means that the kernel has crashed. 154 | raise CompleteCrash(cell_index, char_index) 155 | if reply['header']['msg_type'] == 'status' and \ 156 | reply['content']['execution_state'] == 'idle': 157 | break 158 | 159 | def _init_kernel(self): 160 | km, kc = start_new_kernel(kernel_name='swift') 161 | self.km = km 162 | self.kc = kc 163 | 164 | # Runs each code cell in order, asking for completions in each cell along 165 | # the way. Raises an exception if there is an error or crash. Otherwise, 166 | # returns. 167 | def _run_notebook_once(self, failed_completions): 168 | for cell_index, cell in enumerate(self.code_cells): 169 | completion_times = [] 170 | 171 | # Don't do completions when `char_step` is 0. 172 | # Don't do completions when we already have 3 completion failures 173 | # in this cell. 174 | # Otherwise, ask for a completion every `char_step` chars. 175 | if self.char_step > 0 and \ 176 | len(failed_completions[cell_index]) < 3: 177 | for char_index in range(0, len(cell.source), self.char_step): 178 | if char_index in failed_completions[cell_index]: 179 | continue 180 | if self.verbose: 181 | print('Cell %d/%d: completing char %d/%d' % ( 182 | cell_index, len(self.code_cells), char_index, 183 | len(cell.source)), 184 | end='\r') 185 | start_time = time.time() 186 | self._complete(cell_index, char_index) 187 | completion_times.append(1000 * (time.time() - start_time)) 188 | 189 | 190 | # Execute the cell. 191 | if self.verbose: 192 | print('Cell %d/%d: executing ' % ( 193 | cell_index, len(self.code_cells)), 194 | end='\r') 195 | start_time = time.time() 196 | self._execute_cell(cell_index) 197 | execute_time = 1000 * (time.time() - start_time) 198 | 199 | # Report the results. 200 | report = 'Cell %d/%d: done' % (cell_index, len(self.code_cells)) 201 | report += ' - execute %.0f ms' % execute_time 202 | if len(failed_completions[cell_index]) > 0: 203 | # Don't report completion timings in cells with failed 204 | # completions, because they might be misleading. 205 | report += ' - completion error(s) occurred' 206 | elif len(completion_times) == 0: 207 | report += ' - no completions performed' 208 | else: 209 | report += ' - complete p50 %.0f ms' % ( 210 | numpy.percentile(completion_times, 50)) 211 | report += ' - complete p90 %.0f ms' % ( 212 | numpy.percentile(completion_times, 90)) 213 | report += ' - complete p99 %.0f ms' % ( 214 | numpy.percentile(completion_times, 99)) 215 | if self.verbose: 216 | print(report) 217 | 218 | def _record_error(self, e): 219 | cell = self.code_cells[e.cell_index] 220 | if hasattr(e, 'char_index'): 221 | code = cell.source[:e.char_index] 222 | else: 223 | code = cell.source 224 | 225 | error_description = { 226 | 'error': e, 227 | 'code': code, 228 | } 229 | 230 | if self.verbose: 231 | print('ERROR!\n%s\n\nCode:\n%s\n' % (e, code)) 232 | self.unexpected_errors.append(error_description) 233 | 234 | def run(self): 235 | # map from cell index to set of char indexes where completions failed 236 | failed_completions = defaultdict(set) 237 | 238 | while True: 239 | self._init_kernel() 240 | try: 241 | for _ in range(self.repeat_times): 242 | self._run_notebook_once(failed_completions) 243 | break 244 | except ExecuteException as ee: 245 | # Execution exceptions can't be recovered, so take note of the 246 | # error and stop the stress test. 247 | self._record_error(ee) 248 | break 249 | except CompleteException as ce: 250 | # Completion exceptions can be recovered! Restart the kernel 251 | # and don't ask for the broken completion next time. 252 | self._record_error(ce) 253 | failed_completions[ce.cell_index].add(ce.char_index) 254 | finally: 255 | self.km.shutdown_kernel(now=True) 256 | 257 | 258 | def parse_args(): 259 | parser = argparse.ArgumentParser( 260 | description='Executes all the cells in a Jupyter notebook, and ' 261 | 'requests completions along the way. Records and ' 262 | 'reports errors and kernel crashes that occur.') 263 | parser.add_argument('notebook', 264 | help='path to a notebook to run the test on') 265 | parser.add_argument('--char-step', type=int, default=1, 266 | help='number of chars to step per completion request. ' 267 | '0 disables completion requests') 268 | parser.add_argument('--repeat-times', type=int, default=1, 269 | help='run the notebook this many times, in the same ' 270 | 'kernel instance') 271 | parser.add_argument('--execute-timeout', type=int, default=15, 272 | help='number of seconds to wait for cell execution') 273 | parser.add_argument('--complete-timeout', type=int, default=5, 274 | help='number of seconds to wait for completion') 275 | return parser.parse_args() 276 | 277 | 278 | def _main(): 279 | args = parse_args() 280 | runner = NotebookTestRunner(**args.__dict__) 281 | runner.run() 282 | print(runner.unexpected_errors) 283 | 284 | 285 | if __name__ == '__main__': 286 | _main() 287 | -------------------------------------------------------------------------------- /test/test.py: -------------------------------------------------------------------------------- 1 | """Copy of "all_test_docker.py", for backwards-compatibility with CI scripts 2 | that call "test.py". 3 | 4 | TODO: Delete this after updating CI scripts. 5 | """ 6 | 7 | import unittest 8 | 9 | from tests.kernel_tests import * 10 | from tests.simple_notebook_tests import * 11 | from tests.tutorial_notebook_tests import * 12 | 13 | 14 | if __name__ == '__main__': 15 | unittest.main() 16 | -------------------------------------------------------------------------------- /test/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liuliu/swift-jupyter/534a9515a752d24e24dafb96de21da33f19ee512/test/tests/__init__.py -------------------------------------------------------------------------------- /test/tests/kernel_tests.py: -------------------------------------------------------------------------------- 1 | """Manually crafted tests testing specific features of the kernel. 2 | """ 3 | 4 | import unittest 5 | import jupyter_kernel_test 6 | import time 7 | 8 | from jupyter_client.manager import start_new_kernel 9 | 10 | class SwiftKernelTests(jupyter_kernel_test.KernelTests): 11 | language_name = 'swift' 12 | kernel_name = 'swift' 13 | 14 | code_hello_world = 'print("hello, world!")' 15 | 16 | code_execute_result = [ 17 | {'code': 'let x = 2; x', 'result': 'Use `print()` to show values.\n'}, 18 | ] 19 | 20 | code_generate_error = 'varThatIsntDefined' 21 | 22 | def setUp(self): 23 | self.flush_channels() 24 | 25 | def test_graphics_matplotlib(self): 26 | reply, output_msgs = self.execute_helper(code=""" 27 | %include "EnableIPythonDisplay.swift" 28 | """) 29 | self.assertEqual(reply['content']['status'], 'ok') 30 | 31 | reply, output_msgs = self.execute_helper(code=""" 32 | let np = Python.import("numpy") 33 | let plt = Python.import("matplotlib.pyplot") 34 | IPythonDisplay.shell.enable_matplotlib("inline") 35 | """) 36 | self.assertEqual(reply['content']['status'], 'ok') 37 | 38 | reply, output_msgs = self.execute_helper(code=""" 39 | let ys = np.arange(0, 10, 0.01) 40 | plt.plot(ys) 41 | plt.show() 42 | """) 43 | self.assertEqual(reply['content']['status'], 'ok') 44 | self.assertIn('image/png', output_msgs[0]['content']['data']) 45 | 46 | def test_extensions(self): 47 | reply, output_msgs = self.execute_helper(code=""" 48 | struct Foo{} 49 | """) 50 | self.assertEqual(reply['content']['status'], 'ok') 51 | reply, output_msgs = self.execute_helper(code=""" 52 | extension Foo { func f() -> Int { return 1 } } 53 | """) 54 | self.assertEqual(reply['content']['status'], 'ok') 55 | reply, output_msgs = self.execute_helper(code=""" 56 | print("Value of Foo().f() is", Foo().f()) 57 | """) 58 | self.assertEqual(reply['content']['status'], 'ok') 59 | self.assertIn("Value of Foo().f() is 1", output_msgs[0]['content']['text']) 60 | reply, output_msgs = self.execute_helper(code=""" 61 | extension Foo { func f() -> Int { return 2 } } 62 | """) 63 | self.assertEqual(reply['content']['status'], 'ok') 64 | reply, output_msgs = self.execute_helper(code=""" 65 | print("Value of Foo().f() is", Foo().f()) 66 | """) 67 | self.assertEqual(reply['content']['status'], 'ok') 68 | self.assertIn("Value of Foo().f() is 2", output_msgs[0]['content']['text']) 69 | 70 | def test_gradient_across_cells_error(self): 71 | reply, output_msgs = self.execute_helper(code=""" 72 | func square(_ x : Float) -> Float { return x * x } 73 | """) 74 | self.assertEqual(reply['content']['status'], 'ok') 75 | reply, output_msgs = self.execute_helper(code=""" 76 | print("5^2 is", square(5)) 77 | """) 78 | self.assertEqual(reply['content']['status'], 'ok') 79 | self.assertIn("5^2 is 25.0", output_msgs[0]['content']['text']) 80 | reply, output_msgs = self.execute_helper(code=""" 81 | print("gradient of square at 5 is", gradient(at: 5, in: square)) 82 | """) 83 | self.assertEqual(reply['content']['status'], 'error') 84 | self.assertIn("cannot differentiate functions that have not been marked '@differentiable'", 85 | reply['content']['traceback'][0]) 86 | 87 | def test_gradient_across_cells(self): 88 | reply, output_msgs = self.execute_helper(code=""" 89 | @differentiable 90 | func square(_ x : Float) -> Float { return x * x } 91 | """) 92 | self.assertEqual(reply['content']['status'], 'ok') 93 | reply, output_msgs = self.execute_helper(code=""" 94 | print("5^2 is", square(5)) 95 | """) 96 | self.assertEqual(reply['content']['status'], 'ok') 97 | self.assertIn("5^2 is 25.0", output_msgs[0]['content']['text']) 98 | reply, output_msgs = self.execute_helper(code=""" 99 | print("gradient of square at 5 is", gradient(at: 5, in: square)) 100 | """) 101 | self.assertEqual(reply['content']['status'], 'ok') 102 | self.assertIn("gradient of square at 5 is 10.0", output_msgs[0]['content']['text']) 103 | 104 | def test_error_runtime(self): 105 | reply, output_msgs = self.execute_helper(code=""" 106 | func a() { fatalError("oops") } 107 | """) 108 | self.assertEqual(reply['content']['status'], 'ok') 109 | a_cell = reply['content']['execution_count'] 110 | reply, output_msgs = self.execute_helper(code=""" 111 | print("hello") 112 | print("world") 113 | func b() { a() } 114 | """) 115 | self.assertEqual(reply['content']['status'], 'ok') 116 | b_cell = reply['content']['execution_count'] 117 | reply, output_msgs = self.execute_helper(code=""" 118 | b() 119 | """) 120 | self.assertEqual(reply['content']['status'], 'error') 121 | call_cell = reply['content']['execution_count'] 122 | 123 | stdout = output_msgs[0]['content']['text'] 124 | self.assertIn('Fatal error: oops', stdout) 125 | traceback = output_msgs[1]['content']['traceback'] 126 | all_tracebacks = '\n'.join(traceback) 127 | self.assertIn('Current stack trace:', all_tracebacks) 128 | self.assertIn('a()', all_tracebacks) 129 | # TODO(TF-495): Reenable these assertions. 130 | # self.assertIn('b() at :4:24' % b_cell, all_tracebacks) 131 | # self.assertIn('main at :2:13' % call_cell, all_tracebacks) 132 | 133 | def test_interrupt_execution(self): 134 | # Execute something to trigger debugger initialization, so that the 135 | # next cell executes quickly. 136 | self.execute_helper(code='1 + 1') 137 | 138 | msg_id = self.kc.execute(code="""while true {}""") 139 | 140 | # Give the kernel some time to actually start execution, because it 141 | # ignores interrupts that arrive when it's not actually executing. 142 | time.sleep(1) 143 | 144 | msg = self.kc.iopub_channel.get_msg(timeout=1) 145 | self.assertEqual(msg['content']['execution_state'], 'busy') 146 | 147 | self.km.interrupt_kernel() 148 | reply = self.kc.get_shell_msg(timeout=1) 149 | self.assertEqual(reply['content']['status'], 'error') 150 | 151 | while True: 152 | msg = self.kc.iopub_channel.get_msg(timeout=1) 153 | if msg['msg_type'] == 'status': 154 | self.assertEqual(msg['content']['execution_state'], 'idle') 155 | break 156 | 157 | # Check that the kernel can still execute things after handling an 158 | # interrupt. 159 | reply, output_msgs = self.execute_helper( 160 | code="""print("Hello world")""") 161 | self.assertEqual(reply['content']['status'], 'ok') 162 | for msg in output_msgs: 163 | if msg['msg_type'] == 'stream' and \ 164 | msg['content']['name'] == 'stdout': 165 | self.assertIn('Hello world', msg['content']['text']) 166 | break 167 | 168 | def test_async_stdout(self): 169 | # Execute something to trigger debugger initialization, so that the 170 | # next cell executes quickly. 171 | self.execute_helper(code='1 + 1') 172 | 173 | # Test that we receive stdout while execution is happening by printing 174 | # something and then entering an infinite loop. 175 | msg_id = self.kc.execute(code=""" 176 | print("some stdout") 177 | while true {} 178 | """) 179 | 180 | # Give the kernel some time to send out the stdout. 181 | time.sleep(1) 182 | 183 | # Check that the kernel has sent out the stdout. 184 | while True: 185 | msg = self.kc.iopub_channel.get_msg(timeout=1) 186 | if msg['msg_type'] == 'stream' and \ 187 | msg['content']['name'] == 'stdout': 188 | self.assertIn('some stdout', msg['content']['text']) 189 | break 190 | 191 | # Interrupt execution and consume all messages, so that subsequent 192 | # tests can run. (All the tests in this class run against the same 193 | # instance of the kernel.) 194 | self.km.interrupt_kernel() 195 | self.kc.get_shell_msg(timeout=1) 196 | while True: 197 | msg = self.kc.iopub_channel.get_msg(timeout=1) 198 | if msg['msg_type'] == 'status': 199 | break 200 | 201 | def test_swift_completion(self): 202 | reply, output_msgs = self.execute_helper(code=""" 203 | func aFunctionToComplete() {} 204 | """) 205 | self.assertEqual(reply['content']['status'], 'ok') 206 | 207 | self.kc.complete('aFunctionToC') 208 | reply = self.kc.get_shell_msg() 209 | self.assertEqual(reply['content']['matches'], 210 | ['aFunctionToComplete()']) 211 | self.flush_channels() 212 | 213 | reply, output_msgs = self.execute_helper(code=""" 214 | %disableCompletion 215 | """) 216 | self.assertEqual(reply['content']['status'], 'ok') 217 | 218 | self.kc.complete('aFunctionToC') 219 | reply = self.kc.get_shell_msg() 220 | self.assertEqual(reply['content']['matches'], []) 221 | self.flush_channels() 222 | 223 | reply, output_msgs = self.execute_helper(code=""" 224 | %enableCompletion 225 | """) 226 | self.assertEqual(reply['content']['status'], 'ok') 227 | 228 | self.kc.complete('aFunctionToC') 229 | reply = self.kc.get_shell_msg() 230 | self.assertEqual(reply['content']['matches'], 231 | ['aFunctionToComplete()']) 232 | self.flush_channels() 233 | 234 | def test_swift_clear_output(self): 235 | reply, output_msgs = self.execute_helper(code=r""" 236 | print("before the clear") 237 | print("\u{001B}[2J") 238 | print("after the clear") 239 | """) 240 | self.assertEqual(reply['content']['status'], 'ok') 241 | self.assertEqual( 242 | dict((k, output_msgs[0][k]) for k in ['msg_type', 'content']), 243 | { 244 | 'msg_type': 'stream', 245 | 'content': { 246 | 'name': 'stdout', 247 | 'text': 'before the clear\r\n', 248 | }, 249 | }) 250 | self.assertEqual( 251 | dict((k, output_msgs[1][k]) for k in ['msg_type', 'content']), 252 | { 253 | 'msg_type': 'clear_output', 254 | 'content': { 255 | 'wait': False 256 | } 257 | }) 258 | self.assertEqual( 259 | dict((k, output_msgs[2][k]) for k in ['msg_type', 'content']), 260 | { 261 | 'msg_type': 'stream', 262 | 'content': { 263 | 'name': 'stdout', 264 | 'text': '\r\nafter the clear\r\n', 265 | }, 266 | }) 267 | 268 | def test_show_tensor(self): 269 | reply, output_msgs = self.execute_helper(code=""" 270 | import TensorFlow 271 | Tensor([1, 2, 3]) 272 | """) 273 | self.assertEqual(reply['content']['status'], 'ok') 274 | if 'data' in output_msgs[0]['content']: 275 | self.assertIn( 276 | "Use `print()` to show values", 277 | output_msgs[0]['content']['data']['text/plain']) 278 | 279 | 280 | # Class for tests that need their own kernel. (`SwiftKernelTestsBase` uses one 281 | # kernel for all the tests.) 282 | class OwnKernelTests(unittest.TestCase): 283 | def test_process_killed(self): 284 | km, kc = start_new_kernel(kernel_name='swift') 285 | kc.execute(""" 286 | import Glibc 287 | exit(0) 288 | """) 289 | messages = self.wait_for_idle(kc) 290 | 291 | had_error = False 292 | for message in messages: 293 | if message['header']['msg_type'] == 'error': 294 | had_error = True 295 | self.assertEqual(['Process killed'], 296 | message['content']['traceback']) 297 | self.assertTrue(had_error) 298 | 299 | def test_install_after_execute(self): 300 | # The kernel is supposed to refuse to install package after executing 301 | # code. 302 | 303 | km, kc = start_new_kernel(kernel_name='swift') 304 | kc.execute('1 + 1') 305 | self.wait_for_idle(kc) 306 | kc.execute(""" 307 | %install DummyPackage DummyPackage 308 | """) 309 | messages = self.wait_for_idle(kc) 310 | 311 | had_error = False 312 | for message in messages: 313 | if message['header']['msg_type'] == 'error': 314 | had_error = True 315 | self.assertIn('Install Error: Packages can only be installed ' 316 | 'during the first cell execution.', 317 | message['content']['traceback'][0]) 318 | self.assertTrue(had_error) 319 | 320 | def test_install_after_execute_blank(self): 321 | # If the user executes blank code, the kernel is supposed to try 322 | # to install packages. In particular, Colab sends a blank execution 323 | # request to the kernel when it starts up, and it's important that this 324 | # doesn't block package installation. 325 | 326 | km, kc = start_new_kernel(kernel_name='swift') 327 | kc.execute('\n\n\n') 328 | self.wait_for_idle(kc) 329 | kc.execute(""" 330 | %install DummyPackage DummyPackage 331 | """) 332 | messages = self.wait_for_idle(kc) 333 | 334 | # DummyPackage doesn't exist, so package installation won't actually 335 | # succeed. So we just assert that the kernel tries to install it. 336 | stdout = '' 337 | for message in messages: 338 | if message['header']['msg_type'] == 'stream' and \ 339 | message['content']['name'] == 'stdout': 340 | stdout += message['content']['text'] 341 | self.assertIn('Installing packages:', stdout) 342 | 343 | def wait_for_idle(self, kc): 344 | messages = [] 345 | while True: 346 | message = kc.get_iopub_msg(timeout=30) 347 | messages.append(message) 348 | if message['header']['msg_type'] == 'status' and \ 349 | message['content']['execution_state'] == 'idle': 350 | break 351 | return messages 352 | -------------------------------------------------------------------------------- /test/tests/notebooks/PackageWithC/Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:4.2 2 | 3 | import PackageDescription 4 | 5 | let package = Package( 6 | name: "PackageWithC", 7 | products: [ 8 | .library(name: "PackageWithC", targets: ["PackageWithC1", "PackageWithC2"]), 9 | ], 10 | dependencies: [], 11 | targets: [ 12 | .target( 13 | name: "PackageWithC1", 14 | dependencies: []), 15 | .target( 16 | name: "PackageWithC2", 17 | dependencies: []), 18 | ] 19 | ) 20 | -------------------------------------------------------------------------------- /test/tests/notebooks/PackageWithC/Sources/PackageWithC1/include/sillyfunction1.h: -------------------------------------------------------------------------------- 1 | int sillyfunction1(); 2 | -------------------------------------------------------------------------------- /test/tests/notebooks/PackageWithC/Sources/PackageWithC1/sillyfunction1.c: -------------------------------------------------------------------------------- 1 | int sillyfunction1() { 2 | return 42; 3 | } 4 | -------------------------------------------------------------------------------- /test/tests/notebooks/PackageWithC/Sources/PackageWithC2/include/sillyfunction2.h: -------------------------------------------------------------------------------- 1 | int sillyfunction2(); 2 | -------------------------------------------------------------------------------- /test/tests/notebooks/PackageWithC/Sources/PackageWithC2/sillyfunction2.c: -------------------------------------------------------------------------------- 1 | int sillyfunction2() { 2 | return MACRO_DEFINED_IN_COMPILER_FLAG; 3 | } 4 | -------------------------------------------------------------------------------- /test/tests/notebooks/SimplePackage/Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:4.2 2 | 3 | import PackageDescription 4 | 5 | let package = Package( 6 | name: "SimplePackage", 7 | products: [ 8 | .library(name: "SimplePackage", targets: ["SimplePackage"]), 9 | ], 10 | dependencies: [], 11 | targets: [ 12 | .target( 13 | name: "SimplePackage", 14 | dependencies: []), 15 | ] 16 | ) 17 | -------------------------------------------------------------------------------- /test/tests/notebooks/SimplePackage/Sources/SimplePackage/SimplePackage.swift: -------------------------------------------------------------------------------- 1 | public let publicIntThatIsInSimplePackage: Int = 42 2 | -------------------------------------------------------------------------------- /test/tests/notebooks/install_package.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "%install '.package(path: \"$cwd/SimplePackage\")' SimplePackage" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "import SimplePackage" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "print(publicIntThatIsInSimplePackage)" 28 | ] 29 | } 30 | ], 31 | "metadata": { 32 | "kernelspec": { 33 | "display_name": "Swift", 34 | "language": "swift", 35 | "name": "swift" 36 | } 37 | }, 38 | "nbformat": 4, 39 | "nbformat_minor": 2 40 | } 41 | -------------------------------------------------------------------------------- /test/tests/notebooks/install_package_with_c.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "%install-swiftpm-flags -Xcc -DMACRO_DEFINED_IN_COMPILER_FLAG=1337\n", 10 | "%install '.package(path: \"$cwd/PackageWithC\")' PackageWithC" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "import PackageWithC1\n", 20 | "import PackageWithC2" 21 | ] 22 | }, 23 | { 24 | "cell_type": "code", 25 | "execution_count": null, 26 | "metadata": {}, 27 | "outputs": [], 28 | "source": [ 29 | "print(sillyfunction1())" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "execution_count": null, 35 | "metadata": {}, 36 | "outputs": [], 37 | "source": [ 38 | "print(sillyfunction2())" 39 | ] 40 | } 41 | ], 42 | "metadata": { 43 | "kernelspec": { 44 | "display_name": "Swift", 45 | "language": "swift", 46 | "name": "swift" 47 | }, 48 | "language_info": { 49 | "file_extension": ".swift", 50 | "mimetype": "text/x-swift", 51 | "name": "swift", 52 | "version": "" 53 | } 54 | }, 55 | "nbformat": 4, 56 | "nbformat_minor": 2 57 | } 58 | -------------------------------------------------------------------------------- /test/tests/notebooks/install_package_with_user_location.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "%install-location $cwd/swift-modules\n", 10 | "%install '.package(path: \"$cwd/SimplePackage\")' SimplePackage" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "import SimplePackage" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "print(publicIntThatIsInSimplePackage)" 29 | ] 30 | } 31 | ], 32 | "metadata": { 33 | "kernelspec": { 34 | "display_name": "Swift", 35 | "language": "swift", 36 | "name": "swift" 37 | }, 38 | "language_info": { 39 | "file_extension": ".swift", 40 | "mimetype": "text/x-swift", 41 | "name": "swift", 42 | "version": "" 43 | } 44 | }, 45 | "nbformat": 4, 46 | "nbformat_minor": 2 47 | } 48 | -------------------------------------------------------------------------------- /test/tests/notebooks/intentional_compile_error.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "Hello World\r\n" 13 | ] 14 | } 15 | ], 16 | "source": [ 17 | "print(\"Hello World\")" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 2, 23 | "metadata": {}, 24 | "outputs": [ 25 | { 26 | "ename": "", 27 | "evalue": "", 28 | "output_type": "error", 29 | "traceback": [ 30 | "error: :1:1: error: use of unresolved identifier 'intentionalCompileError'\nintentionalCompileError\n^~~~~~~~~~~~~~~~~~~~~~~\n\n" 31 | ] 32 | } 33 | ], 34 | "source": [ 35 | "intentionalCompileError" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": null, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [] 44 | } 45 | ], 46 | "metadata": { 47 | "kernelspec": { 48 | "display_name": "Swift", 49 | "language": "swift", 50 | "name": "swift" 51 | }, 52 | "language_info": { 53 | "file_extension": ".swift", 54 | "mimetype": "text/x-swift", 55 | "name": "swift", 56 | "version": "" 57 | } 58 | }, 59 | "nbformat": 4, 60 | "nbformat_minor": 2 61 | } 62 | -------------------------------------------------------------------------------- /test/tests/notebooks/intentional_runtime_error.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "Hello World\r\n" 13 | ] 14 | } 15 | ], 16 | "source": [ 17 | "print(\"Hello World\")" 18 | ] 19 | }, 20 | { 21 | "cell_type": "code", 22 | "execution_count": 2, 23 | "metadata": {}, 24 | "outputs": [ 25 | { 26 | "name": "stdout", 27 | "output_type": "stream", 28 | "text": [ 29 | "Fatal error: intentional runtime error: file , line 1\r\n", 30 | "Current stack trace:\r\n", 31 | "0 libswiftCore.so 0x00007f01af2f9320 _swift_stdlib_reportFatalErrorInFile + 115\r\n", 32 | "1 libswiftCore.so 0x00007f01af240dec + 3079660\r\n", 33 | "2 libswiftCore.so 0x00007f01af240ede + 3079902\r\n", 34 | "3 libswiftCore.so 0x00007f01af088752 + 1275730\r\n", 35 | "4 libswiftCore.so 0x00007f01af20b0c2 + 2859202\r\n", 36 | "5 libswiftCore.so 0x00007f01af087b99 + 1272729\r\n" 37 | ] 38 | }, 39 | { 40 | "ename": "", 41 | "evalue": "", 42 | "output_type": "error", 43 | "traceback": [ 44 | "Current stack trace:", 45 | "\tframe #2: 0x00007f01ba316d8b $__lldb_expr22`main at :1:1" 46 | ] 47 | } 48 | ], 49 | "source": [ 50 | "fatalError(\"intentional runtime error\")" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": null, 56 | "metadata": {}, 57 | "outputs": [], 58 | "source": [] 59 | } 60 | ], 61 | "metadata": { 62 | "kernelspec": { 63 | "display_name": "Swift", 64 | "language": "swift", 65 | "name": "swift" 66 | }, 67 | "language_info": { 68 | "file_extension": ".swift", 69 | "mimetype": "text/x-swift", 70 | "name": "swift", 71 | "version": "" 72 | } 73 | }, 74 | "nbformat": 4, 75 | "nbformat_minor": 2 76 | } 77 | -------------------------------------------------------------------------------- /test/tests/notebooks/simple_successful.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "let x = 1" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 2, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "let y = 2" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 3, 24 | "metadata": {}, 25 | "outputs": [ 26 | { 27 | "name": "stdout", 28 | "output_type": "stream", 29 | "text": [ 30 | "Hello World: 3\r\n" 31 | ] 32 | } 33 | ], 34 | "source": [ 35 | "print(\"Hello World: \\(x+y)\")" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": null, 41 | "metadata": {}, 42 | "outputs": [], 43 | "source": [] 44 | } 45 | ], 46 | "metadata": { 47 | "kernelspec": { 48 | "display_name": "Swift", 49 | "language": "swift", 50 | "name": "swift" 51 | }, 52 | "language_info": { 53 | "file_extension": ".swift", 54 | "mimetype": "text/x-swift", 55 | "name": "swift", 56 | "version": "" 57 | } 58 | }, 59 | "nbformat": 4, 60 | "nbformat_minor": 2 61 | } 62 | -------------------------------------------------------------------------------- /test/tests/simple_notebook_tests.py: -------------------------------------------------------------------------------- 1 | """Checks that simple notebooks behave as expected. 2 | """ 3 | 4 | import unittest 5 | import os 6 | 7 | from notebook_tester import ExecuteError 8 | from notebook_tester import NotebookTestRunner 9 | 10 | 11 | THIS_DIR = os.path.dirname(os.path.abspath(__file__)) 12 | NOTEBOOK_DIR = os.path.join(THIS_DIR, 'notebooks') 13 | 14 | 15 | class SimpleNotebookTests(unittest.TestCase): 16 | def test_simple_successful(self): 17 | notebook = os.path.join(NOTEBOOK_DIR, 'simple_successful.ipynb') 18 | runner = NotebookTestRunner(notebook, verbose=False) 19 | runner.run() 20 | self.assertEqual([], runner.unexpected_errors) 21 | self.assertIn('Hello World: 3', runner.stdout[2]) 22 | 23 | def test_intentional_compile_error(self): 24 | notebook = os.path.join(NOTEBOOK_DIR, 'intentional_compile_error.ipynb') 25 | runner = NotebookTestRunner(notebook, verbose=False) 26 | runner.run() 27 | self.assertEqual(1, len(runner.unexpected_errors)) 28 | self.assertIsInstance(runner.unexpected_errors[0]['error'], 29 | ExecuteError) 30 | self.assertEqual(1, runner.unexpected_errors[0]['error'].cell_index) 31 | 32 | def test_intentional_runtime_error(self): 33 | notebook = os.path.join(NOTEBOOK_DIR, 'intentional_runtime_error.ipynb') 34 | runner = NotebookTestRunner(notebook, verbose=False) 35 | runner.run() 36 | self.assertEqual(1, len(runner.unexpected_errors)) 37 | self.assertIsInstance(runner.unexpected_errors[0]['error'], 38 | ExecuteError) 39 | self.assertEqual(1, runner.unexpected_errors[0]['error'].cell_index) 40 | 41 | def test_install_package(self): 42 | notebook = os.path.join(NOTEBOOK_DIR, 'install_package.ipynb') 43 | runner = NotebookTestRunner(notebook, char_step=0, verbose=False) 44 | runner.run() 45 | self.assertIn('Installation complete', runner.stdout[0]) 46 | self.assertIn('42', runner.stdout[2]) 47 | 48 | def test_install_package_with_c(self): 49 | notebook = os.path.join(NOTEBOOK_DIR, 'install_package_with_c.ipynb') 50 | runner = NotebookTestRunner(notebook, char_step=0, verbose=False) 51 | runner.run() 52 | self.assertIn('Installation complete', runner.stdout[0]) 53 | self.assertIn('42', runner.stdout[2]) 54 | self.assertIn('1337', runner.stdout[3]) 55 | -------------------------------------------------------------------------------- /test/tests/tutorial_notebook_tests.py: -------------------------------------------------------------------------------- 1 | # TODO(TF-747): Reenable. 2 | 3 | # """Checks that tutorial notebooks behave as expected. 4 | # """ 5 | # 6 | # import unittest 7 | # import os 8 | # import shutil 9 | # import tempfile 10 | # 11 | # from flaky import flaky 12 | # 13 | # from notebook_tester import NotebookTestRunner 14 | # 15 | # 16 | # class TutorialNotebookTests(unittest.TestCase): 17 | # @classmethod 18 | # def setUpClass(cls): 19 | # cls.tmp_dir = tempfile.mkdtemp() 20 | # git_url = 'https://github.com/tensorflow/swift.git' 21 | # os.system('git clone %s %s -b jupyter-test-branch' % (git_url, cls.tmp_dir)) 22 | # 23 | # @classmethod 24 | # def tearDownClass(cls): 25 | # shutil.rmtree(cls.tmp_dir) 26 | # 27 | # @flaky(max_runs=5, min_passes=1) 28 | # def test_iris(self): 29 | # notebook = os.path.join(self.tmp_dir, 'docs', 'site', 'tutorials', 30 | # 'model_training_walkthrough.ipynb') 31 | # runner = NotebookTestRunner(notebook, verbose=False) 32 | # runner.run() 33 | # self.assertEqual([], runner.unexpected_errors) 34 | # all_stdout = '\n\n'.join(runner.stdout) 35 | # self.assertIn('Epoch 100:', all_stdout) 36 | # self.assertIn('Example 2 prediction:', all_stdout) 37 | --------------------------------------------------------------------------------