├── LICENSE.md
├── README.md
├── julia-ts-misc.el
├── julia-ts-mode.el
└── test
    ├── ArgTools.jl
    ├── ArgTools.jl.faceup
    ├── Downloads.jl
    ├── Downloads.jl.faceup
    ├── test.el
    ├── tree-sitter-corpus.jl
    └── tree-sitter-corpus.jl.faceup


/LICENSE.md:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2022-2023 Ronan Arraes Jardim Chagas
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Julia major mode using tree-sitter
 2 | 
 3 | Emacs major mode for the Julia programming language using tree-sitter for
 4 | font-lock, indentation, imenu, and navigation. It is derived from
 5 | [`julia-mode`](https://github.com/JuliaEditorSupport/julia-emacs).
 6 | 
 7 | # Installation
 8 | 
 9 | ## Dependencies
10 | 
11 | This package requires:
12 | 
13 | 1. Emacs 29 or newer built with tree-sitter support;
14 | 2. [Julia tree-sitter grammar](https://github.com/tree-sitter/tree-sitter-julia); and
15 | 3. The package [`julia-mode`](https://github.com/JuliaEditorSupport/julia-emacs).
16 | 
17 | ## Installation from MELPA
18 | 
19 | `julia-ts-mode` is available on MELPA. The recommended way to install it is
20 | using `use-package`. The following code installs `julia-ts-mode` and selects it
21 | as the default major mode for Julia files:
22 | 
23 | ``` emacs-lisp
24 | (use-package julia-ts-mode
25 |   :ensure t
26 |   :mode "\\.jl$")
27 | ```
28 | 
29 | ## Installation from source
30 | 
31 | You can install this package from source by cloning this directory and adding
32 | the following lines to your Emacs configuration:
33 | 
34 | ``` emacs-lisp
35 | (add-to-list 'load-path "<path to the source-code tree>")
36 | (require 'julia-ts-mode)
37 | ```
38 | 
39 | ## Installing Julia Tree-Sitter Grammar
40 | 
41 | The recommended way to install the tree-sitter grammar is using the package
42 | [treesit-auto](https://github.com/renzmann/treesit-auto). In this case, the
43 | Julia tree-sitter grammar will be automatically downloaded and compiled when a
44 | Julia file is opened using the major mode `julia-ts-mode`. For more information
45 | on how to install and configure `treesit-auto`, check the package documentation.
46 | 
47 | ### Upgrading grammar after a package update
48 | 
49 | Note that the [tree-sitter grammar for
50 | Julia](https://github.com/tree-sitter/tree-sitter-julia) is under active
51 | development, and breaking changes are often introduced between versions. To
52 | provide maximum compatibility, the exact version of the grammar library is fixed
53 | in `julia-ts-mode.el`, with planned regular updates to both the grammar version
54 | and the tree-sitter queries built on top of it. However, the grammar library
55 | itself isn't updated automatically by Emacs. If, after a `julia-ts-mode` package
56 | update, you get some quirky tree-sitter error messages, and font-locking fails
57 | to work in a `julia-ts-mode` buffer, run `M-x treesit-install-language-grammar
58 | RET julia RET` to upgrade the grammar library. This should resolve the issues.
59 | 
60 | # LSP Configuration
61 | 
62 | This mode is derived from `julia-mode`. Hence, most of the feature available for
63 | it will also work in `julia-ts-mode`. However, the LSP requires additional
64 | configuration. First, it is necessary to install the package
65 | [`lsp-julia`](https://github.com/gdkrmr/lsp-julia), and apply the desired
66 | configuration as stated in its documentation. Afterward, add the following code
67 | to your Emacs configuration file:
68 | 
69 | ``` emacs-lisp
70 | (add-to-list 'lsp-language-id-configuration '(julia-ts-mode . "julia"))
71 | (lsp-register-client
72 | (make-lsp-client :new-connection (lsp-stdio-connection 'lsp-julia--rls-command)
73 |                  :major-modes '(julia-mode ess-julia-mode julia-ts-mode)
74 |                  :server-id 'julia-ls
75 |                  :multi-root t))
76 | ```
77 | 


--------------------------------------------------------------------------------
/julia-ts-misc.el:
--------------------------------------------------------------------------------
  1 | ;;; julia-ts-misc.el --- Miscellaneous functions for the julia-ts-mode -*- lexical-binding: t; -*-
  2 | 
  3 | ;;; License:
  4 | ;; Permission is hereby granted, free of charge, to any person obtaining
  5 | ;; a copy of this software and associated documentation files (the
  6 | ;; "Software"), to deal in the Software without restriction, including
  7 | ;; without limitation the rights to use, copy, modify, merge, publish,
  8 | ;; distribute, sublicense, and/or sell copies of the Software, and to
  9 | ;; permit persons to whom the Software is furnished to do so, subject to
 10 | ;; the following conditions:
 11 | ;;
 12 | ;; The above copyright notice and this permission notice shall be
 13 | ;; included in all copies or substantial portions of the Software.
 14 | ;;
 15 | ;; THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 16 | ;; EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 17 | ;; MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 18 | ;; NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 19 | ;; LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 20 | ;; OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 21 | ;; WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 22 | ;;
 23 | ;;; Commentary:
 24 | ;; This file defines miscellaneous functions used in the `julia-ts-mode'.
 25 | 
 26 | ;;;; Code:
 27 | 
 28 | (defun julia-ts--ancestor-bol (regexp)
 29 |   "Return the BOL of the current node's ancestor that matches REGEXP."
 30 |   (lambda (node &rest _)
 31 |       (treesit-node-start (julia-ts--ancestor-node node regexp))))
 32 | 
 33 | (defun julia-ts--ancestor-is (regexp)
 34 |   "Return the ancestor of NODE that matches `REGEXP', if it exists."
 35 |   (lambda (node &rest _)
 36 |     (julia-ts--ancestor-node node regexp)))
 37 | 
 38 | (defun julia-ts--ancestor-is-and-sibling-on-same-line (ancestor-type sibling-index)
 39 |   "Check the type of the node's ancestor and if a sibling is on the same line.
 40 | 
 41 | Return t if the node's ancestor type is ANCESTOR-TYPE and if the sibling with
 42 | index SIBLING-INDEX is on the same line of the ancestor."
 43 |   (lambda (node &rest _)
 44 |     (let ((ancestor (julia-ts--ancestor-node node ancestor-type)))
 45 |       (and ancestor
 46 |            (julia-ts--same-line? (treesit-node-start ancestor)
 47 |                                  (treesit-node-start (treesit-node-child ancestor sibling-index)))))))
 48 | 
 49 | (defun julia-ts--ancestor-is-and-sibling-not-on-same-line (ancestor-type sibling-index)
 50 |   "Check the type of the node's ancestor and if a sibling is not on the same line.
 51 | 
 52 | Return t if the node's ancestor type is ANCESTOR-TYPE and if the sibling with
 53 | index SIBLING-INDEX is not on the same line of the ancestor."
 54 |   (lambda (node &rest _)
 55 |     (let ((ancestor (julia-ts--ancestor-node node ancestor-type)))
 56 |       (and ancestor
 57 |            (not (julia-ts--same-line? (treesit-node-start ancestor)
 58 |                                       (treesit-node-start (treesit-node-child ancestor sibling-index))))))))
 59 | 
 60 | (defun julia-ts--ancestor-node (node regexp)
 61 |   "Return the ancestor NODE that matches REGEXP, if it exists."
 62 |   (treesit-parent-until node
 63 |                         (lambda (node)
 64 |                           (string-match-p regexp (treesit-node-type node)))))
 65 | 
 66 | (defun julia-ts--grand-parent-bol (_n parent &rest _)
 67 |   "Return the beginning of the line (non-space char) where the node's PARENT is on."
 68 |   (save-excursion
 69 |     (goto-char (treesit-node-start (treesit-node-parent parent)))
 70 |     (back-to-indentation)
 71 |     (point)))
 72 | 
 73 | (defun julia-ts--grand-parent-first-sibling (_n parent &rest _)
 74 |   "Return the start of the first child of the parent of the node PARENT."
 75 |   (treesit-node-start (treesit-node-child (treesit-node-parent parent) 0)))
 76 | 
 77 | (defun julia-ts--line-beginning-position-of-point (point)
 78 |   "Return the position of the beginning of the line of POINT."
 79 |   (save-mark-and-excursion
 80 |     (goto-char point)
 81 |     (line-beginning-position)))
 82 | 
 83 | (defun julia-ts--parent-is-and-sibling-on-same-line (parent-type sibling-index)
 84 |   "Check the type of the node's parent and if a sibling is on the same line.
 85 | 
 86 | Return t if the node's parent type is PARENT-TYPE and if the sibling with
 87 | index SIBLING-INDEX is on the same line of the current node's parent."
 88 |   (lambda (_node parent &rest _)
 89 |     (and (string-match-p (treesit-node-type parent) parent-type)
 90 |          (julia-ts--same-line? (treesit-node-start parent)
 91 |                                (treesit-node-start (treesit-node-child parent sibling-index))))))
 92 | 
 93 | (defun julia-ts--parent-is-and-sibling-not-on-same-line (parent-type sibling-index)
 94 |   "Check the type of the node's parent and if a sibling is not on the same line.
 95 | 
 96 | Return t if the node's parent type is PARENT-TYPE and if the sibling with
 97 | index SIBLING-INDEX is not on the same line of the current node's parent."
 98 |   (lambda (_node parent &rest _)
 99 |     (and (string-match-p (treesit-node-type parent) parent-type)
100 |          (not (julia-ts--same-line? (treesit-node-start parent)
101 |                                     (treesit-node-start (treesit-node-child parent sibling-index)))))))
102 | 
103 | (defun julia-ts--same-line? (point-1 point-2)
104 |   "Return t if POINT-1 and POINT-2 are on the same line."
105 |   (equal (julia-ts--line-beginning-position-of-point point-1)
106 |          (julia-ts--line-beginning-position-of-point point-2)))
107 | 
108 | (provide 'julia-ts-misc)
109 | 
110 | ;; Local Variables:
111 | ;; coding: utf-8
112 | ;; byte-compile-warnings: (not obsolete)
113 | ;; End:
114 | ;;; julia-ts-misc.el ends here
115 | 


--------------------------------------------------------------------------------
/julia-ts-mode.el:
--------------------------------------------------------------------------------
  1 | ;;; julia-ts-mode.el --- Major mode for Julia source code using tree-sitter -*- lexical-binding: t; -*-
  2 | 
  3 | ;; Copyright (C) 2022, 2023  Ronan Arraes Jardim Chagas
  4 | ;;
  5 | ;; Author           : Ronan Arraes Jardim Chagas
  6 | ;; Created          : December 2022
  7 | ;; Keywords         : julia languages tree-sitter
  8 | ;; Package-Requires : ((emacs "29.1") (julia-mode "0.4"))
  9 | ;; URL              : https://github.com/ronisbr/julia-ts-mode
 10 | ;; Version          : 0.3
 11 | ;;
 12 | ;;; License:
 13 | ;; Permission is hereby granted, free of charge, to any person obtaining
 14 | ;; a copy of this software and associated documentation files (the
 15 | ;; "Software"), to deal in the Software without restriction, including
 16 | ;; without limitation the rights to use, copy, modify, merge, publish,
 17 | ;; distribute, sublicense, and/or sell copies of the Software, and to
 18 | ;; permit persons to whom the Software is furnished to do so, subject to
 19 | ;; the following conditions:
 20 | ;;
 21 | ;; The above copyright notice and this permission notice shall be
 22 | ;; included in all copies or substantial portions of the Software.
 23 | ;;
 24 | ;; THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 25 | ;; EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 26 | ;; MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 27 | ;; NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 28 | ;; LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 29 | ;; OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 30 | ;; WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 31 | ;;
 32 | ;;; Commentary:
 33 | ;; This major modes uses tree-sitter for font-lock, indentation, imenu, and
 34 | ;; navigation. It is derived from `julia-mode'.
 35 | 
 36 | ;;;; Code:
 37 | 
 38 | (require 'treesit)
 39 | (eval-when-compile (require 'rx))
 40 | (require 'julia-mode)
 41 | (require 'julia-ts-misc)
 42 | 
 43 | ;; Fix grammar version to 0.23.1, which is known to be compatible with the
 44 | ;; treesitter queries defined in the present version of the mode definition.
 45 | (add-to-list 'treesit-language-source-alist
 46 |              '(julia "https://github.com/tree-sitter/tree-sitter-julia" "v0.23.1"))
 47 | 
 48 | (declare-function treesit-parser-create "treesit.c")
 49 | 
 50 | (defgroup julia-ts nil
 51 |   "Major mode for the julia programming language using tree-sitter."
 52 |   :group 'languages
 53 |   :prefix "julia-ts-")
 54 | ;; Notice that all custom variables and faces are automatically added to the
 55 | ;; most recent group.
 56 | 
 57 | (defcustom julia-ts-align-argument-list-to-first-sibling nil
 58 |   "Align the argument list to the first sibling.
 59 | 
 60 | If it is set to t, the following indentation is used:
 61 | 
 62 |     myfunc(a, b,
 63 |            c, d)
 64 | 
 65 | Otherwise, the indentation is:
 66 | 
 67 |     myfunc(a, b
 68 |         c, d)"
 69 |   :version "29.1"
 70 |   :type 'boolean)
 71 | 
 72 | (defcustom julia-ts-align-assignment-expressions-to-first-sibling nil
 73 |   "Align the expressions after an assignment to the first sibling.
 74 | 
 75 | If it is set to t, the following indentation is used:
 76 | 
 77 |     var = a + b + c +
 78 |           d + e +
 79 |           f
 80 | 
 81 | Otherwise, the indentation is:
 82 | 
 83 |     var = a + b + c +
 84 |         d + e +
 85 |         f"
 86 |   :version "29.1"
 87 |   :type 'boolean)
 88 | 
 89 | ;; As of the grammar version 0.22, it uses the argument_list node for both
 90 | ;; function definitions and calls.
 91 | (define-obsolete-variable-alias
 92 |   'julia-ts-align-parameter-list-to-first-sibling
 93 |   'julia-ts-align-argument-list-to-first-sibling
 94 |   "0.3")
 95 | 
 96 | (defcustom julia-ts-align-curly-brace-expressions-to-first-sibling nil
 97 |   "Align curly brace expressions to the first sibling.
 98 | 
 99 | If it is set to t, the following indentation is used:
100 | 
101 |     MyType{A, B,
102 |            C, D}
103 | 
104 | Otherwise, the indentation is:
105 | 
106 |     MyType{A, B
107 |         C, D}"
108 |   :version "29.1"
109 |   :type 'boolean)
110 | 
111 | (defcustom julia-ts-indent-offset 4
112 |   "Number of spaces for each indentation step in `julia-ts-mode'."
113 |   :version "29.1"
114 |   :type 'integer
115 |   :safe 'intergerp)
116 | 
117 | (defface julia-ts-macro-face
118 |   '((t :inherit font-lock-preprocessor-face))
119 |   "Face for Julia macro invocations in `julia-ts-mode'.")
120 | 
121 | (defface julia-ts-quoted-symbol-face
122 |   '((t :inherit font-lock-constant-face))
123 |   "Face for quoted Julia symbols in `julia-ts-mode', e.g. :foo.")
124 | 
125 | (defface julia-ts-keyword-argument-face
126 |   '((t :inherit font-lock-constant-face))
127 |   "Face for keyword argument names in `julia-ts-mode'.")
128 | 
129 | (defface julia-ts-interpolation-expression-face
130 |   '((t :inherit font-lock-constant-face))
131 |   "Face for interpolation expressions in `julia-ts-mode', e.g. $foo.")
132 | 
133 | (defface julia-ts-string-interpolation-face
134 |   '((t :inherit font-lock-constant-face :weight bold))
135 |   "Face for string interpolations in `julia-ts-mode', e.g. \"$foo\".")
136 | 
137 | (defvar julia-ts--keywords
138 |   '("baremodule" "begin" "catch" "const" "do" "else" "elseif" "end" "export"
139 |     "finally" "for" "function" "global" "if" "import" "let" "local" "macro"
140 |     "module" "outer" "public" "quote" "return" "try" "using" "where" "while")
141 |   "Keywords for `julia-ts-mode'.")
142 | 
143 | (defvar julia-ts--treesit-font-lock-settings
144 |   (treesit-font-lock-rules
145 |    :language 'julia
146 |    :feature 'assignment
147 |    `((assignment :anchor [(identifier) (operator)] @font-lock-variable-name-face)
148 |      (assignment
149 |       :anchor
150 |       (field_expression
151 |        value: (identifier) "." (identifier) @font-lock-variable-name-face))
152 |      (assignment (open_tuple (identifier) @font-lock-variable-name-face))
153 |      (assignment
154 |       :anchor
155 |       (open_tuple
156 |        (field_expression
157 |         value: (identifier) "." (identifier) @font-lock-variable-name-face)))
158 |      (local_statement (identifier) @font-lock-variable-name-face)
159 |      (let_statement :anchor (identifier) @font-lock-variable-name-face)
160 |      ((let_statement _ @comma :anchor (identifier) @font-lock-variable-name-face)
161 |       (:equal "," @comma))
162 |      (let_binding :anchor (identifier) @font-lock-variable-name-face)
163 |      (global_statement (identifier) @font-lock-variable-name-face)
164 |      (named_argument (identifier) @julia-ts-keyword-argument-face (operator)))
165 | 
166 |    :language 'julia
167 |    :feature 'constant
168 |    `(((identifier) @font-lock-constant-face
169 |       (:match
170 |        "^\\(NaN\\|NaN16\\|NaN32\\|NaN64\\|Inf\\|Inf16\\|Inf32\\|Inf64\\|nothing\\|missing\\|undef\\)$"
171 |        @font-lock-constant-face)))
172 | 
173 |    :language 'julia
174 |    :feature 'comment
175 |    `((line_comment) @font-lock-comment-face
176 |      (block_comment) @font-lock-comment-face)
177 | 
178 |    :language 'julia
179 |    :feature 'definition
180 |    `((function_definition
181 |       (signature (identifier) @font-lock-function-name-face))
182 |      (function_definition
183 |       (signature
184 |        (call_expression [(identifier) (operator)] @font-lock-function-name-face)))
185 |      (function_definition
186 |       (signature
187 |        (typed_expression
188 |         (call_expression [(identifier) (operator)] @font-lock-function-name-face))))
189 |      (function_definition
190 |       (signature
191 |        (where_expression
192 |         (call_expression [(identifier) (operator)] @font-lock-function-name-face))))
193 |      (function_definition
194 |       (signature
195 |        (where_expression
196 |         (typed_expression
197 |         (call_expression [(identifier) (operator)] @font-lock-function-name-face)))))
198 |      (function_definition
199 |       (signature
200 |        (call_expression
201 |         (field_expression
202 |          value: (identifier) "." (identifier) @font-lock-function-name-face))))
203 |      (function_definition
204 |       (signature
205 |        (typed_expression
206 |         (call_expression
207 |          (field_expression
208 |           value: (identifier) "." (identifier) @font-lock-function-name-face)))))
209 |      (macro_definition
210 |       (signature
211 |        (call_expression (identifier) @font-lock-function-name-face)))
212 |      (macro_definition
213 |       (signature
214 |        (call_expression
215 |         (field_expression
216 |          value: (identifier) "." (identifier) @font-lock-function-name-face))))
217 |      (abstract_definition
218 |       (type_head (identifier) @font-lock-type-face))
219 |      (abstract_definition
220 |       (type_head (binary_expression (identifier) @font-lock-type-face)))
221 |      (primitive_definition
222 |       (type_head (identifier) @font-lock-type-face))
223 |      (primitive_definition
224 |       (type_head (binary_expression (identifier) @font-lock-type-face)))
225 |      (struct_definition
226 |       (type_head (identifier) @font-lock-type-face))
227 |      (struct_definition
228 |       (type_head (binary_expression (identifier) @font-lock-type-face)))
229 |      (assignment
230 |       :anchor
231 |       (call_expression [(identifier) (operator)] @font-lock-function-name-face))
232 |      (assignment
233 |       :anchor
234 |       (call_expression
235 |        (parenthesized_expression
236 |         [(identifier) (operator)] @font-lock-function-name-face)))
237 |      (assignment
238 |       :anchor
239 |       (call_expression
240 |        (field_expression
241 |         value: (identifier) "." (identifier) @font-lock-function-name-face)))
242 |      (assignment
243 |       :anchor
244 |       (where_expression
245 |        (call_expression (identifier) @font-lock-function-name-face)))
246 |      (assignment
247 |       :anchor
248 |       (where_expression
249 |        (call_expression
250 |         (field_expression
251 |          value: (identifier) "." (identifier) @font-lock-function-name-face))))
252 |      (assignment
253 |       :anchor
254 |       (binary_expression _ (operator) @font-lock-function-name-face)))
255 | 
256 |    :language 'julia
257 |    :feature 'error
258 |    :override t
259 |    `((ERROR) @font-lock-warning-face)
260 | 
261 |    :language 'julia
262 |    :feature 'keyword
263 |    `((abstract_definition ["abstract" "type"] @font-lock-keyword-face)
264 |      (primitive_definition ["primitive" "type"] @font-lock-keyword-face)
265 |      (struct_definition ["mutable" "struct"] @font-lock-keyword-face)
266 |      (break_statement) @font-lock-keyword-face
267 |      (continue_statement) @font-lock-keyword-face
268 |      ((operator) @font-lock-keyword-face
269 |       (:equal "in" @font-lock-keyword-face))
270 |      ([,@julia-ts--keywords]) @font-lock-keyword-face)
271 | 
272 |    :language 'julia
273 |    :feature 'literal
274 |    `([(boolean_literal)
275 |       (character_literal)
276 |       (integer_literal)
277 |       (float_literal)] @font-lock-constant-face)
278 | 
279 |    :language 'julia
280 |    :feature 'macro_call
281 |    `((macro_identifier) @julia-ts-macro-face)
282 | 
283 |    :language 'julia
284 |    :feature 'operator
285 |    `((adjoint_expression "'" @font-lock-operator-face)
286 |      (let_binding (operator) @font-lock-operator-face)
287 |      ((for_binding (operator) @font-lock-keyword-face)
288 |       (:match "^\\[=∈\\]$" @font-lock-keyword-face))
289 |      (arrow_function_expression "->" @font-lock-keyword-face)
290 |      (operator) @font-lock-operator-face
291 |      (splat_expression "..." @font-lock-misc-punctuation-face)
292 |      (ternary_expression ["?" ":"] @font-lock-keyword-face)
293 |      (["." "::"] @font-lock-operator-face))
294 | 
295 |    :language 'julia
296 |    :feature 'interpolation
297 |    :override 'keep
298 |    `((interpolation_expression
299 |       "$" @julia-ts-interpolation-expression-face)
300 |      (interpolation_expression
301 |       (identifier) @default)
302 |      (interpolation_expression
303 |       (parenthesized_expression
304 |        "(" @julia-ts-interpolation-expression-face
305 |        _ @default
306 |        ")" @julia-ts-interpolation-expression-face))
307 |      (string_interpolation
308 |       "$" @julia-ts-string-interpolation-face
309 |       "(":? @julia-ts-string-interpolation-face
310 |       _ @default
311 |       ")":? @julia-ts-string-interpolation-face))
312 | 
313 |    :language 'julia
314 |    :feature 'constant
315 |    :override 'keep
316 |    `((quote_expression) @julia-ts-quoted-symbol-face)
317 | 
318 |    :language 'julia
319 |    :feature 'string
320 |    :override 'keep
321 |    `([(command_literal)
322 |       (prefixed_command_literal)
323 |       (string_literal)
324 |       (prefixed_string_literal)] @font-lock-string-face)
325 | 
326 |    ;; We need to override this feature because otherwise in statements like:
327 |    ;;     a::Union{Int, NTuple{4, Char}}
328 |    ;; the type is not fontified correctly due to the integer literal.
329 |    :language 'julia
330 |    :feature 'type
331 |    :override t
332 |    `((typed_expression (_) "::" (identifier) @font-lock-type-face)
333 |      (typed_expression (_) "::" (field_expression "." (identifier) @font-lock-type-face))
334 |      (unary_typed_expression "::" (_) @font-lock-type-face)
335 |      (parametrized_type_expression (identifier) @font-lock-type-face)
336 |      (parametrized_type_expression
337 |       (curly_expression "{" (identifier) @font-lock-type-face))
338 |      (parametrized_type_expression
339 |       (curly_expression "{" (unary_expression (identifier) @font-lock-type-face)))
340 |      (parametrized_type_expression
341 |       (curly_expression "{" (binary_expression (identifier) @font-lock-type-face)))
342 |      (where_expression "where" (identifier) @font-lock-type-face)
343 |      (where_expression
344 |       "where"
345 |       (curly_expression "{" (identifier) @font-lock-type-face))
346 |      (where_expression
347 |       "where"
348 |       (curly_expression "{" (binary_expression (identifier) @font-lock-type-face)))))
349 | 
350 |   "Tree-sitter font-lock settings for `julia-ts-mode'.")
351 | 
352 | (defvar julia-ts--treesit-indent-rules
353 |   `((julia
354 |      ((parent-is "abstract_definition") parent-bol 0)
355 |      ((parent-is "module_definition") parent-bol 0)
356 |      ((node-is "end") (and parent parent-bol) 0)
357 |      ((node-is "elseif") parent-bol 0)
358 |      ((node-is "else") parent-bol 0)
359 |      ((node-is "catch") parent-bol 0)
360 |      ((node-is "finally") parent-bol 0)
361 |      ((node-is ")") parent-bol 0)
362 |      ((node-is "]") parent-bol 0)
363 |      ((node-is "}") parent-bol 0)
364 | 
365 |      ;; Alignment of parenthesized expressions.
366 |      ((parent-is "parenthesized_expression") parent-bol julia-ts-indent-offset)
367 | 
368 |      ;; Alignment of tuples.
369 |      ((julia-ts--parent-is-and-sibling-on-same-line "tuple_expression" 1) first-sibling 1)
370 |      ((julia-ts--parent-is-and-sibling-not-on-same-line "tuple_expression" 1) parent-bol julia-ts-indent-offset)
371 | 
372 |      ;; Alignment of arrays.
373 |      ((julia-ts--parent-is-and-sibling-on-same-line "vector_expression" 1) first-sibling 1)
374 |      ((julia-ts--parent-is-and-sibling-not-on-same-line "vector_expression" 1) parent-bol julia-ts-indent-offset)
375 |      ((julia-ts--parent-is-and-sibling-on-same-line "matrix_expression" 1) first-sibling 1)
376 |      ((julia-ts--parent-is-and-sibling-not-on-same-line "matrix_expression" 1) parent-bol julia-ts-indent-offset)
377 | 
378 |      ;; Alignment of curly brace expressions.
379 |      ,(if julia-ts-align-curly-brace-expressions-to-first-sibling
380 |           `((julia-ts--parent-is-and-sibling-on-same-line "curly_expression" 1) first-sibling 1)
381 |         `((julia-ts--parent-is-and-sibling-on-same-line "curly_expression" 1) parent-bol julia-ts-indent-offset))
382 |      ((julia-ts--parent-is-and-sibling-not-on-same-line "curly_expression" 1) parent-bol julia-ts-indent-offset)
383 | 
384 |      ;; Align the expressions in the if statement conditions.
385 |      ((parent-is "if_statement") parent-bol julia-ts-indent-offset)
386 | 
387 |      ;; For all other expressions, keep the indentation as the parent.
388 |      ((parent-is "_expression") parent 0)
389 | 
390 |      ;; General indentation rules for blocks.
391 |      ((parent-is "_statement") parent-bol julia-ts-indent-offset)
392 |      ((parent-is "_definition") parent-bol julia-ts-indent-offset)
393 |      ((parent-is "_clause") parent-bol julia-ts-indent-offset)
394 | 
395 |      ;; Alignment of argument lists.
396 |      ,(if julia-ts-align-argument-list-to-first-sibling
397 |           `((julia-ts--parent-is-and-sibling-on-same-line "argument_list" 1) first-sibling 1)
398 |         `((julia-ts--parent-is-and-sibling-on-same-line "argument_list" 1) parent-bol julia-ts-indent-offset))
399 |      ((julia-ts--parent-is-and-sibling-not-on-same-line "argument_list" 1) parent-bol julia-ts-indent-offset)
400 | 
401 |      ;; Match if the node is inside an assignment.
402 |      ;; Note that if the user wants to align the assignment expressions on the
403 |      ;; first sibling, we should only check if the first sibling is not on the
404 |      ;; same line of its parent. The other rules already perform the correct
405 |      ;; indentation.
406 |      ,@(unless julia-ts-align-assignment-expressions-to-first-sibling
407 |          (list `((julia-ts--ancestor-is-and-sibling-on-same-line "assignment" 2) (julia-ts--ancestor-bol "assignment") julia-ts-indent-offset)))
408 |      ((julia-ts--ancestor-is-and-sibling-not-on-same-line "assignment" 2) (julia-ts--ancestor-bol "assignment") julia-ts-indent-offset)
409 | 
410 |      ;; This rule takes care of blank lines most of the time.
411 |      (no-node parent-bol 0)))
412 |   "Tree-sitter indent rules for `julia-ts-mode'.")
413 | 
414 | (defun julia-ts--defun-name (node)
415 |   "Return the defun name of NODE.
416 | Return nil if there is no name or if NODE is not a defun node."
417 |   (pcase (treesit-node-type node)
418 |     ((or "abstract_definition" "struct_definition")
419 |      (treesit-node-text
420 |       (treesit-node-child-by-field-name node "name")
421 |       t))
422 |     ("function_definition"
423 |      (when-let* ((node1 (julia-ts--child-of-type node "signature"))
424 |                  (node2 (julia-ts--child-of-type node1 "call_expression"))
425 |                  (node3 (treesit-node-child node2 0)))
426 |        (treesit-node-text node3)))))
427 | 
428 | (defun julia-ts--child-of-type (node type)
429 |   "Return first child of NODE that has TYPE."
430 |   (car (treesit-filter-child
431 |         node
432 |         (lambda (child)
433 |           (equal (treesit-node-type child) type)))))
434 | 
435 | ;;;###autoload
436 | (add-to-list 'auto-mode-alist '("\\.jl\\'" . julia-ts-mode))
437 | 
438 | ;;;###autoload
439 | (define-derived-mode julia-ts-mode julia-mode "Julia (TS)"
440 |   "Major mode for Julia files using tree-sitter."
441 |   :group 'julia
442 | 
443 |   (unless (treesit-ready-p 'julia)
444 |     (error "Tree-sitter for Julia is not available"))
445 | 
446 |   ;; Override the functions in `julia-mode' that are not needed when using
447 |   ;; tree-sitter.
448 |   (setq-local syntax-propertize-function nil)
449 |   (setq-local indent-line-function nil)
450 | 
451 |   (treesit-parser-create 'julia)
452 | 
453 |   ;; Comments.
454 |   (setq-local comment-start "# ")
455 |   (setq-local comment-end "")
456 |   (setq-local comment-start-skip (rx "#" (* (syntax whitespace))))
457 | 
458 |   ;; Indent.
459 |   (setq-local treesit-simple-indent-rules julia-ts--treesit-indent-rules)
460 | 
461 |   ;; Navigation.
462 |   (setq-local treesit-defun-type-regexp
463 |               (rx (or "function_definition"
464 |                       "struct_definition")))
465 |   (setq-local treesit-defun-name-function #'julia-ts--defun-name)
466 | 
467 |   ;; Imenu.
468 |   (setq-local treesit-simple-imenu-settings
469 |               `(("Function" "\\`function_definition\\'" nil nil)
470 |                 ("Struct" "\\`struct_definition\\'" nil nil)
471 |                 ("Type" "\\`abstract_definition\\'" nil nil)))
472 | 
473 |   ;; Fontification
474 |   (setq-local treesit-font-lock-settings julia-ts--treesit-font-lock-settings)
475 |   (setq-local treesit-font-lock-feature-list
476 |               '((comment definition)
477 |                 (constant keyword string type)
478 |                 (assignment literal interpolation macro_call)
479 |                 (error operator)))
480 | 
481 |   (treesit-major-mode-setup))
482 | 
483 | (provide 'julia-ts-mode)
484 | 
485 | ;; Local Variables:
486 | ;; coding: utf-8
487 | ;; byte-compile-warnings: (not obsolete)
488 | ;; End:
489 | ;;; julia-ts-mode.el ends here
490 | 


--------------------------------------------------------------------------------
/test/ArgTools.jl:
--------------------------------------------------------------------------------
  1 | module ArgTools
  2 | 
  3 | export
  4 |     arg_read,  ArgRead,  arg_readers,
  5 |     arg_write, ArgWrite, arg_writers,
  6 |     arg_isdir, arg_mkdir, @arg_test
  7 | 
  8 | import Base: AbstractCmd, CmdRedirect, Process
  9 | 
 10 | if isdefined(Base.Filesystem, :prepare_for_deletion)
 11 |     using Base.Filesystem: prepare_for_deletion
 12 | else
 13 |     function prepare_for_deletion(path::AbstractString)
 14 |         try prepare_for_deletion_core(path)
 15 |         catch
 16 |         end
 17 |     end
 18 |     function prepare_for_deletion_core(path::AbstractString)
 19 |         isdir(path) || return
 20 |         chmod(path, filemode(path) | 0o333)
 21 |         for name in readdir(path)
 22 |             path′ = joinpath(path, name)
 23 |             prepare_for_deletion_core(path′)
 24 |         end
 25 |     end
 26 | end
 27 | 
 28 | const nolock_docstring = """
 29 | Note: when opening a file, ArgTools will pass `lock = false` to the file `open(...)` call.
 30 | Therefore, the object returned by this function should not be used from multiple threads.
 31 | This restriction may be relaxed in the future, which would not break any working code.
 32 | """
 33 | 
 34 | if VERSION ≥ v"1.5"
 35 |     open_nolock(args...; kws...) = open(args...; kws..., lock=false)
 36 | else
 37 |     open_nolock(args...; kws...) = open(args...; kws...)
 38 | end
 39 | 
 40 | ## main API ##
 41 | 
 42 | """
 43 |     ArgRead = Union{AbstractString, AbstractCmd, IO}
 44 | 
 45 | The `ArgRead` types is a union of the types that the `arg_read` function knows
 46 | how to convert into readable IO handles. See [`arg_read`](@ref) for details.
 47 | """
 48 | const ArgRead = Union{AbstractString, AbstractCmd, IO}
 49 | 
 50 | """
 51 |     ArgWrite = Union{AbstractString, AbstractCmd, IO}
 52 | 
 53 | The `ArgWrite` types is a union of the types that the `arg_write` function knows
 54 | how to convert into writeable IO handles, except for `Nothing` which `arg_write`
 55 | handles by generating a temporary file. See [`arg_write`](@ref) for details.
 56 | """
 57 | const ArgWrite = Union{AbstractString, AbstractCmd, IO}
 58 | 
 59 | """
 60 |     arg_read(f::Function, arg::ArgRead) -> f(arg_io)
 61 | 
 62 | The `arg_read` function accepts an argument `arg` that can be any of these:
 63 | 
 64 | - `AbstractString`: a file path to be opened for reading
 65 | - `AbstractCmd`: a command to be run, reading from its standard output
 66 | - `IO`: an open IO handle to be read from
 67 | 
 68 | Whether the body returns normally or throws an error, a path which is opened
 69 | will be closed before returning from `arg_read` and an `IO` handle will be
 70 | flushed but not closed before returning from `arg_read`.
 71 | 
 72 | $(nolock_docstring)
 73 | """
 74 | arg_read(f::Function, arg::AbstractString) = open_nolock(f, arg)
 75 | arg_read(f::Function, arg::ArgRead) = open(f, arg)
 76 | arg_read(f::Function, arg::IO) = f(arg)
 77 | 
 78 | """
 79 |     arg_write(f::Function, arg::ArgWrite) -> arg
 80 |     arg_write(f::Function, arg::Nothing) -> tempname()
 81 | 
 82 | The `arg_read` function accepts an argument `arg` that can be any of these:
 83 | 
 84 | - `AbstractString`: a file path to be opened for writing
 85 | - `AbstractCmd`: a command to be run, writing to its standard input
 86 | - `IO`: an open IO handle to be written to
 87 | - `Nothing`: a temporary path should be written to
 88 | 
 89 | If the body returns normally, a path that is opened will be closed upon
 90 | completion; an IO handle argument is left open but flushed before return. If the
 91 | argument is `nothing` then a temporary path is opened for writing and closed
 92 | open completion and the path is returned from `arg_write`. In all other cases,
 93 | `arg` itself is returned. This is a useful pattern since you can consistently
 94 | return whatever was written, whether an argument was passed or not.
 95 | 
 96 | If there is an error during the evaluation of the body, a path that is opened by
 97 | `arg_write` for writing will be deleted, whether it's passed in as a string or a
 98 | temporary path generated when `arg` is `nothing`.
 99 | 
100 | $(nolock_docstring)
101 | """
102 | function arg_write(f::Function, arg::AbstractString)
103 |     try open_nolock(f, arg, write=true)
104 |     catch
105 |         rm(arg, force=true)
106 |         rethrow()
107 |     end
108 |     return arg
109 | end
110 | 
111 | function arg_write(f::Function, arg::AbstractCmd)
112 |     open(f, arg, write=true)
113 |     return arg
114 | end
115 | 
116 | function arg_write(f::Function, arg::Nothing)
117 |     @static if VERSION ≥ v"1.5"
118 |         file = tempname()
119 |         io = open_nolock(file, write=true)
120 |     else
121 |         file, io = mktemp()
122 |     end
123 |     try f(io)
124 |     catch
125 |         close(io)
126 |         rm(file, force=true)
127 |         rethrow()
128 |     end
129 |     close(io)
130 |     return file
131 | end
132 | 
133 | function arg_write(f::Function, arg::IO)
134 |     try f(arg)
135 |     finally
136 |         flush(arg)
137 |     end
138 |     return arg
139 | end
140 | 
141 | """
142 |     arg_isdir(f::Function, arg::AbstractString) -> f(arg)
143 | 
144 | The `arg_isdir` function takes `arg` which must be the path to an existing
145 | directory (an error is raised otherwise) and passes that path to `f` finally
146 | returning the result of `f(arg)`. This is definitely the least useful tool
147 | offered by `ArgTools` and mostly exists for symmetry with `arg_mkdir` and to
148 | give consistent error messages.
149 | """
150 | function arg_isdir(f::Function, arg::AbstractString)
151 |     isdir(arg) || error("arg_isdir: $(repr(arg)) not a directory")
152 |     return f(arg)
153 | end
154 | 
155 | """
156 |     arg_mkdir(f::Function, arg::AbstractString) -> arg
157 |     arg_mkdir(f::Function, arg::Nothing) -> mktempdir()
158 | 
159 | The `arg_mkdir` function takes `arg` which must either be one of:
160 | 
161 | - a path to an already existing empty directory,
162 | - a non-existent path which can be created as a directory, or
163 | - `nothing` in which case a temporary directory is created.
164 | 
165 | In all cases the path to the directory is returned. If an error occurs during
166 | `f(arg)`, the directory is returned to its original state: if it already existed
167 | but was empty, it will be emptied; if it did not exist it will be deleted.
168 | """
169 | function arg_mkdir(f::Function, arg::Union{AbstractString, Nothing})
170 |     existed = false
171 |     if arg === nothing
172 |         arg = mktempdir()
173 |     else
174 |         st = stat(arg)
175 |         if !ispath(st)
176 |             mkdir(arg)
177 |         elseif !isdir(st)
178 |             error("arg_mkdir: $(repr(arg)) not a directory")
179 |         else
180 |             isempty(readdir(arg)) ||
181 |                 error("arg_mkdir: $(repr(arg)) directory not empty")
182 |             existed = true
183 |         end
184 |     end
185 |     try f(arg)
186 |     catch
187 |         if existed
188 |             for name in readdir(arg)
189 |                 path = joinpath(arg, name)
190 |                 prepare_for_deletion(path)
191 |                 rm(path, force=true, recursive=true)
192 |             end
193 |         else
194 |             prepare_for_deletion(arg)
195 |             rm(arg, force=true, recursive=true)
196 |         end
197 |         rethrow()
198 |     end
199 |     return arg
200 | end
201 | 
202 | ## test utilities ##
203 | 
204 | const ARG_READERS = [
205 |     String      => path -> f -> f(path)
206 |     Cmd         => path -> f -> f(`cat $path`)
207 |     CmdRedirect => path -> f -> f(pipeline(path, `cat`))
208 |     IOStream    => path -> f -> open(f, path)
209 |     Process     => path -> f -> open(f, `cat $path`)
210 | ]
211 | 
212 | const ARG_WRITERS = [
213 |     String      => path -> f -> f(path)
214 |     Cmd         => path -> f -> f(`tee $path`)
215 |     CmdRedirect => path -> f -> f(pipeline(`cat`, path))
216 |     IOStream    => path -> f -> open(f, path, write=true)
217 |     Process     => path -> f -> open(f, pipeline(`cat`, path), write=true)
218 | ]
219 | 
220 | @assert all(t <: ArgRead  for t in map(first, ARG_READERS))
221 | @assert all(t <: ArgWrite for t in map(first, ARG_WRITERS))
222 | 
223 | """
224 |     arg_readers(arg :: AbstractString, [ type = ArgRead ]) do arg::Function
225 |         ## pre-test setup ##
226 |         @arg_test arg begin
227 |             arg :: ArgRead
228 |             ## test using `arg` ##
229 |         end
230 |         ## post-test cleanup ##
231 |     end
232 | 
233 | The `arg_readers` function takes a path to be read and a single-argument do
234 | block, which is invoked once for each test reader type that `arg_read` can
235 | handle. If the optional `type` argument is given then the do block is only
236 | invoked for readers that produce arguments of that type.
237 | 
238 | The `arg` passed to the do block is not the argument value itself, because some
239 | of test argument types need to be initialized and finalized for each test case.
240 | Consider an open file handle argument: once you've used it for one test, you
241 | can't use it again; you need to close it and open the file again for the next
242 | test. This function `arg` can be converted into an `ArgRead` instance using
243 | `@arg_test arg begin ... end`.
244 | """
245 | function arg_readers(
246 |     body::Function,
247 |     path::AbstractString,
248 |     type::Type = ArgRead,
249 | )
250 |     for (t, reader) in ARG_READERS
251 |         t <: type || continue
252 |         body(reader(path))
253 |     end
254 | end
255 | 
256 | """
257 |     arg_writers([ type = ArgWrite ]) do path::String, arg::Function
258 |         ## pre-test setup ##
259 |         @arg_test arg begin
260 |             arg :: ArgWrite
261 |             ## test using `arg` ##
262 |         end
263 |         ## post-test cleanup ##
264 |     end
265 | 
266 | The `arg_writers` function takes a do block, which is invoked once for each test
267 | writer type that `arg_write` can handle with a temporary (non-existent) `path`
268 | and `arg` which can be converted into various writable argument types which
269 | write to `path`. If the optional `type` argument is given then the do block is
270 | only invoked for writers that produce arguments of that type.
271 | 
272 | The `arg` passed to the do block is not the argument value itself, because some
273 | of test argument types need to be initialized and finalized for each test case.
274 | Consider an open file handle argument: once you've used it for one test, you
275 | can't use it again; you need to close it and open the file again for the next
276 | test. This function `arg` can be converted into an `ArgWrite` instance using
277 | `@arg_test arg begin ... end`.
278 | 
279 | There is also an `arg_writers` method that takes a path name like `arg_readers`:
280 | 
281 |     arg_writers(path::AbstractString, [ type = ArgWrite ]) do arg::Function
282 |         ## pre-test setup ##
283 |         @arg_test arg begin
284 |             # here `arg :: ArgWrite`
285 |             ## test using `arg` ##
286 |         end
287 |         ## post-test cleanup ##
288 |     end
289 | 
290 | This method is useful if you need to specify `path` instead of using path name
291 | generated by `tempname()`. Since `path` is passed from outside of `arg_writers`,
292 | the path is not an argument to the do block in this form.
293 | """
294 | function arg_writers(
295 |     body::Function,
296 |     type::Type = ArgWrite,
297 | )
298 |     for (t, writer) in ARG_WRITERS
299 |         t <: type || continue
300 |         path = tempname()
301 |         try body(path, writer(path))
302 |         finally
303 |             rm(path, force=true)
304 |         end
305 |     end
306 | end
307 | 
308 | function arg_writers(
309 |     body::Function,
310 |     path::AbstractString,
311 |     type::Type = ArgWrite,
312 | )
313 |     for (t, writer) in ARG_WRITERS
314 |         t <: type || continue
315 |         body(writer(path))
316 |     end
317 | end
318 | 
319 | """
320 |     @arg_test arg1 arg2 ... body
321 | 
322 | The `@arg_test` macro is used to convert `arg` functions provided by
323 | `arg_readers` and `arg_writers` into actual argument values. When you write
324 | `@arg_test arg body` it is equivalent to `arg(arg -> body)`.
325 | """
326 | macro arg_test(args...)
327 |     arg_test(args...)
328 | end
329 | 
330 | function arg_test(var::Symbol, args...)
331 |     var = esc(var)
332 |     body = arg_test(args...)
333 |     :($var($var -> $body))
334 | end
335 | 
336 | arg_test(ex::Expr) = esc(ex)
337 | 
338 | end # module
339 | 


--------------------------------------------------------------------------------
/test/ArgTools.jl.faceup:
--------------------------------------------------------------------------------
  1 | «k:module» ArgTools
  2 | 
  3 | «k:export»
  4 |     arg_read,  ArgRead,  arg_readers,
  5 |     arg_write, ArgWrite, arg_writers,
  6 |     arg_isdir, arg_mkdir, «:julia-ts-macro-face:@arg_test»
  7 | 
  8 | «k:import» Base: AbstractCmd, CmdRedirect, Process
  9 | 
 10 | «k:if» isdefined(Base.Filesystem, «:julia-ts-quoted-symbol-face::prepare_for_deletion»)
 11 |     «k:using» Base.Filesystem: prepare_for_deletion
 12 | «k:else»
 13 |     «k:function» «f:prepare_for_deletion»(path::«t:AbstractString»)
 14 |         «k:try» prepare_for_deletion_core(path)
 15 |         «k:catch»
 16 |         «k:end»
 17 |     «k:end»
 18 |     «k:function» «f:prepare_for_deletion_core»(path::«t:AbstractString»)
 19 |         isdir(path) || «k:return»
 20 |         chmod(path, filemode(path) | «c:0o333»)
 21 |         «k:for» name «k:in» readdir(path)
 22 |             «v:path′» = joinpath(path, name)
 23 |             prepare_for_deletion_core(path′)
 24 |         «k:end»
 25 |     «k:end»
 26 | «k:end»
 27 | 
 28 | «k:const» «v:nolock_docstring» = «s:"""
 29 | Note: when opening a file, ArgTools will pass `lock = false` to the file `open(...)` call.
 30 | Therefore, the object returned by this function should not be used from multiple threads.
 31 | This restriction may be relaxed in the future, which would not break any working code.
 32 | """»
 33 | 
 34 | «k:if» VERSION ≥ «s:v"1.5"»
 35 |     «f:open_nolock»(args...; kws...) = open(args...; kws..., «:julia-ts-keyword-argument-face:lock»=«c:false»)
 36 | «k:else»
 37 |     «f:open_nolock»(args...; kws...) = open(args...; kws...)
 38 | «k:end»
 39 | 
 40 | «x:## main API ##»
 41 | 
 42 | «s:"""
 43 |     ArgRead = Union{AbstractString, AbstractCmd, IO}
 44 | 
 45 | The `ArgRead` types is a union of the types that the `arg_read` function knows
 46 | how to convert into readable IO handles. See [`arg_read`](@ref) for details.
 47 | """»
 48 | «k:const» «v:ArgRead» = «t:Union»{«t:AbstractString», «t:AbstractCmd», «t:IO»}
 49 | 
 50 | «s:"""
 51 |     ArgWrite = Union{AbstractString, AbstractCmd, IO}
 52 | 
 53 | The `ArgWrite` types is a union of the types that the `arg_write` function knows
 54 | how to convert into writeable IO handles, except for `Nothing` which `arg_write`
 55 | handles by generating a temporary file. See [`arg_write`](@ref) for details.
 56 | """»
 57 | «k:const» «v:ArgWrite» = «t:Union»{«t:AbstractString», «t:AbstractCmd», «t:IO»}
 58 | 
 59 | «s:"""
 60 |     arg_read(f::Function, arg::ArgRead) -> f(arg_io)
 61 | 
 62 | The `arg_read` function accepts an argument `arg` that can be any of these:
 63 | 
 64 | - `AbstractString`: a file path to be opened for reading
 65 | - `AbstractCmd`: a command to be run, reading from its standard output
 66 | - `IO`: an open IO handle to be read from
 67 | 
 68 | Whether the body returns normally or throws an error, a path which is opened
 69 | will be closed before returning from `arg_read` and an `IO` handle will be
 70 | flushed but not closed before returning from `arg_read`.
 71 | 
 72 | »«:julia-ts-string-interpolation-face:$(»«D:nolock_docstring»«:julia-ts-string-interpolation-face:)»«s:
 73 | """»
 74 | «f:arg_read»(f::«t:Function», arg::«t:AbstractString») = open_nolock(f, arg)
 75 | «f:arg_read»(f::«t:Function», arg::«t:ArgRead») = open(f, arg)
 76 | «f:arg_read»(f::«t:Function», arg::«t:IO») = f(arg)
 77 | 
 78 | «s:"""
 79 |     arg_write(f::Function, arg::ArgWrite) -> arg
 80 |     arg_write(f::Function, arg::Nothing) -> tempname()
 81 | 
 82 | The `arg_read` function accepts an argument `arg` that can be any of these:
 83 | 
 84 | - `AbstractString`: a file path to be opened for writing
 85 | - `AbstractCmd`: a command to be run, writing to its standard input
 86 | - `IO`: an open IO handle to be written to
 87 | - `Nothing`: a temporary path should be written to
 88 | 
 89 | If the body returns normally, a path that is opened will be closed upon
 90 | completion; an IO handle argument is left open but flushed before return. If the
 91 | argument is `nothing` then a temporary path is opened for writing and closed
 92 | open completion and the path is returned from `arg_write`. In all other cases,
 93 | `arg` itself is returned. This is a useful pattern since you can consistently
 94 | return whatever was written, whether an argument was passed or not.
 95 | 
 96 | If there is an error during the evaluation of the body, a path that is opened by
 97 | `arg_write` for writing will be deleted, whether it's passed in as a string or a
 98 | temporary path generated when `arg` is `nothing`.
 99 | 
100 | »«:julia-ts-string-interpolation-face:$(»«D:nolock_docstring»«:julia-ts-string-interpolation-face:)»«s:
101 | """»
102 | «k:function» «f:arg_write»(f::«t:Function», arg::«t:AbstractString»)
103 |     «k:try» open_nolock(f, arg, «:julia-ts-keyword-argument-face:write»=«c:true»)
104 |     «k:catch»
105 |         rm(arg, «:julia-ts-keyword-argument-face:force»=«c:true»)
106 |         rethrow()
107 |     «k:end»
108 |     «k:return» arg
109 | «k:end»
110 | 
111 | «k:function» «f:arg_write»(f::«t:Function», arg::«t:AbstractCmd»)
112 |     open(f, arg, «:julia-ts-keyword-argument-face:write»=«c:true»)
113 |     «k:return» arg
114 | «k:end»
115 | 
116 | «k:function» «f:arg_write»(f::«t:Function», arg::«t:Nothing»)
117 |     «:julia-ts-macro-face:@static» «k:if» VERSION ≥ «s:v"1.5"»
118 |         «v:file» = tempname()
119 |         «v:io» = open_nolock(file, «:julia-ts-keyword-argument-face:write»=«c:true»)
120 |     «k:else»
121 |         «v:file», «v:io» = mktemp()
122 |     «k:end»
123 |     «k:try» f(io)
124 |     «k:catch»
125 |         close(io)
126 |         rm(file, «:julia-ts-keyword-argument-face:force»=«c:true»)
127 |         rethrow()
128 |     «k:end»
129 |     close(io)
130 |     «k:return» file
131 | «k:end»
132 | 
133 | «k:function» «f:arg_write»(f::«t:Function», arg::«t:IO»)
134 |     «k:try» f(arg)
135 |     «k:finally»
136 |         flush(arg)
137 |     «k:end»
138 |     «k:return» arg
139 | «k:end»
140 | 
141 | «s:"""
142 |     arg_isdir(f::Function, arg::AbstractString) -> f(arg)
143 | 
144 | The `arg_isdir` function takes `arg` which must be the path to an existing
145 | directory (an error is raised otherwise) and passes that path to `f` finally
146 | returning the result of `f(arg)`. This is definitely the least useful tool
147 | offered by `ArgTools` and mostly exists for symmetry with `arg_mkdir` and to
148 | give consistent error messages.
149 | """»
150 | «k:function» «f:arg_isdir»(f::«t:Function», arg::«t:AbstractString»)
151 |     isdir(arg) || error(«s:"arg_isdir: »«:julia-ts-string-interpolation-face:$(»«D:repr(arg)»«:julia-ts-string-interpolation-face:)»«s: not a directory"»)
152 |     «k:return» f(arg)
153 | «k:end»
154 | 
155 | «s:"""
156 |     arg_mkdir(f::Function, arg::AbstractString) -> arg
157 |     arg_mkdir(f::Function, arg::Nothing) -> mktempdir()
158 | 
159 | The `arg_mkdir` function takes `arg` which must either be one of:
160 | 
161 | - a path to an already existing empty directory,
162 | - a non-existent path which can be created as a directory, or
163 | - `nothing` in which case a temporary directory is created.
164 | 
165 | In all cases the path to the directory is returned. If an error occurs during
166 | `f(arg)`, the directory is returned to its original state: if it already existed
167 | but was empty, it will be emptied; if it did not exist it will be deleted.
168 | """»
169 | «k:function» «f:arg_mkdir»(f::«t:Function», arg::«t:Union»{«t:AbstractString», «t:Nothing»})
170 |     «v:existed» = «c:false»
171 |     «k:if» arg === «c:nothing»
172 |         «v:arg» = mktempdir()
173 |     «k:else»
174 |         «v:st» = stat(arg)
175 |         «k:if» !ispath(st)
176 |             mkdir(arg)
177 |         «k:elseif» !isdir(st)
178 |             error(«s:"arg_mkdir: »«:julia-ts-string-interpolation-face:$(»«D:repr(arg)»«:julia-ts-string-interpolation-face:)»«s: not a directory"»)
179 |         «k:else»
180 |             isempty(readdir(arg)) ||
181 |                 error(«s:"arg_mkdir: »«:julia-ts-string-interpolation-face:$(»«D:repr(arg)»«:julia-ts-string-interpolation-face:)»«s: directory not empty"»)
182 |             «v:existed» = «c:true»
183 |         «k:end»
184 |     «k:end»
185 |     «k:try» f(arg)
186 |     «k:catch»
187 |         «k:if» existed
188 |             «k:for» name «k:in» readdir(arg)
189 |                 «v:path» = joinpath(arg, name)
190 |                 prepare_for_deletion(path)
191 |                 rm(path, «:julia-ts-keyword-argument-face:force»=«c:true», «:julia-ts-keyword-argument-face:recursive»=«c:true»)
192 |             «k:end»
193 |         «k:else»
194 |             prepare_for_deletion(arg)
195 |             rm(arg, «:julia-ts-keyword-argument-face:force»=«c:true», «:julia-ts-keyword-argument-face:recursive»=«c:true»)
196 |         «k:end»
197 |         rethrow()
198 |     «k:end»
199 |     «k:return» arg
200 | «k:end»
201 | 
202 | «x:## test utilities ##»
203 | 
204 | «k:const» «v:ARG_READERS» = [
205 |     String      => path -> f -> f(path)
206 |     Cmd         => path -> f -> f(«s:`cat »«:julia-ts-string-interpolation-face:$»«D:path»«s:`»)
207 |     CmdRedirect => path -> f -> f(pipeline(path, «s:`cat`»))
208 |     IOStream    => path -> f -> open(f, path)
209 |     Process     => path -> f -> open(f, «s:`cat »«:julia-ts-string-interpolation-face:$»«D:path»«s:`»)
210 | ]
211 | 
212 | «k:const» «v:ARG_WRITERS» = [
213 |     String      => path -> f -> f(path)
214 |     Cmd         => path -> f -> f(«s:`tee »«:julia-ts-string-interpolation-face:$»«D:path»«s:`»)
215 |     CmdRedirect => path -> f -> f(pipeline(«s:`cat`», path))
216 |     IOStream    => path -> f -> open(f, path, «:julia-ts-keyword-argument-face:write»=«c:true»)
217 |     Process     => path -> f -> open(f, pipeline(«s:`cat`», path), «:julia-ts-keyword-argument-face:write»=«c:true»)
218 | ]
219 | 
220 | «:julia-ts-macro-face:@assert» all(t <: ArgRead  «k:for» t «k:in» map(first, ARG_READERS))
221 | «:julia-ts-macro-face:@assert» all(t <: ArgWrite «k:for» t «k:in» map(first, ARG_WRITERS))
222 | 
223 | «s:"""
224 |     arg_readers(arg :: AbstractString, [ type = ArgRead ]) do arg::Function
225 |         ## pre-test setup ##
226 |         @arg_test arg begin
227 |             arg :: ArgRead
228 |             ## test using `arg` ##
229 |         end
230 |         ## post-test cleanup ##
231 |     end
232 | 
233 | The `arg_readers` function takes a path to be read and a single-argument do
234 | block, which is invoked once for each test reader type that `arg_read` can
235 | handle. If the optional `type` argument is given then the do block is only
236 | invoked for readers that produce arguments of that type.
237 | 
238 | The `arg` passed to the do block is not the argument value itself, because some
239 | of test argument types need to be initialized and finalized for each test case.
240 | Consider an open file handle argument: once you've used it for one test, you
241 | can't use it again; you need to close it and open the file again for the next
242 | test. This function `arg` can be converted into an `ArgRead` instance using
243 | `@arg_test arg begin ... end`.
244 | """»
245 | «k:function» «f:arg_readers»(
246 |     body::«t:Function»,
247 |     path::«t:AbstractString»,
248 |     type::«t:Type» = ArgRead,
249 | )
250 |     «k:for» (t, reader) «k:in» ARG_READERS
251 |         t <: type || «k:continue»
252 |         body(reader(path))
253 |     «k:end»
254 | «k:end»
255 | 
256 | «s:"""
257 |     arg_writers([ type = ArgWrite ]) do path::String, arg::Function
258 |         ## pre-test setup ##
259 |         @arg_test arg begin
260 |             arg :: ArgWrite
261 |             ## test using `arg` ##
262 |         end
263 |         ## post-test cleanup ##
264 |     end
265 | 
266 | The `arg_writers` function takes a do block, which is invoked once for each test
267 | writer type that `arg_write` can handle with a temporary (non-existent) `path`
268 | and `arg` which can be converted into various writable argument types which
269 | write to `path`. If the optional `type` argument is given then the do block is
270 | only invoked for writers that produce arguments of that type.
271 | 
272 | The `arg` passed to the do block is not the argument value itself, because some
273 | of test argument types need to be initialized and finalized for each test case.
274 | Consider an open file handle argument: once you've used it for one test, you
275 | can't use it again; you need to close it and open the file again for the next
276 | test. This function `arg` can be converted into an `ArgWrite` instance using
277 | `@arg_test arg begin ... end`.
278 | 
279 | There is also an `arg_writers` method that takes a path name like `arg_readers`:
280 | 
281 |     arg_writers(path::AbstractString, [ type = ArgWrite ]) do arg::Function
282 |         ## pre-test setup ##
283 |         @arg_test arg begin
284 |             # here `arg :: ArgWrite`
285 |             ## test using `arg` ##
286 |         end
287 |         ## post-test cleanup ##
288 |     end
289 | 
290 | This method is useful if you need to specify `path` instead of using path name
291 | generated by `tempname()`. Since `path` is passed from outside of `arg_writers`,
292 | the path is not an argument to the do block in this form.
293 | """»
294 | «k:function» «f:arg_writers»(
295 |     body::«t:Function»,
296 |     type::«t:Type» = ArgWrite,
297 | )
298 |     «k:for» (t, writer) «k:in» ARG_WRITERS
299 |         t <: type || «k:continue»
300 |         «v:path» = tempname()
301 |         «k:try» body(path, writer(path))
302 |         «k:finally»
303 |             rm(path, «:julia-ts-keyword-argument-face:force»=«c:true»)
304 |         «k:end»
305 |     «k:end»
306 | «k:end»
307 | 
308 | «k:function» «f:arg_writers»(
309 |     body::«t:Function»,
310 |     path::«t:AbstractString»,
311 |     type::«t:Type» = ArgWrite,
312 | )
313 |     «k:for» (t, writer) «k:in» ARG_WRITERS
314 |         t <: type || «k:continue»
315 |         body(writer(path))
316 |     «k:end»
317 | «k:end»
318 | 
319 | «s:"""
320 |     @arg_test arg1 arg2 ... body
321 | 
322 | The `@arg_test` macro is used to convert `arg` functions provided by
323 | `arg_readers` and `arg_writers` into actual argument values. When you write
324 | `@arg_test arg body` it is equivalent to `arg(arg -> body)`.
325 | """»
326 | «k:macro» «f:arg_test»(args...)
327 |     arg_test(args...)
328 | «k:end»
329 | 
330 | «k:function» «f:arg_test»(var::«t:Symbol», args...)
331 |     «v:var» = esc(var)
332 |     «v:body» = arg_test(args...)
333 |     «:julia-ts-quoted-symbol-face::(»«:julia-ts-interpolation-expression-face:$»«D:var»«:julia-ts-quoted-symbol-face:(»«:julia-ts-interpolation-expression-face:$»«D:var»«:julia-ts-quoted-symbol-face: -> »«:julia-ts-interpolation-expression-face:$»«D:body»«:julia-ts-quoted-symbol-face:))»
334 | «k:end»
335 | 
336 | «f:arg_test»(ex::«t:Expr») = esc(ex)
337 | 
338 | «k:end» «x:# module»
339 | 


--------------------------------------------------------------------------------
/test/Downloads.jl:
--------------------------------------------------------------------------------
  1 | """
  2 | The `Downloads` module exports a function [`download`](@ref), which provides cross-platform, multi-protocol,
  3 | in-process download functionality implemented with [libcurl](https://curl.haxx.se/libcurl/).   It is used
  4 | for the `Base.download` function in Julia 1.6 or later.
  5 | 
  6 | More generally, the module exports functions and types that provide lower-level control and diagnostic information
  7 | for file downloading:
  8 | - [`download`](@ref) — download a file from a URL, erroring if it can't be downloaded
  9 | - [`request`](@ref) — request a URL, returning a `Response` object indicating success
 10 | - [`Response`](@ref) — a type capturing the status and other metadata about a request
 11 | - [`RequestError`](@ref) — an error type thrown by `download` and `request` on error
 12 | - [`Downloader`](@ref) — an object encapsulating shared resources for downloading
 13 | """
 14 | module Downloads
 15 | 
 16 | using Base.Experimental: @sync
 17 | using NetworkOptions
 18 | using ArgTools
 19 | 
 20 | include("Curl/Curl.jl")
 21 | using .Curl
 22 | 
 23 | export download, request, Downloader, Response, RequestError, default_downloader!
 24 | 
 25 | ## public API types ##
 26 | 
 27 | """
 28 |     Downloader(; [ grace::Real = 30 ])
 29 | 
 30 | `Downloader` objects are used to perform individual `download` operations.
 31 | Connections, name lookups and other resources are shared within a `Downloader`.
 32 | These connections and resources are cleaned up after a configurable grace period
 33 | (default: 30 seconds) since anything was downloaded with it, or when it is
 34 | garbage collected, whichever comes first. If the grace period is set to zero,
 35 | all resources will be cleaned up immediately as soon as there are no more
 36 | ongoing downloads in progress. If the grace period is set to `Inf` then
 37 | resources are not cleaned up until `Downloader` is garbage collected.
 38 | """
 39 | mutable struct Downloader
 40 |     multi::Multi
 41 |     ca_roots::Union{String, Nothing}
 42 |     easy_hook::Union{Function, Nothing}
 43 | 
 44 |     Downloader(multi::Multi) = new(multi, get_ca_roots(), EASY_HOOK[])
 45 | end
 46 | Downloader(; grace::Real=30) = Downloader(Multi(grace_ms(grace)))
 47 | 
 48 | function grace_ms(grace::Real)
 49 |     grace < 0 && throw(ArgumentError("grace period cannot be negative: $grace"))
 50 |     grace <= typemax(UInt64) ÷ 1000 ? round(UInt64, 1000*grace) : typemax(UInt64)
 51 | end
 52 | 
 53 | function easy_hook(downloader::Downloader, easy::Easy, info::NamedTuple)
 54 |     hook = downloader.easy_hook
 55 |     hook !== nothing && Base.invokelatest(hook, easy, info)
 56 | end
 57 | 
 58 | get_ca_roots() = Curl.SYSTEM_SSL ? ca_roots() : ca_roots_path()
 59 | 
 60 | function set_ca_roots(downloader::Downloader, easy::Easy)
 61 |     ca_roots = downloader.ca_roots
 62 |     ca_roots !== nothing && set_ca_roots_path(easy, ca_roots)
 63 | end
 64 | 
 65 | const DOWNLOAD_LOCK = ReentrantLock()
 66 | const DOWNLOADER = Ref{Union{Downloader, Nothing}}(nothing)
 67 | 
 68 | """
 69 | `EASY_HOOK` is a modifable global hook to used as the default `easy_hook` on
 70 | new `Downloader` objects. This supplies a mechanism to set options for the
 71 | `Downloader` via `Curl.setopt`
 72 | 
 73 | It is expected to be function taking two arguments: an `Easy` struct and an
 74 | `info` NamedTuple with names `url`, `method` and `headers`.
 75 | """
 76 | const EASY_HOOK = Ref{Union{Function, Nothing}}(nothing)
 77 | 
 78 | """
 79 |     struct Response
 80 |         proto   :: String
 81 |         url     :: String
 82 |         status  :: Int
 83 |         message :: String
 84 |         headers :: Vector{Pair{String,String}}
 85 |     end
 86 | 
 87 | `Response` is a type capturing the properties of a successful response to a
 88 | request as an object. It has the following fields:
 89 | 
 90 | - `proto`: the protocol that was used to get the response
 91 | - `url`: the URL that was ultimately requested after following redirects
 92 | - `status`: the status code of the response, indicating success, failure, etc.
 93 | - `message`: a textual message describing the nature of the response
 94 | - `headers`: any headers that were returned with the response
 95 | 
 96 | The meaning and availability of some of these responses depends on the protocol
 97 | used for the request. For many protocols, including HTTP/S and S/FTP, a 2xx
 98 | status code indicates a successful response. For responses in protocols that do
 99 | not support headers, the headers vector will be empty. HTTP/2 does not include a
100 | status message, only a status code, so the message will be empty.
101 | """
102 | struct Response
103 |     proto   :: Union{String, Nothing}
104 |     url     :: String # redirected URL
105 |     status  :: Int
106 |     message :: String
107 |     headers :: Vector{Pair{String,String}}
108 | end
109 | 
110 | Curl.status_ok(response::Response) = status_ok(response.proto, response.status)
111 | 
112 | """
113 |     struct RequestError <: ErrorException
114 |         url      :: String
115 |         code     :: Int
116 |         message  :: String
117 |         response :: Response
118 |     end
119 | 
120 | `RequestError` is a type capturing the properties of a failed response to a
121 | request as an exception object:
122 | 
123 | - `url`: the original URL that was requested without any redirects
124 | - `code`: the libcurl error code; `0` if a protocol-only error occurred
125 | - `message`: the libcurl error message indicating what went wrong
126 | - `response`: response object capturing what response info is available
127 | 
128 | The same `RequestError` type is thrown by `download` if the request was
129 | successful but there was a protocol-level error indicated by a status code that
130 | is not in the 2xx range, in which case `code` will be zero and the `message`
131 | field will be the empty string. The `request` API only throws a `RequestError`
132 | if the libcurl error `code` is non-zero, in which case the included `response`
133 | object is likely to have a `status` of zero and an empty message. There are,
134 | however, situations where a curl-level error is thrown due to a protocol error,
135 | in which case both the inner and outer code and message may be of interest.
136 | """
137 | struct RequestError <: Exception
138 |     url      :: String # original URL
139 |     code     :: Int
140 |     message  :: String
141 |     response :: Response
142 | end
143 | 
144 | function Base.showerror(io::IO, err::RequestError)
145 |     print(io, "RequestError: $(error_message(err)) while requesting $(err.url)")
146 | end
147 | 
148 | function error_message(err::RequestError)
149 |     errstr = err.message
150 |     status = err.response.status
151 |     message = err.response.message
152 |     status_re = Regex(status == 0 ? "" : "\\b$status\\b")
153 | 
154 |     err.code == Curl.CURLE_OK &&
155 |         return isempty(message) ? "Error status $status" :
156 |             contains(message, status_re) ? message :
157 |                 "$message (status $status)"
158 | 
159 |     isempty(message) && !isempty(errstr) &&
160 |         return status == 0 ? errstr : "$errstr (status $status)"
161 | 
162 |     isempty(message) && (message = "Status $status")
163 |     isempty(errstr)  && (errstr = "curl error $(err.code)")
164 | 
165 |     !contains(message, status_re) && !contains(errstr, status_re) &&
166 |         (errstr = "status $status; $errstr")
167 | 
168 |     return "$message ($errstr)"
169 | end
170 | 
171 | ## download API ##
172 | 
173 | """
174 |     download(url, [ output = tempname() ];
175 |         [ method = "GET", ]
176 |         [ headers = <none>, ]
177 |         [ timeout = <none>, ]
178 |         [ progress = <none>, ]
179 |         [ verbose = false, ]
180 |         [ debug = <none>, ]
181 |         [ downloader = <default>, ]
182 |     ) -> output
183 | 
184 |         url        :: AbstractString
185 |         output     :: Union{AbstractString, AbstractCmd, IO}
186 |         method     :: AbstractString
187 |         headers    :: Union{AbstractVector, AbstractDict}
188 |         timeout    :: Real
189 |         progress   :: (total::Integer, now::Integer) --> Any
190 |         verbose    :: Bool
191 |         debug      :: (type, message) --> Any
192 |         downloader :: Downloader
193 | 
194 | Download a file from the given url, saving it to `output` or if not specified, a
195 | temporary path. The `output` can also be an `IO` handle, in which case the body
196 | of the response is streamed to that handle and the handle is returned. If
197 | `output` is a command, the command is run and output is sent to it on stdin.
198 | 
199 | If the `downloader` keyword argument is provided, it must be a `Downloader`
200 | object. Resources and connections will be shared between downloads performed by
201 | the same `Downloader` and cleaned up automatically when the object is garbage
202 | collected or there have been no downloads performed with it for a grace period.
203 | See `Downloader` for more info about configuration and usage.
204 | 
205 | If the `headers` keyword argument is provided, it must be a vector or dictionary
206 | whose elements are all pairs of strings. These pairs are passed as headers when
207 | downloading URLs with protocols that supports them, such as HTTP/S.
208 | 
209 | The `timeout` keyword argument specifies a timeout for the download to complete in
210 | seconds, with a resolution of milliseconds. By default no timeout is set, but this
211 | can also be explicitly requested by passing a timeout value of `Inf`. Separately,
212 | if 20 seconds elapse without receiving any data, the download will timeout. See
213 | extended help for how to disable this timeout.
214 | 
215 | If the `progress` keyword argument is provided, it must be a callback function
216 | which will be called whenever there are updates about the size and status of the
217 | ongoing download. The callback must take two integer arguments: `total` and
218 | `now` which are the total size of the download in bytes, and the number of bytes
219 | which have been downloaded so far. Note that `total` starts out as zero and
220 | remains zero until the server gives an indication of the total size of the
221 | download (e.g. with a `Content-Length` header), which may never happen. So a
222 | well-behaved progress callback should handle a total size of zero gracefully.
223 | 
224 | If the `verbose` option is set to true, `libcurl`, which is used to implement
225 | the download functionality will print debugging information to `stderr`. If the
226 | `debug` option is set to a function accepting two `String` arguments, then the
227 | verbose option is ignored and instead the data that would have been printed to
228 | `stderr` is passed to the `debug` callback with `type` and `message` arguments.
229 | The `type` argument indicates what kind of event has occurred, and is one of:
230 | `TEXT`, `HEADER IN`, `HEADER OUT`, `DATA IN`, `DATA OUT`, `SSL DATA IN` or `SSL
231 | DATA OUT`. The `message` argument is the description of the debug event.
232 | 
233 | ## Extended Help
234 | 
235 | For further customization, use a [`Downloader`](@ref) and
236 | [`easy_hook`s](https://github.com/JuliaLang/Downloads.jl#mutual-tls-using-downloads).
237 | For example, to disable the 20 second timeout when no data is received, you may
238 | use the following:
239 | 
240 | ```jl
241 | downloader = Downloads.Downloader()
242 | downloader.easy_hook = (easy, info) -> Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_LOW_SPEED_TIME, 0)
243 | 
244 | Downloads.download("https://httpbingo.julialang.org/delay/30"; downloader)
245 | ```
246 | """
247 | function download(
248 |     url        :: AbstractString,
249 |     output     :: Union{ArgWrite, Nothing} = nothing;
250 |     method     :: Union{AbstractString, Nothing} = nothing,
251 |     headers    :: Union{AbstractVector, AbstractDict} = Pair{String,String}[],
252 |     timeout    :: Real = Inf,
253 |     progress   :: Union{Function, Nothing} = nothing,
254 |     verbose    :: Bool = false,
255 |     debug      :: Union{Function, Nothing} = nothing,
256 |     downloader :: Union{Downloader, Nothing} = nothing,
257 | ) :: ArgWrite
258 |     arg_write(output) do output
259 |         response = request(
260 |             url,
261 |             output = output,
262 |             method = method,
263 |             headers = headers,
264 |             timeout = timeout,
265 |             progress = progress,
266 |             verbose = verbose,
267 |             debug = debug,
268 |             downloader = downloader,
269 |         )::Response
270 |         status_ok(response) && return output
271 |         throw(RequestError(url, Curl.CURLE_OK, "", response))
272 |     end
273 | end
274 | 
275 | ## request API ##
276 | 
277 | """
278 |     request(url;
279 |         [ input = <none>, ]
280 |         [ output = <none>, ]
281 |         [ method = input ? "PUT" : output ? "GET" : "HEAD", ]
282 |         [ headers = <none>, ]
283 |         [ timeout = <none>, ]
284 |         [ progress = <none>, ]
285 |         [ verbose = false, ]
286 |         [ debug = <none>, ]
287 |         [ throw = true, ]
288 |         [ downloader = <default>, ]
289 |         [ interrupt = <none>, ]
290 |     ) -> Union{Response, RequestError}
291 | 
292 |         url        :: AbstractString
293 |         input      :: Union{AbstractString, AbstractCmd, IO}
294 |         output     :: Union{AbstractString, AbstractCmd, IO}
295 |         method     :: AbstractString
296 |         headers    :: Union{AbstractVector, AbstractDict}
297 |         timeout    :: Real
298 |         progress   :: (dl_total, dl_now, ul_total, ul_now) --> Any
299 |         verbose    :: Bool
300 |         debug      :: (type, message) --> Any
301 |         throw      :: Bool
302 |         downloader :: Downloader
303 |         interrupt  :: Base.Event
304 | 
305 | Make a request to the given url, returning a `Response` object capturing the
306 | status, headers and other information about the response. The body of the
307 | response is written to `output` if specified and discarded otherwise. For HTTP/S
308 | requests, if an `input` stream is given, a `PUT` request is made; otherwise if
309 | an `output` stream is given, a `GET` request is made; if neither is given a
310 | `HEAD` request is made. For other protocols, appropriate default methods are
311 | used based on what combination of input and output are requested. The following
312 | options differ from the `download` function:
313 | 
314 | - `input` allows providing a request body; if provided default to `PUT` request
315 | - `progress` is a callback taking four integers for upload and download progress
316 | - `throw` controls whether to throw or return a `RequestError` on request error
317 | 
318 | Note that unlike `download` which throws an error if the requested URL could not
319 | be downloaded (indicated by non-2xx status code), `request` returns a `Response`
320 | object no matter what the status code of the response is. If there is an error
321 | with getting a response at all, then a `RequestError` is thrown or returned.
322 | 
323 | If the `interrupt` keyword argument is provided, it must be a `Base.Event` object.
324 | If the event is triggered while the request is in progress, the request will be
325 | cancelled and an error will be thrown. This can be used to interrupt a long
326 | running request, for example if the user wants to cancel a download.
327 | """
328 | function request(
329 |     url        :: AbstractString;
330 |     input      :: Union{ArgRead, Nothing} = nothing,
331 |     output     :: Union{ArgWrite, Nothing} = nothing,
332 |     method     :: Union{AbstractString, Nothing} = nothing,
333 |     headers    :: Union{AbstractVector, AbstractDict} = Pair{String,String}[],
334 |     timeout    :: Real = Inf,
335 |     progress   :: Union{Function, Nothing} = nothing,
336 |     verbose    :: Bool = false,
337 |     debug      :: Union{Function, Nothing} = nothing,
338 |     throw      :: Bool = true,
339 |     downloader :: Union{Downloader, Nothing} = nothing,
340 |     interrupt  :: Union{Nothing, Base.Event} = nothing,
341 | ) :: Union{Response, RequestError}
342 |     if downloader === nothing
343 |         lock(DOWNLOAD_LOCK) do
344 |             downloader = DOWNLOADER[]
345 |             if downloader === nothing
346 |                 downloader = DOWNLOADER[] = Downloader()
347 |             end
348 |         end
349 |     end
350 |     local response
351 |     have_input = input !== nothing
352 |     have_output = output !== nothing
353 |     input = something(input, devnull)
354 |     output = something(output, devnull)
355 |     input_size = arg_read_size(input)
356 |     if input_size === nothing
357 |         # take input_size from content-length header if one is supplied
358 |         input_size = content_length(headers)
359 |     end
360 |     progress = p_func(progress, input, output)
361 |     arg_read(input) do input
362 |         arg_write(output) do output
363 |             with_handle(Easy()) do easy
364 |                 # setup the request
365 |                 set_url(easy, url)
366 |                 set_timeout(easy, timeout)
367 |                 set_verbose(easy, verbose)
368 |                 set_debug(easy, debug)
369 |                 add_headers(easy, headers)
370 | 
371 |                 # libcurl does not set the default header reliably so set it
372 |                 # explicitly unless user has specified it, xref
373 |                 # https://github.com/JuliaLang/Pkg.jl/pull/2357
374 |                 if !any(kv -> lowercase(kv[1]) == "user-agent", headers)
375 |                     Curl.add_header(easy, "User-Agent", Curl.USER_AGENT)
376 |                 end
377 | 
378 |                 if have_input
379 |                     enable_upload(easy)
380 |                     if input_size !== nothing
381 |                         set_upload_size(easy, input_size)
382 |                     end
383 |                     if applicable(seek, input, 0)
384 |                         set_seeker(easy) do offset
385 |                             seek(input, Int(offset))
386 |                         end
387 |                     end
388 |                 else
389 |                     set_body(easy, have_output && method != "HEAD")
390 |                 end
391 |                 method !== nothing && set_method(easy, method)
392 |                 progress !== nothing && enable_progress(easy)
393 |                 set_ca_roots(downloader, easy)
394 |                 info = (url = url, method = method, headers = headers)
395 |                 easy_hook(downloader, easy, info)
396 | 
397 |                 # do the request
398 |                 add_handle(downloader.multi, easy)
399 |                 interrupted = Threads.Atomic{Bool}(false)
400 |                 if interrupt !== nothing
401 |                     interrupt_task = @async begin
402 |                         # wait for the interrupt event
403 |                         wait(interrupt)
404 |                         # cancel the request
405 |                         remove_handle(downloader.multi, easy)
406 |                         close(easy.output)
407 |                         close(easy.progress)
408 |                         interrupted[] = true
409 |                         close(input)
410 |                         notify(easy.ready)
411 |                     end
412 |                 else
413 |                     interrupt_task = nothing
414 |                 end
415 |                 try # ensure handle is removed
416 |                     @sync begin
417 |                         @async for buf in easy.output
418 |                             write(output, buf)
419 |                         end
420 |                         if progress !== nothing
421 |                             @async for prog in easy.progress
422 |                                 progress(prog...)
423 |                             end
424 |                         end
425 |                         if have_input
426 |                             @async upload_data(easy, input)
427 |                         end
428 |                     end
429 |                 finally
430 |                     if !(interrupted[])
431 |                         if interrupt_task !== nothing
432 |                             # trigger interrupt
433 |                             notify(interrupt)
434 |                             wait(interrupt_task)
435 |                         else
436 |                             remove_handle(downloader.multi, easy)
437 |                         end
438 |                     end
439 |                 end
440 | 
441 |                 # return the response or throw an error
442 |                 response = Response(get_response_info(easy)...)
443 |                 easy.code == Curl.CURLE_OK && return response
444 |                 message = get_curl_errstr(easy)
445 |                 if easy.code == typemax(Curl.CURLcode)
446 |                     # uninitialized code, likely a protocol error
447 |                     code = Int(0)
448 |                 else
449 |                     code = Int(easy.code)
450 |                 end
451 |                 response = RequestError(url, code, message, response)
452 |                 throw && Base.throw(response)
453 |             end
454 |         end
455 |     end
456 |     return response
457 | end
458 | 
459 | ## helper functions ##
460 | 
461 | function p_func(progress::Function, input::ArgRead, output::ArgWrite)
462 |     hasmethod(progress, NTuple{4,Int}) && return progress
463 |     hasmethod(progress, NTuple{2,Int}) ||
464 |         throw(ArgumentError("invalid progress callback"))
465 | 
466 |     input === devnull && output !== devnull &&
467 |         return (total, now, _, _) -> progress(total, now)
468 |     input !== devnull && output === devnull &&
469 |         return (_, _, total, now) -> progress(total, now)
470 | 
471 |     (dl_total, dl_now, ul_total, ul_now) ->
472 |         progress(dl_total + ul_total, dl_now + ul_now)
473 | end
474 | p_func(progress::Nothing, input::ArgRead, output::ArgWrite) = nothing
475 | 
476 | arg_read_size(path::AbstractString) = filesize(path)
477 | arg_read_size(io::Base.GenericIOBuffer) = bytesavailable(io)
478 | arg_read_size(::Base.DevNull) = 0
479 | arg_read_size(::Any) = nothing
480 | 
481 | function content_length(headers::Union{AbstractVector, AbstractDict})
482 |     for (key, value) in headers
483 |         if lowercase(key) == "content-length" && isa(value, AbstractString)
484 |             return tryparse(Int, value)
485 |         end
486 |     end
487 |     return nothing
488 | end
489 | 
490 | """
491 |     default_downloader!(
492 |         downloader = <none>
493 |     )
494 | 
495 |         downloader :: Downloader
496 | 
497 | Set the default `Downloader`. If no argument is provided, resets the default downloader so that a fresh one is created the next time the default downloader is needed.
498 | """
499 | function default_downloader!(
500 |     downloader :: Union{Downloader, Nothing} = nothing
501 | )
502 |     lock(DOWNLOAD_LOCK) do
503 |         DOWNLOADER[] = downloader
504 |     end
505 | end
506 | 
507 | # Precompile
508 | let
509 |     Curl.__init__()
510 |     d = Downloader()
511 |     f = mktemp()[1]
512 |     download("file://" * f; downloader=d)
513 |     precompile(Tuple{typeof(Downloads.download), String, String})
514 |     precompile(Tuple{typeof(Downloads.Curl.status_2xx_ok), Int64})
515 | end
516 | 
517 | end # module
518 | 


--------------------------------------------------------------------------------
/test/Downloads.jl.faceup:
--------------------------------------------------------------------------------
  1 | «s:"""
  2 | The `Downloads` module exports a function [`download`](@ref), which provides cross-platform, multi-protocol,
  3 | in-process download functionality implemented with [libcurl](https://curl.haxx.se/libcurl/).   It is used
  4 | for the `Base.download` function in Julia 1.6 or later.
  5 | 
  6 | More generally, the module exports functions and types that provide lower-level control and diagnostic information
  7 | for file downloading:
  8 | - [`download`](@ref) — download a file from a URL, erroring if it can't be downloaded
  9 | - [`request`](@ref) — request a URL, returning a `Response` object indicating success
 10 | - [`Response`](@ref) — a type capturing the status and other metadata about a request
 11 | - [`RequestError`](@ref) — an error type thrown by `download` and `request` on error
 12 | - [`Downloader`](@ref) — an object encapsulating shared resources for downloading
 13 | """»
 14 | «k:module» Downloads
 15 | 
 16 | «k:using» Base.Experimental: «:julia-ts-macro-face:@sync»
 17 | «k:using» NetworkOptions
 18 | «k:using» ArgTools
 19 | 
 20 | include(«s:"Curl/Curl.jl"»)
 21 | «k:using» .Curl
 22 | 
 23 | «k:export» download, request, Downloader, Response, RequestError, default_downloader!
 24 | 
 25 | «x:## public API types ##»
 26 | 
 27 | «s:"""
 28 |     Downloader(; [ grace::Real = 30 ])
 29 | 
 30 | `Downloader` objects are used to perform individual `download` operations.
 31 | Connections, name lookups and other resources are shared within a `Downloader`.
 32 | These connections and resources are cleaned up after a configurable grace period
 33 | (default: 30 seconds) since anything was downloaded with it, or when it is
 34 | garbage collected, whichever comes first. If the grace period is set to zero,
 35 | all resources will be cleaned up immediately as soon as there are no more
 36 | ongoing downloads in progress. If the grace period is set to `Inf` then
 37 | resources are not cleaned up until `Downloader` is garbage collected.
 38 | """»
 39 | «k:mutable» «k:struct» «t:Downloader»
 40 |     multi::«t:Multi»
 41 |     ca_roots::«t:Union»{«t:String», «t:Nothing»}
 42 |     easy_hook::«t:Union»{«t:Function», «t:Nothing»}
 43 | 
 44 |     «f:Downloader»(multi::«t:Multi») = new(multi, get_ca_roots(), EASY_HOOK[])
 45 | «k:end»
 46 | «f:Downloader»(; grace::«t:Real»=«c:30») = Downloader(Multi(grace_ms(grace)))
 47 | 
 48 | «k:function» «f:grace_ms»(grace::«t:Real»)
 49 |     grace < «c:0» && throw(ArgumentError(«s:"grace period cannot be negative: »«:julia-ts-string-interpolation-face:$»«D:grace»«s:"»))
 50 |     grace <= typemax(UInt64) ÷ «c:1000» ? round(UInt64, «c:1000»*grace) : typemax(UInt64)
 51 | «k:end»
 52 | 
 53 | «k:function» «f:easy_hook»(downloader::«t:Downloader», easy::«t:Easy», info::«t:NamedTuple»)
 54 |     «v:hook» = downloader.easy_hook
 55 |     hook !== «c:nothing» && Base.invokelatest(hook, easy, info)
 56 | «k:end»
 57 | 
 58 | «f:get_ca_roots»() = Curl.SYSTEM_SSL ? ca_roots() : ca_roots_path()
 59 | 
 60 | «k:function» «f:set_ca_roots»(downloader::«t:Downloader», easy::«t:Easy»)
 61 |     «v:ca_roots» = downloader.ca_roots
 62 |     ca_roots !== «c:nothing» && set_ca_roots_path(easy, ca_roots)
 63 | «k:end»
 64 | 
 65 | «k:const» «v:DOWNLOAD_LOCK» = ReentrantLock()
 66 | «k:const» «v:DOWNLOADER» = «t:Ref»{«t:Union»{«t:Downloader», «t:Nothing»}}(«c:nothing»)
 67 | 
 68 | «s:"""
 69 | `EASY_HOOK` is a modifable global hook to used as the default `easy_hook` on
 70 | new `Downloader` objects. This supplies a mechanism to set options for the
 71 | `Downloader` via `Curl.setopt`
 72 | 
 73 | It is expected to be function taking two arguments: an `Easy` struct and an
 74 | `info` NamedTuple with names `url`, `method` and `headers`.
 75 | """»
 76 | «k:const» «v:EASY_HOOK» = «t:Ref»{«t:Union»{«t:Function», «t:Nothing»}}(«c:nothing»)
 77 | 
 78 | «s:"""
 79 |     struct Response
 80 |         proto   :: String
 81 |         url     :: String
 82 |         status  :: Int
 83 |         message :: String
 84 |         headers :: Vector{Pair{String,String}}
 85 |     end
 86 | 
 87 | `Response` is a type capturing the properties of a successful response to a
 88 | request as an object. It has the following fields:
 89 | 
 90 | - `proto`: the protocol that was used to get the response
 91 | - `url`: the URL that was ultimately requested after following redirects
 92 | - `status`: the status code of the response, indicating success, failure, etc.
 93 | - `message`: a textual message describing the nature of the response
 94 | - `headers`: any headers that were returned with the response
 95 | 
 96 | The meaning and availability of some of these responses depends on the protocol
 97 | used for the request. For many protocols, including HTTP/S and S/FTP, a 2xx
 98 | status code indicates a successful response. For responses in protocols that do
 99 | not support headers, the headers vector will be empty. HTTP/2 does not include a
100 | status message, only a status code, so the message will be empty.
101 | """»
102 | «k:struct» «t:Response»
103 |     proto   :: «t:Union»{«t:String», «t:Nothing»}
104 |     url     :: «t:String» «x:# redirected URL»
105 |     status  :: «t:Int»
106 |     message :: «t:String»
107 |     headers :: «t:Vector»{«t:Pair»{«t:String»,«t:String»}}
108 | «k:end»
109 | 
110 | Curl.«f:status_ok»(response::«t:Response») = status_ok(response.proto, response.status)
111 | 
112 | «s:"""
113 |     struct RequestError <: ErrorException
114 |         url      :: String
115 |         code     :: Int
116 |         message  :: String
117 |         response :: Response
118 |     end
119 | 
120 | `RequestError` is a type capturing the properties of a failed response to a
121 | request as an exception object:
122 | 
123 | - `url`: the original URL that was requested without any redirects
124 | - `code`: the libcurl error code; `0` if a protocol-only error occurred
125 | - `message`: the libcurl error message indicating what went wrong
126 | - `response`: response object capturing what response info is available
127 | 
128 | The same `RequestError` type is thrown by `download` if the request was
129 | successful but there was a protocol-level error indicated by a status code that
130 | is not in the 2xx range, in which case `code` will be zero and the `message`
131 | field will be the empty string. The `request` API only throws a `RequestError`
132 | if the libcurl error `code` is non-zero, in which case the included `response`
133 | object is likely to have a `status` of zero and an empty message. There are,
134 | however, situations where a curl-level error is thrown due to a protocol error,
135 | in which case both the inner and outer code and message may be of interest.
136 | """»
137 | «k:struct» «t:RequestError» <: «t:Exception»
138 |     url      :: «t:String» «x:# original URL»
139 |     code     :: «t:Int»
140 |     message  :: «t:String»
141 |     response :: «t:Response»
142 | «k:end»
143 | 
144 | «k:function» Base.«f:showerror»(io::«t:IO», err::«t:RequestError»)
145 |     print(io, «s:"RequestError: »«:julia-ts-string-interpolation-face:$(»«D:error_message(err)»«:julia-ts-string-interpolation-face:)»«s: while requesting »«:julia-ts-string-interpolation-face:$(»«D:err.url»«:julia-ts-string-interpolation-face:)»«s:"»)
146 | «k:end»
147 | 
148 | «k:function» «f:error_message»(err::«t:RequestError»)
149 |     «v:errstr» = err.message
150 |     «v:status» = err.response.status
151 |     «v:message» = err.response.message
152 |     «v:status_re» = Regex(status == «c:0» ? «s:""» : «s:"\\b»«:julia-ts-string-interpolation-face:$»«D:status»«s:\\b"»)
153 | 
154 |     err.code == Curl.CURLE_OK &&
155 |         «k:return» isempty(message) ? «s:"Error status »«:julia-ts-string-interpolation-face:$»«D:status»«s:"» :
156 |             contains(message, status_re) ? message :
157 |                 «s:"»«:julia-ts-string-interpolation-face:$»«D:message»«s: (status »«:julia-ts-string-interpolation-face:$»«D:status»«s:)"»
158 | 
159 |     isempty(message) && !isempty(errstr) &&
160 |         «k:return» status == «c:0» ? errstr : «s:"»«:julia-ts-string-interpolation-face:$»«D:errstr»«s: (status »«:julia-ts-string-interpolation-face:$»«D:status»«s:)"»
161 | 
162 |     isempty(message) && («v:message» = «s:"Status »«:julia-ts-string-interpolation-face:$»«D:status»«s:"»)
163 |     isempty(errstr)  && («v:errstr» = «s:"curl error »«:julia-ts-string-interpolation-face:$(»«D:err.code»«:julia-ts-string-interpolation-face:)»«s:"»)
164 | 
165 |     !contains(message, status_re) && !contains(errstr, status_re) &&
166 |         («v:errstr» = «s:"status »«:julia-ts-string-interpolation-face:$»«D:status»«s:; »«:julia-ts-string-interpolation-face:$»«D:errstr»«s:"»)
167 | 
168 |     «k:return» «s:"»«:julia-ts-string-interpolation-face:$»«D:message»«s: (»«:julia-ts-string-interpolation-face:$»«D:errstr»«s:)"»
169 | «k:end»
170 | 
171 | «x:## download API ##»
172 | 
173 | «s:"""
174 |     download(url, [ output = tempname() ];
175 |         [ method = "GET", ]
176 |         [ headers = <none>, ]
177 |         [ timeout = <none>, ]
178 |         [ progress = <none>, ]
179 |         [ verbose = false, ]
180 |         [ debug = <none>, ]
181 |         [ downloader = <default>, ]
182 |     ) -> output
183 | 
184 |         url        :: AbstractString
185 |         output     :: Union{AbstractString, AbstractCmd, IO}
186 |         method     :: AbstractString
187 |         headers    :: Union{AbstractVector, AbstractDict}
188 |         timeout    :: Real
189 |         progress   :: (total::Integer, now::Integer) --> Any
190 |         verbose    :: Bool
191 |         debug      :: (type, message) --> Any
192 |         downloader :: Downloader
193 | 
194 | Download a file from the given url, saving it to `output` or if not specified, a
195 | temporary path. The `output` can also be an `IO` handle, in which case the body
196 | of the response is streamed to that handle and the handle is returned. If
197 | `output` is a command, the command is run and output is sent to it on stdin.
198 | 
199 | If the `downloader` keyword argument is provided, it must be a `Downloader`
200 | object. Resources and connections will be shared between downloads performed by
201 | the same `Downloader` and cleaned up automatically when the object is garbage
202 | collected or there have been no downloads performed with it for a grace period.
203 | See `Downloader` for more info about configuration and usage.
204 | 
205 | If the `headers` keyword argument is provided, it must be a vector or dictionary
206 | whose elements are all pairs of strings. These pairs are passed as headers when
207 | downloading URLs with protocols that supports them, such as HTTP/S.
208 | 
209 | The `timeout` keyword argument specifies a timeout for the download to complete in
210 | seconds, with a resolution of milliseconds. By default no timeout is set, but this
211 | can also be explicitly requested by passing a timeout value of `Inf`. Separately,
212 | if 20 seconds elapse without receiving any data, the download will timeout. See
213 | extended help for how to disable this timeout.
214 | 
215 | If the `progress` keyword argument is provided, it must be a callback function
216 | which will be called whenever there are updates about the size and status of the
217 | ongoing download. The callback must take two integer arguments: `total` and
218 | `now` which are the total size of the download in bytes, and the number of bytes
219 | which have been downloaded so far. Note that `total` starts out as zero and
220 | remains zero until the server gives an indication of the total size of the
221 | download (e.g. with a `Content-Length` header), which may never happen. So a
222 | well-behaved progress callback should handle a total size of zero gracefully.
223 | 
224 | If the `verbose` option is set to true, `libcurl`, which is used to implement
225 | the download functionality will print debugging information to `stderr`. If the
226 | `debug` option is set to a function accepting two `String` arguments, then the
227 | verbose option is ignored and instead the data that would have been printed to
228 | `stderr` is passed to the `debug` callback with `type` and `message` arguments.
229 | The `type` argument indicates what kind of event has occurred, and is one of:
230 | `TEXT`, `HEADER IN`, `HEADER OUT`, `DATA IN`, `DATA OUT`, `SSL DATA IN` or `SSL
231 | DATA OUT`. The `message` argument is the description of the debug event.
232 | 
233 | ## Extended Help
234 | 
235 | For further customization, use a [`Downloader`](@ref) and
236 | [`easy_hook`s](https://github.com/JuliaLang/Downloads.jl#mutual-tls-using-downloads).
237 | For example, to disable the 20 second timeout when no data is received, you may
238 | use the following:
239 | 
240 | ```jl
241 | downloader = Downloads.Downloader()
242 | downloader.easy_hook = (easy, info) -> Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_LOW_SPEED_TIME, 0)
243 | 
244 | Downloads.download("https://httpbingo.julialang.org/delay/30"; downloader)
245 | ```
246 | """»
247 | «k:function» «f:download»(
248 |     url        :: «t:AbstractString»,
249 |     output     :: «t:Union»{«t:ArgWrite», «t:Nothing»} = «c:nothing»;
250 |     method     :: «t:Union»{«t:AbstractString», «t:Nothing»} = «c:nothing»,
251 |     headers    :: «t:Union»{«t:AbstractVector», «t:AbstractDict»} = «t:Pair»{«t:String»,«t:String»}[],
252 |     timeout    :: «t:Real» = «c:Inf»,
253 |     progress   :: «t:Union»{«t:Function», «t:Nothing»} = «c:nothing»,
254 |     verbose    :: «t:Bool» = «c:false»,
255 |     debug      :: «t:Union»{«t:Function», «t:Nothing»} = «c:nothing»,
256 |     downloader :: «t:Union»{«t:Downloader», «t:Nothing»} = «c:nothing»,
257 | ) :: «t:ArgWrite»
258 |     arg_write(output) «k:do» output
259 |         «v:response» = request(
260 |             url,
261 |             «:julia-ts-keyword-argument-face:output» = output,
262 |             «:julia-ts-keyword-argument-face:method» = method,
263 |             «:julia-ts-keyword-argument-face:headers» = headers,
264 |             «:julia-ts-keyword-argument-face:timeout» = timeout,
265 |             «:julia-ts-keyword-argument-face:progress» = progress,
266 |             «:julia-ts-keyword-argument-face:verbose» = verbose,
267 |             «:julia-ts-keyword-argument-face:debug» = debug,
268 |             «:julia-ts-keyword-argument-face:downloader» = downloader,
269 |         )::«t:Response»
270 |         status_ok(response) && «k:return» output
271 |         throw(RequestError(url, Curl.CURLE_OK, «s:""», response))
272 |     «k:end»
273 | «k:end»
274 | 
275 | «x:## request API ##»
276 | 
277 | «s:"""
278 |     request(url;
279 |         [ input = <none>, ]
280 |         [ output = <none>, ]
281 |         [ method = input ? "PUT" : output ? "GET" : "HEAD", ]
282 |         [ headers = <none>, ]
283 |         [ timeout = <none>, ]
284 |         [ progress = <none>, ]
285 |         [ verbose = false, ]
286 |         [ debug = <none>, ]
287 |         [ throw = true, ]
288 |         [ downloader = <default>, ]
289 |         [ interrupt = <none>, ]
290 |     ) -> Union{Response, RequestError}
291 | 
292 |         url        :: AbstractString
293 |         input      :: Union{AbstractString, AbstractCmd, IO}
294 |         output     :: Union{AbstractString, AbstractCmd, IO}
295 |         method     :: AbstractString
296 |         headers    :: Union{AbstractVector, AbstractDict}
297 |         timeout    :: Real
298 |         progress   :: (dl_total, dl_now, ul_total, ul_now) --> Any
299 |         verbose    :: Bool
300 |         debug      :: (type, message) --> Any
301 |         throw      :: Bool
302 |         downloader :: Downloader
303 |         interrupt  :: Base.Event
304 | 
305 | Make a request to the given url, returning a `Response` object capturing the
306 | status, headers and other information about the response. The body of the
307 | response is written to `output` if specified and discarded otherwise. For HTTP/S
308 | requests, if an `input` stream is given, a `PUT` request is made; otherwise if
309 | an `output` stream is given, a `GET` request is made; if neither is given a
310 | `HEAD` request is made. For other protocols, appropriate default methods are
311 | used based on what combination of input and output are requested. The following
312 | options differ from the `download` function:
313 | 
314 | - `input` allows providing a request body; if provided default to `PUT` request
315 | - `progress` is a callback taking four integers for upload and download progress
316 | - `throw` controls whether to throw or return a `RequestError` on request error
317 | 
318 | Note that unlike `download` which throws an error if the requested URL could not
319 | be downloaded (indicated by non-2xx status code), `request` returns a `Response`
320 | object no matter what the status code of the response is. If there is an error
321 | with getting a response at all, then a `RequestError` is thrown or returned.
322 | 
323 | If the `interrupt` keyword argument is provided, it must be a `Base.Event` object.
324 | If the event is triggered while the request is in progress, the request will be
325 | cancelled and an error will be thrown. This can be used to interrupt a long
326 | running request, for example if the user wants to cancel a download.
327 | """»
328 | «k:function» «f:request»(
329 |     url        :: «t:AbstractString»;
330 |     input      :: «t:Union»{«t:ArgRead», «t:Nothing»} = «c:nothing»,
331 |     output     :: «t:Union»{«t:ArgWrite», «t:Nothing»} = «c:nothing»,
332 |     method     :: «t:Union»{«t:AbstractString», «t:Nothing»} = «c:nothing»,
333 |     headers    :: «t:Union»{«t:AbstractVector», «t:AbstractDict»} = «t:Pair»{«t:String»,«t:String»}[],
334 |     timeout    :: «t:Real» = «c:Inf»,
335 |     progress   :: «t:Union»{«t:Function», «t:Nothing»} = «c:nothing»,
336 |     verbose    :: «t:Bool» = «c:false»,
337 |     debug      :: «t:Union»{«t:Function», «t:Nothing»} = «c:nothing»,
338 |     throw      :: «t:Bool» = «c:true»,
339 |     downloader :: «t:Union»{«t:Downloader», «t:Nothing»} = «c:nothing»,
340 |     interrupt  :: «t:Union»{«t:Nothing», Base.Event} = «c:nothing»,
341 | ) :: «t:Union»{«t:Response», «t:RequestError»}
342 |     «k:if» downloader === «c:nothing»
343 |         lock(DOWNLOAD_LOCK) «k:do»
344 |             «v:downloader» = DOWNLOADER[]
345 |             «k:if» downloader === «c:nothing»
346 |                 «v:downloader» = DOWNLOADER[] = Downloader()
347 |             «k:end»
348 |         «k:end»
349 |     «k:end»
350 |     «k:local» «v:response»
351 |     «v:have_input» = input !== «c:nothing»
352 |     «v:have_output» = output !== «c:nothing»
353 |     «v:input» = something(input, devnull)
354 |     «v:output» = something(output, devnull)
355 |     «v:input_size» = arg_read_size(input)
356 |     «k:if» input_size === «c:nothing»
357 |         «x:# take input_size from content-length header if one is supplied»
358 |         «v:input_size» = content_length(headers)
359 |     «k:end»
360 |     «v:progress» = p_func(progress, input, output)
361 |     arg_read(input) «k:do» input
362 |         arg_write(output) «k:do» output
363 |             with_handle(Easy()) «k:do» easy
364 |                 «x:# setup the request»
365 |                 set_url(easy, url)
366 |                 set_timeout(easy, timeout)
367 |                 set_verbose(easy, verbose)
368 |                 set_debug(easy, debug)
369 |                 add_headers(easy, headers)
370 | 
371 |                 «x:# libcurl does not set the default header reliably so set it»
372 |                 «x:# explicitly unless user has specified it, xref»
373 |                 «x:# https://github.com/JuliaLang/Pkg.jl/pull/2357»
374 |                 «k:if» !any(kv -> lowercase(kv[«c:1»]) == «s:"user-agent"», headers)
375 |                     Curl.add_header(easy, «s:"User-Agent"», Curl.USER_AGENT)
376 |                 «k:end»
377 | 
378 |                 «k:if» have_input
379 |                     enable_upload(easy)
380 |                     «k:if» input_size !== «c:nothing»
381 |                         set_upload_size(easy, input_size)
382 |                     «k:end»
383 |                     «k:if» applicable(seek, input, «c:0»)
384 |                         set_seeker(easy) «k:do» offset
385 |                             seek(input, Int(offset))
386 |                         «k:end»
387 |                     «k:end»
388 |                 «k:else»
389 |                     set_body(easy, have_output && method != «s:"HEAD"»)
390 |                 «k:end»
391 |                 method !== «c:nothing» && set_method(easy, method)
392 |                 progress !== «c:nothing» && enable_progress(easy)
393 |                 set_ca_roots(downloader, easy)
394 |                 «v:info» = (url = url, method = method, headers = headers)
395 |                 easy_hook(downloader, easy, info)
396 | 
397 |                 «x:# do the request»
398 |                 add_handle(downloader.multi, easy)
399 |                 «v:interrupted» = Threads.Atomic{«t:Bool»}(«c:false»)
400 |                 «k:if» interrupt !== «c:nothing»
401 |                     «v:interrupt_task» = «:julia-ts-macro-face:@async» «k:begin»
402 |                         «x:# wait for the interrupt event»
403 |                         wait(interrupt)
404 |                         «x:# cancel the request»
405 |                         remove_handle(downloader.multi, easy)
406 |                         close(easy.output)
407 |                         close(easy.progress)
408 |                         interrupted[] = «c:true»
409 |                         close(input)
410 |                         notify(easy.ready)
411 |                     «k:end»
412 |                 «k:else»
413 |                     «v:interrupt_task» = «c:nothing»
414 |                 «k:end»
415 |                 «k:try» «x:# ensure handle is removed»
416 |                     «:julia-ts-macro-face:@sync» «k:begin»
417 |                         «:julia-ts-macro-face:@async» «k:for» buf «k:in» easy.output
418 |                             write(output, buf)
419 |                         «k:end»
420 |                         «k:if» progress !== «c:nothing»
421 |                             «:julia-ts-macro-face:@async» «k:for» prog «k:in» easy.progress
422 |                                 progress(prog...)
423 |                             «k:end»
424 |                         «k:end»
425 |                         «k:if» have_input
426 |                             «:julia-ts-macro-face:@async» upload_data(easy, input)
427 |                         «k:end»
428 |                     «k:end»
429 |                 «k:finally»
430 |                     «k:if» !(interrupted[])
431 |                         «k:if» interrupt_task !== «c:nothing»
432 |                             «x:# trigger interrupt»
433 |                             notify(interrupt)
434 |                             wait(interrupt_task)
435 |                         «k:else»
436 |                             remove_handle(downloader.multi, easy)
437 |                         «k:end»
438 |                     «k:end»
439 |                 «k:end»
440 | 
441 |                 «x:# return the response or throw an error»
442 |                 «v:response» = Response(get_response_info(easy)...)
443 |                 easy.code == Curl.CURLE_OK && «k:return» response
444 |                 «v:message» = get_curl_errstr(easy)
445 |                 «k:if» easy.code == typemax(Curl.CURLcode)
446 |                     «x:# uninitialized code, likely a protocol error»
447 |                     «v:code» = Int(«c:0»)
448 |                 «k:else»
449 |                     «v:code» = Int(easy.code)
450 |                 «k:end»
451 |                 «v:response» = RequestError(url, code, message, response)
452 |                 throw && Base.throw(response)
453 |             «k:end»
454 |         «k:end»
455 |     «k:end»
456 |     «k:return» response
457 | «k:end»
458 | 
459 | «x:## helper functions ##»
460 | 
461 | «k:function» «f:p_func»(progress::«t:Function», input::«t:ArgRead», output::«t:ArgWrite»)
462 |     hasmethod(progress, «t:NTuple»{«c:4»,«t:Int»}) && «k:return» progress
463 |     hasmethod(progress, «t:NTuple»{«c:2»,«t:Int»}) ||
464 |         throw(ArgumentError(«s:"invalid progress callback"»))
465 | 
466 |     input === devnull && output !== devnull &&
467 |         «k:return» (total, now, _, _) -> progress(total, now)
468 |     input !== devnull && output === devnull &&
469 |         «k:return» (_, _, total, now) -> progress(total, now)
470 | 
471 |     (dl_total, dl_now, ul_total, ul_now) ->
472 |         progress(dl_total + ul_total, dl_now + ul_now)
473 | «k:end»
474 | «f:p_func»(progress::«t:Nothing», input::«t:ArgRead», output::«t:ArgWrite») = «c:nothing»
475 | 
476 | «f:arg_read_size»(path::«t:AbstractString») = filesize(path)
477 | «f:arg_read_size»(io::Base.«t:GenericIOBuffer») = bytesavailable(io)
478 | «f:arg_read_size»(::«t:Base.DevNull») = «c:0»
479 | «f:arg_read_size»(::«t:Any») = «c:nothing»
480 | 
481 | «k:function» «f:content_length»(headers::«t:Union»{«t:AbstractVector», «t:AbstractDict»})
482 |     «k:for» (key, value) «k:in» headers
483 |         «k:if» lowercase(key) == «s:"content-length"» && isa(value, AbstractString)
484 |             «k:return» tryparse(Int, value)
485 |         «k:end»
486 |     «k:end»
487 |     «k:return» «c:nothing»
488 | «k:end»
489 | 
490 | «s:"""
491 |     default_downloader!(
492 |         downloader = <none>
493 |     )
494 | 
495 |         downloader :: Downloader
496 | 
497 | Set the default `Downloader`. If no argument is provided, resets the default downloader so that a fresh one is created the next time the default downloader is needed.
498 | """»
499 | «k:function» «f:default_downloader!»(
500 |     downloader :: «t:Union»{«t:Downloader», «t:Nothing»} = «c:nothing»
501 | )
502 |     lock(DOWNLOAD_LOCK) «k:do»
503 |         DOWNLOADER[] = downloader
504 |     «k:end»
505 | «k:end»
506 | 
507 | «x:# Precompile»
508 | «k:let»
509 |     Curl.__init__()
510 |     «v:d» = Downloader()
511 |     «v:f» = mktemp()[«c:1»]
512 |     download(«s:"file://"» * f; «:julia-ts-keyword-argument-face:downloader»=d)
513 |     precompile(«t:Tuple»{typeof(Downloads.download), «t:String», «t:String»})
514 |     precompile(«t:Tuple»{typeof(Downloads.Curl.status_2xx_ok), «t:Int64»})
515 | «k:end»
516 | 
517 | «k:end» «x:# module»
518 | 


--------------------------------------------------------------------------------
/test/test.el:
--------------------------------------------------------------------------------
 1 | ;;; test.el --- ert tests for font-locking with julia-ts-mode
 2 | ;;; Commentary:
 3 | ;; Load this package then run M-x ert RET t RET
 4 | 
 5 | ;;; Code:
 6 | (require 'faceup)
 7 | 
 8 | (defvar julia-font-lock-test-dir (faceup-this-file-directory))
 9 | 
10 | (defun julia-font-lock-test (file)
11 |   "Test that the julia FILE is fontifies as the .faceup file describes."
12 |   (faceup-test-font-lock-file 'julia-ts-mode
13 |                               (concat julia-font-lock-test-dir file)))
14 | 
15 | (faceup-defexplainer julia-font-lock-test)
16 | 
17 | (ert-deftest julia-font-lock-file-test ()
18 |   (should (julia-font-lock-test "ArgTools.jl"))
19 |   (should (julia-font-lock-test "Downloads.jl"))
20 |   (should (julia-font-lock-test "tree-sitter-corpus.jl")))
21 | 
22 | (provide 'test)
23 | ;;; test.el ends here
24 | 


--------------------------------------------------------------------------------
/test/tree-sitter-corpus.jl:
--------------------------------------------------------------------------------
  1 | # ==============================
  2 | # tuple collections
  3 | # ==============================
  4 | 
  5 | ()
  6 | # There's no (,)
  7 | 
  8 | (1) # NOT a tuple
  9 | (1,)
 10 | (2,3,4,)
 11 | 
 12 | # ==============================
 13 | # named tuple collections
 14 | # ==============================
 15 | 
 16 | (a = 1) # NOT a tuple
 17 | (a = 1,)
 18 | (a = 1, b = 2)
 19 | (;)
 20 | (; a)
 21 | (; a = 1)
 22 | (; a = 1, b = 2)
 23 | (; a, foo.b)
 24 | 
 25 | # ==============================
 26 | # vector array collections
 27 | # ==============================
 28 | 
 29 | []
 30 | # There's no [,]
 31 | [x]
 32 | [x,]
 33 | [1, 2]
 34 | 
 35 | # Check unary-and-binary-operators
 36 | [x.-y, 2]
 37 | 
 38 | # ==============================
 39 | # matrix array collections
 40 | # ==============================
 41 | 
 42 | [x;]
 43 | [1 2]
 44 | [1 2; 3 4]
 45 | [1 2
 46 |  3 4]
 47 | [1
 48 |  2
 49 |  3
 50 | ]
 51 | [
 52 |  a;
 53 |  b;
 54 |  c;
 55 | ]
 56 | [1;; 2;; 3;;; 4;; 5;; 6;;]
 57 | Int[1 2 3 4]
 58 | 
 59 | # ========================================
 60 | # comprehension array collections
 61 | # ========================================
 62 | 
 63 | [x for x in xs]
 64 | [x
 65 |   for x in xs
 66 |   if x > 0
 67 | ]
 68 | UInt[b(c, e) for c in d for e in f]
 69 | 
 70 | f(1, 2, i for i in iter)
 71 | (b(c, e) for c in d, e = 5 if e)
 72 | 
 73 | # ==============================
 74 | # module definitions
 75 | # ==============================
 76 | 
 77 | module A
 78 | 
 79 | baremodule B end
 80 | 
 81 | module C
 82 | end
 83 | 
 84 | end
 85 | 
 86 | # ==============================
 87 | # abstract type definitions
 88 | # ==============================
 89 | 
 90 | abstract type T end
 91 | abstract type T <: S end
 92 | abstract type T{S} <: U end
 93 | 
 94 | # ==============================
 95 | # primitive type definitions
 96 | # ==============================
 97 | 
 98 | primitive type T 8 end
 99 | primitive type T <: S 16 end
100 | primitive type Ptr{T} 32 end
101 | 
102 | # ==============================
103 | # struct definitions
104 | # ==============================
105 | 
106 | struct Unit end
107 | 
108 | struct MyInt field::Int end
109 | 
110 | mutable struct Foo
111 |   bar
112 |   baz::Float64
113 | end
114 | 
115 | # ==============================
116 | # parametric struct definitions
117 | # ==============================
118 | 
119 | struct Point{T}
120 |   x::T
121 |   y::T
122 | end
123 | 
124 | struct Rational{T<:Integer} <: Real
125 |   num::T
126 |   den::T
127 | end
128 | 
129 | mutable struct MyVec <: AbstractArray
130 |   foos::Vector{Foo}
131 | end
132 | 
133 | # ==============================
134 | # function definitions
135 | # ==============================
136 | 
137 | function f end
138 | 
139 | function nop() end
140 | 
141 | function I(x) x end
142 | 
143 | function Base.rand(n::MyInt)
144 |     return 4
145 | end
146 | 
147 | function Γ(z)
148 |     gamma(z)
149 | end
150 | 
151 | function ⊕(x, y)
152 |     x + y
153 | end
154 | 
155 | function fix2(f, x)
156 |     return function(y)
157 |         f(x, y)
158 |     end
159 | end
160 | 
161 | function (foo::Foo)()
162 | end
163 | 
164 | # ==============================
165 | # short function definitions
166 | # ==============================
167 | 
168 | s(n) = n + 1
169 | 
170 | Base.foo(x) = x
171 | 
172 | ι(n) = range(1, n)
173 | 
174 | ⊗(x, y) = x * y
175 | 
176 | (+)(x, y) = x + y
177 | 
178 | # ==============================
179 | # function definition parameters
180 | # ==============================
181 | 
182 | function f(x, y::Int, z=1, ws...) end
183 | 
184 | function (::Type{Int}, x::Int = 1, y::Int...) end
185 | 
186 | function apply(f, args...; kwargs...)
187 | end
188 | 
189 | function g(; x, y::Int, z = 1, kwargs...) nothing end
190 | 
191 | # ==================================================
192 | # function definition return types
193 | # ==================================================
194 | 
195 | function s(n)::MyInt
196 |     MyInt(n + 1)
197 | end
198 | 
199 | function bar(f, xs::Foo.Bar)::Foo.Bar
200 |     map(f, xs)
201 | end
202 | 
203 | # ==================================================
204 | # function definition tuple parameters
205 | # ==================================================
206 | 
207 | function swap((x, y))
208 |     (y, x)
209 | end
210 | 
211 | function f((x, y)=(1,2))
212 |     (x, y)
213 | end
214 | 
215 | function car((x, y)::Tuple{T, T}) where T
216 |     x
217 | end
218 | 
219 | # ==================================================
220 | # type parametric function definition parameters
221 | # ==================================================
222 | 
223 | function f(x::T) where T
224 | end
225 | 
226 | function f(n::N) where {N <: Integer}
227 |     n
228 | end
229 | 
230 | f(n::N, m::M) where {N <: Number} where {M <: Integer} = n^m
231 | 
232 | Foo{T}(x::T) where {T} = x
233 | 
234 | function norm(p::Point{T} where T<:Real)
235 |     norm2(p)
236 | end
237 | 
238 | Base.show(io::IO, ::MIME"text/plain", m::Method; kwargs...) = show_method(io, m, kwargs)
239 | 
240 | # ==============================
241 | # macro definitions
242 | # ==============================
243 | 
244 | macro name(s::Symbol)
245 |     String(s)
246 | end
247 | 
248 | macro count(args...) length(args) end
249 | 
250 | # ==============================
251 | # identifiers
252 | # ==============================
253 | 
254 | abc_123_ABC
255 | _fn!
256 | ρ; φ; z
257 | ℝ
258 | x′
259 | θ̄
260 | logŷ
261 | ϵ
262 | ŷ
263 | 🙋
264 | 🦀
265 | 
266 | # ==============================
267 | # field expressions
268 | # ==============================
269 | 
270 | foo.x
271 | bar.x.y.z
272 | 
273 | (a[1].b().c).d
274 | 
275 | Base.:+
276 | 
277 | df."a"
278 | 
279 | # ==============================
280 | # index expressions
281 | # ==============================
282 | 
283 | a[1, 2, 3]
284 | a[1, :]
285 | "foo"[1]
286 | 
287 | # ==============================
288 | # type parametrized expressions
289 | # ==============================
290 | 
291 | Vector{Int}
292 | Vector{<:Number}
293 | $(usertype){T}
294 | 
295 | {:x} ~ normal(0, 1)
296 | 
297 | # ==============================
298 | # function call expressions
299 | # ==============================
300 | 
301 | f()
302 | g("hi", 2)
303 | h(d...)
304 | 
305 | f(e; f = g)
306 | g(arg; kwarg)
307 | 
308 | new{typeof(xs)}(xs)
309 | 
310 | # ========================================
311 | # function call expressions with do blocks
312 | # ========================================
313 | 
314 | reduce(xs) do x, y
315 |   f(x, y)
316 | end
317 | 
318 | # ==============================
319 | # macro call expressions
320 | # ==============================
321 | 
322 | @assert x == y "a message"
323 | 
324 | @testset "a" begin
325 |   b = c
326 | end
327 | 
328 | @. a * x + b
329 | 
330 | joinpath(@__DIR__, "grammar.js")
331 | 
332 | @macroexpand @async accept(socket)
333 | 
334 | @Meta.dump 1, 2
335 | Meta.@dump x = 1
336 | 
337 | # ==============================
338 | # closed macro call expressions
339 | # ==============================
340 | 
341 | @enum(Light, red, yellow, green)
342 | f(@nospecialize(x)) = x
343 | 
344 | @m[1, 2] + 1
345 | @m [1, 2] + 1
346 | 
347 | # ==============================
348 | # quote expressions
349 | # ==============================
350 | 
351 | :foo
352 | :const
353 | 
354 | :(x; y)
355 | :(x, y)
356 | :[x, y, z]
357 | 
358 | :+
359 | :->
360 | :(::)
361 | 
362 | # ==============================
363 | # interpolation expressions
364 | # ==============================
365 | 
366 | $foo
367 | $obj.field
368 | $(obj.field)
369 | $f(x)
370 | $f[1, 2]
371 | $"foo"
372 | 
373 | using $Package: $(name)
374 | 
375 | # Similar definitions in Gadfly/src/varset.jl
376 | mutable struct $(name)
377 |   $(vars...)
378 | end
379 | function $(name)($(parameters_expr))
380 |     $(name)($(parsed_vars...))
381 | end
382 | 
383 | # ==============================
384 | # adjoint expressions
385 | # ==============================
386 | 
387 | [u, v]'
388 | A'[i]
389 | (x, y)'
390 | f'(x)
391 | :a'
392 | 
393 | # ==============================
394 | # juxtaposition expressions
395 | # ==============================
396 | 
397 | 1x
398 | 2v[i]
399 | 3f(x)
400 | 4df.a
401 | 5u"kg"
402 | x'x
403 | 2x^2 - .3x
404 | 2(x-1)^2 - 3(x-1)
405 | 
406 | # =============================
407 | # arrow function expressions
408 | # =============================
409 | 
410 | x -> x^2
411 | (x,y,z)-> 2*x + y - z
412 | ()->3
413 | () -> (sleep(0.1); i += 1; l)
414 | a -> a = 2, 3
415 | 
416 | # ==============================
417 | # boolean literals
418 | # ==============================
419 | 
420 | true
421 | false
422 | 
423 | # ==============================
424 | # integer number literals
425 | # ==============================
426 | 
427 | 0b01
428 | 0o01234567
429 | 0123456789
430 | 123_456_789
431 | 0x0123456789_abcdef_ABCDEF
432 | 
433 | # ==============================
434 | # float number literals
435 | # ==============================
436 | 
437 | 0123456789.
438 | .0123456789
439 | 0123456789.0123456789
440 | 
441 | 9e10
442 | 9E-1
443 | 9f10
444 | 9f-1
445 | 
446 | .1e10
447 | 1.1e10
448 | 1.e10
449 | 
450 | 0x0123456789_abcdef.ABCDEFp0
451 | 0x0123456789_abcdef_ABCDEF.p-1
452 | 0x.0123456789_abcdef_ABCDEFp1
453 | 
454 | # ==============================
455 | # character literals
456 | # ==============================
457 | 
458 | ' '
459 | 'o'
460 | '\t'
461 | '\uffff'
462 | '\U10ffff'
463 | 
464 | # ==============================
465 | # string literals
466 | # ==============================
467 | 
468 | ""
469 | "\""
470 | "foo
471 |  bar"
472 | "this is a \"string\"."
473 | """this is also a "string"."""
474 | band = "Interpol"
475 | "$band is a cool band"
476 | "$(2π) is a cool number"
477 | "cells interlinked within $("cells interlinked whithin $("cells interlinked whithin one stem")")"
478 | 
479 | # ==============================
480 | # command string literals
481 | # ==============================
482 | 
483 | `pwd`
484 | m`pwd`
485 | `cd $dir`
486 | `echo \`cmd\``
487 | ```
488 | echo "\033[31mred\033[m"
489 | ```
490 | 
491 | # ==============================
492 | # non-standard string literals
493 | # ==============================
494 | 
495 | # FIXME: \s shouldn't be an escape_sequence here
496 | trailing_ws = r"\s+$"
497 | version = v"1.0"
498 | K"\\"
499 | 
500 | # ==============================
501 | # comments
502 | # ==============================
503 | 
504 | # comment
505 | #= comment =#
506 | #=
507 | comment
508 | =#
509 | x = #= comment =# 1
510 | 
511 | #=
512 | nested #= comments =# =#
513 | #==#
514 | 
515 | # ==============================
516 | # assignment operators
517 | # ==============================
518 | 
519 | a = b
520 | a .. b = a * b
521 | tup = 1, 2, 3
522 | car, cdr... = list
523 | c &= d ÷= e
524 | 
525 | # ==============================
526 | # binary arithmetic operators
527 | # ==============================
528 | 
529 | a + b
530 | a ++ 1 × b ⥌ 2 → c
531 | a // b
532 | x = A \ (v × w)
533 | 
534 | # ==============================
535 | # other binary operators
536 | # ==============================
537 | 
538 | a & b | c
539 | (x >>> 16, x >>> 8, x) .& 0xff
540 | 
541 | Dict(b => c, d => e)
542 | 
543 | x |>
544 |   f |>
545 |   g
546 | 
547 | 1..10
548 | (1:10...,)
549 | 
550 | # ==============================
551 | # binary comparison operators
552 | # ==============================
553 | 
554 | a === 1
555 | a! != 0
556 | 
557 | A ⊆ B ⊆ C
558 | x ≥ 0 ≥ z
559 | 
560 | # ==============================
561 | # unary operators
562 | # ==============================
563 | 
564 | -A'
565 | +a
566 | -b
567 | √9
568 | !p === !(p)
569 | 1 ++ +2
570 | 
571 | # =============================
572 | # operator broadcasting
573 | # =============================
574 | 
575 | a .* b .+ c
576 | .~[x]
577 | 
578 | # ==============================
579 | # ternary operator
580 | # ==============================
581 | 
582 | x = batch_size == 1 ?
583 |   rand(10) :
584 |   rand(10, batch_size)
585 | 
586 | # ==============================
587 | # operators as values
588 | # ==============================
589 | 
590 | x = +
591 | ⪯ = .≤
592 | print(:)
593 | foo(^, ÷, -)
594 | 
595 | # ==============================
596 | # compound statements
597 | # ==============================
598 | 
599 | begin
600 | end
601 | 
602 | begin
603 |     foo
604 |     bar
605 |     baz
606 | end
607 | 
608 | # ==============================
609 | # quote statements
610 | # ==============================
611 | 
612 | quote end
613 | 
614 | quote
615 |   x = 1
616 |   y = 2
617 |   x + y
618 | end
619 | 
620 | # ==============================
621 | # let statements
622 | # ==============================
623 | 
624 | let
625 | end
626 | 
627 | let var1 = value1, var2, var3 = value3
628 |     code
629 | end
630 | 
631 | # ==============================
632 | # if statements
633 | # ==============================
634 | 
635 | if a
636 | elseif b
637 | else
638 | end
639 | 
640 | if true 1 else 0 end
641 | 
642 | if a
643 |   b()
644 | elseif c
645 |   d()
646 |   d()
647 | else
648 |   e()
649 | end
650 | 
651 | # ==============================
652 | # try statements
653 | # ==============================
654 | 
655 | try catch end
656 | try finally end
657 | 
658 | try
659 |     sqrt(x)
660 | catch
661 |     sqrt(complex(x, 0))
662 | end
663 | 
664 | try
665 |     operate_on_file(f)
666 | finally
667 |     close(f)
668 | end
669 | 
670 | try
671 |     # fallible
672 | catch
673 |     # handle errors
674 | else
675 |     # do something if there were no exceptions
676 | end
677 | 
678 | # ==============================
679 | # for statements
680 | # ==============================
681 | 
682 | for x in xs end
683 | 
684 | for x in xs foo!(x) end
685 | 
686 | for i in [1, 2, 3]
687 |   print(i)
688 | end
689 | 
690 | for (a, b) in c
691 |   print(a, b)
692 | end
693 | 
694 | # ==============================
695 | # for outer statements
696 | # ==============================
697 | 
698 | n = 1
699 | for outer n in range
700 |   body
701 | end
702 | 
703 | for outer x = iter1, outer y = iter2
704 |   body
705 | end
706 | 
707 | # ==============================
708 | # while statements
709 | # ==============================
710 | 
711 | while true end
712 | 
713 | while i < 5
714 |   print(i)
715 |   continue
716 |   break
717 | end
718 | 
719 | while a(); b(); end
720 | 
721 | # ==============================
722 | # return statements
723 | # ==============================
724 | 
725 | return
726 | return a
727 | return a || b
728 | return a, b, c
729 | 
730 | # ==============================
731 | # export statements
732 | # ==============================
733 | 
734 | export a
735 | export a, b, +, (*)
736 | export @macroMcAtface
737 | public a
738 | public a, b, +, (*)
739 | public @macroMcAtface
740 | 
741 | # ==============================
742 | # import statements
743 | # ==============================
744 | 
745 | import Pkg
746 | 
747 | using Sockets
748 | 
749 | using ..Foo, ..Bar
750 | 
751 | import CSV, Chain, DataFrames
752 | 
753 | import Base: show, @kwdef, +, (*)
754 | 
755 | import LinearAlgebra as la
756 | 
757 | import Base: @view as @v
758 | 
759 | # ===============================
760 | # const statements
761 | # ===============================
762 | 
763 | const x = 5
764 | const y, z = 1, 2
765 | 
766 | (0, const x, y = 1, 2)
767 | 
768 | # ===============================
769 | # local statements
770 | # ===============================
771 | 
772 | local x
773 | local y, z = 1, 2
774 | local foo() = 3
775 | local function bar() 4 end
776 | 
777 | # ===============================
778 | # global statements
779 | # ===============================
780 | 
781 | global X
782 | global Y, Z = 11, 42
783 | global foo() = 3
784 | global function bar() 4 end
785 | 


--------------------------------------------------------------------------------
/test/tree-sitter-corpus.jl.faceup:
--------------------------------------------------------------------------------
  1 | «x:# ==============================»
  2 | «x:# tuple collections»
  3 | «x:# ==============================»
  4 | 
  5 | ()
  6 | «x:# There's no (,)»
  7 | 
  8 | («c:1») «x:# NOT a tuple»
  9 | («c:1»,)
 10 | («c:2»,«c:3»,«c:4»,)
 11 | 
 12 | «x:# ==============================»
 13 | «x:# named tuple collections»
 14 | «x:# ==============================»
 15 | 
 16 | («v:a» = «c:1») «x:# NOT a tuple»
 17 | (a = «c:1»,)
 18 | (a = «c:1», b = «c:2»)
 19 | (;)
 20 | (; a)
 21 | (; a = «c:1»)
 22 | (; a = «c:1», b = «c:2»)
 23 | (; a, foo.b)
 24 | 
 25 | «x:# ==============================»
 26 | «x:# vector array collections»
 27 | «x:# ==============================»
 28 | 
 29 | []
 30 | «x:# There's no [,]»
 31 | [x]
 32 | [x,]
 33 | [«c:1», «c:2»]
 34 | 
 35 | «x:# Check unary-and-binary-operators»
 36 | [x.-y, «c:2»]
 37 | 
 38 | «x:# ==============================»
 39 | «x:# matrix array collections»
 40 | «x:# ==============================»
 41 | 
 42 | [x;]
 43 | [«c:1» «c:2»]
 44 | [«c:1» «c:2»; «c:3» «c:4»]
 45 | [«c:1» «c:2»
 46 |  «c:3» «c:4»]
 47 | [«c:1»
 48 |  «c:2»
 49 |  «c:3»
 50 | ]
 51 | [
 52 |  a;
 53 |  b;
 54 |  c;
 55 | ]
 56 | [«c:1»;; «c:2»;; «c:3»;;; «c:4»;; «c:5»;; «c:6»;;]
 57 | Int[«c:1» «c:2» «c:3» «c:4»]
 58 | 
 59 | «x:# ========================================»
 60 | «x:# comprehension array collections»
 61 | «x:# ========================================»
 62 | 
 63 | [x «k:for» x «k:in» xs]
 64 | [x
 65 |   «k:for» x «k:in» xs
 66 |   «k:if» x > «c:0»
 67 | ]
 68 | UInt[b(c, e) «k:for» c «k:in» d «k:for» e «k:in» f]
 69 | 
 70 | f(«c:1», «c:2», i «k:for» i «k:in» iter)
 71 | (b(c, e) «k:for» c «k:in» d, e = «c:5» «k:if» e)
 72 | 
 73 | «x:# ==============================»
 74 | «x:# module definitions»
 75 | «x:# ==============================»
 76 | 
 77 | «k:module» A
 78 | 
 79 | «k:baremodule» B «k:end»
 80 | 
 81 | «k:module» C
 82 | «k:end»
 83 | 
 84 | «k:end»
 85 | 
 86 | «x:# ==============================»
 87 | «x:# abstract type definitions»
 88 | «x:# ==============================»
 89 | 
 90 | «k:abstract» «k:type» «t:T» «k:end»
 91 | «k:abstract» «k:type» «t:T» <: «t:S» «k:end»
 92 | «k:abstract» «k:type» «t:T»{«t:S»} <: «t:U» «k:end»
 93 | 
 94 | «x:# ==============================»
 95 | «x:# primitive type definitions»
 96 | «x:# ==============================»
 97 | 
 98 | «k:primitive» «k:type» «t:T» «c:8» «k:end»
 99 | «k:primitive» «k:type» «t:T» <: «t:S» «c:16» «k:end»
100 | «k:primitive» «k:type» «t:Ptr»{«t:T»} «c:32» «k:end»
101 | 
102 | «x:# ==============================»
103 | «x:# struct definitions»
104 | «x:# ==============================»
105 | 
106 | «k:struct» «t:Unit» «k:end»
107 | 
108 | «k:struct» «t:MyInt» field::«t:Int» «k:end»
109 | 
110 | «k:mutable» «k:struct» «t:Foo»
111 |   bar
112 |   baz::«t:Float64»
113 | «k:end»
114 | 
115 | «x:# ==============================»
116 | «x:# parametric struct definitions»
117 | «x:# ==============================»
118 | 
119 | «k:struct» «t:Point»{«t:T»}
120 |   x::«t:T»
121 |   y::«t:T»
122 | «k:end»
123 | 
124 | «k:struct» «t:Rational»{«t:T»<:«t:Integer»} <: «t:Real»
125 |   num::«t:T»
126 |   den::«t:T»
127 | «k:end»
128 | 
129 | «k:mutable» «k:struct» «t:MyVec» <: «t:AbstractArray»
130 |   foos::«t:Vector»{«t:Foo»}
131 | «k:end»
132 | 
133 | «x:# ==============================»
134 | «x:# function definitions»
135 | «x:# ==============================»
136 | 
137 | «k:function» «f:f» «k:end»
138 | 
139 | «k:function» «f:nop»() «k:end»
140 | 
141 | «k:function» «f:I»(x) x «k:end»
142 | 
143 | «k:function» Base.«f:rand»(n::«t:MyInt»)
144 |     «k:return» «c:4»
145 | «k:end»
146 | 
147 | «k:function» «f:Γ»(z)
148 |     gamma(z)
149 | «k:end»
150 | 
151 | «k:function» «f:⊕»(x, y)
152 |     x + y
153 | «k:end»
154 | 
155 | «k:function» «f:fix2»(f, x)
156 |     «k:return» «k:function»(y)
157 |         f(x, y)
158 |     «k:end»
159 | «k:end»
160 | 
161 | «k:function» (foo::«t:Foo»)()
162 | «k:end»
163 | 
164 | «x:# ==============================»
165 | «x:# short function definitions»
166 | «x:# ==============================»
167 | 
168 | «f:s»(n) = n + «c:1»
169 | 
170 | Base.«f:foo»(x) = x
171 | 
172 | «f:ι»(n) = range(«c:1», n)
173 | 
174 | «f:⊗»(x, y) = x * y
175 | 
176 | («f:+»)(x, y) = x + y
177 | 
178 | «x:# ==============================»
179 | «x:# function definition parameters»
180 | «x:# ==============================»
181 | 
182 | «k:function» «f:f»(x, y::«t:Int», «:julia-ts-keyword-argument-face:z»=«c:1», ws...) «k:end»
183 | 
184 | «k:function» (::«t:Type{Int}», x::«t:Int» = «c:1», y::«t:Int»...) «k:end»
185 | 
186 | «k:function» «f:apply»(f, args...; kwargs...)
187 | «k:end»
188 | 
189 | «k:function» «f:g»(; x, y::«t:Int», «:julia-ts-keyword-argument-face:z» = «c:1», kwargs...) «c:nothing» «k:end»
190 | 
191 | «x:# ==================================================»
192 | «x:# function definition return types»
193 | «x:# ==================================================»
194 | 
195 | «k:function» «f:s»(n)::«t:MyInt»
196 |     MyInt(n + «c:1»)
197 | «k:end»
198 | 
199 | «k:function» «f:bar»(f, xs::Foo.«t:Bar»)::Foo.«t:Bar»
200 |     map(f, xs)
201 | «k:end»
202 | 
203 | «x:# ==================================================»
204 | «x:# function definition tuple parameters»
205 | «x:# ==================================================»
206 | 
207 | «k:function» «f:swap»((x, y))
208 |     (y, x)
209 | «k:end»
210 | 
211 | «k:function» «f:f»((x, y)=(«c:1»,«c:2»))
212 |     (x, y)
213 | «k:end»
214 | 
215 | «k:function» «f:car»((x, y)::«t:Tuple»{«t:T», «t:T»}) «k:where» «t:T»
216 |     x
217 | «k:end»
218 | 
219 | «x:# ==================================================»
220 | «x:# type parametric function definition parameters»
221 | «x:# ==================================================»
222 | 
223 | «k:function» «f:f»(x::«t:T») «k:where» «t:T»
224 | «k:end»
225 | 
226 | «k:function» «f:f»(n::«t:N») «k:where» {«t:N» <: «t:Integer»}
227 |     n
228 | «k:end»
229 | 
230 | f(n::«t:N», m::«t:M») «k:where» {«t:N» <: «t:Number»} «k:where» {«t:M» <: «t:Integer»} = n^m
231 | 
232 | «t:Foo»{«t:T»}(x::«t:T») «k:where» {«t:T»} = x
233 | 
234 | «k:function» «f:norm»(p::«t:Point»{«t:T»} «k:where» T<:Real)
235 |     norm2(p)
236 | «k:end»
237 | 
238 | Base.«f:show»(io::«t:IO», ::«t:MIME"text/plain"», m::«t:Method»; kwargs...) = show_method(io, m, kwargs)
239 | 
240 | «x:# ==============================»
241 | «x:# macro definitions»
242 | «x:# ==============================»
243 | 
244 | «k:macro» «f:name»(s::«t:Symbol»)
245 |     String(s)
246 | «k:end»
247 | 
248 | «k:macro» «f:count»(args...) length(args) «k:end»
249 | 
250 | «x:# ==============================»
251 | «x:# identifiers»
252 | «x:# ==============================»
253 | 
254 | abc_123_ABC
255 | _fn!
256 | ρ; φ; z
257 | ℝ
258 | x′
259 | θ̄
260 | logŷ
261 | ϵ
262 | ŷ
263 | 🙋
264 | 🦀
265 | 
266 | «x:# ==============================»
267 | «x:# field expressions»
268 | «x:# ==============================»
269 | 
270 | foo.x
271 | bar.x.y.z
272 | 
273 | (a[«c:1»].b().c).d
274 | 
275 | Base.«:julia-ts-quoted-symbol-face::+»
276 | 
277 | df.«s:"a"»
278 | 
279 | «x:# ==============================»
280 | «x:# index expressions»
281 | «x:# ==============================»
282 | 
283 | a[«c:1», «c:2», «c:3»]
284 | a[«c:1», :]
285 | «s:"foo"»[«c:1»]
286 | 
287 | «x:# ==============================»
288 | «x:# type parametrized expressions»
289 | «x:# ==============================»
290 | 
291 | «t:Vector»{«t:Int»}
292 | «t:Vector»{<:«t:Number»}
293 | «:julia-ts-interpolation-expression-face:$(»«D:usertype»«:julia-ts-interpolation-expression-face:)»{«t:T»}
294 | 
295 | {«:julia-ts-quoted-symbol-face::x»} ~ normal(«c:0», «c:1»)
296 | 
297 | «x:# ==============================»
298 | «x:# function call expressions»
299 | «x:# ==============================»
300 | 
301 | f()
302 | g(«s:"hi"», «c:2»)
303 | h(d...)
304 | 
305 | f(e; «:julia-ts-keyword-argument-face:f» = g)
306 | g(arg; kwarg)
307 | 
308 | «t:new»{typeof(xs)}(xs)
309 | 
310 | «x:# ========================================»
311 | «x:# function call expressions with do blocks»
312 | «x:# ========================================»
313 | 
314 | reduce(xs) «k:do» x, y
315 |   f(x, y)
316 | «k:end»
317 | 
318 | «x:# ==============================»
319 | «x:# macro call expressions»
320 | «x:# ==============================»
321 | 
322 | «:julia-ts-macro-face:@assert» x == y «s:"a message"»
323 | 
324 | «:julia-ts-macro-face:@testset» «s:"a"» «k:begin»
325 |   «v:b» = c
326 | «k:end»
327 | 
328 | «:julia-ts-macro-face:@.» a * x + b
329 | 
330 | joinpath(«:julia-ts-macro-face:@__DIR__», «s:"grammar.js"»)
331 | 
332 | «:julia-ts-macro-face:@macroexpand» «:julia-ts-macro-face:@async» accept(socket)
333 | 
334 | «:julia-ts-macro-face:@Meta.dump» «c:1», «c:2»
335 | Meta.«:julia-ts-macro-face:@dump» «v:x» = «c:1»
336 | 
337 | «x:# ==============================»
338 | «x:# closed macro call expressions»
339 | «x:# ==============================»
340 | 
341 | «:julia-ts-macro-face:@enum»(Light, red, yellow, green)
342 | «f:f»(«:julia-ts-macro-face:@nospecialize»(x)) = x
343 | 
344 | «:julia-ts-macro-face:@m»[«c:1», «c:2»] + «c:1»
345 | «:julia-ts-macro-face:@m» [«c:1», «c:2»] + «c:1»
346 | 
347 | «x:# ==============================»
348 | «x:# quote expressions»
349 | «x:# ==============================»
350 | 
351 | «:julia-ts-quoted-symbol-face::foo»
352 | «:julia-ts-quoted-symbol-face::const»
353 | 
354 | «:julia-ts-quoted-symbol-face::(x; y)»
355 | «:julia-ts-quoted-symbol-face::(x, y)»
356 | «:julia-ts-quoted-symbol-face::[x, y, z]»
357 | 
358 | «:julia-ts-quoted-symbol-face::+»
359 | «:julia-ts-quoted-symbol-face::->»
360 | «:julia-ts-quoted-symbol-face::(::)»
361 | 
362 | «x:# ==============================»
363 | «x:# interpolation expressions»
364 | «x:# ==============================»
365 | 
366 | «:julia-ts-interpolation-expression-face:$»«D:foo»
367 | «:julia-ts-interpolation-expression-face:$»«D:obj».field
368 | «:julia-ts-interpolation-expression-face:$(»«D:obj.field»«:julia-ts-interpolation-expression-face:)»
369 | «:julia-ts-interpolation-expression-face:$»«D:f»(x)
370 | «:julia-ts-interpolation-expression-face:$»«D:f»[«c:1», «c:2»]
371 | «:julia-ts-interpolation-expression-face:$»«s:"foo"»
372 | 
373 | «k:using» «:julia-ts-interpolation-expression-face:$»«D:Package»: «:julia-ts-interpolation-expression-face:$(»«D:name»«:julia-ts-interpolation-expression-face:)»
374 | 
375 | «x:# Similar definitions in Gadfly/src/varset.jl»
376 | «k:mutable» «k:struct» «:julia-ts-interpolation-expression-face:$(»«D:name»«:julia-ts-interpolation-expression-face:)»
377 |   «:julia-ts-interpolation-expression-face:$(»«D:vars...»«:julia-ts-interpolation-expression-face:)»
378 | «k:end»
379 | «k:function» «:julia-ts-interpolation-expression-face:$(»«D:name»«:julia-ts-interpolation-expression-face:)»(«:julia-ts-interpolation-expression-face:$(»«D:parameters_expr»«:julia-ts-interpolation-expression-face:)»)
380 |     «:julia-ts-interpolation-expression-face:$(»«D:name»«:julia-ts-interpolation-expression-face:)»(«:julia-ts-interpolation-expression-face:$(»«D:parsed_vars...»«:julia-ts-interpolation-expression-face:)»)
381 | «k:end»
382 | 
383 | «x:# ==============================»
384 | «x:# adjoint expressions»
385 | «x:# ==============================»
386 | 
387 | [u, v]'
388 | A'[i]
389 | (x, y)'
390 | f'(x)
391 | «:julia-ts-quoted-symbol-face::a»'
392 | 
393 | «x:# ==============================»
394 | «x:# juxtaposition expressions»
395 | «x:# ==============================»
396 | 
397 | «c:1»x
398 | «c:2»v[i]
399 | «c:3»f(x)
400 | «c:4»df.a
401 | «c:5»«s:u"kg"»
402 | x'x
403 | «c:2»x^«c:2» - «c:.3»x
404 | «c:2»(x-«c:1»)^«c:2» - «c:3»(x-«c:1»)
405 | 
406 | «x:# =============================»
407 | «x:# arrow function expressions»
408 | «x:# =============================»
409 | 
410 | x -> x^«c:2»
411 | (x,y,z)-> «c:2»*x + y - z
412 | ()->«c:3»
413 | () -> (sleep(«c:0.1»); i += «c:1»; l)
414 | a -> «v:a» = «c:2», «c:3»
415 | 
416 | «x:# ==============================»
417 | «x:# boolean literals»
418 | «x:# ==============================»
419 | 
420 | «c:true»
421 | «c:false»
422 | 
423 | «x:# ==============================»
424 | «x:# integer number literals»
425 | «x:# ==============================»
426 | 
427 | «c:0b01»
428 | «c:0o01234567»
429 | «c:0123456789»
430 | «c:123_456_789»
431 | «c:0x0123456789_abcdef_ABCDEF»
432 | 
433 | «x:# ==============================»
434 | «x:# float number literals»
435 | «x:# ==============================»
436 | 
437 | «c:0123456789.»
438 | «c:.0123456789»
439 | «c:0123456789.0123456789»
440 | 
441 | «c:9e10»
442 | «c:9E-1»
443 | «c:9f10»
444 | «c:9f-1»
445 | 
446 | «c:.1e10»
447 | «c:1.1e10»
448 | «c:1.e10»
449 | 
450 | «c:0x0123456789_abcdef.ABCDEFp0»
451 | «c:0x0123456789_abcdef_ABCDEF.p-1»
452 | «c:0x.0123456789_abcdef_ABCDEFp1»
453 | 
454 | «x:# ==============================»
455 | «x:# character literals»
456 | «x:# ==============================»
457 | 
458 | «c:' '»
459 | «c:'o'»
460 | «c:'\t'»
461 | «c:'\uffff'»
462 | «c:'\U10ffff'»
463 | 
464 | «x:# ==============================»
465 | «x:# string literals»
466 | «x:# ==============================»
467 | 
468 | «s:""»
469 | «s:"\""»
470 | «s:"foo
471 |  bar"»
472 | «s:"this is a \"string\"."»
473 | «s:"""this is also a "string"."""»
474 | «v:band» = «s:"Interpol"»
475 | «s:"»«:julia-ts-string-interpolation-face:$»«D:band»«s: is a cool band"»
476 | «s:"»«:julia-ts-string-interpolation-face:$(»«c:2»«D:π»«:julia-ts-string-interpolation-face:)»«s: is a cool number"»
477 | «s:"cells interlinked within »«:julia-ts-string-interpolation-face:$(»«D:"cells interlinked whithin »«:julia-ts-string-interpolation-face:$(»«D:"cells interlinked whithin one stem"»«:julia-ts-string-interpolation-face:)»«D:"»«:julia-ts-string-interpolation-face:)»«s:"»
478 | 
479 | «x:# ==============================»
480 | «x:# command string literals»
481 | «x:# ==============================»
482 | 
483 | «s:`pwd`»
484 | «s:m`pwd`»
485 | «s:`cd »«:julia-ts-string-interpolation-face:$»«D:dir»«s:`»
486 | «s:`echo \`cmd\``»
487 | «s:```
488 | echo "\033[31mred\033[m"
489 | ```»
490 | 
491 | «x:# ==============================»
492 | «x:# non-standard string literals»
493 | «x:# ==============================»
494 | 
495 | «x:# FIXME: \s shouldn't be an escape_sequence here»
496 | «v:trailing_ws» = «s:r"\s+$"»
497 | «v:version» = «s:v"1.0"»
498 | «s:K"\\"»
499 | 
500 | «x:# ==============================»
501 | «x:# comments»
502 | «x:# ==============================»
503 | 
504 | «x:# comment»
505 | «x:#= comment =#»
506 | «x:#=
507 | comment
508 | =#»
509 | «v:x» = «x:#= comment =#» «c:1»
510 | 
511 | «x:#=
512 | nested #= comments =# =#»
513 | «x:#==#»
514 | 
515 | «x:# ==============================»
516 | «x:# assignment operators»
517 | «x:# ==============================»
518 | 
519 | «v:a» = b
520 | a «f:..» b = a * b
521 | «v:tup» = «c:1», «c:2», «c:3»
522 | «v:car», cdr... = list
523 | c &= d ÷= e
524 | 
525 | «x:# ==============================»
526 | «x:# binary arithmetic operators»
527 | «x:# ==============================»
528 | 
529 | a + b
530 | a ++ «c:1» × b ⥌ «c:2» → c
531 | a // b
532 | «v:x» = A \ (v × w)
533 | 
534 | «x:# ==============================»
535 | «x:# other binary operators»
536 | «x:# ==============================»
537 | 
538 | a & b | c
539 | (x >>> «c:16», x >>> «c:8», x) .& «c:0xff»
540 | 
541 | Dict(b => c, d => e)
542 | 
543 | x |>
544 |   f |>
545 |   g
546 | 
547 | «c:1»..«c:10»
548 | («c:1»:«c:10»...,)
549 | 
550 | «x:# ==============================»
551 | «x:# binary comparison operators»
552 | «x:# ==============================»
553 | 
554 | a === «c:1»
555 | a! != «c:0»
556 | 
557 | A ⊆ B ⊆ C
558 | x ≥ «c:0» ≥ z
559 | 
560 | «x:# ==============================»
561 | «x:# unary operators»
562 | «x:# ==============================»
563 | 
564 | -A'
565 | +a
566 | -b
567 | √«c:9»
568 | !p === !(p)
569 | «c:1» ++ +«c:2»
570 | 
571 | «x:# =============================»
572 | «x:# operator broadcasting»
573 | «x:# =============================»
574 | 
575 | a .* b .+ c
576 | .~[x]
577 | 
578 | «x:# ==============================»
579 | «x:# ternary operator»
580 | «x:# ==============================»
581 | 
582 | «v:x» = batch_size == «c:1» ?
583 |   rand(«c:10») :
584 |   rand(«c:10», batch_size)
585 | 
586 | «x:# ==============================»
587 | «x:# operators as values»
588 | «x:# ==============================»
589 | 
590 | «v:x» = +
591 | «v:⪯» = .≤
592 | print(:)
593 | foo(^, ÷, -)
594 | 
595 | «x:# ==============================»
596 | «x:# compound statements»
597 | «x:# ==============================»
598 | 
599 | «k:begin»
600 | «k:end»
601 | 
602 | «k:begin»
603 |     foo
604 |     bar
605 |     baz
606 | «k:end»
607 | 
608 | «x:# ==============================»
609 | «x:# quote statements»
610 | «x:# ==============================»
611 | 
612 | «k:quote» «k:end»
613 | 
614 | «k:quote»
615 |   «v:x» = «c:1»
616 |   «v:y» = «c:2»
617 |   x + y
618 | «k:end»
619 | 
620 | «x:# ==============================»
621 | «x:# let statements»
622 | «x:# ==============================»
623 | 
624 | «k:let»
625 | «k:end»
626 | 
627 | «k:let» «v:var1» = value1, «v:var2», «v:var3» = value3
628 |     code
629 | «k:end»
630 | 
631 | «x:# ==============================»
632 | «x:# if statements»
633 | «x:# ==============================»
634 | 
635 | «k:if» a
636 | «k:elseif» b
637 | «k:else»
638 | «k:end»
639 | 
640 | «k:if» «c:true» «c:1» «k:else» «c:0» «k:end»
641 | 
642 | «k:if» a
643 |   b()
644 | «k:elseif» c
645 |   d()
646 |   d()
647 | «k:else»
648 |   e()
649 | «k:end»
650 | 
651 | «x:# ==============================»
652 | «x:# try statements»
653 | «x:# ==============================»
654 | 
655 | «k:try» «k:catch» «k:end»
656 | «k:try» «k:finally» «k:end»
657 | 
658 | «k:try»
659 |     sqrt(x)
660 | «k:catch»
661 |     sqrt(complex(x, «c:0»))
662 | «k:end»
663 | 
664 | «k:try»
665 |     operate_on_file(f)
666 | «k:finally»
667 |     close(f)
668 | «k:end»
669 | 
670 | «k:try»
671 |     «x:# fallible»
672 | «k:catch»
673 |     «x:# handle errors»
674 | «k:else»
675 |     «x:# do something if there were no exceptions»
676 | «k:end»
677 | 
678 | «x:# ==============================»
679 | «x:# for statements»
680 | «x:# ==============================»
681 | 
682 | «k:for» x «k:in» xs «k:end»
683 | 
684 | «k:for» x «k:in» xs foo!(x) «k:end»
685 | 
686 | «k:for» i «k:in» [«c:1», «c:2», «c:3»]
687 |   print(i)
688 | «k:end»
689 | 
690 | «k:for» (a, b) «k:in» c
691 |   print(a, b)
692 | «k:end»
693 | 
694 | «x:# ==============================»
695 | «x:# for outer statements»
696 | «x:# ==============================»
697 | 
698 | «v:n» = «c:1»
699 | «k:for» «k:outer» n «k:in» range
700 |   body
701 | «k:end»
702 | 
703 | «k:for» «k:outer» x = iter1, «k:outer» y = iter2
704 |   body
705 | «k:end»
706 | 
707 | «x:# ==============================»
708 | «x:# while statements»
709 | «x:# ==============================»
710 | 
711 | «k:while» «c:true» «k:end»
712 | 
713 | «k:while» i < «c:5»
714 |   print(i)
715 |   «k:continue»
716 |   «k:break»
717 | «k:end»
718 | 
719 | «k:while» a(); b(); «k:end»
720 | 
721 | «x:# ==============================»
722 | «x:# return statements»
723 | «x:# ==============================»
724 | 
725 | «k:return»
726 | «k:return» a
727 | «k:return» a || b
728 | «k:return» a, b, c
729 | 
730 | «x:# ==============================»
731 | «x:# export statements»
732 | «x:# ==============================»
733 | 
734 | «k:export» a
735 | «k:export» a, b, +, (*)
736 | «k:export» «:julia-ts-macro-face:@macroMcAtface»
737 | «k:public» a
738 | «k:public» a, b, +, (*)
739 | «k:public» «:julia-ts-macro-face:@macroMcAtface»
740 | 
741 | «x:# ==============================»
742 | «x:# import statements»
743 | «x:# ==============================»
744 | 
745 | «k:import» Pkg
746 | 
747 | «k:using» Sockets
748 | 
749 | «k:using» ..Foo, ..Bar
750 | 
751 | «k:import» CSV, Chain, DataFrames
752 | 
753 | «k:import» Base: show, «:julia-ts-macro-face:@kwdef», +, (*)
754 | 
755 | «k:import» LinearAlgebra as la
756 | 
757 | «k:import» Base: «:julia-ts-macro-face:@view» as «:julia-ts-macro-face:@v»
758 | 
759 | «x:# ===============================»
760 | «x:# const statements»
761 | «x:# ===============================»
762 | 
763 | «k:const» «v:x» = «c:5»
764 | «k:const» «v:y», «v:z» = «c:1», «c:2»
765 | 
766 | («c:0», «k:const» «v:x», «v:y» = «c:1», «c:2»)
767 | 
768 | «x:# ===============================»
769 | «x:# local statements»
770 | «x:# ===============================»
771 | 
772 | «k:local» «v:x»
773 | «k:local» «v:y», «v:z» = «c:1», «c:2»
774 | «k:local» «f:foo»() = «c:3»
775 | «k:local» «k:function» «f:bar»() «c:4» «k:end»
776 | 
777 | «x:# ===============================»
778 | «x:# global statements»
779 | «x:# ===============================»
780 | 
781 | «k:global» «v:X»
782 | «k:global» «v:Y», «v:Z» = «c:11», «c:42»
783 | «k:global» «f:foo»() = «c:3»
784 | «k:global» «k:function» «f:bar»() «c:4» «k:end»
785 | 


--------------------------------------------------------------------------------