├── .gitignore ├── Curve Macro.ipynb ├── From Macros to DSLs in Julia - Part 1 - Macros.ipynb ├── From Macros to DSLs in Julia - Part 2 - DSLs.ipynb ├── LICENSE ├── README.md ├── Rewriting Expressions to Tuple Functions.ipynb ├── Statistics in Julia - Maximum Likelihood Estimation.ipynb └── solutions └── part_1 └── three_valued_logic ├── src.jl └── tests.jl /.gitignore: -------------------------------------------------------------------------------- 1 | # Files generated by invoking Julia with --code-coverage 2 | *.jl.cov 3 | *.jl.*.cov 4 | 5 | # Files generated by invoking Julia with --track-allocation 6 | *.jl.mem 7 | 8 | # System-specific files and directories generated by the BinaryProvider and BinDeps packages 9 | # They contain absolute paths specific to the host computer, and so should not be committed 10 | deps/deps.jl 11 | deps/build.log 12 | deps/downloads/ 13 | deps/usr/ 14 | deps/src/ 15 | 16 | # Build artifacts for creating documentation generated by the Documenter package 17 | docs/build/ 18 | docs/site/ 19 | 20 | # File generated by Pkg, the package manager, based on a corresponding Project.toml 21 | # It records a fixed state of all packages used by the project. As such, it should not be 22 | # committed for packages, but should be committed for applications that require a static 23 | # environment. 24 | Manifest.toml 25 | -------------------------------------------------------------------------------- /Curve Macro.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import Plots: plot" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 2, 15 | "metadata": {}, 16 | "outputs": [ 17 | { 18 | "data": { 19 | "text/plain": [ 20 | "expr_to_function_and_string (generic function with 1 method)" 21 | ] 22 | }, 23 | "execution_count": 2, 24 | "metadata": {}, 25 | "output_type": "execute_result" 26 | } 27 | ], 28 | "source": [ 29 | "function expr_to_function_and_string(e::Expr)::Tuple{Expr, String}\n", 30 | " (\n", 31 | " :(x -> $e),\n", 32 | " String(Symbol(e)),\n", 33 | " )\n", 34 | "end" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 3, 40 | "metadata": {}, 41 | "outputs": [ 42 | { 43 | "data": { 44 | "text/plain": [ 45 | "(:(x->begin\n", 46 | " #= In[2]:3 =#\n", 47 | " x + sin(x ^ 2)\n", 48 | " end), \"x + sin(x ^ 2)\")" 49 | ] 50 | }, 51 | "execution_count": 3, 52 | "metadata": {}, 53 | "output_type": "execute_result" 54 | } 55 | ], 56 | "source": [ 57 | "let e = :(x + sin(x^2))\n", 58 | " expr_to_function_and_string(e)\n", 59 | "end" 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": 4, 65 | "metadata": {}, 66 | "outputs": [ 67 | { 68 | "data": { 69 | "text/plain": [ 70 | "@curve (macro with 1 method)" 71 | ] 72 | }, 73 | "execution_count": 4, 74 | "metadata": {}, 75 | "output_type": "execute_result" 76 | } 77 | ], 78 | "source": [ 79 | "macro curve(e, from, to)\n", 80 | " @assert isa(from, Number)\n", 81 | " @assert isa(to, Number)\n", 82 | " func_expr, expr_str = expr_to_function_and_string(e)\n", 83 | " quote\n", 84 | " plot(\n", 85 | " range($from, $to, length = 1000),\n", 86 | " $func_expr,\n", 87 | " xlabel=\"x\",\n", 88 | " ylabel=$expr_str,\n", 89 | " leg=false,\n", 90 | " )\n", 91 | " end\n", 92 | "end" 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 5, 98 | "metadata": {}, 99 | "outputs": [ 100 | { 101 | "data": { 102 | "image/svg+xml": [ 103 | "\n", 104 | "\n", 105 | "\n", 106 | " \n", 107 | " \n", 108 | " \n", 109 | "\n", 110 | "\n", 113 | "\n", 114 | " \n", 115 | " \n", 116 | " \n", 117 | "\n", 118 | "\n", 121 | "\n", 122 | " \n", 123 | " \n", 124 | " \n", 125 | "\n", 126 | "\n", 129 | "\n", 132 | "\n", 135 | "\n", 138 | "\n", 141 | "\n", 144 | "\n", 147 | "\n", 150 | "\n", 153 | "\n", 156 | "\n", 159 | "\n", 162 | "\n", 165 | "\n", 168 | "\n", 171 | "\n", 174 | "\n", 177 | "\n", 180 | "\n", 183 | "\n", 186 | "\n", 189 | "\n", 192 | "\n", 295 | "\n" 296 | ] 297 | }, 298 | "execution_count": 5, 299 | "metadata": {}, 300 | "output_type": "execute_result" 301 | } 302 | ], 303 | "source": [ 304 | "@curve(x + sin(x^2), -5.0, 5.0)" 305 | ] 306 | }, 307 | { 308 | "cell_type": "code", 309 | "execution_count": 6, 310 | "metadata": {}, 311 | "outputs": [ 312 | { 313 | "data": { 314 | "image/svg+xml": [ 315 | "\n", 316 | "\n", 317 | "\n", 318 | " \n", 319 | " \n", 320 | " \n", 321 | "\n", 322 | "\n", 325 | "\n", 326 | " \n", 327 | " \n", 328 | " \n", 329 | "\n", 330 | "\n", 333 | "\n", 334 | " \n", 335 | " \n", 336 | " \n", 337 | "\n", 338 | "\n", 341 | "\n", 344 | "\n", 347 | "\n", 350 | "\n", 353 | "\n", 356 | "\n", 359 | "\n", 362 | "\n", 365 | "\n", 368 | "\n", 371 | "\n", 374 | "\n", 377 | "\n", 380 | "\n", 383 | "\n", 386 | "\n", 389 | "\n", 392 | "\n", 395 | "\n", 398 | "\n", 401 | "\n", 404 | "\n", 507 | "\n" 508 | ] 509 | }, 510 | "execution_count": 6, 511 | "metadata": {}, 512 | "output_type": "execute_result" 513 | } 514 | ], 515 | "source": [ 516 | "@curve(sin(x^2) + x, -5.0, 5.0)" 517 | ] 518 | } 519 | ], 520 | "metadata": { 521 | "@webio": { 522 | "lastCommId": null, 523 | "lastKernelId": null 524 | }, 525 | "kernelspec": { 526 | "display_name": "Julia 1.4.1", 527 | "language": "julia", 528 | "name": "julia-1.4" 529 | }, 530 | "language_info": { 531 | "file_extension": ".jl", 532 | "mimetype": "application/julia", 533 | "name": "julia", 534 | "version": "1.5.0" 535 | } 536 | }, 537 | "nbformat": 4, 538 | "nbformat_minor": 4 539 | } 540 | -------------------------------------------------------------------------------- /From Macros to DSLs in Julia - Part 1 - Macros.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Introduction" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "Many new users find Julia's Lisp-style macros confusing. For people without experience using syntactic macros in another language, Julia's macros present several challenges: macros introduce multiple pieces of unfamiliar syntax (i.e. `@` and `$`); macros differ from functions in fairly subtle ways that make it unclear when to use which; and authoring macros requires a fairly deep understanding of Julia's syntax because Julia has a large range of expressions that can occur in normal code.\n", 15 | "\n", 16 | "The goal of this series of notebooks is to help users overcome those challenges so they can learn to write macros with confidence. By the end of the series, you should understand macros well enough to implement a DSL in Julia for yourself. In this first notebook, we won't implement any DSL's and will instead focus on writing simpler macros. But the second notebook will entirely focus on issues that arise in developing a DSL using macros.\n", 17 | "\n", 18 | "To avoid unhelpful redundancies with existing documentation, this document assumes that you've already read the [entire manual section on metaprogramming](https://docs.julialang.org/en/v1/manual/metaprogramming/). Although I assume you've read that section of the language manual, I also assume that you're reading this document because:\n", 19 | "\n", 20 | "1. You might not have fully understood some of the subtler points in the manual. For example, you might not be entirely sure that you understand how `$`, `esc` or `QuoteNode` work.\n", 21 | "2. You're not entirely comfortable writing macros on your own.\n", 22 | "\n", 23 | "To improve your understanding of the core concepts relevant to writing macros, we're going to proceed as follows:\n", 24 | "\n", 25 | "1. We'll review Julia's representations of quoted code and consider the kinds of expressions that you'll need to be able to reason about to produce reliable macros that work correctly across a broad range of inputs.\n", 26 | "2. We'll learn to distinguish quotation from quasiquotation to help us understand how `$` works in macros.\n", 27 | "3. We'll talk about writing functions over expressions, which should be used to make macro definitions easier to read.\n", 28 | "4. We'll discuss what hygiene means and understand how `esc` lets you tell Julia to not enforce hygiene.\n", 29 | "5. We'll contrast the order in which macros are expanded with the order in which functions are evaluated.\n", 30 | "6. We'll discuss when to use macros instead of functions.\n", 31 | "\n", 32 | "The approach we'll take is driven by focusing on examples that illustrate the core issues. Every time you see a snippet of code, think about what it does before executing it. Then execute it to check whether your understanding was correct. Keep thinking and checking until your own predictions about what should happen coincide with what happens.\n", 33 | "\n", 34 | "In addition to many executable examples, there are also several exercises in this notebook. You should try to complete as many of them as possible to solidify your grasp on the core concepts." 35 | ] 36 | }, 37 | { 38 | "cell_type": "markdown", 39 | "metadata": {}, 40 | "source": [ 41 | "## What is a Macro?" 42 | ] 43 | }, 44 | { 45 | "cell_type": "markdown", 46 | "metadata": {}, 47 | "source": [ 48 | "At their core, macros are functions that:\n", 49 | "\n", 50 | "* Take in one or more pieces of quoted code as their arguments.\n", 51 | "* Produce one piece of quoted code as their return value.\n", 52 | "* Are evaluated exactly once at compile time. After evaluating the macro this one time, the returned quoted code is substituted in the calling code in place of the original macro call. Once this substitution occurs, there is no trace left behind of the macro's presence. (*Footnote: A few very exceptional macros generate source code that could never be written without using macros. You should assume that you will not be writing any such macros in the near future.*)\n", 53 | "\n", 54 | "Most of what makes writing macros challenging stems from a few subtleties related to these three issues. It is especially easy to be confused about:\n", 55 | "\n", 56 | "* How macro arguments are quoted before macro execution begins.\n", 57 | "* How the quoted code that your macro produces is munged/postprocessed by Julia to prevent accidentally creating or mutating any local variables.\n", 58 | "\n", 59 | "We'll therefore start by focusing on those two topics." 60 | ] 61 | }, 62 | { 63 | "cell_type": "markdown", 64 | "metadata": {}, 65 | "source": [ 66 | "## Quoting Code Produces More Types than Just Expr Objects" 67 | ] 68 | }, 69 | { 70 | "cell_type": "markdown", 71 | "metadata": {}, 72 | "source": [ 73 | "As noted earlier, a macro is, at core, a function that takes in quoted code as its arguments. As such, we need to be sure we understand how quoted code behaves in Julia. This is a little messy for two reasons:\n", 74 | "\n", 75 | "* Not every argument to a macro will be an instance of Julia's `Expr` type. Quoted code can consist of several types of values beyond the `Expr` type.\n", 76 | "* The way in which code is quoted before passing it as an argument to a macro is different than what can be achieved using the quotation mechanisms that are available for use outside of macros.\n", 77 | "\n", 78 | "To understand both issues, let's start by looking at a few examples of quoting code using the `:()` and `quote` operators that are built into Julia." 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": null, 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [ 87 | ":(1)" 88 | ] 89 | }, 90 | { 91 | "cell_type": "code", 92 | "execution_count": null, 93 | "metadata": {}, 94 | "outputs": [], 95 | "source": [ 96 | ":(:x)" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": null, 102 | "metadata": {}, 103 | "outputs": [], 104 | "source": [ 105 | ":(1 + x)" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": null, 111 | "metadata": {}, 112 | "outputs": [], 113 | "source": [ 114 | "quote\n", 115 | " 1 + x\n", 116 | " x + 1\n", 117 | "end" 118 | ] 119 | }, 120 | { 121 | "cell_type": "code", 122 | "execution_count": null, 123 | "metadata": {}, 124 | "outputs": [], 125 | "source": [ 126 | "typeof(:(1))" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "metadata": {}, 133 | "outputs": [], 134 | "source": [ 135 | "typeof(:(:x))" 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": null, 141 | "metadata": {}, 142 | "outputs": [], 143 | "source": [ 144 | "typeof(:(1 + x))" 145 | ] 146 | }, 147 | { 148 | "cell_type": "code", 149 | "execution_count": null, 150 | "metadata": {}, 151 | "outputs": [], 152 | "source": [ 153 | "typeof(\n", 154 | " quote\n", 155 | " 1 + x\n", 156 | " x + 1\n", 157 | " end\n", 158 | ")" 159 | ] 160 | }, 161 | { 162 | "cell_type": "markdown", 163 | "metadata": {}, 164 | "source": [ 165 | "All of these snippets are examples of what I'll call quoted code going forward. But only `:(1 + x)` and the `quote` block produce objects of type `Expr`. The wide range of types produced by quoting code confuses many people who start metaprogramming in Julia because they assume that any piece of quoted code will be some kind of `Expr`. Other languages often wrap all quoted code to ensure that everything is some kind of `Expr` object. **Julia does not wrap all values in some kind of `Expr` object and so macros need to be ready to process non-`Expr` inputs.**" 166 | ] 167 | }, 168 | { 169 | "cell_type": "markdown", 170 | "metadata": {}, 171 | "source": [ 172 | "## The Many Kinds of Expr Objects" 173 | ] 174 | }, 175 | { 176 | "cell_type": "markdown", 177 | "metadata": {}, 178 | "source": [ 179 | "In addition to the diversity of types that be produced by quoting code, metaprogramming in Julia requires that you become familiar with the internal structure of the many kinds of `Expr` objects that exist. Unlike Lisp, in which there exists a small number of special forms, Julia's rich syntax generates many kinds of `Expr` objects and you unfortunately need to have some strategy for dealing with this complexity -- even if your strategy is just ignoring most kinds of expressions in the macros you write.\n", 180 | "\n", 181 | "To see the range of internal structures that occur in `Expr` objects, let's look at a few examples:" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": {}, 188 | "outputs": [], 189 | "source": [ 190 | "dump(:(1 + x))" 191 | ] 192 | }, 193 | { 194 | "cell_type": "code", 195 | "execution_count": null, 196 | "metadata": {}, 197 | "outputs": [], 198 | "source": [ 199 | "dump(:(sin(x)))" 200 | ] 201 | }, 202 | { 203 | "cell_type": "code", 204 | "execution_count": null, 205 | "metadata": {}, 206 | "outputs": [], 207 | "source": [ 208 | "dump(:(x == y))" 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": null, 214 | "metadata": {}, 215 | "outputs": [], 216 | "source": [ 217 | "dump(\n", 218 | " quote\n", 219 | " let x = 1\n", 220 | " x + 1\n", 221 | " end\n", 222 | " end\n", 223 | ")" 224 | ] 225 | }, 226 | { 227 | "cell_type": "markdown", 228 | "metadata": {}, 229 | "source": [ 230 | "This last example also illustrates that quoted blocks of multiple lines of code include `LineNumberNode` objects in addition to `Expr` objects that refer to the code itself. These `LineNumberNode` are important for error reporting, but are mostly a source of confusion when you first starting authoring macros. If you want to simplify things by ignoring these nodes, you can install the [MacroTools.jl](https://github.com/MikeInnes/MacroTools.jl) package and use the [`rmlines` function](https://mikeinnes.github.io/MacroTools.jl/stable/utilities/#MacroTools.rmlines), although I have the impression that `rmlines` doesn't apply itself recursively." 231 | ] 232 | }, 233 | { 234 | "cell_type": "code", 235 | "execution_count": null, 236 | "metadata": {}, 237 | "outputs": [], 238 | "source": [ 239 | "import MacroTools: rmlines\n", 240 | "\n", 241 | "dump(\n", 242 | " rmlines(\n", 243 | " quote\n", 244 | " let x = 1\n", 245 | " x + 1\n", 246 | " end\n", 247 | " end\n", 248 | " )\n", 249 | ")" 250 | ] 251 | }, 252 | { 253 | "cell_type": "markdown", 254 | "metadata": {}, 255 | "source": [ 256 | "To really master macros, it's good to have a basic sense of what kinds of `Expr` objects could come up. See [this gist](https://gist.github.com/johnmyleswhite/17d8e897e995874ce04f2fc102b59991) for the kinds of `Expr` objects that occur in Julia's base code along with their relative frequencies. See [ast.scm in Julia's parser](https://github.com/JuliaLang/julia/blob/a645d7f256c2d1634869fa927c90d4e282ff0a47/src/ast.scm#L75) for what seems to be an exhaustive list of what the parser understands." 257 | ] 258 | }, 259 | { 260 | "cell_type": "markdown", 261 | "metadata": {}, 262 | "source": [ 263 | "In addition to quoting code and examining the results, it's also valuable to spend some time constructing a few expressions manually and check that match the expressions you think they do. Some examples to get you started:" 264 | ] 265 | }, 266 | { 267 | "cell_type": "code", 268 | "execution_count": null, 269 | "metadata": {}, 270 | "outputs": [], 271 | "source": [ 272 | ":(x - y) == Expr(:call, :-, :x, :y)" 273 | ] 274 | }, 275 | { 276 | "cell_type": "code", 277 | "execution_count": null, 278 | "metadata": {}, 279 | "outputs": [], 280 | "source": [ 281 | ":(sin(x)) == Expr(:call, :sin, :x)" 282 | ] 283 | }, 284 | { 285 | "cell_type": "code", 286 | "execution_count": null, 287 | "metadata": { 288 | "scrolled": true 289 | }, 290 | "outputs": [], 291 | "source": [ 292 | ":(1 + sin(x)) == Expr(:call, :+, 1, Expr(:call, :sin, :x))" 293 | ] 294 | }, 295 | { 296 | "cell_type": "markdown", 297 | "metadata": {}, 298 | "source": [ 299 | "### Exercises\n", 300 | "\n", 301 | "Now that we've discussed our first topic, it's time for you to do some exercises. To make sure you're familiar with Julia's `Expr` objects, try the following:\n", 302 | "\n", 303 | "1. Write out a `for` loop and quote it using `quote`, then print it out using `dump` to inspect its structure as an `Expr` object.\n", 304 | "2. Write out a `if` expression and inspect it.\n", 305 | "3. Write out an anonymous function (e.g. `x -> x + 1`) and inspect it.\n", 306 | "4. Take the code from [David Sanders for printing AST's as graphs](https://gist.github.com/dpsanders/5cc1acff2471d27bc583916e00d43387) and try it on a few expressions.\n", 307 | "5. Install the MacroTools package and read about the pattern matching macro called [`@capture`](https://mikeinnes.github.io/MacroTools.jl/stable/pattern-matching/) that can help you avoid dealing with the internal structure of `Expr` objects." 308 | ] 309 | }, 310 | { 311 | "cell_type": "markdown", 312 | "metadata": {}, 313 | "source": [ 314 | "## Quotation vs Quasiquotation" 315 | ] 316 | }, 317 | { 318 | "cell_type": "markdown", 319 | "metadata": {}, 320 | "source": [ 321 | "We mentioned earlier that Julia's built-in syntax for quotation (i.e. `:()` and `quote`) actually perform quotation in a different way than macros quote their arguments. In particular, macros employ true quotation whereas both `:()` and `quote` perform [quasiquotation](https://courses.cs.washington.edu/courses/cse341/04wi/lectures/14-scheme-quote.html). The distinction between these two kinds of quotation has to do with how [interpolation/splicing](https://docs.julialang.org/en/v1/manual/metaprogramming/#man-expression-interpolation-1) works.\n", 322 | "\n", 323 | "In quasiquotation, the unary `$` unary operator allows you to substitute a value inside of a quoted expression. In true quotation, you just see a call to the `$` unary operator without any interpolation.\n", 324 | "\n", 325 | "Let's see this in action." 326 | ] 327 | }, 328 | { 329 | "cell_type": "code", 330 | "execution_count": null, 331 | "metadata": {}, 332 | "outputs": [], 333 | "source": [ 334 | "x = 2" 335 | ] 336 | }, 337 | { 338 | "cell_type": "code", 339 | "execution_count": null, 340 | "metadata": {}, 341 | "outputs": [], 342 | "source": [ 343 | ":(1 + x)" 344 | ] 345 | }, 346 | { 347 | "cell_type": "code", 348 | "execution_count": null, 349 | "metadata": {}, 350 | "outputs": [], 351 | "source": [ 352 | ":(1 + $x)" 353 | ] 354 | }, 355 | { 356 | "cell_type": "code", 357 | "execution_count": null, 358 | "metadata": {}, 359 | "outputs": [], 360 | "source": [ 361 | "let y = :x\n", 362 | " :(1 + y), :(1 + $y)\n", 363 | "end" 364 | ] 365 | }, 366 | { 367 | "cell_type": "markdown", 368 | "metadata": {}, 369 | "source": [ 370 | "In contrast to the behavior of `:()`, true quotation would not perform interpolation where unary `$` occurs. Instead, we would capture the syntax that describes interpolation and produce something like the following:" 371 | ] 372 | }, 373 | { 374 | "cell_type": "code", 375 | "execution_count": null, 376 | "metadata": {}, 377 | "outputs": [], 378 | "source": [ 379 | "(\n", 380 | " :(1 + $x), # Quasiquotation\n", 381 | " Expr(:call, :+, 1, Expr(:$, :x)), # True quotation\n", 382 | ")" 383 | ] 384 | }, 385 | { 386 | "cell_type": "markdown", 387 | "metadata": {}, 388 | "source": [ 389 | "Unfortunately, Julia does not provide built-in syntax for this operation, which is why we had to write out an `Expr` object by hand. But we can write a macro to perform true quotation. This will be our first macro, so let's get excited. Before we write our macro, we need to get familiar with `QuoteNode`, which is a way of representing something that's been truly quoted. To see why it exists at all, consider the following two examples:" 390 | ] 391 | }, 392 | { 393 | "cell_type": "code", 394 | "execution_count": null, 395 | "metadata": {}, 396 | "outputs": [], 397 | "source": [ 398 | ":x, typeof(:(x))" 399 | ] 400 | }, 401 | { 402 | "cell_type": "code", 403 | "execution_count": null, 404 | "metadata": {}, 405 | "outputs": [], 406 | "source": [ 407 | ":(:x), typeof(:(:x))" 408 | ] 409 | }, 410 | { 411 | "cell_type": "markdown", 412 | "metadata": {}, 413 | "source": [ 414 | "When applied to symbols, this `QuoteNode` wrapper might seem useless since it's unclear how wrapping a `Symbol` in a `QuoteNode` helps. But `QuoteNode` has clear value when it's used inside a macro to indicate that something should stay quoted even after the macro finishes executing. To see what this means, note how the following macro behaves:" 415 | ] 416 | }, 417 | { 418 | "cell_type": "code", 419 | "execution_count": null, 420 | "metadata": {}, 421 | "outputs": [], 422 | "source": [ 423 | "macro true_quote(e)\n", 424 | " QuoteNode(e)\n", 425 | "end" 426 | ] 427 | }, 428 | { 429 | "cell_type": "code", 430 | "execution_count": null, 431 | "metadata": {}, 432 | "outputs": [], 433 | "source": [ 434 | "let y = :x\n", 435 | " (\n", 436 | " @true_quote(1 + $y),\n", 437 | " :(1 + $y),\n", 438 | " )\n", 439 | "end" 440 | ] 441 | }, 442 | { 443 | "cell_type": "markdown", 444 | "metadata": {}, 445 | "source": [ 446 | "Why have we made such a big deal about all of this? We've focused on this seemingly obscure issue because Julia macros apply true quotation to their arguments before evaluating the macro body. They do not apply quasiquotation because interpolation/splicing is a fundamentally runtime concept that involves capturing runtime values.\n", 447 | "\n", 448 | "An important implication of this is that **you cannot write macros that use standard interpolation/splicing**. If you want that behavior, you need to take steps inside of your macro to simulate it. When you see macros like `@btime` from BenchmarkTools.jl encourage using `$`, [they are doing this simulation](https://discourse.julialang.org/t/interpolation-in-macro-calls/25530).\n", 449 | "\n", 450 | "On the flip side, the fact that macros apply true quotation rather than quasiquotation means that you can assign non-standard semantics to the interpolation/splice operator." 451 | ] 452 | }, 453 | { 454 | "cell_type": "markdown", 455 | "metadata": {}, 456 | "source": [ 457 | "### Exercises\n", 458 | "\n", 459 | "1. Read the code in [BenchmarkTools.jl](https://github.com/JuliaCI/BenchmarkTools.jl) to see how they make use of the splicing operator syntax to capture local variables without incurring performance penalties that would distort benchmarking results." 460 | ] 461 | }, 462 | { 463 | "cell_type": "markdown", 464 | "metadata": {}, 465 | "source": [ 466 | "## Functions over Expressions" 467 | ] 468 | }, 469 | { 470 | "cell_type": "markdown", 471 | "metadata": {}, 472 | "source": [ 473 | "Returning again to our earlier definition, a macro is essentially a function that maps input expressions into an output expression at compile time. Because macros behave like functions, it's often easier to write macros in terms of functions over expressions. In large part, this simplifies things because these functions behave the same way whether you're inside of a macro or not in one.\n", 474 | "\n", 475 | "To build up skills writing such functions, we're going to write a few now. We'll start by writing a function to count the number of arguments in an `Expr` object:" 476 | ] 477 | }, 478 | { 479 | "cell_type": "code", 480 | "execution_count": null, 481 | "metadata": {}, 482 | "outputs": [], 483 | "source": [ 484 | "nargs(e::Expr) = length(e.args)" 485 | ] 486 | }, 487 | { 488 | "cell_type": "code", 489 | "execution_count": null, 490 | "metadata": {}, 491 | "outputs": [], 492 | "source": [ 493 | "nargs(:(1 + 2 + 3))" 494 | ] 495 | }, 496 | { 497 | "cell_type": "markdown", 498 | "metadata": {}, 499 | "source": [ 500 | "If you're surprised by this result, use `dump` to look at the expression and count the arguments by hand. You'll see there's an argument there you might not be thinking about." 501 | ] 502 | }, 503 | { 504 | "cell_type": "markdown", 505 | "metadata": {}, 506 | "source": [ 507 | "Let's do something a bit deeper and write a function that computes the maximum number of arguments in an expression recursively." 508 | ] 509 | }, 510 | { 511 | "cell_type": "code", 512 | "execution_count": null, 513 | "metadata": {}, 514 | "outputs": [], 515 | "source": [ 516 | "maxargs(x::Any) = 0\n", 517 | "maxargs(e::Expr) = max(nargs(e), maximum(map(maxargs, e.args)))" 518 | ] 519 | }, 520 | { 521 | "cell_type": "code", 522 | "execution_count": null, 523 | "metadata": {}, 524 | "outputs": [], 525 | "source": [ 526 | "e = quote\n", 527 | " z = 1 + log(x + y)\n", 528 | " exp(z)\n", 529 | "end\n", 530 | "\n", 531 | "maxargs(e)" 532 | ] 533 | }, 534 | { 535 | "cell_type": "markdown", 536 | "metadata": {}, 537 | "source": [ 538 | "Notee that we had to define both `maxargs(x::Any)` and `maxargs(e::Expr)` here. Without the definition for `maxargs(x::Any)`, the recursive definition we've used would throw a missing method error whenever any of `e.args` is not an `Expr` object.\n", 539 | "\n", 540 | "In general, the diversity of kinds of quoted code means that we need to define functions over expressions and also over other types that might occur. Many functions over expressions will need to perform recursion over the arguments to an expression; to make this work, any functions we call need to be defined over `Expr` and also more broadly. I often default to using `Any` for the broader case and letting Julia's specificity rules apply the appropriate method." 541 | ] 542 | }, 543 | { 544 | "cell_type": "markdown", 545 | "metadata": {}, 546 | "source": [ 547 | "### Exercises\n", 548 | "\n", 549 | "Getting good at writing macros is fairly simple once you're good at writing functions over quoted code. So here's some exercises to practice that more foundational skill before you start writing macros:\n", 550 | "\n", 551 | "* Write a function, `symbols(e)`, that returns a `Set{Symbol}` containing all of the symbols in an expression. Remember that this requires descending the entire expression tree recursively.\n", 552 | "* Write a function, `free_vars(e)`, that returns a `Set{Symbol}` containing all of the symbols that are unbound in an anonymous function like `x -> x + y`. The output in that case should contain `:+` and `:y` because function names are themselves symbols. As a hint, try using the `symbols` function from the previous exercise.\n", 553 | "* Write a function, `function_names(e)`, that returns all of the symbols in an anonymous function definition that must refer to something callable because they are called in the expression. For example, `x -> sin(x)` should return something containing `:sin`. But note that `x -> Float64(x)` should return `Float64`, so the results of this function aren't necessarily object whose type is a function -- you're just finding symbols bound to callables. \n", 554 | "* Write a function, `math_eval(e)`, that evaluates a purely mathematical expression like `1 + sin(2.0)` and returns the result as a `Float64` value. Do not just call `eval`; walk the expression manually and call out to `+`, `*`, etc. manually." 555 | ] 556 | }, 557 | { 558 | "cell_type": "markdown", 559 | "metadata": {}, 560 | "source": [ 561 | "**Once you've written a few functions over `Expr` objects, it will be clear that a repeated pattern is recursively descending an AST while performing some action repeatedly. The MacroTools package captures this pattern in two functions: [prewalk](https://mikeinnes.github.io/MacroTools.jl/stable/pattern-matching/#Expression-Walking-1) and [postwalk](https://mikeinnes.github.io/MacroTools.jl/stable/pattern-matching/#Expression-Walking-1).**" 562 | ] 563 | }, 564 | { 565 | "cell_type": "markdown", 566 | "metadata": {}, 567 | "source": [ 568 | "## A Simplified Model of Macro Expansion" 569 | ] 570 | }, 571 | { 572 | "cell_type": "markdown", 573 | "metadata": {}, 574 | "source": [ 575 | "Now that we've discussed a few of the main issues that arise in writing macros, let's walk through a simplified model of how macros are expanded/executed.\n", 576 | "\n", 577 | "1. Treat all of the macro arguments as code and quote that code using true quotation rather than quasiquotation.\n", 578 | "2. Evaluate the macro body just like it was a normal function body, but (a) require that the output be quoted/quotable code and (b) execute this function body exactly once.\n", 579 | "3. Take the output quoted code and run a hygiene-enforcing pass on it. We'll define hygiene in the next section.\n", 580 | "4. Insert the hygiene-enforced quoted code in place of the macro call. This happens **exactly once** at compile time; once that time has passed, it is **impossible** to distinguish the result from code written without using a macro unless the macro generates code that could never be written without using macros." 581 | ] 582 | }, 583 | { 584 | "cell_type": "markdown", 585 | "metadata": {}, 586 | "source": [ 587 | "To make this last point clear, consider the following macro that might surprise a user:" 588 | ] 589 | }, 590 | { 591 | "cell_type": "code", 592 | "execution_count": null, 593 | "metadata": {}, 594 | "outputs": [], 595 | "source": [ 596 | "macro bad_macro()\n", 597 | " x = rand()\n", 598 | " :($x)\n", 599 | "end" 600 | ] 601 | }, 602 | { 603 | "cell_type": "code", 604 | "execution_count": null, 605 | "metadata": {}, 606 | "outputs": [], 607 | "source": [ 608 | "for i in 1:10\n", 609 | " println((i, @bad_macro()))\n", 610 | "end" 611 | ] 612 | }, 613 | { 614 | "cell_type": "markdown", 615 | "metadata": {}, 616 | "source": [ 617 | "Why does `@bad_macro` have this behavior? Use `@macroexpand(@bad_macro())` to see for yourself." 618 | ] 619 | }, 620 | { 621 | "cell_type": "markdown", 622 | "metadata": {}, 623 | "source": [ 624 | "## What's Hygiene?" 625 | ] 626 | }, 627 | { 628 | "cell_type": "markdown", 629 | "metadata": {}, 630 | "source": [ 631 | "The one core macro concept we've mentioned so far without explanation is hygiene. Let's try to understand hygiene now. Hygiene is a slightly tricky topic, so you'll likely want to reread this section a few times until you're sure you understand it.\n", 632 | "\n", 633 | "The problem of hygiene is this: if you're automatically generating code, it's possible that you will introduce variable names in your generated code that will clash with existing variable names in the scope in which a macro is called. These clashes might cause your generated code to read from or write to variables that you should not interacting with. **A macro is hygienic when it does not interact with existing variables; it is non-hygienic when it does.**\n", 634 | "\n", 635 | "Because you don't know (and can't know) which variables will be in scope when your macro is called, there is a non-trivial problem to be solved here. Julia's approach to solving this hygiene problem is too automatically enforce hygiene, but offer macro authors a mechanism for writing non-hygienic macros that intentionally clash with existing variables.\n", 636 | "\n", 637 | "To understand these issues, we're intentionally going to write an incorrect macro to develop an intuition for the problems that arise. Our macro is intended to be non-hygienic and to perform assign in the calling scope, but it will succeed because the Julia hygiene-enforcing pass will change our generated code before substituting it in place of our macro's call site." 638 | ] 639 | }, 640 | { 641 | "cell_type": "code", 642 | "execution_count": null, 643 | "metadata": {}, 644 | "outputs": [], 645 | "source": [ 646 | "macro assign(name, e)\n", 647 | " Expr(:(=), name, e)\n", 648 | "end" 649 | ] 650 | }, 651 | { 652 | "cell_type": "code", 653 | "execution_count": null, 654 | "metadata": {}, 655 | "outputs": [], 656 | "source": [ 657 | "@assign(z, 1)" 658 | ] 659 | }, 660 | { 661 | "cell_type": "code", 662 | "execution_count": null, 663 | "metadata": {}, 664 | "outputs": [], 665 | "source": [ 666 | "z" 667 | ] 668 | }, 669 | { 670 | "cell_type": "markdown", 671 | "metadata": {}, 672 | "source": [ 673 | "What went wrong? And why did the assignment expression produce a value even though the assignment seems to have not worked as we intended rather than throw an error?\n", 674 | "\n", 675 | "To debug these kinds of things, it's best to use the `@macroexpand` macro to see what the `@assign` macro is being expanded to." 676 | ] 677 | }, 678 | { 679 | "cell_type": "code", 680 | "execution_count": null, 681 | "metadata": {}, 682 | "outputs": [], 683 | "source": [ 684 | "@macroexpand(@assign(z, 1))" 685 | ] 686 | }, 687 | { 688 | "cell_type": "markdown", 689 | "metadata": {}, 690 | "source": [ 691 | "What's this strange variable name? It's the result of Julia's hygiene-enforcing pass, which is intended to prevent us from overwriting existing variables during macro expansion. This pass usually makes our macros safer, but it is also a source of confusion because it introduces a gap between the expressions we generate and the expressions that end up in the resulting source code.\n", 692 | "\n", 693 | "In particular, we generated the following expression when our macro executed:" 694 | ] 695 | }, 696 | { 697 | "cell_type": "code", 698 | "execution_count": null, 699 | "metadata": {}, 700 | "outputs": [], 701 | "source": [ 702 | "let name = :z, e = :(1)\n", 703 | " Expr(:(=), name, e)\n", 704 | "end" 705 | ] 706 | }, 707 | { 708 | "cell_type": "markdown", 709 | "metadata": {}, 710 | "source": [ 711 | "But the expression that is generated as the result of executing our `@assign` macro had hygiene enforced automatically before substituting it into the calling code. So Julia effectively wrote our expression as follows using the `gensym` function, which generates a new variable name that is guaranteed to not clash with any existing variable names:" 712 | ] 713 | }, 714 | { 715 | "cell_type": "code", 716 | "execution_count": null, 717 | "metadata": {}, 718 | "outputs": [], 719 | "source": [ 720 | "let name = :z, e = :(1)\n", 721 | " Expr(:(=), gensym(name), e)\n", 722 | "end" 723 | ] 724 | }, 725 | { 726 | "cell_type": "markdown", 727 | "metadata": {}, 728 | "source": [ 729 | "If we want to prevent the hygiene-enforcing pass from changing the expression we generate, we need to use `esc` to wrap the expression in a special escape expression that tells the hygiene system to pass through the results without editing. Think of `esc` as the special all access pass you see staff use at airports that lets them skip through security screeening without undergoing a full security check. Arguably a more clear, but much longer, name for `esc` would be `no_hygiene`.\n", 730 | "\n", 731 | "Let's see what `esc` does outside of a macro and how it works to prevent the hygiene post-processing step from managling our macro's output:" 732 | ] 733 | }, 734 | { 735 | "cell_type": "code", 736 | "execution_count": null, 737 | "metadata": {}, 738 | "outputs": [], 739 | "source": [ 740 | "esc(:x)" 741 | ] 742 | }, 743 | { 744 | "cell_type": "code", 745 | "execution_count": null, 746 | "metadata": {}, 747 | "outputs": [], 748 | "source": [ 749 | "esc(:(1 + x))" 750 | ] 751 | }, 752 | { 753 | "cell_type": "code", 754 | "execution_count": null, 755 | "metadata": {}, 756 | "outputs": [], 757 | "source": [ 758 | "macro assign(name, e)\n", 759 | " Expr(:(=), esc(name), e)\n", 760 | "end" 761 | ] 762 | }, 763 | { 764 | "cell_type": "code", 765 | "execution_count": null, 766 | "metadata": {}, 767 | "outputs": [], 768 | "source": [ 769 | "@assign(z, 1)" 770 | ] 771 | }, 772 | { 773 | "cell_type": "code", 774 | "execution_count": null, 775 | "metadata": {}, 776 | "outputs": [], 777 | "source": [ 778 | "z" 779 | ] 780 | }, 781 | { 782 | "cell_type": "markdown", 783 | "metadata": {}, 784 | "source": [ 785 | "Note that we can apply this escaping at many levels:" 786 | ] 787 | }, 788 | { 789 | "cell_type": "code", 790 | "execution_count": null, 791 | "metadata": {}, 792 | "outputs": [], 793 | "source": [ 794 | "macro assign(name, e)\n", 795 | " esc(Expr(:(=), name, e))\n", 796 | "end" 797 | ] 798 | }, 799 | { 800 | "cell_type": "code", 801 | "execution_count": null, 802 | "metadata": {}, 803 | "outputs": [], 804 | "source": [ 805 | "@assign(z, 1)" 806 | ] 807 | }, 808 | { 809 | "cell_type": "code", 810 | "execution_count": null, 811 | "metadata": {}, 812 | "outputs": [], 813 | "source": [ 814 | "z" 815 | ] 816 | }, 817 | { 818 | "cell_type": "markdown", 819 | "metadata": {}, 820 | "source": [ 821 | "In general, it's wise to apply the escaping mechanism to the smallest expression that requires escaping because the hygiene-enforcement mechanism is often useful." 822 | ] 823 | }, 824 | { 825 | "cell_type": "markdown", 826 | "metadata": {}, 827 | "source": [ 828 | "## Macro Expansion Order vs Function Evaluation Order" 829 | ] 830 | }, 831 | { 832 | "cell_type": "markdown", 833 | "metadata": {}, 834 | "source": [ 835 | "One final issue we'll discuss is the order in which macros are evaluated. To see this directly, let's consider these two macros that we'll compose:" 836 | ] 837 | }, 838 | { 839 | "cell_type": "code", 840 | "execution_count": null, 841 | "metadata": {}, 842 | "outputs": [], 843 | "source": [ 844 | "macro foo(e)\n", 845 | " println(\"In foo\")\n", 846 | " e\n", 847 | "end\n", 848 | "\n", 849 | "macro bar(e)\n", 850 | " println(\"In bar\")\n", 851 | " e\n", 852 | "end" 853 | ] 854 | }, 855 | { 856 | "cell_type": "code", 857 | "execution_count": null, 858 | "metadata": {}, 859 | "outputs": [], 860 | "source": [ 861 | "@foo(@bar(1))" 862 | ] 863 | }, 864 | { 865 | "cell_type": "markdown", 866 | "metadata": {}, 867 | "source": [ 868 | "Contrast this with how functions are evaluated:" 869 | ] 870 | }, 871 | { 872 | "cell_type": "code", 873 | "execution_count": null, 874 | "metadata": {}, 875 | "outputs": [], 876 | "source": [ 877 | "function foo(e)\n", 878 | " println(\"In foo\")\n", 879 | " e\n", 880 | "end\n", 881 | "\n", 882 | "function bar(e)\n", 883 | " println(\"In bar\")\n", 884 | " e\n", 885 | "end" 886 | ] 887 | }, 888 | { 889 | "cell_type": "code", 890 | "execution_count": null, 891 | "metadata": {}, 892 | "outputs": [], 893 | "source": [ 894 | "foo(bar(1))" 895 | ] 896 | }, 897 | { 898 | "cell_type": "markdown", 899 | "metadata": {}, 900 | "source": [ 901 | "I assure you that this issue will trip you up at some point if you rely on your intuitions from calling functions, so it's good to have it pointed out to you." 902 | ] 903 | }, 904 | { 905 | "cell_type": "markdown", 906 | "metadata": {}, 907 | "source": [ 908 | "## When to Use Macros" 909 | ] 910 | }, 911 | { 912 | "cell_type": "markdown", 913 | "metadata": {}, 914 | "source": [ 915 | "As we've noted repeatedly, macros are similar to functions in many ways. In general, you should use functions instead of macros whenever possible because they are easier to reason about and are more familiar to most Julia users.\n", 916 | "\n", 917 | "That said, the cases in which macros are required are outlined well in [Chapter 8 of Paul Graham's On Lisp](https://www.csee.umbc.edu/courses/undergraduate/331/resources/lisp/onLisp/08whenToUseMacros.pdf).\n", 918 | "\n", 919 | "In Julia, some broad classes of cases include:\n", 920 | "\n", 921 | "* You want to employ existing Julia syntax, but provide non-standard semantics that cannot be achieved through function overloading. Almost all DSL's fall under this category.\n", 922 | "* You want to provide a function that operates on expressions, but doesn't require the user to quote the expressions. Our `@true_quote` macro was a variant on this idea.\n", 923 | "* You want to force some computation to occur exactly once at compile-time. Our `@bac_macro` was a bad example of this.\n", 924 | "* You want to change the values bound to a variable in the caller's scope. Julia functions cannot do this, although they can mutate the contents of mutable values passed in to them as arguments. Our `@assign` macro was an example of this.\n", 925 | "* You want to write a function that does not evaluate all of its arguments eagerly. You'll write an example of this in the upcoming exercises.\n", 926 | "* You want to write a function that captures both an expression's value and the surface syntax in which it is written. This will come up in Part 2 of this series.\n", 927 | "\n", 928 | "So there are quite a few cases in which macros are useful. But it's worth remembering the limitations of macros as well:\n", 929 | "\n", 930 | "* Essentially every macro call gets translated into Julia code that does not make use of macros. The implication of this is that macros never allow you to express computations that you can't express without macros -- they are purely syntactic sugar. This makes them very different from concepts like structs, parametric types, etc. that are not immediately reducible to other existing language features. The few exceptions to this rule are macros that produce expressions as output that have no surface syntax in Julia. Our true quotation versus quasiquotation example is almost an example of this, except that there is surface syntax based on `Expr` constructors. The `@inbounds` macro in Julia is a better example in which the relevant `Expr` objects can be constructed, but could only be used in combination with `eval` to actually take effect and therefore cannot occur in normal Julia code (where normal is defined as not making use of `eval`). In general, you should assume you will write very few of these macros because they usually can't work without changes to the Julia compiler itself. As such, it is best to stick to the intuitive rule that **macros should be used to provide a more ergonomic solution to a problem than would be possible without them**.\n", 931 | "\n", 932 | "Some closing tips for writing macros:\n", 933 | "\n", 934 | "* Before you write a macro, write an example of the code you want to generate.\n", 935 | "* Do all of the important work in functions over expressions. This is especially important if you will hit hygiene issues because the macro will produce output that may surprise you, but the function will not." 936 | ] 937 | }, 938 | { 939 | "cell_type": "markdown", 940 | "metadata": {}, 941 | "source": [ 942 | "### Exercise: A @loop Macro\n", 943 | "\n", 944 | "Write a `@loop` macro that translates:\n", 945 | "\n", 946 | "```\n", 947 | "@loop begin\n", 948 | " [LOOP BODY]\n", 949 | "end\n", 950 | "```\n", 951 | "\n", 952 | "into:\n", 953 | "\n", 954 | "```\n", 955 | "while true\n", 956 | " [LOOP BODY]\n", 957 | "end\n", 958 | "```\n", 959 | "\n", 960 | "This is a good example of how Julia's macros let you \"create new syntax\", but in a way that isn't as clean as first-class syntax because you have to employ a `begin` block. Once you've got something, think through the following design decision: should your macro throw an error if the loop body never contains a `break` expression that could end the infinite loop?" 961 | ] 962 | }, 963 | { 964 | "cell_type": "markdown", 965 | "metadata": {}, 966 | "source": [ 967 | "### Exercise: An @and Macro\n", 968 | "\n", 969 | "Write a macro that behaves like the `&&` operator, which short-circuits evaluation and only evaluates the lefthand side if it evaluates to `false`. (This is the contrast with the `&` function, which allows evaluates both the lefthand and righthand sides.\n", 970 | "\n", 971 | "Your macro should translate something like:\n", 972 | "\n", 973 | "```\n", 974 | "@and(e1, e2)\n", 975 | "```\n", 976 | "\n", 977 | "into something like:\n", 978 | "\n", 979 | "```\n", 980 | "if !e1\n", 981 | " false\n", 982 | "else\n", 983 | " e2\n", 984 | "end\n", 985 | "```\n", 986 | "\n", 987 | "It's also valuable to convince yourself that you cannot write a function that behaves this way." 988 | ] 989 | }, 990 | { 991 | "cell_type": "markdown", 992 | "metadata": {}, 993 | "source": [ 994 | "### Exercise: Macros Generating Macro Calls" 995 | ] 996 | }, 997 | { 998 | "cell_type": "markdown", 999 | "metadata": {}, 1000 | "source": [ 1001 | "It's possible for a macro to generate an expression that represents calling a macro. Use this ability to write an `@infinite_loop` macro that generates a call to itself." 1002 | ] 1003 | }, 1004 | { 1005 | "cell_type": "code", 1006 | "execution_count": 1, 1007 | "metadata": {}, 1008 | "outputs": [ 1009 | { 1010 | "data": { 1011 | "text/plain": [ 1012 | "@infinite_loop (macro with 1 method)" 1013 | ] 1014 | }, 1015 | "execution_count": 1, 1016 | "metadata": {}, 1017 | "output_type": "execute_result" 1018 | } 1019 | ], 1020 | "source": [ 1021 | "macro infinite_loop()\n", 1022 | " Expr(:macrocall, esc(Symbol(\"@infinite_loop\")), LineNumberNode(0, nothing))\n", 1023 | "end" 1024 | ] 1025 | }, 1026 | { 1027 | "cell_type": "markdown", 1028 | "metadata": {}, 1029 | "source": [ 1030 | "You can verify this macro generates an infinite loop by running it:\n", 1031 | "\n", 1032 | "```\n", 1033 | "@infinite_loop()\n", 1034 | "```" 1035 | ] 1036 | }, 1037 | { 1038 | "cell_type": "markdown", 1039 | "metadata": {}, 1040 | "source": [ 1041 | "### Exercise: Three-Valued Logic Macro" 1042 | ] 1043 | }, 1044 | { 1045 | "cell_type": "markdown", 1046 | "metadata": {}, 1047 | "source": [ 1048 | "Julia has a `missing` value that is meant to behave like the `NULL` value in SQL. In SQL, there is an extended version of Boolean logic called [three-valued logic](https://en.wikipedia.org/wiki/Three-valued_logic). To make this logic available in Julia is not possible without macros because the `&&` and `||` operators are not functions that can be overloaded to support three-valued logic. Instead, you'll need to write macro. It should take an expression like:\n", 1049 | "\n", 1050 | "```\n", 1051 | "@tvl x && y || z\n", 1052 | "```\n", 1053 | "\n", 1054 | "and generate something that computes the result using three-valued logic. This is quite challenging to do and you should expect this exercise will take you some time.\n", 1055 | "\n", 1056 | "To approach this, I suggest you implement helper macros for `@tvl_and` and `@tvl_or`. `@tvl_or` would generate something like the following:\n", 1057 | "\n", 1058 | "```\n", 1059 | "let tmp1 = x\n", 1060 | " if !ismissing(tmp1)\n", 1061 | " if tmp1\n", 1062 | " true\n", 1063 | " else\n", 1064 | " tmp2 = y\n", 1065 | " if !ismissing(tmp2)\n", 1066 | " tmp2\n", 1067 | " else\n", 1068 | " missing\n", 1069 | " end\n", 1070 | " end\n", 1071 | " else\n", 1072 | " tmp2 = y\n", 1073 | " if !ismissing(tmp2)\n", 1074 | " if tmp2\n", 1075 | " true\n", 1076 | " else\n", 1077 | " tmp1\n", 1078 | " end\n", 1079 | " else\n", 1080 | " missing\n", 1081 | " end\n", 1082 | " end\n", 1083 | "end\n", 1084 | "```" 1085 | ] 1086 | }, 1087 | { 1088 | "cell_type": "markdown", 1089 | "metadata": {}, 1090 | "source": [ 1091 | "Implementing just `@tvl_or` will be a good exercise in using `esc` and `$`." 1092 | ] 1093 | }, 1094 | { 1095 | "cell_type": "markdown", 1096 | "metadata": {}, 1097 | "source": [ 1098 | "## Other Topics" 1099 | ] 1100 | }, 1101 | { 1102 | "cell_type": "markdown", 1103 | "metadata": {}, 1104 | "source": [ 1105 | "* Getting really good at writing macros requires you to understand what information is available at macro expansion time and what information is missing. Explore the `__module__` and `__lineno__` arguments that implicitly passed to every macro to help you think about this issue. Investigate the `@isdefined` and `Base.@locals` macros as well.\n", 1106 | "* Julia's string macros let you create custom string literals that can invoke arbitrary macros at compile time. Explore this topic by using `@macroexpand` to see how the `r\"x*\"` regex works." 1107 | ] 1108 | }, 1109 | { 1110 | "cell_type": "markdown", 1111 | "metadata": {}, 1112 | "source": [ 1113 | "## References\n", 1114 | "\n", 1115 | "* [Chapter 8 of Paul Graham's On Lisp](https://www.csee.umbc.edu/courses/undergraduate/331/resources/lisp/onLisp/08whenToUseMacros.pdf)\n", 1116 | "* [Jeff Bezanson commeent on quotation versus quasiquotation at 51:00](https://www.youtube.com/watch?v=vfxS6_Sx1Pk&feature=youtu.be)\n", 1117 | "* [Wikipedia on DSL's](https://en.wikipedia.org/wiki/Domain-specific_language)" 1118 | ] 1119 | } 1120 | ], 1121 | "metadata": { 1122 | "kernelspec": { 1123 | "display_name": "Julia 1.4.1", 1124 | "language": "julia", 1125 | "name": "julia-1.4" 1126 | }, 1127 | "language_info": { 1128 | "file_extension": ".jl", 1129 | "mimetype": "application/julia", 1130 | "name": "julia", 1131 | "version": "1.4.1" 1132 | } 1133 | }, 1134 | "nbformat": 4, 1135 | "nbformat_minor": 4 1136 | } 1137 | -------------------------------------------------------------------------------- /From Macros to DSLs in Julia - Part 2 - DSLs.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "## Building Simple DSL's" 8 | ] 9 | }, 10 | { 11 | "cell_type": "markdown", 12 | "metadata": {}, 13 | "source": [ 14 | "Now that you've worked through Part 1 of this tutorial, it's time to start putting your newly acquired knowledge to use by solving more interesting problems. To that end, we'll be writing substantially more complicated macros going forward. As macros get more complicated, they often start to resemble simple [domain-specific languages](https://en.wikipedia.org/wiki/Domain-specific_language), which I'll always refer to as DSL's going forward.\n", 15 | "\n", 16 | "Macros that operate on Julia expressions can be used to write [embedded DSL's](https://en.wikipedia.org/wiki/Domain-specific_language#External_and_Embedded_Domain_Specific_Languages) that provide alternative semantics for existing Julia syntax, even though these DSL's must still strictly conform to Julia's existing syntax rules. If you want to provide novel syntax that isn't present in Julia, you can do so by writing a macro that takes in one or more strings as inputs. Julia's [non-standard string literals](https://docs.julialang.org/en/v1/manual/metaprogramming/#Non-Standard-String-Literals-1) are examples of this approach that come with additional syntactic sugar to make them seem closely integrated into the rest of the language. This approach to custom syntax via macros that take in strings instead of AST's is closely related to [reader macros in Lisp](https://letoverlambda.com/index.cl/guest/chap4.html). We won't discuss string macros in depth here, although there's an exercise at the end that you can do to experiment with them.\n", 17 | "\n", 18 | "In what follows, you're going to implement several DSL's. To make it easier to handle the complexity of writing DSL's from scratch, we'll generally construct a sequence of increasingly complicated DSL's that are nested inside of each other. Adding complexity incrementality is a good way to make macro authoring tractable since handling the entirety of Julia's syntax by default can be overwhelming.\n", 19 | "\n", 20 | "Note that these DSL's are not formally specified because we won't spell out a formal grammar or specification. This is common practice for Julia DSL's, although it also accounts for many cases in which Julia DSL's have surprising edge cases that were not considered by the DSL creators. Because these DSL's are just regular macros, they conceivably could be passed arbitrarily complicated quoted code, but they often won't be capable of processing such code. Providing good error messages is one of the tricks to writing great macros, but we won't go into it in detail. The provided solutions to some of the exercises demonstrate a few approaches to producing usable error messages." 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": {}, 26 | "source": [ 27 | "## Our First DSL: Graph Literals\n", 28 | "\n", 29 | "Programming languages typically offer literals for many of their primitive types; Julia offers literals integers (`1`), floating point numbers (`1.0`), strings (`\"foo\"`), symbols (`:foo`) and more. Using these literals can make code more readable; to see why you benefit from them, imagine a language that only had string literals and required you to write `1` as something like `parse(Int, \"1\")`. You would probably find this pretty tedious.\n", 30 | "\n", 31 | "Macros provide one mechanism for defining something like literals for more complex types. We're going to explore this topic by writing a macro that lets users write out graphs as macro calls. For example, we want something like:\n", 32 | "\n", 33 | "```\n", 34 | "@graph begin\n", 35 | " 1 -> 2\n", 36 | " 2 -> 3\n", 37 | "end\n", 38 | "```\n", 39 | "\n", 40 | "to expand to the following substantially more verbose graph constructor:\n", 41 | "\n", 42 | "\n", 43 | "```\n", 44 | "let edges = ((1, 2), (2, 3)), g = LightGraphs.SimpleDiGraph(3)\n", 45 | " for e in edges\n", 46 | " LightGraphs.add_edge!(g, e[1], e[2])\n", 47 | " end\n", 48 | " g\n", 49 | "end\n", 50 | "```\n", 51 | "\n", 52 | "We'll also want to support undirected graphs like the following:\n", 53 | "\n", 54 | "```\n", 55 | "@graph begin\n", 56 | " 1 - 2\n", 57 | " 2 - 3\n", 58 | "end\n", 59 | "```\n", 60 | "\n", 61 | "We won't support mixed graphs that contain both directed and undirected edges." 62 | ] 63 | }, 64 | { 65 | "cell_type": "markdown", 66 | "metadata": {}, 67 | "source": [ 68 | "### Implementation Stragegy\n", 69 | "\n", 70 | "To get started, we'll restrict our focus to directed graphs and then cycle back to support undirected graphs. The high-level approach will be:\n", 71 | "\n", 72 | "1. Assume we're being passed a block in which each expression is a directed edge. Anything else will be an error on the user's part, so we'll throw an error if it occurs.\n", 73 | "2. Given each of directed edge expressions, we'll transform it into a form that only contains information needed for the semantics of defining a graph. In our case, `:(1 -> 2)` will be transformed into the tuple `(1, 2)` assuming background information about the edge being directed.\n", 74 | "3. Given all of the edges represented as tuples, we'll generate the suggested graph constructor expression based on the specified edges." 75 | ] 76 | }, 77 | { 78 | "cell_type": "markdown", 79 | "metadata": {}, 80 | "source": [ 81 | "### Step 1: Processing a Block\n", 82 | "\n", 83 | "We'll do the bare minimum needed to get something working and then suggest improvements in the exercises. To start, we'll write code to process a full graph as a block expression:\n", 84 | "\n", 85 | "1. We'll assert the passed in block really is a block.\n", 86 | "2. We'll output the edges and nodes as `Vector{NTuple{2, Int}}` and `Set{Int}` respectively.\n", 87 | "3. We'll just ignore line numbers.\n", 88 | "4. We'll assume the block only contains valid directed edge expressions.\n", 89 | "5. We'll extract the edge from the expression.\n", 90 | "6. We'll return the edges and nodes." 91 | ] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": 1, 96 | "metadata": {}, 97 | "outputs": [ 98 | { 99 | "data": { 100 | "text/plain": [ 101 | "extract_graph (generic function with 1 method)" 102 | ] 103 | }, 104 | "execution_count": 1, 105 | "metadata": {}, 106 | "output_type": "execute_result" 107 | } 108 | ], 109 | "source": [ 110 | "function extract_graph(e::Expr)\n", 111 | " @assert e.head == :block\n", 112 | " edges = NTuple{2, Int}[]\n", 113 | " nodes = Set{Int}()\n", 114 | " for ex in e.args\n", 115 | " if isa(ex, LineNumberNode)\n", 116 | " continue\n", 117 | " end\n", 118 | " edge = extract_edge(ex)\n", 119 | " push!(edges, edge)\n", 120 | " push!(nodes, edge[1])\n", 121 | " push!(nodes, edge[2])\n", 122 | " end\n", 123 | " edges, nodes\n", 124 | "end" 125 | ] 126 | }, 127 | { 128 | "cell_type": "markdown", 129 | "metadata": {}, 130 | "source": [ 131 | "### Step 2: Processing an Edge\n", 132 | "\n", 133 | "To make `extract_graph` work, we need to implement `extract_edge`. This is fairly simple code that mostly requires dealing with the specific format of `Expr` objects that represent anonymous functions. Because we need to make many assumptions, we'll include a lot of `@assert` calls that should be turned into proper error messages that users could use to fix their code." 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": 2, 139 | "metadata": {}, 140 | "outputs": [ 141 | { 142 | "data": { 143 | "text/plain": [ 144 | "extract_edge (generic function with 1 method)" 145 | ] 146 | }, 147 | "execution_count": 2, 148 | "metadata": {}, 149 | "output_type": "execute_result" 150 | } 151 | ], 152 | "source": [ 153 | "function extract_edge(e::Expr)\n", 154 | " @assert e.head == :->\n", 155 | " @assert isa(e.args[1], Integer)\n", 156 | " e′ = e.args[2]\n", 157 | " @assert e′.head == :block\n", 158 | " @assert length(e′.args) == 2\n", 159 | " @assert isa(e′.args[2], Integer)\n", 160 | " (e.args[1], e′.args[2])\n", 161 | "end" 162 | ] 163 | }, 164 | { 165 | "cell_type": "markdown", 166 | "metadata": {}, 167 | "source": [ 168 | "Let's test our code on a few cases to see that it can handle the simplest examples we can think of:" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": 3, 174 | "metadata": {}, 175 | "outputs": [ 176 | { 177 | "data": { 178 | "text/plain": [ 179 | "(1, 2)" 180 | ] 181 | }, 182 | "execution_count": 3, 183 | "metadata": {}, 184 | "output_type": "execute_result" 185 | } 186 | ], 187 | "source": [ 188 | "extract_edge(:(1 -> 2))" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": 4, 194 | "metadata": {}, 195 | "outputs": [ 196 | { 197 | "data": { 198 | "text/plain": [ 199 | "(1, 3)" 200 | ] 201 | }, 202 | "execution_count": 4, 203 | "metadata": {}, 204 | "output_type": "execute_result" 205 | } 206 | ], 207 | "source": [ 208 | "extract_edge(:(1 -> 3))" 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": 5, 214 | "metadata": {}, 215 | "outputs": [ 216 | { 217 | "data": { 218 | "text/plain": [ 219 | "(2, 1)" 220 | ] 221 | }, 222 | "execution_count": 5, 223 | "metadata": {}, 224 | "output_type": "execute_result" 225 | } 226 | ], 227 | "source": [ 228 | "extract_edge(:(2 -> 1))" 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "execution_count": 6, 234 | "metadata": {}, 235 | "outputs": [ 236 | { 237 | "data": { 238 | "text/plain": [ 239 | "([(1, 2), (2, 3)], Set([2, 3, 1]))" 240 | ] 241 | }, 242 | "execution_count": 6, 243 | "metadata": {}, 244 | "output_type": "execute_result" 245 | } 246 | ], 247 | "source": [ 248 | "extract_graph(:(\n", 249 | " begin\n", 250 | " 1 -> 2\n", 251 | " 2 -> 3\n", 252 | " end\n", 253 | "))" 254 | ] 255 | }, 256 | { 257 | "cell_type": "markdown", 258 | "metadata": {}, 259 | "source": [ 260 | "### Step 3: Write the Macro\n", 261 | "\n", 262 | "Now that we have the core logic in place for transforming a block expression representing a graph into a condensed form, writing the `@graph` macro is very simple. To make the macro easier to read, we produce our final expression using quasiquotation splicing:" 263 | ] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": 7, 268 | "metadata": {}, 269 | "outputs": [ 270 | { 271 | "data": { 272 | "text/plain": [ 273 | "@graph (macro with 1 method)" 274 | ] 275 | }, 276 | "execution_count": 7, 277 | "metadata": {}, 278 | "output_type": "execute_result" 279 | } 280 | ], 281 | "source": [ 282 | "macro graph(e)\n", 283 | " edges, nodes = extract_graph(e)\n", 284 | " n = length(nodes)\n", 285 | " quote\n", 286 | " import LightGraphs\n", 287 | " let edges = $edges, g = LightGraphs.SimpleDiGraph($n)\n", 288 | " for e in edges\n", 289 | " LightGraphs.add_edge!(g, e[1], e[2])\n", 290 | " end\n", 291 | " g\n", 292 | " end\n", 293 | " end\n", 294 | "end" 295 | ] 296 | }, 297 | { 298 | "cell_type": "markdown", 299 | "metadata": {}, 300 | "source": [ 301 | "Now we can confirm that our macro works:" 302 | ] 303 | }, 304 | { 305 | "cell_type": "code", 306 | "execution_count": 8, 307 | "metadata": {}, 308 | "outputs": [ 309 | { 310 | "data": { 311 | "text/plain": [ 312 | "{3, 2} directed simple Int64 graph" 313 | ] 314 | }, 315 | "execution_count": 8, 316 | "metadata": {}, 317 | "output_type": "execute_result" 318 | } 319 | ], 320 | "source": [ 321 | "@graph begin\n", 322 | " 1 -> 2\n", 323 | " 2 -> 3\n", 324 | "end" 325 | ] 326 | }, 327 | { 328 | "cell_type": "markdown", 329 | "metadata": {}, 330 | "source": [ 331 | "### Exercises\n", 332 | "\n", 333 | "1. Extend the DSL to support multiple edges per line like the following:\n", 334 | "\n", 335 | "```\n", 336 | "1 -> 2 -> 3\n", 337 | "```\n", 338 | "\n", 339 | "2. Support writing edges in the other direction:\n", 340 | "\n", 341 | "```\n", 342 | "1 <- 2\n", 343 | "```\n", 344 | "\n", 345 | "3. Mix multiple edges per line with support for both left and right directed edges:\n", 346 | "\n", 347 | "```\n", 348 | "1 <- 2 -> 3\n", 349 | "```\n", 350 | "\n", 351 | "4. Suppport undirected graphs using `1 - 2` to represent an edge instead of `1 -> 2`.\n", 352 | "\n", 353 | "5. Think through the issues for and against allowing the macro to decide for itself whether the output graph is directed or undirected. Instead of allowing the macro to decide, do you want to use two distinct macros called `@graph` and `@digraph`? Does type stability matter for macros that act like literals? Why or why not?\n", 354 | "\n", 355 | "6. Ensure that your macro emits useful error information. At the least, provide error messages that describe which expression caused an error, what kind of error was encountered and what was expected to occur in that expression. Even better, add line numbers to improve reporting.\n", 356 | "\n", 357 | "7. Much more ambitious exercise: make `@graph` into a string macro that processes raw strings that you parse with a custom parser. Invent a piece of graph syntax that Julia's normal syntax doesn't support.\n", 358 | "\n", 359 | "8. Convince yourself `@graph` can't be a function unless you pass in strings. What are some of the reasons that something like the following wouldn't work?\n", 360 | "\n", 361 | "```\n", 362 | "graph(1 -> 2, 2 -> 3)\n", 363 | "```\n", 364 | "\n", 365 | "8. (Cont.) What problems would we hit if we tried this instead?\n", 366 | "\n", 367 | "```\n", 368 | "graph(1 => 2, 2 => 3)\n", 369 | "```" 370 | ] 371 | }, 372 | { 373 | "cell_type": "markdown", 374 | "metadata": {}, 375 | "source": [ 376 | "## Named Tuple Rand" 377 | ] 378 | }, 379 | { 380 | "cell_type": "markdown", 381 | "metadata": {}, 382 | "source": [ 383 | "Now that we've built our first simple DSL, let's explore a richer space. In what follows, we'll build a DSL that lets users describe a probabilistic model using the notation found in probabistic programming languages (aka PPL's) like BUGS, JAGS, Stan or Turing. But, unlike a true PPL, we're only going to use this model specification to generate a single sample from the model, which we'll return as a named tuple.\n", 384 | "\n", 385 | "Just this simplified topic is very deep and we'll explore only a piece of it in what follows; there's essentially no limit to how rich the modeling language could be made if you decided to invest more time into it." 386 | ] 387 | }, 388 | { 389 | "cell_type": "markdown", 390 | "metadata": {}, 391 | "source": [ 392 | "### Exercise 1: Named Tuple RNG without Dependencies\n", 393 | "\n", 394 | "To get started write a macro `@rng` that takes in a simple model with no dependencies between variables and generates a named tuple. For example, the following model:\n", 395 | "\n", 396 | "```\n", 397 | "@rng begin\n", 398 | " mu ~ Normal(0, 1)\n", 399 | " sigma ~ Gamma(1, 1)\n", 400 | "end\n", 401 | "```\n", 402 | "\n", 403 | "should generate the following named tuple:\n", 404 | "\n", 405 | "```\n", 406 | "(\n", 407 | " mu = rand(Distributions.Normal(0, 1)),\n", 408 | " sigma = rand(Distributions.Gamma(1, 1)),\n", 409 | ")\n", 410 | "```\n", 411 | "\n", 412 | "Do this for yourself. The logic here is quite similar to the graph literals in the previous section. If you get stuck, consult the [solutions](https://github.com/johnmyleswhite/julia_tutorials/tree/master/solutions/)." 413 | ] 414 | }, 415 | { 416 | "cell_type": "markdown", 417 | "metadata": {}, 418 | "source": [ 419 | "### Exercise 2: Named Tuple RNG with Dependencies\n", 420 | "\n", 421 | "Now that you have something simple working, let's introduce dependencies between variables. To make things simpler for now, you can assume that the model block is written in the order in which calls to `rand` must happen:\n", 422 | "\n", 423 | "```\n", 424 | "mu ~ Normal(0, 1)\n", 425 | "sigma ~ Gamma(1, 1)\n", 426 | "x ~ Normal(mu, sigma)\n", 427 | "```\n", 428 | "\n", 429 | "Note that you'll need to deal with scoping issues here because named tuples can't refer to previous tuple elements (unlike keyword arguments)." 430 | ] 431 | }, 432 | { 433 | "cell_type": "markdown", 434 | "metadata": {}, 435 | "source": [ 436 | "### Exercise 3: Named Tuple RNG with Out-of-Order Dependencies\n", 437 | "\n", 438 | "Once you've finished Exercise 2, modify your code to support a block in which variables are defined in an arbitrary order, requiring you, the macro author, to topologically sort them before emitting calls to `rand`.\n", 439 | "\n", 440 | "```\n", 441 | "x ~ Normal(mu, sigma)\n", 442 | "mu ~ Normal(0, 1)\n", 443 | "sigma ~ Gamma(1, 1)\n", 444 | "```" 445 | ] 446 | }, 447 | { 448 | "cell_type": "markdown", 449 | "metadata": {}, 450 | "source": [ 451 | "### Exercise 4: Support Expressions as Distribution Parameters\n", 452 | "\n", 453 | "None of the examples so far passed in expressions as arguments to the distribution constructors. Add support for things like:\n", 454 | "\n", 455 | "```\n", 456 | "x ~ Normal(mu + 1, sigma)\n", 457 | "mu ~ Normal(0, 1)\n", 458 | "sigma ~ Gamma(1, 1)\n", 459 | "```" 460 | ] 461 | }, 462 | { 463 | "cell_type": "markdown", 464 | "metadata": {}, 465 | "source": [ 466 | "### Exercise 5: Introducing Branches\n", 467 | "\n", 468 | "Enrich your modeling language by support `if`/`else` expressions:\n", 469 | "\n", 470 | "```\n", 471 | "x ~ Bernoulli(0.5)\n", 472 | "if x == 1\n", 473 | " y ~ Normal(+1, 1.0)\n", 474 | "else\n", 475 | " y ~ Normal(-1, 1.0)\n", 476 | "end\n", 477 | "```\n", 478 | "\n", 479 | "Sometimes this can be done trivially using existing dependencies if the parameters of a distribution are arbitrary expressions (as in the following example), but this is not sufficient in the distribution itself varies across the branches of the `if`/`else` expression:\n", 480 | "\n", 481 | "```\n", 482 | "x ~ Bernoulli(0.5)\n", 483 | "y ~ Normal(+1 * x + -1 * (1 - x), 1.0)\n", 484 | "```" 485 | ] 486 | }, 487 | { 488 | "cell_type": "markdown", 489 | "metadata": {}, 490 | "source": [ 491 | "### Exercise 6: Support Statically Bounded For Loops\n", 492 | "\n", 493 | "Further enrich your language by adding support for static for loops whose loop bounds are known at macro expansion time like the following example in which the bounds are statically known to be `1` and `10`:\n", 494 | "\n", 495 | "```\n", 496 | "mu ~ Normal(0, 1)\n", 497 | "sigma ~ Gamma(1, 1)\n", 498 | "for i in 1:10\n", 499 | " x[i] ~ Normal(mu, sigma)\n", 500 | "end\n", 501 | "```\n", 502 | "\n", 503 | "You should decide whether you want to generate elements of the tuple like `Symbol(\"x[1]\")` or whether `x` should be a vector. What are arguments for and against each approach?\n", 504 | "\n", 505 | "There's also an interesting question here about the kind of code you want to emit: do you want to emit code uses for loops? Or do you want to emit code in which each variable defined over the course of the loop generates its own expression? The latter is called [unrolling a loop](https://en.wikipedia.org/wiki/Loop_unrolling)." 506 | ] 507 | }, 508 | { 509 | "cell_type": "markdown", 510 | "metadata": {}, 511 | "source": [ 512 | "### Exercise 7: Support Dynamically Bounded For Loops\n", 513 | "\n", 514 | "Go a step further and support dynamic for loops whose bounds are **not** known at macro expansion time like the following in which the uppper loop bound is a variable:\n", 515 | "\n", 516 | "```\n", 517 | "mu ~ Normal(0, 1)\n", 518 | "sigma ~ Gamma(1, 1)\n", 519 | "for i in 1:n\n", 520 | " x[i] ~ Normal(mu, sigma)\n", 521 | "end\n", 522 | "```\n", 523 | "\n", 524 | "Because this computation depends on information that cannot be known at compile time, you need to produce a function of `n` that evaluates to a named tuple; this is introduces a breaking change relative to your previous work.\n", 525 | "\n", 526 | "Note that this kind of dynamic bounds completely rules out manual unrolling at macro compile time." 527 | ] 528 | }, 529 | { 530 | "cell_type": "markdown", 531 | "metadata": {}, 532 | "source": [ 533 | "### Exercise 8: Non-IID Loops\n", 534 | "\n", 535 | "So far all of the loops we've considered (whether statically bound or dynamically bound) have not introduced dependencies between the output values generated by the loop. Extend your language (if necessary) to support dependencies like the following one:\n", 536 | "\n", 537 | "```\n", 538 | "mu ~ Normal(0, 1)\n", 539 | "sigma ~ Gamma(1, 1)\n", 540 | "x[1] ~ Normal(mu, sigma)\n", 541 | "for i in 2:n\n", 542 | " x[i] ~ x[i - 1] + Normal(mu, sigma)\n", 543 | "end\n", 544 | "```" 545 | ] 546 | }, 547 | { 548 | "cell_type": "markdown", 549 | "metadata": {}, 550 | "source": [ 551 | "## More Ideas for Macros and DSL's to Implement\n", 552 | "\n", 553 | "If you're worked through these DSL's and want to work on more projects, here's a few to consider. Some of these are really just more complicated macros rather than DSL's.\n", 554 | "\n", 555 | "1. **Symbolic Differentiation**\n", 556 | "\n", 557 | "Write a macro that uses [Calculus.jl](https://github.com/JuliaMath/Calculus.jl) to perform symbolic differentiation on expressions to generate their derivatives. Your macro should look something like:\n", 558 | "\n", 559 | "```\n", 560 | "@derivative(x + sin(exp(x)))\n", 561 | "```\n", 562 | "\n", 563 | "and it should generate output like:\n", 564 | "\n", 565 | "```\n", 566 | "x -> 1 + cos(exp(x)) * exp(x)\n", 567 | "```\n", 568 | "\n", 569 | "2. **Embedded Lisp**\n", 570 | "\n", 571 | "As noted earlier, you can support novel syntax using string macros. Based on ideas [from Peter Norvig's blog](https://norvig.com/lispy.html), implement a basic version of Lispy using a Julia string macro:\n", 572 | "\n", 573 | "```\n", 574 | "lisp\"(+ 1 (* 2 3))\"\n", 575 | "```\n", 576 | "\n", 577 | "3. **Curve Plotting**\n", 578 | "\n", 579 | "Write a macro that plots a curve specified in terms of the assumed variable `x`. Make sure that you both make use of the expression as a function while also representing the expression incorrectly in the axis labels for your plot. For example, be sure the labels for these two calls produce different outputs even though they define the same functions:\n", 580 | "\n", 581 | "```\n", 582 | "@curve(sin(x + y), xlim = (-1, 1), ylim = (-1, 1))\n", 583 | "```\n", 584 | "\n", 585 | "```\n", 586 | "@curve(sin(y + x), xlim = (-1, 1), ylim = (-1, 1))\n", 587 | "```\n", 588 | "\n", 589 | "4. **Model Transformer DSL**\n", 590 | "\n", 591 | "One of the main uses of [non-standard evaluation](http://adv-r.had.co.nz/Computing-on-the-language.html) in the R programming language is to support [Wilkinson notation](https://www.jstor.org/stable/2346786?seq=1) for transforming tabular data into design matrices for modeling. Implement something like:\n", 592 | "\n", 593 | "```\n", 594 | "@transformer(z ~ 1 + x + factor(y))\n", 595 | "```\n", 596 | "\n", 597 | "This should generate a transformer object that, when applied to a DataFrame, would generate the appropriate matrix. If you're unfamiliar with linear models, you should probably skip this exercise as the modeling issues are much deeper than the macro authoring issues." 598 | ] 599 | } 600 | ], 601 | "metadata": { 602 | "kernelspec": { 603 | "display_name": "Julia 1.4.1", 604 | "language": "julia", 605 | "name": "julia-1.4" 606 | }, 607 | "language_info": { 608 | "file_extension": ".jl", 609 | "mimetype": "application/julia", 610 | "name": "julia", 611 | "version": "1.4.1" 612 | } 613 | }, 614 | "nbformat": 4, 615 | "nbformat_minor": 4 616 | } 617 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 John Myles White 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Tutorials on Topics in Julia Programming 2 | 3 | * From Macros to DSLs in Julia 4 | * [Part 1](https://github.com/johnmyleswhite/julia_tutorials/blob/master/From%20Macros%20to%20DSLs%20in%20Julia%20-%20Part%201%20-%20Macros.ipynb) 5 | * [Part 2](https://github.com/johnmyleswhite/julia_tutorials/blob/master/From%20Macros%20to%20DSLs%20in%20Julia%20-%20Part%202%20-%20DSLs.ipynb) 6 | * Statistics in Julia 7 | * [Maximum Likelihood Estimation](https://github.com/johnmyleswhite/julia_tutorials/blob/master/Statistics%20in%20Julia%20-%20Maximum%20Likelihood%20Estimation.ipynb) 8 | * [Implementing a dplyr-like Interface for DataFrames](https://github.com/johnmyleswhite/julia_tutorials/blob/master/Rewriting%20Expressions%20to%20Tuple%20Functions.ipynb) 9 | -------------------------------------------------------------------------------- /Rewriting Expressions to Tuple Functions.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import MacroTools: postwalk\n", 10 | "import DataFrames: DataFrame" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": {}, 16 | "source": [ 17 | "### Rewriting Expressions as Functions over Tuples or Named Tuples" 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "metadata": {}, 23 | "source": [ 24 | "Suppose that we want to offer an API like `@select(df, col3 = sin(col1) + cos(col2))` for working with DataFrames. One approach we can take is to take the expression `sin(col1) + cos(col2)` and rewrite it into an anoymous function that maps tuples to scalars. This transformation is easiest to understand with named tuples, but we will write code to support working with either tuples or named tuples in this notebook.\n", 25 | "\n", 26 | "In the named tuple case, the transformation looks like:\n", 27 | "\n", 28 | "```\n", 29 | "row -> sin(row.col1) + cos(row.col2)\n", 30 | "```\n", 31 | "\n", 32 | "In what follows, we'll show how to do this. Our approach can easily be extended to offer additional functionality such as automatic lifting of functions to ensure that they can process the `missing` value or even the introduction of three-valued logic." 33 | ] 34 | }, 35 | { 36 | "cell_type": "markdown", 37 | "metadata": {}, 38 | "source": [ 39 | "Our approach will involve the following rules:\n", 40 | "\n", 41 | "1. If a symbol in an expression occurs in a syntactic position that implies it's function name, we'll assume it's a function name.\n", 42 | "2. All other symbols are assumed to be column names.\n", 43 | "\n", 44 | "Based on these rules, we'll rewrite expressions by making three passes through the expression:\n", 45 | "\n", 46 | "1. Find all function names by finding all symbols that are syntactically treated like function names in the expression and accumulating them in a `Set{Symbol}`.\n", 47 | "2. Pass through the expression again and find all column names by accumulating all symbols that are not function names in a `Set{Symbol}`.\n", 48 | "3. Rewrite all column name symbools as either (a) tuple numeric indexing or (b) named tupled field access depending on a Boolean flag." 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 2, 54 | "metadata": {}, 55 | "outputs": [ 56 | { 57 | "data": { 58 | "text/plain": [ 59 | "expr_to_tuple_function_expr (generic function with 1 method)" 60 | ] 61 | }, 62 | "execution_count": 2, 63 | "metadata": {}, 64 | "output_type": "execute_result" 65 | } 66 | ], 67 | "source": [ 68 | "function expr_to_tuple_function_expr(e::Any, named::Bool)\n", 69 | " function_names = find_function_names(e)\n", 70 | " column_names = find_column_names(e, function_names)\n", 71 | " column_name_to_index = Dict(column_names .=> 1:length(column_names))\n", 72 | " tuple_name = gensym()\n", 73 | " anon_func_body = postwalk(\n", 74 | " e′ -> symbol_to_tuple_index(\n", 75 | " e′,\n", 76 | " function_names,\n", 77 | " column_names,\n", 78 | " column_name_to_index,\n", 79 | " tuple_name,\n", 80 | " named,\n", 81 | " ),\n", 82 | " e,\n", 83 | " )\n", 84 | " (\n", 85 | " :($tuple_name -> $anon_func_body),\n", 86 | " collect(column_names),\n", 87 | " )\n", 88 | "end" 89 | ] 90 | }, 91 | { 92 | "cell_type": "markdown", 93 | "metadata": {}, 94 | "source": [ 95 | "To make this work, we need to define the core functions: we'll start with `find_function_names`, which is easy to write using the `postwalk` function in the MacroTools package." 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": 3, 101 | "metadata": {}, 102 | "outputs": [ 103 | { 104 | "data": { 105 | "text/plain": [ 106 | "find_function_names (generic function with 1 method)" 107 | ] 108 | }, 109 | "execution_count": 3, 110 | "metadata": {}, 111 | "output_type": "execute_result" 112 | } 113 | ], 114 | "source": [ 115 | "function find_function_names(e::Any)\n", 116 | " function_names = Set{Symbol}()\n", 117 | " postwalk(\n", 118 | " e′ -> update_function_names!(function_names, e′),\n", 119 | " e,\n", 120 | " )\n", 121 | " function_names\n", 122 | "end" 123 | ] 124 | }, 125 | { 126 | "cell_type": "code", 127 | "execution_count": 4, 128 | "metadata": {}, 129 | "outputs": [ 130 | { 131 | "data": { 132 | "text/plain": [ 133 | "update_function_names! (generic function with 1 method)" 134 | ] 135 | }, 136 | "execution_count": 4, 137 | "metadata": {}, 138 | "output_type": "execute_result" 139 | } 140 | ], 141 | "source": [ 142 | "function update_function_names!(function_names::Set{Symbol}, e::Any)\n", 143 | " if isa(e, Expr) && e.head == :call\n", 144 | " push!(function_names, e.args[1])\n", 145 | " end\n", 146 | " e\n", 147 | "end" 148 | ] 149 | }, 150 | { 151 | "cell_type": "markdown", 152 | "metadata": {}, 153 | "source": [ 154 | "Let's try it out:" 155 | ] 156 | }, 157 | { 158 | "cell_type": "code", 159 | "execution_count": 5, 160 | "metadata": {}, 161 | "outputs": [ 162 | { 163 | "data": { 164 | "text/plain": [ 165 | "Set{Symbol} with 2 elements:\n", 166 | " :+\n", 167 | " :sin" 168 | ] 169 | }, 170 | "execution_count": 5, 171 | "metadata": {}, 172 | "output_type": "execute_result" 173 | } 174 | ], 175 | "source": [ 176 | "find_function_names(:(a + sin(b)))" 177 | ] 178 | }, 179 | { 180 | "cell_type": "markdown", 181 | "metadata": {}, 182 | "source": [ 183 | "Now we'll implement column names:" 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": 6, 189 | "metadata": {}, 190 | "outputs": [ 191 | { 192 | "data": { 193 | "text/plain": [ 194 | "find_column_names (generic function with 1 method)" 195 | ] 196 | }, 197 | "execution_count": 6, 198 | "metadata": {}, 199 | "output_type": "execute_result" 200 | } 201 | ], 202 | "source": [ 203 | "function find_column_names(e::Any, function_names::Set{Symbol})\n", 204 | " column_names = Set{Symbol}()\n", 205 | " postwalk(\n", 206 | " e′ -> update_column_names!(column_names, e′, function_names),\n", 207 | " e,\n", 208 | " )\n", 209 | " column_names\n", 210 | "end" 211 | ] 212 | }, 213 | { 214 | "cell_type": "markdown", 215 | "metadata": {}, 216 | "source": [ 217 | "In this case, we want to distinguish two cases" 218 | ] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "execution_count": 7, 223 | "metadata": {}, 224 | "outputs": [ 225 | { 226 | "data": { 227 | "text/plain": [ 228 | "update_column_names! (generic function with 1 method)" 229 | ] 230 | }, 231 | "execution_count": 7, 232 | "metadata": {}, 233 | "output_type": "execute_result" 234 | } 235 | ], 236 | "source": [ 237 | "function update_column_names!(column_names::Set{Symbol}, e::Any, function_names::Set{Symbol})\n", 238 | " if isa(e, Symbol) && !(e in function_names)\n", 239 | " push!(column_names, e)\n", 240 | " end\n", 241 | " e\n", 242 | "end" 243 | ] 244 | }, 245 | { 246 | "cell_type": "code", 247 | "execution_count": 8, 248 | "metadata": {}, 249 | "outputs": [ 250 | { 251 | "data": { 252 | "text/plain": [ 253 | "Set{Symbol} with 2 elements:\n", 254 | " :a\n", 255 | " :b" 256 | ] 257 | }, 258 | "execution_count": 8, 259 | "metadata": {}, 260 | "output_type": "execute_result" 261 | } 262 | ], 263 | "source": [ 264 | "let e = :(a + sin(b))\n", 265 | " find_column_names(e, find_function_names(e))\n", 266 | "end" 267 | ] 268 | }, 269 | { 270 | "cell_type": "code", 271 | "execution_count": 9, 272 | "metadata": {}, 273 | "outputs": [ 274 | { 275 | "data": { 276 | "text/plain": [ 277 | "symbol_to_tuple_index (generic function with 1 method)" 278 | ] 279 | }, 280 | "execution_count": 9, 281 | "metadata": {}, 282 | "output_type": "execute_result" 283 | } 284 | ], 285 | "source": [ 286 | "function symbol_to_tuple_index(\n", 287 | " e::Any,\n", 288 | " function_names::Set{Symbol},\n", 289 | " column_names::Set{Symbol},\n", 290 | " column_name_to_index::Dict{Symbol, Int},\n", 291 | " tuple_name::Symbol,\n", 292 | " named::Bool,\n", 293 | ")\n", 294 | " if isa(e, Symbol) && e in column_names\n", 295 | " if !named\n", 296 | " :($(tuple_name)[$(column_name_to_index[e])])\n", 297 | " else\n", 298 | " :($(tuple_name).$e)\n", 299 | " end\n", 300 | " else\n", 301 | " e\n", 302 | " end\n", 303 | "end" 304 | ] 305 | }, 306 | { 307 | "cell_type": "code", 308 | "execution_count": 10, 309 | "metadata": {}, 310 | "outputs": [ 311 | { 312 | "data": { 313 | "text/plain": [ 314 | "(:(var\"##254\"->begin\n", 315 | " #= In[2]:18 =#\n", 316 | " var\"##254\"[1] + sin(var\"##254\"[2])\n", 317 | " end), [:a, :b])" 318 | ] 319 | }, 320 | "execution_count": 10, 321 | "metadata": {}, 322 | "output_type": "execute_result" 323 | } 324 | ], 325 | "source": [ 326 | "expr_to_tuple_function_expr(:(a + sin(b)), false)" 327 | ] 328 | }, 329 | { 330 | "cell_type": "code", 331 | "execution_count": 11, 332 | "metadata": {}, 333 | "outputs": [ 334 | { 335 | "data": { 336 | "text/plain": [ 337 | "(:(var\"##255\"->begin\n", 338 | " #= In[2]:18 =#\n", 339 | " (var\"##255\").a + sin((var\"##255\").b)\n", 340 | " end), [:a, :b])" 341 | ] 342 | }, 343 | "execution_count": 11, 344 | "metadata": {}, 345 | "output_type": "execute_result" 346 | } 347 | ], 348 | "source": [ 349 | "expr_to_tuple_function_expr(:(a + sin(b)), true)" 350 | ] 351 | }, 352 | { 353 | "cell_type": "markdown", 354 | "metadata": {}, 355 | "source": [ 356 | "To get a sense how we might use this, let's write a macro that performs SQL-like select in which users write something like:\n", 357 | "\n", 358 | "```\n", 359 | "@select(df, c = a + sin(b), d = a - b)\n", 360 | "```\n", 361 | "\n", 362 | "To make this work, we'll do a few things:\n", 363 | "\n", 364 | "1. We'll define a method to construct a tuple iterator from a DataFrame. The iterator can be used to give us tuple that we can apply a tuple function to.\n", 365 | "2. For every expression in the list of macro arguments, we'll translate it into a tuple function, then we'll apply that function to the tuple iterator.\n", 366 | "3. We'll construct a new DataFrame from the generated columns." 367 | ] 368 | }, 369 | { 370 | "cell_type": "code", 371 | "execution_count": 12, 372 | "metadata": {}, 373 | "outputs": [ 374 | { 375 | "data": { 376 | "text/plain": [ 377 | "@select (macro with 1 method)" 378 | ] 379 | }, 380 | "execution_count": 12, 381 | "metadata": {}, 382 | "output_type": "execute_result" 383 | } 384 | ], 385 | "source": [ 386 | "macro select(df, es...)\n", 387 | " kwargs = Any[]\n", 388 | " for assignment_e in es\n", 389 | " @assert isa(assignment_e, Expr) && assignment_e.head == :(=)\n", 390 | " res_name = assignment_e.args[1]\n", 391 | " e = assignment_e.args[2]\n", 392 | " anon_func_expr, column_names = expr_to_tuple_function_expr(e, false)\n", 393 | " res_column = quote\n", 394 | " map(\n", 395 | " $anon_func_expr,\n", 396 | " get_tuple_iterator($(esc(df)), $column_names),\n", 397 | " )\n", 398 | " end\n", 399 | " push!(kwargs, Expr(:kw, res_name, res_column))\n", 400 | " end\n", 401 | " quote\n", 402 | " DataFrame(\n", 403 | " $(kwargs...),\n", 404 | " )\n", 405 | " end\n", 406 | "end" 407 | ] 408 | }, 409 | { 410 | "cell_type": "code", 411 | "execution_count": 13, 412 | "metadata": {}, 413 | "outputs": [ 414 | { 415 | "data": { 416 | "text/plain": [ 417 | "get_tuple_iterator (generic function with 1 method)" 418 | ] 419 | }, 420 | "execution_count": 13, 421 | "metadata": {}, 422 | "output_type": "execute_result" 423 | } 424 | ], 425 | "source": [ 426 | "function get_tuple_iterator(df::DataFrame, names::Vector{Symbol})\n", 427 | " requested_columns = [df[name] for name in names]\n", 428 | " zip(requested_columns...)\n", 429 | "end" 430 | ] 431 | }, 432 | { 433 | "cell_type": "code", 434 | "execution_count": 14, 435 | "metadata": {}, 436 | "outputs": [ 437 | { 438 | "data": { 439 | "text/html": [ 440 | "

3 rows × 2 columns

ab
Int64Float64?
112.1
223.4
33missing
" 441 | ], 442 | "text/latex": [ 443 | "\\begin{tabular}{r|cc}\n", 444 | "\t& a & b\\\\\n", 445 | "\t\\hline\n", 446 | "\t& Int64 & Float64?\\\\\n", 447 | "\t\\hline\n", 448 | "\t1 & 1 & 2.1 \\\\\n", 449 | "\t2 & 2 & 3.4 \\\\\n", 450 | "\t3 & 3 & \\emph{missing} \\\\\n", 451 | "\\end{tabular}\n" 452 | ], 453 | "text/plain": [ 454 | "3×2 DataFrame\n", 455 | "│ Row │ a │ b │\n", 456 | "│ │ \u001b[90mInt64\u001b[39m │ \u001b[90mFloat64?\u001b[39m │\n", 457 | "├─────┼───────┼──────────┤\n", 458 | "│ 1 │ 1 │ 2.1 │\n", 459 | "│ 2 │ 2 │ 3.4 │\n", 460 | "│ 3 │ 3 │ \u001b[90mmissing\u001b[39m │" 461 | ] 462 | }, 463 | "execution_count": 14, 464 | "metadata": {}, 465 | "output_type": "execute_result" 466 | } 467 | ], 468 | "source": [ 469 | "df = DataFrame(a = [1, 2, 3], b = [2.1, 3.4, missing])" 470 | ] 471 | }, 472 | { 473 | "cell_type": "code", 474 | "execution_count": 15, 475 | "metadata": {}, 476 | "outputs": [ 477 | { 478 | "data": { 479 | "text/html": [ 480 | "

3 rows × 2 columns

cd
Float64?Float64?
11.86321-1.1
21.74446-1.4
3missingmissing
" 481 | ], 482 | "text/latex": [ 483 | "\\begin{tabular}{r|cc}\n", 484 | "\t& c & d\\\\\n", 485 | "\t\\hline\n", 486 | "\t& Float64? & Float64?\\\\\n", 487 | "\t\\hline\n", 488 | "\t1 & 1.86321 & -1.1 \\\\\n", 489 | "\t2 & 1.74446 & -1.4 \\\\\n", 490 | "\t3 & \\emph{missing} & \\emph{missing} \\\\\n", 491 | "\\end{tabular}\n" 492 | ], 493 | "text/plain": [ 494 | "3×2 DataFrame\n", 495 | "│ Row │ c │ d │\n", 496 | "│ │ \u001b[90mFloat64?\u001b[39m │ \u001b[90mFloat64?\u001b[39m │\n", 497 | "├─────┼──────────┼──────────┤\n", 498 | "│ 1 │ 1.86321 │ -1.1 │\n", 499 | "│ 2 │ 1.74446 │ -1.4 │\n", 500 | "│ 3 │ \u001b[90mmissing\u001b[39m │ \u001b[90mmissing\u001b[39m │" 501 | ] 502 | }, 503 | "execution_count": 15, 504 | "metadata": {}, 505 | "output_type": "execute_result" 506 | } 507 | ], 508 | "source": [ 509 | "@select(df, c = a + sin(b), d = a - b)" 510 | ] 511 | }, 512 | { 513 | "cell_type": "markdown", 514 | "metadata": {}, 515 | "source": [ 516 | "Extensions include:\n", 517 | "\n", 518 | "1. Support for referencing columns already introduced left-to-right. \n", 519 | "2. Support for pulling in local variables with `$`.\n", 520 | "3. Support for passing `*` as an argument that returns all existing columns.\n", 521 | "4. Handle lifting of functions to ensure they process `missing` correctly.\n", 522 | "5. Instead of an iterator approach, rewrite everything in broadcasting form." 523 | ] 524 | } 525 | ], 526 | "metadata": { 527 | "@webio": { 528 | "lastCommId": null, 529 | "lastKernelId": null 530 | }, 531 | "kernelspec": { 532 | "display_name": "Julia 1.4.1", 533 | "language": "julia", 534 | "name": "julia-1.4" 535 | }, 536 | "language_info": { 537 | "file_extension": ".jl", 538 | "mimetype": "application/julia", 539 | "name": "julia", 540 | "version": "1.5.0" 541 | } 542 | }, 543 | "nbformat": 4, 544 | "nbformat_minor": 4 545 | } 546 | -------------------------------------------------------------------------------- /Statistics in Julia - Maximum Likelihood Estimation.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Intended Audience\n", 8 | "\n", 9 | "This notebook is intended to be read by people with an interest in statistical computing that already have:\n", 10 | "* An intermediate knowledge of statistics. For example, the reader should already understand most of the material in a book like [\"All of Statistics\"](https://www.stat.cmu.edu/~larry/all-of-statistics/).\n", 11 | "* An intermediate knowledge of programming. For example, the reader should already understand most of the material in a book like [\"Advanced R\"](https://adv-r.hadley.nz/).\n", 12 | "\n", 13 | "In contrast, little to no knowledge of Julia is assumed. Characteristics of Julia that are not shared with languages like Python and R will be explicitly called out." 14 | ] 15 | }, 16 | { 17 | "cell_type": "markdown", 18 | "metadata": {}, 19 | "source": [ 20 | "# Mastering Julia for Statistical Computing" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": {}, 26 | "source": [ 27 | "The core Julia language has exceptionally [good documentation](https://docs.julialang.org/en/v1/), but the documentation is focused on describing the language itself rather than on how to use the language to solve problems. Because of this gap, I find that many potential Julia users wish there were learning materials that (a) work through solving specific problems in detail rather than through language features in the abstract and (b) discuss best practices for programming in Julia as part of the exposition of solving specific problems. For statistics, this kind of documentation for Julia is, as of April 2020, still largely underdeveloped, and some of what does exist is not freely available online.\n", 28 | "\n", 29 | "This notebook represents my attempt to help just a little bit by writing up one specific problem in great detail. My hope is that reading through this notebook and internalizing the main ideas will help bring a user from \"able to write Julia that mostly works\" to \"writing Julia code that is competitive with code written by experts\". To do that, I'll describe how to implement maximum likelihood estimation for the logistic regression model. The final result is not intended to serve as the state of the art implementation of logistic regression for Julia, but, despite that, it should be clear to readers how to make the final implementation competitive with any other implementation in terms of numerical accuracy or execution performance.\n", 30 | "\n", 31 | "One major goal I have for this document is to put down in writing the many alternative ways of writing code that are plausible and help the reader understand the pros and cons of each. I find that too little of the educational material that programmers are exposed to focuses on comparisons between implementations, even though such comparisons often form the essence of how effective mentors educate their mentees. Learning to program well requires that one learn a causal mental model about how small changes in code lead to large changes in accuracy or performance; developing such a causal mental model requires that one see many small changes alongside the effects those changes have.\n", 32 | "\n", 33 | "In addition to these goals, the approach I'll take in this notebook is intended to emphasize broadly applicable principles that apply to model fitting for any probabilistic model. As such, we will not dig into special propreties of the logistic regression model." 34 | ] 35 | }, 36 | { 37 | "cell_type": "markdown", 38 | "metadata": {}, 39 | "source": [ 40 | "# Review of the Mathematics of Logistic Regression via MLE" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": {}, 46 | "source": [ 47 | "Before we begin, let's review the logistic regression in purely mathematical terms, which I assume you've seen before. The formulation we'll use here works as follows:\n", 48 | "\n", 49 | "* There's a constant (e.g. non-random) design matrix, $X$, that has $n$ rows and $d + 1$ columns. Each row describes an observation; each observation consists of $d$ features and a constant term that defines the intercept term of the logistic regression model.\n", 50 | "* There's a random, but observed, vector, $y$, of length $n$ that contains the binary outcome of the logistic regression model.\n", 51 | "* There's a constant (e.g. non-random) vector of parameters, $\\beta$, of length $d + 1$ that defines the coefficients of the logistic regression model.\n", 52 | "* Given the design matrix and the coefficients, we can construct the matrix-vector product $X \\beta$ and call it $z$. This vector defines the probabilities that $y_i = 1$ for all $i$, but the probabilities are in the logit-space. To get probabilities in probability space, we map $z_i \\to y_i$ via the inverse logit function, $\\text{logit}^{-1}(z) = (1 + \\exp(-z))^{-1}$.\n", 53 | "\n", 54 | "Summarizing all of that in distributional notation:\n", 55 | "\n", 56 | "$$\n", 57 | "X \\in \\mathbb{R}^{n, d + 1} \\\\\n", 58 | "\\beta \\in \\mathbb{R}^{d + 1} \\\\\n", 59 | "z = X \\beta \\\\\n", 60 | "p_i = \\text{logit}^{-1}(z_i) \\\\\n", 61 | "y_i \\sim \\text{Bernoulli}(p_i)\n", 62 | "$$\n", 63 | "\n", 64 | "The likelihood function for the full dataset of $n$ observations is therefore:\n", 65 | "\n", 66 | "$$\n", 67 | "L(\\beta) = \\prod_{i = 1}^{n} p_i^{y_i} (1 - p_i)^{1 - y_i}\n", 68 | "$$\n", 69 | "\n", 70 | "The log likelihood function, which is both mathematically and numerically better suited for use, is:\n", 71 | "\n", 72 | "$$\n", 73 | "\\mathcal{L}(\\beta) = \\sum_{i = 1}^{n} y_i \\log(p_i) + (1 - y_i) \\log(1 - p_i)\n", 74 | "$$\n", 75 | "\n", 76 | "We find the maximum likelihood estimate of $\\beta$ by maximizing the log likelihood. This is equivalent to minimizing the negative log likelihood. Because blackbox optimization API's usually default to minimization, we'll work with the negative log likelihood in our implementation." 77 | ] 78 | }, 79 | { 80 | "cell_type": "markdown", 81 | "metadata": {}, 82 | "source": [ 83 | "# Step 1: Import Relevant Julia Libraries" 84 | ] 85 | }, 86 | { 87 | "cell_type": "markdown", 88 | "metadata": {}, 89 | "source": [ 90 | "A common belief among programmers is that it's usually better to re-use existing code (especially public libraries that have been used by many other people) than to write code from scratch. For learning purposes, we will write several pieces of code from scratch in this notebook. But we will also try to use existing libraries as much as possible.\n", 91 | "\n", 92 | "In what follows, I'm going to assume you're using Julia 1.4, which is what I have installed on my machine. In addition to making use of the basic Julia installation, we'll use a few packages in this example that need to be manually installed by the user. If you do not already have these required packages installed, change `false` to `true` below and execute this block of code to install the missing packages." 93 | ] 94 | }, 95 | { 96 | "cell_type": "code", 97 | "execution_count": 1, 98 | "metadata": {}, 99 | "outputs": [], 100 | "source": [ 101 | "new_install = false\n", 102 | "\n", 103 | "if new_install\n", 104 | " import Pkg\n", 105 | "\n", 106 | " Pkg.add(\"Distributions\")\n", 107 | " Pkg.add(\"ForwardDiff\")\n", 108 | " Pkg.add(\"Optim\")\n", 109 | " Pkg.add(\"StatsFuns\")\n", 110 | "end" 111 | ] 112 | }, 113 | { 114 | "cell_type": "markdown", 115 | "metadata": {}, 116 | "source": [ 117 | "In this example, we're going to use the following packages:\n", 118 | "\n", 119 | "* *Standard Library*\n", 120 | " * *LinearAlgebra*: The package provides functions and types for linear algebra, including computing dot products and matrix-vector multiplication.\n", 121 | " * *Statistics*: This package provides functions for computing the most fundamental statistics like `mean` and `var.\n", 122 | "* *User-Installed Libraries*\n", 123 | " * *Distributions*: This package provides functions and types for working with probability distributions. We'll use it to define Bernoulli and Normal distributions.\n", 124 | " * *ForwardDiff*: This package provides functions for automatically differentation quasi-arbitrary Julia functions.\n", 125 | " * *Optim*: This packages provides functions for optimization of quasi-arbitrary Julia functions.\n", 126 | " * *StatsFuns*: This package provides implementations of common statistical functions like the CDF of the logistic distribution (i.e. the inverse logit function).\n", 127 | "\n", 128 | "We'll going to pull in each of these packages using Julia's `import` keyword, which doesn't import any names into the local scope except for the names explicitly specified by the user. If you prefer to get access to everything in the package in the way that Python's `from foo import *` works or R's `library(foo)`, you can do `using Foo` instead." 129 | ] 130 | }, 131 | { 132 | "cell_type": "code", 133 | "execution_count": 2, 134 | "metadata": {}, 135 | "outputs": [], 136 | "source": [ 137 | "import LinearAlgebra: diag, dot, mul!\n", 138 | "import Statistics: cov, mean, var\n", 139 | "\n", 140 | "import Distributions: Bernoulli, Normal, cquantile\n", 141 | "import ForwardDiff: hessian\n", 142 | "import Optim: LBFGS, minimizer, optimize\n", 143 | "import StatsFuns: logistic, log1pexp, logit" 144 | ] 145 | }, 146 | { 147 | "cell_type": "markdown", 148 | "metadata": {}, 149 | "source": [ 150 | "# Step 2: Write a Data Generation Function" 151 | ] 152 | }, 153 | { 154 | "cell_type": "markdown", 155 | "metadata": {}, 156 | "source": [ 157 | "Before we implement a function for fitting a logistic regression via maximum likelhood, we're going to implement a function to generate a set of samples from the model given the design matrix, $X$, and the parameter vector, $\\beta$. I see starting this way as a broad principle for writing correct code for probabilistic models, so let's call it out as an explicit principle:\n", 158 | "\n", 159 | "**When implementing a probabilistic model, write the data generating function first.**\n", 160 | "\n", 161 | "More broadly, whenever possible, I like to work in the following order:\n", 162 | "\n", 163 | "* _Step 1: Write the generative model code._\n", 164 | "* _Step 2: Use that code to generate data that is truly generated by the model you're estimating._\n", 165 | "* _Step 3: Write the model fitting code._\n", 166 | "* _Step 4: Evaluate the model fitting code by checking how it behaves on data generated by the model. Exploit the fact that the parameters are fully known when you assess performance using generated data._\n", 167 | "\n", 168 | "The virtue of taking this approach is that it becomes far easier to test your model fitting code; all of the frequentist statistical theory about the quality of estimated parameters can be applied directly to your code as unit tests. If, in contrast, you work with an existing dataset whose data generating process you don't know, you can only check whether your code produces the same answers as existing software or analytic calculations. Since analytic calculations for logistic regressions are not generally tractable, that appproach is not particularly fulfilling." 169 | ] 170 | }, 171 | { 172 | "cell_type": "markdown", 173 | "metadata": {}, 174 | "source": [ 175 | "With all that in mind, let's start to implement our data generation function:" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": 3, 181 | "metadata": {}, 182 | "outputs": [ 183 | { 184 | "data": { 185 | "text/plain": [ 186 | "generate! (generic function with 1 method)" 187 | ] 188 | }, 189 | "execution_count": 3, 190 | "metadata": {}, 191 | "output_type": "execute_result" 192 | } 193 | ], 194 | "source": [ 195 | "function generate!(y, X, β)\n", 196 | " for i in eachindex(y)\n", 197 | " zᵢ = dot(X[i, :], β)\n", 198 | " pᵢ = logistic(zᵢ)\n", 199 | " y[i] = rand(Bernoulli(pᵢ))\n", 200 | " end\n", 201 | " y\n", 202 | "end" 203 | ] 204 | }, 205 | { 206 | "cell_type": "markdown", 207 | "metadata": {}, 208 | "source": [ 209 | "Let's walk through this function line-by-line to understand what's happening and why we've implemented things this way." 210 | ] 211 | }, 212 | { 213 | "cell_type": "markdown", 214 | "metadata": {}, 215 | "source": [ 216 | "### Line 1\n", 217 | "\n", 218 | "```\n", 219 | "function generate!(y, X, β)\n", 220 | "```\n", 221 | "\n", 222 | "The first line indicates that we're defining a function called `generate!` that takes in three arguments: `y`, `X` and `β`. The function name has an exclamation mark at the end to indicate that the function mutates at least one of its arguments. In particular, the function mutates `y`, which is the first argument because there's a convention in Julia of placing the arguments that will be mutated at the start of the argument list.\n", 223 | "\n", 224 | "Why are we writing this function so that it operates via mutation? Because mutating functions generally have better performance since they make it possible to remove memory allocations from the function body. If we want a pure function, we can write a wrapper for this mutating function that allocates a new arrray for `y`, calls this mutating function and then return the newly allocated `y`. This argument derives from another design principle:\n", 225 | "\n", 226 | "* **Write a mutating function that performs no allocations first and then write a pure wrapper for it that automatically allocates memory.**\n", 227 | "\n", 228 | "This principle is itself a special case of a broader principle:\n", 229 | "\n", 230 | "* **When faced with a tradeoff between performance and safety between functions X and Y, consider whether X can be built on top of Y or Y can be built on top of X. If, for example, Y can be built from X but X cannot be built from Y, implement X and provide it -- then expose Y built on top of X to users who prefer a safer, slower approach.**\n", 231 | "\n", 232 | "Note that we did not specify the types of any arguments. We could have written something like this instead:\n", 233 | "\n", 234 | "```\n", 235 | "function generate!(y::Vector{Float64}, X::Matrix{Float64}, β::Vector{Float64})\n", 236 | "```\n", 237 | "\n", 238 | "Why did we not specify types in that way? Because we want to keep our code generic. This reflects a general tension in how types are used in Julia. We can specify types either because:\n", 239 | "\n", 240 | "* We want to block certain types from being passed to the function by forcing a method error.\n", 241 | "* We want to make use of multiple dispatch to overload the function name to do slightly different things for different argument types.\n", 242 | "\n", 243 | "We don't want to do either of those things right now. Later on we might want to restrict our input types a bit more after understanding the space of valid inputs we want to allow. But for now we want to keep our code generic as we explore the design space. To give a sense of inputs we probably want to accept, all of the following seem reasonable:\n", 244 | "\n", 245 | "* `y::Vector{Int8}`\n", 246 | "* `y::Vector{Int32}`\n", 247 | "* `y::Vector{Int64}`\n", 248 | "* `y::Vector{Float32}`\n", 249 | "\n", 250 | "By not writing any input types, we allow all of these. If we had chosen the monomorphic `y::Vector{Float64}` declaration, we would have banned all of them from use.\n", 251 | "\n", 252 | "Note also that the absence of types will not introduce any performance problems. There are important cases in writing Julia code in which type information should be specified for maximum performance. The argument definitions for a function is never one of those cases. This is important enough to deserve being called out explicitly:\n", 253 | "\n", 254 | "**You do not need to annotate the types of function parameters to write fast Julia code.**" 255 | ] 256 | }, 257 | { 258 | "cell_type": "markdown", 259 | "metadata": {}, 260 | "source": [ 261 | "### Line 2\n", 262 | "\n", 263 | "```\n", 264 | "for i in eachindex(y)\n", 265 | "```\n", 266 | "\n", 267 | "In this line, we start a loop over the indices for `y`. The `eachindex(y)` function is a popular way to extract indices for arrays in Julia because it can provide the most efficient indexing strategy depending on the type of array you're using, whereas a more straightforward `for i in 1:length(y)` idiom might be sub-optimal for more complex array types like sparse arrays. If you're interested to see more, try running `eachindex` on a 10x10 dense matrix like `rand(10, 10)` versus running it on `SparseArrays.spzeros(10, 10)` from the `SparseArrays` package that's part of Julia's standard library.\n", 268 | "\n", 269 | "One thing to note about this line is that we're going to implicitly assume in the body of the looop that the following invariants hold, but we will not be testing them: `length(y) == size(X, 1)` and `size(X, 2) == length(β)`. We are skipping these checks for the same reason we're mutating our inputs: they can be hoisted out of this part of the code and tested before we enter any loop that invokes this code. We make code like this fast by not doing work we don't have to do, although the result is code that's less safe. In some settings, Julia might be smart enough to remove redundant checks, but we're going to take a bit more control for ourselves.\n", 270 | "\n", 271 | "Stated as a design principle: **it's better to write an inner function that assumes invariants hold and then write outer functions to enforce them than it is to constantly recheck them inside the inner function, especially if the inner function could only throw an exception if the invariants failed in the outer functions**. This principle and the previous principle of starting with a mutating function reflect a broader principle: acknowledge asymmetries in what can be built on top of what. It's easy to build a safe, slower function on top of an unsafe, faster function -- but the reverse is not true. Likewise it's easy to build a pure wrapper that allocates outputs on top of a mutating function, but it's not possible to avoid allocations if your pure function always performs them.\n", 272 | "\n", 273 | "Having set the loop start and end, we enter the loop body where we'll generate our observations one-by-one. For each observation, we generate some scalar intermediates. (We could also generate all of them at once using a mutating operation. We can even get away without needing to allocate memory because we can reuse the same vector.)" 274 | ] 275 | }, 276 | { 277 | "cell_type": "markdown", 278 | "metadata": {}, 279 | "source": [ 280 | "### Line 3\n", 281 | "\n", 282 | "```\n", 283 | "zᵢ = dot(X[i, :], β)\n", 284 | "```\n", 285 | "\n", 286 | "Here we extract the i-th row of `X` and take its dot product with `β` using the `LinearAlgebra.dot` function, which takes in two vectors and produces a scalar. The result is stored in the scalar variable `zᵢ`." 287 | ] 288 | }, 289 | { 290 | "cell_type": "markdown", 291 | "metadata": {}, 292 | "source": [ 293 | "### Lines 4-5\n", 294 | "\n", 295 | "```\n", 296 | "pᵢ = logistic(zᵢ)\n", 297 | "y[i] = rand(Bernoulli(pᵢ))\n", 298 | "```\n", 299 | "\n", 300 | "Here we use the `logistic` function from the StatsFuns package to transform `zᵢ` into `pᵢ`. The benefit of using the version of this function from the StatsFuns package is that it's been written to support the largest set of inputs possible; a naive `logistic(z) = 1 / (1 + exp(-z))` implementation will generate exact `0.0` probabilities earlier than it should.\n", 301 | "\n", 302 | "After computing `pᵢ`, we generate the observed `y[i]` Bernoulli outcomes. We do this by constructing a Bernoulli distribution object and calling the `rand` function on it. This is very efficient because the `Bernoulli` object is immutable." 303 | ] 304 | }, 305 | { 306 | "cell_type": "markdown", 307 | "metadata": {}, 308 | "source": [ 309 | "### Lines 6-8\n", 310 | "\n", 311 | "These lines mostly contain the ends of blocks, which Julia marks with `end`.\n", 312 | "\n", 313 | "The interesting part is Line 7, where we return `y` at the end to make it easy to use our function with inputs that aren't named variables." 314 | ] 315 | }, 316 | { 317 | "cell_type": "markdown", 318 | "metadata": {}, 319 | "source": [ 320 | "# Some Small Possible Variations" 321 | ] 322 | }, 323 | { 324 | "cell_type": "markdown", 325 | "metadata": {}, 326 | "source": [ 327 | "There are a couple of minor changes we could make that might be important in some settings. For example, we can use views instead of copies to replace `X[i, :]` with `@view(X[i, :])`. For very large arrays, this can help reduce the amount of memory we allocate, but it's worth noting that complex views can cause downstream code to be slow because they have to constantly perform quirky indexing operations that are not cache-friendly.\n", 328 | "\n", 329 | "We could also replace `LinearAlgebra.dot(X[i, :], β)` with `X[i, :]' * β` using Julia's lazy transpose operation.\n", 330 | "\n", 331 | "Finally we could draw random samples from the logistic distribution and compare them with `zᵢ` instead of ever generating `pᵢ`. This is the latent variable representation of logistic regression, but it's not clear that it would provide performance benefits to make this change.\n", 332 | "\n", 333 | "In what follows, we make a few of these changes and also use the builtin Julia macro `@inbounds` to turn off bounds checking inside of a code block. This changes the code from throwing a bounds-checking exception if the invariants we mentioned earlier are false to segfaulting. There are non-trivial performance benefits to doing this in some cases since the Julia compiler isn't always able to convince itself that bounds-checks are safe to eliminate automatically." 334 | ] 335 | }, 336 | { 337 | "cell_type": "code", 338 | "execution_count": 4, 339 | "metadata": {}, 340 | "outputs": [ 341 | { 342 | "data": { 343 | "text/plain": [ 344 | "generate! (generic function with 1 method)" 345 | ] 346 | }, 347 | "execution_count": 4, 348 | "metadata": {}, 349 | "output_type": "execute_result" 350 | } 351 | ], 352 | "source": [ 353 | "function generate!(y, X, β)\n", 354 | " @inbounds for i in eachindex(y)\n", 355 | " zᵢ = @view(X[i, :])' * β\n", 356 | " pᵢ = logistic(zᵢ)\n", 357 | " y[i] = rand(Bernoulli(pᵢ))\n", 358 | " end\n", 359 | " y\n", 360 | "end" 361 | ] 362 | }, 363 | { 364 | "cell_type": "markdown", 365 | "metadata": {}, 366 | "source": [ 367 | "If you're interested in deciding which of the many combinatorial variants is best, you should use the `@btime` macro from the BenchmarkTools package to compare them explicitly in terms of speed and memory usage." 368 | ] 369 | }, 370 | { 371 | "cell_type": "markdown", 372 | "metadata": {}, 373 | "source": [ 374 | "## A Different Coding Style" 375 | ] 376 | }, 377 | { 378 | "cell_type": "markdown", 379 | "metadata": {}, 380 | "source": [ 381 | "All of the variants we've seen so far have explicit iterative loops over the data. This is often the easiest way to write code like this and it's quite fast because of Julia's language design. But there are other approaches:\n", 382 | "\n", 383 | "* We could exploit the embarassingly parallel nature of the sum we're computing: there's no relationship between the i-th term in our inputs or outputs to any other term. We could employ threads that operate on disjoint set of observations to take advantage of this independence, for example.\n", 384 | "* Write a \"vectorized\" solution that operates on the entire dataset at once at the function call level (but where implementation still has to think about how to process individual elements)\n", 385 | "* Make use of BLAS calls to get some peformance improvements at the cost of potentially requiring allocating memory. In this example, there is no such cost because we get away with writing the value of `z` temporarily into the `y` array and then writing over that array with the value of `y`.\n", 386 | "\n", 387 | "Below, we'll use this BLAS approach by calling `LinearAlgebra.mul!`. At that same time, we'll show a \"vectorized\" approach to evaluating `logistic` and calling `rand` by using Julia's dot broadcasting notation, which explicitly \"vectorizes\" any scalar function we already have. Dot broadcasting is very special part of Julia because it uses syntax to lift scalar functions to vectorized functions, but it strictly more performant than traditional vectorization because it sees a whole sequence of operations at once and can perform loop fusion to avoid having to loop over an array multiple times." 388 | ] 389 | }, 390 | { 391 | "cell_type": "code", 392 | "execution_count": 5, 393 | "metadata": {}, 394 | "outputs": [ 395 | { 396 | "data": { 397 | "text/plain": [ 398 | "generate! (generic function with 1 method)" 399 | ] 400 | }, 401 | "execution_count": 5, 402 | "metadata": {}, 403 | "output_type": "execute_result" 404 | } 405 | ], 406 | "source": [ 407 | "function generate!(y, X, β)\n", 408 | " mul!(y, X, β)\n", 409 | " y .= rand.(Bernoulli.(logistic.(y)))\n", 410 | " y\n", 411 | "end" 412 | ] 413 | }, 414 | { 415 | "cell_type": "markdown", 416 | "metadata": {}, 417 | "source": [ 418 | "Testing our implementations of the `generate!` function to ensure they're equivalent is really hard. We can check they generate similar data according to summary statistics and use frequentist bounds to ensure the summaries are credible, but that's all I know how to do. In this example, I'm very confident all the modifications are safe." 419 | ] 420 | }, 421 | { 422 | "cell_type": "markdown", 423 | "metadata": {}, 424 | "source": [ 425 | "# Step 3: Generate Data" 426 | ] 427 | }, 428 | { 429 | "cell_type": "markdown", 430 | "metadata": {}, 431 | "source": [ 432 | "Now that we have a function to sample data, let's create some data with it and use it to test the model fitting code we'll write. We'll start with a decent number of observations so that our estimates are tolerably precise without being so accurate that confidence intervals are uninteresting." 433 | ] 434 | }, 435 | { 436 | "cell_type": "code", 437 | "execution_count": 6, 438 | "metadata": {}, 439 | "outputs": [ 440 | { 441 | "data": { 442 | "text/plain": [ 443 | "simulate_data (generic function with 1 method)" 444 | ] 445 | }, 446 | "execution_count": 6, 447 | "metadata": {}, 448 | "output_type": "execute_result" 449 | } 450 | ], 451 | "source": [ 452 | "function simulate_data(n, d)\n", 453 | " X = hcat(ones(n), rand(Normal(0, 1), n, d));\n", 454 | " β = rand(Normal(0, 1), d + 1)\n", 455 | " y = Array{Float64}(undef, n)\n", 456 | " generate!(y, X, β)\n", 457 | " y, X, β\n", 458 | "end" 459 | ] 460 | }, 461 | { 462 | "cell_type": "code", 463 | "execution_count": 7, 464 | "metadata": {}, 465 | "outputs": [ 466 | { 467 | "data": { 468 | "text/plain": [ 469 | "([0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0 … 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0], [1.0 0.10781238613877225 0.9185577169564151; 1.0 0.2875710507117643 0.29447235851120307; … ; 1.0 1.4970548123092562 0.25385694441532525; 1.0 -2.371265555206343 0.8649145476870038], [-1.1382399429248256, -0.25633479736844866, -0.29936682624813693])" 470 | ] 471 | }, 472 | "execution_count": 7, 473 | "metadata": {}, 474 | "output_type": "execute_result" 475 | } 476 | ], 477 | "source": [ 478 | "y, X, β = simulate_data(10_000, 2)" 479 | ] 480 | }, 481 | { 482 | "cell_type": "markdown", 483 | "metadata": {}, 484 | "source": [ 485 | "# Step 4: Implementing the Log Likelihood Function" 486 | ] 487 | }, 488 | { 489 | "cell_type": "markdown", 490 | "metadata": {}, 491 | "source": [ 492 | "Now that we have data in hand, let's code up the log likelihood function. I like to do this by copying the body of the `generate!` function and then changing it appropriately so the two pieces of code are maximally similar. A formal PPL would make it easier to reuse code between them." 493 | ] 494 | }, 495 | { 496 | "cell_type": "code", 497 | "execution_count": 8, 498 | "metadata": {}, 499 | "outputs": [ 500 | { 501 | "data": { 502 | "text/plain": [ 503 | "log_likelihood_naive (generic function with 1 method)" 504 | ] 505 | }, 506 | "execution_count": 8, 507 | "metadata": {}, 508 | "output_type": "execute_result" 509 | } 510 | ], 511 | "source": [ 512 | "function log_likelihood_naive(X, y, β)\n", 513 | " ll = 0.0\n", 514 | " @inbounds for i in eachindex(y)\n", 515 | " zᵢ = dot(X[i, :], β)\n", 516 | " pᵢ = logistic(zᵢ)\n", 517 | " ll += y[i] * log(pᵢ) + (1 - y[i]) * log(1 - pᵢ)\n", 518 | " end\n", 519 | " ll\n", 520 | "end" 521 | ] 522 | }, 523 | { 524 | "cell_type": "markdown", 525 | "metadata": {}, 526 | "source": [ 527 | "This code has some problems of numerical accuracy and it also perform computations we don't really need to perform to produce the correct output. The essence of the problem with this naive translation of the log likelihoood equation is that we're doing work to compute `logistic(zᵢ)` when we really only need `log(logistic(zᵢ))`. We can improve on this using the `log1pexp` method we imported earlier, which also provides substantially better numerical accuracy when any of the `zᵢ < -710.0`." 528 | ] 529 | }, 530 | { 531 | "cell_type": "code", 532 | "execution_count": 9, 533 | "metadata": {}, 534 | "outputs": [ 535 | { 536 | "data": { 537 | "text/plain": [ 538 | "log_likelihood (generic function with 1 method)" 539 | ] 540 | }, 541 | "execution_count": 9, 542 | "metadata": {}, 543 | "output_type": "execute_result" 544 | } 545 | ], 546 | "source": [ 547 | "function log_likelihood(X, y, β)\n", 548 | " ll = 0.0\n", 549 | " @inbounds for i in eachindex(y)\n", 550 | " zᵢ = dot(X[i, :], β)\n", 551 | " c = -log1pexp(-zᵢ) # Conceptually equivalent to log(1 / (1 + exp(-zᵢ))) == -log(1 + exp(-zᵢ))\n", 552 | " ll += y[i] * c + (1 - y[i]) * (-zᵢ + c) # Conceptually equivalent to log(exp(-zᵢ) / (1 + exp(-zᵢ)))\n", 553 | " end\n", 554 | " ll\n", 555 | "end" 556 | ] 557 | }, 558 | { 559 | "cell_type": "markdown", 560 | "metadata": {}, 561 | "source": [ 562 | "We can reassure ourselves that our changes are correct:" 563 | ] 564 | }, 565 | { 566 | "cell_type": "code", 567 | "execution_count": 10, 568 | "metadata": {}, 569 | "outputs": [ 570 | { 571 | "data": { 572 | "text/plain": [ 573 | "(-5490.842415218151, -5490.842415218151)" 574 | ] 575 | }, 576 | "execution_count": 10, 577 | "metadata": {}, 578 | "output_type": "execute_result" 579 | } 580 | ], 581 | "source": [ 582 | "(\n", 583 | " log_likelihood_naive(X, y, β),\n", 584 | " log_likelihood(X, y, β),\n", 585 | ")" 586 | ] 587 | }, 588 | { 589 | "cell_type": "markdown", 590 | "metadata": {}, 591 | "source": [ 592 | "The log likelihood as we've written it is a function of both the data and the parameters, but mathematically it should only depend on the parameters, $\\beta$. In addition to that mathetical reason for creating a new function, we want a function only of the parameters because the optimization algorithms in Optim assume the inputs have that property. To achieve both goals, we'll construct a closure that partially applies the log likelihood function for us and negates it to give us the negative log likelihood we want to minimize." 593 | ] 594 | }, 595 | { 596 | "cell_type": "markdown", 597 | "metadata": {}, 598 | "source": [ 599 | "# The Log Likelihood Function Should Be a Closure" 600 | ] 601 | }, 602 | { 603 | "cell_type": "code", 604 | "execution_count": 11, 605 | "metadata": {}, 606 | "outputs": [ 607 | { 608 | "data": { 609 | "text/plain": [ 610 | "make_closures (generic function with 1 method)" 611 | ] 612 | }, 613 | "execution_count": 11, 614 | "metadata": {}, 615 | "output_type": "execute_result" 616 | } 617 | ], 618 | "source": [ 619 | "make_closures(X, y) = β -> -log_likelihood(X, y, β)" 620 | ] 621 | }, 622 | { 623 | "cell_type": "code", 624 | "execution_count": 12, 625 | "metadata": {}, 626 | "outputs": [ 627 | { 628 | "data": { 629 | "text/plain": [ 630 | "#3 (generic function with 1 method)" 631 | ] 632 | }, 633 | "execution_count": 12, 634 | "metadata": {}, 635 | "output_type": "execute_result" 636 | } 637 | ], 638 | "source": [ 639 | "nll = make_closures(X, y)" 640 | ] 641 | }, 642 | { 643 | "cell_type": "markdown", 644 | "metadata": {}, 645 | "source": [ 646 | "# Step 5: Minimizing the Negative Log Likelihood Function" 647 | ] 648 | }, 649 | { 650 | "cell_type": "markdown", 651 | "metadata": {}, 652 | "source": [ 653 | "Now that we have the negative log likelihood we'll want to minimize it starting from some point. It's common to initialize all of the parameters to zero, so let's start there:" 654 | ] 655 | }, 656 | { 657 | "cell_type": "code", 658 | "execution_count": 13, 659 | "metadata": {}, 660 | "outputs": [], 661 | "source": [ 662 | "β₀ = zeros(2 + 1); # d = 2 and we want an intercept term" 663 | ] 664 | }, 665 | { 666 | "cell_type": "markdown", 667 | "metadata": {}, 668 | "source": [ 669 | "We can then check whether the negative log likelihood evaluated relative to the zero parameter function gives a value that seems plausible:" 670 | ] 671 | }, 672 | { 673 | "cell_type": "code", 674 | "execution_count": 14, 675 | "metadata": {}, 676 | "outputs": [ 677 | { 678 | "data": { 679 | "text/plain": [ 680 | "6931.471805600547" 681 | ] 682 | }, 683 | "execution_count": 14, 684 | "metadata": {}, 685 | "output_type": "execute_result" 686 | } 687 | ], 688 | "source": [ 689 | "nll(β₀)" 690 | ] 691 | }, 692 | { 693 | "cell_type": "markdown", 694 | "metadata": {}, 695 | "source": [ 696 | "Now we want to minimixe the negative log likelihood. To do that, we'll use the Optim.jl library, which provides an `optimize` method for minimization of blackbox functions. We'll pass two options to `optimize` to improve our results:\n", 697 | "1. We'll use the L-BFGS algorithm to exploit the gradient that can be compute for the negative log likelihood function rather than let the algorithm default to Nelder-Mead.\n", 698 | "2. We'll pass in an argument to use forward-mode automatic differentation to ensure that we get exact gradients rather than approximate ones. Without this, the algorithm will sometimes (or even often) fail to converge to a highly precise result because the finite-difference gradients that calculated by default will become inaccurate." 699 | ] 700 | }, 701 | { 702 | "cell_type": "code", 703 | "execution_count": 15, 704 | "metadata": {}, 705 | "outputs": [ 706 | { 707 | "data": { 708 | "text/plain": [ 709 | " * Status: success\n", 710 | "\n", 711 | " * Candidate solution\n", 712 | " Minimizer: [-1.13e+00, -2.44e-01, -3.23e-01]\n", 713 | " Minimum: 5.490175e+03\n", 714 | "\n", 715 | " * Found with\n", 716 | " Algorithm: L-BFGS\n", 717 | " Initial Point: [0.00e+00, 0.00e+00, 0.00e+00]\n", 718 | "\n", 719 | " * Convergence measures\n", 720 | " |x - x'| = 1.21e-08 ≰ 0.0e+00\n", 721 | " |x - x'|/|x'| = 1.07e-08 ≰ 0.0e+00\n", 722 | " |f(x) - f(x')| = 3.09e-11 ≰ 0.0e+00\n", 723 | " |f(x) - f(x')|/|f(x')| = 5.63e-15 ≰ 0.0e+00\n", 724 | " |g(x)| = 5.16e-10 ≤ 1.0e-08\n", 725 | "\n", 726 | " * Work counters\n", 727 | " Seconds run: 1 (vs limit Inf)\n", 728 | " Iterations: 7\n", 729 | " f(x) calls: 28\n", 730 | " ∇f(x) calls: 28\n" 731 | ] 732 | }, 733 | "execution_count": 15, 734 | "metadata": {}, 735 | "output_type": "execute_result" 736 | } 737 | ], 738 | "source": [ 739 | "res = optimize(nll, β₀, LBFGS(), autodiff=:forward)" 740 | ] 741 | }, 742 | { 743 | "cell_type": "markdown", 744 | "metadata": {}, 745 | "source": [ 746 | "I personally like to initialize the parameters to values that I'm hopeful will let the algorithm converge faster without being costly to compute. One heuristic I've used for logistic regression is the following:\n", 747 | "\n", 748 | "* Set the intercept to be exactly right if there were no other parameters.\n", 749 | "* Set the other coefficients based on doing a standard univariate OLS fit to the logit-transformed data after replacing 0's with $\\epsilon$ and 1's with $1 - \\epsilon$. I use a large $\\epsilon = 0.1$." 750 | ] 751 | }, 752 | { 753 | "cell_type": "code", 754 | "execution_count": 16, 755 | "metadata": {}, 756 | "outputs": [ 757 | { 758 | "data": { 759 | "text/plain": [ 760 | "3-element Array{Float64,1}:\n", 761 | " -1.0932860443854355\n", 762 | " -0.1939078725087912\n", 763 | " -0.2582119475758624" 764 | ] 765 | }, 766 | "execution_count": 16, 767 | "metadata": {}, 768 | "output_type": "execute_result" 769 | } 770 | ], 771 | "source": [ 772 | "function initialize!(β₀, X, y, ϵ = 0.1)\n", 773 | " β₀[1] = logit(mean(y))\n", 774 | " logit_y = [ifelse(y_i == 1.0, logit(1 - ϵ), logit(ϵ)) for y_i in y]\n", 775 | " for j in 2:length(β₀)\n", 776 | " β₀[j] = cov(logit_y, @view(X[:, j])) / var(@view(X[:, j]))\n", 777 | " end\n", 778 | " β₀\n", 779 | "end\n", 780 | "\n", 781 | "initialize!(β₀, X, y)" 782 | ] 783 | }, 784 | { 785 | "cell_type": "code", 786 | "execution_count": 17, 787 | "metadata": {}, 788 | "outputs": [ 789 | { 790 | "data": { 791 | "text/plain": [ 792 | " * Status: success\n", 793 | "\n", 794 | " * Candidate solution\n", 795 | " Minimizer: [-1.13e+00, -2.44e-01, -3.23e-01]\n", 796 | " Minimum: 5.490175e+03\n", 797 | "\n", 798 | " * Found with\n", 799 | " Algorithm: L-BFGS\n", 800 | " Initial Point: [-1.09e+00, -1.94e-01, -2.58e-01]\n", 801 | "\n", 802 | " * Convergence measures\n", 803 | " |x - x'| = 5.79e-11 ≰ 0.0e+00\n", 804 | " |x - x'|/|x'| = 5.10e-11 ≰ 0.0e+00\n", 805 | " |f(x) - f(x')| = 2.36e-11 ≰ 0.0e+00\n", 806 | " |f(x) - f(x')|/|f(x')| = 4.31e-15 ≰ 0.0e+00\n", 807 | " |g(x)| = 1.39e-13 ≤ 1.0e-08\n", 808 | "\n", 809 | " * Work counters\n", 810 | " Seconds run: 0 (vs limit Inf)\n", 811 | " Iterations: 6\n", 812 | " f(x) calls: 17\n", 813 | " ∇f(x) calls: 17\n" 814 | ] 815 | }, 816 | "execution_count": 17, 817 | "metadata": {}, 818 | "output_type": "execute_result" 819 | } 820 | ], 821 | "source": [ 822 | "res = optimize(nll, β₀, LBFGS(), autodiff=:forward)" 823 | ] 824 | }, 825 | { 826 | "cell_type": "markdown", 827 | "metadata": {}, 828 | "source": [ 829 | "If you compare the work counters, you can see that the optimization procedure had to compute the log likelihood and its gradient fewer times. I don't know how to prove that my initialization approach will always have this effect and think it's very possible that this initialization won't always find an optimum faster. But in practice I've found it can make things a bit faster because the initialization step costs much less than evaluating `f(x)` and `∇f(x)` a few times." 830 | ] 831 | }, 832 | { 833 | "cell_type": "markdown", 834 | "metadata": {}, 835 | "source": [ 836 | "Given the results of optimization, we can extract our estimates using the `Optim.minimizer` method:" 837 | ] 838 | }, 839 | { 840 | "cell_type": "code", 841 | "execution_count": 18, 842 | "metadata": {}, 843 | "outputs": [ 844 | { 845 | "data": { 846 | "text/plain": [ 847 | "3-element Array{Float64,1}:\n", 848 | " -1.1341732805409002\n", 849 | " -0.24414707752560674\n", 850 | " -0.3229958057866628" 851 | ] 852 | }, 853 | "execution_count": 18, 854 | "metadata": {}, 855 | "output_type": "execute_result" 856 | } 857 | ], 858 | "source": [ 859 | "β̂ = minimizer(res)" 860 | ] 861 | }, 862 | { 863 | "cell_type": "markdown", 864 | "metadata": {}, 865 | "source": [ 866 | "# Step 6: Testing Our Estimates via Confidence Interval Coverage Checks" 867 | ] 868 | }, 869 | { 870 | "cell_type": "markdown", 871 | "metadata": {}, 872 | "source": [ 873 | "Now we have estimates, but how do we know if the estimates are good enough?\n", 874 | "\n", 875 | "Given results about the convergence in probability of logistic regression coefficients, we know that as `n` goes to infinity, β̂ converges to β. Unfortunately, we can't simulate infinite data. In finite sample the convergence isn't complete, so there's some error.\n", 876 | "\n", 877 | "So the question for evaluating our code becomes: how do we know the error is reasonable? This is a place where frequentist statistics is very useful -- if the model is true (which we're trying to ensure occurs by construction), then asymptotically we can use the Fisher Information Matrix to compute confidence intervals for β̂ and check whether they contain β. We'll follow a standard of using the observed Fisher information matrix instead, since that only requires us to evaluate the Hesssian of the negative log likelihood function." 878 | ] 879 | }, 880 | { 881 | "cell_type": "code", 882 | "execution_count": 19, 883 | "metadata": {}, 884 | "outputs": [ 885 | { 886 | "data": { 887 | "text/plain": [ 888 | "compute_ses (generic function with 1 method)" 889 | ] 890 | }, 891 | "execution_count": 19, 892 | "metadata": {}, 893 | "output_type": "execute_result" 894 | } 895 | ], 896 | "source": [ 897 | "function compute_ses(nll, β̂)\n", 898 | " H = hessian(nll, β̂)\n", 899 | " ses = sqrt.(diag(inv(H)))\n", 900 | " ses\n", 901 | "end" 902 | ] 903 | }, 904 | { 905 | "cell_type": "markdown", 906 | "metadata": {}, 907 | "source": [ 908 | "See Chapter 9 of Wasserman's All of Statistics for details on the math we're using here." 909 | ] 910 | }, 911 | { 912 | "cell_type": "code", 913 | "execution_count": 20, 914 | "metadata": {}, 915 | "outputs": [ 916 | { 917 | "data": { 918 | "text/plain": [ 919 | "compute_cis (generic function with 1 method)" 920 | ] 921 | }, 922 | "execution_count": 20, 923 | "metadata": {}, 924 | "output_type": "execute_result" 925 | } 926 | ], 927 | "source": [ 928 | "function compute_cis(nll, β̂, α)\n", 929 | " ses = compute_ses(nll, β̂)\n", 930 | " τ = cquantile(Normal(0, 1), α)\n", 931 | " lower = β̂ - τ * ses\n", 932 | " upper = β̂ + τ * ses\n", 933 | " lower, upper\n", 934 | "end" 935 | ] 936 | }, 937 | { 938 | "cell_type": "markdown", 939 | "metadata": {}, 940 | "source": [ 941 | "Standard CI computation using quantiles from the normal distribution." 942 | ] 943 | }, 944 | { 945 | "cell_type": "code", 946 | "execution_count": 21, 947 | "metadata": {}, 948 | "outputs": [ 949 | { 950 | "data": { 951 | "text/plain": [ 952 | "check_cis (generic function with 1 method)" 953 | ] 954 | }, 955 | "execution_count": 21, 956 | "metadata": {}, 957 | "output_type": "execute_result" 958 | } 959 | ], 960 | "source": [ 961 | "check_cis(β, lower, upper) = all(lower .<= β .<= upper)" 962 | ] 963 | }, 964 | { 965 | "cell_type": "code", 966 | "execution_count": 22, 967 | "metadata": {}, 968 | "outputs": [ 969 | { 970 | "data": { 971 | "text/plain": [ 972 | "true" 973 | ] 974 | }, 975 | "execution_count": 22, 976 | "metadata": {}, 977 | "output_type": "execute_result" 978 | } 979 | ], 980 | "source": [ 981 | "α = 0.001\n", 982 | "check_cis(β, compute_cis(nll, β̂, α)...)" 983 | ] 984 | }, 985 | { 986 | "cell_type": "markdown", 987 | "metadata": {}, 988 | "source": [ 989 | "# Conclusion" 990 | ] 991 | }, 992 | { 993 | "cell_type": "markdown", 994 | "metadata": {}, 995 | "source": [ 996 | "Hopefully this short tutorial gives a flavor of how to fit models via MLE in Julia. There's many more topics that we could have explored, but I wanted to keep things relatively short. A few topics that I'd encourage the reader to investigate:\n", 997 | "\n", 998 | "* Is it better to use the analytic gradient in `optimize` in terms of accuracy or performance?\n", 999 | "* Do our results match the results from Julia's GLM package or R's `glm` function?\n", 1000 | "* Should we modify the log_likelihood function to automatically scale inputs so they have a standard deviation of 1?\n", 1001 | "* How do we construct robust standard errors when the model is misspecified? There's an intro in Julia to robust standard errors [here](https://github.com/PaulSoderlind/FinancialEconometrics/blob/master/Ch12_MLE.ipynb)." 1002 | ] 1003 | } 1004 | ], 1005 | "metadata": { 1006 | "kernelspec": { 1007 | "display_name": "Julia 1.4.1", 1008 | "language": "julia", 1009 | "name": "julia-1.4" 1010 | }, 1011 | "language_info": { 1012 | "file_extension": ".jl", 1013 | "mimetype": "application/julia", 1014 | "name": "julia", 1015 | "version": "1.5.0" 1016 | } 1017 | }, 1018 | "nbformat": 4, 1019 | "nbformat_minor": 4 1020 | } 1021 | -------------------------------------------------------------------------------- /solutions/part_1/three_valued_logic/src.jl: -------------------------------------------------------------------------------- 1 | macro _tvl_or(x, y) 2 | t1 = gensym() 3 | quote 4 | let $t1 = $x 5 | if $t1 === true 6 | true 7 | elseif $t1 === false 8 | $y 9 | else 10 | if $y === true 11 | true 12 | else 13 | missing 14 | end 15 | end 16 | end 17 | end 18 | end 19 | 20 | macro _tvl_and(x, y) 21 | t1 = gensym() 22 | quote 23 | let $t1 = $x 24 | if $t1 === false 25 | false 26 | elseif $t1 === true 27 | $y 28 | else 29 | if $y === false 30 | false 31 | else 32 | missing 33 | end 34 | end 35 | end 36 | end 37 | end 38 | 39 | replace_and_or(e::Any) = e 40 | 41 | function replace_and_or(e::Expr) 42 | if e.head == :&& 43 | Expr( 44 | :macrocall, 45 | Symbol("@_tvl_and"), 46 | LineNumberNode(0, nothing), 47 | esc(replace_and_or(e.args[1])), 48 | esc(replace_and_or(e.args[2])), 49 | ) 50 | elseif e.head == :|| 51 | Expr( 52 | :macrocall, 53 | Symbol("@_tvl_or"), 54 | LineNumberNode(0, nothing), 55 | esc(replace_and_or(e.args[1])), 56 | esc(replace_and_or(e.args[2])), 57 | ) 58 | else 59 | Expr( 60 | e.head, 61 | map(ex->esc(replace_and_or(ex)), e.args)... 62 | ) 63 | end 64 | end 65 | 66 | macro tvl(e) 67 | replace_and_or(e) 68 | end 69 | -------------------------------------------------------------------------------- /solutions/part_1/three_valued_logic/tests.jl: -------------------------------------------------------------------------------- 1 | import Test: @testset, @test 2 | 3 | include("src.jl") 4 | 5 | @testset "tvl_or truth table" begin 6 | @test @_tvl_or(true, true) === true 7 | @test @_tvl_or(true, false) === true 8 | @test @_tvl_or(true, missing) === true 9 | @test @_tvl_or(false, true) === true 10 | @test @_tvl_or(false, false) === false 11 | @test @_tvl_or(false, missing) === missing 12 | @test @_tvl_or(missing, true) === true 13 | @test @_tvl_or(missing, false) === missing 14 | @test @_tvl_or(missing, missing) === missing 15 | end 16 | 17 | @testset "tvl_and truth table" begin 18 | @test @_tvl_and(true, true) === true 19 | @test @_tvl_and(true, false) === false 20 | @test @_tvl_and(true, missing) === missing 21 | @test @_tvl_and(false, true) === false 22 | @test @_tvl_and(false, false) === false 23 | @test @_tvl_and(false, missing) === false 24 | @test @_tvl_and(missing, true) === missing 25 | @test @_tvl_and(missing, false) === false 26 | @test @_tvl_and(missing, missing) === missing 27 | end 28 | 29 | # We define a function that prints out a unique ID for each argument 30 | # to a Boolean operator. By wrapping all Boolean values in calls to this 31 | # function, we're able to check that the order of evaluation and 32 | # side-effects of the short-circuiting operators are retained by our 33 | # macro rewrites. 34 | 35 | function f(io, i, x) 36 | print(io, i) 37 | x 38 | end 39 | 40 | @testset "Order of evaluation for tvl" begin 41 | for x in (true, false) 42 | for y in (true, false) 43 | for z in (true, false) 44 | io = IOBuffer() 45 | 46 | a = f(io, 1, x) && f(io, 2, y) || f(io, 3, z) 47 | order_a = String(take!(io)) 48 | b = @tvl f(io, 1, x) && f(io, 2, y) || f(io, 3, z) 49 | order_b = String(take!(io)) 50 | 51 | @test a === b 52 | @test order_a === order_b 53 | 54 | a = f(io, 1, x) || f(io, 2, y) && f(io, 3, z) 55 | order_a = String(take!(io)) 56 | b = @tvl f(io, 1, x) || f(io, 2, y) && f(io, 3, z) 57 | order_b = String(take!(io)) 58 | 59 | @test a === b 60 | @test order_a === order_b 61 | end 62 | end 63 | end 64 | end 65 | --------------------------------------------------------------------------------